package controllers import ( "encoding/json" "eta/eta_index_lib/models" "eta/eta_index_lib/services/sci99" "eta/eta_index_lib/utils" "fmt" "github.com/mozillazg/go-pinyin" "strings" "time" ) // 卓创资讯爬虫 type Sci99Crawler struct { BaseAuthController } var IndexCodeMap = make(map[string]string) var IndexMap = make(map[string]*models.BaseFromSci99Index) var ClassifyMap = make(map[string]*models.BaseFromSci99Classify) // @Title 刷新数据 // @Description 刷新数据接口 // @Param request body models.AddEdbClassifyReq true "type json string" // @Success 200 {object} models.EdbClassify // @router /refresh/list [post] func (this *Sci99Crawler) RefreshSci99Classify() { br := new(models.BaseResponse).Init() defer func() { this.Data["json"] = br this.ServeJSON() }() var req sci99.RefreshSci99CrawlerReq err := json.Unmarshal(this.Ctx.Input.RequestBody, &req) if err != nil { br.Msg = "参数解析异常!" br.ErrMsg = "参数解析失败,Err:" + err.Error() return } allClassify, err := models.GetBaseFromSci99Classify() if err != nil { fmt.Println("select Code err:", err) utils.FileLog.Info("GetBaseFromSci99Index err:", err) return } for _, item := range allClassify { ClassifyMap[item.ClassifyName] = item } if _, ok := ClassifyMap[req.ClassifyName]; !ok { classifyItem := models.BaseFromSci99Classify{ ClassifyName: req.ClassifyName, CreateTime: time.Now(), ModifyTime: time.Now(), } id, e := models.AddBaseFromSci99Classify(&classifyItem) if e != nil { err = e fmt.Println("Error inserting into database:", err) utils.FileLog.Info("Error inserting into database:", err) return } fmt.Println("新增分类:", classifyItem.ClassifyName) classifyItem.BaseFromSciClassifyId = int(id) ClassifyMap[req.ClassifyName] = &classifyItem } allCode, e := models.GetBaseFromSci99Index() if e != nil { err = e fmt.Println("select Code err:", err) utils.FileLog.Info("GetBaseFromSci99Index err:", err) return } for _, item := range allCode { IndexCodeMap[item.IndexName] = item.IndexCode IndexMap[item.IndexName] = item } v := req.ListData // 组成指标 indexName := req.IndexName fmt.Println("indexName:", indexName) indexCodeStr := "卓创资讯" + v.ProductName + v.PriceType indexCode, needAdd := Sci99IndexCodeGenerator(indexName, indexCodeStr, v.MarketSampleName, v.FactorySampleName+v.Model) if needAdd { index := models.BaseFromSci99Index{ IndexCode: indexCode, IndexName: indexName, ClassifyId: ClassifyMap[req.ClassifyName].BaseFromSciClassifyId, Unit: v.Unit, Frequency: "日度", Describe: v.DataItemName, CreateTime: time.Now(), ModifyTime: time.Now(), } id, e := models.AddBaseFromSci99Index(&index) if e != nil { err = e fmt.Println("Error inserting into database:", err) utils.FileLog.Info("Error inserting into database:", err) return } fmt.Println("新增指标:", index.IndexName) index.BaseFromSciIndexId = int(id) IndexMap[indexName] = &index } detailResponse := req.DetailData existDataMap := make(map[string]*models.BaseFromSci99Data) //获取所有指标信息 allData, e := models.GetBaseFromTradeSci99DataAll(indexCode) if e != nil { err = e fmt.Println("select err:", err) utils.FileLog.Info("GetBaseFromTradeSci99IndexAll err:", err) } for _, vv := range allData { indexKey := vv.IndexCode + vv.DataTime existDataMap[indexKey] = vv } // 新增data数据 addList := make([]*models.BaseFromSci99Data, 0) for _, v := range detailResponse.Data.List { if dataItem, ok := existDataMap[indexCode+v.RealDate]; ok { if dataItem.Value != v.MDataValue { // 更新 fmt.Println("更新指标:", indexCode+v.RealDate) e = models.UpdateBaseFromSci99Data(dataItem.Value, indexCode, v.RealDate) if e != nil { err = e fmt.Println("Error update into database:", err) utils.FileLog.Info("Error update into database:", err) return } } } else { // 新增 dataItem := models.BaseFromSci99Data{ BaseFromSciIndexId: IndexMap[indexName].BaseFromSciIndexId, IndexCode: indexCode, DataTime: v.RealDate, Value: v.MDataValue, CreateTime: time.Now(), ModifyTime: time.Now(), } addList = append(addList, &dataItem) fmt.Println("新增数据:", indexCode+v.RealDate) } } if len(addList) > 0 { e = models.AddBaseFromSci99DataMulti(addList) if e != nil { err = e fmt.Println("Error inserting into database:", err) utils.FileLog.Info("Error inserting into database:", err) return } } time.Sleep(1 * time.Second) br.Ret = 200 br.Msg = "获取成功" br.Success = true } func Sci99IndexCodeGenerator(indexName, indexCodeStr, marketSampleName, model string) (indexCode string, needAdd bool) { strResult := "" indexCode, _ = IndexCodeMap[indexName] if indexCode == "" { //首字母 a := pinyin.NewArgs() a.Fallback = func(r rune, a pinyin.Args) []string { return []string{string(r)} } rows := pinyin.Pinyin(indexCodeStr, a) for i := 0; i < len(rows); i++ { //strResult += rows[i][0] if len(rows[i]) != 0 { str := rows[i][0] pi := str[0:1] strResult += pi } } // 处理市场名称 if province, ok := ProvinceMap[marketSampleName]; ok { strResult += province } else { a := pinyin.NewArgs() rows := pinyin.LazyPinyin(marketSampleName, a) for i := 0; i < len(rows); i++ { strResult += rows[i] } } // 去除特殊符号 model = strings.Replace(model, " ", "", -1) model = strings.Replace(model, "-", "", -1) model = strings.Replace(model, "/", "", -1) model = strings.Replace(model, "#", "", -1) model = strings.Replace(model, ":", "", -1) model = strings.Replace(model, "(", "", -1) model = strings.Replace(model, ")", "", -1) // 拼接型号 modelRows := pinyin.Pinyin(model, a) for i := 0; i < len(modelRows); i++ { if len(modelRows[i]) != 0 { str := modelRows[i][0] pi := str[0:1] strResult += pi } } needAdd = true indexCode = strings.Replace(strResult, " ", "", -1) IndexCodeMap[indexName] = indexCode } return } var ProvinceMap = map[string]string{ "上海": "shanghai", "云南": "yunnan", "内蒙古": "innermongolia", "北京": "beijing", "台湾": "taiwan", "吉林": "jilin", "四川": "sichuan", "天津": "tianjin", "宁夏": "ningxia", "安徽": "anhui", "山东": "shandong", "山西": "shanxi", "广东": "guangdong", "广西": "guangxi", "新疆": "xinjiang", "江苏": "jiangsu", "江西": "jiangxi", "河北": "hebei", "河南": "henan", "浙江": "zhejiang", "海南": "hainan", "湖北": "hubei", "湖南": "hunan", "澳门": "macao", "甘肃": "gansu", "福建": "fujian", "西藏": "tibet", "贵州": "guizhou", "辽宁": "liaoning", "重庆": "chongqing", "陕西": "shaanxi", "青海": "qinhai", "香港": "hongkong", "黑龙江": "heilongjiang", } var requestList = []sci99.ListRequestBody{ { Ppname: "天然橡胶原料", PPIDs: "13676", Navid: "593", }, { Ppname: "混合胶", PPIDs: "12973", Navid: "591", }, { Ppname: "标准胶", PPIDs: "12960", Navid: "590", }, { Ppname: "烟片胶", PPIDs: "12985", Navid: "592", }, { Ppname: "天然乳胶", PPIDs: "12947", Navid: "595", }, { Ppname: "丁苯橡胶", PPIDs: "12951", Navid: "596", }, { Ppname: "顺丁橡胶", PPIDs: "12964", Navid: "597", }, { Ppname: "SBS", PPIDs: "12948", Navid: "598", }, { Ppname: "丁腈橡胶", PPIDs: "12945", Navid: "605", }, }