package sci99 import ( "bytes" "context" "encoding/json" "eta/eta_crawler/utils" "fmt" "io/ioutil" "net/http" "strconv" "strings" "time" "github.com/mozillazg/go-pinyin" ) var IndexCodeMap = make(map[string]string) var PriceTypeIDs = []int{34320, 34318} var requestList = []ListRequestBody{ { Ppname: "天然橡胶原料", PPIDs: "13676", Navid: "593", }, { Ppname: "混合胶", PPIDs: "12973", Navid: "591", }, { Ppname: "标准胶", PPIDs: "12960", Navid: "590", }, { Ppname: "烟片胶", PPIDs: "12985", Navid: "592", }, { Ppname: "天然乳胶", PPIDs: "12947", Navid: "595", }, { Ppname: "丁苯橡胶", PPIDs: "12951", Navid: "596", }, { Ppname: "顺丁橡胶", PPIDs: "12964", Navid: "597", }, { Ppname: "SBS", PPIDs: "12948", Navid: "598", }, { Ppname: "丁腈橡胶", PPIDs: "12945", Navid: "605", }, } func Scii99(cont context.Context) (err error) { filePath := utils.SCI99_COOKIE_PATH // 打开文件 file, err := ioutil.ReadFile(filePath) if err != nil { utils.FileLog.Info("ReadFile err:", err) return } cookie := string(file) // 定义请求的URL listUrl := "https://prices.sci99.com/api/zh-cn/product/datavalue" detailUrl := "https://prices.sci99.com/api/zh-cn/dataitem/datavalue" for _, request := range requestList { classifyName := request.Ppname // 分别获取国内价格和国际价格 for _, priceTypeID := range PriceTypeIDs { // 创建请求体实例并填充数据,这里的参数都是一样的,所以这边统一写 request.CycleType = "day" request.SiteType = 1 request.PageNo = 1 request.PageSize = "1000" request.PriceTypeID = priceTypeID // 将请求体序列化为JSON jsonData, e := json.Marshal(request) if e != nil { err = e fmt.Println("Error marshalling JSON:", err) utils.FileLog.Info("Error marshalling JSON:", err) return } // 创建一个HTTP POST请求 req, e := http.NewRequest("POST", listUrl, bytes.NewBuffer(jsonData)) if e != nil { err = e fmt.Println("Error creating request:", err) utils.FileLog.Info("Error creating request:", err) return } // 设置请求头信息 req.Header.Set("Content-Type", "application/json") req.Header.Set("Accept", "application/json") req.Header.Set("Cookie", cookie) // 发送请求 client := &http.Client{} resp, e := client.Do(req) if e != nil { err = e fmt.Println("Error sending request:", err) utils.FileLog.Info("Error sending request:", err) return } defer resp.Body.Close() // 检查响应状态码 if resp.StatusCode != http.StatusOK { fmt.Printf("Server returned non-200 status: %d\n", resp.StatusCode) utils.FileLog.Info("Server returned non-200 status: %d", resp.StatusCode) return } // 打印响应体 body, _ := ioutil.ReadAll(resp.Body) //fmt.Println("Response body:", string(body)) var listResponse ListResponse err = json.Unmarshal(body, &listResponse) if err != nil { fmt.Println(err) utils.FileLog.Info("Error unmarshalling JSON:", err) return } fmt.Println("获取到的产品数量:", len(listResponse.FirstData.SecondData.Items)) for _, v := range listResponse.FirstData.SecondData.Items { if v.FromArea != "" && len(listResponse.FirstData.SecondData.Items) > 1 { continue } // 组成指标 var indexName string if v.FactorySampleName == "" || len(listResponse.FirstData.SecondData.Items) == 1 { indexName = v.ProductName + ":" + v.PriceType + ":" + v.MarketSampleName + ":" + v.Model } else { indexName = v.ProductName + ":" + v.PriceType + ":" + v.MarketSampleName + ":" + v.FactorySampleName + ":" + v.Model } ppid := strconv.Itoa(v.ProductID) dataTypeID := strconv.Itoa(v.DataTypeID) diidd := strconv.Itoa(v.DIID) deailReq := DetailRequest{ PPID: ppid, DataTypeID: dataTypeID, DIIDD: diidd, APIStart: "2021/01/01", APIEnd: time.Now().AddDate(1, 0, 0).Format(utils.FormatDate2), CycleType: "day", } // 初始化是不加日期获取全部,初始化后只拿前三天的 if utils.IS_INIT_SCI99 != "true" { deailReq.Start = time.Now().AddDate(0, 0, -5).Format(utils.FormatDate2) deailReq.End = time.Now().Format(utils.FormatDate2) } // 将请求体序列化为JSON reqJson, e := json.Marshal(deailReq) if e != nil { err = e fmt.Println("Error marshalling JSON:", err) utils.FileLog.Info("Error marshalling JSON:", err) return } // 创建一个HTTP POST请求 req, err = http.NewRequest("POST", detailUrl, bytes.NewBuffer(reqJson)) if err != nil { fmt.Println("Error creating request:", err) utils.FileLog.Info("Error creating request:", err) return } // 设置请求头信息 req.Header.Set("Content-Type", "application/json") req.Header.Set("Accept", "application/json") req.Header.Set("Cookie", cookie) // 发送请求 resp, err = client.Do(req) if err != nil { fmt.Println("Error sending request:", err) utils.FileLog.Info("Error sending request:", err) return } defer resp.Body.Close() // 检查响应状态码 if resp.StatusCode != http.StatusOK { fmt.Printf("Server returned non-200 status: %d\n", resp.StatusCode) utils.FileLog.Info("Server returned non-200 status: %d", resp.StatusCode) return } // 打印响应体 body, _ = ioutil.ReadAll(resp.Body) //fmt.Println("Response Detail body:", string(body)) var detailResponse DetailResponse err = json.Unmarshal(body, &detailResponse) if err != nil { fmt.Println(err) utils.FileLog.Info("Error unmarshalling JSON:", err) return } fmt.Println("classifyName:", classifyName) fmt.Println("indexName:", indexName) fmt.Println("数据长度:", len(detailResponse.Data.List)) param := make(map[string]interface{}) param["ListData"] = v param["DetailData"] = detailResponse param["IndexName"] = indexName param["ClassifyName"] = classifyName urlStr := `sci99_crawler/refresh/list` postUrl := utils.EDB_LIB_URL + urlStr postData, e := json.Marshal(param) if e != nil { err = e utils.FileLog.Info("Marshal Err:" + err.Error()) fmt.Println(err) return } result, e := HttpPost(postUrl, string(postData), "application/json") if e != nil { err = e fmt.Println(err) utils.FileLog.Info("HttpPost Err:" + err.Error()) return } utils.FileLog.Info("postRefreshEdbData:" + postUrl + ";" + string(postData) + ";result:" + string(result)) err = json.Unmarshal(result, &resp) if err != nil { fmt.Println(err) utils.FileLog.Info("Unmarshal resp Err:" + err.Error()) return } time.Sleep(1 * time.Second) } } } return } func Sci99IndexCodeGenerator(indexName, indexCodeStr, marketSampleName, model string) (indexCode string, needAdd bool) { strResult := "" indexCode, _ = IndexCodeMap[indexName] if indexCode == "" { //首字母 a := pinyin.NewArgs() a.Fallback = func(r rune, a pinyin.Args) []string { return []string{string(r)} } rows := pinyin.Pinyin(indexCodeStr, a) for i := 0; i < len(rows); i++ { //strResult += rows[i][0] if len(rows[i]) != 0 { str := rows[i][0] pi := str[0:1] strResult += pi } } // 处理市场名称 if province, ok := ProvinceMap[marketSampleName]; ok { strResult += province } else { a := pinyin.NewArgs() rows := pinyin.LazyPinyin(marketSampleName, a) for i := 0; i < len(rows); i++ { strResult += rows[i] } } // 去除特殊符号 model = strings.Replace(model, " ", "", -1) model = strings.Replace(model, "-", "", -1) model = strings.Replace(model, "/", "", -1) model = strings.Replace(model, "#", "", -1) model = strings.Replace(model, ":", "", -1) model = strings.Replace(model, "(", "", -1) model = strings.Replace(model, ")", "", -1) // 拼接型号 modelRows := pinyin.Pinyin(model, a) for i := 0; i < len(modelRows); i++ { if len(modelRows[i]) != 0 { str := modelRows[i][0] pi := str[0:1] strResult += pi } } needAdd = true indexCode = strings.Replace(strResult, " ", "", -1) IndexCodeMap[indexName] = indexCode } return } var ProvinceMap = map[string]string{ "上海": "shanghai", "云南": "yunnan", "内蒙古": "innermongolia", "北京": "beijing", "台湾": "taiwan", "吉林": "jilin", "四川": "sichuan", "天津": "tianjin", "宁夏": "ningxia", "安徽": "anhui", "山东": "shandong", "山西": "shanxi", "广东": "guangdong", "广西": "guangxi", "新疆": "xinjiang", "江苏": "jiangsu", "江西": "jiangxi", "河北": "hebei", "河南": "henan", "浙江": "zhejiang", "海南": "hainan", "湖北": "hubei", "湖南": "hunan", "澳门": "macao", "甘肃": "gansu", "福建": "fujian", "西藏": "tibet", "贵州": "guizhou", "辽宁": "liaoning", "重庆": "chongqing", "陕西": "shaanxi", "青海": "qinhai", "香港": "hongkong", "黑龙江": "heilongjiang", } func HttpPost(url, postData string, params ...string) ([]byte, error) { body := ioutil.NopCloser(strings.NewReader(postData)) client := &http.Client{} req, err := http.NewRequest("POST", url, body) if err != nil { return nil, err } contentType := "application/x-www-form-urlencoded;charset=utf-8" if len(params) > 0 && params[0] != "" { contentType = params[0] } req.Header.Set("Content-Type", contentType) req.Header.Set("authorization", utils.MD5(utils.APP_EDB_LIB_NAME_EN+utils.EDB_LIB_Md5_KEY)) resp, err := client.Do(req) if err != nil { return nil, err } defer resp.Body.Close() b, err := ioutil.ReadAll(resp.Body) fmt.Println("HttpPost:" + string(b)) return b, err }