|
|
@@ -0,0 +1,366 @@
|
|
|
+package sci99
|
|
|
+
|
|
|
+import (
|
|
|
+ "bytes"
|
|
|
+ "context"
|
|
|
+ "encoding/json"
|
|
|
+ "eta/eta_crawler/utils"
|
|
|
+ "fmt"
|
|
|
+ "github.com/mozillazg/go-pinyin"
|
|
|
+ "io/ioutil"
|
|
|
+ "net/http"
|
|
|
+ "strconv"
|
|
|
+ "strings"
|
|
|
+ "time"
|
|
|
+)
|
|
|
+
|
|
|
+var IndexCodeMap = make(map[string]string)
|
|
|
+
|
|
|
+var PriceTypeIDs = []int{34320, 34318}
|
|
|
+var requestList = []ListRequestBody{
|
|
|
+ {
|
|
|
+ Ppname: "天然橡胶原料",
|
|
|
+ PPIDs: "13676",
|
|
|
+ Navid: "593",
|
|
|
+ }, {
|
|
|
+ Ppname: "混合胶",
|
|
|
+ PPIDs: "12973",
|
|
|
+ Navid: "591",
|
|
|
+ }, {
|
|
|
+ Ppname: "标准胶",
|
|
|
+ PPIDs: "12960",
|
|
|
+ Navid: "590",
|
|
|
+ }, {
|
|
|
+ Ppname: "烟片胶",
|
|
|
+ PPIDs: "12985",
|
|
|
+ Navid: "592",
|
|
|
+ }, {
|
|
|
+ Ppname: "天然乳胶",
|
|
|
+ PPIDs: "12947",
|
|
|
+ Navid: "595",
|
|
|
+ }, {
|
|
|
+ Ppname: "丁苯橡胶",
|
|
|
+ PPIDs: "12951",
|
|
|
+ Navid: "596",
|
|
|
+ }, {
|
|
|
+ Ppname: "顺丁橡胶",
|
|
|
+ PPIDs: "12964",
|
|
|
+ Navid: "597",
|
|
|
+ }, {
|
|
|
+ Ppname: "SBS",
|
|
|
+ PPIDs: "12948",
|
|
|
+ Navid: "598",
|
|
|
+ }, {
|
|
|
+ Ppname: "丁腈橡胶",
|
|
|
+ PPIDs: "12945",
|
|
|
+ Navid: "605",
|
|
|
+ },
|
|
|
+}
|
|
|
+
|
|
|
+func Scii99(cont context.Context) (err error) {
|
|
|
+ filePath := utils.SCI99_COOKIE_PATH
|
|
|
+
|
|
|
+ // 打开文件
|
|
|
+ file, err := ioutil.ReadFile(filePath)
|
|
|
+ if err != nil {
|
|
|
+ utils.FileLog.Info("ReadFile err:", err)
|
|
|
+ return
|
|
|
+ }
|
|
|
+
|
|
|
+ cookie := string(file)
|
|
|
+
|
|
|
+ // 定义请求的URL
|
|
|
+ listUrl := "https://prices.sci99.com/api/zh-cn/product/datavalue"
|
|
|
+ detailUrl := "https://prices.sci99.com/api/zh-cn/dataitem/datavalue"
|
|
|
+
|
|
|
+ for _, request := range requestList {
|
|
|
+ classifyName := request.Ppname
|
|
|
+
|
|
|
+ // 分别获取国内价格和国际价格
|
|
|
+ for _, priceTypeID := range PriceTypeIDs {
|
|
|
+ // 创建请求体实例并填充数据,这里的参数都是一样的，所以这边统一写
|
|
|
+ request.CycleType = "day"
|
|
|
+ request.SiteType = 1
|
|
|
+ request.PageNo = 1
|
|
|
+ request.PageSize = "1000"
|
|
|
+ request.PriceTypeID = priceTypeID
|
|
|
+
|
|
|
+ // 将请求体序列化为JSON
|
|
|
+ jsonData, e := json.Marshal(request)
|
|
|
+ if e != nil {
|
|
|
+ err = e
|
|
|
+ fmt.Println("Error marshalling JSON:", err)
|
|
|
+ utils.FileLog.Info("Error marshalling JSON:", err)
|
|
|
+ return
|
|
|
+ }
|
|
|
+
|
|
|
+ // 创建一个HTTP POST请求
|
|
|
+ req, e := http.NewRequest("POST", listUrl, bytes.NewBuffer(jsonData))
|
|
|
+ if e != nil {
|
|
|
+ err = e
|
|
|
+ fmt.Println("Error creating request:", err)
|
|
|
+ utils.FileLog.Info("Error creating request:", err)
|
|
|
+ return
|
|
|
+ }
|
|
|
+
|
|
|
+ // 设置请求头信息
|
|
|
+ req.Header.Set("Content-Type", "application/json")
|
|
|
+ req.Header.Set("Accept", "application/json")
|
|
|
+ req.Header.Set("Cookie", cookie)
|
|
|
+
|
|
|
+ // 发送请求
|
|
|
+ client := &http.Client{}
|
|
|
+ resp, e := client.Do(req)
|
|
|
+ if e != nil {
|
|
|
+ err = e
|
|
|
+ fmt.Println("Error sending request:", err)
|
|
|
+ utils.FileLog.Info("Error sending request:", err)
|
|
|
+ return
|
|
|
+ }
|
|
|
+ defer resp.Body.Close()
|
|
|
+
|
|
|
+ // 检查响应状态码
|
|
|
+ if resp.StatusCode != http.StatusOK {
|
|
|
+ fmt.Printf("Server returned non-200 status: %d\n", resp.StatusCode)
|
|
|
+ utils.FileLog.Info("Server returned non-200 status: %d", resp.StatusCode)
|
|
|
+ return
|
|
|
+ }
|
|
|
+ // 打印响应体
|
|
|
+ body, _ := ioutil.ReadAll(resp.Body)
|
|
|
+ //fmt.Println("Response body:", string(body))
|
|
|
+
|
|
|
+ var listResponse ListResponse
|
|
|
+
|
|
|
+ err = json.Unmarshal(body, &listResponse)
|
|
|
+ if err != nil {
|
|
|
+ fmt.Println(err)
|
|
|
+ utils.FileLog.Info("Error unmarshalling JSON:", err)
|
|
|
+ return
|
|
|
+ }
|
|
|
+
|
|
|
+ for _, v := range listResponse.FirstData.SecondData.Items {
|
|
|
+ if v.FromArea != "" && len(listResponse.FirstData.SecondData.Items) > 1 {
|
|
|
+ continue
|
|
|
+ }
|
|
|
+ // 组成指标
|
|
|
+ var indexName string
|
|
|
+ if v.FactorySampleName == "" || len(listResponse.FirstData.SecondData.Items) == 1 {
|
|
|
+ indexName = v.ProductName + "：" + v.PriceType + "：" + v.MarketSampleName + "：" + v.Model
|
|
|
+ } else {
|
|
|
+ indexName = v.ProductName + "：" + v.PriceType + "：" + v.MarketSampleName + "：" + v.FactorySampleName + "：" + v.Model
|
|
|
+ }
|
|
|
+
|
|
|
+ ppid := strconv.Itoa(v.ProductID)
|
|
|
+ dataTypeID := strconv.Itoa(v.DataTypeID)
|
|
|
+ diidd := strconv.Itoa(v.DIID)
|
|
|
+ deailReq := DetailRequest{
|
|
|
+ PPID: ppid,
|
|
|
+ DataTypeID: dataTypeID,
|
|
|
+ DIIDD: diidd,
|
|
|
+ APIStart: "2021/01/01",
|
|
|
+ APIEnd: time.Now().AddDate(1, 0, 0).Format(utils.FormatDate2),
|
|
|
+ CycleType: "day",
|
|
|
+ }
|
|
|
+
|
|
|
+ // 初始化是不加日期获取全部，初始化后只拿前三天的
|
|
|
+ if utils.IS_INIT_SCI99 != "true" {
|
|
|
+ deailReq.Start = time.Now().AddDate(0,0,-3).Format(utils.FormatDate2)
|
|
|
+ deailReq.End = time.Now().Format(utils.FormatDate2)
|
|
|
+ }
|
|
|
+
|
|
|
+ // 将请求体序列化为JSON
|
|
|
+ reqJson, e := json.Marshal(deailReq)
|
|
|
+ if e != nil {
|
|
|
+ err = e
|
|
|
+ fmt.Println("Error marshalling JSON:", err)
|
|
|
+ utils.FileLog.Info("Error marshalling JSON:", err)
|
|
|
+ return
|
|
|
+ }
|
|
|
+
|
|
|
+ // 创建一个HTTP POST请求
|
|
|
+ req, err = http.NewRequest("POST", detailUrl, bytes.NewBuffer(reqJson))
|
|
|
+ if err != nil {
|
|
|
+ fmt.Println("Error creating request:", err)
|
|
|
+ utils.FileLog.Info("Error creating request:", err)
|
|
|
+ return
|
|
|
+ }
|
|
|
+
|
|
|
+ // 设置请求头信息
|
|
|
+ req.Header.Set("Content-Type", "application/json")
|
|
|
+ req.Header.Set("Accept", "application/json")
|
|
|
+ req.Header.Set("Cookie", cookie)
|
|
|
+
|
|
|
+ // 发送请求
|
|
|
+ resp, err = client.Do(req)
|
|
|
+ if err != nil {
|
|
|
+ fmt.Println("Error sending request:", err)
|
|
|
+ utils.FileLog.Info("Error sending request:", err)
|
|
|
+ return
|
|
|
+ }
|
|
|
+ defer resp.Body.Close()
|
|
|
+ // 检查响应状态码
|
|
|
+ if resp.StatusCode != http.StatusOK {
|
|
|
+ fmt.Printf("Server returned non-200 status: %d\n", resp.StatusCode)
|
|
|
+ utils.FileLog.Info("Server returned non-200 status: %d", resp.StatusCode)
|
|
|
+ return
|
|
|
+ }
|
|
|
+ // 打印响应体
|
|
|
+ body, _ = ioutil.ReadAll(resp.Body)
|
|
|
+ //fmt.Println("Response Detail body:", string(body))
|
|
|
+
|
|
|
+ var detailResponse DetailResponse
|
|
|
+
|
|
|
+ err = json.Unmarshal(body, &detailResponse)
|
|
|
+ if err != nil {
|
|
|
+ fmt.Println(err)
|
|
|
+ utils.FileLog.Info("Error unmarshalling JSON:", err)
|
|
|
+ return
|
|
|
+ }
|
|
|
+
|
|
|
+ param := make(map[string]interface{})
|
|
|
+ param["ListData"] = v
|
|
|
+ param["DetailData"] = detailResponse
|
|
|
+ param["IndexName"] = indexName
|
|
|
+ param["ClassifyName"] = classifyName
|
|
|
+ urlStr := `sci99_crawler/refresh/list`
|
|
|
+ postUrl := utils.EDB_LIB_URL + urlStr
|
|
|
+ postData, e := json.Marshal(param)
|
|
|
+ if e != nil {
|
|
|
+ err = e
|
|
|
+ utils.FileLog.Info("Marshal Err:" + err.Error())
|
|
|
+ fmt.Println(err)
|
|
|
+ return
|
|
|
+ }
|
|
|
+ result, e := HttpPost(postUrl, string(postData), "application/json")
|
|
|
+ if e != nil {
|
|
|
+ err = e
|
|
|
+ fmt.Println(err)
|
|
|
+ utils.FileLog.Info("HttpPost Err:" + err.Error())
|
|
|
+ return
|
|
|
+ }
|
|
|
+ utils.FileLog.Info("postRefreshEdbData:" + postUrl + ";" + string(postData) + ";result:" + string(result))
|
|
|
+ err = json.Unmarshal(result, &resp)
|
|
|
+ if err != nil {
|
|
|
+ fmt.Println(err)
|
|
|
+ utils.FileLog.Info("Unmarshal resp Err:" + err.Error())
|
|
|
+ return
|
|
|
+ }
|
|
|
+ time.Sleep(1 * time.Second)
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+ return
|
|
|
+}
|
|
|
+
|
|
|
+func Sci99IndexCodeGenerator(indexName, indexCodeStr, marketSampleName, model string) (indexCode string, needAdd bool) {
|
|
|
+ strResult := ""
|
|
|
+ indexCode, _ = IndexCodeMap[indexName]
|
|
|
+ if indexCode == "" {
|
|
|
+ //首字母
|
|
|
+ a := pinyin.NewArgs()
|
|
|
+ a.Fallback = func(r rune, a pinyin.Args) []string {
|
|
|
+ return []string{string(r)}
|
|
|
+ }
|
|
|
+ rows := pinyin.Pinyin(indexCodeStr, a)
|
|
|
+ for i := 0; i < len(rows); i++ {
|
|
|
+ //strResult += rows[i][0]
|
|
|
+ if len(rows[i]) != 0 {
|
|
|
+ str := rows[i][0]
|
|
|
+ pi := str[0:1]
|
|
|
+ strResult += pi
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ // 处理市场名称
|
|
|
+ if province, ok := ProvinceMap[marketSampleName]; ok {
|
|
|
+ strResult += province
|
|
|
+ } else {
|
|
|
+ a := pinyin.NewArgs()
|
|
|
+ rows := pinyin.LazyPinyin(marketSampleName, a)
|
|
|
+ for i := 0; i < len(rows); i++ {
|
|
|
+ strResult += rows[i]
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ // 去除特殊符号
|
|
|
+ model = strings.Replace(model, " ", "", -1)
|
|
|
+ model = strings.Replace(model, "-", "", -1)
|
|
|
+ model = strings.Replace(model, "/", "", -1)
|
|
|
+ model = strings.Replace(model, "#", "", -1)
|
|
|
+ model = strings.Replace(model, "：", "", -1)
|
|
|
+ model = strings.Replace(model, "(", "", -1)
|
|
|
+ model = strings.Replace(model, ")", "", -1)
|
|
|
+
|
|
|
+ // 拼接型号
|
|
|
+ modelRows := pinyin.Pinyin(model, a)
|
|
|
+ for i := 0; i < len(modelRows); i++ {
|
|
|
+ if len(modelRows[i]) != 0 {
|
|
|
+ str := modelRows[i][0]
|
|
|
+ pi := str[0:1]
|
|
|
+ strResult += pi
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ needAdd = true
|
|
|
+ indexCode = strings.Replace(strResult, " ", "", -1)
|
|
|
+ IndexCodeMap[indexName] = indexCode
|
|
|
+ }
|
|
|
+ return
|
|
|
+}
|
|
|
+
|
|
|
+var ProvinceMap = map[string]string{
|
|
|
+ "上海": "shanghai",
|
|
|
+ "云南": "yunnan",
|
|
|
+ "内蒙古": "innermongolia",
|
|
|
+ "北京": "beijing",
|
|
|
+ "台湾": "taiwan",
|
|
|
+ "吉林": "jilin",
|
|
|
+ "四川": "sichuan",
|
|
|
+ "天津": "tianjin",
|
|
|
+ "宁夏": "ningxia",
|
|
|
+ "安徽": "anhui",
|
|
|
+ "山东": "shandong",
|
|
|
+ "山西": "shanxi",
|
|
|
+ "广东": "guangdong",
|
|
|
+ "广西": "guangxi",
|
|
|
+ "新疆": "xinjiang",
|
|
|
+ "江苏": "jiangsu",
|
|
|
+ "江西": "jiangxi",
|
|
|
+ "河北": "hebei",
|
|
|
+ "河南": "henan",
|
|
|
+ "浙江": "zhejiang",
|
|
|
+ "海南": "hainan",
|
|
|
+ "湖北": "hubei",
|
|
|
+ "湖南": "hunan",
|
|
|
+ "澳门": "macao",
|
|
|
+ "甘肃": "gansu",
|
|
|
+ "福建": "fujian",
|
|
|
+ "西藏": "tibet",
|
|
|
+ "贵州": "guizhou",
|
|
|
+ "辽宁": "liaoning",
|
|
|
+ "重庆": "chongqing",
|
|
|
+ "陕西": "shaanxi",
|
|
|
+ "青海": "qinhai",
|
|
|
+ "香港": "hongkong",
|
|
|
+ "黑龙江": "heilongjiang",
|
|
|
+}
|
|
|
+
|
|
|
+func HttpPost(url, postData string, params ...string) ([]byte, error) {
|
|
|
+ body := ioutil.NopCloser(strings.NewReader(postData))
|
|
|
+ client := &http.Client{}
|
|
|
+ req, err := http.NewRequest("POST", url, body)
|
|
|
+ if err != nil {
|
|
|
+ return nil, err
|
|
|
+ }
|
|
|
+ contentType := "application/x-www-form-urlencoded;charset=utf-8"
|
|
|
+ if len(params) > 0 && params[0] != "" {
|
|
|
+ contentType = params[0]
|
|
|
+ }
|
|
|
+ req.Header.Set("Content-Type", contentType)
|
|
|
+ req.Header.Set("authorization", utils.MD5(utils.APP_EDB_LIB_NAME_EN+utils.EDB_LIB_Md5_KEY))
|
|
|
+ resp, err := client.Do(req)
|
|
|
+ defer resp.Body.Close()
|
|
|
+ b, err := ioutil.ReadAll(resp.Body)
|
|
|
+ fmt.Println("HttpPost:" + string(b))
|
|
|
+ return b, err
|
|
|
+}
|
zwxi