|
|
@@ -0,0 +1,433 @@
|
|
|
+package sci99
|
|
|
+
|
|
|
+import (
|
|
|
+ "bytes"
|
|
|
+ "context"
|
|
|
+ "encoding/json"
|
|
|
+ "eta/eta_crawler/models"
|
|
|
+ "eta/eta_crawler/utils"
|
|
|
+ "fmt"
|
|
|
+ "github.com/mozillazg/go-pinyin"
|
|
|
+ "io/ioutil"
|
|
|
+ "net/http"
|
|
|
+ "strconv"
|
|
|
+ "strings"
|
|
|
+ "time"
|
|
|
+)
|
|
|
+
|
|
|
+var IndexCodeMap = make(map[string]string)
|
|
|
+var IndexMap = make(map[string]*models.BaseFromSci99Index)
|
|
|
+var ClassifyMap = make(map[string]*models.BaseFromSci99Classify)
|
|
|
+
|
|
|
+var PriceTypeIDs = []int{34320, 34318}
|
|
|
+var requestList = []ListRequestBody{
|
|
|
+ {
|
|
|
+ Ppname: "天然橡胶原料",
|
|
|
+ PPIDs: "13676",
|
|
|
+ Navid: "593",
|
|
|
+ }, {
|
|
|
+ Ppname: "混合胶",
|
|
|
+ PPIDs: "12973",
|
|
|
+ Navid: "591",
|
|
|
+ }, {
|
|
|
+ Ppname: "标准胶",
|
|
|
+ PPIDs: "12960",
|
|
|
+ Navid: "590",
|
|
|
+ }, {
|
|
|
+ Ppname: "烟片胶",
|
|
|
+ PPIDs: "12985",
|
|
|
+ Navid: "592",
|
|
|
+ }, {
|
|
|
+ Ppname: "天然乳胶",
|
|
|
+ PPIDs: "12947",
|
|
|
+ Navid: "595",
|
|
|
+ }, {
|
|
|
+ Ppname: "丁苯橡胶",
|
|
|
+ PPIDs: "12951",
|
|
|
+ Navid: "596",
|
|
|
+ }, {
|
|
|
+ Ppname: "顺丁橡胶",
|
|
|
+ PPIDs: "12964",
|
|
|
+ Navid: "597",
|
|
|
+ }, {
|
|
|
+ Ppname: "SBS",
|
|
|
+ PPIDs: "12948",
|
|
|
+ Navid: "598",
|
|
|
+ }, {
|
|
|
+ Ppname: "丁腈橡胶",
|
|
|
+ PPIDs: "12945",
|
|
|
+ Navid: "605",
|
|
|
+ },
|
|
|
+}
|
|
|
+
|
|
|
+func Scii99(cont context.Context) (err error) {
|
|
|
+ filePath := utils.SCI99_COOKIE_PATH
|
|
|
+
|
|
|
+ // 打开文件
|
|
|
+ file, err := ioutil.ReadFile(filePath)
|
|
|
+ if err != nil {
|
|
|
+ utils.FileLog.Info("ReadFile err:", err)
|
|
|
+ return
|
|
|
+ }
|
|
|
+
|
|
|
+ cookie := string(file)
|
|
|
+
|
|
|
+ // 定义请求的URL
|
|
|
+ listUrl := "https://prices.sci99.com/api/zh-cn/product/datavalue"
|
|
|
+ detailUrl := "https://prices.sci99.com/api/zh-cn/dataitem/datavalue"
|
|
|
+
|
|
|
+ allClassify, err := models.GetBaseFromSci99Classify()
|
|
|
+ if err != nil {
|
|
|
+ fmt.Println("select Code err:", err)
|
|
|
+ utils.FileLog.Info("GetBaseFromSci99Index err:", err)
|
|
|
+ return
|
|
|
+ }
|
|
|
+ for _, item := range allClassify {
|
|
|
+ ClassifyMap[item.ClassifyName] = item
|
|
|
+ }
|
|
|
+
|
|
|
+ for _, request := range requestList {
|
|
|
+ classifyId := 0
|
|
|
+ if classify, ok := ClassifyMap[request.Ppname]; !ok {
|
|
|
+ classifyItem := models.BaseFromSci99Classify{
|
|
|
+ ClassifyName: request.Ppname,
|
|
|
+ CreateTime: time.Now(),
|
|
|
+ ModifyTime: time.Now(),
|
|
|
+ }
|
|
|
+
|
|
|
+ id, err := models.AddBaseFromSci99Classify(&classifyItem)
|
|
|
+ if err != nil {
|
|
|
+ fmt.Println("Error inserting into database:", err)
|
|
|
+ utils.FileLog.Info("Error inserting into database:", err)
|
|
|
+ return
|
|
|
+ }
|
|
|
+ classifyId = int(id)
|
|
|
+ classifyItem.BaseFromSciClassifyId = int(id)
|
|
|
+ ClassifyMap[request.Ppname] = &classifyItem
|
|
|
+ } else {
|
|
|
+ classifyId = classify.BaseFromSciClassifyId
|
|
|
+ }
|
|
|
+
|
|
|
+ // 分别获取国内价格和国际价格
|
|
|
+ for _, priceTypeID := range PriceTypeIDs {
|
|
|
+ // 创建请求体实例并填充数据,这里的参数都是一样的，所以这边统一写
|
|
|
+ request.CycleType = "day"
|
|
|
+ request.SiteType = 1
|
|
|
+ request.PageNo = 1
|
|
|
+ request.PageSize = "1000"
|
|
|
+ request.PriceTypeID = priceTypeID
|
|
|
+
|
|
|
+ // 将请求体序列化为JSON
|
|
|
+ jsonData, err := json.Marshal(request)
|
|
|
+ if err != nil {
|
|
|
+ fmt.Println("Error marshalling JSON:", err)
|
|
|
+ utils.FileLog.Info("Error marshalling JSON:", err)
|
|
|
+ return
|
|
|
+ }
|
|
|
+
|
|
|
+ // 创建一个HTTP POST请求
|
|
|
+ req, err := http.NewRequest("POST", listUrl, bytes.NewBuffer(jsonData))
|
|
|
+ if err != nil {
|
|
|
+ fmt.Println("Error creating request:", err)
|
|
|
+ utils.FileLog.Info("Error creating request:", err)
|
|
|
+ return
|
|
|
+ }
|
|
|
+
|
|
|
+ // 设置请求头信息
|
|
|
+ req.Header.Set("Content-Type", "application/json")
|
|
|
+ req.Header.Set("Accept", "application/json")
|
|
|
+ req.Header.Set("Cookie", cookie)
|
|
|
+
|
|
|
+ // 发送请求
|
|
|
+ client := &http.Client{}
|
|
|
+ resp, err := client.Do(req)
|
|
|
+ if err != nil {
|
|
|
+ fmt.Println("Error sending request:", err)
|
|
|
+ utils.FileLog.Info("Error sending request:", err)
|
|
|
+ return
|
|
|
+ }
|
|
|
+ defer resp.Body.Close()
|
|
|
+
|
|
|
+ // 检查响应状态码
|
|
|
+ if resp.StatusCode != http.StatusOK {
|
|
|
+ fmt.Printf("Server returned non-200 status: %d\n", resp.StatusCode)
|
|
|
+ utils.FileLog.Info("Server returned non-200 status: %d", resp.StatusCode)
|
|
|
+ return
|
|
|
+ }
|
|
|
+ // 打印响应体
|
|
|
+ body, _ := ioutil.ReadAll(resp.Body)
|
|
|
+ //fmt.Println("Response body:", string(body))
|
|
|
+
|
|
|
+ var listResponse ListResponse
|
|
|
+
|
|
|
+ err = json.Unmarshal(body, &listResponse)
|
|
|
+ if err != nil {
|
|
|
+ fmt.Println(err)
|
|
|
+ utils.FileLog.Info("Error unmarshalling JSON:", err)
|
|
|
+ return
|
|
|
+ }
|
|
|
+ allCode, err := models.GetBaseFromSci99Index()
|
|
|
+ if err != nil {
|
|
|
+ fmt.Println("select Code err:", err)
|
|
|
+ utils.FileLog.Info("GetBaseFromSci99Index err:", err)
|
|
|
+ return
|
|
|
+ }
|
|
|
+ for _, item := range allCode {
|
|
|
+ IndexCodeMap[item.IndexName] = item.IndexCode
|
|
|
+ IndexMap[item.IndexName] = item
|
|
|
+ }
|
|
|
+
|
|
|
+ for _, v := range listResponse.FirstData.SecondData.Items {
|
|
|
+ if v.FromArea != "" && len(listResponse.FirstData.SecondData.Items) > 1 {
|
|
|
+ continue
|
|
|
+ }
|
|
|
+ // 组成指标
|
|
|
+ var indexName string
|
|
|
+ if v.FactorySampleName == "" || len(listResponse.FirstData.SecondData.Items) == 1 {
|
|
|
+ indexName = v.ProductName + "：" + v.PriceType + "：" + v.MarketSampleName + "：" + v.Model
|
|
|
+ } else {
|
|
|
+ indexName = v.ProductName + "：" + v.PriceType + "：" + v.MarketSampleName + "：" + v.FactorySampleName + "：" + v.Model
|
|
|
+ }
|
|
|
+ fmt.Println("indexName:", indexName)
|
|
|
+ indexCodeStr := "卓创资讯" + v.ProductName + v.PriceType
|
|
|
+ indexCode, needAdd := Sci99IndexCodeGenerator(indexName, indexCodeStr, v.MarketSampleName, v.FactorySampleName+v.Model)
|
|
|
+ if needAdd {
|
|
|
+ index := models.BaseFromSci99Index{
|
|
|
+ IndexCode: indexCode,
|
|
|
+ IndexName: indexName,
|
|
|
+ ClassifyId: classifyId,
|
|
|
+ Unit: v.Unit,
|
|
|
+ Frequency: "日度",
|
|
|
+ Describe: v.DataItemName,
|
|
|
+ CreateTime: time.Now(),
|
|
|
+ ModifyTime: time.Now(),
|
|
|
+ }
|
|
|
+ id, err := models.AddBaseFromSci99Index(&index)
|
|
|
+ if err != nil {
|
|
|
+ fmt.Println("Error inserting into database:", err)
|
|
|
+ utils.FileLog.Info("Error inserting into database:", err)
|
|
|
+ return
|
|
|
+ }
|
|
|
+ index.BaseFromSciIndexId = int(id)
|
|
|
+ IndexMap[indexName] = &index
|
|
|
+ }
|
|
|
+
|
|
|
+ ppid := strconv.Itoa(v.ProductID)
|
|
|
+ dataTypeID := strconv.Itoa(v.DataTypeID)
|
|
|
+ diidd := strconv.Itoa(v.DIID)
|
|
|
+ deailReq := DetailRequest{
|
|
|
+ PPID: ppid,
|
|
|
+ DataTypeID: dataTypeID,
|
|
|
+ DIIDD: diidd,
|
|
|
+ APIStart: "2021/01/01",
|
|
|
+ APIEnd: time.Now().AddDate(1, 0, 0).Format(utils.FormatDate2),
|
|
|
+ CycleType: "day",
|
|
|
+ }
|
|
|
+
|
|
|
+ // 获取最新日期
|
|
|
+ dateTime, err := models.GetBaseFromTradeSci99LatestData(indexCode)
|
|
|
+ if err != nil && err.Error() != utils.ErrNoRow() {
|
|
|
+ utils.FileLog.Info("GetBaseFromTradeSci99LatestData err:", err)
|
|
|
+ return
|
|
|
+ }
|
|
|
+ if !dateTime.IsZero() {
|
|
|
+ deailReq.Start = dateTime.Format(utils.FormatDate2)
|
|
|
+ deailReq.End = time.Now().Format(utils.FormatDate2)
|
|
|
+ }
|
|
|
+
|
|
|
+ // 将请求体序列化为JSON
|
|
|
+ reqJson, e := json.Marshal(deailReq)
|
|
|
+ if e != nil {
|
|
|
+ err = e
|
|
|
+ fmt.Println("Error marshalling JSON:", err)
|
|
|
+ utils.FileLog.Info("Error marshalling JSON:", err)
|
|
|
+ return
|
|
|
+ }
|
|
|
+
|
|
|
+ // 创建一个HTTP POST请求
|
|
|
+ req, err = http.NewRequest("POST", detailUrl, bytes.NewBuffer(reqJson))
|
|
|
+ if err != nil {
|
|
|
+ fmt.Println("Error creating request:", err)
|
|
|
+ utils.FileLog.Info("Error creating request:", err)
|
|
|
+ return
|
|
|
+ }
|
|
|
+
|
|
|
+ // 设置请求头信息
|
|
|
+ req.Header.Set("Content-Type", "application/json")
|
|
|
+ req.Header.Set("Accept", "application/json")
|
|
|
+ req.Header.Set("Cookie", cookie)
|
|
|
+
|
|
|
+ // 发送请求
|
|
|
+ resp, err = client.Do(req)
|
|
|
+ if err != nil {
|
|
|
+ fmt.Println("Error sending request:", err)
|
|
|
+ utils.FileLog.Info("Error sending request:", err)
|
|
|
+ return
|
|
|
+ }
|
|
|
+ defer resp.Body.Close()
|
|
|
+ // 检查响应状态码
|
|
|
+ if resp.StatusCode != http.StatusOK {
|
|
|
+ fmt.Printf("Server returned non-200 status: %d\n", resp.StatusCode)
|
|
|
+ utils.FileLog.Info("Server returned non-200 status: %d", resp.StatusCode)
|
|
|
+ return
|
|
|
+ }
|
|
|
+ // 打印响应体
|
|
|
+ body, _ = ioutil.ReadAll(resp.Body)
|
|
|
+ //fmt.Println("Response Detail body:", string(body))
|
|
|
+
|
|
|
+ var detailResponse DetailResponse
|
|
|
+
|
|
|
+ err = json.Unmarshal(body, &detailResponse)
|
|
|
+ if err != nil {
|
|
|
+ fmt.Println(err)
|
|
|
+ utils.FileLog.Info("Error unmarshalling JSON:", err)
|
|
|
+ return
|
|
|
+ }
|
|
|
+
|
|
|
+
|
|
|
+ existDataMap := make(map[string]*models.BaseFromSci99Data)
|
|
|
+
|
|
|
+ //获取所有指标信息
|
|
|
+ allIndex, err := models.GetBaseFromTradeSci99IndexAll(indexCode)
|
|
|
+ if err != nil {
|
|
|
+ fmt.Println("select err:", err)
|
|
|
+ utils.FileLog.Info("GetBaseFromTradeSci99IndexAll err:", err)
|
|
|
+ }
|
|
|
+ for _, v := range allIndex {
|
|
|
+ indexKey := v.IndexCode + v.DataTime
|
|
|
+ existDataMap[indexKey] = v
|
|
|
+ }
|
|
|
+
|
|
|
+
|
|
|
+ // 新增data数据
|
|
|
+ addList := make([]*models.BaseFromSci99Data, 0)
|
|
|
+ for _, v := range detailResponse.Data.List {
|
|
|
+ if dataItem, ok := existDataMap[indexCode + v.RealDate]; ok {
|
|
|
+ if dataItem.Value != v.MDataValue {
|
|
|
+ // 更新
|
|
|
+ err = models.UpdateBaseFromSci99Data(dataItem.Value, indexCode, v.RealDate)
|
|
|
+ if err != nil {
|
|
|
+ fmt.Println("Error update into database:", err)
|
|
|
+ utils.FileLog.Info("Error update into database:", err)
|
|
|
+ return
|
|
|
+ }
|
|
|
+ }
|
|
|
+ } else {
|
|
|
+ // 新增
|
|
|
+ dataItem := models.BaseFromSci99Data{
|
|
|
+ BaseFromSciIndexId: IndexMap[indexName].BaseFromSciIndexId,
|
|
|
+ IndexCode: indexCode,
|
|
|
+ DataTime: v.RealDate,
|
|
|
+ Value: v.MDataValue,
|
|
|
+ CreateTime: time.Now(),
|
|
|
+ ModifyTime: time.Now(),
|
|
|
+ }
|
|
|
+ addList = append(addList, &dataItem)
|
|
|
+ }
|
|
|
+ }
|
|
|
+ if len(addList) > 0 {
|
|
|
+ err = models.AddBaseFromSci99DataMulti(addList)
|
|
|
+ if err != nil {
|
|
|
+ fmt.Println("Error inserting into database:", err)
|
|
|
+ utils.FileLog.Info("Error inserting into database:", err)
|
|
|
+ return
|
|
|
+ }
|
|
|
+ }
|
|
|
+ time.Sleep(1 * time.Second)
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+}
|
|
|
+
|
|
|
+func Sci99IndexCodeGenerator(indexName, indexCodeStr, marketSampleName, model string) (indexCode string, needAdd bool) {
|
|
|
+ strResult := ""
|
|
|
+ indexCode, _ = IndexCodeMap[indexName]
|
|
|
+ if indexCode == "" {
|
|
|
+ //首字母
|
|
|
+ a := pinyin.NewArgs()
|
|
|
+ a.Fallback = func(r rune, a pinyin.Args) []string {
|
|
|
+ return []string{string(r)}
|
|
|
+ }
|
|
|
+ rows := pinyin.Pinyin(indexCodeStr, a)
|
|
|
+ for i := 0; i < len(rows); i++ {
|
|
|
+ //strResult += rows[i][0]
|
|
|
+ if len(rows[i]) != 0 {
|
|
|
+ str := rows[i][0]
|
|
|
+ pi := str[0:1]
|
|
|
+ strResult += pi
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ // 处理市场名称
|
|
|
+ if province, ok := ProvinceMap[marketSampleName]; ok {
|
|
|
+ strResult += province
|
|
|
+ } else {
|
|
|
+ a := pinyin.NewArgs()
|
|
|
+ rows := pinyin.LazyPinyin(marketSampleName, a)
|
|
|
+ for i := 0; i < len(rows); i++ {
|
|
|
+ strResult += rows[i]
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ // 去除特殊符号
|
|
|
+ model = strings.Replace(model, " ", "", -1)
|
|
|
+ model = strings.Replace(model, "-", "", -1)
|
|
|
+ model = strings.Replace(model, "/", "", -1)
|
|
|
+ model = strings.Replace(model, "#", "", -1)
|
|
|
+ model = strings.Replace(model, "：", "", -1)
|
|
|
+ model = strings.Replace(model, "(", "", -1)
|
|
|
+ model = strings.Replace(model, ")", "", -1)
|
|
|
+
|
|
|
+ // 拼接型号
|
|
|
+ modelRows := pinyin.Pinyin(model, a)
|
|
|
+ for i := 0; i < len(modelRows); i++ {
|
|
|
+ if len(modelRows[i]) != 0 {
|
|
|
+ str := modelRows[i][0]
|
|
|
+ pi := str[0:1]
|
|
|
+ strResult += pi
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ needAdd = true
|
|
|
+ indexCode = strings.Replace(strResult, " ", "", -1)
|
|
|
+ IndexCodeMap[indexName] = indexCode
|
|
|
+ }
|
|
|
+ return
|
|
|
+}
|
|
|
+
|
|
|
+var ProvinceMap = map[string]string{
|
|
|
+ "上海": "shanghai",
|
|
|
+ "云南": "yunnan",
|
|
|
+ "内蒙古": "innermongolia",
|
|
|
+ "北京": "beijing",
|
|
|
+ "台湾": "taiwan",
|
|
|
+ "吉林": "jilin",
|
|
|
+ "四川": "sichuan",
|
|
|
+ "天津": "tianjin",
|
|
|
+ "宁夏": "ningxia",
|
|
|
+ "安徽": "anhui",
|
|
|
+ "山东": "shandong",
|
|
|
+ "山西": "shanxi",
|
|
|
+ "广东": "guangdong",
|
|
|
+ "广西": "guangxi",
|
|
|
+ "新疆": "xinjiang",
|
|
|
+ "江苏": "jiangsu",
|
|
|
+ "江西": "jiangxi",
|
|
|
+ "河北": "hebei",
|
|
|
+ "河南": "henan",
|
|
|
+ "浙江": "zhejiang",
|
|
|
+ "海南": "hainan",
|
|
|
+ "湖北": "hubei",
|
|
|
+ "湖南": "hunan",
|
|
|
+ "澳门": "macao",
|
|
|
+ "甘肃": "gansu",
|
|
|
+ "福建": "fujian",
|
|
|
+ "西藏": "tibet",
|
|
|
+ "贵州": "guizhou",
|
|
|
+ "辽宁": "liaoning",
|
|
|
+ "重庆": "chongqing",
|
|
|
+ "陕西": "shaanxi",
|
|
|
+ "青海": "qinhai",
|
|
|
+ "香港": "hongkong",
|
|
|
+ "黑龙江": "heilongjiang",
|
|
|
+}
|
zwxi