|
|
@@ -0,0 +1,256 @@
|
|
|
+package sci99
|
|
|
+
|
|
|
+import (
|
|
|
+ "bytes"
|
|
|
+ "encoding/json"
|
|
|
+ "eta/eta_crawler/models"
|
|
|
+ "eta/eta_crawler/utils"
|
|
|
+ "fmt"
|
|
|
+ "github.com/mozillazg/go-pinyin"
|
|
|
+ "io/ioutil"
|
|
|
+ "net/http"
|
|
|
+ "strconv"
|
|
|
+ "strings"
|
|
|
+ "time"
|
|
|
+)
|
|
|
+
|
|
|
+var IndexCodeMap = make(map[string]string)
|
|
|
+var IndexMap = make(map[string]*models.BaseFromSci99Index)
|
|
|
+
|
|
|
+func Scii99() {
|
|
|
+ // 定义请求的URL
|
|
|
+ listUrl := "https://prices.sci99.com/api/zh-cn/product/datavalue"
|
|
|
+ detailUrl := "https://prices.sci99.com/api/zh-cn/dataitem/datavalue"
|
|
|
+
|
|
|
+ // 创建请求体实例并填充数据
|
|
|
+ requestData := ListRequestBody{
|
|
|
+ Ppname: "天然橡胶原料",
|
|
|
+ PriceTypeID: 34320,
|
|
|
+ PPIDs: "13676",
|
|
|
+ Navid: "593",
|
|
|
+ SiteType: 1,
|
|
|
+ PageNo: 1,
|
|
|
+ PageSize: "1000",
|
|
|
+ }
|
|
|
+
|
|
|
+ // 将请求体序列化为JSON
|
|
|
+ jsonData, err := json.Marshal(requestData)
|
|
|
+ if err != nil {
|
|
|
+ fmt.Println("Error marshalling JSON:", err)
|
|
|
+ return
|
|
|
+ }
|
|
|
+
|
|
|
+ // 创建一个HTTP POST请求
|
|
|
+ req, err := http.NewRequest("POST", listUrl, bytes.NewBuffer(jsonData))
|
|
|
+ if err != nil {
|
|
|
+ fmt.Println("Error creating request:", err)
|
|
|
+ return
|
|
|
+ }
|
|
|
+
|
|
|
+ // 设置请求头信息
|
|
|
+ req.Header.Set("Content-Type", "application/json")
|
|
|
+ req.Header.Set("Accept", "application/json")
|
|
|
+ req.Header.Set("Cookie", Cookie)
|
|
|
+
|
|
|
+ // 发送请求
|
|
|
+ client := &http.Client{}
|
|
|
+ resp, err := client.Do(req)
|
|
|
+ if err != nil {
|
|
|
+ fmt.Println("Error sending request:", err)
|
|
|
+ utils.FileLog.Info("Error sending request:", err)
|
|
|
+ return
|
|
|
+ }
|
|
|
+ defer resp.Body.Close()
|
|
|
+
|
|
|
+ // 检查响应状态码
|
|
|
+ if resp.StatusCode != http.StatusOK {
|
|
|
+ fmt.Printf("Server returned non-200 status: %d\n", resp.StatusCode)
|
|
|
+ utils.FileLog.Info("Server returned non-200 status: %d", resp.StatusCode)
|
|
|
+ return
|
|
|
+ }
|
|
|
+ // 打印响应体
|
|
|
+ body, _ := ioutil.ReadAll(resp.Body)
|
|
|
+ fmt.Println("Response body:", string(body))
|
|
|
+
|
|
|
+ var listResponse ListResponse
|
|
|
+
|
|
|
+ err = json.Unmarshal(body, &listResponse)
|
|
|
+ if err != nil {
|
|
|
+ fmt.Println(err)
|
|
|
+ utils.FileLog.Info("Error unmarshalling JSON:", err)
|
|
|
+ return
|
|
|
+ }
|
|
|
+ allCode, err := models.GetBaseFromSci99Index()
|
|
|
+ if err != nil {
|
|
|
+ fmt.Println("select Code err:", err)
|
|
|
+ }
|
|
|
+ for _, item := range allCode {
|
|
|
+ IndexCodeMap[item.IndexName] = item.IndexCode
|
|
|
+ IndexMap[item.IndexName] = item
|
|
|
+ }
|
|
|
+
|
|
|
+ for _, v := range listResponse.FirstData.SecondData.Items {
|
|
|
+ // 组成指标
|
|
|
+ indexName := v.ProductName + v.PriceType + v.MarketSampleName + v.Model
|
|
|
+ indexCodeStr := "卓创资讯" + indexName
|
|
|
+ indexCode, needAdd := Sci99IndexCodeGenerator(indexName, indexCodeStr, v.MarketSampleName)
|
|
|
+ if needAdd {
|
|
|
+ index := models.BaseFromSci99Index{
|
|
|
+ IndexCode: indexCode,
|
|
|
+ IndexName: indexName,
|
|
|
+ ClassifyId: 0,
|
|
|
+ Unit: v.Unit,
|
|
|
+ Frequency: "day",
|
|
|
+ Describe: v.DataItemName,
|
|
|
+ CreateTime: time.Now(),
|
|
|
+ ModifyTime: time.Now(),
|
|
|
+ }
|
|
|
+ id, err := models.AddBaseFromSci99Index(&index)
|
|
|
+ if err != nil {
|
|
|
+ fmt.Println("Error inserting into database:", err)
|
|
|
+ utils.FileLog.Info("Error inserting into database:", err)
|
|
|
+ return
|
|
|
+ }
|
|
|
+ index.BaseFromSciIndexId = int(id)
|
|
|
+ IndexMap[indexName] = &index
|
|
|
+ }
|
|
|
+
|
|
|
+ ppid := strconv.Itoa(v.ProductID)
|
|
|
+ dataTypeID := strconv.Itoa(v.DataTypeID)
|
|
|
+ diidd := strconv.Itoa(v.DIID)
|
|
|
+ deailReq := DetailRequest{
|
|
|
+ PPID: ppid,
|
|
|
+ DataTypeID: dataTypeID,
|
|
|
+ DIIDD: diidd,
|
|
|
+ APIStart: "2021/01/01",
|
|
|
+ APIEnd: "2024/09/11",
|
|
|
+ CycleType: "day",
|
|
|
+ }
|
|
|
+
|
|
|
+ // 将请求体序列化为JSON
|
|
|
+ reqJson, e := json.Marshal(deailReq)
|
|
|
+ if e != nil {
|
|
|
+ err = e
|
|
|
+ fmt.Println("Error marshalling JSON:", err)
|
|
|
+ return
|
|
|
+ }
|
|
|
+
|
|
|
+ // 创建一个HTTP POST请求
|
|
|
+ req, err = http.NewRequest("POST", detailUrl, bytes.NewBuffer(reqJson))
|
|
|
+ if err != nil {
|
|
|
+ fmt.Println("Error creating request:", err)
|
|
|
+ return
|
|
|
+ }
|
|
|
+
|
|
|
+ // 设置请求头信息
|
|
|
+ req.Header.Set("Content-Type", "application/json")
|
|
|
+ req.Header.Set("Accept", "application/json")
|
|
|
+ req.Header.Set("Cookie", Cookie)
|
|
|
+
|
|
|
+ // 发送请求
|
|
|
+ resp, err = client.Do(req)
|
|
|
+ if err != nil {
|
|
|
+ fmt.Println("Error sending request:", err)
|
|
|
+ utils.FileLog.Info("Error sending request:", err)
|
|
|
+ return
|
|
|
+ }
|
|
|
+ defer resp.Body.Close()
|
|
|
+ // 检查响应状态码
|
|
|
+ if resp.StatusCode != http.StatusOK {
|
|
|
+ fmt.Printf("Server returned non-200 status: %d\n", resp.StatusCode)
|
|
|
+ utils.FileLog.Info("Server returned non-200 status: %d", resp.StatusCode)
|
|
|
+ return
|
|
|
+ }
|
|
|
+ // 打印响应体
|
|
|
+ body, _ = ioutil.ReadAll(resp.Body)
|
|
|
+ fmt.Println("Response Detail body:", string(body))
|
|
|
+
|
|
|
+ var detailResponse DetailResponse
|
|
|
+
|
|
|
+ err = json.Unmarshal(body, &detailResponse)
|
|
|
+ if err != nil {
|
|
|
+ fmt.Println(err)
|
|
|
+ utils.FileLog.Info("Error unmarshalling JSON:", err)
|
|
|
+ return
|
|
|
+ }
|
|
|
+
|
|
|
+ // 新增data数据
|
|
|
+ addList := make([]*models.BaseFromSci99Data, 0)
|
|
|
+ for _, v := range detailResponse.Data.List {
|
|
|
+ dataItem := models.BaseFromSci99Data{
|
|
|
+ BaseFromSciIndexId: IndexMap[indexName].BaseFromSciIndexId,
|
|
|
+ IndexCode: indexCode,
|
|
|
+ DataTime: v.RealDate,
|
|
|
+ Value: v.MDataValue,
|
|
|
+ CreateTime: time.Now(),
|
|
|
+ ModifyTime: time.Now(),
|
|
|
+ }
|
|
|
+ addList = append(addList, &dataItem)
|
|
|
+ }
|
|
|
+ if len(addList) > 0 {
|
|
|
+ err = models.AddBaseFromSci99DataMulti(addList)
|
|
|
+ if err != nil {
|
|
|
+ fmt.Println("Error inserting into database:", err)
|
|
|
+ utils.FileLog.Info("Error inserting into database:", err)
|
|
|
+ return
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+}
|
|
|
+
|
|
|
+func Sci99IndexCodeGenerator(indexName, indexCodeStr, marketSampleName string) (indexCode string, needAdd bool) {
|
|
|
+ strResult := ""
|
|
|
+ indexCode, _ = IndexCodeMap[indexName]
|
|
|
+ if indexCode == "" {
|
|
|
+ //首字母
|
|
|
+ a := pinyin.NewArgs()
|
|
|
+ a.Style = pinyin.FirstLetter
|
|
|
+ rows := pinyin.LazyPinyin(indexCodeStr, a)
|
|
|
+ for i := 0; i < len(rows); i++ {
|
|
|
+ strResult += rows[i]
|
|
|
+ }
|
|
|
+ strResult += ProvinceMap[marketSampleName]
|
|
|
+
|
|
|
+ needAdd = true
|
|
|
+ indexCode = strings.Replace(strResult, " ", "", -1)
|
|
|
+ IndexCodeMap[indexName] = indexCode
|
|
|
+ }
|
|
|
+ return
|
|
|
+}
|
|
|
+
|
|
|
+var ProvinceMap = map[string]string{
|
|
|
+ "上海": "shanghai",
|
|
|
+ "云南": "yunnan",
|
|
|
+ "内蒙古": "innermongolia",
|
|
|
+ "北京": "beijing",
|
|
|
+ "台湾": "taiwan",
|
|
|
+ "吉林": "jilin",
|
|
|
+ "四川": "sichuan",
|
|
|
+ "天津": "tianjin",
|
|
|
+ "宁夏": "ningxia",
|
|
|
+ "安徽": "anhui",
|
|
|
+ "山东": "shandong",
|
|
|
+ "山西": "shanxi",
|
|
|
+ "广东": "guangdong",
|
|
|
+ "广西": "guangxi",
|
|
|
+ "新疆": "xinjiang",
|
|
|
+ "江苏": "jiangsu",
|
|
|
+ "江西": "jiangxi",
|
|
|
+ "河北": "hebei",
|
|
|
+ "河南": "henan",
|
|
|
+ "浙江": "zhejiang",
|
|
|
+ "海南": "hainan",
|
|
|
+ "湖北": "hubei",
|
|
|
+ "湖南": "hunan",
|
|
|
+ "澳门": "macao",
|
|
|
+ "甘肃": "gansu",
|
|
|
+ "福建": "fujian",
|
|
|
+ "西藏": "tibet",
|
|
|
+ "贵州": "guizhou",
|
|
|
+ "辽宁": "liaoning",
|
|
|
+ "重庆": "chongqing",
|
|
|
+ "陕西": "shaanxi",
|
|
|
+ "青海": "qinhai",
|
|
|
+ "香港": "hongkong",
|
|
|
+ "黑龙江": "heilongjiang",
|
|
|
+}
|
zwxi