eta_crawler/services/sci99/sci99.go

package sci99

import (
	"bytes"
	"context"
	"encoding/json"
	"eta/eta_crawler/utils"
	"fmt"
	"io/ioutil"
	"net/http"
	"strconv"
	"strings"
	"time"

	"github.com/mozillazg/go-pinyin"
)

var IndexCodeMap = make(map[string]string)

var PriceTypeIDs = []int{34320, 34318}
var requestList = []ListRequestBody{
	{
		Ppname: "天然橡胶原料",
		PPIDs:  "13676",
		Navid:  "593",
	}, {
		Ppname: "混合胶",
		PPIDs:  "12973",
		Navid:  "591",
	}, {
		Ppname: "标准胶",
		PPIDs:  "12960",
		Navid:  "590",
	}, {
		Ppname: "烟片胶",
		PPIDs:  "12985",
		Navid:  "592",
	}, {
		Ppname: "天然乳胶",
		PPIDs:  "12947",
		Navid:  "595",
	}, {
		Ppname: "丁苯橡胶",
		PPIDs:  "12951",
		Navid:  "596",
	}, {
		Ppname: "顺丁橡胶",
		PPIDs:  "12964",
		Navid:  "597",
	}, {
		Ppname: "SBS",
		PPIDs:  "12948",
		Navid:  "598",
	}, {
		Ppname: "丁腈橡胶",
		PPIDs:  "12945",
		Navid:  "605",
	},
}

func Scii99(cont context.Context) (err error) {
	filePath := utils.SCI99_COOKIE_PATH

	// 打开文件
	file, err := ioutil.ReadFile(filePath)
	if err != nil {
		utils.FileLog.Info("ReadFile err:", err)
		return
	}

	cookie := string(file)

	// 定义请求的URL
	listUrl := "https://prices.sci99.com/api/zh-cn/product/datavalue"
	detailUrl := "https://prices.sci99.com/api/zh-cn/dataitem/datavalue"

	for _, request := range requestList {
		classifyName := request.Ppname

		// 分别获取国内价格和国际价格
		for _, priceTypeID := range PriceTypeIDs {
			// 创建请求体实例并填充数据,这里的参数都是一样的，所以这边统一写
			request.CycleType = "day"
			request.SiteType = 1
			request.PageNo = 1
			request.PageSize = "1000"
			request.PriceTypeID = priceTypeID

			// 将请求体序列化为JSON
			jsonData, e := json.Marshal(request)
			if e != nil {
				err = e
				fmt.Println("Error marshalling JSON:", err)
				utils.FileLog.Info("Error marshalling JSON:", err)
				return
			}

			// 创建一个HTTP POST请求
			req, e := http.NewRequest("POST", listUrl, bytes.NewBuffer(jsonData))
			if e != nil {
				err = e
				fmt.Println("Error creating request:", err)
				utils.FileLog.Info("Error creating request:", err)
				return
			}

			// 设置请求头信息
			req.Header.Set("Content-Type", "application/json")
			req.Header.Set("Accept", "application/json")
			req.Header.Set("Cookie", cookie)

			// 发送请求
			client := &http.Client{}
			resp, e := client.Do(req)
			if e != nil {
				err = e
				fmt.Println("Error sending request:", err)
				utils.FileLog.Info("Error sending request:", err)
				return
			}
			defer resp.Body.Close()

			// 检查响应状态码
			if resp.StatusCode != http.StatusOK {
				fmt.Printf("Server returned non-200 status: %d\n", resp.StatusCode)
				utils.FileLog.Info("Server returned non-200 status: %d", resp.StatusCode)
				return
			}
			// 打印响应体
			body, _ := ioutil.ReadAll(resp.Body)
			//fmt.Println("Response body:", string(body))

			var listResponse ListResponse

			err = json.Unmarshal(body, &listResponse)
			if err != nil {
				fmt.Println(err)
				utils.FileLog.Info("Error unmarshalling JSON:", err)
				return
			}
			fmt.Println("获取到的产品数量：", len(listResponse.FirstData.SecondData.Items))
			for _, v := range listResponse.FirstData.SecondData.Items {
				if v.FromArea != "" && len(listResponse.FirstData.SecondData.Items) > 1 {
					continue
				}
				// 组成指标
				var indexName string
				if v.FactorySampleName == "" || len(listResponse.FirstData.SecondData.Items) == 1 {
					indexName = v.ProductName + "：" + v.PriceType + "：" + v.MarketSampleName + "：" + v.Model
				} else {
					indexName = v.ProductName + "：" + v.PriceType + "：" + v.MarketSampleName + "：" + v.FactorySampleName + "：" + v.Model
				}

				ppid := strconv.Itoa(v.ProductID)
				dataTypeID := strconv.Itoa(v.DataTypeID)
				diidd := strconv.Itoa(v.DIID)
				deailReq := DetailRequest{
					PPID:       ppid,
					DataTypeID: dataTypeID,
					DIIDD:      diidd,
					APIStart:   "2021/01/01",
					APIEnd:     time.Now().AddDate(1, 0, 0).Format(utils.FormatDate2),
					CycleType:  "day",
				}

				// 初始化是不加日期获取全部，初始化后只拿前三天的
				if utils.IS_INIT_SCI99 != "true" {
					deailReq.Start = time.Now().AddDate(0, 0, -5).Format(utils.FormatDate2)
					deailReq.End = time.Now().Format(utils.FormatDate2)
				}

				// 将请求体序列化为JSON
				reqJson, e := json.Marshal(deailReq)
				if e != nil {
					err = e
					fmt.Println("Error marshalling JSON:", err)
					utils.FileLog.Info("Error marshalling JSON:", err)
					return
				}

				// 创建一个HTTP POST请求
				req, err = http.NewRequest("POST", detailUrl, bytes.NewBuffer(reqJson))
				if err != nil {
					fmt.Println("Error creating request:", err)
					utils.FileLog.Info("Error creating request:", err)
					return
				}

				// 设置请求头信息
				req.Header.Set("Content-Type", "application/json")
				req.Header.Set("Accept", "application/json")
				req.Header.Set("Cookie", cookie)

				// 发送请求
				resp, err = client.Do(req)
				if err != nil {
					fmt.Println("Error sending request:", err)
					utils.FileLog.Info("Error sending request:", err)
					return
				}
				defer resp.Body.Close()
				// 检查响应状态码
				if resp.StatusCode != http.StatusOK {
					fmt.Printf("Server returned non-200 status: %d\n", resp.StatusCode)
					utils.FileLog.Info("Server returned non-200 status: %d", resp.StatusCode)
					return
				}
				// 打印响应体
				body, _ = ioutil.ReadAll(resp.Body)
				//fmt.Println("Response Detail body:", string(body))

				var detailResponse DetailResponse

				err = json.Unmarshal(body, &detailResponse)
				if err != nil {
					fmt.Println(err)
					utils.FileLog.Info("Error unmarshalling JSON:", err)
					return
				}
				fmt.Println("classifyName：", classifyName)
				fmt.Println("indexName：", indexName)
				fmt.Println("数据长度：", len(detailResponse.Data.List))
				param := make(map[string]interface{})
				param["ListData"] = v
				param["DetailData"] = detailResponse
				param["IndexName"] = indexName
				param["ClassifyName"] = classifyName
				urlStr := `sci99_crawler/refresh/list`
				postUrl := utils.EDB_LIB_URL + urlStr
				postData, e := json.Marshal(param)
				if e != nil {
					err = e
					utils.FileLog.Info("Marshal Err:" + err.Error())
					fmt.Println(err)
					return
				}
				result, e := HttpPost(postUrl, string(postData), "application/json")
				if e != nil {
					err = e
					fmt.Println(err)
					utils.FileLog.Info("HttpPost Err:" + err.Error())
					return
				}
				utils.FileLog.Info("postRefreshEdbData:" + postUrl + ";" + string(postData) + ";result:" + string(result))
				err = json.Unmarshal(result, &resp)
				if err != nil {
					fmt.Println(err)
					utils.FileLog.Info("Unmarshal resp Err:" + err.Error())
					return
				}
				time.Sleep(1 * time.Second)
			}
		}
	}
	return
}

func Sci99IndexCodeGenerator(indexName, indexCodeStr, marketSampleName, model string) (indexCode string, needAdd bool) {
	strResult := ""
	indexCode, _ = IndexCodeMap[indexName]
	if indexCode == "" {
		//首字母
		a := pinyin.NewArgs()
		a.Fallback = func(r rune, a pinyin.Args) []string {
			return []string{string(r)}
		}
		rows := pinyin.Pinyin(indexCodeStr, a)
		for i := 0; i < len(rows); i++ {
			//strResult += rows[i][0]
			if len(rows[i]) != 0 {
				str := rows[i][0]
				pi := str[0:1]
				strResult += pi
			}
		}

		// 处理市场名称
		if province, ok := ProvinceMap[marketSampleName]; ok {
			strResult += province
		} else {
			a := pinyin.NewArgs()
			rows := pinyin.LazyPinyin(marketSampleName, a)
			for i := 0; i < len(rows); i++ {
				strResult += rows[i]
			}
		}

		// 去除特殊符号
		model = strings.Replace(model, " ", "", -1)
		model = strings.Replace(model, "-", "", -1)
		model = strings.Replace(model, "/", "", -1)
		model = strings.Replace(model, "#", "", -1)
		model = strings.Replace(model, "：", "", -1)
		model = strings.Replace(model, "(", "", -1)
		model = strings.Replace(model, ")", "", -1)

		// 拼接型号
		modelRows := pinyin.Pinyin(model, a)
		for i := 0; i < len(modelRows); i++ {
			if len(modelRows[i]) != 0 {
				str := modelRows[i][0]
				pi := str[0:1]
				strResult += pi
			}
		}

		needAdd = true
		indexCode = strings.Replace(strResult, " ", "", -1)
		IndexCodeMap[indexName] = indexCode
	}
	return
}

var ProvinceMap = map[string]string{
	"上海":  "shanghai",
	"云南":  "yunnan",
	"内蒙古": "innermongolia",
	"北京":  "beijing",
	"台湾":  "taiwan",
	"吉林":  "jilin",
	"四川":  "sichuan",
	"天津":  "tianjin",
	"宁夏":  "ningxia",
	"安徽":  "anhui",
	"山东":  "shandong",
	"山西":  "shanxi",
	"广东":  "guangdong",
	"广西":  "guangxi",
	"新疆":  "xinjiang",
	"江苏":  "jiangsu",
	"江西":  "jiangxi",
	"河北":  "hebei",
	"河南":  "henan",
	"浙江":  "zhejiang",
	"海南":  "hainan",
	"湖北":  "hubei",
	"湖南":  "hunan",
	"澳门":  "macao",
	"甘肃":  "gansu",
	"福建":  "fujian",
	"西藏":  "tibet",
	"贵州":  "guizhou",
	"辽宁":  "liaoning",
	"重庆":  "chongqing",
	"陕西":  "shaanxi",
	"青海":  "qinhai",
	"香港":  "hongkong",
	"黑龙江": "heilongjiang",
}

func HttpPost(url, postData string, params ...string) ([]byte, error) {
	body := ioutil.NopCloser(strings.NewReader(postData))
	client := &http.Client{}
	req, err := http.NewRequest("POST", url, body)
	if err != nil {
		return nil, err
	}
	contentType := "application/x-www-form-urlencoded;charset=utf-8"
	if len(params) > 0 && params[0] != "" {
		contentType = params[0]
	}
	req.Header.Set("Content-Type", contentType)
	req.Header.Set("authorization", utils.MD5(utils.APP_EDB_LIB_NAME_EN+utils.EDB_LIB_Md5_KEY))
	resp, err := client.Do(req)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()
	b, err := ioutil.ReadAll(resp.Body)
	fmt.Println("HttpPost:" + string(b))
	return b, err
}