Merge remote-tracking branch 'origin/debug' into debug

services/smm_shanghai.go

@@ -0,0 +1,550 @@
+package services
+
+import (
+	"context"
+	"encoding/json"
+	"eta/eta_crawler/models"
+	"eta/eta_crawler/services/sci99"
+	"eta/eta_crawler/utils"
+	"fmt"
+	"io"
+	"net/http"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/xuri/excelize/v2"
+)
+
+type SmmRequest struct {
+	Name      string
+	IndexCode string
+	Url       string
+	Frequency string
+	Unit      string
+	Value     string
+}
+
+var nickelReqs = []SmmRequest{
+	{
+		Name:      "SMM 1#电解镍",
+		IndexCode: "",
+		Url:       "https://hq.smm.cn/ajax/spot/history/201102250239",
+		Frequency: "日度",
+		Unit:      "元/吨",
+	},
+	{
+		Name:      "1#金川镍",
+		Url:       "https://hq.smm.cn/ajax/spot/history/201102250174",
+		Frequency: "日度",
+		Unit:      "元/吨",
+	},
+	{
+		Name:      "1#进口镍",
+		Url:       "https://hq.smm.cn/ajax/spot/history/201102250423",
+		Frequency: "日度",
+		Unit:      "元/吨",
+	},
+	{
+		Name:      "镍豆",
+		Url:       "https://hq.smm.cn/ajax/spot/history/202008270001",
+		Frequency: "日度",
+		Unit:      "元/吨",
+	},
+	{
+		Name:      `印尼内贸红土镍矿1.2%(到厂价)`,
+		Url:       "https://hq.smm.cn/ajax/spot/history/202311230010",
+		Frequency: "周度",
+		Unit:      "美元/湿吨",
+	},
+	{
+		Name:      `印尼内贸红土镍矿1.6%(到厂价)`,
+		Url:       "https://hq.smm.cn/ajax/spot/history/202311230011",
+		Frequency: "周度",
+		Unit:      "美元/湿吨",
+	},
+	{
+		Name:      `菲律宾红土镍矿0.9%,Al≥7%(CIF)`,
+		Url:       "https://hq.smm.cn/ajax/spot/history/201608170001",
+		Frequency: "日度",
+		Unit:      "美元/湿吨",
+	},
+	{
+		Name:      `菲律宾红土镍矿0.9%,Al<7%(CIF) `, // 重名
+		Url:       "https://hq.smm.cn/ajax/spot/history/202109140006",
+		Frequency: "日度",
+		Unit:      "美元/湿吨",
+	},
+
+	{
+		Name:      `菲律宾红土镍矿1.3%(CIF)`,
+		Url:       "https://hq.smm.cn/ajax/spot/history/202109140001",
+		Frequency: "日度",
+		Unit:      "美元/湿吨",
+	},
+	{
+		Name:      `菲律宾红土镍矿1.4%(CIF)`,
+		Url:       "https://hq.smm.cn/ajax/spot/history/202109140003",
+		Frequency: "日度",
+		Unit:      "美元/湿吨",
+	},
+	{
+		Name:      `菲律宾红土镍矿1.5%(CIF)`,
+		Url:       "https://hq.smm.cn/ajax/spot/history/201608170002",
+		Frequency: "日度",
+		Unit:      "美元/湿吨",
+	},
+	{
+		Name:      `红土镍矿1.8%(CIF)`,
+		Url:       "https://hq.smm.cn/ajax/spot/history/201608170003",
+		Frequency: "日度",
+		Unit:      "美元/湿吨",
+	},
+
+	{
+		Name:      `菲律宾红土镍矿0.9%,Al≥7%(FOB)`,
+		Url:       "https://hq.smm.cn/ajax/spot/history/201509220001",
+		Frequency: "日度",
+		Unit:      "美元/湿吨",
+	},
+	{
+		Name:      `菲律宾红土镍矿0.9%,Al<7%(FOB)`, //重名
+		Url:       "https://hq.smm.cn/ajax/spot/history/202109140005",
+		Frequency: "日度",
+		Unit:      "美元/湿吨",
+	},
+	{
+		Name:      `菲律宾红土镍矿1.3%(FOB)`,
+		Url:       "https://hq.smm.cn/ajax/spot/history/202109140002",
+		Frequency: "日度",
+		Unit:      "美元/湿吨",
+	},
+	{
+		Name:      `菲律宾红土镍矿1.4%(FOB)`,
+		Url:       "https://hq.smm.cn/ajax/spot/history/202109140004",
+		Frequency: "日度",
+		Unit:      "美元/湿吨",
+	},
+	{
+		Name:      `菲律宾红土镍矿1.5%(FOB)`,
+		Url:       "https://hq.smm.cn/ajax/spot/history/201509220002",
+		Frequency: "日度",
+		Unit:      "美元/湿吨",
+	},
+	{
+		Name:      `红土镍矿1.8%(FOB)`,
+		Url:       "https://hq.smm.cn/ajax/spot/history/201509220003",
+		Frequency: "日度",
+		Unit:      "美元/湿吨",
+	},
+	{
+		Name:      `8-12%高镍生铁(出厂价)`,
+		Url:       "https://hq.smm.cn/ajax/spot/history/201106150005",
+		Frequency: "日度",
+		Unit:      "元/镍点",
+	},
+	{
+		Name:      `1.5-1.7%镍生铁(出厂价)`,
+		Url:       "https://hq.smm.cn/ajax/spot/history/201106150007",
+		Frequency: "日度",
+		Unit:      "元/吨",
+	},
+}
+
+var copperReqs = []SmmRequest{
+	{
+		Name:      "SMM 1#电解铜",
+		Url:       "https://hq.smm.cn/ajax/spot/history/201102250376",
+		Unit:      "元/吨",
+		Frequency: "日度",
+	},
+	{
+		Name:      "SMM 1#电解铜升贴水最大值",
+		Url:       "https://hq.smm.cn/ajax/premium/history/201102250185",
+		Value:     "highs",
+		Frequency: "日度",
+	},
+	{
+		Name:      "SMM 1#电解铜升贴水最小值",
+		Url:       "https://hq.smm.cn/ajax/premium/history/201102250185",
+		Value:     "low",
+		Frequency: "日度",
+	},
+	{
+		Name:      "SMM 广东1#电解铜",
+		Url:       "https://hq.smm.cn/ajax/spot/history/201912300001",
+		Unit:      "元/吨",
+		Frequency: "日度",
+	},
+	{
+		Name:      "SMM 广东1#电解铜升贴水最大值",
+		Url:       "https://hq.smm.cn/ajax/premium/history/201912300002",
+		Value:     "highs",
+		Frequency: "日度",
+	},
+	{
+		Name:      "SMM 广东1#电解铜升贴水最小值",
+		Url:       "https://hq.smm.cn/ajax/premium/history/201912300002",
+		Value:     "low",
+		Frequency: "日度",
+	},
+	{
+		Name:      "进口铜精矿指数(周)",
+		Url:       "https://hq.smm.cn/ajax/spot/history/201910240001",
+		Unit:      "美元/吨",
+		Frequency: "周度",
+	},
+	{
+		Name:      "废铜:广东:1#光亮铜线",
+		Url:       "https://hq.smm.cn/ajax/spot/history/201108090036",
+		Unit:      "元/吨",
+		Frequency: "日度",
+	},
+}
+
+var zincReqs = []SmmRequest{
+	{
+		Name:      "SMM 0#锌锭",
+		Url:       "https://hq.smm.cn/ajax/spot/history/201102250173",
+		Unit:      "元/吨",
+		Frequency: "日度",
+	},
+	{
+		Name:      "SMM 1#锌锭",
+		Url:       "https://hq.smm.cn/ajax/spot/history/201102250418",
+		Unit:      "元/吨",
+		Frequency: "日度",
+	},
+	// {
+	// 	Name:      "SMM 0#锌锭溢价(广东)",
+	// 	Url:       "https://hq.smm.cn/ajax/spot/history/202010210003",
+	// 	Unit:      "元/吨",
+	// 	Frequency: "日度",
+	// },
+	{
+		Name:      "SMM 0#锌锭(广东)",
+		Url:       "https://hq.smm.cn/ajax/spot/history/201102250231",
+		Unit:      "元/吨",
+		Frequency: "日度",
+	},
+	// {
+	// 	Name:      "SMM 1#锌锭溢价(广东)",
+	// 	Url:       "https://hq.smm.cn/ajax/spot/history/202010210004",
+	// 	Unit:      "元/吨",
+	// 	Frequency: "日度",
+	// },
+	{
+		Name:      "SMM 1#锌锭(广东)",
+		Url:       "https://hq.smm.cn/ajax/spot/history/201102250069",
+		Unit:      "元/吨",
+		Frequency: "日度",
+	},
+	// {
+	// 	Name:      "SMM 0#锌锭溢价(天津)",
+	// 	Url:       "https://hq.smm.cn/ajax/spot/history/202010210001",
+	// 	Unit:      "元/吨",
+	// 	Frequency: "日度",
+	// },
+	{
+		Name:      "SMM 1#锌锭(天津)",
+		Url:       "https://hq.smm.cn/ajax/spot/history/201102250556",
+		Unit:      "元/吨",
+		Frequency: "日度",
+	},
+	{
+		Name:      "SMM 0#锌锭(天津)",
+		Url:       "https://hq.smm.cn/ajax/spot/history/201102250399",
+		Unit:      "元/吨",
+		Frequency: "日度",
+	},
+	// {
+	// 	Name:      "SMM 1#锌锭溢价(天津)",
+	// 	Url:       "https://hq.smm.cn/ajax/spot/history/202010210002",
+	// 	Unit:      "元/吨",
+	// 	Frequency: "日度",
+	// },
+	// {
+	// 	Name:      "SMM 0#锌锭溢价(宁波)",
+	// 	Url:       "https://hq.smm.cn/ajax/spot/history/202010210005",
+	// 	Unit:      "元/吨",
+	// 	Frequency: "日度",
+	// },
+	// {
+	// 	Name:      "SMM 0#锌锭(宁波)",
+	// 	Url:       "https://hq.smm.cn/ajax/spot/history/202004070006",
+	// 	Unit:      "元/吨",
+	// 	Frequency: "日度",
+	// },
+	{
+		Name:      "Zn50国产TC(月)",
+		Url:       "https://hq.smm.cn/ajax/spot/history/201312030008",
+		Unit:      "元/金属吨",
+		Frequency: "月度",
+	},
+	{
+		Name:      "Zn50进口TC(月)",
+		Url:       "https://hq.smm.cn/ajax/spot/history/201312030009",
+		Unit:      "美元/千吨",
+		Frequency: "月度",
+	},
+	{
+		Name:      "Zn50国产TC(周)",
+		Url:       "https://hq.smm.cn/ajax/spot/history/202004070002",
+		Unit:      "元/金属吨",
+		Frequency: "周度",
+	},
+	{
+		Name:      "Zn50进口TC(周)",
+		Url:       "https://hq.smm.cn/ajax/spot/history/202004070001",
+		Unit:      "美元/千吨",
+		Frequency: "周度",
+	},
+	{
+		Name:      "Zn50内蒙古国产TC(周)",
+		Url:       "https://hq.smm.cn/ajax/spot/history/201504080006",
+		Unit:      "元/金属吨",
+		Frequency: "周度",
+	},
+	{
+		Name:      "Zn50陕西国产TC(周)",
+		Url:       "https://hq.smm.cn/ajax/spot/history/201504080007",
+		Unit:      "元/金属吨",
+		Frequency: "周度",
+	},
+	{
+		Name:      "Zn50甘肃国产TC(周)",
+		Url:       "https://hq.smm.cn/ajax/spot/history/202108050001",
+		Unit:      "元/金属吨",
+		Frequency: "周度",
+	},
+	{
+		Name:      "Zn50四川国产TC(周)",
+		Url:       "https://hq.smm.cn/ajax/spot/history/201504080008",
+		Unit:      "元/金属吨",
+		Frequency: "周度",
+	},
+	{
+		Name:      "Zn50广西国产TC(周)",
+		Url:       "https://hq.smm.cn/ajax/spot/history/201504080009",
+		Unit:      "元/金属吨",
+		Frequency: "周度",
+	},
+	{
+		Name:      "Zn50云南国产TC(周)",
+		Url:       "https://hq.smm.cn/ajax/spot/history/201504080010",
+		Unit:      "元/金属吨",
+		Frequency: "周度",
+	},
+	{
+		Name:      "Zn50湖南国产TC(周)",
+		Url:       "https://hq.smm.cn/ajax/spot/history/201504080011",
+		Unit:      "元/金属吨",
+		Frequency: "周度",
+	},
+}
+
+type SmmResponse struct {
+	Code int
+	Data []*SmmData
+	Msg  string
+}
+
+type SmmData struct {
+	Highs           float64 `json:"highs"`
+	Low             float64 `json:"low"`
+	Average         float64 `json:"average"`
+	VchangeRate     float64 `json:"vchange_rate"`
+	LowShow         string  `json:"low_show"`
+	HighShow        string  `json:"high_show"`
+	AverageShow     string  `json:"average_show"`
+	ProductId       string  `json:"product_id"`
+	Vchange         float64 `json:"vchange"`
+	RenewDate       string  `json:"renew_date"`
+	ChangeValueShow string  `json:"change_value_show"`
+	ChangeRateShow  string  `json:"change_rate_show"`
+}
+
+type EdbInfoData struct {
+	ClassifyName string
+	IndexName    string
+	IndexCode    string
+	Frequency    string
+	Unit         string
+	Value        string
+	LastDate     time.Time
+	OldDate      time.Time
+	Data         map[string]float64
+	SmmData
+}
+
+func SyncShangHaiSmm(cont context.Context) (err error) {
+	upMonth := time.Now().AddDate(0, -1, 0).Format(utils.FormatDate)
+	curDate := time.Now().Format(utils.FormatDate)
+	reqList := make([]SmmRequest, 0)
+	reqList = append(reqList, zincReqs...)
+	reqList = append(reqList, copperReqs...)
+	reqList = append(reqList, nickelReqs...)
+
+	edbInfoList := make([]*EdbInfoData, 0)
+	for _, v := range reqList {
+		url := fmt.Sprintf("%s/%s/%s", v.Url, upMonth, curDate)
+		req, e := http.NewRequest("GET", url, nil)
+		if e != nil {
+			fmt.Println(e)
+			return
+		}
+		// 发送请求
+		client := &http.Client{}
+		resp, e := client.Do(req)
+		if e != nil {
+			fmt.Println(e)
+			return
+		}
+		var respData SmmResponse
+		body, er := io.ReadAll(resp.Body)
+		if er != nil {
+			fmt.Println(er)
+			return er
+		}
+		if err = json.Unmarshal(body, &respData); err != nil {
+			fmt.Println(err)
+			return
+		}
+		if respData.Code != 0 {
+			fmt.Printf("上海有色爬虫请求失败, 指标名称:%s, 指标地址:%s", v.Name, v.Url)
+			continue
+		}
+		data := respData.Data
+		edbInfoList = append(edbInfoList, &EdbInfoData{
+			IndexName: v.Name,
+			Frequency: v.Frequency,
+			Unit:      v.Unit,
+			Value:     v.Value,
+			SmmData:   *data[len(data)-1],
+		})
+	}
+
+	urlStr := `shanghai_smm/refresh/list`
+	postUrl := utils.EDB_LIB_URL + urlStr
+	postData, err := json.Marshal(edbInfoList)
+	if err != nil {
+		utils.FileLog.Info("Marshal Err:" + err.Error())
+		return
+	}
+	result, err := sci99.HttpPost(postUrl, string(postData), "application/json")
+	if err != nil {
+		utils.FileLog.Info("HttpPost Err:" + err.Error())
+		return
+	}
+	resp := new(models.BaseResponse)
+	err = json.Unmarshal(result, &resp)
+	if err != nil {
+		fmt.Println(err)
+		utils.FileLog.Info("Unmarshal resp Err:" + err.Error())
+		return
+	}
+	if resp.Ret != 200 {
+		fmt.Println("上海有色爬虫更新失败")
+		return
+	}
+	return
+}
+
+// 解析读取历史excel数据,将历史数据转化为Smm结构体
+func ExcelToSmm(f *excelize.File, s string) (edbInfoList []*EdbInfoData) {
+	cols, err := f.GetCols(s)
+	if err != nil {
+		fmt.Println(err)
+		return
+	}
+	date := cols[0][4:]
+	dateFormat := make([]string, 0)
+	for _, v := range date {
+		if v == "" {
+			continue
+		}
+		var parseDate time.Time
+		var err error
+		if strings.Contains(s, "锌") {
+			parseDate, err = time.Parse(utils.FormatDate, v)
+		} else {
+			parseDate, err = time.Parse("01-02-06", v)
+		}
+		if err != nil {
+			fmt.Println(err)
+			return
+		}
+		dateFormat = append(dateFormat, parseDate.Format(utils.FormatDate))
+	}
+
+	for i := 1; i < len(cols); i++ {
+		tmpEdbInfo := new(EdbInfoData)
+		tmpEdbInfo.IndexName = cols[i][0]
+		tmpEdbInfo.IndexCode = cols[i][1]
+		tmpEdbInfo.Frequency = cols[i][2]
+		tmpEdbInfo.Unit = cols[i][3]
+		tmpEdbInfo.Data = make(map[string]float64)
+		tmpEdbInfo.OldDate = time.Now()
+		for j := 4; j < len(cols[i]); j++ {
+			floatVal, err := strconv.ParseFloat(cols[i][j], 64)
+			if err != nil {
+				fmt.Println("无效字符", cols[i][j])
+				continue
+			}
+			tmpEdbInfo.Data[dateFormat[j-4]] = floatVal
+			curTime, _ := time.Parse(utils.FormatDate, dateFormat[j-4])
+			if curTime.After(tmpEdbInfo.LastDate) {
+				tmpEdbInfo.LastDate = curTime
+			}
+			if curTime.Before(tmpEdbInfo.OldDate) {
+				tmpEdbInfo.OldDate = curTime
+			}
+		}
+		edbInfoList = append(edbInfoList, tmpEdbInfo)
+	}
+	return
+}
+
+func OldExcel() {
+	excelPath := ""
+	f, err := excelize.OpenFile(excelPath)
+	if err != nil {
+		fmt.Print(err)
+		return
+	}
+	var edbClassify = []string{`镍`, `铜`, `锌`, `锌(日度)`}
+	for _, v := range edbClassify {
+		edbInfoList := ExcelToSmm(f, v)
+		_, err := json.Marshal(edbInfoList)
+		if err != nil {
+			fmt.Println(err)
+			return
+		}
+		fmt.Println("成功:", v)
+		urlStr := `shanghai_smm/refresh/excel`
+		postUrl := utils.EDB_LIB_URL + urlStr
+		postData, err := json.Marshal(edbInfoList)
+		if err != nil {
+			utils.FileLog.Info("Marshal Err:" + err.Error())
+			return
+		}
+		result, err := sci99.HttpPost(postUrl, string(postData), "application/json")
+		if err != nil {
+			utils.FileLog.Info("HttpPost Err:" + err.Error())
+			return
+		}
+		resp := new(models.BaseResponse)
+		err = json.Unmarshal(result, &resp)
+		if err != nil {
+			fmt.Println(err)
+			utils.FileLog.Info("Unmarshal resp Err:" + err.Error())
+			return
+		}
+		if resp.Ret != 200 {
+			fmt.Println("上海有色网excel历史有色数据更新失败")
+			utils.FileLog.Info("上海有色网excel历史有色数据更新失败")
+			return
+		}
+		time.Sleep(10 * time.Second)
+	}
+}

services/task.go

@@ -5,9 +5,10 @@ import (
 	"eta/eta_crawler/services/sci99"
 	"eta/eta_crawler/utils"
 	"fmt"
-	"github.com/beego/beego/v2/task"
 	"strconv"
 	"time"
+
+	"github.com/beego/beego/v2/task"
 )
 
 func Task() {
@@ -59,6 +60,9 @@ func Task() {
 		task.AddTask("卓创资讯", sci99)
 	}
 
+	crawlerSmm := task.NewTask("refreshShangHaiSmmData", "0 0 10-18 * * *", SyncShangHaiSmm) // 上海有色网爬虫
+
+	task.AddTask("上海有色网指标爬取", crawlerSmm) //每天10-18点, 每小时爬一次
 	//task.AddTask("统计局数据爬取-月度A", refreshNationalMonthA) // 每月10号2:15执行
 	//task.AddTask("统计局数据爬取-月度B", refreshNationalMonthB) // 每月16号2:15执行
 	//task.AddTask("统计局数据爬取-季度", refreshNationalQuarter) // 每月15号1:25执行