eta_crawler_local/services/commodity_trade_ine.go

package services

import (
	"encoding/json"
	"eta/eta_crawler_local/services/alarm_msg"
	"eta/eta_crawler_local/utils"
	"fmt"
	"io/ioutil"
	"net/http"
	"strings"
	"time"
)

type JSONData struct {
	OCursor    []OCursor   `json:"o_cursor"`
	OCode      interface{} `json:"o_code"`
	OMsg       string      `json:"o_msg"`
	ReportDate string      `json:"report_date"`
	UpdateDate string      `json:"update_date"`
}
type OCursor struct {
	Instrumentid     string      `json:"INSTRUMENTID"`
	Participantid3   string      `json:"PARTICIPANTID3"`
	Participantid2   string      `json:"PARTICIPANTID2"`
	Participantid1   string      `json:"PARTICIPANTID1"`
	Participantabbr3 string      `json:"PARTICIPANTABBR3"`
	Participantabbr2 string      `json:"PARTICIPANTABBR2"`
	Rank             int         `json:"RANK"`
	Participantabbr1 string      `json:"PARTICIPANTABBR1"`
	BuyIn            interface{} `json:"CJ2"`
	Deal             interface{} `json:"CJ1"`
	Change1          interface{} `json:"CJ1_CHG"`
	Change3          interface{} `json:"CJ3_CHG"`
	Productname      string      `json:"Productname"`
	Productsortno    interface{} `json:"PRODUCTSORTNO"`
	SoldOut          interface{} `json:"CJ3"`
	Change2          interface{} `json:"CJ2_CHG"`
}

// SyncRankingFromIne 上海能源交易中心持单排名
func SyncRankingFromIne() {
	var err error
	defer func() {
		if err != nil {
			msg := "失败提醒" + "SyncRankingFromIne ErrMsg:" + err.Error()
			go alarm_msg.SendAlarmMsg(msg, 3)
		}
	}()
	for i := 25; i >= 0; i-- {
		var resp BaseResponse
		var message JSONData

		zzUrl := "https://www.ine.com.cn/data/tradedata/future/dailydata/pm%s.dat"
		date := time.Now().AddDate(0, 0, -i)
		dateStr := date.Format(utils.FormatDateUnSpace)
		zzUrl = fmt.Sprintf(zzUrl, dateStr)
		fmt.Println(zzUrl)
		utils.FileLog.Info("zzUrl:" + zzUrl)

		req, _ := http.NewRequest("GET", zzUrl, nil)
		req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.111 Safari/537.36")
		client := http.Client{}
		res, err := client.Do(req)
		if err != nil {
			fmt.Println(err)
			return
		}
		defer res.Body.Close()

		fmt.Println("resp:", res.Body)
		// 读取响应的内容
		body, err := ioutil.ReadAll(res.Body)
		if err != nil {
			fmt.Println(err)
			return
		}
		err = json.Unmarshal(body, &message)
		if err != nil {
			fmt.Println("Unmarshal Err:", err)
			utils.FileLog.Info("Unmarshal Err:" + err.Error())
			continue
		}
		//fmt.Println("body:",string(body))
		param := make(map[string]interface{})
		param["Date"] = date.Format(utils.FormatDate)
		param["Data"] = message
		urlStr := `exchange_crawler/refresh/ine`
		postUrl := utils.EDB_LIB_URL + urlStr
		postData, err := json.Marshal(param)
		if err != nil {
			utils.FileLog.Info("Marshal Err:" + err.Error())
			fmt.Println(err)
			return
		}
		result, err := HttpPost(postUrl, string(postData), "application/json")
		if err != nil {
			fmt.Println(err)
			utils.FileLog.Info("HttpPost Err:" + err.Error())
			return
		}
		utils.FileLog.Info("postRefreshEdbData:" + postUrl + ";" + string(postData) + ";result:" + string(result))
		err = json.Unmarshal(result, &resp)
		if err != nil {
			fmt.Println(err)
			utils.FileLog.Info("Unmarshal resp Err:" + err.Error())
			return
		}
	}
	utils.FileLog.Info("end")
	fmt.Println("end")
}

func HttpPost(url, postData string, params ...string) ([]byte, error) {
	body := ioutil.NopCloser(strings.NewReader(postData))
	client := &http.Client{}
	req, err := http.NewRequest("POST", url, body)
	if err != nil {
		return nil, err
	}
	contentType := "application/x-www-form-urlencoded;charset=utf-8"
	if len(params) > 0 && params[0] != "" {
		contentType = params[0]
	}
	req.Header.Set("Content-Type", contentType)
	req.Header.Set("authorization", utils.MD5(utils.APP_EDB_LIB_NAME_EN+utils.EDB_LIB_Md5_KEY))
	resp, err := client.Do(req)
	defer resp.Body.Close()
	b, err := ioutil.ReadAll(resp.Body)
	fmt.Println("HttpPost:" + string(b))
	return b, err
}