package services import ( "encoding/xml" "eta/eta_crawler/models" "eta/eta_crawler/utils" "fmt" "github.com/mozillazg/go-pinyin" "io/ioutil" netHttp "net/http" "strings" "time" ) type XmlItem struct { XMLName xml.Name `xml:"positionRank"` Data []struct { ContractCode string `xml:"instrumentid"` Value string `xml:"Value,attr"` Text string `xml:"Text,attr"` Tradingday string `xml:"tradingday"` Datatypeid string `xml:"datatypeid"` Rank int `xml:"rank"` ShortName string `xml:"shortname"` Volume int `xml:"volume"` Varvolume int `xml:"varvolume"` ProductId string `xml:"productid"` } `xml:"data"` } var cffexIndexCode string var cffexIndexCodeMap = make(map[string]string) var cffexActionCodeMap = make(map[string]map[string]int) func cffexIndexCodeGenerator(shortName, indexName, contractCode, suffix string) string { if shortName == "" { cffexIndexCode = "" return cffexIndexCode } //取公司全拼 strResult := "" a := pinyin.NewArgs() rows := pinyin.LazyPinyin(shortName, a) for i := 0; i < len(rows); i++ { strResult += rows[i] } cffexIndexCode, _ := cffexIndexCodeMap[indexName] if cffexIndexCode == "" { cffexIndexCode = strResult + contractCode + suffix cffexIndexCodeMap[indexName] = cffexIndexCode err := models.AddBaseFromTradeMapping(indexName, cffexIndexCode, "CFFEX") if err != nil { fmt.Println("add Code err:", err) } } return cffexIndexCode } // SyncRankingFromCffex 中国金融期货交易所 func SyncRankingFromCffex() { n := utils.GetRandInt(10, 120) time.Sleep(time.Duration(n) * time.Second) allCode, err := models.GetIndexCodeFromMapping("CFFEX") if err != nil { fmt.Println("select Code err:", err) } for _, item := range allCode { cffexIndexCodeMap[item.IndexName] = item.IndexCode } for i := 2; i >= 0; i-- { zzUrl := "http://www.cffex.com.cn/sj/ccpm/%s/%s/" date := time.Now().AddDate(0, 0, -i) dateStr := date.Format(utils.FormatDateUnSpace) pre := dateStr[:6] sub := dateStr[6:] zzUrl = fmt.Sprintf(zzUrl, pre, sub) fmt.Println(zzUrl) var xmlItems = make([]*XmlItem, 8) xmlItems[0] = GetXmlItem(zzUrl, "IF") xmlItems[1] = GetXmlItem(zzUrl, "IC") xmlItems[2] = GetXmlItem(zzUrl, "IH") xmlItems[3] = GetXmlItem(zzUrl, "TS") xmlItems[4] = GetXmlItem(zzUrl, "TF") xmlItems[5] = GetXmlItem(zzUrl, "T") xmlItems[6] = GetXmlItem(zzUrl, "IM") xmlItems[7] = GetXmlItem(zzUrl, "TL") //获取新的指标信息 var sRank string //获取所有指标信息 allIndex, err := models.GetBaseFromTradeCffexIndexAll(dateStr) if err != nil { fmt.Println("select err:", err) } existDealMap := make(map[string]int) existBuyMap := make(map[string]int) existSoldMap := make(map[string]int) for _, v := range allIndex { existDealMap[v.DealName] = v.BaseFromTradeCffexIndexId existBuyMap[v.BuyName] = v.BaseFromTradeCffexIndexId existSoldMap[v.SoldName] = v.BaseFromTradeCffexIndexId sRank = fmt.Sprintf("%d", v.Rank) time := strings.Replace(v.DataTime, "-", "", -1) cffexActionCodeMap[sRank+v.ClassifyType+time] = map[string]int{"0": v.DealValue, "1": v.BuyValue, "2": v.SoldValue, "id": v.BaseFromTradeCffexIndexId} } shortNameColum := map[string]string{"0": "deal_short_name", "1": "buy_short_name", "2": "sold_short_name"} nameColum := map[string]string{"0": "deal_name", "1": "buy_name", "2": "sold_name"} codeColum := map[string]string{"0": "deal_code", "1": "buy_code", "2": "sold_code"} valueColum := map[string]string{"0": "deal_value", "1": "buy_value", "2": "sold_value"} changeColum := map[string]string{"0": "deal_change", "1": "buy_change", "2": "sold_change"} var dataName string var dataCode string var indexKey string for _, xmlItem := range xmlItems { for _, i := range xmlItem.Data { i.ShortName = strings.Replace(i.ShortName,"(经纪)","",-1) i.ShortName = strings.Replace(i.ShortName,"(代客)","",-1) var item = new(models.BaseFromTradeCffexIndex) item.DealValue = -1 item.BuyValue = -1 item.SoldValue = -1 item.Rank = i.Rank switch i.Value { case "0": item.DealShortName = i.ShortName item.DealName = fmt.Sprintf("%s", i.ShortName+"_"+i.ContractCode+"_成交量(手)") item.DealCode = cffexIndexCodeGenerator(item.DealShortName, item.DealName, i.ContractCode, "deal") item.DealValue = i.Volume item.DealChange = i.Varvolume dataName = item.DealName dataCode = item.DealCode case "1": item.BuyShortName = i.ShortName item.BuyName = fmt.Sprintf("%s", i.ShortName+"_"+i.ContractCode+"_持买单量(手)") item.BuyCode = cffexIndexCodeGenerator(item.BuyShortName, item.BuyName, i.ContractCode, "buy") item.BuyValue = i.Volume item.BuyChange = i.Varvolume dataName = item.BuyName dataCode = item.BuyCode case "2": item.SoldShortName = i.ShortName item.SoldName = fmt.Sprintf("%s", i.ShortName+"_"+i.ContractCode+"_持卖单量(手)") item.SoldCode = cffexIndexCodeGenerator(item.SoldShortName, item.SoldName, i.ContractCode, "sold") item.SoldValue = i.Volume item.SoldChange = i.Varvolume dataName = item.SoldName dataCode = item.SoldCode default: fmt.Println("No data value") } item.Frequency = "日度" item.ClassifyName = i.ProductId item.ClassifyType = i.ContractCode item.CreateTime = time.Now() item.ModifyTime = time.Now() item.DataTime = i.Tradingday valueMap := map[string]int{"0": item.DealValue, "1": item.BuyValue, "2": item.SoldValue} //检查在actionCodeMap中是否已经有了 indexKey = fmt.Sprintf("%d", item.Rank) + item.ClassifyType + item.DataTime if existMap, ok := cffexActionCodeMap[indexKey]; !ok { //没有,新增 newID, err := models.AddBaseFromTradeCffexIndex(item) if err != nil { fmt.Println("insert error:", err) } fmt.Println("insert new indexID:", newID) existMap = make(map[string]int) existMap[i.Value] = i.Volume existMap["id"] = int(newID) cffexActionCodeMap[indexKey] = existMap } else if existMap != nil && existMap[i.Value] != i.Volume { //更新 var columnList = [5]string{ shortNameColum[i.Value], nameColum[i.Value], codeColum[i.Value], valueColum[i.Value], changeColum[i.Value], } var dataList = [5]interface{}{ i.ShortName, dataName, dataCode, valueMap[i.Value], i.Varvolume, } err := models.ModifyBaseFromTradeCffexIndex(columnList, dataList, existMap["id"]) if err != nil { fmt.Println("data update err:", err) } cffexActionCodeMap[indexKey][i.Value] = i.Volume } else if _, ok := existDealMap[item.DealName]; !ok { //更新 var columnList = [5]string{ shortNameColum[i.Value], nameColum[i.Value], codeColum[i.Value], valueColum[i.Value], changeColum[i.Value], } var dataList = [5]interface{}{ i.ShortName, dataName, dataCode, valueMap[i.Value], i.Varvolume, } err := models.ModifyBaseFromTradeCffexIndex(columnList, dataList, existMap["id"]) if err != nil { fmt.Println("data update err:", err) } cffexActionCodeMap[indexKey][i.Value] = i.Volume } else if _, ok := existBuyMap[item.BuyName]; !ok { //更新 var columnList = [5]string{ shortNameColum[i.Value], nameColum[i.Value], codeColum[i.Value], valueColum[i.Value], changeColum[i.Value], } var dataList = [5]interface{}{ i.ShortName, dataName, dataCode, valueMap[i.Value], i.Varvolume, } err := models.ModifyBaseFromTradeCffexIndex(columnList, dataList, existMap["id"]) if err != nil { fmt.Println("data update err:", err) } cffexActionCodeMap[indexKey][i.Value] = i.Volume } else if _, ok := existSoldMap[item.SoldName]; !ok { //更新 var columnList = [5]string{ shortNameColum[i.Value], nameColum[i.Value], codeColum[i.Value], valueColum[i.Value], changeColum[i.Value], } var dataList = [5]interface{}{ i.ShortName, dataName, dataCode, valueMap[i.Value], i.Varvolume, } err := models.ModifyBaseFromTradeCffexIndex(columnList, dataList, existMap["id"]) if err != nil { fmt.Println("data update err:", err) } cffexActionCodeMap[indexKey][i.Value] = i.Volume } } } } } func GetXmlItem(url, contract string) *XmlItem { zzUrl := url + contract + ".xml" fmt.Println(zzUrl) //body, err := http.Get(zzUrl) //if err != nil { // fmt.Println("err:", err) //} req, _ := netHttp.NewRequest("GET", zzUrl, nil) req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.111 Safari/537.36") client := netHttp.Client{} res, err := client.Do(req) if err != nil { fmt.Println(err) return nil } defer res.Body.Close() // 读取响应的内容 body, err := ioutil.ReadAll(res.Body) if err != nil { fmt.Println(err) return nil } xmlItem := new(XmlItem) err = xml.Unmarshal(body, &xmlItem) if err != nil { fmt.Println("xml.Unmarshal:", err.Error()) } return xmlItem }