package services import ( "encoding/json" "eta/eta_crawler/models" "eta/eta_crawler/services/alarm_msg" "eta/eta_crawler/utils" "fmt" "github.com/mozillazg/go-pinyin" "github.com/rdlucklib/rdluck_tools/http" "strings" "time" ) type Position []struct { ContractCode string `json:"INSTRUMENTID"` ProductSortNo int `json:"PRODUCTSORTNO"` Rank int `json:"RANK"` ParticipantID1 string `json:"PARTICIPANTID1"` ParticipantName1 string `json:"PARTICIPANTABBR1"` Deal interface{} `json:"CJ1"` Change1 interface{} `json:"CJ1_CHG"` ParticipantID2 string `json:"PARTICIPANTID2"` ParticipantName2 string `json:"PARTICIPANTABBR2"` BuyIn interface{} `json:"CJ2"` Change2 interface{} `json:"CJ2_CHG"` ParticipantID3 string `json:"PARTICIPANTID3"` ParticipantName3 string `json:"PARTICIPANTABBR3"` SoldOut interface{} `json:"CJ3"` Change3 interface{} `json:"CJ3_CHG"` ProductName string `json:"PRODUCTNAME"` } type Message struct { Position Position `json:"o_cursor"` Length string `json:"showlength"` Code int `json:"o_code"` Msg string `json:"o_msg"` ReportDate string `json:"report_date"` UpdateDate string `json:"update_date"` PrintDate string `json:"print_date"` } var indexCode string var indexCodeMap = make(map[string]string) func shIndexCodeGenerator(shortName, indexName, contractCode, suffix string) string { if shortName == "" { return "" } strResult := "" if shortName != "top20" { //取公司全拼 a := pinyin.NewArgs() rows := pinyin.LazyPinyin(shortName, a) for i := 0; i < len(rows); i++ { strResult += rows[i] } } else { strResult = "top20" } indexCode, _ := indexCodeMap[indexName] if indexCode == "" { indexCode = strResult + contractCode + suffix indexCode = strings.Replace(indexCode, " ", "", -1) indexCodeMap[indexName] = indexCode err := models.AddBaseFromTradeMapping(indexName, indexCode, "SH") if err != nil { fmt.Println("add Code err:", err) } } return strings.Replace(indexCode, " ", "", -1) } // SyncRankingFromShangHai 上海商品交易所持单排名 func SyncRankingFromShangHai() { var err error defer func() { if err != nil { msg := "失败提醒" + "SyncRankingFromShangHai ErrMsg:" + err.Error() fmt.Println("msg:",msg) go alarm_msg.SendAlarmMsg(msg, 3) } }() n := utils.GetRandInt(10, 120) time.Sleep(time.Duration(n) * time.Second) allCode, err := models.GetIndexCodeFromMapping("SH") if err != nil { fmt.Println("select Code err:", err) return } for _, item := range allCode { indexCodeMap[item.IndexName] = item.IndexCode } //获取新的指标信息 for i := 10; i >= 0; i-- { var message Message zzUrl := "http://www.shfe.com.cn/data/dailydata/kx/pm%s.dat" date := time.Now().AddDate(0, 0, -i) dateStr := date.Format(utils.FormatDateUnSpace) zzUrl = fmt.Sprintf(zzUrl, dateStr) fmt.Println(zzUrl) body, e := http.Get(zzUrl) if e != nil { err = e fmt.Println("err:", err) return } err = json.Unmarshal(body, &message) if err != nil { fmt.Println("Unmarshal Err:", err) continue } var position = message.Position var tradeDate = message.ReportDate //获取所有指标信息 allIndex, e := models.GetBaseFromTradeShangHaiIndexAll(dateStr) if e != nil { err = e return } existIndexMap := make(map[string]*models.BaseFromTradeShanghaiIndex) for _, v := range allIndex { indexKey := v.DealName + v.BuyName + v.SoldName existIndexMap[indexKey] = v } var itemVerifyCode int //处理指标 for _, p := range position { var item = new(models.BaseFromTradeShanghaiIndex) if p.Rank > 0 && p.Rank < 40 && p.ParticipantName1 != "" { if strings.Replace(p.ProductName, " ", "", -1) != "20号胶" && strings.Replace(p.ProductName, " ", "", -1) != "低硫燃料油" { contractCode := strings.Replace(p.ContractCode, " ", "", -1) //成交量 item.Rank = p.Rank item.DealShortName = strings.Replace(p.ParticipantName1, " ", "", -1) item.BuyShortName = strings.Replace(p.ParticipantName2, " ", "", -1) item.SoldShortName = strings.Replace(p.ParticipantName3, " ", "", -1) item.DealName = strings.Replace(fmt.Sprintf("%s", p.ParticipantName1+"_"+p.ContractCode+"_成交量(手)"), " ", "", -1) item.BuyName = strings.Replace(fmt.Sprintf("%s", p.ParticipantName2+"_"+p.ContractCode+"_持买单量(手)"), " ", "", -1) item.SoldName = strings.Replace(fmt.Sprintf("%s", p.ParticipantName3+"_"+p.ContractCode+"_持卖单量(手)"), " ", "", -1) item.DealCode = shIndexCodeGenerator(item.DealShortName, item.DealName, contractCode, "deal") item.BuyCode = shIndexCodeGenerator(item.BuyShortName, item.BuyName, contractCode, "buy") item.SoldCode = shIndexCodeGenerator(item.SoldShortName, item.SoldName, contractCode, "sold") item.ClassifyName = strings.Replace(p.ProductName, " ", "", -1) item.ClassifyType = strings.Replace(p.ContractCode, " ", "", -1) item.Frequency = "日度" item.CreateTime = time.Now() item.ModifyTime = time.Now() item.DataTime = tradeDate if deal, ok := p.Deal.(float64); ok{ item.DealValue = int(deal) } if change1, ok := p.Change1.(float64); ok{ item.DealChange = int(change1) } if buyIn, ok := p.BuyIn.(float64); ok{ item.BuyValue = int(buyIn) } if change2, ok := p.Change2.(float64); ok{ item.BuyChange = int(change2) } if soldOut, ok := p.SoldOut.(float64); ok{ item.SoldValue = int(soldOut) } if change3, ok := p.Change3.(float64); ok{ item.SoldChange = int(change3) } itemVerifyCode = item.BuyValue + item.DealValue + item.SoldValue if existIndex, ok := existIndexMap[item.DealName+item.BuyName+item.SoldName]; !ok { newID, e := models.AddBaseFromTradeShangHaiIndex(item) if e != nil { err = e fmt.Println("insert error:", err) } fmt.Println("insert new indexID:", newID) } else if existIndex != nil && itemVerifyCode != (existIndex.DealValue+existIndex.BuyValue+existIndex.SoldValue) { //更新 err = models.ModifyBaseFromTradeShangHaiIndex(item.DealValue, item.BuyValue, item.SoldValue, existIndex.BaseFromTradeShangHaiIndexId) if err != nil { fmt.Println("data update err:", err) } } } } else if p.Rank == 999 { if strings.Replace(p.ProductName, " ", "", -1) != "20号胶" && strings.Replace(p.ProductName, " ", "", -1) != "低硫燃料油" { contractCode := strings.Replace(p.ContractCode, " ", "", -1) //Top 20 item.Rank = p.Rank item.DealShortName = strings.Replace(p.ParticipantName1, " ", "", -1) item.BuyShortName = strings.Replace(p.ParticipantName2, " ", "", -1) item.SoldShortName = strings.Replace(p.ParticipantName3, " ", "", -1) item.DealName = strings.Replace(fmt.Sprintf("%s", "top20_"+p.ContractCode+"_成交量(手)"), " ", "", -1) item.BuyName = strings.Replace(fmt.Sprintf("%s", "top20_"+p.ContractCode+"_持买单量(手)"), " ", "", -1) item.SoldName = strings.Replace(fmt.Sprintf("%s", "top20_"+p.ContractCode+"_持卖单量(手)"), " ", "", -1) item.DealCode = shIndexCodeGenerator("top20", item.DealName, contractCode, "deal") item.BuyCode = shIndexCodeGenerator("top20", item.BuyName, contractCode, "buy") item.SoldCode = shIndexCodeGenerator("top20", item.SoldName, contractCode, "sold") item.ClassifyName = strings.Replace(p.ProductName, " ", "", -1) item.ClassifyType = strings.Replace(p.ContractCode, " ", "", -1) item.Frequency = "日度" item.CreateTime = time.Now() item.ModifyTime = time.Now() item.DataTime = tradeDate if deal, ok := p.Deal.(float64); ok{ item.DealValue = int(deal) } if change1, ok := p.Change1.(float64); ok{ item.DealChange = int(change1) } if buyIn, ok := p.BuyIn.(float64); ok{ item.BuyValue = int(buyIn) } if change2, ok := p.Change2.(float64); ok{ item.BuyChange = int(change2) } if soldOut, ok := p.SoldOut.(float64); ok{ item.SoldValue = int(soldOut) } if change3, ok := p.Change3.(float64); ok{ item.SoldChange = int(change3) } itemVerifyCode = item.BuyValue + item.DealValue + item.SoldValue if existIndex, ok := existIndexMap[item.DealName+item.BuyName+item.SoldName]; !ok { newID, e := models.AddBaseFromTradeShangHaiIndex(item) if e != nil { err = e fmt.Println("insert error:", err) } fmt.Println("insert new indexID:", newID) } else if existIndex != nil && itemVerifyCode != (existIndex.DealValue+existIndex.BuyValue+existIndex.SoldValue) { //更新 err = models.ModifyBaseFromTradeShangHaiIndex(item.DealValue, item.BuyValue, item.SoldValue, existIndex.BaseFromTradeShangHaiIndexId) if err != nil { fmt.Println("data update err:", err) } } } } } } fmt.Println("end") }