package controllers import ( "encoding/json" "eta/eta_index_lib/models" "fmt" "github.com/mozillazg/go-pinyin" "strings" "time" ) // 交易所爬虫 type ExchangeCrawler struct { BaseAuthController } var ineIndexCodeMap = make(map[string]string) // @Title 刷新数据 // @Description 刷新数据接口 // @Param request body models.AddEdbClassifyReq true "type json string" // @Success 200 {object} models.EdbClassify // @router /refresh/ine [post] func (this *ExchangeCrawler) RefreshIne() { br := new(models.BaseResponse).Init() defer func() { this.Data["json"] = br this.ServeJSON() }() var req models.RefreshINEExchangeReq err := json.Unmarshal(this.Ctx.Input.RequestBody, &req) if err != nil { br.Msg = "参数解析异常!" br.ErrMsg = "参数解析失败,Err:" + err.Error() return } allCode, err := models.GetIndexCodeFromMapping("Ine") if err != nil { fmt.Println("select Code err:", err) } for _, item := range allCode { ineIndexCodeMap[item.IndexName] = item.IndexCode } message := req.Data var position = message.OCursor var tradeDate = message.ReportDate existIndexMap := make(map[string]*models.BaseFromTradeIneIndex) //获取所有指标信息 allIndex, err := models.GetBaseFromTradeIneIndexAll(req.Date) if err != nil { fmt.Println("select err:", err) } for _, v := range allIndex { indexKey := v.DealName + v.BuyName + v.SoldName existIndexMap[indexKey] = v } var itemVerifyCode int //处理指标 for _, p := range position { var item = new(models.BaseFromTradeIneIndex) if p.Rank > 0 && p.Rank < 40 && p.Participantabbr1 != "" { //成交量 item.Rank = p.Rank item.DealShortName = strings.Trim(p.Participantabbr1, " ") item.BuyShortName = strings.Trim(p.Participantabbr2, " ") item.SoldShortName = strings.Trim(p.Participantabbr3, " ") item.DealName = strings.Replace(fmt.Sprintf("%s", item.DealShortName+"_"+p.Instrumentid+"_成交量(手)"), " ", "", -1) item.BuyName = strings.Replace(fmt.Sprintf("%s", item.BuyShortName+"_"+p.Instrumentid+"_持买单量(手)"), " ", "", -1) item.SoldName = strings.Replace(fmt.Sprintf("%s", item.SoldShortName+"_"+p.Instrumentid+"_持卖单量(手)"), " ", "", -1) item.DealCode = IneIndexCodeGenerator(item.DealShortName, item.DealName, p.Instrumentid, "deal") item.BuyCode = IneIndexCodeGenerator(item.BuyShortName, item.BuyName, p.Instrumentid, "buy") item.SoldCode = IneIndexCodeGenerator(item.SoldShortName, item.SoldName, p.Instrumentid, "sold") item.ClassifyName = strings.Replace(p.Productname, " ", "", -1) item.ClassifyType = strings.Replace(p.Instrumentid, " ", "", -1) item.Frequency = "日度" item.CreateTime = time.Now() item.ModifyTime = time.Now() item.DataTime = tradeDate if deal, ok := p.Deal.(float64); ok { item.DealValue = int(deal) } if change1, ok := p.Change1.(float64); ok { item.DealChange = int(change1) } if buyIn, ok := p.BuyIn.(float64); ok { item.BuyValue = int(buyIn) } if change2, ok := p.Change2.(float64); ok { item.BuyChange = int(change2) } if soldOut, ok := p.SoldOut.(float64); ok { item.SoldValue = int(soldOut) } if change3, ok := p.Change3.(float64); ok { item.SoldChange = int(change3) } itemVerifyCode = item.BuyValue + item.DealValue + item.SoldValue if existIndex, ok := existIndexMap[item.DealName+item.BuyName+item.SoldName]; !ok { newID, err := models.AddBaseFromTradeIneIndex(item) if err != nil { fmt.Println("insert error:", err) } fmt.Println("insert new indexID:", newID) } else if existIndex != nil && itemVerifyCode != (existIndex.DealValue+existIndex.BuyValue+existIndex.SoldValue) { //更新 err := models.ModifyBaseFromTradeIneIndex(item.DealValue, item.BuyValue, item.SoldValue, existIndex.BaseFromTradeIneIndexId) if err != nil { fmt.Println("data update err:", err) } } } else if p.Rank == 999 { //Top 20 item.Rank = p.Rank item.DealShortName = strings.Trim(p.Participantabbr1, " ") item.BuyShortName = strings.Trim(p.Participantabbr2, " ") item.SoldShortName = strings.Trim(p.Participantabbr3, " ") item.DealName = strings.Replace(fmt.Sprintf("%s", "top20_"+p.Instrumentid+"_成交量(手)"), " ", "", -1) item.BuyName = strings.Replace(fmt.Sprintf("%s", "top20_"+p.Instrumentid+"_持买单量(手)"), " ", "", -1) item.SoldName = strings.Replace(fmt.Sprintf("%s", "top20_"+p.Instrumentid+"_持卖单量(手)"), " ", "", -1) item.DealCode = IneIndexCodeGenerator("top20", item.DealName, p.Instrumentid, "deal") item.BuyCode = IneIndexCodeGenerator("top20", item.BuyName, p.Instrumentid, "buy") item.SoldCode = IneIndexCodeGenerator("top20", item.SoldName, p.Instrumentid, "sold") item.ClassifyName = strings.Replace(p.Productname, " ", "", -1) item.ClassifyType = strings.Replace(p.Instrumentid, " ", "", -1) item.Frequency = "日度" item.CreateTime = time.Now() item.ModifyTime = time.Now() item.DataTime = tradeDate if deal, ok := p.Deal.(float64); ok { item.DealValue = int(deal) } if change1, ok := p.Change1.(float64); ok { item.DealChange = int(change1) } if buyIn, ok := p.BuyIn.(float64); ok { item.BuyValue = int(buyIn) } if change2, ok := p.Change2.(float64); ok { item.BuyChange = int(change2) } if soldOut, ok := p.SoldOut.(float64); ok { item.SoldValue = int(soldOut) } if change3, ok := p.Change3.(float64); ok { item.SoldChange = int(change3) } itemVerifyCode = item.BuyValue + item.DealValue + item.SoldValue if existIndex, ok := existIndexMap[item.DealName+item.BuyName+item.SoldName]; !ok { newID, err := models.AddBaseFromTradeIneIndex(item) if err != nil { fmt.Println("insert error:", err) } fmt.Println("insert new indexID:", newID) } else if existIndex != nil && itemVerifyCode != (existIndex.DealValue+existIndex.BuyValue+existIndex.SoldValue) { //更新 err := models.ModifyBaseFromTradeIneIndex(item.DealValue, item.BuyValue, item.SoldValue, existIndex.BaseFromTradeIneIndexId) if err != nil { fmt.Println("data update err:", err) } } } } br.Ret = 200 br.Msg = "获取成功" br.Success = true } var indexCodeMap = make(map[string]string) // @Title 刷新数据 // @Description 刷新数据接口 // @Param request body models.AddEdbClassifyReq true "type json string" // @Success 200 {object} models.EdbClassify // @router /refresh/sh [post] func (this *ExchangeCrawler) RefreshSH() { br := new(models.BaseResponse).Init() defer func() { this.Data["json"] = br this.ServeJSON() }() var req models.RefreshSHExchangeReq err := json.Unmarshal(this.Ctx.Input.RequestBody, &req) if err != nil { br.Msg = "参数解析异常!" br.ErrMsg = "参数解析失败,Err:" + err.Error() return } allCode, err := models.GetIndexCodeFromMapping("SH") if err != nil { fmt.Println("select Code err:", err) return } for _, item := range allCode { indexCodeMap[item.IndexName] = item.IndexCode } message := req.Data var position = message.Position var tradeDate = message.ReportDate existIndexMap := make(map[string]*models.BaseFromTradeShanghaiIndex) //获取所有指标信息 allIndex, err := models.GetBaseFromTradeShangHaiIndexAll(req.Date) if err != nil { fmt.Println("select err:", err) } for _, v := range allIndex { indexKey := v.DealName + v.BuyName + v.SoldName existIndexMap[indexKey] = v } var itemVerifyCode int //处理指标 for _, p := range position { var item = new(models.BaseFromTradeShanghaiIndex) if p.Rank > 0 && p.Rank < 40 && p.ParticipantName1 != "" { if strings.Replace(p.ProductName, " ", "", -1) != "20号胶" && strings.Replace(p.ProductName, " ", "", -1) != "低硫燃料油" { contractCode := strings.Replace(p.ContractCode, " ", "", -1) //成交量 item.Rank = p.Rank item.DealShortName = strings.Replace(p.ParticipantName1, " ", "", -1) item.BuyShortName = strings.Replace(p.ParticipantName2, " ", "", -1) item.SoldShortName = strings.Replace(p.ParticipantName3, " ", "", -1) item.DealName = strings.Replace(fmt.Sprintf("%s", p.ParticipantName1+"_"+p.ContractCode+"_成交量(手)"), " ", "", -1) item.BuyName = strings.Replace(fmt.Sprintf("%s", p.ParticipantName2+"_"+p.ContractCode+"_持买单量(手)"), " ", "", -1) item.SoldName = strings.Replace(fmt.Sprintf("%s", p.ParticipantName3+"_"+p.ContractCode+"_持卖单量(手)"), " ", "", -1) item.DealCode = ShIndexCodeGenerator(item.DealShortName, item.DealName, contractCode, "deal") item.BuyCode = ShIndexCodeGenerator(item.BuyShortName, item.BuyName, contractCode, "buy") item.SoldCode = ShIndexCodeGenerator(item.SoldShortName, item.SoldName, contractCode, "sold") item.ClassifyName = strings.Replace(p.ProductName, " ", "", -1) item.ClassifyType = strings.Replace(p.ContractCode, " ", "", -1) item.Frequency = "日度" item.CreateTime = time.Now() item.ModifyTime = time.Now() item.DataTime = tradeDate if deal, ok := p.Deal.(float64); ok { item.DealValue = int(deal) } if change1, ok := p.Change1.(float64); ok { item.DealChange = int(change1) } if buyIn, ok := p.BuyIn.(float64); ok { item.BuyValue = int(buyIn) } if change2, ok := p.Change2.(float64); ok { item.BuyChange = int(change2) } if soldOut, ok := p.SoldOut.(float64); ok { item.SoldValue = int(soldOut) } if change3, ok := p.Change3.(float64); ok { item.SoldChange = int(change3) } itemVerifyCode = item.BuyValue + item.DealValue + item.SoldValue if existIndex, ok := existIndexMap[item.DealName+item.BuyName+item.SoldName]; !ok { newID, e := models.AddBaseFromTradeShangHaiIndex(item) if e != nil { err = e fmt.Println("insert error:", err) } fmt.Println("insert new indexID:", newID) } else if existIndex != nil && itemVerifyCode != (existIndex.DealValue+existIndex.BuyValue+existIndex.SoldValue) { //更新 err = models.ModifyBaseFromTradeShangHaiIndex(item.DealValue, item.BuyValue, item.SoldValue, existIndex.BaseFromTradeShangHaiIndexId) if err != nil { fmt.Println("data update err:", err) } } } } else if p.Rank == 999 { if strings.Replace(p.ProductName, " ", "", -1) != "20号胶" && strings.Replace(p.ProductName, " ", "", -1) != "低硫燃料油" { contractCode := strings.Replace(p.ContractCode, " ", "", -1) //Top 20 item.Rank = p.Rank item.DealShortName = strings.Replace(p.ParticipantName1, " ", "", -1) item.BuyShortName = strings.Replace(p.ParticipantName2, " ", "", -1) item.SoldShortName = strings.Replace(p.ParticipantName3, " ", "", -1) item.DealName = strings.Replace(fmt.Sprintf("%s", "top20_"+p.ContractCode+"_成交量(手)"), " ", "", -1) item.BuyName = strings.Replace(fmt.Sprintf("%s", "top20_"+p.ContractCode+"_持买单量(手)"), " ", "", -1) item.SoldName = strings.Replace(fmt.Sprintf("%s", "top20_"+p.ContractCode+"_持卖单量(手)"), " ", "", -1) item.DealCode = ShIndexCodeGenerator("top20", item.DealName, contractCode, "deal") item.BuyCode = ShIndexCodeGenerator("top20", item.BuyName, contractCode, "buy") item.SoldCode = ShIndexCodeGenerator("top20", item.SoldName, contractCode, "sold") item.ClassifyName = strings.Replace(p.ProductName, " ", "", -1) item.ClassifyType = strings.Replace(p.ContractCode, " ", "", -1) item.Frequency = "日度" item.CreateTime = time.Now() item.ModifyTime = time.Now() item.DataTime = tradeDate if deal, ok := p.Deal.(float64); ok { item.DealValue = int(deal) } if change1, ok := p.Change1.(float64); ok { item.DealChange = int(change1) } if buyIn, ok := p.BuyIn.(float64); ok { item.BuyValue = int(buyIn) } if change2, ok := p.Change2.(float64); ok { item.BuyChange = int(change2) } if soldOut, ok := p.SoldOut.(float64); ok { item.SoldValue = int(soldOut) } if change3, ok := p.Change3.(float64); ok { item.SoldChange = int(change3) } itemVerifyCode = item.BuyValue + item.DealValue + item.SoldValue if existIndex, ok := existIndexMap[item.DealName+item.BuyName+item.SoldName]; !ok { newID, e := models.AddBaseFromTradeShangHaiIndex(item) if e != nil { err = e fmt.Println("insert error:", err) } fmt.Println("insert new indexID:", newID) } else if existIndex != nil && itemVerifyCode != (existIndex.DealValue+existIndex.BuyValue+existIndex.SoldValue) { //更新 err = models.ModifyBaseFromTradeShangHaiIndex(item.DealValue, item.BuyValue, item.SoldValue, existIndex.BaseFromTradeShangHaiIndexId) if err != nil { fmt.Println("data update err:", err) } } } } } br.Ret = 200 br.Msg = "获取成功" br.Success = true } func ShIndexCodeGenerator(shortName, indexName, contractCode, suffix string) string { if shortName == "" { return "" } strResult := "" if shortName != "top20" { //取公司全拼 a := pinyin.NewArgs() rows := pinyin.LazyPinyin(shortName, a) for i := 0; i < len(rows); i++ { strResult += rows[i] } } else { strResult = "top20" } indexCode, _ := indexCodeMap[indexName] if indexCode == "" { indexCode = strResult + contractCode + suffix indexCode = strings.Replace(indexCode, " ", "", -1) indexCodeMap[indexName] = indexCode err := models.AddBaseFromTradeMapping(indexName, indexCode, "SH") if err != nil { fmt.Println("add Code err:", err) } } return strings.Replace(indexCode, " ", "", -1) } func IneIndexCodeGenerator(shortName, indexName, Instrumentid, suffix string) string { if shortName == "" { return "" } strResult := "" if shortName != "top20" { //取公司全拼 a := pinyin.NewArgs() rows := pinyin.LazyPinyin(shortName, a) for i := 0; i < len(rows); i++ { strResult += rows[i] } } else { strResult = "top20" } ineIndexCode, _ := ineIndexCodeMap[indexName] if ineIndexCode == "" { ineIndexCode = strResult + Instrumentid + suffix ineIndexCode = strings.Replace(ineIndexCode, " ", "", -1) ineIndexCodeMap[indexName] = ineIndexCode err := models.AddBaseFromTradeMapping(indexName, ineIndexCode, "INE") if err != nil { fmt.Println("add Code err:", err) } } return strings.Replace(ineIndexCode, " ", "", -1) }