指标计算方式优化

models/base_from_trade_dalian.go

@@ -85,3 +85,12 @@ func UpdateBaseFromTradeDalianIndex(item *BaseFromTradeDalianIndex, dataId int)
 	_, err = o.Raw(sql, item.Rank, item.DealShortName, item.DealName, item.DealCode, item.DealValue, item.BuyShortName, item.DealChange, item.BuyName, item.BuyCode, item.BuyValue, item.BuyChange, item.SoldShortName, item.SoldName, item.SoldCode, item.SoldValue, item.SoldChange, item.Frequency, item.ClassifyName, item.ClassifyType, item.ModifyTime, item.DataTime, dataId).Exec()
 	return
 }
+
+//获取所有指标Id
+func GetIndexCodeMapList(exchange string) (items []*BaseFromTradeMapping, err error) {
+	o := orm.NewOrm()
+	o.Using("data")
+	sql := `SELECT * FROM base_from_trade_mapping WHERE exchange=?`
+	_, err = o.Raw(sql, exchange).QueryRows(&items)
+	return
+}

models/db.go

@@ -31,7 +31,7 @@ func init() {
 		new(BaseFromTradeShanghaiIndex),
 		new(BaseFromTradeIneIndex),
 		new(BaseFromTradeCffexIndex),
-		new(BaseFromEicIndex),
+		//new(BaseFromEicIndex),
 		//
 		new(BaseFromTradeDalianIndex),
 		//new(BaseFromTradeDalianData),

services/commodity_trade_dalian.go

@@ -230,7 +230,7 @@ func DoSearchMap(body string) (items []*SearchContractId) {
 }
 
 //处理解析Html
-func DoHtml(body, name, contractId string, dateTime time.Time, listDataMap map[string]int) (err error) {
+func DoHtml(body, name, contractId string, dateTime time.Time, listDataMap map[string]int, listIndexCodeMap map[string]string) (err error) {
 	defer func() {
 		if err != nil {
 			fmt.Println("RefreshDataFromDaLian Err:" + err.Error())
@@ -291,17 +291,29 @@ func DoHtml(body, name, contractId string, dateTime time.Time, listDataMap map[s
 			item.Rank = rank
 			item.DealShortName = shortName
 			item.DealName = shortName + "_成交量_" + contractId
-			item.DealCode = DlIndexCodeGenerator(item.DealName, "deal")
+			if val, ok := listIndexCodeMap[item.DealName]; ok {
+				item.DealCode = val
+			} else {
+				item.DealCode = DlIndexCodeGenerator(item.DealName, "deal")
+			}
 			item.DealValue = dealValue
 			item.BuyShortName = buyName
 			item.DealChange = dealChange
 			item.BuyName = buyName + "_持买单量_" + contractId
-			item.BuyCode = DlIndexCodeGenerator(item.BuyName, "buy")
+			if val, ok := listIndexCodeMap[item.BuyName]; ok {
+				item.BuyCode = val
+			} else {
+				item.BuyCode = DlIndexCodeGenerator(item.BuyName, "buy")
+			}
 			item.BuyValue = buyValue
 			item.BuyChange = buyChange
 			item.SoldShortName = soldName
 			item.SoldName = soldName + "_持卖单量_" + contractId
-			item.SoldCode = DlIndexCodeGenerator(item.SoldName, "sold")
+			if val, ok := listIndexCodeMap[item.SoldName]; ok {
+				item.SoldCode = val
+			} else {
+				item.SoldCode = DlIndexCodeGenerator(item.SoldName, "sold")
+			}
 			item.SoldValue = soldValue
 			item.SoldChange = soldChange
 			item.Frequency = "日度"
@@ -332,15 +344,11 @@ func DoHtml(body, name, contractId string, dateTime time.Time, listDataMap map[s
 	return
 }
 
-func DlIndexCodeGenerator(indexName, suffix string) string {
-	ineIndexCode, _ := ineIndexCodeMap[indexName]
-	if ineIndexCode == "" {
-		ineIndexCode = fmt.Sprintf("DL%s", time.Now().Format(utils.FormatDateTimeUnSpace)+strconv.Itoa(utils.GetRandInt(1, 100))+suffix)
-		ineIndexCodeMap[indexName] = ineIndexCode
-		err := models.AddBaseFromTradeMapping(indexName, ineIndexCode, "DL")
-		if err != nil {
-			fmt.Println("add Code err:", err)
-		}
+func DlIndexCodeGenerator(indexName, suffix string) (ineIndexCode string) {
+	ineIndexCode = fmt.Sprintf("DL%s", strconv.FormatInt(time.Now().UnixNano(), 10)+suffix)
+	err := models.AddBaseFromTradeMapping(indexName, ineIndexCode, "DL")
+	if err != nil {
+		fmt.Println("add Code err:", err)
 	}
 	return ineIndexCode
 }
@@ -430,29 +438,6 @@ func SyncRankingFromDalian(dayNum int, searchList []*SearchList) (err error) {
 	}()
 	//n := utils.GetRandInt(10, 120)
 	//time.Sleep(time.Duration(n) * time.Second)
-	//searchList := []SearchList{
-	//	SearchList{VarietyName: "豆一", CarietyCode: "a", List: []SearchContractId{SearchContractId{ContractId: "a2111"}, {ContractId: "a2201"}, {ContractId: "a2203"}}},
-	//	SearchList{VarietyName: "豆二", CarietyCode: "b", List: []SearchContractId{SearchContractId{ContractId: "b2112"}, {ContractId: "b2201"}}},
-	//	SearchList{VarietyName: "豆粕", CarietyCode: "m", List: []SearchContractId{SearchContractId{ContractId: "m2111"}, {ContractId: "m2112"}, {ContractId: "m2201"}, {ContractId: "m2203"}, {ContractId: "m2205"}, {ContractId: "m2207"}, {ContractId: "m2208"}, {ContractId: "m2209"}}},
-	//	SearchList{VarietyName: "豆油", CarietyCode: "y", List: []SearchContractId{SearchContractId{ContractId: "y2112"}, {ContractId: "y2201"}, {ContractId: "y2203"}, {ContractId: "y2205"}, {ContractId: "y2207"}, {ContractId: "y2208"}}},
-	//	SearchList{VarietyName: "棕榈油", CarietyCode: "p", List: []SearchContractId{SearchContractId{ContractId: "p2112"}, {ContractId: "p2201"}, {ContractId: "p2202"}, {ContractId: "p2203"}, {ContractId: "p2204"}, {ContractId: "p2205"}}},
-	//	SearchList{VarietyName: "玉米", CarietyCode: "c", List: []SearchContractId{SearchContractId{ContractId: "c2111"}, {ContractId: "c2201"}, {ContractId: "c2203"}, {ContractId: "c2205"}, {ContractId: "c2207"}}},
-	//	SearchList{VarietyName: "玉米淀粉", CarietyCode: "cs", List: []SearchContractId{SearchContractId{ContractId: "cs2111"}, {ContractId: "cs2201"}, {ContractId: "cs2203"}}},
-	//	SearchList{VarietyName: "鸡蛋", CarietyCode: "jd", List: []SearchContractId{SearchContractId{ContractId: "jd2201"}, {ContractId: "jd2202"}, {ContractId: "jd2203"}, {ContractId: "jd2204"}, {ContractId: "jd2205"}}},
-	//	SearchList{VarietyName: "粳米", CarietyCode: "rr", List: []SearchContractId{SearchContractId{ContractId: "rr2112"}}},
-	//	//SearchList{VarietyName: "纤维板", CarietyCode: "fb", List:[]SearchContractId{}}, //无数据
-	//	//SearchList{VarietyName: "胶合板", CarietyCode: "bb", List:[]SearchContractId{}}, //无数据
-	//	SearchList{VarietyName: "生猪", CarietyCode: "lh", List: []SearchContractId{SearchContractId{ContractId: "lh2201"}, {ContractId: "lh2203"}, {ContractId: "lh2205"}}},
-	//	SearchList{VarietyName: "聚乙烯", CarietyCode: "l", List: []SearchContractId{SearchContractId{ContractId: "l2112"}, {ContractId: "l2201"}, {ContractId: "l2202"}, {ContractId: "l2203"}, {ContractId: "l2204"}, {ContractId: "l2205"}}},
-	//	SearchList{VarietyName: "聚氯乙烯", CarietyCode: "v", List: []SearchContractId{SearchContractId{ContractId: "v2111"}, {ContractId: "v2112"}, {ContractId: "v2201"}, {ContractId: "v2202"}, {ContractId: "v2203"}, {ContractId: "v2204"}, {ContractId: "v2205"}}},
-	//	SearchList{VarietyName: "聚丙烯", CarietyCode: "pp", List: []SearchContractId{SearchContractId{ContractId: "pp2112"}, {ContractId: "pp2201"}, {ContractId: "pp2202"}, {ContractId: "pp2203"}, {ContractId: "pp2204"}, {ContractId: "pp2205"}}},
-	//	SearchList{VarietyName: "苯乙烯", CarietyCode: "eb", List: []SearchContractId{SearchContractId{ContractId: "eb2111"}, {ContractId: "eb2112"}, {ContractId: "eb2201"}, {ContractId: "eb2202"}}},
-	//	SearchList{VarietyName: "焦炭", CarietyCode: "j", List: []SearchContractId{SearchContractId{ContractId: "j2201"}, {ContractId: "j2205"}}},
-	//	SearchList{VarietyName: "焦煤", CarietyCode: "jm", List: []SearchContractId{SearchContractId{ContractId: "jm2201"}, {ContractId: "jm2205"}}},
-	//	SearchList{VarietyName: "铁矿石", CarietyCode: "i", List: []SearchContractId{SearchContractId{ContractId: "i2112"}, {ContractId: "i2201"}, {ContractId: "i2202"}, {ContractId: "i2203"}, {ContractId: "i2204"}, {ContractId: "i2205"}, {ContractId: "i2206"}, {ContractId: "i2207"}, {ContractId: "i2208"}, {ContractId: "i2209"}}},
-	//	SearchList{VarietyName: "乙二醇", CarietyCode: "eg", List: []SearchContractId{SearchContractId{ContractId: "eg2112"}, {ContractId: "eg2201"}, {ContractId: "eg2202"}, {ContractId: "eg2203"}, {ContractId: "eg2205"}}},
-	//	SearchList{VarietyName: "液化石油气", CarietyCode: "pg", List: []SearchContractId{SearchContractId{ContractId: "pg2111"}, {ContractId: "pg2112"}, {ContractId: "pg2201"}, {ContractId: "pg2202"}}},
-	//}
 
 	//定义爬取时间
 	endDate := time.Now().AddDate(0, 0, -dayNum).Format(utils.FormatDateTime)
@@ -480,6 +465,15 @@ func SyncRankingFromDalian(dayNum int, searchList []*SearchList) (err error) {
 		fmt.Println(err)
 		return err
 	}
+	listIndexCode, err := models.GetIndexCodeMapList("DL") //获取往期指标
+	if err != nil {
+		fmt.Println(err)
+		return err
+	}
+	listIndexCodeMap := make(map[string]string)
+	for _, v := range listIndexCode {
+		listIndexCodeMap[v.IndexName] = v.IndexCode
+	}
 	for _, v := range searchList {
 		for _, v2 := range v.ListSearch {
 			//模拟form表单请求
@@ -519,7 +513,7 @@ func SyncRankingFromDalian(dayNum int, searchList []*SearchList) (err error) {
 				go utils.SendEmail(utils.APPNAME+"【"+utils.RunMode+"】"+"失败提醒", "RefreshDataFromDalian ErrMsg:"+err.Error()+"获取指标失败："+currDate+v.VarietyName+v2.ContractId, utils.EmailSendToUsers)
 				return err
 			}
-			err = DoHtml(string(body), v.VarietyName, v2.ContractId, timeDate, listDataMap)
+			err = DoHtml(string(body), v.VarietyName, v2.ContractId, timeDate, listDataMap, listIndexCodeMap)
 			if err != nil {
 				return err
 			}

services/task.go

@@ -1,7 +1,9 @@
 package services
 
 import (
+	"context"
 	"fmt"
+	"github.com/beego/beego/v2/task"
 )
 
 func Task() {
@@ -16,20 +18,19 @@ func Task() {
 	//time.Sleep(20 * time.Second)
 	//SyncRankingFromDalianDo()
 	//SyncStorageFromEic()
-
-	//refreshData := task.NewTask("refreshData", "0 0 3 * * *", RefreshData)
-	//task.AddTask("数据爬取", refreshData)
-	//task.StartTask()
+	refreshData := task.NewTask("refreshData", "0 0 3 * * *", RefreshData)
+	task.AddTask("数据爬取", refreshData)
+	task.StartTask()
 	fmt.Println("end crawler")
 }
 
-//func RefreshData(cont context.Context) (err error) {
-//	//大连
-//	SyncRankingFromDalian(1)
-//	SyncRankingFromZhengzhou()
-//	SyncRankingFromShangHai()
-//	SyncRankingFromIne()
-//	SyncRankingFromCffex()
-//
-//	return
-//}
+func RefreshData(cont context.Context) (err error) {
+	//大连
+	SyncRankingFromDalianSearch(1)
+	SyncRankingFromZhengzhou()
+	SyncRankingFromShangHai()
+	SyncRankingFromIne()
+	SyncRankingFromCffex()
+
+	return
+}

static/dalian.html

@@ -556,6 +556,30 @@
 //}
 
 
+//searchList := []SearchList{
+//	SearchList{VarietyName: "豆一", CarietyCode: "a", List: []SearchContractId{SearchContractId{ContractId: "a2111"}, {ContractId: "a2201"}, {ContractId: "a2203"}}},
+//	SearchList{VarietyName: "豆二", CarietyCode: "b", List: []SearchContractId{SearchContractId{ContractId: "b2112"}, {ContractId: "b2201"}}},
+//	SearchList{VarietyName: "豆粕", CarietyCode: "m", List: []SearchContractId{SearchContractId{ContractId: "m2111"}, {ContractId: "m2112"}, {ContractId: "m2201"}, {ContractId: "m2203"}, {ContractId: "m2205"}, {ContractId: "m2207"}, {ContractId: "m2208"}, {ContractId: "m2209"}}},
+//	SearchList{VarietyName: "豆油", CarietyCode: "y", List: []SearchContractId{SearchContractId{ContractId: "y2112"}, {ContractId: "y2201"}, {ContractId: "y2203"}, {ContractId: "y2205"}, {ContractId: "y2207"}, {ContractId: "y2208"}}},
+//	SearchList{VarietyName: "棕榈油", CarietyCode: "p", List: []SearchContractId{SearchContractId{ContractId: "p2112"}, {ContractId: "p2201"}, {ContractId: "p2202"}, {ContractId: "p2203"}, {ContractId: "p2204"}, {ContractId: "p2205"}}},
+//	SearchList{VarietyName: "玉米", CarietyCode: "c", List: []SearchContractId{SearchContractId{ContractId: "c2111"}, {ContractId: "c2201"}, {ContractId: "c2203"}, {ContractId: "c2205"}, {ContractId: "c2207"}}},
+//	SearchList{VarietyName: "玉米淀粉", CarietyCode: "cs", List: []SearchContractId{SearchContractId{ContractId: "cs2111"}, {ContractId: "cs2201"}, {ContractId: "cs2203"}}},
+//	SearchList{VarietyName: "鸡蛋", CarietyCode: "jd", List: []SearchContractId{SearchContractId{ContractId: "jd2201"}, {ContractId: "jd2202"}, {ContractId: "jd2203"}, {ContractId: "jd2204"}, {ContractId: "jd2205"}}},
+//	SearchList{VarietyName: "粳米", CarietyCode: "rr", List: []SearchContractId{SearchContractId{ContractId: "rr2112"}}},
+//	//SearchList{VarietyName: "纤维板", CarietyCode: "fb", List:[]SearchContractId{}}, //无数据
+//	//SearchList{VarietyName: "胶合板", CarietyCode: "bb", List:[]SearchContractId{}}, //无数据
+//	SearchList{VarietyName: "生猪", CarietyCode: "lh", List: []SearchContractId{SearchContractId{ContractId: "lh2201"}, {ContractId: "lh2203"}, {ContractId: "lh2205"}}},
+//	SearchList{VarietyName: "聚乙烯", CarietyCode: "l", List: []SearchContractId{SearchContractId{ContractId: "l2112"}, {ContractId: "l2201"}, {ContractId: "l2202"}, {ContractId: "l2203"}, {ContractId: "l2204"}, {ContractId: "l2205"}}},
+//	SearchList{VarietyName: "聚氯乙烯", CarietyCode: "v", List: []SearchContractId{SearchContractId{ContractId: "v2111"}, {ContractId: "v2112"}, {ContractId: "v2201"}, {ContractId: "v2202"}, {ContractId: "v2203"}, {ContractId: "v2204"}, {ContractId: "v2205"}}},
+//	SearchList{VarietyName: "聚丙烯", CarietyCode: "pp", List: []SearchContractId{SearchContractId{ContractId: "pp2112"}, {ContractId: "pp2201"}, {ContractId: "pp2202"}, {ContractId: "pp2203"}, {ContractId: "pp2204"}, {ContractId: "pp2205"}}},
+//	SearchList{VarietyName: "苯乙烯", CarietyCode: "eb", List: []SearchContractId{SearchContractId{ContractId: "eb2111"}, {ContractId: "eb2112"}, {ContractId: "eb2201"}, {ContractId: "eb2202"}}},
+//	SearchList{VarietyName: "焦炭", CarietyCode: "j", List: []SearchContractId{SearchContractId{ContractId: "j2201"}, {ContractId: "j2205"}}},
+//	SearchList{VarietyName: "焦煤", CarietyCode: "jm", List: []SearchContractId{SearchContractId{ContractId: "jm2201"}, {ContractId: "jm2205"}}},
+//	SearchList{VarietyName: "铁矿石", CarietyCode: "i", List: []SearchContractId{SearchContractId{ContractId: "i2112"}, {ContractId: "i2201"}, {ContractId: "i2202"}, {ContractId: "i2203"}, {ContractId: "i2204"}, {ContractId: "i2205"}, {ContractId: "i2206"}, {ContractId: "i2207"}, {ContractId: "i2208"}, {ContractId: "i2209"}}},
+//	SearchList{VarietyName: "乙二醇", CarietyCode: "eg", List: []SearchContractId{SearchContractId{ContractId: "eg2112"}, {ContractId: "eg2201"}, {ContractId: "eg2202"}, {ContractId: "eg2203"}, {ContractId: "eg2205"}}},
+//	SearchList{VarietyName: "液化石油气", CarietyCode: "pg", List: []SearchContractId{SearchContractId{ContractId: "pg2111"}, {ContractId: "pg2112"}, {ContractId: "pg2201"}, {ContractId: "pg2202"}}},
+//}
+
 
 type SearchList struct {
 VarietyName         string `description:"商品名称"`