大连数据爬取，更新优化

services/commodity_trade_dalian.go

@@ -67,6 +67,8 @@ func SyncRankingFromDalianSearch(dayNum int) (err error) {
 	for _, v := range list {
 		listDataMap[v.DealShortName+v.ClassifyType+v.DataTime] = v.BaseFromTradeDalianIndexId
 	}
+	n := utils.GetRandInt(10, 120)
+	time.Sleep(time.Duration(n) * time.Second)
 
 	var ContractId string
 	var CarietyCode string
@@ -230,7 +232,7 @@ func DoSearchMap(body string) (items []*SearchContractId) {
 }
 
 //处理解析Html
-func DoHtml(body, name, contractId string, dateTime time.Time, listDataMap map[string]int, listIndexCodeMap map[string]string) (err error) {
+func DoHtml(body, name, contractId string, dateTime time.Time, listDataMap map[string]int, listIndexCodeMap map[string]string, listDataMapVal map[string]int) (err error) {
 	defer func() {
 		if err != nil {
 			fmt.Println("RefreshDataFromDaLian Err:" + err.Error())
@@ -327,9 +329,11 @@ func DoHtml(body, name, contractId string, dateTime time.Time, listDataMap map[s
 				isAdd = true
 			} else {
 				//更新
-				err := models.UpdateBaseFromTradeDalianIndex(item, val)
-				if err != nil {
-					fmt.Println("UpdateBaseFromTradeDalianIndex err:", err)
+				if listDataMapVal[item.DealValue+item.BuyValue+item.SoldValue] != val {
+					err := models.UpdateBaseFromTradeDalianIndex(item, val)
+					if err != nil {
+						fmt.Println("UpdateBaseFromTradeDalianIndex err:", err)
+					}
 				}
 			}
 		}
@@ -458,9 +462,13 @@ func SyncRankingFromDalian(dayNum int, searchList []*SearchList) (err error) {
 
 	list, err := models.GetBaseFromTradeDalianDataList(timeDate.Format(utils.FormatDate))
 	listDataMap := make(map[string]int)
+	listDataMapVal := make(map[string]int)
 	for _, v := range list {
 		listDataMap[v.DealShortName+v.ClassifyType+v.DataTime] = v.BaseFromTradeDalianIndexId
 	}
+	for _, v := range list {
+		listDataMapVal[v.DealValue+v.BuyValue+v.SoldValue] = v.BaseFromTradeDalianIndexId
+	}
 	if err != nil {
 		fmt.Println(err)
 		return err
@@ -513,7 +521,7 @@ func SyncRankingFromDalian(dayNum int, searchList []*SearchList) (err error) {
 				go utils.SendEmail(utils.APPNAME+"【"+utils.RunMode+"】"+"失败提醒", "RefreshDataFromDalian ErrMsg:"+err.Error()+"获取指标失败："+currDate+v.VarietyName+v2.ContractId, utils.EmailSendToUsers)
 				return err
 			}
-			err = DoHtml(string(body), v.VarietyName, v2.ContractId, timeDate, listDataMap, listIndexCodeMap)
+			err = DoHtml(string(body), v.VarietyName, v2.ContractId, timeDate, listDataMap, listIndexCodeMap, listDataMapVal)
 			if err != nil {
 				return err
 			}