fix

models/com_trade_data.go

@@ -61,6 +61,15 @@ func AddComTradeCodeMapping(item *ComTradeCodeMapping) (err error) {
 	return
 }
 
+func UpdateComTradeCodeMapping(item *ComTradeCodeMapping,cols []string) (err error) {
+	o := orm.NewOrmUsingDB("data")
+	_, err = o.Update(item, cols...)
+	if err != nil {
+		return
+	}
+	return
+}
+
 func AddMultiComTradeCodeMapping(items []*ComTradeCodeMapping) (err error) {
 	o := orm.NewOrmUsingDB("data")
 	_, err = o.InsertMulti(len(items), items)

services/com_trade.go

@@ -228,13 +228,13 @@ func sync3() {
 	//	`866,732,736,364,459,698,706,12,674,234,516,891,849,260,304,388,899,492,612,660,772,694,862,697,592,472,324,531,226,92,28,232,583,647,654,166,64,838,336,430,58`,
 	//}
 
-	codeMappingListMap := make(map[string]bool)
+	codeMappingListMap := make(map[string]string)
 	codeMappingList, err := models.GetAllComTradeCodeMappingList()
 	if err != nil {
 		return
 	}
 	for _, v := range codeMappingList {
-		codeMappingListMap[v.Code] = true
+		codeMappingListMap[v.Code] = v.Name
 	}
 
 	countryMap, err := GetAllCountry()
@@ -262,13 +262,13 @@ func sync3() {
 }
 
 func syncAllComTradeData2(year, month int, flowCode, dateType string) (err error) {
-	codeMappingListMap := make(map[string]bool)
+	codeMappingListMap := make(map[string]string)
 	codeMappingList, err := models.GetAllComTradeCodeMappingList()
 	if err != nil {
 		return
 	}
 	for _, v := range codeMappingList {
-		codeMappingListMap[v.Code] = true
+		codeMappingListMap[v.Code] = v.Name
 	}
 
 	comTradeList, err := models.GetAllComTradeIndexList()
@@ -309,13 +309,13 @@ func syncAllComTradeData2(year, month int, flowCode, dateType string) (err error
 }
 
 func syncAllComTradeData() (err error) {
-	codeMappingListMap := make(map[string]bool)
+	codeMappingListMap := make(map[string]string)
 	codeMappingList, err := models.GetAllComTradeCodeMappingList()
 	if err != nil {
 		return
 	}
 	for _, v := range codeMappingList {
-		codeMappingListMap[v.Code] = true
+		codeMappingListMap[v.Code] = v.Name
 	}
 
 	comTradeList, err := models.GetAllComTradeIndexList()
@@ -354,7 +354,7 @@ func syncAllComTradeData() (err error) {
 	return
 }
 
-func syncSingleComTradeData(comTradeId, year, month int, flowCode, indexCode, indexNameCn, dateType string, countryMap map[int]models.ComTradeCountry, codeMappingListMap map[string]bool, countryGroup map[int][]models.ComTradeCountry) {
+func syncSingleComTradeData(comTradeId, year, month int, flowCode, indexCode, indexNameCn, dateType string, countryMap map[int]models.ComTradeCountry, codeMappingListMap map[string]string, countryGroup map[int][]models.ComTradeCountry) {
 	////flowCode := "X"
 	//flowCode := "M"
 	//year := 2020
@@ -400,13 +400,13 @@ func SyncSingleComTradeData3() {
 	indexCode := "290121"
 	indexNameCn := `乙烯（出口退税率13%）`
 	dateType := `month`
-	codeMappingListMap := make(map[string]bool)
+	codeMappingListMap := make(map[string]string)
 	codeMappingList, err := models.GetAllComTradeCodeMappingList()
 	if err != nil {
 		return
 	}
 	for _, v := range codeMappingList {
-		codeMappingListMap[v.Code] = true
+		codeMappingListMap[v.Code] = v.Name
 	}
 
 	countryMap, err := GetAllCountry()
@@ -437,7 +437,7 @@ func SyncSingleComTradeData3() {
 	fmt.Println(indexCode, "结束了")
 }
 
-func syncComTradeData(comTradeId int, indexCode, indexNameCn, flowCode, partnerCode, dateType string, year, month int, codeMappingListMap map[string]bool, countryMap map[int]models.ComTradeCountry) (err error) {
+func syncComTradeData(comTradeId int, indexCode, indexNameCn, flowCode, partnerCode, dateType string, year, month int, codeMappingListMap map[string]string, countryMap map[int]models.ComTradeCountry) (err error) {
 	// 获取数据
 
 	//https://comtradeapi.un.org/public/v1/preview/C/A/HS?period=2021&partnerCode=842&cmdCode=290244&flowCode=X&customsCode=C00&motCode=0
@@ -582,7 +582,7 @@ func syncComTradeData(comTradeId int, indexCode, indexNameCn, flowCode, partnerC
 			valueName := tmpName + "贸易金额"
 			weightName := tmpName + "贸易重量"
 
-			if _, ok := codeMappingListMap[tmpData.IndexTradeCode]; !ok {
+			if name, ok := codeMappingListMap[tmpData.IndexTradeCode]; !ok {
 				comTradeCodeMappingType := 1
 				if codeStr == "M" {
 					comTradeCodeMappingType = 3
@@ -599,11 +599,25 @@ func syncComTradeData(comTradeId int, indexCode, indexNameCn, flowCode, partnerC
 				if tmpErr != nil {
 					fmt.Println(indexCode, "添加贸易金额失败,err:", err)
 				} else {
-					codeMappingListMap[tmpData.IndexTradeCode] = true
+					codeMappingListMap[tmpData.IndexTradeCode] = valueName
+				}
+			} else {
+				if name != valueName{
+					tmpComTradeCodeMapping := &models.ComTradeCodeMapping{
+						Code:       tmpData.IndexTradeCode,
+						Name:       valueName,
+					}
+					e := models.UpdateComTradeCodeMapping(tmpComTradeCodeMapping, []string{"Name"})
+					if e != nil {
+						err = e
+						return
+					}
+					utils.FileLog.Info("更新指标名称","旧："+name, "新：" + valueName,"code:" +tmpData.IndexTradeCode)
+					fmt.Println("更新指标名称","旧："+name, "新：" + valueName)
 				}
 			}
 
-			if _, ok := codeMappingListMap[tmpData.IndexNetWeightCode]; !ok {
+			if name, ok := codeMappingListMap[tmpData.IndexNetWeightCode]; !ok {
 				comTradeCodeMappingType := 2
 				if codeStr == "M" {
 					comTradeCodeMappingType = 3
@@ -619,7 +633,21 @@ func syncComTradeData(comTradeId int, indexCode, indexNameCn, flowCode, partnerC
 				if tmpErr != nil {
 					fmt.Println(indexCode, "添加贸易重量失败,err:", err)
 				} else {
-					codeMappingListMap[tmpData.IndexNetWeightCode] = true
+					codeMappingListMap[tmpData.IndexNetWeightCode] = weightName
+				}
+			} else {
+				if name != weightName{
+					tmpComTradeCodeMapping := &models.ComTradeCodeMapping{
+						Code:       tmpData.IndexTradeCode,
+						Name:       valueName,
+					}
+					e := models.UpdateComTradeCodeMapping(tmpComTradeCodeMapping, []string{"Name"})
+					if e != nil {
+						err = e
+						return
+					}
+					utils.FileLog.Info("更新指标名称","旧："+name, "新：" + valueName ,"code:" +tmpData.IndexNetWeightCode)
+					fmt.Println("更新指标名称","旧："+name, "新：" + valueName)
 				}
 			}
 		}
@@ -892,3 +920,160 @@ func syncAllComTradeMapping2() (err error) {
 	fmt.Println("整体结束了=============")
 	return
 }
+
+func syncAllComTradeMapping3()  {
+	codeMappingListMap := make(map[string]*models.ComTradeCodeMapping)
+	codeMappingList, err := models.GetAllComTradeCodeMappingList()
+	if err != nil {
+		return
+	}
+	for _, v := range codeMappingList {
+		codeMappingListMap[v.Code] = v
+	}
+
+	comTradeIndexMap := make(map[string]string)
+	comTradeIndexList, err := models.GetAllComTradeIndexList()
+	if err != nil {
+		return
+	}
+	for _, v := range comTradeIndexList {
+		comTradeIndexMap[v.IndexCode] = v.IndexNameCn
+	}
+
+	list, err := models.GetAllComTradeDataList()
+	if err != nil {
+		return
+	}
+
+	countryMap, err := GetAllCountry()
+	if err != nil {
+		return
+	}
+
+	lenList := len(list)
+	fmt.Println("总共", lenList, "条")
+
+	dataList := make([]*models.ComTradeCodeMapping, 0)
+	for k, v := range list {
+		fmt.Println("剩余：", lenList-k-1, "条")
+		var reporterCnName, partnerCnName, partner2CnName string
+		if reporter, ok := countryMap[v.ReporterCode]; ok {
+			reporterCnName = reporter.NameCn
+		}
+		if partner, ok := countryMap[v.PartnerCode]; ok {
+			partnerCnName = partner.NameCn
+		}
+		if partner, ok := countryMap[v.Partner2Code]; ok {
+			partner2CnName = partner.NameCn
+		}
+
+		indexNameCn := comTradeIndexMap[v.IndexCode]
+		if indexNameCn == "" {
+			fmt.Println(indexNameCn)
+			fmt.Println(v.IndexCode)
+		}
+		codeStr := "Y"
+		tmpName := indexNameCn + "年度" + reporterCnName
+		if v.DateType == 2 {
+			codeStr = "M"
+			tmpName = indexNameCn + "月度" + reporterCnName
+		}
+		flow := ``
+		if v.Flow == `X` {
+			flow = "出口"
+		} else {
+			flow = "进口"
+		}
+		tmpName += flow
+
+		finalpartnerName := ``
+		if partnerCnName == partner2CnName {
+			if partnerCnName == "世界" {
+				//若对手国和第二对手国是world时，则翻译为总量；
+				finalpartnerName = "总量"
+			}else {
+				//若对手国或第二对手国是相同的，只需翻译一个；
+				finalpartnerName = partnerCnName
+			}
+		} else {
+			if partnerCnName == "世界" || partner2CnName == "世界" {
+				//若对手国或第二对手国是world的，只需翻译非world的对手国；
+				if partnerCnName == "世界" {
+					finalpartnerName = partner2CnName
+				} else {
+					finalpartnerName = partnerCnName
+				}
+			} else {
+				//若对手国和第二对手国不一样，翻译成：对手国和第二对手国
+				finalpartnerName = partnerCnName + "和" + partner2CnName
+			}
+		}
+		tmpName += finalpartnerName
+
+		valueName := tmpName + "贸易金额"
+		weightName := tmpName + "贸易重量"
+
+		if _, ok := codeMappingListMap[v.IndexTradeCode]; !ok {
+			comTradeCodeMappingType := 1
+			if codeStr == "M" {
+				comTradeCodeMappingType = 3
+			}
+			tmpData := &models.ComTradeCodeMapping{
+				Code:       v.IndexTradeCode,
+				IndexCode:  v.IndexCode,
+				Type:       comTradeCodeMappingType,
+				Name:       valueName,
+				CreateTime: time.Now(),
+			}
+			dataList = append(dataList, tmpData)
+			codeMappingListMap[v.IndexTradeCode] = tmpData
+			//tmpErr := models.AddComTradeCodeMapping(tmpData)
+			//if tmpErr != nil {
+			//	fmt.Println(indexCode, "添加贸易金额失败,err:", err)
+			//} else {
+			//	codeMappingListMap[v.IndexTradeCode] = true
+			//}
+		}
+
+		if _, ok := codeMappingListMap[v.IndexNetWeightCode]; !ok {
+			comTradeCodeMappingType := 2
+			if codeStr == "M" {
+				comTradeCodeMappingType = 4
+			}
+			tmpData := &models.ComTradeCodeMapping{
+				Code:       v.IndexNetWeightCode,
+				IndexCode:  v.IndexCode,
+				Type:       comTradeCodeMappingType,
+				Name:       weightName,
+				CreateTime: time.Now(),
+			}
+			dataList = append(dataList, tmpData)
+			codeMappingListMap[v.IndexNetWeightCode] = tmpData
+			//tmpErr := models.AddComTradeCodeMapping(tmpData)
+			//if tmpErr != nil {
+			//	fmt.Println(indexCode, "添加贸易重量失败,err:", err)
+			//} else {
+			//	codeMappingListMap[v.IndexNetWeightCode] = true
+			//}
+		}
+
+
+		if len(dataList) >= 2000 {
+			tmpErr := models.AddMultiComTradeCodeMapping(dataList)
+			if tmpErr != nil {
+				fmt.Println(indexCode, "添加贸易重量失败,err:", tmpErr)
+			}
+			dataList = make([]*models.ComTradeCodeMapping, 0)
+		}
+	}
+
+	if len(dataList) > 0 {
+		tmpErr := models.AddMultiComTradeCodeMapping(dataList)
+		if tmpErr != nil {
+			fmt.Println(indexCode, "添加贸易重量失败,err:", tmpErr)
+		}
+		dataList = make([]*models.ComTradeCodeMapping, 0)
+	}
+	fmt.Println("整体结束了=============")
+	return
+}

services/task.go

@@ -12,6 +12,7 @@ func Task() {
 	//sync3()
 	//syncAllComTradeMapping()
 	//syncAllComTradeMapping2()
+	syncAllComTradeMapping3()
 	//return
 	fmt.Println("start crawler")
 	refreshData := task.NewTask("refreshData", "0 0,30 16-20 * * *", RefreshData)