eta_crawler/services/com_trade.go

package services

import (
	"context"
	"encoding/json"
	"eta/eta_crawler/models"
	"eta/eta_crawler/services/alarm_msg"
	"eta/eta_crawler/utils"
	"fmt"
	"github.com/rdlucklib/rdluck_tools/http"
	"strconv"
	"strings"
	"time"
)

func SyncComTradeData(cont context.Context) (err error) {
	//comTradeId := 1
	//indexCode := "270730"
	//flowCode := "X"
	//year := 2021
	//partnerCode := 842
	//err = syncComTradeData(comTradeId, indexCode, flowCode, partnerCode, year)
	return
}

const ComTradeUrl = "https://comtradeapi.un.org/public/v1/preview/C/A/HS?"
const ComTradeMonthUrl = "https://comtradeapi.un.org/public/v1/preview/C/M/HS?"

//func sync() {
//	//yearList := []int{2009, 2008, 2007, 2006, 2005, 2004, 2003, 2002, 2001, 2000}
//	yearList := []int{2022}
//	flowCodeList := []string{"X", "M"}
//
//	for _, year := range yearList {
//		for _, flow := range flowCodeList {
//			fmt.Println("开始", year, "的", flow)
//			syncAllComTradeData2(year, 0, flow, "year")
//			fmt.Println("结束", year, "的", flow)
//		}
//	}
//}
//
//func sync2() {
//	defer func() {
//		//go utils.SendEmail(utils.APPNAME+"【"+utils.RunMode+"】"+"失败提醒", "RefreshDataFromCoalCoastal ErrMsg:"+err.Error(), utils.EmailSendToUsers)
//		msg := "UN数据同步完成"
//		go alarm_msg.SendAlarmMsg(msg, 3)
//	}()
//	//yearList := []int{2009, 2008, 2007, 2006, 2005, 2004, 2003, 2002, 2001, 2000}
//	//yearList := []int{2022, 2021, 2020, 2019, 2018, 2017, 2016, 2015, 2014, 2013, 2012, 2011, 2010}
//	yearList := []int{2023}
//	//monthList := []int{12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1}
//	monthList := []int{1, 2, 3, 4, 5, 6}
//	flowCodeList := []string{"X", "M"}
//
//	for _, year := range yearList {
//		for _, month := range monthList {
//			for _, flow := range flowCodeList {
//				fmt.Println("开始", year, month, "的", flow)
//				syncAllComTradeData2(year, month, flow, "month")
//				fmt.Println("结束", year, "的", flow)
//			}
//		}
//	}
//}

// SyncYearComTrade 同步年度数据
func SyncYearComTrade(cont context.Context) (err error) {
	errMsg := make([]string, 0)
	defer func() {
		if len(errMsg) > 0 {
			go alarm_msg.SendAlarmMsg("UN年度度数据同步失败："+strings.Join(errMsg, "\n"), 3)
		}
	}()
	flowCodeList := []string{"X", "M"}
	year := time.Now().Year()
	for _, flow := range flowCodeList {
		fmt.Println("开始", year, "的", flow)
		err = syncAllComTradeData2(year, 0, flow, "year")
		if err != nil {
			errMsg = append(errMsg, fmt.Sprint(year, "的", flow, "数据爬取失败：", err.Error()))
		}
		fmt.Println("结束", year, "的", flow)
	}

	return
}

// SyncYearMonthComTrade 同步月度数据
func SyncYearMonthComTrade(cont context.Context) (err error) {
	errMsg := make([]string, 0)
	defer func() {
		if len(errMsg) > 0 {
			go alarm_msg.SendAlarmMsg("UN月度数据同步失败："+strings.Join(errMsg, "\n"), 3)
		}
	}()
	flowCodeList := []string{"X", "M"}

	year := time.Now().Year()
	month := int(time.Now().Month())
	for _, flow := range flowCodeList {
		fmt.Println("开始", year, month, "的", flow)
		err := syncAllComTradeData2(year, month, flow, "month")
		if err != nil {
			errMsg = append(errMsg, fmt.Sprint(year, month, "的", flow, "数据爬取失败：", err.Error()))
		}
		fmt.Println("结束", year, "的", flow)
	}

	return
}

type TmpStruct struct {
	IndexCode   string
	Year        int
	Month       int
	FlowCode    string
	PartnerCode string
}

func sync3() {
	defer func() {
		//go utils.SendEmail(utils.APPNAME+"【"+utils.RunMode+"】"+"失败提醒", "RefreshDataFromCoalCoastal ErrMsg:"+err.Error(), utils.EmailSendToUsers)
		//msg := "UN数据同步完成"
		//go alarm_msg.SendAlarmMsg(msg, 3)
	}()
	str := `2023/01/04 15:16:51.837  [I] [com_trade.go:183]  271114获取数据失败，err:map[error:Something went wrong. Please try again or contact comtrade@un.org];errObject:<nil>；年份：2022;月份：10;flow:M;partnerCode:170,410,246,703,826,233,699,784
2023/01/04 15:16:53.139  [I] [com_trade.go:183]  271114获取数据失败，err:map[error:Something went wrong. Please try again or contact comtrade@un.org];errObject:<nil>；年份：2022;月份：10;flow:M;partnerCode:757,218,764,710,124,579,70,528
2023/01/04 15:25:22.880  [I] [com_trade.go:183]  290121获取数据失败，err:map[error:Something went wrong. Please try again or contact comtrade@un.org];errObject:<nil>；年份：2022;月份：9;flow:X;partnerCode:178,275,328,768,748,84,762,496,108,270,533,894,136,670,418,174,478,446,462,72,204,690,854,562,417,678,104,60,716,52,882
2023/01/04 15:25:24.233  [I] [com_trade.go:183]  290121获取数据失败，err:map[error:Something went wrong. Please try again or contact comtrade@un.org];errObject:<nil>；年份：2022;月份：9;flow:X;partnerCode:470,222,400,268,450,686,860,96,484,352,68,480,512,591,740,780,508,24,858,558,634,646,242,834,340,807,308,116,524,231
2023/01/04 15:25:25.609  [I] [com_trade.go:183]  381900获取数据失败，err:map[error:Something went wrong. Please try again or contact comtrade@un.org];errObject:<nil>；年份：2022;月份：9;flow:X;partnerCode:392,458,600,40,300,608,498,191,360,214,36,100,704,376,32,348
2023/01/04 15:45:47.877  [I] [com_trade.go:183]  271114获取数据失败，err:map[error:Something went wrong. Please try again or contact comtrade@un.org];errObject:<nil>；年份：2022;月份：8;flow:M;partnerCode:156,170,804,764,246
2023/01/04 15:45:49.197  [I] [com_trade.go:183]  271114获取数据失败，err:map[error:Something went wrong. Please try again or contact comtrade@un.org];errObject:<nil>；年份：2022;月份：8;flow:M;partnerCode:440,100,196,31,498,36,392,348,376,40,208,360,76,682,191,214
2023/01/04 15:45:50.491  [I] [com_trade.go:183]  271114获取数据失败，err:map[error:Something went wrong. Please try again or contact comtrade@un.org];errObject:<nil>；年份：2022;月份：8;flow:M;partnerCode:300,600,32,458,554,608,704,112
2022/12/30 08:31:53.846  [I] [com_trade.go:176]  270730获取数据失败，err:map[error:Something went wrong. Please try again or contact comtrade@un.org];errObject:<nil>；年份：2019;月份：1;flow:X;partnerCode:458,36,360,376,76,214,40,348,608,498,112,682,208,600,392,196
2022/12/30 08:31:55.167  [I] [com_trade.go:176]  270730获取数据失败，err:map[error:Something went wrong. Please try again or contact comtrade@un.org];errObject:<nil>；年份：2019;月份：1;flow:X;partnerCode:100,554,191,300,32,704,31,440
2022/12/30 08:53:56.893  [I] [com_trade.go:176]  290230获取数据失败，err:map[error:Something went wrong. Please try again or contact comtrade@un.org];errObject:<nil>；年份：2018;月份：12;flow:X;partnerCode:882,72,136,894,178,417,854,174,670,690,748,496,328,418,104,762,533,678,462,446,716,52,478,270,562,768,275,84,60,204,108
2022/12/30 08:53:59.079  [I] [com_trade.go:176]  290230获取数据失败，err:map[error:Something went wrong. Please try again or contact comtrade@un.org];errObject:<nil>；年份：2018;月份：12;flow:X;partnerCode:40,458,600,76,376,682,608,31,32,348,704,100,300,498,392,360
2022/12/30 09:16:34.790  [I] [com_trade.go:176]  270750获取数据失败，err:map[error:Something went wrong. Please try again or contact comtrade@un.org];errObject:<nil>；年份：2018;月份：11;flow:X;partnerCode:703,826,702,757,218,428,752,56
2022/12/30 09:16:36.185  [I] [com_trade.go:176]  270750获取数据失败，err:map[error:Something went wrong. Please try again or contact comtrade@un.org];errObject:<nil>；年份：2018;月份：11;flow:X;partnerCode:410,842,528,246,170,70,579,792
2022/12/30 09:16:37.641  [I] [com_trade.go:176]  270750获取数据失败，err:map[error:Something went wrong. Please try again or contact comtrade@un.org];errObject:<nil>；年份：2018;月份：11;flow:X;partnerCode:156,620,251,616,764
2022/12/30 09:23:56.668  [I] [com_trade.go:176]  290244获取数据失败，err:map[error:Something went wrong. Please try again or contact comtrade@un.org];errObject:<nil>；年份：2018;月份：11;flow:X;partnerCode:624,368,891,837,266,527,50,278,500,662,180,226,238,638,312,175,532,492,666,698,434,868,254,674,90,4,184,577,776,140,588,296,890,388,706,899,866,288,800,850,697,720,472,316,221,336,585,64,292,48,200,258,332,466,548,454,879,408,356,836,516,568,334,637,728,120,290,239,86,647,426,459,12,232,583,262,772,849,92,582,574,534,16,660,398,520,862,535,166,473,10,471,234,760,570,457,28,129,230,58,192,364,304,430,732,581,148,461,711,796,798,590,717,598,8,474,658,729,654,20,384,736,835,626,531,592,260,324,841,580,530,132,414,536,652,74,212,612,540,887,694,162,886,659,795,876,44,80,280,839,810,838,584,636
2022/12/30 09:23:58.075  [I] [com_trade.go:176]  290244获取数据失败，err:map[error:Something went wrong. Please try again or contact comtrade@un.org];errObject:<nil>；年份：2018;月份：11;flow:X;partnerCode:705,710,642,699,490,724,442,784
2022/12/30 09:31:23.532  [I] [com_trade.go:176]  290122获取数据失败，err:map[error:Something went wrong. Please try again or contact comtrade@un.org];errObject:<nil>；年份：2018;月份：11;flow:M;partnerCode:757,752,620,642,702
2022/12/30 09:31:24.940  [I] [com_trade.go:176]  290122获取数据失败，err:map[error:Something went wrong. Please try again or contact comtrade@un.org];errObject:<nil>；年份：2018;月份：11;flow:M;partnerCode:174,328,748,84,562,882,204,104,670,60,418,678,108,768,178,275,533,462,690,446,496,716,270,478,894,417,854,72,762,52,136
2022/12/30 10:17:07.994  [I] [com_trade.go:176]  270750获取数据失败，err:map[error:Something went wrong. Please try again or contact comtrade@un.org];errObject:<nil>；年份：2018;月份：8;flow:X;partnerCode:583,368,711,810,838,28,652,666,849,868,772,175,304,570,577,638,12,80,592,580,798,132,288,356,332,239,492,461,4,180,588,887,474,516,659,534,886,312,760,862,58,527,582,732,472,324,520,662,729,166,531,698,90,388,879,674,624,500,891,626,140,795,20,200,736,316,568,234,426,162,336,850,636,776,530,64,148,835,800,184,262,612,120,260,232,890,414,796,585,292,540,192,258,10,44,384,536,654,658,899,590,841,876,334,50,697,278,74,836,434,454,647,837,408,266,457,8,86,296,866,584,212,728,535,471,129,398,430,459,48,238,706,581,473,466,694,92,226,548,364,574,660,720,230,717,598,221,839,280,532,290,637,16,254
2022/12/30 10:46:04.444  [I] [com_trade.go:176]  290244获取数据失败，err:map[error:Something went wrong. Please try again or contact comtrade@un.org];errObject:<nil>；年份：2018;月份：7;flow:X;partnerCode:780,340,116,834,858,558,524,24,686,400,860,512,268,96,508,242,222,231,646,740,470,634,480,484,308,68,352,450,591,807
2022/12/30 10:46:05.864  [I] [com_trade.go:176]  290244获取数据失败，err:map[error:Something went wrong. Please try again or contact comtrade@un.org];errObject:<nil>；年份：2018;月份：7;flow:X;partnerCode:682,76,704,348,376,498,600,608,360,214,300,392,100,40,32,31
2022/12/30 11:22:44.563  [I] [com_trade.go:176]  290243获取数据失败，err:map[error:Something went wrong. Please try again or contact comtrade@un.org];errObject:<nil>；年份：2018;月份：5;flow:M;partnerCode:276,688,784,792,56,616,124,703
2022/12/30 11:22:45.936  [I] [com_trade.go:176]  290243获取数据失败，err:map[error:Something went wrong. Please try again or contact comtrade@un.org];errObject:<nil>；年份：2018;月份：5;flow:M;partnerCode:428,579,702,170,620,528,643,764
2022/12/30 11:40:43.264  [I] [com_trade.go:176]  290244获取数据失败，err:map[error:Something went wrong. Please try again or contact comtrade@un.org];errObject:<nil>；年份：2018;月份：4;flow:M;partnerCode:56,703,218,490,170,156,616,410
2022/12/30 11:40:44.553  [I] [com_trade.go:176]  290244获取数据失败，err:map[error:Something went wrong. Please try again or contact comtrade@un.org];errObject:<nil>；年份：2018;月份：4;flow:M;partnerCode:784,70,579,251,699,233,705,752
2022/12/30 22:41:22.216  [I] [com_trade.go:176]  290230获取数据失败，err:map[error:Something went wrong. Please try again or contact comtrade@un.org];errObject:<nil>；年份：2017;月份：7;flow:M;partnerCode:528,642,70,784,792,710,842,56
2022/12/30 22:41:23.516  [I] [com_trade.go:176]  290230获取数据失败，err:map[error:Something went wrong. Please try again or contact comtrade@un.org];errObject:<nil>；年份：2017;月份：7;flow:M;partnerCode:246,705,251,218,757
2022/12/30 22:41:24.824  [I] [com_trade.go:176]  270750获取数据失败，err:map[error:Something went wrong. Please try again or contact comtrade@un.org];errObject:<nil>；年份：2017;月份：7;flow:M;partnerCode:0
2022/12/30 22:41:26.138  [I] [com_trade.go:176]  270750获取数据失败，err:map[error:Something went wrong. Please try again or contact comtrade@un.org];errObject:<nil>；年份：2017;月份：7;flow:M;partnerCode:68,512,470,222,558,858,508,308,352,591,231,242,780,24,484,807,740,646,524,686,340,268,116,634,860,480,96,400,450,834
2022/12/30 22:41:27.457  [I] [com_trade.go:176]  270750获取数据失败，err:map[error:Something went wrong. Please try again or contact comtrade@un.org];errObject:<nil>；年份：2017;月份：7;flow:M;partnerCode:304,624,332,654,795,841,258,577,879,732,835,527,647,86,866,706,232,796,638,4,80,717,12,492,454,798,180,44,697,838,520,776,262,473,887,16,540,129,192,772,850,280,637,581,570,839,810,90,20,334,474,459,230,388,662,534,50,120,288,466,530,531,590,316,398,471,221,175,74,580,891,711,212,720,598,666,800,886,8,238,414,28,64,312,658,296,849,408,278,694,568,698,48,140,584,461,736,836,292,430,659,760,336,472,899,583,536,636,588,239,166,384,457,516,574,837,58,868,254,368,652,226,612,862,266,434,728,200,592,290,674,548,585,729,92,162,364,532,626,10,184,426,148,660,535,234,324,500,356,132,582,890,260,876
2022/12/30 23:13:14.705  [I] [com_trade.go:176]  271114获取数据失败，err:map[error:Something went wrong. Please try again or contact comtrade@un.org];errObject:<nil>；年份：2017;月份：6;flow:M;partnerCode:200,624,636,776,384,140,266,304,12,278,28,260,835,674,531,841,837,430,659,296,472,612,474,520,86,120,457,10,74,500,132,368,637,454,316,434,221,536,459,839,585,810,890,706,862,148,582,698,492,336,795,581,592,577,80,162,548,868,232,426,568,626,262,580,570,654,729,866,175,598,732,64,717,736,838,899,720,398,290,658,212,292,527,638,129,8,92,662,728,879,588,234,58,414,711,760,408,324,388,583,584,534,530,471,192,886,666,697,574,184,90,4,20,516,647,312,364,473,652,694,48,50,590,254,258,772,796,230,288,332,334,849,876,660,540,887,16,466,891,44,166,180,532,850,800,535,226,238,356,239,798,836,461,280
2022/12/30 23:13:16.091  [I] [com_trade.go:176]  271114获取数据失败，err:map[error:Something went wrong. Please try again or contact comtrade@un.org];errObject:<nil>；年份：2017;月份：6;flow:M;partnerCode:222,524,470,231,634,740,24,780,512,508,807,340,484,686,834,352,558,268,68,242,96,116,860,480,858,591,450,646,308,400
2022/12/30 23:13:17.486  [I] [com_trade.go:176]  290122获取数据失败，err:map[error:Something went wrong. Please try again or contact comtrade@un.org];errObject:<nil>；年份：2017;月份：6;flow:M;partnerCode:608,498,112,36,682,704,40,348,208,392,600,554,300,440,196,458
2022/12/31 00:05:18.625  [I] [com_trade.go:176]  271114获取数据失败，err:map[error:Something went wrong. Please try again or contact comtrade@un.org];errObject:<nil>；年份：2017;月份：3;flow:M;partnerCode:826,56,757,724,233,251,752,702
2022/12/31 00:05:19.966  [I] [com_trade.go:176]  271114获取数据失败，err:map[error:Something went wrong. Please try again or contact comtrade@un.org];errObject:<nil>；年份：2017;月份：3;flow:M;partnerCode:410,688,703,616,276,620,579,380
2022/12/31 00:05:21.315  [I] [com_trade.go:176]  271114获取数据失败，err:map[error:Something went wrong. Please try again or contact comtrade@un.org];errObject:<nil>；年份：2017;月份：3;flow:M;partnerCode:170,710,528,490,699,428,124,792
2022/12/31 00:46:55.568  [I] [com_trade.go:176]  290122获取数据失败，err:map[error:Something went wrong. Please try again or contact comtrade@un.org];errObject:<nil>；年份：2017;月份：2;flow:M;partnerCode:0
2022/12/31 00:46:56.919  [I] [com_trade.go:176]  290121获取数据失败，err:map[error:Something went wrong. Please try again or contact comtrade@un.org];errObject:<nil>；年份：2017;月份：2;flow:M;partnerCode:697,886,254,80,166,536,324,426,839,796,50,221,800,12,461,364,728,16,48,132,230,660,212,798,548,568,662,184,585,414,500,760,772,527,86,574,288,459,534,590,582,850,368,570,666,408,334,530,336,866,810,90,887,588,332,148,581,612,532,260,388,838,659,472,238,520,795,180,835,296,706,652,278,836,698,717,540,531,862,8,384,290,434,454,636,736,535,262,473,292,624,140,162,304,776,74,234,266,891,20,899,58,841,879,120,654,694,44,466,356,474,658,674,457,10,258,890,647,729,583,638,876,92,471,129,626,732,280,398,637,430,516,200,232,192,312,175,239,837,720,711,849,64,580,4,592,28,577,492,316,584,598,226,868
2022/12/31 00:46:58.356  [I] [com_trade.go:176]  290121获取数据失败，err:map[error:Something went wrong. Please try again or contact comtrade@un.org];errObject:<nil>；年份：2017;月份：2;flow:M;partnerCode:246,643,156,688,826,528,804,616
2022/12/31 04:15:22.974  [I] [com_trade.go:176]  290122获取数据失败，err:map[error:Something went wrong. Please try again or contact comtrade@un.org];errObject:<nil>；年份：2016;月份：10;flow:M;partnerCode:404,372,604,422,504,788,344,188,51,320,566,586,144,152,818,499
2022/12/31 04:15:24.490  [I] [com_trade.go:176]  290122获取数据失败，err:map[error:Something went wrong. Please try again or contact comtrade@un.org];errObject:<nil>；年份：2016;月份：10;flow:M;partnerCode:234,296,58,192,16,212,772,694,659,536,457,92,612,760,798,837,527,654,132,592,12,408,582,568,711,384,879,364,532,162,166,200,638,20,574,891,254,535,492,466,266,140,890,74,86,4,660,44,540,776,584,580,585,80,221,796,570,534,239,414,500,636,148,238,312,666,129,232,64,332,292,48,262,581,471,862,304,278,583,530,626,8,520,838,280,800,90,588,28,324,473,662,697,652,720,290,334,598,474,10,899,368,461,180,388,430,835,184,531,647,728,732,288,454,850,624,434,698,398,868,839,356,226,472,336,120,230,717,887,637,577,795,674,706,548,516,50,810,729,836,459,886,258,590,260,866,849,316,426,658,175,841,876,736
2022/12/31 04:15:26.001  [I] [com_trade.go:176]  290122获取数据失败，err:map[error:Something went wrong. Please try again or contact comtrade@un.org];errObject:<nil>；年份：2016;月份：10;flow:M;partnerCode:231,646,834,591,340,480,68,524,24,308,558,400,352,268,450,116,96,740,860,222,634,780,484,512,242,807,858,470,508,686
2022/12/31 08:06:11.934  [I] [com_trade.go:176]  271114获取数据失败，err:map[error:Something went wrong. Please try again or contact comtrade@un.org];errObject:<nil>；年份：2015;月份：12;flow:M;partnerCode:699,642,705,380,579,490,643,703
2022/12/31 08:06:13.619  [I] [com_trade.go:176]  271114获取数据失败，err:map[error:Something went wrong. Please try again or contact comtrade@un.org];errObject:<nil>；年份：2015;月份：12;flow:M;partnerCode:724,710,156,428,620,70,784,616
2022/12/31 09:51:10.568  [I] [com_trade.go:176]  290244获取数据失败，err:map[error:Something went wrong. Please try again or contact comtrade@un.org];errObject:<nil>；年份：2015;月份：4;flow:X;partnerCode:826,70,156,702,428,705,792,764
2022/12/31 09:51:12.117  [I] [com_trade.go:176]  290244获取数据失败，err:map[error:Something went wrong. Please try again or contact comtrade@un.org];errObject:<nil>；年份：2015;月份：4;flow:X;partnerCode:642,203,579,620,752,410,528,804
2022/12/31 09:58:54.330  [I] [com_trade.go:176]  381900获取数据失败，err:map[error:Something went wrong. Please try again or contact comtrade@un.org];errObject:<nil>；年份：2015;月份：4;flow:M;partnerCode:218,276,826,842,764,699,124,643
2023/01/03 22:26:25.230  [I] [com_trade.go:176]  270730获取数据失败，err:map[error:Something went wrong. Please try again or contact comtrade@un.org];errObject:<nil>；年份：2003;月份：8;flow:X;partnerCode:682,31,348,40,208,458,704,376,32,191,392,360,36,196,554,600
2023/01/03 22:34:20.413  [I] [com_trade.go:176]  270750获取数据失败，err:map[error:Something went wrong. Please try again or contact comtrade@un.org];errObject:<nil>；年份：2003;月份：8;flow:M;partnerCode:591,308,352,268,450,524,231,484,807,558,68,646,24,834,740,222,480,470,96,508,858,400,116,340,634,780,242,860,512,686
2023/01/03 22:34:22.005  [I] [com_trade.go:176]  270750获取数据失败，err:map[error:Something went wrong. Please try again or contact comtrade@un.org];errObject:<nil>；年份：2003;月份：8;flow:M;partnerCode:894,174,104,84,108,270,418,678,716,204,178,462,496,478,670,60,328,690,533,136,446,768,417,762,562,72,748,854,882,275,52
2023/01/03 22:47:23.927  [I] [com_trade.go:176]  290243获取数据失败，err:map[error:Something went wrong. Please try again or contact comtrade@un.org];errObject:<nil>；年份：2003;月份：7;flow:M;partnerCode:591,646,508,24,308,268,558,634,807,480,512,834,780,242,524,352,450,484,470,96,686,116,231,858,340,740,222,860,400,68
2023/01/03 22:47:25.509  [I] [com_trade.go:176]  290243获取数据失败，err:map[error:Something went wrong. Please try again or contact comtrade@un.org];errObject:<nil>；年份：2003;月份：7;flow:M;partnerCode:32,214,704,376,208,360,191,112,600,348,554,300,100,76,40,196
2023/01/03 22:47:26.957  [I] [com_trade.go:176]  290243获取数据失败，err:map[error:Something went wrong. Please try again or contact comtrade@un.org];errObject:<nil>；年份：2003;月份：7;flow:M;partnerCode:682,392,36,608,440,31,458,498
2023/01/03 22:56:11.582  [I] [com_trade.go:176]  290230获取数据失败，err:map[error:Something went wrong. Please try again or contact comtrade@un.org];errObject:<nil>；年份：2003;月份：6;flow:X;partnerCode:68,508,400,860,231,740,242,512,96,807,470,268,858,524,222,450,308,646,686,591,480,558,780,634,834,24,340,116,484,352
2023/01/03 22:56:12.894  [I] [com_trade.go:176]  290230获取数据失败，err:map[error:Something went wrong. Please try again or contact comtrade@un.org];errObject:<nil>；年份：2003;月份：6;flow:X;partnerCode:136,446,496,882,690,204,60,275,462,533,108,178,670,716,52,104,762,328,418,270,84,894,768,72,854,174,417,562,678,478,748
2023/01/03 23:09:42.931  [I] [com_trade.go:176]  290122获取数据失败，err:map[error:Something went wrong. Please try again or contact comtrade@un.org];errObject:<nil>；年份：2003;月份：5;flow:M;partnerCode:608,112,40,214,682,392,458,31,348,498,76,600,554,704,191,300
2023/01/03 23:09:44.481  [I] [com_trade.go:176]  290122获取数据失败，err:map[error:Something went wrong. Please try again or contact comtrade@un.org];errObject:<nil>；年份：2003;月份：5;flow:M;partnerCode:32,36,360,440,208,196,100,376
2023/01/03 23:17:27.853  [I] [com_trade.go:176]  270730获取数据失败，err:map[error:Something went wrong. Please try again or contact comtrade@un.org];errObject:<nil>；年份：2003;月份：5;flow:M;partnerCode:608,112,40,214,682,392,458,31,348,498,76,600,554,704,191,300
2023/01/03 23:17:29.534  [I] [com_trade.go:176]  270730获取数据失败，err:map[error:Something went wrong. Please try again or contact comtrade@un.org];errObject:<nil>；年份：2003;月份：5;flow:M;partnerCode:32,36,360,440,208,196,100,376
2023/01/03 23:25:00.828  [I] [com_trade.go:176]  381900获取数据失败，err:map[error:Something went wrong. Please try again or contact comtrade@un.org];errObject:<nil>；年份：2003;月份：4;flow:M;partnerCode:40,76,704,208,458,608,214,31,300,112,196,191,600,360,348,682
2023/01/04 01:02:18.808  [I] [com_trade.go:176]  290230获取数据失败，err:map[error:Something went wrong. Please try again or contact comtrade@un.org];errObject:<nil>；年份：2002;月份：6;flow:M;partnerCode:835,92,772,230,624,694,192,527,899,384,534,711,80,226,520,647,887,10,434,536,570,254,278,332,530,166,584,876,44,368,414,221,810,636,471,180,583,388,585,64,836,736,212,532,638,706,866,334,408,839,548,292,74,304,598,891,28,426,20,398,473,238,540,666,796,862,4,868,175,492,890,500,850,262,461,290,472,336,296,580,577,459,232,457,8,886,717,90,776,129,430,652,48,760,50,720,728,588,590,637,654,732,474,260,612,581,841,531,837,132,697,592,266,316,140,148,184,356,454,12,364,466,698,795,658,674,838,568,312,86,582,729,879,16,200,288,120,234,574,535,662,798,162,258,239,660,280,659,800,58,516,626,849,324
2023/01/04 01:02:20.180  [I] [com_trade.go:176]  290230获取数据失败，err:map[error:Something went wrong. Please try again or contact comtrade@un.org];errObject:<nil>；年份：2002;月份：6;flow:M;partnerCode:32,31,191,554,76,300,100,348,458,214,40,196,392,704,376,360
2023/01/04 01:09:53.528  [I] [com_trade.go:176]  290121获取数据失败，err:map[error:Something went wrong. Please try again or contact comtrade@un.org];errObject:<nil>；年份：2002;月份：5;flow:X;partnerCode:233,428,203,528,724,792,703,380
2023/01/04 01:52:58.039  [I] [com_trade.go:176]  270750获取数据失败，err:map[error:Something went wrong. Please try again or contact comtrade@un.org];errObject:<nil>；年份：2002;月份：1;flow:X;partnerCode:0
2023/01/04 02:00:21.549  [I] [com_trade.go:176]  290122获取数据失败，err:map[error:Something went wrong. Please try again or contact comtrade@un.org];errObject:<nil>；年份：2002;月份：1;flow:M;partnerCode:0
2023/01/04 02:12:09.179  [I] [com_trade.go:176]  290243获取数据失败，err:map[error:Something went wrong. Please try again or contact comtrade@un.org];errObject:<nil>；年份：2001;月份：12;flow:X;partnerCode:688,579,752,380,764,842,156,70
2023/01/04 02:29:32.109  [I] [com_trade.go:176]  290243获取数据失败，err:map[error:Something went wrong. Please try again or contact comtrade@un.org];errObject:<nil>；年份：2001;月份：11;flow:M;partnerCode:710,442,699,616,703
2023/01/04 02:29:33.532  [I] [com_trade.go:176]  290243获取数据失败，err:map[error:Something went wrong. Please try again or contact comtrade@un.org];errObject:<nil>；年份：2001;月份：11;flow:M;partnerCode:0
2023/01/04 02:29:34.888  [I] [com_trade.go:176]  290243获取数据失败，err:map[error:Something went wrong. Please try again or contact comtrade@un.org];errObject:<nil>；年份：2001;月份：11;flow:M;partnerCode:768,204,84,854,60,72,716,136,275,748,178,762,562,496,446,690,104,678,894,478,52,328,533,882,108,270,462,417,670,418,174
2023/01/04 02:37:00.528  [I] [com_trade.go:176]  381900获取数据失败，err:map[error:Something went wrong. Please try again or contact comtrade@un.org];errObject:<nil>；年份：2001;月份：10;flow:X;partnerCode:152,372,504,188,499,818,344,404,422,566,144,604,320,51,788,586
2023/01/04 02:37:01.816  [I] [com_trade.go:176]  381900获取数据失败，err:map[error:Something went wrong. Please try again or contact comtrade@un.org];errObject:<nil>；年份：2001;月份：10;flow:X;partnerCode:882,533,690,84,270,275,496,446,104,462,60,762,204,417,418,328,748,478,72,52,854,136,178,562,670,716,108,678,894,174,768
2023/01/04 02:37:03.143  [I] [com_trade.go:176]  381900获取数据失败，err:map[error:Something went wrong. Please try again or contact comtrade@un.org];errObject:<nil>；年份：2001;月份：10;flow:X;partnerCode:0
2023/01/04 02:44:23.164  [I] [com_trade.go:176]  290243获取数据失败，err:map[error:Something went wrong. Please try again or contact comtrade@un.org];errObject:<nil>；年份：2001;月份：10;flow:M;partnerCode:528,710,688,642,784,442,124,705
2023/01/04 02:44:24.971  [I] [com_trade.go:176]  290243获取数据失败，err:map[error:Something went wrong. Please try again or contact comtrade@un.org];errObject:<nil>；年份：2001;月份：10;flow:M;partnerCode:616,410,724,804,490,826,380,620
2023/01/04 02:58:29.473  [I] [com_trade.go:176]  270750获取数据失败，err:map[error:Something went wrong. Please try again or contact comtrade@un.org];errObject:<nil>；年份：2001;月份：9;flow:M;partnerCode:498,682,600,458,392,32,440,360
2023/01/04 02:58:31.133  [I] [com_trade.go:176]  270750获取数据失败，err:map[error:Something went wrong. Please try again or contact comtrade@un.org];errObject:<nil>；年份：2001;月份：9;flow:M;partnerCode:566,422,144,788,504,404,152,188,604,372,818,320,586,499,51,344
2023/01/04 03:24:13.204  [I] [com_trade.go:176]  270730获取数据失败，err:map[error:Something went wrong. Please try again or contact comtrade@un.org];errObject:<nil>；年份：2001;月份：7;flow:M;partnerCode:604,144,320,372,586,566,422,344,499,188,788,504,818,51,404,152
2023/01/04 03:24:14.661  [I] [com_trade.go:176]  270730获取数据失败，err:map[error:Something went wrong. Please try again or contact comtrade@un.org];errObject:<nil>；年份：2001;月份：7;flow:M;partnerCode:882,72,462,768,270,690,678,894,478,108,52,104,84,60,670,762,496,446,417,204,562,533,328,716,174,178,418,275,854,136,748
2023/01/04 03:33:55.137  [I] [com_trade.go:176]  270730获取数据失败，err:map[error:Something went wrong. Please try again or contact comtrade@un.org];errObject:<nil>；年份：2001;月份：6;flow:M;partnerCode:191,360,682,600,704,440,554,392,196,40,36,100,32,76,112,214`
	tmpList := strings.Split(str, "\n")

	list := make([]TmpStruct, 0)

	for _, v := range tmpList {
		//fmt.Println(v)
		tmp := strings.Split(v, "获取数据失败")
		//fmt.Println(tmp)

		// 指标编码
		codeList := strings.Split(tmp[0], "  ")
		indexCode := codeList[2]
		//fmt.Println(indexCode)

		// 年度
		//年份：2001;月份：6;flow:M;partnerCode:191,360,682,600,704,440,554,392,196,40,36,100,32,76,112,214`
		tmp = strings.Split(tmp[1], "年份：")
		// 指标编码
		tmpList2 := strings.Split(tmp[1], ";")
		yearStr := tmpList2[0]
		year, _ := strconv.Atoi(yearStr)
		//fmt.Println(year)

		// 月度
		//月份：6;flow:M;partnerCode:191,360,682,600,704,440,554,392,196,40,36,100,32,76,112,214`
		tmp = strings.Split(tmp[1], "月份：")
		// 指标编码
		tmpList2 = strings.Split(tmp[1], ";")
		monthStr := tmpList2[0]
		month, _ := strconv.Atoi(monthStr)
		//fmt.Println(month)

		// flow
		//flow:M;partnerCode:191,360,682,600,704,440,554,392,196,40,36,100,32,76,112,214`
		tmp = strings.Split(tmp[1], "flow:")
		// 指标编码
		tmpList2 = strings.Split(tmp[1], ";")
		flowCode := tmpList2[0]
		//fmt.Println(flowCode)

		// partnerCode
		//flow:M;partnerCode:191,360,682,600,704,440,554,392,196,40,36,100,32,76,112,214`
		tmp = strings.Split(tmp[1], "partnerCode:")
		// 指标编码
		partnerCode := tmp[1]
		//fmt.Println(partnerCode)

		list = append(list, TmpStruct{
			IndexCode:   indexCode,
			Year:        year,
			Month:       month,
			FlowCode:    flowCode,
			PartnerCode: partnerCode,
		})
	}

	//year := 2021
	//month := 6
	//flowCode := "M"
	//comTradeId := 6
	//indexCode := `381900`
	//indexNameCn := `石脑油（出口退税率0%）`
	//thirdIdStr := `636,278,624,839,384,234,296,694,866,835,50,230,662,48,697,290,12,388,454,457,582,798,80,364,674,58,836,838,44,86,500,471,90,368,795,647,899,720,336,520,226,800,612,548,16,531,332,398,652,472,473,581,430,536,568,879,717,598,706,8,258,638,886,810,312,334,534,254,324,841,120,492,148,577,760,592,666`
	//thirdIdStr := `588,583,316,849,887,626,129,580,530,288,414,796,466,584,891,292,356,426,540,74,184,459,10,535,166,4,266,585,658,862,660,232,92,239,0,776,474,280,736,64,221,516,132,698,162,434,461,180,654,262,570,850,238,175,659,637,711,527,732,20,140,868,532,200,728,729,772,192,260,28,212,890,876,837,590,408,304,574`
	//thirdIdStr := `203,218,276,757`
	//thirdIdStr := `380,784,56,528`
	dateType := `month`
	//codeList := []string{
	//	`835,729,120,221,500,184,200,0,666,258,624,426,532,574,720,536,534,471,581,312,434,568,659,408,334,590,162,296,530,140,80,90,8,711`,
	//	`866,732,736,364,459,698,706,12,674,234,516,891,849,260,304,388,899,492,612,660,772,694,862,697,592,472,324,531,226,92,28,232,583,647,654,166,64,838,336,430,58`,
	//}

	codeMappingListMap := make(map[string]string)
	codeMappingList, err := models.GetAllComTradeCodeMappingList()
	if err != nil {
		return
	}
	for _, v := range codeMappingList {
		codeMappingListMap[v.Code] = v.Name
	}

	countryMap, err := GetAllCountry()

	comTradeList, err := models.GetAllComTradeIndexList()
	if err != nil {
		return
	}
	comTradeMap := make(map[string]*models.ComTradeIndex)
	for _, v := range comTradeList {
		comTradeMap[v.IndexCode] = v
	}

	for key, v := range list {
		comTradeInfo, ok := comTradeMap[v.IndexCode]
		if !ok {
			fmt.Println("找不到指标")
		}
		//fmt.Println(v)
		fmt.Println(key, ";开始：", ";code：", v.IndexCode, "；年份：", v.Year, "；月份：", v.Month, "的", v.FlowCode, ";国家：", v.PartnerCode)
		syncComTradeData(comTradeInfo.ComTradeId, v.IndexCode, comTradeInfo.IndexNameCn, v.FlowCode, v.PartnerCode, dateType, v.Year, v.Month, codeMappingListMap, countryMap)
		fmt.Println(key, ";结束：", ";code：", v.IndexCode, "；年份：", v.Year, "；月份：", v.Month, "的", v.FlowCode, ";国家：", v.PartnerCode)
		time.Sleep(1 * time.Second)
	}
}

func syncAllComTradeData2(year, month int, flowCode, dateType string) (err error) {
	codeMappingListMap := make(map[string]string)
	codeMappingList, err := models.GetAllComTradeCodeMappingList()
	if err != nil {
		return
	}
	for _, v := range codeMappingList {
		codeMappingListMap[v.Code] = v.Name
	}

	comTradeList, err := models.GetAllComTradeIndexList()
	if err != nil {
		return
	}
	lenComTradeList := len(comTradeList)

	countryMap, err := GetAllCountry()
	if err != nil {
		return
	}

	countryGroup := make(map[int][]models.ComTradeCountry)
	for _, v := range countryMap {
		togetherCountry := v.TogetherCountry
		if dateType == "month" {
			togetherCountry = v.MonthTogetherCountry
		}
		tmpList, ok := countryGroup[togetherCountry]
		if !ok {
			tmpList = make([]models.ComTradeCountry, 0)
		}
		countryGroup[togetherCountry] = append(tmpList, v)
	}
	//fmt.Println(countryGroup)
	i := 0
	startTime := time.Now()
	fmt.Println("整体开始了=============")
	for _, v := range comTradeList {
		fmt.Println("剩余", lenComTradeList-i, "个指标=============")
		syncSingleComTradeData(v.ComTradeId, year, month, flowCode, v.IndexCode, v.IndexNameCn, dateType, countryMap, codeMappingListMap, countryGroup)
		i++
	}
	fmt.Println("整体结束了=============")
	fmt.Println("耗时", time.Now().Sub(startTime).String())
	return
}

func syncAllComTradeData() (err error) {
	codeMappingListMap := make(map[string]string)
	codeMappingList, err := models.GetAllComTradeCodeMappingList()
	if err != nil {
		return
	}
	for _, v := range codeMappingList {
		codeMappingListMap[v.Code] = v.Name
	}

	comTradeList, err := models.GetAllComTradeIndexList()
	if err != nil {
		return
	}
	lenComTradeList := len(comTradeList)

	countryMap, err := GetAllCountry()
	if err != nil {
		return
	}

	countryGroup := make(map[int][]models.ComTradeCountry)
	for _, v := range countryMap {
		tmpList, ok := countryGroup[v.TogetherCountry]
		if !ok {
			tmpList = make([]models.ComTradeCountry, 0)
		}
		countryGroup[v.TogetherCountry] = append(tmpList, v)
	}
	//fmt.Println(countryGroup)
	//flowCode := "X"
	flowCode := "M"
	year := 2022
	i := 0
	startTime := time.Now()
	fmt.Println("整体开始了=============")
	for _, v := range comTradeList {
		fmt.Println("剩余", lenComTradeList-i, "个指标=============")
		syncSingleComTradeData(v.ComTradeId, year, 0, flowCode, v.IndexCode, v.IndexNameCn, "year", countryMap, codeMappingListMap, countryGroup)
		i++
	}
	fmt.Println("整体结束了=============")
	fmt.Println("耗时", time.Now().Sub(startTime).String())
	return
}

func syncSingleComTradeData(comTradeId, year, month int, flowCode, indexCode, indexNameCn, dateType string, countryMap map[int]models.ComTradeCountry, codeMappingListMap map[string]string, countryGroup map[int][]models.ComTradeCountry) {
	////flowCode := "X"
	//flowCode := "M"
	//year := 2020
	//partnerCode := 842
	lenCountry := len(countryMap)

	i := 0
	fmt.Println(indexCode, "开始了")

	for needMergeNum, countryList := range countryGroup {
		mergeNum := 0
		thirdIdList := make([]string, 0)
		lenCountryList := len(countryList) - 1

		for countryIndex, v := range countryList {
			fmt.Println(indexCode, "剩余", lenCountry-i, "条数据")
			i++

			// 开始处理
			mergeNum++
			thirdIdList = append(thirdIdList, strconv.Itoa(v.ThirdId))
			if mergeNum < needMergeNum && countryIndex < lenCountryList {
				continue
			}
			syncComTradeData(comTradeId, indexCode, indexNameCn, flowCode, strings.Join(thirdIdList, ","), dateType, year, month, codeMappingListMap, countryMap)

			// 重置
			mergeNum = 0
			thirdIdList = make([]string, 0)

			time.Sleep(1 * time.Second)
		}

	}
	fmt.Println(indexCode, "结束了")
}

func SyncSingleComTradeData3() {
	comTradeId := 7
	year := 2022
	month := 11
	flowCode := "M"
	indexCode := "290121"
	indexNameCn := `乙烯（出口退税率13%）`
	dateType := `month`
	codeMappingListMap := make(map[string]string)
	codeMappingList, err := models.GetAllComTradeCodeMappingList()
	if err != nil {
		return
	}
	for _, v := range codeMappingList {
		codeMappingListMap[v.Code] = v.Name
	}

	countryMap, err := GetAllCountry()
	if err != nil {
		return
	}

	countryGroup := make(map[int][]models.ComTradeCountry)
	for _, v := range countryMap {
		togetherCountry := v.TogetherCountry
		if dateType == "month" {
			togetherCountry = v.MonthTogetherCountry
		}
		tmpList, ok := countryGroup[togetherCountry]
		if !ok {
			tmpList = make([]models.ComTradeCountry, 0)
		}
		countryGroup[togetherCountry] = append(tmpList, v)
	}
	//fmt.Println(countryGroup)

	fmt.Println(indexCode, "开始了")

	thirdIdStr := `849,192,12,296,212,473,234,364,879,580,698,20,582,850,316,568,459,697,866,540,4,280,288,461,658,44,430,598,798,868,92,180,536,810,574,266,841,16,140,148,652,334,637,660,232,760,471,332,520,548,516,839,414,531,200,64,166,581,837,221,592,500,466,292,336,48,886,290,80,120,262,583,795,612,836,258,530,736,838,368,8,28,129,278,90,800,184,674,588,492,887,230,570,534,776,239,527,891,0,356,729,226,324,876,626,720,426,654,590,312,772,706,50,304,457,796,408,434,624,454,474,254,532,584,10,132,835,662,666,717,260,728,398,711,238,890,899,74,388,585,472,58,577,647,659,638,862,732,535,86,175,636,694,162,384`

	syncComTradeData(comTradeId, indexCode, indexNameCn, flowCode, thirdIdStr, dateType, year, month, codeMappingListMap, countryMap)

	fmt.Println(indexCode, "结束了")
}

func syncComTradeData(comTradeId int, indexCode, indexNameCn, flowCode, partnerCode, dateType string, year, month int, codeMappingListMap map[string]string, countryMap map[int]models.ComTradeCountry) (err error) {
	// 获取数据

	//https://comtradeapi.un.org/public/v1/preview/C/A/HS?period=2021&partnerCode=842&cmdCode=290244&flowCode=X&customsCode=C00&motCode=0
	params := make(map[string]string)
	//period=2021&partnerCode=842
	params["motCode"] = "0"       //运输方式
	params["customsCode"] = "C00" //海关编码

	params["cmdCode"] = indexCode       //指标code
	params["flowCode"] = flowCode       //贸易流向，X：出口（Export）；M：进口（Import）
	params["partnerCode"] = partnerCode //进口国id

	params["period"] = strconv.Itoa(year) //年份（月份）
	dataTime := fmt.Sprint(year, "-12-31")
	codeStr := "Y"
	dataType := 1
	if dateType == "month" {
		monthStr := strconv.Itoa(month)
		if month < 10 {
			monthStr = "0" + monthStr
		}
		params["period"] = strconv.Itoa(year) + monthStr //年份（月份）
		dataTime = fmt.Sprint(year, "-", monthStr, "-01")
		dataTimeT, tmpErr := time.ParseInLocation(utils.FormatDate, dataTime, time.Local)
		if tmpErr != nil {
			fmt.Println(dataTime, "时间转换异常")
			return
		}
		dataTime = dataTimeT.AddDate(0, 1, -1).Format(utils.FormatDate)
		codeStr = "M"
		dataType = 2
	}
	// 这是获取数据的链接（月度的）
	comeData, err := queryComeTradeData(params, dateType)
	if err != nil {
		fmt.Println("读取失败", err)
		return
	}
	//fmt.Println(comeData)
	if comeData.Count >= 500 {
		msg := fmt.Sprint(indexCode, "；数据超限了", "；年份：", year, ";月份：", month, ";flow:", flowCode, ";partnerCode:", partnerCode)
		fmt.Println(msg)
		fmt.Println(comeData.Count)
		utils.FileLog.Info(msg)
	} else if comeData.Count == -1 {
		msg := fmt.Sprint(indexCode, "；获取数据失败，err:", comeData.Error, ";errObject:", comeData.ErrorObject, "；年份：", year, ";月份：", month, ";flow:", flowCode, ";partnerCode:", partnerCode)
		fmt.Println(msg, comeData)
		utils.FileLog.Info(msg)
		return
	}

	listMap := make(map[string][]*models.ComTradeData, 0)
	for _, v := range comeData.Data {
		//fmt.Println(v)
		var reporterName, partnerName, partner2Name, reporterCnName, partnerCnName, partner2CnName string
		if reporter, ok := countryMap[v.ReporterCode]; ok {
			reporterCnName = reporter.NameCn
			reporterName = reporter.Name
		}
		if partner, ok := countryMap[v.PartnerCode]; ok {
			partnerCnName = partner.NameCn
			partnerName = partner.Name
		}
		if partner, ok := countryMap[v.Partner2Code]; ok {
			partner2CnName = partner.NameCn
			partner2Name = partner.Name
		}

		//ValueCode：HS+Reporter+trade flow+Partner+Value  例如：270730ChinaXUSAValue
		//WeightCode：HS+Reporter+trade flow+Partner+Weight  ,例如：270730ChinaXUSAWeight
		tmpIndexTradeCode := utils.TrimStr(fmt.Sprint(v.CmdCode, codeStr, reporterName, flowCode, partnerName, "2nd", partner2Name, "Value"))
		tmpIndexNetWeightCode := utils.TrimStr(fmt.Sprint(v.CmdCode, codeStr, reporterName, flowCode, partnerName, "2nd", partner2Name, "Weight"))
		tmpData := &models.ComTradeData{
			ComTradeDataId:     0,
			ComTradeId:         comTradeId,
			IndexCode:          v.CmdCode,
			Flow:               flowCode,
			ReporterCode:       v.ReporterCode,
			ReporterName:       reporterCnName,
			PartnerCode:        v.PartnerCode,
			PartnerName:        partnerCnName,
			Partner2Code:       v.Partner2Code,
			Partner2Name:       partner2CnName,
			DateType:           dataType,
			DataTime:           dataTime,
			IndexTradeCode:     tmpIndexTradeCode,
			TradeValue:         v.PrimaryValue,
			IndexNetWeightCode: tmpIndexNetWeightCode,
			NetWeightValue:     v.NetWgt,
			ModifyTime:         time.Now(),
			CreateTime:         time.Now(),
		}

		tmpList, ok := listMap[v.CmdCode]
		if !ok {
			tmpList = make([]*models.ComTradeData, 0)
		}
		listMap[v.CmdCode] = append(tmpList, tmpData)

		//指标关系
		{

			tmpName := indexNameCn + "年度" + reporterCnName
			if dateType == "month" {
				tmpName = indexNameCn + "月度" + reporterCnName
			}

			flow := ``
			if tmpData.Flow == `X` {
				flow = "出口"
			} else {
				flow = "进口"
			}
			tmpName += flow

			finalpartnerName := ``
			if partnerCnName == partner2CnName {
				if partnerCnName == "世界" {
					//若对手国和第二对手国是world时，则翻译为总量；
					finalpartnerName = "总量"
				} else {
					//若对手国或第二对手国是相同的，只需翻译一个；
					finalpartnerName = partnerCnName
				}
			} else {
				if partnerCnName == "世界" || partner2CnName == "世界" {
					//若对手国或第二对手国是world的，只需翻译非world的对手国；
					if partnerCnName == "世界" {
						finalpartnerName = partner2CnName
					} else {
						finalpartnerName = partnerCnName
					}
				} else {
					//若对手国和第二对手国不一样，翻译成：对手国和第二对手国
					finalpartnerName = partnerCnName + "和" + partner2CnName
				}
			}
			tmpName += finalpartnerName
			//fmt.Println(tmpData.IndexTradeCode)
			//fmt.Println(tmpName)

			valueName := tmpName + "贸易金额"
			weightName := tmpName + "贸易重量"

			if name, ok := codeMappingListMap[tmpData.IndexTradeCode]; !ok {
				comTradeCodeMappingType := 1
				if codeStr == "M" {
					comTradeCodeMappingType = 3
				}
				tmpComTradeCodeMapping := &models.ComTradeCodeMapping{
					Code:       tmpData.IndexTradeCode,
					IndexCode:  tmpData.IndexCode,
					Type:       comTradeCodeMappingType,
					Name:       valueName,
					CreateTime: time.Now(),
				}

				tmpErr := models.AddComTradeCodeMapping(tmpComTradeCodeMapping)
				if tmpErr != nil {
					fmt.Println(indexCode, "添加贸易金额失败,err:", err)
				} else {
					codeMappingListMap[tmpData.IndexTradeCode] = valueName
				}
			} else {
				if name != valueName {
					tmpComTradeCodeMapping := &models.ComTradeCodeMapping{
						Code: tmpData.IndexTradeCode,
						Name: valueName,
					}
					e := models.UpdateComTradeCodeMapping(tmpComTradeCodeMapping, []string{"Name"})
					if e != nil {
						err = e
						return
					}
					utils.FileLog.Info("更新指标名称", "旧："+name, "新："+valueName, "code:"+tmpData.IndexTradeCode)
					fmt.Println("更新指标名称", "旧："+name, "新："+valueName)
				}
			}

			if name, ok := codeMappingListMap[tmpData.IndexNetWeightCode]; !ok {
				comTradeCodeMappingType := 2
				if codeStr == "M" {
					comTradeCodeMappingType = 3
				}
				tmpComTradeCodeMapping := &models.ComTradeCodeMapping{
					Code:       tmpData.IndexNetWeightCode,
					IndexCode:  tmpData.IndexCode,
					Type:       comTradeCodeMappingType,
					Name:       weightName,
					CreateTime: time.Now(),
				}
				tmpErr := models.AddComTradeCodeMapping(tmpComTradeCodeMapping)
				if tmpErr != nil {
					fmt.Println(indexCode, "添加贸易重量失败,err:", err)
				} else {
					codeMappingListMap[tmpData.IndexNetWeightCode] = weightName
				}
			} else {
				if name != weightName {
					tmpComTradeCodeMapping := &models.ComTradeCodeMapping{
						Code: tmpData.IndexTradeCode,
						Name: valueName,
					}
					e := models.UpdateComTradeCodeMapping(tmpComTradeCodeMapping, []string{"Name"})
					if e != nil {
						err = e
						return
					}
					utils.FileLog.Info("更新指标名称", "旧："+name, "新："+valueName, "code:"+tmpData.IndexNetWeightCode)
					fmt.Println("更新指标名称", "旧："+name, "新："+valueName)
				}
			}
		}
	}

	for indexCode, list := range listMap {
		models.HandleComTradeData(list, indexCode, flowCode, dataTime, partnerCode)
	}

	return
}

// queryComeTradeData 接口请求网站数据
func queryComeTradeData(params map[string]string, dateType string) (comTradeResp ComTradeResp, err error) {
	//ComTradeUrl
	// 待签名字符串
	paramStr := ``
	for index, val := range params {
		paramStr += index + `=` + val + `&`
	}
	paramStr = paramStr[:len(paramStr)-1]
	urlStr := ComTradeUrl + paramStr
	if dateType == "month" {
		urlStr = ComTradeMonthUrl + paramStr
	}
	//fmt.Println("urlStr:", urlStr)
	body, err := http.Get(urlStr)
	if err != nil {
		fmt.Println("err:", err)
	}
	//utils.FileLog.Info("queryComeTradeData 数据：" + ComTradeUrl + paramStr)
	err = json.Unmarshal(body, &comTradeResp)
	return
}

type ComTradeResp struct {
	ElapsedTime string `json:"elapsedTime"`
	Count       int    `json:"count"`
	Data        []struct {
		TypeCode                 string      `json:"typeCode"`
		FreqCode                 string      `json:"freqCode"`
		RefPeriodId              int         `json:"refPeriodId"`
		RefYear                  int         `json:"refYear"`
		RefMonth                 int         `json:"refMonth"`
		Period                   string      `json:"period"`
		ReporterCode             int         `json:"reporterCode"`
		ReporterISO              interface{} `json:"reporterISO"`
		ReporterDesc             interface{} `json:"reporterDesc"`
		FlowCode                 string      `json:"flowCode"`
		FlowDesc                 interface{} `json:"flowDesc"`
		PartnerCode              int         `json:"partnerCode"`
		PartnerISO               interface{} `json:"partnerISO"`
		PartnerDesc              interface{} `json:"partnerDesc"`
		Partner2Code             int         `json:"partner2Code"`
		Partner2ISO              interface{} `json:"partner2ISO"`
		Partner2Desc             interface{} `json:"partner2Desc"`
		ClassificationCode       string      `json:"classificationCode"`
		ClassificationSearchCode string      `json:"classificationSearchCode"`
		IsOriginalClassification bool        `json:"isOriginalClassification"`
		CmdCode                  string      `json:"cmdCode"`
		CmdDesc                  interface{} `json:"cmdDesc"`
		AggrLevel                int         `json:"aggrLevel"`
		IsLeaf                   interface{} `json:"isLeaf"`
		CustomsCode              string      `json:"customsCode"`
		CustomsDesc              interface{} `json:"customsDesc"`
		MosCode                  string      `json:"mosCode"`
		MotCode                  int         `json:"motCode"`
		MotDesc                  interface{} `json:"motDesc"`
		QtyUnitCode              int         `json:"qtyUnitCode"`
		QtyUnitAbbr              interface{} `json:"qtyUnitAbbr"`
		Qty                      float64     `json:"qty"`
		IsQtyEstimated           bool        `json:"isQtyEstimated"`
		AltQtyUnitCode           int         `json:"altQtyUnitCode"`
		AltQtyUnitAbbr           interface{} `json:"altQtyUnitAbbr"`
		AltQty                   float64     `json:"altQty"`
		IsAltQtyEstimated        bool        `json:"isAltQtyEstimated"`
		NetWgt                   float64     `json:"netWgt"`
		IsNetWgtEstimated        bool        `json:"isNetWgtEstimated"`
		GrossWgt                 float64     `json:"grossWgt"`
		IsGrossWgtEstimated      bool        `json:"isGrossWgtEstimated"`
		Cifvalue                 float64     `json:"cifvalue"`
		Fobvalue                 float64     `json:"fobvalue"`
		PrimaryValue             float64     `json:"primaryValue"`
		LegacyEstimationFlag     int         `json:"legacyEstimationFlag"`
		IsReported               bool        `json:"isReported"`
		IsAggregate              bool        `json:"isAggregate"`
	} `json:"data"`
	Error       interface{} `json:"error"`
	ErrorObject interface{} `json:"errorObject"`
}

func GetAllCountry() (countryMap map[int]models.ComTradeCountry, err error) {
	countryMap = make(map[int]models.ComTradeCountry)
	tmpList, err := models.GetComTradeCountryAll()
	if err != nil {
		return
	}

	for _, v := range tmpList {
		countryMap[v.ThirdId] = *v
	}

	return
}

func syncAllComTradeMapping() (err error) {
	codeMappingListMap := make(map[string]bool)
	codeMappingList, err := models.GetAllComTradeCodeMappingList()
	if err != nil {
		return
	}
	for _, v := range codeMappingList {
		codeMappingListMap[v.Code] = true
	}

	comTradeIndexMap := make(map[string]string)
	comTradeIndexList, err := models.GetAllComTradeIndexList()
	if err != nil {
		return
	}
	for _, v := range comTradeIndexList {
		comTradeIndexMap[v.IndexCode] = v.IndexNameCn
	}

	list, err := models.GetAllComTradeDataList()
	if err != nil {
		return
	}

	countryMap, err := GetAllCountry()
	if err != nil {
		return
	}

	lenList := len(list)
	fmt.Println("总共", lenList, "条")
	for k, v := range list {
		fmt.Println("剩余：", lenList-k-1, "条")
		var reporterName, partnerName, partner2Name string
		if reporter, ok := countryMap[v.ReporterCode]; ok {
			reporterName = reporter.Name
		}
		if partner, ok := countryMap[v.PartnerCode]; ok {
			partnerName = partner.Name
		}
		if partner, ok := countryMap[v.Partner2Code]; ok {
			partner2Name = partner.Name
		}

		tmpValueCode := fmt.Sprint(v.IndexCode, "Y", reporterName, v.Flow, partnerName, "2nd", partner2Name, "Value")
		tmpNetWeightCode := fmt.Sprint(v.IndexCode, "Y", reporterName, v.Flow, partnerName, "2nd", partner2Name, "Weight")
		//v.IndexTradeCode = utils.TrimStr(tmpValueCode)
		//v.IndexNetWeightCode = utils.TrimStr(tmpNetWeightCode)
		//v.Update([]string{"IndexTradeCode", "IndexNetWeightCode"})

		models.UpdateComTradeData(utils.TrimStr(tmpValueCode), utils.TrimStr(tmpNetWeightCode), v.IndexTradeCode)
	}
	fmt.Println("整体结束了=============")
	return
}

func syncAllComTradeMapping2() (err error) {
	codeMappingListMap := make(map[string]bool)
	codeMappingList, err := models.GetAllComTradeCodeMappingList()
	if err != nil {
		return
	}
	for _, v := range codeMappingList {
		codeMappingListMap[v.Code] = true
	}

	comTradeIndexMap := make(map[string]string)
	comTradeIndexList, err := models.GetAllComTradeIndexList()
	if err != nil {
		return
	}
	for _, v := range comTradeIndexList {
		comTradeIndexMap[v.IndexCode] = v.IndexNameCn
	}

	list, err := models.GetAllComTradeDataList()
	if err != nil {
		return
	}

	countryMap, err := GetAllCountry()
	if err != nil {
		return
	}

	lenList := len(list)
	fmt.Println("总共", lenList, "条")

	dataList := make([]*models.ComTradeCodeMapping, 0)
	for k, v := range list {
		fmt.Println("剩余：", lenList-k-1, "条")
		var reporterName string
		if reporter, ok := countryMap[v.ReporterCode]; ok {
			reporterName = reporter.NameCn
		}
		tmpName := ``
		if comTradeIndexName, ok := comTradeIndexMap[v.IndexCode]; ok {
			tmpName = comTradeIndexName
		}
		tmpName += reporterName

		flow := ``
		if v.Flow == `X` {
			flow = "出口"
		} else {
			flow = "进口"
		}
		tmpName += flow

		valueName := tmpName + "贸易金额"
		weightName := tmpName + "贸易重量"

		if _, ok := codeMappingListMap[v.IndexTradeCode]; !ok {
			tmpData := &models.ComTradeCodeMapping{
				Code:       v.IndexTradeCode,
				IndexCode:  v.IndexCode,
				Type:       1,
				Name:       valueName,
				CreateTime: time.Now(),
			}
			dataList = append(dataList, tmpData)
			codeMappingListMap[v.IndexTradeCode] = true
			//tmpErr := models.AddComTradeCodeMapping(tmpData)
			//if tmpErr != nil {
			//	fmt.Println(indexCode, "添加贸易金额失败,err:", err)
			//} else {
			//	codeMappingListMap[v.IndexTradeCode] = true
			//}
		}

		if _, ok := codeMappingListMap[v.IndexNetWeightCode]; !ok {
			tmpData := &models.ComTradeCodeMapping{
				Code:       v.IndexNetWeightCode,
				IndexCode:  v.IndexCode,
				Type:       2,
				Name:       weightName,
				CreateTime: time.Now(),
			}
			dataList = append(dataList, tmpData)
			codeMappingListMap[v.IndexNetWeightCode] = true
			//tmpErr := models.AddComTradeCodeMapping(tmpData)
			//if tmpErr != nil {
			//	fmt.Println(indexCode, "添加贸易重量失败,err:", err)
			//} else {
			//	codeMappingListMap[v.IndexNetWeightCode] = true
			//}
		}

		if len(dataList) >= 2000 {
			tmpErr := models.AddMultiComTradeCodeMapping(dataList)
			if tmpErr != nil {
				fmt.Println(indexCode, "添加贸易重量失败,err:", tmpErr)
			}
			dataList = make([]*models.ComTradeCodeMapping, 0)
		}
	}

	if len(dataList) > 0 {
		tmpErr := models.AddMultiComTradeCodeMapping(dataList)
		if tmpErr != nil {
			fmt.Println(indexCode, "添加贸易重量失败,err:", tmpErr)
		}
		dataList = make([]*models.ComTradeCodeMapping, 0)
	}
	fmt.Println("整体结束了=============")
	return
}

func syncAllComTradeMapping3() {
	codeMappingListMap := make(map[string]*models.ComTradeCodeMapping)
	codeMappingList, err := models.GetAllComTradeCodeMappingList()
	if err != nil {
		return
	}
	for _, v := range codeMappingList {
		codeMappingListMap[v.Code] = v
	}

	comTradeIndexMap := make(map[string]string)
	comTradeIndexList, err := models.GetAllComTradeIndexList()
	if err != nil {
		return
	}
	for _, v := range comTradeIndexList {
		comTradeIndexMap[v.IndexCode] = v.IndexNameCn
	}

	list, err := models.GetAllComTradeDataList()
	if err != nil {
		return
	}

	countryMap, err := GetAllCountry()
	if err != nil {
		return
	}

	lenList := len(list)
	fmt.Println("总共", lenList, "条")

	dataList := make([]*models.ComTradeCodeMapping, 0)
	for k, v := range list {
		fmt.Println("剩余：", lenList-k-1, "条")
		var reporterCnName, partnerCnName, partner2CnName string
		if reporter, ok := countryMap[v.ReporterCode]; ok {
			reporterCnName = reporter.NameCn
		}
		if partner, ok := countryMap[v.PartnerCode]; ok {
			partnerCnName = partner.NameCn
		}
		if partner, ok := countryMap[v.Partner2Code]; ok {
			partner2CnName = partner.NameCn
		}

		indexNameCn := comTradeIndexMap[v.IndexCode]
		if indexNameCn == "" {
			fmt.Println(indexNameCn)
			fmt.Println(v.IndexCode)
		}
		codeStr := "Y"
		tmpName := indexNameCn + "年度" + reporterCnName
		if v.DateType == 2 {
			codeStr = "M"
			tmpName = indexNameCn + "月度" + reporterCnName
		}
		flow := ``
		if v.Flow == `X` {
			flow = "出口"
		} else {
			flow = "进口"
		}
		tmpName += flow

		finalpartnerName := ``
		if partnerCnName == partner2CnName {
			if partnerCnName == "世界" {
				//若对手国和第二对手国是world时，则翻译为总量；
				finalpartnerName = "总量"
			} else {
				//若对手国或第二对手国是相同的，只需翻译一个；
				finalpartnerName = partnerCnName
			}
		} else {
			if partnerCnName == "世界" || partner2CnName == "世界" {
				//若对手国或第二对手国是world的，只需翻译非world的对手国；
				if partnerCnName == "世界" {
					finalpartnerName = partner2CnName
				} else {
					finalpartnerName = partnerCnName
				}
			} else {
				//若对手国和第二对手国不一样，翻译成：对手国和第二对手国
				finalpartnerName = partnerCnName + "和" + partner2CnName
			}
		}
		tmpName += finalpartnerName

		valueName := tmpName + "贸易金额"
		weightName := tmpName + "贸易重量"

		if _, ok := codeMappingListMap[v.IndexTradeCode]; !ok {
			comTradeCodeMappingType := 1
			if codeStr == "M" {
				comTradeCodeMappingType = 3
			}
			tmpData := &models.ComTradeCodeMapping{
				Code:       v.IndexTradeCode,
				IndexCode:  v.IndexCode,
				Type:       comTradeCodeMappingType,
				Name:       valueName,
				CreateTime: time.Now(),
			}
			dataList = append(dataList, tmpData)
			codeMappingListMap[v.IndexTradeCode] = tmpData
			//tmpErr := models.AddComTradeCodeMapping(tmpData)
			//if tmpErr != nil {
			//	fmt.Println(indexCode, "添加贸易金额失败,err:", err)
			//} else {
			//	codeMappingListMap[v.IndexTradeCode] = true
			//}
		}

		if _, ok := codeMappingListMap[v.IndexNetWeightCode]; !ok {
			comTradeCodeMappingType := 2
			if codeStr == "M" {
				comTradeCodeMappingType = 4
			}
			tmpData := &models.ComTradeCodeMapping{
				Code:       v.IndexNetWeightCode,
				IndexCode:  v.IndexCode,
				Type:       comTradeCodeMappingType,
				Name:       weightName,
				CreateTime: time.Now(),
			}
			dataList = append(dataList, tmpData)
			codeMappingListMap[v.IndexNetWeightCode] = tmpData
			//tmpErr := models.AddComTradeCodeMapping(tmpData)
			//if tmpErr != nil {
			//	fmt.Println(indexCode, "添加贸易重量失败,err:", err)
			//} else {
			//	codeMappingListMap[v.IndexNetWeightCode] = true
			//}
		}

		if len(dataList) >= 2000 {
			tmpErr := models.AddMultiComTradeCodeMapping(dataList)
			if tmpErr != nil {
				fmt.Println(indexCode, "添加贸易重量失败,err:", tmpErr)
			}
			dataList = make([]*models.ComTradeCodeMapping, 0)
		}
	}

	if len(dataList) > 0 {
		tmpErr := models.AddMultiComTradeCodeMapping(dataList)
		if tmpErr != nil {
			fmt.Println(indexCode, "添加贸易重量失败,err:", tmpErr)
		}
		dataList = make([]*models.ComTradeCodeMapping, 0)
	}
	fmt.Println("整体结束了=============")
	return
}