package services import ( "context" "encoding/json" "eta/eta_crawler/models" "eta/eta_crawler/services/alarm_msg" "eta/eta_crawler/utils" "fmt" "github.com/rdlucklib/rdluck_tools/http" "strconv" "strings" "time" ) func SyncComTradeData(cont context.Context) (err error) { //comTradeId := 1 //indexCode := "270730" //flowCode := "X" //year := 2021 //partnerCode := 842 //err = syncComTradeData(comTradeId, indexCode, flowCode, partnerCode, year) return } const ComTradeUrl = "https://comtradeapi.un.org/public/v1/preview/C/A/HS?" const ComTradeMonthUrl = "https://comtradeapi.un.org/public/v1/preview/C/M/HS?" //func sync() { // //yearList := []int{2009, 2008, 2007, 2006, 2005, 2004, 2003, 2002, 2001, 2000} // yearList := []int{2022} // flowCodeList := []string{"X", "M"} // // for _, year := range yearList { // for _, flow := range flowCodeList { // fmt.Println("开始", year, "的", flow) // syncAllComTradeData2(year, 0, flow, "year") // fmt.Println("结束", year, "的", flow) // } // } //} // //func sync2() { // defer func() { // //go utils.SendEmail(utils.APPNAME+"【"+utils.RunMode+"】"+"失败提醒", "RefreshDataFromCoalCoastal ErrMsg:"+err.Error(), utils.EmailSendToUsers) // msg := "UN数据同步完成" // go alarm_msg.SendAlarmMsg(msg, 3) // }() // //yearList := []int{2009, 2008, 2007, 2006, 2005, 2004, 2003, 2002, 2001, 2000} // //yearList := []int{2022, 2021, 2020, 2019, 2018, 2017, 2016, 2015, 2014, 2013, 2012, 2011, 2010} // yearList := []int{2023} // //monthList := []int{12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1} // monthList := []int{1, 2, 3, 4, 5, 6} // flowCodeList := []string{"X", "M"} // // for _, year := range yearList { // for _, month := range monthList { // for _, flow := range flowCodeList { // fmt.Println("开始", year, month, "的", flow) // syncAllComTradeData2(year, month, flow, "month") // fmt.Println("结束", year, "的", flow) // } // } // } //} // SyncYearComTrade 同步年度数据 func SyncYearComTrade(cont context.Context) (err error) { errMsg := make([]string, 0) defer func() { if len(errMsg) > 0 { go alarm_msg.SendAlarmMsg("UN年度度数据同步失败："+strings.Join(errMsg, "\n"), 3) } }() flowCodeList := []string{"X", "M"} year := time.Now().Year() for _, flow := range flowCodeList { fmt.Println("开始", year, "的", flow) err = syncAllComTradeData2(year, 0, flow, "year") if err != nil { errMsg = append(errMsg, fmt.Sprint(year, "的", flow, "数据爬取失败：", err.Error())) } fmt.Println("结束", year, "的", flow) } return } // SyncYearMonthComTrade 同步月度数据 func SyncYearMonthComTrade(cont context.Context) (err error) { errMsg := make([]string, 0) defer func() { if len(errMsg) > 0 { go alarm_msg.SendAlarmMsg("UN月度数据同步失败："+strings.Join(errMsg, "\n"), 3) } }() flowCodeList := []string{"X", "M"} year := time.Now().Year() month := int(time.Now().Month()) for _, flow := range flowCodeList { fmt.Println("开始", year, month, "的", flow) err := syncAllComTradeData2(year, month, flow, "month") if err != nil { errMsg = append(errMsg, fmt.Sprint(year, month, "的", flow, "数据爬取失败：", err.Error())) } fmt.Println("结束", year, "的", flow) } return } type TmpStruct struct { IndexCode string Year int Month int FlowCode string PartnerCode string } func sync3() { defer func() { //go utils.SendEmail(utils.APPNAME+"【"+utils.RunMode+"】"+"失败提醒", "RefreshDataFromCoalCoastal ErrMsg:"+err.Error(), utils.EmailSendToUsers) //msg := "UN数据同步完成" //go alarm_msg.SendAlarmMsg(msg, 3) }() str := `2023/01/04 15:16:51.837 [I] [com_trade.go:183] 271114获取数据失败，err:map[error:Something went wrong. Please try again or contact comtrade@un.org];errObject:；年份：2022;月份：10;flow:M;partnerCode:170,410,246,703,826,233,699,784 2023/01/04 15:16:53.139 [I] [com_trade.go:183] 271114获取数据失败，err:map[error:Something went wrong. Please try again or contact comtrade@un.org];errObject:；年份：2022;月份：10;flow:M;partnerCode:757,218,764,710,124,579,70,528 2023/01/04 15:25:22.880 [I] [com_trade.go:183] 290121获取数据失败，err:map[error:Something went wrong. Please try again or contact comtrade@un.org];errObject:；年份：2022;月份：9;flow:X;partnerCode:178,275,328,768,748,84,762,496,108,270,533,894,136,670,418,174,478,446,462,72,204,690,854,562,417,678,104,60,716,52,882 2023/01/04 15:25:24.233 [I] [com_trade.go:183] 290121获取数据失败，err:map[error:Something went wrong. Please try again or contact comtrade@un.org];errObject:；年份：2022;月份：9;flow:X;partnerCode:470,222,400,268,450,686,860,96,484,352,68,480,512,591,740,780,508,24,858,558,634,646,242,834,340,807,308,116,524,231 2023/01/04 15:25:25.609 [I] [com_trade.go:183] 381900获取数据失败，err:map[error:Something went wrong. Please try again or contact comtrade@un.org];errObject:；年份：2022;月份：9;flow:X;partnerCode:392,458,600,40,300,608,498,191,360,214,36,100,704,376,32,348 2023/01/04 15:45:47.877 [I] [com_trade.go:183] 271114获取数据失败，err:map[error:Something went wrong. Please try again or contact comtrade@un.org];errObject:；年份：2022;月份：8;flow:M;partnerCode:156,170,804,764,246 2023/01/04 15:45:49.197 [I] [com_trade.go:183] 271114获取数据失败，err:map[error:Something went wrong. Please try again or contact comtrade@un.org];errObject:；年份：2022;月份：8;flow:M;partnerCode:440,100,196,31,498,36,392,348,376,40,208,360,76,682,191,214 2023/01/04 15:45:50.491 [I] [com_trade.go:183] 271114获取数据失败，err:map[error:Something went wrong. Please try again or contact comtrade@un.org];errObject:；年份：2022;月份：8;flow:M;partnerCode:300,600,32,458,554,608,704,112 2022/12/30 08:31:53.846 [I] [com_trade.go:176] 270730获取数据失败，err:map[error:Something went wrong. Please try again or contact comtrade@un.org];errObject:；年份：2019;月份：1;flow:X;partnerCode:458,36,360,376,76,214,40,348,608,498,112,682,208,600,392,196 2022/12/30 08:31:55.167 [I] [com_trade.go:176] 270730获取数据失败，err:map[error:Something went wrong. Please try again or contact comtrade@un.org];errObject:；年份：2019;月份：1;flow:X;partnerCode:100,554,191,300,32,704,31,440 2022/12/30 08:53:56.893 [I] [com_trade.go:176] 290230获取数据失败，err:map[error:Something went wrong. Please try again or contact comtrade@un.org];errObject:；年份：2018;月份：12;flow:X;partnerCode:882,72,136,894,178,417,854,174,670,690,748,496,328,418,104,762,533,678,462,446,716,52,478,270,562,768,275,84,60,204,108 2022/12/30 08:53:59.079 [I] [com_trade.go:176] 290230获取数据失败，err:map[error:Something went wrong. Please try again or contact comtrade@un.org];errObject:；年份：2018;月份：12;flow:X;partnerCode:40,458,600,76,376,682,608,31,32,348,704,100,300,498,392,360 2022/12/30 09:16:34.790 [I] [com_trade.go:176] 270750获取数据失败，err:map[error:Something went wrong. Please try again or contact comtrade@un.org];errObject:；年份：2018;月份：11;flow:X;partnerCode:703,826,702,757,218,428,752,56 2022/12/30 09:16:36.185 [I] [com_trade.go:176] 270750获取数据失败，err:map[error:Something went wrong. Please try again or contact comtrade@un.org];errObject:；年份：2018;月份：11;flow:X;partnerCode:410,842,528,246,170,70,579,792 2022/12/30 09:16:37.641 [I] [com_trade.go:176] 270750获取数据失败，err:map[error:Something went wrong. Please try again or contact comtrade@un.org];errObject:；年份：2018;月份：11;flow:X;partnerCode:156,620,251,616,764 2022/12/30 09:23:56.668 [I] [com_trade.go:176] 290244获取数据失败，err:map[error:Something went wrong. Please try again or contact comtrade@un.org];errObject:；年份：2018;月份：11;flow:X;partnerCode:624,368,891,837,266,527,50,278,500,662,180,226,238,638,312,175,532,492,666,698,434,868,254,674,90,4,184,577,776,140,588,296,890,388,706,899,866,288,800,850,697,720,472,316,221,336,585,64,292,48,200,258,332,466,548,454,879,408,356,836,516,568,334,637,728,120,290,239,86,647,426,459,12,232,583,262,772,849,92,582,574,534,16,660,398,520,862,535,166,473,10,471,234,760,570,457,28,129,230,58,192,364,304,430,732,581,148,461,711,796,798,590,717,598,8,474,658,729,654,20,384,736,835,626,531,592,260,324,841,580,530,132,414,536,652,74,212,612,540,887,694,162,886,659,795,876,44,80,280,839,810,838,584,636 2022/12/30 09:23:58.075 [I] [com_trade.go:176] 290244获取数据失败，err:map[error:Something went wrong. Please try again or contact comtrade@un.org];errObject:；年份：2018;月份：11;flow:X;partnerCode:705,710,642,699,490,724,442,784 2022/12/30 09:31:23.532 [I] [com_trade.go:176] 290122获取数据失败，err:map[error:Something went wrong. Please try again or contact comtrade@un.org];errObject:；年份：2018;月份：11;flow:M;partnerCode:757,752,620,642,702 2022/12/30 09:31:24.940 [I] [com_trade.go:176] 290122获取数据失败，err:map[error:Something went wrong. Please try again or contact comtrade@un.org];errObject:；年份：2018;月份：11;flow:M;partnerCode:174,328,748,84,562,882,204,104,670,60,418,678,108,768,178,275,533,462,690,446,496,716,270,478,894,417,854,72,762,52,136 2022/12/30 10:17:07.994 [I] [com_trade.go:176] 270750获取数据失败，err:map[error:Something went wrong. Please try again or contact comtrade@un.org];errObject:；年份：2018;月份：8;flow:X;partnerCode:583,368,711,810,838,28,652,666,849,868,772,175,304,570,577,638,12,80,592,580,798,132,288,356,332,239,492,461,4,180,588,887,474,516,659,534,886,312,760,862,58,527,582,732,472,324,520,662,729,166,531,698,90,388,879,674,624,500,891,626,140,795,20,200,736,316,568,234,426,162,336,850,636,776,530,64,148,835,800,184,262,612,120,260,232,890,414,796,585,292,540,192,258,10,44,384,536,654,658,899,590,841,876,334,50,697,278,74,836,434,454,647,837,408,266,457,8,86,296,866,584,212,728,535,471,129,398,430,459,48,238,706,581,473,466,694,92,226,548,364,574,660,720,230,717,598,221,839,280,532,290,637,16,254 2022/12/30 10:46:04.444 [I] [com_trade.go:176] 290244获取数据失败，err:map[error:Something went wrong. Please try again or contact comtrade@un.org];errObject:；年份：2018;月份：7;flow:X;partnerCode:780,340,116,834,858,558,524,24,686,400,860,512,268,96,508,242,222,231,646,740,470,634,480,484,308,68,352,450,591,807 2022/12/30 10:46:05.864 [I] [com_trade.go:176] 290244获取数据失败，err:map[error:Something went wrong. Please try again or contact comtrade@un.org];errObject:；年份：2018;月份：7;flow:X;partnerCode:682,76,704,348,376,498,600,608,360,214,300,392,100,40,32,31 2022/12/30 11:22:44.563 [I] [com_trade.go:176] 290243获取数据失败，err:map[error:Something went wrong. Please try again or contact comtrade@un.org];errObject:；年份：2018;月份：5;flow:M;partnerCode:276,688,784,792,56,616,124,703 2022/12/30 11:22:45.936 [I] [com_trade.go:176] 290243获取数据失败，err:map[error:Something went wrong. Please try again or contact comtrade@un.org];errObject:；年份：2018;月份：5;flow:M;partnerCode:428,579,702,170,620,528,643,764 2022/12/30 11:40:43.264 [I] [com_trade.go:176] 290244获取数据失败，err:map[error:Something went wrong. Please try again or contact comtrade@un.org];errObject:；年份：2018;月份：4;flow:M;partnerCode:56,703,218,490,170,156,616,410 2022/12/30 11:40:44.553 [I] [com_trade.go:176] 290244获取数据失败，err:map[error:Something went wrong. Please try again or contact comtrade@un.org];errObject:；年份：2018;月份：4;flow:M;partnerCode:784,70,579,251,699,233,705,752 2022/12/30 22:41:22.216 [I] [com_trade.go:176] 290230获取数据失败，err:map[error:Something went wrong. Please try again or contact comtrade@un.org];errObject:；年份：2017;月份：7;flow:M;partnerCode:528,642,70,784,792,710,842,56 2022/12/30 22:41:23.516 [I] [com_trade.go:176] 290230获取数据失败，err:map[error:Something went wrong. Please try again or contact comtrade@un.org];errObject:；年份：2017;月份：7;flow:M;partnerCode:246,705,251,218,757 2022/12/30 22:41:24.824 [I] [com_trade.go:176] 270750获取数据失败，err:map[error:Something went wrong. Please try again or contact comtrade@un.org];errObject:；年份：2017;月份：7;flow:M;partnerCode:0 2022/12/30 22:41:26.138 [I] [com_trade.go:176] 270750获取数据失败，err:map[error:Something went wrong. Please try again or contact comtrade@un.org];errObject:；年份：2017;月份：7;flow:M;partnerCode:68,512,470,222,558,858,508,308,352,591,231,242,780,24,484,807,740,646,524,686,340,268,116,634,860,480,96,400,450,834 2022/12/30 22:41:27.457 [I] [com_trade.go:176] 270750获取数据失败，err:map[error:Something went wrong. Please try again or contact comtrade@un.org];errObject:；年份：2017;月份：7;flow:M;partnerCode:304,624,332,654,795,841,258,577,879,732,835,527,647,86,866,706,232,796,638,4,80,717,12,492,454,798,180,44,697,838,520,776,262,473,887,16,540,129,192,772,850,280,637,581,570,839,810,90,20,334,474,459,230,388,662,534,50,120,288,466,530,531,590,316,398,471,221,175,74,580,891,711,212,720,598,666,800,886,8,238,414,28,64,312,658,296,849,408,278,694,568,698,48,140,584,461,736,836,292,430,659,760,336,472,899,583,536,636,588,239,166,384,457,516,574,837,58,868,254,368,652,226,612,862,266,434,728,200,592,290,674,548,585,729,92,162,364,532,626,10,184,426,148,660,535,234,324,500,356,132,582,890,260,876 2022/12/30 23:13:14.705 [I] [com_trade.go:176] 271114获取数据失败，err:map[error:Something went wrong. Please try again or contact comtrade@un.org];errObject:；年份：2017;月份：6;flow:M;partnerCode:200,624,636,776,384,140,266,304,12,278,28,260,835,674,531,841,837,430,659,296,472,612,474,520,86,120,457,10,74,500,132,368,637,454,316,434,221,536,459,839,585,810,890,706,862,148,582,698,492,336,795,581,592,577,80,162,548,868,232,426,568,626,262,580,570,654,729,866,175,598,732,64,717,736,838,899,720,398,290,658,212,292,527,638,129,8,92,662,728,879,588,234,58,414,711,760,408,324,388,583,584,534,530,471,192,886,666,697,574,184,90,4,20,516,647,312,364,473,652,694,48,50,590,254,258,772,796,230,288,332,334,849,876,660,540,887,16,466,891,44,166,180,532,850,800,535,226,238,356,239,798,836,461,280 2022/12/30 23:13:16.091 [I] [com_trade.go:176] 271114获取数据失败，err:map[error:Something went wrong. Please try again or contact comtrade@un.org];errObject:；年份：2017;月份：6;flow:M;partnerCode:222,524,470,231,634,740,24,780,512,508,807,340,484,686,834,352,558,268,68,242,96,116,860,480,858,591,450,646,308,400 2022/12/30 23:13:17.486 [I] [com_trade.go:176] 290122获取数据失败，err:map[error:Something went wrong. Please try again or contact comtrade@un.org];errObject:；年份：2017;月份：6;flow:M;partnerCode:608,498,112,36,682,704,40,348,208,392,600,554,300,440,196,458 2022/12/31 00:05:18.625 [I] [com_trade.go:176] 271114获取数据失败，err:map[error:Something went wrong. Please try again or contact comtrade@un.org];errObject:；年份：2017;月份：3;flow:M;partnerCode:826,56,757,724,233,251,752,702 2022/12/31 00:05:19.966 [I] [com_trade.go:176] 271114获取数据失败，err:map[error:Something went wrong. Please try again or contact comtrade@un.org];errObject:；年份：2017;月份：3;flow:M;partnerCode:410,688,703,616,276,620,579,380 2022/12/31 00:05:21.315 [I] [com_trade.go:176] 271114获取数据失败，err:map[error:Something went wrong. Please try again or contact comtrade@un.org];errObject:；年份：2017;月份：3;flow:M;partnerCode:170,710,528,490,699,428,124,792 2022/12/31 00:46:55.568 [I] [com_trade.go:176] 290122获取数据失败，err:map[error:Something went wrong. Please try again or contact comtrade@un.org];errObject:；年份：2017;月份：2;flow:M;partnerCode:0 2022/12/31 00:46:56.919 [I] [com_trade.go:176] 290121获取数据失败，err:map[error:Something went wrong. Please try again or contact comtrade@un.org];errObject:；年份：2017;月份：2;flow:M;partnerCode:697,886,254,80,166,536,324,426,839,796,50,221,800,12,461,364,728,16,48,132,230,660,212,798,548,568,662,184,585,414,500,760,772,527,86,574,288,459,534,590,582,850,368,570,666,408,334,530,336,866,810,90,887,588,332,148,581,612,532,260,388,838,659,472,238,520,795,180,835,296,706,652,278,836,698,717,540,531,862,8,384,290,434,454,636,736,535,262,473,292,624,140,162,304,776,74,234,266,891,20,899,58,841,879,120,654,694,44,466,356,474,658,674,457,10,258,890,647,729,583,638,876,92,471,129,626,732,280,398,637,430,516,200,232,192,312,175,239,837,720,711,849,64,580,4,592,28,577,492,316,584,598,226,868 2022/12/31 00:46:58.356 [I] [com_trade.go:176] 290121获取数据失败，err:map[error:Something went wrong. Please try again or contact comtrade@un.org];errObject:；年份：2017;月份：2;flow:M;partnerCode:246,643,156,688,826,528,804,616 2022/12/31 04:15:22.974 [I] [com_trade.go:176] 290122获取数据失败，err:map[error:Something went wrong. Please try again or contact comtrade@un.org];errObject:；年份：2016;月份：10;flow:M;partnerCode:404,372,604,422,504,788,344,188,51,320,566,586,144,152,818,499 2022/12/31 04:15:24.490 [I] [com_trade.go:176] 290122获取数据失败，err:map[error:Something went wrong. Please try again or contact comtrade@un.org];errObject:；年份：2016;月份：10;flow:M;partnerCode:234,296,58,192,16,212,772,694,659,536,457,92,612,760,798,837,527,654,132,592,12,408,582,568,711,384,879,364,532,162,166,200,638,20,574,891,254,535,492,466,266,140,890,74,86,4,660,44,540,776,584,580,585,80,221,796,570,534,239,414,500,636,148,238,312,666,129,232,64,332,292,48,262,581,471,862,304,278,583,530,626,8,520,838,280,800,90,588,28,324,473,662,697,652,720,290,334,598,474,10,899,368,461,180,388,430,835,184,531,647,728,732,288,454,850,624,434,698,398,868,839,356,226,472,336,120,230,717,887,637,577,795,674,706,548,516,50,810,729,836,459,886,258,590,260,866,849,316,426,658,175,841,876,736 2022/12/31 04:15:26.001 [I] [com_trade.go:176] 290122获取数据失败，err:map[error:Something went wrong. Please try again or contact comtrade@un.org];errObject:；年份：2016;月份：10;flow:M;partnerCode:231,646,834,591,340,480,68,524,24,308,558,400,352,268,450,116,96,740,860,222,634,780,484,512,242,807,858,470,508,686 2022/12/31 08:06:11.934 [I] [com_trade.go:176] 271114获取数据失败，err:map[error:Something went wrong. Please try again or contact comtrade@un.org];errObject:；年份：2015;月份：12;flow:M;partnerCode:699,642,705,380,579,490,643,703 2022/12/31 08:06:13.619 [I] [com_trade.go:176] 271114获取数据失败，err:map[error:Something went wrong. Please try again or contact comtrade@un.org];errObject:；年份：2015;月份：12;flow:M;partnerCode:724,710,156,428,620,70,784,616 2022/12/31 09:51:10.568 [I] [com_trade.go:176] 290244获取数据失败，err:map[error:Something went wrong. Please try again or contact comtrade@un.org];errObject:；年份：2015;月份：4;flow:X;partnerCode:826,70,156,702,428,705,792,764 2022/12/31 09:51:12.117 [I] [com_trade.go:176] 290244获取数据失败，err:map[error:Something went wrong. Please try again or contact comtrade@un.org];errObject:；年份：2015;月份：4;flow:X;partnerCode:642,203,579,620,752,410,528,804 2022/12/31 09:58:54.330 [I] [com_trade.go:176] 381900获取数据失败，err:map[error:Something went wrong. Please try again or contact comtrade@un.org];errObject:；年份：2015;月份：4;flow:M;partnerCode:218,276,826,842,764,699,124,643 2023/01/03 22:26:25.230 [I] [com_trade.go:176] 270730获取数据失败，err:map[error:Something went wrong. Please try again or contact comtrade@un.org];errObject:；年份：2003;月份：8;flow:X;partnerCode:682,31,348,40,208,458,704,376,32,191,392,360,36,196,554,600 2023/01/03 22:34:20.413 [I] [com_trade.go:176] 270750获取数据失败，err:map[error:Something went wrong. Please try again or contact comtrade@un.org];errObject:；年份：2003;月份：8;flow:M;partnerCode:591,308,352,268,450,524,231,484,807,558,68,646,24,834,740,222,480,470,96,508,858,400,116,340,634,780,242,860,512,686 2023/01/03 22:34:22.005 [I] [com_trade.go:176] 270750获取数据失败，err:map[error:Something went wrong. Please try again or contact comtrade@un.org];errObject:；年份：2003;月份：8;flow:M;partnerCode:894,174,104,84,108,270,418,678,716,204,178,462,496,478,670,60,328,690,533,136,446,768,417,762,562,72,748,854,882,275,52 2023/01/03 22:47:23.927 [I] [com_trade.go:176] 290243获取数据失败，err:map[error:Something went wrong. Please try again or contact comtrade@un.org];errObject:；年份：2003;月份：7;flow:M;partnerCode:591,646,508,24,308,268,558,634,807,480,512,834,780,242,524,352,450,484,470,96,686,116,231,858,340,740,222,860,400,68 2023/01/03 22:47:25.509 [I] [com_trade.go:176] 290243获取数据失败，err:map[error:Something went wrong. Please try again or contact comtrade@un.org];errObject:；年份：2003;月份：7;flow:M;partnerCode:32,214,704,376,208,360,191,112,600,348,554,300,100,76,40,196 2023/01/03 22:47:26.957 [I] [com_trade.go:176] 290243获取数据失败，err:map[error:Something went wrong. Please try again or contact comtrade@un.org];errObject:；年份：2003;月份：7;flow:M;partnerCode:682,392,36,608,440,31,458,498 2023/01/03 22:56:11.582 [I] [com_trade.go:176] 290230获取数据失败，err:map[error:Something went wrong. Please try again or contact comtrade@un.org];errObject:；年份：2003;月份：6;flow:X;partnerCode:68,508,400,860,231,740,242,512,96,807,470,268,858,524,222,450,308,646,686,591,480,558,780,634,834,24,340,116,484,352 2023/01/03 22:56:12.894 [I] [com_trade.go:176] 290230获取数据失败，err:map[error:Something went wrong. Please try again or contact comtrade@un.org];errObject:；年份：2003;月份：6;flow:X;partnerCode:136,446,496,882,690,204,60,275,462,533,108,178,670,716,52,104,762,328,418,270,84,894,768,72,854,174,417,562,678,478,748 2023/01/03 23:09:42.931 [I] [com_trade.go:176] 290122获取数据失败，err:map[error:Something went wrong. Please try again or contact comtrade@un.org];errObject:；年份：2003;月份：5;flow:M;partnerCode:608,112,40,214,682,392,458,31,348,498,76,600,554,704,191,300 2023/01/03 23:09:44.481 [I] [com_trade.go:176] 290122获取数据失败，err:map[error:Something went wrong. Please try again or contact comtrade@un.org];errObject:；年份：2003;月份：5;flow:M;partnerCode:32,36,360,440,208,196,100,376 2023/01/03 23:17:27.853 [I] [com_trade.go:176] 270730获取数据失败，err:map[error:Something went wrong. Please try again or contact comtrade@un.org];errObject:；年份：2003;月份：5;flow:M;partnerCode:608,112,40,214,682,392,458,31,348,498,76,600,554,704,191,300 2023/01/03 23:17:29.534 [I] [com_trade.go:176] 270730获取数据失败，err:map[error:Something went wrong. Please try again or contact comtrade@un.org];errObject:；年份：2003;月份：5;flow:M;partnerCode:32,36,360,440,208,196,100,376 2023/01/03 23:25:00.828 [I] [com_trade.go:176] 381900获取数据失败，err:map[error:Something went wrong. Please try again or contact comtrade@un.org];errObject:；年份：2003;月份：4;flow:M;partnerCode:40,76,704,208,458,608,214,31,300,112,196,191,600,360,348,682 2023/01/04 01:02:18.808 [I] [com_trade.go:176] 290230获取数据失败，err:map[error:Something went wrong. Please try again or contact comtrade@un.org];errObject:；年份：2002;月份：6;flow:M;partnerCode:835,92,772,230,624,694,192,527,899,384,534,711,80,226,520,647,887,10,434,536,570,254,278,332,530,166,584,876,44,368,414,221,810,636,471,180,583,388,585,64,836,736,212,532,638,706,866,334,408,839,548,292,74,304,598,891,28,426,20,398,473,238,540,666,796,862,4,868,175,492,890,500,850,262,461,290,472,336,296,580,577,459,232,457,8,886,717,90,776,129,430,652,48,760,50,720,728,588,590,637,654,732,474,260,612,581,841,531,837,132,697,592,266,316,140,148,184,356,454,12,364,466,698,795,658,674,838,568,312,86,582,729,879,16,200,288,120,234,574,535,662,798,162,258,239,660,280,659,800,58,516,626,849,324 2023/01/04 01:02:20.180 [I] [com_trade.go:176] 290230获取数据失败，err:map[error:Something went wrong. Please try again or contact comtrade@un.org];errObject:；年份：2002;月份：6;flow:M;partnerCode:32,31,191,554,76,300,100,348,458,214,40,196,392,704,376,360 2023/01/04 01:09:53.528 [I] [com_trade.go:176] 290121获取数据失败，err:map[error:Something went wrong. Please try again or contact comtrade@un.org];errObject:；年份：2002;月份：5;flow:X;partnerCode:233,428,203,528,724,792,703,380 2023/01/04 01:52:58.039 [I] [com_trade.go:176] 270750获取数据失败，err:map[error:Something went wrong. Please try again or contact comtrade@un.org];errObject:；年份：2002;月份：1;flow:X;partnerCode:0 2023/01/04 02:00:21.549 [I] [com_trade.go:176] 290122获取数据失败，err:map[error:Something went wrong. Please try again or contact comtrade@un.org];errObject:；年份：2002;月份：1;flow:M;partnerCode:0 2023/01/04 02:12:09.179 [I] [com_trade.go:176] 290243获取数据失败，err:map[error:Something went wrong. Please try again or contact comtrade@un.org];errObject:；年份：2001;月份：12;flow:X;partnerCode:688,579,752,380,764,842,156,70 2023/01/04 02:29:32.109 [I] [com_trade.go:176] 290243获取数据失败，err:map[error:Something went wrong. Please try again or contact comtrade@un.org];errObject:；年份：2001;月份：11;flow:M;partnerCode:710,442,699,616,703 2023/01/04 02:29:33.532 [I] [com_trade.go:176] 290243获取数据失败，err:map[error:Something went wrong. Please try again or contact comtrade@un.org];errObject:；年份：2001;月份：11;flow:M;partnerCode:0 2023/01/04 02:29:34.888 [I] [com_trade.go:176] 290243获取数据失败，err:map[error:Something went wrong. Please try again or contact comtrade@un.org];errObject:；年份：2001;月份：11;flow:M;partnerCode:768,204,84,854,60,72,716,136,275,748,178,762,562,496,446,690,104,678,894,478,52,328,533,882,108,270,462,417,670,418,174 2023/01/04 02:37:00.528 [I] [com_trade.go:176] 381900获取数据失败，err:map[error:Something went wrong. Please try again or contact comtrade@un.org];errObject:；年份：2001;月份：10;flow:X;partnerCode:152,372,504,188,499,818,344,404,422,566,144,604,320,51,788,586 2023/01/04 02:37:01.816 [I] [com_trade.go:176] 381900获取数据失败，err:map[error:Something went wrong. Please try again or contact comtrade@un.org];errObject:；年份：2001;月份：10;flow:X;partnerCode:882,533,690,84,270,275,496,446,104,462,60,762,204,417,418,328,748,478,72,52,854,136,178,562,670,716,108,678,894,174,768 2023/01/04 02:37:03.143 [I] [com_trade.go:176] 381900获取数据失败，err:map[error:Something went wrong. Please try again or contact comtrade@un.org];errObject:；年份：2001;月份：10;flow:X;partnerCode:0 2023/01/04 02:44:23.164 [I] [com_trade.go:176] 290243获取数据失败，err:map[error:Something went wrong. Please try again or contact comtrade@un.org];errObject:；年份：2001;月份：10;flow:M;partnerCode:528,710,688,642,784,442,124,705 2023/01/04 02:44:24.971 [I] [com_trade.go:176] 290243获取数据失败，err:map[error:Something went wrong. Please try again or contact comtrade@un.org];errObject:；年份：2001;月份：10;flow:M;partnerCode:616,410,724,804,490,826,380,620 2023/01/04 02:58:29.473 [I] [com_trade.go:176] 270750获取数据失败，err:map[error:Something went wrong. Please try again or contact comtrade@un.org];errObject:；年份：2001;月份：9;flow:M;partnerCode:498,682,600,458,392,32,440,360 2023/01/04 02:58:31.133 [I] [com_trade.go:176] 270750获取数据失败，err:map[error:Something went wrong. Please try again or contact comtrade@un.org];errObject:；年份：2001;月份：9;flow:M;partnerCode:566,422,144,788,504,404,152,188,604,372,818,320,586,499,51,344 2023/01/04 03:24:13.204 [I] [com_trade.go:176] 270730获取数据失败，err:map[error:Something went wrong. Please try again or contact comtrade@un.org];errObject:；年份：2001;月份：7;flow:M;partnerCode:604,144,320,372,586,566,422,344,499,188,788,504,818,51,404,152 2023/01/04 03:24:14.661 [I] [com_trade.go:176] 270730获取数据失败，err:map[error:Something went wrong. Please try again or contact comtrade@un.org];errObject:；年份：2001;月份：7;flow:M;partnerCode:882,72,462,768,270,690,678,894,478,108,52,104,84,60,670,762,496,446,417,204,562,533,328,716,174,178,418,275,854,136,748 2023/01/04 03:33:55.137 [I] [com_trade.go:176] 270730获取数据失败，err:map[error:Something went wrong. Please try again or contact comtrade@un.org];errObject:；年份：2001;月份：6;flow:M;partnerCode:191,360,682,600,704,440,554,392,196,40,36,100,32,76,112,214` tmpList := strings.Split(str, "\n") list := make([]TmpStruct, 0) for _, v := range tmpList { //fmt.Println(v) tmp := strings.Split(v, "获取数据失败") //fmt.Println(tmp) // 指标编码 codeList := strings.Split(tmp[0], " ") indexCode := codeList[2] //fmt.Println(indexCode) // 年度 //年份：2001;月份：6;flow:M;partnerCode:191,360,682,600,704,440,554,392,196,40,36,100,32,76,112,214` tmp = strings.Split(tmp[1], "年份：") // 指标编码 tmpList2 := strings.Split(tmp[1], ";") yearStr := tmpList2[0] year, _ := strconv.Atoi(yearStr) //fmt.Println(year) // 月度 //月份：6;flow:M;partnerCode:191,360,682,600,704,440,554,392,196,40,36,100,32,76,112,214` tmp = strings.Split(tmp[1], "月份：") // 指标编码 tmpList2 = strings.Split(tmp[1], ";") monthStr := tmpList2[0] month, _ := strconv.Atoi(monthStr) //fmt.Println(month) // flow //flow:M;partnerCode:191,360,682,600,704,440,554,392,196,40,36,100,32,76,112,214` tmp = strings.Split(tmp[1], "flow:") // 指标编码 tmpList2 = strings.Split(tmp[1], ";") flowCode := tmpList2[0] //fmt.Println(flowCode) // partnerCode //flow:M;partnerCode:191,360,682,600,704,440,554,392,196,40,36,100,32,76,112,214` tmp = strings.Split(tmp[1], "partnerCode:") // 指标编码 partnerCode := tmp[1] //fmt.Println(partnerCode) list = append(list, TmpStruct{ IndexCode: indexCode, Year: year, Month: month, FlowCode: flowCode, PartnerCode: partnerCode, }) } //year := 2021 //month := 6 //flowCode := "M" //comTradeId := 6 //indexCode := `381900` //indexNameCn := `石脑油（出口退税率0%）` //thirdIdStr := `636,278,624,839,384,234,296,694,866,835,50,230,662,48,697,290,12,388,454,457,582,798,80,364,674,58,836,838,44,86,500,471,90,368,795,647,899,720,336,520,226,800,612,548,16,531,332,398,652,472,473,581,430,536,568,879,717,598,706,8,258,638,886,810,312,334,534,254,324,841,120,492,148,577,760,592,666` //thirdIdStr := `588,583,316,849,887,626,129,580,530,288,414,796,466,584,891,292,356,426,540,74,184,459,10,535,166,4,266,585,658,862,660,232,92,239,0,776,474,280,736,64,221,516,132,698,162,434,461,180,654,262,570,850,238,175,659,637,711,527,732,20,140,868,532,200,728,729,772,192,260,28,212,890,876,837,590,408,304,574` //thirdIdStr := `203,218,276,757` //thirdIdStr := `380,784,56,528` dateType := `month` //codeList := []string{ // `835,729,120,221,500,184,200,0,666,258,624,426,532,574,720,536,534,471,581,312,434,568,659,408,334,590,162,296,530,140,80,90,8,711`, // `866,732,736,364,459,698,706,12,674,234,516,891,849,260,304,388,899,492,612,660,772,694,862,697,592,472,324,531,226,92,28,232,583,647,654,166,64,838,336,430,58`, //} codeMappingListMap := make(map[string]string) codeMappingList, err := models.GetAllComTradeCodeMappingList() if err != nil { return } for _, v := range codeMappingList { codeMappingListMap[v.Code] = v.Name } countryMap, err := GetAllCountry() comTradeList, err := models.GetAllComTradeIndexList() if err != nil { return } comTradeMap := make(map[string]*models.ComTradeIndex) for _, v := range comTradeList { comTradeMap[v.IndexCode] = v } for key, v := range list { comTradeInfo, ok := comTradeMap[v.IndexCode] if !ok { fmt.Println("找不到指标") } //fmt.Println(v) fmt.Println(key, ";开始：", ";code：", v.IndexCode, "；年份：", v.Year, "；月份：", v.Month, "的", v.FlowCode, ";国家：", v.PartnerCode) syncComTradeData(comTradeInfo.ComTradeId, v.IndexCode, comTradeInfo.IndexNameCn, v.FlowCode, v.PartnerCode, dateType, v.Year, v.Month, codeMappingListMap, countryMap) fmt.Println(key, ";结束：", ";code：", v.IndexCode, "；年份：", v.Year, "；月份：", v.Month, "的", v.FlowCode, ";国家：", v.PartnerCode) time.Sleep(1 * time.Second) } } func syncAllComTradeData2(year, month int, flowCode, dateType string) (err error) { codeMappingListMap := make(map[string]string) codeMappingList, err := models.GetAllComTradeCodeMappingList() if err != nil { return } for _, v := range codeMappingList { codeMappingListMap[v.Code] = v.Name } comTradeList, err := models.GetAllComTradeIndexList() if err != nil { return } lenComTradeList := len(comTradeList) countryMap, err := GetAllCountry() if err != nil { return } countryGroup := make(map[int][]models.ComTradeCountry) for _, v := range countryMap { togetherCountry := v.TogetherCountry if dateType == "month" { togetherCountry = v.MonthTogetherCountry } tmpList, ok := countryGroup[togetherCountry] if !ok { tmpList = make([]models.ComTradeCountry, 0) } countryGroup[togetherCountry] = append(tmpList, v) } //fmt.Println(countryGroup) i := 0 startTime := time.Now() fmt.Println("整体开始了=============") for _, v := range comTradeList { fmt.Println("剩余", lenComTradeList-i, "个指标=============") syncSingleComTradeData(v.ComTradeId, year, month, flowCode, v.IndexCode, v.IndexNameCn, dateType, countryMap, codeMappingListMap, countryGroup) i++ } fmt.Println("整体结束了=============") fmt.Println("耗时", time.Now().Sub(startTime).String()) return } func syncAllComTradeData() (err error) { codeMappingListMap := make(map[string]string) codeMappingList, err := models.GetAllComTradeCodeMappingList() if err != nil { return } for _, v := range codeMappingList { codeMappingListMap[v.Code] = v.Name } comTradeList, err := models.GetAllComTradeIndexList() if err != nil { return } lenComTradeList := len(comTradeList) countryMap, err := GetAllCountry() if err != nil { return } countryGroup := make(map[int][]models.ComTradeCountry) for _, v := range countryMap { tmpList, ok := countryGroup[v.TogetherCountry] if !ok { tmpList = make([]models.ComTradeCountry, 0) } countryGroup[v.TogetherCountry] = append(tmpList, v) } //fmt.Println(countryGroup) //flowCode := "X" flowCode := "M" year := 2022 i := 0 startTime := time.Now() fmt.Println("整体开始了=============") for _, v := range comTradeList { fmt.Println("剩余", lenComTradeList-i, "个指标=============") syncSingleComTradeData(v.ComTradeId, year, 0, flowCode, v.IndexCode, v.IndexNameCn, "year", countryMap, codeMappingListMap, countryGroup) i++ } fmt.Println("整体结束了=============") fmt.Println("耗时", time.Now().Sub(startTime).String()) return } func syncSingleComTradeData(comTradeId, year, month int, flowCode, indexCode, indexNameCn, dateType string, countryMap map[int]models.ComTradeCountry, codeMappingListMap map[string]string, countryGroup map[int][]models.ComTradeCountry) { ////flowCode := "X" //flowCode := "M" //year := 2020 //partnerCode := 842 lenCountry := len(countryMap) i := 0 fmt.Println(indexCode, "开始了") for needMergeNum, countryList := range countryGroup { mergeNum := 0 thirdIdList := make([]string, 0) lenCountryList := len(countryList) - 1 for countryIndex, v := range countryList { fmt.Println(indexCode, "剩余", lenCountry-i, "条数据") i++ // 开始处理 mergeNum++ thirdIdList = append(thirdIdList, strconv.Itoa(v.ThirdId)) if mergeNum < needMergeNum && countryIndex < lenCountryList { continue } syncComTradeData(comTradeId, indexCode, indexNameCn, flowCode, strings.Join(thirdIdList, ","), dateType, year, month, codeMappingListMap, countryMap) // 重置 mergeNum = 0 thirdIdList = make([]string, 0) time.Sleep(1 * time.Second) } } fmt.Println(indexCode, "结束了") } func SyncSingleComTradeData3() { comTradeId := 7 year := 2022 month := 11 flowCode := "M" indexCode := "290121" indexNameCn := `乙烯（出口退税率13%）` dateType := `month` codeMappingListMap := make(map[string]string) codeMappingList, err := models.GetAllComTradeCodeMappingList() if err != nil { return } for _, v := range codeMappingList { codeMappingListMap[v.Code] = v.Name } countryMap, err := GetAllCountry() if err != nil { return } countryGroup := make(map[int][]models.ComTradeCountry) for _, v := range countryMap { togetherCountry := v.TogetherCountry if dateType == "month" { togetherCountry = v.MonthTogetherCountry } tmpList, ok := countryGroup[togetherCountry] if !ok { tmpList = make([]models.ComTradeCountry, 0) } countryGroup[togetherCountry] = append(tmpList, v) } //fmt.Println(countryGroup) fmt.Println(indexCode, "开始了") thirdIdStr := `849,192,12,296,212,473,234,364,879,580,698,20,582,850,316,568,459,697,866,540,4,280,288,461,658,44,430,598,798,868,92,180,536,810,574,266,841,16,140,148,652,334,637,660,232,760,471,332,520,548,516,839,414,531,200,64,166,581,837,221,592,500,466,292,336,48,886,290,80,120,262,583,795,612,836,258,530,736,838,368,8,28,129,278,90,800,184,674,588,492,887,230,570,534,776,239,527,891,0,356,729,226,324,876,626,720,426,654,590,312,772,706,50,304,457,796,408,434,624,454,474,254,532,584,10,132,835,662,666,717,260,728,398,711,238,890,899,74,388,585,472,58,577,647,659,638,862,732,535,86,175,636,694,162,384` syncComTradeData(comTradeId, indexCode, indexNameCn, flowCode, thirdIdStr, dateType, year, month, codeMappingListMap, countryMap) fmt.Println(indexCode, "结束了") } func syncComTradeData(comTradeId int, indexCode, indexNameCn, flowCode, partnerCode, dateType string, year, month int, codeMappingListMap map[string]string, countryMap map[int]models.ComTradeCountry) (err error) { // 获取数据 //https://comtradeapi.un.org/public/v1/preview/C/A/HS?period=2021&partnerCode=842&cmdCode=290244&flowCode=X&customsCode=C00&motCode=0 params := make(map[string]string) //period=2021&partnerCode=842 params["motCode"] = "0" //运输方式 params["customsCode"] = "C00" //海关编码 params["cmdCode"] = indexCode //指标code params["flowCode"] = flowCode //贸易流向，X：出口（Export）；M：进口（Import） params["partnerCode"] = partnerCode //进口国id params["period"] = strconv.Itoa(year) //年份（月份） dataTime := fmt.Sprint(year, "-12-31") codeStr := "Y" dataType := 1 if dateType == "month" { monthStr := strconv.Itoa(month) if month < 10 { monthStr = "0" + monthStr } params["period"] = strconv.Itoa(year) + monthStr //年份（月份） dataTime = fmt.Sprint(year, "-", monthStr, "-01") dataTimeT, tmpErr := time.ParseInLocation(utils.FormatDate, dataTime, time.Local) if tmpErr != nil { fmt.Println(dataTime, "时间转换异常") return } dataTime = dataTimeT.AddDate(0, 1, -1).Format(utils.FormatDate) codeStr = "M" dataType = 2 } // 这是获取数据的链接（月度的） comeData, err := queryComeTradeData(params, dateType) if err != nil { fmt.Println("读取失败", err) return } //fmt.Println(comeData) if comeData.Count >= 500 { msg := fmt.Sprint(indexCode, "；数据超限了", "；年份：", year, ";月份：", month, ";flow:", flowCode, ";partnerCode:", partnerCode) fmt.Println(msg) fmt.Println(comeData.Count) utils.FileLog.Info(msg) } else if comeData.Count == -1 { msg := fmt.Sprint(indexCode, "；获取数据失败，err:", comeData.Error, ";errObject:", comeData.ErrorObject, "；年份：", year, ";月份：", month, ";flow:", flowCode, ";partnerCode:", partnerCode) fmt.Println(msg, comeData) utils.FileLog.Info(msg) return } listMap := make(map[string][]*models.ComTradeData, 0) for _, v := range comeData.Data { //fmt.Println(v) var reporterName, partnerName, partner2Name, reporterCnName, partnerCnName, partner2CnName string if reporter, ok := countryMap[v.ReporterCode]; ok { reporterCnName = reporter.NameCn reporterName = reporter.Name } if partner, ok := countryMap[v.PartnerCode]; ok { partnerCnName = partner.NameCn partnerName = partner.Name } if partner, ok := countryMap[v.Partner2Code]; ok { partner2CnName = partner.NameCn partner2Name = partner.Name } //ValueCode：HS+Reporter+trade flow+Partner+Value 例如：270730ChinaXUSAValue //WeightCode：HS+Reporter+trade flow+Partner+Weight ,例如：270730ChinaXUSAWeight tmpIndexTradeCode := utils.TrimStr(fmt.Sprint(v.CmdCode, codeStr, reporterName, flowCode, partnerName, "2nd", partner2Name, "Value")) tmpIndexNetWeightCode := utils.TrimStr(fmt.Sprint(v.CmdCode, codeStr, reporterName, flowCode, partnerName, "2nd", partner2Name, "Weight")) tmpData := &models.ComTradeData{ ComTradeDataId: 0, ComTradeId: comTradeId, IndexCode: v.CmdCode, Flow: flowCode, ReporterCode: v.ReporterCode, ReporterName: reporterCnName, PartnerCode: v.PartnerCode, PartnerName: partnerCnName, Partner2Code: v.Partner2Code, Partner2Name: partner2CnName, DateType: dataType, DataTime: dataTime, IndexTradeCode: tmpIndexTradeCode, TradeValue: v.PrimaryValue, IndexNetWeightCode: tmpIndexNetWeightCode, NetWeightValue: v.NetWgt, ModifyTime: time.Now(), CreateTime: time.Now(), } tmpList, ok := listMap[v.CmdCode] if !ok { tmpList = make([]*models.ComTradeData, 0) } listMap[v.CmdCode] = append(tmpList, tmpData) //指标关系 { tmpName := indexNameCn + "年度" + reporterCnName if dateType == "month" { tmpName = indexNameCn + "月度" + reporterCnName } flow := `` if tmpData.Flow == `X` { flow = "出口" } else { flow = "进口" } tmpName += flow finalpartnerName := `` if partnerCnName == partner2CnName { if partnerCnName == "世界" { //若对手国和第二对手国是world时，则翻译为总量； finalpartnerName = "总量" } else { //若对手国或第二对手国是相同的，只需翻译一个； finalpartnerName = partnerCnName } } else { if partnerCnName == "世界" || partner2CnName == "世界" { //若对手国或第二对手国是world的，只需翻译非world的对手国； if partnerCnName == "世界" { finalpartnerName = partner2CnName } else { finalpartnerName = partnerCnName } } else { //若对手国和第二对手国不一样，翻译成：对手国和第二对手国 finalpartnerName = partnerCnName + "和" + partner2CnName } } tmpName += finalpartnerName //fmt.Println(tmpData.IndexTradeCode) //fmt.Println(tmpName) valueName := tmpName + "贸易金额" weightName := tmpName + "贸易重量" if name, ok := codeMappingListMap[tmpData.IndexTradeCode]; !ok { comTradeCodeMappingType := 1 if codeStr == "M" { comTradeCodeMappingType = 3 } tmpComTradeCodeMapping := &models.ComTradeCodeMapping{ Code: tmpData.IndexTradeCode, IndexCode: tmpData.IndexCode, Type: comTradeCodeMappingType, Name: valueName, CreateTime: time.Now(), } tmpErr := models.AddComTradeCodeMapping(tmpComTradeCodeMapping) if tmpErr != nil { fmt.Println(indexCode, "添加贸易金额失败,err:", err) } else { codeMappingListMap[tmpData.IndexTradeCode] = valueName } } else { if name != valueName { tmpComTradeCodeMapping := &models.ComTradeCodeMapping{ Code: tmpData.IndexTradeCode, Name: valueName, } e := models.UpdateComTradeCodeMapping(tmpComTradeCodeMapping, []string{"Name"}) if e != nil { err = e return } utils.FileLog.Info("更新指标名称", "旧："+name, "新："+valueName, "code:"+tmpData.IndexTradeCode) fmt.Println("更新指标名称", "旧："+name, "新："+valueName) } } if name, ok := codeMappingListMap[tmpData.IndexNetWeightCode]; !ok { comTradeCodeMappingType := 2 if codeStr == "M" { comTradeCodeMappingType = 3 } tmpComTradeCodeMapping := &models.ComTradeCodeMapping{ Code: tmpData.IndexNetWeightCode, IndexCode: tmpData.IndexCode, Type: comTradeCodeMappingType, Name: weightName, CreateTime: time.Now(), } tmpErr := models.AddComTradeCodeMapping(tmpComTradeCodeMapping) if tmpErr != nil { fmt.Println(indexCode, "添加贸易重量失败,err:", err) } else { codeMappingListMap[tmpData.IndexNetWeightCode] = weightName } } else { if name != weightName { tmpComTradeCodeMapping := &models.ComTradeCodeMapping{ Code: tmpData.IndexTradeCode, Name: valueName, } e := models.UpdateComTradeCodeMapping(tmpComTradeCodeMapping, []string{"Name"}) if e != nil { err = e return } utils.FileLog.Info("更新指标名称", "旧："+name, "新："+valueName, "code:"+tmpData.IndexNetWeightCode) fmt.Println("更新指标名称", "旧："+name, "新："+valueName) } } } } for indexCode, list := range listMap { models.HandleComTradeData(list, indexCode, flowCode, dataTime, partnerCode) } return } // queryComeTradeData 接口请求网站数据 func queryComeTradeData(params map[string]string, dateType string) (comTradeResp ComTradeResp, err error) { //ComTradeUrl // 待签名字符串 paramStr := `` for index, val := range params { paramStr += index + `=` + val + `&` } paramStr = paramStr[:len(paramStr)-1] urlStr := ComTradeUrl + paramStr if dateType == "month" { urlStr = ComTradeMonthUrl + paramStr } //fmt.Println("urlStr:", urlStr) body, err := http.Get(urlStr) if err != nil { fmt.Println("err:", err) } //utils.FileLog.Info("queryComeTradeData 数据：" + ComTradeUrl + paramStr) err = json.Unmarshal(body, &comTradeResp) return } type ComTradeResp struct { ElapsedTime string `json:"elapsedTime"` Count int `json:"count"` Data []struct { TypeCode string `json:"typeCode"` FreqCode string `json:"freqCode"` RefPeriodId int `json:"refPeriodId"` RefYear int `json:"refYear"` RefMonth int `json:"refMonth"` Period string `json:"period"` ReporterCode int `json:"reporterCode"` ReporterISO interface{} `json:"reporterISO"` ReporterDesc interface{} `json:"reporterDesc"` FlowCode string `json:"flowCode"` FlowDesc interface{} `json:"flowDesc"` PartnerCode int `json:"partnerCode"` PartnerISO interface{} `json:"partnerISO"` PartnerDesc interface{} `json:"partnerDesc"` Partner2Code int `json:"partner2Code"` Partner2ISO interface{} `json:"partner2ISO"` Partner2Desc interface{} `json:"partner2Desc"` ClassificationCode string `json:"classificationCode"` ClassificationSearchCode string `json:"classificationSearchCode"` IsOriginalClassification bool `json:"isOriginalClassification"` CmdCode string `json:"cmdCode"` CmdDesc interface{} `json:"cmdDesc"` AggrLevel int `json:"aggrLevel"` IsLeaf interface{} `json:"isLeaf"` CustomsCode string `json:"customsCode"` CustomsDesc interface{} `json:"customsDesc"` MosCode string `json:"mosCode"` MotCode int `json:"motCode"` MotDesc interface{} `json:"motDesc"` QtyUnitCode int `json:"qtyUnitCode"` QtyUnitAbbr interface{} `json:"qtyUnitAbbr"` Qty float64 `json:"qty"` IsQtyEstimated bool `json:"isQtyEstimated"` AltQtyUnitCode int `json:"altQtyUnitCode"` AltQtyUnitAbbr interface{} `json:"altQtyUnitAbbr"` AltQty float64 `json:"altQty"` IsAltQtyEstimated bool `json:"isAltQtyEstimated"` NetWgt float64 `json:"netWgt"` IsNetWgtEstimated bool `json:"isNetWgtEstimated"` GrossWgt float64 `json:"grossWgt"` IsGrossWgtEstimated bool `json:"isGrossWgtEstimated"` Cifvalue float64 `json:"cifvalue"` Fobvalue float64 `json:"fobvalue"` PrimaryValue float64 `json:"primaryValue"` LegacyEstimationFlag int `json:"legacyEstimationFlag"` IsReported bool `json:"isReported"` IsAggregate bool `json:"isAggregate"` } `json:"data"` Error interface{} `json:"error"` ErrorObject interface{} `json:"errorObject"` } func GetAllCountry() (countryMap map[int]models.ComTradeCountry, err error) { countryMap = make(map[int]models.ComTradeCountry) tmpList, err := models.GetComTradeCountryAll() if err != nil { return } for _, v := range tmpList { countryMap[v.ThirdId] = *v } return } func syncAllComTradeMapping() (err error) { codeMappingListMap := make(map[string]bool) codeMappingList, err := models.GetAllComTradeCodeMappingList() if err != nil { return } for _, v := range codeMappingList { codeMappingListMap[v.Code] = true } comTradeIndexMap := make(map[string]string) comTradeIndexList, err := models.GetAllComTradeIndexList() if err != nil { return } for _, v := range comTradeIndexList { comTradeIndexMap[v.IndexCode] = v.IndexNameCn } list, err := models.GetAllComTradeDataList() if err != nil { return } countryMap, err := GetAllCountry() if err != nil { return } lenList := len(list) fmt.Println("总共", lenList, "条") for k, v := range list { fmt.Println("剩余：", lenList-k-1, "条") var reporterName, partnerName, partner2Name string if reporter, ok := countryMap[v.ReporterCode]; ok { reporterName = reporter.Name } if partner, ok := countryMap[v.PartnerCode]; ok { partnerName = partner.Name } if partner, ok := countryMap[v.Partner2Code]; ok { partner2Name = partner.Name } tmpValueCode := fmt.Sprint(v.IndexCode, "Y", reporterName, v.Flow, partnerName, "2nd", partner2Name, "Value") tmpNetWeightCode := fmt.Sprint(v.IndexCode, "Y", reporterName, v.Flow, partnerName, "2nd", partner2Name, "Weight") //v.IndexTradeCode = utils.TrimStr(tmpValueCode) //v.IndexNetWeightCode = utils.TrimStr(tmpNetWeightCode) //v.Update([]string{"IndexTradeCode", "IndexNetWeightCode"}) models.UpdateComTradeData(utils.TrimStr(tmpValueCode), utils.TrimStr(tmpNetWeightCode), v.IndexTradeCode) } fmt.Println("整体结束了=============") return } func syncAllComTradeMapping2() (err error) { codeMappingListMap := make(map[string]bool) codeMappingList, err := models.GetAllComTradeCodeMappingList() if err != nil { return } for _, v := range codeMappingList { codeMappingListMap[v.Code] = true } comTradeIndexMap := make(map[string]string) comTradeIndexList, err := models.GetAllComTradeIndexList() if err != nil { return } for _, v := range comTradeIndexList { comTradeIndexMap[v.IndexCode] = v.IndexNameCn } list, err := models.GetAllComTradeDataList() if err != nil { return } countryMap, err := GetAllCountry() if err != nil { return } lenList := len(list) fmt.Println("总共", lenList, "条") dataList := make([]*models.ComTradeCodeMapping, 0) for k, v := range list { fmt.Println("剩余：", lenList-k-1, "条") var reporterName string if reporter, ok := countryMap[v.ReporterCode]; ok { reporterName = reporter.NameCn } tmpName := `` if comTradeIndexName, ok := comTradeIndexMap[v.IndexCode]; ok { tmpName = comTradeIndexName } tmpName += reporterName flow := `` if v.Flow == `X` { flow = "出口" } else { flow = "进口" } tmpName += flow valueName := tmpName + "贸易金额" weightName := tmpName + "贸易重量" if _, ok := codeMappingListMap[v.IndexTradeCode]; !ok { tmpData := &models.ComTradeCodeMapping{ Code: v.IndexTradeCode, IndexCode: v.IndexCode, Type: 1, Name: valueName, CreateTime: time.Now(), } dataList = append(dataList, tmpData) codeMappingListMap[v.IndexTradeCode] = true //tmpErr := models.AddComTradeCodeMapping(tmpData) //if tmpErr != nil { // fmt.Println(indexCode, "添加贸易金额失败,err:", err) //} else { // codeMappingListMap[v.IndexTradeCode] = true //} } if _, ok := codeMappingListMap[v.IndexNetWeightCode]; !ok { tmpData := &models.ComTradeCodeMapping{ Code: v.IndexNetWeightCode, IndexCode: v.IndexCode, Type: 2, Name: weightName, CreateTime: time.Now(), } dataList = append(dataList, tmpData) codeMappingListMap[v.IndexNetWeightCode] = true //tmpErr := models.AddComTradeCodeMapping(tmpData) //if tmpErr != nil { // fmt.Println(indexCode, "添加贸易重量失败,err:", err) //} else { // codeMappingListMap[v.IndexNetWeightCode] = true //} } if len(dataList) >= 2000 { tmpErr := models.AddMultiComTradeCodeMapping(dataList) if tmpErr != nil { fmt.Println(indexCode, "添加贸易重量失败,err:", tmpErr) } dataList = make([]*models.ComTradeCodeMapping, 0) } } if len(dataList) > 0 { tmpErr := models.AddMultiComTradeCodeMapping(dataList) if tmpErr != nil { fmt.Println(indexCode, "添加贸易重量失败,err:", tmpErr) } dataList = make([]*models.ComTradeCodeMapping, 0) } fmt.Println("整体结束了=============") return } func syncAllComTradeMapping3() { codeMappingListMap := make(map[string]*models.ComTradeCodeMapping) codeMappingList, err := models.GetAllComTradeCodeMappingList() if err != nil { return } for _, v := range codeMappingList { codeMappingListMap[v.Code] = v } comTradeIndexMap := make(map[string]string) comTradeIndexList, err := models.GetAllComTradeIndexList() if err != nil { return } for _, v := range comTradeIndexList { comTradeIndexMap[v.IndexCode] = v.IndexNameCn } list, err := models.GetAllComTradeDataList() if err != nil { return } countryMap, err := GetAllCountry() if err != nil { return } lenList := len(list) fmt.Println("总共", lenList, "条") dataList := make([]*models.ComTradeCodeMapping, 0) for k, v := range list { fmt.Println("剩余：", lenList-k-1, "条") var reporterCnName, partnerCnName, partner2CnName string if reporter, ok := countryMap[v.ReporterCode]; ok { reporterCnName = reporter.NameCn } if partner, ok := countryMap[v.PartnerCode]; ok { partnerCnName = partner.NameCn } if partner, ok := countryMap[v.Partner2Code]; ok { partner2CnName = partner.NameCn } indexNameCn := comTradeIndexMap[v.IndexCode] if indexNameCn == "" { fmt.Println(indexNameCn) fmt.Println(v.IndexCode) } codeStr := "Y" tmpName := indexNameCn + "年度" + reporterCnName if v.DateType == 2 { codeStr = "M" tmpName = indexNameCn + "月度" + reporterCnName } flow := `` if v.Flow == `X` { flow = "出口" } else { flow = "进口" } tmpName += flow finalpartnerName := `` if partnerCnName == partner2CnName { if partnerCnName == "世界" { //若对手国和第二对手国是world时，则翻译为总量； finalpartnerName = "总量" } else { //若对手国或第二对手国是相同的，只需翻译一个； finalpartnerName = partnerCnName } } else { if partnerCnName == "世界" || partner2CnName == "世界" { //若对手国或第二对手国是world的，只需翻译非world的对手国； if partnerCnName == "世界" { finalpartnerName = partner2CnName } else { finalpartnerName = partnerCnName } } else { //若对手国和第二对手国不一样，翻译成：对手国和第二对手国 finalpartnerName = partnerCnName + "和" + partner2CnName } } tmpName += finalpartnerName valueName := tmpName + "贸易金额" weightName := tmpName + "贸易重量" if _, ok := codeMappingListMap[v.IndexTradeCode]; !ok { comTradeCodeMappingType := 1 if codeStr == "M" { comTradeCodeMappingType = 3 } tmpData := &models.ComTradeCodeMapping{ Code: v.IndexTradeCode, IndexCode: v.IndexCode, Type: comTradeCodeMappingType, Name: valueName, CreateTime: time.Now(), } dataList = append(dataList, tmpData) codeMappingListMap[v.IndexTradeCode] = tmpData //tmpErr := models.AddComTradeCodeMapping(tmpData) //if tmpErr != nil { // fmt.Println(indexCode, "添加贸易金额失败,err:", err) //} else { // codeMappingListMap[v.IndexTradeCode] = true //} } if _, ok := codeMappingListMap[v.IndexNetWeightCode]; !ok { comTradeCodeMappingType := 2 if codeStr == "M" { comTradeCodeMappingType = 4 } tmpData := &models.ComTradeCodeMapping{ Code: v.IndexNetWeightCode, IndexCode: v.IndexCode, Type: comTradeCodeMappingType, Name: weightName, CreateTime: time.Now(), } dataList = append(dataList, tmpData) codeMappingListMap[v.IndexNetWeightCode] = tmpData //tmpErr := models.AddComTradeCodeMapping(tmpData) //if tmpErr != nil { // fmt.Println(indexCode, "添加贸易重量失败,err:", err) //} else { // codeMappingListMap[v.IndexNetWeightCode] = true //} } if len(dataList) >= 2000 { tmpErr := models.AddMultiComTradeCodeMapping(dataList) if tmpErr != nil { fmt.Println(indexCode, "添加贸易重量失败,err:", tmpErr) } dataList = make([]*models.ComTradeCodeMapping, 0) } } if len(dataList) > 0 { tmpErr := models.AddMultiComTradeCodeMapping(dataList) if tmpErr != nil { fmt.Println(indexCode, "添加贸易重量失败,err:", tmpErr) } dataList = make([]*models.ComTradeCodeMapping, 0) } fmt.Println("整体结束了=============") return }