package base_from_hisugar import ( "context" "encoding/json" "eta/eta_data_analysis/models" "eta/eta_data_analysis/utils" "fmt" "io/ioutil" "log" "net/http" "net/url" "strconv" "strings" "time" ) // 国产糖 var GuochantangList = []string{ "国内食糖月度累计销售均价", "国内食糖月度销售均价", "全国食糖累计产销率", "全国单月产糖量", "全国单月销糖量", "国内月度工业库存", "全国月度累计糖产量", "全国月度累计糖销量", "国内白糖产区现货价格", "国内制糖集团报价", "国内加工糖集团报价", "全国单月甘蔗糖产量", "全国单月甘蔗糖销量", "全国单月甜菜产糖量", "全国单月甜菜糖销量", "广西单月产糖量", "广西单月销糖量", "广西月度库存", "云南单月产糖量", "云南单月销糖量", "云南月度库存", "海南单月产糖量", "海南单月销糖量", "海南月度库存", "广东单月产糖量", "广东单月销糖量", "广东月度库存", "新疆单月产糖量", "新疆单月销糖量", "新疆月度库存", "内蒙单月产糖量", "内蒙单月销糖量", "内蒙月度库存", "食糖月度进口量", "巴西港口等待船只数量", "巴西港口最大等待天数", "巴西港口待装运食糖数量", "周度含水乙醇折糖价", "巴西双周累计产糖量", "巴西双周食糖库存", "巴西食糖月度出口量", "中南部双周累计产糖量", "中南部双周糖库存", "中南部双周乙醇产量", "巴西双周甘蔗累计入榨量", "中南部双周甘蔗压榨量", "中南部双周制糖比", "巴西双周甘蔗入榨量", "巴西中南部乙醇月度销售情况", "巴西中南部乙醇国内销售情况", "巴西双周无水乙醇库存", "巴西双周水合乙醇库存", "印度食糖月度出口量", "印度双周累计产糖量", "泰国双周糖累计产量", "泛糖副产品平台桔水成交价格指数", } var HisugarGuoChanTangParmsMap = map[string]string{ "国内食糖月度累计销售均价": shiTangYueDuLeiJiXiaoShouJunJia, "国内食糖月度销售均价": shiTangYueDuXiaoShouJunJia, "全国食糖累计产销率": shiTangLeiJiChanXiaoLv, "全国单月产糖量": quanGuoDanYueChanTangLiang, "全国单月销糖量": quanGuoDanYueXiaoTangLiang, "国内月度工业库存": guoNeiYueDuGongYeKuCun, "全国月度累计糖产量": quanGuoYueDuLeiJiTangChanLiang, "全国月度累计糖销量": quanGuoYueDuLeiJiTangXiaoLiang, "国内白糖产区现货价格": guoNeiBaiTangChanQuXianHuoJiaGe, "国内制糖集团报价": guoNeiZhiTangJiTuanBaoJia, "国内加工糖集团报价": guoNeiJiaGongTangJiTuanBaoJia, "广西单月产糖量": guangXiDanYueChanTangLiang, "广西单月销糖量": guangXiDanYueXiaoTangLiang, "广西月度库存": guangXiYueDuKuCun, "云南单月产糖量": yunNanDanYueChanTangLiang, "云南单月销糖量": yunNanDanYueXiaoTangLiang, "云南月度库存": yunNanYueDukuCun, "海南单月产糖量": haiNanDanYueChanTangLiang, "海南单月销糖量": haiNanDanyueXiaoTangliang, "海南月度库存": haiNanYueduKuCun, "广东单月产糖量": guangDongDanYueChanTangLiang, "广东单月销糖量": guangDongDanYueXiaoTangLiang, "广东月度库存": guangDongYueDuKuCun, "新疆单月产糖量": xinJiangDanYueChanTangLiang, "新疆单月销糖量": xinJiangDanYueXiaoTangLiang, "新疆月度库存": xinJiangYueDuKuCun, "内蒙单月产糖量": neiMengDanYueChanTangLiang, "内蒙单月销糖量": neiMengDanYueXiaoTangLiang, "内蒙月度库存": neiMengYueDuKuCun, "食糖月度进口量": shiTangYueDuJinKouLiang, "巴西港口等待船只数量": baXiGangKouDengDaiChuanZhiShuLiang, "巴西港口最大等待天数": baXiGangKouZuiDaDengDaiTianShu, "巴西港口待装运食糖数量": baXiGangKouDaiZhuangYunShiTiaoShangShuLiang, "周度含水乙醇折糖价": zhouDuHanShuiYiChunZheTangJia, "巴西双周累计产糖量": baXiShuangZhouLeiJiChanTangLiang, "巴西双周食糖库存": baXiShuangZhouShiTangKuCun, "巴西食糖月度出口量": baXiShiTangYueDuChuKouLiang, "中南部双周累计产糖量": zhongNanShuangZhouLiJiChanTangLiang, "中南部双周糖库存": zhongNanShuangZhouTangKuCun, "中南部双周乙醇产量": zhongNanShuangZhouXianHuoChanLiang, "巴西双周甘蔗累计入榨量": baXiShuangZhouGanZheLeiJiRuZhaLiang, "中南部双周甘蔗压榨量": zhongNanBuShuangZhouGanZheYaZhaLiang, "中南部双周制糖比": zhongNanBuShuangZhouZHiTangBi, "巴西双周甘蔗入榨量": baXiShuangZhouGanZheRuZhaLiang, "巴西中南部乙醇月度销售情况": baXiZhongNanBuYiChunYueDuXiaoShouQingKuang, "巴西中南部乙醇国内销售情况": baXiZhongNanBuYiChunGuoNeiXiaoShouQingKuang, "巴西双周无水乙醇库存": baXiShuangZhouWuShuiYiChunKuCun, "巴西双周水合乙醇库存": baXiShuangZhouShuiHeYiChunKuCun, "印度食糖月度出口量": yinDuShiTangYueDuChuKouLiang, "印度双周累计产糖量": yinDuShuangZhouLeiJiChanTangLiang, "泰国双周糖累计产量": taiGuoShuangZhouTangLeiJiChanLiang, "泛糖副产品平台桔水成交价格指数": JuShuiChengJIaoJiaGeZhiShu, "全国单月甘蔗糖产量": quanGuoDanYueGanZheTangChanLiang, "全国单月甘蔗糖销量": quanGuoDanYueGanZheTangXiaoLiang, "全国单月甜菜产糖量": quanGuoDanYueTianCaiChanLiang, "全国单月甜菜糖销量": quanGuoDanYueTianCaiXiaoLiang, } var HisugarGuoChanTangUnitMap = map[string]string{ "国内食糖月度累计销售均价": "元/吨", "国内食糖月度销售均价": "元/吨", "全国食糖累计产销率": "%", "全国单月产糖量": "万吨", "全国单月销糖量": "万吨", "国内月度工业库存": "万吨", "全国月度累计糖产量": "万吨", "全国月度累计糖销量": "万吨", "国内白糖产区现货价格": "元/吨", "国内制糖集团报价": "元/吨", "国内加工糖集团报价": "元/吨", "广西单月产糖量": "万吨", "广西单月销糖量": "万吨", "广西月度库存": "万吨", "云南单月产糖量": "万吨", "云南单月销糖量": "万吨", "云南月度库存": "万吨", "海南单月产糖量": "万吨", "海南单月销糖量": "万吨", "海南月度库存": "万吨", "广东单月产糖量": "万吨", "广东单月销糖量": "万吨", "广东月度库存": "万吨", "新疆单月产糖量": "万吨", "新疆单月销糖量": "万吨", "新疆月度库存": "万吨", "内蒙单月产糖量": "万吨", "内蒙单月销糖量": "万吨", "内蒙月度库存": "万吨", "食糖月度进口量": "万吨", "巴西港口等待船只数量": "艘", "巴西港口最大等待天数": "天", "巴西港口待装运食糖数量": "万吨", "周度含水乙醇折糖价": "美分/磅", "巴西双周累计产糖量": "万吨", "巴西双周食糖库存": "万吨", "巴西食糖月度出口量": "万吨", "中南部双周累计产糖量": "万吨", "中南部双周糖库存": "万吨", "中南部双周乙醇产量": "立方米", "巴西双周甘蔗累计入榨量": "万吨", "中南部双周甘蔗压榨量": "吨", "中南部双周制糖比": "%", "巴西双周甘蔗入榨量": "万吨", "巴西中南部乙醇月度销售情况": "万立方米", "巴西中南部乙醇国内销售情况": "万立方米", "巴西双周无水乙醇库存": "万立方米", "巴西双周水合乙醇库存": "万立方米", "印度食糖月度出口量": "万吨", "印度双周累计产糖量": "万吨", "泰国双周糖累计产量": "万吨", "全国单月甘蔗糖产量": "万吨", "全国单月甘蔗糖销量": "万吨", "全国单月甜菜产糖量": "万吨", "全国单月甜菜糖销量": "万吨", } var HisugarTaskAnalysisHandlers = map[string]func(name, reqParam, body string) (err error){ "国内食糖月度累计销售均价": AnalysisHiSugar, "国内食糖月度销售均价": AnalysisHiSugar, "全国食糖累计产销率": AnalysisHiSugar3, "全国单月产糖量": AnalysisHiSugar3, "全国单月销糖量": AnalysisHiSugar3, "国内月度工业库存": AnalysisHiSugar3, "全国月度累计糖产量": AnalysisHiSugar4, "全国月度累计糖销量": AnalysisHiSugar4, "国内白糖产区现货价格": AnalysisHiSugar5, "国内制糖集团报价": AnalysisHiSugar5, "国内加工糖集团报价": AnalysisHiSugar5, "全国单月甘蔗糖产量": AnalysisHiSugar3, "全国单月甘蔗糖销量": AnalysisHiSugar3, "全国单月甜菜产糖量": AnalysisHiSugar3, "全国单月甜菜糖销量": AnalysisHiSugar3, "广西单月产糖量": AnalysisHiSugar, "广西单月销糖量": AnalysisHiSugar, "广西月度库存": AnalysisHiSugar, "云南单月产糖量": AnalysisHiSugar, "云南单月销糖量": AnalysisHiSugar, "云南月度库存": AnalysisHiSugar, "海南单月产糖量": AnalysisHiSugar, "海南单月销糖量": AnalysisHiSugar, "海南月度库存": AnalysisHiSugar, "广东单月产糖量": AnalysisHiSugar, "广东单月销糖量": AnalysisHiSugar, "广东月度库存": AnalysisHiSugar, "新疆单月产糖量": AnalysisHiSugar, "新疆单月销糖量": AnalysisHiSugar, "新疆月度库存": AnalysisHiSugar, "内蒙单月产糖量": AnalysisHiSugar, "内蒙单月销糖量": AnalysisHiSugar, "内蒙月度库存": AnalysisHiSugar, "食糖月度进口量": AnalysisHiSugar6, "巴西港口等待船只数量": AnalysisHiSugar7, "巴西港口最大等待天数": AnalysisHiSugar7, "巴西港口待装运食糖数量": AnalysisHiSugar7, "周度含水乙醇折糖价": AnalysisHiSugar5, "巴西双周累计产糖量": AnalysisHiSugar11, "巴西双周食糖库存": AnalysisHiSugar12, "巴西食糖月度出口量": AnalysisHiSugar6, "中南部双周累计产糖量": AnalysisHiSugar12, "中南部双周糖库存": AnalysisHiSugar12, "中南部双周乙醇产量": AnalysisHiSugar12, "巴西双周甘蔗累计入榨量": AnalysisHiSugar12, "中南部双周甘蔗压榨量": AnalysisHiSugar12, "中南部双周制糖比": AnalysisHiSugar12, "巴西双周甘蔗入榨量": AnalysisHiSugar12, "巴西中南部乙醇月度销售情况": AnalysisHiSugar13, "巴西中南部乙醇国内销售情况": AnalysisHiSugar13, "巴西双周无水乙醇库存": AnalysisHiSugar12, "巴西双周水合乙醇库存": AnalysisHiSugar12, "印度食糖月度出口量": AnalysisHiSugar8, "印度双周累计产糖量": AnalysisHiSugar10, "泰国双周糖累计产量": AnalysisHiSugar9, "泛糖副产品平台桔水成交价格指数": AnalysisHiSugar3, } var HisugarGuoChanTangClassifyMap = map[string]uint{ "国内食糖月度累计销售均价": 5, "国内食糖月度销售均价": 5, "全国食糖累计产销率": 5, "全国单月产糖量": 5, "全国单月销糖量": 5, "国内月度工业库存": 5, "全国月度累计糖产量": 5, "全国月度累计糖销量": 5, "国内白糖产区现货价格": 5, "国内制糖集团报价": 5, "国内加工糖集团报价": 5, "全国单月甘蔗糖产量": 5, "全国单月甘蔗糖销量": 5, "全国单月甜菜产糖量": 5, "全国单月甜菜糖销量": 5, "广西单月产糖量": 6, "广西单月销糖量": 6, "广西月度库存": 6, "云南单月产糖量": 8, "云南单月销糖量": 8, "云南月度库存": 8, "海南单月产糖量": 9, "海南单月销糖量": 9, "海南月度库存": 9, "广东单月产糖量": 7, "广东单月销糖量": 7, "广东月度库存": 7, "新疆单月产糖量": 10, "新疆单月销糖量": 10, "新疆月度库存": 10, "内蒙单月产糖量": 11, "内蒙单月销糖量": 11, "内蒙月度库存": 11, "食糖月度进口量": 12, "印度食糖月度出口量": 14, "印度双周累计产糖量": 14, "泰国双周糖累计产量": 15, "泛糖副产品平台桔水成交价格指数": 16, "巴西食糖月度出口量": 13, "巴西中南部乙醇月度销售情况": 13, "巴西中南部乙醇国内销售情况": 13, } var DescribeMap = map[string]string{ "国内食糖月度累计销售均价": "中国糖业协会", "国内食糖月度销售均价": "中国糖业协会", "全国食糖累计产销率": "中国糖业协会", "全国单月产糖量": "中国糖业协会", "全国单月销糖量": "中国糖业协会", "国内月度工业库存": "中国糖业协会", "全国月度累计糖产量": "中国糖业协会", "全国月度累计糖销量": "中国糖业协会", "广西单月产糖量": "广西糖业协会", "广西单月销糖量": "广西糖业协会", "广西月度库存": "广西糖业协会", "云南单月产糖量": "云南省糖业协会", "云南单月销糖量": "云南省糖业协会", "云南月度库存": "云南省糖业协会", "海南单月产糖量": "海南省糖业协会", "海南单月销糖量": "海南省糖业协会", "海南月度库存": "海南省糖业协会", "广东单月产糖量": "广东省单月产糖量", "广东单月销糖量": "广东省单月产糖量", "广东月度库存": "广东省单月产糖量", "新疆单月产糖量": "中国糖业协会", "新疆单月销糖量": "中国糖业协会", "新疆月度库存": "中国糖业协会", "内蒙单月产糖量": "内蒙古糖业协会", "内蒙单月销糖量": "内蒙古糖业协会", "内蒙月度库存": "内蒙古糖业协会", "食糖月度进口量": "海关总署", "巴西港口等待船只数量": "泛糖科技", "巴西港口最大等待天数": "泛糖科技", "巴西港口待装运食糖数量": "泛糖科技", "周度含水乙醇折糖价": "泛糖科技", "巴西双周累计产糖量": "巴西农业部", "巴西双周食糖库存": "巴西农业部", "巴西食糖月度出口量": "巴西农业部", "中南部双周累计产糖量": "巴西农业部", "中南部双周糖库存": "巴西农业部", "中南部双周乙醇产量": "Unica", "巴西双周甘蔗累计入榨量": "巴西农业部", "中南部双周甘蔗压榨量": "Unica", "中南部双周制糖比": "Unica", "巴西双周甘蔗入榨量": "巴西农业部", "巴西中南部乙醇月度销售情况": "巴西农业部", "巴西中南部乙醇国内销售情况": "巴西农业部", "巴西双周无水乙醇库存": "巴西农业部", "巴西双周水合乙醇库存": "巴西农业部", "印度食糖月度出口量": "印度商务部", "印度双周累计产糖量": "ISMA", "泰国双周糖累计产量": "OCSB", "全国单月甘蔗糖产量": "中国糖业协会", "全国单月甘蔗糖销量": "中国糖业协会", "全国单月甜菜产糖量": "中国糖业协会", "全国单月甜菜糖销量": "中国糖业协会", "泛糖副产品平台桔水成交价格指数": "泛糖科技", } // 爬虫主体 func HiugarCrawler(context.Context) (err error) { // 模拟登陆获取cookie cookie, err := HisugarLogin() if err != nil { return } // 目标 URL urlStr := "http://222.84.157.44:12880/tempo/gridController/loadData" for _, name := range GuochantangList { reqParam := HisugarGuoChanTangParmsMap[name] // 构建表单数据 formData := url.Values{ "chartParams": {reqParam}, "isPager": {"true"}, "isFooterrow": {"false"}, "subTotal": {"false"}, //"colSubtotal": {"{\"CC787774162__tablecol_\":false,\"CC787774161__tablecol_\":false,\"CC787774160__tablecol_\":false,\"CC787774159__tablecol_\":false,\"CC787774158__tablecol_\":false,\"CC787774157__tablecol_\":false,\"CC787774156__tablecol_\":false,\"CC787774155__tablecol_\":false,\"CC787774154__tablecol_\":false,\"C1348804802__tablecol_\":false,\"C1348804803__tablecol_\":false,\"C1348804804__tablecol_\":false,\"C1348804805__tablecol_\":false}"}, "colToRow": {"false"}, "subTotalTop": {"false"}, "dataLimit": {"{\"enable\":false,\"max\":\"10000\"}"}, "_isSqlTrace": {"false"}, "_search": {"false"}, "nd": {"1723802570612"}, "rows": {"1000"}, "page": {"1"}, "sord": {"desc}"}, } // 创建请求 req, e := http.NewRequest("POST", urlStr, strings.NewReader(formData.Encode())) if e != nil { err = e log.Fatal(err) } // 设置 Content-Type 为 application/x-www-form-urlencoded req.Header.Add("Content-Type", "application/x-www-form-urlencoded") req.Header.Add("Cookie", cookie) // 发送请求 client := &http.Client{} resp, e := client.Do(req) if e != nil { err = e log.Fatal(err) } defer resp.Body.Close() // 读取响应 body, e := ioutil.ReadAll(resp.Body) if e != nil { err = e log.Fatal(err) } // 打印响应体 //fmt.Println(string(body)) handler, ok := HisugarTaskAnalysisHandlers[name] if !ok { utils.FileLog.Info(fmt.Sprintf("%s无解析函数\n", name)) err = fmt.Errorf("%s无解析函数", name) return } err = handler(name, reqParam, string(body)) if err != nil { utils.FileLog.Error(fmt.Sprintf("%s解析失败\n", name)) return } time.Sleep(5 * time.Second) } return } // 11，12是上一年的 func AnalysisHiSugar(name, reqParam, body string) (err error) { secName := "" columMap := make(map[string]string) var req ChartRequestSimple if e := json.Unmarshal([]byte(reqParam), &req); e != nil { err = e log.Println(err) return } else { secName = req.SceneName fmt.Println(secName) for _, column := range req.Columns { columMap[column.AliasName] = column.Title } } var resp HisugarResponseDataSimple if e := json.Unmarshal([]byte(body), &resp); e != nil { err = e log.Println(err) return } else { indexList := make([]*models.BaseFromHisugarIndex, 0) for _, row := range resp.Rows { monthValueMap := make(map[int]string) yearStrSlice := make([]string, 0) year := 0 dataTime := time.Time{} for k, v := range row { columTitle := columMap[k] yearSlice := strings.Split(v, "/") monthStr := "" month := 0 //value := "" if strings.Contains(v, "/") && (columTitle == "榨季" || columTitle == "榨季_万吨") { // 年份 for _, s := range yearSlice { yearStrSlice = append(yearStrSlice, "20"+s) } } else if strings.Contains(columTitle, "月") { monthStr = strings.TrimRight(columTitle, "月") month, _ = strconv.Atoi(monthStr) monthValueMap[month] = v } } for month, value := range monthValueMap { if value == "" || value == "" { continue } if month == 11 || month == 12 { year, _ = strconv.Atoi(yearStrSlice[0]) dataTime = time.Date(year, time.Month(month), 1, 0, 0, 0, 0, time.Local) dataTime = dataTime.AddDate(0, 1, -1) } else { year, _ = strconv.Atoi(yearStrSlice[1]) dataTime = time.Date(year, time.Month(month), 1, 0, 0, 0, 0, time.Local) dataTime = dataTime.AddDate(0, 1, -1) } item := models.BaseFromHisugarIndex{ DataTime: dataTime.Format(utils.FormatDate), Value: value, IndexName: secName, Unit: HisugarGuoChanTangUnitMap[name], CreateTime: time.Now(), ModifyTime: time.Now(), ClassifyId: HisugarGuoChanTangClassifyMap[name], Frequency: "月度", } fmt.Println(item.DataTime, value) indexList = append(indexList, &item) } } // 写入数据库 err = PostHandleHisugar(indexList) if err != nil { utils.FileLog.Error(fmt.Sprintf("PostHandleHisugar err:%v", err)) fmt.Println("PostHandleHisugar err", err) return } } return } // 10，11，12月是上一年的 // 全国食糖累计产销率，全国单月产糖量，全国单月销糖量 func AnalysisHiSugar3(name, reqParam, body string) (err error) { secName := "" columMap := make(map[string]string) var req ChartRequestSimple if e := json.Unmarshal([]byte(reqParam), &req); e != nil { err = e log.Println(err) return } else { secName = req.SceneName fmt.Println(secName) for _, column := range req.Columns { columMap[column.AliasName] = column.Title } } var resp HisugarResponseDataSimple if e := json.Unmarshal([]byte(body), &resp); e != nil { err = e log.Println(err) return } else { indexList := make([]*models.BaseFromHisugarIndex, 0) for _, row := range resp.Rows { monthValueMap := make(map[int]string) yearStrSlice := make([]string, 0) year := 0 dataTime := time.Time{} for k, v := range row { columTitle := columMap[k] yearSlice := strings.Split(v, "/") monthStr := "" month := 0 //value := "" if strings.Contains(v, "/") && (columTitle == "榨季" || columTitle == "榨季_万吨") { // 年份 for _, s := range yearSlice { yearStrSlice = append(yearStrSlice, "20"+s) } } else if strings.Contains(columTitle, "月") { monthStr = strings.TrimRight(columTitle, "月") month, _ = strconv.Atoi(monthStr) monthValueMap[month] = v } } for month, value := range monthValueMap { if value == "" || value == "" || value == "-" { continue } value = strings.ReplaceAll(value, "%", "") if month > 9 && month < 13 { year, _ = strconv.Atoi(yearStrSlice[0]) dataTime = time.Date(year, time.Month(month), 1, 0, 0, 0, 0, time.Local) dataTime = dataTime.AddDate(0, 1, -1) } else { year, _ = strconv.Atoi(yearStrSlice[1]) dataTime = time.Date(year, time.Month(month), 1, 0, 0, 0, 0, time.Local) dataTime = dataTime.AddDate(0, 1, -1) } item := models.BaseFromHisugarIndex{ IndexName: secName, ClassifyId: HisugarGuoChanTangClassifyMap[name], Unit: HisugarGuoChanTangUnitMap[name], Frequency: "月度", DataTime: dataTime.Format(utils.FormatDate), CreateTime: time.Now(), ModifyTime: time.Now(), Value: value, Describe: DescribeMap[name], } fmt.Println(item.DataTime, value) indexList = append(indexList, &item) } } // 写入数据库 err = PostHandleHisugar(indexList) if err != nil { utils.FileLog.Error(fmt.Sprintf("PostHandleHisugar err:%v", err)) fmt.Println("PostHandleHisugar err", err) return } } return } // 分产地 // 全国月度累计糖产量，全国月度累计糖销量 func AnalysisHiSugar4(name, reqParam, body string) (err error) { secName := "" columMap := make(map[string]string) dateColum := "" var req ChartRequestSimple if e := json.Unmarshal([]byte(reqParam), &req); e != nil { err = e log.Println(err) return } else { secName = req.SceneName fmt.Println(secName) for _, column := range req.Columns { if column.AliasTitle == "" { columMap[column.AliasName] = column.Title } else { columMap[column.AliasName] = column.AliasTitle } if columMap[column.AliasName] == "月份" { dateColum = column.AliasName } //fmt.Println(columMap[column.AliasName]) } } var resp HisugarResponseDataSimple if e := json.Unmarshal([]byte(body), &resp); e != nil { err = e log.Println(err) return } else { indexList := make([]*models.BaseFromHisugarIndex, 0) for _, row := range resp.Rows { //columTitleMap := make(map[string]string) dataTime := time.Time{} if date, ok := row[dateColum]; ok { dataTime, err = time.Parse("200601", date) if err != nil { fmt.Println(err) utils.FileLog.Error(fmt.Sprintf("时间解析错误:%v", err)) return } } for k, v := range row { columTitle := columMap[k] //value := "" if columTitle != "月份" && strings.Contains(columTitle, "量") { if v == "" || v == "" || v == "-" { continue } value := strings.ReplaceAll(v, "%", "") item := models.BaseFromHisugarIndex{ IndexName: "全国月度累计" + columTitle, ClassifyId: 5, Unit: HisugarGuoChanTangUnitMap[name], Frequency: "月度", Describe: DescribeMap[name], DataTime: dataTime.AddDate(0, 1, -1).Format(utils.FormatDate), CreateTime: time.Now(), ModifyTime: time.Now(), Value: value, IndexNameStr: "全国月度累计" + columTitle, MarketName: columTitle, } fmt.Println(item.DataTime, "全国月度累计"+columTitle, value) indexList = append(indexList, &item) } } } //写入数据库 err = PostHandleHisugar(indexList) if err != nil { utils.FileLog.Error(fmt.Sprintf("PostHandleHisugar err:%v", err)) fmt.Println("PostHandleHisugar err", err) return } } return } // 分产地 // 国内白糖产区现货价格5000条,国内制糖集团报价1000条,国内加工糖集团报价1000条，周度含水乙醇折糖价400条 func AnalysisHiSugar5(name, reqParam, body string) (err error) { secName := "" columMap := make(map[string]string) dateColum := "" var req ChartRequestSimple if e := json.Unmarshal([]byte(reqParam), &req); e != nil { err = e log.Println(err) return } else { secName = req.SceneName fmt.Println(secName) for _, column := range req.Columns { if column.AliasTitle == "" { columMap[column.AliasName] = column.Title } else { columMap[column.AliasName] = column.AliasTitle } if columMap[column.AliasName] == "日期" { dateColum = column.AliasName } //fmt.Println(columMap[column.AliasName]) } } var resp HisugarResponseDataSimple if e := json.Unmarshal([]byte(body), &resp); e != nil { err = e log.Println(e) return } else { indexList := make([]*models.BaseFromHisugarIndex, 0) for _, row := range resp.Rows { //columTitleMap := make(map[string]string) dataTime := time.Time{} classifyId := 5 frequency := "日度" if date, ok := row[dateColum]; ok { date = strings.ReplaceAll(date, " ", "") if name == "周度含水乙醇折糖价" { dataTime, err = time.Parse("2006/1/2", date) if err != nil { fmt.Println(err) utils.FileLog.Error(fmt.Sprintf("时间解析错误:%v", err)) continue } classifyId = 13 frequency = "周度" } else { dataTime, err = time.Parse("20060102", date) if err != nil { fmt.Println(err) utils.FileLog.Error(fmt.Sprintf("时间解析错误:%v", err)) continue } } } for k, v := range row { columTitle := columMap[k] //value := "" indexName := secName + columTitle if columTitle != "日期" { if v == "" || v == "" || v == "-" { continue } if strings.Contains(indexName, "日照凌云海") { continue } value := strings.ReplaceAll(v, "%", "") item := models.BaseFromHisugarIndex{ CreateTime: time.Now(), DataTime: dataTime.Format(utils.FormatDate), ModifyTime: time.Now(), Value: value, IndexName: indexName, Unit: HisugarGuoChanTangUnitMap[name], Frequency: frequency, ClassifyId: uint(classifyId), IndexNameStr: secName, MarketName: columTitle, Describe: DescribeMap[name], } if name == "周度含水乙醇折糖价" { item.IndexName = secName item.MarketName = "" } fmt.Println(item.DataTime, item.IndexName, value) indexList = append(indexList, &item) } } } // 写入数据库 err = PostHandleHisugar(indexList) if err != nil { utils.FileLog.Error(fmt.Sprintf("PostHandleHisugar err:%v", err)) fmt.Println("PostHandleHisugar err", err) return } } return } // 一年一行，1-12月 // 食糖月度进口量 func AnalysisHiSugar6(name, reqParam, body string) (err error) { secName := "" columMap := make(map[string]string) var req ChartRequestSimple if e := json.Unmarshal([]byte(reqParam), &req); e != nil { err = e log.Println(err) return } else { secName = req.SceneName fmt.Println(secName) for _, column := range req.Columns { columMap[column.AliasName] = column.Title } } var resp HisugarResponseDataSimple if e := json.Unmarshal([]byte(body), &resp); e != nil { err = e log.Println(err) return } else { indexList := make([]*models.BaseFromHisugarIndex, 0) for _, row := range resp.Rows { monthValueMap := make(map[int]string) year := 0 dataTime := time.Time{} for k, v := range row { columTitle := columMap[k] monthStr := "" month := 0 //value := "" if strings.Contains(columTitle, "年份") { // 年份 year, _ = strconv.Atoi(v) } else if strings.Contains(columTitle, "月") { monthStr = strings.TrimRight(columTitle, "月") month, _ = strconv.Atoi(monthStr) monthValueMap[month] = v } } for month, value := range monthValueMap { if value == "" || value == "" { continue } value = strings.ReplaceAll(value, "%", "") dataTime = time.Date(year, time.Month(month), 1, 0, 0, 0, 0, time.Local) dataTime = dataTime.AddDate(0, 1, -1) item := models.BaseFromHisugarIndex{ IndexName: secName, ClassifyId: 12, Unit: HisugarGuoChanTangUnitMap[name], Frequency: "月度", DataTime: dataTime.Format(utils.FormatDate), CreateTime: time.Now(), ModifyTime: time.Now(), Value: value, Describe: DescribeMap[name], } if name == "巴西食糖月度出口量" { item.ClassifyId = 13 } fmt.Println(item.DataTime, secName, value) indexList = append(indexList, &item) } } // 写入数据库 //err = PostHandleHisugar(indexList) //if err != nil { // utils.FileLog.Error(fmt.Sprintf("PostHandleHisugar err:%v", err)) // fmt.Println("PostHandleHisugar err", err) // return //} } return } // 第一列月日，后面每一列是年份 // 巴西港口等待船只数量 func AnalysisHiSugar7(name, reqParam, body string) (err error) { secName := "" columMap := make(map[string]string) var req ChartRequestSimple if e := json.Unmarshal([]byte(reqParam), &req); e != nil { err = e log.Println(err) return } else { secName = req.SceneName fmt.Println(secName) for _, column := range req.Columns { columMap[column.AliasName] = column.Title } } var resp HisugarResponseDataSimple if e := json.Unmarshal([]byte(body), &resp); e != nil { err = e log.Println(err) return } else { indexList := make([]*models.BaseFromHisugarIndex, 0) for _, row := range resp.Rows { yeareMap := make(map[int]string) year := 0 monthDay := time.Time{} for k, v := range row { columTitle := columMap[k] yearStr := "" //month := 0 //day := 0 //value := "" if strings.Contains(columTitle, "日期") { // 日期 //year, _ = strconv.Atoi(v) v = strings.ReplaceAll(v, " ", "") monthDay, err = utils.ParseTimeShort(v) if err != nil { fmt.Println(err) utils.FileLog.Error(fmt.Sprintf("时间解析错误:%v", err)) return } //month = int(dataTime.Month()) //day = dataTime.Day() } else if strings.Contains(columTitle, "年") { yearStr = strings.TrimRight(columTitle, "年") year, _ = strconv.Atoi(yearStr) yeareMap[year] = v } } for year, value := range yeareMap { if value == "" || value == "" { continue } value = strings.ReplaceAll(value, "%", "") dataTime := time.Date(year, monthDay.Month(), monthDay.Day(), 0, 0, 0, 0, time.Local) //dataTime = dataTime.AddDate(0, 1, -1) item := models.BaseFromHisugarIndex{ IndexName: secName, ClassifyId: 13, Unit: HisugarGuoChanTangUnitMap[name], Frequency: "周度", DataTime: dataTime.Format(utils.FormatDate), CreateTime: time.Now(), ModifyTime: time.Now(), Value: value, Describe: DescribeMap[name], } fmt.Println(item.DataTime, secName, value) indexList = append(indexList, &item) } } // 写入数据库 err = PostHandleHisugar(indexList) if err != nil { utils.FileLog.Error(fmt.Sprintf("PostHandleHisugar err:%v", err)) fmt.Println("PostHandleHisugar err", err) return } } return } // 10，11，12月是上一年的,年份格式2023/24 // 印度食糖月度出口量 func AnalysisHiSugar8(name, reqParam, body string) (err error) { secName := "" columMap := make(map[string]string) var req ChartRequestSimple if e := json.Unmarshal([]byte(reqParam), &req); e != nil { err = e log.Println(err) return } else { secName = req.SceneName fmt.Println(secName) for _, column := range req.Columns { columMap[column.AliasName] = column.Title } } var resp HisugarResponseDataSimple if e := json.Unmarshal([]byte(body), &resp); e != nil { err = e log.Println(err) return } else { indexList := make([]*models.BaseFromHisugarIndex, 0) for _, row := range resp.Rows { monthValueMap := make(map[int]string) yearStrSlice := make([]string, 0) year := 0 dataTime := time.Time{} for k, v := range row { columTitle := columMap[k] yearSlice := strings.Split(v, "/") monthStr := "" month := 0 //value := "" if strings.Contains(v, "/") && (columTitle == "榨季" || columTitle == "榨季_万吨") { // 年份 for i, s := range yearSlice { if i == 1 { yearStrSlice = append(yearStrSlice, "20"+s) } else { yearStrSlice = append(yearStrSlice, s) } } } else if strings.Contains(columTitle, "月") { monthStr = strings.TrimRight(columTitle, "月") month, _ = strconv.Atoi(monthStr) monthValueMap[month] = v } } for month, value := range monthValueMap { if value == "" || value == "" { continue } value = strings.ReplaceAll(value, "%", "") if month > 9 && month < 13 { year, _ = strconv.Atoi(yearStrSlice[0]) dataTime = time.Date(year, time.Month(month), 1, 0, 0, 0, 0, time.Local) dataTime = dataTime.AddDate(0, 1, -1) } else { year, _ = strconv.Atoi(yearStrSlice[1]) dataTime = time.Date(year, time.Month(month), 1, 0, 0, 0, 0, time.Local) dataTime = dataTime.AddDate(0, 1, -1) } item := models.BaseFromHisugarIndex{ IndexName: secName, ClassifyId: 14, Unit: HisugarGuoChanTangUnitMap[name], Frequency: "月度", DataTime: dataTime.Format(utils.FormatDate), CreateTime: time.Now(), ModifyTime: time.Now(), Value: value, Describe: DescribeMap[name], } fmt.Println(item.DataTime, secName, value) indexList = append(indexList, &item) } } // 写入数据库 err = PostHandleHisugar(indexList) if err != nil { utils.FileLog.Error(fmt.Sprintf("PostHandleHisugar err:%v", err)) fmt.Println("PostHandleHisugar err", err) return } } return } // 10，11，12月是上一年的,年份格式23/24，旬度 // 泰国双周糖累计产量 func AnalysisHiSugar9(name, reqParam, body string) (err error) { secName := "" columMap := make(map[string]string) var req ChartRequestSimple if e := json.Unmarshal([]byte(reqParam), &req); e != nil { err = e log.Println(err) return } else { secName = req.SceneName fmt.Println(secName) for _, column := range req.Columns { columMap[column.AliasName] = column.Title } } var resp HisugarResponseDataSimple if e := json.Unmarshal([]byte(body), &resp); e != nil { err = e log.Println(err) return } else { indexList := make([]*models.BaseFromHisugarIndex, 0) for _, row := range resp.Rows { monthValueMap := make(map[int]string) dayValueMap := make(map[int]int) yearStrSlice := make([]string, 0) year := 0 dataTime := time.Time{} for k, v := range row { columTitle := columMap[k] yearSlice := strings.Split(v, "/") monthStr := "" month := 0 //value := "" if strings.Contains(v, "/") && (columTitle == "榨季" || columTitle == "榨季_吨") { // 年份 for _, s := range yearSlice { yearStrSlice = append(yearStrSlice, "20"+s) } } else if strings.Contains(columTitle, "月") { if strings.Contains(columTitle, "月上半月") { monthStr = strings.TrimRight(columTitle, "月上半月") month, _ = strconv.Atoi(monthStr) dayValueMap[month] = 15 } else if strings.Contains(columTitle, "月下半月") { monthStr = strings.TrimRight(columTitle, "月下半月") month, _ = strconv.Atoi(monthStr) dayValueMap[month] = 25 } monthValueMap[month] = v } } for month, value := range monthValueMap { if value == "" || value == "" { continue } value = strings.ReplaceAll(value, "%", "") if month > 11 { year, _ = strconv.Atoi(yearStrSlice[0]) } else { year, _ = strconv.Atoi(yearStrSlice[1]) } if dayValueMap[month] > 20 { dataTime = time.Date(year, time.Month(month), 1, 0, 0, 0, 0, time.Local) dataTime = dataTime.AddDate(0, 1, -1) } else { dataTime = time.Date(year, time.Month(month), 15, 0, 0, 0, 0, time.Local) } item := models.BaseFromHisugarIndex{ IndexName: secName, ClassifyId: 15, Unit: HisugarGuoChanTangUnitMap[name], Frequency: "月度", DataTime: dataTime.Format(utils.FormatDate), CreateTime: time.Now(), ModifyTime: time.Now(), Value: value, Describe: DescribeMap[name], } fmt.Println(item.DataTime, secName, value) indexList = append(indexList, &item) } } // 写入数据库 err = PostHandleHisugar(indexList) if err != nil { utils.FileLog.Error(fmt.Sprintf("PostHandleHisugar err:%v", err)) fmt.Println("PostHandleHisugar err", err) return } } return } // 11，12是上一年的，年份23/24,横轴日期11月15，11月30 // 印度双周累计产糖量 func AnalysisHiSugar10(name, reqParam, body string) (err error) { secName := "" columMap := make(map[string]string) var req ChartRequestSimple if e := json.Unmarshal([]byte(reqParam), &req); e != nil { err = e log.Println(err) return } else { secName = req.SceneName fmt.Println(secName) for _, column := range req.Columns { columMap[column.AliasName] = column.Title } } var resp HisugarResponseDataSimple if e := json.Unmarshal([]byte(body), &resp); e != nil { err = e log.Println(err) return } else { indexList := make([]*models.BaseFromHisugarIndex, 0) for _, row := range resp.Rows { //monthValueMap := make(map[int]string) //dayValueMap := make(map[int]int) dateValueMap := make(map[time.Time]string) yearStrSlice := make([]string, 0) year := 0 dataTime := time.Time{} for k, v := range row { columTitle := columMap[k] yearSlice := strings.Split(v, "/") //monthStr := "" month := 0 day := 0 //value := "" if strings.Contains(v, "/") && (columTitle == "榨季" || columTitle == "榨季_万吨") { // 年份 for _, s := range yearSlice { yearStrSlice = append(yearStrSlice, "20"+s) } } else if strings.Contains(columTitle, "月") { dateSlice := strings.Split(columTitle, "月") if len(dateSlice) == 2 { month, _ = strconv.Atoi(dateSlice[0]) day, _ = strconv.Atoi(dateSlice[1]) } // 这个时候不一定有年份，所以先搞个假的年份，后面赋值真年份 dataTime = time.Date(2000, time.Month(month), day, 0, 0, 0, 0, time.Local) dateValueMap[dataTime] = v } } for date, value := range dateValueMap { if value == "" || value == "" || value == "-" { continue } if date.Month() == 11 || date.Month() == 12 { year, _ = strconv.Atoi(yearStrSlice[0]) //dataTime = time.Date(year, time.Month(month), 1, 0, 0, 0, 0, time.Local) //dataTime = dataTime.AddDate(0, 1, -1) } else { year, _ = strconv.Atoi(yearStrSlice[1]) //dataTime = time.Date(year, time.Month(month), 1, 0, 0, 0, 0, time.Local) //dataTime = dataTime.AddDate(0, 1, -1) } date = time.Date(year, date.Month(), date.Day(), 0, 0, 0, 0, time.Local) item := models.BaseFromHisugarIndex{ DataTime: date.Format(utils.FormatDate), Value: value, IndexName: secName, Unit: HisugarGuoChanTangUnitMap[name], CreateTime: time.Now(), ModifyTime: time.Now(), ClassifyId: HisugarGuoChanTangClassifyMap[name], Frequency: "旬度", Describe: DescribeMap[name], } fmt.Println(item.DataTime, value) indexList = append(indexList, &item) } } // 写入数据库 err = PostHandleHisugar(indexList) if err != nil { utils.FileLog.Error(fmt.Sprintf("PostHandleHisugar err:%v", err)) fmt.Println("PostHandleHisugar err", err) return } } return } // 4月16到12月是上一年的,年份格式23/24，旬度 // 巴西双周累计产糖量 func AnalysisHiSugar11(name, reqParam, body string) (err error) { secName := "" columMap := make(map[string]string) var req ChartRequestSimple if e := json.Unmarshal([]byte(reqParam), &req); e != nil { err = e log.Println(err) return } else { secName = req.SceneName fmt.Println(secName) for _, column := range req.Columns { columMap[column.AliasName] = column.Title } } var resp HisugarResponseDataSimple if e := json.Unmarshal([]byte(body), &resp); e != nil { err = e log.Println(err) return } else { indexList := make([]*models.BaseFromHisugarIndex, 0) for _, row := range resp.Rows { monthValueMap := make(map[int]string) dayValueMap := make(map[int]int) yearStrSlice := make([]string, 0) year := 0 dataTime := time.Time{} for k, v := range row { columTitle := columMap[k] yearSlice := strings.Split(v, "/") monthStr := "" month := 0 //value := "" if strings.Contains(v, "/") && (columTitle == "榨季" || columTitle == "榨季_吨") { // 年份 for _, s := range yearSlice { yearStrSlice = append(yearStrSlice, "20"+s) } } else if strings.Contains(columTitle, "月") { if strings.Contains(columTitle, "月16日") { monthStr = strings.TrimRight(columTitle, "月16日") month, _ = strconv.Atoi(monthStr) dayValueMap[month] = 16 } else if strings.Contains(columTitle, "月1日") { monthStr = strings.TrimRight(columTitle, "月1日") month, _ = strconv.Atoi(monthStr) dayValueMap[month] = 1 } monthValueMap[month] = v } } for month, value := range monthValueMap { if value == "" || value == "" { continue } value = strings.ReplaceAll(value, "%", "") if month > 4 && dayValueMap[month] > 15 { year, _ = strconv.Atoi(yearStrSlice[0]) } else { year, _ = strconv.Atoi(yearStrSlice[1]) } dataTime = time.Date(year, time.Month(month), dayValueMap[month], 0, 0, 0, 0, time.Local) item := models.BaseFromHisugarIndex{ IndexName: secName, ClassifyId: 13, Unit: HisugarGuoChanTangUnitMap[name], Frequency: "旬度", DataTime: dataTime.Format(utils.FormatDate), CreateTime: time.Now(), ModifyTime: time.Now(), Value: value, Describe: DescribeMap[name], } fmt.Println(item.DataTime, secName, value) indexList = append(indexList, &item) } } //写入数据库 err = PostHandleHisugar(indexList) if err != nil { utils.FileLog.Error(fmt.Sprintf("PostHandleHisugar err:%v", err)) fmt.Println("PostHandleHisugar err", err) return } } return } // 4月16到12月是上一年的,年份格式23/24，旬度 // 巴西双周食糖库存,中南部双周累计产糖量,中南部双周糖库存,中南部双周乙醇产量 func AnalysisHiSugar12(name, reqParam, body string) (err error) { secName := "" columMap := make(map[string]string) var req ChartRequestSimple if e := json.Unmarshal([]byte(reqParam), &req); e != nil { err = e log.Println(err) return } else { secName = req.SceneName fmt.Println(secName) for _, column := range req.Columns { columMap[column.AliasName] = column.Title } } var resp HisugarResponseDataSimple if e := json.Unmarshal([]byte(body), &resp); e != nil { err = e log.Println(err) return } else { indexList := make([]*models.BaseFromHisugarIndex, 0) for _, row := range resp.Rows { monthValueMap := make(map[int]string) dayValueMap := make(map[int]int) yearStrSlice := make([]string, 0) year := 0 dataTime := time.Time{} for k, v := range row { columTitle := columMap[k] yearSlice := strings.Split(v, "/") //monthStr := "" month := 0 //value := "" if strings.Contains(v, "/") && (columTitle == "榨季" || columTitle == "榨季_万吨" || columTitle == "榨季_立方米" || columTitle == "榨季_吨" || columTitle == "榨季_百分比") { // 年份 for _, s := range yearSlice { yearStrSlice = append(yearStrSlice, "20"+s) } } else if strings.Contains(columTitle, "月") { date, _ := time.Parse("1月2日", columTitle) month = int(date.Month()) dayValueMap[month] = date.Day() monthValueMap[month] = v //if strings.Contains(columTitle, "月16日") { // monthStr = strings.TrimRight(columTitle, "月16日") // month, _ = strconv.Atoi(monthStr) // dayValueMap[month] = 16 //} else if strings.Contains(columTitle, "月1日") { // monthStr = strings.TrimRight(columTitle, "月1日") // month, _ = strconv.Atoi(monthStr) // dayValueMap[month] = 1 //} //monthValueMap[month] = v } } for month, value := range monthValueMap { if value == "" || value == "" { continue } value = strings.ReplaceAll(value, "%", "") if month > 3 { year, _ = strconv.Atoi(yearStrSlice[0]) } else { year, _ = strconv.Atoi(yearStrSlice[1]) } dataTime = time.Date(year, time.Month(month), dayValueMap[month], 0, 0, 0, 0, time.Local) item := models.BaseFromHisugarIndex{ IndexName: secName, ClassifyId: 13, Unit: HisugarGuoChanTangUnitMap[name], Frequency: "旬度", DataTime: dataTime.Format(utils.FormatDate), CreateTime: time.Now(), ModifyTime: time.Now(), Value: value, Describe: DescribeMap[name], } fmt.Println(item.DataTime, secName, value) indexList = append(indexList, &item) } } //写入数据库 err = PostHandleHisugar(indexList) if err != nil { utils.FileLog.Error(fmt.Sprintf("PostHandleHisugar err:%v", err)) fmt.Println("PostHandleHisugar err", err) return } } return } // 4-12月是上一年的 // 巴西中南部乙醇月度销售情况，巴西中南部乙醇国内销售情况 func AnalysisHiSugar13(name, reqParam, body string) (err error) { secName := "" columMap := make(map[string]string) var req ChartRequestSimple if e := json.Unmarshal([]byte(reqParam), &req); e != nil { err = e log.Println(err) return } else { secName = req.SceneName fmt.Println(secName) for _, column := range req.Columns { columMap[column.AliasName] = column.Title } } var resp HisugarResponseDataSimple if e := json.Unmarshal([]byte(body), &resp); e != nil { err = e log.Println(err) return } else { indexList := make([]*models.BaseFromHisugarIndex, 0) for _, row := range resp.Rows { monthValueMap := make(map[int]string) yearStrSlice := make([]string, 0) year := 0 dataTime := time.Time{} for k, v := range row { columTitle := columMap[k] yearSlice := strings.Split(v, "/") monthStr := "" month := 0 //value := "" if strings.Contains(v, "/") && (columTitle == "榨季" || columTitle == "榨季_万吨") { // 年份 for _, s := range yearSlice { yearStrSlice = append(yearStrSlice, "20"+s) } } else if strings.Contains(columTitle, "月") { monthStr = strings.TrimRight(columTitle, "月") month, _ = strconv.Atoi(monthStr) monthValueMap[month] = v } } for month, value := range monthValueMap { if value == "" || value == "" || value == "-" { continue } value = strings.ReplaceAll(value, "%", "") if month > 3 && month < 13 { year, _ = strconv.Atoi(yearStrSlice[0]) dataTime = time.Date(year, time.Month(month), 1, 0, 0, 0, 0, time.Local) dataTime = dataTime.AddDate(0, 1, -1) } else { year, _ = strconv.Atoi(yearStrSlice[1]) dataTime = time.Date(year, time.Month(month), 1, 0, 0, 0, 0, time.Local) dataTime = dataTime.AddDate(0, 1, -1) } item := models.BaseFromHisugarIndex{ IndexName: secName, ClassifyId: HisugarGuoChanTangClassifyMap[name], Unit: HisugarGuoChanTangUnitMap[name], Frequency: "月度", DataTime: dataTime.Format(utils.FormatDate), CreateTime: time.Now(), ModifyTime: time.Now(), Value: value, Describe: DescribeMap[name], } fmt.Println(item.DataTime, value) indexList = append(indexList, &item) } } // 写入数据库 err = PostHandleHisugar(indexList) if err != nil { utils.FileLog.Error(fmt.Sprintf("PostHandleHisugar err:%v", err)) fmt.Println("PostHandleHisugar err", err) return } } return }