解析美国农业部

services/usda_psd.go

@@ -8,10 +8,13 @@ import (
 	"fmt"
 	"github.com/PuerkitoBio/goquery"
 	"io"
+	"io/ioutil"
 	"mime/multipart"
 	"net/http"
 	"os"
+	"strconv"
 	"strings"
+	"time"
 )
 
 // UsdaPsdDataQueryParams 定义了JSON到Go结构体的映射
@@ -50,6 +53,20 @@ type UsdaPsdDataAttribute struct {
 	AttributeId int `json:"attributeId"`
 }
 
+// UsdaFasIndex 美国农业部指标数据
+type UsdaFasIndex struct {
+	ClassifyName       string `description:"指标目录"`
+	ParentClassifyName string `description:"父级指标目录"`
+	ClassifySort       int    `description:"指标目录排序号"`
+	IndexName          string `description:"指标名称"`
+	IndexCode          string `description:"指标编码"`
+	Unit               string `description:"单位"`
+	Sort               int    `description:"排序号"`
+	Frequency          string `description:"频度"`
+	TerminalCode       string `description:"编码"`
+	ExcelDataMap       map[string]string
+}
+
 // Meal, Palm Kernel:0813800
 // Meal, Peanut:0813200
 // Meal, Rapeseed:0813600
@@ -73,9 +90,22 @@ type UsdaPsdDataAttribute struct {
 // Oilseed, Soybean:2222000
 // Oilseed, Sunflowerseed:2224000
 // 美国农业部月度供需平衡表数据
-func DownloadUsdaPsdData() {
-	var err error
-	defer func() {
+func DownloadUsdaPsdData() (indexList []*UsdaFasIndex, err error) {
+	// 从test.json文件中读取json串
+	body, err := ioutil.ReadFile("test.json")
+	if err != nil {
+		return
+	}
+	// 解析json串
+	item := new(UsdaPsdData)
+	err = json.Unmarshal(body, &item)
+	if err != nil {
+		fmt.Println("json.Unmarshal err:" + err.Error())
+		return
+	}
+	indexList, err = handleUsdaFasPsd(item)
+	return
+	/*defer func() {
 		if err != nil {
 			msg := "失败提醒" + "downloadUsdaPsdData ErrMsg:" + err.Error()
 			fmt.Println("msg:", msg)
@@ -148,16 +178,7 @@ func DownloadUsdaPsdData() {
 		fmt.Println("json.Unmarshal err:" + err.Error())
 		return
 	}
-	// 解析
-	for k, v := range item.TableHeaders {
-		// 键值对的值
-		fmt.Println("key:", k, "value:", v)
-	}
-	// 解析
-	for k, v := range item.QueryResult {
-		// 键值对的值
-		fmt.Println("key:", k, "value:", v)
-	}
+	indexList, err = handleUsdaFasPsd(item)*/
 	return
 }
 
@@ -302,3 +323,159 @@ func DownloadUsdaFmsData() {
 	fmt.Println("Excel file downloaded successfully")
 	return
 }
+
+func handleUsdaFasPsd(item *UsdaPsdData) (indexList []*UsdaFasIndex, err error) {
+	// 解析
+	headerSlice := make([]string, 0)
+	for index, v := range item.TableHeaders {
+		// 键值对的值
+		fmt.Println("key:", index, "value:", v)
+		if !strings.Contains(v, "/") && !strings.Contains(v, " ") {
+			v = strings.ToLower(v)
+		}
+		if v == "Unit Description" {
+			v = "unit Description"
+		}
+		headerSlice = append(headerSlice, v)
+	}
+	// 解析
+	// 遍历行读取
+	indexList = make([]*UsdaFasIndex, 0)
+	sort := 0
+	// 指标名称
+	indexMap := make(map[string]*UsdaFasIndex)
+	// 键值对的值
+	commodityRow := ""
+	countriesRow := ""
+	attributesRow := ""
+	errMsg := ""
+	for _, row := range item.QueryResult {
+		unitK := headerSlice[len(headerSlice)-1]
+		unit := row[unitK].(string)
+		// unit 去掉左右两边的括号，去掉中间的空格
+		unit = strings.Replace(unit, " ", "", -1)
+		unit = strings.Trim(unit, "()")
+		for _, k := range headerSlice {
+			col, ok := row[k]
+			if !ok || col == nil {
+				continue
+			}
+			if k == "commodity" {
+				commodityRow = col.(string)
+			} else if k == "country" {
+				countriesRow = col.(string)
+			} else if k == "attribute" {
+				attributesRow = col.(string)
+			} else if k == "unit Description" {
+				// unit = col.(string)
+			} else {
+				//数据列
+				year, _ := strconv.Atoi(strings.Split(k, "/")[0])
+				month := 0
+				indexName := ""
+				classifyName := ""
+				classifySort := 0
+				inCode := ""
+				fre := "年度"
+				lastStr := "Yearly"
+				// year年度的最后一天日期
+				dateT := time.Date(year, time.December, 31, 0, 0, 0, 0, time.UTC)
+				if strings.Contains(k, "(") {
+					fre = "月度"
+					lastStr = "Monthly"
+					// 截取括号中间的月度数据
+					monthStr := strings.Split(k, "(")[1]
+					monthStr = strings.Split(monthStr, ")")[0]
+					// 将Jul英文月份前缀转成数字月份
+					monthT, e := time.ParseInLocation("Jan", monthStr, time.Local)
+					if e != nil {
+						errMsg += fmt.Sprintf("月份转换错误:%s%s\n", monthStr, e.Error())
+						continue
+					}
+					month = int(monthT.Month())
+					// 将year和month拼接成日期，该月的最后一天日期
+					dateT = time.Date(year, time.Month(month), 31, 0, 0, 0, 0, time.UTC)
+				}
+				date := dateT.Format("2006-01-02")
+				// 封装成指标数据
+				if commodityRow != "" && countriesRow != "" && attributesRow != "" {
+					indexName = commodityRow + ": " + countriesRow + ": " + attributesRow + ": " + lastStr
+				} else {
+					fmt.Println("commodityRow:", commodityRow, "countriesRow:", countriesRow, "attributesRow:", attributesRow)
+					errMsg += fmt.Sprintf("指标名称为空 commodityRow:%s,countriesRow:%s,attributesRow:%s\n", commodityRow, countriesRow, attributesRow)
+					continue
+				}
+
+				inCode = "usda" + utils.GetFirstLetter(indexName)
+				indexItem, okIndex := indexMap[indexName]
+				// 首字母大写
+				classifyName = commodityRow
+				if !okIndex {
+					// 新增指标
+					indexItem = new(UsdaFasIndex)
+					indexItem.IndexName = indexName
+					indexItem.ClassifyName = classifyName
+					indexItem.ParentClassifyName = "月度供需"
+					indexItem.ClassifySort = classifySort
+					indexItem.IndexCode = inCode
+					indexItem.Frequency = fre
+					indexItem.Sort = sort
+					indexItem.Unit = unit
+					indexItem.ExcelDataMap = make(map[string]string)
+					sort++
+				}
+				val := col.(float64)
+				indexItem.ExcelDataMap[date] = fmt.Sprintf("%.4f", val)
+				indexMap[indexName] = indexItem
+				continue
+			}
+		}
+
+	}
+
+	for _, v := range indexMap {
+		fmt.Printf("IndexName: %s \n", v.IndexName)
+		fmt.Printf("IndexCode: %s \n", v.IndexCode)
+		indexList = append(indexList, v)
+		if len(indexList) > 500 {
+			err = addUsdaFasPsdData(indexList)
+			if err != nil {
+				return
+			}
+			indexList = []*UsdaFasIndex{}
+		}
+	}
+	err = addUsdaFasPsdData(indexList)
+	if err != nil {
+		return
+	}
+	return
+}
+
+func addUsdaFasPsdData(indexList []*UsdaFasIndex) (err error) {
+	sheetName := "月度供需"
+	if len(indexList) > 0 {
+		params := make(map[string]interface{})
+		params["List"] = indexList
+		params["TerminalCode"] = ""
+		result, e := utils.PostEdbLib(params, "usda_fas/handle/excel_data")
+		if e != nil {
+			err = fmt.Errorf("sheet ：%s PostEdbLib err: %s", sheetName, e.Error())
+			b, _ := json.Marshal(params)
+			utils.FileLog.Info(fmt.Sprintf("sheet ：%s PostEdbLib err: %s, params: %s", sheetName, e.Error(), string(b)))
+			return
+		}
+		resp := new(utils.BaseEdbLibResponse)
+		if e := json.Unmarshal(result, &resp); e != nil {
+			err = fmt.Errorf("sheet ：%s json.Unmarshal err: %s", sheetName, e)
+			utils.FileLog.Info(fmt.Sprintf("sheet ：%s json.Unmarshal err: %s", sheetName, e))
+			return
+		}
+		if resp.Ret != 200 {
+			err = fmt.Errorf("sheet ：%s Msg: %s, ErrMsg: %s", sheetName, resp.Msg, resp.ErrMsg)
+			utils.FileLog.Info(fmt.Sprintf("sheet ：%s Msg: %s, ErrMsg: %s", sheetName, resp.Msg, resp.ErrMsg))
+			return
+		}
+	}
+	return
+}

utils/common.go

@@ -1128,6 +1128,21 @@ func ChineseToPinyinInitials(input string) string {
 	return result.String()
 }
 
+// GetFirstLetter 英文单词首字母
+func GetFirstLetter(name string) string {
+	// 定义一个正则表达式，它匹配由字母组成的单词，并捕获每个单词的首字母
+	// 注意：这个正则表达式假设单词由字母组成，并且被非字母字符分隔
+	// 你可能需要根据实际情况调整这个正则表达式
+	re := regexp.MustCompile(`\b[a-zA-Z]`)
+
+	// FindAllString 方法会返回所有匹配的字符串（在这个例子中，是单词的首字母）
+	matches := re.FindAllString(name, -1)
+
+	// 将匹配的首字母连接成一个字符串
+	newStr := strings.Join(matches, "")
+	return strings.ToLower(newStr)
+}
+
 // 修改供应商信息
 func HttpPostNoCookie(url string, reqParam string, headersParams map[string]string) ([]byte, error) {
 
@@ -1182,3 +1197,54 @@ func HttpGetNoCookie(url string) ([]byte, error) {
 	defer res.Body.Close()
 	return ioutil.ReadAll(res.Body)
 }
+
+// PostEdbLib 调用指标接口
+func PostEdbLib(param map[string]interface{}, method string) (result []byte, err error) {
+	// todo postUrl := EDB_LIB_URL + method
+	postUrl := "http://127.0.0.1:8900/edbapi/" + method
+	postData, err := json.Marshal(param)
+	if err != nil {
+		return
+	}
+	result, err = httpPostEtaLib(postUrl, string(postData), "application/json")
+	if err != nil {
+		return
+	}
+	return
+}
+
+type BaseEdbLibResponse struct {
+	Ret     int
+	Msg     string
+	ErrMsg  string
+	ErrCode string
+	Data    interface{}
+}
+
+func httpPostEtaLib(url, postData string, params ...string) ([]byte, error) {
+	fmt.Println("HttpPost Url:" + url)
+	body := ioutil.NopCloser(strings.NewReader(postData))
+	client := &http.Client{}
+	req, err := http.NewRequest("POST", url, body)
+	if err != nil {
+		return nil, err
+	}
+	contentType := "application/x-www-form-urlencoded;charset=utf-8"
+	if len(params) > 0 && params[0] != "" {
+		contentType = params[0]
+	}
+	req.Header.Set("Content-Type", contentType)
+	// todo req.Header.Set("authorization", MD5(APP_EDB_LIB_NAME_EN+"EDB_LIB_Md5_KEY"))
+	req.Header.Set("authorization", MD5("eta_index_lib"+"2342342341asd"))
+	resp, err := client.Do(req)
+	if err != nil {
+		fmt.Println("client.Do err:" + err.Error())
+		return nil, err
+	}
+	defer resp.Body.Close()
+	b, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		fmt.Println("HttpPost:" + string(b))
+	}
+	return b, err
+}