eta_task/services/usda_fas.go

package services

import (
	"encoding/json"
	"errors"
	"eta/eta_task/models/data_manage"
	"eta/eta_task/utils"
	"fmt"
	"time"
)

// UsdaFas美国农业部指标数据同步
func SyncBaseFromUsdaFas() (err error) {
	var startDate string
	maxDate, err := data_manage.GetBaseFromUsdaFasIndexMaxDate()
	if err != nil || maxDate.IsZero() {
		startDate = time.Now().AddDate(0, 0, -1).Format(utils.FormatDate) + " 00:00:00"
	} else {
		startDate = maxDate.Format(utils.FormatDateTime)
	}

	method := `index/list`
	data := make(map[string]interface{})
	data["Source"] = utils.DATA_SOURCE_USDA_FAS
	data["StartDate"] = startDate
	//data["EndDate"] = endDate
	result, err := HttpPost("SyncRankingFromUsdaFas", method, data)
	utils.FileLog.Info(result)
	fmt.Println(result)

	respObj := new(data_manage.UsdaFasIndexResp)
	err = json.Unmarshal([]byte(result), &respObj)
	if err != nil {
		utils.FileLog.Info("err:", err.Error())
		fmt.Println("err:", err.Error())
		return err
	}

	UsdaFasObj := new(data_manage.BaseFromUsdaFasIndex)

	for _, zv := range respObj.Data {
		newID, err := UsdaFasObj.InsertOrUpdateBaseFromUsdaFasIndex(zv)
		if err != nil {
			fmt.Println("InsertOrUpdateBaseFromUsdaFasIndex error:", err)
		}
		fmt.Println("InsertOrUpdateBaseFromUsdaFasIndex new indexID:", newID)
	}
	return err
}

// UsdaFas美国农业部-分类
func SyncBaseFromUsdaFasClassify() (err error) {
	data := make(map[string]interface{})
	data["Source"] = utils.DATA_SOURCE_USDA_FAS
	method := `classify/list`
	result, err := HttpPost("SyncBaseFromUsdaFasClassify", method, data)
	if err != nil {
		fmt.Println("SyncBaseFromUsdaFasClassify HttpPost Err:", err.Error())
	}
	utils.FileLog.Info(result)
	fmt.Println("SyncBaseFromUsdaFasClassify result:", result)

	respObj := new(data_manage.UsdaFasClassifyResp)
	err = json.Unmarshal([]byte(result), &respObj)
	if err != nil {
		return err
	}
	//获取所有分类
	/*	allClassify, err := data_manage.GetBaseFromUsdaFasClassifyAll()
		if err != nil {
			return
		}

		existIndexMap := make(map[int]*data_manage.BaseFromUsdaFasClassify)
		for _, v := range allClassify {
			existIndexMap[v.BaseFromUsdaFasClassifyId] = v
		}*/

	for _, item := range respObj.Data {
		//if _, ok := existIndexMap[item.BaseFromUsdaFasClassifyId]; !ok {
		newID, err := data_manage.InsertOrUpdateBaseFromUsdaFasClassify(item)
		if err != nil {
			fmt.Println("InsertOrUpdateBaseFromUsdaFasClassify error:", err)
		}
		fmt.Println("InsertOrUpdateBaseFromUsdaFasClassify new indexID:", newID)
		//}
	}
	return err
}

// UsdaFas美国农业部-数据
func SyncBaseFromUsdaFasData() (err error) {
	startDate := time.Now().Format(utils.FormatDate) + " 00:00:00"
	//var startDate string
	maxDate, err := data_manage.GetBaseFromUsdaFasMaxDate()
	if err != nil || maxDate.IsZero() {
		startDate = time.Now().AddDate(0, 0, -1).Format(utils.FormatDate) + " 00:00:00"
	} else {
		startDate = maxDate.Format(utils.FormatDateTime)
	}

	method := `index/data/list_page`

	existDataMap := make(map[string]*data_manage.BaseFromUsdaFasData)
	allData, err1 := data_manage.GetAllBaseFromUsdaFasDataList(startDate)
	if err1 != nil {
		fmt.Println("get GetAllBaseFromUsdaFasDataList err:" + err1.Error())
		return
	}
	for _, dv := range allData {
		tmpKey := dv.IndexCode + "_" + dv.DataTime
		existDataMap[tmpKey] = dv
	}

	//获取所有指标信息  某一天的
	maxPage := 1

	for currPage := 0; currPage < maxPage; currPage++ {
		data := make(map[string]interface{})
		data["Source"] = utils.DATA_SOURCE_USDA_FAS
		data["StartDate"] = startDate
		data["CurrPage"] = currPage
		data["PageSize"] = 500 //

		var result string
		result, err = HttpPost("SyncBaseFromUsdaFasData", method, data)
		utils.FileLog.Info(result)
		fmt.Println(result)

		respObj := new(data_manage.BaseFromUsdaFasDataResp)
		err = json.Unmarshal([]byte(result), &respObj)
		if err != nil {
			fmt.Println("json.Unmarshal err:" + err.Error())
			return err
		}
		if respObj.Ret != 200 {
			err = errors.New(respObj.ErrMsg)
			return
		}
		// 总页码数
		maxPage = respObj.Data.Paging.Pages

		addDataList := make([]*data_manage.BaseFromUsdaFasData, 0)
		if respObj.Data.List != nil && len(respObj.Data.List) > 0 {
			for _, dv := range respObj.Data.List {
				tmpKey := dv.IndexCode + "_" + dv.DataTime
				if _, ok := existDataMap[tmpKey]; !ok {
					addDataList = append(addDataList, dv)
					existDataMap[tmpKey] = dv
				}
			}
		}

		// 最后如果还有数据未插入，那么继续插入吧
		if len(addDataList) > 0 {
			_, err = data_manage.MultiAddBaseFromUsdaFasDataIndex(addDataList)
			if err != nil {
				fmt.Println("MultiAddBaseFromUsdaFasDataIndex error:", err)
			}
		}
	}

	return err
}