Merge branch 'feature/eta2.2.1_usda_fas' of eta_server/eta_task into master

models/data_manage/base_from_usda_fas.go

@@ -0,0 +1,150 @@
+package data_manage
+
+import (
+	"github.com/beego/beego/v2/client/orm"
+	"github.com/rdlucklib/rdluck_tools/paging"
+	"time"
+)
+
+type BaseFromUsdaFasIndex struct {
+	BaseFromUsdaFasIndexId int `orm:"column(base_from_usda_fas_index_id);pk"`
+	ClassifyId             int
+	IndexCode              string
+	IndexName              string
+	Frequency              string
+	Unit                   string
+	Sort                   int
+	StartDate              string `description:"开始日期"`
+	EndDate                string `description:"结束日期"`
+	EndValue               float64
+	Country                string `description:"国家"`
+	Commodity              string `description:"属性"`
+	CreateTime             time.Time
+	ModifyTime             time.Time
+}
+
+// BaseFromUsdaFasClassify UsdaFas原始数据分类表
+type BaseFromUsdaFasClassify struct {
+	ClassifyId      int       `orm:"column(classify_id);pk"`
+	ClassifyName    string    `description:"分类名称"`
+	ParentId        int       `description:"父级id"`
+	SysUserId       int       `description:"创建人id"`
+	SysUserRealName string    `description:"创建人姓名"`
+	Level           int       `description:"层级"`
+	Sort            int       `description:"排序字段，越小越靠前，默认值：10"`
+	ModifyTime      time.Time `description:"修改时间"`
+	CreateTime      time.Time `description:"创建时间"`
+}
+
+func GetBaseFromUsdaFasIndexMaxDate() (max_date time.Time, err error) {
+	o := orm.NewOrm()
+	sql := ` SELECT max(a.modify_time)as max_date FROM base_from_usda_fas_data as a `
+	err = o.Raw(sql).QueryRow(&max_date)
+	return
+}
+
+type UsdaFasIndexResp struct {
+	Ret     int
+	Msg     string
+	ErrMsg  string
+	ErrCode string
+	Data    []*BaseFromUsdaFasIndex
+}
+
+func GetBaseFromUsdaFasIndexAll(dateStr string) (list []*BaseFromUsdaFasIndex, err error) {
+	o := orm.NewOrm()
+	sql := `SELECT * FROM base_from_usda_fas_index WHERE end_date>=?`
+	_, err = o.Raw(sql, dateStr).QueryRows(&list)
+	return
+}
+
+func (obj *BaseFromUsdaFasIndex) AddBaseFromUsdaFasIndex(item *BaseFromUsdaFasIndex) (lastId int64, err error) {
+	o := orm.NewOrm()
+	lastId, err = o.Insert(item)
+	return
+}
+
+func (obj *BaseFromUsdaFasIndex) InsertOrUpdateBaseFromUsdaFasIndex(item *BaseFromUsdaFasIndex) (lastId int64, err error) {
+	o := orm.NewOrm()
+	lastId, err = o.InsertOrUpdate(item)
+	return
+}
+
+type UsdaFasClassifyResp struct {
+	Ret     int
+	Msg     string
+	ErrMsg  string
+	ErrCode string
+	Data    []*BaseFromUsdaFasClassify
+}
+
+func GetBaseFromUsdaFasClassifyAll() (list []*BaseFromUsdaFasClassify, err error) {
+	o := orm.NewOrm()
+	sql := `SELECT * FROM base_from_usda_fas_classify `
+	_, err = o.Raw(sql).QueryRows(&list)
+	return
+}
+
+func AddBaseFromUsdaFasClassify(item *BaseFromUsdaFasClassify) (lastId int64, err error) {
+	o := orm.NewOrm()
+	lastId, err = o.Insert(item)
+	return
+}
+
+func InsertOrUpdateBaseFromUsdaFasClassify(item *BaseFromUsdaFasClassify) (lastId int64, err error) {
+	o := orm.NewOrm()
+	lastId, err = o.InsertOrUpdate(item)
+	return
+}
+
+// GetBaseFromComTradeMaxDate 获取UsdaFas消费者指数最大数据
+func GetBaseFromUsdaFasMaxDate() (max_date time.Time, err error) {
+	o := orm.NewOrm()
+	sql := ` SELECT max(a.modify_time)as max_date FROM base_from_usda_fas_data as a `
+	err = o.Raw(sql).QueryRow(&max_date)
+	return
+}
+
+type BaseFromUsdaFasData struct {
+	BaseFromUsdaFasDataId  int       `orm:"column(base_from_usda_fas_data_id);pk"`
+	BaseFromUsdaFasIndexId int       `description:"指标id"`
+	IndexCode              string    `description:"指标编码"`
+	DataTime               string    `description:"日期"`
+	Value                  string    `description:"值"`
+	CreateTime             time.Time `description:"创建时间"`
+	ModifyTime             time.Time `description:"修改时间"`
+}
+
+// GetAllComTradeDataList 获取UsdaFas消费者指数数据
+func GetAllBaseFromUsdaFasDataList(startDate string) (list []*BaseFromUsdaFasData, err error) {
+	o := orm.NewOrm()
+	sql := `SELECT * FROM base_from_usda_fas_data WHERE modify_time>=?  ORDER BY base_from_usda_fas_data_id ASC `
+	_, err = o.Raw(sql, startDate).QueryRows(&list)
+	return
+}
+
+type BaseFromUsdaFasDataResp struct {
+	Ret     int
+	Msg     string
+	ErrMsg  string
+	ErrCode string
+	Data    BaseFromUsdaFasDataIndexAndDataResp
+}
+
+// ComTradeIndexDataResp 分页列表响应体
+type BaseFromUsdaFasDataIndexAndDataResp struct {
+	List   []*BaseFromUsdaFasData
+	Paging *paging.PagingItem `description:"分页数据"`
+}
+
+// MultiAddBaseFromComTradeData 批量添加数据
+func MultiAddBaseFromUsdaFasDataIndex(items []*BaseFromUsdaFasData) (lastId int64, err error) {
+	num := len(items)
+	if num <= 0 {
+		return
+	}
+	o := orm.NewOrm()
+	lastId, err = o.InsertMulti(num, items)
+
+	return
+}

models/db.go

@@ -145,6 +145,10 @@ func initEdbDataTable() {
 
 		new(data_manage.EdbInfoRelation), //指标引用关联表
 		new(data_manage.BaseFromThsHfIndex),
+		//美国农业部
+		new(data_manage.BaseFromUsdaFasIndex),
+		new(data_manage.BaseFromUsdaFasClassify),
+		new(data_manage.BaseFromUsdaFasData),
 	)
 }
 

services/sync_hz_data.go

@@ -134,6 +134,27 @@ func SyncHzDataIndex(cont context.Context) (err error) {
 		}
 	}
 
+	//美国农业部
+	{
+		//分类信息
+		err = SyncBaseFromUsdaFasClassify()
+		if err != nil {
+			fmt.Println("SyncBaseFromUsdaFasClassify Err:" + err.Error())
+			return
+		}
+		//指标信息
+		err = SyncBaseFromUsdaFas()
+		if err != nil {
+			fmt.Println("SyncBaseFromUsdaFas Err:" + err.Error())
+			return
+		}
+		err = SyncBaseFromUsdaFasData()
+		if err != nil {
+			fmt.Println("SyncBaseFromUsdaFasData Err:" + err.Error())
+			return
+		}
+	}
+
 	// 同步指标数据
 	SyncHzDataIndexData()
 

services/usda_fas.go

@@ -0,0 +1,164 @@
+package services
+
+import (
+	"encoding/json"
+	"errors"
+	"eta/eta_task/models/data_manage"
+	"eta/eta_task/utils"
+	"fmt"
+	"time"
+)
+
+// UsdaFas美国农业部指标数据同步
+func SyncBaseFromUsdaFas() (err error) {
+	var startDate string
+	maxDate, err := data_manage.GetBaseFromUsdaFasIndexMaxDate()
+	if err != nil || maxDate.IsZero() {
+		startDate = time.Now().AddDate(0, 0, -1).Format(utils.FormatDate) + " 00:00:00"
+	} else {
+		startDate = maxDate.Format(utils.FormatDateTime)
+	}
+
+	method := `index/list`
+	data := make(map[string]interface{})
+	data["Source"] = utils.DATA_SOURCE_USDA_FAS
+	data["StartDate"] = startDate
+	//data["EndDate"] = endDate
+	result, err := HttpPost("SyncRankingFromUsdaFas", method, data)
+	utils.FileLog.Info(result)
+	fmt.Println(result)
+
+	respObj := new(data_manage.UsdaFasIndexResp)
+	err = json.Unmarshal([]byte(result), &respObj)
+	if err != nil {
+		utils.FileLog.Info("err:", err.Error())
+		fmt.Println("err:", err.Error())
+		return err
+	}
+
+	UsdaFasObj := new(data_manage.BaseFromUsdaFasIndex)
+
+	for _, zv := range respObj.Data {
+		newID, err := UsdaFasObj.InsertOrUpdateBaseFromUsdaFasIndex(zv)
+		if err != nil {
+			fmt.Println("InsertOrUpdateBaseFromUsdaFasIndex error:", err)
+		}
+		fmt.Println("InsertOrUpdateBaseFromUsdaFasIndex new indexID:", newID)
+	}
+	return err
+}
+
+// UsdaFas美国农业部-分类
+func SyncBaseFromUsdaFasClassify() (err error) {
+	data := make(map[string]interface{})
+	data["Source"] = utils.DATA_SOURCE_USDA_FAS
+	method := `classify/list`
+	result, err := HttpPost("SyncBaseFromUsdaFasClassify", method, data)
+	if err != nil {
+		fmt.Println("SyncBaseFromUsdaFasClassify HttpPost Err:", err.Error())
+	}
+	utils.FileLog.Info(result)
+	fmt.Println("SyncBaseFromUsdaFasClassify result:", result)
+
+	respObj := new(data_manage.UsdaFasClassifyResp)
+	err = json.Unmarshal([]byte(result), &respObj)
+	if err != nil {
+		return err
+	}
+	//获取所有分类
+	/*	allClassify, err := data_manage.GetBaseFromUsdaFasClassifyAll()
+		if err != nil {
+			return
+		}
+
+		existIndexMap := make(map[int]*data_manage.BaseFromUsdaFasClassify)
+		for _, v := range allClassify {
+			existIndexMap[v.BaseFromUsdaFasClassifyId] = v
+		}*/
+
+	for _, item := range respObj.Data {
+		//if _, ok := existIndexMap[item.BaseFromUsdaFasClassifyId]; !ok {
+		newID, err := data_manage.InsertOrUpdateBaseFromUsdaFasClassify(item)
+		if err != nil {
+			fmt.Println("InsertOrUpdateBaseFromUsdaFasClassify error:", err)
+		}
+		fmt.Println("InsertOrUpdateBaseFromUsdaFasClassify new indexID:", newID)
+		//}
+	}
+	return err
+}
+
+// UsdaFas美国农业部-数据
+func SyncBaseFromUsdaFasData() (err error) {
+	startDate := time.Now().Format(utils.FormatDate) + " 00:00:00"
+	//var startDate string
+	maxDate, err := data_manage.GetBaseFromUsdaFasMaxDate()
+	if err != nil || maxDate.IsZero() {
+		startDate = time.Now().AddDate(0, 0, -1).Format(utils.FormatDate) + " 00:00:00"
+	} else {
+		startDate = maxDate.Format(utils.FormatDateTime)
+	}
+
+	method := `index/data/list_page`
+
+	existDataMap := make(map[string]*data_manage.BaseFromUsdaFasData)
+	allData, err1 := data_manage.GetAllBaseFromUsdaFasDataList(startDate)
+	if err1 != nil {
+		fmt.Println("get GetAllBaseFromUsdaFasDataList err:" + err1.Error())
+		return
+	}
+	for _, dv := range allData {
+		tmpKey := dv.IndexCode + "_" + dv.DataTime
+		existDataMap[tmpKey] = dv
+	}
+
+	//获取所有指标信息  某一天的
+	maxPage := 1
+
+	for currPage := 0; currPage < maxPage; currPage++ {
+		data := make(map[string]interface{})
+		data["Source"] = utils.DATA_SOURCE_USDA_FAS
+		data["StartDate"] = startDate
+		data["CurrPage"] = currPage
+		data["PageSize"] = 500 //
+
+		var result string
+		result, err = HttpPost("SyncBaseFromUsdaFasData", method, data)
+		utils.FileLog.Info(result)
+		fmt.Println(result)
+
+		respObj := new(data_manage.BaseFromUsdaFasDataResp)
+		err = json.Unmarshal([]byte(result), &respObj)
+		if err != nil {
+			fmt.Println("json.Unmarshal err:" + err.Error())
+			return err
+		}
+		if respObj.Ret != 200 {
+			err = errors.New(respObj.ErrMsg)
+			return
+		}
+		// 总页码数
+		maxPage = respObj.Data.Paging.Pages
+
+		addDataList := make([]*data_manage.BaseFromUsdaFasData, 0)
+		if respObj.Data.List != nil && len(respObj.Data.List) > 0 {
+			for _, dv := range respObj.Data.List {
+				tmpKey := dv.IndexCode + "_" + dv.DataTime
+				if _, ok := existDataMap[tmpKey]; !ok {
+					addDataList = append(addDataList, dv)
+					existDataMap[tmpKey] = dv
+				}
+			}
+		}
+
+		// 最后如果还有数据未插入，那么继续插入吧
+		if len(addDataList) > 0 {
+			_, err = data_manage.MultiAddBaseFromUsdaFasDataIndex(addDataList)
+			if err != nil {
+				fmt.Println("MultiAddBaseFromUsdaFasDataIndex error:", err)
+			}
+		}
+	}
+
+	return err
+}

utils/constants.go

@@ -107,6 +107,7 @@ const (
 	DATA_SOURCE_BUSINESS                             = 84       // 来源于自有数据
 	DATA_SOURCE_CCF                                  = 86       // CCF化纤信息
 	DATA_SOURCE_TRADE_ANALYSIS                       = 92       // 持仓分析
+	DATA_SOURCE_USDA_FAS                             = 96       //美国农业部
 )
 
 // 数据刷新频率