Merge branch 'feature/eta2.4.4_gpr_risk' into debug

# Conflicts:
#	models/db.go
#	routers/router.go
#	utils/config.go
#	utils/constants.go

controllers/base_from_gpr_risk.go

@@ -0,0 +1,174 @@
+package controllers
+
+import (
+	"encoding/json"
+	"eta/eta_index_lib/logic"
+	"eta/eta_index_lib/models"
+	"eta/eta_index_lib/services"
+	"eta/eta_index_lib/utils"
+	"fmt"
+	"strconv"
+	"time"
+)
+
+// GprRiskController GPR地缘风险指数
+type GprRiskController struct {
+	BaseAuthController
+}
+
+// Add
+// @Title 新增GPR地缘风险指数指标接口
+// @Description 新增GPR地缘风险指数指标接口
+// @Success 200 {object} models.AddEdbInfoReq
+// @router /add [post]
+func (this *GprRiskController) Add() {
+	br := new(models.BaseResponse).Init()
+	var cacheKey string
+	defer func() {
+		this.Data["json"] = br
+		this.ServeJSON()
+	}()
+	gprRisk := new(models.BaseFromGprRisk)
+	source := gprRisk.GetSource()
+	var req models.AddEdbInfoReq
+	err := json.Unmarshal(this.Ctx.Input.RequestBody, &req)
+	if err != nil {
+		br.Msg = "参数解析异常！"
+		br.ErrMsg = "参数解析失败，Err:" + err.Error()
+		return
+	}
+	if req.EdbCode == "" {
+		br.Msg = "请输入指标编码！"
+		br.ErrMsg = "请输入指标编码，指标编码为空"
+		return
+	}
+	cacheKey = utils.CACHE_EDB_DATA_ADD + strconv.Itoa(source) + "_" + req.EdbCode
+	if utils.Rc.IsExist(cacheKey) {
+		br.Ret = 501
+		br.Success = true
+		br.Msg = "系统处理中，请稍后重试"
+		return
+	}
+	utils.Rc.SetNX(cacheKey, 1, 1*time.Minute)
+	defer func() {
+		utils.Rc.Delete(cacheKey)
+	}()
+	err = gprRisk.Add(req.EdbCode)
+	if err != nil {
+		br.Msg = "获取指标信息失败！"
+		br.ErrMsg = "获取指标信息失败 AddEdbDataFromGprRisk，Err:" + err.Error()
+		return
+	}
+	br.Ret = 200
+	br.Success = true
+	br.Msg = "获取成功"
+	return
+}
+
+// Refresh
+// @Title 刷新GPR地缘风险指数指标接口
+// @Description 刷新GPR地缘风险指数指标接口
+// @Success 200 {object} models.RefreshEdbInfoReq
+// @router /refresh [post]
+func (this *GprRiskController) Refresh() {
+	br := new(models.BaseResponse).Init()
+	var cacheKey string
+	defer func() {
+		this.Data["json"] = br
+		this.ServeJSON()
+	}()
+	GprRisk := new(models.BaseFromGprRisk)
+	source := GprRisk.GetSource()
+	var req models.RefreshEdbInfoReq
+	err := json.Unmarshal(this.Ctx.Input.RequestBody, &req)
+	if err != nil {
+		br.Msg = "参数解析异常！"
+		br.ErrMsg = "参数解析失败，Err:" + err.Error()
+		return
+	}
+	if req.EdbCode == "" {
+		br.Msg = "请输入指标编码！"
+		br.ErrMsg = "请输入指标编码，指标编码为空"
+		return
+	}
+	if req.EdbInfoId < 0 {
+		br.Msg = "请输入指标ID！"
+		br.ErrMsg = "请输入指标ID"
+		return
+	}
+
+	// 获取指标详情
+	edbInfo, err := models.GetEdbInfoByEdbCode(source, req.EdbCode)
+	if err != nil {
+		br.Msg = "指标不存在！"
+		br.ErrMsg = "指标不存在"
+		return
+	}
+	cacheKey = utils.CACHE_EDB_DATA_REFRESH + strconv.Itoa(source) + "_" + req.EdbCode
+	if utils.Rc.IsExist(cacheKey) {
+		br.Ret = 501
+		br.Success = true
+		br.Msg = "系统处理中，请稍后重试"
+		return
+	}
+	utils.Rc.SetNX(cacheKey, 1, 1*time.Minute)
+	defer func() {
+		utils.Rc.Delete(cacheKey)
+	}()
+	if req.EdbInfoId <= 0 {
+		req.EdbInfoId = edbInfo.EdbInfoId
+	}
+	err = GprRisk.Refresh(req.EdbInfoId, req.EdbCode, req.StartDate)
+	if err != nil && err.Error() != utils.ErrNoRow() {
+		br.Msg = "刷新指标信息失败！"
+		br.ErrMsg = "刷新指标信息失败 RefreshEdbDataFromGprRisk，Err:" + err.Error()
+		return
+	}
+	// 更新指标最大最小值
+	err, errMsg := models.UnifiedModifyEdbInfoMaxAndMinInfo(edbInfo)
+	if err != nil {
+		br.Msg = errMsg
+		br.ErrMsg = err.Error()
+		return
+	}
+
+	// 更新ES
+	go logic.UpdateEs(edbInfo.EdbInfoId)
+
+	br.Ret = 200
+	br.Success = true
+	br.Msg = "获取成功"
+}
+
+// HandleExcelData
+// @Title 处理GPR地缘风险指数指标的接口
+// @Description 处理GPR地缘风险指数指标的接口
+// @Success 200 {object} models.HandleGprRiskExcelDataReq
+// @router /handle/excel_data [post]
+func (this *GprRiskController) HandleExcelData() {
+	br := new(models.BaseResponse).Init()
+	defer func() {
+		this.Data["json"] = br
+		this.ServeJSON()
+	}()
+	var req models.HandleGprRiskExcelDataReq
+	err := json.Unmarshal(this.Ctx.Input.RequestBody, &req)
+	if err != nil {
+		br.Msg = "参数解析异常！"
+		br.ErrMsg = "参数解析失败，Err:" + err.Error()
+		return
+	}
+
+	// 处理excel表数据
+	err = services.HandleGprRiskIndex(&req)
+	if err != nil {
+		fmt.Println("HandleMysteelIndex Err:" + err.Error())
+		br.Msg = "处理失败"
+		br.ErrMsg = "处理失败,Err:" + err.Error()
+		return
+	}
+
+	br.Ret = 200
+	br.Success = true
+	br.Msg = "处理成功"
+}

controllers/exchange_crawler.go

@@ -62,12 +62,12 @@ func (this *ExchangeCrawler) RefreshIne() {
 		if p.Rank > 0 && p.Rank < 40 && p.Participantabbr1 != "" {
 			//成交量
 			item.Rank = p.Rank
-			item.DealShortName = p.Participantabbr1
-			item.BuyShortName = p.Participantabbr2
-			item.SoldShortName = p.Participantabbr3
-			item.DealName = strings.Replace(fmt.Sprintf("%s", p.Participantabbr1+"_"+p.Instrumentid+"_成交量(手)"), " ", "", -1)
-			item.BuyName = strings.Replace(fmt.Sprintf("%s", p.Participantabbr2+"_"+p.Instrumentid+"_持买单量(手)"), " ", "", -1)
-			item.SoldName = strings.Replace(fmt.Sprintf("%s", p.Participantabbr3+"_"+p.Instrumentid+"_持卖单量(手)"), " ", "", -1)
+			item.DealShortName = strings.Trim(p.Participantabbr1, " ")
+			item.BuyShortName = strings.Trim(p.Participantabbr2, " ")
+			item.SoldShortName = strings.Trim(p.Participantabbr3, " ")
+			item.DealName = strings.Replace(fmt.Sprintf("%s", item.DealShortName+"_"+p.Instrumentid+"_成交量(手)"), " ", "", -1)
+			item.BuyName = strings.Replace(fmt.Sprintf("%s", item.BuyShortName+"_"+p.Instrumentid+"_持买单量(手)"), " ", "", -1)
+			item.SoldName = strings.Replace(fmt.Sprintf("%s", item.SoldShortName+"_"+p.Instrumentid+"_持卖单量(手)"), " ", "", -1)
 			item.DealCode = IneIndexCodeGenerator(item.DealShortName, item.DealName, p.Instrumentid, "deal")
 			item.BuyCode = IneIndexCodeGenerator(item.BuyShortName, item.BuyName, p.Instrumentid, "buy")
 			item.SoldCode = IneIndexCodeGenerator(item.SoldShortName, item.SoldName, p.Instrumentid, "sold")
@@ -114,9 +114,9 @@ func (this *ExchangeCrawler) RefreshIne() {
 		} else if p.Rank == 999 {
 			//Top 20
 			item.Rank = p.Rank
-			item.DealShortName = p.Participantabbr1
-			item.BuyShortName = p.Participantabbr2
-			item.SoldShortName = p.Participantabbr3
+			item.DealShortName = strings.Trim(p.Participantabbr1, " ")
+			item.BuyShortName = strings.Trim(p.Participantabbr2, " ")
+			item.SoldShortName = strings.Trim(p.Participantabbr3, " ")
 			item.DealName = strings.Replace(fmt.Sprintf("%s", "top20_"+p.Instrumentid+"_成交量(手)"), " ", "", -1)
 			item.BuyName = strings.Replace(fmt.Sprintf("%s", "top20_"+p.Instrumentid+"_持买单量(手)"), " ", "", -1)
 			item.SoldName = strings.Replace(fmt.Sprintf("%s", "top20_"+p.Instrumentid+"_持卖单量(手)"), " ", "", -1)
@@ -392,4 +392,4 @@ func IneIndexCodeGenerator(shortName, indexName, Instrumentid, suffix string) st
 		}
 	}
 	return strings.Replace(ineIndexCode, " ", "", -1)
-}
+}

models/base_from_gpr_risk.go

@@ -0,0 +1,349 @@
+package models
+
+import (
+	"eta/eta_index_lib/utils"
+	"fmt"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/beego/beego/v2/client/orm"
+)
+
+// BaseFromGprRisk 美国农业部
+type BaseFromGprRisk struct{}
+
+type BaseFromGprRiskData struct {
+	BaseFromGprRiskDataId  int `orm:"column(base_from_gpr_risk_data_id);pk"`
+	BaseFromGprRiskIndexId int
+	IndexCode              string
+	DataTime               string
+	Value                  string
+	CreateTime             time.Time
+	ModifyTime             time.Time
+	DataTimestamp          int64
+}
+
+func GetBaseFromGprRiskDataByCondition(condition string, pars []interface{}) (list []*BaseFromGprRiskData, err error) {
+	o := orm.NewOrm()
+	sql := `SELECT * FROM base_from_gpr_risk_data WHERE 1=1 `
+	if condition != "" {
+		sql += condition
+	}
+	_, err = o.Raw(sql, pars).QueryRows(&list)
+	return
+}
+
+// Add 添加
+func (obj BaseFromGprRisk) Add(edbCode string) (err error) {
+	o := orm.NewOrm()
+
+	var condition string
+	var pars []interface{}
+	if edbCode != "" {
+		condition += " AND index_code=? "
+		pars = append(pars, edbCode)
+	}
+	GprRiskBaseDataAll, err := GetBaseFromGprRiskDataByCondition(condition, pars)
+	if err != nil && err.Error() != utils.ErrNoRow() {
+		return
+	}
+	var isAdd bool
+	addSql := ` INSERT INTO edb_data_gpr_risk(edb_info_id,edb_code,data_time,value,create_time,modify_time,data_timestamp) values `
+	existMap := make(map[string]string)
+	for _, sv := range GprRiskBaseDataAll {
+		eDate := sv.DataTime
+		dataTime, err := time.Parse(utils.FormatDate, eDate)
+		if err != nil {
+			return err
+		}
+		timestamp := dataTime.UnixNano() / 1e6
+		timeStr := fmt.Sprintf("%d", timestamp)
+		if _, ok := existMap[eDate]; !ok {
+			addSql += GetAddSql("0", edbCode, eDate, timeStr, sv.Value)
+			isAdd = true
+		}
+		existMap[eDate] = sv.Value
+	}
+	if isAdd {
+		addSql = strings.TrimRight(addSql, ",")
+		utils.FileLog.Info("addSql:" + addSql)
+		_, err = o.Raw(addSql).Exec()
+		if err != nil {
+			return err
+		}
+	}
+	return
+}
+
+// Refresh 刷新涌益咨询指标数据
+func (obj BaseFromGprRisk) Refresh(edbInfoId int, edbCode, startDate string) (err error) {
+	source := obj.GetSource()
+	o := orm.NewOrm()
+	if err != nil {
+		return
+	}
+	edbInfoIdStr := strconv.Itoa(edbInfoId)
+	//计算数据
+	var condition string
+	var pars []interface{}
+
+	if edbCode != "" {
+		condition += " AND index_code=? "
+		pars = append(pars, edbCode)
+	}
+
+	if startDate != "" {
+		condition += " AND data_time>=? "
+		pars = append(pars, startDate)
+	}
+
+	GprRiskDataList, err := GetBaseFromGprRiskDataByCondition(condition, pars)
+	if err != nil {
+		return
+	}
+
+	// 真实数据的最大日期  ， 插入规则配置的日期
+	var realDataMaxDate, edbDataInsertConfigDate time.Time
+	var edbDataInsertConfig *EdbDataInsertConfig
+	var isFindConfigDateRealData bool //是否找到配置日期的实际数据的值
+	{
+		edbDataInsertConfig, err = GetEdbDataInsertConfigByEdbId(edbInfoId)
+		if err != nil && err.Error() != utils.ErrNoRow() {
+			return
+		}
+		if edbDataInsertConfig != nil {
+			edbDataInsertConfigDate = edbDataInsertConfig.Date
+		}
+	}
+
+	var existCondition string
+	var existPars []interface{}
+
+	existCondition += " AND edb_info_id=? "
+	existPars = append(existPars, edbInfoId)
+	if startDate != "" {
+		existCondition += " AND data_time>=? "
+		existPars = append(existPars, startDate)
+	}
+
+	existList, err := GetEdbDataByCondition(source, 0, existCondition, existPars)
+	if err != nil {
+		return err
+	}
+	existMap := make(map[string]*EdbInfoSearchData)
+	for _, v := range existList {
+		existMap[v.DataTime] = v
+	}
+	addSql := ` INSERT INTO edb_data_gpr_risk(edb_info_id,edb_code,data_time,value,create_time,modify_time,data_timestamp) values `
+	var isAdd bool
+	for _, v := range GprRiskDataList {
+		item := v
+		eDate := item.DataTime
+		dataTime, err := time.ParseInLocation(utils.FormatDate, eDate, time.Local)
+		if err != nil {
+			return err
+		}
+		if findItem, ok := existMap[v.DataTime]; !ok {
+			sValue := item.Value
+
+			timestamp := dataTime.UnixNano() / 1e6
+			timeStr := fmt.Sprintf("%d", timestamp)
+
+			addSql += GetAddSql(edbInfoIdStr, edbCode, eDate, timeStr, sValue)
+			isAdd = true
+		} else {
+			if findItem != nil && utils.SubFloatToString(findItem.Value, 30) != item.Value {
+				err = ModifyEdbDataById(source, 0, findItem.EdbDataId, item.Value)
+				if err != nil {
+					return err
+				}
+			}
+		}
+
+		// 下面代码主要目的是处理掉手动插入的数据判断
+		{
+			if realDataMaxDate.IsZero() || dataTime.After(realDataMaxDate) {
+				realDataMaxDate = dataTime
+			}
+			if edbDataInsertConfigDate.IsZero() || dataTime.Equal(edbDataInsertConfigDate) {
+				isFindConfigDateRealData = true
+			}
+		}
+	}
+
+	// 处理手工数据补充的配置
+	HandleConfigInsertEdbData(realDataMaxDate, edbDataInsertConfig, edbInfoId, source, 0, existMap, isFindConfigDateRealData)
+
+	if isAdd {
+		addSql = strings.TrimRight(addSql, ",")
+		_, err = o.Raw(addSql).Exec()
+		if err != nil {
+			fmt.Println("RefreshEdbDataFromBaiinfo add Err", err.Error())
+			return
+		}
+	}
+	return
+}
+
+// GetSource 获取来源编码id
+func (obj BaseFromGprRisk) GetSource() int {
+	return utils.DATA_SOURCE_GPR_RISK
+}
+
+// GetSourceName 获取来源名称
+func (obj BaseFromGprRisk) GetSourceName() string {
+	return utils.DATA_SOURCE_NAME_GPR_RISK
+}
+
+type BaseFromGprRiskIndex struct {
+	BaseFromGprRiskIndexId int64 `orm:"column(base_from_gpr_risk_index_id);pk"`
+	IndexCode              string
+	IndexName              string
+	Frequency              string
+	Unit                   string
+	StartDate              string
+	EndDate                string
+	ClassifyId             int64
+	Sort                   int
+	BaseFileName           string
+	RenameFileName         string
+	TerminalCode           string
+	CreateTime             time.Time
+	ModifyTime             time.Time
+}
+
+type BaseFromGprRiskIndexList struct {
+	BaseFromGprRiskIndexId int64 `orm:"column(base_from_gpr_risk_index_id);pk"`
+	IndexCode              string
+	IndexName              string
+	Frequency              string
+	Unit                   string
+	Sort                   int
+	ClassifyId             int64
+	StartDate              string
+	EndDate                string
+	TerminalCode           string
+	CreateTime             string
+	ModifyTime             string
+}
+
+func (y *BaseFromGprRiskData) GetByIndexCode(indexCode string) (list []*BaseFromGprRiskData, err error) {
+	o := orm.NewOrm()
+	sql := ` SELECT * FROM base_from_gpr_risk_data WHERE index_code=? `
+	_, err = o.Raw(sql, indexCode).QueryRows(&list)
+	return
+}
+
+func (y *BaseFromGprRiskData) AddMulti(item []*BaseFromGprRiskData) (err error) {
+	o := orm.NewOrm()
+	_, err = o.InsertMulti(1, item)
+	return
+}
+
+// Update 修改
+func (y *BaseFromGprRiskData) Update(updateCols []string) (err error) {
+	o := orm.NewOrm()
+	_, err = o.Update(y, updateCols...)
+	return
+}
+
+// HandleGprRiskExcelData 涌益咨询的excel数据
+type HandleGprRiskExcelData struct {
+	ClassifyName       string `description:"指标目录"`
+	ParentClassifyName string `description:"父级指标目录"`
+	ClassifySort       int    `description:"指标目录排序号"`
+	IndexName          string `description:"指标名称"`
+	IndexCode          string `description:"指标编码"`
+	Unit               string `description:"单位"`
+	Sort               int    `description:"排序号"`
+	Frequency          string `description:"频度"`
+	ExcelDataMap       map[string]string
+}
+
+type HandleGprRiskExcelDataReq struct {
+	List         []*HandleGprRiskExcelData
+	TerminalCode string `description:"编码"`
+}
+
+func (y *BaseFromGprRiskData) GetMaxAndMinDateByIndexCode(indexCode string) (item *EdbInfoMaxAndMinInfo, err error) {
+	o := orm.NewOrm()
+	sql := ` SELECT MIN(data_time) AS min_date,MAX(data_time) AS max_date,MIN(value) AS min_value,MAX(value) AS max_value FROM base_from_gpr_risk_data WHERE index_code=? `
+	err = o.Raw(sql, indexCode).QueryRow(&item)
+	var latest_value float64
+	sql = ` SELECT value AS latest_value FROM %s WHERE index_code=? ORDER BY data_time DESC LIMIT 1 `
+	sql = fmt.Sprintf(sql, "base_from_gpr_risk_data")
+	err = o.Raw(sql, indexCode).QueryRow(&latest_value)
+	item.LatestValue = latest_value
+	return
+}
+
+func (y *BaseFromGprRiskIndex) ModifyIndexMaxAndMinDate(indexCode string, item *EdbInfoMaxAndMinInfo) (err error) {
+	o := orm.NewOrm()
+	sql := ` UPDATE base_from_gpr_risk_index SET start_date=?,end_date=?, end_value=?, modify_time=NOW() WHERE index_code=? `
+	_, err = o.Raw(sql, item.MinDate, item.MaxDate, item.LatestValue, indexCode).Exec()
+	return
+}
+
+func (y *BaseFromGprRiskIndex) GetByIndexCode(indexCode string) (item *BaseFromGprRiskIndex, err error) {
+	o := orm.NewOrm()
+	sql := ` SELECT * FROM base_from_gpr_risk_index WHERE index_code=? `
+	err = o.Raw(sql, indexCode).QueryRow(&item)
+	return
+}
+
+func (y *BaseFromGprRiskIndex) Add() (lastId int64, err error) {
+	o := orm.NewOrm()
+	lastId, err = o.Insert(y)
+	return
+}
+
+// Update 修改
+func (y *BaseFromGprRiskIndex) Update(updateCols []string) (err error) {
+	o := orm.NewOrm()
+	_, err = o.Update(y, updateCols...)
+
+	return
+}
+
+// BaseFromGprRiskClassify GprRisk原始数据分类表
+type BaseFromGprRiskClassify struct {
+	ClassifyId      int64     `orm:"column(classify_id);pk"`
+	ClassifyName    string    `description:"分类名称"`
+	ClassifyNameEn  string    `description:"分类名称"`
+	ParentId        int       `description:"父级id"`
+	SysUserId       int       `description:"创建人id"`
+	SysUserRealName string    `description:"创建人姓名"`
+	Level           int       `description:"层级"`
+	Sort            int       `description:"排序字段，越小越靠前，默认值：10"`
+	ModifyTime      time.Time `description:"修改时间"`
+	CreateTime      time.Time `description:"创建时间"`
+}
+
+func (y *BaseFromGprRiskClassify) Add() (lastId int64, err error) {
+	o := orm.NewOrm()
+	lastId, err = o.Insert(y)
+	return
+}
+
+// Update 修改
+func (y *BaseFromGprRiskClassify) Update(updateCols []string) (err error) {
+	o := orm.NewOrm()
+	_, err = o.Update(y, updateCols...)
+
+	return
+}
+
+func (y *BaseFromGprRiskClassify) GetByClassifyName(classifyName string) (item *BaseFromGprRiskClassify, err error) {
+	o := orm.NewOrm()
+	sql := ` SELECT * FROM base_from_gpr_risk_classify WHERE classify_name=? `
+	err = o.Raw(sql, classifyName).QueryRow(&item)
+	return
+}
+
+func (y *BaseFromGprRiskClassify) GetParentClassify() (items []*BaseFromGprRiskClassify, err error) {
+	o := orm.NewOrm()
+	sql := ` SELECT * FROM base_from_gpr_risk_classify WHERE parent_id=0 `
+	_, err = o.Raw(sql).QueryRows(&items)
+	return
+}

models/db.go

@@ -179,6 +179,9 @@ func initBaseIndex() {
 		new(BaseFromUsdaFasClassify),
 		new(BaseFromClarksonsIndex),
 		new(BaseFromClarksonsData),
+		new(BaseFromGprRiskIndex),
+		new(BaseFromGprRiskData),
+		new(BaseFromGprRiskClassify),
 	)
 }
 

routers/commentsRouter.go

@@ -1033,6 +1033,33 @@ func init() {
             Filters: nil,
             Params: nil})
 
+    beego.GlobalControllerRouter["eta/eta_index_lib/controllers:GprRiskController"] = append(beego.GlobalControllerRouter["eta/eta_index_lib/controllers:GprRiskController"],
+        beego.ControllerComments{
+            Method: "Add",
+            Router: `/add`,
+            AllowHTTPMethods: []string{"post"},
+            MethodParams: param.Make(),
+            Filters: nil,
+            Params: nil})
+
+    beego.GlobalControllerRouter["eta/eta_index_lib/controllers:GprRiskController"] = append(beego.GlobalControllerRouter["eta/eta_index_lib/controllers:GprRiskController"],
+        beego.ControllerComments{
+            Method: "HandleExcelData",
+            Router: `/handle/excel_data`,
+            AllowHTTPMethods: []string{"post"},
+            MethodParams: param.Make(),
+            Filters: nil,
+            Params: nil})
+
+    beego.GlobalControllerRouter["eta/eta_index_lib/controllers:GprRiskController"] = append(beego.GlobalControllerRouter["eta/eta_index_lib/controllers:GprRiskController"],
+        beego.ControllerComments{
+            Method: "Refresh",
+            Router: `/refresh`,
+            AllowHTTPMethods: []string{"post"},
+            MethodParams: param.Make(),
+            Filters: nil,
+            Params: nil})
+
     beego.GlobalControllerRouter["eta/eta_index_lib/controllers:GzController"] = append(beego.GlobalControllerRouter["eta/eta_index_lib/controllers:GzController"],
         beego.ControllerComments{
             Method: "Add",

routers/router.go

@@ -342,6 +342,11 @@ func init() {
 				&controllers.ClarksonsDataController{},
 			),
 		),
+		beego.NSNamespace("/gpr_risk",
+			beego.NSInclude(
+				&controllers.GprRiskController{},
+			),
+		),
 	)
 	beego.AddNamespace(ns)
 }

services/base_from_gpr_risk.go

@@ -0,0 +1,309 @@
+package services
+
+import (
+	"eta/eta_index_lib/logic"
+	"eta/eta_index_lib/models"
+	"eta/eta_index_lib/services/alarm_msg"
+	"eta/eta_index_lib/utils"
+	"fmt"
+	"strconv"
+	"strings"
+	"time"
+)
+
+// HandleGprRiskIndex 处理GPR地缘风险指数的excel数据
+func HandleGprRiskIndex(req *models.HandleGprRiskExcelDataReq) (err error) {
+	errMsgList := make([]string, 0)
+	defer func() {
+		if len(errMsgList) > 0 {
+			msg := fmt.Sprint("数据源-GPR地缘风险指数数据处理失败,err:", strings.Join(errMsgList, "\n"))
+			utils.FileLog.Info(msg)
+			go alarm_msg.SendAlarmMsg(msg, 3)
+		}
+	}()
+	// 查询所有的一级分类
+	classifyObj := new(models.BaseFromGprRiskClassify)
+	classifyList, err := classifyObj.GetParentClassify()
+	if err != nil {
+		err = fmt.Errorf("查询一级目录信息失败 Err：%s", err)
+		return
+	}
+	classifyMap := make(map[string]int, 0)
+	for _, v := range classifyList {
+		classifyMap[v.ClassifyName] = int(v.ClassifyId)
+	}
+
+	for _, v := range req.List {
+		if v.IndexName == "" || v.IndexCode == "" {
+			errMsgList = append(errMsgList, fmt.Sprintf("新增指标异常，指标编码%s或者指标ID%s为空：", v.IndexCode, v.IndexName))
+			continue
+		}
+		err = handleGprRiskIndex(v, req.TerminalCode, classifyMap)
+		if err != nil {
+			errMsgList = append(errMsgList, fmt.Sprintf("新增指标异常，指标编码：%s, Err: %s", v.IndexCode, err))
+			return
+		}
+	}
+	return
+}
+
+func handleGprRiskIndex(req *models.HandleGprRiskExcelData, terminalCode string, classifyMap map[string]int) (err error) {
+	indexName := req.IndexName
+	indexCode := req.IndexCode
+	excelDataMap := req.ExcelDataMap
+	errMsgList := make([]string, 0)
+	defer func() {
+		if len(errMsgList) > 0 {
+			msg := fmt.Sprint("数据源-GPR地缘风险指数数据处理失败,err:", strings.Join(errMsgList, "\n"))
+			utils.FileLog.Info(msg)
+			go alarm_msg.SendAlarmMsg(msg, 3)
+		}
+	}()
+	indexObj := new(models.BaseFromGprRiskIndex)
+	dataObj := new(models.BaseFromGprRiskData)
+	classifyObj := new(models.BaseFromGprRiskClassify)
+
+	var indexId int64
+
+	addDataList := make([]*models.BaseFromGprRiskData, 0)
+
+	exitDataMap := make(map[string]*models.BaseFromGprRiskData)
+
+	// 修改指标信息
+	if indexName == "" {
+		utils.FileLog.Info("未刷新到指标数据:indexName:" + indexName)
+		return
+	}
+	// 判断目录是否存在
+	var classifyId int64
+	now := time.Now()
+	if req.ClassifyName != "" {
+		classifyParentId := 0
+		level := 1
+		classifyParentId, _ = classifyMap[req.ParentClassifyName]
+		if classifyParentId > 0 {
+			level = 2
+		}
+		classifyObj, err = classifyObj.GetByClassifyName(req.ClassifyName)
+		if err != nil {
+			if err.Error() == utils.ErrNoRow() {
+				//新增分类
+				classifyObj = &models.BaseFromGprRiskClassify{
+					ClassifyName:    req.ClassifyName,
+					ClassifyNameEn:  req.ClassifyName,
+					ParentId:        classifyParentId,
+					SysUserId:       0,
+					SysUserRealName: "",
+					Level:           level,
+					Sort:            req.ClassifySort,
+					ModifyTime:      now,
+					CreateTime:      now,
+				}
+
+				classifyId, err = classifyObj.Add()
+				if err != nil {
+					err = fmt.Errorf("新增分类失败 Err：%s", err)
+					return
+				}
+				classifyObj.ClassifyId = classifyId
+			} else {
+				return
+			}
+		} else {
+			classifyId = classifyObj.ClassifyId
+			classifyObj.ModifyTime = now
+			//classifyObj.Sort = req.ClassifySort
+			classifyObj.ParentId = classifyParentId
+			//e := classifyObj.Update([]string{"ParentId", "Sort", "ModifyTime"})
+			e := classifyObj.Update([]string{"ParentId", "ModifyTime"})
+			if e != nil {
+				fmt.Println("classifyObj Update Err:" + e.Error())
+				return
+			}
+		}
+	}
+
+	//判断指标是否存在
+	var isAdd int
+	item, err := indexObj.GetByIndexCode(indexCode)
+	if err != nil {
+		if err.Error() == utils.ErrNoRow() {
+			isAdd = 1
+			err = nil
+		} else {
+			isAdd = -1
+			err = fmt.Errorf("查询数据源指标库失败 GetByIndexCode Err：%s", err)
+			return
+		}
+	}
+	if item != nil && item.BaseFromGprRiskIndexId > 0 {
+		fmt.Println("item:", item)
+		isAdd = 2
+	} else {
+		isAdd = 1
+	}
+
+	if isAdd == 1 {
+		indexObj.IndexCode = indexCode
+		indexObj.IndexName = indexName
+		indexObj.Frequency = req.Frequency
+		indexObj.ClassifyId = classifyId
+		indexObj.Unit = req.Unit
+		indexObj.Sort = req.Sort
+		indexObj.ModifyTime = time.Now()
+		indexObj.CreateTime = time.Now()
+		indexObj.TerminalCode = terminalCode
+		indexId, err = indexObj.Add()
+		if err != nil {
+			err = fmt.Errorf("数据源新增GPR地缘风险指数指标失败 Err：%s", err)
+			return
+		}
+		indexObj.BaseFromGprRiskIndexId = indexId
+	} else if isAdd == 2 {
+		indexId = item.BaseFromGprRiskIndexId
+		if item.TerminalCode == `` && terminalCode != `` {
+			item.TerminalCode = terminalCode
+			err = item.Update([]string{"TerminalCode"})
+			if err != nil {
+				err = fmt.Errorf("数据源更新GPR地缘风险指数指标失败 Err：%s", err)
+				return
+			}
+		}
+
+		indexObj.BaseFromGprRiskIndexId = item.BaseFromGprRiskIndexId
+		indexObj.IndexName = indexName
+		indexObj.Frequency = req.Frequency
+		indexObj.ClassifyId = classifyId
+		indexObj.Unit = req.Unit
+		indexObj.Sort = req.Sort
+		indexObj.ModifyTime = time.Now()
+
+		//修改数据
+		updateColsArr := make([]string, 0)
+		updateColsArr = append(updateColsArr, "index_name")
+		updateColsArr = append(updateColsArr, "classify_id")
+		updateColsArr = append(updateColsArr, "frequency")
+		updateColsArr = append(updateColsArr, "sort")
+		updateColsArr = append(updateColsArr, "unit")
+		updateColsArr = append(updateColsArr, "modify_time")
+
+		e := indexObj.Update(updateColsArr)
+		if e != nil {
+			fmt.Println("Index Update Err:" + e.Error())
+			return
+		}
+	}
+
+	//获取已存在的所有数据
+	var exitDataList []*models.BaseFromGprRiskData
+	exitDataList, err = dataObj.GetByIndexCode(indexCode)
+	if err != nil {
+		err = fmt.Errorf("数据源查询GPR地缘风险指数指标数据失败 Err：%s", err)
+		return
+	}
+	fmt.Println("exitDataListLen:", len(exitDataList))
+	for _, v := range exitDataList {
+		dateStr := v.DataTime
+		exitDataMap[dateStr] = v
+	}
+
+	// 遍历excel数据，然后跟现有的数据做校验，不存在则入库
+	for date, value := range excelDataMap {
+		if findData, ok := exitDataMap[date]; !ok {
+			_, err = time.ParseInLocation(utils.FormatDate, date, time.Local)
+			if err != nil {
+				err = fmt.Errorf("%s 转换日期格式失败 Err：%s", date, err)
+				return
+			}
+			//if !strings.Contains(value, "#N/A") {
+			var saveDataTime time.Time
+			if strings.Contains(date, "00:00:00") {
+				saveDataTime, err = time.Parse(utils.FormatDateTime, date)
+			} else {
+				saveDataTime, err = time.Parse(utils.FormatDate, date)
+			}
+			if err != nil {
+				err = fmt.Errorf("%s 转换日期格式失败 Err：%s", date, err)
+				continue
+			}
+			timestamp := saveDataTime.UnixNano() / 1e6
+
+			dataItem := new(models.BaseFromGprRiskData)
+			dataItem.BaseFromGprRiskIndexId = int(indexId)
+			dataItem.IndexCode = indexCode
+			dataItem.DataTime = date
+			dataItem.Value = value
+			dataItem.CreateTime = time.Now()
+			dataItem.ModifyTime = time.Now()
+			dataItem.DataTimestamp = timestamp
+			addDataList = append(addDataList, dataItem)
+			if len(addDataList) > 500 {
+				err = dataObj.AddMulti(addDataList)
+				if err != nil {
+					err = fmt.Errorf("批量新增指标失败 Err：%s", err)
+					return
+				}
+				addDataList = make([]*models.BaseFromGprRiskData, 0)
+			}
+			//}
+		} else {
+			if findData != nil && findData.Value != value && !strings.Contains(value, "#N/A") { //修改数据
+				// 过滤0.50和0.5的比较
+				oldV, _ := strconv.ParseFloat(findData.Value, 64)
+				newV, _ := strconv.ParseFloat(value, 64)
+				if oldV == newV {
+					continue
+				}
+				dataObj.BaseFromGprRiskIndexId = findData.BaseFromGprRiskIndexId
+				dataObj.Value = value
+				dataObj.ModifyTime = time.Now()
+
+				updateDataColsArr := make([]string, 0)
+				updateDataColsArr = append(updateDataColsArr, "value")
+				updateDataColsArr = append(updateDataColsArr, "modify_time")
+				dataObj.Update(updateDataColsArr)
+			}
+		}
+	}
+
+	if len(addDataList) > 0 {
+		err = dataObj.AddMulti(addDataList)
+		if err != nil {
+			err = fmt.Errorf("批量新增指标失败 Err：%s", err)
+			return
+		}
+	}
+
+	var dateItem *models.EdbInfoMaxAndMinInfo
+	dateItem, err = dataObj.GetMaxAndMinDateByIndexCode(indexCode)
+	if err != nil {
+		err = fmt.Errorf("查询指标最新日期失败 Err：%s", err)
+		return
+	}
+
+	go func() {
+		indexObj.ModifyIndexMaxAndMinDate(indexCode, dateItem)
+	}()
+
+	// 同步刷新ETA指标库的指标
+	{
+		// 获取指标详情
+		baseObj := new(models.BaseFromGprRisk)
+		var edbInfo *models.EdbInfo
+		edbInfo, err = models.GetEdbInfoByEdbCode(baseObj.GetSource(), indexCode)
+		if err != nil {
+			if err.Error() != utils.ErrNoRow() {
+				errMsgList = append(errMsgList, fmt.Sprint("刷新ETA指标异常，指标编码：", indexCode, err.Error()))
+				return
+			} else {
+				err = nil
+			}
+		}
+
+		// 已经加入到指标库的话，那么就去更新ETA指标库吧
+		if edbInfo != nil {
+			go logic.RefreshBaseEdbInfo(edbInfo, ``)
+		}
+	}
+	return
+}

services/base_from_mysteel_chemical.go

@@ -553,7 +553,12 @@ func GetEdbDataFromMySteelChemical(indexCodes []string, startTime, endTime, orde
 		err = er
 		return
 	}
-	postUrl := `https://mds.mysteel.com/dynamic/order/api/fcAbRA`
+	// postUrl := `https://mds.mysteel.com/dynamic/order/api/fcAbRA`
+	postUrl := utils.MySteelChemicalApiUrl
+	if postUrl == "" {
+		err = errors.New("钢联化工接口url未配置")
+		return
+	}
 	body, err := MySteelChemicalPost(postUrl, "data", postData)
 	if err != nil {
 		return
@@ -605,7 +610,12 @@ func getPageIndexInfoMap(pageNum, pageSize int, includeInfo bool) (item *models.
 		err = er
 		return
 	}
-	postUrl := `https://mds.mysteel.com/dynamic/order/api/fcAbRA`
+	// postUrl := `https://mds.mysteel.com/dynamic/order/api/fcAbRA`
+	postUrl := utils.MySteelChemicalApiUrl
+	if postUrl == "" {
+		err = errors.New("钢联化工接口url未配置")
+		return
+	}
 	body, er := MySteelChemicalPost(postUrl, "info", postData)
 	if er != nil {
 		err = er

utils/config.go

@@ -74,7 +74,8 @@ var (
 
 	ThsDataMethod           string //同花顺数据获取的方式，app是通过终端；api是通过接口
 	ThsRefreshToken         string // 同花顺的刷新token
-	MysteelChemicalApiToken string // 上海钢联的api数据token
+	MysteelChemicalApiToken string // 钢联化工的api数据token
+	MySteelChemicalApiUrl   string // 钢联化工的api数据地址
 )
 
 type WindUrlMap struct {
@@ -232,6 +233,7 @@ func init() {
 			ThsDataMethod = "api"
 		}
 		MysteelChemicalApiToken = config["mysteel_chemical_api_token"]
+		MySteelChemicalApiUrl = config["mysteel_chemical_api_url"]
 	}
 
 	// ES配置

utils/constants.go

@@ -122,6 +122,7 @@ const (
 	DATA_SOURCE_CALCULATE_PHASE_SHIFT                = 95  // 期数移动
 	DATA_SOURCE_RZD                                  = 97  // 睿咨得科技 -> 93
 	DATA_SOURCE_CLARKSONS                            = 101 // 克拉克森 -> 101
+	DATA_SOURCE_GPR_RISK                             = 102
 )
 
 // 指标来源的中文展示
@@ -207,14 +208,15 @@ const (
 	DATA_SOURCE_NAME_CALCULATE_SUM                        = `多指标求和`
 	DATA_SOURCE_NAME_CALCULATE_AVG                        = `多指标求平均`
 	DATA_SOURCE_NAME_BUSINESS                             = `自有数据`
-	DATA_SOURCE_NAME_CALCULATE_RANGEANLYSIS               = `区间计算`   //区间计算->87
-	DATA_SOURCE_NAME_PREDICT_CALCULATE_RANGEANLYSIS       = `预测区间计算` //区间计算->90
-	DATA_SOURCE_NAME_TRADE_ANALYSIS                       = `持仓分析`   // 持仓分析
-	DATA_SOURCE_NAME_USDA_FAS                             = `美国农业部`  //美国农业部->96
-	DATA_SOURCE_NAME_CCF                                  = `CCF`    // CCF化纤信息
-	DATA_SOURCE_NAME_SCI_HQ                               = `卓创红期`   // 卓创红期
-	DATA_SOURCE_NAME_OILCHEM                              = `隆众资讯`   // 隆众资讯 -> 89
-	DATA_SOURCE_NAME_HISUGAR                              = `泛糖科技`   // 泛糖科技 -> 93
+	DATA_SOURCE_NAME_CALCULATE_RANGEANLYSIS               = `区间计算`      //区间计算->87
+	DATA_SOURCE_NAME_PREDICT_CALCULATE_RANGEANLYSIS       = `预测区间计算`    //区间计算->90
+	DATA_SOURCE_NAME_TRADE_ANALYSIS                       = `持仓分析`      // 持仓分析
+	DATA_SOURCE_NAME_USDA_FAS                             = `美国农业部`     //美国农业部->96
+	DATA_SOURCE_NAME_CCF                                  = `CCF`       // CCF化纤信息
+	DATA_SOURCE_NAME_SCI_HQ                               = `卓创红期`      // 卓创红期
+	DATA_SOURCE_NAME_OILCHEM                              = `隆众资讯`      // 隆众资讯 -> 89
+	DATA_SOURCE_NAME_HISUGAR                              = `泛糖科技`      // 泛糖科技 -> 93
+	DATA_SOURCE_NAME_GPR_RISK                             = `GPR地缘风险指数` // GPR地缘风险指数 -> 102
 )
 
 // 基础数据初始化日期