Merge remote-tracking branch 'origin/master' into fix/terminal

# Conflicts:
#	controllers/base_from_smm.go
#	services/base_from_smm.go

.gitignore

@@ -8,4 +8,7 @@ binlog/
 /eta_index_lib.exe
 /eta_index_lib.exe~
 eta_index_lib
-/static/imgs/python/*
+/static/imgs/python/*
+/etalogs
+*.gz
+*.exe

controllers/base_from_jiayue.go

@@ -0,0 +1,262 @@
+package controllers
+
+import (
+	"encoding/json"
+	"eta/eta_index_lib/logic"
+	"eta/eta_index_lib/models"
+	"eta/eta_index_lib/services"
+	"eta/eta_index_lib/utils"
+	"fmt"
+	"strconv"
+	"time"
+)
+
+// JiaYueController 嘉悦物产数据源
+type JiaYueController struct {
+	BaseAuthController
+}
+
+// Add
+// @Title 嘉悦物产-新增指标接口
+// @Description  新增指标接口
+// @Success 200 {object} models.AddEdbInfoReq
+// @router /add [post]
+func (this *JiaYueController) Add() {
+	br := new(models.BaseResponse).Init()
+	var cacheKey string
+	defer func() {
+		if br.ErrMsg == "" {
+			br.IsSendEmail = false
+		}
+		_ = utils.Rc.Delete(cacheKey)
+		this.Data["json"] = br
+		this.ServeJSON()
+	}()
+	var req models.AddEdbInfoReq
+	if e := json.Unmarshal(this.Ctx.Input.RequestBody, &req); e != nil {
+		br.Msg = "操作失败"
+		br.ErrMsg = "参数解析异常, err: " + e.Error()
+		return
+	}
+	if req.EdbCode == "" {
+		br.Msg = "请输入指标编码"
+		br.ErrMsg = "指标编码为空"
+		return
+	}
+	if req.Source <= 0 {
+		br.Msg = "请输入指标来源"
+		br.ErrMsg = "指标来源为空"
+		return
+	}
+
+	// 加锁
+	cacheKey = utils.CACHE_EDB_DATA_ADD + strconv.Itoa(req.Source) + "_" + req.EdbCode
+	if utils.Rc.IsExist(cacheKey) {
+		br.Ret = 501
+		br.Success = true
+		br.Msg = "系统处理中，请稍后重试"
+		return
+	}
+	utils.Rc.SetNX(cacheKey, 1, 1*time.Minute)
+	defer func() {
+		_ = utils.Rc.Delete(cacheKey)
+	}()
+
+	// 校验数据源
+	sourceItem := models.EdbSourceIdMap[req.Source]
+	if sourceItem == nil {
+		br.Msg = "数据源有误"
+		br.ErrMsg = fmt.Sprintf("数据源有误, Source: %d", req.Source)
+		return
+	}
+
+	// 从桥接服务获取指标和数据
+	var params models.BridgeJiaYueIndexDataParams
+	params.IndexCode = req.EdbCode
+	params.SourceExtend = sourceItem.SourceExtend
+	params.StartDate = utils.BASE_START_DATE
+	params.EndDate = utils.BASE_END_DATE
+	params.IndexCodeRequired = sourceItem.EdbCodeRequired
+	indexData, e := services.GetJiaYueIndexDataFromBridge(params)
+	if e != nil {
+		br.Msg = "获取指标失败"
+		br.ErrMsg = "获取指标数据失败, Err: " + e.Error()
+		return
+	}
+	if indexData.Id <= 0 {
+		br.Msg = "指标不存在"
+		br.Success = true
+		return
+	}
+
+	// 新增指标数据
+	e = models.AddEdbDataFromJiaYue(sourceItem.TableName, req.EdbCode, indexData.IndexData)
+	if e != nil {
+		br.Msg = "操作失败"
+		br.ErrMsg = "新增指标数据失败, Err: " + e.Error()
+		return
+	}
+
+	br.Ret = 200
+	br.Success = true
+	br.Msg = "操作成功"
+}
+
+// Refresh
+// @Title 嘉悦物产-刷新指标接口
+// @Description 刷新指标接口
+// @Success 200 {object} models.RefreshEdbInfoReq
+// @router /refresh [post]
+func (this *JiaYueController) Refresh() {
+	br := new(models.BaseResponse).Init()
+	var cacheKey string
+	defer func() {
+		if br.ErrMsg == "" {
+			br.IsSendEmail = false
+		}
+		this.Data["json"] = br
+		this.ServeJSON()
+	}()
+	var req models.RefreshEdbInfoReq
+	err := json.Unmarshal(this.Ctx.Input.RequestBody, &req)
+	if err != nil {
+		br.Msg = "参数解析异常！"
+		br.ErrMsg = "参数解析失败，Err:" + err.Error()
+		return
+	}
+	if req.EdbCode == "" {
+		br.Msg = "请输入指标编码！"
+		br.ErrMsg = "请输入指标编码，指标编码为空"
+		return
+	}
+	if req.EdbInfoId <= 0 {
+		br.Msg = "请输入指标ID！"
+		br.ErrMsg = "请输入指标ID"
+		return
+	}
+	if req.Source <= 0 {
+		br.Msg = "请输入指标来源"
+		br.ErrMsg = "指标来源为空"
+		return
+	}
+	startDate := req.StartDate
+	if startDate == "" {
+		startDate = utils.BASE_START_DATE
+	}
+
+	// 加锁
+	cacheKey = utils.CACHE_EDB_DATA_REFRESH + strconv.Itoa(req.Source) + "_" + req.EdbCode
+	if utils.Rc.IsExist(cacheKey) {
+		br.Ret = 501
+		br.Success = true
+		br.Msg = "系统处理中，请稍后重试"
+		return
+	}
+	utils.Rc.SetNX(cacheKey, 1, 1*time.Minute)
+	defer func() {
+		_ = utils.Rc.Delete(cacheKey)
+	}()
+
+	// 校验指标和数据源
+	edbInfo, e := models.GetEdbInfoById(req.EdbInfoId)
+	if e != nil {
+		br.Msg = "指标信息有误"
+		br.ErrMsg = "查询指标信息失败, Err: " + e.Error()
+		return
+	}
+	sourceItem := models.EdbSourceIdMap[req.Source]
+	if sourceItem == nil {
+		br.Msg = "数据源有误"
+		br.ErrMsg = fmt.Sprintf("数据源有误, Source: %d", req.Source)
+		return
+	}
+
+	// 从桥接服务获取指标和数据
+	var params models.BridgeJiaYueIndexDataParams
+	params.IndexCode = req.EdbCode
+	params.SourceExtend = sourceItem.SourceExtend
+	params.StartDate = startDate
+	params.EndDate = utils.BASE_END_DATE
+	params.IndexCodeRequired = sourceItem.EdbCodeRequired
+	indexData, e := services.GetJiaYueIndexDataFromBridge(params)
+	if e != nil {
+		br.Msg = "获取指标失败"
+		br.ErrMsg = "获取指标数据失败, Err: " + e.Error()
+		return
+	}
+	if indexData.Id <= 0 {
+		br.Msg = "指标不存在"
+		br.Success = true
+		return
+	}
+
+	// 刷新指标数据
+	e = models.RefreshEdbDataFromJiaYue(req.Source, req.EdbInfoId, sourceItem.TableName, req.EdbCode, startDate, indexData.IndexData)
+	if e != nil {
+		br.Msg = "刷新失败"
+		br.ErrMsg = "刷新嘉悦指标失败, Err: " + e.Error()
+		return
+	}
+
+	// 更新指标最大最小值
+	e, errMsg := models.UnifiedModifyEdbInfoMaxAndMinInfo(edbInfo)
+	if e != nil {
+		br.Msg = errMsg
+		br.ErrMsg = e.Error()
+		return
+	}
+
+	// 更新ES
+	go logic.UpdateEs(edbInfo.EdbInfoId)
+
+	br.Ret = 200
+	br.Success = true
+	br.Msg = "操作成功"
+}
+
+// SyncNewIndex
+// @Title 嘉悦物产-同步增量指标
+// @Description  同步增量指标
+// @Success 200 {object} models.AddEdbInfoReq
+// @router /sync_new_index [post]
+func (this *JiaYueController) SyncNewIndex() {
+	br := new(models.BaseResponse).Init()
+	defer func() {
+		if br.ErrMsg == "" {
+			br.IsSendEmail = false
+		}
+		this.Data["json"] = br
+		this.ServeJSON()
+	}()
+
+	// 从桥接服务获取增量指标
+	newIndexes, e := services.GetJiaYueNewIndexFromBridge()
+	if e != nil {
+		br.Msg = "同步增量指标失败"
+		br.ErrMsg = "同步增量指标失败, Err: " + e.Error()
+		return
+	}
+	if len(newIndexes) == 0 {
+		utils.FileLog.Info("无增量指标同步")
+		return
+	}
+
+	// 获取指标目录
+	indexMenus, e := services.GetJiaYueMenuListFromBridge()
+	if e != nil {
+		br.Msg = "获取指标目录失败"
+		br.ErrMsg = "获取指标目录失败, Err: " + e.Error()
+		return
+	}
+
+	go func() {
+		for _, v := range newIndexes {
+			// 错误信息在FileLog中这边不做返回
+			_ = services.SyncJiaYueNewIndex(v, indexMenus)
+		}
+	}()
+
+	br.Ret = 200
+	br.Success = true
+	br.Msg = "操作成功"
+}

controllers/base_from_smm.go

@@ -6,6 +6,7 @@ import (
 	"eta/eta_index_lib/models"
 	"eta/eta_index_lib/services"
 	"eta/eta_index_lib/utils"
+	"fmt"
 	"strconv"
 	"time"
 )
@@ -153,7 +154,12 @@ func (this *SmmController) HandleExcelData() {
 	}
 
 	// 处理excel表数据
-	services.SmmIndexHandle(req.BaseFilePath, req.RenameFilePath, req.IndexName, req.IndexCode, req.Unit, req.Frequency, req.Source, req.ExcelDataMap, req.TerminalCode)
+	e, errMsg := services.SmmIndexHandle(req.BaseFilePath, req.RenameFilePath, req.IndexName, req.IndexCode, req.Unit, req.Frequency, req.Source, req.ExcelDataMap, req.TerminalCode)
+	if e != nil {
+		br.Msg = "处理失败"
+		br.ErrMsg = fmt.Sprintf("处理有色excel表数据失败, Err: %s, ErrMsg: %s", e.Error(), errMsg)
+		return
+	}
 
 	br.Ret = 200
 	br.Success = true

models/base_from_bridge.go

@@ -0,0 +1,27 @@
+package models
+
+type EdbDataFromWindBridge struct {
+	Code int                        `json:"code"`
+	Msg  string                     `json:"msg"`
+	Data EdbDataFromWindBridgeIndex `json:"data"`
+}
+
+type EdbDataFromWindBridgeIndex struct {
+	IndexCode      string                      `json:"index_code"`
+	LastDate       string                      `description:"指标最新时间" json:"last_date"`
+	LastUpdateTime string                      `description:"最新更新时间" json:"last_update_time"`
+	Status         int                         `description:"指标状态" json:"status"`
+	IndexData      []EdbDataFromWindBridgeData `json:"index_data"`
+}
+
+type EdbDataFromWindBridgeData struct {
+	Val        float64 `json:"val"`
+	DataTime   string  `json:"data_time"`
+	UpdateTime string  `json:"update_time"`
+}
+
+type EdbDataFromWindSimple struct {
+	Val          string
+	DateTime     string
+	TimestampStr string
+}

models/base_from_jiayue.go

@@ -0,0 +1,228 @@
+package models
+
+import (
+	"eta/eta_index_lib/utils"
+	"fmt"
+	"github.com/beego/beego/v2/client/orm"
+	"strconv"
+	"strings"
+	"time"
+)
+
+// BridgeJiaYueIndexDataParams 桥接服务-获取嘉悦指标数据入参
+type BridgeJiaYueIndexDataParams struct {
+	IndexCode         string `json:"index_code" form:"index_code" description:"指标编码"`
+	SourceExtend      string `json:"source_extend" form:"source_extend" description:"来源"`
+	StartDate         string `json:"start_date" form:"start_date" description:"开始日期"`
+	EndDate           string `json:"end_date" form:"end_date" description:"结束日期"`
+	IndexCodeRequired int    `json:"index_code_required" form:"index_code_required" description:"指标编码是否必填: 0-否; 1-是"`
+}
+
+// BridgeJiaYueResultIndexData 桥接服务-获取嘉悦指标数据响应体
+type BridgeJiaYueResultIndexData struct {
+	Code int                      `json:"code" description:"状态码"`
+	Msg  string                   `json:"msg" description:"提示信息"`
+	Data BridgeJiaYueIndexAndData `json:"data" description:"返回数据"`
+}
+
+// BridgeJiaYueIndexAndData 桥接服务-嘉悦指标和数据
+type BridgeJiaYueIndexAndData struct {
+	Id             int                       `description:"指标自增ID" json:"id"`
+	IndexCode      string                    `description:"指标编码" json:"index_code"`
+	IndexName      string                    `description:"指标名称" json:"index_name"`
+	SourceType     string                    `description:"指标来源" json:"source_type"`
+	Unit           string                    `description:"单位" json:"unit"`
+	Frequency      string                    `description:"频度" json:"frequency"`
+	LastDate       time.Time                 `description:"指标最新时间" json:"last_date"`
+	LastUpdateTime time.Time                 `description:"最新更新时间" json:"last_update_time"`
+	Status         int                       `description:"指标状态" json:"status"`
+	IndexData      []BridgeJiaYueIndexData   `description:"指标数据" json:"index_data"`
+	MenuData       BridgeJiaYueIndexMenuData `description:"指标目录信息" json:"menu_data"`
+}
+
+// BridgeJiaYueIndexData 桥接服务-嘉悦指标数据
+type BridgeJiaYueIndexData struct {
+	Val        float64   `json:"val"`
+	DataTime   time.Time `json:"data_time"`
+	UpdateTime time.Time `json:"update_time"`
+}
+
+// BridgeJiaYueIndexMenuData 桥接服务-嘉悦指标目录信息
+type BridgeJiaYueIndexMenuData struct {
+	Id         int    `description:"目录ID" json:"id"`
+	Type       string `description:"目录类型" json:"type"`
+	Code       string `description:"目录编码" json:"code"`
+	Name       string `description:"目录名称" json:"name"`
+	Icon       string `description:"目录图标" json:"icon"`
+	Sort       int    `description:"排序" json:"sort"`
+	ParentId   int    `description:"父级目录ID" json:"parent_id"`
+	ParentName string `description:"父级目录名称" json:"parent_name"`
+	Path       string `description:"目录全路径" json:"path"`
+}
+
+// BridgeJiaYueIndexMenuWithLevel 桥接服务-嘉悦指标目录带层级
+type BridgeJiaYueIndexMenuWithLevel struct {
+	Level int `description:"层级"`
+	Menu  BridgeJiaYueIndexMenuData
+}
+
+// BridgeJiaYueResultNewIndexData 桥接服务-获取嘉悦增量指标数据响应体
+type BridgeJiaYueResultNewIndexData struct {
+	Code int                        `json:"code" description:"状态码"`
+	Msg  string                     `json:"msg" description:"提示信息"`
+	Data []BridgeJiaYueIndexAndData `json:"data" description:"返回数据"`
+}
+
+// BridgeJiaYueResultMenuListData 桥接服务-获取嘉悦指标目录数据响应体
+type BridgeJiaYueResultMenuListData struct {
+	Code int                         `json:"code" description:"状态码"`
+	Msg  string                      `json:"msg" description:"提示信息"`
+	Data []BridgeJiaYueIndexMenuData `json:"data" description:"返回数据"`
+}
+
+// AddEdbDataFromJiaYue 新增嘉悦指标数据
+func AddEdbDataFromJiaYue(tableName, edbCode string, dataList []BridgeJiaYueIndexData) (err error) {
+	if tableName == "" {
+		err = fmt.Errorf("数据表名为空")
+		return
+	}
+	if edbCode == "" {
+		err = fmt.Errorf("指标编码为空")
+		return
+	}
+	if len(dataList) == 0 {
+		return
+	}
+
+	sql := fmt.Sprintf(`INSERT INTO %s(edb_info_id, edb_code, data_time, value, create_time, modify_time, data_timestamp) VALUES `, tableName)
+	for _, v := range dataList {
+		val := utils.SubFloatToString(v.Val, 20)
+		stamp := fmt.Sprint(v.DataTime.UnixMilli())
+		sql += GetAddSql("0", edbCode, v.DataTime.Format(utils.FormatDate), stamp, val)
+	}
+	sql = strings.TrimRight(sql, ",")
+
+	// 新增入库
+	o := orm.NewOrm()
+	_, e := o.Raw(sql).Exec()
+	if e != nil {
+		err = fmt.Errorf("insert data err: %s", e.Error())
+	}
+	return
+}
+
+// RefreshEdbDataFromJiaYue 刷新嘉悦指标数据
+func RefreshEdbDataFromJiaYue(source, edbInfoId int, tableName, edbCode, startDate string, dataList []BridgeJiaYueIndexData) (err error) {
+	if source <= 0 {
+		err = fmt.Errorf("指标来源有误")
+		return
+	}
+	if edbInfoId <= 0 {
+		err = fmt.Errorf("指标ID有误")
+		return
+	}
+	if tableName == "" {
+		err = fmt.Errorf("数据表名为空")
+		return
+	}
+
+	// 真实数据的最大日期, 插入规则配置的日期
+	var realDataMaxDate, edbDataInsertConfigDate time.Time
+	var edbDataInsertConfig *EdbDataInsertConfig
+	var isFindConfigDateRealData bool //是否找到配置日期的实际数据的值
+	{
+		conf, e := GetEdbDataInsertConfigByEdbId(edbInfoId)
+		if e != nil && e.Error() != utils.ErrNoRow() {
+			return
+		}
+		edbDataInsertConfig = conf
+		if edbDataInsertConfig != nil {
+			edbDataInsertConfigDate = edbDataInsertConfig.Date
+		}
+	}
+
+	// 获取已有数据
+	cond := ` AND edb_info_id = ?`
+	pars := make([]interface{}, 0)
+	pars = append(pars, edbInfoId)
+	var startDateTime time.Time
+	if startDate != "" {
+		cond += ` AND data_time >= ?`
+		pars = append(pars, startDate)
+		startDateTime, _ = time.ParseInLocation(utils.FormatDate, startDate, time.Local)
+	}
+	existList, e := GetEdbDataByCondition(source, cond, pars)
+	if e != nil {
+		err = fmt.Errorf("获取指标已有数据失败, Err: %s", e.Error())
+		return
+	}
+	existMap := make(map[string]*EdbInfoSearchData)
+	for _, v := range existList {
+		existMap[v.DataTime] = v
+	}
+
+	// 比对数据
+	hasNew := false
+	strEdbInfoId := strconv.Itoa(edbInfoId)
+	addExists := make(map[string]bool)
+	sqlInsert := fmt.Sprintf(`INSERT INTO %s(edb_info_id, edb_code, data_time, value, create_time, modify_time, data_timestamp) VALUES `, tableName)
+	for _, v := range dataList {
+		val := utils.SubFloatToString(v.Val, 30)
+		stamp := fmt.Sprint(v.DataTime.UnixMilli())
+		dataTime := v.DataTime.Format(utils.FormatDate)
+
+		// 如果传入的开始时间是空的, 且当前数据日期早于传入的开始日期, 那么需要判断下当前日期的数据是否存在
+		if !startDateTime.IsZero() && v.DataTime.Before(startDateTime) {
+			t, e := GetEdbDataByDate(source, edbCode, dataTime)
+			if e == nil && t != nil {
+				existMap[t.DataTime] = t
+			}
+		}
+
+		// 下面代码主要目的是处理掉手动插入的数据判断
+		{
+			if realDataMaxDate.IsZero() || v.DataTime.After(realDataMaxDate) {
+				realDataMaxDate = v.DataTime
+			}
+			if edbDataInsertConfigDate.IsZero() || v.DataTime.Equal(edbDataInsertConfigDate) {
+				isFindConfigDateRealData = true
+			}
+		}
+
+		// 新增数据
+		exist, ok := existMap[dataTime]
+		if !ok {
+			// 不在历史数据中且与新增中的数据不重复
+			if _, o := addExists[dataTime]; !o {
+				hasNew = true
+				sqlInsert += GetAddSql(strEdbInfoId, edbCode, dataTime, stamp, val)
+				addExists[dataTime] = true
+			}
+			continue
+		}
+
+		// 更新数据
+		if exist != nil && utils.SubFloatToString(exist.Value, 30) != val {
+			if e = ModifyEdbDataById(source, exist.EdbDataId, val); e != nil {
+				err = fmt.Errorf("modify edb data err: %s", e.Error())
+				return
+			}
+		}
+	}
+
+	// 处理手工数据补充的配置
+	HandleConfigInsertEdbData(realDataMaxDate, edbDataInsertConfig, edbInfoId, source, existMap, isFindConfigDateRealData)
+
+	// 执行新增
+	if !hasNew {
+		return
+	}
+	o := orm.NewOrm()
+	sqlInsert = strings.TrimRight(sqlInsert, ",")
+	_, e = o.Raw(sqlInsert).Exec()
+	if e != nil {
+		err = fmt.Errorf("insert edb data err: %s", e.Error())
+		return
+	}
+	return
+}

models/db.go

@@ -68,6 +68,9 @@ func init() {
 
 	// Eta表格相关
 	initExcel()
+
+	// 初始化部分数据表变量(直接init会有顺序问题=_=!)
+	InitEdbSource()
 }
 
 // initFutureGood 注册期货数据 数据表

models/edb_data_base.go

@@ -44,6 +44,7 @@ func GetAddSql(edbInfoId, edbCode, dataTime, timestampStr string, value string)
 
 type AddEdbInfoReq struct {
 	EdbCode string `description:"指标编码"`
+	Source  int    `description:"指标来源ID"`
 }
 
 // GetEdbInfoCountByCondition 获取指标数量
@@ -120,6 +121,7 @@ type RefreshEdbInfoReq struct {
 	EdbInfoId int    `description:"指标ID"`
 	EdbCode   string `description:"指标编码"`
 	StartDate string `description:"开始日期"`
+	Source    int    `description:"指标来源ID"`
 }
 
 // GetAllEdbDataList 获取所有的指标数据列表

models/edb_data_table.go

@@ -4,7 +4,7 @@ import (
 	"eta/eta_index_lib/utils"
 )
 
-// 指标数据->存储表
+// GetEdbDataTableName 指标数据->存储表
 func GetEdbDataTableName(source int) (tableName string) {
 	switch source {
 	case utils.DATA_SOURCE_THS:
@@ -154,7 +154,10 @@ func GetEdbDataTableName(source int) (tableName string) {
 	case utils.DATA_SOURCE_CALCULATE_RJZ: //日均值75
 		tableName = "edb_data_calculate_rjz"
 	default:
-		tableName = ""
+		edbSource := EdbSourceIdMap[source]
+		if edbSource != nil {
+			tableName = edbSource.TableName
+		}
 	}
 	return
 }

models/edb_info.go

@@ -1071,8 +1071,14 @@ func EdbInfoAdd(req *AddEdbInfoParams, serverUrl string, sysUserId int, sysUserR
 
 	sourceName, ok := sourceNameMap[source]
 	if !ok {
-		err = errors.New("指标来源异常")
-		return
+		edbSource := EdbSourceIdMap[source]
+		if edbSource != nil {
+			sourceName = edbSource.SourceName
+		}
+		if sourceName == "" {
+			err = errors.New("指标来源异常")
+			return
+		}
 	}
 	edbInfo.SourceName = sourceName
 

models/edb_source.go

@@ -0,0 +1,74 @@
+package models
+
+import (
+	"eta/eta_index_lib/utils"
+	"fmt"
+	"github.com/beego/beego/v2/client/orm"
+	"strings"
+)
+
+var (
+	EdbSourceIdMap       map[int]*EdbSource // 指标来源
+	EdbSourceExtendIdMap map[string]int     // 指标来源字符串对应来源ID
+)
+
+// EdbSource 指标来源表
+type EdbSource struct {
+	EdbSourceId      int    `orm:"column(edb_source_id);pk"`
+	SourceName       string `description:"指标来源名称"`
+	TableName        string `description:"数据表名"`
+	EdbAddMethod     string `description:"指标新增接口"`
+	EdbRefreshMethod string `description:"指标刷新接口"`
+	IsBase           int    `description:"是否为基础指标: 0-否; 1-是"`
+	FromBridge       int    `description:"是否来源于桥接服务: 0-否; 1-是"`
+	BridgeFlag       string `description:"桥接服务对象标识"`
+	SourceExtend     string `description:"扩展字段做查询用"`
+	EdbCodeRequired  int    `description:"指标编码是否必填: 0-否; 1-是"`
+}
+
+// GetEdbSourceItemsByCondition 获取指标来源列表
+func GetEdbSourceItemsByCondition(condition string, pars []interface{}, fieldArr []string, orderRule string) (items []*EdbSource, err error) {
+	o := orm.NewOrm()
+	fields := strings.Join(fieldArr, ",")
+	if len(fieldArr) == 0 {
+		fields = `*`
+	}
+	order := `ORDER BY edb_source_id ASC`
+	if orderRule != "" {
+		order = ` ORDER BY ` + orderRule
+	}
+	sql := fmt.Sprintf(`SELECT %s FROM edb_source WHERE 1=1 %s %s`, fields, condition, order)
+	_, err = o.Raw(sql, pars).QueryRows(&items)
+	return
+}
+
+// GetEdbSourceItemByCondition 获取指标来源
+func GetEdbSourceItemByCondition(condition string, pars []interface{}) (item *EdbSource, err error) {
+	o := orm.NewOrm()
+	sql := fmt.Sprintf(`SELECT * FROM edb_source WHERE 1=1 %s`, condition)
+	err = o.Raw(sql, pars).QueryRow(&item)
+	return
+}
+
+// InitEdbSource 初始化时加载指标来源对应信息, 避免循环中查库, 注意edb_source表修改table_name的话需要重启服务
+func InitEdbSource() {
+	EdbSourceIdMap = make(map[int]*EdbSource)
+	EdbSourceExtendIdMap = make(map[string]int)
+	sources, e := GetEdbSourceItemsByCondition(``, make([]interface{}, 0), []string{}, "")
+	if e != nil {
+		utils.FileLog.Info("init source table err: %s", e.Error())
+		return
+	}
+	for _, v := range sources {
+		EdbSourceIdMap[v.EdbSourceId] = v
+		if v.SourceExtend != "" {
+			arr := strings.Split(v.SourceExtend, ",")
+			if len(arr) == 0 {
+				continue
+			}
+			for _, s := range arr {
+				EdbSourceExtendIdMap[s] = v.EdbSourceId
+			}
+		}
+	}
+}

routers/commentsRouter.go

@@ -376,6 +376,33 @@ func init() {
             Filters: nil,
             Params: nil})
 
+    beego.GlobalControllerRouter["eta/eta_index_lib/controllers:JiaYueController"] = append(beego.GlobalControllerRouter["eta/eta_index_lib/controllers:JiaYueController"],
+        beego.ControllerComments{
+            Method: "Add",
+            Router: `/add`,
+            AllowHTTPMethods: []string{"post"},
+            MethodParams: param.Make(),
+            Filters: nil,
+            Params: nil})
+
+    beego.GlobalControllerRouter["eta/eta_index_lib/controllers:JiaYueController"] = append(beego.GlobalControllerRouter["eta/eta_index_lib/controllers:JiaYueController"],
+        beego.ControllerComments{
+            Method: "Refresh",
+            Router: `/refresh`,
+            AllowHTTPMethods: []string{"post"},
+            MethodParams: param.Make(),
+            Filters: nil,
+            Params: nil})
+
+    beego.GlobalControllerRouter["eta/eta_index_lib/controllers:JiaYueController"] = append(beego.GlobalControllerRouter["eta/eta_index_lib/controllers:JiaYueController"],
+        beego.ControllerComments{
+            Method: "SyncNewIndex",
+            Router: `/sync_new_index`,
+            AllowHTTPMethods: []string{"post"},
+            MethodParams: param.Make(),
+            Filters: nil,
+            Params: nil})
+
     beego.GlobalControllerRouter["eta/eta_index_lib/controllers:LtController"] = append(beego.GlobalControllerRouter["eta/eta_index_lib/controllers:LtController"],
         beego.ControllerComments{
             Method: "Add",

routers/router.go

@@ -197,6 +197,11 @@ func init() {
 				&fix.CustomAnalysisController{},
 			),
 		),
+		beego.NSNamespace("/jiayue_index",
+			beego.NSInclude(
+				&controllers.JiaYueController{},
+			),
+		),
 	)
 	beego.AddNamespace(ns)
 }

services/base_from_jiayue.go

@@ -0,0 +1,363 @@
+package services
+
+import (
+	"encoding/json"
+	"eta/eta_index_lib/logic"
+	"eta/eta_index_lib/models"
+	"eta/eta_index_lib/services/alarm_msg"
+	"eta/eta_index_lib/utils"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"sort"
+	"strconv"
+	"strings"
+)
+
+var (
+	BridgeApiJiaYueIndexDataUrl = "/api/index_data/jiayue/index_data" // 获取指标数据API
+	BridgeApiJiaYueNewIndexUrl  = "/api/index_data/jiayue/new_index"  // 获取增量指标API
+	BridgeApiJiaYueMenuListUrl  = "/api/index_data/jiayue/menu_list"  // 获取指标目录API
+)
+
+// GetJiaYueIndexDataFromBridge 从桥接服务获取指标数据
+func GetJiaYueIndexDataFromBridge(param models.BridgeJiaYueIndexDataParams) (indexData models.BridgeJiaYueIndexAndData, err error) {
+	defer func() {
+		if err != nil {
+			b, _ := json.Marshal(param)
+			tips := fmt.Sprintf("桥接服务-获取嘉悦指标数据失败, err: %s, params: %s", err.Error(), string(b))
+			utils.FileLog.Info(tips)
+			go alarm_msg.SendAlarmMsg(tips, 3)
+		}
+	}()
+
+	url := fmt.Sprint(utils.EtaBridgeUrl, BridgeApiJiaYueIndexDataUrl)
+	data, e := json.Marshal(param)
+	if e != nil {
+		err = fmt.Errorf("data json marshal err: %s", e.Error())
+		return
+	}
+	body := ioutil.NopCloser(strings.NewReader(string(data)))
+	client := &http.Client{}
+	req, e := http.NewRequest("POST", url, body)
+	if e != nil {
+		err = fmt.Errorf("http create request err: %s", e.Error())
+		return
+	}
+
+	checkToken := utils.MD5(utils.EtaBridgeAppNameEn + utils.EtaBridgeMd5Key)
+	contentType := "application/json;charset=utf-8"
+	req.Header.Set("Content-Type", contentType)
+	req.Header.Set("Authorization", checkToken)
+	resp, e := client.Do(req)
+	if e != nil {
+		err = fmt.Errorf("http client do err: %s", e.Error())
+		return
+	}
+	defer func() {
+		_ = resp.Body.Close()
+	}()
+	b, e := ioutil.ReadAll(resp.Body)
+	if e != nil {
+		err = fmt.Errorf("resp body read err: %s", e.Error())
+		return
+	}
+	if len(b) == 0 {
+		err = fmt.Errorf("resp body is empty")
+		return
+	}
+	// 生产环境解密
+	if utils.RunMode == "release" {
+		str := string(b)
+		str = strings.Trim(str, `"`)
+		b = utils.DesBase64Decrypt([]byte(str), utils.EtaBridgeDesKey)
+	}
+
+	result := new(models.BridgeJiaYueResultIndexData)
+	if e = json.Unmarshal(b, &result); e != nil {
+		err = fmt.Errorf("result unmarshal err: %s\nresult: %s", e.Error(), string(b))
+		return
+	}
+	if result.Code != 200 {
+		err = fmt.Errorf("result: %s", string(b))
+		return
+	}
+	indexData = result.Data
+	return
+}
+
+// GetJiaYueNewIndexFromBridge 从桥接服务获取增量指标
+func GetJiaYueNewIndexFromBridge() (indexData []models.BridgeJiaYueIndexAndData, err error) {
+	defer func() {
+		if err != nil {
+			tips := fmt.Sprintf("桥接服务-获取嘉悦增量指标失败, err: %s", err.Error())
+			utils.FileLog.Info(tips)
+			go alarm_msg.SendAlarmMsg(tips, 3)
+		}
+	}()
+
+	url := fmt.Sprint(utils.EtaBridgeUrl, BridgeApiJiaYueNewIndexUrl)
+	body := ioutil.NopCloser(strings.NewReader(""))
+	client := &http.Client{}
+	req, e := http.NewRequest("POST", url, body)
+	if e != nil {
+		err = fmt.Errorf("http create request err: %s", e.Error())
+		return
+	}
+
+	checkToken := utils.MD5(utils.EtaBridgeAppNameEn + utils.EtaBridgeMd5Key)
+	contentType := "application/json;charset=utf-8"
+	req.Header.Set("Content-Type", contentType)
+	req.Header.Set("Authorization", checkToken)
+	resp, e := client.Do(req)
+	if e != nil {
+		err = fmt.Errorf("http client do err: %s", e.Error())
+		return
+	}
+	defer func() {
+		_ = resp.Body.Close()
+	}()
+	b, e := ioutil.ReadAll(resp.Body)
+	if e != nil {
+		err = fmt.Errorf("resp body read err: %s", e.Error())
+		return
+	}
+	if len(b) == 0 {
+		err = fmt.Errorf("resp body is empty")
+		return
+	}
+	// 生产环境解密
+	if utils.RunMode == "release" {
+		str := string(b)
+		str = strings.Trim(str, `"`)
+		b = utils.DesBase64Decrypt([]byte(str), utils.EtaBridgeDesKey)
+	}
+
+	result := new(models.BridgeJiaYueResultNewIndexData)
+	if e = json.Unmarshal(b, &result); e != nil {
+		err = fmt.Errorf("result unmarshal err: %s\nresult: %s", e.Error(), string(b))
+		return
+	}
+	if result.Code != 200 {
+		err = fmt.Errorf("result: %s", string(b))
+		return
+	}
+	indexData = result.Data
+	return
+}
+
+// GetJiaYueMenuListFromBridge 从桥接服务获取指标目录列表
+func GetJiaYueMenuListFromBridge() (indexData []models.BridgeJiaYueIndexMenuData, err error) {
+	defer func() {
+		if err != nil {
+			tips := fmt.Sprintf("桥接服务-获取嘉悦增量指标失败, err: %s", err.Error())
+			utils.FileLog.Info(tips)
+			go alarm_msg.SendAlarmMsg(tips, 3)
+		}
+	}()
+
+	url := fmt.Sprint(utils.EtaBridgeUrl, BridgeApiJiaYueMenuListUrl)
+	body := ioutil.NopCloser(strings.NewReader(""))
+	client := &http.Client{}
+	req, e := http.NewRequest("POST", url, body)
+	if e != nil {
+		err = fmt.Errorf("http create request err: %s", e.Error())
+		return
+	}
+
+	checkToken := utils.MD5(utils.EtaBridgeAppNameEn + utils.EtaBridgeMd5Key)
+	contentType := "application/json;charset=utf-8"
+	req.Header.Set("Content-Type", contentType)
+	req.Header.Set("Authorization", checkToken)
+	resp, e := client.Do(req)
+	if e != nil {
+		err = fmt.Errorf("http client do err: %s", e.Error())
+		return
+	}
+	defer func() {
+		_ = resp.Body.Close()
+	}()
+	b, e := ioutil.ReadAll(resp.Body)
+	if e != nil {
+		err = fmt.Errorf("resp body read err: %s", e.Error())
+		return
+	}
+	if len(b) == 0 {
+		err = fmt.Errorf("resp body is empty")
+		return
+	}
+	// 生产环境解密
+	if utils.RunMode == "release" {
+		str := string(b)
+		str = strings.Trim(str, `"`)
+		b = utils.DesBase64Decrypt([]byte(str), utils.EtaBridgeDesKey)
+	}
+
+	result := new(models.BridgeJiaYueResultMenuListData)
+	if e = json.Unmarshal(b, &result); e != nil {
+		err = fmt.Errorf("result unmarshal err: %s\nresult: %s", e.Error(), string(b))
+		return
+	}
+	if result.Code != 200 {
+		err = fmt.Errorf("result: %s", string(b))
+		return
+	}
+	indexData = result.Data
+	return
+}
+
+// SyncJiaYueNewIndex 同步嘉悦增量指标
+func SyncJiaYueNewIndex(item models.BridgeJiaYueIndexAndData, menus []models.BridgeJiaYueIndexMenuData) (err error) {
+	defer func() {
+		if err != nil {
+			b, _ := json.Marshal(item)
+			tips := fmt.Sprintf("SyncJiaYueNewIndex同步增量指标失败, Err: %s\nIndex: %s", err.Error(), string(b))
+			utils.FileLog.Info(tips)
+			go alarm_msg.SendAlarmMsg(tips, 3)
+		}
+	}()
+
+	// 校验指标来源
+	sourceId := models.EdbSourceExtendIdMap[item.SourceType]
+	if sourceId <= 0 {
+		utils.FileLog.Info(fmt.Sprintf("指标来源ID有误, 忽略同步, sourceId: %d; sourceType: %s", sourceId, item.SourceType))
+		return
+	}
+	edbSource := models.EdbSourceIdMap[sourceId]
+	if edbSource == nil {
+		utils.FileLog.Info(fmt.Sprintf("指标来源有误, sourceId: %d", sourceId))
+		return
+	}
+	sourceTableName := edbSource.TableName
+	if sourceTableName == "" {
+		utils.FileLog.Info(fmt.Sprintf("指标数据表有误, 忽略同步, sourceId: %d", sourceId))
+		return
+	}
+
+	// 指标编码
+	indexCode := ""
+	if edbSource.EdbCodeRequired == 1 {
+		indexCode = item.IndexCode
+	} else {
+		indexCode = strconv.Itoa(item.Id)
+	}
+	if indexCode == "" {
+		utils.FileLog.Info("指标编码不存在")
+		return
+	}
+
+	// 从桥接服务获取指标和数据
+	var params models.BridgeJiaYueIndexDataParams
+	params.IndexCode = indexCode
+	params.SourceExtend = item.SourceType
+	params.StartDate = utils.BASE_START_DATE
+	params.EndDate = utils.BASE_END_DATE
+	params.IndexCodeRequired = edbSource.EdbCodeRequired
+	indexData, e := GetJiaYueIndexDataFromBridge(params)
+	if e != nil {
+		err = fmt.Errorf("GetJiaYueIndexDataFromBridge err: %s", e.Error())
+		return
+	}
+
+	// 没找到数据, 频度有误的忽略掉
+	if indexData.Id <= 0 {
+		utils.FileLog.Info(fmt.Sprintf("SyncJiaYueNewIndex indexCode empty, IndexCode: %s", indexCode))
+		return
+	}
+	frequency := TransJiaYueFrequency(item.Frequency)
+	if frequency == "" {
+		utils.FileLog.Info(fmt.Sprintf("SyncJiaYueNewIndex frequency empty, Frequency: %s", item.Frequency))
+		return
+	}
+
+	// 获取指标目录对应的所有父级目录
+	parentId := 0
+	parentMenus := GetJiaYueParentMenusByMenu(indexData.MenuData, menus, 0)
+	level := len(parentMenus)
+	if level > 0 {
+		// 将父级目录按照Level倒序排序(因为是从子目录往上找的, 所以Level最大的对应为顶级目录), 再进行遍历
+		sort.Slice(parentMenus, func(i, j int) bool {
+			return parentMenus[i].Level > parentMenus[j].Level
+		})
+		for _, p := range parentMenus {
+			l := level - p.Level - 1
+			c, _, msg := models.SaveEdbClassify(p.Menu.Name, parentId, l, 0, utils.InitAdminId, utils.InitAdminName)
+			if msg != "" {
+				err = fmt.Errorf("SaveEdbClassify err: %s", msg)
+				return
+			}
+			parentId = c.ClassifyId
+		}
+	}
+	classify, _, msg := models.SaveEdbClassify(indexData.MenuData.Name, parentId, level, 0, utils.InitAdminId, utils.InitAdminName)
+	if msg != "" {
+		err = fmt.Errorf("SaveEdbClassify err: %s", msg)
+		return
+	}
+
+	// 新增指标
+	addParams := new(models.AddEdbInfoParams)
+	addParams.EdbCode = indexCode
+	addParams.EdbName = item.IndexName
+	addParams.Frequency = frequency
+	addParams.Unit = item.Unit
+	addParams.ClassifyId = classify.ClassifyId
+	addParams.Source = sourceId
+	edbInfo, e := models.EdbInfoAdd(addParams, "", 0, "")
+	if e != nil {
+		err = fmt.Errorf("EdbInfoAdd err: %s", e.Error())
+		return
+	}
+
+	// 刷新指标数据
+	e = models.RefreshEdbDataFromJiaYue(sourceId, edbInfo.EdbInfoId, sourceTableName, edbInfo.EdbCode, params.StartDate, indexData.IndexData)
+	if e != nil {
+		err = fmt.Errorf("RefreshEdbDataFromJiaYue err: %s", e.Error())
+		return
+	}
+
+	// 更新指标最大最小值
+	e, errMsg := models.UnifiedModifyEdbInfoMaxAndMinInfo(edbInfo)
+	if e != nil {
+		err = fmt.Errorf("UnifiedModifyEdbInfoMaxAndMinInfo err: %s, errMsg: %s", e.Error(), errMsg)
+		return
+	}
+
+	// 更新ES
+	go logic.UpdateEs(edbInfo.EdbInfoId)
+
+	return
+}
+
+// TransJiaYueFrequency 频度转换
+func TransJiaYueFrequency(origin string) string {
+	mapping := map[string]string{
+		"日":  "日度",
+		"周":  "周度",
+		"旬":  "旬度",
+		"半月": "旬度",
+		"月":  "月度",
+		"季":  "季度",
+		"半年": "半年度",
+		"年":  "年度",
+	}
+	return mapping[origin]
+}
+
+// GetJiaYueParentMenusByMenu 获取指定目录的父级目录
+func GetJiaYueParentMenusByMenu(menu models.BridgeJiaYueIndexMenuData, menus []models.BridgeJiaYueIndexMenuData, level int) (results []models.BridgeJiaYueIndexMenuWithLevel) {
+	results = make([]models.BridgeJiaYueIndexMenuWithLevel, 0)
+	for _, m := range menus {
+		if menu.ParentId == m.Id {
+			results = append(results, models.BridgeJiaYueIndexMenuWithLevel{
+				Level: level,
+				Menu:  m,
+			})
+			ps := GetJiaYueParentMenusByMenu(m, menus, level+1)
+			if len(ps) > 0 {
+				results = append(results, ps...)
+			}
+		}
+	}
+	return
+}

services/base_from_manual.go

@@ -0,0 +1,79 @@
+package services
+
+/*
+import (
+	"errors"
+	"eta/eta_index_lib/models"
+	"eta/eta_index_lib/utils"
+)
+
+func AddEdbDataFromManual(edbCode string) (err error) {
+	// 判断是否调用桥接服务
+	var manualDataList []*models.ManualEdbdata
+	url, _ := GetWindBridgeUrl()
+	if url != "" {
+		manualDataList, err = GetManualEdbDataFromBridgeJy(url, edbCode, "", "")
+		if err != nil {
+			return
+		}
+		// 设置指标与终端关系的缓存
+		windUrlCacheKey := utils.CACHE_WIND_URL + ":" + edbCode
+		_ = utils.Rc.SetNX(windUrlCacheKey, url, utils.GetTodayLastSecond())
+	} else {
+		var condition string
+		var pars []interface{}
+
+		if edbCode != "" {
+			condition += " AND TRADE_CODE=? "
+			pars = append(pars, edbCode)
+		}
+		manualDataList, err = models.GetEdbdataManualByCondition(condition, pars)
+		if err != nil {
+			return
+		}
+	}
+
+	err = models.AddEdbDataFromManual(edbCode, manualDataList)
+	return
+}
+
+func RefreshEdbDataFromManual(req models.RefreshEdbInfoReq, edbinfo *models.EdbInfo) (err error) {
+	var manualDataList []*models.ManualEdbdata
+	url, _ := GetManualBridgeJYUrl()
+	if url != "" && edbinfo.ServerUrl == url {
+		manualDataList, err = GetManualEdbDataFromBridgeJy(url, req.EdbCode, "", "")
+		if err != nil {
+			err = errors.New("查询桥接服务手工数据失败，Err:" + err.Error())
+			return
+		}
+	} else {
+		var condition string
+		var pars []interface{}
+
+		if req.EdbCode != "" {
+			condition += " AND TRADE_CODE=? "
+			pars = append(pars, req.EdbCode)
+		}
+
+		if req.StartDate != "" {
+			condition += " AND DT>=? "
+			pars = append(pars, req.StartDate)
+		} else {
+			condition += " AND DT != ? "
+			pars = append(pars, `0000-00-00`)
+		}
+
+		manualDataList, err = models.GetEdbdataManualByCondition(condition, pars)
+		if err != nil {
+			err = errors.New("查询手工数据失败，Err:" + err.Error())
+			return
+		}
+	}
+	err = models.RefreshEdbDataFromManual(req.EdbInfoId, req.EdbCode, req.StartDate, manualDataList)
+	if err != nil && err.Error() != utils.ErrNoRow() {
+		err = errors.New("刷新指标信息失败 service RefreshEdbDataFromManual，Err:" + err.Error())
+		return
+	}
+	return
+}
+*/

services/base_from_smm.go

@@ -1,7 +1,6 @@
 package services
 
 import (
-	"errors"
 	"eta/eta_index_lib/logic"
 	"eta/eta_index_lib/models"
 	"eta/eta_index_lib/services/alarm_msg"
@@ -11,14 +10,17 @@ import (
 	"time"
 )
 
-func SmmIndexHandle(baseFilePath, renameFilePath, indexName, indexCode, unit, frequency, source string, excelDataMap map[string]string, terminalCode string) {
-	var err error
-
-	errMsgList := make([]string, 0)
+func SmmIndexHandle(baseFilePath, renameFilePath, indexName, indexCode, unit, frequency, source string, excelDataMap map[string]string, terminalCode string) (err error, errMsg string) {
+	messages := make([]string, 0)
 	defer func() {
-		if len(errMsgList) > 0 {
-			fmt.Println(fmt.Sprint("SMM有色实际数据处理失败,err:", strings.Join(errMsgList, "\n")))
-			go alarm_msg.SendAlarmMsg(fmt.Sprint("SMM有色实际数据处理失败,err:", strings.Join(errMsgList, "\n")), 3)
+		if len(messages) > 0 {
+			errMsg = strings.Join(messages, "\n")
+			utils.FileLog.Info("SmmIndexHandle ErrMsg: %s", errMsg)
+		}
+		if err != nil {
+			tips := fmt.Sprintf("SMM有色excel数据处理失败, Err: %s, ErrMsg: %s", err.Error(), errMsg)
+			utils.FileLog.Info(tips)
+			go alarm_msg.SendAlarmMsg(tips, 3)
 		}
 	}()
 
@@ -35,26 +37,24 @@ func SmmIndexHandle(baseFilePath, renameFilePath, indexName, indexCode, unit, fr
 		utils.FileLog.Info("未刷新到指标数据:indexName:" + indexName)
 		return
 	}
-	//判断指标是否存在
+	// 判断指标是否存在
 	var isAdd int
-	item, err := indexObj.GetSmmIndexItem(indexCode)
-	if err != nil {
-		if err.Error() == utils.ErrNoRow() {
+	item, e := indexObj.GetSmmIndexItem(indexCode)
+	if e != nil {
+		if e.Error() == utils.ErrNoRow() {
 			isAdd = 1
 		} else {
 			isAdd = -1
-			fmt.Println("GetSmmIndexItem Err:" + err.Error())
+			err = fmt.Errorf("GetSmmIndexItem Err: %s", e.Error())
 			return
 		}
 	}
 	if item != nil && item.BaseFromSmmIndexId > 0 {
-		fmt.Println("item:", item)
 		isAdd = 2
 	} else {
 		isAdd = 1
 	}
 
-	fmt.Println("isAdd:", isAdd)
 	if !strings.Contains(frequency, "度") {
 		frequency = frequency + "度"
 	}
@@ -70,18 +70,18 @@ func SmmIndexHandle(baseFilePath, renameFilePath, indexName, indexCode, unit, fr
 		indexObj.BaseFileName = baseFilePath
 		indexObj.RenameFileName = renameFilePath
 		indexObj.TerminalCode = terminalCode
-		lastId, err := indexObj.AddBaseFromSmmIndex()
-		if err != nil {
-			fmt.Println("add err:" + err.Error())
+		lastId, e := indexObj.AddBaseFromSmmIndex()
+		if e != nil {
+			err = fmt.Errorf("AddBaseFromSmmIndex err: %s", e.Error())
 			return
 		}
 		indexId = lastId
 		go models.ModifySmmIndexSort(indexId)
 	} else if isAdd == 2 {
 		//获取已存在的所有数据
-		exitDataList, err := models.GetBaseFromSmmDataBySmmCode(indexCode)
-		if err != nil {
-			fmt.Println("GetIndexDataList Err:" + err.Error())
+		exitDataList, e := models.GetBaseFromSmmDataBySmmCode(indexCode)
+		if e != nil {
+			err = fmt.Errorf("GetBaseFromSmmDataBySmmCode err: %s", e.Error())
 			return
 		}
 		fmt.Println("exitDataListLen:", len(exitDataList))
@@ -103,9 +103,9 @@ func SmmIndexHandle(baseFilePath, renameFilePath, indexName, indexCode, unit, fr
 		whereParam["index_code"] = indexCode
 
 		smmIndexObj := new(models.BaseFromSmmIndex)
-		err = smmIndexObj.Update(updateParams, whereParam)
-		if err != nil {
-			err = errors.New("smm index update err:" + err.Error())
+		e = smmIndexObj.Update(updateParams, whereParam)
+		if e != nil {
+			err = fmt.Errorf("smm index update err: %s", e.Error())
 			return
 		}
 	}
@@ -114,20 +114,27 @@ func SmmIndexHandle(baseFilePath, renameFilePath, indexName, indexCode, unit, fr
 	for date, value := range excelDataMap {
 		dateTime, e := time.ParseInLocation(utils.FormatDate, date, time.Local)
 		if e != nil {
-			fmt.Println("time.ParseInLocation Err:" + e.Error())
-			return
+			messages = append(messages, fmt.Sprintf("data time parse err: %s, date: %s", e.Error(), date))
+			continue
 		}
 		if _, ok := exitDataMap[date]; !ok {
 			if !strings.Contains(value, "#N/A") {
 				var saveDataTime time.Time
 				if strings.Contains(date, "00:00:00") {
-					saveDataTime, err = time.Parse(utils.FormatDateTime, date)
+					saveDataTime, e = time.Parse(utils.FormatDateTime, date)
+					if e != nil {
+						messages = append(messages, fmt.Sprintf("save data time parse err: %s, date: %s", e.Error(), date))
+						continue
+					}
 				} else {
-					saveDataTime, err = time.Parse(utils.FormatDate, date)
+					saveDataTime, e = time.Parse(utils.FormatDate, date)
+					if e != nil {
+						messages = append(messages, fmt.Sprintf("save data time parse err: %s, date: %s", e.Error(), date))
+						continue
+					}
 				}
-				if err != nil {
-					errMsg := "saveDataTime parse err:" + err.Error() + " dataTime:" + date
-					fmt.Println(errMsg)
+				if saveDataTime.IsZero() {
+					messages = append(messages, fmt.Sprintf("save data time empty"))
 					continue
 				}
 				timestamp := saveDataTime.UnixNano() / 1e6
@@ -144,7 +151,12 @@ func SmmIndexHandle(baseFilePath, renameFilePath, indexName, indexCode, unit, fr
 			}
 		} else {
 			// 更新对应日期数据
-			if strings.Contains(value, "#N/A") {
+			if value == "" || strings.Contains(value, "#N/A") {
+				continue
+			}
+			// 若数值相同则忽略更新
+			existData := exitDataMap[date]
+			if existData != nil && existData.Value == value {
 				continue
 			}
 
@@ -157,38 +169,40 @@ func SmmIndexHandle(baseFilePath, renameFilePath, indexName, indexCode, unit, fr
 		}
 	}
 
+	//fmt.Println("addDataList len: ", len(addDataList))
 	if len(addDataList) > 0 {
-		err = models.AddBaseFromSmmData(addDataList)
-		if err != nil {
-			fmt.Println("AddBaseFromSmmData Err:" + err.Error())
-			errMsgList = append(errMsgList, "AddBaseFromSmmData err: "+err.Error())
+		e = models.AddBaseFromSmmData(addDataList)
+		if e != nil {
+			err = fmt.Errorf("AddBaseFromSmmData err: %s", e.Error())
+			return
 		}
 	}
 
 	// 已存在的日期数据更新
+	//fmt.Println("updateDataList len: ", len(updateDataList))
 	if len(updateDataList) > 0 {
-		err = models.MultiUpdateBaseFromSmmDataValue(updateDataList)
-		if err != nil {
-			fmt.Println("MultiUpdateBaseFromSmmDataValue Err:" + err.Error())
-			errMsgList = append(errMsgList, "MultiUpdateBaseFromSmmDataValue err: "+err.Error())
+		e = models.MultiUpdateBaseFromSmmDataValue(updateDataList)
+		if e != nil {
+			err = fmt.Errorf("MultiUpdateBaseFromSmmDataValue err: %s", e.Error())
+			return
 		}
 	}
 
 	itemInfo, err := models.GetSmmIndexInfoMaxAndMinInfo(indexCode)
 	if err == nil && item != nil {
-		e := models.ModifySmmIndexMaxAndMinInfo(indexCode, itemInfo)
+		e = models.ModifySmmIndexMaxAndMinInfo(indexCode, itemInfo)
 		if e != nil {
-			fmt.Println("ModifySmmIndexMaxAndMinInfo Err:" + e.Error())
-			errMsgList = append(errMsgList, "ModifySmmIndexMaxAndMinInfo err: "+e.Error())
+			err = fmt.Errorf("ModifySmmIndexMaxAndMinInfo err: %s", e.Error())
+			return
 		}
 	}
 
 	// 同步刷新ETA图库有色的指标
 	{
 		// 获取指标详情
-		edbInfo, err := models.GetEdbInfoByEdbCode(utils.DATA_SOURCE_YS, indexCode)
-		if err != nil && err.Error() != utils.ErrNoRow() {
-			errMsgList = append(errMsgList, fmt.Sprint("刷新ETA指标异常，指标编码：", indexCode, err.Error()))
+		edbInfo, e := models.GetEdbInfoByEdbCode(utils.DATA_SOURCE_YS, indexCode)
+		if e != nil && e.Error() != utils.ErrNoRow() {
+			messages = append(messages, fmt.Sprintf("刷新ETA指标异常, indexCode: %s, err: %s", indexCode, e.Error()))
 		}
 
 		// 已经加入到指标库的话，那么就去更新ETA指标库吧
@@ -196,4 +210,5 @@ func SmmIndexHandle(baseFilePath, renameFilePath, indexName, indexCode, unit, fr
 			go logic.RefreshBaseEdbInfo(edbInfo, ``)
 		}
 	}
+	return
 }

services/base_from_ths.go

@@ -159,7 +159,7 @@ type FutureGoodDataFromThsInterface struct {
 }
 
 func GetFutureGoodDataFromThs(edbCode, startDate, endDate string) (item future_good.FutureGoodDataFromThs, err error) {
-	if utils.RunMode == `release` { // 生产环境走官方http请求，测试环境走终端
+	if utils.ThsDataMethod == "" || utils.ThsDataMethod == "api" { // 生产环境走官方http请求，测试环境走终端
 		return getFutureGoodDataFromThsHttp(edbCode, startDate, endDate)
 	} else {
 		return getFutureGoodDataFromThsApp(edbCode, startDate, endDate, 0)

utils/config.go

@@ -51,6 +51,11 @@ var (
 
 	// AlarmMsgUrl 报警服务地址
 	AlarmMsgUrl string
+
+	EtaBridgeUrl       string // 桥接服务地址
+	EtaBridgeAppNameEn string // 桥接服务英文名称-鉴权用
+	EtaBridgeMd5Key    string // 桥接服务Md5密钥-鉴权用
+	EtaBridgeDesKey    string // 桥接服务Des密钥-解密数据用
 )
 
 // 第三方服务地址配置
@@ -171,6 +176,11 @@ func init() {
 
 		// AlarmMsgUrl 报警服务地址
 		AlarmMsgUrl = config["alarm_msg_url"]
+
+		EtaBridgeUrl = config["eta_bridge_url"]               // 桥接服务地址
+		EtaBridgeAppNameEn = config["eta_bridge_app_name_en"] // 桥接服务英文名称-鉴权用
+		EtaBridgeMd5Key = config["eta_bridge_md5_key"]        // 桥接服务Md5密钥-鉴权用
+		EtaBridgeDesKey = config["eta_bridge_des_key"]        // 桥接服务Des密钥-解密数据用
 	}
 
 	// 第三方服务地址配置

utils/des3.go

@@ -0,0 +1,187 @@
+// 加密工具类，用了3des和base64
+package utils
+
+import (
+	"bytes"
+	"crypto/cipher"
+	"crypto/des"
+	"encoding/base64"
+	"encoding/hex"
+	"errors"
+	"strings"
+)
+
+// des3 + base64 encrypt
+func DesBase64Encrypt(origData []byte, desKey string) []byte {
+	result, err := TripleDesEncrypt(origData, []byte(desKey))
+	if err != nil {
+		panic(any(err))
+	}
+	return []byte(base64.StdEncoding.EncodeToString(result))
+}
+
+func DesBase64Decrypt(crypted []byte, desKey string) []byte {
+	result, _ := base64.StdEncoding.DecodeString(string(crypted))
+	remain := len(result) % 8
+	if remain > 0 {
+		mod := 8 - remain
+		for i := 0; i < mod; i++ {
+			result = append(result, 0)
+		}
+	}
+	origData, err := TripleDesDecrypt(result, []byte(desKey))
+	if err != nil {
+		panic(any(err))
+	}
+	return origData
+}
+
+// 3DES加密
+func TripleDesEncrypt(origData, key []byte) ([]byte, error) {
+	block, err := des.NewTripleDESCipher(key)
+	if err != nil {
+		return nil, err
+	}
+	origData = PKCS5Padding(origData, block.BlockSize())
+	// origData = ZeroPadding(origData, block.BlockSize())
+	blockMode := cipher.NewCBCEncrypter(block, key[:8])
+	crypted := make([]byte, len(origData))
+	blockMode.CryptBlocks(crypted, origData)
+	return crypted, nil
+}
+
+// 3DES解密
+func TripleDesDecrypt(crypted, key []byte) ([]byte, error) {
+	block, err := des.NewTripleDESCipher(key)
+	if err != nil {
+		return nil, err
+	}
+	blockMode := cipher.NewCBCDecrypter(block, key[:8])
+	origData := make([]byte, len(crypted))
+	// origData := crypted
+	blockMode.CryptBlocks(origData, crypted)
+	origData = PKCS5UnPadding(origData)
+	// origData = ZeroUnPadding(origData)
+	return origData, nil
+}
+
+func ZeroPadding(ciphertext []byte, blockSize int) []byte {
+	padding := blockSize - len(ciphertext)%blockSize
+	padtext := bytes.Repeat([]byte{0}, padding)
+	return append(ciphertext, padtext...)
+}
+
+func ZeroUnPadding(origData []byte) []byte {
+	length := len(origData)
+	unpadding := int(origData[length-1])
+	return origData[:(length - unpadding)]
+}
+
+func PKCS5Padding(ciphertext []byte, blockSize int) []byte {
+	padding := blockSize - len(ciphertext)%blockSize
+	padtext := bytes.Repeat([]byte{byte(padding)}, padding)
+	return append(ciphertext, padtext...)
+}
+
+func PKCS5UnPadding(origData []byte) []byte {
+	length := len(origData)
+	// 去掉最后一个字节 unpadding 次
+	unpadding := int(origData[length-1])
+	return origData[:(length - unpadding)]
+}
+
+// DES加密
+func DesEncrypt(content string, key string) string {
+	contents := []byte(content)
+	keys := []byte(key)
+	block, err := des.NewCipher(keys)
+	if err != nil {
+		return ""
+	}
+	contents = PKCS5Padding(contents, block.BlockSize())
+	blockMode := cipher.NewCBCEncrypter(block, keys)
+	crypted := make([]byte, len(contents))
+	blockMode.CryptBlocks(crypted, contents)
+	return byteToHexString(crypted)
+}
+
+func byteToHexString(bytes []byte) string {
+	str := ""
+	for i := 0; i < len(bytes); i++ {
+		sTemp := hex.EncodeToString([]byte{bytes[i]})
+		if len(sTemp) < 2 {
+			str += string(0)
+		}
+		str += strings.ToUpper(sTemp)
+	}
+	return str
+}
+
+// DES解密
+func DesDecrypt(content string, key string) string {
+	contentBytes, err := hex.DecodeString(content)
+	if err != nil {
+		return "字符串转换16进制数组失败" + err.Error()
+	}
+	keys := []byte(key)
+	block, err := des.NewCipher(keys)
+	if err != nil {
+		return "解密失败" + err.Error()
+	}
+	blockMode := cipher.NewCBCDecrypter(block, keys)
+	origData := contentBytes
+	blockMode.CryptBlocks(origData, contentBytes)
+	origData = ZeroUnPadding(origData)
+	return string(origData)
+}
+
+// DES ECB PKCK5Padding
+func EntryptDesECB(data, key []byte) (string, error) {
+	if len(key) > 8 {
+		key = key[:8]
+	}
+	block, err := des.NewCipher(key)
+	if err != nil {
+		return "", errors.New("des.NewCipher " + err.Error())
+	}
+	bs := block.BlockSize()
+	data = PKCS5Padding(data, bs)
+	if len(data)%bs != 0 {
+		return "", errors.New("EntryptDesECB Need a multiple of the blocksize")
+	}
+	out := make([]byte, len(data))
+	dst := out
+	for len(data) > 0 {
+		block.Encrypt(dst, data[:bs])
+		data = data[bs:]
+		dst = dst[bs:]
+	}
+	return base64.StdEncoding.EncodeToString(out), nil
+}
+
+func DecryptDESECB(d string, key []byte) ([]byte, error) {
+	data, err := base64.StdEncoding.DecodeString(d)
+	if err != nil {
+		return nil, errors.New("decodebase64 " + err.Error())
+	}
+	if len(key) > 8 {
+		key = key[:8]
+	}
+	block, err := des.NewCipher(key)
+	if err != nil {
+		return nil, errors.New("des.NewCipher " + err.Error())
+	}
+	bs := block.BlockSize()
+	if len(data)%bs != 0 {
+		return nil, errors.New("DecryptDES crypto/cipher: input not full blocks")
+	}
+	out := make([]byte, len(data))
+	dst := out
+	for len(data) > 0 {
+		block.Decrypt(dst, data[:bs])
+		data = data[bs:]
+		dst = dst[bs:]
+	}
+	out = PKCS5UnPadding(out)
+	return out, nil
+}

utils/elastic.go

@@ -13,7 +13,7 @@ func initEs() {
 		elastic.SetSniff(false))
 	EsClient = client
 	if err != nil {
-		panic("ElasticSearch连接失败")
+		panic("ElasticSearch连接失败,Err:" + err.Error())
 		//go alarm_msg.SendAlarmMsg("ElasticSearch连接失败", 2)
 	}
 	return