新增指标编辑/刷新日志

controllers/base_from_calculate.go

@@ -1068,6 +1068,8 @@ func (this *CalculateController) BatchEdit() {
 		}
 	}
 
+	dataUpdateTime := time.Now().Format(utils.FormatDateTime)
+
 	var sourName, errMsg string
 	var edbInfoId int
 	var baseEdbInfoModel models.BaseEdbInfoInterface
@@ -1266,11 +1268,14 @@ func (this *CalculateController) BatchEdit() {
 		err, errMsg = baseEdbInfoModel.Edit(editParams)
 	}
 	if err != nil {
+		// todo 添加更新失败记录
 		br.Msg = "生成" + sourName + "失败"
 		if errMsg != `` {
 			br.Msg = errMsg
 		}
 		br.Msg = "生成" + sourName + "失败 Err:" + err.Error()
+		// 添加指标刷新失败日志
+		_ = services.AddEdbInfoUpdateLog(edbInfo.EdbInfoId, 2, br.Msg)
 		return
 	}
 	if edbInfo == nil {
@@ -1302,13 +1307,17 @@ func (this *CalculateController) BatchEdit() {
 	}
 
 	// 更新指标最大最小值
-	err, errMsg = models.UnifiedModifyEdbInfoMaxAndMinInfo(edbInfo)
+	err, errMsg = models.UnifiedModifyEdbInfoMaxAndMinInfoTmp(edbInfo, dataUpdateTime)
 	if err != nil {
 		br.Msg = errMsg
 		br.ErrMsg = err.Error()
 		return
 	}
 
+	// todo 新增更新记录
+	// 添加指标刷新成功日志
+	_ = services.AddEdbInfoUpdateLog(edbInfo.EdbInfoId, 1, "")
+
 	resp := models.AddEdbInfoResp{
 		EdbInfoId:  edbInfo.EdbInfoId,
 		UniqueCode: edbInfo.UniqueCode,

controllers/base_from_pb.go

@@ -121,19 +121,24 @@ func (this *PbController) Refresh() {
 		br.ErrMsg = "获取指标信息失败 GetEdbDataFromPb，Err:" + err.Error()
 		return
 	}
+	dataUpdateTime := time.Now().Format(utils.FormatDateTime)
 	err = models.RefreshEdbDataFromPb(req.EdbInfoId, req.EdbCode, req.StartDate, dataItem)
 	if err != nil && err.Error() != utils.ErrNoRow() {
+		// 添加指标刷新失败日志
+		_ = services.AddEdbInfoUpdateLog(edbInfo.EdbInfoId, 2, err.Error())
 		br.Msg = "刷新指标信息失败！"
 		br.ErrMsg = "刷新指标信息失败 RefreshEdbDataFromPb，Err:" + err.Error()
 		return
 	}
 	// 更新指标最大最小值
-	err, errMsg := models.UnifiedModifyEdbInfoMaxAndMinInfo(edbInfo)
+	err, errMsg := models.UnifiedModifyEdbInfoMaxAndMinInfoTmp(edbInfo, dataUpdateTime)
 	if err != nil {
 		br.Msg = errMsg
 		br.ErrMsg = err.Error()
 		return
 	}
+	// 添加指标刷新成功日志
+	_ = services.AddEdbInfoUpdateLog(edbInfo.EdbInfoId, 1, "")
 
 	// 更新ES
 	go logic.UpdateEs(edbInfo.EdbInfoId)

controllers/base_from_wind.go

@@ -209,20 +209,25 @@ func (this *WindController) Refresh() {
 		br.ErrMsg = "获取指标信息失败 GetEdbDataFromWind，Err:" + err.Error()
 		return
 	}
+	dataUpdateTime := time.Now().Format(utils.FormatDateTime)
 	err = models.RefreshEdbDataFromWind(req.EdbInfoId, req.EdbCode, req.StartDate, dataItem)
 	if err != nil && err.Error() != utils.ErrNoRow() {
+		// 添加指标刷新失败日志
+		_ = services.AddEdbInfoUpdateLog(edbInfo.EdbInfoId, 2, err.Error())
 		br.Msg = "刷新指标信息失败！"
 		br.ErrMsg = "刷新指标信息失败 RefreshEdbDataFromWind，Err:" + err.Error()
 		return
 	}
 	// 更新指标最大最小值
-	err, errMsg := models.UnifiedModifyEdbInfoMaxAndMinInfo(edbInfo)
+	err, errMsg := models.UnifiedModifyEdbInfoMaxAndMinInfoTmp(edbInfo, dataUpdateTime)
 	if err != nil {
 		br.Msg = errMsg
 		br.ErrMsg = err.Error()
 		return
 	}
 
+	// 添加指标刷新成功日志
+	_ = services.AddEdbInfoUpdateLog(edbInfo.EdbInfoId, 1, "")
 	// 更新ES
 	go logic.UpdateEs(edbInfo.EdbInfoId)
 

models/data_stat/edb_info_update_log.go

@@ -0,0 +1,41 @@
+package data_stat
+
+import (
+	"github.com/beego/beego/v2/client/orm"
+	"time"
+)
+
+// EdbInfoUpdateLog 指标更新/刷新日志列表
+type EdbInfoUpdateLog struct {
+	Id                 uint64    `orm:"column(id);pk"`
+	EdbInfoId          int       `description:"指标ID"`
+	SourceName         string    `description:"来源名称"`
+	Source             int       `description:"来源id"`
+	EdbCode            string    `description:"指标编码"`
+	EdbName            string    `description:"指标名称"`
+	EdbNameSource      string    `description:"指标名称来源"`
+	Frequency          string    `description:"频率"`
+	Unit               string    `description:"单位"`
+	StartDate          string    `description:"起始日期"`
+	EndDate            string    `description:"终止日期"`
+	SysUserId          int       `description:"创建人ID"`
+	SysUserRealName    string    `description:"创建人姓名"`
+	UniqueCode         string    `description:"指标唯一编码"`
+	EdbCreateTime      time.Time `description:"指标创建时间"`
+	EdbModifyTime      time.Time `description:"指标修改时间"`
+	CreateTime         time.Time `description:"创建时间"`
+	UpdateTime         time.Time `description:"刷新/修改时间"`
+	LatestDate         string    `description:"数据最新日期"`
+	LatestValue        float64   `description:"数据最新值"`
+	TerminalCode       string    `description:"终端编码，用于配置在机器上"`
+	DataUpdateTime     string    `description:"最近一次数据发生变化的时间"`
+	ErDataUpdateDate   string    `description:"本次更新，数据发生变化的最早日期"`
+	UpdateResult       int       `description:"本次刷新结果：1成功，2失败"`
+	UpdateFailedReason string    `description:"数据未正常刷新原因"`
+}
+
+func AddEdbUpdateLog(item *EdbInfoUpdateLog) (lastId int64, err error) {
+	o := orm.NewOrm()
+	lastId, err = o.Insert(item)
+	return
+}

models/data_stat/edb_info_update_stat.go

@@ -0,0 +1,45 @@
+package data_stat
+
+import (
+	"github.com/beego/beego/v2/client/orm"
+	"time"
+)
+
+// EdbInfoUpdateStat 指标更新/刷新 汇总表（数据源明细表）
+type EdbInfoUpdateStat struct {
+	Id                 uint64    `orm:"column(id);pk"`
+	EdbInfoId          int       `description:"指标ID"`
+	SourceName         string    `description:"来源名称"`
+	Source             int       `description:"来源id"`
+	EdbCode            string    `description:"指标编码"`
+	EdbName            string    `description:"指标名称"`
+	EdbNameSource      string    `description:"指标名称来源"`
+	Frequency          string    `description:"频率"`
+	Unit               string    `description:"单位"`
+	StartDate          string    `description:"起始日期"`
+	EndDate            string    `description:"终止日期"`
+	SysUserId          int       `description:"创建人ID"`
+	SysUserRealName    string    `description:"创建人姓名"`
+	UniqueCode         string    `description:"指标唯一编码"`
+	EdbCreateTime      time.Time `description:"指标创建时间"`
+	EdbModifyTime      time.Time `description:"指标修改时间"`
+	CreateTime         time.Time
+	LatestDate         string    `description:"数据最新日期"`
+	LatestValue        float64   `description:"数据最新值"`
+	TerminalCode       string    `description:"终端编码，用于配置在机器上"`
+	DataUpdateTime     time.Time `description:"最近一次数据发生变化的时间"`
+	ErDataUpdateDate   time.Time `description:"本次更新，数据发生变化的最早日期"`
+	UpdateResult       int32     `description:"本次刷新结果：1成功，2失败"`
+	UpdateFailedReason string    `description:"数据未正常刷新原因"`
+	ModifyTime         string    `description:"修改时间"`
+	UpdateTime         string    `description:"刷新/修改时间"`
+	IsAdd              int8      `description:"是否是当日新增的指标：1是，2否"`
+	NeedRefresh        int8      `description:"今日是否需要定时刷新，1需要，0不需要"` //todo 确认该字段
+	HasRefresh         int8      `description:"今日是否已刷新，1是，0否"`
+}
+
+func AddEdbUpdateStat(item *EdbInfoUpdateStat) (lastId int64, err error) {
+	o := orm.NewOrm()
+	lastId, err = o.Insert(item)
+	return
+}

models/db.go

@@ -1,6 +1,7 @@
 package models
 
 import (
+	"eta/eta_index_lib/models/data_stat"
 	"eta/eta_index_lib/models/future_good"
 	"eta/eta_index_lib/models/supply_analysis"
 	"eta/eta_index_lib/utils"
@@ -64,6 +65,9 @@ func init() {
 
 	// 基础指标表
 	initBaseIndex()
+
+	// initDataStat 数据源统计管理相关表
+	initDataStat()
 }
 
 // initFutureGood 注册期货数据 数据表
@@ -103,3 +107,10 @@ func initBaseIndex() {
 		new(BaseFromSmmClassify),
 	)
 }
+
+// initDataStat 数据源统计管理相关表
+func initDataStat() {
+	orm.RegisterModel(
+		new(data_stat.EdbInfoUpdateLog), // 指标更新/刷新日志列表
+	)
+}

models/edb_info.go

@@ -48,6 +48,8 @@ type EdbInfo struct {
 	DataDateType     string  `orm:"column(data_date_type);size(255);null;default(交易日)"`
 	ManualSave       int     `description:"是否有手动保存过上下限: 0-否; 1-是"`
 	TerminalCode     string  `description:"终端编码，用于配置在机器上"`
+	DataUpdateTime   string  `description:"最近一次数据发生变化的时间"`
+	ErDataUpdateDate string  `description:"本次更新，数据发生变化的最早日期"`
 }
 
 func (e *EdbInfo) Add() (err error) {
@@ -231,6 +233,21 @@ func GetEdbInfoMaxAndMinInfo(source int, edbCode string) (item *EdbInfoMaxAndMin
 	return
 }
 
+type EdbIndoDataUpdateTime struct {
+	MinDate string `description:"本次更新，数据发生变化的最早日期"`
+}
+
+// GetEdbDataUpdateTimeByModify 获取指标的刷新时间获取最早的一条记录
+func GetEdbDataUpdateTimeByModify(source int, edbCode string, dataUpdateTime string) (item *EdbIndoDataUpdateTime, err error) {
+	o := orm.NewOrm()
+	sql := ``
+	tableName := GetEdbDataTableName(source)
+	sql = ` SELECT MIN(data_time) AS min_date FROM %s WHERE edb_code=? and modify_time>=?`
+	sql = fmt.Sprintf(sql, tableName)
+	err = o.Raw(sql, edbCode, dataUpdateTime).QueryRow(&item)
+	return
+}
+
 // ModifyEdbInfoMaxAndMinInfo 修改指标的最新数据信息
 func ModifyEdbInfoMaxAndMinInfo(edbInfoId int, item *EdbInfoMaxAndMinInfo) (err error) {
 	o := orm.NewOrm()
@@ -247,6 +264,14 @@ func ModifyEdbTimeAndLastInfo(edbInfoId int, item *EdbInfoMaxAndMinInfo) (err er
 	return
 }
 
+// ModifyEdbDataUpdateTime 修改指标刷新，本次数据刷新的最早日期
+func ModifyEdbDataUpdateTime(edbInfoId int, dataUpdateTime, erDataUpdateDate string) (err error) {
+	o := orm.NewOrm()
+	sql := `UPDATE edb_info SET data_update_time = ?, er_data_update_date = ?, modify_time = NOW() WHERE edb_info_id = ?`
+	_, err = o.Raw(sql, dataUpdateTime, erDataUpdateDate, edbInfoId).Exec()
+	return
+}
+
 // GetEdbDataCount 获取edb指标数据的数量； order:1升序，其余值为降序
 func GetEdbDataCount(condition string, pars []interface{}, source int) (count int, err error) {
 	o := orm.NewOrm()
@@ -413,6 +438,78 @@ func UnifiedModifyEdbInfoMaxAndMinInfo(edbInfo *EdbInfo) (err error, errMsg stri
 			}
 		}
 	}
+	var dataUpdateTime string
+	// 获取本次刷新，指标数据更新的最早日期
+	erDataUpdateTime, tErr := GetEdbDataUpdateTimeByModify(edbInfo.Source, edbInfo.EdbCode, dataUpdateTime)
+	if tErr != nil {
+		if tErr.Error() != utils.ErrNoRow() { //本次刷新指标数据无变化
+			errMsg = "刷新指标失败！"
+			err = errors.New("查询指标刷新，数据更新的最早日期失败，err:" + tErr.Error())
+			return
+		}
+	} else if erDataUpdateTime.MinDate != "" {
+		err = ModifyEdbDataUpdateTime(edbInfo.EdbInfoId, dataUpdateTime, erDataUpdateTime.MinDate)
+		if err != nil {
+			errMsg = "刷新指标失败！"
+			err = errors.New("修改指标数据更新的最早日期失败，err:" + tErr.Error())
+			return
+		}
+	}
+	// 修改关联的预测指标
+	go ModifyPredictEdbInfoMaxAndMinInfoBySourceEdbInfoId(edbInfo.EdbInfoId, maxAndMinItem)
+	return
+}
+
+// UnifiedModifyEdbInfoMaxAndMinInfo 统一修改指标的最大最小值
+func UnifiedModifyEdbInfoMaxAndMinInfoTmp(edbInfo *EdbInfo, dataUpdateTime string) (err error, errMsg string) {
+	// 修改最大最小值
+	maxAndMinItem, err := GetEdbInfoMaxAndMinInfo(edbInfo.Source, edbInfo.EdbCode)
+	if err != nil {
+		if err.Error() == utils.ErrNoRow() {
+			err = nil
+			return
+		}
+
+		errMsg = "刷新指标失败！"
+		err = errors.New("获取指标最大最小值失败，err:" + err.Error())
+		return
+	}
+
+	if maxAndMinItem != nil {
+		// ETA1.0.3改-如果指标有手动保存过, 那么就不更新指标的最大最小值
+		if edbInfo.ManualSave == 1 {
+			err = ModifyEdbTimeAndLastInfo(edbInfo.EdbInfoId, maxAndMinItem)
+			if err != nil {
+				errMsg = "刷新指标失败！"
+				err = errors.New("修改指标最新日期和数据失败，err:" + err.Error())
+				return
+			}
+		} else {
+			err = ModifyEdbInfoMaxAndMinInfo(edbInfo.EdbInfoId, maxAndMinItem)
+			if err != nil {
+				errMsg = "刷新指标失败！"
+				err = errors.New("修改指标最大最小值失败，err:" + err.Error())
+				return
+			}
+		}
+	}
+
+	// 获取本次刷新，指标数据更新的最早日期
+	erDataUpdateTime, tErr := GetEdbDataUpdateTimeByModify(edbInfo.Source, edbInfo.EdbCode, dataUpdateTime)
+	if tErr != nil {
+		if tErr.Error() != utils.ErrNoRow() { //本次刷新指标数据无变化
+			errMsg = "刷新指标失败！"
+			err = errors.New("查询指标刷新，数据更新的最早日期失败，err:" + tErr.Error())
+			return
+		}
+	} else if erDataUpdateTime.MinDate != "" {
+		err = ModifyEdbDataUpdateTime(edbInfo.EdbInfoId, dataUpdateTime, erDataUpdateTime.MinDate)
+		if err != nil {
+			errMsg = "刷新指标失败！"
+			err = errors.New("修改指标数据更新的最早日期失败，err:" + tErr.Error())
+			return
+		}
+	}
 	// 修改关联的预测指标
 	go ModifyPredictEdbInfoMaxAndMinInfoBySourceEdbInfoId(edbInfo.EdbInfoId, maxAndMinItem)
 	return

models/edb_terminal.go

@@ -21,7 +21,7 @@ type EdbTerminal struct {
 
 // GetEdbTerminalListBySource 根据指标来源类型获取所有的终端列表
 func GetEdbTerminalListBySource(source int) (items []*EdbTerminal, err error) {
-	o := orm.NewOrmUsingDB("data")
+	o := orm.NewOrm()
 	sql := ` SELECT *  FROM edb_terminal WHERE source = ? and status=1 ORDER BY terminal_id ASC `
 	_, err = o.Raw(sql, source).QueryRows(&items)
 	return
@@ -29,7 +29,7 @@ func GetEdbTerminalListBySource(source int) (items []*EdbTerminal, err error) {
 
 // GetEdbTerminalByCode 根据终端编码获取终端信息
 func GetEdbTerminalByCode(terminalCode string) (item *EdbTerminal, err error) {
-	o := orm.NewOrmUsingDB("data")
+	o := orm.NewOrm()
 	sql := ` SELECT *  FROM edb_terminal WHERE terminal_code = ? `
 	err = o.Raw(sql, terminalCode).QueryRow(&item)
 	return
@@ -37,7 +37,7 @@ func GetEdbTerminalByCode(terminalCode string) (item *EdbTerminal, err error) {
 
 // GetEdbCountGroupByTerminal 获取终端code分组总数
 func GetEdbCountGroupByTerminal(source int) (list []ServerUrlCountGroup, err error) {
-	o := orm.NewOrmUsingDB("data")
+	o := orm.NewOrm()
 	sql := `select server_url,count(1) total from edb_info where source = ? AND no_update=0 AND terminal_code != "" group by terminal_code; `
 	_, err = o.Raw(sql, source).QueryRows(&list)
 	return

services/edb_info_stat.go

@@ -0,0 +1,52 @@
+package services
+
+import (
+	"eta/eta_index_lib/models"
+	"eta/eta_index_lib/models/data_stat"
+	"fmt"
+	"time"
+)
+
+// AddEdbInfoUpdateLog 添加指标编辑/刷新日志
+func AddEdbInfoUpdateLog(edbInfoId int, updateResult int, updateFailedReason string) (err error) {
+	var edbInfo *models.EdbInfo
+	if edbInfoId > 0 {
+		// 获取指标详情
+		edbInfo, err = models.GetEdbInfoById(edbInfoId)
+		if err != nil {
+			err = fmt.Errorf("指标不存在")
+			return
+		}
+		log := new(data_stat.EdbInfoUpdateLog)
+		log.EdbInfoId = edbInfo.EdbInfoId
+		log.SourceName = edbInfo.SourceName
+		log.Source = edbInfo.Source
+		log.EdbCode = edbInfo.EdbCode
+		log.EdbName = edbInfo.EdbName
+		log.EdbNameSource = edbInfo.EdbNameSource
+		log.Frequency = edbInfo.Frequency
+		log.Unit = edbInfo.Unit
+		log.StartDate = edbInfo.StartDate
+		log.EndDate = edbInfo.EndDate
+		log.SysUserId = edbInfo.SysUserId
+		log.SysUserRealName = edbInfo.SysUserRealName
+		log.UniqueCode = edbInfo.UniqueCode
+		log.EdbCreateTime = edbInfo.CreateTime
+		log.EdbModifyTime = edbInfo.ModifyTime
+		log.CreateTime = time.Now()
+		log.UpdateTime = time.Now()
+		log.LatestDate = edbInfo.LatestDate
+		log.LatestValue = edbInfo.LatestValue
+		log.TerminalCode = edbInfo.TerminalCode
+		log.UpdateResult = updateResult
+		log.UpdateFailedReason = updateFailedReason
+		log.DataUpdateTime = edbInfo.DataUpdateTime
+		log.ErDataUpdateDate = edbInfo.ErDataUpdateDate
+		_, err = data_stat.AddEdbUpdateLog(log)
+		if err != nil {
+			err = fmt.Errorf("新增指标更新日志失败，Err: %s", err)
+			return
+		}
+	}
+	return
+}