feat:binlog日志

go.mod

@@ -27,6 +27,7 @@ require (
 	github.com/garyburd/redigo v1.6.3 // indirect
 	github.com/go-redis/redis/v8 v8.11.5 // indirect
 	github.com/golang/protobuf v1.5.2 // indirect
+	github.com/google/uuid v1.6.0 // indirect
 	github.com/hashicorp/golang-lru v0.5.4 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect

go.sum

@@ -167,6 +167,8 @@ github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm4
 github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
+github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
 github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg=
 github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=

models/data_manage/base_from_mysteel_chemical_index.go

@@ -0,0 +1,49 @@
+package data_manage
+
+import "github.com/beego/beego/v2/client/orm"
+
+type BaseFromMysteelChemicalIndexItem struct {
+	BaseFromMysteelChemicalIndexId    int32   `json:"base_from_mysteel_chemical_index_id"`
+	BaseFromMysteelChemicalClassifyId int32   `json:"base_from_mysteel_chemical_classify_id"` // 钢联化工指标分类id
+	IndexCode                         string  `json:"index_code"`                             // 指标编码
+	IndexName                         string  `json:"index_name"`                             // 指标名称
+	Unit                              string  `json:"unit"`                                   // 单位
+	Source                            string  `json:"source"`                                 // 数据来源
+	Frequency                         string  `json:"frequency"`                              // 频度
+	StartDate                         string  `json:"start_date"`                             // 开始日期
+	EndDate                           string  `json:"end_date"`                               // 结束日期
+	Describe                          string  `json:"describe"`                               // 指标描述
+	UpdateWeek                        string  `json:"update_week"`                            // 更新周期
+	UpdateTime                        string  `json:"update_time"`                            // 更新时间，多个时间点用英文,隔开
+	UpdateTime2                       string  `json:"update_time2"`                           // 更新时间2
+	SysUserId                         int32   `json:"sys_user_id"`                            // 创建人id
+	SysUserRealName                   string  `json:"sys_user_real_name"`                     // 创建人姓名
+	CreateTime                        string  `json:"create_time"`                            // 创建时间
+	ModifyTime                        string  `json:"modify_time"`                            // 修改时间
+	FilePath                          string  `json:"file_path"`                              // 文件存储路径
+	Sort                              int32   `json:"sort"`                                   // 排序
+	MergeFilePath                     string  `json:"merge_file_path"`
+	FileIndex                         int32   `json:"file_index"`
+	MergeUpdateWeek                   string  `json:"merge_update_week"`    // 合并文件的更新周
+	MergeFilePathWeek                 string  `json:"merge_file_path_week"` // 更新文件
+	UpdateDate                        string  `json:"update_date"`          // 更新日期
+	IsRefresh                         int32   `json:"is_refresh"`           // 0:不需要及时刷新,1:需要及时刷新
+	IsStop                            int32   `json:"is_stop"`              // 是否停更:1:停更,0:未停更
+	TerminalCode                      string  `json:"terminal_code"`        // 所属终端编码
+	EndValue                          float64 `json:"end_value"`            // 指标的最新值
+}
+
+// GetBaseFromMysteelChemicalIndexItemByCode
+// @Description: 根据指标编码获取钢联指标详情
+// @author: Roc
+// @datetime 2024-03-11 16:42:56
+// @param edbCode string
+// @return item *BaseFromMysteelChemicalIndexItem
+// @return err error
+func GetBaseFromMysteelChemicalIndexItemByCode(edbCode string) (item *BaseFromMysteelChemicalIndexItem, err error) {
+	o := orm.NewOrmUsingDB("data")
+	sql := ` SELECT * FROM base_from_mysteel_chemical_index WHERE index_code = ?`
+	err = o.Raw(sql, edbCode).QueryRow(&item)
+
+	return
+}

models/data_manage/base_from_smm_index.go

@@ -0,0 +1,50 @@
+package data_manage
+
+import "github.com/beego/beego/v2/client/orm"
+
+type BaseFromSmmIndexItem struct {
+	BaseFromSmmIndexId int32   `json:"base_from_smm_index_id"`
+	ClassifyId         int32   `json:"classify_id"` // Smm原始数据指标分类id
+	Interface          string  `json:"interface"`
+	Name               string  `json:"name"`
+	IndexCode          string  `json:"index_code"`      // 指标编码
+	IndexName          string  `json:"index_name"`      // 指标名称
+	Type1              string  `json:"type_1"`          // 类型1
+	Type2              string  `json:"type_2"`          // 类型2
+	Type3              string  `json:"type_3"`          // 类型3
+	Frequency          string  `json:"frequency"`       // 频度
+	Unit               string  `json:"unit"`            // 单位
+	ApiStartTime       string  `json:"api_start_time"`  // 接口开始日期
+	ApiUpdateTime      string  `json:"api_update_time"` // 接口更新日期
+	StartTime          string  `json:"start_time"`      // 开始日期
+	FinishTime         string  `json:"finish_time"`     // 完成日期
+	CreateTime         string  `json:"create_time"`
+	ModifyTime         string  `json:"modify_time"`
+	StartDate          string  `json:"start_date"`
+	EndDate            string  `json:"end_date"`
+	IsGet              int32   `json:"is_get"`           // 0:未获取数据,1:已获取数据
+	Sort               int32   `json:"sort"`             // 排序
+	BaseFileName       string  `json:"base_file_name"`   // 文件目录
+	RenameFileName     string  `json:"rename_file_name"` // 重命名文件目录
+	TerminalCode       string  `json:"terminal_code"`    // 所属终端编码
+	DataState          string  `json:"data_state"`       // 指标数据状态 normal/空白-正常更新 ceased-已停更irregular-更新依赖外部会有不定期延迟
+	ReleaseTime        string  `json:"release_time"`     // 预计发布时间
+	IsStop             int32   `json:"is_stop"`          // 是否停更:1:停更,0:未停更
+	EndValue           float64 `json:"end_value"`        // 指标的最新值
+}
+
+// GetBaseFromSmmIndexItemItemByCode
+// @Description: 根据指标编码获取指标信息
+// @author: Roc
+// @datetime 2024-03-11 16:41:02
+// @param source int
+// @param edbCode string
+// @return item *BaseFromSmmIndexItem
+// @return err error
+func GetBaseFromSmmIndexItemItemByCode(edbCode string) (item *BaseFromSmmIndexItem, err error) {
+	o := orm.NewOrmUsingDB("data")
+	sql := ` SELECT * FROM base_from_smm_index WHERE index_code=? `
+	err = o.Raw(sql, edbCode).QueryRow(&item)
+
+	return
+}

models/data_manage/edb_classify.go

@@ -6,21 +6,21 @@ import (
 )
 
 type EdbClassify struct {
-	ClassifyID         int64     `orm:"column(classify_id);pk"`
-	ClassifyType       uint32    `orm:"column:classify_type;default:0;comment:'分类类型，0：普通指标分类，1：预测指标分类'"`
-	ClassifyName       string    `orm:"column:classify_name;not null;default:'';comment:'分类名称'"`
-	ParentID           int64     `orm:"column:parent_id;not null;default:0;comment:'父级id'"`
-	HasData            int32     `orm:"column:has_data;default:0;comment:'是否存在指标数据:1有，2：无'"`
-	CreateTime         time.Time `orm:"column:create_time"`
-	ModifyTime         time.Time `orm:"column:modify_time"`
-	SysUserID          int64     `orm:"column:sys_user_id;comment:'创建人id'"`
-	SysUserRealName    string    `orm:"column:sys_user_real_name;size:50;comment:'创建人姓名'"`
-	Level              int64     `orm:"column:level;default:0;comment:'层级'"`
-	UniqueCode         string    `orm:"column:unique_code;size:50"`
-	Sort               uint32    `orm:"column:sort;unsigned;default:10;comment:'排序字段，越小越靠前，默认值：10'"`
-	RootID             int64     `orm:"column:root_id;not null;default:0;comment:'顶级ID'"`
-	LastModifyUserID   uint32    `orm:"column:last_modify_user_id;unsigned;not null;default:0;comment:'最后修改人ID'"`
-	LastModifyUserName string    `orm:"column:last_modify_user_real_name;size:64;not null;default:'';comment:'最后修改人姓名'"`
+	ClassifyID         int64 `orm:"column(classify_id);pk"`
+	ClassifyType       uint32
+	ClassifyName       string
+	ParentID           int64
+	HasData            int32
+	CreateTime         time.Time
+	ModifyTime         time.Time
+	SysUserID          int64
+	SysUserRealName    string
+	Level              int64
+	UniqueCode         string
+	Sort               uint32
+	RootID             int64
+	LastModifyUserID   uint32
+	LastModifyUserName string
 }
 
 // GetAllEdbClassifyListByCondition

models/data_manage/edb_data_base.go

@@ -2,40 +2,25 @@ package data_manage
 
 import (
 	"eta/eta_task/utils"
-	"fmt"
-	"github.com/beego/beego/v2/client/orm"
 	"time"
 )
 
-var (
-	EdbDataTableNameMap     map[int]string        // 指标来源对应数据表名
-	EdbDataRefreshMethodMap map[int]string        // 指标来源对应的刷新指标方法
-	EdbTableNameSourceMap   map[string]*EdbSource // 数据表名对应的指标来源
-)
-
-// InitEdbSourceVar 初始化时加载指标来源对应信息, 避免循环中查库, 注意edb_source表修改table_name的话需要重启服务
-func InitEdbSourceVar() {
-	EdbDataTableNameMap = make(map[int]string)
-	EdbDataRefreshMethodMap = make(map[int]string)
-	EdbTableNameSourceMap = make(map[string]*EdbSource)
-	sources, e := GetEdbSourceItemsByCondition(``, make([]interface{}, 0), []string{}, "")
-	if e != nil {
-		utils.FileLog.Info("init source table err: %s", e.Error())
-		return
-	}
-	for _, v := range sources {
-		EdbDataTableNameMap[v.EdbSourceId] = v.TableName
-		EdbDataRefreshMethodMap[v.EdbSourceId] = v.EdbRefreshMethod
-		EdbTableNameSourceMap[v.TableName] = v
-	}
-}
-
-func GetEdbDataTableName(source int) (tableName string) {
+// GetEdbDataTableName 指标数据->存储表
+func GetEdbDataTableName(source, subSource int) (tableName string) {
 	switch source {
 	case utils.DATA_SOURCE_THS:
 		tableName = "edb_data_ths"
+		if subSource == utils.DATA_SUB_SOURCE_DATE {
+			tableName = "edb_data_ths_ds"
+		} else {
+			tableName = "edb_data_ths"
+		}
 	case utils.DATA_SOURCE_WIND:
-		tableName = "edb_data_wind"
+		if subSource == utils.DATA_SUB_SOURCE_DATE {
+			tableName = "edb_data_wind_wsd"
+		} else {
+			tableName = "edb_data_wind"
+		}
 	case utils.DATA_SOURCE_PB, utils.DATA_SOURCE_PB_FINANCE:
 		tableName = "edb_data_pb"
 	case utils.DATA_SOURCE_CALCULATE:
@@ -84,6 +69,8 @@ func GetEdbDataTableName(source int) (tableName string) {
 		tableName = "edb_data_lt"
 	case utils.DATA_SOURCE_COAL:
 		tableName = "edb_data_coal"
+	case utils.DATA_SOURCE_PYTHON:
+		tableName = "edb_data_python"
 	case utils.DATA_SOURCE_GOOGLE_TRAVEL:
 		tableName = "edb_data_google_travel"
 	case utils.DATA_SOURCE_PREDICT_CALCULATE:
@@ -98,6 +85,16 @@ func GetEdbDataTableName(source int) (tableName string) {
 		tableName = "edb_data_calculate_cjjx"
 	case utils.DATA_SOURCE_EIA_STEO:
 		tableName = "edb_data_eia_steo"
+	case utils.DATA_SOURCE_CALCULATE_NHCC:
+		tableName = "edb_data_calculate_nhcc"
+	case utils.DATA_SOURCE_COM_TRADE:
+		tableName = "edb_data_com_trade"
+	case utils.DATA_SOURCE_PREDICT_CALCULATE_NSZYDPJJS:
+		tableName = "edb_data_predict_calculate_nszydpjjs"
+	case utils.DATA_SOURCE_CALCULATE_ADJUST:
+		tableName = "edb_data_calculate_adjust"
+	case utils.DATA_SOURCE_SCI:
+		tableName = "edb_data_sci"
 	case utils.DATA_SOURCE_PREDICT_CALCULATE_LJZZY:
 		tableName = "edb_data_predict_calculate_ljzzy"
 	case utils.DATA_SOURCE_PREDICT_CALCULATE_TIME_SHIFT:
@@ -116,9 +113,61 @@ func GetEdbDataTableName(source int) (tableName string) {
 		tableName = "edb_data_predict_calculate_hcz"
 	case utils.DATA_SOURCE_PREDICT_CALCULATE_BP:
 		tableName = "edb_data_predict_calculate_bp"
+	case utils.DATA_SOURCE_CALCULATE_JP:
+		tableName = "edb_data_calculate_jp"
+	case utils.DATA_SOURCE_CALCULATE_NH:
+		tableName = "edb_data_calculate_nh"
+	case utils.DATA_SOURCE_CALCULATE_KSZS:
+		tableName = "edb_data_calculate_kszs"
+	case utils.DATA_SOURCE_PREDICT_CALCULATE_JP:
+		tableName = "edb_data_predict_calculate_jp"
+	case utils.DATA_SOURCE_PREDICT_CALCULATE_NH:
+		tableName = "edb_data_predict_calculate_nh"
+	case utils.DATA_SOURCE_PREDICT_CALCULATE_KSZS:
+		tableName = "edb_data_predict_calculate_kszs"
+	case utils.DATA_SOURCE_BAIINFO:
+		tableName = "edb_data_baiinfo"
+	case utils.DATA_SOURCE_STOCK_PLANT:
+		tableName = "edb_data_stock_plant"
+	case utils.DATA_SOURCE_CALCULATE_CORRELATION:
+		tableName = "edb_data_calculate_correlation"
+	case utils.DATA_SOURCE_NATIONAL_STATISTICS:
+		tableName = "edb_data_national_statistics"
+	case utils.DATA_SOURCE_CALCULATE_LJZZJ: //累计值转季 -> 61
+		tableName = "edb_data_calculate_ljzzj"
+	case utils.DATA_SOURCE_CALCULATE_LJZ: //累计值 -> 62
+		tableName = "edb_data_calculate_ljz"
+	case utils.DATA_SOURCE_CALCULATE_LJZNCZJ: //累计值(年初至今） -> 63
+		tableName = "edb_data_calculate_ljznczj"
+	case utils.DATA_SOURCE_PREDICT_CALCULATE_LJZZJ: // 预测指标 - 累计值 -> 65
+		tableName = "edb_data_predict_calculate_ljzzj"
+	case utils.DATA_SOURCE_PREDICT_CALCULATE_LJZ: //预测指标 - 累计值转季->64
+		tableName = "edb_data_predict_calculate_ljz"
+	case utils.DATA_SOURCE_PREDICT_CALCULATE_LJZNCZJ: //预测指标 - 累计值(年初至今） -> 66
+		tableName = "edb_data_predict_calculate_ljznczj"
+	case utils.DATA_SOURCE_CALCULATE_STANDARD_DEVIATION: //标准差->67
+		tableName = "edb_data_calculate_standard_deviation"
+	case utils.DATA_SOURCE_CALCULATE_PERCENTILE: //百分位->68
+		tableName = "edb_data_calculate_percentile"
+	case utils.DATA_SOURCE_PREDICT_CALCULATE_STANDARD_DEVIATION: //预测标准差->69
+		tableName = "edb_data_predict_ccalculate_standard_deviation"
+	case utils.DATA_SOURCE_PREDICT_CALCULATE_PERCENTILE: //预测百分位->70
+		tableName = "edb_data_predict_ccalculate_percentile"
+	case utils.DATA_SOURCE_FUBAO:
+		tableName = "edb_data_fubao"
+	case utils.DATA_SOURCE_CALCULATE_ZSXY:
+		tableName = "edb_data_calculate_zsxy" // 指数修匀->72
+	case utils.DATA_SOURCE_PREDICT_CALCULATE_ZSXY:
+		tableName = "edb_data_predict_calculate_zsxy" // 预测指数修匀->73
+	case utils.DATA_SOURCE_CALCULATE_ZDYFX:
+		tableName = "edb_data_calculate_zdyfx" // 自定义分析->74
+	case utils.DATA_SOURCE_CALCULATE_RJZ: //日均值75
+		tableName = "edb_data_calculate_rjz"
 	default:
-		tableName = EdbDataTableNameMap[source] // 没有对应的从edb_source中取
-		return
+		edbSource := EdbSourceIdMap[source]
+		if edbSource != nil {
+			tableName = edbSource.TableName
+		}
 	}
 	return
 }
@@ -158,15 +207,6 @@ func GetAddSql(edbInfoId, edbCode, dataTime, timestampStr, value string) (addSql
 	return
 }
 
-func GetEdbDataBaseByCodeAndDate(source int, edbCode string, startDate string) (count int, err error) {
-	o := orm.NewOrm()
-	tableName := GetEdbDataTableName(source)
-	sql := ` SELECT COUNT(1) AS count FROM %s WHERE edb_code=? AND data_time=? `
-	sql = fmt.Sprintf(sql, tableName)
-	err = o.Raw(sql, edbCode, startDate).QueryRow(&count)
-	return
-}
-
 type EdbDataBase struct {
 	EdbDataId     int `orm:"column(edb_data_id);pk"`
 	EdbInfoId     int

models/data_manage/edb_data_calculate_time_shift.go

@@ -1,265 +0,0 @@
-package data_manage
-
-import (
-	"eta/eta_task/utils"
-	"fmt"
-	"github.com/beego/beego/v2/client/orm"
-	"github.com/shopspring/decimal"
-	"strconv"
-	"strings"
-	"time"
-)
-
-//时间移位
-
-type EdbInfoCalculateMove struct {
-	EdbInfoCalculateBpId int       `orm:"column(edb_info_calculate_bp_id);pk"`
-	EdbInfoId            int       `description:"指标id"`
-	EdbCode              string    `description:"指标编码"`
-	FromEdbInfoId        int       `description:"计算指标id"`
-	FromEdbCode          string    `description:"计算指标编码"`
-	FromEdbName          string    `description:"计算指标名称"`
-	FromSource           int       `description:"计算指标来源"`
-	FromSourceName       string    `description:"计算指标来源名称"`
-	FromTag              string    `description:"来源指标标签"`
-	Sort                 int       `description:"计算指标名称排序"`
-	CreateTime           time.Time `description:"创建时间"`
-	ModifyTime           time.Time `description:"修改时间"`
-}
-
-// 变频
-func RefreshCalculateTimeShift(edbInfoId, formulaInt, moveType int, fromEdbInfo *EdbInfo, edbCode, startDate, endDate, moveFrequency string) (err error) {
-	o := orm.NewOrm()
-	tx, err := o.Begin()
-	if err != nil {
-		return
-	}
-	defer func() {
-		if err != nil {
-			tx.Rollback()
-		} else {
-			tx.Commit()
-		}
-	}()
-
-	if err != nil {
-		return
-	}
-	edbInfoIdStr := strconv.Itoa(edbInfoId)
-	//计算数据
-
-	//计算数据
-	var condition string
-	var pars []interface{}
-	condition += " AND edb_info_id=? "
-	pars = append(pars, fromEdbInfo.EdbInfoId)
-
-	if startDate != "" {
-		condition += " AND data_time>=? "
-		pars = append(pars, startDate)
-	}
-	if endDate != "" {
-		condition += " AND data_time<=? "
-		pars = append(pars, endDate)
-	}
-
-	var shiftDay int
-	switch moveFrequency {
-	case "天":
-		shiftDay = formulaInt
-	case "周":
-		shiftDay = formulaInt * 7
-	case "月":
-		shiftDay = formulaInt * 30
-	case "季":
-		shiftDay = formulaInt * 90
-	case "年":
-		shiftDay = formulaInt * 365
-	default:
-		shiftDay = formulaInt
-	}
-	if moveType == 2 {
-		shiftDay = -shiftDay
-	}
-	dataList, err := GetEdbDataListAll(condition, pars, fromEdbInfo.Source, 0)
-	if err != nil {
-		return err
-	}
-	existMap := make(map[string]string)
-	dataLen := len(dataList)
-	addSql := ` INSERT INTO edb_data_calculate_bp(edb_info_id,edb_code,data_time,value,create_time,modify_time,status,data_timestamp) values `
-	var isAdd bool
-	for i := 0; i < dataLen; i++ {
-		//当期
-		currentItem := dataList[i]
-		existKey := edbCode + currentItem.DataTime
-		if _, ok := existMap[existKey]; !ok {
-			currentDate, _ := time.Parse(utils.FormatDate, currentItem.DataTime)
-			newDate := currentDate.AddDate(0, 0, -shiftDay)
-			timestamp := newDate.UnixNano() / 1e6
-			timestampStr := fmt.Sprintf("%d", timestamp)
-			valStr := decimal.NewFromFloat(currentItem.Value).String()
-			addSql += GetAddSql(edbInfoIdStr, edbCode, newDate.Format(utils.FormatDate), timestampStr, valStr)
-		}
-		existMap[existKey] = currentItem.DataTime
-	}
-	if isAdd {
-		addSql = strings.TrimRight(addSql, ",")
-		_, err = tx.Raw(addSql).Exec()
-		if err != nil {
-			return err
-		}
-	}
-	return
-}
-
-type EdbInfoCalculateTimeShiftDetail struct {
-	EdbInfoCalculateBpId int       `orm:"column(edb_info_calculate_bp_id);pk"`
-	EdbInfoId            int       `description:"指标id"`
-	EdbCode              string    `description:"指标编码"`
-	FromEdbInfoId        int       `description:"计算指标id"`
-	FromEdbCode          string    `description:"计算指标编码"`
-	FromEdbName          string    `description:"计算指标名称"`
-	FromSource           int       `description:"计算指标来源"`
-	FromSourceName       string    `description:"计算指标来源名称"`
-	FromTag              string    `description:"来源指标标签"`
-	Sort                 int       `description:"计算指标名称排序"`
-	CreateTime           time.Time `description:"创建时间"`
-	ModifyTime           time.Time `description:"修改时间"`
-	StartDate            string    `description:"开始日期"`
-	EndDate              string    `description:"结束日期"`
-}
-
-func GetEdbInfoCalculateTimeShiftDetail(edbInfoId int) (item *EdbInfoCalculateTbzDetail, err error) {
-	o := orm.NewOrm()
-	sql := ` SELECT a.*,b.start_date,b.end_date FROM edb_info_calculate_mapping AS a
-			INNER JOIN edb_info AS b ON a.from_edb_info_id=b.edb_info_id
-			WHERE a.edb_info_id=? `
-	err = o.Raw(sql, edbInfoId).QueryRow(&item)
-	return
-}
-
-// 刷新所有变频数据
-func RefreshAllCalculateTimeShift(edbInfoId, source, formulaInt, moveType int, fromEdbInfo *EdbInfo, edbCode, moveFrequency string) (err error) {
-	o := orm.NewOrm()
-	tx, err := o.Begin()
-	if err != nil {
-		return err
-	}
-	defer func() {
-		if err != nil {
-			tx.Rollback()
-		} else {
-			tx.Commit()
-		}
-	}()
-
-	if err != nil {
-		return
-	}
-	edbInfoIdStr := strconv.Itoa(edbInfoId)
-	//计算数据
-
-	//计算数据
-	var condition string
-	var pars []interface{}
-	condition += " AND edb_info_id=? "
-	pars = append(pars, fromEdbInfo.EdbInfoId)
-
-	//if startDate != "" {
-	//	condition += " AND data_time>=? "
-	//	pars = append(pars, startDate)
-	//}
-	//if endDate != "" {
-	//	condition += " AND data_time<=? "
-	//	pars = append(pars, endDate)
-	//}
-
-	var shiftDay int
-	switch moveFrequency {
-	case "天":
-		shiftDay = formulaInt
-	case "周":
-		shiftDay = formulaInt * 7
-	case "月":
-		shiftDay = formulaInt * 30
-	case "季":
-		shiftDay = formulaInt * 90
-	case "年":
-		shiftDay = formulaInt * 365
-	default:
-		shiftDay = formulaInt
-	}
-
-	if moveType == 2 {
-		shiftDay = -shiftDay
-	}
-
-	dataList, err := GetEdbDataListAll(condition, pars, fromEdbInfo.Source, 0)
-	if err != nil {
-		return err
-	}
-	var dateArr []string
-	dataMap := make(map[string]*EdbInfoSearchData)
-	for _, v := range dataList {
-		dateArr = append(dateArr, v.DataTime)
-		dataMap[v.DataTime] = v
-	}
-	fmt.Println("source:", source)
-	//获取指标所有数据
-	existDataList := make([]*EdbDataBase, 0)
-	dataTableName := GetEdbDataTableName(source)
-	fmt.Println("dataTableName:", dataTableName)
-	sql := `SELECT * FROM %s WHERE edb_info_id=? `
-	sql = fmt.Sprintf(sql, dataTableName)
-	_, err = o.Raw(sql, edbInfoId).QueryRows(&existDataList)
-	if err != nil {
-		return err
-	}
-	existDataMap := make(map[string]string)
-	for _, v := range existDataList {
-		existDataMap[v.DataTime] = v.Value
-	}
-	fmt.Println("existDataMap:", existDataMap)
-	addSql := ` INSERT INTO edb_data_calculate_time_shift (edb_info_id,edb_code,data_time,value,create_time,modify_time,status,data_timestamp) values `
-	var isAdd bool
-
-	existMap := make(map[string]string)
-	dataLen := len(dataList)
-	for i := 0; i < dataLen; i++ {
-		//当期
-		currentItem := dataList[i]
-		existKey := edbCode + currentItem.DataTime
-		if _, ok := existMap[existKey]; !ok {
-			currentDate, _ := time.Parse(utils.FormatDate, currentItem.DataTime)
-			newDate := currentDate.AddDate(0, 0, shiftDay)
-
-			timestamp := newDate.UnixNano() / 1e6
-			timestampStr := fmt.Sprintf("%d", timestamp)
-			valStr := decimal.NewFromFloat(currentItem.Value).String()
-			if existVal, ok := existDataMap[newDate.Format(utils.FormatDate)]; !ok {
-				isAdd = true
-				addSql += GetAddSql(edbInfoIdStr, edbCode, newDate.Format(utils.FormatDate), timestampStr, valStr)
-			} else {
-				if existVal != valStr {
-					sql := ` UPDATE %s SET value=?,modify_time=NOW() WHERE edb_info_id=? AND data_time=? `
-					sql = fmt.Sprintf(sql, dataTableName)
-					_, err = tx.Raw(sql, valStr, edbInfoId, newDate.Format(utils.FormatDate)).Exec()
-					if err != nil {
-						return err
-					}
-				}
-			}
-		}
-		existMap[existKey] = currentItem.DataTime
-	}
-
-	if isAdd {
-		addSql = strings.TrimRight(addSql, ",")
-		_, err = tx.Raw(addSql).Exec()
-		if err != nil {
-			return err
-		}
-	}
-	return
-}

models/data_manage/edb_data_cffex.go

@@ -168,126 +168,6 @@ func RefreshEdbDataByCffex(edbInfoId int, edbCode, startDate, endDate string) (e
 	return
 }
 
-// RefreshAllEdbDataByCffex 全部刷新中金所
-func RefreshAllEdbDataByCffex(edbInfoId, source int, edbCode, startDate, endDate string) (err error) {
-	o := orm.NewOrm()
-	tx, err := o.Begin()
-	if err != nil {
-		return err
-	}
-	defer func() {
-		if err != nil {
-			tx.Rollback()
-		} else {
-			tx.Commit()
-		}
-	}()
-	if err != nil {
-		return
-	}
-	var suffix string
-	if strings.Contains(edbCode, "deal") {
-		suffix = "deal"
-	} else if strings.Contains(edbCode, "buy") {
-		suffix = "buy"
-	} else if strings.Contains(edbCode, "sold") {
-		suffix = "sold"
-	}
-	edbInfoIdStr := strconv.Itoa(edbInfoId)
-	//计算数据
-	var condition string
-	var pars []interface{}
-
-	if edbCode != "" {
-		if suffix == "deal" {
-			condition += " AND deal_code=? "
-		} else if suffix == "buy" {
-			condition += " AND buy_code=? "
-		} else {
-			condition += " AND sold_code=? "
-		}
-		pars = append(pars, edbCode)
-	}
-
-	if startDate != "" {
-		condition += " AND data_time>=? "
-		pars = append(pars, startDate)
-	}
-
-	if endDate != "" {
-		condition += " AND data_time<=? "
-		pars = append(pars, endDate)
-	}
-
-	glDataList, err := GetCffexDataByTradeCode(condition, pars)
-
-	//获取指标所有数据
-	dataList := make([]*EdbDataBase, 0)
-	dataTableName := GetEdbDataTableName(source)
-	sql := `SELECT * FROM %s WHERE edb_info_id=? `
-	sql = fmt.Sprintf(sql, dataTableName)
-	_, err = o.Raw(sql, edbInfoId).QueryRows(&dataList)
-	if err != nil {
-		return err
-	}
-	dataMap := make(map[string]string)
-	for _, v := range dataList {
-		dataMap[v.DataTime] = v.Value
-	}
-
-	addSql := ` INSERT INTO edb_data_cffex(edb_info_id,edb_code,data_time,value,create_time,modify_time,status,data_timestamp) values `
-	var isAdd bool
-	existMap := make(map[string]string)
-	for _, v := range glDataList {
-		var value string
-		if suffix == "deal" {
-			value = v.DealValue
-		} else if suffix == "buy" {
-			value = v.BuyValue
-		} else {
-			value = v.SoldValue
-		}
-		item := v
-		itemValue := value
-		if _, ok := existMap[v.DataTime]; !ok {
-			eDate := item.DataTime
-			sValue := itemValue
-			if sValue != "" {
-				dataTime, err := time.Parse(utils.FormatDate, eDate)
-				if err != nil {
-					return err
-				}
-				timestamp := dataTime.UnixNano() / 1e6
-				timeStr := fmt.Sprintf("%d", timestamp)
-				saveValue := sValue
-
-				if existVal, ok := dataMap[eDate]; !ok {
-					addSql += GetAddSql(edbInfoIdStr, edbCode, eDate, timeStr, saveValue)
-					isAdd = true
-				} else {
-					if existVal != saveValue {
-						sql := ` UPDATE %s SET value=?,modify_time=NOW() WHERE edb_info_id=? AND data_time=? `
-						sql = fmt.Sprintf(sql, dataTableName)
-						_, err = tx.Raw(sql, sValue, edbInfoId, eDate).Exec()
-						if err != nil {
-							return err
-						}
-					}
-				}
-			}
-		}
-		existMap[v.DataTime] = v.DataTime
-	}
-	if isAdd {
-		addSql = strings.TrimRight(addSql, ",")
-		_, err = tx.Raw(addSql).Exec()
-		if err != nil {
-			return err
-		}
-	}
-	return
-}
-
 // GetBaseInfoFromCffexByIndexCode 获取指标信息
 func GetBaseInfoFromCffexByIndexCode(indexCode, suffix string) (list []*BaseInfoFromCffex, err error) {
 	o := orm.NewOrm()

models/data_manage/edb_data_dl.go

@@ -142,126 +142,6 @@ func RefreshEdbDataByDl(edbInfoId int, edbCode, startDate, endDate string) (err
 	return
 }
 
-// RefreshAllEdbDataByDl 全部刷新大商所
-func RefreshAllEdbDataByDl(edbInfoId, source int, edbCode, startDate, endDate string) (err error) {
-	o := orm.NewOrm()
-	tx, err := o.Begin()
-	if err != nil {
-		return err
-	}
-	defer func() {
-		if err != nil {
-			tx.Rollback()
-		} else {
-			tx.Commit()
-		}
-	}()
-	if err != nil {
-		return
-	}
-	var suffix string
-	if strings.Contains(edbCode, "deal") {
-		suffix = "deal"
-	} else if strings.Contains(edbCode, "buy") {
-		suffix = "buy"
-	} else if strings.Contains(edbCode, "sold") {
-		suffix = "sold"
-	}
-	edbInfoIdStr := strconv.Itoa(edbInfoId)
-	//计算数据
-	var condition string
-	var pars []interface{}
-
-	if edbCode != "" {
-		if suffix == "deal" {
-			condition += " AND deal_code=? "
-		} else if suffix == "buy" {
-			condition += " AND buy_code=? "
-		} else {
-			condition += " AND sold_code=? "
-		}
-		pars = append(pars, edbCode)
-	}
-
-	if startDate != "" {
-		condition += " AND data_time>=? "
-		pars = append(pars, startDate)
-	}
-
-	if endDate != "" {
-		condition += " AND data_time<=? "
-		pars = append(pars, endDate)
-	}
-
-	glDataList, err := GetDlDataByTradeCode(condition, pars)
-
-	//获取指标所有数据
-	dataList := make([]*EdbDataBase, 0)
-	dataTableName := GetEdbDataTableName(source)
-	sql := `SELECT * FROM %s WHERE edb_info_id=? `
-	sql = fmt.Sprintf(sql, dataTableName)
-	_, err = o.Raw(sql, edbInfoId).QueryRows(&dataList)
-	if err != nil {
-		return err
-	}
-	dataMap := make(map[string]string)
-	for _, v := range dataList {
-		dataMap[v.DataTime] = v.Value
-	}
-
-	addSql := ` INSERT INTO edb_data_dl(edb_info_id,edb_code,data_time,value,create_time,modify_time,status,data_timestamp) values `
-	var isAdd bool
-	existMap := make(map[string]string)
-	for _, v := range glDataList {
-		var value string
-		if suffix == "deal" {
-			value = v.DealValue
-		} else if suffix == "buy" {
-			value = v.BuyValue
-		} else {
-			value = v.SoldValue
-		}
-		item := v
-		itemValue := value
-		if _, ok := existMap[v.DataTime]; !ok {
-			eDate := item.DataTime
-			sValue := itemValue
-			if sValue != "" {
-				dataTime, err := time.Parse(utils.FormatDate, eDate)
-				if err != nil {
-					return err
-				}
-				timestamp := dataTime.UnixNano() / 1e6
-				timeStr := fmt.Sprintf("%d", timestamp)
-				saveValue := sValue
-
-				if existVal, ok := dataMap[eDate]; !ok {
-					addSql += GetAddSql(edbInfoIdStr, edbCode, eDate, timeStr, saveValue)
-					isAdd = true
-				} else {
-					if existVal != saveValue {
-						sql := ` UPDATE %s SET value=?,modify_time=NOW() WHERE edb_info_id=? AND data_time=? `
-						sql = fmt.Sprintf(sql, dataTableName)
-						_, err = tx.Raw(sql, sValue, edbInfoId, eDate).Exec()
-						if err != nil {
-							return err
-						}
-					}
-				}
-			}
-		}
-		existMap[v.DataTime] = v.DataTime
-	}
-	if isAdd {
-		addSql = strings.TrimRight(addSql, ",")
-		_, err = tx.Raw(addSql).Exec()
-		if err != nil {
-			return err
-		}
-	}
-	return
-}
-
 // GetBaseInfoFromDalianByIndexCode 获取指标信息
 func GetBaseInfoFromDalianByIndexCode(indexCode, suffix string) (list []*BaseInfoFromDl, err error) {
 	o := orm.NewOrm()

models/data_manage/edb_data_gl.go

@@ -95,7 +95,7 @@ func RefreshEdbDataByGl(edbInfoId int, edbCode, startDate, endDate string) (err
 
 	//获取已存在指标所有数据
 	existDataList := make([]*EdbDataBase, 0)
-	dataTableName := GetEdbDataTableName(utils.DATA_SOURCE_GL)
+	dataTableName := GetEdbDataTableName(utils.DATA_SOURCE_GL, utils.DATA_SUB_SOURCE_EDB)
 	sql := `SELECT * FROM %s WHERE edb_info_id=? `
 	sql = fmt.Sprintf(sql, dataTableName)
 	_, err = o.Raw(sql, edbInfoId).QueryRows(&existDataList)

models/data_manage/edb_data_lt.go

@@ -6,9 +6,6 @@ import (
 	"fmt"
 	"github.com/beego/beego/v2/client/orm"
 	"github.com/rdlucklib/rdluck_tools/http"
-	"reflect"
-	"strconv"
-	"strings"
 	"time"
 )
 
@@ -60,89 +57,6 @@ func GetEdbDataLtByCode(edbCode string, size int) (items []*EdbInfoSearchData, e
 	return
 }
 
-// 刷新所有数据
-func RefreshAllEdbDataByLt(edbInfoId, source int, edbCode, startDate, endDate string) (err error) {
-	// 获取路透社数据
-	ltDataList, err := QueryEdbDataByLt(edbCode, startDate, endDate)
-	if err != nil {
-		return
-	}
-
-	o := orm.NewOrm()
-	tx, err := o.Begin()
-	defer func() {
-		if err != nil {
-			tx.Rollback()
-		} else {
-			tx.Commit()
-		}
-	}()
-
-	//获取指标所有数据
-	dataList := make([]*EdbDataBase, 0)
-	dataTableName := GetEdbDataTableName(source)
-	sql := `SELECT * FROM %s WHERE edb_info_id=? `
-	sql = fmt.Sprintf(sql, dataTableName)
-	_, err = o.Raw(sql, edbInfoId).QueryRows(&dataList)
-	if err != nil {
-		return err
-	}
-	dataMap := make(map[string]string)
-	for _, v := range dataList {
-		dataMap[v.DataTime] = v.Value
-	}
-	edbInfoIdStr := strconv.Itoa(edbInfoId)
-
-	addSql := ` INSERT INTO edb_data_lt(edb_info_id,edb_code,data_time,value,create_time,modify_time,status,data_timestamp) values `
-	var isAdd bool
-
-	for timestampInt, edbValue := range ltDataList {
-		dataTime := time.Unix(timestampInt/1000, 0)
-
-		//校验数据类型对不对
-		valType := reflect.TypeOf(edbValue)
-		if valType == nil {
-			continue
-		}
-		if valType.String() != "float64" {
-			continue
-		}
-		sValue := edbValue.(float64)
-
-		eDate := dataTime.Format(utils.FormatDate)
-		if err != nil {
-			return err
-		}
-
-		saveValue := utils.SubFloatToString(sValue, 30)
-		if existVal, ok := dataMap[eDate]; !ok {
-			timestamp := dataTime.UnixNano() / 1e6
-			timeStr := fmt.Sprintf("%d", timestamp)
-
-			addSql += GetAddSql(edbInfoIdStr, edbCode, eDate, timeStr, saveValue)
-			isAdd = true
-		} else {
-			if existVal != saveValue {
-				sql := ` UPDATE %s SET value=?,modify_time=NOW() WHERE edb_info_id=? AND data_time=? `
-				sql = fmt.Sprintf(sql, dataTableName)
-				_, err = tx.Raw(sql, sValue, edbInfoId, eDate).Exec()
-				if err != nil {
-					return err
-				}
-			}
-		}
-	}
-	if isAdd {
-		addSql = strings.TrimRight(addSql, ",")
-		_, err = tx.Raw(addSql).Exec()
-		if err != nil {
-			fmt.Println("RefreshAllEdbDataByLt add Err", err.Error())
-			return
-		}
-	}
-	return
-}
-
 type EdbDataFromLt struct {
 	Close map[int64]interface{} `json:"CLOSE"`
 }

models/data_manage/edb_data_lz.go

@@ -1,12 +1,7 @@
 package data_manage
 
 import (
-	"eta/eta_task/utils"
-	"fmt"
 	"github.com/beego/beego/v2/client/orm"
-	"strconv"
-	"strings"
-	"time"
 )
 
 type lzSurveyData struct {
@@ -81,96 +76,3 @@ func ModifyEdbDataLz(edbInfoId int64, dataTime, value string) (err error) {
 	_, err = o.Raw(sql, value, edbInfoId, dataTime).Exec()
 	return
 }
-
-// 刷新隆众指标数据
-func RefreshEdbDataByLz(edbInfoId int, edbCode, startDate, endDate string) (err error) {
-	o := orm.NewOrm()
-	tx, err := o.Begin()
-	defer func() {
-		if err != nil {
-			tx.Rollback()
-		} else {
-			tx.Commit()
-		}
-	}()
-
-	if err != nil {
-		return
-	}
-	edbInfoIdStr := strconv.Itoa(edbInfoId)
-	//计算数据
-	var condition string
-	var pars []interface{}
-
-	if edbCode != "" {
-		condition += " AND b.lz_code=? "
-		pars = append(pars, edbCode)
-	}
-
-	if startDate != "" {
-		condition += " AND a.data_time>=? "
-		pars = append(pars, startDate)
-	}
-
-	if endDate != "" {
-		condition += " AND a.data_time<=? "
-		pars = append(pars, endDate)
-	}
-
-	lzDataList, err := GetLzSurveyDataByTradeCode(condition, pars)
-
-	//获取已存在指标所有数据
-	existDataList := make([]*EdbDataBase, 0)
-	dataTableName := GetEdbDataTableName(utils.DATA_SOURCE_LZ)
-	sql := `SELECT * FROM %s WHERE edb_info_id=? `
-	sql = fmt.Sprintf(sql, dataTableName)
-	_, err = o.Raw(sql, edbInfoId).QueryRows(&existDataList)
-	if err != nil {
-		return err
-	}
-	existDataMap := make(map[string]string)
-	for _, v := range existDataList {
-		existDataMap[v.DataTime] = v.Value
-	}
-
-	addSql := ` INSERT INTO edb_data_lz(edb_info_id,edb_code,data_time,value,create_time,modify_time,status,data_timestamp) values `
-	var isAdd bool
-	addExistMap := make(map[string]string)
-	for _, v := range lzDataList {
-		item := v
-		if _, ok := addExistMap[v.DataTime]; !ok {
-			if existVal, existOk := existDataMap[v.DataTime]; !existOk {
-				eDate := item.DataTime
-				sValue := item.InputValue
-				if sValue != "" {
-					dataTime, err := time.Parse(utils.FormatDate, eDate)
-					if err != nil {
-						return err
-					}
-					timestamp := dataTime.UnixNano() / 1e6
-					timeStr := fmt.Sprintf("%d", timestamp)
-					addSql += GetAddSql(edbInfoIdStr, edbCode, eDate, timeStr, sValue)
-					isAdd = true
-				}
-			} else {
-				if existVal != item.InputValue {
-					sql := ` UPDATE %s SET value=?,modify_time=NOW() WHERE edb_info_id=? AND data_time=? `
-					sql = fmt.Sprintf(sql, dataTableName)
-					_, err = tx.Raw(sql, item.InputValue, edbInfoId, v.DataTime).Exec()
-					if err != nil {
-						return err
-					}
-				}
-			}
-		}
-		addExistMap[v.DataTime] = v.InputValue
-	}
-	if isAdd {
-		addSql = strings.TrimRight(addSql, ",")
-		_, err = tx.Raw(addSql).Exec()
-		if err != nil {
-			return err
-		}
-	}
-	return
-}

models/data_manage/edb_data_manual.go

@@ -95,7 +95,7 @@ func RefreshManual(item *EdbInfoList) (err error) {
 		existPars = append(existPars, edbCode)
 	}
 	//获取ETA指标库已存在的所有数据
-	existDataList, err := GetEdbDataListAll(existCondition, existPars, utils.DATA_SOURCE_MANUAL, 0)
+	existDataList, err := GetEdbDataListAll(existCondition, existPars, utils.DATA_SOURCE_MANUAL, utils.DATA_SUB_SOURCE_EDB, 0)
 	if err != nil {
 		return err
 	}
@@ -138,7 +138,7 @@ func RefreshManual(item *EdbInfoList) (err error) {
 
 	for _, v := range existDataList {
 		if _, ok := manualMap[v.DataTime]; !ok {
-			go DeleteEdbDataByIdAndSource(v.EdbDataId, utils.DATA_SOURCE_MANUAL)
+			go DeleteEdbDataByIdAndSource(v.EdbDataId, utils.DATA_SOURCE_MANUAL, utils.DATA_SUB_SOURCE_EDB)
 		}
 	}
 
@@ -150,7 +150,7 @@ func RefreshManual(item *EdbInfoList) (err error) {
 		}
 	}
 
-	maxAndMinItem, err := GetEdbInfoMaxAndMinInfo(utils.DATA_SOURCE_MANUAL, edbCode)
+	maxAndMinItem, err := GetEdbInfoMaxAndMinInfo(utils.DATA_SOURCE_MANUAL, utils.DATA_SUB_SOURCE_EDB, edbCode)
 	if err != nil {
 		if err.Error() == utils.ErrNoRow() {
 			err = nil

models/data_manage/edb_data_sh.go

@@ -168,126 +168,6 @@ func RefreshEdbDataBySh(edbInfoId int, edbCode, startDate, endDate string) (err
 	return
 }
 
-// RefreshAllEdbDataBySh 全部刷新上期所
-func RefreshAllEdbDataBySh(edbInfoId, source int, edbCode, startDate, endDate string) (err error) {
-	o := orm.NewOrm()
-	tx, err := o.Begin()
-	if err != nil {
-		return err
-	}
-	defer func() {
-		if err != nil {
-			tx.Rollback()
-		} else {
-			tx.Commit()
-		}
-	}()
-	if err != nil {
-		return
-	}
-	var suffix string
-	if strings.Contains(edbCode, "deal") {
-		suffix = "deal"
-	} else if strings.Contains(edbCode, "buy") {
-		suffix = "buy"
-	} else if strings.Contains(edbCode, "sold") {
-		suffix = "sold"
-	}
-	edbInfoIdStr := strconv.Itoa(edbInfoId)
-	//计算数据
-	var condition string
-	var pars []interface{}
-
-	if edbCode != "" {
-		if suffix == "deal" {
-			condition += " AND deal_code=? "
-		} else if suffix == "buy" {
-			condition += " AND buy_code=? "
-		} else {
-			condition += " AND sold_code=? "
-		}
-		pars = append(pars, edbCode)
-	}
-
-	if startDate != "" {
-		condition += " AND data_time>=? "
-		pars = append(pars, startDate)
-	}
-
-	if endDate != "" {
-		condition += " AND data_time<=? "
-		pars = append(pars, endDate)
-	}
-
-	glDataList, err := GetShDataByTradeCode(condition, pars)
-
-	//获取指标所有数据
-	dataList := make([]*EdbDataBase, 0)
-	dataTableName := GetEdbDataTableName(source)
-	sql := `SELECT * FROM %s WHERE edb_info_id=? `
-	sql = fmt.Sprintf(sql, dataTableName)
-	_, err = o.Raw(sql, edbInfoId).QueryRows(&dataList)
-	if err != nil {
-		return err
-	}
-	dataMap := make(map[string]string)
-	for _, v := range dataList {
-		dataMap[v.DataTime] = v.Value
-	}
-
-	addSql := ` INSERT INTO edb_data_sh(edb_info_id,edb_code,data_time,value,create_time,modify_time,status,data_timestamp) values `
-	var isAdd bool
-	existMap := make(map[string]string)
-	for _, v := range glDataList {
-		var value string
-		if suffix == "deal" {
-			value = v.DealValue
-		} else if suffix == "buy" {
-			value = v.BuyValue
-		} else {
-			value = v.SoldValue
-		}
-		item := v
-		itemValue := value
-		if _, ok := existMap[v.DataTime]; !ok {
-			eDate := item.DataTime
-			sValue := itemValue
-			if sValue != "" {
-				dataTime, err := time.Parse(utils.FormatDate, eDate)
-				if err != nil {
-					return err
-				}
-				timestamp := dataTime.UnixNano() / 1e6
-				timeStr := fmt.Sprintf("%d", timestamp)
-				saveValue := sValue
-
-				if existVal, ok := dataMap[eDate]; !ok {
-					addSql += GetAddSql(edbInfoIdStr, edbCode, eDate, timeStr, saveValue)
-					isAdd = true
-				} else {
-					if existVal != saveValue {
-						sql := ` UPDATE %s SET value=?,modify_time=NOW() WHERE edb_info_id=? AND data_time=? `
-						sql = fmt.Sprintf(sql, dataTableName)
-						_, err = tx.Raw(sql, sValue, edbInfoId, eDate).Exec()
-						if err != nil {
-							return err
-						}
-					}
-				}
-			}
-		}
-		existMap[v.DataTime] = v.DataTime
-	}
-	if isAdd {
-		addSql = strings.TrimRight(addSql, ",")
-		_, err = tx.Raw(addSql).Exec()
-		if err != nil {
-			return err
-		}
-	}
-	return
-}
-
 // GetBaseInfoFromShByIndexCode 获取指标信息
 func GetBaseInfoFromShByIndexCode(indexCode, suffix string) (list []*BaseInfoFromSh, err error) {
 	o := orm.NewOrm()

models/data_manage/edb_data_shfe.go

@@ -168,126 +168,6 @@ func RefreshEdbDataByShfe(edbInfoId int, edbCode, startDate, endDate string) (er
 	return
 }
 
-// RefreshAllEdbDataByShfe 全部刷新上期能源
-func RefreshAllEdbDataByShfe(edbInfoId, source int, edbCode, startDate, endDate string) (err error) {
-	o := orm.NewOrm()
-	tx, err := o.Begin()
-	if err != nil {
-		return err
-	}
-	defer func() {
-		if err != nil {
-			tx.Rollback()
-		} else {
-			tx.Commit()
-		}
-	}()
-	if err != nil {
-		return
-	}
-	var suffix string
-	if strings.Contains(edbCode, "deal") {
-		suffix = "deal"
-	} else if strings.Contains(edbCode, "buy") {
-		suffix = "buy"
-	} else if strings.Contains(edbCode, "sold") {
-		suffix = "sold"
-	}
-	edbInfoIdStr := strconv.Itoa(edbInfoId)
-	//计算数据
-	var condition string
-	var pars []interface{}
-
-	if edbCode != "" {
-		if suffix == "deal" {
-			condition += " AND deal_code=? "
-		} else if suffix == "buy" {
-			condition += " AND buy_code=? "
-		} else {
-			condition += " AND sold_code=? "
-		}
-		pars = append(pars, edbCode)
-	}
-
-	if startDate != "" {
-		condition += " AND data_time>=? "
-		pars = append(pars, startDate)
-	}
-
-	if endDate != "" {
-		condition += " AND data_time<=? "
-		pars = append(pars, endDate)
-	}
-
-	glDataList, err := GetShfeDataByTradeCode(condition, pars)
-
-	//获取指标所有数据
-	dataList := make([]*EdbDataBase, 0)
-	dataTableName := GetEdbDataTableName(source)
-	sql := `SELECT * FROM %s WHERE edb_info_id=? `
-	sql = fmt.Sprintf(sql, dataTableName)
-	_, err = o.Raw(sql, edbInfoId).QueryRows(&dataList)
-	if err != nil {
-		return err
-	}
-	dataMap := make(map[string]string)
-	for _, v := range dataList {
-		dataMap[v.DataTime] = v.Value
-	}
-
-	addSql := ` INSERT INTO edb_data_ine(edb_info_id,edb_code,data_time,value,create_time,modify_time,status,data_timestamp) values `
-	var isAdd bool
-	existMap := make(map[string]string)
-	for _, v := range glDataList {
-		var value string
-		if suffix == "deal" {
-			value = v.DealValue
-		} else if suffix == "buy" {
-			value = v.BuyValue
-		} else {
-			value = v.SoldValue
-		}
-		item := v
-		itemValue := value
-		if _, ok := existMap[v.DataTime]; !ok {
-			eDate := item.DataTime
-			sValue := itemValue
-			if sValue != "" {
-				dataTime, err := time.Parse(utils.FormatDate, eDate)
-				if err != nil {
-					return err
-				}
-				timestamp := dataTime.UnixNano() / 1e6
-				timeStr := fmt.Sprintf("%d", timestamp)
-				saveValue := sValue
-
-				if existVal, ok := dataMap[eDate]; !ok {
-					addSql += GetAddSql(edbInfoIdStr, edbCode, eDate, timeStr, saveValue)
-					isAdd = true
-				} else {
-					if existVal != saveValue {
-						sql := ` UPDATE %s SET value=?,modify_time=NOW() WHERE edb_info_id=? AND data_time=? `
-						sql = fmt.Sprintf(sql, dataTableName)
-						_, err = tx.Raw(sql, sValue, edbInfoId, eDate).Exec()
-						if err != nil {
-							return err
-						}
-					}
-				}
-			}
-		}
-		existMap[v.DataTime] = v.DataTime
-	}
-	if isAdd {
-		addSql = strings.TrimRight(addSql, ",")
-		_, err = tx.Raw(addSql).Exec()
-		if err != nil {
-			return err
-		}
-	}
-	return
-}
-
 // GetBaseInfoFromShfeByIndexCode 获取指标信息
 func GetBaseInfoFromShfeByIndexCode(indexCode, suffix string) (list []*BaseInfoFromShfe, err error) {
 	o := orm.NewOrm()

models/data_manage/edb_data_ys.go

@@ -45,74 +45,6 @@ func ModifyEdbDataSmm(edbCode, dataTime string, value float64) (err error) {
 	return
 }
 
-// 刷新有色指标数据
-func RefreshEdbDataByYs(edbInfoId int, edbCode, startDate, endDate string) (err error) {
-	var condition string
-	var pars []interface{}
-	condition += " AND index_code=? "
-	pars = append(pars, edbCode)
-
-	if startDate != "" {
-		condition += " AND data_time>=? "
-		pars = append(pars, startDate)
-	}
-
-	if endDate != "" {
-		condition += " AND data_time<=? "
-		pars = append(pars, endDate)
-	}
-
-	ysBaseDataAll, err := GetBaseFromSmmDataByCondition(condition, pars)
-	if err != nil {
-		err = errors.New("GetBaseFromSmmDataByCondition Err:" + err.Error())
-		return
-	}
-	var existCondition string
-	var existPars []interface{}
-	existCondition += " AND edb_code=? "
-	existPars = append(existPars, edbCode)
-
-	existDataList, err := GetEdbDataListAll(existCondition, existPars, utils.DATA_SOURCE_YS, 1)
-	if err != nil {
-		err = errors.New("GetEdbDataListAll Err:" + err.Error())
-		return err
-	}
-
-	existMap := make(map[string]float64)
-	for _, ev := range existDataList {
-		existMap[ev.DataTime] = ev.Value
-	}
-	ysList := make([]*EdbDataYs, 0)
-	for _, dv := range ysBaseDataAll {
-		if existItem, ok := existMap[dv.DataTime]; !ok { //新增
-			dataDate, _ := time.Parse(utils.FormatDate, dv.DataTime)
-			timestamp := dataDate.UnixNano() / 1e6
-			item := new(EdbDataYs)
-			item.EdbCode = edbCode
-			item.EdbInfoId = edbInfoId
-			item.CreateTime = time.Now()
-			item.ModifyTime = time.Now()
-			item.DataTime = dv.DataTime
-			item.Value = dv.Value
-			item.Status = 1
-			item.DataTimestamp = timestamp
-			ysList = append(ysList, item)
-		} else {
-			if dv.Value != existItem { //修改
-				fmt.Println("ModifyEdbDataSmm:", edbCode, dv.DataTime, dv.Value)
-				ModifyEdbDataSmm(edbCode, dv.DataTime, dv.Value)
-			}
-		}
-	}
-	if len(ysList) > 0 {
-		err = AddEdbDataSmm(ysList)
-		if err != nil {
-			return
-		}
-	}
-	return
-}
-
 /*
 
 //有色基础指标同步

models/data_manage/edb_data_zz.go

@@ -142,126 +142,6 @@ func RefreshEdbDataByZz(edbInfoId int, edbCode, startDate, endDate string) (err
 	return
 }
 
-// RefreshAllEdbDataByZz 全部刷新郑商所
-func RefreshAllEdbDataByZz(edbInfoId, source int, edbCode, startDate, endDate string) (err error) {
-	o := orm.NewOrm()
-	tx, err := o.Begin()
-	if err != nil {
-		return err
-	}
-	defer func() {
-		if err != nil {
-			tx.Rollback()
-		} else {
-			tx.Commit()
-		}
-	}()
-	if err != nil {
-		return
-	}
-	var suffix string
-	if strings.Contains(edbCode, "deal") {
-		suffix = "deal"
-	} else if strings.Contains(edbCode, "buy") {
-		suffix = "buy"
-	} else if strings.Contains(edbCode, "sold") {
-		suffix = "sold"
-	}
-	edbInfoIdStr := strconv.Itoa(edbInfoId)
-	//计算数据
-	var condition string
-	var pars []interface{}
-
-	if edbCode != "" {
-		if suffix == "deal" {
-			condition += " AND deal_code=? "
-		} else if suffix == "buy" {
-			condition += " AND buy_code=? "
-		} else {
-			condition += " AND sold_code=? "
-		}
-		pars = append(pars, edbCode)
-	}
-
-	if startDate != "" {
-		condition += " AND data_time>=? "
-		pars = append(pars, startDate)
-	}
-
-	if endDate != "" {
-		condition += " AND data_time<=? "
-		pars = append(pars, endDate)
-	}
-
-	glDataList, err := GetZzDataByTradeCode(condition, pars)
-
-	//获取指标所有数据
-	dataList := make([]*EdbDataBase, 0)
-	dataTableName := GetEdbDataTableName(source)
-	sql := `SELECT * FROM %s WHERE edb_info_id=? `
-	sql = fmt.Sprintf(sql, dataTableName)
-	_, err = o.Raw(sql, edbInfoId).QueryRows(&dataList)
-	if err != nil {
-		return err
-	}
-	dataMap := make(map[string]string)
-	for _, v := range dataList {
-		dataMap[v.DataTime] = v.Value
-	}
-
-	addSql := ` INSERT INTO edb_data_zz(edb_info_id,edb_code,data_time,value,create_time,modify_time,status,data_timestamp) values `
-	var isAdd bool
-	existMap := make(map[string]string)
-	for _, v := range glDataList {
-		var value string
-		if suffix == "deal" {
-			value = v.DealValue
-		} else if suffix == "buy" {
-			value = v.BuyValue
-		} else {
-			value = v.SoldValue
-		}
-		item := v
-		itemValue := value
-		if _, ok := existMap[v.DataTime]; !ok {
-			eDate := item.DataTime
-			sValue := itemValue
-			if sValue != "" {
-				dataTime, err := time.Parse(utils.FormatDate, eDate)
-				if err != nil {
-					return err
-				}
-				timestamp := dataTime.UnixNano() / 1e6
-				timeStr := fmt.Sprintf("%d", timestamp)
-				saveValue := sValue
-
-				if existVal, ok := dataMap[eDate]; !ok {
-					addSql += GetAddSql(edbInfoIdStr, edbCode, eDate, timeStr, saveValue)
-					isAdd = true
-				} else {
-					if existVal != saveValue {
-						sql := ` UPDATE %s SET value=?,modify_time=NOW() WHERE edb_info_id=? AND data_time=? `
-						sql = fmt.Sprintf(sql, dataTableName)
-						_, err = tx.Raw(sql, sValue, edbInfoId, eDate).Exec()
-						if err != nil {
-							return err
-						}
-					}
-				}
-			}
-		}
-		existMap[v.DataTime] = v.DataTime
-	}
-	if isAdd {
-		addSql = strings.TrimRight(addSql, ",")
-		_, err = o.Raw(addSql).Exec()
-		if err != nil {
-			return err
-		}
-	}
-	return
-}
-
 // GetBaseInfoFromZhengzhouByIndexCode 获取指标信息
 func GetBaseInfoFromZhengzhouByIndexCode(indexCode, suffix string) (list []*BaseInfoFromZz, err error) {
 	o := orm.NewOrm()

models/data_manage/edb_info.go

@@ -91,10 +91,10 @@ type EdbInfoMaxAndMinInfo struct {
 	LatestValue float64 `description:"最新值"`
 }
 
-func GetEdbInfoMaxAndMinInfo(source int, edbCode string) (item *EdbInfoMaxAndMinInfo, err error) {
+func GetEdbInfoMaxAndMinInfo(source, subSource int, edbCode string) (item *EdbInfoMaxAndMinInfo, err error) {
 	o := orm.NewOrm()
 	sql := ``
-	tableName := GetEdbDataTableName(source)
+	tableName := GetEdbDataTableName(source, subSource)
 	if tableName == "" {
 		err = errors.New("无效的表名称:source:" + strconv.Itoa(source))
 		return nil, err
@@ -119,10 +119,10 @@ func ModifyEdbInfoMaxAndMinInfo(edbInfoId int, item *EdbInfoMaxAndMinInfo) (err
 }
 
 //order:1升序，其余值为降序
-func GetEdbDataListAll(condition string, pars []interface{}, source, order int) (item []*EdbInfoSearchData, err error) {
+func GetEdbDataListAll(condition string, pars []interface{}, source, subSource, order int) (item []*EdbInfoSearchData, err error) {
 	o := orm.NewOrm()
 	sql := ``
-	tableName := GetEdbDataTableName(source)
+	tableName := GetEdbDataTableName(source, subSource)
 	sql = ` SELECT * FROM %s WHERE 1=1 `
 	sql = fmt.Sprintf(sql, tableName)
 
@@ -138,10 +138,10 @@ func GetEdbDataListAll(condition string, pars []interface{}, source, order int)
 	return
 }
 
-func GetEdbDataListAllV1(condition string, pars []interface{}, source, order int) (item []*EdbInfoSearchDataV1, err error) {
+func GetEdbDataListAllV1(condition string, pars []interface{}, source, subSource, order int) (item []*EdbInfoSearchDataV1, err error) {
 	o := orm.NewOrm()
 	sql := ``
-	tableName := GetEdbDataTableName(source)
+	tableName := GetEdbDataTableName(source, subSource)
 	sql = ` SELECT * FROM %s WHERE 1=1 `
 	sql = fmt.Sprintf(sql, tableName)
 
@@ -201,9 +201,9 @@ func GetEdbInfoCalculateListByCondition(condition string, pars []interface{}) (i
 	return
 }
 
-func DeleteEdbDataByIdAndSource(edbDataId, source int) (err error) {
+func DeleteEdbDataByIdAndSource(edbDataId, source, subSource int) (err error) {
 	sql := ` DELETE FROM %s WHERE edb_data_id=? `
-	tableName := GetEdbDataTableName(source)
+	tableName := GetEdbDataTableName(source, subSource)
 	sql = fmt.Sprintf(sql, tableName)
 
 	o := orm.NewOrm()
@@ -243,3 +243,42 @@ func GetAllEdbInfoClassifyListByCondition(condition string, pars []interface{})
 
 	return
 }
+
+// GetEdbInfoItemByCodeAndSource
+// @Description: 根据指标编码和来源id获取指标信息
+// @author: Roc
+// @datetime 2024-03-11 16:26:23
+// @param source int
+// @param edbCode string
+// @return item *EdbInfo
+// @return err error
+func GetEdbInfoItemByCodeAndSource(source int, edbCode string) (item *EdbInfoItem, err error) {
+	o := orm.NewOrm()
+	sql := ` SELECT * FROM edb_info WHERE edb_code=? AND source = ?`
+	err = o.Raw(sql, source, edbCode).QueryRow(&item)
+
+	return
+}
+
+// GetEdbInfoMaxModifyTime
+// @Description:  根据指标来源和编码获取该指标数据最晚修改时间
+// @author: Roc
+// @datetime 2024-03-11 17:01:01
+// @param source int
+// @param edbCode string
+// @return modifyTime string
+// @return err error
+func GetEdbInfoMaxModifyTime(source, subSource int, edbCode string) (modifyTime string, err error) {
+	o := orm.NewOrmUsingDB("data")
+	tableName := GetEdbDataTableName(source, subSource)
+	if tableName == "" {
+		err = errors.New("无效的表名称:source:" + strconv.Itoa(source))
+		return
+	}
+
+	sql := ` SELECT MAX(modify_time) AS modify_time FROM %s WHERE edb_code=? `
+	sql = fmt.Sprintf(sql, tableName)
+	err = o.Raw(sql, edbCode).QueryRow(&modifyTime)
+
+	return
+}

models/data_manage/edb_info_calculate_bp.go

@@ -1,166 +0,0 @@
-package data_manage
-
-import (
-	"eta/eta_task/utils"
-	"fmt"
-	"github.com/beego/beego/v2/client/orm"
-	"github.com/shopspring/decimal"
-	"strconv"
-	"strings"
-	"time"
-)
-
-type EdbInfoCalculateBp struct {
-	EdbInfoCalculateBpId int       `orm:"column(edb_info_calculate_bp_id);pk"`
-	EdbInfoId            int       `description:"指标id"`
-	EdbCode              string    `description:"指标编码"`
-	FromEdbInfoId        int       `description:"计算指标id"`
-	FromEdbCode          string    `description:"计算指标编码"`
-	FromEdbName          string    `description:"计算指标名称"`
-	FromSource           int       `description:"计算指标来源"`
-	FromSourceName       string    `description:"计算指标来源名称"`
-	FromTag              string    `description:"来源指标标签"`
-	Sort                 int       `description:"计算指标名称排序"`
-	CreateTime           time.Time `description:"创建时间"`
-	ModifyTime           time.Time `description:"修改时间"`
-}
-
-// 变频
-func RefreshCalculateBp(edbInfoId int, fromEdbInfo *EdbInfo, edbCode, startDate, endDate string) (err error) {
-	o := orm.NewOrm()
-	tx, err := o.Begin()
-	if err != nil {
-		return err
-	}
-	defer func() {
-		if err != nil {
-			tx.Rollback()
-		} else {
-			tx.Commit()
-		}
-	}()
-
-	if err != nil {
-		return
-	}
-	edbInfoIdStr := strconv.Itoa(edbInfoId)
-	//计算数据
-	var condition string
-	var pars []interface{}
-	condition += " AND edb_info_id=? "
-	pars = append(pars, fromEdbInfo.EdbInfoId)
-	if startDate != "" {
-		condition += " AND data_time>=? "
-		pars = append(pars, startDate)
-	}
-
-	if endDate != "" {
-		condition += " AND data_time<=? "
-		pars = append(pars, endDate)
-	}
-	dataList, err := GetEdbDataListAll(condition, pars, fromEdbInfo.Source, 0)
-	if err != nil {
-		return err
-	}
-
-	var existCondition string
-	var existPars []interface{}
-	existCondition += " AND edb_info_id=? "
-	existPars = append(existPars, edbInfoId)
-	dataExistList, err := GetEdbDataListAll(existCondition, existPars, utils.DATA_SOURCE_CALCULATE_BP, 0)
-	if err != nil {
-		return err
-	}
-
-	existDataMap := make(map[string]*EdbInfoSearchData)
-	for _, v := range dataExistList {
-		existDataMap[v.DataTime] = v
-	}
-
-	addExistMap := make(map[string]string)
-	dataLen := len(dataList)
-	addSql := ` INSERT INTO edb_data_calculate_bp(edb_info_id,edb_code,data_time,value,create_time,modify_time,status,data_timestamp) values `
-	var isAdd bool
-	for i := 0; i < dataLen; i++ {
-		//当期
-		currentItem := dataList[i]
-		currentDate, _ := time.Parse(utils.FormatDate, currentItem.DataTime)
-		var day int
-		if i == 0 {
-			day = int(time.Now().Sub(currentDate).Hours() / float64(24))
-		} else {
-			j := i + 1
-			if j < dataLen {
-				preItem := dataList[j]
-				preDate, _ := time.Parse(utils.FormatDate, preItem.DataTime)
-				day = int(currentDate.Sub(preDate).Hours() / float64(24))
-			}
-		}
-		for k := 1; k <= day; k++ {
-			needDayForm := currentDate.AddDate(0, 0, k)
-			if needDayForm.Before(time.Now()) {
-				needDay := needDayForm.Format(utils.FormatDate)
-				if _, ok := existDataMap[needDay]; !ok {
-					existKey := edbCode + needDay
-					if _, ok := addExistMap[existKey]; !ok {
-						currentDate, _ := time.Parse(utils.FormatDate, currentItem.DataTime)
-						timestamp := currentDate.UnixNano() / 1e6
-						timestampStr := fmt.Sprintf("%d", timestamp)
-						valStr := decimal.NewFromFloat(currentItem.Value).String()
-						isAdd = true
-						addSql += GetAddSql(edbInfoIdStr, edbCode, needDay, timestampStr, valStr)
-					}
-					addExistMap[existKey] = needDay
-				}
-			}
-		}
-
-		if _, ok := existDataMap[currentItem.DataTime]; !ok {
-			existKey := edbCode + currentItem.DataTime
-			if _, ok := addExistMap[existKey]; !ok {
-				currentDate, _ := time.Parse(utils.FormatDate, currentItem.DataTime)
-				timestamp := currentDate.UnixNano() / 1e6
-				timestampStr := fmt.Sprintf("%d", timestamp)
-				valStr := decimal.NewFromFloat(currentItem.Value).String()
-				isAdd = true
-				addSql += GetAddSql(edbInfoIdStr, edbCode, currentItem.DataTime, timestampStr, valStr)
-			}
-			addExistMap[existKey] = currentItem.DataTime
-		}
-	}
-	if isAdd {
-		addSql = strings.TrimRight(addSql, ",")
-		_, err = tx.Raw(addSql).Exec()
-		if err != nil {
-			fmt.Println("add err:" + err.Error())
-			return err
-		}
-	}
-	return
-}
-
-type EdbInfoCalculateBpDetail struct {
-	EdbInfoCalculateBpId int       `orm:"column(edb_info_calculate_bp_id);pk"`
-	EdbInfoId            int       `description:"指标id"`
-	EdbCode              string    `description:"指标编码"`
-	FromEdbInfoId        int       `description:"计算指标id"`
-	FromEdbCode          string    `description:"计算指标编码"`
-	FromEdbName          string    `description:"计算指标名称"`
-	FromSource           int       `description:"计算指标来源"`
-	FromSourceName       string    `description:"计算指标来源名称"`
-	FromTag              string    `description:"来源指标标签"`
-	Sort                 int       `description:"计算指标名称排序"`
-	CreateTime           time.Time `description:"创建时间"`
-	ModifyTime           time.Time `description:"修改时间"`
-	StartDate            string    `description:"开始日期"`
-	EndDate              string    `description:"结束日期"`
-}
-
-func GetEdbInfoCalculateBpDetail(edbInfoId int) (item *EdbInfoCalculateTbzDetail, err error) {
-	o := orm.NewOrm()
-	sql := ` SELECT a.*,b.start_date,b.end_date FROM edb_info_calculate_mapping AS a
-			INNER JOIN edb_info AS b ON a.edb_info_id=b.edb_info_id
-			WHERE a.edb_info_id=? `
-	err = o.Raw(sql, edbInfoId).QueryRow(&item)
-	return
-}

models/data_manage/edb_info_calculate_hbz.go

@@ -1,165 +0,0 @@
-package data_manage
-
-import (
-	"eta/eta_task/utils"
-	"fmt"
-	"github.com/beego/beego/v2/client/orm"
-	"github.com/shopspring/decimal"
-	"strconv"
-	"strings"
-	"time"
-)
-
-type EdbInfoCalculateHbz struct {
-	EdbInfoCalculateHbzId int       `orm:"column(edb_info_calculate_hbz_id);pk"`
-	EdbInfoId             int       `description:"指标id"`
-	EdbCode               string    `description:"指标编码"`
-	FromEdbInfoId         int       `description:"计算指标id"`
-	FromEdbCode           string    `description:"计算指标编码"`
-	FromEdbName           string    `description:"计算指标名称"`
-	FromSource            int       `description:"计算指标来源"`
-	FromSourceName        string    `description:"计算指标来源名称"`
-	FromTag               string    `description:"来源指标标签"`
-	Sort                  int       `description:"计算指标名称排序"`
-	CreateTime            time.Time `description:"创建时间"`
-	ModifyTime            time.Time `description:"修改时间"`
-}
-
-// 环比值，current:当期,pre:上期 公式: (当期-上期）/上期
-func HbzDiv(current, pre float64) string {
-	if pre == 0 {
-		return ""
-	}
-	currentVal := decimal.NewFromFloat(float64(current))
-	preVal := decimal.NewFromFloat(float64(pre))
-	val, _ := currentVal.Sub(preVal).Div(preVal).Float64()
-	valStr := utils.SubFloatToString(val, 4)
-	return valStr
-}
-
-// 刷新环比值数据
-func RefreshCalculateHbz(edbInfoId, source int, fromEdbInfo *EdbInfo, edbCode, startDate, endDate string, formulaInt int) (err error) {
-	o := orm.NewOrm()
-	tx, err := o.Begin()
-	if err != nil {
-		return err
-	}
-	defer func() {
-		if err != nil {
-			tx.Rollback()
-		} else {
-			tx.Commit()
-		}
-	}()
-
-	if err != nil {
-		return
-	}
-	edbInfoIdStr := strconv.Itoa(edbInfoId)
-
-	var condition string
-	var pars []interface{}
-	condition += " AND edb_info_id=? "
-	pars = append(pars, fromEdbInfo.EdbInfoId)
-
-	if startDate != "" {
-		condition += " AND data_time>=? "
-		pars = append(pars, startDate)
-	}
-	if endDate != "" {
-		condition += " AND data_time<=? "
-		pars = append(pars, endDate)
-	}
-
-	dataList, err := GetEdbDataListAll(condition, pars, fromEdbInfo.Source, 0)
-	if err != nil {
-		return err
-	}
-
-	//获取指标所有数据
-	existDataList := make([]*EdbDataBase, 0)
-	dataTableName := GetEdbDataTableName(source)
-	fmt.Println("dataTableName:", dataTableName)
-	sql := `SELECT * FROM %s WHERE edb_info_id=? `
-	sql = fmt.Sprintf(sql, dataTableName)
-	_, err = o.Raw(sql, edbInfoId).QueryRows(&existDataList)
-	if err != nil {
-		return err
-	}
-	existDataMap := make(map[string]string)
-	for _, v := range existDataList {
-		existDataMap[v.DataTime] = v.Value
-	}
-	addSql := ` INSERT INTO edb_data_calculate_hbz(edb_info_id,edb_code,data_time,value,create_time,modify_time,status,data_timestamp) values `
-	var isAdd bool
-	existMap := make(map[string]string)
-	dataLen := len(dataList)
-	fmt.Println("dataLen:", dataLen)
-	for i := 0; i < dataLen; i++ {
-		j := i + formulaInt
-		if j < dataLen {
-			//当期
-			currentItem := dataList[i]
-			preItem := dataList[j]
-			if currentItem != nil && preItem != nil {
-				existKey := edbCode + currentItem.DataTime
-				if _, ok := existMap[existKey]; !ok {
-					currentDate, _ := time.Parse(utils.FormatDate, currentItem.DataTime)
-					timestamp := currentDate.UnixNano() / 1e6
-					timestampStr := fmt.Sprintf("%d", timestamp)
-					val := HbzDiv(currentItem.Value, preItem.Value)
-					if val != "" {
-						if existVal, findOk := existDataMap[currentItem.DataTime]; !findOk {
-							addSql += GetAddSql(edbInfoIdStr, edbCode, currentItem.DataTime, timestampStr, val)
-							isAdd = true
-						} else {
-							if existVal != val {
-								sql := ` UPDATE %s SET value=?,modify_time=NOW() WHERE edb_info_id=? AND data_time=? `
-								sql = fmt.Sprintf(sql, dataTableName)
-								_, err = tx.Raw(sql, val, edbInfoId, currentItem.DataTime).Exec()
-								if err != nil {
-									return err
-								}
-							}
-						}
-					}
-				}
-				existMap[existKey] = currentItem.DataTime
-			}
-		}
-	}
-	if isAdd {
-		addSql = strings.TrimRight(addSql, ",")
-		_, err = o.Raw(addSql).Exec()
-		if err != nil {
-			return err
-		}
-	}
-	return
-}
-
-type EdbInfoCalculateHbzDetail struct {
-	EdbInfoCalculateTbzId int       `orm:"column(edb_info_calculate_tbz_id);pk"`
-	EdbInfoId             int       `description:"指标id"`
-	EdbCode               string    `description:"指标编码"`
-	FromEdbInfoId         int       `description:"计算指标id"`
-	FromEdbCode           string    `description:"计算指标编码"`
-	FromEdbName           string    `description:"计算指标名称"`
-	FromSource            int       `description:"计算指标来源"`
-	FromSourceName        string    `description:"计算指标来源名称"`
-	FromTag               string    `description:"来源指标标签"`
-	Sort                  int       `description:"计算指标名称排序"`
-	CreateTime            time.Time `description:"创建时间"`
-	ModifyTime            time.Time `description:"修改时间"`
-	StartDate             string    `description:"开始日期"`
-	EndDate               string    `description:"结束日期"`
-}
-
-func GetEdbInfoCalculateHbzDetail(edbInfoId int) (item *EdbInfoCalculateTbzDetail, err error) {
-	o := orm.NewOrm()
-	sql := ` SELECT a.*,b.start_date,b.end_date FROM edb_info_calculate_mapping AS a
-			INNER JOIN edb_info AS b ON a.edb_info_id=b.edb_info_id
-			WHERE a.edb_info_id=? `
-	err = o.Raw(sql, edbInfoId).QueryRow(&item)
-	return
-}

models/data_manage/edb_info_calculate_hcz.go

@@ -1,164 +0,0 @@
-package data_manage
-
-import (
-	"eta/eta_task/utils"
-	"fmt"
-	"github.com/beego/beego/v2/client/orm"
-	"github.com/shopspring/decimal"
-	"strconv"
-	"strings"
-	"time"
-)
-
-type EdbInfoCalculateHcz struct {
-	EdbInfoCalculateHczId int       `orm:"column(edb_info_calculate_hcz_id);pk"`
-	EdbInfoId             int       `description:"指标id"`
-	EdbCode               string    `description:"指标编码"`
-	FromEdbInfoId         int       `description:"计算指标id"`
-	FromEdbCode           string    `description:"计算指标编码"`
-	FromEdbName           string    `description:"计算指标名称"`
-	FromSource            int       `description:"计算指标来源"`
-	FromSourceName        string    `description:"计算指标来源名称"`
-	FromTag               string    `description:"来源指标标签"`
-	Sort                  int       `description:"计算指标名称排序"`
-	CreateTime            time.Time `description:"创建时间"`
-	ModifyTime            time.Time `description:"修改时间"`
-}
-
-// 环差值，current:当期,pre:上期 公式:当期-上期
-func HczDiv(current, pre float64) string {
-	if pre == 0 {
-		return ""
-	}
-	currentVal := decimal.NewFromFloat(current)
-	preVal := decimal.NewFromFloat(pre)
-	val, _ := currentVal.Sub(preVal).Float64()
-	valStr := utils.SubFloatToString(val, 4)
-	return valStr
-}
-
-// 刷新环比值数据
-func RefreshCalculateHcz(edbInfoId, source int, fromEdbInfo *EdbInfo, edbCode, startDate, endDate string) (err error) {
-	o := orm.NewOrm()
-	tx, err := o.Begin()
-	if err != nil {
-		return err
-	}
-	defer func() {
-		if err != nil {
-			tx.Rollback()
-		} else {
-			tx.Commit()
-		}
-	}()
-
-	if err != nil {
-		return
-	}
-	edbInfoIdStr := strconv.Itoa(edbInfoId)
-
-	var condition string
-	var pars []interface{}
-	condition += " AND edb_info_id=? "
-	pars = append(pars, fromEdbInfo.EdbInfoId)
-
-	if startDate != "" {
-		condition += " AND data_time>=? "
-		pars = append(pars, startDate)
-	}
-	if endDate != "" {
-		condition += " AND data_time<=? "
-		pars = append(pars, endDate)
-	}
-
-	dataList, err := GetEdbDataListAll(condition, pars, fromEdbInfo.Source, 0)
-	if err != nil {
-		return err
-	}
-
-	//获取指标所有数据
-	existDataList := make([]*EdbDataBase, 0)
-	dataTableName := GetEdbDataTableName(source)
-	fmt.Println("dataTableName:", dataTableName)
-	sql := `SELECT * FROM %s WHERE edb_info_id=? `
-	sql = fmt.Sprintf(sql, dataTableName)
-	_, err = o.Raw(sql, edbInfoId).QueryRows(&existDataList)
-	if err != nil {
-		return err
-	}
-	existDataMap := make(map[string]string)
-	for _, v := range existDataList {
-		existDataMap[v.DataTime] = v.Value
-	}
-	addSql := ` INSERT INTO edb_data_calculate_hcz(edb_info_id,edb_code,data_time,value,create_time,modify_time,status,data_timestamp) values `
-	var isAdd bool
-	existMap := make(map[string]string)
-	dataLen := len(dataList)
-	for i := 0; i < dataLen; i++ {
-		j := i + 1
-		if j < dataLen {
-			//当期
-			currentItem := dataList[i]
-			preItem := dataList[j]
-			if currentItem != nil && preItem != nil {
-				existKey := edbCode + currentItem.DataTime
-				if _, ok := existMap[existKey]; !ok {
-					currentDate, _ := time.Parse(utils.FormatDate, currentItem.DataTime)
-					timestamp := currentDate.UnixNano() / 1e6
-					timestampStr := fmt.Sprintf("%d", timestamp)
-					val := HczDiv(currentItem.Value, preItem.Value)
-					if val != "" {
-						if existVal, findOk := existDataMap[currentItem.DataTime]; !findOk {
-							addSql += GetAddSql(edbInfoIdStr, edbCode, currentItem.DataTime, timestampStr, val)
-							isAdd = true
-						} else {
-							if existVal != val {
-								sql := ` UPDATE %s SET value=?,modify_time=NOW() WHERE edb_info_id=? AND data_time=? `
-								sql = fmt.Sprintf(sql, dataTableName)
-								_, err = tx.Raw(sql, val, edbInfoId, currentItem.DataTime).Exec()
-								if err != nil {
-									return err
-								}
-							}
-						}
-					}
-				}
-				existMap[existKey] = currentItem.DataTime
-			}
-		}
-	}
-	if isAdd {
-		addSql = strings.TrimRight(addSql, ",")
-		_, err = o.Raw(addSql).Exec()
-		if err != nil {
-			return err
-		}
-	}
-	return
-}
-
-type EdbInfoCalculateHczDetail struct {
-	EdbInfoCalculateHczId int       `orm:"column(edb_info_calculate_hcz_id);pk"`
-	EdbInfoId             int       `description:"指标id"`
-	EdbCode               string    `description:"指标编码"`
-	FromEdbInfoId         int       `description:"计算指标id"`
-	FromEdbCode           string    `description:"计算指标编码"`
-	FromEdbName           string    `description:"计算指标名称"`
-	FromSource            int       `description:"计算指标来源"`
-	FromSourceName        string    `description:"计算指标来源名称"`
-	FromTag               string    `description:"来源指标标签"`
-	Sort                  int       `description:"计算指标名称排序"`
-	CreateTime            time.Time `description:"创建时间"`
-	ModifyTime            time.Time `description:"修改时间"`
-	StartDate             string    `description:"开始日期"`
-	EndDate               string    `description:"结束日期"`
-}
-
-func GetEdbInfoCalculateHczDetail(edbInfoId int) (item *EdbInfoCalculateHczDetail, err error) {
-	o := orm.NewOrm()
-	sql := ` SELECT a.*,b.start_date,b.end_date FROM edb_info_calculate_mapping AS a
-			INNER JOIN edb_info AS b ON a.edb_info_id=b.edb_info_id
-			WHERE a.edb_info_id=? `
-	err = o.Raw(sql, edbInfoId).QueryRow(&item)
-	return
-}

models/data_manage/edb_info_calculate_ljztbpj.go

@@ -1,289 +0,0 @@
-package data_manage
-
-import (
-	"errors"
-	"eta/eta_task/utils"
-	"fmt"
-	"github.com/beego/beego/v2/client/orm"
-	"github.com/shopspring/decimal"
-	"sort"
-	"strings"
-	"time"
-)
-
-type EdbInfoCalculateLjztbpjDetail struct {
-	EdbInfoCalculateBpId int       `orm:"column(edb_info_calculate_bp_id);pk"`
-	EdbInfoId            int       `description:"指标id"`
-	EdbCode              string    `description:"指标编码"`
-	FromEdbInfoId        int       `description:"计算指标id"`
-	FromEdbCode          string    `description:"计算指标编码"`
-	FromEdbName          string    `description:"计算指标名称"`
-	FromSource           int       `description:"计算指标来源"`
-	FromSourceName       string    `description:"计算指标来源名称"`
-	FromTag              string    `description:"来源指标标签"`
-	Sort                 int       `description:"计算指标名称排序"`
-	CreateTime           time.Time `description:"创建时间"`
-	ModifyTime           time.Time `description:"修改时间"`
-	StartDate            string    `description:"开始日期"`
-	EndDate              string    `description:"结束日期"`
-}
-
-// RefreshAllCalculateLjztbpj 刷新所有 累计值同比拼接 数据
-func RefreshAllCalculateLjztbpj(edbInfo *EdbInfo) (err error) {
-	o := orm.NewOrm()
-	tx, err := o.Begin()
-	if err != nil {
-		return err
-	}
-	defer func() {
-		if err != nil {
-			tx.Rollback()
-		} else {
-			tx.Commit()
-		}
-	}()
-
-	//查询当前指标现有的数据
-	var condition string
-	var pars []interface{}
-	condition += " AND edb_info_id=? "
-	pars = append(pars, edbInfo.EdbInfoId)
-
-	//所有的数据
-	dataList, err := GetAllEdbDataCalculateLjztbpjByEdbInfoId(edbInfo.EdbInfoId)
-	if err != nil {
-		return err
-	}
-
-	//待拼接指标map
-	pjDataMap := make(map[string]float64)     //需要入库的数据
-	nowEdbDataMap := make(map[string]float64) //当前指标的数据（已经在库里了，不需要重新）
-	//拼接指标的日期切片数据
-	pjEdbDataTimeList := make([]string, 0)
-
-	dataMap := make(map[string]*EdbDataCalculateLjztbpj)
-	for _, v := range dataList {
-		pjEdbDataTimeList = append(pjEdbDataTimeList, v.DataTime)
-		dataMap[v.DataTime] = v
-		nowEdbDataMap[v.DataTime] = v.Value
-	}
-
-	//查询关联指标信息
-	var existCondition string
-	var existPars []interface{}
-	existCondition += " AND edb_info_id=? "
-	existPars = append(existPars, edbInfo.EdbInfoId)
-	existList, err := GetEdbInfoCalculateListByCondition(existCondition, existPars)
-	if err != nil {
-		err = errors.New("判断指标是否改变失败,Err:" + err.Error())
-		return
-	}
-
-	var existItemA, existItemB *EdbInfoCalculateMapping
-	for _, existItem := range existList {
-		if existItem.FromTag == "A" {
-			existItemA = existItem
-		} else if existItem.FromTag == "B" {
-			existItemB = existItem
-		}
-	}
-	// 原数据开始计算日期
-	startCalculationDate, _ := time.ParseInLocation(utils.FormatDate, edbInfo.CalculateFormula, time.Local)
-
-	//待拼接指标
-	{
-		var condition string
-		var pars []interface{}
-
-		condition += " AND data_time <= ? AND edb_info_id=? "
-		pars = append(pars, startCalculationDate, existItemA.FromEdbInfoId)
-
-		//第一个指标的数据列表
-		firstDataList, tmpErr := GetEdbDataListAll(condition, pars, existItemA.FromSource, 0)
-		if tmpErr != nil {
-			return tmpErr
-		}
-
-		for _, v := range firstDataList {
-			//时间戳
-			if edbData, ok := dataMap[v.DataTime]; ok {
-				if edbData.Value != v.Value {
-					//更新指标数据
-					edbData.Value = v.Value
-					o.Update(edbData, "Value")
-
-					//将新的数据存入已入库指标map里面，以便后续计算
-					nowEdbDataMap[edbData.DataTime] = v.Value
-				}
-			}
-		}
-	}
-
-	//同比值指标map
-	tbzEdbDataMap := make(map[string]float64)
-
-	//同比值日期切片列表
-	tbzEdbDataTimeList := make([]string, 0)
-
-	//同比值指标
-	{
-		var condition string
-		var pars []interface{}
-
-		condition += " AND data_time > ? AND edb_info_id = ? "
-		pars = append(pars, startCalculationDate, existItemB.FromEdbInfoId)
-
-		//第二个指标的数据列表
-		secondDataList, tmpErr := GetEdbDataListAll(condition, pars, existItemB.FromSource, 0)
-		if tmpErr != nil {
-			return tmpErr
-		}
-
-		for _, v := range secondDataList {
-			tbzEdbDataMap[v.DataTime] = v.Value
-			tbzEdbDataTimeList = append(tbzEdbDataTimeList, v.DataTime)
-		}
-	}
-
-	sort.Strings(tbzEdbDataTimeList)
-
-	// 遍历现有的数据，判断拼接指标中是否存在该日期数据，如果拼接指标无此数据，那么需要删除该日期数据（日期的判断：需要在开始计算日期之后）
-	removeDateList := make([]string, 0)
-	for nowEdbDate := range nowEdbDataMap {
-		nowEdbDateTime, _ := time.ParseInLocation(utils.FormatDate, nowEdbDate, time.Local)
-		//校验日期 需要 大于 拼接前日期
-		if startCalculationDate.Before(nowEdbDateTime) {
-			if _, ok := tbzEdbDataMap[nowEdbDate]; !ok {
-				// 同比指标中，不存在该日期数据，那么需要移除 现有数据 中该日期的数据
-				removeDateList = append(removeDateList, nowEdbDate)
-			}
-		}
-	}
-
-	//待修改的指标数据map（index：日期，value：值）
-	updateEdbDataMap := make(map[string]float64)
-	for _, v := range tbzEdbDataTimeList {
-		tbzDataTime, _ := time.ParseInLocation(utils.FormatDate, v, time.Local)
-
-		//获取拼接指标上一年同一天的数据
-		var pjDataTime time.Time
-		if tbzDataTime.Month() == 2 {
-			pjDataTime = tbzDataTime.AddDate(0, -11, 0)
-			pjDataTime = time.Date(pjDataTime.Year(), pjDataTime.Month(), 1, 0, 0, 0, 0, time.Now().Location()).AddDate(0, 0, -1)
-		} else {
-			pjDataTime = tbzDataTime.AddDate(-1, 0, 0)
-		}
-
-		//校验现有数据中，是否存在该日期的数据，如果存在的话，那么就要去校验 最新计算数据 与 现有数据 是否一致
-		if nowEdbDataValue, isHas := nowEdbDataMap[v]; isHas {
-			//获取去年今日的数据，获取到后，然后是去修改该日期的数据
-			if lastYearEdbDataValue, ok := nowEdbDataMap[pjDataTime.Format(utils.FormatDate)]; ok {
-
-				if v == "2021-08-31" {
-					fmt.Println("进来了")
-				}
-				tbzDataValue := tbzEdbDataMap[v] //同比值
-				currValue := lastYearEdbDataValue * (1 + tbzDataValue/100)
-				currValue, _ = decimal.NewFromFloat(currValue).Truncate(4).Float64() //保留4位小数
-				//如果计算出来的值与库里面的值不匹配，那么就去修改该值
-				if nowEdbDataValue != currValue {
-					//将计算后的数据存入待拼接指标map里面，以便后续计算
-					updateEdbDataMap[v] = currValue
-				}
-			}
-		} else {
-			//因为 现有数据中 不存在该日期数据，那么需要做新增数据处理
-
-			//如果去年今日存在该数据，那么就去计算当前的数据
-			if pjDataValue, ok := nowEdbDataMap[pjDataTime.Format(utils.FormatDate)]; ok {
-				tbzDataValue := tbzEdbDataMap[v] //同比值
-				currValue := pjDataValue * (1 + tbzDataValue/100)
-
-				currValue, _ = decimal.NewFromFloat(currValue).Truncate(4).Float64()
-				//将计算后的数据存入已入库指标map里面，以便后续计算
-				nowEdbDataMap[v] = currValue
-
-				//将计算后的数据存入待拼接指标map里面，以便后续入库
-				pjDataMap[v] = currValue
-				pjEdbDataTimeList = append(pjEdbDataTimeList, v)
-			}
-		}
-	}
-
-	//新增的数据入库
-	{
-		addDataList := make([]*EdbDataCalculateLjztbpj, 0)
-		for dataTime, dataValue := range pjDataMap {
-			//时间戳
-			currentDate, _ := time.Parse(utils.FormatDate, dataTime)
-			timestamp := currentDate.UnixNano() / 1e6
-
-			edbDataLjztbpj := &EdbDataCalculateLjztbpj{
-				EdbInfoId:     edbInfo.EdbInfoId,
-				EdbCode:       edbInfo.EdbCode,
-				DataTime:      dataTime,
-				Value:         dataValue,
-				Status:        1,
-				CreateTime:    time.Now(),
-				ModifyTime:    time.Now(),
-				DataTimestamp: timestamp,
-			}
-			addDataList = append(addDataList, edbDataLjztbpj)
-		}
-
-		tmpAddDataList := make([]*EdbDataCalculateLjztbpj, 0)
-		for _, v := range addDataList {
-			tmpAddDataList = append(tmpAddDataList, v)
-
-			if len(tmpAddDataList) >= 200 {
-				_, tmpErr := tx.InsertMulti(len(tmpAddDataList), tmpAddDataList)
-				if tmpErr != nil {
-					err = tmpErr
-					return
-				}
-				//重新初始化需要加入的数据切片
-				tmpAddDataList = make([]*EdbDataCalculateLjztbpj, 0)
-			}
-		}
-		//最后如果还有需要新增的数据，那么就统一入库
-		if len(tmpAddDataList) > 0 {
-			_, tmpErr := tx.InsertMulti(len(tmpAddDataList), tmpAddDataList)
-			if tmpErr != nil {
-				err = tmpErr
-				return
-			}
-		}
-	}
-
-	//删除已经不存在的累计同比拼接指标数据（由于同比值当日的数据删除了）
-	{
-		if len(removeDateList) > 0 {
-			removeDateStr := strings.Join(removeDateList, `","`)
-			removeDateStr = `"` + removeDateStr + `"`
-			//如果拼接指标变更了，那么需要删除所有的指标数据
-			tableName := GetEdbDataTableName(edbInfo.Source)
-			sql := fmt.Sprintf(` DELETE FROM %s WHERE edb_info_id = ? and data_time in (%s) `, tableName, removeDateStr)
-
-			_, err = tx.Raw(sql, edbInfo.EdbInfoId).Exec()
-			if err != nil {
-				err = errors.New("删除不存在的累计值同比拼接指标数据失败,Err:" + err.Error())
-				return
-			}
-		}
-	}
-
-	//修改现有的数据中对应的值
-	{
-		tableName := GetEdbDataTableName(edbInfo.Source)
-		for edbDate, edbDataValue := range updateEdbDataMap {
-			sql := fmt.Sprintf(` UPDATE  %s set value = ?,modify_time=now() WHERE edb_info_id = ? and data_time = ? `, tableName)
-
-			_, err = tx.Raw(sql, edbDataValue, edbInfo.EdbInfoId, edbDate).Exec()
-			if err != nil {
-				err = errors.New("更新现有的累计值同比拼接指标数据失败,Err:" + err.Error())
-				return
-			}
-		}
-	}
-	return
-}

models/data_manage/edb_info_calculate_ljzzy.go

@@ -1,217 +0,0 @@
-package data_manage
-
-import (
-	"eta/eta_task/utils"
-	"fmt"
-	"github.com/beego/beego/v2/client/orm"
-	"github.com/shopspring/decimal"
-	"strconv"
-	"strings"
-	"time"
-)
-
-type EdbInfoCalculateLjzzy struct {
-	EdbInfoCalculateLjzzyId int       `orm:"column(edb_info_calculate_ljzzy_id);pk"`
-	EdbInfoId               int       `description:"指标id"`
-	EdbCode                 string    `description:"指标编码"`
-	FromEdbInfoId           int       `description:"计算指标id"`
-	FromEdbCode             string    `description:"计算指标编码"`
-	FromEdbName             string    `description:"计算指标名称"`
-	FromSource              int       `description:"计算指标来源"`
-	FromSourceName          string    `description:"计算指标来源名称"`
-	FromTag                 string    `description:"来源指标标签"`
-	Sort                    int       `description:"计算指标名称排序"`
-	CreateTime              time.Time `description:"创建时间"`
-	ModifyTime              time.Time `description:"修改时间"`
-}
-
-// 刷新累计值转月值数据
-func RefreshCalculateLjzzy(edbInfoId int, fromEdbInfo *EdbInfo, edbCode, startDate, endDate string) (err error) {
-	o := orm.NewOrm()
-	tx, err := o.Begin()
-	if err != nil {
-		return err
-	}
-	defer func() {
-		if err != nil {
-			tx.Rollback()
-		} else {
-			tx.Commit()
-		}
-	}()
-
-	if err != nil {
-		return
-	}
-	edbInfoIdStr := strconv.Itoa(edbInfoId)
-	//计算数据
-	var condition string
-	var pars []interface{}
-	condition += " AND edb_info_id=? "
-	pars = append(pars, fromEdbInfo.EdbInfoId)
-
-	if startDate != "" {
-		condition += " AND data_time>=? "
-		pars = append(pars, startDate)
-	}
-	if endDate != "" {
-		condition += " AND data_time<=? "
-		pars = append(pars, endDate)
-	}
-
-	dataList, err := GetEdbDataListAll(condition, pars, fromEdbInfo.Source, 1)
-	if err != nil {
-		return err
-	}
-
-	yearMap := make(map[int]map[int]*EdbInfoSearchData)
-	dataLen := len(dataList)
-	for i := 0; i < dataLen; i++ {
-		item := dataList[i]
-		//日其中获取年
-		itemDate, err := time.Parse(utils.FormatDate, item.DataTime)
-		if err != nil {
-			return err
-		}
-		year := itemDate.Year()
-		month := int(itemDate.Month())
-		if monthMap, yok := yearMap[year]; yok {
-			monthMap[month] = item
-			yearMap[year] = monthMap
-		} else {
-			monthMap = make(map[int]*EdbInfoSearchData)
-			monthMap[month] = item
-			yearMap[year] = monthMap
-		}
-	}
-
-	addSql := ` INSERT INTO edb_data_calculate_ljzzy(edb_info_id,edb_code,data_time,value,create_time,modify_time,status,data_timestamp) values `
-	nowStr := time.Now().Format(utils.FormatDateTime)
-	var isAdd bool
-
-	for yk, yv := range yearMap {
-		_, oneMonthOk := yv[1]
-		_, twoMonthOk := yv[2]
-		if !oneMonthOk && !twoMonthOk {
-			continue
-		}
-		for i := 1; i <= 12; i++ {
-			fmt.Println(yk, i, yv[i])
-			dataCurrentItem := yv[i]
-			var date string
-			var val float64
-			if i == 1 || i == 2 {
-				if _, mok := yv[1]; mok { //1月有值
-					if i == 1 {
-						date = dataCurrentItem.DataTime
-						val, _ = decimal.NewFromFloat(dataCurrentItem.Value).Float64() //a.Div(b).Float64()
-					}
-					if i == 2 {
-						dataOneItem := yv[1]
-						date = dataCurrentItem.DataTime
-						twoMonth := decimal.NewFromFloat(dataCurrentItem.Value)
-						oneMonth := decimal.NewFromFloat(dataOneItem.Value)
-						val, _ = twoMonth.Sub(oneMonth).Float64()
-					}
-				} else { //1月无值
-					dataTwoItem := yv[2]
-					if i == 1 {
-						date = strconv.Itoa(yk) + "-01-31"
-						a := decimal.NewFromFloat(dataTwoItem.Value)
-						b := decimal.NewFromFloat(2.0)
-						val, _ = a.Div(b).Float64()
-					}
-					if i == 2 {
-						date = dataCurrentItem.DataTime
-						a := decimal.NewFromFloat(dataTwoItem.Value)
-						b := decimal.NewFromFloat(2.0)
-						val, _ = a.Div(b).Float64()
-					}
-				}
-			} else {
-				dataPreItem := yv[i-1]
-				if dataCurrentItem != nil && dataPreItem != nil {
-					date = dataCurrentItem.DataTime
-					//val, _ = decimal.NewFromFloat(dataCurrentItem.Value).Sub(decimal.NewFromFloat(dataPreItem.Value)).Float64()
-					a := decimal.NewFromFloat(dataCurrentItem.Value)
-					b := decimal.NewFromFloat(dataPreItem.Value)
-					val, _ = a.Sub(b).Float64()
-				}
-			}
-			if date != "" {
-				//判断数据是否存在
-				count, err := GetEdbDataCalculateLjzzyByCodeAndDate(edbCode, date)
-				if err != nil && err.Error() != utils.ErrNoRow() {
-					return err
-				}
-				if count <= 0 {
-					dataTime, _ := time.Parse(utils.FormatDate, date)
-					timestamp := dataTime.UnixNano() / 1e6
-					timeStr := fmt.Sprintf("%d", timestamp)
-
-					addSql += "("
-					addSql += edbInfoIdStr + "," + "'" + edbCode + "'" + "," + "'" + date + "'" + "," + utils.SubFloatToString(val, 4) + "," + "'" + nowStr + "'" +
-						"," + "'" + nowStr + "'" + "," + "1"
-					addSql += "," + "'" + timeStr + "'"
-					addSql += "),"
-					isAdd = true
-				} else {
-					//val = utils.FixFloat(val, 4)
-					valStr := utils.SubFloatToString(val, 4)
-					err = ModifyEdbDataCalculateLjzzy(int64(edbInfoId), date, valStr)
-					if err != nil {
-						return err
-					}
-				}
-			}
-		}
-	}
-	if isAdd {
-		addSql = strings.TrimRight(addSql, ",")
-		_, err = tx.Raw(addSql).Exec()
-		if err != nil {
-			return err
-		}
-	}
-	return
-}
-
-type EdbInfoCalculateLjzzyDetail struct {
-	EdbInfoCalculateLjzzyId int       `orm:"column(edb_info_calculate_ljzzy_id);pk"`
-	EdbInfoId               int       `description:"指标id"`
-	EdbCode                 string    `description:"指标编码"`
-	FromEdbInfoId           int       `description:"计算指标id"`
-	FromEdbCode             string    `description:"计算指标编码"`
-	FromEdbName             string    `description:"计算指标名称"`
-	FromSource              int       `description:"计算指标来源"`
-	FromSourceName          string    `description:"计算指标来源名称"`
-	FromTag                 string    `description:"来源指标标签"`
-	Sort                    int       `description:"计算指标名称排序"`
-	CreateTime              time.Time `description:"创建时间"`
-	ModifyTime              time.Time `description:"修改时间"`
-	StartDate               string    `description:"开始日期"`
-	EndDate                 string    `description:"结束日期"`
-}
-
-func GetEdbInfoCalculateLjzzyDetail(edbInfoId int) (item *EdbInfoCalculateLjzzyDetail, err error) {
-	o := orm.NewOrm()
-	sql := ` SELECT a.*,b.start_date,b.end_date FROM edb_info_calculate_mapping AS a
-			INNER JOIN edb_info AS b ON a.edb_info_id=b.edb_info_id
-			WHERE a.edb_info_id=? `
-	err = o.Raw(sql, edbInfoId).QueryRow(&item)
-	return
-}
-
-func GetEdbDataCalculateLjzzyByCodeAndDate(edbCode string, startDate string) (count int, err error) {
-	o := orm.NewOrm()
-	sql := ` SELECT COUNT(1) AS count FROM edb_data_calculate_ljzzy WHERE edb_code=? AND data_time=? `
-	err = o.Raw(sql, edbCode, startDate).QueryRow(&count)
-	return
-}
-
-func ModifyEdbDataCalculateLjzzy(edbInfoId int64, dataTime, value string) (err error) {
-	o := orm.NewOrm()
-	sql := ` UPDATE edb_data_calculate_ljzzy SET value=?,modify_time=NOW() WHERE edb_info_id=? AND data_time=? `
-	_, err = o.Raw(sql, value, edbInfoId, dataTime).Exec()
-	return
-}

models/data_manage/edb_info_calculate_nszydpjjs.go

@@ -1,192 +0,0 @@
-package data_manage
-
-import (
-	"eta/eta_task/utils"
-	"fmt"
-	"github.com/beego/beego/v2/client/orm"
-	"github.com/shopspring/decimal"
-	"strconv"
-	"strings"
-	"time"
-)
-
-type EdbInfoCalculateNszydpjjs struct {
-	EdbInfoCalculateNszydpjjsId int       `orm:"column(edb_info_calculate_nszydpjjs_id);pk"`
-	EdbInfoId                   int       `description:"指标id"`
-	EdbCode                     string    `description:"指标编码"`
-	FromEdbInfoId               int       `description:"计算指标id"`
-	FromEdbCode                 string    `description:"计算指标编码"`
-	FromEdbName                 string    `description:"计算指标名称"`
-	FromSource                  int       `description:"计算指标来源"`
-	FromSourceName              string    `description:"计算指标来源名称"`
-	FromTag                     string    `description:"来源指标标签"`
-	Sort                        int       `description:"计算指标名称排序"`
-	CreateTime                  time.Time `description:"创建时间"`
-	ModifyTime                  time.Time `description:"修改时间"`
-}
-
-// 刷新 N数值移动平均计算
-func RefreshCalculateNszydpjjs(edbInfoId, source, formulaInt int, fromEdbInfo *EdbInfo, edbCode string) (err error) {
-	o := orm.NewOrm()
-	tx, err := o.Begin()
-	if err != nil {
-		return err
-	}
-	defer func() {
-		if err != nil {
-			tx.Rollback()
-		} else {
-			tx.Commit()
-		}
-	}()
-
-	if err != nil {
-		return
-	}
-	edbInfoIdStr := strconv.Itoa(edbInfoId)
-
-	//计算数据
-	var condition string
-	var pars []interface{}
-	condition += " AND edb_info_id=? "
-	pars = append(pars, edbInfoId)
-
-	dataList, err := GetEdbDataListAllV1(condition, pars, source, 0)
-	if err != nil {
-		return err
-	}
-
-	existDataMap := make(map[string]*EdbInfoSearchDataV1)
-	for _, v := range dataList {
-		existDataMap[v.DataTime] = v
-	}
-
-	//计算来源数据
-
-	var fromCondition string
-	var fromPars []interface{}
-	fromCondition += " AND edb_info_id=? "
-	fromPars = append(fromPars, fromEdbInfo.EdbInfoId)
-
-	fromDataList, err := GetEdbDataListAll(fromCondition, fromPars, fromEdbInfo.Source, 0)
-	if err != nil {
-		return err
-	}
-
-	var fromDateArr []string
-	fromDataMap := make(map[string]*EdbInfoSearchData)
-	for _, v := range fromDataList {
-		fromDateArr = append(fromDateArr, v.DataTime)
-		fromDataMap[v.DataTime] = v
-	}
-
-	addSql := ` INSERT INTO edb_data_calculate_nszydpjjs(edb_info_id,edb_code,data_time,value,create_time,modify_time,status,data_timestamp) values `
-	var isAdd bool
-
-	arrLen := len(fromDateArr)
-	for ak, av := range fromDateArr {
-		//处理第一个值
-		var valArr []float64
-		if findItem, ok := fromDataMap[av]; ok {
-			valArr = append(valArr, findItem.Value)
-		} else {
-			continue
-		}
-		if ak+1 != arrLen {
-			//处理除第一个值之外的N-1个值
-			for i := 1; i < formulaInt; i++ {
-				arrIndex := ak + i
-				if arrIndex >= arrLen {
-					break
-				}
-				arrVal := fromDateArr[arrIndex]
-				if findItem, ok := fromDataMap[arrVal]; ok {
-					valArr = append(valArr, findItem.Value)
-				} else {
-					continue
-				}
-			}
-		}
-		valArrLen := len(valArr)
-		totalVal := decimal.NewFromFloat(0.00)
-		for _, v := range valArr {
-			//totalVal = totalVal + v
-			newDecimal := decimal.NewFromFloat(v)
-			totalVal = totalVal.Add(newDecimal)
-		}
-		//af := decimal.NewFromFloat(totalVal)
-		af := totalVal
-
-		bf := decimal.NewFromFloat(float64(valArrLen))
-		val, _ := af.Div(bf).Float64()
-		currentDate, err := time.Parse(utils.FormatDate, av)
-		if err != nil {
-			return err
-		}
-
-		timestamp := currentDate.UnixNano() / 1e6
-		timestampStr := fmt.Sprintf("%d", timestamp)
-		valStr := utils.SubFloatToString(val, 4)
-
-		if existItem, existOk := existDataMap[av]; !existOk {
-			addSql += GetAddSql(edbInfoIdStr, edbCode, av, timestampStr, valStr)
-			isAdd = true
-		} else {
-			if existItem != nil && existItem.Value != valStr {
-				err = ModifyEdbDataCalculateNszydpjjs(int64(edbInfoId), av, valStr)
-				if err != nil {
-					return err
-				}
-			}
-		}
-	}
-
-	if isAdd {
-		addSql = strings.TrimRight(addSql, ",")
-		_, err = tx.Raw(addSql).Exec()
-		if err != nil {
-			return err
-		}
-	}
-	return
-}
-
-type EdbInfoCalculateNszydpjjsDetail struct {
-	EdbInfoCalculateNszydpjjsId int       `orm:"column(edb_info_calculate_nszydpjjs_id);pk"`
-	EdbInfoId                   int       `description:"指标id"`
-	EdbCode                     string    `description:"指标编码"`
-	FromEdbInfoId               int       `description:"计算指标id"`
-	FromEdbCode                 string    `description:"计算指标编码"`
-	FromEdbName                 string    `description:"计算指标名称"`
-	FromSource                  int       `description:"计算指标来源"`
-	FromSourceName              string    `description:"计算指标来源名称"`
-	FromTag                     string    `description:"来源指标标签"`
-	Sort                        int       `description:"计算指标名称排序"`
-	CreateTime                  time.Time `description:"创建时间"`
-	ModifyTime                  time.Time `description:"修改时间"`
-	StartDate                   string    `description:"开始日期"`
-	EndDate                     string    `description:"结束日期"`
-}
-
-func GetEdbInfoCalculateNszydpjjsDetail(edbInfoId int) (item *EdbInfoCalculateNszydpjjsDetail, err error) {
-	o := orm.NewOrm()
-	sql := ` SELECT a.*,b.start_date,b.end_date FROM edb_info_calculate_mapping AS a
-			INNER JOIN edb_info AS b ON a.edb_info_id=b.edb_info_id
-			WHERE a.edb_info_id=? `
-	err = o.Raw(sql, edbInfoId).QueryRow(&item)
-	return
-}
-
-func GetEdbDataCalculateNszydpjjsByCodeAndDate(edbCode string, startDate string) (count int, err error) {
-	o := orm.NewOrm()
-	sql := ` SELECT COUNT(1) AS count FROM edb_data_calculate_nszydpjjs WHERE edb_code=? AND data_time=? `
-	err = o.Raw(sql, edbCode, startDate).QueryRow(&count)
-	return
-}
-
-func ModifyEdbDataCalculateNszydpjjs(edbInfoId int64, dataTime, value string) (err error) {
-	o := orm.NewOrm()
-	sql := ` UPDATE edb_data_calculate_nszydpjjs SET value=?,modify_time=NOW() WHERE edb_info_id=? AND data_time=? `
-	_, err = o.Raw(sql, value, edbInfoId, dataTime).Exec()
-	return
-}

models/data_manage/edb_info_calculate_tbz.go

@@ -1,302 +0,0 @@
-package data_manage
-
-import (
-	"eta/eta_task/utils"
-	"fmt"
-	"github.com/beego/beego/v2/client/orm"
-	"github.com/shopspring/decimal"
-	"strconv"
-	"strings"
-	"time"
-)
-
-type EdbInfoCalculateTbz struct {
-	EdbInfoCalculateTbzId int       `orm:"column(edb_info_calculate_tbz_id);pk"`
-	EdbInfoId             int       `description:"指标id"`
-	EdbCode               string    `description:"指标编码"`
-	FromEdbInfoId         int       `description:"计算指标id"`
-	FromEdbCode           string    `description:"计算指标编码"`
-	FromEdbName           string    `description:"计算指标名称"`
-	FromSource            int       `description:"计算指标来源"`
-	FromSourceName        string    `description:"计算指标来源名称"`
-	FromTag               string    `description:"来源指标标签"`
-	Sort                  int       `description:"计算指标名称排序"`
-	CreateTime            time.Time `description:"创建时间"`
-	ModifyTime            time.Time `description:"修改时间"`
-}
-
-func TbzDiv(a, b float64) string {
-	af := decimal.NewFromFloat(float64(a))
-	bf := decimal.NewFromFloat(float64(b))
-	val, _ := af.Div(bf).Float64()
-	val = val - 1
-	valStr := utils.SubFloatToString(val, 4)
-	return valStr
-}
-
-// 刷新同比值数据
-func RefreshCalculateTbz(edbInfoId int, fromEdbInfo *EdbInfo, edbCode, startDate, endDate string) (err error) {
-	o := orm.NewOrm()
-	tx, err := o.Begin()
-	if err != nil {
-		return err
-	}
-	defer func() {
-		if err != nil {
-			tx.Rollback()
-		} else {
-			tx.Commit()
-		}
-	}()
-
-	if err != nil {
-		return
-	}
-	edbInfoIdStr := strconv.Itoa(edbInfoId)
-	//计算数据
-
-	//计算数据
-	var condition string
-	var pars []interface{}
-	condition += " AND edb_info_id=? "
-	pars = append(pars, fromEdbInfo.EdbInfoId)
-
-	if startDate != "" {
-		condition += " AND data_time>=? "
-		pars = append(pars, startDate)
-	}
-	if endDate != "" {
-		condition += " AND data_time<=? "
-		pars = append(pars, endDate)
-	}
-
-	dataList, err := GetEdbDataListAll(condition, pars, fromEdbInfo.Source, 0)
-	if err != nil {
-		return err
-	}
-	var dateArr []string
-	dataMap := make(map[string]*EdbInfoSearchData)
-	for _, v := range dataList {
-		dateArr = append(dateArr, v.DataTime)
-		dataMap[v.DataTime] = v
-	}
-
-	addSql := ` INSERT INTO edb_data_calculate_tbz(edb_info_id,edb_code,data_time,value,create_time,modify_time,status,data_timestamp) values `
-	var isAdd bool
-	for ak, av := range dateArr {
-		fmt.Println(ak, av)
-		currentItem := dataMap[av]
-		if currentItem != nil {
-			//当前日期
-			currentDate, err := time.Parse(utils.FormatDate, av)
-			if err != nil {
-				return err
-			}
-			//上一年的日期
-			preDate := currentDate.AddDate(-1, 0, 0)
-			preDateStr := preDate.Format(utils.FormatDate)
-			if findItem, ok := dataMap[preDateStr]; ok && findItem.Value != 0 { //上一年同期找到
-				//dataTime, _ := time.Parse(utils.FormatDate, date)
-				timestamp := currentDate.UnixNano() / 1e6
-				timestampStr := fmt.Sprintf("%d", timestamp)
-				val := TbzDiv(currentItem.Value, findItem.Value)
-
-				count, err := GetEdbDataCalculateTbzByCodeAndDate(edbCode, av)
-				if err != nil && err.Error() != utils.ErrNoRow() {
-					return err
-				}
-				if count <= 0 {
-					addSql += GetAddSql(edbInfoIdStr, edbCode, av, timestampStr, val)
-					isAdd = true
-				} else {
-					err = ModifyEdbDataCalculateTbz(int64(edbInfoId), av, val)
-					if err != nil {
-						return err
-					}
-				}
-				utils.FileLog.Info("同期找到:" + av + ";" + preDateStr)
-				continue
-			} else {
-				if fromEdbInfo.Frequency == "月度" { //向上和向下，各找一个月
-					nextDateDay := preDate.AddDate(0, 1, 0)
-					nextDateDayStr := nextDateDay.Format(utils.FormatDate)
-
-					preDateDay := preDate.AddDate(0, -1, 0)
-					preDateDayStr := preDateDay.Format(utils.FormatDate)
-
-					for i := 0; i <= 6; i++ {
-						if i >= 1 {
-							nextDateDay = nextDateDay.AddDate(0, 0, i)
-							nextDateDayStr = nextDateDay.Format(utils.FormatDate)
-						}
-						if findItem, ok := dataMap[nextDateDayStr]; ok && findItem.Value != 0 { //上一年同期->下一个月找到
-							timestamp := currentDate.UnixNano() / 1e6
-							timestampStr := fmt.Sprintf("%d", timestamp)
-							val := TbzDiv(currentItem.Value, findItem.Value)
-
-							count, err := GetEdbDataCalculateTbzByCodeAndDate(edbCode, av)
-							if err != nil && err.Error() != utils.ErrNoRow() {
-								return err
-							}
-							if count <= 0 {
-								addSql += GetAddSql(edbInfoIdStr, edbCode, av, timestampStr, val)
-								isAdd = true
-							} else {
-								err = ModifyEdbDataCalculateTbz(int64(edbInfoId), av, val)
-								if err != nil {
-									return err
-								}
-							}
-							break
-						} else {
-							if i >= 1 {
-								preDateDay = preDate.AddDate(0, 0, -i)
-								preDateDayStr = nextDateDay.Format(utils.FormatDate)
-							}
-							if findItem, ok := dataMap[preDateDayStr]; ok && findItem.Value != 0 { //上一年同期->上一个月找到
-								timestamp := currentDate.UnixNano() / 1e6
-								timestampStr := fmt.Sprintf("%d", timestamp)
-								val := TbzDiv(currentItem.Value, findItem.Value)
-
-								count, err := GetEdbDataCalculateTbzByCodeAndDate(edbCode, av)
-								if err != nil && err.Error() != utils.ErrNoRow() {
-									return err
-								}
-								if count <= 0 {
-									addSql += GetAddSql(edbInfoIdStr, edbCode, av, timestampStr, val)
-									isAdd = true
-								} else {
-									err = ModifyEdbDataCalculateTbz(int64(edbInfoId), av, val)
-									if err != nil {
-										return err
-									}
-								}
-								break
-							}
-						}
-					}
-				} else if fromEdbInfo.Frequency == "季度" || fromEdbInfo.Frequency == "年度" {
-					if findItem, ok := dataMap[preDateStr]; ok && findItem.Value != 0 { //上一年同期->下一个月找到
-						timestamp := currentDate.UnixNano() / 1e6
-						timestampStr := fmt.Sprintf("%d", timestamp)
-						val := TbzDiv(currentItem.Value, findItem.Value)
-
-						count, err := GetEdbDataCalculateTbzByCodeAndDate(edbCode, av)
-						if err != nil && err.Error() != utils.ErrNoRow() {
-							return err
-						}
-						if count <= 0 {
-							addSql += GetAddSql(edbInfoIdStr, edbCode, av, timestampStr, val)
-							isAdd = true
-						} else {
-							err = ModifyEdbDataCalculateTbz(int64(edbInfoId), av, val)
-							if err != nil {
-								return err
-							}
-						}
-						break
-					}
-				} else {
-					nextDateDay := preDate
-					preDateDay := preDate
-
-					for i := 0; i < 35; i++ {
-						nextDateDayStr := nextDateDay.Format(utils.FormatDate)
-						if findItem, ok := dataMap[nextDateDayStr]; ok && findItem.Value != 0 { //上一年同期->下一个月找到
-							timestamp := currentDate.UnixNano() / 1e6
-							timestampStr := fmt.Sprintf("%d", timestamp)
-							val := TbzDiv(currentItem.Value, findItem.Value)
-
-							count, err := GetEdbDataCalculateTbzByCodeAndDate(edbCode, av)
-							if err != nil && err.Error() != utils.ErrNoRow() {
-								return err
-							}
-							if count <= 0 {
-								addSql += GetAddSql(edbInfoIdStr, edbCode, av, timestampStr, val)
-								isAdd = true
-							} else {
-								err = ModifyEdbDataCalculateTbz(int64(edbInfoId), av, val)
-								if err != nil {
-									return err
-								}
-							}
-							break
-						} else {
-							preDateDayStr := preDateDay.Format(utils.FormatDate)
-							if findItem, ok := dataMap[preDateDayStr]; ok && findItem.Value != 0 { //上一年同期->上一个月找到
-								timestamp := currentDate.UnixNano() / 1e6
-								timestampStr := fmt.Sprintf("%d", timestamp)
-								val := TbzDiv(currentItem.Value, findItem.Value)
-
-								count, err := GetEdbDataCalculateTbzByCodeAndDate(edbCode, av)
-								if err != nil && err.Error() != utils.ErrNoRow() {
-									return err
-								}
-								if count <= 0 {
-									addSql += GetAddSql(edbInfoIdStr, edbCode, av, timestampStr, val)
-									isAdd = true
-								} else {
-									err = ModifyEdbDataCalculateTbz(int64(edbInfoId), av, val)
-									if err != nil {
-										return err
-									}
-								}
-								break
-							}
-						}
-						nextDateDay = nextDateDay.AddDate(0, 0, 1)
-						preDateDay = preDateDay.AddDate(0, 0, -1)
-					}
-				}
-			}
-		}
-	}
-	if isAdd {
-		addSql = strings.TrimRight(addSql, ",")
-		_, err = tx.Raw(addSql).Exec()
-		if err != nil {
-			return err
-		}
-	}
-	return
-}
-
-type EdbInfoCalculateTbzDetail struct {
-	EdbInfoCalculateTbzId int       `orm:"column(edb_info_calculate_tbz_id);pk"`
-	EdbInfoId             int       `description:"指标id"`
-	EdbCode               string    `description:"指标编码"`
-	FromEdbInfoId         int       `description:"计算指标id"`
-	FromEdbCode           string    `description:"计算指标编码"`
-	FromEdbName           string    `description:"计算指标名称"`
-	FromSource            int       `description:"计算指标来源"`
-	FromSourceName        string    `description:"计算指标来源名称"`
-	FromTag               string    `description:"来源指标标签"`
-	Sort                  int       `description:"计算指标名称排序"`
-	CreateTime            time.Time `description:"创建时间"`
-	ModifyTime            time.Time `description:"修改时间"`
-	StartDate             string    `description:"开始日期"`
-	EndDate               string    `description:"结束日期"`
-}
-
-func GetEdbInfoCalculateTbzDetail(edbInfoId int) (item *EdbInfoCalculateTbzDetail, err error) {
-	o := orm.NewOrm()
-	sql := ` SELECT a.*,b.start_date,b.end_date FROM edb_info_calculate_mapping AS a
-			INNER JOIN edb_info AS b ON a.edb_info_id=b.edb_info_id
-			WHERE a.edb_info_id=? `
-	err = o.Raw(sql, edbInfoId).QueryRow(&item)
-	return
-}
-
-func GetEdbDataCalculateTbzByCodeAndDate(edbCode string, startDate string) (count int, err error) {
-	o := orm.NewOrm()
-	sql := ` SELECT COUNT(1) AS count FROM edb_data_calculate_tbz WHERE edb_code=? AND data_time=? `
-	err = o.Raw(sql, edbCode, startDate).QueryRow(&count)
-	return
-}
-
-func ModifyEdbDataCalculateTbz(edbInfoId int64, dataTime, value string) (err error) {
-	o := orm.NewOrm()
-	sql := ` UPDATE edb_data_calculate_tbz SET value=?,modify_time=NOW() WHERE edb_info_id=? AND data_time=? `
-	_, err = o.Raw(sql, value, edbInfoId, dataTime).Exec()
-	return
-}

models/data_manage/edb_info_calculate_tcz.go

@@ -1,303 +0,0 @@
-package data_manage
-
-import (
-	"eta/eta_task/utils"
-	"fmt"
-	"github.com/beego/beego/v2/client/orm"
-	"github.com/shopspring/decimal"
-	"strconv"
-	"strings"
-	"time"
-)
-
-type EdbInfoCalculateTcz struct {
-	EdbInfoCalculateTbzId int       `orm:"column(edb_info_calculate_tcz_id);pk"`
-	EdbInfoId             int       `description:"指标id"`
-	EdbCode               string    `description:"指标编码"`
-	FromEdbInfoId         int       `description:"计算指标id"`
-	FromEdbCode           string    `description:"计算指标编码"`
-	FromEdbName           string    `description:"计算指标名称"`
-	FromSource            int       `description:"计算指标来源"`
-	FromSourceName        string    `description:"计算指标来源名称"`
-	FromTag               string    `description:"来源指标标签"`
-	Sort                  int       `description:"计算指标名称排序"`
-	CreateTime            time.Time `description:"创建时间"`
-	ModifyTime            time.Time `description:"修改时间"`
-}
-
-func TczSub(a, b float64) string {
-	af := decimal.NewFromFloat(float64(a))
-	fmt.Println(af)
-	bf := decimal.NewFromFloat(float64(b))
-	val, _ := af.Sub(bf).Float64()
-	valStr := utils.SubFloatToString(val, 4)
-	return valStr
-}
-
-// 刷新同差值数据
-func RefreshCalculateTcz(edbInfoId int, fromEdbInfo *EdbInfo, edbCode, startDate, endDate string) (err error) {
-	o := orm.NewOrm()
-	tx, err := o.Begin()
-	if err != nil {
-		return err
-	}
-	defer func() {
-		if err != nil {
-			tx.Rollback()
-		} else {
-			tx.Commit()
-		}
-	}()
-
-	if err != nil {
-		return
-	}
-	edbInfoIdStr := strconv.Itoa(edbInfoId)
-
-	fmt.Println(edbInfoId, fromEdbInfo.EdbInfoId)
-	//计算数据
-	var condition string
-	var pars []interface{}
-	condition += " AND edb_info_id=? "
-	pars = append(pars, fromEdbInfo.EdbInfoId)
-
-	if startDate != "" {
-		condition += " AND data_time>=? "
-		pars = append(pars, startDate)
-	}
-	if endDate != "" {
-		condition += " AND data_time<=? "
-		pars = append(pars, endDate)
-	}
-
-	dataList, err := GetEdbDataListAll(condition, pars, fromEdbInfo.Source, 0)
-	if err != nil {
-		return err
-	}
-	var dateArr []string
-	dataMap := make(map[string]*EdbInfoSearchData)
-	for _, v := range dataList {
-		dateArr = append(dateArr, v.DataTime)
-		dataMap[v.DataTime] = v
-	}
-
-	addSql := ` INSERT INTO edb_data_calculate_tcz(edb_info_id,edb_code,data_time,value,create_time,modify_time,status,data_timestamp) values `
-	var isAdd bool
-	for _, av := range dateArr {
-		currentItem := dataMap[av]
-		if currentItem != nil {
-			//当前日期
-			currentDate, err := time.Parse(utils.FormatDate, av)
-			if err != nil {
-				return err
-			}
-			//上一年的日期
-			preDate := currentDate.AddDate(-1, 0, 0)
-			preDateStr := preDate.Format(utils.FormatDate)
-			if findItem, ok := dataMap[preDateStr]; ok { //上一年同期找到
-				//dataTime, _ := time.Parse(utils.FormatDate, date)
-				timestamp := currentDate.UnixNano() / 1e6
-				timestampStr := fmt.Sprintf("%d", timestamp)
-				val := TczSub(currentItem.Value, findItem.Value)
-
-				count, err := GetEdbDataCalculateTczByCodeAndDate(edbCode, av)
-				if err != nil && err.Error() != utils.ErrNoRow() {
-					return err
-				}
-				if count <= 0 {
-					addSql += GetAddSql(edbInfoIdStr, edbCode, av, timestampStr, val)
-					isAdd = true
-				} else {
-					err = ModifyEdbDataCalculateTcz(int64(edbInfoId), av, val)
-					if err != nil {
-						return err
-					}
-				}
-				utils.FileLog.Info("同期找到:" + av + ";" + preDateStr)
-				continue
-			} else {
-				if fromEdbInfo.Frequency == "月度" { //向上和向下，各找一个月
-					nextDateDay := preDate.AddDate(0, 1, 0)
-					nextDateDayStr := nextDateDay.Format(utils.FormatDate)
-
-					preDateDay := preDate.AddDate(0, -1, 0)
-					preDateDayStr := preDateDay.Format(utils.FormatDate)
-
-					for i := 0; i <= 6; i++ {
-						if i >= 1 {
-							nextDateDay = nextDateDay.AddDate(0, 0, i)
-							nextDateDayStr = nextDateDay.Format(utils.FormatDate)
-						}
-						if findItem, ok := dataMap[nextDateDayStr]; ok { //上一年同期->下一个月找到
-							timestamp := currentDate.UnixNano() / 1e6
-							timestampStr := fmt.Sprintf("%d", timestamp)
-							val := TczSub(currentItem.Value, findItem.Value)
-
-							count, err := GetEdbDataCalculateTczByCodeAndDate(edbCode, av)
-							if err != nil && err.Error() != utils.ErrNoRow() {
-								return err
-							}
-							if count <= 0 {
-								addSql += GetAddSql(edbInfoIdStr, edbCode, av, timestampStr, val)
-								isAdd = true
-							} else {
-								err = ModifyEdbDataCalculateTcz(int64(edbInfoId), av, val)
-								if err != nil {
-									return err
-								}
-							}
-							break
-						} else {
-							if i >= 1 {
-								preDateDay = preDate.AddDate(0, 0, -i)
-								preDateDayStr = nextDateDay.Format(utils.FormatDate)
-							}
-							if findItem, ok := dataMap[preDateDayStr]; ok { //上一年同期->上一个月找到
-								timestamp := currentDate.UnixNano() / 1e6
-								timestampStr := fmt.Sprintf("%d", timestamp)
-								val := TczSub(currentItem.Value, findItem.Value)
-
-								count, err := GetEdbDataCalculateTczByCodeAndDate(edbCode, av)
-								if err != nil && err.Error() != utils.ErrNoRow() {
-									return err
-								}
-								if count <= 0 {
-									addSql += GetAddSql(edbInfoIdStr, edbCode, av, timestampStr, val)
-									isAdd = true
-								} else {
-									err = ModifyEdbDataCalculateTcz(int64(edbInfoId), av, val)
-									if err != nil {
-										return err
-									}
-								}
-								break
-							}
-						}
-					}
-				} else if fromEdbInfo.Frequency == "季度" || fromEdbInfo.Frequency == "年度" {
-					if findItem, ok := dataMap[preDateStr]; ok { //上一年同期->下一个月找到
-						timestamp := currentDate.UnixNano() / 1e6
-						timestampStr := fmt.Sprintf("%d", timestamp)
-						val := TczSub(currentItem.Value, findItem.Value)
-
-						count, err := GetEdbDataCalculateTczByCodeAndDate(edbCode, av)
-						if err != nil && err.Error() != utils.ErrNoRow() {
-							return err
-						}
-						if count <= 0 {
-							addSql += GetAddSql(edbInfoIdStr, edbCode, av, timestampStr, val)
-							isAdd = true
-						} else {
-							err = ModifyEdbDataCalculateTcz(int64(edbInfoId), av, val)
-							if err != nil {
-								return err
-							}
-						}
-						break
-					}
-				} else {
-					nextDateDay := preDate
-					preDateDay := preDate
-
-					for i := 0; i < 35; i++ {
-						nextDateDayStr := nextDateDay.Format(utils.FormatDate)
-						if findItem, ok := dataMap[nextDateDayStr]; ok { //上一年同期->下一个月找到
-							timestamp := currentDate.UnixNano() / 1e6
-							timestampStr := fmt.Sprintf("%d", timestamp)
-							val := TczSub(currentItem.Value, findItem.Value)
-
-							count, err := GetEdbDataCalculateTczByCodeAndDate(edbCode, av)
-							if err != nil && err.Error() != utils.ErrNoRow() {
-								return err
-							}
-							if count <= 0 {
-								addSql += GetAddSql(edbInfoIdStr, edbCode, av, timestampStr, val)
-								isAdd = true
-							} else {
-								err = ModifyEdbDataCalculateTcz(int64(edbInfoId), av, val)
-								if err != nil {
-									return err
-								}
-							}
-							break
-						} else {
-							preDateDayStr := preDateDay.Format(utils.FormatDate)
-							if findItem, ok := dataMap[preDateDayStr]; ok { //上一年同期->上一个月找到
-								timestamp := currentDate.UnixNano() / 1e6
-								timestampStr := fmt.Sprintf("%d", timestamp)
-								val := TczSub(currentItem.Value, findItem.Value)
-
-								count, err := GetEdbDataCalculateTczByCodeAndDate(edbCode, av)
-								if err != nil && err.Error() != utils.ErrNoRow() {
-									return err
-								}
-								if count <= 0 {
-									addSql += GetAddSql(edbInfoIdStr, edbCode, av, timestampStr, val)
-									isAdd = true
-								} else {
-									err = ModifyEdbDataCalculateTcz(int64(edbInfoId), av, val)
-									if err != nil {
-										return err
-									}
-								}
-								break
-							}
-						}
-						nextDateDay = nextDateDay.AddDate(0, 0, 1)
-						preDateDay = preDateDay.AddDate(0, 0, -1)
-					}
-				}
-			}
-		}
-	}
-	fmt.Println("isAdd:", isAdd)
-
-	if isAdd {
-		addSql = strings.TrimRight(addSql, ",")
-		_, err = tx.Raw(addSql).Exec()
-		if err != nil {
-			return err
-		}
-	}
-	return
-}
-
-type EdbInfoCalculateTczDetail struct {
-	EdbInfoCalculateTczId int       `orm:"column(edb_info_calculate_tcz_id);pk"`
-	EdbInfoId             int       `description:"指标id"`
-	EdbCode               string    `description:"指标编码"`
-	FromEdbInfoId         int       `description:"计算指标id"`
-	FromEdbCode           string    `description:"计算指标编码"`
-	FromEdbName           string    `description:"计算指标名称"`
-	FromSource            int       `description:"计算指标来源"`
-	FromSourceName        string    `description:"计算指标来源名称"`
-	FromTag               string    `description:"来源指标标签"`
-	Sort                  int       `description:"计算指标名称排序"`
-	CreateTime            time.Time `description:"创建时间"`
-	ModifyTime            time.Time `description:"修改时间"`
-	StartDate             string    `description:"开始日期"`
-	EndDate               string    `description:"结束日期"`
-}
-
-func GetEdbInfoCalculateTczDetail(edbInfoId int) (item *EdbInfoCalculateTczDetail, err error) {
-	o := orm.NewOrm()
-	sql := ` SELECT a.*,b.start_date,b.end_date FROM edb_info_calculate_mapping AS a
-			INNER JOIN edb_info AS b ON a.edb_info_id=b.edb_info_id
-			WHERE a.edb_info_id=? `
-	err = o.Raw(sql, edbInfoId).QueryRow(&item)
-	return
-}
-
-func GetEdbDataCalculateTczByCodeAndDate(edbCode string, startDate string) (count int, err error) {
-	o := orm.NewOrm()
-	sql := ` SELECT COUNT(1) AS count FROM edb_data_calculate_tcz WHERE edb_code=? AND data_time=? `
-	err = o.Raw(sql, edbCode, startDate).QueryRow(&count)
-	return
-}
-
-func ModifyEdbDataCalculateTcz(edbInfoId int64, dataTime, value string) (err error) {
-	o := orm.NewOrm()
-	sql := ` UPDATE edb_data_calculate_tcz SET value=?,modify_time=NOW() WHERE edb_info_id=? AND data_time=? `
-	_, err = o.Raw(sql, value, edbInfoId, dataTime).Exec()
-	return
-}

models/data_manage/edb_info_calculate_time_shift.go

@@ -1,3 +0,0 @@
-package data_manage
-
-//时间移位

models/data_manage/edb_info_calculate_zjpj.go

@@ -1,225 +0,0 @@
-package data_manage
-
-import (
-	"errors"
-	"eta/eta_task/utils"
-	"fmt"
-	"github.com/beego/beego/v2/client/orm"
-	"strings"
-	"time"
-)
-
-type EdbInfoCalculateZjpjDetail struct {
-	EdbInfoCalculateBpId int       `orm:"column(edb_info_calculate_bp_id);pk"`
-	EdbInfoId            int       `description:"指标id"`
-	EdbCode              string    `description:"指标编码"`
-	FromEdbInfoId        int       `description:"计算指标id"`
-	FromEdbCode          string    `description:"计算指标编码"`
-	FromEdbName          string    `description:"计算指标名称"`
-	FromSource           int       `description:"计算指标来源"`
-	FromSourceName       string    `description:"计算指标来源名称"`
-	FromTag              string    `description:"来源指标标签"`
-	Sort                 int       `description:"计算指标名称排序"`
-	CreateTime           time.Time `description:"创建时间"`
-	ModifyTime           time.Time `description:"修改时间"`
-	StartDate            string    `description:"开始日期"`
-	EndDate              string    `description:"结束日期"`
-}
-
-// RefreshAllCalculateZjpj 刷新所有 直接拼接 数据
-func RefreshAllCalculateZjpj(edbInfo *EdbInfo) (err error) {
-	o := orm.NewOrm()
-	tx, err := o.Begin()
-	if err != nil {
-		return err
-	}
-	defer func() {
-		if err != nil {
-			tx.Rollback()
-		} else {
-			tx.Commit()
-		}
-	}()
-
-	//查询当前指标现有的数据
-	var condition string
-	var pars []interface{}
-	condition += " AND edb_info_id=? "
-	pars = append(pars, edbInfo.EdbInfoId)
-
-	dataList, err := GetAllEdbDataCalculateZjpjByEdbInfoId(edbInfo.EdbInfoId)
-	if err != nil {
-		return err
-	}
-	var dateArr []string
-	dataMap := make(map[string]*EdbDataCalculateZjpj)
-	removeDataTimeMap := make(map[string]int) //需要移除的日期数据
-	for _, v := range dataList {
-		dateArr = append(dateArr, v.DataTime)
-		dataMap[v.DataTime] = v
-		removeDataTimeMap[v.DataTime] = 1
-	}
-
-	//查询关联指标信息
-	var existCondition string
-	var existPars []interface{}
-	existCondition += " AND edb_info_id=? "
-	existPars = append(existPars, edbInfo.EdbInfoId)
-	existList, err := GetEdbInfoCalculateListByCondition(existCondition, existPars)
-	if err != nil {
-		err = errors.New("判断指标是否改变失败,Err:" + err.Error())
-		return
-	}
-
-	var existItemA, existItemB *EdbInfoCalculateMapping
-	for _, existItem := range existList {
-		if existItem.FromTag == "A" {
-			existItemA = existItem
-		} else if existItem.FromTag == "B" {
-			existItemB = existItem
-		}
-	}
-
-	addDataList := make([]*EdbDataCalculateZjpj, 0)
-	//第一个指标
-	{
-		var condition string
-		var pars []interface{}
-
-		condition += " AND data_time < ? AND edb_info_id=? "
-		pars = append(pars, edbInfo.CalculateFormula, existItemA.FromEdbInfoId)
-
-		//第一个指标的数据列表
-		firstDataList, tmpErr := GetEdbDataListAll(condition, pars, existItemA.FromSource, 0)
-		if tmpErr != nil {
-			return tmpErr
-		}
-
-		for _, v := range firstDataList {
-			//校验待删除日期数据里面是否存在该元素，如果存在的话，那么移除该元素
-			if _, ok := removeDataTimeMap[v.DataTime]; ok {
-				delete(removeDataTimeMap, v.DataTime)
-			}
-			//时间戳
-			if edbData, ok := dataMap[v.DataTime]; ok {
-				if edbData.Value != v.Value {
-					//更新指标数据
-					edbData.Value = v.Value
-					o.Update(edbData, "Value")
-				}
-			} else {
-				//时间戳
-				currentDate, _ := time.Parse(utils.FormatDate, v.DataTime)
-				timestamp := currentDate.UnixNano() / 1e6
-
-				edbDataZjpj := &EdbDataCalculateZjpj{
-					EdbInfoId:     edbInfo.EdbInfoId,
-					EdbCode:       edbInfo.EdbCode,
-					DataTime:      v.DataTime,
-					Value:         v.Value,
-					Status:        1,
-					CreateTime:    time.Now(),
-					ModifyTime:    time.Now(),
-					DataTimestamp: timestamp,
-				}
-				addDataList = append(addDataList, edbDataZjpj)
-			}
-		}
-	}
-
-	//第二个指标
-	{
-		condition = ``
-		pars = make([]interface{}, 0)
-
-		condition += " AND data_time >= ? AND edb_info_id = ? "
-		pars = append(pars, edbInfo.CalculateFormula, existItemB.FromEdbInfoId)
-
-		//第二个指标的数据列表
-		secondDataList, tmpErr := GetEdbDataListAll(condition, pars, existItemB.FromSource, 0)
-		if tmpErr != nil {
-			return tmpErr
-		}
-
-		for _, v := range secondDataList {
-			//校验待删除日期数据里面是否存在该元素，如果存在的话，那么移除该元素
-			if _, ok := removeDataTimeMap[v.DataTime]; ok {
-				delete(removeDataTimeMap, v.DataTime)
-			}
-
-			if edbData, ok := dataMap[v.DataTime]; ok {
-				if edbData.Value != v.Value {
-					//更新指标数据
-					edbData.Value = v.Value
-					edbData.ModifyTime = time.Now()
-					_, tmpErr := o.Update(edbData, "Value", "ModifyTime")
-					if tmpErr != nil {
-						fmt.Println("tmpErr:", tmpErr)
-					}
-				}
-			} else {
-				//时间戳
-				currentDate, _ := time.Parse(utils.FormatDate, v.DataTime)
-				timestamp := currentDate.UnixNano() / 1e6
-
-				edbDataZjpj := &EdbDataCalculateZjpj{
-					EdbInfoId:     edbInfo.EdbInfoId,
-					EdbCode:       edbInfo.EdbCode,
-					DataTime:      v.DataTime,
-					Value:         v.Value,
-					Status:        1,
-					CreateTime:    time.Now(),
-					ModifyTime:    time.Now(),
-					DataTimestamp: timestamp,
-				}
-				addDataList = append(addDataList, edbDataZjpj)
-			}
-		}
-	}
-
-	//删除已经不存在的累计同比拼接指标数据（由于同比值当日的数据删除了）
-	{
-		removeDateList := make([]string, 0)
-		for dateTime := range removeDataTimeMap {
-			removeDateList = append(removeDateList, dateTime)
-		}
-		if len(removeDateList) > 0 {
-			removeDateStr := strings.Join(removeDateList, `","`)
-			removeDateStr = `"` + removeDateStr + `"`
-			//如果拼接指标变更了，那么需要删除所有的指标数据
-			tableName := GetEdbDataTableName(edbInfo.Source)
-			sql := fmt.Sprintf(` DELETE FROM %s WHERE edb_info_id = ? and data_time in (%s) `, tableName, removeDateStr)
-
-			_, err = o.Raw(sql, edbInfo.EdbInfoId).Exec()
-			if err != nil {
-				err = errors.New("删除不存在的直接拼接指标数据失败,Err:" + err.Error())
-				return
-			}
-		}
-	}
-
-	//数据入库
-	tmpAddDataList := make([]*EdbDataCalculateZjpj, 0)
-	for _, v := range addDataList {
-		tmpAddDataList = append(tmpAddDataList, v)
-
-		if len(tmpAddDataList) >= 200 {
-			_, tmpErr := o.InsertMulti(len(tmpAddDataList), tmpAddDataList)
-			if tmpErr != nil {
-				err = tmpErr
-				return
-			}
-			//重新初始化需要加入的数据切片
-			tmpAddDataList = make([]*EdbDataCalculateZjpj, 0)
-		}
-	}
-	//最后如果还有需要新增的数据，那么就统一入库
-	if len(tmpAddDataList) > 0 {
-		_, tmpErr := tx.InsertMulti(len(tmpAddDataList), tmpAddDataList)
-		if tmpErr != nil {
-			err = tmpErr
-			return
-		}
-	}
-	return
-}

models/data_manage/edb_source.go

@@ -1,11 +1,20 @@
 package data_manage
 
 import (
+	"eta/eta_task/utils"
 	"fmt"
 	"github.com/beego/beego/v2/client/orm"
 	"strings"
 )
 
+var (
+	EdbDataTableNameMap     map[int]string        // 指标来源对应数据表名
+	EdbDataRefreshMethodMap map[int]string        // 指标来源对应的刷新指标方法
+	EdbTableNameSourceMap   map[string]*EdbSource // 数据表名对应的指标来源
+	EdbSourceIdMap          map[int]*EdbSource    // 指标来源
+	EdbSourceExtendIdMap    map[string]int        // 指标来源字符串对应来源ID
+)
+
 // EdbSource 指标来源表
 type EdbSource struct {
 	EdbSourceId      int    `orm:"column(edb_source_id);pk"`
@@ -42,3 +51,33 @@ func GetEdbSourceItemByCondition(condition string, pars []interface{}) (item *Ed
 	err = o.Raw(sql, pars).QueryRow(&item)
 	return
 }
+
+// InitEdbSourceVar 初始化时加载指标来源对应信息, 避免循环中查库, 注意edb_source表修改table_name的话需要重启服务
+func InitEdbSourceVar() {
+	EdbDataTableNameMap = make(map[int]string)
+	EdbDataRefreshMethodMap = make(map[int]string)
+	EdbTableNameSourceMap = make(map[string]*EdbSource)
+	EdbSourceIdMap = make(map[int]*EdbSource)
+	EdbSourceExtendIdMap = make(map[string]int)
+	sources, e := GetEdbSourceItemsByCondition(``, make([]interface{}, 0), []string{}, "")
+	if e != nil {
+		utils.FileLog.Info("init source table err: %s", e.Error())
+		return
+	}
+	for _, v := range sources {
+		EdbDataTableNameMap[v.EdbSourceId] = v.TableName
+		EdbDataRefreshMethodMap[v.EdbSourceId] = v.EdbRefreshMethod
+		EdbTableNameSourceMap[v.TableName] = v
+
+		EdbSourceIdMap[v.EdbSourceId] = v
+		if v.SourceExtend != "" {
+			arr := strings.Split(v.SourceExtend, ",")
+			if len(arr) == 0 {
+				continue
+			}
+			for _, s := range arr {
+				EdbSourceExtendIdMap[s] = v.EdbSourceId
+			}
+		}
+	}
+}

models/data_manage/edb_update_log.go

@@ -1,6 +1,7 @@
 package data_manage
 
 import (
+	"eta/eta_task/utils"
 	"github.com/beego/beego/v2/client/orm"
 	"time"
 )
@@ -12,28 +13,139 @@ type EdbUpdateLog struct {
 	OpType      string    `json:"op_type"`       // 变更类型
 	OldData     string    `json:"old_data"`      // 历史数据
 	NewData     string    `json:"new_data"`      // 新数据
+	IsHandle    int       `json:"is_handle"`     // 是否处理，0：未处理，1：已处理；默认：0
+	ModifyTime  time.Time `json:"modify_time"`
 	CreateTime  time.Time `json:"create_time"`
 }
 
+// GetEdbUpdateLogMaxId
+// @Description: 获取最大的数据id
+// @author: Roc
+// @datetime 2024-03-07 18:38:23
+// @param condition string
+// @param pars []interface{}
+// @return maxId int
+// @return err error
+func GetEdbUpdateLogMaxId() (maxId int64, err error) {
+	o := orm.NewOrmUsingDB("data")
+	sql := ` SELECT id FROM edb_update_log WHERE 1=1 ORDER BY id desc limit 1 `
+	err = o.Raw(sql).QueryRow(&maxId)
+
+	return
+}
+
+// GetEdbUpdateLogMaxHandleId
+// @Description: 获取已经操作了的最大的数据id
+// @author: Roc
+// @datetime 2024-03-08 17:53:29
+// @return maxId int64
+// @return err error
+func GetEdbUpdateLogMaxHandleId() (maxId int64, err error) {
+	o := orm.NewOrmUsingDB("data")
+	sql := ` SELECT id FROM edb_update_log WHERE 1=1 AND is_handle = 1 ORDER BY id desc limit 1 `
+	err = o.Raw(sql).QueryRow(&maxId)
+
+	return
+}
+
+// GetEdbUpdateLogByCondition
+// @Description: 获取指标更新列表
+// @author: Roc
+// @datetime 2024-03-07 18:36:44
+// @param condition string
+// @param pars []interface{}
+// @return item []*EdbUpdateLog
+// @return err error
 func GetEdbUpdateLogByCondition(condition string, pars []interface{}) (item []*EdbUpdateLog, err error) {
 	o := orm.NewOrmUsingDB("data")
 	sql := ` SELECT * FROM edb_update_log WHERE 1=1 `
 	if condition != "" {
 		sql += condition
 	}
-	sql += `ORDER BY id ASC limit 500 `
+	sql += ` ORDER BY id ASC limit 500 `
 	_, err = o.Raw(sql, pars).QueryRows(&item)
 
 	return
 }
 
+// HandleUpdateLogByIds
+// @Description: 批量处理指标更新记录
+// @author: Roc
+// @datetime 2024-03-08 17:30:52
+// @param idList []int
+// @return err error
+func HandleUpdateLogByIds(idList []int64, modifyTime string) (err error) {
+	num := len(idList)
+	if num <= 0 {
+		return
+	}
+	o := orm.NewOrmUsingDB("data")
+	sql := ` UPDATE edb_update_log set is_handle=1 ,modify_time = ?  WHERE id in (` + utils.GetOrmInReplace(num) + `) `
+	_, err = o.Raw(sql, modifyTime, idList).Exec()
+
+	return
+}
+
+type EdbInfoItem struct {
+	CalculateFormula string  `json:"calculate_formula"`
+	Calendar         string  `json:"calendar"`
+	ChartImage       string  `json:"chart_image"`
+	ClassifyId       int     `json:"classify_id"`
+	CreateTime       string  `json:"create_time"`
+	DataDateType     string  `json:"data_date_type"`
+	DataUpdateTime   string  `json:"data_update_time"`
+	EdbCode          string  `json:"edb_code"`
+	EdbInfoId        int     `json:"edb_info_id"`
+	EdbInfoType      int     `json:"edb_info_type"`
+	EdbName          string  `json:"edb_name"`
+	EdbNameEn        string  `json:"edb_name_en"`
+	EdbNameSource    string  `json:"edb_name_source"`
+	EdbType          int     `json:"edb_type"`
+	EmptyType        int     `json:"empty_type"`
+	EndDate          string  `json:"end_date"`
+	EndValue         float64 `json:"end_value"`
+	ErDataUpdateDate string  `json:"er_data_update_date"`
+	Extra            string  `json:"extra"`
+	Frequency        string  `json:"frequency"`
+	IndicatorCode    string  `json:"indicator_code"`
+	IsUpdate         int     `json:"is_update"`
+	LatestDate       string  `json:"latest_date"`
+	LatestValue      float64 `json:"latest_value"`
+	ManualSave       int     `json:"manual_save"`
+	MaxEmptyType     int     `json:"max_empty_type"`
+	MaxValue         float64 `json:"max_value"`
+	MinValue         float64 `json:"min_value"`
+	ModifyTime       string  `json:"modify_time"`
+	MoveFrequency    string  `json:"move_frequency"`
+	MoveType         int     `json:"move_type"`
+	NoUpdate         int     `json:"no_update"`
+	ServerUrl        string  `json:"server_url"`
+	Sort             int     `json:"sort"`
+	Source           int     `json:"source"`
+	SourceIndexName  string  `json:"source_index_name"`
+	SourceName       string  `json:"source_name"`
+	StartDate        string  `json:"start_date"`
+	StockCode        string  `json:"stock_code"`
+	SubSource        int     `json:"sub_source"`
+	SubSourceName    string  `json:"sub_source_name"`
+	SysUserId        int     `json:"sys_user_id"`
+	SysUserRealName  string  `json:"sys_user_real_name"`
+	TerminalCode     string  `json:"terminal_code"`
+	UniqueCode       string  `json:"unique_code"`
+	Unit             string  `json:"unit"`
+	UnitEn           string  `json:"unit_en"`
+}
+
 type EdbData struct {
-	EdbDataId     int32     `orm:"column(edb_data_id);pk" json:"edb_data_id"`
-	EdbInfoId     int32     `json:"edb_info_id"`    // 指标id
-	EdbCode       string    `json:"edb_code"`       // 指标编码
-	DataTime      time.Time `json:"data_time"`      // 数据日期
-	Value         float64   `json:"value"`          // 数据值
-	CreateTime    time.Time `json:"create_time"`    // 创建时间
-	ModifyTime    time.Time `json:"modify_time"`    // 修改时间
-	DataTimestamp int64     `json:"data_timestamp"` // 数据日期时间戳
+	EdbDataId int32  `orm:"column(edb_data_id);pk" json:"edb_data_id"`
+	EdbInfoId int32  `json:"edb_info_id"` // 指标id
+	EdbCode   string `json:"edb_code"`    // 指标编码
+	//DataTime      time.Time `json:"data_time"`      // 数据日期
+	DataTime string  `json:"data_time"` // 数据日期
+	Value    float64 `json:"value"`     // 数据值
+	//CreateTime    time.Time `json:"create_time"`    // 创建时间
+	//ModifyTime    time.Time `json:"modify_time"`    // 修改时间
+	CreateTime    string `json:"create_time"`    // 创建时间
+	ModifyTime    string `json:"modify_time"`    // 修改时间
+	DataTimestamp int64  `json:"data_timestamp"` // 数据日期时间戳
 }

services/data/data_calculate.go

@@ -1,132 +0,0 @@
-package data
-
-import (
-	"errors"
-	"eta/eta_task/models/data_manage"
-	"eta/eta_task/utils"
-	"fmt"
-	"strconv"
-	"strings"
-	"time"
-
-	"github.com/yidane/formula"
-)
-
-// 指标运算刷新数据
-func RefreshCalculate(edbInfoIdArr []*data_manage.EdbInfo, edbInfoId int, edbCode, formulaStr, startDate, endDate string, edbInfoIdBytes []string) (err error) {
-	defer func() {
-		if err != nil {
-			utils.FileLog.Info("Calculate Err:%s" + err.Error())
-			go utils.SendEmail(utils.APPNAME+"【"+utils.RunMode+"】"+"失败提醒", "RefreshCalculate ErrMsg:"+err.Error(), utils.EmailSendToUsers)
-		}
-	}()
-	saveDataMap := make(map[string]map[int]float64)
-	for _, v := range edbInfoIdArr {
-		var condition string
-		var pars []interface{}
-		condition += " AND edb_info_id=? "
-		pars = append(pars, v.EdbInfoId)
-		if startDate != "" {
-			condition += " AND data_time>=? "
-			pars = append(pars, startDate)
-		}
-		if endDate != "" {
-			condition += " AND data_time<=? "
-			pars = append(pars, endDate)
-		}
-		dataList, err := data_manage.GetEdbDataListAll(condition, pars, v.Source, 1)
-
-		if err != nil {
-			return err
-		}
-		dataMap := make(map[string]float64)
-		for _, dv := range dataList {
-			if val, ok := saveDataMap[dv.DataTime]; ok {
-				if _, ok := val[v.EdbInfoId]; !ok {
-					val[v.EdbInfoId] = dv.Value
-				}
-			} else {
-				temp := make(map[int]float64)
-				temp[v.EdbInfoId] = dv.Value
-				saveDataMap[dv.DataTime] = temp
-			}
-		}
-		item := new(CalculateItems)
-		item.EdbInfoId = v.EdbInfoId
-		item.DataMap = dataMap
-	}
-
-	formulaMap := CheckFormula(formulaStr)
-	addSql := ` INSERT INTO edb_data_calculate(edb_info_id,edb_code,data_time,value,create_time,modify_time,status,data_timestamp) values `
-	nowStr := time.Now().Format(utils.FormatDateTime)
-	var isAdd bool
-	removeDateList := make([]string, 0) //需要移除的日期
-
-	for sk, sv := range saveDataMap {
-		formulaStr = strings.ToUpper(formulaStr)
-		formulaFormStr := ReplaceFormula(edbInfoIdArr, sv, formulaMap, formulaStr, edbInfoIdBytes)
-		//if formulaFormStr != "" && !strings.Contains(formulaFormStr, "/0") {
-		if formulaFormStr != "" {
-			expression := formula.NewExpression(formulaFormStr)
-			calResult, err := expression.Evaluate()
-			if err != nil {
-				// 分母为0的报错
-				if strings.Contains(err.Error(), "divide by zero") {
-					removeDateList = append(removeDateList, sk)
-					continue
-				}
-				err = errors.New("计算失败,edbCode：" + fmt.Sprint(edbCode) + "：Err:" + err.Error() + ";formulaStr:" + formulaFormStr)
-				fmt.Println(err)
-				return err
-			}
-			calVal, err := calResult.Float64()
-			if err != nil {
-				err = errors.New("计算失败：获取计算值失败 Err:" + err.Error() + ";formulaStr:" + formulaFormStr)
-				fmt.Println(err)
-				return err
-			}
-
-			count, err := data_manage.GetEdbDataCalculateByCodeAndDate(edbCode, sk)
-			if err != nil && err.Error() != utils.ErrNoRow() {
-				return err
-			}
-			if count <= 0 { //需要存入的数据
-				dataTime, _ := time.Parse(utils.FormatDate, sk)
-				timestamp := dataTime.UnixNano() / 1e6
-				timeStr := fmt.Sprintf("%d", timestamp)
-				addSql += "("
-				addSql += strconv.Itoa(edbInfoId) + "," + "'" + edbCode + "'" + "," + "'" + sk + "'" + "," + utils.SubFloatToString(calVal, 4) + "," + "'" + nowStr + "'" +
-					"," + "'" + nowStr + "'" + "," + "1"
-				addSql += "," + "'" + timeStr + "'"
-				addSql += "),"
-				isAdd = true
-			} else {
-				calVal = utils.FixFloat(calVal, 4)
-				err = data_manage.ModifyEdbDataCalculate(int64(edbInfoId), sk, calVal)
-				if err != nil {
-					return err
-				}
-			}
-		}
-	}
-	if isAdd {
-		addSql = strings.TrimRight(addSql, ",")
-		err = data_manage.AddEdbDataCalculateBySql(addSql)
-		if err != nil {
-			fmt.Println("AddEdbDataCalculate Err:" + err.Error())
-			return err
-		}
-	}
-	if len(removeDateList) > 0 {
-		removeDateStr := strings.Join(removeDateList, `","`)
-		removeDateStr = `"` + removeDateStr + `"`
-		//如果拼接指标变更了，那么需要删除所有的指标数据
-		sql := fmt.Sprintf(` DELETE FROM edb_data_calculate WHERE edb_info_id = ? and data_time in (%s) `, removeDateStr)
-		err = data_manage.DeleteEdbDataCalculateBySql(sql, edbInfoId)
-		if err != nil {
-			err = fmt.Errorf("删除计算失败的计算指标数据失败,Err:" + err.Error())
-			return
-		}
-	}
-	return
-}

services/data/edb_info_sync.go

@@ -32,7 +32,7 @@ func SyncGlDataBase() {
 			return
 		}
 
-		item, err := data_manage.GetEdbInfoMaxAndMinInfo(v.Source, v.EdbCode)
+		item, err := data_manage.GetEdbInfoMaxAndMinInfo(v.Source, utils.DATA_SUB_SOURCE_EDB, v.EdbCode)
 		if err != nil {
 			return
 		}

services/eta_bridge/demo.go

@@ -0,0 +1,227 @@
+package eta_bridge
+
+import (
+	"context"
+	"eta/eta_task/models/data_manage"
+	"eta/eta_task/services/alarm_msg"
+	"eta/eta_task/utils"
+	"fmt"
+)
+
+// SyncIndexList
+// @Description: 定时同步ETA指标信息变更数据至第三方
+// @author: Roc
+// @datetime 2024-02-28 14:00:45
+// @param cont context.Context
+// @return err error
+func SyncIndexList(cont context.Context) (err error) {
+	defer func() {
+		if err != nil {
+			tips := "SyncIndexList-定时同步ETA指标信息变更数据至第三方失败, ErrMsg:\n" + err.Error()
+			utils.FileLog.Info(tips)
+			go alarm_msg.SendAlarmMsg(tips, 3)
+		}
+	}()
+
+	var condition string
+	var pars []interface{}
+	condition += " AND update_type in (?,?) "
+	pars = append(pars, 1, 2)
+
+	list, err := data_manage.GetEdbInfoUpdateLogByCondition(condition, pars)
+	if err != nil {
+		fmt.Println(err)
+		return
+	}
+
+	dataList := make([]PushIndexParamDataReq, 0)
+	for _, v := range list {
+		dataList = append(dataList, PushIndexParamDataReq{
+			SourceIndexCode:  v.EdbCode,
+			IndexCode:        fmt.Sprint(v.Source, "_", v.EdbCode),
+			IndexName:        v.EdbName, //todo
+			IndexShortName:   v.EdbName,
+			FrequenceName:    v.Frequency,
+			UnitName:         v.Unit,
+			AssetBeginDate:   v.StartDate,
+			AssetEndDate:     v.EndDate,
+			CreateUser:       v.SysUserRealName,
+			IndexCreateTime:  v.CreateTime.Format(utils.FormatDateTime), //todo
+			UpdateUser:       v.UpdateSysUserRealName,
+			DetailUpdateTime: v.CreateTime.Format(utils.FormatDateTime), //todo
+			IndexUpdateTime:  v.CreateTime.Format(utils.FormatDateTime), //todo
+			OrginSource:      v.SourceName,
+			OrginSysSource:   v.SourceName, // todo
+			SysSource:        "产研平台",       //todo
+			SourceType:       "RPA",        //TODO
+			Status:           1,
+		})
+	}
+
+	lenData := len(dataList)
+	if lenData <= 0 {
+		return
+	}
+
+	req := PushBaseParamReq{
+		SerialID:    utils.GetRandString(32), //todo
+		TableCode:   "",
+		Total:       lenData,
+		IsEmailWarn: 0,
+		Data:        dataList,
+	}
+
+	uri := "/xy/index/pushIndexData"
+	_, e, errMsg := HttpEtaBridgePost(uri, req)
+	if e != nil {
+		err = fmt.Errorf("postRefreshEdbData err: %s", e.Error())
+		fmt.Println(err)
+		return
+	}
+	fmt.Println(errMsg)
+	//if res != nil && res.Ret != 200 {
+	//	err = fmt.Errorf("postRefreshEdbData fail")
+	//	return
+	//}
+	return
+}
+
+// SyncIndexValueList
+// @Description: 定时同步ETA指标日期值的变更数据至第三方
+// @author: Roc
+// @datetime 2024-02-28 14:00:45
+// @param cont context.Context
+// @return err error
+func SyncIndexValueList(cont context.Context) (err error) {
+	defer func() {
+		if err != nil {
+			tips := "SyncIndexList-定时同步ETA指标信息变更数据至第三方失败, ErrMsg:\n" + err.Error()
+			utils.FileLog.Info(tips)
+			go alarm_msg.SendAlarmMsg(tips, 3)
+		}
+	}()
+
+	var condition string
+	var pars []interface{}
+	condition += " AND update_type = ? "
+	pars = append(pars, 0)
+
+	list, err := data_manage.GetEdbInfoUpdateLogByCondition(condition, pars)
+	if err != nil {
+		fmt.Println(err)
+		return
+	}
+
+	dataList := make([]PushIndexValueItemReq, 0)
+	for _, v := range list {
+		dataList = append(dataList, PushIndexValueItemReq{
+			Id:           utils.MD5(fmt.Sprint(v.Source, "_", v.SourceName, "_", v.Id)),
+			IndexCode:    fmt.Sprint(v.Source, "_", v.EdbCode),
+			Value:        fmt.Sprint(v.LatestValue),
+			BusinessDate: v.LatestDate,
+			CreateTime:   v.EdbModifyTime,
+			UpdateTime:   v.CreateTime.Format(utils.FormatDateTime), //todo,
+			Status:       "1",
+		})
+	}
+
+	lenData := len(dataList)
+	if lenData <= 0 {
+		return
+	}
+
+	req := PushBaseParamReq{
+		SerialID:    utils.GetRandString(32), //todo
+		TableCode:   "",
+		Total:       lenData,
+		IsEmailWarn: 0,
+		Data:        dataList,
+	}
+
+	uri := "/xy/index/pushIndexValue"
+	_, e, errMsg := HttpEtaBridgePost(uri, req)
+	if e != nil {
+		err = fmt.Errorf("postRefreshEdbData err: %s", e.Error())
+		fmt.Println(err)
+		return
+	}
+	fmt.Println(errMsg)
+	//if res != nil && res.Ret != 200 {
+	//	err = fmt.Errorf("postRefreshEdbData fail")
+	//	return
+	//}
+	return
+}
+
+// SyncEdbClassifyList
+// @Description: 定时同步ETA指标与分类的关系至第三方
+// @author: Roc
+// @datetime 2024-02-28 14:00:45
+// @param cont context.Context
+// @return err error
+func SyncEdbClassifyList(cont context.Context) (err error) {
+	defer func() {
+		if err != nil {
+			tips := "SyncIndexList-定时同步ETA指标信息变更数据至第三方失败, ErrMsg:\n" + err.Error()
+			utils.FileLog.Info(tips)
+			go alarm_msg.SendAlarmMsg(tips, 3)
+		}
+	}()
+
+	var condition string
+	var pars []interface{}
+
+	list, err := data_manage.GetAllEdbInfoClassifyListByCondition(condition, pars)
+	if err != nil {
+		fmt.Println(err)
+		return
+	}
+
+	dataLimitList := make([][]PushEdbClassifyItemReq, 0)
+
+	dataList := make([]PushEdbClassifyItemReq, 0)
+	for _, v := range list {
+		dataList = append(dataList, PushEdbClassifyItemReq{
+			Id:         fmt.Sprint(v.EdbInfoId),
+			ClassifyId: v.ClassifyId,
+			IndexCode:  fmt.Sprint(v.Source, "_", v.EdbCode),
+			CreateTime: v.CreateTime.Format(utils.FormatDateTime), //todo
+			CreateUser: v.SysUserRealName,
+			UpdateTime: v.ModifyTime.Format(utils.FormatDateTime), //todo
+			UpdateUser: v.SysUserRealName,
+		})
+		if len(dataList) >= 100 {
+			dataLimitList = append(dataLimitList, dataList)
+			dataList = make([]PushEdbClassifyItemReq, 0)
+		}
+	}
+
+	lenData := len(dataList)
+	if lenData > 0 {
+		dataLimitList = append(dataLimitList, dataList)
+	}
+
+	if len(dataLimitList) < 0 {
+		fmt.Println("无分类数据推送")
+		return
+	}
+	uri := utils.SyncIndexPath + "/pushEdbClassify"
+	for k, tmpDataList := range dataLimitList {
+		req := PushBaseParamReq{
+			SerialID:    utils.GetRandString(32), //todo
+			TableCode:   "",
+			Total:       len(tmpDataList),
+			IsEmailWarn: 0,
+			Data:        tmpDataList,
+		}
+
+		_, e, _ := HttpEtaBridgePost(uri, req)
+		if e != nil {
+			err = fmt.Errorf("第%d组分类数据推送失败，postRefreshEdbData err: %s", k+1, e.Error())
+			fmt.Println(err)
+			continue
+		}
+	}
+
+	return
+}

services/eta_bridge/handle_data.go

@@ -0,0 +1,646 @@
+package eta_bridge
+
+import (
+	"encoding/json"
+	"eta/eta_task/models/data_manage"
+	"eta/eta_task/utils"
+	"fmt"
+	"strings"
+)
+
+// handleData
+// @Description: 监听的数据处理
+// @author: Roc
+// @datetime 2024-03-11 14:52:49
+// @param edbUpdateLog *data_manage.EdbUpdateLog
+// @return pushIndexData *PushIndexParamDataReq
+// @return pushEdbClassify *PushEdbClassifyItemReq
+// @return pushIndexValue *PushIndexValueItemReq
+// @return err error
+func handleData(edbUpdateLog *data_manage.EdbUpdateLog) (pushIndexData *PushIndexParamDataReq, pushEdbClassify *PushEdbClassifyItemReq, pushIndexValue *PushIndexValueItemReq, err error) {
+	switch edbUpdateLog.OpType {
+	case "insert":
+		return handleInsert(edbUpdateLog)
+	case "update":
+		return handleUpdate(edbUpdateLog)
+	case "delete":
+		return handleDelete(edbUpdateLog)
+	}
+
+	return
+}
+
+// handleInsert
+// @Description: 新增数据处理
+// @author: Roc
+// @datetime 2024-03-11 14:53:03
+// @param edbUpdateLog *data_manage.EdbUpdateLog
+// @return pushIndexData *PushIndexParamDataReq
+// @return pushEdbClassify *PushEdbClassifyItemReq
+// @return pushIndexValue *PushIndexValueItemReq
+// @return err error
+func handleInsert(edbUpdateLog *data_manage.EdbUpdateLog) (pushIndexData *PushIndexParamDataReq, pushEdbClassify *PushEdbClassifyItemReq, pushIndexValue *PushIndexValueItemReq, err error) {
+	data := edbUpdateLog.NewData
+
+	//指标信息
+	if edbUpdateLog.OpTableName == "edb_info" {
+		var edbInfo *data_manage.EdbInfoItem
+		err = json.Unmarshal([]byte(data), &edbInfo)
+		if err != nil {
+			return
+		}
+
+		// 预测指标不处理
+		if edbInfo.EdbInfoType == 1 {
+			return
+		}
+		// 计算指标不处理
+		if edbInfo.EdbType == 2 {
+			return
+		}
+
+		// 获取数据源中指标的基础信息
+		origInfo := getOrigInfo(edbInfo.Source, edbInfo.SubSource, edbInfo.EdbCode)
+
+		// 指标信息
+		pushIndexData = &PushIndexParamDataReq{
+			SourceIndexCode:  edbInfo.EdbCode,
+			IndexCode:        fmt.Sprint(edbInfo.Source, "_", edbInfo.EdbCode),
+			IndexName:        origInfo.EdbName,
+			IndexShortName:   edbInfo.EdbName,
+			FrequenceName:    edbInfo.Frequency,
+			UnitName:         edbInfo.Unit,
+			AssetBeginDate:   edbInfo.StartDate,
+			AssetEndDate:     edbInfo.EndDate,
+			CreateUser:       edbInfo.SysUserRealName,
+			IndexCreateTime:  edbInfo.CreateTime,
+			UpdateUser:       edbInfo.SysUserRealName,
+			DetailUpdateTime: getMaxModifyTime(edbInfo.Source, edbInfo.SubSource, edbInfo.EdbCode, edbUpdateLog.CreateTime.Format(utils.FormatDateTime)),
+			IndexUpdateTime:  edbInfo.ModifyTime,
+			OrginSource:      edbInfo.SourceName,
+			OrginSysSource:   origInfo.SourceName,
+			SysSource:        "产研平台",
+			SourceType:       getSourceType(edbInfo.Source),
+			Status:           1,
+		}
+
+		// 指标与分类的关系信息
+		pushEdbClassify = &PushEdbClassifyItemReq{
+			Id:         fmt.Sprint(edbInfo.EdbInfoId),
+			ClassifyId: edbInfo.ClassifyId,
+			IndexCode:  fmt.Sprint(edbInfo.Source, "_", edbInfo.EdbCode),
+			CreateTime: edbInfo.CreateTime,
+			CreateUser: edbInfo.SysUserRealName,
+			UpdateTime: edbInfo.ModifyTime,
+			UpdateUser: edbInfo.SysUserRealName,
+		}
+
+		return
+	}
+
+	// 分类信息
+	//if edbUpdateLog.OpTableName == "edb_classify" {
+	//	var edbClassify *data_manage.EdbClassify
+	//	err = json.Unmarshal([]byte(data), &edbClassify)
+	//	if err != nil {
+	//		return
+	//	}
+	//
+	//	// 指标信息
+	//	pushClassify = PushClassifyItemReq{
+	//		ClassifyId:      int(edbClassify.ClassifyID),
+	//		ClassifyType:    int(edbClassify.ClassifyType),
+	//		ClassifyName:    edbClassify.ClassifyName,
+	//		ParentId:        int(edbClassify.ParentID),
+	//		HasData:         int(edbClassify.HasData),
+	//		CreateTime:      edbClassify.CreateTime.Format(utils.FormatDateTime),
+	//		UpdateTime:      edbClassify.ModifyTime.Format(utils.FormatDateTime),
+	//		SysUserId:       int(edbClassify.SysUserID),
+	//		SysUserRealName: edbClassify.SysUserRealName,
+	//		Level:           int(edbClassify.Level),
+	//		UniqueCode:      edbClassify.UniqueCode,
+	//		SortColumn:      int(edbClassify.Sort),
+	//	}
+	//
+	//	return
+	//}
+
+	// 数据信息
+	if strings.HasPrefix(edbUpdateLog.OpTableName, "edb_data_") {
+		// 计算指标不处理
+		if strings.HasPrefix(edbUpdateLog.OpTableName, "edb_data_calculate") {
+			return
+		}
+		// 预测指标不处理
+		if strings.HasPrefix(edbUpdateLog.OpTableName, "edb_data_predict") {
+			return
+		}
+
+		var edbData *data_manage.EdbData
+		err = json.Unmarshal([]byte(data), &edbData)
+		if err != nil {
+			return
+		}
+
+		edbSource, ok := data_manage.EdbTableNameSourceMap[edbUpdateLog.OpTableName]
+		if !ok {
+			// 没有找到来源，那就过滤
+			return
+		}
+		// 数据信息
+		pushIndexValue = &PushIndexValueItemReq{
+			Id:           utils.MD5(fmt.Sprint(edbSource.EdbSourceId, "_", edbSource.SourceName, "_", edbData.EdbDataId)),
+			IndexCode:    fmt.Sprint(edbSource.EdbSourceId, "_", edbData.EdbCode),
+			Value:        fmt.Sprint(edbData.Value),
+			BusinessDate: edbData.DataTime,
+			CreateTime:   edbData.CreateTime,
+			UpdateTime:   edbData.ModifyTime,
+			Status:       "1",
+		}
+
+		return
+	}
+
+	return
+}
+
+// handleDelete
+// @Description: 删除数据处理
+// @author: Roc
+// @datetime 2024-03-11 14:53:20
+// @param edbUpdateLog *data_manage.EdbUpdateLog
+// @return pushIndexData *PushIndexParamDataReq
+// @return pushEdbClassify *PushEdbClassifyItemReq
+// @return pushIndexValue *PushIndexValueItemReq
+// @return err error
+func handleDelete(edbUpdateLog *data_manage.EdbUpdateLog) (pushIndexData *PushIndexParamDataReq, pushEdbClassify *PushEdbClassifyItemReq, pushIndexValue *PushIndexValueItemReq, err error) {
+	data := edbUpdateLog.OldData
+
+	//指标信息
+	if edbUpdateLog.OpTableName == "edb_info" {
+		var edbInfo *data_manage.EdbInfoItem
+		err = json.Unmarshal([]byte(data), &edbInfo)
+		if err != nil {
+			return
+		}
+
+		// 预测指标不处理
+		if edbInfo.EdbInfoType == 1 {
+			return
+		}
+		// 计算指标不处理
+		if edbInfo.EdbType == 2 {
+			return
+		}
+
+		// 获取数据源中指标的基础信息
+		origInfo := getOrigInfo(edbInfo.Source, edbInfo.SubSource, edbInfo.EdbCode)
+
+		// 指标信息
+		pushIndexData = &PushIndexParamDataReq{
+			SourceIndexCode:  edbInfo.EdbCode,
+			IndexCode:        fmt.Sprint(edbInfo.Source, "_", edbInfo.EdbCode),
+			IndexName:        origInfo.EdbName,
+			IndexShortName:   edbInfo.EdbName,
+			FrequenceName:    edbInfo.Frequency,
+			UnitName:         edbInfo.Unit,
+			AssetBeginDate:   edbInfo.StartDate,
+			AssetEndDate:     edbInfo.EndDate,
+			CreateUser:       edbInfo.SysUserRealName,
+			IndexCreateTime:  edbInfo.CreateTime,
+			UpdateUser:       edbInfo.SysUserRealName,
+			DetailUpdateTime: edbUpdateLog.CreateTime.Format(utils.FormatDateTime),
+			IndexUpdateTime:  edbInfo.ModifyTime,
+			OrginSource:      edbInfo.SourceName,
+			OrginSysSource:   origInfo.SourceName,
+			SysSource:        "产研平台",
+			SourceType:       getSourceType(edbInfo.Source),
+			Status:           0,
+		}
+
+		// 指标与分类的关系信息
+		pushEdbClassify = &PushEdbClassifyItemReq{
+			Id:         fmt.Sprint(edbInfo.EdbInfoId),
+			ClassifyId: edbInfo.ClassifyId,
+			IndexCode:  fmt.Sprint(edbInfo.Source, "_", edbInfo.EdbCode),
+			CreateTime: edbInfo.CreateTime,
+			CreateUser: edbInfo.SysUserRealName,
+			UpdateTime: edbInfo.ModifyTime,
+			UpdateUser: edbInfo.SysUserRealName,
+		}
+
+		return
+	}
+
+	// 分类信息
+	//if edbUpdateLog.OpTableName == "edb_classify" {
+	//	var edbClassify *data_manage.EdbClassify
+	//	err = json.Unmarshal([]byte(data), &edbClassify)
+	//	if err != nil {
+	//		return
+	//	}
+	//
+	//	// 指标信息
+	//	pushClassify = PushClassifyItemReq{
+	//		ClassifyId:      int(edbClassify.ClassifyID),
+	//		ClassifyType:    int(edbClassify.ClassifyType),
+	//		ClassifyName:    edbClassify.ClassifyName,
+	//		ParentId:        int(edbClassify.ParentID),
+	//		HasData:         int(edbClassify.HasData),
+	//		CreateTime:      edbClassify.CreateTime.Format(utils.FormatDateTime),
+	//		UpdateTime:      edbClassify.ModifyTime.Format(utils.FormatDateTime),
+	//		SysUserId:       int(edbClassify.SysUserID),
+	//		SysUserRealName: edbClassify.SysUserRealName,
+	//		Level:           int(edbClassify.Level),
+	//		UniqueCode:      edbClassify.UniqueCode,
+	//		SortColumn:      int(edbClassify.Sort),
+	//	}
+	//
+	//	return
+	//}
+
+	// 数据信息
+	if strings.HasPrefix(edbUpdateLog.OpTableName, "edb_data_") {
+		// 计算指标不处理
+		if strings.HasPrefix(edbUpdateLog.OpTableName, "edb_data_calculate") {
+			return
+		}
+		// 预测指标不处理
+		if strings.HasPrefix(edbUpdateLog.OpTableName, "edb_data_predict") {
+			return
+		}
+
+		var edbData *data_manage.EdbData
+		err = json.Unmarshal([]byte(data), &edbData)
+		if err != nil {
+			return
+		}
+
+		edbSource, ok := data_manage.EdbTableNameSourceMap[edbUpdateLog.OpTableName]
+		if !ok {
+			// 没有找到来源，那就过滤
+			return
+		}
+		// 数据信息
+		pushIndexValue = &PushIndexValueItemReq{
+			Id:           utils.MD5(fmt.Sprint(edbSource.EdbSourceId, "_", edbSource.SourceName, "_", edbData.EdbDataId)),
+			IndexCode:    fmt.Sprint(edbSource.EdbSourceId, "_", edbData.EdbCode),
+			Value:        fmt.Sprint(edbData.Value),
+			BusinessDate: edbData.DataTime,
+			CreateTime:   edbData.CreateTime,
+			UpdateTime:   edbData.ModifyTime,
+			Status:       "0",
+		}
+
+		return
+	}
+
+	return
+}
+
+// handleUpdate
+// @Description: 更新数据处理
+// @author: Roc
+// @datetime 2024-03-11 14:53:35
+// @param edbUpdateLog *data_manage.EdbUpdateLog
+// @return pushIndexData *PushIndexParamDataReq
+// @return pushEdbClassify *PushEdbClassifyItemReq
+// @return pushIndexValue *PushIndexValueItemReq
+// @return err error
+func handleUpdate(edbUpdateLog *data_manage.EdbUpdateLog) (pushIndexData *PushIndexParamDataReq, pushEdbClassify *PushEdbClassifyItemReq, pushIndexValue *PushIndexValueItemReq, err error) {
+	oldData := edbUpdateLog.OldData
+	newData := edbUpdateLog.NewData
+
+	//指标信息
+	if edbUpdateLog.OpTableName == "edb_info" {
+		var oldEdbInfo *data_manage.EdbInfoItem
+		err = json.Unmarshal([]byte(oldData), &oldEdbInfo)
+		if err != nil {
+			return
+		}
+
+		// 预测指标不处理
+		if oldEdbInfo.EdbInfoType == 1 {
+			return
+		}
+		// 计算指标不处理
+		if oldEdbInfo.EdbType == 2 {
+			return
+		}
+
+		var newEdbInfo *data_manage.EdbInfoItem
+		err = json.Unmarshal([]byte(newData), &newEdbInfo)
+		if err != nil {
+			return
+		}
+
+		isUpdateEdbInfo := checkUpdateType(oldEdbInfo, newEdbInfo)
+		// 指标信息
+		if isUpdateEdbInfo {
+
+			// 获取数据源中指标的基础信息
+			origInfo := getOrigInfo(newEdbInfo.Source, newEdbInfo.SubSource, newEdbInfo.EdbCode)
+
+			pushIndexData = &PushIndexParamDataReq{
+				SourceIndexCode:  newEdbInfo.EdbCode,
+				IndexCode:        fmt.Sprint(newEdbInfo.Source, "_", newEdbInfo.EdbCode),
+				IndexName:        origInfo.EdbName,
+				IndexShortName:   newEdbInfo.EdbName,
+				FrequenceName:    newEdbInfo.Frequency,
+				UnitName:         newEdbInfo.Unit,
+				AssetBeginDate:   newEdbInfo.StartDate,
+				AssetEndDate:     newEdbInfo.EndDate,
+				CreateUser:       newEdbInfo.SysUserRealName,
+				IndexCreateTime:  newEdbInfo.CreateTime,
+				UpdateUser:       newEdbInfo.SysUserRealName,
+				DetailUpdateTime: getMaxModifyTime(newEdbInfo.Source, newEdbInfo.SubSource, newEdbInfo.EdbCode, edbUpdateLog.CreateTime.Format(utils.FormatDateTime)),
+				IndexUpdateTime:  newEdbInfo.ModifyTime,
+				OrginSource:      newEdbInfo.SourceName,
+				OrginSysSource:   origInfo.SourceName,
+				SysSource:        "产研平台",
+				SourceType:       getSourceType(newEdbInfo.Source),
+				Status:           1,
+			}
+		}
+
+		// 指标与分类的关系信息
+		if oldEdbInfo.ClassifyId != newEdbInfo.ClassifyId {
+			pushEdbClassify = &PushEdbClassifyItemReq{
+				Id:         fmt.Sprint(newEdbInfo.EdbInfoId),
+				ClassifyId: newEdbInfo.ClassifyId,
+				IndexCode:  fmt.Sprint(newEdbInfo.Source, "_", newEdbInfo.EdbCode),
+				CreateTime: newEdbInfo.CreateTime,
+				CreateUser: newEdbInfo.SysUserRealName,
+				UpdateTime: newEdbInfo.ModifyTime,
+				UpdateUser: newEdbInfo.SysUserRealName,
+			}
+		}
+
+		return
+	}
+
+	// 数据信息
+	if strings.HasPrefix(edbUpdateLog.OpTableName, "edb_data_") {
+		// 计算指标不处理
+		if strings.HasPrefix(edbUpdateLog.OpTableName, "edb_data_calculate") {
+			return
+		}
+		// 预测指标不处理
+		if strings.HasPrefix(edbUpdateLog.OpTableName, "edb_data_predict") {
+			return
+		}
+
+		var edbData *data_manage.EdbData
+		err = json.Unmarshal([]byte(newData), &edbData)
+		if err != nil {
+			return
+		}
+
+		edbSource, ok := data_manage.EdbTableNameSourceMap[edbUpdateLog.OpTableName]
+		if !ok {
+			// 没有找到来源，那就过滤
+			return
+		}
+		// 数据信息
+		pushIndexValue = &PushIndexValueItemReq{
+			Id:           utils.MD5(fmt.Sprint(edbSource.EdbSourceId, "_", edbSource.SourceName, "_", edbData.EdbDataId)),
+			IndexCode:    fmt.Sprint(edbSource.EdbSourceId, "_", edbData.EdbCode),
+			Value:        fmt.Sprint(edbData.Value),
+			BusinessDate: edbData.DataTime,
+			CreateTime:   edbData.CreateTime,
+			UpdateTime:   edbData.ModifyTime,
+			Status:       "1",
+		}
+
+		return
+	}
+
+	// 上面都没有匹配到的话，那就是数据源的
+	return handleUpdateDataSource(edbUpdateLog)
+
+}
+
+// handleUpdateDataSource
+// @Description: 更新数据处理（数据源的基础数据）
+// @author: Roc
+// @datetime 2024-03-11 14:53:35
+// @param edbUpdateLog *data_manage.EdbUpdateLog
+// @return pushIndexData *PushIndexParamDataReq
+// @return pushEdbClassify *PushEdbClassifyItemReq
+// @return pushIndexValue *PushIndexValueItemReq
+// @return err error
+func handleUpdateDataSource(edbUpdateLog *data_manage.EdbUpdateLog) (pushIndexData *PushIndexParamDataReq, pushEdbClassify *PushEdbClassifyItemReq, pushIndexValue *PushIndexValueItemReq, err error) {
+	oldData := edbUpdateLog.OldData
+	newData := edbUpdateLog.NewData
+
+	isUpdateIndexInfo := false
+	indexName := ``
+	indexSourceName := ``
+
+	source := 0
+	edbCode := ``
+
+	// 钢联
+	if edbUpdateLog.OpTableName == "base_from_mysteel_chemical_index" {
+		var oldEdbInfo *data_manage.BaseFromMysteelChemicalIndexItem
+		err = json.Unmarshal([]byte(oldData), &oldEdbInfo)
+		if err != nil {
+			return
+		}
+
+		var newEdbInfo *data_manage.BaseFromMysteelChemicalIndexItem
+		err = json.Unmarshal([]byte(newData), &newEdbInfo)
+		if err != nil {
+			return
+		}
+
+		if oldEdbInfo.IndexName != newEdbInfo.IndexName {
+			isUpdateIndexInfo = true
+		} else if oldEdbInfo.Source != newEdbInfo.Source {
+			isUpdateIndexInfo = true
+		}
+		if isUpdateIndexInfo {
+			indexName = newEdbInfo.IndexName
+			indexSourceName = newEdbInfo.Source
+			edbCode = newEdbInfo.IndexCode
+			source = utils.DATA_SOURCE_MYSTEEL_CHEMICAL
+		}
+	} else if edbUpdateLog.OpTableName == "base_from_smm_index" {
+		// 有色
+		var oldEdbInfo *data_manage.BaseFromSmmIndexItem
+		err = json.Unmarshal([]byte(oldData), &oldEdbInfo)
+		if err != nil {
+			return
+		}
+
+		var newEdbInfo *data_manage.BaseFromSmmIndexItem
+		err = json.Unmarshal([]byte(newData), &newEdbInfo)
+		if err != nil {
+			return
+		}
+
+		if oldEdbInfo.IndexName != newEdbInfo.IndexName {
+			isUpdateIndexInfo = true
+		} else if oldEdbInfo.Interface != newEdbInfo.Interface {
+			isUpdateIndexInfo = true
+		}
+		if isUpdateIndexInfo {
+			indexName = newEdbInfo.IndexName
+			indexSourceName = newEdbInfo.Interface
+			edbCode = newEdbInfo.IndexCode
+			source = utils.DATA_SOURCE_YS
+		}
+	}
+
+	// 数据源的指标信息变更了
+	if isUpdateIndexInfo {
+		newEdbInfo, tmpErr := data_manage.GetEdbInfoItemByCodeAndSource(source, edbCode)
+		if tmpErr != nil {
+			if tmpErr.Error() != utils.ErrNoRow() {
+				err = tmpErr
+				return
+			}
+			return
+		}
+		pushIndexData = &PushIndexParamDataReq{
+			SourceIndexCode:  newEdbInfo.EdbCode,
+			IndexCode:        fmt.Sprint(newEdbInfo.Source, "_", newEdbInfo.EdbCode),
+			IndexName:        indexName,
+			IndexShortName:   newEdbInfo.EdbName,
+			FrequenceName:    newEdbInfo.Frequency,
+			UnitName:         newEdbInfo.Unit,
+			AssetBeginDate:   newEdbInfo.StartDate,
+			AssetEndDate:     newEdbInfo.EndDate,
+			CreateUser:       newEdbInfo.SysUserRealName,
+			IndexCreateTime:  newEdbInfo.CreateTime,
+			UpdateUser:       newEdbInfo.SysUserRealName,
+			DetailUpdateTime: getMaxModifyTime(newEdbInfo.Source, newEdbInfo.SubSource, newEdbInfo.EdbCode, edbUpdateLog.CreateTime.Format(utils.FormatDateTime)),
+			IndexUpdateTime:  newEdbInfo.ModifyTime,
+			OrginSource:      newEdbInfo.SourceName,
+			OrginSysSource:   indexSourceName,
+			SysSource:        "产研平台",
+			SourceType:       getSourceType(newEdbInfo.Source),
+			Status:           1,
+		}
+	}
+
+	return
+}
+
+// getSourceType
+// @Description: 获取指标来源类型
+// @author: Roc
+// @datetime 2024-03-01 13:40:03
+// @param source int
+// @return string
+func getSourceType(source int) string {
+	switch source {
+	case utils.DATA_SOURCE_MYSTEEL_CHEMICAL, utils.DATA_SOURCE_YS, utils.DATA_SOURCE_BAIINFO, utils.DATA_SOURCE_SCI: //钢联,有色，百川盈孚，红桃3
+		return "RPA"
+	case utils.DATA_SOURCE_MANUAL:
+		return "手工"
+	default:
+		return "接口"
+	}
+}
+
+// checkUpdateType
+// @Description: 检查指标更新状态
+// @author: Roc
+// @datetime 2024-03-11 16:35:22
+// @param oldEdbInfo *data_manage.EdbInfoItem
+// @param newEdbInfo *data_manage.EdbInfoItem
+// @return isUpdateEdbInfo bool
+func checkUpdateType(oldEdbInfo, newEdbInfo *data_manage.EdbInfoItem) (isUpdateEdbInfo bool) {
+	// eta内部名称
+	if oldEdbInfo.EdbName != newEdbInfo.EdbName {
+		isUpdateEdbInfo = true
+		return
+	}
+	if oldEdbInfo.Frequency != newEdbInfo.Frequency {
+		isUpdateEdbInfo = true
+		return
+	}
+	if oldEdbInfo.Unit != newEdbInfo.Unit {
+		isUpdateEdbInfo = true
+		return
+	}
+	if oldEdbInfo.StartDate != newEdbInfo.StartDate {
+		isUpdateEdbInfo = true
+		return
+	}
+	if oldEdbInfo.EndDate != newEdbInfo.EndDate {
+		isUpdateEdbInfo = true
+		return
+	}
+	if oldEdbInfo.SysUserId != newEdbInfo.SysUserId {
+		isUpdateEdbInfo = true
+		return
+	}
+	if oldEdbInfo.SysUserRealName != newEdbInfo.SysUserRealName {
+		isUpdateEdbInfo = true
+		return
+	}
+
+	return
+}
+
+// OrigInfo
+// @Description: 数据源中的指标基础信息
+type OrigInfo struct {
+	EdbName    string
+	SourceName string
+}
+
+// getOrigInfo
+// @Description: 获取数据源中的指标基础信息
+// @author: Roc
+// @datetime 2024-03-11 16:45:34
+// @param source int
+// @param subSource int
+// @param edbCode string
+// @return origInfo OrigInfo
+func getOrigInfo(source, subSource int, edbCode string) (origInfo OrigInfo) {
+	switch source {
+	case utils.DATA_SOURCE_THS, utils.DATA_SOURCE_WIND: // 同花顺、wind
+		origInfo.SourceName = "经济数据库"
+		if subSource == utils.DATA_SUB_SOURCE_DATE {
+			origInfo.SourceName = "日期序列"
+		}
+	case utils.DATA_SOURCE_MYSTEEL_CHEMICAL:
+		edbInfo, err := data_manage.GetBaseFromMysteelChemicalIndexItemByCode(edbCode)
+		if err != nil {
+			return
+		}
+		origInfo.EdbName = edbInfo.IndexName
+		origInfo.SourceName = edbInfo.Source
+	// 钢联化工
+	case utils.DATA_SOURCE_YS: // 有色
+		edbInfo, err := data_manage.GetBaseFromSmmIndexItemItemByCode(edbCode)
+		if err != nil {
+			return
+		}
+		origInfo.EdbName = edbInfo.IndexName
+		origInfo.SourceName = edbInfo.Interface
+	}
+
+	return
+}
+
+// getMaxModifyTime
+// @Description:  获取eta指标明细中的指标最大修改时间
+// @author: Roc
+// @datetime 2024-03-11 17:10:28
+// @param source int
+// @param edbCode string
+// @param addUpdateTime string
+// @return modifyTime string
+func getMaxModifyTime(source, subSource int, edbCode, addUpdateTime string) (modifyTime string) {
+	modifyTime, err := data_manage.GetEdbInfoMaxModifyTime(source, subSource, edbCode)
+	if err != nil {
+		modifyTime = addUpdateTime
+		return
+	}
+
+	return
+}

services/eta_bridge/xy.go

@@ -7,10 +7,13 @@ import (
 	"eta/eta_task/services/alarm_msg"
 	"eta/eta_task/utils"
 	"fmt"
+	"github.com/rdlucklib/rdluck_tools/uuid"
 	"strings"
 	"sync"
+	"time"
 )
 
+// 用户同步的锁
 var lockSyncUser sync.Mutex
 
 // SyncUser
@@ -53,11 +56,11 @@ type PushBaseParamReq struct {
 // @Description: 指标数据结构
 type PushIndexParamDataReq struct {
 	SourceIndexCode string `json:"source_index_code" description:"上游来源指标ID"`
-	IndexCode       string `json:"index_code" description:""`
-	IndexName       string `json:"index_name" description:""`
-	IndexShortName  string `json:"index_short_name" description:""`
-	FrequenceName   string `json:"frequence_name" description:""`
-	UnitName        string `json:"unit_name" description:""`
+	IndexCode       string `json:"index_code" description:"数仓加工的指标ID，来源+来源ID，使用下划线连接，MST000ID00013242"`
+	IndexName       string `json:"index_name" description:"外部来源的指标名称"`
+	IndexShortName  string `json:"index_short_name" description:"系统内的指标名称/简称"`
+	FrequenceName   string `json:"frequence_name" description:"指标频度，如：日度、周度、月度"`
+	UnitName        string `json:"unit_name" description:"指标单位，如：元/吨、千克、立方米"`
 	//CountryName          string `json:"country_name" description:""`
 	//ProvinceName         string `json:"province_name" description:""`
 	//AreaName             string `json:"area_name" description:""`
@@ -73,99 +76,21 @@ type PushIndexParamDataReq struct {
 	//ContractName         string `json:"contract_name" description:""`
 	//AuthKindName         string `json:"auth_kind_name" description:""`
 	//CustomSmallClassName string `json:"custom_small_class_name" description:""`
-	AssetBeginDate   string `json:"asset_begin_date" description:""`
-	AssetEndDate     string `json:"asset_end_date" description:""`
-	CreateUser       string `json:"create_user" description:""`
-	IndexCreateTime  string `json:"index_create_time" description:""`
-	UpdateUser       string `json:"update_user" description:""`
-	DetailUpdateTime string `json:"detail_update_time" description:""`
-	IndexUpdateTime  string `json:"index_update_time" description:""`
+	AssetBeginDate   string `json:"asset_begin_date" description:"业务字段，指标明细数据的业务日期开始时间;格式yyyy-mm-dd"`
+	AssetEndDate     string `json:"asset_end_date" description:"业务字段，指标明细数据的业务日期结束时间;格式yyyy-mm-dd"`
+	CreateUser       string `json:"create_user" description:"创建人姓名"`
+	IndexCreateTime  string `json:"index_create_time" description:"指标基础信息创建时间戳;格式yyyy-mm-dd hh:mi:ss"`
+	UpdateUser       string `json:"update_user" description:"更新人姓名"`
+	DetailUpdateTime string `json:"detail_update_time" description:"指标明细信息更新时间戳;格式yyyy-mm-dd hh:mi:ss"`
+	IndexUpdateTime  string `json:"index_update_time" description:"指标基础信息更新时间戳;格式yyyy-mm-dd hh:mi:ss"`
 	//DutyDept             string `json:"duty_dept" description:""`
 	//BusinessDept         string `json:"business_dept" description:""`
-	OrginSource    string `json:"orgin_source" description:""`
-	OrginSysSource string `json:"orgin_sys_source" description:""`
-	SysSource      string `json:"sys_source" description:""`
-	SourceType     string `json:"source_type" description:""`
+	OrginSource    string `json:"orgin_source" description:"外部数据原始来源，如国家统计局、钢联等"`
+	OrginSysSource string `json:"orgin_sys_source" description:"外部来源系统，即数据供应商，如钢联、wind、同花顺"`
+	SysSource      string `json:"sys_source" description:"内部来源系统，如产研平台、市价平台"`
+	SourceType     string `json:"source_type" description:"数据接入方式，手工、接口、RPA"`
 	//EtlTime              string `json:"etl_time" description:""`
-	Status int `json:"status" description:""`
-}
-
-// SyncIndexList
-// @Description: 定时同步ETA指标信息变更数据至第三方
-// @author: Roc
-// @datetime 2024-02-28 14:00:45
-// @param cont context.Context
-// @return err error
-func SyncIndexList(cont context.Context) (err error) {
-	defer func() {
-		if err != nil {
-			tips := "SyncIndexList-定时同步ETA指标信息变更数据至第三方失败, ErrMsg:\n" + err.Error()
-			utils.FileLog.Info(tips)
-			go alarm_msg.SendAlarmMsg(tips, 3)
-		}
-	}()
-
-	var condition string
-	var pars []interface{}
-	condition += " AND update_type in (?,?) "
-	pars = append(pars, 1, 2)
-
-	list, err := data_manage.GetEdbInfoUpdateLogByCondition(condition, pars)
-	if err != nil {
-		fmt.Println(err)
-		return
-	}
-
-	dataList := make([]PushIndexParamDataReq, 0)
-	for _, v := range list {
-		dataList = append(dataList, PushIndexParamDataReq{
-			SourceIndexCode:  v.EdbCode,
-			IndexCode:        fmt.Sprint(v.Source, "_", v.EdbCode),
-			IndexName:        v.EdbName,
-			IndexShortName:   v.EdbName, //todo
-			FrequenceName:    v.Frequency,
-			UnitName:         v.Unit,
-			AssetBeginDate:   v.StartDate,
-			AssetEndDate:     v.EndDate,
-			CreateUser:       v.SysUserRealName,
-			IndexCreateTime:  v.CreateTime.Format(utils.FormatDateTime), //todo
-			UpdateUser:       v.UpdateSysUserRealName,
-			DetailUpdateTime: v.CreateTime.Format(utils.FormatDateTime), //todo
-			IndexUpdateTime:  v.CreateTime.Format(utils.FormatDateTime), //todo
-			OrginSource:      v.SourceName,                              // todo
-			OrginSysSource:   v.SourceName,
-			SysSource:        "产研平台", //todo
-			SourceType:       "RPA",      //TODO
-			Status:           1,
-		})
-	}
-
-	lenData := len(dataList)
-	if lenData <= 0 {
-		return
-	}
-
-	req := PushBaseParamReq{
-		SerialID:    utils.GetRandString(32), //todo
-		TableCode:   "",
-		Total:       lenData,
-		IsEmailWarn: 0,
-		Data:        dataList,
-	}
-
-	uri := "/xy/index/pushIndexData"
-	_, e, errMsg := HttpEtaBridgePost(uri, req)
-	if e != nil {
-		err = fmt.Errorf("postRefreshEdbData err: %s", e.Error())
-		fmt.Println(err)
-		return
-	}
-	fmt.Println(errMsg)
-	//if res != nil && res.Ret != 200 {
-	//	err = fmt.Errorf("postRefreshEdbData fail")
-	//	return
-	//}
-	return
+	Status int `json:"status" description:"逻辑删除：0-失效，1-有效"`
 }
 
 // PushIndexValueItemReq
@@ -180,73 +105,6 @@ type PushIndexValueItemReq struct {
 	Status       string `json:"status" description:"逻辑删除使用，0-禁用，1-启用"`
 }
 
-// SyncIndexValueList
-// @Description: 定时同步ETA指标日期值的变更数据至第三方
-// @author: Roc
-// @datetime 2024-02-28 14:00:45
-// @param cont context.Context
-// @return err error
-func SyncIndexValueList(cont context.Context) (err error) {
-	defer func() {
-		if err != nil {
-			tips := "SyncIndexList-定时同步ETA指标信息变更数据至第三方失败, ErrMsg:\n" + err.Error()
-			utils.FileLog.Info(tips)
-			go alarm_msg.SendAlarmMsg(tips, 3)
-		}
-	}()
-
-	var condition string
-	var pars []interface{}
-	condition += " AND update_type = ? "
-	pars = append(pars, 0)
-
-	list, err := data_manage.GetEdbInfoUpdateLogByCondition(condition, pars)
-	if err != nil {
-		fmt.Println(err)
-		return
-	}
-
-	dataList := make([]PushIndexValueItemReq, 0)
-	for _, v := range list {
-		dataList = append(dataList, PushIndexValueItemReq{
-			Id:           utils.MD5(fmt.Sprint(v.Source, "_", v.SourceName, "_", v.Id)),
-			IndexCode:    fmt.Sprint(v.Source, "_", v.EdbCode),
-			Value:        fmt.Sprint(v.LatestValue),
-			BusinessDate: v.LatestDate,
-			CreateTime:   v.EdbModifyTime,
-			UpdateTime:   v.CreateTime.Format(utils.FormatDateTime), //todo,
-			Status:       "1",
-		})
-	}
-
-	lenData := len(dataList)
-	if lenData <= 0 {
-		return
-	}
-
-	req := PushBaseParamReq{
-		SerialID:    utils.GetRandString(32), //todo
-		TableCode:   "",
-		Total:       lenData,
-		IsEmailWarn: 0,
-		Data:        dataList,
-	}
-
-	uri := "/xy/index/pushIndexValue"
-	_, e, errMsg := HttpEtaBridgePost(uri, req)
-	if e != nil {
-		err = fmt.Errorf("postRefreshEdbData err: %s", e.Error())
-		fmt.Println(err)
-		return
-	}
-	fmt.Println(errMsg)
-	//if res != nil && res.Ret != 200 {
-	//	err = fmt.Errorf("postRefreshEdbData fail")
-	//	return
-	//}
-	return
-}
-
 // PushClassifyItemReq
 // @Description: 指标分类数据结构
 type PushClassifyItemReq struct {
@@ -264,6 +122,21 @@ type PushClassifyItemReq struct {
 	SortColumn      int    `json:"sort_column" description:"排序字段,越小越靠前，默认值：10"`
 }
 
+// PushEdbClassifyItemReq
+// @Description: 指标与目录的关系请求结构
+type PushEdbClassifyItemReq struct {
+	Id         string `json:"id" description:"唯一主键"`
+	ClassifyId int    `json:"classify_id" description:"目录分类ID"`
+	IndexCode  string `json:"index_code" description:"指标ID"`
+	CreateTime string `json:"create_time" description:"创建时间"`
+	CreateUser string `json:"create_user" description:"创建人"`
+	UpdateTime string `json:"update_time" description:"修改时间"`
+	UpdateUser string `json:"update_user" description:"修改人"`
+}
+
+// 同步指标分类锁
+var lockSyncClassify sync.Mutex
+
 // SyncClassifyList
 // @Description: 定时同步ETA分类数据至第三方
 // @author: Roc
@@ -271,17 +144,23 @@ type PushClassifyItemReq struct {
 // @param cont context.Context
 // @return err error
 func SyncClassifyList(cont context.Context) (err error) {
+	lockSyncClassify.Lock()
 	defer func() {
 		if err != nil {
-			tips := "SyncIndexList-定时同步ETA指标信息变更数据至第三方失败, ErrMsg:\n" + err.Error()
+			tips := "SyncIndexList-定时同步ETA指标分类变更数据至第三方失败, ErrMsg:\n" + err.Error()
 			utils.FileLog.Info(tips)
 			go alarm_msg.SendAlarmMsg(tips, 3)
 		}
+		lockSyncClassify.Unlock()
 	}()
 
 	var condition string
 	var pars []interface{}
 
+	// 普通指标分类
+	condition = " AND classify_type = ? "
+	pars = append(pars, 0)
+
 	list, err := data_manage.GetAllEdbClassifyListByCondition(condition, pars)
 	if err != nil {
 		fmt.Println(err)
@@ -324,7 +203,7 @@ func SyncClassifyList(cont context.Context) (err error) {
 
 	for k, tmpDataList := range dataLimitList {
 		req := PushBaseParamReq{
-			SerialID:    utils.GetRandString(32), //todo
+			SerialID:    uuid.NewUUID().Hex32(),
 			TableCode:   "",
 			Total:       len(tmpDataList),
 			IsEmailWarn: 0,
@@ -343,107 +222,77 @@ func SyncClassifyList(cont context.Context) (err error) {
 	return
 }
 
-// PushEdbClassifyItemReq
-// @Description: 指标与目录的关系请求结构
-type PushEdbClassifyItemReq struct {
-	Id         string `json:"id" description:"唯一主键"`
-	ClassifyId int    `json:"classify_id" description:"目录分类ID"`
-	IndexCode  string `json:"index_code" description:"指标ID"`
-	CreateTime string `json:"create_time" description:"创建时间"`
-	CreateUser string `json:"create_user" description:"创建人"`
-	UpdateTime string `json:"update_time" description:"修改时间"`
-	UpdateUser string `json:"update_user" description:"修改人"`
-}
+// 同步指标信息锁
+var lockSyncIndex sync.Mutex
 
-// SyncEdbClassifyList
-// @Description: 定时同步ETA指标与分类的关系至第三方
+// SyncIndex
+// @Description: 定时同步指标信息
 // @author: Roc
-// @datetime 2024-02-28 14:00:45
+// @datetime 2024-03-07 17:39:34
 // @param cont context.Context
 // @return err error
-func SyncEdbClassifyList(cont context.Context) (err error) {
+func SyncIndex(cont context.Context) (err error) {
+	lockSyncIndex.Lock()
 	defer func() {
 		if err != nil {
-			tips := "SyncIndexList-定时同步ETA指标信息变更数据至第三方失败, ErrMsg:\n" + err.Error()
+			tips := "SyncIndex-定时同步ETA指标信息变更数据至第三方失败, ErrMsg:\n" + err.Error()
 			utils.FileLog.Info(tips)
 			go alarm_msg.SendAlarmMsg(tips, 3)
 		}
+		lockSyncIndex.Unlock()
 	}()
 
-	var condition string
-	var pars []interface{}
-
-	list, err := data_manage.GetAllEdbInfoClassifyListByCondition(condition, pars)
+	// 获取当前最大ID
+	logMaxId, err := data_manage.GetEdbUpdateLogMaxId()
 	if err != nil {
-		fmt.Println(err)
 		return
 	}
 
-	dataLimitList := make([][]PushEdbClassifyItemReq, 0)
-
-	dataList := make([]PushEdbClassifyItemReq, 0)
-	for _, v := range list {
-		dataList = append(dataList, PushEdbClassifyItemReq{
-			Id:         fmt.Sprint(v.EdbInfoId),
-			ClassifyId: v.ClassifyId,
-			IndexCode:  fmt.Sprint(v.Source, "_", v.EdbCode),
-			CreateTime: v.CreateTime.Format(utils.FormatDateTime), //todo
-			CreateUser: v.SysUserRealName,
-			UpdateTime: v.ModifyTime.Format(utils.FormatDateTime), //todo
-			UpdateUser: v.SysUserRealName,
-		})
-		if len(dataList) >= 100 {
-			dataLimitList = append(dataLimitList, dataList)
-			dataList = make([]PushEdbClassifyItemReq, 0)
+	var currLogId int64
+	// 当前已经操作的最大ID
+	currLogId, err = utils.Rc.GetInt64(utils.CACHE_EDB_UPDATE_LOG_ID)
+	if err != nil {
+		// 如果不是没找到key，那么说明是redis报错
+		if err.Error() != utils.RedisNoKeyErr {
+			return
 		}
-	}
-
-	lenData := len(dataList)
-	if lenData > 0 {
-		dataLimitList = append(dataLimitList, dataList)
-	}
-
-	if len(dataLimitList) < 0 {
-		fmt.Println("无分类数据推送")
-		return
-	}
-
-	for k, tmpDataList := range dataLimitList {
-		req := PushBaseParamReq{
-			SerialID:    utils.GetRandString(32), //todo
-			TableCode:   "",
-			Total:       len(tmpDataList),
-			IsEmailWarn: 0,
-			Data:        tmpDataList,
+		err = nil
+		// 查找当前已经处理了的日志最大ID
+		currLogId, err = data_manage.GetEdbUpdateLogMaxHandleId()
+		if err != nil {
+			if err.Error() != utils.ErrNoRow() {
+				utils.FileLog.Error("查找当前已经处理了的日志最大ID失败：" + err.Error())
+			} else {
+				err = nil
+			}
+			currLogId = 0
 		}
+	}
 
-		uri := "/xy/index/pushEdbClassify"
-		_, e, _ := HttpEtaBridgePost(uri, req)
-		if e != nil {
-			err = fmt.Errorf("第%d组分类数据推送失败，postRefreshEdbData err: %s", k+1, e.Error())
-			fmt.Println(err)
-			continue
-		}
+	// 遍历获取下一页的数据
+	for currId := currLogId; currId < logMaxId; {
+		currId = handlePush(currId, logMaxId)
 	}
 
 	return
 }
 
-func SyncIndex(cont context.Context) (err error) {
-	defer func() {
-		if err != nil {
-			tips := "SyncIndex-定时同步ETA指标信息变更数据至第三方失败, ErrMsg:\n" + err.Error()
-			utils.FileLog.Info(tips)
-			go alarm_msg.SendAlarmMsg(tips, 3)
-		}
-	}()
-
+// handlePush
+// @Description: 推送处理
+// @author: Roc
+// @datetime 2024-03-07 19:20:19
+// @param currLogIdStr int64
+// @param logMaxId int64
+// @return lastId int64
+func handlePush(currLogIdStr, logMaxId int64) (lastId int64) {
+	lastId = currLogIdStr
+
+	// 查询当次需要同步的数据
 	var condition string
 	var pars []interface{}
-	//condition += " AND update_type in (?,?) "
-	//pars = append(pars, 1, 2)
-	condition += " AND id > ?"
-	pars = append(pars, 50)
+
+	condition += " AND id > ? AND id < ?"
+	pars = append(pars, currLogIdStr, logMaxId)
 
 	list, err := data_manage.GetEdbUpdateLogByCondition(condition, pars)
 	if err != nil {
@@ -451,15 +300,16 @@ func SyncIndex(cont context.Context) (err error) {
 		return
 	}
 
-	//pushIndexData *PushIndexParamDataReq, pushEdbClassify *PushEdbClassifyItemReq, pushIndexValue *PushIndexValueItemReq
-
 	pushIndexList := make([]*PushIndexParamDataReq, 0)
 	pushEdbClassifyList := make([]*PushEdbClassifyItemReq, 0)
 	pushIndexValueList := make([]*PushIndexValueItemReq, 0)
 
+	idList := make([]int64, 0)
 	for _, v := range list {
-		pushIndexData, pushEdbClassify, pushIndexValue, err := handleData(v)
-		if err != nil {
+		idList = append(idList, v.Id)
+		pushIndexData, pushEdbClassifyData, pushIndexValueData, tmpErr := handleData(v)
+		if tmpErr != nil {
+			err = tmpErr
 			continue
 		}
 
@@ -467,445 +317,193 @@ func SyncIndex(cont context.Context) (err error) {
 			pushIndexList = append(pushIndexList, pushIndexData)
 		}
 
-		if pushEdbClassify != nil {
-			pushEdbClassifyList = append(pushEdbClassifyList, pushEdbClassify)
+		if pushEdbClassifyData != nil {
+			pushEdbClassifyList = append(pushEdbClassifyList, pushEdbClassifyData)
 		}
 
-		if pushIndexValue != nil {
-			pushIndexValueList = append(pushIndexValueList, pushIndexValue)
+		if pushIndexValueData != nil {
+			pushIndexValueList = append(pushIndexValueList, pushIndexValueData)
 		}
+
+		lastId = v.Id
 	}
 
-	return
-}
+	pushIndex(pushIndexList)
+	pushIndexClassify(pushEdbClassifyList)
+	pushIndexValue(pushIndexValueList)
 
-func handleData(edbUpdateLog *data_manage.EdbUpdateLog) (pushIndexData *PushIndexParamDataReq, pushEdbClassify *PushEdbClassifyItemReq, pushIndexValue *PushIndexValueItemReq, err error) {
-	switch edbUpdateLog.OpType {
-	case "insert":
-		return handleInsert(edbUpdateLog)
-	case "update":
-		return handleUpdate(edbUpdateLog)
-	case "delete":
-		return handleDelete(edbUpdateLog)
+	// 标记处理
+	err = data_manage.HandleUpdateLogByIds(idList, time.Now().Format(utils.FormatDateTime))
+	if err != nil {
+		utils.FileLog.Error("批量处理指标更新记录失败:" + err.Error())
 	}
 
+	utils.Rc.Put(utils.CACHE_EDB_UPDATE_LOG_ID, lastId, 31*24*time.Hour)
+
 	return
 }
 
-func handleInsert(edbUpdateLog *data_manage.EdbUpdateLog) (pushIndexData *PushIndexParamDataReq, pushEdbClassify *PushEdbClassifyItemReq, pushIndexValue *PushIndexValueItemReq, err error) {
-	data := edbUpdateLog.NewData
+const pushBatchSize = 100
 
-	//指标信息
-	if edbUpdateLog.OpTableName == "edb_info" {
-		var edbInfo *data_manage.EdbInfo
-		err = json.Unmarshal([]byte(data), &edbInfo)
-		if err != nil {
-			return
-		}
+// pushIndex
+// @Description: 指标信息数据推送
+// @author: Roc
+// @datetime 2024-03-07 16:35:02
+// @param allPushList []*PushIndexParamDataReq
+func pushIndex(allPushList []*PushIndexParamDataReq) {
+	lenDataList := len(allPushList)
+	if lenDataList <= 0 {
+		return
+	}
+	uri := utils.SyncIndexPath + "/pushIndexData"
 
-		// 指标信息
-		pushIndexData = &PushIndexParamDataReq{
-			SourceIndexCode:  edbInfo.EdbCode,
-			IndexCode:        fmt.Sprint(edbInfo.Source, "_", edbInfo.EdbCode),
-			IndexName:        edbInfo.EdbName,
-			IndexShortName:   edbInfo.EdbName, //todo
-			FrequenceName:    edbInfo.Frequency,
-			UnitName:         edbInfo.Unit,
-			AssetBeginDate:   edbInfo.StartDate,
-			AssetEndDate:     edbInfo.EndDate,
-			CreateUser:       edbInfo.SysUserRealName,
-			IndexCreateTime:  edbInfo.CreateTime.Format(utils.FormatDateTime),
-			UpdateUser:       edbInfo.SysUserRealName,                         // todo
-			DetailUpdateTime: edbInfo.CreateTime.Format(utils.FormatDateTime), //todo
-			IndexUpdateTime:  edbInfo.ModifyTime.Format(utils.FormatDateTime), //todo
-			OrginSource:      edbInfo.SourceName,                              // todo
-			OrginSysSource:   edbInfo.SourceName,
-			SysSource:        "产研平台",
-			SourceType:       getSourceType(edbInfo.Source),
-			Status:           1,
+	errDataList := make([]*PushIndexParamDataReq, 0)
+	errList := make([]string, 0)
+	defer func() {
+		if len(errList) > 0 {
+			dataByte, err := json.Marshal(errDataList)
+			if err != nil {
+				dataByte = []byte("序列化指标信息数据失败" + err.Error())
+			}
+			utils.FileLog.Info("pushIndex errList:%s；推送失败的指标信息数据列表：%s", strings.Join(errList, "\n"), string(dataByte))
 		}
+	}()
 
-		// 指标与分类的关系信息
-		pushEdbClassify = &PushEdbClassifyItemReq{
-			Id:         fmt.Sprint(edbInfo.EdbInfoId),
-			ClassifyId: edbInfo.ClassifyId,
-			IndexCode:  fmt.Sprint(edbInfo.Source, "_", edbInfo.EdbCode),
-			CreateTime: edbInfo.CreateTime.Format(utils.FormatDateTime),
-			CreateUser: edbInfo.SysUserRealName,
-			UpdateTime: edbInfo.ModifyTime.Format(utils.FormatDateTime),
-			UpdateUser: edbInfo.SysUserRealName,
-		}
+	dataLimitList := make([][]*PushIndexParamDataReq, 0)
 
-		return
+	for i := 0; i < lenDataList; i += pushBatchSize {
+		endIndex := min(i+pushBatchSize, lenDataList)
+		tempSlice := allPushList[i:endIndex]
+		dataLimitList = append(dataLimitList, tempSlice)
 	}
 
-	// 分类信息
-	//if edbUpdateLog.OpTableName == "edb_classify" {
-	//	var edbClassify *data_manage.EdbClassify
-	//	err = json.Unmarshal([]byte(data), &edbClassify)
-	//	if err != nil {
-	//		return
-	//	}
-	//
-	//	// 指标信息
-	//	pushClassify = PushClassifyItemReq{
-	//		ClassifyId:      int(edbClassify.ClassifyID),
-	//		ClassifyType:    int(edbClassify.ClassifyType),
-	//		ClassifyName:    edbClassify.ClassifyName,
-	//		ParentId:        int(edbClassify.ParentID),
-	//		HasData:         int(edbClassify.HasData),
-	//		CreateTime:      edbClassify.CreateTime.Format(utils.FormatDateTime),
-	//		UpdateTime:      edbClassify.ModifyTime.Format(utils.FormatDateTime),
-	//		SysUserId:       int(edbClassify.SysUserID),
-	//		SysUserRealName: edbClassify.SysUserRealName,
-	//		Level:           int(edbClassify.Level),
-	//		UniqueCode:      edbClassify.UniqueCode,
-	//		SortColumn:      int(edbClassify.Sort),
-	//	}
-	//
-	//	return
-	//}
-
-	// 数据信息
-	if strings.HasPrefix(edbUpdateLog.OpTableName, "edb_data_") {
-		var edbData *data_manage.EdbData
-		err = json.Unmarshal([]byte(data), &edbData)
-		if err != nil {
-			return
+	for k, dataList := range dataLimitList {
+		req := PushBaseParamReq{
+			SerialID:    uuid.NewUUID().Hex32(),
+			TableCode:   "",
+			Total:       len(dataList),
+			IsEmailWarn: 0,
+			Data:        dataList,
 		}
 
-		edbSource, ok := data_manage.EdbTableNameSourceMap[edbUpdateLog.OpTableName]
-		if !ok {
-			// 没有找到来源，那就过滤
-			return
-		}
-		// 数据信息
-		pushIndexValue = &PushIndexValueItemReq{
-			Id:           utils.MD5(fmt.Sprint(edbSource.EdbSourceId, "_", edbSource.SourceName, "_", edbData.EdbDataId)),
-			IndexCode:    fmt.Sprint(edbSource, "_", edbData.EdbCode),
-			Value:        fmt.Sprint(edbData.Value),
-			BusinessDate: edbData.DataTime.Format(utils.FormatDate),
-			CreateTime:   edbData.CreateTime.Format(utils.FormatDateTime),
-			UpdateTime:   edbData.ModifyTime.Format(utils.FormatDateTime),
-			Status:       "1",
+		_, e, _ := HttpEtaBridgePost(uri, req)
+		if e != nil {
+			errList = append(errList, fmt.Sprintf("第%d组指标信息数据推送失败，postRefreshEdbData err: %s", k+1, e.Error()))
+			errDataList = append(errDataList, dataList...)
+			continue
 		}
-
-		return
 	}
 
-	return
 }
 
-func handleDelete(edbUpdateLog *data_manage.EdbUpdateLog) (pushIndexData *PushIndexParamDataReq, pushEdbClassify *PushEdbClassifyItemReq, pushIndexValue *PushIndexValueItemReq, err error) {
-	data := edbUpdateLog.OldData
+// pushIndexValue
+// @Description: 指标明细数据推送
+// @author: Roc
+// @datetime 2024-03-07 16:32:47
+// @param allPushList []*PushIndexValueItemReq
+func pushIndexValue(allPushList []*PushIndexValueItemReq) {
+	lenDataList := len(allPushList)
+	if lenDataList <= 0 {
+		return
+	}
+	uri := utils.SyncIndexPath + "/pushIndexValue"
 
-	//指标信息
-	if edbUpdateLog.OpTableName == "edb_info" {
-		var edbInfo *data_manage.EdbInfo
-		err = json.Unmarshal([]byte(data), &edbInfo)
-		if err != nil {
-			return
+	errDataList := make([]*PushIndexValueItemReq, 0)
+	errList := make([]string, 0)
+	defer func() {
+		if len(errList) > 0 {
+			dataByte, err := json.Marshal(errDataList)
+			if err != nil {
+				dataByte = []byte("序列化指标明细数据失败" + err.Error())
+			}
+			utils.FileLog.Info("pushIndexValue errList:%s；推送失败的指标信息数据列表：%s", strings.Join(errList, "\n"), string(dataByte))
 		}
+	}()
 
-		// 指标信息
-		pushIndexData = &PushIndexParamDataReq{
-			SourceIndexCode:  edbInfo.EdbCode,
-			IndexCode:        fmt.Sprint(edbInfo.Source, "_", edbInfo.EdbCode),
-			IndexName:        edbInfo.EdbName,
-			IndexShortName:   edbInfo.EdbName, //todo
-			FrequenceName:    edbInfo.Frequency,
-			UnitName:         edbInfo.Unit,
-			AssetBeginDate:   edbInfo.StartDate,
-			AssetEndDate:     edbInfo.EndDate,
-			CreateUser:       edbInfo.SysUserRealName,
-			IndexCreateTime:  edbInfo.CreateTime.Format(utils.FormatDateTime),
-			UpdateUser:       edbInfo.SysUserRealName,                         // todo
-			DetailUpdateTime: edbInfo.CreateTime.Format(utils.FormatDateTime), //todo
-			IndexUpdateTime:  edbInfo.ModifyTime.Format(utils.FormatDateTime), //todo
-			OrginSource:      edbInfo.SourceName,                              // todo
-			OrginSysSource:   edbInfo.SourceName,
-			SysSource:        "产研平台",
-			SourceType:       getSourceType(edbInfo.Source),
-			Status:           0,
-		}
+	dataLimitList := make([][]*PushIndexValueItemReq, 0)
 
-		// 指标与分类的关系信息
-		pushEdbClassify = &PushEdbClassifyItemReq{
-			Id:         fmt.Sprint(edbInfo.EdbInfoId),
-			ClassifyId: edbInfo.ClassifyId,
-			IndexCode:  fmt.Sprint(edbInfo.Source, "_", edbInfo.EdbCode),
-			CreateTime: edbInfo.CreateTime.Format(utils.FormatDateTime),
-			CreateUser: edbInfo.SysUserRealName,
-			UpdateTime: edbInfo.ModifyTime.Format(utils.FormatDateTime),
-			UpdateUser: edbInfo.SysUserRealName,
-		}
-
-		return
+	for i := 0; i < lenDataList; i += pushBatchSize {
+		endIndex := min(i+pushBatchSize, lenDataList)
+		tempSlice := allPushList[i:endIndex]
+		dataLimitList = append(dataLimitList, tempSlice)
 	}
 
-	// 分类信息
-	//if edbUpdateLog.OpTableName == "edb_classify" {
-	//	var edbClassify *data_manage.EdbClassify
-	//	err = json.Unmarshal([]byte(data), &edbClassify)
-	//	if err != nil {
-	//		return
-	//	}
-	//
-	//	// 指标信息
-	//	pushClassify = PushClassifyItemReq{
-	//		ClassifyId:      int(edbClassify.ClassifyID),
-	//		ClassifyType:    int(edbClassify.ClassifyType),
-	//		ClassifyName:    edbClassify.ClassifyName,
-	//		ParentId:        int(edbClassify.ParentID),
-	//		HasData:         int(edbClassify.HasData),
-	//		CreateTime:      edbClassify.CreateTime.Format(utils.FormatDateTime),
-	//		UpdateTime:      edbClassify.ModifyTime.Format(utils.FormatDateTime),
-	//		SysUserId:       int(edbClassify.SysUserID),
-	//		SysUserRealName: edbClassify.SysUserRealName,
-	//		Level:           int(edbClassify.Level),
-	//		UniqueCode:      edbClassify.UniqueCode,
-	//		SortColumn:      int(edbClassify.Sort),
-	//	}
-	//
-	//	return
-	//}
-
-	// 数据信息
-	if strings.HasPrefix(edbUpdateLog.OpTableName, "edb_data_") {
-		var edbData *data_manage.EdbData
-		err = json.Unmarshal([]byte(data), &edbData)
-		if err != nil {
-			return
+	for k, dataList := range dataLimitList {
+		req := PushBaseParamReq{
+			SerialID:    uuid.NewUUID().Hex32(),
+			TableCode:   "",
+			Total:       len(dataList),
+			IsEmailWarn: 0,
+			Data:        dataList,
 		}
 
-		edbSource, ok := data_manage.EdbTableNameSourceMap[edbUpdateLog.OpTableName]
-		if !ok {
-			// 没有找到来源，那就过滤
-			return
-		}
-		// 数据信息
-		pushIndexValue = &PushIndexValueItemReq{
-			Id:           utils.MD5(fmt.Sprint(edbSource.EdbSourceId, "_", edbSource.SourceName, "_", edbData.EdbDataId)),
-			IndexCode:    fmt.Sprint(edbSource, "_", edbData.EdbCode),
-			Value:        fmt.Sprint(edbData.Value),
-			BusinessDate: edbData.DataTime.Format(utils.FormatDate),
-			CreateTime:   edbData.CreateTime.Format(utils.FormatDateTime),
-			UpdateTime:   edbData.ModifyTime.Format(utils.FormatDateTime),
-			Status:       "0",
+		_, e, _ := HttpEtaBridgePost(uri, req)
+		if e != nil {
+			errList = append(errList, fmt.Sprintf("第%d组指标明细数据推送失败，postRefreshEdbData err: %s", k+1, e.Error()))
+			errDataList = append(errDataList, dataList...)
+			continue
 		}
-
-		return
 	}
 
 	return
 }
 
-func handleUpdate(edbUpdateLog *data_manage.EdbUpdateLog) (pushIndexData *PushIndexParamDataReq, pushEdbClassify *PushEdbClassifyItemReq, pushIndexValue *PushIndexValueItemReq, err error) {
-	oldData := edbUpdateLog.OldData
-	newData := edbUpdateLog.NewData
-
-	//指标信息
-	if edbUpdateLog.OpTableName == "edb_info" {
-		var oldEdbInfo *data_manage.EdbInfo
-		err = json.Unmarshal([]byte(oldData), &oldEdbInfo)
-		if err != nil {
-			return
-		}
-
-		var newEdbInfo *data_manage.EdbInfo
-		err = json.Unmarshal([]byte(newData), &newEdbInfo)
-		if err != nil {
-			return
-		}
+// pushIndexClassify
+// @Description: 指标与分类的关系推送
+// @author: Roc
+// @datetime 2024-03-07 16:32:47
+// @param allPushList []*PushIndexValueItemReq
+func pushIndexClassify(allPushList []*PushEdbClassifyItemReq) {
+	lenDataList := len(allPushList)
+	if lenDataList <= 0 {
+		return
+	}
+	uri := utils.SyncIndexPath + "/pushEdbClassify"
 
-		isUpdateEdbInfo := checkUpdateType(oldEdbInfo, newEdbInfo)
-		// 指标信息
-		if isUpdateEdbInfo {
-			pushIndexData = &PushIndexParamDataReq{
-				SourceIndexCode:  newEdbInfo.EdbCode,
-				IndexCode:        fmt.Sprint(newEdbInfo.Source, "_", newEdbInfo.EdbCode),
-				IndexName:        newEdbInfo.EdbName,
-				IndexShortName:   newEdbInfo.EdbName, //todo
-				FrequenceName:    newEdbInfo.Frequency,
-				UnitName:         newEdbInfo.Unit,
-				AssetBeginDate:   newEdbInfo.StartDate,
-				AssetEndDate:     newEdbInfo.EndDate,
-				CreateUser:       newEdbInfo.SysUserRealName,
-				IndexCreateTime:  newEdbInfo.CreateTime.Format(utils.FormatDateTime),
-				UpdateUser:       newEdbInfo.SysUserRealName,                         // todo
-				DetailUpdateTime: newEdbInfo.CreateTime.Format(utils.FormatDateTime), //todo
-				IndexUpdateTime:  newEdbInfo.ModifyTime.Format(utils.FormatDateTime), //todo
-				OrginSource:      newEdbInfo.SourceName,                              // todo
-				OrginSysSource:   newEdbInfo.SourceName,
-				SysSource:        "产研平台",
-				SourceType:       getSourceType(newEdbInfo.Source),
-				Status:           1,
+	errDataList := make([]*PushEdbClassifyItemReq, 0)
+	errList := make([]string, 0)
+	defer func() {
+		if len(errList) > 0 {
+			dataByte, err := json.Marshal(errDataList)
+			if err != nil {
+				dataByte = []byte("序列化指标明细数据失败" + err.Error())
 			}
+			utils.FileLog.Info("pushIndexClassify errList:%s；推送失败的指标所属分类明细数据列表：%s", strings.Join(errList, "\n"), string(dataByte))
 		}
+	}()
 
-		// 指标与分类的关系信息
-		if oldEdbInfo.ClassifyId != newEdbInfo.ClassifyId {
-			pushEdbClassify = &PushEdbClassifyItemReq{
-				Id:         fmt.Sprint(newEdbInfo.EdbInfoId),
-				ClassifyId: newEdbInfo.ClassifyId,
-				IndexCode:  fmt.Sprint(newEdbInfo.Source, "_", newEdbInfo.EdbCode),
-				CreateTime: newEdbInfo.CreateTime.Format(utils.FormatDateTime),
-				CreateUser: newEdbInfo.SysUserRealName,
-				UpdateTime: newEdbInfo.ModifyTime.Format(utils.FormatDateTime),
-				UpdateUser: newEdbInfo.SysUserRealName,
-			}
-		}
+	dataLimitList := make([][]*PushEdbClassifyItemReq, 0)
 
-		return
+	for i := 0; i < lenDataList; i += pushBatchSize {
+		endIndex := min(i+pushBatchSize, lenDataList)
+		tempSlice := allPushList[i:endIndex]
+		dataLimitList = append(dataLimitList, tempSlice)
 	}
 
-	// 分类信息
-	//if edbUpdateLog.OpTableName == "edb_classify" {
-	//	var edbClassify *data_manage.EdbClassify
-	//	err = json.Unmarshal([]byte(newData), &edbClassify)
-	//	if err != nil {
-	//		return
-	//	}
-	//
-	//	// 指标信息
-	//	pushClassify = PushClassifyItemReq{
-	//		ClassifyId:      int(edbClassify.ClassifyID),
-	//		ClassifyType:    int(edbClassify.ClassifyType),
-	//		ClassifyName:    edbClassify.ClassifyName,
-	//		ParentId:        int(edbClassify.ParentID),
-	//		HasData:         int(edbClassify.HasData),
-	//		CreateTime:      edbClassify.CreateTime.Format(utils.FormatDateTime),
-	//		UpdateTime:      edbClassify.ModifyTime.Format(utils.FormatDateTime),
-	//		SysUserId:       int(edbClassify.SysUserID),
-	//		SysUserRealName: edbClassify.SysUserRealName,
-	//		Level:           int(edbClassify.Level),
-	//		UniqueCode:      edbClassify.UniqueCode,
-	//		SortColumn:      int(edbClassify.Sort),
-	//	}
-	//
-	//	return
-	//}
-
-	// 数据信息
-	if strings.HasPrefix(edbUpdateLog.OpTableName, "edb_data_") {
-		var edbData *data_manage.EdbData
-		err = json.Unmarshal([]byte(newData), &edbData)
-		if err != nil {
-			return
+	for k, dataList := range dataLimitList {
+		req := PushBaseParamReq{
+			SerialID:    uuid.NewUUID().Hex32(),
+			TableCode:   "",
+			Total:       len(dataList),
+			IsEmailWarn: 0,
+			Data:        dataList,
 		}
 
-		edbSource, ok := data_manage.EdbTableNameSourceMap[edbUpdateLog.OpTableName]
-		if !ok {
-			// 没有找到来源，那就过滤
-			return
-		}
-		// 数据信息
-		pushIndexValue = &PushIndexValueItemReq{
-			Id:           utils.MD5(fmt.Sprint(edbSource.EdbSourceId, "_", edbSource.SourceName, "_", edbData.EdbDataId)),
-			IndexCode:    fmt.Sprint(edbSource, "_", edbData.EdbCode),
-			Value:        fmt.Sprint(edbData.Value),
-			BusinessDate: edbData.DataTime.Format(utils.FormatDate),
-			CreateTime:   edbData.CreateTime.Format(utils.FormatDateTime),
-			UpdateTime:   edbData.ModifyTime.Format(utils.FormatDateTime),
-			Status:       "1",
+		_, e, _ := HttpEtaBridgePost(uri, req)
+		if e != nil {
+			errList = append(errList, fmt.Sprintf("第%d组指标所属分类数据推送失败，postRefreshEdbData err: %s", k+1, e.Error()))
+			errDataList = append(errDataList, dataList...)
+			continue
 		}
-
-		return
 	}
 
 	return
 }
 
-//PushIndexParamDataReq{
-//SourceIndexCode:  edbInfo.EdbCode,
-//IndexCode:        fmt.Sprint(edbInfo.Source, "_", edbInfo.EdbCode),
-//IndexName:        edbInfo.EdbName,
-//IndexShortName:   edbInfo.EdbName, //todo
-//FrequenceName:    edbInfo.Frequency,
-//UnitName:         edbInfo.Unit,
-//AssetBeginDate:   edbInfo.StartDate,
-//AssetEndDate:     edbInfo.EndDate,
-//CreateUser:       edbInfo.SysUserRealName,
-//IndexCreateTime:  edbInfo.CreateTime.Format(utils.FormatDateTime),
-//UpdateUser:       edbInfo.SysUserRealName,                         // todo
-//DetailUpdateTime: edbInfo.CreateTime.Format(utils.FormatDateTime), //todo
-//IndexUpdateTime:  edbInfo.ModifyTime.Format(utils.FormatDateTime), //todo
-//OrginSource:      edbInfo.SourceName,                              // todo
-//OrginSysSource:   edbInfo.SourceName,
-//SysSource:        "产研平台",
-//SourceType:       getSourceType(edbInfo.Source),
-//Status:           1,
-//}
-
-// getSourceType
-// @Description: 获取指标来源类型
-// @author: Roc
-// @datetime 2024-03-01 13:40:03
-// @param source int
-// @return string
-func getSourceType(source int) string {
-	switch source {
-	case utils.DATA_SOURCE_MYSTEEL_CHEMICAL, utils.DATA_SOURCE_YS, utils.DATA_SOURCE_BAIINFO, utils.DATA_SOURCE_SCI: //钢联,有色，百川盈孚，红桃3
-		return "RPA"
-	case utils.DATA_SOURCE_MANUAL:
-		return "手工"
-	default:
-		return "接口"
+// 辅助函数：返回a和b中的较小值
+func min(a, b int) int {
+	if a < b {
+		return a
 	}
+	return b
 }
-
-func checkUpdateType(oldEdbInfo, newEdbInfo *data_manage.EdbInfo) (isUpdateEdbInfo bool) {
-	//todo 外部来源名称需要处理；更新人姓名，指标更新时间
-
-	// eta内部名称
-	if oldEdbInfo.EdbName != newEdbInfo.EdbName {
-		isUpdateEdbInfo = true
-		return
-	}
-	if oldEdbInfo.Frequency != newEdbInfo.Frequency {
-		isUpdateEdbInfo = true
-		return
-	}
-	if oldEdbInfo.Unit != newEdbInfo.Unit {
-		isUpdateEdbInfo = true
-		return
-	}
-	if oldEdbInfo.StartDate != newEdbInfo.StartDate {
-		isUpdateEdbInfo = true
-		return
-	}
-	if oldEdbInfo.EndDate != newEdbInfo.EndDate {
-		isUpdateEdbInfo = true
-		return
-	}
-	if oldEdbInfo.SysUserId != newEdbInfo.SysUserId {
-		isUpdateEdbInfo = true
-		return
-	}
-	if oldEdbInfo.SysUserRealName != newEdbInfo.SysUserRealName {
-		isUpdateEdbInfo = true
-		return
-	}
-
-	return
-}
-
-//func pushIndex(pushIndexList []*PushIndexParamDataReq) {
-//	if len(pushIndexList) <= 0 {
-//		return
-//	}
-//
-//	uri := "/xy/index/pushIndexData"
-//	req := PushBaseParamReq{
-//		SerialID:    utils.GetRandString(32), //todo
-//		TableCode:   "",
-//		Total:       lenData,
-//		IsEmailWarn: 0,
-//		Data:        dataList,
-//	}
-//
-//	var pushIndexList []*PushIndexParamDataReq
-//	dataLimitList := make([][]PushBaseParamReq, 0)
-//
-//	_, e, errMsg := HttpEtaBridgePost(uri, req)
-//}

services/task_xy.go

@@ -15,18 +15,36 @@ func XyTask() {
 	//eta_bridge.SyncIndexValueList(nil)
 	//eta_bridge.SyncClassifyList(nil)
 	//eta_bridge.SyncEdbClassifyList(nil)
+	//eta_bridge.SyncIndex(nil)
 	//return
 
+	// 用户信息
 	if utils.SyncUserPath != `` {
 		syncUserTime := utils.SyncUserTime
 		if syncUserTime == `` {
 			syncUserTime = "0 0 * * * *"
 		}
-		// 每天同步一次指标列表
-		syncUser := task.NewTask("SyncUser", syncUserTime, eta_bridge.SyncUser)
+		// 每小时同步一次用户列表
+		syncUser := task.NewTask("syncUser", syncUserTime, eta_bridge.SyncUser)
 		task.AddTask("定时同步用户账号", syncUser)
 	}
-	// 每天同步一次指标列表
-	//syncIndex := task.NewTask("syncZhongJiIndexList", "0 0 17 * * *", eta_bridge.SyncIndexList)
-	//task.AddTask("定时同步指标列表", syncIndex)
+
+	// 指标相关
+	if utils.SyncIndexPath != `` {
+		// 每天同步一次指标与分类关系的列表
+		syncIndexClassifyTime := utils.SyncIndexClassifyTime
+		if syncIndexClassifyTime == `` {
+			syncIndexClassifyTime = "0 0 0 * * *"
+		}
+		syncIndexClassify := task.NewTask("syncIndexClassify", syncIndexClassifyTime, eta_bridge.SyncClassifyList)
+		task.AddTask("定时同步指标与分类关系", syncIndexClassify)
+
+		// 每小时同步一次指标列表
+		syncIndexTime := utils.SyncIndexTime
+		if syncIndexTime == `` {
+			syncIndexTime = "0 0 * * * *"
+		}
+		syncIndex := task.NewTask("syncIndex", syncIndexTime, eta_bridge.SyncIndex)
+		task.AddTask("定时同步指标信息", syncIndex)
+	}
 }

utils/config.go

@@ -121,12 +121,15 @@ var (
 )
 
 var (
-	EtaBridgeUrl       string // 桥接服务地址
-	EtaBridgeAppNameEn string // 桥接服务英文名称-鉴权用
-	EtaBridgeMd5Key    string // 桥接服务Md5密钥-鉴权用
-	EtaBridgeDesKey    string // 桥接服务Des密钥-解密数据用
-	SyncUserPath       string // 桥接服务-用户同步服务地址
-	SyncUserTime       string // 桥接服务-用户同步的时间配置
+	EtaBridgeUrl          string // 桥接服务地址
+	EtaBridgeAppNameEn    string // 桥接服务英文名称-鉴权用
+	EtaBridgeMd5Key       string // 桥接服务Md5密钥-鉴权用
+	EtaBridgeDesKey       string // 桥接服务Des密钥-解密数据用
+	SyncUserPath          string // 桥接服务-用户同步服务地址
+	SyncUserTime          string // 桥接服务-用户同步的时间配置
+	SyncIndexPath         string // 桥接服务-指标同步服务地址
+	SyncIndexTime         string // 桥接服务-指标同步的时间配置
+	SyncIndexClassifyTime string // 桥接服务-指标分类同步的时间配置
 )
 
 var HzDataApi string
@@ -266,12 +269,15 @@ func init() {
 
 	// 桥接服务
 	{
-		EtaBridgeUrl = config["eta_bridge_url"]               // 桥接服务地址
-		EtaBridgeAppNameEn = config["eta_bridge_app_name_en"] // 桥接服务英文名称-鉴权用
-		EtaBridgeMd5Key = config["eta_bridge_md5_key"]        // 桥接服务Md5密钥-鉴权用
-		EtaBridgeDesKey = config["eta_bridge_des_key"]        // 桥接服务Des密钥-解密数据用
-		SyncUserPath = config["sync_user_path"]               // 桥接服务-用户同步服务地址
-		SyncUserTime = config["sync_user_time"]               // 桥接服务-用户同步的时间配置
+		EtaBridgeUrl = config["eta_bridge_url"]                    // 桥接服务地址
+		EtaBridgeAppNameEn = config["eta_bridge_app_name_en"]      // 桥接服务英文名称-鉴权用
+		EtaBridgeMd5Key = config["eta_bridge_md5_key"]             // 桥接服务Md5密钥-鉴权用
+		EtaBridgeDesKey = config["eta_bridge_des_key"]             // 桥接服务Des密钥-解密数据用
+		SyncUserPath = config["sync_user_path"]                    // 桥接服务-用户同步服务地址
+		SyncUserTime = config["sync_user_time"]                    // 桥接服务-用户同步的时间配置
+		SyncIndexPath = config["sync_index_path"]                  // 桥接服务-指标同步服务地址
+		SyncIndexTime = config["sync_index_time"]                  // 桥接服务-指标同步的时间配置
+		SyncIndexClassifyTime = config["sync_index_classify_time"] // 桥接服务-指标分类同步的时间配置
 	}
 
 	HzDataApi = config["hz_data_api"]

utils/constants.go

@@ -26,16 +26,16 @@ const (
 // 数据来源渠道
 const (
 	DATA_SOURCE_THS                                  = iota + 1 //同花顺
-	DATA_SOURCE_WIND                                            //wind
-	DATA_SOURCE_PB                                              //彭博
-	DATA_SOURCE_CALCULATE                                       //计算指标
-	DATA_SOURCE_CALCULATE_LJZZY                                 //累计值转月
-	DATA_SOURCE_CALCULATE_TBZ                                   //同比值
-	DATA_SOURCE_CALCULATE_TCZ                                   //同差值
-	DATA_SOURCE_CALCULATE_NSZYDPJJS                             //N数值移动平均计算
-	DATA_SOURCE_MANUAL                                          //手工指标
-	DATA_SOURCE_LZ                                              //隆众
-	DATA_SOURCE_YS                                              //有色
+	DATA_SOURCE_WIND                                            //wind->2
+	DATA_SOURCE_PB                                              //彭博->3
+	DATA_SOURCE_CALCULATE                                       //指标运算->4
+	DATA_SOURCE_CALCULATE_LJZZY                                 //累计值转月->5
+	DATA_SOURCE_CALCULATE_TBZ                                   //同比值->6
+	DATA_SOURCE_CALCULATE_TCZ                                   //同差值->7
+	DATA_SOURCE_CALCULATE_NSZYDPJJS                             //N数值移动平均计算->8
+	DATA_SOURCE_MANUAL                                          //手工指标->9
+	DATA_SOURCE_LZ                                              //隆众->10
+	DATA_SOURCE_YS                                              //有色->11
 	DATA_SOURCE_CALCULATE_HBZ                                   //环比值->12
 	DATA_SOURCE_CALCULATE_HCZ                                   //环差值->13
 	DATA_SOURCE_CALCULATE_BP                                    //变频->14
@@ -50,14 +50,14 @@ const (
 	DATA_SOURCE_CALCULATE_ZJPJ                                  //直接拼接->23
 	DATA_SOURCE_CALCULATE_LJZTBPJ                               //累计值同比拼接->24
 	DATA_SOURCE_LT                                              //路透->25
-	DATA_SOURCE_COAL                                            //煤炭网->26
+	DATA_SOURCE_COAL                                            //中国煤炭市场网->26
 	DATA_SOURCE_PYTHON                                          //python代码->27
 	DATA_SOURCE_PB_FINANCE                                      //彭博财务数据->28
 	DATA_SOURCE_GOOGLE_TRAVEL                                   //谷歌出行->29
 	DATA_SOURCE_PREDICT                                         //普通预测指标->30
 	DATA_SOURCE_PREDICT_CALCULATE                               //预测指标运算->31
-	DATA_SOURCE_PREDICT_CALCULATE_TBZ                           //预测指标同比值->32
-	DATA_SOURCE_PREDICT_CALCULATE_TCZ                           //预测指标同差值->33
+	DATA_SOURCE_PREDICT_CALCULATE_TBZ                           //预测同比值->32
+	DATA_SOURCE_PREDICT_CALCULATE_TCZ                           //预测同差值->33
 	DATA_SOURCE_MYSTEEL_CHEMICAL                                //钢联化工->34
 	DATA_SOURCE_CALCULATE_CJJX                                  //超季节性->35
 	DATA_SOURCE_EIA_STEO                                        //eia steo报告->36
@@ -83,9 +83,9 @@ const (
 	DATA_SOURCE_PREDICT_CALCULATE_KSZS                          //预测指标 - 计算指标(扩散指数)->56
 	DATA_SOURCE_BAIINFO                                         //百川盈孚 ->57
 	DATA_SOURCE_STOCK_PLANT                                     //存量装置 ->58
-	DATA_SOURCE_CALCULATE_CORRELATION                           //相关性计算->59
+	DATA_SOURCE_CALCULATE_CORRELATION                           //滚动相关性->59
 	DATA_SOURCE_NATIONAL_STATISTICS                             //国家统计局->60
-	DATA_SOURCE_MEETING_PROBABILITIES                           //美联储加息概率->61
+	DATA_SOURCE_CALCULATE_LJZZJ                                 //累计值转季 -> 61
 	DATA_SOURCE_CALCULATE_LJZ                                   //累计值 -> 62
 	DATA_SOURCE_CALCULATE_LJZNCZJ                               //累计值(年初至今） -> 63
 	DATA_SOURCE_PREDICT_CALCULATE_LJZZJ                         //预测指标 - 累计值转季->64
@@ -144,3 +144,12 @@ const (
 	BusinessCodeJiaYue  = "E2023092201" // 嘉悦物产
 	BusinessCodeZhongJi = "E2023110300" // 中基宁波
 )
+
+// 子数据来源渠道
+const (
+	DATA_SUB_SOURCE_EDB  = iota //经济数据库
+	DATA_SUB_SOURCE_DATE        //日期序列
+)
+
+// 已经处理了的变更id
+const CACHE_EDB_UPDATE_LOG_ID = "eta:edb_update_log:id"

utils/redis.go

@@ -7,6 +7,9 @@ import (
 
 type RedisClient interface {
 	Get(key string) interface{}
+	GetStr(key string) string
+	GetInt64(key string) (int64, error)
+	GetUInt64(key string) (uint64, error)
 	RedisBytes(key string) (data []byte, err error)
 	RedisString(key string) (data string, err error)
 	RedisInt(key string) (data int, err error)
@@ -31,3 +34,6 @@ func initRedis(redisType string, conf string) (redisClient RedisClient, err erro
 
 	return
 }
+
+// RedisNoKeyErr redis没有key的错误
+const RedisNoKeyErr = "redis: nil"

utils/redis/cluster_redis.go

@@ -86,6 +86,35 @@ func (rc *ClusterRedisClient) Get(key string) interface{} {
 	return data
 }
 
+// GetStr
+// @Description: 根据key获取字符串数据
+// @receiver rc
+// @param key
+// @return string
+func (rc *ClusterRedisClient) GetStr(key string) string {
+	return rc.redisClient.Get(context.TODO(), key).Val()
+}
+
+// GetInt
+// @Description: 根据key获取int数据
+// @receiver rc
+// @param key
+// @return int
+// @return error
+func (rc *ClusterRedisClient) GetInt64(key string) (int64, error) {
+	return rc.redisClient.Get(context.TODO(), key).Int64()
+}
+
+// GetInt
+// @Description: 根据key获取int数据
+// @receiver rc
+// @param key
+// @return int
+// @return error
+func (rc *ClusterRedisClient) GetUInt64(key string) (uint64, error) {
+	return rc.redisClient.Get(context.TODO(), key).Uint64()
+}
+
 // RedisBytes
 // @Description: 根据key获取字节编码数据
 // @receiver rc

utils/redis/standalone_redis.go

@@ -78,6 +78,35 @@ func (rc *StandaloneRedisClient) Get(key string) interface{} {
 	return data
 }
 
+// GetStr
+// @Description: 根据key获取字符串数据
+// @receiver rc
+// @param key
+// @return string
+func (rc *StandaloneRedisClient) GetStr(key string) string {
+	return rc.redisClient.Get(context.TODO(), key).Val()
+}
+
+// GetInt
+// @Description: 根据key获取int数据
+// @receiver rc
+// @param key
+// @return int
+// @return error
+func (rc *StandaloneRedisClient) GetInt64(key string) (int64, error) {
+	return rc.redisClient.Get(context.TODO(), key).Int64()
+}
+
+// GetInt
+// @Description: 根据key获取int数据
+// @receiver rc
+// @param key
+// @return int
+// @return error
+func (rc *StandaloneRedisClient) GetUInt64(key string) (uint64, error) {
+	return rc.redisClient.Get(context.TODO(), key).Uint64()
+}
+
 // RedisBytes
 // @Description: 根据key获取字节编码数据
 // @receiver rc