feat：数据调整逻辑变更

models/base_from_adjust.go

@@ -1,6 +1,7 @@
 package models
 
 import (
+	"errors"
 	"fmt"
 	"github.com/beego/beego/v2/client/orm"
 	"hongze/hongze_edb_lib/services/alarm_msg"
@@ -107,6 +108,23 @@ func SaveAdjustEdb(req SaveAdjustEdbReq) (edbInfo *EdbInfo, err error, errMsg st
 				return
 			}
 		}
+		//相关配置
+		{
+			var fromEdbEndDate time.Time
+			if fromEdbInfo.EndDate != `0000-00-00` {
+				fromEdbEndDate, _ = time.Parse(utils.FormatDate, fromEdbInfo.EndDate)
+			}
+			edbAdjustConf := &EdbAdjustConf{
+				EdbInfoId:     edbInfo.EdbInfoId,
+				SourceEndDate: fromEdbEndDate,
+				ModifyTime:    time.Now(),
+				CreateTime:    time.Now(),
+			}
+			_, err = to.Insert(edbAdjustConf)
+			if err != nil {
+				return
+			}
+		}
 	} else {
 		edbInfo, err = GetEdbInfoById(req.EdbInfoId)
 		if err != nil {
@@ -136,6 +154,45 @@ func SaveAdjustEdb(req SaveAdjustEdbReq) (edbInfo *EdbInfo, err error, errMsg st
 		if err != nil {
 			return
 		}
+
+		//相关配置
+		{
+			// 获取来源指标
+			calculateMapping, tmpErr := GetEdbInfoCalculateMappingDetail(edbInfo.EdbInfoId)
+			if tmpErr != nil {
+				errMsg = `关联指标获取异常`
+				err = tmpErr
+				return
+			}
+			fromEdbInfo, tmpErr := GetEdbInfoById(calculateMapping.FromEdbInfoId)
+			if tmpErr != nil {
+				errMsg = `来源指标不存在`
+				err = tmpErr
+				return
+			}
+			var fromEdbEndDate time.Time
+			if fromEdbInfo.EndDate != `0000-00-00` {
+				fromEdbEndDate, _ = time.Parse(utils.FormatDate, fromEdbInfo.EndDate)
+			}
+
+			// 获取之前的配置
+			var edbAdjustConf *EdbAdjustConf
+			sql := ` SELECT * FROM edb_adjust_conf WHERE edb_info_id=? `
+			err = o.Raw(sql, edbInfo.EdbInfoId).QueryRow(&edbAdjustConf)
+			if err != nil {
+				errMsg = "获取指标配置失败"
+				return
+			}
+
+			// 修改配置的最晚日期
+			edbAdjustConf.SourceEndDate = fromEdbEndDate
+			edbAdjustConf.ModifyTime = time.Now()
+			_, err = to.Update(edbAdjustConf, "SourceEndDate", "ModifyTime")
+			if err != nil {
+				errMsg = "更新指标配置失败"
+				return
+			}
+		}
 	}
 	dataList := req.DataList
 
@@ -200,13 +257,15 @@ func RefreshAllAdjustEdb(edbInfo *EdbInfo, fromEdbInfo *EdbInfo) (err error) {
 		}
 	}()
 
-	// 获取当前指标的最晚的日期
+	// 当前数据表名
 	dataTableName := GetEdbDataTableName(edbInfo.Source)
-	var maxDate string
-	sql := `SELECT max(data_time) data_time FROM %s WHERE edb_info_id=? `
-	sql = fmt.Sprintf(sql, dataTableName)
-	err = to.Raw(sql, edbInfo.EdbInfoId).QueryRow(&maxDate)
+
+	// 获取之前的配置
+	var edbAdjustConf *EdbAdjustConf
+	sql := ` SELECT * FROM edb_adjust_conf WHERE edb_info_id=? `
+	err = o.Raw(sql, edbInfo.EdbInfoId).QueryRow(&edbAdjustConf)
 	if err != nil {
+		err = errors.New("获取指标配置失败")
 		return
 	}
 
@@ -214,31 +273,60 @@ func RefreshAllAdjustEdb(edbInfo *EdbInfo, fromEdbInfo *EdbInfo) (err error) {
 	var condition string
 	var pars []interface{}
 	condition += " AND edb_info_id=? AND data_time > ? "
-	pars = append(pars, fromEdbInfo.EdbInfoId, maxDate)
-
+	pars = append(pars, fromEdbInfo.EdbInfoId, edbAdjustConf.SourceEndDate)
 	dataList, err := GetEdbDataListAllByTo(to, condition, pars, fromEdbInfo.Source, 1)
 	if err != nil {
 		return err
 	}
 
-	var isAdd bool
-	addSql := ` INSERT INTO ` + dataTableName + `(edb_info_id,edb_code,data_time,value,create_time,modify_time,data_timestamp) values `
-	for _, item := range dataList {
-		//格式化时间
-		currentDate, tmpErr := time.Parse(utils.FormatDate, item.DataTime)
+	// 已经入库的日期数据
+	existDataMap := make(map[string]float64)
+	{
+		var tmpCondition string
+		var tmpPars []interface{}
+		condition += " AND edb_info_id=? AND data_time > ? "
+		pars = append(pars, fromEdbInfo.EdbInfoId, edbAdjustConf.SourceEndDate)
+		existDataList, tmpErr := GetEdbDataListAllByTo(to, tmpCondition, tmpPars, edbInfo.Source, 1)
 		if tmpErr != nil {
 			err = tmpErr
 			return
 		}
-		timestamp := currentDate.UnixNano() / 1e6
-		timestampStr := fmt.Sprintf("%d", timestamp)
+		for _, v := range existDataList {
+			existDataMap[v.DataTime] = v.Value
+		}
+	}
 
+	var isAdd bool
+	addSql := ` INSERT INTO ` + dataTableName + `(edb_info_id,edb_code,data_time,value,create_time,modify_time,data_timestamp) values `
+	for _, item := range dataList {
+		currDay := item.DataTime
 		//值
 		val := item.Value
 		saveVal := utils.SubFloatToString(val, 20)
-		addSql += GetAddSql(fmt.Sprint(edbInfo.EdbInfoId), edbInfo.EdbCode, item.DataTime, timestampStr, saveVal)
+		if existVal, ok := existDataMap[currDay]; !ok {
+			//格式化时间
+			currentDate, tmpErr := time.Parse(utils.FormatDate, item.DataTime)
+			if tmpErr != nil {
+				err = tmpErr
+				return
+			}
+			timestamp := currentDate.UnixNano() / 1e6
+			timestampStr := fmt.Sprintf("%d", timestamp)
+
+			addSql += GetAddSql(fmt.Sprint(edbInfo.EdbInfoId), edbInfo.EdbCode, item.DataTime, timestampStr, saveVal)
+
+			isAdd = true
+		} else {
+			if existVal != item.Value {
+				sql = ` UPDATE %s SET value=?,modify_time=NOW() WHERE edb_info_id=? AND data_time=? `
+				sql = fmt.Sprintf(sql, dataTableName)
+				_, err = to.Raw(sql, saveVal, edbInfo.EdbInfoId, currDay).Exec()
+				if err != nil {
+					return err
+				}
+			}
+		}
 
-		isAdd = true
 	}
 	if isAdd {
 		addSql = strings.TrimRight(addSql, ",")
@@ -248,6 +336,17 @@ func RefreshAllAdjustEdb(edbInfo *EdbInfo, fromEdbInfo *EdbInfo) (err error) {
 		}
 	}
 
+	// 变更数据调整配置表的实际结束日期
+	{
+		var fromEdbEndDate time.Time
+		if fromEdbInfo.EndDate != `0000-00-00` {
+			fromEdbEndDate, _ = time.Parse(utils.FormatDate, fromEdbInfo.EndDate)
+		}
+		edbAdjustConf.SourceEndDate = fromEdbEndDate
+		edbAdjustConf.ModifyTime = time.Now()
+		_, err = to.Update(edbAdjustConf, "SourceEndDate", "ModifyTime")
+	}
+
 	return
 }
 
@@ -269,3 +368,43 @@ type SaveAdjustEdbDataReq struct {
 	Date  string  `description:"数据日期"`
 	Value float64 `description:"数据值"`
 }
+
+// FixData 修复数据
+func FixData() {
+	var list []*EdbInfo
+	o := orm.NewOrm()
+	sql := ` SELECT * FROM edb_info WHERE source=? `
+	_, err := o.Raw(sql, 40).QueryRows(&list)
+	for _, edbInfo := range list {
+		// 获取来源指标
+		calculateMapping, tmpErr := GetEdbInfoCalculateMappingDetail(edbInfo.EdbInfoId)
+		if tmpErr != nil {
+			fmt.Println(edbInfo.EdbInfoId, "关联指标获取异常；err:", tmpErr)
+			continue
+		}
+		fromEdbInfo, tmpErr := GetEdbInfoById(calculateMapping.FromEdbInfoId)
+		if tmpErr != nil {
+			fmt.Println(edbInfo.EdbInfoId, "来源指标不存在；err:", tmpErr)
+			continue
+		}
+		var fromEdbEndDate time.Time
+		if fromEdbInfo.EndDate != `0000-00-00` {
+			fromEdbEndDate, _ = time.Parse(utils.FormatDate, fromEdbInfo.EndDate)
+		}
+
+		// 获取之前的配置
+		edbAdjustConf := &EdbAdjustConf{
+			EdbInfoId:     edbInfo.EdbInfoId,
+			SourceEndDate: fromEdbEndDate,
+			ModifyTime:    time.Now(),
+			CreateTime:    time.Now(),
+		}
+		_, err = o.Insert(edbAdjustConf)
+		if err != nil {
+			fmt.Println(edbInfo.EdbInfoId, "添加失败；err:", tmpErr)
+			continue
+		}
+
+	}
+	fmt.Println("end")
+}

models/base_from_manual.go

@@ -29,7 +29,7 @@ func GetEdbdataManualByCondition(condition string, pars []interface{}) (item []*
 	return
 }
 
-//新增弘则手工指标数据
+// AddEdbDataFromManual 新增弘则手工指标数据
 func AddEdbDataFromManual(edbCode string) (err error) {
 	o := orm.NewOrm()
 
@@ -74,7 +74,7 @@ func AddEdbDataFromManual(edbCode string) (err error) {
 	return
 }
 
-//刷新手工指标数据
+// RefreshEdbDataFromManual 刷新手工指标数据
 func RefreshEdbDataFromManual(edbInfoId int, edbCode, startDate string) (err error) {
 	source := utils.DATA_SOURCE_MANUAL
 	o := orm.NewOrm()
@@ -117,7 +117,7 @@ func RefreshEdbDataFromManual(edbInfoId int, edbCode, startDate string) (err err
 		existMap[v.DataTime] = v
 	}
 
-	addSql := ` INSERT INTO edb_data_manual(edb_info_id,edb_code,data_time,value,create_time,modify_time,data_timestamp) values `
+	addSql := ` INSERT INTO edb_data_manual (edb_info_id,edb_code,data_time,value,create_time,modify_time,data_timestamp) values `
 	var isAdd bool
 	manualMap := make(map[string]*ManualEdbdata)
 	//fmt.Println("manualDataList:", len(manualDataList))

models/db.go

@@ -50,6 +50,7 @@ func init() {
 		new(EdbDataPredictCalculateLjztbpj),
 		new(EdbDataPredictCalculateNhcc),
 		new(EdbDataPredictCalculateZjpj),
+		new(EdbAdjustConf), // 数据调整的配置
 	)
 
 	// 注册期货数据 数据表

models/edb_adjust_conf.go

@@ -0,0 +1,13 @@
+package models
+
+import (
+	"time"
+)
+
+// EdbAdjustConf 计算指标于基础指标，关系表
+type EdbAdjustConf struct {
+	EdbInfoId     int       `orm:"column(edb_info_id);pk" description:"计算指标id"`
+	SourceEndDate time.Time `description:"来源指标的实际数据日期"`
+	ModifyTime    time.Time `description:"修改时间"`
+	CreateTime    time.Time `description:"创建时间"`
+}