Merge branch 'debug' of http://8.136.199.33:3000/eta_server/eta_index_lib into bzq1/sci_hq_custom_cf

controllers/base_from_calculate.go

@@ -118,6 +118,9 @@ func (this *CalculateController) Add() {
 	if err != nil {
 		return
 	}
+
+	// 判断是否需要禁用
+	go services.DisableEdbInfoNoUpdate(edbInfo)
 	resp := models.AddEdbInfoResp{
 		EdbInfoId:  edbInfo.EdbInfoId,
 		UniqueCode: uniqueCode,
@@ -248,6 +251,11 @@ func (this *CalculateController) Edit() {
 		br.Msg = "指标信息不存在，EdbInfoId:" + strconv.Itoa(req.EdbInfoId)
 		return
 	}
+	// 记录指标原始的基本信息
+	oldEdbName := edbInfoDetail.EdbName
+	oldFrequency := edbInfoDetail.Frequency
+	oldUnit := edbInfoDetail.Unit
+
 	var needCalculate bool
 
 	if edbInfoDetail.CalculateFormula != req.CalculateFormula || edbInfoDetail.EmptyType != req.EmptyType || edbInfoDetail.MaxEmptyType != req.MaxEmptyType || edbInfoDetail.Extra != req.Extra {
@@ -332,10 +340,31 @@ func (this *CalculateController) Edit() {
 		br.ErrMsg = err.Error()
 		return
 	}
+	// 记录基础信息变更日志
+	oldEdbInfo := new(models.EdbInfo)
+	oldEdbInfo.EdbInfoId = edbInfoDetail.EdbInfoId
+	oldEdbInfo.EdbName = oldEdbName
+	oldEdbInfo.Frequency = oldFrequency
+	oldEdbInfo.Unit = oldUnit
+	newEdbInfoEditRecord := new(models.EdbInfoEditRecord)
+	newEdbInfoEditRecord.EdbName = req.EdbName
+	newEdbInfoEditRecord.Frequency = req.Frequency
+	newEdbInfoEditRecord.Unit = req.Unit
+	newEdbInfoEditRecord.OperateUserId = req.AdminId
+	newEdbInfoEditRecord.OperateUserRealName = req.AdminName
+	err = services.AddEditEdbInfoRecord(oldEdbInfo, newEdbInfoEditRecord)
+	if err != nil {
+		br.Msg = "记录基础信息变更日志失败"
+		br.ErrMsg = err.Error()
+		return
+	}
+
 	resp := models.AddEdbInfoResp{
 		EdbInfoId:  edbInfoDetail.EdbInfoId,
 		UniqueCode: edbInfoDetail.UniqueCode,
 	}
+	// 重置计算指标中的引用关系
+	go services.ResetEdbRelation(edbInfoDetail.EdbInfoId)
 	br.Ret = 200
 	br.Success = true
 	br.Msg = "保存成功"
@@ -778,6 +807,22 @@ func (this *CalculateController) BatchSave() {
 		return
 	}
 
+	newEdbInfo := new(models.EdbInfoEditRecord)
+	newEdbInfo.EdbName = req.EdbName
+	newEdbInfo.Frequency = req.Frequency
+	newEdbInfo.Unit = req.Unit
+	newEdbInfo.OperateUserId = req.AdminId
+	newEdbInfo.OperateUserRealName = req.AdminName
+	err = services.AddEditEdbInfoRecord(edbInfo, newEdbInfo)
+	if err != nil {
+		br.Msg = "保存失败"
+		br.ErrMsg = "保存失败,Err:" + err.Error()
+		return
+	}
+
+	// 判断是否需要禁用
+	go services.DisableEdbInfoNoUpdate(edbInfo)
+
 	resp := models.AddEdbInfoResp{
 		EdbInfoId:  edbInfo.EdbInfoId,
 		UniqueCode: edbInfo.UniqueCode,
@@ -861,6 +906,10 @@ func (this *CalculateController) BatchEdit() {
 		br.ErrMsg = "获取指标信息失败:Err:" + err.Error()
 		return
 	}
+	// 记录原始指标信息
+	oldEdbName := edbInfo.EdbName
+	oldFrequency := edbInfo.Frequency
+	oldUnit := edbInfo.Unit
 
 	// 基础指标id
 	fromEdbInfoId := req.FromEdbInfoId
@@ -1207,6 +1256,26 @@ func (this *CalculateController) BatchEdit() {
 		br.ErrMsg = err.Error()
 		return
 	}
+	// 记录指标的操作记录
+	oldEdbInfo := new(models.EdbInfo)
+	oldEdbInfo.EdbInfoId = edbInfo.EdbInfoId
+	oldEdbInfo.EdbName = oldEdbName
+	oldEdbInfo.Frequency = oldFrequency
+	oldEdbInfo.Unit = oldUnit
+	newEdbInfoRecord := new(models.EdbInfoEditRecord)
+	newEdbInfoRecord.EdbName = req.EdbName
+	newEdbInfoRecord.Frequency = req.Frequency
+	newEdbInfoRecord.Unit = req.Unit
+	newEdbInfoRecord.OperateUserId = req.AdminId
+	newEdbInfoRecord.OperateUserRealName = req.AdminName
+	err = services.AddEditEdbInfoRecord(oldEdbInfo, newEdbInfoRecord)
+	if err != nil {
+		br.Msg = "保存失败"
+		br.ErrMsg = "记录指标操作记录失败，Err:" + err.Error()
+	}
+
+	// 重置计算指标中的引用关系
+	go services.ResetEdbRelation(edbInfoId)
 
 	resp := models.AddEdbInfoResp{
 		EdbInfoId:  edbInfo.EdbInfoId,
@@ -1440,7 +1509,7 @@ func (this *CalculateController) Refresh() {
 		}
 		//startDate = edbInfo.StartDate
 		endDate = time.Now().Format(utils.FormatDate)
-		err = models.RefreshAllCalculateBp(edbInfoId, source, subSource, fromEdbInfo, calculateTbz.EdbCode, startDate, endDate)
+		err = models.RefreshAllCalculateBp(edbInfoId, source, subSource, fromEdbInfo, calculateTbz.EdbCode, startDate, endDate, edbInfo.EmptyType)
 		if err != nil && err.Error() != utils.ErrNoRow() {
 			errMsg = "RefreshAllCalculateBp Err:" + err.Error()
 			break
@@ -1705,30 +1774,21 @@ func (this *CalculateController) SaveAdjust() {
 		}()
 	}
 
-	//获取指标数据
-	var condition string
-	var pars []interface{}
-	condition += " AND edb_name=? "
-	pars = append(pars, req.EdbName)
-
-	if req.EdbInfoId > 0 {
-		condition += " AND edb_info_id != ? "
-		pars = append(pars, req.EdbInfoId)
-	}
-
-	count, err := models.GetEdbInfoCountByCondition(condition, pars)
+	// 校验指标名称是否存在
+	existEdbName, err := logic.CheckExistByEdbNameAndEdbInfoId(0, req.EdbInfoId, req.EdbName, this.Lang)
 	if err != nil {
 		br.Msg = "判断指标名称是否存在失败"
 		br.ErrMsg = "判断指标名称是否存在失败,Err:" + err.Error()
 		return
 	}
-	if count > 0 {
+
+	if existEdbName {
 		br.Msg = "指标名称已存在，请重新填写"
 		br.ErrMsg = "指标名称已存在，请重新填写"
 		return
 	}
 
-	edbInfo, err, errMsg := models.SaveAdjustEdb(req)
+	edbInfo, err, errMsg := models.SaveAdjustEdb(req, this.Lang)
 	if err != nil {
 		br.Msg = errMsg
 		br.Msg = "添加指标失败,Err:" + err.Error()
@@ -1767,6 +1827,11 @@ func (this *CalculateController) SaveAdjust() {
 		EdbInfoId:  edbInfo.EdbInfoId,
 		UniqueCode: edbInfo.UniqueCode,
 	}
+
+	// 判断是否需要禁用
+	go services.DisableEdbInfoNoUpdate(edbInfo)
+	// 重置计算指标中的引用关系
+	go services.ResetEdbRelation(edbInfo.EdbInfoId)
 	br.Ret = 200
 	br.Success = true
 	br.Msg = "保存成功"
@@ -2247,6 +2312,8 @@ func (this *CalculateController) BatchEditMulti() {
 			Frequency:        item.Frequency,
 			Unit:             item.Unit,
 			ClassifyId:       item.ClassifyId,
+			AdminId:          req.AdminId,
+			AdminName:        req.AdminName,
 			Formula:          req.Formula, //N数值移动平均计算、环比值、环差值
 			FromEdbInfoId:    item.FromEdbInfoId,
 			CalculateFormula: req.CalculateFormula,
@@ -2302,3 +2369,119 @@ func (this *CalculateController) BatchEditMulti() {
 	br.Data = resp
 	br.IsAddLog = true
 }
+
+// StepCalculate
+// @Title 多步骤计算
+// @Description 多步骤计算
+// @Param request body models.StepCalculateBySearchData true "type json string"
+// @Success Ret=200 返回指标id
+// @router /base/step_calculate [post]
+func (this *CalculateController) StepCalculate() {
+	br := new(models.BaseResponse).Init()
+	defer func() {
+		if br.ErrMsg == "" {
+			br.IsSendEmail = false
+		}
+		this.Data["json"] = br
+		this.ServeJSON()
+	}()
+	var req models.StepCalculateBySearchData
+	if e := json.Unmarshal(this.Ctx.Input.RequestBody, &req); e != nil {
+		br.Msg = "参数解析异常"
+		br.ErrMsg = fmt.Sprintf("参数解析失败, Err: %v", e)
+		return
+	}
+	//sort.Slice(req.Calculates, func(i, j int) bool {
+	//	return req.Calculates[i].Sort < req.Calculates[j].Sort
+	//})
+	//
+	//var errMsg string
+	//originData, e := models.EdbInfoSearchDataToData(req.DataList)
+	//if e != nil {
+	//	br.Msg = "基础数据异常"
+	//	br.ErrMsg = fmt.Sprintf("计算失败, 基础数据异常, Err: %v", e)
+	//	return
+	//}
+	//
+	//calculateData := originData
+	//dateDataMap := make(map[time.Time]float64)
+	//for _, v := range req.Calculates {
+	//	baseCalculate := models.BaseCalculate{
+	//		DataList:      calculateData,
+	//		Frequency:     v.Frequency,
+	//		Formula:       v.Formula,
+	//		Calendar:      v.Calendar,
+	//		MoveType:      v.MoveType,
+	//		MoveFrequency: v.MoveFrequency,
+	//		FromFrequency: v.FromFrequency,
+	//		Source:        v.Source,
+	//	}
+	//
+	//	// 计算方式
+	//	switch baseCalculate.Source {
+	//	case utils.EdbBaseCalculateLjzzy:
+	//		dateDataMap, e, errMsg = baseCalculate.Ljzzy()
+	//	case utils.EdbBaseCalculateLjzzj:
+	//		dateDataMap, e, errMsg = baseCalculate.Ljzzj()
+	//	case utils.EdbBaseCalculateTbz:
+	//		dateDataMap, e, errMsg = baseCalculate.Tbz()
+	//	case utils.EdbBaseCalculateTcz:
+	//		dateDataMap, e, errMsg = baseCalculate.Tcz()
+	//	case utils.EdbBaseCalculateNszydpjjs:
+	//		dateDataMap, e, errMsg = baseCalculate.Nszydpjjs()
+	//	case utils.EdbBaseCalculateHbz:
+	//		dateDataMap, e, errMsg = baseCalculate.Hbz()
+	//	case utils.EdbBaseCalculateHcz:
+	//		dateDataMap, e, errMsg = baseCalculate.Hcz()
+	//	case utils.EdbBaseCalculateUpFrequency:
+	//		dateDataMap, e, errMsg = baseCalculate.UpFrequency()
+	//	case utils.EdbBaseCalculateDownFrequency:
+	//		dateDataMap, e, errMsg = baseCalculate.DownFrequency()
+	//	case utils.EdbBaseCalculateTimeShift:
+	//		dateDataMap, e, errMsg = baseCalculate.TimeShift()
+	//	case utils.EdbBaseCalculateCjjx:
+	//		dateDataMap, e, errMsg = baseCalculate.Cjjx()
+	//	case utils.EdbBaseCalculateAnnualized:
+	//		dateDataMap, e, errMsg = baseCalculate.Annualized()
+	//	case utils.EdbBaseCalculateLjz:
+	//		dateDataMap, e, errMsg = baseCalculate.Ljz()
+	//	case utils.EdbBaseCalculateLjzNczj:
+	//		dateDataMap, e, errMsg = baseCalculate.LjzNczj()
+	//	case utils.EdbBaseCalculateExponentialSmoothing:
+	//		dateDataMap, e, errMsg = baseCalculate.ExponentialSmoothing()
+	//	case utils.EdbBaseCalculateRjz:
+	//		dateDataMap, e, errMsg = baseCalculate.Rjz()
+	//	default:
+	//		errMsg = "计算方式无效"
+	//		e = fmt.Errorf("%s:%d", errMsg, baseCalculate.Source)
+	//	}
+	//	if e != nil {
+	//		br.Msg = "计算失败"
+	//		if errMsg != "" {
+	//			br.Msg = errMsg
+	//		}
+	//		br.ErrMsg = e.Error()
+	//		return
+	//	}
+	//
+	//	calculateData = models.TransDateData2EdbData(dateDataMap)
+	//}
+	resultData, dates, errMsg, e := services.StepCalculate(req.DataList, req.Calculates)
+	if e != nil {
+		br.Msg = "计算失败"
+		if errMsg != "" {
+			br.Msg = errMsg
+		}
+		br.ErrMsg = fmt.Sprintf("分步骤计算失败, Err: %v", e)
+		return
+	}
+
+	br.Ret = 200
+	br.Success = true
+	br.Msg = "计算成功"
+	br.Data = models.BaseCalculateResp{
+		DataMap:  resultData,
+		DateList: dates,
+	}
+	br.IsAddLog = true
+}

controllers/base_from_mysteel_chemical.go

@@ -87,19 +87,12 @@ func (this *MySteelChemicalController) Refresh() {
 		br.ErrMsg = "请输入指标编码，指标编码为空"
 		return
 	}
-	if req.EdbInfoId <= 0 {
+	if req.EdbInfoId < 0 {
 		br.Msg = "请输入指标ID！"
 		br.ErrMsg = "请输入指标ID"
 		return
 	}
 
-	// 获取指标详情
-	edbInfo, err := models.GetEdbInfoByEdbCode(source, req.EdbCode)
-	if err != nil {
-		br.Msg = "指标不存在！"
-		br.ErrMsg = "指标不存在"
-		return
-	}
 	cacheKey = utils.CACHE_EDB_DATA_REFRESH + strconv.Itoa(source) + "_" + req.EdbCode
 	if utils.Rc.IsExist(cacheKey) {
 		br.Ret = 501
@@ -109,30 +102,55 @@ func (this *MySteelChemicalController) Refresh() {
 	}
 	dataUpdateTime := time.Now().Format(utils.FormatDateTime)
 	utils.Rc.SetNX(cacheKey, 1, 1*time.Minute)
+	// 将数据刷新到BaseFromMysteelChemicalData
+	err = services.RefreshDataFromMysteelChemical(req.EdbCode, utils.GetEdbRefreshStartDate(req.StartDate), utils.BASE_END_DATE)
+	if err != nil {
+		br.Msg = "获取指标信息失败！"
+		br.ErrMsg = "获取指标信息失败 RefreshDataFromMysteelChemical，Err:" + err.Error()
+		return
+	}
 	err = models.RefreshEdbDataFromMysteelChemical(req.EdbInfoId, req.EdbCode, req.StartDate)
 	if err != nil && err.Error() != utils.ErrNoRow() {
 		br.Msg = "刷新指标信息失败！"
 		br.ErrMsg = "刷新指标信息失败 RefreshEdbDataFromMysteel，Err:" + err.Error()
 		return
 	}
+	if req.EdbInfoId != 0 {
+		// 获取指标详情
+		edbInfo, err := models.GetEdbInfoByEdbCode(source, req.EdbCode)
+		if err != nil {
+			br.Msg = "指标不存在！"
+			br.ErrMsg = "指标不存在"
+			return
+		}
 
-	// 更新指标最大最小值
-	erDataUpdateDate, err, errMsg := models.UnifiedModifyEdbInfoMaxAndMinInfoDataUpdate(edbInfo, dataUpdateTime)
-	if err != nil {
-		br.Msg = errMsg
-		br.ErrMsg = err.Error()
-		return
-	}
-	// 添加指标刷新成功日志
-	if erDataUpdateDate != "" {
-		_ = services.AddEdbInfoUpdateLog(edbInfo.EdbInfoId, 1, "", 1, "", 0, 0)
-	} else {
-		_ = services.AddEdbInfoUpdateLog(edbInfo.EdbInfoId, 1, "", 2, "未刷新到数据", 0, 0)
-	}
+		// 更新指标最大最小值
+		erDataUpdateDate, err, errMsg := models.UnifiedModifyEdbInfoMaxAndMinInfoDataUpdate(edbInfo, dataUpdateTime)
+		if err != nil {
+			br.Msg = errMsg
+			br.ErrMsg = err.Error()
+			return
+		}
+		// 添加指标刷新成功日志
+		isSourceRefresh := 0
+		if utils.MysteelChemicalApiToken != "" {
+			isSourceRefresh = 1
+		}
+		if erDataUpdateDate != "" {
+			_ = services.AddEdbInfoUpdateLog(edbInfo.EdbInfoId, 1, "", 1, "", isSourceRefresh, 0)
+		} else {
+			_ = services.AddEdbInfoUpdateLog(edbInfo.EdbInfoId, 1, "", 2, "未刷新到数据", isSourceRefresh, 0)
+		}
 
-	// 更新ES
-	go logic.UpdateEs(edbInfo.EdbInfoId)
+		// 更新ES
+		go logic.UpdateEs(edbInfo.EdbInfoId)
+	}
 
+	if utils.MysteelChemicalApiToken != "" {
+		// 钢联走api接口，那就更新数据源明细表
+		_ = services.SetMysteelChemicalEdbInfoUpdateStat(false)
+		_ = services.SetEdbSourceStat(false)
+	}
 	br.Ret = 200
 	br.Success = true
 	br.Msg = "获取成功"
@@ -196,6 +214,41 @@ func (this *MySteelChemicalController) HandleMysteelIndex() {
 	br.Msg = "处理成功"
 }
 
+// @Title 处理钢联指标的接口
+// @Description 处理钢联指标的接口
+// @Success 200 {object} models.HandleMysteelIndexResp
+// @router /handle/api/mysteel/index [post]
+func (this *MySteelChemicalController) HandleApiMysteelIndex() {
+	br := new(models.BaseResponse).Init()
+	defer func() {
+		this.Data["json"] = br
+		this.ServeJSON()
+	}()
+	body := this.Ctx.Input.RequestBody
+	var req models.HandleMysteelIndexResp
+	err := json.Unmarshal(body, &req)
+	if err != nil {
+		br.Msg = "参数解析异常！"
+		br.ErrMsg = "参数解析失败，Err:" + err.Error()
+		return
+	}
+
+	errMsg, err := services.HandleApiMysteelIndex(&req)
+	if err != nil {
+		fmt.Println("HandleMysteelIndex Err:" + err.Error())
+		if errMsg == "" {
+			br.Msg = "数据获取失败"
+		} else {
+			br.Msg = errMsg
+		}
+		br.ErrMsg = "处理失败,Err:" + err.Error()
+		return
+	}
+	br.Ret = 200
+	br.Success = true
+	br.Msg = "处理成功"
+}
+
 // GetMaxFileIndex
 // @Title 获取最大的文件编号下标
 // @Description 获取最大的文件编号下标

controllers/base_from_predict_calculate.go

@@ -4,6 +4,7 @@ import (
 	"encoding/json"
 	"eta/eta_index_lib/logic"
 	"eta/eta_index_lib/models"
+	"eta/eta_index_lib/services"
 	"eta/eta_index_lib/utils"
 	"fmt"
 	"strconv"
@@ -224,6 +225,7 @@ func addPredictCalculate(br *models.BaseResponse, req models.EdbInfoCalculateSav
 		Extra:            req.Extra,
 		EdbNameEn:        req.EdbName,
 		UnitEn:           req.Unit,
+		Sort:             models.GetAddEdbMaxSortByClassifyId(req.ClassifyId, utils.PREDICT_EDB_INFO_TYPE),
 	}
 	edbInfoId, err := models.AddEdbInfo(edbInfo)
 	if err != nil {
@@ -383,6 +385,11 @@ func editPredictCalculate(br *models.BaseResponse, req models.EdbInfoCalculateSa
 		return
 	}
 
+	// 记录旧的指标信息
+	oldEdbName := edbInfo.EdbName
+	oldFrequency := edbInfo.Frequency
+	oldUnit := edbInfo.Unit
+
 	// 根据指标名称和指标ID校验库中是否还存在其他同名指标
 	existEdbName, err := logic.CheckExistByEdbNameAndEdbInfoId(edbInfo.EdbInfoType, edbInfo.EdbInfoId, req.EdbName, lang)
 	if err != nil {
@@ -506,6 +513,26 @@ func editPredictCalculate(br *models.BaseResponse, req models.EdbInfoCalculateSa
 		br.ErrMsg = err.Error()
 		return
 	}
+
+	// 记录基础信息变更日志
+	oldEdbInfo := new(models.EdbInfo)
+	oldEdbInfo.EdbInfoId = req.EdbInfoId
+	oldEdbInfo.EdbName = oldEdbName
+	oldEdbInfo.Frequency = oldFrequency
+	oldEdbInfo.Unit = oldUnit
+	newEdbInfoEditRecord := new(models.EdbInfoEditRecord)
+	newEdbInfoEditRecord.EdbName = req.EdbName
+	newEdbInfoEditRecord.Frequency = req.Frequency
+	newEdbInfoEditRecord.Unit = req.Unit
+	newEdbInfoEditRecord.OperateUserId = req.AdminId
+	newEdbInfoEditRecord.OperateUserRealName = req.AdminName
+	err = services.AddEditEdbInfoRecord(oldEdbInfo, newEdbInfoEditRecord)
+	if err != nil {
+		br.Msg = "记录基础信息变更日志失败"
+		br.ErrMsg = err.Error()
+		return
+	}
+
 	resp := models.AddEdbInfoResp{
 		EdbInfoId:  edbInfo.EdbInfoId,
 		UniqueCode: edbInfo.UniqueCode,
@@ -687,6 +714,17 @@ func (this *PredictCalculateController) CalculateBatchSave() {
 		return
 	}
 
+	// 记录原始指标信息
+	var oldEdbInfo *models.EdbInfo
+	if req.EdbInfoId > 0 {
+		oldEdbInfo, err = models.GetEdbInfoById(req.EdbInfoId)
+		if err != nil {
+			br.Msg = "获取指标信息失败"
+			br.ErrMsg = "获取指标信息失败:Err:" + err.Error()
+			return
+		}
+	}
+
 	// 来源预测指标信息
 	var fromEdbInfo *models.EdbInfo
 	if fromEdbInfoId > 0 {
@@ -875,6 +913,22 @@ func (this *PredictCalculateController) CalculateBatchSave() {
 		return
 	}
 
+	if req.EdbInfoId > 0 {
+		// 记录操作变更记录
+		newEdbInfo := new(models.EdbInfoEditRecord)
+		newEdbInfo.EdbName = req.EdbName
+		newEdbInfo.Frequency = req.Frequency
+		newEdbInfo.Unit = req.Unit
+		newEdbInfo.OperateUserId = req.AdminId
+		newEdbInfo.OperateUserRealName = req.AdminName
+		err = services.AddEditEdbInfoRecord(oldEdbInfo, newEdbInfo)
+		if err != nil {
+			br.Msg = "保存失败"
+			br.ErrMsg = "保存失败,Err:" + err.Error()
+			return
+		}
+	}
+
 	resp := models.AddEdbInfoResp{
 		EdbInfoId:  edbInfo.EdbInfoId,
 		UniqueCode: uniqueCode,
@@ -1049,7 +1103,7 @@ func (this *PredictCalculateController) Refresh() {
 			break
 		}
 		endDate = time.Now().Format(utils.FormatDate)
-		latestDateStr, latestValue, err = models.RefreshAllPredictCalculateBp(edbInfoId, source, edbInfo.SubSource, fromEdbInfo, calculateTbz.EdbCode, startDate, endDate)
+		latestDateStr, latestValue, err = models.RefreshAllPredictCalculateBp(edbInfoId, source, edbInfo.SubSource, fromEdbInfo, calculateTbz.EdbCode, startDate, endDate, edbInfo.EmptyType)
 		if err != nil && err.Error() != utils.ErrNoRow() {
 			errMsg = "RefreshAllPredictCalculateBp Err:" + err.Error()
 			break

controllers/base_from_python.go

@@ -180,6 +180,7 @@ func (this *PythonController) Add() {
 		ModifyTime:      time.Now(),
 		UniqueCode:      uniqueCode,
 		EdbType:         2,
+		Sort:            models.GetAddEdbMaxSortByClassifyId(req.ClassifyId, utils.EDB_INFO_TYPE),
 	}
 
 	edbInfoId, err := models.AddEdbInfo(edbInfo)

controllers/base_from_ths_ds.go

@@ -6,6 +6,7 @@ import (
 	"eta/eta_index_lib/models"
 	"eta/eta_index_lib/services"
 	"eta/eta_index_lib/utils"
+	"fmt"
 	"strconv"
 	"time"
 )
@@ -46,7 +47,7 @@ func (this *ThsDsController) Add() {
 	cacheKey = utils.CACHE_EDB_DATA_ADD + strconv.Itoa(source) + "_" + req.StockCode + req.EdbCode
 	if !utils.Rc.IsExist(cacheKey) {
 		utils.Rc.SetNX(cacheKey, 1, 1*time.Minute)
-		dataItem, err := services.GetEdbDataFromThsDs(req.StockCode, req.EdbCode, utils.BASE_START_DATE, endDate, "")
+		dataItem, err := services.GetEdbDataFromThsDs(req.StockCode, req.EdbCode, utils.BASE_START_DATE, endDate, "", req.ExtraPars)
 		if err != nil {
 			br.Msg = "获取指标信息失败！"
 			br.ErrMsg = "获取指标信息失败 GetEdbDataFromThsDs，Err:" + err.Error()
@@ -118,7 +119,17 @@ func (this *ThsDsController) Refresh() {
 	defer func() {
 		utils.Rc.Delete(cacheKey)
 	}()
-	dataItem, err := services.GetEdbDataFromThsDs(edbInfo.StockCode, edbInfo.IndicatorCode, utils.GetEdbRefreshStartDate(req.StartDate), endDate, edbInfo.TerminalCode)
+
+	var extra models.EdbInfoExtra
+	if edbInfo.Extra != "" {
+		if e := json.Unmarshal([]byte(edbInfo.Extra), &extra); e != nil {
+			br.Msg = "刷新失败"
+			br.ErrMsg = fmt.Sprintf("API额外参数解析失败, %v", e)
+			return
+		}
+	}
+
+	dataItem, err := services.GetEdbDataFromThsDs(edbInfo.StockCode, edbInfo.IndicatorCode, utils.GetEdbRefreshStartDate(req.StartDate), endDate, edbInfo.TerminalCode, extra.ApiExtraPars)
 	if err != nil {
 		br.Msg = "获取指标信息失败！"
 		br.ErrMsg = "获取指标信息失败 GetEdbDataFromThsDs，Err:" + err.Error()

controllers/base_from_ths_hf.go

@@ -0,0 +1,818 @@
+package controllers
+
+import (
+	"encoding/json"
+	"eta/eta_index_lib/logic"
+	"eta/eta_index_lib/models"
+	"eta/eta_index_lib/models/mgo"
+	"eta/eta_index_lib/services"
+	"eta/eta_index_lib/utils"
+	"fmt"
+	"strconv"
+	"strings"
+	"time"
+)
+
+// ThsHfController 同花顺高频数据
+type ThsHfController struct {
+	BaseAuthController
+}
+
+// GetData
+// @Title 同花顺高频数据-获取数据
+// @Description 同花顺高频数据-获取数据
+// @Success 200 {object} models.ThsHfSearchEdbReq
+// @router /hf/edb_data [post]
+func (this *ThsHfController) GetData() {
+	br := new(models.BaseResponse).Init()
+	defer func() {
+		if br.ErrMsg == "" {
+			br.IsSendEmail = false
+		}
+		this.Data["json"] = br
+		this.ServeJSON()
+	}()
+	var params models.ThsHfSearchEdbReq
+	if e := json.Unmarshal(this.Ctx.Input.RequestBody, &params); e != nil {
+		br.Msg = "参数解析异常"
+		br.ErrMsg = fmt.Sprintf("参数解析失败, %v", e)
+		return
+	}
+	params.StockCode = strings.TrimSpace(params.StockCode)
+	if params.StockCode == "" {
+		br.Msg = "请输入证券代码"
+		return
+	}
+	stockCodes := strings.Split(params.StockCode, ",")
+	if len(stockCodes) == 0 {
+		br.Msg = "请输入证券代码"
+		return
+	}
+	if len(stockCodes) > 10 {
+		br.Msg = "最多输入10个证券代码"
+		return
+	}
+	params.EdbCode = strings.TrimSpace(params.EdbCode)
+	if params.EdbCode == "" {
+		br.Msg = "请输入指标代码"
+		return
+	}
+	edbCodes := strings.Split(params.EdbCode, ",")
+	if len(edbCodes) == 0 {
+		br.Msg = "请输入指标代码"
+		return
+	}
+	if len(edbCodes) > 20 {
+		br.Msg = "最多选择/输入20个指标代码"
+		return
+	}
+	if params.StartTime == "" {
+		br.Msg = "请选择起始时间"
+		return
+	}
+	_, e := time.ParseInLocation(utils.FormatDateTime, params.StartTime, time.Local)
+	if e != nil {
+		br.Msg = "起始时间格式有误"
+		br.ErrMsg = fmt.Sprintf("起始时间格式有误, %v", e)
+		return
+	}
+	// 结束时间选填, 不填则为当前时间
+	if params.EndTime != "" {
+		_, e := time.ParseInLocation(utils.FormatDateTime, params.EndTime, time.Local)
+		if e != nil {
+			br.Msg = "截止时间格式有误"
+			br.ErrMsg = fmt.Sprintf("截止时间格式有误, %v", e)
+			return
+		}
+	}
+	if params.EndTime == "" {
+		params.EndTime = time.Now().Local().Format(utils.FormatDateTime)
+	}
+	if !utils.InArrayByInt(models.ThsHfPeriodArr, params.Interval) {
+		br.Msg = "时间周期有误"
+		br.ErrMsg = fmt.Sprintf("时间周期有误, Interval: %d", params.Interval)
+		return
+	}
+	if params.CPS != "" && !utils.InArrayByStr(models.ThsHfCPSArr, params.CPS) {
+		br.Msg = "复权方式有误"
+		br.ErrMsg = fmt.Sprintf("复权方式有误, CPS: %s", params.CPS)
+		return
+	}
+	if params.BaseDate != "" {
+		_, e = time.ParseInLocation(utils.FormatDate, params.BaseDate, time.Local)
+		if e != nil {
+			br.Msg = "复权基点格式有误"
+			br.ErrMsg = fmt.Sprintf("复权基点格式有误, %v", e)
+			return
+		}
+	}
+	if params.Fill != "" && !utils.InArrayByStr(models.ThsHfFillArr, params.Fill) {
+		br.Msg = "非交易间隔处理有误"
+		br.ErrMsg = fmt.Sprintf("非交易间隔处理有误, Fill: %s", params.Fill)
+		return
+	}
+
+	// 根据配置获取指标数据
+	indexes, e := services.GetEdbDataFromThsHf(params, "")
+	if e != nil {
+		br.Msg = "获取失败"
+		br.ErrMsg = fmt.Sprintf("获取同花顺高频指标失败, %v", e)
+		return
+	}
+
+	br.Data = indexes
+	br.Ret = 200
+	br.Success = true
+	br.Msg = "获取成功"
+}
+
+// BaseAdd
+// @Title 同花顺高频数据-新增至数据源
+// @Description 同花顺高频数据-新增至数据源
+// @Success 200 {object} models.ThsHfBaseAddReq
+// @router /hf/base/add [post]
+func (this *ThsHfController) BaseAdd() {
+	br := new(models.BaseResponse).Init()
+	defer func() {
+		if br.ErrMsg == "" {
+			br.IsSendEmail = false
+		}
+		this.Data["json"] = br
+		this.ServeJSON()
+	}()
+	var params models.ThsHfBaseAddReq
+	if e := json.Unmarshal(this.Ctx.Input.RequestBody, &params); e != nil {
+		br.Msg = "参数解析异常"
+		br.ErrMsg = fmt.Sprintf("参数解析失败, %v", e)
+		return
+	}
+	params.StockCode = strings.TrimSpace(params.StockCode)
+	if params.StockCode == "" {
+		br.Msg = "请输入证券代码"
+		return
+	}
+	params.EdbCode = strings.TrimSpace(params.EdbCode)
+	if params.EdbCode == "" {
+		br.Msg = "请输入指标代码"
+		return
+	}
+	if params.StartTime == "" {
+		br.Msg = "请选择起始时间"
+		return
+	}
+	startTime, e := time.ParseInLocation(utils.FormatDateTime, params.StartTime, time.Local)
+	if e != nil {
+		br.Msg = "起始时间格式有误"
+		br.ErrMsg = fmt.Sprintf("起始时间格式有误, %v", e)
+		return
+	}
+	var endTime time.Time
+	if params.EndTime != "" {
+		ed, e := time.ParseInLocation(utils.FormatDateTime, params.EndTime, time.Local)
+		if e != nil {
+			br.Msg = "截止时间格式有误"
+			br.ErrMsg = fmt.Sprintf("截止时间格式有误, %v", e)
+			return
+		}
+		endTime = ed
+	}
+	if !utils.InArrayByInt(models.ThsHfPeriodArr, params.Interval) {
+		br.Msg = "时间周期有误"
+		br.ErrMsg = fmt.Sprintf("时间周期有误, Interval: %d", params.Interval)
+		return
+	}
+	if params.CPS != "" && !utils.InArrayByStr(models.ThsHfCPSArr, params.CPS) {
+		br.Msg = "复权方式有误"
+		br.ErrMsg = fmt.Sprintf("复权方式有误, CPS: %s", params.CPS)
+		return
+	}
+	if params.BaseDate != "" {
+		_, e = time.ParseInLocation(utils.FormatDate, params.BaseDate, time.Local)
+		if e != nil {
+			br.Msg = "复权基点格式有误"
+			br.ErrMsg = fmt.Sprintf("复权基点格式有误, %v", e)
+			return
+		}
+	}
+	if params.Fill != "" && !utils.InArrayByStr(models.ThsHfFillArr, params.Fill) {
+		br.Msg = "非交易间隔处理有误"
+		br.ErrMsg = fmt.Sprintf("非交易间隔处理有误, Fill: %s", params.Fill)
+		return
+	}
+	if params.ClassifyId <= 0 {
+		br.Msg = "请选择分类"
+		return
+	}
+	params.IndexName = strings.TrimSpace(params.IndexName)
+	if params.IndexName == "" {
+		br.Msg = "请输入指标名称"
+		return
+	}
+	if params.Frequency == "" {
+		br.Msg = "请输入频度"
+		return
+	}
+
+	// 缓存
+	source := utils.DATA_SOURCE_THS
+	subSource := utils.DATA_SUB_SOURCE_HIGH_FREQUENCY
+	cacheKey := fmt.Sprintf("%s_%d_%d_%s_%s", utils.CACHE_BASE_EDB_ADD, source, subSource, params.StockCode, params.EdbCode)
+	defer func() {
+		_ = utils.Rc.Delete(cacheKey)
+	}()
+	if utils.Rc.IsExist(cacheKey) {
+		br.Ret = 501
+		br.Success = true
+		br.Msg = "系统处理中，请稍后重试"
+		return
+	}
+	utils.Rc.SetNX(cacheKey, 1, 3*time.Minute)
+
+	// 已添加则忽略
+	indexOb := new(models.BaseFromThsHfIndex)
+	{
+		cond := fmt.Sprintf(" AND %s = ? AND %s = ?", indexOb.Cols().StockCode, indexOb.Cols().Indicator)
+		pars := make([]interface{}, 0)
+		pars = append(pars, params.StockCode, params.EdbCode)
+		item, e := indexOb.GetItemByCondition(cond, pars, "")
+		if e != nil && e.Error() != utils.ErrNoRow() {
+			br.Msg = "操作失败"
+			br.ErrMsg = fmt.Sprintf("获取原始指标失败, %v", e)
+			return
+		}
+		if item != nil {
+			br.Ret = 200
+			br.Success = true
+			br.Msg = "操作成功"
+			return
+		}
+	}
+
+	// 获取指标数据
+	var apiPars models.ThsHfSearchEdbReq
+	apiPars.StockCode = params.StockCode
+	apiPars.EdbCode = params.EdbCode
+	apiPars.StartTime = params.StartTime
+	apiPars.EndTime = params.EndTime
+	apiPars.Interval = params.Interval
+	apiPars.Fill = params.Fill
+	apiPars.CPS = params.CPS
+	apiPars.BaseDate = params.BaseDate
+	indexes, e := services.GetEdbDataFromThsHf(apiPars, "")
+	if e != nil {
+		br.Msg = "操作失败"
+		br.ErrMsg = fmt.Sprintf("获取同花顺高频指标失败, %v", e)
+		return
+	}
+	if len(indexes) == 0 {
+		br.Msg = "未搜索到指标"
+		br.ErrMsg = "未搜索到指标"
+		return
+	}
+	indexWithData := indexes[0]
+
+	indexItem := new(models.BaseFromThsHfIndex)
+	indexItem.BaseFromThsHfClassifyId = params.ClassifyId
+	indexItem.IndexCode = fmt.Sprintf("%s%s%s%s", utils.ThsHf, params.StockCode, params.EdbCode, params.Frequency)
+	indexItem.IndexName = params.IndexName
+	indexItem.Unit = params.Unit
+	indexItem.Frequency = params.Frequency
+	indexItem.StartDate = startTime
+	indexItem.EndDate = endTime
+	indexItem.SysUserId = params.SysAdminId
+	indexItem.SysUserRealName = params.SysAdminName
+	terminal, e := services.GetFirstTerminal(utils.DATA_SOURCE_THS, "")
+	if e != nil {
+		br.Msg = "终端未配置"
+		br.ErrMsg = fmt.Sprintf("终端未配置, %v", e)
+		return
+	}
+	indexItem.TerminalCode = terminal.TerminalCode
+	indexItem.StockCode = params.StockCode
+	indexItem.Indicator = params.EdbCode
+	b, e := json.Marshal(apiPars)
+	if e != nil {
+		br.Msg = "操作失败"
+		br.ErrMsg = fmt.Sprintf("API入参JSON格式化失败, %v", e)
+		return
+	}
+	indexItem.ApiPars = string(b)
+	if len(indexWithData.IndexData) > 0 {
+		indexItem.StartDate = indexWithData.IndexData[0].DataTime
+		indexItem.EndDate = indexWithData.IndexData[len(indexWithData.IndexData)-1].DataTime
+		lastVal, e := utils.FormatFloatPlaces(indexWithData.IndexData[0].Value, 4)
+		if e != nil {
+			br.Msg = "操作失败"
+			br.ErrMsg = fmt.Sprintf("格式化最新值失败, val: %v, err: %v", indexWithData.IndexData[0].Value, e)
+			return
+		}
+		indexItem.LatestValue = lastVal
+	}
+	indexItem.CreateTime = time.Now().Local()
+	indexItem.ModifyTime = time.Now().Local()
+
+	// 新增指标
+	if e := indexItem.Create(); e != nil {
+		br.Msg = "操作失败"
+		br.ErrMsg = fmt.Sprintf("新增指标失败, %v", e)
+		return
+	}
+
+	// 新增数据
+	if utils.UseMongo {
+		dataList := make([]interface{}, 0)
+		for _, v := range indexWithData.IndexData {
+			newVal, e := utils.FormatFloatPlaces(v.Value, 4)
+			if e != nil {
+				utils.FileLog.Info(fmt.Sprintf("FormatFloatPlaces err: %v", e))
+				continue
+			}
+
+			dataList = append(dataList, &mgo.BaseFromThsHfData{
+				BaseFromThsHfIndexId: int64(indexItem.BaseFromThsHfIndexId),
+				IndexCode:            indexItem.IndexCode,
+				DataTime:             v.DataTime,
+				Value:                newVal,
+				UniqueCode:           utils.MD5(fmt.Sprint(indexItem.IndexCode, v.DataTime.Format(utils.FormatDateTimeMinute))),
+				CreateTime:           time.Now().Local(),
+				ModifyTime:           time.Now().Local(),
+				DataTimestamp:        v.DataTime.UnixNano() / 1e6,
+			})
+		}
+		dataOb := new(mgo.BaseFromThsHfData)
+		if e = dataOb.BatchInsertData(500, dataList); e != nil {
+			br.Msg = "操作失败"
+			br.ErrMsg = fmt.Sprintf("批量新增数据失败-Mongo, %v", e)
+			return
+		}
+	} else {
+		dataOb := new(models.BaseFromThsHfData)
+		itemData := make([]*models.BaseFromThsHfData, 0)
+		for _, v := range indexWithData.IndexData {
+			newVal, e := utils.FormatFloatPlaces(v.Value, 4)
+			if e != nil {
+				utils.FileLog.Info(fmt.Sprintf("FormatFloatPlaces err: %v", e))
+				continue
+			}
+
+			t := new(models.BaseFromThsHfData)
+			t.BaseFromThsHfIndexId = indexItem.BaseFromThsHfIndexId
+			t.IndexCode = indexItem.IndexCode
+			t.DataTime = v.DataTime
+			t.Value = newVal
+			t.UniqueCode = utils.MD5(fmt.Sprint(indexItem.IndexCode, v.DataTime.Format(utils.FormatDateTimeMinute)))
+			t.CreateTime = time.Now().Local()
+			t.ModifyTime = time.Now().Local()
+			t.DataTimestamp = v.DataTime.UnixNano() / 1e6
+			itemData = append(itemData, t)
+		}
+		if e = dataOb.CreateMulti(itemData); e != nil {
+			br.Msg = "操作失败"
+			br.ErrMsg = fmt.Sprintf("批量新增数据失败-MySQL, %v", e)
+			return
+		}
+	}
+
+	br.Ret = 200
+	br.Success = true
+	br.Msg = "操作成功"
+}
+
+// BaseRefresh
+// @Title 同花顺高频数据-数据源刷新
+// @Description 同花顺高频数据-数据源刷新
+// @Success 200 {object} models.ThsHfBaseRefreshReq
+// @router /hf/base/refresh [post]
+func (this *ThsHfController) BaseRefresh() {
+	br := new(models.BaseResponse).Init()
+	defer func() {
+		if br.ErrMsg == "" {
+			br.IsSendEmail = false
+		}
+		this.Data["json"] = br
+		this.ServeJSON()
+	}()
+	var params models.ThsHfBaseRefreshReq
+	if e := json.Unmarshal(this.Ctx.Input.RequestBody, &params); e != nil {
+		br.Msg = "参数解析异常"
+		br.ErrMsg = fmt.Sprintf("参数解析失败, %v", e)
+		return
+	}
+	params.BaseIndexCode = strings.TrimSpace(params.BaseIndexCode)
+	if params.BaseIndexCode == "" {
+		br.Msg = "参数异常"
+		br.ErrMsg = fmt.Sprintf("参数异常, BaseIndexCode: %s", params.BaseIndexCode)
+		return
+	}
+	if params.RefreshType <= 0 {
+		params.RefreshType = 1
+	}
+
+	indexItem := new(models.BaseFromThsHfIndex)
+	{
+		ob := new(models.BaseFromThsHfIndex)
+		cond := fmt.Sprintf(" AND %s = ?", ob.Cols().IndexCode)
+		pars := make([]interface{}, 0)
+		pars = append(pars, params.BaseIndexCode)
+		item, e := ob.GetItemByCondition(cond, pars, "")
+		if e != nil {
+			if e.Error() == utils.ErrNoRow() {
+				br.Msg = "指标不存在"
+				return
+			}
+			br.Msg = "操作失败"
+			br.ErrMsg = fmt.Sprintf("获取源指标失败, %v", e)
+			return
+		}
+		indexItem = item
+	}
+
+	source := utils.DATA_SOURCE_THS
+	subSource := utils.DATA_SUB_SOURCE_HIGH_FREQUENCY
+	cacheKey := fmt.Sprintf("%s_%d_%d_%s_%s", utils.CACHE_BASE_EDB_REFRESH, source, subSource, indexItem.StockCode, indexItem.Indicator)
+	defer func() {
+		_ = utils.Rc.Delete(cacheKey)
+	}()
+	if utils.Rc.IsExist(cacheKey) {
+		br.Ret = 501
+		br.Success = true
+		br.Msg = "系统处理中，请稍后重试"
+		return
+	}
+	utils.Rc.SetNX(cacheKey, 1, 3*time.Minute)
+
+	// API参数
+	var apiPars models.ThsHfSearchEdbReq
+	if e := json.Unmarshal([]byte(indexItem.ApiPars), &apiPars); e != nil {
+		br.Msg = "操作失败"
+		br.ErrMsg = fmt.Sprintf("源指标API参数异常, %v", e)
+		return
+	}
+	// 刷新6小时: 指标开始时间前推6小时; 全部: API参数中的开始时间
+	if params.RefreshType == 1 {
+		apiPars.StartTime = indexItem.StartDate.Add(-6 * time.Hour).Format(utils.FormatDateTime)
+	}
+	// 若API参数中的结束时间不为空, 且不在EndDate之后, 那么不再刷新该指标
+	if apiPars.EndTime != "" {
+		apiEnd, e := time.ParseInLocation(utils.FormatDateTime, apiPars.EndTime, time.Local)
+		if e != nil {
+			br.Msg = "操作失败"
+			br.ErrMsg = fmt.Sprintf("API参数结束时间有误, %v", e)
+			return
+		}
+		if !apiEnd.After(indexItem.EndDate) {
+			br.Ret = 200
+			br.Success = true
+			br.Msg = "该指标无需刷新"
+			return
+		}
+	}
+
+	// API-获取指标数据
+	indexes, e := services.GetEdbDataFromThsHf(apiPars, indexItem.TerminalCode)
+	if e != nil {
+		br.Msg = "操作失败"
+		br.ErrMsg = fmt.Sprintf("获取同花顺高频指标失败, %v", e)
+		return
+	}
+	if len(indexes) == 0 {
+		br.Msg = "未搜索到指标"
+		br.ErrMsg = fmt.Sprintf("未搜索到指标, StockCode: %s, Indicator: %s", indexItem.StockCode, indexItem.Indicator)
+		return
+	}
+	indexWithData := indexes[0]
+
+	// 写入指标数据
+	if utils.UseMongo {
+		if e = services.RefreshThsHfBaseIndexMgo(indexItem, indexWithData, apiPars.StartTime); e != nil {
+			br.Msg = "操作失败"
+			br.ErrMsg = fmt.Sprintf("写入源指标数据失败-Mongo, %v", e)
+			return
+		}
+	} else {
+		if e = services.RefreshThsHfBaseIndex(indexItem, indexWithData, apiPars.StartTime); e != nil {
+			br.Msg = "操作失败"
+			br.ErrMsg = fmt.Sprintf("写入源指标数据失败, %v", e)
+			return
+		}
+	}
+
+	br.Ret = 200
+	br.Success = true
+	br.Msg = "操作成功"
+}
+
+// EdbAdd
+// @Title 同花顺高频数据-新增至指标库
+// @Description 同花顺高频数据-新增至指标库
+// @Success 200 {object} models.ThsHfEdbAddReq
+// @router /hf/edb/add [post]
+func (this *ThsHfController) EdbAdd() {
+	br := new(models.BaseResponse).Init()
+	defer func() {
+		if br.ErrMsg == "" {
+			br.IsSendEmail = false
+		}
+		this.Data["json"] = br
+		this.ServeJSON()
+	}()
+	var params models.ThsHfEdbAddReq
+	if e := json.Unmarshal(this.Ctx.Input.RequestBody, &params); e != nil {
+		br.Msg = "参数解析异常"
+		br.ErrMsg = fmt.Sprintf("参数解析失败, %v", e)
+		return
+	}
+	if params.NewIndex == nil {
+		br.Msg = "参数有误"
+		br.ErrMsg = "参数有误, 指标信息有误"
+		return
+	}
+	params.NewIndex.NewIndexName = strings.TrimSpace(params.NewIndex.NewIndexName)
+	if params.NewIndex.NewIndexName == "" {
+		br.Msg = "请输入指标名称"
+		return
+	}
+	if params.NewIndex.ClassifyId <= 0 {
+		br.Msg = "请选择分类"
+		return
+	}
+	if params.NewIndex.Unit == "" {
+		params.NewIndex.Unit = "无"
+	}
+	if params.NewIndex.NewFrequency == "" {
+		br.Msg = "请输入频度"
+		return
+	}
+
+	// 校验转换规则
+	convertRule := params.ConvertRule
+	if convertRule.ConvertType != 1 && convertRule.ConvertType != 2 {
+		br.Msg = "请选择数据转换方式"
+		return
+	}
+	if convertRule.ConvertType == 1 {
+		if convertRule.ConvertFixed.FixedDay != 1 && convertRule.ConvertFixed.FixedDay != 2 {
+			br.Msg = "请选择指定时间"
+			return
+		}
+		if convertRule.ConvertFixed.FixedTime == "" {
+			br.Msg = "请选择指定时间"
+			return
+		}
+		timePrefix := time.Now().Local().Format(utils.FormatDate)
+		st := fmt.Sprintf("%s %s", timePrefix, convertRule.ConvertFixed.FixedTime)
+		_, e := time.Parse(utils.FormatDateTime, st)
+		if e != nil {
+			br.Msg = "指定时间格式有误"
+			return
+		}
+	}
+	if convertRule.ConvertType == 2 {
+		if convertRule.ConvertArea.StartDay != 1 && convertRule.ConvertArea.StartDay != 2 {
+			br.Msg = "请选择起始时间"
+			return
+		}
+		if convertRule.ConvertArea.StartTime == "" {
+			br.Msg = "请选择起始时间"
+			return
+		}
+		var startTimePre string
+		if convertRule.ConvertArea.StartDay == 1 {
+			startTimePre = time.Now().Local().Format(utils.FormatDate)
+		}
+		if convertRule.ConvertArea.StartDay == 2 {
+			startTimePre = time.Now().Local().AddDate(0, 0, -1).Format(utils.FormatDate)
+		}
+		st := fmt.Sprintf("%s %s", startTimePre, convertRule.ConvertArea.StartTime)
+		startTime, e := time.Parse(utils.FormatDateTime, st)
+		if e != nil {
+			br.Msg = "起始时间格式有误"
+			return
+		}
+
+		if convertRule.ConvertArea.EndDay != 1 && convertRule.ConvertArea.EndDay != 2 {
+			br.Msg = "请选择截止时间"
+			return
+		}
+		if convertRule.ConvertArea.EndTime == "" {
+			br.Msg = "请选择截止时间"
+			return
+		}
+		var endTimePre string
+		if convertRule.ConvertArea.EndDay == 1 {
+			endTimePre = time.Now().Local().Format(utils.FormatDate)
+		}
+		if convertRule.ConvertArea.EndDay == 2 {
+			endTimePre = time.Now().Local().AddDate(0, 0, -1).Format(utils.FormatDate)
+		}
+		ed := fmt.Sprintf("%s %s", endTimePre, convertRule.ConvertArea.EndTime)
+		endTime, e := time.Parse(utils.FormatDateTime, ed)
+		if e != nil {
+			br.Msg = "截止时间格式有误"
+			return
+		}
+		if startTime.After(endTime) {
+			br.Msg = "起始日期不可早于截止日期"
+			return
+		}
+	}
+	convertRuleByte, e := json.Marshal(params.ConvertRule)
+	if e != nil {
+		br.Msg = "操作失败"
+		br.ErrMsg = fmt.Sprintf("转换规则JSON格式化失败, %v", e)
+		return
+	}
+
+	// 缓存
+	source := utils.DATA_SOURCE_THS
+	subSource := utils.DATA_SUB_SOURCE_HIGH_FREQUENCY
+	cacheKey := fmt.Sprintf("%s_%d_%d_%s_%s_%s", utils.CACHE_EDB_DATA_ADD, source, subSource, params.NewIndex.StockCode, params.NewIndex.EdbCode, params.NewIndex.NewFrequency)
+	defer func() {
+		_ = utils.Rc.Delete(cacheKey)
+	}()
+	if utils.Rc.IsExist(cacheKey) {
+		br.Ret = 501
+		br.Success = true
+		br.Msg = "系统处理中，请稍后重试"
+		return
+	}
+	utils.Rc.SetNX(cacheKey, 1, 3*time.Minute)
+
+	// 校验指标/分类
+	baseIndexOb := new(models.BaseFromThsHfIndex)
+	baseIndex, e := baseIndexOb.GetItemById(params.NewIndex.IndexId)
+	if e != nil {
+		br.Msg = "原指标不存在"
+		br.ErrMsg = fmt.Sprintf("原指标不存在, %v", e)
+		return
+	}
+	_, e = models.GetEdbClassifyById(params.NewIndex.ClassifyId)
+	if e != nil {
+		br.Msg = "分类信息有误"
+		br.ErrMsg = fmt.Sprintf("获取分类失败, %v", e)
+		return
+	}
+
+	// 判断指标名称是否已存在
+	{
+		var cond string
+		var pars []interface{}
+		if this.Lang == utils.EnLangVersion {
+			cond += " AND edb_name_en = ? "
+		} else {
+			cond += " AND edb_name = ?"
+		}
+		pars = append(pars, params.NewIndex.NewIndexName)
+		count, e := models.GetEdbInfoCountByCondition(cond, pars)
+		if e != nil {
+			br.Msg = "操作失败"
+			br.ErrMsg = fmt.Sprintf("获取重名指标失败, %v", e)
+			return
+		}
+		if count > 0 {
+			br.Msg = "指标名称已存在"
+			return
+		}
+	}
+
+	// 排序/指标编码
+	sortMax, e := models.GetEdbClassifyMaxSort(params.NewIndex.ClassifyId, 0)
+	if e != nil {
+		br.Msg = "操作失败"
+		br.ErrMsg = fmt.Sprintf("获取最大排序失败, %v", e)
+		return
+	}
+	edbCode, e := utils.GenerateEdbCode(1, "")
+	if e != nil {
+		br.Msg = "操作失败"
+		br.ErrMsg = fmt.Sprintf("生成指标编码失败, %v", e)
+		return
+	}
+	timestamp := strconv.FormatInt(time.Now().UnixNano(), 10)
+	uniqueCode := utils.MD5(utils.DATA_PREFIX + "_" + timestamp)
+
+	thsOb := new(models.EdbThsHf)
+	var addPars models.ThsHfAddBaseParams
+	addPars.EdbCode = edbCode
+	addPars.EdbName = params.NewIndex.NewIndexName
+	addPars.Unit = params.NewIndex.Unit
+	addPars.Frequency = params.NewIndex.NewFrequency
+	addPars.Sort = sortMax + 1
+	addPars.ClassifyId = params.NewIndex.ClassifyId
+	addPars.SysUserId = params.NewIndex.SysAdminId
+	addPars.SysUserRealName = params.NewIndex.SysAdminName
+	addPars.UniqueCode = uniqueCode
+	addPars.ConvertRule = string(convertRuleByte)
+	edbInfo, e := thsOb.Add(addPars, baseIndex)
+	if e != nil {
+		br.Msg = "新增失败"
+		br.ErrMsg = fmt.Sprintf("新增指标失败, %v", e)
+		return
+	}
+
+	// 更新指标最值
+	if e = thsOb.UnifiedModifyEdbInfoMaxAndMinInfo(edbInfo); e != nil {
+		br.Msg = "刷新指标失败"
+		br.ErrMsg = fmt.Sprintf("更新指标最值失败, %v", e)
+		return
+	}
+
+	// 添加到es
+	go logic.UpdateEs(edbInfo.EdbInfoId)
+
+	br.Ret = 200
+	br.Success = true
+	br.Msg = "操作成功"
+}
+
+// EdbRefresh
+// @Title 同花顺高频数据-指标库刷新
+// @Description 同花顺高频数据-指标库刷新
+// @Success 200 {object} models.RefreshEdbInfoReq
+// @router /hf/edb/refresh [post]
+func (this *ThsHfController) EdbRefresh() {
+	br := new(models.BaseResponse).Init()
+	defer func() {
+		if br.ErrMsg == "" {
+			br.IsSendEmail = false
+		}
+		this.Data["json"] = br
+		this.ServeJSON()
+	}()
+	var req models.RefreshEdbInfoReq
+	err := json.Unmarshal(this.Ctx.Input.RequestBody, &req)
+	if err != nil {
+		br.Msg = "参数解析异常！"
+		br.ErrMsg = "参数解析失败，Err:" + err.Error()
+		return
+	}
+	if req.EdbCode == "" {
+		br.Msg = "请输入指标编码！"
+		br.ErrMsg = "请输入指标编码，指标编码为空"
+		return
+	}
+	if req.EdbInfoId < 0 {
+		br.Msg = "请输入指标ID！"
+		br.ErrMsg = "请输入指标ID"
+		return
+	}
+	thsOb := new(models.EdbThsHf)
+	source := thsOb.GetSource()
+	subSource := thsOb.GetSubSource()
+	cacheKey := fmt.Sprintf("%s_%d_%d_%s", utils.CACHE_EDB_DATA_REFRESH, source, subSource, req.EdbCode)
+	if utils.Rc.IsExist(cacheKey) {
+		br.Ret = 501
+		br.Success = true
+		br.Msg = "系统处理中，请稍后重试"
+		return
+	}
+	utils.Rc.SetNX(cacheKey, 1, 1*time.Minute)
+	defer func() {
+		_ = utils.Rc.Delete(cacheKey)
+	}()
+
+	// 获取指标详情
+	edbInfo, e := models.GetEdbInfoByEdbCode(source, req.EdbCode)
+	if e != nil {
+		br.Msg = "指标不存在"
+		br.ErrMsg = fmt.Sprintf("指标不存在, %v", e)
+		return
+	}
+
+	// 获取指标关联信息
+	baseMapping := new(models.BaseFromEdbMapping)
+	{
+		ob := new(models.BaseFromEdbMapping)
+		cond := fmt.Sprintf(" AND %s = ? AND %s = ? AND %s = ?", ob.Cols().EdbCode, ob.Cols().Source, ob.Cols().SubSource)
+		pars := make([]interface{}, 0)
+		pars = append(pars, req.EdbCode, thsOb.GetSource(), thsOb.GetSubSource())
+		mapping, e := ob.GetItemByCondition(cond, pars, "")
+		if e != nil {
+			br.Msg = "刷新失败"
+			br.ErrMsg = fmt.Sprintf("指标关联信息有误, %v", e)
+			return
+		}
+		baseMapping = mapping
+	}
+
+	// 刷新指标
+	if e = thsOb.Refresh(edbInfo, baseMapping, req.StartDate); e != nil {
+		br.Msg = "刷新指标失败"
+		br.ErrMsg = fmt.Sprintf("刷新指标失败, %v", e)
+		return
+	}
+
+	// 更新指标最值
+	if e = thsOb.UnifiedModifyEdbInfoMaxAndMinInfo(edbInfo); e != nil {
+		br.Msg = "刷新指标失败"
+		br.ErrMsg = fmt.Sprintf("更新指标最值失败, %v", e)
+		return
+	}
+
+	// 更新ES
+	go logic.UpdateEs(edbInfo.EdbInfoId)
+
+	br.Ret = 200
+	br.Success = true
+	br.Msg = "操作成功"
+}

controllers/base_from_wind.go

@@ -57,6 +57,7 @@ func (this *WindController) Add() {
 
 		// 试用平台的话，需要额外从弘则这边获取下地址
 		if utils.BusinessCode == utils.HZ_TRIAL_BUSSINESS_CODE {
+			// todo 从体验版中获取对应的终端信息
 			tmpWindUrl, tmpTerminalCode, err, errMsg := services.GetHzWindUrl(req.EdbCode, source)
 			if err != nil {
 				br.Msg = "添加失败！"

controllers/base_from_wind_wsd.go

@@ -53,7 +53,6 @@ func (this *WindWsdController) Add() {
 
 		windUrl := ``
 		terminalCode := ""
-
 		// 试用平台的话，需要额外从弘则这边获取下地址
 		if utils.BusinessCode == utils.HZ_TRIAL_BUSSINESS_CODE {
 			tmpWindUrl, tmpTerminalCode, err, errMsg := services.GetHzWindUrl(req.EdbCode, source)

controllers/commodity_coal.go

@@ -1059,14 +1059,17 @@ func (this *CoalMineDataController) Coastal() {
 	for _, sheet := range req.SheetData {
 		sheetName = sheet.Name
 		//遍历行读取
-		maxCol := sheet.MaxCol
-		for i := 0; i < maxCol; i++ {
+		maxRow := sheet.MaxRow
+		for i := 0; i < maxRow; i++ {
 			if i == 0 {
 				row := sheet.Rows[i]
 				cells := row.Cells
 				for k, cell := range cells {
 					if k > 0 && k < 9 {
 						text := cell.Value
+						if text == "" {
+							continue
+						}
 						groupMap[k] = text
 						var item models.BaseFromCoalmineMapping
 						//合计命名

controllers/factor_edb_series/factor_edb_series.go

@@ -0,0 +1,416 @@
+package factor_edb_series
+
+import (
+	"encoding/json"
+	"eta/eta_index_lib/controllers"
+	"eta/eta_index_lib/logic"
+	"eta/eta_index_lib/models"
+	"eta/eta_index_lib/services"
+	"eta/eta_index_lib/utils"
+	"fmt"
+	"strconv"
+	"sync"
+	"time"
+)
+
+// FactorEdbSeriesController 因子指标系列
+type FactorEdbSeriesController struct {
+	controllers.BaseAuthController
+}
+
+// Recalculate
+// @Title 因子指标系列-重计算
+// @Description 因子指标系列-重计算
+// @Success 200 {object} models.FactorEdbRecalculateReq
+// @router /recalculate [post]
+func (this *FactorEdbSeriesController) Recalculate() {
+	br := new(models.BaseResponse).Init()
+	defer func() {
+		if br.ErrMsg == "" {
+			br.IsSendEmail = false
+		}
+		this.Data["json"] = br
+		this.ServeJSON()
+	}()
+	var req models.FactorEdbRecalculateReq
+	if e := json.Unmarshal(this.Ctx.Input.RequestBody, &req); e != nil {
+		br.Msg = "参数解析异常！"
+		br.ErrMsg = fmt.Sprintf("参数解析失败, Err: %v", e)
+		return
+	}
+	if req.EdbInfoId <= 0 {
+		br.Msg = "参数有误"
+		br.ErrMsg = fmt.Sprintf("指标ID有误, EdbInfoId: %d", req.EdbInfoId)
+		return
+	}
+
+	cacheKey := utils.CACHE_EDB_DATA_REFRESH + "_factor_edb_recalculate_" + req.EdbCode
+	if utils.Rc.IsExist(cacheKey) {
+		br.Ret = 501
+		br.Success = true
+		br.Msg = fmt.Sprintf("系统处理中, 请稍后重试, 指标编码: %s", req.EdbCode)
+		return
+	}
+	utils.Rc.SetNX(cacheKey, 1, 10*time.Minute)
+	defer func() {
+		_ = utils.Rc.Delete(cacheKey)
+	}()
+
+	// 获取指标信息
+	edbInfo, e := models.GetEdbInfoById(req.EdbInfoId)
+	if e != nil {
+		if e.Error() == utils.ErrNoRow() {
+			br.Msg = "指标不存在"
+			br.ErrMsg = fmt.Sprintf("指标不存在, EdbInfoId: %d, EdbCode: %s", req.EdbInfoId, req.EdbCode)
+			return
+		}
+		br.Msg = "操作失败"
+		br.ErrMsg = fmt.Sprintf("获取指标信息失败, Err: %v", e)
+		return
+	}
+
+	// 查询指标关联的系列
+	mappings := make([]*models.FactorEdbSeriesMapping, 0)
+	{
+		ob := new(models.FactorEdbSeriesMapping)
+		cond := fmt.Sprintf(" AND %s = ?", ob.Cols().EdbInfoId)
+		pars := make([]interface{}, 0)
+		pars = append(pars, req.EdbInfoId)
+		fields := []string{ob.Cols().FactorEdbSeriesId, ob.Cols().EdbInfoId, ob.Cols().EdbCode}
+		list, e := ob.GetItemsByCondition(cond, pars, fields, "")
+		if e != nil {
+			br.Msg = "操作失败"
+			br.ErrMsg = fmt.Sprintf("获取指标系列关联失败, Err: %v", e)
+			return
+		}
+		mappings = list
+	}
+	if len(mappings) == 0 {
+		br.Ret = 200
+		br.Success = true
+		br.Msg = "操作成功"
+		return
+	}
+
+	// 获取系列信息
+	seriesIds := make([]int, 0)
+	seriesExist := make(map[int]bool)
+	for _, v := range mappings {
+		if !seriesExist[v.FactorEdbSeriesId] {
+			seriesExist[v.FactorEdbSeriesId] = true
+			seriesIds = append(seriesIds, v.FactorEdbSeriesId)
+		}
+	}
+	if len(seriesIds) == 0 {
+		br.Ret = 200
+		br.Success = true
+		br.Msg = "操作成功"
+		return
+	}
+	seriesMap := make(map[int]*models.FactorEdbSeries, 0)
+	{
+		ob := new(models.FactorEdbSeries)
+		cond := fmt.Sprintf(" AND %s IN (%s)", ob.Cols().PrimaryId, utils.GetOrmInReplace(len(seriesIds)))
+		pars := make([]interface{}, 0)
+		pars = append(pars, seriesIds)
+		fields := []string{ob.Cols().PrimaryId, ob.Cols().EdbInfoType, ob.Cols().CalculateStep, ob.Cols().CalculateState}
+		list, e := ob.GetItemsByCondition(cond, pars, fields, "")
+		if e != nil {
+			br.Msg = "操作失败"
+			br.ErrMsg = fmt.Sprintf("获取指标系列失败, Err: %v", e)
+			return
+		}
+		for _, v := range list {
+			seriesMap[v.FactorEdbSeriesId] = v
+		}
+	}
+
+	// 获取指标原数据
+	edbData, e := models.GetEdbDataAllByEdbCode(edbInfo.EdbCode, edbInfo.Source, edbInfo.SubSource, 0)
+	if e != nil {
+		br.Msg = "操作失败"
+		br.ErrMsg = fmt.Sprintf("获取指标数据失败, EdbCode: %s, Err: %v", edbInfo.EdbCode, e)
+		return
+	}
+	if len(edbData) == 0 {
+		br.Msg = "指标无数据"
+		br.ErrMsg = fmt.Sprintf("指标无数据, EdbCode: %s", edbInfo.EdbCode)
+		return
+	}
+
+	for _, v := range mappings {
+		series := seriesMap[v.FactorEdbSeriesId]
+		if series == nil {
+			continue
+		}
+		// 系列无计算则忽略
+		if series.CalculateState != models.FactorEdbSeriesCalculateNone || series.CalculateStep == "" {
+			continue
+		}
+		var calculates []models.CalculatesReq
+		if e = json.Unmarshal([]byte(series.CalculateStep), &calculates); e != nil {
+			br.Msg = "计算步骤异常"
+			br.ErrMsg = fmt.Sprintf("计算步骤异常, SeriesId: %d, Err: %v", series.FactorEdbSeriesId, e)
+			return
+		}
+
+		// 重新计算相关数据
+		e = services.FactorEdbStepCalculate(v.FactorEdbSeriesId, v.EdbInfoId, v.EdbCode, edbData, calculates)
+		if e != nil {
+			br.Msg = "因子指标计算失败"
+			br.ErrMsg = fmt.Sprintf("因子指标计算失败, SeriesId: %d, EdbCode: %s, Err: %v", v.FactorEdbSeriesId, v.EdbCode, e)
+			return
+		}
+	}
+
+	br.Ret = 200
+	br.Success = true
+	br.Msg = "操作成功"
+}
+
+// ChartRecalculate
+// @Title 图表数据-重计算
+// @Description 图表数据-重计算
+// @Success 200 {object} models.FactorEdbRecalculateReq
+// @router /chart_recalculate [post]
+func (this *FactorEdbSeriesController) ChartRecalculate() {
+	br := new(models.BaseResponse).Init()
+	defer func() {
+		if br.ErrMsg == "" {
+			br.IsSendEmail = false
+		}
+		this.Data["json"] = br
+		this.ServeJSON()
+	}()
+	var req models.FactorEdbChartRecalculateReq
+	if e := json.Unmarshal(this.Ctx.Input.RequestBody, &req); e != nil {
+		br.Msg = "参数解析异常！"
+		br.ErrMsg = fmt.Sprintf("参数解析失败, Err: %v", e)
+		return
+	}
+	if req.ChartInfoId <= 0 {
+		br.Msg = "参数有误"
+		br.ErrMsg = fmt.Sprintf("指标ID有误, ChartInfoId: %d", req.ChartInfoId)
+		return
+	}
+
+	cacheKey := utils.CACHE_CHART_INFO_DATA + "_factor_chart_recalculate_" + strconv.Itoa(req.ChartInfoId)
+	if utils.Rc.IsExist(cacheKey) {
+		br.Ret = 501
+		br.Success = true
+		br.Msg = fmt.Sprintf("系统处理中, 请稍后重试, 图表ID: %s", req.ChartInfoId)
+		return
+	}
+	utils.Rc.SetNX(cacheKey, 1, 10*time.Minute)
+	defer func() {
+		_ = utils.Rc.Delete(cacheKey)
+	}()
+
+	// 查询图表关联的系列指标
+	mappings := make([]*models.FactorEdbSeriesChartMapping, 0)
+	{
+		ob := new(models.FactorEdbSeriesChartMapping)
+		cond := fmt.Sprintf(" AND %s = ?", ob.Cols().ChartInfoId)
+		pars := make([]interface{}, 0)
+		pars = append(pars, req.ChartInfoId)
+		fields := []string{ob.Cols().PrimaryId, ob.Cols().CalculateType, ob.Cols().CalculatePars, ob.Cols().FactorEdbSeriesId, ob.Cols().EdbInfoId}
+		list, e := ob.GetItemsByCondition(cond, pars, fields, "")
+		if e != nil {
+			br.Msg = "操作失败"
+			br.ErrMsg = fmt.Sprintf("获取指标系列关联失败, Err: %v", e)
+			return
+		}
+		mappings = list
+	}
+	if len(mappings) == 0 {
+		br.Ret = 200
+		br.Success = true
+		br.Msg = "操作成功"
+		return
+	}
+	seriesIds := make([]int, 0)
+	edbInfoIds := make([]int, 0)
+	for _, v := range mappings {
+		seriesIds = append(seriesIds, v.FactorEdbSeriesId)
+		edbInfoIds = append(edbInfoIds, v.EdbInfoId)
+	}
+
+	// 获取因子指标及系列信息
+	seriesIdItem := make(map[int]*models.FactorEdbSeries)
+	{
+		ob := new(models.FactorEdbSeries)
+		cond := fmt.Sprintf(" AND %s IN (%s)", ob.Cols().PrimaryId, utils.GetOrmInReplace(len(seriesIds)))
+		pars := make([]interface{}, 0)
+		pars = append(pars, seriesIds)
+		items, e := ob.GetItemsByCondition(cond, pars, []string{}, fmt.Sprintf("%s ASC", ob.Cols().PrimaryId))
+		if e != nil {
+			br.Msg = "获取失败"
+			br.ErrMsg = "获取因子指标系列失败, Err: " + e.Error()
+			return
+		}
+		for _, v := range items {
+			seriesIdItem[v.FactorEdbSeriesId] = v
+		}
+	}
+	edbIdItem := make(map[int]*models.EdbInfo)
+	edbItems, e := models.GetEdbInfoByIdList(edbInfoIds)
+	if e != nil {
+		br.Msg = "获取失败"
+		br.ErrMsg = "获取因子指标失败, Err: " + e.Error()
+		return
+	}
+	for _, v := range edbItems {
+		edbIdItem[v.EdbInfoId] = v
+	}
+
+	// 重新计算
+	calculateWorkers := make(chan struct{}, 10)
+	wg := sync.WaitGroup{}
+	calculateDataOb := new(models.FactorEdbSeriesCalculateData)
+	for _, v := range mappings {
+		edbItem := edbIdItem[v.EdbInfoId]
+		if edbItem == nil {
+			continue
+		}
+		seriesItem := seriesIdItem[v.FactorEdbSeriesId]
+		if seriesItem == nil {
+			continue
+		}
+
+		wg.Add(1)
+		go func(chartMapping *models.FactorEdbSeriesChartMapping, edb *models.EdbInfo, series *models.FactorEdbSeries) {
+			defer func() {
+				wg.Done()
+				<-calculateWorkers
+			}()
+			calculateWorkers <- struct{}{}
+
+			// 相关性计算
+			if chartMapping.CalculateType == models.FactorEdbSeriesChartCalculateTypeCorrelation {
+				// 解析计算参数
+				if chartMapping.CalculatePars == "" {
+					utils.FileLog.Info(fmt.Sprintf("相关性-计算参数为空, MappingId: %d", chartMapping.FactorEdbSeriesChartMappingId))
+					return
+				}
+				var calculatePars models.FactorEdbSeriesChartCalculateCorrelationReq
+				if e := json.Unmarshal([]byte(chartMapping.CalculatePars), &calculatePars); e != nil {
+					utils.FileLog.Info(fmt.Sprintf("相关性-计算参数解析失败, MappingId: %d, err: %v", chartMapping.FactorEdbSeriesChartMappingId, e))
+					return
+				}
+
+				// 获取标的指标信息及数据
+				baseEdb, e := models.GetEdbInfoById(calculatePars.BaseEdbInfoId)
+				if e != nil {
+					utils.FileLog.Info(fmt.Sprintf("相关性-获取标的指标失败, MappingId: %d, err: %v", chartMapping.FactorEdbSeriesChartMappingId, e))
+					return
+				}
+				calculateUnitDays, ok := utils.FrequencyDaysMap[calculatePars.CalculateUnit]
+				if !ok {
+					utils.FileLog.Info(fmt.Sprintf("相关性-错误的计算窗口频度, MappingId: %d", chartMapping.FactorEdbSeriesChartMappingId))
+					return
+				}
+				calculateDays := calculatePars.CalculateValue * calculateUnitDays
+
+				dataListA := make([]*models.EdbInfoSearchData, 0)
+				{
+					// 标的指标数据日期区间
+					startDate := time.Now().AddDate(0, 0, -calculateDays).Format(utils.FormatDate)
+					endDate := time.Now().Format(utils.FormatDate)
+					startDateTime, _ := time.ParseInLocation(utils.FormatDate, startDate, time.Local)
+					startDate = startDateTime.AddDate(0, 0, 1).Format(utils.FormatDate) // 不包含第一天
+					switch baseEdb.EdbInfoType {
+					case 0:
+						list, e := models.GetEdbDataList(baseEdb.Source, baseEdb.SubSource, baseEdb.EdbInfoId, startDate, endDate)
+						if e != nil {
+							utils.FileLog.Info(fmt.Sprintf("相关性-获取标的指标数据失败, EdbInfoId: %d, err: %v", baseEdb.EdbInfoId, e))
+							return
+						}
+						dataListA = models.TransEdbInfoDataList2SearchData(list)
+					case 1:
+						dataListA, _, e, _ = models.GetPredictDataListByPredictEdbInfo(baseEdb, 1, startDate)
+						if e != nil {
+							utils.FileLog.Info(fmt.Sprintf("相关性-获取标的指标数据失败, EdbInfoId: %d, err: %v", baseEdb.EdbInfoId, e))
+							return
+						}
+					default:
+						utils.FileLog.Info(fmt.Sprintf("相关性-标的指标类型异常, EdbInfoId: %d", baseEdb.EdbInfoId))
+						return
+					}
+				}
+
+				// 获取指标数据
+				dataListB := make([]*models.EdbInfoSearchData, 0)
+				if series.CalculateState == models.FactorEdbSeriesCalculated {
+					cond := fmt.Sprintf(" AND %s = ? AND %s = ?", calculateDataOb.Cols().FactorEdbSeriesId, calculateDataOb.Cols().EdbInfoId)
+					pars := make([]interface{}, 0)
+					pars = append(pars, chartMapping.FactorEdbSeriesId, chartMapping.EdbInfoId)
+					dataItems, e := calculateDataOb.GetItemsByCondition(cond, pars, []string{calculateDataOb.Cols().DataTime, calculateDataOb.Cols().Value}, fmt.Sprintf("%s ASC", calculateDataOb.Cols().DataTime))
+					if e != nil {
+						utils.FileLog.Info(fmt.Sprintf("相关性-获取因子指标数据失败, MappingId: %d, err: %v", chartMapping.FactorEdbSeriesChartMappingId, e))
+						return
+					}
+					dataListB = models.TransEdbSeriesCalculateData2EdbDataList(dataItems)
+				} else {
+					switch edb.EdbInfoType {
+					case 0:
+						list, e := models.GetEdbDataList(edb.Source, edb.SubSource, edb.EdbInfoId, "", "")
+						if e != nil {
+							utils.FileLog.Info(fmt.Sprintf("相关性-获取因子指标数据失败, EdbInfoId: %d, err: %v", edb.EdbInfoId, e))
+							return
+						}
+						dataListB = models.TransEdbInfoDataList2SearchData(list)
+					case 1:
+						dataListB, e = models.GetPredictEdbDataListAll(baseEdb, 1)
+						if e != nil {
+							utils.FileLog.Info(fmt.Sprintf("相关性-获取因子指标数据失败, EdbInfoId: %d, err: %v", edb.EdbInfoId, e))
+							return
+						}
+					default:
+						utils.FileLog.Info(fmt.Sprintf("相关性-因子指标类型异常, EdbInfoId: %d", edb.EdbInfoId))
+						return
+					}
+				}
+
+				// 计算相关性
+				xEdbIdValue, yDataList, e := logic.CalculateCorrelation(calculatePars.LeadValue, calculatePars.LeadUnit, baseEdb.Frequency, edb.Frequency, dataListA, dataListB)
+				if e != nil {
+					utils.FileLog.Info(fmt.Sprintf("相关性-计算失败, EdbInfoId: %d", edb.EdbInfoId))
+					return
+				}
+
+				// X及Y轴数据
+				yData := yDataList[0].Value
+				yLen := len(yData)
+				values := make([]models.FactorEdbSeriesCorrelationMatrixValues, len(xEdbIdValue))
+				for k, x := range xEdbIdValue {
+					var y float64
+					if k >= 0 && k < yLen {
+						y = yData[k]
+					}
+					y = utils.SubFloatToFloat(y, 2)
+					values[k] = models.FactorEdbSeriesCorrelationMatrixValues{
+						XData: x, YData: y,
+					}
+				}
+
+				// 更新计算结果
+				b, e := json.Marshal(values)
+				if e != nil {
+					utils.FileLog.Info(fmt.Sprintf("相关性-计算结果JSON格式化失败, err: %v", e))
+					return
+				}
+				chartMapping.CalculateData = string(b)
+				chartMapping.ModifyTime = time.Now().Local()
+				if e = chartMapping.Update([]string{chartMapping.Cols().CalculateData, chartMapping.Cols().ModifyTime}); e != nil {
+					utils.FileLog.Info(fmt.Sprintf("相关性-更新矩阵数据失败, err: %v", e))
+					return
+				}
+			}
+		}(v, edbItem, seriesItem)
+	}
+	wg.Wait()
+
+	br.Ret = 200
+	br.Success = true
+	br.Msg = "操作成功"
+}

go.mod

@@ -1,35 +1,36 @@
 module eta/eta_index_lib
 
-go 1.19
+go 1.21.7
 
 require (
-	github.com/beego/bee/v2 v2.0.2
-	github.com/beego/beego/v2 v2.0.2
+	github.com/beego/bee/v2 v2.1.0
+	github.com/beego/beego/v2 v2.2.2
+	github.com/dengsgo/math-engine v0.0.0-20230823154425-78f211b48149
 	github.com/dgrijalva/jwt-go v3.2.0+incompatible
 	github.com/go-redis/redis/v8 v8.11.5
-	github.com/go-sql-driver/mysql v1.6.0
+	github.com/go-sql-driver/mysql v1.8.1
 	github.com/gonum/stat v0.0.0-20181125101827-41a0da705a5b
 	github.com/mozillazg/go-pinyin v0.20.0
 	github.com/nosixtools/solarlunar v0.0.0-20211112060703-1b6dea7b4a19
 	github.com/olivere/elastic/v7 v7.0.32
 	github.com/qiniu/qmgo v1.1.8
 	github.com/rdlucklib/rdluck_tools v1.0.3
-	github.com/shopspring/decimal v1.3.1
+	github.com/shopspring/decimal v1.4.0
 	github.com/yidane/formula v0.0.0-20210902154546-0782e1736717
-	go.mongodb.org/mongo-driver v1.15.0
+	go.mongodb.org/mongo-driver v1.16.0
 	gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df
 )
 
 require (
+	filippo.io/edwards25519 v1.1.0 // indirect
 	github.com/antlr/antlr4/runtime/Go/antlr v0.0.0-20211223021540-a6a1e1173105 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
-	github.com/cespare/xxhash/v2 v2.1.2 // indirect
+	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
 	github.com/go-playground/locales v0.13.0 // indirect
 	github.com/go-playground/universal-translator v0.17.0 // indirect
 	github.com/go-playground/validator/v10 v10.4.1 // indirect
-	github.com/golang/protobuf v1.5.2 // indirect
-	github.com/golang/snappy v0.0.1 // indirect
+	github.com/golang/snappy v0.0.4 // indirect
 	github.com/gonum/blas v0.0.0-20181208220705-f22b278b28ac // indirect
 	github.com/gonum/floats v0.0.0-20181209220543-c233463c7e82 // indirect
 	github.com/gonum/integrate v0.0.0-20181209220457-a422b5c0fdf2 // indirect
@@ -41,26 +42,25 @@ require (
 	github.com/klauspost/compress v1.13.6 // indirect
 	github.com/leodido/go-urn v1.2.0 // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect
-	github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
-	github.com/mitchellh/mapstructure v1.4.1 // indirect
-	github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe // indirect
+	github.com/mitchellh/mapstructure v1.5.0 // indirect
+	github.com/montanaflynn/stats v0.7.1 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
-	github.com/prometheus/client_golang v1.11.0 // indirect
-	github.com/prometheus/client_model v0.2.0 // indirect
-	github.com/prometheus/common v0.26.0 // indirect
-	github.com/prometheus/procfs v0.6.0 // indirect
+	github.com/prometheus/client_golang v1.19.0 // indirect
+	github.com/prometheus/client_model v0.5.0 // indirect
+	github.com/prometheus/common v0.48.0 // indirect
+	github.com/prometheus/procfs v0.12.0 // indirect
 	github.com/shiena/ansicolor v0.0.0-20200904210342-c7312218db18 // indirect
+	github.com/valyala/bytebufferpool v1.0.0 // indirect
 	github.com/xdg-go/pbkdf2 v1.0.0 // indirect
 	github.com/xdg-go/scram v1.1.2 // indirect
 	github.com/xdg-go/stringprep v1.0.4 // indirect
 	github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect
-	golang.org/x/crypto v0.17.0 // indirect
-	golang.org/x/net v0.10.0 // indirect
-	golang.org/x/sync v0.1.0 // indirect
-	golang.org/x/sys v0.15.0 // indirect
-	golang.org/x/text v0.14.0 // indirect
-	google.golang.org/protobuf v1.26.0 // indirect
+	golang.org/x/crypto v0.23.0 // indirect
+	golang.org/x/net v0.23.0 // indirect
+	golang.org/x/sync v0.7.0 // indirect
+	golang.org/x/sys v0.20.0 // indirect
+	golang.org/x/text v0.15.0 // indirect
+	google.golang.org/protobuf v1.34.1 // indirect
 	gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc // indirect
-	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

go.sum

@@ -1,147 +1,61 @@
-cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
-cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
-cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
-cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU=
-cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY=
-cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc=
-cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0=
-cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
-cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
-cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk=
-cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=
-cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw=
-dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
+filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA=
+filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
-github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
 github.com/Knetic/govaluate v3.0.0+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0=
-github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0=
-github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
-github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo=
-github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI=
-github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/4+TcAqDqk/vUH7g=
-github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5/go.mod h1:SkGFH1ia65gfNATL8TAiHDNxPzPdmEL5uirI2Uyuz6c=
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
 github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
-github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho=
 github.com/alicebob/gopher-json v0.0.0-20180125190556-5a6b3ba71ee6/go.mod h1:SGnFV6hVsYE877CKEZ6tDNTjaSXYUk6QqoIK6PrAtcc=
 github.com/alicebob/miniredis v2.5.0+incompatible/go.mod h1:8HZjEj4yU0dwhYHky+DxYx+6BMjkBbe5ONFIF1MXffk=
-github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
 github.com/antlr/antlr4/runtime/Go/antlr v0.0.0-20211223021540-a6a1e1173105 h1:80Iu0Rujat7jabX6Egx/dU/ijgsMp4WtDr9LAv3dMmo=
 github.com/antlr/antlr4/runtime/Go/antlr v0.0.0-20211223021540-a6a1e1173105/go.mod h1:F7bn7fEU90QkQ3tnmaTx3LTKLEDqnwWODIYppRQ5hnY=
-github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ=
-github.com/apache/thrift v0.13.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ=
-github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=
-github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
-github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
-github.com/aryann/difflib v0.0.0-20170710044230-e206f873d14a/go.mod h1:DAHtR1m6lCRdSC2Tm3DSWRPvIPr6xNKyeHdqDQSQT+A=
 github.com/astaxie/beego v1.12.3/go.mod h1:p3qIm0Ryx7zeBHLljmd7omloyca1s4yu1a8kM1FkpIA=
-github.com/aws/aws-lambda-go v1.13.3/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQwij/eHl5CU=
-github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
-github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g=
-github.com/beego/bee/v2 v2.0.2 h1:xWARyIqdnnbNMDBDUdb6Gvr9S/yGXC6Ni43kKdS1/eg=
-github.com/beego/bee/v2 v2.0.2/go.mod h1:rfZa899qLAF8SYBRvE7mWNPZTU7/qysOBhaCLmZrMX4=
-github.com/beego/beego/v2 v2.0.1/go.mod h1:8zyHi1FnWO1mZLwTn62aKRIZF/aIKvkCBB2JYs+eqQI=
-github.com/beego/beego/v2 v2.0.2 h1:Mx2MWMHJN1oFBHewHWyIhR25tXB9IPceIK8X7OuMdZM=
-github.com/beego/beego/v2 v2.0.2/go.mod h1:4pxstbxq+2qE8IUzFsVK8X9BsqfRjbp7ohbapTrTLho=
+github.com/beego/bee/v2 v2.1.0 h1:4WngbAnkvVOyKy74WXcRH3clon76wkjhuzrV2mx2fQU=
+github.com/beego/bee/v2 v2.1.0/go.mod h1:wDhKy5TNxv46LHKsK2gyxo38ObCOm9PbCN89lWHK3EU=
+github.com/beego/beego/v2 v2.2.2 h1:h6TNybAiMPXx9RXxK71Wz+JkPE7rpsL+ctjSZpv5yB0=
+github.com/beego/beego/v2 v2.2.2/go.mod h1:A3BC73uulBnqW3O1uBEN7q+oykprxipZTYRdZtEuKyY=
 github.com/beego/goyaml2 v0.0.0-20130207012346-5545475820dd/go.mod h1:1b+Y/CofkYwXMUU0OhQqGvsY2Bvgr4j6jfT699wyZKQ=
 github.com/beego/x2j v0.0.0-20131220205130-a0352aadc542/go.mod h1:kSeGC/p1AbBiEp5kat81+DSQrZenVBZXklMLaELspWU=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
-github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
-github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84=
 github.com/bradfitz/gomemcache v0.0.0-20180710155616-bc664df96737/go.mod h1:PmM6Mmwb0LSuEubjR8N7PtNe1KxZLtOUHtbeikc5h60=
-github.com/bradfitz/gomemcache v0.0.0-20190913173617-a41fca850d0b/go.mod h1:H0wQNHz2YrLsuXOZozoeDmnHXkNCRmMW0gwFWDfEZDA=
 github.com/casbin/casbin v1.7.0/go.mod h1:c67qKN6Oum3UF5Q1+BByfFxkwKvhwW57ITjqwtzR1KE=
-github.com/casbin/casbin v1.9.1/go.mod h1:z8uPsfBJGUsnkagrt3G8QvjgTKFMBJ32UP8HpZllfog=
-github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ=
-github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM=
-github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
-github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
 github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
-github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE=
-github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
-github.com/clbanning/x2j v0.0.0-20191024224557-825249438eec/go.mod h1:jMjuTZXRI4dUb/I5gc9Hdhagfvm9+RyrPryS/auMzxE=
-github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
+github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
+github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58/go.mod h1:EOBUe0h4xcZ5GoxqC5SDxFQ8gwyZPKQoEzownBlhI80=
-github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
-github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
-github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8=
-github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI=
-github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
-github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
-github.com/coreos/etcd v3.3.25+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
-github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
-github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
-github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
-github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
-github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
-github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
-github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
-github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
-github.com/cosiner/argv v0.1.0/go.mod h1:EusR6TucWKX+zFgtdUsKT2Cvg45K5rtpCcWz4hK06d8=
 github.com/couchbase/go-couchbase v0.0.0-20200519150804-63f3cdb75e0d/go.mod h1:TWI8EKQMs5u5jLKW/tsb9VwauIrMIxQG1r5fMsswK5U=
-github.com/couchbase/go-couchbase v0.1.0/go.mod h1:+/bddYDxXsf9qt0xpDUtRR47A2GjaXmGGAqQ/k3GJ8A=
 github.com/couchbase/gomemcached v0.0.0-20200526233749-ec430f949808/go.mod h1:srVSlQLB8iXBVXHgnqemxUXqN6FCvClgCMPCsjBDR7c=
-github.com/couchbase/gomemcached v0.1.3/go.mod h1:mxliKQxOv84gQ0bJWbI+w9Wxdpt9HjDvgW9MjCym5Vo=
 github.com/couchbase/goutils v0.0.0-20180530154633-e865a1461c8a/go.mod h1:BQwMFlJzDjFDG3DJUdU0KORxn88UlsOULuxLExMh3Hs=
-github.com/couchbase/goutils v0.1.0/go.mod h1:BQwMFlJzDjFDG3DJUdU0KORxn88UlsOULuxLExMh3Hs=
-github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE=
-github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
-github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY=
-github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/cupcake/rdb v0.0.0-20161107195141-43ba34106c76/go.mod h1:vYwsqCOLxGiisLwp9rITslkFNpZD5rz43tf41QFkTWY=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dengsgo/math-engine v0.0.0-20230823154425-78f211b48149 h1:TkVfb0s14IUHDGvjQfq3f0PZnV1zh609did4DrnD4q4=
+github.com/dengsgo/math-engine v0.0.0-20230823154425-78f211b48149/go.mod h1:zkR27k4K0I8FS6rkEd8qBhPeS8i3X2FKfvSPdF64OpQ=
 github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM=
 github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
 github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
 github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
-github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
-github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
-github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
-github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs=
-github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU=
-github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I=
 github.com/edsrzf/mmap-go v0.0.0-20170320065105-0bce6a688712/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M=
-github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M=
 github.com/elastic/go-elasticsearch/v6 v6.8.5/go.mod h1:UwaDJsD3rWLM5rKNFzv9hgox93HoX8utj1kxD9aFUcI=
-github.com/elastic/go-elasticsearch/v6 v6.8.10/go.mod h1:UwaDJsD3rWLM5rKNFzv9hgox93HoX8utj1kxD9aFUcI=
 github.com/elazarl/go-bindata-assetfs v1.0.0/go.mod h1:v+YaWX3bdea5J/mo8dSETolEo7R71Vk1u8bnjau5yw4=
 github.com/elazarl/go-bindata-assetfs v1.0.1 h1:m0kkaHRKEu7tUIUFVwhGGGYClXvyl4RE03qmvRTNfbw=
 github.com/elazarl/go-bindata-assetfs v1.0.1/go.mod h1:v+YaWX3bdea5J/mo8dSETolEo7R71Vk1u8bnjau5yw4=
-github.com/envoyproxy/go-control-plane v0.6.9/go.mod h1:SBwIajubJHhxtWwsL9s8ss4safvEdbitLhGGK48rN6g=
-github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
-github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
-github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
-github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
-github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
-github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
-github.com/flosch/pongo2 v0.0.0-20190707114632-bbf5a6c351f4/go.mod h1:T9YF2M40nIgbVgp3rreNmTged+9HrbNTIQf1PsaIiTA=
-github.com/flosch/pongo2 v0.0.0-20200529170236-5abacdfa4915/go.mod h1:fB4mx6dzqFinCxIf3a7Mf5yLk+18Bia9mPAnuejcvDA=
 github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw=
-github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4=
-github.com/franela/goreq v0.0.0-20171204163338-bcd34c9993f8/go.mod h1:ZhphrRTfi2rbfLwlschooIH4+wKKDR4Pdxhh+TRoA20=
+github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
 github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
 github.com/garyburd/redigo v1.6.3/go.mod h1:rTb6epsqigu3kYKBnaF028A7Tf/Aw5s0cqA47doKKqw=
-github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
 github.com/glendc/gopher-json v0.0.0-20170414221815-dc4743023d0c/go.mod h1:Gja1A+xZ9BoviGJNA2E9vFkPjjsl+CoJxSXiQM1UXtw=
-github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclKsC9YodN5RgxqK/VD9HM9JsCSh7rNhMZE98=
-github.com/go-delve/delve v1.5.0/go.mod h1:c6b3a1Gry6x8a4LGCe/CWzrocrfaHvkUxCj3k4bvSUQ=
-github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
-github.com/go-kit/kit v0.10.0/go.mod h1:xUsJbQ/Fp4kEt7AFgCuvyX4a71u8h9jB8tj/ORgOZ7o=
-github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY=
 github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
-github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A=
 github.com/go-playground/assert/v2 v2.0.1 h1:MsBgLAaY856+nPRTKrp3/OZK38U/wa0CcBYNjji3q3A=
 github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
 github.com/go-playground/locales v0.13.0 h1:HyWk6mgj5qFqCT5fjGBuRArbVDfE4hi8+e8ceBS/t7Q=
@@ -151,47 +65,28 @@ github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+
 github.com/go-playground/validator/v10 v10.4.1 h1:pH2c5ADXtd66mxoE0Zm9SUhxE20r7aM3F26W0hOn+GE=
 github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn9GlaMV7XkbRSipzJ0Ii4=
 github.com/go-redis/redis v6.14.2+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA=
-github.com/go-redis/redis/v7 v7.4.0/go.mod h1:JDNMw23GTyLNC4GZu9njt15ctBQVn7xjRfnwdHj/Dcg=
 github.com/go-redis/redis/v8 v8.11.5 h1:AcZZR7igkdvfVmQTPnu9WE37LRrO/YrBH5zWyjDC0oI=
 github.com/go-redis/redis/v8 v8.11.5/go.mod h1:gREzHqY1hg6oD9ngVRbLStwAWKhA0FEgq8Jd4h5lpwo=
-github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
 github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
-github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE=
 github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
+github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpvNJ1Y=
+github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
-github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
-github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
-github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
-github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
-github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
-github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
-github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
-github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
-github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
-github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
-github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
-github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
-github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
-github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
 github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
 github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
 github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
 github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
 github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
-github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
 github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
-github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
-github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
-github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw=
-github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
 github.com/golang/snappy v0.0.0-20170215233205-553a64147049/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
-github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
 github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM=
+github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/gomodule/redigo v2.0.0+incompatible/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4=
 github.com/gonum/blas v0.0.0-20181208220705-f22b278b28ac h1:Q0Jsdxl5jbxouNs1TQYt0gxesYMU4VXRbsTlgDloZ50=
 github.com/gonum/blas v0.0.0-20181208220705-f22b278b28ac/go.mod h1:P32wAyui1PQ58Oce/KYkOqQv8cVw1zAapXOl+dRFGbc=
@@ -207,98 +102,28 @@ github.com/gonum/matrix v0.0.0-20181209220409-c518dec07be9 h1:V2IgdyerlBa/MxaEFR
 github.com/gonum/matrix v0.0.0-20181209220409-c518dec07be9/go.mod h1:0EXg4mc1CNP0HCqCz+K4ts155PXIlUywf0wqN+GfPZw=
 github.com/gonum/stat v0.0.0-20181125101827-41a0da705a5b h1:fbskpz/cPqWH8VqkQ7LJghFkl2KPAiIFUHrTJ2O3RGk=
 github.com/gonum/stat v0.0.0-20181125101827-41a0da705a5b/go.mod h1:Z4GIJBJO3Wa4gD4vbwQxXXZ+WHmW6E9ixmNrwvs0iZs=
-github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
-github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
-github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.7 h1:81/ik6ipDQS2aGcBfIN5dHDB36BwrStyeAQquSYCV4o=
-github.com/google/go-dap v0.2.0/go.mod h1:5q8aYQFnHOAZEMP+6vmq25HKYAEwE+LF5yh7JKrrhSQ=
+github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
+github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
-github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
-github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
-github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
-github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
-github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
-github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
-github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
-github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg=
-github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
-github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
-github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
-github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
-github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
-github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
-github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
-github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
-github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
-github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=
-github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q=
-github.com/hashicorp/consul/api v1.3.0/go.mod h1:MmDNSzIMUjNpY/mQ398R4bk2FnqQLoPndWW5VkKPlCE=
-github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8=
-github.com/hashicorp/consul/sdk v0.3.0/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8=
-github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
-github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
-github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60=
-github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM=
-github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk=
-github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU=
-github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU=
-github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4=
-github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
-github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
-github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
-github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90=
-github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
-github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc=
 github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
-github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
-github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64=
-github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ=
-github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I=
-github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
-github.com/hudl/fargo v1.3.0/go.mod h1:y3CKSmjA+wD2gak7sUSXTAoopbhU08POFhmITJgmKTg=
-github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
-github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo=
-github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
-github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
 github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
 github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
-github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=
 github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
-github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
-github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
-github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
-github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
-github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
-github.com/juju/errors v0.0.0-20181118221551-089d3ea4e4d5/go.mod h1:W54LbzXuIE0boCoNJfwqpmkKJ1O4TCTZMetAt6jGk7Q=
-github.com/juju/errors v0.0.0-20190930114154-d42613fe1ab9/go.mod h1:W54LbzXuIE0boCoNJfwqpmkKJ1O4TCTZMetAt6jGk7Q=
-github.com/juju/loggo v0.0.0-20180524022052-584905176618/go.mod h1:vgyd7OREkbtVEN/8IXZe5Ooef3LQePvuBm9UWj6ZL8U=
-github.com/juju/testing v0.0.0-20180920084828-472a3e8b2073/go.mod h1:63prj8cnj0tU0S9OHjGJn+b1h0ZghCndfnbQolrYTwA=
 github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
-github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
-github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
-github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
-github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
-github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/klauspost/compress v1.13.6 h1:P76CopJELS0TiO2mebmnzgWaajssP/EszplttgQxcgc=
 github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
-github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
+github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
@@ -306,201 +131,100 @@ github.com/ledisdb/ledisdb v0.0.0-20200510135210-d35789ec47e6/go.mod h1:n931TsDu
 github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y=
 github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII=
 github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
-github.com/lib/pq v1.7.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
-github.com/lib/pq v1.10.2/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
-github.com/lib/pq v1.10.4 h1:SO9z7FRPzA03QhHKJrH5BXA6HU1rS4V2nIVrrNC1iYk=
 github.com/lib/pq v1.10.4/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
-github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM=
-github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4=
-github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ=
-github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
+github.com/lib/pq v1.10.5 h1:J+gdV2cUmX7ZqL2B0lFcW0m+egaHC2V3lpO8nWxyYiQ=
+github.com/lib/pq v1.10.5/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
 github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
 github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
-github.com/mattn/go-colorable v0.0.0-20170327083344-ded68f7a9561/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
-github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
-github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
-github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
-github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
-github.com/mattn/go-sqlite3 v1.14.7/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
 github.com/mattn/go-sqlite3 v2.0.3+incompatible h1:gXHsfypPkaMZrKbD5209QV9jbUTJKjyR5WD3HYQSd+U=
 github.com/mattn/go-sqlite3 v2.0.3+incompatible/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
-github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpevwGNQEw=
-github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
-github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
-github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc=
-github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
-github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI=
-github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg=
-github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY=
-github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
-github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
-github.com/mitchellh/mapstructure v1.3.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
-github.com/mitchellh/mapstructure v1.4.1 h1:CpVNEelQCZBooIPDn+AR3NpivK/TIKU8bDxdASFVQag=
-github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
+github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
+github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
-github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe h1:iruDEfMl2E6fbMZ9s0scYfZQ84/6SPL6zC8ACM2oIL0=
 github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc=
+github.com/montanaflynn/stats v0.7.1 h1:etflOAAHORrCC44V+aR6Ftzort912ZU+YLiSTuV8eaE=
+github.com/montanaflynn/stats v0.7.1/go.mod h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt6R8Bnaayow=
 github.com/mozillazg/go-pinyin v0.20.0 h1:BtR3DsxpApHfKReaPO1fCqF4pThRwH9uwvXzm+GnMFQ=
 github.com/mozillazg/go-pinyin v0.20.0/go.mod h1:iR4EnMMRXkfpFVV5FMi4FNB6wGq9NV6uDWbUuPhP4Yc=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
-github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
-github.com/nats-io/jwt v0.3.0/go.mod h1:fRYCDE99xlTsqUzISS1Bi75UBJ6ljOJQOAAu5VglpSg=
-github.com/nats-io/jwt v0.3.2/go.mod h1:/euKqTS1ZD+zzjYrY7pseZrTtWQSjujC7xjPc8wL6eU=
-github.com/nats-io/nats-server/v2 v2.1.2/go.mod h1:Afk+wRZqkMQs/p45uXdrVLuab3gwv3Z8C4HTBu8GD/k=
-github.com/nats-io/nats.go v1.9.1/go.mod h1:ZjDU1L/7fJ09jvUSRVBR2e7+RnLiiIQyqyzEE/Zbp4w=
-github.com/nats-io/nkeys v0.1.0/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w=
-github.com/nats-io/nkeys v0.1.3/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w=
-github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
-github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs=
 github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
 github.com/nosixtools/solarlunar v0.0.0-20211112060703-1b6dea7b4a19 h1:LhWT2dBuNkYexwRSsPpYh67e0ikmH1ebBDaVkGHoMts=
 github.com/nosixtools/solarlunar v0.0.0-20211112060703-1b6dea7b4a19/go.mod h1:LjhyrWzOLJ9l1azMoNr9iCvfNrHEREqvJHzSLQcD0/o=
 github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE=
-github.com/oklog/oklog v0.3.2/go.mod h1:FCV+B7mhrz4o+ueLpx+KqkyXRGMWOYEvfiXtdGtbWGs=
-github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA=
-github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
-github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo=
+github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU=
 github.com/olivere/elastic/v7 v7.0.32 h1:R7CXvbu8Eq+WlsLgxmKVKPox0oOwAE/2T9Si5BnvK6E=
 github.com/olivere/elastic/v7 v7.0.32/go.mod h1:c7PVmLe3Fxq77PIfY/bZmxY/TAamBhCzZ8xDOE09a9k=
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
-github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
-github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.12.0/go.mod h1:oUhWkIvk5aDxtKvDDuw8gItl8pKl42LzjC9KZE0HfGg=
 github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE=
-github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
-github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
+github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU=
 github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
 github.com/onsi/gomega v1.18.1 h1:M1GfJqGRrBrrGGsbxzV5dqM2U2ApXefZCQpkukxYRLE=
-github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk=
-github.com/opentracing-contrib/go-observer v0.0.0-20170622124052-a52f23424492/go.mod h1:Ngi6UdF0k5OKD5t5wlmGhe/EDKPoUM3BXZSSfIuJbis=
-github.com/opentracing/basictracer-go v1.0.0/go.mod h1:QfBfYuafItcjQuMwinw9GhYKwFXS9KnPs5lxoYwgW74=
-github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
-github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
-github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc=
-github.com/openzipkin-contrib/zipkin-go-opentracing v0.4.5/go.mod h1:/wsWhb9smxSfWAKL3wpBW7V8scJMt8N8gnaMCS9E/cA=
-github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw=
-github.com/openzipkin/zipkin-go v0.2.1/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4=
-github.com/openzipkin/zipkin-go v0.2.2/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4=
-github.com/pact-foundation/pact-go v1.0.4/go.mod h1:uExwJY4kCzNPcHRj+hCR/HBbOOIwwtUjcrb0b5/5kLM=
-github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
-github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k=
+github.com/onsi/gomega v1.18.1/go.mod h1:0q+aL8jAiMXy9hbwj2mr5GziHiwhAIQpFmmtT5hitRs=
 github.com/pelletier/go-toml v1.0.1/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
-github.com/pelletier/go-toml v1.8.1/go.mod h1:T2/BmBdy8dvIRq1a/8aqjN41wvWlN4lrapLU/GW4pbc=
-github.com/pelletier/go-toml v1.9.2/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c=
-github.com/performancecopilot/speed v3.0.0+incompatible/go.mod h1:/CLtqpZ5gBg1M9iaPbIdPPGyKcA8hKdoy6hAWba7Yac=
-github.com/peterh/liner v0.0.0-20170317030525-88609521dc4b/go.mod h1:xIteQHvHuaLYG9IFj6mSxM0fCKrs34IrEQUhOYuGPHc=
 github.com/peterh/liner v1.0.1-0.20171122030339-3681c2a91233/go.mod h1:xIteQHvHuaLYG9IFj6mSxM0fCKrs34IrEQUhOYuGPHc=
-github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc=
-github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
-github.com/pkg/profile v1.2.1/go.mod h1:hJw3o1OdXxsrSjjVksARp5W95eeEaEfptyVZyv6JUPA=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI=
 github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
-github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs=
-github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso=
 github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
-github.com/prometheus/client_golang v1.3.0/go.mod h1:hJaj2vgQTGQmVCsAACORcieXFeDPbaTKGT+JTgUa3og=
 github.com/prometheus/client_golang v1.7.0/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M=
-github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M=
-github.com/prometheus/client_golang v1.11.0 h1:HNkLOAEQMIDv/K+04rukrLx6ch7msSRwf3/SASFAGtQ=
-github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0=
+github.com/prometheus/client_golang v1.19.0 h1:ygXvpU1AoN1MhdzckN+PyD9QJOSD4x7kmXYlnfbA6JU=
+github.com/prometheus/client_golang v1.19.0/go.mod h1:ZRM9uEAypZakd+q/x7+gmsvXdURP+DABIEIjnmDdp+k=
 github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
-github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
 github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
-github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
-github.com/prometheus/client_model v0.1.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
-github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M=
 github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
-github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
-github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
-github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
+github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw=
+github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI=
 github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
-github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA=
 github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo=
-github.com/prometheus/common v0.26.0 h1:iMAkS2TDoNWnKM+Kopnx/8tnEStIfpYA0ur0xQzzhMQ=
-github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc=
+github.com/prometheus/common v0.48.0 h1:QO8U2CdOzSn1BBsmXJXduaaW+dY/5QLjfB8svtSzKKE=
+github.com/prometheus/common v0.48.0/go.mod h1:0/KsvlIEfPQCQ5I2iNSAWKPZziNCvRs5EC6ILDTlAPc=
 github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
-github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
-github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
 github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
-github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A=
 github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
-github.com/prometheus/procfs v0.6.0 h1:mxy4L2jP6qMonqmq+aTtOx1ifVWUgG/TAmntgbh3xv4=
-github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
-github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
+github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo=
+github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo=
 github.com/qiniu/qmgo v1.1.8 h1:E64M+P59aqQpXKI24ClVtluYkLaJLkkeD2hTVhrdMks=
 github.com/qiniu/qmgo v1.1.8/go.mod h1:QvZkzWNEv0buWPx0kdZsSs6URhESVubacxFPlITmvB8=
-github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
 github.com/rdlucklib/rdluck_tools v1.0.3 h1:iOtK2QPlPQ6CL6c1htCk5VnFCHzyG6DCfJtunrMswK0=
 github.com/rdlucklib/rdluck_tools v1.0.3/go.mod h1:9Onw9o4w19C8KE5lxb8GyxgRBbZweRVkQSc79v38EaA=
-github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
-github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
-github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
-github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
-github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
-github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
-github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E=
-github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
+github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=
+github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog=
 github.com/shiena/ansicolor v0.0.0-20151119151921-a422bbe96644/go.mod h1:nkxAfR/5quYxwPZhyDxgasBMnRtBZd0FCEpawpjMUFg=
 github.com/shiena/ansicolor v0.0.0-20200904210342-c7312218db18 h1:DAYUYH5869yV94zvCES9F51oYtN5oGlwjxJJz7ZCnik=
 github.com/shiena/ansicolor v0.0.0-20200904210342-c7312218db18/go.mod h1:nkxAfR/5quYxwPZhyDxgasBMnRtBZd0FCEpawpjMUFg=
-github.com/shopspring/decimal v1.3.1 h1:2Usl1nmF/WZucqkFZhnfFYxxxu8LG21F6nPQBE5gKV8=
-github.com/shopspring/decimal v1.3.1/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o=
-github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
+github.com/shopspring/decimal v1.4.0 h1:bxl37RwXBklmTi0C79JfXCEBD1cqqHt0bbgBAGFp81k=
+github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+DMd9qYNcwME=
 github.com/siddontang/go v0.0.0-20170517070808-cb568a3e5cc0/go.mod h1:3yhqj7WBBfRhbBlzyOC3gUxftwsU0u8gqevxwIHQpMw=
 github.com/siddontang/goredis v0.0.0-20150324035039-760763f78400/go.mod h1:DDcKzU3qCuvj/tPnimWSsZZzvk9qvkvrIL5naVBPh5s=
 github.com/siddontang/rdb v0.0.0-20150307021120-fc89ed2e418d/go.mod h1:AMEsy7v5z92TR1JKMkLLoaOQk++LVnOKL3ScbJ8GNGA=
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
 github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
-github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
-github.com/smartwalle/pongo2render v1.0.1/go.mod h1:MGnTzND7nEMz7g194kjlnw8lx/V5JJlb1hr5kDXEO0I=
-github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
-github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
-github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
-github.com/sony/gobreaker v0.4.1/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJOjmxWY=
-github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
-github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
-github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
-github.com/spf13/cobra v0.0.0-20170417170307-b6cb39589372/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
-github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
-github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
-github.com/spf13/pflag v0.0.0-20170417173400-9e4c21054fa1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
-github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
-github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
-github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg=
 github.com/ssdb/gossdb v0.0.0-20180723034631-88f6b59b84ec/go.mod h1:QBvMkMya+gXctz3kmljlUCu/yB3GZ6oee+dUozsezQE=
-github.com/streadway/amqp v0.0.0-20190404075320-75d898a42a94/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw=
-github.com/streadway/amqp v0.0.0-20190827072141-edfb9018d271/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw=
-github.com/streadway/handy v0.0.0-20190108123426-d5acb3125c2a/go.mod h1:qNTQ5P5JnDBl6z3cMAg/SywNDC5ABu5ApDIw6lUbRmI=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.1.1 h1:2vfRuCMp5sSVIDSqO8oNnWJq7mPa6KVP3iPIwFBuy8A=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
-github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
-github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
+github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
+github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
 github.com/syndtr/goleveldb v0.0.0-20160425020131-cfa635847112/go.mod h1:Z4AUp2Km+PwemOoO/VB5AOx9XSsIItzFjoJlOSiYmn0=
 github.com/syndtr/goleveldb v0.0.0-20181127023241-353a9fca669c/go.mod h1:Z4AUp2Km+PwemOoO/VB5AOx9XSsIItzFjoJlOSiYmn0=
 github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
-github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
-github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
 github.com/ugorji/go v0.0.0-20171122102828-84cb69a8af83/go.mod h1:hnLbHMwcvSihnDhEfx2/BzKp2xb0Y+ErdfYcrs9tkJQ=
-github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
-github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
+github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
+github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
 github.com/wendal/errors v0.0.0-20130201093226-f66c77a7882b/go.mod h1:Q12BUT7DqIlHRmgv3RskH+UCM/4eqVMgI0EMmlSpAXc=
 github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c=
 github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
@@ -510,319 +234,105 @@ github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3k
 github.com/xdg-go/stringprep v1.0.3/go.mod h1:W3f5j4i+9rC0kuIEJL0ky1VpHXQU3ocBgklLGvcBnW8=
 github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8=
 github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM=
-github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
 github.com/yidane/formula v0.0.0-20210902154546-0782e1736717 h1:9CTJJpdISGxMAELfVlprj5kZEsJEaNAWiobv8ZAd72U=
 github.com/yidane/formula v0.0.0-20210902154546-0782e1736717/go.mod h1:9/dQiKiN04yPMdgsuFmKGuI2Hdp6OmFV9gSWS1col6g=
 github.com/ylywyn/jpush-api-go-client v0.0.0-20190906031852-8c4466c6e369/go.mod h1:Nv7wKD2/bCdKUFNKcJRa99a+1+aSLlCRJFriFYdjz/I=
 github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d h1:splanxYIlg+5LfHAM6xpdFEAYOk8iySO56hMFq6uLyA=
 github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA=
-github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
-github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
-github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
 github.com/yuin/gopher-lua v0.0.0-20171031051903-609c9cd26973/go.mod h1:aEV29XrmTYFr3CiRxZeGHpkvbwq+prZduBqMaascyCU=
-go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
-go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
-go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg=
-go.etcd.io/etcd v3.3.25+incompatible/go.mod h1:yaeTdrJi5lOmYerz05bd8+V7KubZs8YSFZfzsF9A6aI=
-go.etcd.io/etcd/api/v3 v3.5.0/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs=
-go.etcd.io/etcd/client/pkg/v3 v3.5.0/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g=
-go.etcd.io/etcd/client/v3 v3.5.0/go.mod h1:AIKXXVX/DQXtfTEqBryiLTUXwON+GuvO6Z7lLS/oTh0=
 go.mongodb.org/mongo-driver v1.11.6/go.mod h1:G9TgswdsWjX4tmDA5zfs2+6AEPpYJwqblyjsfuh8oXY=
-go.mongodb.org/mongo-driver v1.15.0 h1:rJCKC8eEliewXjZGf0ddURtl7tTVy1TK3bfl0gkUSLc=
-go.mongodb.org/mongo-driver v1.15.0/go.mod h1:Vzb0Mk/pa7e6cWw85R4F/endUC3u0U9jGcNU603k65c=
-go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk=
-go.opencensus.io v0.20.2/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk=
-go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
-go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
-go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
-go.starlark.net v0.0.0-20190702223751-32f345186213/go.mod h1:c1/X6cHgvdXj6pUlmWKMkuqRnW4K8x2vwt6JAaaircg=
-go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
-go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
-go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
-go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
-go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
-go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
-go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4=
-go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU=
-go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
-go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA=
-go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
-go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM=
-go.uber.org/zap v1.15.0/go.mod h1:Mb2vm2krFEG5DV0W9qcHBYFtp/Wku1cvYaqPsS/WYfc=
-go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo=
-golang.org/x/arch v0.0.0-20190927153633-4e8777c89be4/go.mod h1:flIaEI6LNU6xOCD5PaJvn9wGP0agmIOqjrtsKGRguv4=
+go.mongodb.org/mongo-driver v1.16.0 h1:tpRsfBJMROVHKpdGyc1BBEzzjDUWjItxbVSZ8Ls4BQ4=
+go.mongodb.org/mongo-driver v1.16.0/go.mod h1:oB6AhJQvFQL4LEHyXi6aJzQJtBiTQHiAd83l0GdFaiw=
 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
-golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
-golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
-golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k=
-golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4=
-golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
-golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
-golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
-golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek=
-golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY=
-golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
-golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
-golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
-golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
-golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
-golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
-golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
-golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
-golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
-golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
-golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
-golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
-golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
-golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
-golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY=
-golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
-golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
-golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
-golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI=
+golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
-golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20190125091013-d26f9f9a57f3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
 golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
-golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
-golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
-golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
 golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
-golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M=
-golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
-golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
-golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
-golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
-golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs=
+golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o=
-golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M=
+golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc=
-golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y=
+golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
-golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
-golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
-golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
-golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk=
+golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
-golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
-golang.org/x/tools v0.0.0-20181221001348-537d06c36207/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
-golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
-golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
-golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
-golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
-golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
-golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
-golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
-golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
-golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20191127201027-ecd32218bd7f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20200103221440-774c71fcf114/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
-golang.org/x/tools v0.0.0-20201211185031-d93e913c1a58/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
-golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
-golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
-golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-google.golang.org/api v0.3.1/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk=
-google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
-google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
-google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
-google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
-google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
-google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
-google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
-google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
-google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
-google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
-google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
-google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
-google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
-google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
-google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
-google.golang.org/genproto v0.0.0-20190530194941-fb225487d101/go.mod h1:z3L6/3dTEVtUr6QSP8miRzeRqwQOioJ9I66odjN4I7s=
-google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
-google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
-google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8=
-google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
-google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
-google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
-google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0=
-google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs=
-google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
-google.golang.org/grpc v1.20.0/go.mod h1:chYK+tFQF0nDUGJgXMSgLCQk3phJEuONr2DCgLDdAQM=
-google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
-google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
-google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
-google.golang.org/grpc v1.22.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
-google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
-google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
-google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
-google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
-google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
-google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0=
-google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM=
 google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
 google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
 google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
 google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
 google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
-google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
 google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
-google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
-google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
-google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
-google.golang.org/protobuf v1.26.0 h1:bxAC2xTBsZGibn2RTntX0oH50xLsqy1OxA9tTL3p/lk=
-google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
+google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg=
+google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc h1:2gGKlE2+asNV9m7xrywl36YYNnBG5ZQ0r/BOOxqPpmk=
 gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc/go.mod h1:m7x9LTH6d71AHyAX77c9yqWCCa3UKHcVEj9y7hAtKDk=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU=
 gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw=
-gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
 gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
-gopkg.in/gcfg.v1 v1.2.3/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o=
 gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df h1:n7WqCuqOuCbNr617RXOY0AWRXxgwEyPp2z+p0+hgMuE=
 gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df/go.mod h1:LRQQ+SO6ZHR7tOkpBDuZnXENFzX8qRjMDMyPD6BRkCw=
-gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
-gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA=
 gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA=
-gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
-gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI=
-gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74=
 gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
-honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
-honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
-honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
-honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
-honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
-honnef.co/go/tools v0.0.1-2020.1.5/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
-rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
-rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4=
-sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o=
-sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc=
-sourcegraph.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0/go.mod h1:hI742Nqp5OhwiqlzhgfbWU4mW4yO10fP+LoT9WOswdU=

logic/chart_correlation.go

@@ -0,0 +1,195 @@
+package logic
+
+import (
+	"eta/eta_index_lib/models"
+	"eta/eta_index_lib/utils"
+	"fmt"
+)
+
+// CalculateCorrelation 计算相关性-获取x轴和y轴
+func CalculateCorrelation(leadValue int, leadUnit, frequencyA, frequencyB string, dataListA, dataListB []*models.EdbInfoSearchData) (xEdbIdValue []int, yDataList []YData, err error) {
+	xData := make([]int, 0)
+	yData := make([]float64, 0)
+	if leadValue == 0 {
+		xData = append(xData, 0)
+	}
+	if leadValue > 0 {
+		leadMin := 0 - leadValue
+		xLen := 2*leadValue + 1
+		for i := 0; i < xLen; i++ {
+			n := leadMin + i
+			xData = append(xData, n)
+		}
+	}
+
+	// 计算窗口，不包含第一天
+	//startDateTime, _ := time.ParseInLocation(utils.FormatDate, startDate, time.Local)
+	//startDate = startDateTime.AddDate(0, 0, 1).Format(utils.FormatDate)
+
+	//// 2023-03-02 时间序列始终以指标B为基准, 始终是A进行平移
+	//baseEdbInfo := edbInfoMappingB
+	//changeEdbInfo := edbInfoMappingA
+	// 2023-03-17 时间序列始终以指标A为基准, 始终是B进行平移
+	//baseEdbInfo := edbInfoMappingA
+	//changeEdbInfo := edbInfoMappingB
+
+	// 获取时间基准指标在时间区间内的值
+	//aDataList := make([]*EdbDataList, 0)
+	//switch baseEdbInfo.EdbInfoCategoryType {
+	//case 0:
+	//	aDataList, err = GetEdbDataList(baseEdbInfo.Source, baseEdbInfo.SubSource, baseEdbInfo.EdbInfoId, startDate, endDate)
+	//case 1:
+	//	_, aDataList, _, _, err, _ = data.GetPredictDataListByPredictEdbInfoId(baseEdbInfo.EdbInfoId, startDate, endDate, false)
+	//default:
+	//	err = errors.New("指标base类型异常")
+	//	return
+	//}
+	//
+	//// 获取变频指标所有日期的值, 插值法完善数据
+	//bDataList := make([]*EdbDataList, 0)
+	//switch changeEdbInfo.EdbInfoCategoryType {
+	//case 0:
+	//	bDataList, err = GetEdbDataList(changeEdbInfo.Source, changeEdbInfo.SubSource, changeEdbInfo.EdbInfoId, "", "")
+	//case 1:
+	//	_, bDataList, _, _, err, _ = data.GetPredictDataListByPredictEdbInfoId(changeEdbInfo.EdbInfoId, "", "", false)
+	//default:
+	//	err = errors.New("指标change类型异常")
+	//	return
+	//}
+	//changeDataMap := make(map[string]float64)
+	//newChangeDataList, e := HandleDataByLinearRegression(bDataList, changeDataMap)
+	//if e != nil {
+	//	err = fmt.Errorf("获取变频指标插值法Map失败, Err: %s", e.Error())
+	//	return
+	//}
+
+	// 2023-03-17 时间序列始终以指标A为基准, 始终是B进行平移
+	baseDataList := make([]*models.EdbInfoSearchData, 0)
+	baseDataMap := make(map[string]float64)
+	changeDataList := make([]*models.EdbInfoSearchData, 0)
+	changeDataMap := make(map[string]float64)
+
+	// 先把低频指标升频为高频
+	{
+		frequencyIntMap := map[string]int{
+			"日度": 1,
+			"周度": 2,
+			"旬度": 3,
+			"月度": 4,
+			"季度": 5,
+			"年度": 6,
+		}
+
+		// 如果A指标是高频，那么就需要对B指标进行升频
+		if frequencyIntMap[frequencyA] < frequencyIntMap[frequencyB] {
+			tmpNewChangeDataList, e := models.HandleDataByLinearRegression(dataListB, changeDataMap)
+			if e != nil {
+				err = fmt.Errorf("获取变频指标插值法Map失败, Err: %s", e.Error())
+				return
+			}
+			changeDataList = tmpNewChangeDataList
+			baseDataList = dataListA
+			for _, v := range baseDataList {
+				baseDataMap[v.DataTime] = v.Value
+			}
+
+		} else if frequencyIntMap[frequencyA] > frequencyIntMap[frequencyB] {
+			// 如果B指标是高频，那么就需要对A指标进行升频
+			tmpNewChangeDataList, e := models.HandleDataByLinearRegression(dataListA, baseDataMap)
+			if e != nil {
+				err = fmt.Errorf("获取变频指标插值法Map失败, Err: %s", e.Error())
+				return
+			}
+			baseDataList = tmpNewChangeDataList
+
+			changeDataList = dataListB
+			for _, v := range changeDataList {
+				changeDataMap[v.DataTime] = v.Value
+			}
+		} else {
+			baseDataList = dataListA
+			for _, v := range baseDataList {
+				baseDataMap[v.DataTime] = v.Value
+			}
+			changeDataList = dataListB
+			for _, v := range changeDataList {
+				changeDataMap[v.DataTime] = v.Value
+			}
+		}
+
+	}
+
+	// 计算不领先也不滞后时的相关系数
+	baseCalculateData := make([]float64, 0)
+	baseDataTimeArr := make([]string, 0)
+	for i := range baseDataList {
+		baseDataTimeArr = append(baseDataTimeArr, baseDataList[i].DataTime)
+		baseCalculateData = append(baseCalculateData, baseDataList[i].Value)
+	}
+
+	//zeroBaseData := make([]float64, 0)
+	//zeroCalculateData := make([]float64, 0)
+	//for i := range baseDataTimeArr {
+	//	tmpBaseVal, ok1 := baseDataMap[baseDataTimeArr[i]]
+	//	tmpCalculateVal, ok2 := changeDataMap[baseDataTimeArr[i]]
+	//	if ok1 && ok2 {
+	//		zeroBaseData = append(zeroBaseData, tmpBaseVal)
+	//		zeroCalculateData = append(zeroCalculateData, tmpCalculateVal)
+	//	}
+	//}
+	//if len(zeroBaseData) != len(zeroCalculateData) {
+	//	err = fmt.Errorf("相关系数两组序列元素数不一致, %d-%d", len(baseCalculateData), len(zeroCalculateData))
+	//	return
+	//}
+	//zeroRatio := utils.CalculateCorrelationByIntArr(zeroBaseData, zeroCalculateData)
+	//if leadValue == 0 {
+	//	yData = append(yData, zeroRatio)
+	//}
+
+	// 计算领先/滞后N期
+	if leadValue > 0 {
+		// 平移变频指标领先/滞后的日期(单位天)
+		moveUnitDays := utils.FrequencyDaysMap[leadUnit]
+
+		for i := range xData {
+			//if xData[i] == 0 {
+			//	yData = append(yData, zeroRatio)
+			//	continue
+			//}
+			xCalculateData := make([]float64, 0)
+			yCalculateData := make([]float64, 0)
+
+			// 平移指定天数
+			mDays := int(moveUnitDays) * xData[i]
+			_, dMap := models.MoveDataDaysToNewDataList(changeDataList, mDays)
+
+			// 取出对应的基准日期的值
+			for i2 := range baseDataTimeArr {
+				tmpDate := baseDataTimeArr[i2]
+				if yVal, ok := dMap[tmpDate]; ok {
+					xCalculateData = append(xCalculateData, baseCalculateData[i2])
+					yCalculateData = append(yCalculateData, yVal)
+				}
+			}
+			if len(yCalculateData) <= 0 {
+				//err = fmt.Errorf("领先滞后相关系数两组序列元素数不一致, %d-%d", len(baseCalculateData), len(yCalculateData))
+				//return
+				// 领先滞后后，没有可以计算的数据了
+				continue
+			}
+
+			// 公式计算出领先/滞后频度对应点的相关性系数
+			ratio := utils.CalculateCorrelationByIntArr(xCalculateData, yCalculateData)
+			yData = append(yData, ratio)
+		}
+	}
+
+	xEdbIdValue = xData
+	yDataList = make([]YData, 0)
+	yDate := "0000-00-00"
+	yDataList = append(yDataList, YData{
+		Date:  yDate,
+		Value: yData,
+	})
+	return
+}

logic/predict_edb.go

@@ -113,6 +113,7 @@ func AddPredictEdbInfo(sourceEdbInfoId, classifyId int, edbName, dataDateType st
 		EdbNameEn:     edbName,
 		UnitEn:        sourceEdbInfo.UnitEn,
 		DataDateType:  dataDateType,
+		Sort:          models.GetAddEdbMaxSortByClassifyId(classifyId, utils.PREDICT_EDB_INFO_TYPE),
 	}
 
 	// 关联关系表
@@ -978,7 +979,6 @@ func RefreshPredictEdbInfo(edbInfoId int) (edbInfo *models.EdbInfo, err error, e
 	return
 }
 
-
 // checkExistByEdbName
 // @Description: 根据指标名称校验该指标是否存在库中
 // @author: Roc
@@ -1076,4 +1076,4 @@ func CheckExistByEdbNameAndEdbInfoId(edbInfoType, edbInfoId int, edbName, lang s
 
 	//指标已经入库的情况
 	return checkExistByEdbNameAndEdbInfoId(edbInfoType, edbInfoId, edbName, lang)
-}
+}

logic/profit_chart_info.go

@@ -261,10 +261,10 @@ func GetProfitChartEdbData(baseEdbInfo *models.EdbInfo, zlFutureGoodEdbInfoList
 	// 特殊的商品期货合约（只有M+N的合约，没有固定日期的合约）
 	specialFutureGoodEdbInfoMap := make(map[int]map[int]*future_good.FutureGoodEdbInfo, 0)
 
-	isAllChina := true // 是否都是国内的期货合约
+	hasChina := false // 是否包含国内的期货合约
 	for _, v := range zlFutureGoodEdbInfoList {
-		if v.RegionType != "国内" {
-			isAllChina = false
+		if v.RegionType == "国内" {
+			hasChina = true
 			break
 		}
 	}
@@ -283,7 +283,7 @@ func GetProfitChartEdbData(baseEdbInfo *models.EdbInfo, zlFutureGoodEdbInfoList
 			return
 		}
 
-		childFutureGoodEdbInfoMap, tmpMaxN, tmpErr := getProfitFutureGoodEdbInfoList(earliestDateTime, v, tmpFutureGoodEdbInfoList, isAllChina, monthNum)
+		childFutureGoodEdbInfoMap, tmpMaxN, tmpErr := getProfitFutureGoodEdbInfoList(earliestDateTime, v, tmpFutureGoodEdbInfoList, monthNum)
 		if tmpErr != nil {
 			err = tmpErr
 			return
@@ -322,6 +322,11 @@ func GetProfitChartEdbData(baseEdbInfo *models.EdbInfo, zlFutureGoodEdbInfoList
 		}
 	}
 
+	// 需求池604，只要包含了国内合约，最大必须是12期
+	if hasChina {
+		maxN = 12
+	}
+
 	// 找出所有的N值，并进行正序排列
 	dateList := make([]string, 0)
 	for _, n := range nMap {
@@ -570,7 +575,7 @@ func ProfitChartChartData(baseDataList []*models.EdbDataList, futureGoodEdbInfoM
 }
 
 // getFutureGoodEdbInfoList 获取适用的指标列表
-func getProfitFutureGoodEdbInfoList(earliestDateTime time.Time, zlFutureGoodEdbInfo *future_good.FutureGoodEdbInfo, tmpFutureGoodEdbInfoList []*future_good.FutureGoodEdbInfo, isAllChina bool, monthNum int) (futureGoodEdbInfoDateMap map[string]*future_good.FutureGoodEdbInfo, newMaxN int, err error) {
+func getProfitFutureGoodEdbInfoList(earliestDateTime time.Time, zlFutureGoodEdbInfo *future_good.FutureGoodEdbInfo, tmpFutureGoodEdbInfoList []*future_good.FutureGoodEdbInfo, monthNum int) (futureGoodEdbInfoDateMap map[string]*future_good.FutureGoodEdbInfo, newMaxN int, err error) {
 	maxN := 36 //最大36期合约
 	futureGoodEdbInfoList := make([]*future_good.FutureGoodEdbInfo, 0)
 	futureGoodEdbInfoDateMap = make(map[string]*future_good.FutureGoodEdbInfo)
@@ -635,7 +640,7 @@ func getProfitFutureGoodEdbInfoList(earliestDateTime time.Time, zlFutureGoodEdbI
 		// 如果（当前年-最新日期的年份） * 12个月 + (当前月-最新日期的月份) 小于总月份
 		tmpN := subYear*12 + subMonth
 		if tmpN < newMaxN {
-			tmpDateTime := time.Date(v.Year, time.Month(v.Month), 0, 0, 0, 0, 0, time.Local)
+			tmpDateTime := time.Date(v.Year, time.Month(v.Month), 1, 0, 0, 0, 0, time.Local)
 			futureGoodEdbInfoDateMap[tmpDateTime.Format(utils.FormatYearMonthDate)] = v
 			if tmpN > newMaxN {
 				newMaxN = tmpN

main.go

@@ -6,7 +6,7 @@ import (
 	"eta/eta_index_lib/services/alarm_msg"
 	"eta/eta_index_lib/utils"
 	"fmt"
-	"github.com/beego/beego/v2/adapter/logs"
+	"github.com/beego/beego/v2/core/logs"
 	"github.com/beego/beego/v2/server/web"
 	"github.com/beego/beego/v2/server/web/context"
 	"runtime"

models/base_calculate.go

@@ -1851,3 +1851,35 @@ func GetDateDataAndDateList(dateDataMap map[time.Time]float64) (dateStrDataMap m
 
 	return
 }
+
+// StepCalculateBySearchData
+// @Description: 分步骤计算
+type StepCalculateBySearchData struct {
+	DataList   []*EdbInfoSearchData `description:"基础数据"`
+	Calculates []CalculatesReq
+}
+
+type CalculatesReq struct {
+	Formula       interface{}
+	Calendar      string `description:"公历/农历"`
+	Frequency     string `description:"需要转换的频度"`
+	MoveType      int    `description:"移动方式:1:领先(默认),2:滞后"`
+	MoveFrequency string `description:"移动频度"`
+	FromFrequency string `description:"来源的频度"`
+	Source        int    `description:"1：累计值转月；2：累计值转季；3：同比值；4：同差值；5：N数值移动平均数计算；6：环比值；7：环差值；8：升频；9：降频；10：时间移位；11：超季节性；12：年化；13：累计值；14：累计值年初至今；15：指数修匀；16：日均值"`
+	Sort          int    `description:"计算顺序"`
+}
+
+func TransDateData2EdbData(dateData map[time.Time]float64) (edbData []*EdbInfoData) {
+	edbData = make([]*EdbInfoData, 0)
+	for d, v := range dateData {
+		edbData = append(edbData, &EdbInfoData{
+			DataTime: d,
+			Value:    v,
+		})
+	}
+	sort.Slice(edbData, func(i, j int) bool {
+		return edbData[i].DataTime.Before(edbData[j].DataTime)
+	})
+	return
+}

models/base_from_adjust.go

@@ -30,7 +30,7 @@ type AddAdjustEdbData struct {
 }
 
 // SaveAdjustEdb 保存数据调整指标
-func SaveAdjustEdb(req SaveAdjustEdbReq) (edbInfo *EdbInfo, err error, errMsg string) {
+func SaveAdjustEdb(req SaveAdjustEdbReq, lang string) (edbInfo *EdbInfo, err error, errMsg string) {
 	errMsg = `添加指标失败`
 	o := orm.NewOrm()
 	to, err := o.Begin()
@@ -62,10 +62,12 @@ func SaveAdjustEdb(req SaveAdjustEdbReq) (edbInfo *EdbInfo, err error, errMsg st
 			Source:          utils.DATA_SOURCE_CALCULATE_ADJUST,
 			SourceName:      "数据调整",
 			EdbCode:         edbCode,
-			EdbName:         utils.TrimStr(req.EdbName),
-			EdbNameSource:   utils.TrimStr(req.EdbName),
-			Frequency:       utils.TrimStr(req.Frequency),
-			Unit:            utils.TrimStr(req.Unit),
+			EdbName:         strings.TrimSpace(req.EdbName),
+			EdbNameSource:   strings.TrimSpace(req.EdbName),
+			Frequency:       strings.TrimSpace(req.Frequency),
+			Unit:            strings.TrimSpace(req.Unit),
+			EdbNameEn:       strings.TrimSpace(req.EdbName),
+			UnitEn:          strings.TrimSpace(req.Unit),
 			ClassifyId:      req.ClassifyId,
 			SysUserId:       req.AdminId,
 			SysUserRealName: req.AdminName,
@@ -73,7 +75,9 @@ func SaveAdjustEdb(req SaveAdjustEdbReq) (edbInfo *EdbInfo, err error, errMsg st
 			ModifyTime:      time.Now(),
 			UniqueCode:      uniqueCode,
 			EdbType:         2,
+			Sort:            GetAddEdbMaxSortByClassifyId(req.ClassifyId, utils.EDB_INFO_TYPE),
 		}
+
 		newEdbInfoId, tmpErr := to.Insert(edbInfo)
 		if tmpErr != nil {
 			err = tmpErr
@@ -132,14 +136,29 @@ func SaveAdjustEdb(req SaveAdjustEdbReq) (edbInfo *EdbInfo, err error, errMsg st
 			return
 		}
 
+		// 额外赋值
+		switch lang {
+		case utils.EnLangVersion:
+			req.EdbNameEn = req.EdbName
+			req.UnitEn = req.Unit
+
+			req.EdbName = edbInfo.EdbName
+			req.Unit = edbInfo.Unit
+		default:
+			req.EdbNameEn = edbInfo.EdbNameEn
+			req.UnitEn = edbInfo.UnitEn
+		}
+
 		// 更新指标信息
-		edbInfo.EdbName = utils.TrimStr(req.EdbName)
-		edbInfo.EdbNameSource = utils.TrimStr(req.EdbName)
-		edbInfo.Frequency = utils.TrimStr(req.Frequency)
-		edbInfo.Unit = utils.TrimStr(req.Unit)
+		edbInfo.EdbName = strings.TrimSpace(req.EdbName)
+		edbInfo.EdbNameSource = strings.TrimSpace(req.EdbName)
+		edbInfo.Frequency = strings.TrimSpace(req.Frequency)
+		edbInfo.Unit = strings.TrimSpace(req.Unit)
+		edbInfo.EdbNameEn = strings.TrimSpace(req.EdbNameEn)
+		edbInfo.UnitEn = strings.TrimSpace(req.UnitEn)
 		edbInfo.ClassifyId = req.ClassifyId
 		edbInfo.ModifyTime = time.Now()
-		_, err = to.Update(edbInfo, "EdbName", "EdbNameSource", "Frequency", "Unit", "ClassifyId", "ModifyTime")
+		_, err = to.Update(edbInfo, "EdbName", "EdbNameSource", "Frequency", "Unit", "EdbNameEn", "UnitEn", "ClassifyId", "ModifyTime")
 		if err != nil {
 			return
 		}
@@ -306,7 +325,7 @@ func RefreshAllAdjustEdb(edbInfo *EdbInfo, fromEdbInfo *EdbInfo) (err error) {
 		saveVal := utils.SubFloatToString(val, 20)
 		if existVal, ok := existDataMap[currDay]; !ok {
 			//格式化时间
-			currentDate, tmpErr := time.Parse(utils.FormatDate, item.DataTime)
+			currentDate, tmpErr := time.ParseInLocation(utils.FormatDate, item.DataTime, time.Local)
 			if tmpErr != nil {
 				err = tmpErr
 				return
@@ -362,6 +381,8 @@ type SaveAdjustEdbReq struct {
 	Unit          string                 `description:"单位"`
 	ClassifyId    int                    `description:"分类id"`
 	DataList      []SaveAdjustEdbDataReq `description:"指标对应的数据值"`
+	EdbNameEn     string                 `description:"英文指标名称"`
+	UnitEn        string                 `description:"英文单位"`
 }
 
 // SaveAdjustEdbDataReq 保存数据调整请求的数据的参数

models/base_from_business.go

@@ -211,8 +211,33 @@ func (m *EdbBusinessSource) Add() (err error) {
 	return
 }
 
-// GetEdbInfoMaxAndMinInfo 获取指标的最新数据记录信息
+// GetEdbInfoMaxAndMinInfo
+// @Description: 获取指标的最新数据记录信息
+// @author: Roc
+// @receiver m
+// @datetime 2024-07-02 14:50:50
+// @param edbCode string
+// @return item *EdbInfoMaxAndMinInfo
+// @return err error
 func (m BaseFromBusinessIndex) GetEdbInfoMaxAndMinInfo(edbCode string) (item *EdbInfoMaxAndMinInfo, err error) {
+	if utils.UseMongo {
+		return m.getEdbInfoMaxAndMinInfoByMongo(edbCode)
+	} else {
+		return m.getEdbInfoMaxAndMinInfoByMysql(edbCode)
+	}
+
+	return
+}
+
+// getEdbInfoMaxAndMinInfoByMongo
+// @Description: 获取指标的最新数据记录信息（从mongo中获取）
+// @author: Roc
+// @receiver m
+// @datetime 2024-07-02 14:41:20
+// @param edbCode string
+// @return item *EdbInfoMaxAndMinInfo
+// @return err error
+func (m BaseFromBusinessIndex) getEdbInfoMaxAndMinInfoByMongo(edbCode string) (item *EdbInfoMaxAndMinInfo, err error) {
 	mogDataObj := new(mgo.BaseFromBusinessData)
 	pipeline := []bson.M{
 		{"$match": bson.M{"index_code": edbCode}},
@@ -256,6 +281,34 @@ func (m BaseFromBusinessIndex) GetEdbInfoMaxAndMinInfo(edbCode string) (item *Ed
 	return
 }
 
+// getEdbInfoMaxAndMinInfoByMysql
+// @Description: 获取指标的最新数据记录信息（从mysql中获取）
+// @author: Roc
+// @receiver m
+// @datetime 2024-07-02 14:49:58
+// @param edbCode string
+// @return item *EdbInfoMaxAndMinInfo
+// @return err error
+func (m BaseFromBusinessIndex) getEdbInfoMaxAndMinInfoByMysql(edbCode string) (item *EdbInfoMaxAndMinInfo, err error) {
+	dataObj := BaseFromBusinessData{}
+	result, err := dataObj.GetEdbInfoMaxAndMinInfo(edbCode)
+	if err != nil {
+		return
+	}
+
+	item = &EdbInfoMaxAndMinInfo{
+		MinDate:     result.MinDate,
+		MaxDate:     result.MaxDate,
+		MinValue:    result.MinValue,
+		MaxValue:    result.MaxValue,
+		LatestValue: result.LatestValue,
+		LatestDate:  result.LatestDate,
+		EndValue:    result.EndValue,
+	}
+
+	return
+}
+
 // ModifyIndexMaxAndMinInfo
 // @Description: 修改最大值和最小值信息
 // @author: Roc

models/base_from_business_data.go

@@ -0,0 +1,306 @@
+package models
+
+import (
+	"errors"
+	"eta/eta_index_lib/utils"
+	"fmt"
+	"github.com/beego/beego/v2/client/orm"
+	"strings"
+	"time"
+)
+
+// BaseFromBusinessData
+// @Description: 外部指标（商家系统）原始数据表
+type BaseFromBusinessData struct {
+	BusinessDataId          int       `orm:"column(business_data_id);pk" json:"business_data_id"`
+	BaseFromBusinessIndexId int       `json:"base_from_business_index_id"` // 指标id
+	IndexCode               string    `json:"index_code"`                  // 指标编码
+	DataTime                time.Time `json:"data_time"`                   // 数据日期
+	Value                   float64   `json:"value"`                       // 数据值
+	CreateTime              time.Time `json:"create_time"`                 // 创建时间
+	ModifyTime              time.Time `json:"modify_time"`                 // 修改时间
+}
+
+// TableName
+// @Description:  获取表名
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-26 13:41:36
+// @return string
+func (m *BaseFromBusinessData) TableName() string {
+	return "base_from_business_data"
+}
+
+// CollectionName
+// @Description:  获取集合名称
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-26 13:41:36
+// @return string
+func (m *BaseFromBusinessData) CollectionName() string {
+	return "base_from_business_data"
+}
+
+// DataBaseName
+// @Description: 获取数据库名称
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-26 13:41:33
+// @return string
+func (m *BaseFromBusinessData) DataBaseName() string {
+	return utils.MgoDataDbName
+}
+
+type WhereParams struct {
+	Condition string
+	Pars      []interface{}
+	Order     string `description:"排序字段"`
+}
+
+// GetAllDataList
+// @Description: 根据条件获取所有数据
+// @author: Roc
+// @receiver m
+// @datetime 2024-07-01 17:18:57
+// @param condition []string
+// @param pars []interface{}
+// @param order string
+// @return result []*BaseFromBusinessData
+// @return err error
+func (m *BaseFromBusinessData) GetAllDataList(condition []string, pars []interface{}, order string) (result []*BaseFromBusinessData, err error) {
+	o := orm.NewOrm()
+
+	sql := `SELECT * FROM base_from_business_data `
+	if len(condition) > 0 {
+		sql += ` WHERE ` + strings.Join(condition, " AND ")
+	}
+
+	if order != `` {
+		sql += ` ORDER BY ` + order
+	}
+
+	_, err = o.Raw(sql, pars).QueryRows(&result)
+
+	return
+}
+
+// GetLimitDataList
+// @Description: 根据条件获取指定数量数据列表
+// @author: Roc
+// @receiver m
+// @datetime 2024-07-01 17:19:16
+// @param condition []string
+// @param pars []interface{}
+// @param order string
+// @param size int64
+// @return result []*BaseFromBusinessData
+// @return err error
+func (m *BaseFromBusinessData) GetLimitDataList(condition []string, pars []interface{}, order string, size int64) (result []*BaseFromBusinessData, err error) {
+	o := orm.NewOrm()
+	sql := `SELECT * FROM base_from_business_data `
+	if len(condition) > 0 {
+		sql += ` WHERE ` + strings.Join(condition, " AND ")
+	}
+
+	if order != `` {
+		sql += ` ORDER BY ` + order
+	}
+
+	sql += fmt.Sprintf(` LIMIT %d`, size)
+
+	_, err = o.Raw(sql, pars).QueryRows(&result)
+
+	return
+}
+
+// GetPageDataList
+// @Description: 根据条件获取分页数据列表
+// @author: Roc
+// @receiver m
+// @datetime 2024-07-01 17:19:42
+// @param condition []string
+// @param pars []interface{}
+// @param order string
+// @param startSize int64
+// @param size int64
+// @return result []*BaseFromBusinessData
+// @return err error
+func (m *BaseFromBusinessData) GetPageDataList(condition []string, pars []interface{}, order string, startSize, size int64) (result []*BaseFromBusinessData, err error) {
+	o := orm.NewOrm()
+	sql := `SELECT * FROM base_from_business_data `
+	if len(condition) > 0 {
+		sql += ` WHERE ` + strings.Join(condition, " AND ")
+	}
+
+	if order != `` {
+		sql += ` ORDER BY ` + order
+	}
+
+	sql += fmt.Sprintf(` LIMIT %d,%d`, startSize, size)
+
+	_, err = o.Raw(sql, pars).QueryRows(&result)
+
+	return
+}
+
+// GetCountDataList
+// @Description: 根据条件获取数据列表总数
+// @author: Roc
+// @receiver m
+// @datetime 2024-07-01 17:19:50
+// @param condition []string
+// @param pars []interface{}
+// @return count int64
+// @return err error
+func (m *BaseFromBusinessData) GetCountDataList(condition []string, pars []interface{}) (count int64, err error) {
+	o := orm.NewOrm()
+	sql := `SELECT COUNT(1) FROM base_from_business_data `
+	if len(condition) > 0 {
+		sql += ` WHERE ` + strings.Join(condition, " AND ")
+	}
+	err = o.Raw(sql, pars).QueryRow(&count)
+
+	return
+}
+
+// InsertDataByColl
+// @Description: 写入单条数据（外部传入集合）
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-26 14:22:18
+// @param addData interface{}
+// @return err error
+func (m *BaseFromBusinessData) InsertDataByColl(addData interface{}) (err error) {
+	o := orm.NewOrm()
+	_, err = o.Insert(addData)
+
+	return
+}
+
+// BatchInsertData
+// @Description: 批量写入数据
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-26 14:22:18
+// @param bulk int 每次请求保存的数据量
+// @param dataList []*BaseFromBusinessData
+// @return err error
+func (m *BaseFromBusinessData) BatchInsertData(bulk int, dataList []*BaseFromBusinessData) (err error) {
+	o := orm.NewOrm()
+	_, err = o.InsertMulti(bulk, dataList)
+
+	return
+}
+
+// UpdateData
+// @Description: 单条数据修改
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-26 15:01:51
+// @param whereParams interface{}
+// @param updateParams interface{}
+// @return err error
+func (m *BaseFromBusinessData) UpdateData(updateCols []string) (err error) {
+	o := orm.NewOrm()
+	_, err = o.Update(m, updateCols...)
+	if err != nil {
+		fmt.Println("UpdateDataByColl:Err:" + err.Error())
+		return
+	}
+
+	return
+}
+
+// HandleData
+// @Description: 数据处理
+// @author: Roc
+// @receiver m
+// @datetime 2024-07-01 17:35:54
+// @param addDataList []*BaseFromBusinessData
+// @param updateDataList []*BaseFromBusinessData
+// @return err error
+func (m *BaseFromBusinessData) HandleData(addDataList, updateDataList []*BaseFromBusinessData) (err error) {
+	o := orm.NewOrm()
+	to, err := o.Begin()
+	if err != nil {
+		return
+	}
+	defer func() {
+		if err != nil {
+			fmt.Println("BaseFromBusinessData HandleData,Err:" + err.Error())
+			_ = to.Rollback()
+		} else {
+			_ = to.Commit()
+		}
+	}()
+
+	// 插入数据
+	if len(addDataList) > 0 {
+		_, err = to.InsertMulti(500, addDataList)
+		if err != nil {
+			return
+		}
+	}
+
+	// 修改
+
+	if len(updateDataList) > 0 {
+		for _, v := range updateDataList {
+			_, err = to.Update(v, "Value", "ModifyTime")
+			if err != nil {
+				fmt.Println("BaseFromBusinessData HandleData Update:Err:" + err.Error())
+				return
+			}
+		}
+	}
+
+	return
+}
+
+// GetEdbInfoMaxAndMinInfo
+// @Description: 获取当前指标的最大最小值
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-30 17:15:39
+// @param whereParams interface{}
+// @return result EdbInfoMaxAndMinInfo
+// @return err error
+func (m *BaseFromBusinessData) GetEdbInfoMaxAndMinInfo(indexCode string) (result EdbInfoMaxAndMinInfo, err error) {
+	o := orm.NewOrm()
+	sql := ``
+	sql = ` SELECT MIN(data_time) AS min_date,MAX(data_time) AS max_date,MIN(value) AS min_value,MAX(value) AS max_value FROM base_from_business_data WHERE index_code = ? `
+	err = o.Raw(sql, indexCode).QueryRow(&result)
+	if err != nil {
+		return
+	}
+
+	var latestValue float64
+	sql = ` SELECT value AS latest_value FROM base_from_business_data WHERE index_code = ? ORDER BY data_time DESC LIMIT 1 `
+	err = o.Raw(sql, indexCode).QueryRow(&latestValue)
+	result.LatestValue = latestValue
+
+	return
+}
+
+// DelDataByCond
+// @Description: 根据条件删除多条数据
+// @author: Roc
+// @receiver m
+// @datetime 2024-07-01 17:46:56
+// @param condition []string
+// @param pars []interface{}
+// @return err error
+func (m *BaseFromBusinessData) DelDataByCond(condition []string, pars []interface{}) (err error) {
+	if len(condition) <= 0 {
+		err = errors.New("条件不能为空")
+		return
+	}
+	o := orm.NewOrm()
+	sql := `DELETE FROM base_from_business_data `
+
+	sql += ` WHERE ` + strings.Join(condition, " AND ")
+
+	_, err = o.Raw(sql, pars).Exec()
+
+	return
+}

models/base_from_calculate.go

@@ -6,8 +6,9 @@ import (
 	"eta/eta_index_lib/utils"
 	"fmt"
 	"github.com/beego/beego/v2/client/orm"
+	"github.com/dengsgo/math-engine/engine"
 	"github.com/shopspring/decimal"
-	"github.com/yidane/formula"
+	//"github.com/yidane/formula"
 	"strconv"
 	"strings"
 	"time"
@@ -81,7 +82,7 @@ func AddCalculateInfo(req EdbInfoCalculateSaveReq, calculateMappingList []*EdbIn
 		MaxValue:         0,
 		CalculateFormula: req.CalculateFormula,
 		EdbType:          2,
-		Sort:             0,
+		Sort:             GetAddEdbMaxSortByClassifyId(req.ClassifyId, utils.EDB_INFO_TYPE),
 		MoveType:         0,
 		MoveFrequency:    "",
 		NoUpdate:         0,
@@ -229,7 +230,6 @@ func EditCalculateInfo(edbInfo *EdbInfo, req EdbInfoCalculateSaveReq, formulaSli
 				}
 			}
 		}
-
 		//计算数据
 		err = refreshAllCalculate(to, edbInfoList, edbInfoTag, edbInfo.EdbInfoId, edbInfo.Source, edbInfo.SubSource, edbInfo.EdbCode, edbInfo.CalculateFormula, "", "", edbInfoIdBytes, edbInfo.EmptyType, edbInfo.MaxEmptyType, edbInfo.Extra)
 	}
@@ -460,24 +460,24 @@ func refreshAllCalculate(to orm.TxOrmer, edbInfoIdArr []*EdbInfo, edbInfoTag map
 		}
 		//utils.FileLog.Info(fmt.Sprintf("formulaFormStr:%s", formulaFormStr))
 		fmt.Println(fmt.Sprintf("%s:formulaFormStr:%s", sk, formulaFormStr))
-		expression := formula.NewExpression(formulaFormStr)
-		calResult, err := expression.Evaluate()
-		if err != nil {
-			// 分母为0的报错
-			if strings.Contains(err.Error(), "divide by zero") {
-				//removeDateList = append(removeDateList, sk)
-				continue
-			}
-			err = errors.New("计算失败：Err:" + err.Error() + ";formulaStr:" + formulaFormStr)
-			fmt.Println(err)
-			return err
-		}
-		// 如果计算结果是NAN，那么就退出当前循环
-		if calResult.IsNan() {
-			continue
-		}
-
-		calVal, err := calResult.Float64()
+		//expression := formula.NewExpression(formulaFormStr)
+		//calResult, err := expression.Evaluate()
+		//if err != nil {
+		//	// 分母为0的报错
+		//	if strings.Contains(err.Error(), "divide by zero") {
+		//		//removeDateList = append(removeDateList, sk)
+		//		continue
+		//	}
+		//	err = errors.New("计算失败：Err:" + err.Error() + ";formulaStr:" + formulaFormStr)
+		//	fmt.Println(err)
+		//	return err
+		//}
+		//// 如果计算结果是NAN，那么就退出当前循环
+		//if calResult.IsNan() {
+		//	continue
+		//}
+		calVal, err := engine.ParseAndExec(formulaFormStr)
+		//calVal, err := calResult.Float64()
 		if err != nil {
 			err = errors.New("计算失败：获取计算值失败 Err:" + err.Error() + ";formulaStr:" + formulaFormStr)
 			fmt.Println(err)
@@ -490,7 +490,7 @@ func refreshAllCalculate(to orm.TxOrmer, edbInfoIdArr []*EdbInfo, edbInfoTag map
 		}
 		// 有计算出来值，那么就从待删除指标中移除
 		delete(removeDateMap, sk)
-		saveValue := decimal.NewFromFloat(calVal).RoundCeil(4).String() //utils.SubFloatToString(calVal, 4)
+		saveValue := decimal.NewFromFloat(calVal).Round(4).String() //utils.SubFloatToString(calVal, 4)
 		if existVal, ok := dataMap[sk]; !ok {
 			dataTime, _ := time.ParseInLocation(utils.FormatDate, sk, time.Local)
 			timestamp := dataTime.UnixNano() / 1e6
@@ -571,6 +571,25 @@ func ReplaceFormula(edbInfoIdArr []*EdbInfo, valArr, valArrMax map[int]float64,
 	if strings.Contains(formulaStr, "MAX()") || strings.Contains(formulaStr, "MIN()") {
 		return ""
 	}
+
+	formulaStr = strings.Replace(formulaStr, "MAX", "max", -1)
+	formulaStr = strings.Replace(formulaStr, "MIN", "min", -1)
+	formulaStr = strings.Replace(formulaStr, "LN", "ln", -1)
+	formulaStr = strings.Replace(formulaStr, "LOG", "log", -1)
+	formulaStr = strings.Replace(formulaStr, "SIN", "sin", -1)
+	formulaStr = strings.Replace(formulaStr, "COS", "cos", -1)
+	formulaStr = strings.Replace(formulaStr, "TAN", "tan", -1)
+	formulaStr = strings.Replace(formulaStr, "COT", "cot", -1)
+	formulaStr = strings.Replace(formulaStr, "SEC", "sec", -1)
+	formulaStr = strings.Replace(formulaStr, "CSC", "csc", -1)
+	formulaStr = strings.Replace(formulaStr, "ABS", "abs", -1)
+	formulaStr = strings.Replace(formulaStr, "CEIL", "ceil", -1)
+	formulaStr = strings.Replace(formulaStr, "FLOOR", "floor", -1)
+	formulaStr = strings.Replace(formulaStr, "ROUND", "round", -1)
+	formulaStr = strings.Replace(formulaStr, "SQRT", "sqrt", -1)
+	formulaStr = strings.Replace(formulaStr, "CBRT", "cbrt", -1)
+	formulaStr = strings.Replace(formulaStr, "NOERR", "noerr", -1)
+	formulaStr = strings.Replace(formulaStr, "DOUBLE", "double", -1)
 	if replaceCount == len(formulaMap) {
 		return formulaStr
 	} else {
@@ -683,6 +702,8 @@ type EdbInfoCalculateBatchEditReq struct {
 	Unit             string                         `description:"单位"`
 	UnitEn           string                         `description:"英文单位"`
 	ClassifyId       int                            `description:"分类id"`
+	AdminId          int                            `description:"操作人id"`
+	AdminName        string                         `description:"操作人姓名"`
 	Formula          string                         `description:"N值"`
 	EdbInfoId        int                            `description:"编辑指标id"`
 	FromEdbInfoId    int                            `description:"计算来源指标id"`
@@ -698,6 +719,14 @@ type EdbInfoCalculateBatchEditReq struct {
 	CalculateFormula string                         `description:"计算公式"`
 }
 
+// DeleteCalculateData 删除计算数据
+func DeleteCalculateData(edbInfoId int) (err error) {
+	o := orm.NewOrm()
+	sql := `DELETE FROM edb_data_calculate WHERE edb_info_id=?`
+	_, err = o.Raw(sql, edbInfoId).Exec()
+	return
+}
+
 // CheckFormula2 校验公式是否正常（比如说除法的分母不能为0之类的，实际上就是用预设的字段数据做一次计算）
 func CheckFormula2(edbInfoArr []*EdbInfo, formulaMap map[string]string, formulaStr string, edbInfoIdBytes []string) (ok bool, err error) {
 	valArr := make(map[int]float64)
@@ -712,12 +741,19 @@ func CheckFormula2(edbInfoArr []*EdbInfo, formulaMap map[string]string, formulaS
 	if formulaFormStr == "" {
 		return
 	}
-	expression := formula.NewExpression(formulaFormStr)
-	_, err = expression.Evaluate()
+
+	_, err = engine.ParseAndExec(formulaFormStr)
 	if err != nil {
+		fmt.Println(err)
 	} else {
 		ok = true
 	}
+	//expression := formula.NewExpression(formulaFormStr)
+	//_, err = expression.Evaluate()
+	//if err != nil {
+	//} else {
+	//	ok = true
+	//}
 	return
 }
 

models/base_from_edb_mapping.go

@@ -0,0 +1,162 @@
+package models
+
+import (
+	"eta/eta_index_lib/utils"
+	"fmt"
+	"github.com/beego/beego/v2/client/orm"
+	"strings"
+	"time"
+)
+
+// BaseFromEdbMapping 同花顺高频数据
+type BaseFromEdbMapping struct {
+	Id              int       `orm:"column(id);pk"`
+	BaseFromIndexId int       `description:"源指标ID"`
+	BaseIndexCode   string    `description:"源指标编码"`
+	EdbInfoId       int       `description:"指标ID"`
+	EdbCode         string    `description:"指标编码"`
+	Source          int       `description:"指标来源：1-同花顺..."`
+	SubSource       int       `description:"子数据来源：0-经济数据库；1-日期序列；2-高频数据"`
+	ConvertRule     string    `description:"转换规则"`
+	CreateTime      time.Time `description:"创建时间"`
+	ModifyTime      time.Time `description:"修改时间"`
+}
+
+func (m *BaseFromEdbMapping) TableName() string {
+	return "base_from_edb_mapping"
+}
+
+type BaseFromEdbMappingCols struct {
+	PrimaryId       string
+	BaseFromIndexId string
+	BaseIndexCode   string
+	EdbInfoId       string
+	EdbCode         string
+	Source          string
+	SubSource       string
+	ConvertRule     string
+	CreateTime      string
+	ModifyTime      string
+}
+
+func (m *BaseFromEdbMapping) Cols() BaseFromEdbMappingCols {
+	return BaseFromEdbMappingCols{
+		PrimaryId:       "id",
+		BaseFromIndexId: "base_from_index_id",
+		BaseIndexCode:   "base_index_code",
+		EdbInfoId:       "edb_info_id",
+		EdbCode:         "edb_code",
+		Source:          "source",
+		SubSource:       "sub_source",
+		ConvertRule:     "convert_rule",
+		CreateTime:      "create_time",
+		ModifyTime:      "modify_time",
+	}
+}
+
+func (m *BaseFromEdbMapping) Create() (err error) {
+	o := orm.NewOrm()
+	id, err := o.Insert(m)
+	if err != nil {
+		return
+	}
+	m.Id = int(id)
+	return
+}
+
+func (m *BaseFromEdbMapping) CreateMulti(items []*BaseFromEdbMapping) (err error) {
+	if len(items) == 0 {
+		return
+	}
+	o := orm.NewOrm()
+	_, err = o.InsertMulti(len(items), items)
+	return
+}
+
+func (m *BaseFromEdbMapping) Update(cols []string) (err error) {
+	o := orm.NewOrm()
+	_, err = o.Update(m, cols...)
+	return
+}
+
+func (m *BaseFromEdbMapping) Remove() (err error) {
+	o := orm.NewOrm()
+	sql := fmt.Sprintf(`DELETE FROM %s WHERE %s = ? LIMIT 1`, m.TableName(), m.Cols().PrimaryId)
+	_, err = o.Raw(sql, m.Id).Exec()
+	return
+}
+
+func (m *BaseFromEdbMapping) MultiRemove(ids []int) (err error) {
+	if len(ids) == 0 {
+		return
+	}
+	o := orm.NewOrm()
+	sql := fmt.Sprintf(`DELETE FROM %s WHERE %s IN (%s)`, m.TableName(), m.Cols().PrimaryId, utils.GetOrmInReplace(len(ids)))
+	_, err = o.Raw(sql, ids).Exec()
+	return
+}
+
+func (m *BaseFromEdbMapping) RemoveByCondition(condition string, pars []interface{}) (err error) {
+	if condition == "" {
+		return
+	}
+	o := orm.NewOrm()
+	sql := fmt.Sprintf(`DELETE FROM %s WHERE %s`, m.TableName(), condition)
+	_, err = o.Raw(sql, pars).Exec()
+	return
+}
+
+func (m *BaseFromEdbMapping) GetItemById(id int) (item *BaseFromEdbMapping, err error) {
+	o := orm.NewOrm()
+	sql := fmt.Sprintf(`SELECT * FROM %s WHERE %s = ? LIMIT 1`, m.TableName(), m.Cols().PrimaryId)
+	err = o.Raw(sql, id).QueryRow(&item)
+	return
+}
+
+func (m *BaseFromEdbMapping) GetItemByCondition(condition string, pars []interface{}, orderRule string) (item *BaseFromEdbMapping, err error) {
+	o := orm.NewOrm()
+	order := ``
+	if orderRule != "" {
+		order = ` ORDER BY ` + orderRule
+	}
+	sql := fmt.Sprintf(`SELECT * FROM %s WHERE 1=1 %s %s LIMIT 1`, m.TableName(), condition, order)
+	err = o.Raw(sql, pars).QueryRow(&item)
+	return
+}
+
+func (m *BaseFromEdbMapping) GetCountByCondition(condition string, pars []interface{}) (count int, err error) {
+	o := orm.NewOrm()
+	sql := fmt.Sprintf(`SELECT COUNT(1) FROM %s WHERE 1=1 %s`, m.TableName(), condition)
+	err = o.Raw(sql, pars).QueryRow(&count)
+	return
+}
+
+func (m *BaseFromEdbMapping) GetItemsByCondition(condition string, pars []interface{}, fieldArr []string, orderRule string) (items []*BaseFromEdbMapping, err error) {
+	o := orm.NewOrm()
+	fields := strings.Join(fieldArr, ",")
+	if len(fieldArr) == 0 {
+		fields = `*`
+	}
+	order := fmt.Sprintf(`ORDER BY %s DESC`, m.Cols().CreateTime)
+	if orderRule != "" {
+		order = ` ORDER BY ` + orderRule
+	}
+	sql := fmt.Sprintf(`SELECT %s FROM %s WHERE 1=1 %s %s`, fields, m.TableName(), condition, order)
+	_, err = o.Raw(sql, pars).QueryRows(&items)
+	return
+}
+
+func (m *BaseFromEdbMapping) GetPageItemsByCondition(condition string, pars []interface{}, fieldArr []string, orderRule string, startSize, pageSize int) (items []*BaseFromEdbMapping, err error) {
+	o := orm.NewOrm()
+	fields := strings.Join(fieldArr, ",")
+	if len(fieldArr) == 0 {
+		fields = `*`
+	}
+	order := fmt.Sprintf(`ORDER BY %s DESC`, m.Cols().CreateTime)
+	if orderRule != "" {
+		order = ` ORDER BY ` + orderRule
+	}
+	sql := fmt.Sprintf(`SELECT %s FROM %s WHERE 1=1 %s %s LIMIT ?,?`, fields, m.TableName(), condition, order)
+	_, err = o.Raw(sql, pars, startSize, pageSize).QueryRows(&items)
+	return
+}

models/base_from_mysteel_chemical.go

@@ -243,6 +243,21 @@ func (m *BaseFromMysteelChemicalIndex) GetIndexItem(indexCode string) (item *Bas
 	return
 }
 
+func (m *BaseFromMysteelChemicalIndex) GetBatchIndexItem(indexCodes []string) (items []*BaseFromMysteelChemicalIndex, err error) {
+	if len(indexCodes) <= 0 {
+		return
+	}
+	o := orm.NewOrm()
+	sql := `SELECT * FROM base_from_mysteel_chemical_index WHERE index_code IN (%s) `
+	holder := make([]string, 0, len(indexCodes))
+	for range indexCodes {
+		holder = append(holder, "?")
+	}
+	sql = fmt.Sprintf(sql, strings.Join(holder, ","))
+	_, err = o.Raw(sql, indexCodes).QueryRows(&items)
+	return
+}
+
 func (m *BaseFromMysteelChemicalIndex) GetIndexCreate(terminalCode string) (items []*BaseFromMysteelChemicalIndex, err error) {
 	o := orm.NewOrm()
 	sql := `SELECT * FROM base_from_mysteel_chemical_index WHERE index_name = '' AND terminal_code = ? `
@@ -445,6 +460,16 @@ func (r *BaseFromMysteelChemicalData) Add(list []BaseFromMysteelChemicalData) (e
 	return
 }
 
+// AddV2 新增
+func (r *BaseFromMysteelChemicalData) AddV2(list []*BaseFromMysteelChemicalData) (err error) {
+	if len(list) == 0 {
+		return
+	}
+	o := orm.NewOrm()
+	_, err = o.InsertMulti(500, list)
+	return
+}
+
 type AddMysteelIndexResp struct {
 	EdbCode                           string `description:"指标编码"`
 	TerminalCode                      string `description:"指标终端编码"`
@@ -536,3 +561,58 @@ func (m *BaseFromMysteelChemicalIndex) GetIndexByCondition(condition string, par
 	_, err = o.Raw(sql, pars).QueryRows(&items)
 	return
 }
+
+type BaseFromMysteelChemicalRecord struct {
+	BaseFromMysteelChemicalRecordId int64 `orm:"column(base_from_mysteel_chemical_record_id);pk"`
+	BaseFromMysteelChemicalIndexId  int64
+	OldIndexName                    string    `description:"原始名称"`
+	NewIndexName                    string    `description:"新的名称"`
+	CreateTime                      time.Time `description:"记录创建时间"`
+	Timestamp                       int64     `description:"记录创建时间戳"`
+}
+
+func (m *BaseFromMysteelChemicalRecord) AddBaseFromMysteelChemicalRecord() (err error) {
+	o := orm.NewOrm()
+	_, err = o.Insert(m)
+	return
+}
+
+type MySteelChemicalApiResp struct {
+	Code      string                    `json:"code" description:"200成功，其他失败"`
+	Success   bool                      `json:"success" description:"true 成功，false 失败"`
+	Timestamp int64                     `json:"timestamp" description:"时间戳"`
+	Message   string                    `json:"message" description:"显示执行信息"`
+	Data      []*MySteelChemicalApiData `json:"data" description:"数据"`
+}
+type MySteelChemicalApiInfoResp struct {
+	Code      string                  `json:"code" description:"200成功，其他失败"`
+	Success   bool                    `json:"success" description:"true 成功，false 失败"`
+	Timestamp int64                   `json:"timestamp" description:"时间戳"`
+	Message   string                  `json:"message" description:"显示执行信息"`
+	Data      *MySteelChemicalApiInfo `json:"data" description:"数据"`
+}
+
+type MySteelChemicalApiInfo struct {
+	Total int                           `json:"total" description:"总条数"`
+	Pages int                           `json:"pages" description:"总页数"`
+	List  []*MySteelChemicalApiInfoItem `json:"list" description:"数据列表"`
+}
+
+type MySteelChemicalApiInfoItem struct {
+	IndexCode     string `json:"INDEX_CODE"`
+	IndexName     string `json:"INDEX_NAME"`
+	FrequencyName string `json:"FREQUENCY_NAME"`
+	UnitName      string `json:"UNIT_NAME"`
+}
+
+type MySteelChemicalApiData struct {
+	IndexCode string                        `json:"INDEX_CODE"`
+	DataList  []*MySteelChemicalApiDataList `json:"dataList"`
+}
+
+type MySteelChemicalApiDataList struct {
+	PublishTime int64   `json:"PUBLISH_TIME"`
+	IndexCode   string  `json:"INDEX_CODE"`
+	DataDate    string  `json:"DATA_DATE"`
+	DataValue   float64 `json:"DATA_VALUE"`
+}

models/base_from_smm.go

@@ -612,3 +612,18 @@ func RefreshEdbDataFromSmmToEdb(edbInfoId int, edbCode, startDate string, smmDat
 	}
 	return
 }
+
+type BaseFromSmmRecord struct {
+	BaseFromSmmRecordId int64 `orm:"column(base_from_smm_record_id);pk"`
+	BaseFromSmmIndexId  int64
+	OldIndexName        string    `description:"原始名称"`
+	NewIndexName        string    `description:"新的名称"`
+	CreateTime          time.Time `description:"记录创建时间"`
+	Timestamp           int64     `description:"记录创建时间戳"`
+}
+
+func AddBaseFromSmmRecord(item *BaseFromSmmRecord) (err error) {
+	o := orm.NewOrm()
+	_, err = o.Insert(item)
+	return
+}

models/base_from_ths.go

@@ -1,6 +1,7 @@
 package models
 
 import (
+	"eta/eta_index_lib/services/alarm_msg"
 	"eta/eta_index_lib/utils"
 	"fmt"
 	"github.com/beego/beego/v2/client/orm"
@@ -26,7 +27,14 @@ type Tables struct {
 
 // 新增同花顺指标数据
 func AddEdbDataFromThs(edbCode string, item EdbDataFromThs) (err error) {
+	var errMsg string
 	o := orm.NewOrm()
+	defer func() {
+		if err != nil {
+			//go utils.SendEmail(utils.APP_NAME_CN+"【"+utils.RunMode+"】"+"失败提醒", " 同花顺数据获取失败:err:"+errMsg, utils.EmailSendToUsers)
+			go alarm_msg.SendAlarmMsg("同花顺数据获取失败:err:"+errMsg, 3)
+		}
+	}()
 
 	if len(item.Tables) > 0 {
 		table := item.Tables[0]
@@ -42,6 +50,7 @@ func AddEdbDataFromThs(edbCode string, item EdbDataFromThs) (err error) {
 			sValue := table.Value[i]
 			dataTime, err := time.ParseInLocation(utils.FormatDate, eDate, time.Local)
 			if err != nil {
+				errMsg = " time.Parse :" + err.Error()
 				return err
 			}
 			timestamp := dataTime.UnixNano() / 1e6
@@ -53,6 +62,7 @@ func AddEdbDataFromThs(edbCode string, item EdbDataFromThs) (err error) {
 			addSql = strings.TrimRight(addSql, ",")
 			_, err = o.Raw(addSql).Exec()
 			if err != nil {
+				errMsg = " tx.Exec Err :" + err.Error()
 				return
 			}
 		}

models/base_from_ths_hf.go

@@ -0,0 +1,476 @@
+package models
+
+import (
+	"eta/eta_index_lib/models/mgo"
+	"eta/eta_index_lib/utils"
+	"fmt"
+	"github.com/beego/beego/v2/client/orm"
+	"go.mongodb.org/mongo-driver/bson"
+	"strings"
+	"time"
+)
+
+var (
+	ThsHfPeriodArr = []int{1, 3, 5, 10, 15, 30, 60}                                                                                     // 时间周期
+	ThsHfCPSArr    = []string{"no", "forward1", "forward2", "forward3", "forward4", "backward1", "backward2", "backward3", "backward4"} // 复权方式
+	ThsHfFillArr   = []string{"Original", "Previous", "Blank"}                                                                          // 非交易间隔处理
+)
+
+// BaseFromThsHfIndex 同花顺高频数据
+type BaseFromThsHfIndex struct {
+	BaseFromThsHfIndexId    int       `orm:"column(base_from_ths_hf_index_id);pk"`
+	BaseFromThsHfClassifyId int       `description:"分类ID"`
+	IndexCode               string    `description:"指标编码"`
+	IndexName               string    `description:"指标名称"`
+	Unit                    string    `description:"单位"`
+	Source                  string    `description:"数据来源"`
+	Frequency               string    `description:"频度"`
+	StartDate               time.Time `description:"开始日期（至时分秒）"`
+	EndDate                 time.Time `description:"结束日期（至时分秒）"`
+	Describe                string    `description:"指标描述"`
+	Sort                    int       `description:"排序"`
+	IsStop                  int       `description:"是否停更：0-否；1-停更"`
+	TerminalCode            string    `description:"所属终端编码"`
+	StockCode               string    `description:"证券代码"`
+	Indicator               string    `description:"同花顺指标代码"`
+	ApiPars                 string    `description:"API请求参数"`
+	LatestValue             float64   `description:"最新值"`
+	SysUserId               int       `description:"创建人ID"`
+	SysUserRealName         string    `description:"创建人姓名"`
+	CreateTime              time.Time `description:"创建时间"`
+	ModifyTime              time.Time `description:"修改时间"`
+}
+
+func (m *BaseFromThsHfIndex) TableName() string {
+	return "base_from_ths_hf_index"
+}
+
+type BaseFromThsHfIndexCols struct {
+	PrimaryId               string
+	BaseFromThsHfClassifyId string
+	IndexCode               string
+	IndexName               string
+	Unit                    string
+	Source                  string
+	Frequency               string
+	StartDate               string
+	EndDate                 string
+	Describe                string
+	Sort                    string
+	IsStop                  string
+	TerminalCode            string
+	StockCode               string
+	Indicator               string
+	ApiPars                 string
+	LatestValue             string
+	SysUserId               string
+	SysUserRealName         string
+	CreateTime              string
+	ModifyTime              string
+}
+
+func (m *BaseFromThsHfIndex) Cols() BaseFromThsHfIndexCols {
+	return BaseFromThsHfIndexCols{
+		PrimaryId:               "base_from_ths_hf_index_id",
+		BaseFromThsHfClassifyId: "base_from_ths_hf_classify_id",
+		IndexCode:               "index_code",
+		IndexName:               "index_name",
+		Unit:                    "unit",
+		Source:                  "source",
+		Frequency:               "frequency",
+		StartDate:               "start_date",
+		EndDate:                 "end_date",
+		Describe:                "describe",
+		Sort:                    "sort",
+		IsStop:                  "is_stop",
+		TerminalCode:            "terminal_code",
+		StockCode:               "stock_code",
+		Indicator:               "indicator",
+		ApiPars:                 "api_pars",
+		LatestValue:             "latest_value",
+		SysUserId:               "sys_user_id",
+		SysUserRealName:         "sys_user_real_name",
+		CreateTime:              "create_time",
+		ModifyTime:              "modify_time",
+	}
+}
+
+func (m *BaseFromThsHfIndex) Create() (err error) {
+	o := orm.NewOrm()
+	id, err := o.Insert(m)
+	if err != nil {
+		return
+	}
+	m.BaseFromThsHfIndexId = int(id)
+	return
+}
+
+func (m *BaseFromThsHfIndex) CreateMulti(items []*BaseFromThsHfIndex) (err error) {
+	if len(items) == 0 {
+		return
+	}
+	o := orm.NewOrm()
+	_, err = o.InsertMulti(len(items), items)
+	return
+}
+
+func (m *BaseFromThsHfIndex) Update(cols []string) (err error) {
+	o := orm.NewOrm()
+	_, err = o.Update(m, cols...)
+	return
+}
+
+func (m *BaseFromThsHfIndex) Remove() (err error) {
+	o := orm.NewOrm()
+	sql := fmt.Sprintf(`DELETE FROM %s WHERE %s = ? LIMIT 1`, m.TableName(), m.Cols().PrimaryId)
+	_, err = o.Raw(sql, m.BaseFromThsHfIndexId).Exec()
+	return
+}
+
+func (m *BaseFromThsHfIndex) MultiRemove(ids []int) (err error) {
+	if len(ids) == 0 {
+		return
+	}
+	o := orm.NewOrm()
+	sql := fmt.Sprintf(`DELETE FROM %s WHERE %s IN (%s)`, m.TableName(), m.Cols().PrimaryId, utils.GetOrmInReplace(len(ids)))
+	_, err = o.Raw(sql, ids).Exec()
+	return
+}
+
+func (m *BaseFromThsHfIndex) RemoveByCondition(condition string, pars []interface{}) (err error) {
+	if condition == "" {
+		return
+	}
+	o := orm.NewOrm()
+	sql := fmt.Sprintf(`DELETE FROM %s WHERE %s`, m.TableName(), condition)
+	_, err = o.Raw(sql, pars).Exec()
+	return
+}
+
+func (m *BaseFromThsHfIndex) GetItemById(id int) (item *BaseFromThsHfIndex, err error) {
+	o := orm.NewOrm()
+	sql := fmt.Sprintf(`SELECT * FROM %s WHERE %s = ? LIMIT 1`, m.TableName(), m.Cols().PrimaryId)
+	err = o.Raw(sql, id).QueryRow(&item)
+	return
+}
+
+func (m *BaseFromThsHfIndex) GetItemByCondition(condition string, pars []interface{}, orderRule string) (item *BaseFromThsHfIndex, err error) {
+	o := orm.NewOrm()
+	order := ``
+	if orderRule != "" {
+		order = ` ORDER BY ` + orderRule
+	}
+	sql := fmt.Sprintf(`SELECT * FROM %s WHERE 1=1 %s %s LIMIT 1`, m.TableName(), condition, order)
+	err = o.Raw(sql, pars).QueryRow(&item)
+	return
+}
+
+func (m *BaseFromThsHfIndex) GetCountByCondition(condition string, pars []interface{}) (count int, err error) {
+	o := orm.NewOrm()
+	sql := fmt.Sprintf(`SELECT COUNT(1) FROM %s WHERE 1=1 %s`, m.TableName(), condition)
+	err = o.Raw(sql, pars).QueryRow(&count)
+	return
+}
+
+func (m *BaseFromThsHfIndex) GetItemsByCondition(condition string, pars []interface{}, fieldArr []string, orderRule string) (items []*BaseFromThsHfIndex, err error) {
+	o := orm.NewOrm()
+	fields := strings.Join(fieldArr, ",")
+	if len(fieldArr) == 0 {
+		fields = `*`
+	}
+	order := fmt.Sprintf(`ORDER BY %s DESC`, m.Cols().CreateTime)
+	if orderRule != "" {
+		order = ` ORDER BY ` + orderRule
+	}
+	sql := fmt.Sprintf(`SELECT %s FROM %s WHERE 1=1 %s %s`, fields, m.TableName(), condition, order)
+	_, err = o.Raw(sql, pars).QueryRows(&items)
+	return
+}
+
+func (m *BaseFromThsHfIndex) GetPageItemsByCondition(condition string, pars []interface{}, fieldArr []string, orderRule string, startSize, pageSize int) (items []*BaseFromThsHfIndex, err error) {
+	o := orm.NewOrm()
+	fields := strings.Join(fieldArr, ",")
+	if len(fieldArr) == 0 {
+		fields = `*`
+	}
+	order := fmt.Sprintf(`ORDER BY %s DESC`, m.Cols().CreateTime)
+	if orderRule != "" {
+		order = ` ORDER BY ` + orderRule
+	}
+	sql := fmt.Sprintf(`SELECT %s FROM %s WHERE 1=1 %s %s LIMIT ?,?`, fields, m.TableName(), condition, order)
+	_, err = o.Raw(sql, pars, startSize, pageSize).QueryRows(&items)
+	return
+}
+
+// BaseFromThsHfIndexItem 同花顺高频数据信息
+type BaseFromThsHfIndexItem struct {
+	IndexId    int    `description:"同花顺高频数据ID"`
+	IndexCode  string `description:"指标编码"`
+	IndexName  string `description:"指标名称"`
+	Unit       string `description:"单位"`
+	Source     string `description:"数据来源"`
+	Frequency  string `description:"频度"`
+	StartDate  string `description:"开始日期（至时分秒）"`
+	EndDate    string `description:"结束日期（至时分秒）"`
+	Describe   string `description:"指标描述"`
+	Sort       int    `description:"排序"`
+	CreateTime string `description:"创建时间"`
+	ModifyTime string `description:"修改时间"`
+}
+
+func (m *BaseFromThsHfIndex) Format2Item() (item *BaseFromThsHfIndexItem) {
+	item = new(BaseFromThsHfIndexItem)
+	item.IndexId = m.BaseFromThsHfIndexId
+	item.IndexCode = m.IndexCode
+	item.IndexName = m.IndexName
+	item.Unit = m.Unit
+	item.Source = m.Source
+	item.Frequency = m.Frequency
+	item.StartDate = utils.TimeTransferString(utils.FormatDateTime, m.StartDate)
+	item.EndDate = utils.TimeTransferString(utils.FormatDateTime, m.EndDate)
+	item.Describe = m.Describe
+	item.Sort = m.Sort
+	item.CreateTime = utils.TimeTransferString(utils.FormatDateTime, m.CreateTime)
+	item.ModifyTime = utils.TimeTransferString(utils.FormatDateTime, m.ModifyTime)
+	return
+}
+
+// ThsHfSearchEdbReq 搜索指标请求体
+type ThsHfSearchEdbReq struct {
+	StockCode string `form:"StockCode" description:"证券代码" `
+	EdbCode   string `form:"EdbCode" description:"指标代码"`
+	StartTime string `form:"StartTime" description:"每日数据开始时间"`
+	EndTime   string `form:"EndTime" description:"每日数据结束时间"`
+	Interval  int    `form:"Interval" description:"时间周期"`
+	Fill      string `form:"Fill" description:"非交易间隔处理"`
+	CPS       string `form:"CPS" description:"复权方式"`
+	BaseDate  string `form:"BaseDate" description:"复权基点"`
+}
+
+// ThsHfApiResp 同花顺接口响应
+type ThsHfApiResp struct {
+	ErrorCode int    `json:"errorcode"`
+	ErrMsg    string `json:"errmsg"`
+	Tables    []struct {
+		ThsCode string               `json:"thscode"`
+		Time    []string             `json:"time"`
+		Table   map[string][]float64 `json:"table"`
+	} `json:"tables"`
+}
+
+// ThsHfAppResp 同花顺公用机响应
+type ThsHfAppResp struct {
+	ErrorCode int    `json:"errorcode"`
+	ErrMsg    string `json:"errmsg"`
+	//Data      []interface{} `json:"data"`
+	Data []struct {
+		Time    string  `json:"time"`
+		ThsCode string  `json:"thscode"`
+		Open    float64 `json:"open"`
+		Close   float64 `json:"close"`
+	} `json:"data"`
+}
+
+// ThsHfIndexWithData 同花顺高频指标
+type ThsHfIndexWithData struct {
+	StockCode string            `description:"证券代码"`
+	EdbCode   string            `description:"指标代码"`
+	IndexData []*ThsHfIndexData `description:"指标数据"`
+}
+
+// ThsHfIndexData 同花顺高频指标数据
+type ThsHfIndexData struct {
+	DataTime time.Time `description:"数据时间(2006-01-02 15:04)"`
+	Value    float64   `description:"数据值"`
+}
+
+type ThsHfBaseAddIndexItem struct {
+	ClassifyId int    `description:"分类ID"`
+	Unit       string `description:"单位"`
+	IndexName  string `description:"指标名称"`
+	Frequency  string `description:"频度"`
+	StockCode  string `description:"证券代码"`
+	EdbCode    string `description:"指标代码"`
+}
+
+// ThsHfBaseAddReq 新增至数据源请求
+type ThsHfBaseAddReq struct {
+	StartTime             string `description:"每日数据开始时间"`
+	EndTime               string `description:"每日数据结束时间"`
+	Interval              int    `description:"时间周期"`
+	Fill                  string `description:"非交易间隔处理"`
+	CPS                   string `description:"复权方式"`
+	BaseDate              string `description:"复权基点"`
+	SysAdminId            int    `description:"创建人ID"`
+	SysAdminName          string `description:"创建人姓名"`
+	ThsHfBaseAddIndexItem `description:"指标信息"`
+}
+
+// CreateIndexAndData 新增指标和数据
+//func (m *BaseFromThsHfIndex) CreateIndexAndData(indexItem *BaseFromThsHfIndex, indexData []*BaseFromThsHfData) (err error) {
+//	o := orm.NewOrm()
+//	tx, e := o.Begin()
+//	if e != nil {
+//		err = fmt.Errorf("tx begin err: %v", e)
+//		return
+//	}
+//	defer func() {
+//		if err != nil {
+//			_ = tx.Rollback()
+//			return
+//		}
+//		_ = tx.Commit()
+//	}()
+//
+//	lastId, e := tx.Insert(indexItem)
+//	if e != nil {
+//		err = fmt.Errorf("insert index err: %v", e)
+//		return
+//	}
+//	indexId := int(lastId)
+//	indexItem.BaseFromThsHfIndexId = indexId
+//
+//	if len(indexData) == 0 {
+//		return
+//	}
+//	for _, v := range indexData {
+//		v.BaseFromThsHfIndexId = indexId
+//	}
+//	if _, e = tx.InsertMulti(200, indexData); e != nil {
+//		err = fmt.Errorf("insert index data err: %v", e)
+//		return
+//	}
+//	return
+//}
+
+// ThsHfBaseRefreshReq 数据源刷新请求
+type ThsHfBaseRefreshReq struct {
+	BaseIndexCode string `description:"源指标编码"`
+	RefreshType   int    `description:"刷新类型: 1-最近6小时; 2-全部刷新"`
+}
+
+// ThsHfEdbAddReq 新增至指标库
+type ThsHfEdbAddReq struct {
+	ConvertRule ThsHfIndexConvert2EdbRule
+	NewIndex    *ThsHfIndexMultiSave2EdbPreItem `description:"新增指标"`
+}
+
+type ThsHfIndexConvert2EdbRule struct {
+	ConvertType  int `description:"转换类型: 1-指定时间值; 2-区间计算值"`
+	ConvertFixed struct {
+		FixedDay  int    `description:"指定时间值日期: 1-当日; 2-前一日"`
+		FixedTime string `description:"指定时间值时点(HH:mm:ss)"`
+	} `description:"指定时间值"`
+	ConvertArea struct {
+		StartDay      int    `description:"起始时间日期: 1-当日; 2-前一日"`
+		StartTime     string `description:"起始时间时点(HH:mm:ss)"`
+		EndDay        int    `description:"截止时间日期: 1-当日; 2-前一日"`
+		EndTime       string `description:"截止时间时点(HH:mm:ss)"`
+		CalculateType int    `description:"计算类型: 1-区间均值; 2-最大值; 3-最小值"`
+	} `description:"区间计算值"`
+}
+
+// ThsHfIndexMultiSave2EdbPreItem 批量新增指标库信息
+type ThsHfIndexMultiSave2EdbPreItem struct {
+	IndexId      int    `description:"指标ID"`
+	IndexCode    string `description:"指标编码"`
+	IndexName    string `description:"原指标名称"`
+	NewIndexName string `description:"新指标名称"`
+	StockCode    string `description:"证券代码"`
+	EdbCode      string `description:"指标代码"`
+	Unit         string `description:"单位"`
+	Frequency    string `description:"原频度"`
+	NewFrequency string `description:"新频度(固定日度)"`
+	ClassifyId   int    `description:"指标库分类ID"`
+	SysAdminId   int    `description:"创建人ID"`
+	SysAdminName string `description:"创建人姓名"`
+	Tips         string `description:"提示信息"`
+	ErrMsg       string `description:"错误信息"`
+}
+
+// GetEdbInfoMaxAndMinInfo
+// @Description: 获取指标的最新数据记录信息
+// @author: Roc
+// @receiver m
+// @datetime 2024-07-02 14:50:50
+// @param edbCode string
+// @return item *EdbInfoMaxAndMinInfo
+// @return err error
+func (m BaseFromThsHfIndex) GetEdbInfoMaxAndMinInfo(edbCode string) (item *EdbInfoMaxAndMinInfo, err error) {
+	if utils.UseMongo {
+		return m.getEdbInfoMaxAndMinInfoByMongo(edbCode)
+	}
+	return m.getEdbInfoMaxAndMinInfoByMysql(edbCode)
+}
+
+// getEdbInfoMaxAndMinInfoByMongo
+// @Description: 获取指标的最新数据记录信息（从mongo中获取）
+// @author: Roc
+// @receiver m
+// @datetime 2024-07-02 14:41:20
+// @param edbCode string
+// @return item *EdbInfoMaxAndMinInfo
+// @return err error
+func (m BaseFromThsHfIndex) getEdbInfoMaxAndMinInfoByMongo(edbCode string) (item *EdbInfoMaxAndMinInfo, err error) {
+	mogDataObj := new(mgo.BaseFromThsHfData)
+	pipeline := []bson.M{
+		{"$match": bson.M{"index_code": edbCode}},
+		{"$group": bson.M{
+			"_id":       nil,
+			"min_date":  bson.M{"$min": "$data_time"},
+			"max_date":  bson.M{"$max": "$data_time"},
+			"min_value": bson.M{"$min": "$value"},
+			"max_value": bson.M{"$max": "$value"},
+		}},
+		{"$project": bson.M{"_id": 0}}, // 可选，如果不需要_id字段
+	}
+	result, err := mogDataObj.GetEdbInfoMaxAndMinInfo(pipeline)
+	if err != nil {
+		fmt.Println("BaseFromThsHfIndex GetEdbInfoMaxAndMinInfo Err:" + err.Error())
+		return
+	}
+
+	if !result.MaxDate.IsZero() {
+		whereQuery := bson.M{"index_code": edbCode, "data_time": result.MaxDate}
+		selectParam := bson.D{{"value", 1}, {"_id", 0}}
+		latestValue, tmpErr := mogDataObj.GetLatestValue(whereQuery, selectParam)
+		if tmpErr != nil {
+			err = tmpErr
+			return
+		}
+		result.LatestValue = latestValue.Value
+		result.EndValue = latestValue.Value
+	}
+
+	item = &EdbInfoMaxAndMinInfo{
+		MinDate:     result.MinDate.Format(utils.FormatDateTime),
+		MaxDate:     result.MaxDate.Format(utils.FormatDateTime),
+		MinValue:    result.MinValue,
+		MaxValue:    result.MaxValue,
+		LatestValue: result.LatestValue,
+		LatestDate:  result.LatestDate.Format(utils.FormatDateTime),
+		EndValue:    result.EndValue,
+	}
+	return
+}
+
+// getEdbInfoMaxAndMinInfoByMysql
+// @Description: 获取指标的最新数据记录信息（从mysql中获取）
+func (m BaseFromThsHfIndex) getEdbInfoMaxAndMinInfoByMysql(edbCode string) (item *EdbInfoMaxAndMinInfo, err error) {
+	dataObj := BaseFromThsHfData{}
+	result, err := dataObj.GetIndexMinMax(edbCode)
+	if err != nil {
+		return
+	}
+
+	item = &EdbInfoMaxAndMinInfo{
+		MinDate:     result.MinDate,
+		MaxDate:     result.MaxDate,
+		MinValue:    result.MinValue,
+		MaxValue:    result.MaxValue,
+		LatestValue: result.LatestValue,
+		LatestDate:  result.LatestDate,
+		EndValue:    result.EndValue,
+	}
+	return
+}

models/base_from_ths_hf_data.go

@@ -0,0 +1,229 @@
+package models
+
+import (
+	"eta/eta_index_lib/utils"
+	"fmt"
+	"github.com/beego/beego/v2/client/orm"
+	"strings"
+	"time"
+)
+
+// BaseFromThsHfData 同花顺高频数据-指标数据
+type BaseFromThsHfData struct {
+	BaseFromThsHfDataId  int       `orm:"column(base_from_ths_hf_data_id);pk"`
+	BaseFromThsHfIndexId int       `description:"指标ID"`
+	IndexCode            string    `description:"指标编码"`
+	DataTime             time.Time `description:"数据日期（至时分秒）"`
+	Value                float64   `description:"数据值"`
+	UniqueCode           string    `description:"唯一编码"`
+	CreateTime           time.Time `description:"创建时间"`
+	ModifyTime           time.Time `description:"修改时间"`
+	DataTimestamp        int64     `description:"数据日期时间戳"`
+}
+
+func (m *BaseFromThsHfData) TableName() string {
+	return "base_from_ths_hf_data"
+}
+
+type BaseFromThsHfDataCols struct {
+	PrimaryId            string
+	BaseFromThsHfIndexId string
+	IndexCode            string
+	DataTime             string
+	Value                string
+	UniqueCode           string
+	CreateTime           string
+	ModifyTime           string
+	DataTimestamp        string
+}
+
+func (m *BaseFromThsHfData) Cols() BaseFromThsHfDataCols {
+	return BaseFromThsHfDataCols{
+		PrimaryId:            "base_from_ths_hf_data_id",
+		BaseFromThsHfIndexId: "base_from_ths_hf_index_id",
+		IndexCode:            "index_code",
+		DataTime:             "data_time",
+		Value:                "value",
+		UniqueCode:           "unique_code",
+		CreateTime:           "create_time",
+		ModifyTime:           "modify_time",
+		DataTimestamp:        "data_timestamp",
+	}
+}
+
+func (m *BaseFromThsHfData) Create() (err error) {
+	o := orm.NewOrm()
+	id, err := o.Insert(m)
+	if err != nil {
+		return
+	}
+	m.BaseFromThsHfDataId = int(id)
+	return
+}
+
+func (m *BaseFromThsHfData) CreateMulti(items []*BaseFromThsHfData) (err error) {
+	if len(items) == 0 {
+		return
+	}
+	o := orm.NewOrm()
+	_, err = o.InsertMulti(500, items)
+	return
+}
+
+func (m *BaseFromThsHfData) Update(cols []string) (err error) {
+	o := orm.NewOrm()
+	_, err = o.Update(m, cols...)
+	return
+}
+
+func (m *BaseFromThsHfData) Remove() (err error) {
+	o := orm.NewOrm()
+	sql := fmt.Sprintf(`DELETE FROM %s WHERE %s = ? LIMIT 1`, m.TableName(), m.Cols().PrimaryId)
+	_, err = o.Raw(sql, m.BaseFromThsHfDataId).Exec()
+	return
+}
+
+func (m *BaseFromThsHfData) MultiRemove(ids []int) (err error) {
+	if len(ids) == 0 {
+		return
+	}
+	o := orm.NewOrm()
+	sql := fmt.Sprintf(`DELETE FROM %s WHERE %s IN (%s)`, m.TableName(), m.Cols().PrimaryId, utils.GetOrmInReplace(len(ids)))
+	_, err = o.Raw(sql, ids).Exec()
+	return
+}
+
+func (m *BaseFromThsHfData) RemoveByCondition(condition string, pars []interface{}) (err error) {
+	if condition == "" {
+		return
+	}
+	o := orm.NewOrm()
+	sql := fmt.Sprintf(`DELETE FROM %s WHERE %s`, m.TableName(), condition)
+	_, err = o.Raw(sql, pars).Exec()
+	return
+}
+
+func (m *BaseFromThsHfData) GetItemById(id int) (item *BaseFromThsHfData, err error) {
+	o := orm.NewOrm()
+	sql := fmt.Sprintf(`SELECT * FROM %s WHERE %s = ? LIMIT 1`, m.TableName(), m.Cols().PrimaryId)
+	err = o.Raw(sql, id).QueryRow(&item)
+	return
+}
+
+func (m *BaseFromThsHfData) GetItemByCondition(condition string, pars []interface{}, orderRule string) (item *BaseFromThsHfData, err error) {
+	o := orm.NewOrm()
+	order := ``
+	if orderRule != "" {
+		order = ` ORDER BY ` + orderRule
+	}
+	sql := fmt.Sprintf(`SELECT * FROM %s WHERE 1=1 %s %s LIMIT 1`, m.TableName(), condition, order)
+	err = o.Raw(sql, pars).QueryRow(&item)
+	return
+}
+
+func (m *BaseFromThsHfData) GetCountByCondition(condition string, pars []interface{}) (count int, err error) {
+	o := orm.NewOrm()
+	sql := fmt.Sprintf(`SELECT COUNT(1) FROM %s WHERE 1=1 %s`, m.TableName(), condition)
+	err = o.Raw(sql, pars).QueryRow(&count)
+	return
+}
+
+func (m *BaseFromThsHfData) GetItemsByCondition(condition string, pars []interface{}, fieldArr []string, orderRule string) (items []*BaseFromThsHfData, err error) {
+	o := orm.NewOrm()
+	fields := strings.Join(fieldArr, ",")
+	if len(fieldArr) == 0 {
+		fields = `*`
+	}
+	order := fmt.Sprintf(`ORDER BY %s DESC`, m.Cols().CreateTime)
+	if orderRule != "" {
+		order = ` ORDER BY ` + orderRule
+	}
+	sql := fmt.Sprintf(`SELECT %s FROM %s WHERE 1=1 %s %s`, fields, m.TableName(), condition, order)
+	_, err = o.Raw(sql, pars).QueryRows(&items)
+	return
+}
+
+func (m *BaseFromThsHfData) GetPageItemsByCondition(condition string, pars []interface{}, fieldArr []string, orderRule string, startSize, pageSize int) (items []*BaseFromThsHfData, err error) {
+	o := orm.NewOrm()
+	fields := strings.Join(fieldArr, ",")
+	if len(fieldArr) == 0 {
+		fields = `*`
+	}
+	order := fmt.Sprintf(`ORDER BY %s DESC`, m.Cols().CreateTime)
+	if orderRule != "" {
+		order = ` ORDER BY ` + orderRule
+	}
+	sql := fmt.Sprintf(`SELECT %s FROM %s WHERE 1=1 %s %s LIMIT ?,?`, fields, m.TableName(), condition, order)
+	_, err = o.Raw(sql, pars, startSize, pageSize).QueryRows(&items)
+	return
+}
+
+// BaseFromThsHfDataItem 同花顺高频数据信息
+type BaseFromThsHfDataItem struct {
+	DataId     int     `description:"数据ID"`
+	IndexId    int     `description:"指标ID"`
+	IndexCode  string  `description:"指标编码"`
+	DataTime   string  `description:"数据日期（至时分秒）"`
+	Value      float64 `description:"数据值"`
+	UniqueCode string  `description:"唯一编码"`
+}
+
+func (m *BaseFromThsHfData) Format2Item() (item *BaseFromThsHfDataItem) {
+	item = new(BaseFromThsHfDataItem)
+	item.DataId = m.BaseFromThsHfDataId
+	item.IndexId = m.BaseFromThsHfIndexId
+	item.IndexCode = m.IndexCode
+	item.DataTime = utils.TimeTransferString(utils.FormatDateTime, m.DataTime)
+	item.Value = m.Value
+	item.UniqueCode = m.UniqueCode
+	return
+}
+
+func (m *BaseFromThsHfData) MultiInsertOrUpdate(inserts, updates []*BaseFromThsHfData) (err error) {
+	o := orm.NewOrm()
+	if len(inserts) > 0 {
+		_, e := o.InsertMulti(600, inserts)
+		if e != nil {
+			err = fmt.Errorf("insert multi err: %s", e.Error())
+			return
+		}
+	}
+	if len(updates) > 0 {
+		sql := fmt.Sprintf("UPDATE %s SET %s = ?, modify_time = NOW() WHERE %s = ?", m.TableName(), m.Cols().Value, m.Cols().UniqueCode)
+		p, e := o.Raw(sql).Prepare()
+		if e != nil {
+			err = fmt.Errorf("prepare err: %s", e.Error())
+			return
+		}
+		defer func() {
+			_ = p.Close()
+		}()
+		for _, v := range updates {
+			_, e = p.Exec(v.Value, v.UniqueCode)
+			if e != nil {
+				err = fmt.Errorf("update err: %s", e.Error())
+				return
+			}
+		}
+	}
+	return
+}
+
+func (m *BaseFromThsHfData) GetIndexMinMax(indexCode string) (item *EdbInfoMaxAndMinInfo, err error) {
+	o := orm.NewOrm()
+	sql := fmt.Sprintf(`SELECT MIN(%s) AS min_date, MAX(%s) AS max_date, MIN(%s) AS min_value,MAX(%s) AS max_value FROM %s WHERE %s = ?`, m.Cols().DataTime, m.Cols().DataTime, m.Cols().Value, m.Cols().Value, m.TableName(), m.Cols().IndexCode)
+	err = o.Raw(sql, indexCode).QueryRow(&item)
+	if err != nil {
+		return
+	}
+
+	// 获取最新值
+	var lastVal float64
+	sql = fmt.Sprintf(`SELECT %s AS latest_value FROM %s WHERE %s = ? ORDER BY %s DESC LIMIT 1`, m.Cols().Value, m.TableName(), m.Cols().IndexCode, m.Cols().DataTime)
+	err = o.Raw(sql, indexCode).QueryRow(&lastVal)
+	if err != nil {
+		return
+	}
+	item.LatestValue = lastVal
+	return
+}

models/base_predict_from_calculate.go

@@ -303,6 +303,15 @@ func refreshAllPredictCalculate(to orm.TxOrmer, edbInfoIdList []*EdbInfo, edbInf
 		}
 		if maxStartDate.IsZero() || maxStartDate.Before(tmpStartDate) {
 			maxStartDate = tmpStartDate
+		}
+		tmpEndDate, tmpErr := time.ParseInLocation(utils.FormatDate, v.EndDate, time.Local)
+		if tmpErr != nil {
+			err = errors.New(`最近的日期格式化失败;日期：` + v.EndDate + `;err:` + tmpErr.Error())
+			return
+		}
+		if minEndDate.IsZero() || tmpEndDate.Before(minEndDate) {
+			minEndDate = tmpEndDate
+		}
 		}*/
 
 		// 获取关联指标数据

models/business_conf.go

@@ -0,0 +1,238 @@
+package models
+
+import (
+	"eta/eta_index_lib/utils"
+	"fmt"
+	"github.com/beego/beego/v2/client/orm"
+	"html"
+	"strings"
+	"time"
+)
+
+const (
+	BusinessConfUseXf                     = "UseXf"
+	BusinessConfXfAppid                   = "XfAppid"
+	BusinessConfXfApiKey                  = "XfApiKey"
+	BusinessConfXfApiSecret               = "XfApiSecret"
+	BusinessConfXfVcn                     = "XfVcn"
+	BusinessConfEnPptCoverImgs            = "EnPptCoverImgs"
+	BusinessConfIsReportApprove           = "IsReportApprove"
+	BusinessConfReportApproveType         = "ReportApproveType"
+	BusinessConfCompanyName               = "CompanyName"
+	BusinessConfCompanyWatermark          = "CompanyWatermark"
+	BusinessConfWatermarkChart            = "WatermarkChart"
+	BusinessConfLoginSmsTpId              = "LoginSmsTpId"
+	BusinessConfLoginSmsGjTpId            = "LoginSmsGjTpId"
+	BusinessConfSmsJhgnAppKey             = "SmsJhgnAppKey"
+	BusinessConfSmsJhgjAppKey             = "SmsJhgjAppKey"
+	BusinessConfLdapHost                  = "LdapHost"
+	BusinessConfLdapBase                  = "LdapBase"
+	BusinessConfLdapPort                  = "LdapPort"
+	BusinessConfEmailClient               = "EmailClient"
+	BusinessConfEmailServerHost           = "EmailServerHost"
+	BusinessConfEmailServerPort           = "EmailServerPort"
+	BusinessConfEmailSender               = "EmailSender"
+	BusinessConfEmailSenderUserName       = "EmailSenderUserName"
+	BusinessConfEmailSenderPassword       = "EmailSenderPassword"
+	BusinessConfSmsClient                 = "SmsClient"
+	BusinessConfNanHuaSmsAppKey           = "NanHuaSmsAppKey"
+	BusinessConfNanHuaSmsAppSecret        = "NanHuaSmsAppSecret"
+	BusinessConfNanHuaSmsApiHost          = "NanHuaSmsApiHost"
+	BusinessConfLoginSmsTplContent        = "LoginSmsTplContent"
+	BusinessConfLoginEmailTemplateSubject = "LoginEmailTemplateSubject"
+	BusinessConfLoginEmailTemplateContent = "LoginEmailTemplateContent"
+	BusinessConfLdapBindUserSuffix        = "LdapBindUserSuffix"
+	BusinessConfLdapUserFilter            = "LdapUserFilter"
+
+	BusinessConfTencentApiSecretId           = "TencentApiSecretId"           // 腾讯云API-密钥对
+	BusinessConfTencentApiSecretKey          = "TencentApiSecretKey"          // 腾讯云API-密钥对
+	BusinessConfTencentApiRecTaskCallbackUrl = "TencentApiRecTaskCallbackUrl" // 腾讯云API-语音识别回调地址
+	BusinessConfSmsJhgjVariable              = "SmsJhgjVariable"              // 聚合国际短信变量
+)
+
+const (
+	BusinessConfReportApproveTypeEta   = "eta"
+	BusinessConfReportApproveTypeOther = "other"
+	BusinessConfClientFlagNanHua       = "nhqh" // 南华标记
+	BusinessConfEmailClientSmtp        = "smtp" // 普通邮箱标记
+)
+
+// FromSceneMap 数据源名称与数据源ID的对应关系
+var FromSceneMap = map[int]string{
+	1: "SmartReportSheetSize",
+	2: "ReportSheetSize",
+	3: "EnReportSheetSize",
+	4: "CnPptSheetSize",
+	5: "EnPptSheetSize",
+}
+
+// BusinessConf 商户配置表
+type BusinessConf struct {
+	Id         int    `orm:"column(id);pk"`
+	ConfKey    string `description:"配置Key"`
+	ConfVal    string `description:"配置值"`
+	ValType    int    `description:"1-字符串；2-数值；3-字符串数组；4-富文本；"`
+	Necessary  int    `description:"是否必填：0-否；1-是"`
+	Remark     string `description:"备注"`
+	CreateTime time.Time
+}
+
+func (m *BusinessConf) TableName() string {
+	return "business_conf"
+}
+
+func (m *BusinessConf) PrimaryId() string {
+	return "id"
+}
+
+func (m *BusinessConf) Create() (err error) {
+	o := orm.NewOrmUsingDB("master")
+	id, err := o.Insert(m)
+	if err != nil {
+		return
+	}
+	m.Id = int(id)
+	return
+}
+
+func (m *BusinessConf) CreateMulti(items []*BusinessConf) (err error) {
+	if len(items) == 0 {
+		return
+	}
+	o := orm.NewOrmUsingDB("master")
+	_, err = o.InsertMulti(len(items), items)
+	return
+}
+
+func (m *BusinessConf) Update(cols []string) (err error) {
+	o := orm.NewOrmUsingDB("master")
+	_, err = o.Update(m, cols...)
+	return
+}
+
+func (m *BusinessConf) Del() (err error) {
+	o := orm.NewOrmUsingDB("master")
+	sql := fmt.Sprintf(`DELETE FROM %s WHERE %s = ? LIMIT 1`, m.TableName(), m.PrimaryId())
+	_, err = o.Raw(sql, m.Id).Exec()
+	return
+}
+
+func (m *BusinessConf) GetItemById(id int) (item *BusinessConf, err error) {
+	o := orm.NewOrmUsingDB("master")
+	sql := fmt.Sprintf(`SELECT * FROM %s WHERE %s = ? LIMIT 1`, m.TableName(), m.PrimaryId())
+	err = o.Raw(sql, id).QueryRow(&item)
+	return
+}
+
+func (m *BusinessConf) GetItemByCondition(condition string, pars []interface{}) (item *BusinessConf, err error) {
+	o := orm.NewOrmUsingDB("master")
+	sql := fmt.Sprintf(`SELECT * FROM %s WHERE 1=1 %s LIMIT 1`, m.TableName(), condition)
+	err = o.Raw(sql, pars).QueryRow(&item)
+	return
+}
+
+func (m *BusinessConf) GetCountByCondition(condition string, pars []interface{}) (count int, err error) {
+	o := orm.NewOrmUsingDB("master")
+	sql := fmt.Sprintf(`SELECT COUNT(1) FROM %s WHERE 1=1 %s`, m.TableName(), condition)
+	err = o.Raw(sql, pars).QueryRow(&count)
+	return
+}
+
+func (m *BusinessConf) GetItemsByCondition(condition string, pars []interface{}, fieldArr []string, orderRule string) (items []*BusinessConf, err error) {
+	o := orm.NewOrmUsingDB("master")
+	fields := strings.Join(fieldArr, ",")
+	if len(fieldArr) == 0 {
+		fields = `*`
+	}
+	order := `ORDER BY create_time DESC`
+	if orderRule != "" {
+		order = ` ORDER BY ` + orderRule
+	}
+	sql := fmt.Sprintf(`SELECT %s FROM %s WHERE 1=1 %s %s`, fields, m.TableName(), condition, order)
+	_, err = o.Raw(sql, pars).QueryRows(&items)
+	return
+}
+
+func (m *BusinessConf) GetPageItemsByCondition(condition string, pars []interface{}, fieldArr []string, orderRule string, startSize, pageSize int) (items []*BusinessConf, err error) {
+	o := orm.NewOrmUsingDB("master")
+	fields := strings.Join(fieldArr, ",")
+	if len(fieldArr) == 0 {
+		fields = `*`
+	}
+	order := `ORDER BY create_time DESC`
+	if orderRule != "" {
+		order = ` ORDER BY ` + orderRule
+	}
+	sql := fmt.Sprintf(`SELECT %s FROM %s WHERE 1=1 %s %s LIMIT ?,?`, fields, m.TableName(), condition, order)
+	_, err = o.Raw(sql, pars, startSize, pageSize).QueryRows(&items)
+	return
+}
+
+// GetBusinessConf 获取商家配置
+func GetBusinessConf() (list map[string]string, err error) {
+	list = make(map[string]string)
+
+	var items []*BusinessConf
+	o := orm.NewOrmUsingDB("master")
+	sql := `SELECT * FROM business_conf`
+	_, err = o.Raw(sql).QueryRows(&items)
+	if err != nil {
+		return
+	}
+
+	for _, v := range items {
+		if v.ValType == 4 {
+			list[v.ConfKey] = html.UnescapeString(v.ConfVal)
+			continue
+		}
+		list[v.ConfKey] = v.ConfVal
+	}
+	return
+}
+
+// BusinessConfUpdate 更新配置
+type BusinessConfUpdate struct {
+	ConfKey string
+	ConfVal string
+}
+
+// UpdateBusinessConfMulti 批量修改配置
+func UpdateBusinessConfMulti(items []BusinessConfUpdate) (err error) {
+	o := orm.NewOrmUsingDB("master")
+	p, err := o.Raw("UPDATE business_conf SET conf_val = ? WHERE conf_key = ?").Prepare()
+	if err != nil {
+		return
+	}
+	defer func() {
+		_ = p.Close()
+	}()
+	for _, v := range items {
+		_, err = p.Exec(v.ConfVal, v.ConfKey)
+		if err != nil {
+			return
+		}
+	}
+	return
+}
+
+func GetBusinessConfByKey(key string) (item *BusinessConf, err error) {
+	o := orm.NewOrmUsingDB("master")
+	sql := fmt.Sprintf(`SELECT * FROM business_conf WHERE conf_key = ? LIMIT 1`)
+	err = o.Raw(sql, key).QueryRow(&item)
+	return
+}
+
+// InitUseMongoConf
+// @Description:
+// @author: Roc
+// @datetime 2024-07-01 13:49:09
+func InitUseMongoConf() {
+	useMongo, e := GetBusinessConfByKey("UseMongo")
+	if e != nil {
+		return
+	}
+
+	if useMongo.ConfVal == `true` {
+		utils.UseMongo = true
+	}
+}

models/db.go

@@ -35,6 +35,17 @@ func init() {
 	gl, _ := orm.GetDB("gl")
 	gl.SetConnMaxLifetime(10 * time.Minute)
 
+	// master库
+	{
+		_ = orm.RegisterDataBase("master", "mysql", utils.MYSQL_URL_MASTER)
+		orm.SetMaxIdleConns("master", 50)
+		orm.SetMaxOpenConns("master", 100)
+
+		master, _ := orm.GetDB("master")
+		master.SetConnMaxLifetime(10 * time.Minute)
+
+	}
+
 	orm.Debug = true
 	orm.DebugLog = orm.NewLog(utils.Binlog)
 
@@ -43,6 +54,9 @@ func init() {
 		new(EdbDataCalculateZjpj),
 		new(EdbDataCalculateLjztbpj),
 		new(EdbInfo),
+		new(EdbInfoRecord),
+		new(BaseFromSmmRecord),
+		new(BaseFromMysteelChemicalRecord),
 		new(EdbInfoCalculateMapping),
 		new(EdbPythonCode),
 		new(EdbDataPython),
@@ -55,9 +69,10 @@ func init() {
 		new(EdbDataPredictCalculateLjztbpj),
 		new(EdbDataPredictCalculateNhcc),
 		new(EdbDataPredictCalculateZjpj),
-		new(EdbDataInsertConfig),
 		new(EdbAdjustConf), // 数据调整的配置
+		new(EdbDataInsertConfig),
 		new(BaseFromMysteelChemicalClassify),
+		new(EdbInfoRelation), //指标引用记录
 	)
 
 	// 注册期货数据 数据表
@@ -81,8 +96,11 @@ func init() {
 	// 自有数据指标
 	initBusinessEdb()
 
+	// 初始化因子指标系列
+	initFactorEdbSeries()
+
 	// 初始化部分数据表变量(直接init会有顺序问题=_=!)
-	InitEdbSource()
+	afterInitTable()
 }
 
 // initFutureGood 注册期货数据 数据表
@@ -129,10 +147,10 @@ func initBaseIndex() {
 		new(BaseFromCoalmineInlandIndex),
 		new(BaseFromCoalmineCompanyIndex),
 		new(BaseFromCoalmineFirmIndex),
-		new(BaseFromMtjhMapping),
-		new(BaseFromMtjhIndex),
 		new(BaseFromFenweiIndex),
 		new(BaseFromFenweiData),
+		new(BaseFromMtjhMapping),
+		new(BaseFromMtjhIndex),
 		new(BaseFromBloombergIndex),
 		new(BaseFromBloombergData),
 		new(BaseFromSci99Data),
@@ -143,6 +161,11 @@ func initBaseIndex() {
 		new(CCFStockExcel),
 		new(BaseFromSciHqData),
 		new(BaseFromSciHqIndex),
+		new(BaseFromBusinessData), // 数据源中自有数据的明细数据表
+		new(BaseFromThsHfIndex),
+		new(BaseFromThsHfData),
+		new(BaseFromEdbMapping),
+		new(EdbDataThsHf),
 	)
 }
 
@@ -181,3 +204,25 @@ func initBusinessEdb() {
 		new(EdbBusinessSource),     // 自有数据（商家）指标来源
 	)
 }
+
+// initFactorEdbSeries 因子指标系列数据表
+func initFactorEdbSeries() {
+	orm.RegisterModel(
+		new(FactorEdbSeries),              // 因子指标系列
+		new(FactorEdbSeriesChartMapping),  // 因子指标系列-图表关联
+		new(FactorEdbSeriesMapping),       // 因子指标系列-指标计算数据
+		new(FactorEdbSeriesCalculateData), // 因子指标系列-指标关联
+	)
+}
+
+// afterInitTable
+// @Description: 初始化表结构的的后置操作
+// @author: Roc
+// @datetime 2024-07-01 13:31:09
+func afterInitTable() {
+	// 初始化指标来源配置
+	InitEdbSource()
+
+	// 初始化是否启用mongo配置
+	InitUseMongoConf()
+}

models/edb_classify.go

@@ -115,13 +115,6 @@ func SaveEdbClassify(classifyName string, parentId, level int, classifyType uint
 	}
 	if isAdd {
 		sysUserIdInt, _ := strconv.Atoi(sysUserId)
-		//获取该层级下最大的排序数
-		maxSort, e := GetEdbAndClassifyMaxSort(parentId, classifyType)
-		if e != nil {
-			errMsg = "查询分类信息出错" + e.Error()
-			err = errors.New(errMsg)
-			return
-		}
 		rootId := 0
 		if parentId > 0 {
 			parentClassify, tErr := GetEdbClassifyById(parentId)
@@ -151,7 +144,7 @@ func SaveEdbClassify(classifyName string, parentId, level int, classifyType uint
 			SysUserRealName: sysUserName,
 			Level:           level + 1,
 			UniqueCode:      utils.MD5(utils.DATA_PREFIX + "_" + timestamp),
-			Sort:            maxSort + 1,
+			Sort:            GetAddEdbMaxSortByClassifyId(parentId, classifyType),
 		}
 		var classifyId int64
 		classifyId, err = AddEdbClassify(classifyInfo)

models/edb_data_base.go

@@ -46,6 +46,7 @@ type AddEdbInfoReq struct {
 	Source    int    `description:"指标来源ID"`
 	StockCode string `description:"证券代码"`
 	Frequency string `description:"频度"`
+	ExtraPars string `description:"额外参数(如同花顺日期序列)"`
 }
 
 // GetEdbInfoCountByCondition 获取指标数量

models/edb_data_business.go

@@ -7,8 +7,8 @@ import (
 	"github.com/beego/beego/v2/client/orm"
 	"github.com/shopspring/decimal"
 	"go.mongodb.org/mongo-driver/bson"
-	"go.mongodb.org/mongo-driver/bson/primitive"
 	"reflect"
+	"strings"
 	"time"
 )
 
@@ -95,6 +95,7 @@ func (obj Business) Add(params AddBaseParams, businessIndexItem *BaseFromBusines
 	edbInfo.EdbType = obj.GetEdbType()
 	edbInfo.SubSource = businessIndexItem.Source
 	edbInfo.SubSourceName = businessIndexItem.SourceName
+	edbInfo.Sort = GetAddEdbMaxSortByClassifyId(params.ClassifyId, utils.EDB_INFO_TYPE)
 	newEdbInfoId, tmpErr := to.Insert(edbInfo)
 	if tmpErr != nil {
 		err = tmpErr
@@ -103,7 +104,7 @@ func (obj Business) Add(params AddBaseParams, businessIndexItem *BaseFromBusines
 	edbInfo.EdbInfoId = int(newEdbInfoId)
 
 	// 更新数据
-	err = obj.refresh(edbInfo, "")
+	err = obj.refresh(to, edbInfo, "")
 
 	return
 }
@@ -153,7 +154,7 @@ func (obj Business) Edit(params EditBaseParams, businessIndexItem *BaseFromBusin
 	}
 
 	//计算数据
-	err = obj.refresh(edbInfo, "")
+	err = obj.refresh(to, edbInfo, "")
 
 	return
 }
@@ -174,7 +175,7 @@ func (obj Business) Refresh(params RefreshBaseParams) (err error, errMsg string)
 	}()
 
 	// 计算数据
-	err = obj.refresh(params.EdbInfo, params.StartDate)
+	err = obj.refresh(to, params.EdbInfo, params.StartDate)
 
 	return
 }
@@ -194,7 +195,17 @@ func (obj Business) GetEdbType() int {
 	return utils.DEFAULT_EDB_TYPE
 }
 
-func (obj Business) refresh(edbInfo *EdbInfo, startDate string) (err error) {
+func (obj Business) refresh(to orm.TxOrmer, edbInfo *EdbInfo, startDate string) (err error) {
+	if utils.UseMongo {
+		return obj.refreshByMongo(edbInfo, startDate)
+	} else {
+		return obj.refreshByMysql(to, edbInfo, startDate)
+	}
+
+	return
+}
+
+func (obj Business) refreshByMongo(edbInfo *EdbInfo, startDate string) (err error) {
 	// 真实数据的最大日期  ， 插入规则配置的日期
 	var realDataMaxDate, edbDataInsertConfigDate time.Time
 	var edbDataInsertConfig *EdbDataInsertConfig
@@ -210,7 +221,7 @@ func (obj Business) refresh(edbInfo *EdbInfo, startDate string) (err error) {
 	}
 
 	//获取已存在的所有数据
-	baseDataList, err := obj.getBaseBusinessData(edbInfo, startDate)
+	baseDataList, err := obj.getBaseBusinessDataByMongo(edbInfo, startDate)
 
 	//获取指标所有数据
 	existDataList := make([]*mgo.EdbDataBusiness, 0)
@@ -352,8 +363,144 @@ func (obj Business) refresh(edbInfo *EdbInfo, startDate string) (err error) {
 	return
 }
 
+func (obj Business) refreshByMysql(to orm.TxOrmer, edbInfo *EdbInfo, startDate string) (err error) {
+	dataTableName := GetEdbDataTableName(edbInfo.Source, edbInfo.SubSource)
+	edbInfoIdStr := fmt.Sprint(edbInfo.EdbInfoId)
+
+	// 真实数据的最大日期  ， 插入规则配置的日期
+	var realDataMaxDate, edbDataInsertConfigDate time.Time
+	var edbDataInsertConfig *EdbDataInsertConfig
+	var isFindConfigDateRealData bool //是否找到配置日期的实际数据的值
+	{
+		edbDataInsertConfig, err = GetEdbDataInsertConfigByEdbId(edbInfo.EdbInfoId)
+		if err != nil && err.Error() != utils.ErrNoRow() {
+			return
+		}
+		if edbDataInsertConfig != nil {
+			edbDataInsertConfigDate = edbDataInsertConfig.Date
+		}
+	}
+
+	//获取已存在的所有数据
+	baseDataList, err := obj.getBaseBusinessDataByMysql(edbInfo, startDate)
+
+	//获取指标所有数据
+	var existCondition string
+	var existPars []interface{}
+
+	existCondition += " AND edb_info_id=? "
+	existPars = append(existPars, edbInfo.EdbInfoId)
+	if startDate != "" {
+		existCondition += " AND data_time>=? "
+		existPars = append(existPars, startDate)
+	}
+
+	existList, err := GetEdbDataByCondition(edbInfo.Source, edbInfo.SubSource, existCondition, existPars)
+	if err != nil {
+		fmt.Println(obj.GetSourceName() + ",refreshByMysql err;getEdbDataBusinessList Err:" + err.Error())
+		return err
+	}
+	existDataMap := make(map[string]*EdbInfoSearchData)
+	removeDataTimeMap := make(map[string]bool) //需要移除的日期数据
+	for _, v := range existList {
+		existDataMap[v.DataTime] = v
+		removeDataTimeMap[v.DataTime] = true
+	}
+	needAddDateMap := make(map[time.Time]int)
+
+	// 待添加的数据集
+	addSql := ` INSERT INTO ` + dataTableName + ` (edb_info_id,edb_code,data_time,value,create_time,modify_time,data_timestamp) values `
+	var isAdd bool
+	// 待更新的数据集
+	updateDataList := make([]*EdbInfoSearchData, 0)
+
+	for _, tmpData := range baseDataList {
+		currDate := tmpData.DataTime
+		currDateStr := currDate.Format(utils.FormatDate)
+
+		// 当前的实际值
+		saveValue := decimal.NewFromFloat(tmpData.Value).Round(4).String()
+
+		// 下面代码主要目的是处理掉手动插入的数据判断
+		{
+			if realDataMaxDate.IsZero() || currDate.After(realDataMaxDate) {
+				realDataMaxDate = currDate
+			}
+			if edbDataInsertConfigDate.IsZero() || currDate.Equal(edbDataInsertConfigDate) {
+				isFindConfigDateRealData = true
+			}
+		}
+
+		existData, ok := existDataMap[currDateStr]
+		// 如果库中已经存在该数据的话，那么就进行值的变更操作
+		if ok {
+			// 已经入到指标库的值
+			existValStr := decimal.NewFromFloat(existData.Value).Round(4).String()
+
+			//校验待删除日期数据里面是否存在该元素，如果存在的话，那么移除该日期
+			delete(removeDataTimeMap, currDateStr)
+			if existValStr != saveValue {
+				existData.Value = tmpData.Value
+				updateDataList = append(updateDataList, existData)
+			}
+
+			continue
+		}
+
+		// 库中不存在该日期的数据
+		timestamp := currDate.UnixNano() / 1e6
+		needAddDateMap[currDate] = 1
+		addSql += GetAddSql(edbInfoIdStr, edbInfo.EdbCode, currDateStr, fmt.Sprint(timestamp), saveValue)
+		isAdd = true
+	}
+
+	//删除已经不存在的指标数据（由于该指标当日的数据删除了）
+	{
+		removeDateList := make([]string, 0)
+		for dateTime := range removeDataTimeMap {
+			removeDateList = append(removeDateList, dateTime)
+		}
+		removeNum := len(removeDateList)
+		if removeNum > 0 {
+			//如果拼接指标变更了，那么需要删除所有的指标数据
+			sql := fmt.Sprintf(` DELETE FROM %s WHERE edb_info_id = ? and data_time in (`+utils.GetOrmInReplace(removeNum)+`) `, dataTableName)
+
+			_, err = to.Raw(sql, edbInfo.EdbInfoId, removeDateList).Exec()
+			if err != nil {
+				err = fmt.Errorf("删除自有数据的明细数据失败,Err:" + err.Error())
+				return
+			}
+		}
+	}
+
+	if isAdd {
+		addSql = strings.TrimRight(addSql, ",")
+		_, err = to.Raw(addSql).Exec()
+		if err != nil {
+			fmt.Println("RefreshAllCalculate add Err", err.Error())
+			return
+		}
+	}
+
+	// 修改历史数据
+	if len(updateDataList) > 0 {
+		for _, v := range updateDataList {
+			err = ModifyEdbDataById(edbInfo.Source, edbInfo.SubSource, v.EdbDataId, fmt.Sprint(v.Value))
+			if err != nil {
+				fmt.Println(obj.GetSourceName() + "，refreshByMysql:Err:" + err.Error())
+				return err
+			}
+		}
+	}
+
+	// 处理手工数据补充的配置
+	HandleConfigInsertEdbData(realDataMaxDate, edbDataInsertConfig, edbInfo.EdbInfoId, edbInfo.Source, edbInfo.SubSource, existDataMap, isFindConfigDateRealData)
+
+	return
+}
+
 // GetEdbInfoMaxAndMinInfo 获取指标的最新数据记录信息
-func (obj Business) GetEdbInfoMaxAndMinInfo(edbCode string) (item *EdbInfoMaxAndMinInfo, err error) {
+func (obj Business) getEdbInfoMaxAndMinInfoByMongo(edbCode string) (item *EdbInfoMaxAndMinInfo, err error) {
 	mogDataObj := new(mgo.EdbDataBusiness)
 	pipeline := []bson.M{
 		{"$match": bson.M{"edb_code": edbCode}},
@@ -405,21 +552,29 @@ func (obj Business) GetEdbInfoMaxAndMinInfo(edbCode string) (item *EdbInfoMaxAnd
 // @param edbInfo *EdbInfo
 // @return err error
 func (obj Business) UnifiedModifyEdbInfoMaxAndMinInfo(edbInfo *EdbInfo) (err error) {
-	edbInfoMaxAndMinInfo, err := obj.GetEdbInfoMaxAndMinInfo(edbInfo.EdbCode)
-	if err != nil {
-		return
+	if utils.UseMongo {
+		edbInfoMaxAndMinInfo, tmpErr := obj.getEdbInfoMaxAndMinInfoByMongo(edbInfo.EdbCode)
+		// 如果正常获取到了，那就去修改指标的最大最小值
+		if tmpErr == nil && edbInfoMaxAndMinInfo != nil {
+			err = ModifyEdbInfoMaxAndMinInfo(edbInfo.EdbInfoId, edbInfoMaxAndMinInfo)
+		} else {
+			// 清空的目的是为了避免异常返回
+			err = nil
+		}
+	} else {
+		err, _ = UnifiedModifyEdbInfoMaxAndMinInfo(edbInfo)
 	}
-	err = ModifyEdbInfoMaxAndMinInfo(edbInfo.EdbInfoId, edbInfoMaxAndMinInfo)
+
 	return
 }
 
 // EdbInfoMgoData
 // @Description: mgo里面的数据
 type EdbInfoMgoData struct {
-	EdbDataId primitive.ObjectID `description:"数据ID"`
-	DataTime  time.Time          `description:"数据日期"`
-	Value     float64            `description:"数据"`
-	EdbCode   string             `description:"指标编码"`
+	//EdbDataId primitive.ObjectID `description:"数据ID"`
+	DataTime time.Time `description:"数据日期"`
+	Value    float64   `description:"数据"`
+	EdbCode  string    `description:"指标编码"`
 }
 
 // getBaseBusinessData
@@ -432,6 +587,19 @@ type EdbInfoMgoData struct {
 // @return newDataList []EdbInfoSearchData
 // @return err error
 func (obj Business) getBaseBusinessData(edbInfo *EdbInfo, startDate string) (newDataList []EdbInfoMgoData, err error) {
+	return obj.getBaseBusinessDataByMongo(edbInfo, startDate)
+}
+
+// getBaseBusinessDataByMongo
+// @Description: 从mongo中获取基础的明细数据
+// @author: Roc
+// @receiver obj
+// @datetime 2024-07-02 10:12:02
+// @param edbInfo *EdbInfo
+// @param startDate string
+// @return newDataList []EdbInfoMgoData
+// @return err error
+func (obj Business) getBaseBusinessDataByMongo(edbInfo *EdbInfo, startDate string) (newDataList []EdbInfoMgoData, err error) {
 	newDataList = make([]EdbInfoMgoData, 0)
 
 	// 获取数据源的指标数据
@@ -460,10 +628,53 @@ func (obj Business) getBaseBusinessData(edbInfo *EdbInfo, startDate string) (new
 
 	for _, v := range baseDataList {
 		newDataList = append(newDataList, EdbInfoMgoData{
-			EdbDataId: v.ID,
-			DataTime:  v.DataTime,
-			Value:     v.Value,
-			EdbCode:   v.IndexCode,
+			//EdbDataId: v.ID,
+			DataTime: v.DataTime,
+			Value:    v.Value,
+			EdbCode:  v.IndexCode,
+		})
+	}
+
+	return
+}
+
+// getBaseBusinessDataByMysql
+// @Description: 从mysql中获取基础的明细数据
+// @author: Roc
+// @receiver obj
+// @datetime 2024-07-02 10:12:16
+// @param edbInfo *EdbInfo
+// @param startDate string
+// @return newDataList []EdbInfoMgoData
+// @return err error
+func (obj Business) getBaseBusinessDataByMysql(edbInfo *EdbInfo, startDate string) (newDataList []EdbInfoMgoData, err error) {
+	newDataList = make([]EdbInfoMgoData, 0)
+	// 获取数据源的指标数据
+	baseBusinessDataObj := new(BaseFromBusinessData)
+
+	// 构建查询条件
+	var condition []string
+	var pars []interface{}
+	condition = append(condition, "index_code = ? ")
+	pars = append(pars, edbInfo.EdbCode)
+
+	if startDate != `` {
+		condition = append(condition, " data_time >= ? ")
+		pars = append(pars, startDate)
+	}
+
+	baseDataList, err := baseBusinessDataObj.GetAllDataList(condition, pars, " data_time ASC ")
+	if err != nil {
+		fmt.Println("getBaseBusinessData Err:" + err.Error())
+		return
+	}
+
+	for _, v := range baseDataList {
+		newDataList = append(newDataList, EdbInfoMgoData{
+			//EdbDataId: v.BusinessDataId,
+			DataTime: v.DataTime,
+			Value:    v.Value,
+			EdbCode:  v.IndexCode,
 		})
 	}
 

models/edb_data_calculate_avg.go

@@ -53,6 +53,7 @@ func (obj CalculateAvg) Add(params AddCalculateBatchParams) (edbInfo *EdbInfo, e
 	edbInfo.ModifyTime = time.Now()
 	edbInfo.UniqueCode = params.UniqueCode
 	edbInfo.EdbType = obj.GetEdbType()
+	edbInfo.Sort = GetAddEdbMaxSortByClassifyId(req.ClassifyId, utils.EDB_INFO_TYPE)
 	edbInfo.EmptyType = req.EmptyType
 	edbInfo.MaxEmptyType = req.MaxEmptyType
 	edbInfo.Extra = req.Extra

models/edb_data_calculate_bp.go

@@ -45,6 +45,8 @@ func AddCalculateBp(req *EdbInfoCalculateBatchSaveReq, fromEdbInfo *EdbInfo, edb
 		edbInfo.EdbNameEn = req.EdbName
 		edbInfo.UnitEn = req.Unit
 		edbInfo.EdbType = 2
+		edbInfo.Sort = GetAddEdbMaxSortByClassifyId(req.ClassifyId, utils.EDB_INFO_TYPE)
+		edbInfo.EmptyType = req.EmptyType
 		newEdbInfoId, tmpErr := to.Insert(edbInfo)
 		if tmpErr != nil {
 			err = tmpErr
@@ -89,7 +91,7 @@ func AddCalculateBp(req *EdbInfoCalculateBatchSaveReq, fromEdbInfo *EdbInfo, edb
 	}
 
 	//计算数据
-	err = refreshAllCalculateBp(to, edbInfo.EdbInfoId, edbInfo.Source, edbInfo.SubSource, fromEdbInfo, edbInfo.EdbCode, "", "", 0)
+	err = refreshAllCalculateBp(to, edbInfo.EdbInfoId, edbInfo.Source, edbInfo.SubSource, fromEdbInfo, edbInfo.EdbCode, "", "", 0, edbInfo.EmptyType)
 
 	return
 }
@@ -110,6 +112,7 @@ func EditCalculateBp(edbInfo *EdbInfo, req *EdbInfoCalculateBatchEditReq, fromEd
 		}
 	}()
 
+	oldEmptyType := edbInfo.EmptyType
 	//修改指标信息
 	edbInfo.EdbName = req.EdbName
 	edbInfo.EdbNameSource = req.EdbName
@@ -119,7 +122,8 @@ func EditCalculateBp(edbInfo *EdbInfo, req *EdbInfoCalculateBatchEditReq, fromEd
 	edbInfo.EdbNameEn = req.EdbNameEn
 	edbInfo.UnitEn = req.UnitEn
 	edbInfo.ModifyTime = time.Now()
-	_, err = to.Update(edbInfo, "EdbName", "EdbNameSource", "Frequency", "Unit", "ClassifyId", "ModifyTime", "EdbNameEn", "UnitEn")
+	edbInfo.EmptyType = req.EmptyType
+	_, err = to.Update(edbInfo, "EdbName", "EdbNameSource", "Frequency", "Unit", "ClassifyId", "ModifyTime", "EdbNameEn", "UnitEn", "EmptyType")
 	if err != nil {
 		return
 	}
@@ -135,7 +139,7 @@ func EditCalculateBp(edbInfo *EdbInfo, req *EdbInfoCalculateBatchEditReq, fromEd
 		err = errors.New("判断指标是否改变失败,Err:" + err.Error())
 		return
 	}
-	if count > 0 { // 指标未被替换，无需重新计算
+	if count > 0 && oldEmptyType == req.EmptyType { // 指标未被替换，无需重新计算
 		return
 	}
 
@@ -178,7 +182,7 @@ func EditCalculateBp(edbInfo *EdbInfo, req *EdbInfoCalculateBatchEditReq, fromEd
 	}
 
 	//计算数据
-	err = refreshAllCalculateBp(to, edbInfo.EdbInfoId, edbInfo.Source, edbInfo.SubSource, fromEdbInfo, edbInfo.EdbCode, "", "", 0)
+	err = refreshAllCalculateBp(to, edbInfo.EdbInfoId, edbInfo.Source, edbInfo.SubSource, fromEdbInfo, edbInfo.EdbCode, "", "", 0, edbInfo.EmptyType)
 
 	return
 }
@@ -315,7 +319,7 @@ func RefreshAllCalculateBpBak(edbInfoId, source, subSource int, fromEdbInfo *Edb
 	return
 }
 
-func RefreshAllCalculateBp(edbInfoId, source, subSource int, fromEdbInfo *EdbInfo, edbCode, startDate, endDate string) (err error) {
+func RefreshAllCalculateBp(edbInfoId, source, subSource int, fromEdbInfo *EdbInfo, edbCode, startDate, endDate string, emptyType int) (err error) {
 	o := orm.NewOrm()
 	to, err := o.Begin()
 	if err != nil {
@@ -331,13 +335,13 @@ func RefreshAllCalculateBp(edbInfoId, source, subSource int, fromEdbInfo *EdbInf
 	}()
 
 	// 计算数据
-	err = refreshAllCalculateBp(to, edbInfoId, source, subSource, fromEdbInfo, edbCode, startDate, endDate, 1)
+	err = refreshAllCalculateBp(to, edbInfoId, source, subSource, fromEdbInfo, edbCode, startDate, endDate, 0, emptyType)
 
 	return
 }
 
 // refreshAllCalculateBp 刷新升频数据
-func refreshAllCalculateBp(to orm.TxOrmer, edbInfoId, source, subSource int, fromEdbInfo *EdbInfo, edbCode, startDate, endDate string, order int) (err error) {
+func refreshAllCalculateBp(to orm.TxOrmer, edbInfoId, source, subSource int, fromEdbInfo *EdbInfo, edbCode, startDate, endDate string, order int, emptyType int) (err error) {
 	edbInfoIdStr := strconv.Itoa(edbInfoId)
 	//计算数据
 
@@ -346,127 +350,152 @@ func refreshAllCalculateBp(to orm.TxOrmer, edbInfoId, source, subSource int, fro
 		EdbInfoId: fromEdbInfo.EdbInfoId,
 	}, order)
 	if err != nil {
-		return err
-	}
-
-	// 来源指标没有数据，那么需要删除所有的计算指标数据
-	if len(dataList) <= 0 {
-		// todo 删除所有的计算指标数据
 		return
 	}
-	// 来源指标的第一个日期
-	fromFirstDate, err := time.ParseInLocation(utils.FormatDate, dataList[0].DataTime, time.Local)
-	if err != nil {
-		return
-	}
-	fromFirstDate = time.Date(fromFirstDate.Year(), fromFirstDate.Month(), fromFirstDate.Day(), 0, 0, 0, 0, time.Local)
-
-	// 变频计算
-	newDataList, err := EdbInfoSearchDataToData(dataList)
-	if err != nil {
-		return
-	}
-
-	baseCalculate := BaseCalculate{
-		DataList:      newDataList,
-		Frequency:     "",
-		Formula:       nil,
-		Calendar:      "",
-		MoveType:      0,
-		MoveFrequency: "",
-		FromFrequency: "",
-		Source:        source,
-	}
-	dateDataMap, err, _ := baseCalculate.UpFrequency()
-	if err != nil {
-		return
+	var dateArr []string
+	dataMap := make(map[string]*EdbInfoSearchData)
+	fromDataMap := make(map[string]float64)
+	//来源指指标数据
+	for _, v := range dataList {
+		dateArr = append(dateArr, v.DataTime)
+		dataMap[v.DataTime] = v
+		fromDataMap[v.DataTime] = v.Value
 	}
+	fmt.Println("source:", source)
 
-	// 获取升频指所有已经存在的计算指标数据
+	//获取升频指标所有数据
 	existDataList, err := GetAllEdbDataListByTo(to, edbInfoId, source, subSource)
 	if err != nil {
 		return
 	}
 	//计算指标的map
 	existDataMap := make(map[string]*EdbData, 0)
-	for _, v := range existDataList {
-		existDataMap[v.DataTime] = v
-	}
-
+	removeDateMap := make(map[string]struct{})
 	addSql := ` INSERT INTO edb_data_calculate_bp(edb_info_id,edb_code,data_time,value,create_time,modify_time,data_timestamp) values `
 	var isAdd bool
-
-	now := time.Date(time.Now().Year(), time.Now().Month(), time.Now().Day(), 0, 0, 0, 0, time.Local)
-
-	for currDate := fromFirstDate; !currDate.After(now); currDate = currDate.AddDate(0, 0, 1) {
-		currDateStr := currDate.Format(utils.FormatDate)
-		timestamp := currDate.UnixNano() / 1e6
-		timestampStr := fmt.Sprintf("%d", timestamp)
-
-		// 当前计算的值
-		currValue, ok := dateDataMap[currDate]
-		if !ok {
-			// 没有计算成功就过滤
-			continue
+	//待删除的日期
+	removeDateList := make([]string, 0)
+	if len(existDataList) > 0 && len(dateArr) == 0 {
+		//如果没有来源指标数据，那么已经入库的计算指标数据需要全部删除
+		tableName := GetEdbDataTableName(source, subSource)
+		sql := fmt.Sprintf(` DELETE FROM %s WHERE edb_info_id = ?`, tableName)
+		_, err = to.Raw(sql, edbInfoId).Exec()
+		if err != nil {
+			err = fmt.Errorf("删除所有的升频指标数据失败,Err:" + err.Error())
+			return
 		}
-		lastValueStr := decimal.NewFromFloat(currValue).Round(4).String()
+		return
+	}
 
-		// 已经入库的值
-		existData, ok := existDataMap[currDateStr]
-		if !ok {
-			// 没有入库那么就插入添加
-			isAdd = true
-			addSql += GetAddSql(edbInfoIdStr, edbCode, currDateStr, timestampStr, lastValueStr)
-			continue
-		}
+	existMap := make(map[string]string)
 
-		// 将已经入库的值转换为decimal类型，然后再保留4位小数，目的是为了做匹配，要不然取出来的数据与计算的数据不一致
-		existDataValueDec, tmpErr := decimal.NewFromString(existData.Value)
-		if tmpErr != nil {
-			err = tmpErr
-			return
+	dataLen := len(dataList)
+	//第三步： 已经入库的数据处理
+	for _, v := range existDataList {
+		existDataMap[v.DataTime] = v
+		removeDateMap[v.DataTime] = struct{}{}
+	}
+	for i := 0; i < dataLen; i++ {
+		//当期
+		currentItem := dataList[i]
+		var prevItem *EdbInfoSearchData
+		if emptyType == 3 { //3后值填充,其余前值填充
+			if i >= 1 {
+				prevItem = dataList[i-1]
+			}
 		}
-		existDataValueStr := existDataValueDec.Round(4).String()
-
-		// 如果该日期已经入库了，且两个值不匹配，那么就更新
-		if lastValueStr != existDataValueStr {
-			err = ModifyEdbDataById(source, subSource, existData.EdbDataId, lastValueStr)
-			if err != nil {
-				return err
+		currentDate, _ := time.ParseInLocation(utils.FormatDate, currentItem.DataTime, time.Local)
+		var day int
+		var preItem *EdbInfoSearchData
+		var preDate time.Time
+		if i == 0 {
+			if emptyType == 3 { //后值填充
+				day = 0 //最新的时间就是来源指标的最新日期
+				preDate = currentDate
+			} else {
+				day = int(time.Now().Sub(currentDate).Hours() / float64(24))
+				preDate = time.Date(time.Now().Year(), time.Now().Month(), time.Now().Day(), 0, 0, 0, 0, time.Local)
+			}
+		} else {
+			j := i - 1
+			if j < dataLen {
+				preItem = dataList[j]
+				preDate, _ = time.ParseInLocation(utils.FormatDate, preItem.DataTime, time.Local)
+				day = int(preDate.Sub(currentDate).Hours() / float64(24))
+				utils.FileLog.Info("preItem.DataTime:" + preItem.DataTime + ";currentItem.DataTime" + currentItem.DataTime)
 			}
 		}
+		for k := 0; k <= day; k++ {
+			needDay := preDate.AddDate(0, 0, -k)
+			needDayStr := needDay.Format(utils.FormatDate)
+			delete(removeDateMap, needDayStr)
+			existKey := edbCode + needDayStr
+			if _, ok := existMap[existKey]; !ok {
+				timestamp := needDay.UnixNano() / 1e6
+				timestampStr := fmt.Sprintf("%d", timestamp)
+				valStr := decimal.NewFromFloat(currentItem.Value).String()
+				if prevItem != nil && needDayStr != currentItem.DataTime {
+					valStr = decimal.NewFromFloat(prevItem.Value).String()
+				}
+				tmpExistData, ok2 := existDataMap[needDayStr]
+				if !ok2 {
+					addSql += GetAddSql(edbInfoIdStr, edbCode, needDayStr, timestampStr, valStr)
+					isAdd = true
+				} else {
+					//如果对应的值不匹配
+					if tmpExistData.Value != valStr {
+						err = ModifyEdbDataById(source, subSource, tmpExistData.EdbDataId, valStr)
+						if err != nil {
+							return err
+						}
+					}
+				}
 
-		// 该日期已经处理过了，所以需要移除，如果后面该map还有数据，那么需要删除该map里面的日期数据
-		delete(existDataMap, currDateStr)
+			}
+			existMap[existKey] = needDayStr
+		}
+		existKey := edbCode + currentItem.DataTime
+		if _, ok := existMap[existKey]; !ok {
+			currentDate, _ := time.ParseInLocation(utils.FormatDate, currentItem.DataTime, time.Local)
+			timestamp := currentDate.UnixNano() / 1e6
+			timestampStr := fmt.Sprintf("%d", timestamp)
+			valStr := decimal.NewFromFloat(currentItem.Value).String()
+			tmpExistData, ok2 := existDataMap[currentItem.DataTime]
+			if !ok2 {
+				addSql += GetAddSql(edbInfoIdStr, edbCode, currentItem.DataTime, timestampStr, valStr)
+				isAdd = true
+			} else {
+				//如果对应的值不匹配
+				if tmpExistData.Value != valStr {
+					err = ModifyEdbDataById(source, subSource, tmpExistData.EdbDataId, valStr)
+					if err != nil {
+						return err
+					}
+				}
+			}
 
+		}
+		existMap[existKey] = currentItem.DataTime
 	}
 
+	for k, _ := range removeDateMap {
+		removeDateList = append(removeDateList, k)
+	}
 	// 删除不需要的指标数据
-	if len(existDataMap) > 0 {
-		//待删除的日期
-		removeDateList := make([]string, 0)
-		for date := range existDataMap {
-			removeDateList = append(removeDateList, date)
-		}
-
-		removeDateStr := strings.Join(removeDateList, `","`)
-		removeDateStr = `"` + removeDateStr + `"`
+	if len(removeDateList) > 0 {
 		//如果拼接指标变更了，那么需要删除所有的指标数据
 		tableName := GetEdbDataTableName(source, subSource)
-		sql := fmt.Sprintf(` DELETE FROM %s WHERE edb_info_id = ? and data_time in (%s) `, tableName, removeDateStr)
-
-		_, err = to.Raw(sql, edbInfoId).Exec()
+		sql := fmt.Sprintf(` DELETE FROM %s WHERE edb_info_id = ? and data_time in (`+utils.GetOrmInReplace(len(removeDateList))+`) `, tableName)
+		_, err = to.Raw(sql, edbInfoId, removeDateList).Exec()
 		if err != nil {
 			err = fmt.Errorf("删除不存在的升频指标数据失败,Err:" + err.Error())
 			return
 		}
 	}
 
-	// 新增的数据值
 	if isAdd {
 		addSql = strings.TrimRight(addSql, ",")
 		_, err = to.Raw(addSql).Exec()
 	}
-
 	return
 }

models/edb_data_calculate_cjjx.go

@@ -48,6 +48,8 @@ func AddCalculateCjjx(req *EdbInfoCalculateBatchSaveReq, fromEdbInfo *EdbInfo, e
 		edbInfo.EdbNameEn = req.EdbName
 		edbInfo.UnitEn = req.Unit
 		edbInfo.EdbType = 2
+		edbInfo.Sort = GetAddEdbMaxSortByClassifyId(req.ClassifyId, utils.EDB_INFO_TYPE)
+
 		newEdbInfoId, tmpErr := to.Insert(edbInfo)
 		if tmpErr != nil {
 			err = tmpErr

models/edb_data_calculate_correlation.go

@@ -50,6 +50,7 @@ func AddCalculateCorrelation(req *EdbInfoCalculateBatchSaveReq, edbCode, uniqueC
 	edbInfo.EdbNameEn = req.EdbName
 	edbInfo.UnitEn = req.Unit
 	edbInfo.EdbType = 2
+	edbInfo.Sort = GetAddEdbMaxSortByClassifyId(req.ClassifyId, utils.EDB_INFO_TYPE)
 	newEdbInfoId, tmpErr := to.Insert(edbInfo)
 	if tmpErr != nil {
 		err = tmpErr

models/edb_data_calculate_hbz.go

@@ -45,6 +45,7 @@ func AddCalculateHbz(req *EdbInfoCalculateBatchSaveReq, fromEdbInfo *EdbInfo, ed
 		edbInfo.EdbNameEn = req.EdbName
 		edbInfo.UnitEn = req.Unit
 		edbInfo.EdbType = 2
+		edbInfo.Sort = GetAddEdbMaxSortByClassifyId(req.ClassifyId, utils.EDB_INFO_TYPE)
 		newEdbInfoId, tmpErr := to.Insert(edbInfo)
 		if tmpErr != nil {
 			err = tmpErr

models/edb_data_calculate_hcz.go

@@ -45,6 +45,7 @@ func AddCalculateHcz(req *EdbInfoCalculateBatchSaveReq, fromEdbInfo *EdbInfo, ed
 		edbInfo.EdbNameEn = req.EdbName
 		edbInfo.UnitEn = req.Unit
 		edbInfo.EdbType = 2
+		edbInfo.Sort = GetAddEdbMaxSortByClassifyId(req.ClassifyId, utils.EDB_INFO_TYPE)
 		newEdbInfoId, tmpErr := to.Insert(edbInfo)
 		if tmpErr != nil {
 			err = tmpErr

models/edb_data_calculate_jp.go

@@ -72,6 +72,7 @@ func AddCalculateJp(req *EdbInfoCalculateBatchSaveReq, fromEdbInfo *EdbInfo, edb
 		edbInfo.EdbNameEn = req.EdbName
 		edbInfo.UnitEn = req.Unit
 		edbInfo.EdbType = 2
+		edbInfo.Sort = GetAddEdbMaxSortByClassifyId(req.ClassifyId, utils.EDB_INFO_TYPE)
 		newEdbInfoId, tmpErr := to.Insert(edbInfo)
 		if tmpErr != nil {
 			err = tmpErr
@@ -284,8 +285,7 @@ func refreshAllCalculateJp(to orm.TxOrmer, edbInfoId, source, subSource int, fro
 	startDataTime, _ := time.ParseInLocation(utils.FormatDate, dataList[0].DataTime, time.Local)
 	endDataTime, _ := time.ParseInLocation(utils.FormatDate, dataList[dataLen-1].DataTime, time.Local)
 
-	var lastValue float64     // 最近的值
-	var nextEndDate time.Time // 下一个节点的日期
+	nextEndDate := utils.GetFrequencyEndDay(startDataTime, edbFrequency) // 下一个节点的日期
 	weekDayDataList := make([]float64, 0)
 	for tmpStartDataTime := startDataTime; !tmpStartDataTime.After(endDataTime); tmpStartDataTime = tmpStartDataTime.AddDate(0, 0, 1) {
 		// 将当前数据加入到 weekDayDataList
@@ -294,10 +294,6 @@ func refreshAllCalculateJp(to orm.TxOrmer, edbInfoId, source, subSource int, fro
 			tmpValueFloat, _ := tmpValue.Round(4).Float64()
 			weekDayDataList = append(weekDayDataList, tmpValueFloat)
 		}
-		// 如果下个节点的日期不存在,那么就先给赋值(兼容时间区间内只有一组数据的情况)
-		if nextEndDate.IsZero() {
-			nextEndDate = utils.GetFrequencyEndDay(tmpStartDataTime, edbFrequency)
-		}
 
 		// 日期处理过滤
 		switch edbFrequency {
@@ -355,25 +351,25 @@ func refreshAllCalculateJp(to orm.TxOrmer, edbInfoId, source, subSource int, fro
 			return
 		}
 
+		// 本期的数据值
+		lenWeekDayDataList := len(weekDayDataList)
+		if lenWeekDayDataList <= 0 {
+			continue
+		}
+
 		// 当前时间段内的数据计算,得出实际值
 		var currVal float64
-		lenWeekDayDataList := len(weekDayDataList)
-		// 如果这个时间区间内没有数据,那么就采用上一个时间区间的值
-		if len(weekDayDataList) <= 0 {
-			currVal = lastValue
+		if formula == "期末值" { // 期末值，取区间最后一个日期的数据值
+			currVal = weekDayDataList[lenWeekDayDataList-1]
 		} else {
-			if formula == "期末值" {
-				currVal = weekDayDataList[lenWeekDayDataList-1]
-			} else {
-				// 平均值
-				sumValDeci := decimal.NewFromFloat(0)
-				for _, v := range weekDayDataList {
-					tmpValDeci := decimal.NewFromFloat(v)
-					sumValDeci = sumValDeci.Add(tmpValDeci)
-				}
-				lenDeci := decimal.NewFromInt(int64(lenWeekDayDataList))
-				currVal, _ = sumValDeci.Div(lenDeci).Round(4).Float64()
+			// 平均值 取区间平均值
+			sumValDeci := decimal.NewFromFloat(0)
+			for _, v := range weekDayDataList {
+				tmpValDeci := decimal.NewFromFloat(v)
+				sumValDeci = sumValDeci.Add(tmpValDeci)
 			}
+			lenDeci := decimal.NewFromInt(int64(lenWeekDayDataList))
+			currVal, _ = sumValDeci.Div(lenDeci).Round(4).Float64()
 		}
 
 		tmpStartDataTimeStr := tmpStartDataTime.Format(utils.FormatDate)
@@ -415,30 +411,26 @@ func refreshAllCalculateJp(to orm.TxOrmer, edbInfoId, source, subSource int, fro
 	}
 
 	// 最后已有的日期处理完成后,需要对剩余不在时间段内的数据做处理
-	if len(weekDayDataList) > 0 {
+	lenWeekDayDataList := len(weekDayDataList)
+	if lenWeekDayDataList > 0 {
 		// 当前时间段内的数据计算,得出实际值
 		var currVal float64
-		lenWeekDayDataList := len(weekDayDataList)
-		// 如果这个时间区间内没有数据,那么就采用上一个时间区间的值
-		if len(weekDayDataList) < 0 {
-			currVal = lastValue
+		if formula == "期末值" {
+			currVal = weekDayDataList[lenWeekDayDataList-1]
 		} else {
-			if formula == "期末值" {
-				currVal = weekDayDataList[lenWeekDayDataList-1]
-			} else {
-				// 平均值
-				sumValDeci := decimal.NewFromFloat(0)
-				for _, v := range weekDayDataList {
-					tmpValDeci := decimal.NewFromFloat(v)
-					sumValDeci = sumValDeci.Add(tmpValDeci)
-				}
-				lenDeci := decimal.NewFromInt(int64(lenWeekDayDataList))
-				currVal, _ = sumValDeci.Div(lenDeci).Round(4).Float64()
+			// 平均值
+			sumValDeci := decimal.NewFromFloat(0)
+			for _, v := range weekDayDataList {
+				tmpValDeci := decimal.NewFromFloat(v)
+				sumValDeci = sumValDeci.Add(tmpValDeci)
 			}
+			lenDeci := decimal.NewFromInt(int64(lenWeekDayDataList))
+			currVal, _ = sumValDeci.Div(lenDeci).Round(4).Float64()
 		}
+		nextEndDateStr := nextEndDate.Format(utils.FormatDate)
 
 		// 判断降频指标是否存在数据
-		if existData, ok := existDataMap[nextEndDate.Format(utils.FormatDate)]; ok {
+		if existData, ok := existDataMap[nextEndDateStr]; ok {
 			// 处理降频数据的值
 			existValStr := existData.Value
 			existValDeci, tmpErr := decimal.NewFromString(existValStr)
@@ -455,7 +447,7 @@ func refreshAllCalculateJp(to orm.TxOrmer, edbInfoId, source, subSource int, fro
 				}
 			}
 			// 移除待删除的日期
-			delete(existDelDateMap, nextEndDate.Format(utils.FormatDate))
+			delete(existDelDateMap, nextEndDateStr)
 		} else {
 			// 直接入库
 			timestamp := nextEndDate.UnixNano() / 1e6

models/edb_data_calculate_kszs.go

@@ -50,6 +50,7 @@ func AddCalculateKszs(req *EdbInfoCalculateBatchSaveReq, edbCode, uniqueCode str
 	edbInfo.EdbNameEn = req.EdbName
 	edbInfo.UnitEn = req.Unit
 	edbInfo.EdbType = 2
+	edbInfo.Sort = GetAddEdbMaxSortByClassifyId(req.ClassifyId, utils.EDB_INFO_TYPE)
 	newEdbInfoId, tmpErr := to.Insert(edbInfo)
 	if tmpErr != nil {
 		err = tmpErr

models/edb_data_calculate_ljz.go

@@ -69,7 +69,7 @@ func (obj Ljz) Add(params AddCalculateBatchParams) (edbInfo *EdbInfo, err error,
 		MaxValue:         0,
 		CalculateFormula: req.Formula,
 		EdbType:          obj.GetEdbType(),
-		Sort:             0,
+		Sort:             GetAddEdbMaxSortByClassifyId(req.ClassifyId, utils.EDB_INFO_TYPE),
 		MoveType:         0,
 		MoveFrequency:    "",
 		NoUpdate:         0,

models/edb_data_calculate_ljznczj.go

@@ -68,7 +68,7 @@ func (obj LjzNczj) Add(params AddCalculateBatchParams) (edbInfo *EdbInfo, err er
 		MaxValue:         0,
 		CalculateFormula: req.Formula,
 		EdbType:          obj.GetEdbType(),
-		Sort:             0,
+		Sort:             GetAddEdbMaxSortByClassifyId(req.ClassifyId, utils.EDB_INFO_TYPE),
 		MoveType:         0,
 		MoveFrequency:    "",
 		NoUpdate:         0,

models/edb_data_calculate_ljztbpj.go

@@ -101,6 +101,7 @@ func AddCalculateLjztbpj(req *EdbInfoCalculateBatchSaveReq, firstEdbInfo, second
 			EdbNameEn:        req.EdbName,
 			UnitEn:           req.Unit,
 			EdbType:          2,
+			Sort:             GetAddEdbMaxSortByClassifyId(req.ClassifyId, utils.EDB_INFO_TYPE),
 		}
 		newEdbInfoId, tmpErr := to.Insert(edbInfo)
 		if tmpErr != nil {

models/edb_data_calculate_ljzzj.go

@@ -58,6 +58,7 @@ func (obj Ljzzj) Add(params AddCalculateBatchParams) (edbInfo *EdbInfo, err erro
 	edbInfo.EdbNameEn = req.EdbName
 	edbInfo.UnitEn = req.Unit
 	edbInfo.EdbType = obj.GetEdbType()
+	edbInfo.Sort = GetAddEdbMaxSortByClassifyId(req.ClassifyId, utils.EDB_INFO_TYPE)
 	newEdbInfoId, tmpErr := to.Insert(edbInfo)
 	if tmpErr != nil {
 		err = tmpErr

models/edb_data_calculate_ljzzy.go

@@ -46,6 +46,7 @@ func AddCalculateLjzzy(req *EdbInfoCalculateBatchSaveReq, fromEdbInfo *EdbInfo,
 		edbInfo.EdbNameEn = req.EdbName
 		edbInfo.UnitEn = req.Unit
 		edbInfo.EdbType = 2
+		edbInfo.Sort = GetAddEdbMaxSortByClassifyId(req.ClassifyId, utils.EDB_INFO_TYPE)
 		newEdbInfoId, tmpErr := to.Insert(edbInfo)
 		if tmpErr != nil {
 			err = tmpErr

models/edb_data_calculate_nh.go

@@ -45,6 +45,7 @@ func AddCalculateNh(req *EdbInfoCalculateBatchSaveReq, fromEdbInfo *EdbInfo, edb
 		edbInfo.EdbNameEn = req.EdbName
 		edbInfo.UnitEn = req.Unit
 		edbInfo.EdbType = 2
+		edbInfo.Sort = GetAddEdbMaxSortByClassifyId(req.ClassifyId, utils.EDB_INFO_TYPE)
 		newEdbInfoId, tmpErr := to.Insert(edbInfo)
 		if tmpErr != nil {
 			err = tmpErr

models/edb_data_calculate_nhcc.go

@@ -66,6 +66,7 @@ func AddCalculateNhcc(req *EdbInfoCalculateBatchSaveReq, firstEdbInfo, secondEdb
 			EdbNameEn:        req.EdbName,
 			UnitEn:           req.Unit,
 			EdbType:          2,
+			Sort:             GetAddEdbMaxSortByClassifyId(req.ClassifyId, utils.EDB_INFO_TYPE),
 		}
 		newEdbInfoId, tmpErr := to.Insert(edbInfo)
 		if tmpErr != nil {

models/edb_data_calculate_nszydbpjjs.go

@@ -42,6 +42,7 @@ func AddCalculateNszydpjjs(req *EdbInfoCalculateBatchSaveReq, fromEdbInfo *EdbIn
 		edbInfo.EdbNameEn = req.EdbName
 		edbInfo.UnitEn = req.Unit
 		edbInfo.EdbType = 2
+		edbInfo.Sort = GetAddEdbMaxSortByClassifyId(req.ClassifyId, utils.EDB_INFO_TYPE)
 		newEdbInfoId, tmpErr := to.Insert(edbInfo)
 		if tmpErr != nil {
 			err = tmpErr

models/edb_data_calculate_percentile.go

@@ -54,6 +54,7 @@ func (obj Percentile) Add(params AddCalculateBatchParams) (edbInfo *EdbInfo, err
 	edbInfo.EdbNameEn = req.EdbName
 	edbInfo.UnitEn = req.Unit
 	edbInfo.EdbType = obj.GetEdbType()
+	edbInfo.Sort = GetAddEdbMaxSortByClassifyId(req.ClassifyId, utils.EDB_INFO_TYPE)
 	newEdbInfoId, tmpErr := to.Insert(edbInfo)
 	if tmpErr != nil {
 		err = tmpErr
@@ -442,21 +443,25 @@ func (obj Percentile) getPercentileData(fromEdbInfo *EdbInfo, calculateValue int
 	// 百分位数据个数算法
 	// 数据区间第一个和最后一个数据点的时间和数据分别为(T1,S1)(T2,S2); N=T1到T2指标数据个数, n=小于等于S2的数据个数
 	// 个数百分位=(n-1)/(N-1)
-	maxDay := len(dataList) // 往前找数据的边界
+	var firstDate time.Time
+	if len(dataList) > 0 {
+		d, _ := time.ParseInLocation(utils.FormatDate, dataList[0].DataTime, time.Local)
+		firstDate = d
+	}
 	if percentType == utils.PercentCalculateTypeNum {
 		for i, d := range dataList {
 			// T2为当前日期
 			s2 := decimal.NewFromFloat(d.Value)
 			t2, _ := time.ParseInLocation(utils.FormatDate, d.DataTime, time.Local)
 
-			// 计算N和n
+			// 往前找(时间长度)个有数据的
 			var bigN, tinyN int
-			for k := 0; k < maxDay; k++ {
-				// 往前找(时间长度)个有数据的, N理论上只有最前面几个日期<calculateDay, 后面的N=calculateDay
-				if bigN >= calculateDay {
+			for k := 0; k < calculateDay; k++ {
+				tp := t2.AddDate(0, 0, -k)
+				if tp.Before(firstDate) {
 					break
 				}
-				preVal, preOk := dataMap[t2.AddDate(0, 0, -k)]
+				preVal, preOk := dataMap[tp]
 				if !preOk {
 					continue
 				}

models/edb_data_calculate_rjz.go

@@ -45,6 +45,7 @@ func AddCalculateRjz(req *EdbInfoCalculateBatchSaveReq, fromEdbInfo *EdbInfo, ed
 		edbInfo.EdbNameEn = req.EdbName
 		edbInfo.UnitEn = req.Unit
 		edbInfo.EdbType = 2
+		edbInfo.Sort = GetAddEdbMaxSortByClassifyId(req.ClassifyId, utils.EDB_INFO_TYPE)
 		newEdbInfoId, tmpErr := to.Insert(edbInfo)
 		if tmpErr != nil {
 			err = tmpErr

models/edb_data_calculate_standard_deviation.go

@@ -53,6 +53,7 @@ func (obj StandardDeviation) Add(params AddCalculateBatchParams) (edbInfo *EdbIn
 	edbInfo.EdbNameEn = req.EdbName
 	edbInfo.UnitEn = req.Unit
 	edbInfo.EdbType = obj.GetEdbType()
+	edbInfo.Sort = GetAddEdbMaxSortByClassifyId(req.ClassifyId, utils.EDB_INFO_TYPE)
 	newEdbInfoId, tmpErr := to.Insert(edbInfo)
 	if tmpErr != nil {
 		err = tmpErr

models/edb_data_calculate_sum.go

@@ -54,6 +54,7 @@ func (obj CalculateSum) Add(params AddCalculateBatchParams) (edbInfo *EdbInfo, e
 	edbInfo.ModifyTime = time.Now()
 	edbInfo.UniqueCode = params.UniqueCode
 	edbInfo.EdbType = obj.GetEdbType()
+	edbInfo.Sort = GetAddEdbMaxSortByClassifyId(req.ClassifyId, utils.EDB_INFO_TYPE)
 	edbInfo.EmptyType = req.EmptyType
 	edbInfo.MaxEmptyType = req.MaxEmptyType
 	edbInfo.Extra = req.Extra

models/edb_data_calculate_tbz.go

@@ -45,6 +45,7 @@ func AddCalculateTbz(req *EdbInfoCalculateBatchSaveReq, fromEdbInfo *EdbInfo, ed
 		edbInfo.EdbNameEn = req.EdbName
 		edbInfo.UnitEn = req.Unit
 		edbInfo.EdbType = 2
+		edbInfo.Sort = GetAddEdbMaxSortByClassifyId(req.ClassifyId, utils.EDB_INFO_TYPE)
 		newEdbInfoId, tmpErr := to.Insert(edbInfo)
 		if tmpErr != nil {
 			err = tmpErr

models/edb_data_calculate_tcz.go

@@ -46,6 +46,7 @@ func AddCalculateTcz(req *EdbInfoCalculateBatchSaveReq, fromEdbInfo *EdbInfo, ed
 		edbInfo.EdbNameEn = req.EdbName
 		edbInfo.UnitEn = req.Unit
 		edbInfo.EdbType = 2
+		edbInfo.Sort = GetAddEdbMaxSortByClassifyId(req.ClassifyId, utils.EDB_INFO_TYPE)
 		newEdbInfoId, tmpErr := to.Insert(edbInfo)
 		if tmpErr != nil {
 			err = tmpErr

models/edb_data_calculate_time_shift.go

@@ -46,6 +46,7 @@ func AddCalculateTimeShift(req *EdbInfoCalculateBatchSaveReq, fromEdbInfo *EdbIn
 		edbInfo.EdbNameEn = req.EdbName
 		edbInfo.UnitEn = req.Unit
 		edbInfo.EdbType = 2
+		edbInfo.Sort = GetAddEdbMaxSortByClassifyId(req.ClassifyId, utils.EDB_INFO_TYPE)
 		edbInfo.MoveType = req.MoveType
 		edbInfo.MoveFrequency = req.MoveFrequency
 		newEdbInfoId, tmpErr := to.Insert(edbInfo)

models/edb_data_calculate_zdyfx.go

@@ -71,7 +71,8 @@ func (obj CustomAnalysis) Add(params AddCalculateBatchParams) (edbInfo *EdbInfo,
 	edbInfo.CalculateFormula = req.Formula
 	edbInfo.EdbNameEn = req.EdbName
 	edbInfo.UnitEn = req.Unit
-	edbInfo.EdbType = 2
+	edbInfo.EdbType = obj.GetEdbType()
+	edbInfo.Sort = GetAddEdbMaxSortByClassifyId(req.ClassifyId, utils.EDB_INFO_TYPE)
 	newEdbInfoId, tmpErr := to.Insert(edbInfo)
 	if tmpErr != nil {
 		err = tmpErr

models/edb_data_calculate_zjpj.go

@@ -57,6 +57,7 @@ func AddCalculateZjpj(req *EdbInfoCalculateBatchSaveReq, firstEdbInfo, secondEdb
 			EdbNameEn:        req.EdbName,
 			UnitEn:           req.Unit,
 			EdbType:          2,
+			Sort:             GetAddEdbMaxSortByClassifyId(req.ClassifyId, utils.EDB_INFO_TYPE),
 		}
 		newEdbInfoId, tmpErr := to.Insert(edbInfo)
 		if tmpErr != nil {

models/edb_data_calculate_zsxy.go

@@ -52,6 +52,7 @@ func (obj ExponentialSmoothing) Add(params AddCalculateBatchParams) (edbInfo *Ed
 	edbInfo.EdbNameEn = req.EdbName
 	edbInfo.UnitEn = req.Unit
 	edbInfo.EdbType = obj.GetEdbType()
+	edbInfo.Sort = GetAddEdbMaxSortByClassifyId(req.ClassifyId, utils.EDB_INFO_TYPE)
 	newEdbInfoId, tmpErr := to.Insert(edbInfo)
 	if tmpErr != nil {
 		err = tmpErr

models/edb_data_table.go

@@ -8,10 +8,12 @@ import (
 func GetEdbDataTableName(source, subSource int) (tableName string) {
 	switch source {
 	case utils.DATA_SOURCE_THS:
-		tableName = "edb_data_ths"
-		if subSource == utils.DATA_SUB_SOURCE_DATE {
+		switch subSource {
+		case utils.DATA_SUB_SOURCE_DATE:
 			tableName = "edb_data_ths_ds"
-		} else {
+		case utils.DATA_SUB_SOURCE_HIGH_FREQUENCY:
+			tableName = "edb_data_ths_hf"
+		default:
 			tableName = "edb_data_ths"
 		}
 	case utils.DATA_SOURCE_WIND:
@@ -152,7 +154,7 @@ func GetEdbDataTableName(source, subSource int) (tableName string) {
 		tableName = "edb_data_predict_ccalculate_standard_deviation"
 	case utils.DATA_SOURCE_PREDICT_CALCULATE_PERCENTILE: //预测百分位->70
 		tableName = "edb_data_predict_ccalculate_percentile"
-	case utils.DATA_SOURCE_FUBAO:
+	case utils.DATA_SOURCE_FUBAO: //数宝->71
 		tableName = "edb_data_fubao"
 	case utils.DATA_SOURCE_CALCULATE_ZSXY:
 		tableName = "edb_data_calculate_zsxy" // 指数修匀->72

models/edb_data_ths_hf.go

@@ -0,0 +1,181 @@
+package models
+
+import (
+	"eta/eta_index_lib/utils"
+	"fmt"
+	"github.com/beego/beego/v2/client/orm"
+	"strings"
+	"time"
+)
+
+// EdbDataThsHf 同花顺高频指标数据
+type EdbDataThsHf struct {
+	EdbDataId     int       `orm:"column(edb_data_id);pk"`
+	EdbInfoId     int       `description:"指标ID"`
+	EdbCode       string    `description:"指标编码"`
+	DataTime      time.Time `description:"数据日期"`
+	Value         float64   `description:"数据值"`
+	CreateTime    time.Time `description:"创建时间"`
+	ModifyTime    time.Time `description:"修改时间"`
+	DataTimestamp int64     `description:"数据日期时间戳"`
+}
+
+func (m *EdbDataThsHf) TableName() string {
+	return "edb_data_ths_hf"
+}
+
+type EdbDataThsHfCols struct {
+	PrimaryId     string
+	EdbInfoId     string
+	EdbCode       string
+	DataTime      string
+	Value         string
+	CreateTime    string
+	ModifyTime    string
+	DataTimestamp string
+}
+
+func (m *EdbDataThsHf) Cols() EdbDataThsHfCols {
+	return EdbDataThsHfCols{
+		PrimaryId:     "edb_data_id",
+		EdbInfoId:     "edb_info_id",
+		EdbCode:       "edb_code",
+		DataTime:      "data_time",
+		Value:         "value",
+		CreateTime:    "create_time",
+		ModifyTime:    "modify_time",
+		DataTimestamp: "data_timestamp",
+	}
+}
+
+func (m *EdbDataThsHf) Create() (err error) {
+	o := orm.NewOrm()
+	id, err := o.Insert(m)
+	if err != nil {
+		return
+	}
+	m.EdbDataId = int(id)
+	return
+}
+
+func (m *EdbDataThsHf) CreateMulti(items []*EdbDataThsHf) (err error) {
+	if len(items) == 0 {
+		return
+	}
+	o := orm.NewOrm()
+	_, err = o.InsertMulti(500, items)
+	return
+}
+
+func (m *EdbDataThsHf) Update(cols []string) (err error) {
+	o := orm.NewOrm()
+	_, err = o.Update(m, cols...)
+	return
+}
+
+func (m *EdbDataThsHf) Remove() (err error) {
+	o := orm.NewOrm()
+	sql := fmt.Sprintf(`DELETE FROM %s WHERE %s = ? LIMIT 1`, m.TableName(), m.Cols().PrimaryId)
+	_, err = o.Raw(sql, m.EdbDataId).Exec()
+	return
+}
+
+func (m *EdbDataThsHf) MultiRemove(ids []int) (err error) {
+	if len(ids) == 0 {
+		return
+	}
+	o := orm.NewOrm()
+	sql := fmt.Sprintf(`DELETE FROM %s WHERE %s IN (%s)`, m.TableName(), m.Cols().PrimaryId, utils.GetOrmInReplace(len(ids)))
+	_, err = o.Raw(sql, ids).Exec()
+	return
+}
+
+func (m *EdbDataThsHf) RemoveByCondition(condition string, pars []interface{}) (err error) {
+	if condition == "" {
+		return
+	}
+	o := orm.NewOrm()
+	sql := fmt.Sprintf(`DELETE FROM %s WHERE %s`, m.TableName(), condition)
+	_, err = o.Raw(sql, pars).Exec()
+	return
+}
+
+func (m *EdbDataThsHf) GetItemById(id int) (item *EdbDataThsHf, err error) {
+	o := orm.NewOrm()
+	sql := fmt.Sprintf(`SELECT * FROM %s WHERE %s = ? LIMIT 1`, m.TableName(), m.Cols().PrimaryId)
+	err = o.Raw(sql, id).QueryRow(&item)
+	return
+}
+
+func (m *EdbDataThsHf) GetItemByCondition(condition string, pars []interface{}, orderRule string) (item *EdbDataThsHf, err error) {
+	o := orm.NewOrm()
+	order := ``
+	if orderRule != "" {
+		order = ` ORDER BY ` + orderRule
+	}
+	sql := fmt.Sprintf(`SELECT * FROM %s WHERE 1=1 %s %s LIMIT 1`, m.TableName(), condition, order)
+	err = o.Raw(sql, pars).QueryRow(&item)
+	return
+}
+
+func (m *EdbDataThsHf) GetCountByCondition(condition string, pars []interface{}) (count int, err error) {
+	o := orm.NewOrm()
+	sql := fmt.Sprintf(`SELECT COUNT(1) FROM %s WHERE 1=1 %s`, m.TableName(), condition)
+	err = o.Raw(sql, pars).QueryRow(&count)
+	return
+}
+
+func (m *EdbDataThsHf) GetItemsByCondition(condition string, pars []interface{}, fieldArr []string, orderRule string) (items []*EdbDataThsHf, err error) {
+	o := orm.NewOrm()
+	fields := strings.Join(fieldArr, ",")
+	if len(fieldArr) == 0 {
+		fields = `*`
+	}
+	order := fmt.Sprintf(`ORDER BY %s DESC`, m.Cols().CreateTime)
+	if orderRule != "" {
+		order = ` ORDER BY ` + orderRule
+	}
+	sql := fmt.Sprintf(`SELECT %s FROM %s WHERE 1=1 %s %s`, fields, m.TableName(), condition, order)
+	_, err = o.Raw(sql, pars).QueryRows(&items)
+	return
+}
+
+func (m *EdbDataThsHf) GetPageItemsByCondition(condition string, pars []interface{}, fieldArr []string, orderRule string, startSize, pageSize int) (items []*EdbDataThsHf, err error) {
+	o := orm.NewOrm()
+	fields := strings.Join(fieldArr, ",")
+	if len(fieldArr) == 0 {
+		fields = `*`
+	}
+	order := fmt.Sprintf(`ORDER BY %s DESC`, m.Cols().CreateTime)
+	if orderRule != "" {
+		order = ` ORDER BY ` + orderRule
+	}
+	sql := fmt.Sprintf(`SELECT %s FROM %s WHERE 1=1 %s %s LIMIT ?,?`, fields, m.TableName(), condition, order)
+	_, err = o.Raw(sql, pars, startSize, pageSize).QueryRows(&items)
+	return
+}
+
+// MultiUpdateValue 批量更新数据值
+func (m *EdbDataThsHf) MultiUpdateValue(items []*EdbDataThsHf) (err error) {
+	if len(items) == 0 {
+		return
+	}
+
+	o := orm.NewOrm()
+	sql := fmt.Sprintf("UPDATE %s SET %s = ?, %s = NOW() WHERE %s = ?", m.TableName(), m.Cols().Value, m.Cols().ModifyTime, m.Cols().PrimaryId)
+	p, e := o.Raw(sql).Prepare()
+	if e != nil {
+		err = fmt.Errorf("update sql prepare err: %v", e)
+		return
+	}
+	defer func() {
+		_ = p.Close()
+	}()
+	for _, v := range items {
+		_, err = p.Exec(v.Value, v.EdbDataId)
+		if err != nil {
+			return
+		}
+	}
+	return
+}

models/edb_info.go

@@ -6,11 +6,13 @@ import (
 	"eta/eta_index_lib/models/mgo"
 	"eta/eta_index_lib/utils"
 	"fmt"
-	"github.com/beego/beego/v2/client/orm"
-	"github.com/shopspring/decimal"
-	"go.mongodb.org/mongo-driver/bson"
 	"strconv"
 	"time"
+
+	"go.mongodb.org/mongo-driver/bson"
+
+	"github.com/beego/beego/v2/client/orm"
+	"github.com/shopspring/decimal"
 )
 
 type EdbInfo struct {
@@ -33,6 +35,7 @@ type EdbInfo struct {
 	UniqueCode       string `description:"指标唯一编码"`
 	CreateTime       time.Time
 	ModifyTime       time.Time
+	BaseModifyTime   time.Time
 	MinValue         float64 `description:"指标最小值"`
 	MaxValue         float64 `description:"指标最大值"`
 	CalculateFormula string  `description:"计算公式"`
@@ -209,7 +212,7 @@ type FindEdbDataListAllCond struct {
 // @return item []*EdbInfoSearchData
 // @return err error
 func GetEdbDataListAll(source, subSource int, findEdbDataListAllCond FindEdbDataListAllCond, order int) (item []*EdbInfoSearchData, err error) {
-	if source == utils.DATA_SOURCE_BUSINESS {
+	if source == utils.DATA_SOURCE_BUSINESS && utils.UseMongo {
 		return GetEdbDataListAllByMongo(source, subSource, findEdbDataListAllCond, order)
 	}
 
@@ -277,7 +280,7 @@ func GetEdbDataListAllByMysql(source, subSource int, findEdbDataListAllCond Find
 // @return item []*EdbInfoSearchData
 // @return err error
 func GetEdbDataListAllByTo(to orm.TxOrmer, source, subSource int, findEdbDataListAllCond FindEdbDataListAllCond, order int) (item []*EdbInfoSearchData, err error) {
-	if source == utils.DATA_SOURCE_BUSINESS {
+	if source == utils.DATA_SOURCE_BUSINESS && utils.UseMongo {
 		return GetEdbDataListAllByMongo(source, subSource, findEdbDataListAllCond, order)
 	}
 
@@ -462,7 +465,7 @@ func ModifyEdbDataUpdateTime(edbInfoId int, dataUpdateTime, erDataUpdateDate str
 // @return count int
 // @return err error
 func GetLteZeroEdbDataCount(source, subSource, edbInfoId int) (count int, err error) {
-	if source == utils.DATA_SOURCE_BUSINESS {
+	if source == utils.DATA_SOURCE_BUSINESS && utils.UseMongo {
 		return GetLteZeroEdbDataCountByMongo(source, subSource, edbInfoId)
 	}
 
@@ -1395,12 +1398,6 @@ func EdbInfoAdd(req *AddEdbInfoParams, serverUrl string, sysUserId int, sysUserR
 			}
 		}
 	}
-	//获取该层级下最大的排序数
-	maxSort, err := GetEdbAndClassifyMaxSort(req.ClassifyId, 0)
-	if err != nil {
-		err = errors.New("查询排序信息失败，Err:" + err.Error())
-		return
-	}
 
 	edbInfo.EdbCode = req.EdbCode
 	edbInfo.EdbName = req.EdbName
@@ -1417,7 +1414,7 @@ func EdbInfoAdd(req *AddEdbInfoParams, serverUrl string, sysUserId int, sysUserR
 	edbInfo.ModifyTime = time.Now()
 	edbInfo.ServerUrl = serverUrl
 	edbInfo.TerminalCode = req.TerminalCode
-	edbInfo.Sort = maxSort + 1
+	edbInfo.Sort = GetAddEdbMaxSortByClassifyId(req.ClassifyId, 0)
 	timestamp := strconv.FormatInt(time.Now().UnixNano(), 10)
 	edbInfo.UniqueCode = utils.MD5(utils.DATA_PREFIX + "_" + req.EdbCode + timestamp)
 	itemVal, err := GetEdbInfoMaxAndMinInfo(source, utils.DATA_SUB_SOURCE_EDB, req.EdbCode)
@@ -1436,6 +1433,14 @@ func EdbInfoAdd(req *AddEdbInfoParams, serverUrl string, sysUserId int, sysUserR
 	return
 }
 
+func ModifyEdbInfoBaseTimeById(edbInfoId int, cTime time.Time) (err error) {
+	o := orm.NewOrm()
+	// 更新修改时间
+	sql := ` UPDATE edb_info SET base_modify_time = ? WHERE edb_info_id = ? `
+	_, err = o.Raw(sql, cTime, edbInfoId).Exec()
+	return
+}
+
 // EdbInfoRefreshCheckReq 指标数据更新情况查询
 type EdbInfoRefreshCheckReq struct {
 	Source         int    `description:"来源id"`
@@ -1492,6 +1497,17 @@ type CalculateLjzEdbExtra struct {
 	LastValType int `description:"最新值处理：0默认、均值填充"`
 }
 
+type EdbInfoEditRecord struct {
+	EdbInfoId           int    `description:"指标ID"`
+	EdbName             string `description:"指标名称"`
+	Frequency           string `description:"频率"`
+	Unit                string `description:"单位"`
+	ClassifyId          int    `description:"分类id"`
+	CalculateFormula    string `description:"计算公式"`
+	OperateUserId       int    `description:"操作人id"`
+	OperateUserRealName string `description:"操作人姓名"`
+}
+
 // GetEdbInfoByEdbCodeList
 // @Description: 根据来源和指标编码列表获取指标信息列表
 // @author: Roc
@@ -1511,3 +1527,68 @@ func GetEdbInfoByEdbCodeList(source int, edbCodeList []string) (items []*EdbInfo
 
 	return
 }
+
+// EdbInfoExtra 指标额外数据-extra字段
+type EdbInfoExtra struct {
+	ApiExtraPars string `description:"API-额外参数(如同花顺日期序列)"`
+}
+
+// GetEdbInfoNoUpdateTotalByIdList 根据指标id列表获取指标信息
+func GetEdbInfoNoUpdateTotalByIdList(edbInfoIdList []int) (total int, err error) {
+	num := len(edbInfoIdList)
+	if num <= 0 {
+		return
+	}
+	o := orm.NewOrm()
+	sql := ` SELECT count(*) FROM edb_info WHERE edb_info_id in (` + utils.GetOrmInReplace(num) + `)  and no_update=1`
+	err = o.Raw(sql, edbInfoIdList).QueryRow(&total)
+	return
+}
+
+func TransEdbInfoSearchData2DataList(items []*EdbInfoSearchData) (list []*EdbDataList) {
+	if len(items) == 0 {
+		return
+	}
+	list = make([]*EdbDataList, 0)
+	for _, v := range items {
+		list = append(list, &EdbDataList{
+			EdbDataId:     v.EdbDataId,
+			EdbInfoId:     v.EdbInfoId,
+			DataTime:      v.DataTime,
+			DataTimestamp: v.DataTimestamp,
+			Value:         v.Value,
+		})
+	}
+	return
+}
+
+func TransEdbInfoDataList2SearchData(items []*EdbDataList) (list []*EdbInfoSearchData) {
+	if len(items) == 0 {
+		return
+	}
+	list = make([]*EdbInfoSearchData, 0)
+	for _, v := range items {
+		list = append(list, &EdbInfoSearchData{
+			EdbDataId:     v.EdbDataId,
+			EdbInfoId:     v.EdbInfoId,
+			DataTime:      v.DataTime,
+			DataTimestamp: v.DataTimestamp,
+			Value:         v.Value,
+		})
+	}
+	return
+}
+
+// GetAddEdbMaxSortByClassifyId
+// @Description: 获取添加指标时，该分类下最大的排序（忽略错误信息）
+// @author: Roc
+// @datetime 2024-07-05 09:39:46
+// @param classifyId int
+// @param classifyType uint8
+// @return sort int
+func GetAddEdbMaxSortByClassifyId(classifyId int, classifyType uint8) (sort int) {
+	sort, _ = GetEdbAndClassifyMaxSort(classifyId, classifyType)
+	sort = sort + 1
+
+	return
+}

models/edb_info_calculate_mapping.go

@@ -1,6 +1,7 @@
 package models
 
 import (
+	"eta/eta_index_lib/utils"
 	"github.com/beego/beego/v2/client/orm"
 	"time"
 )
@@ -25,6 +26,31 @@ type EdbInfoCalculateMapping struct {
 	FromSubSource             int       `description:"渠道子数据库来源"`
 }
 
+// EdbInfoCalculateMappingInfo
+// @Description: 计算指标与基础指标关系表
+type EdbInfoCalculateMappingInfo struct {
+	EdbInfoCalculateMappingId int       `orm:"column(edb_info_calculate_mapping_id);pk"`
+	EdbInfoId                 int       `description:"计算指标id"`
+	Source                    int       `description:"计算指标来源"`
+	SourceName                string    `description:"计算指标来源名称"`
+	EdbCode                   string    `description:"计算指标编码"`
+	FromEdbInfoId             int       `description:"基础指标id"`
+	FromEdbCode               string    `description:"基础指标编码"`
+	FromEdbName               string    `description:"基础指标名称"`
+	FromSource                int       `description:"基础指标来源"`
+	FromSourceName            string    `description:"基础指标来源名称"`
+	MoveValue                 int       `description:"领先值"`
+	FromTag                   string    `description:"来源指标标签"`
+	Sort                      int       `description:"计算指标名称排序"`
+	CreateTime                time.Time `description:"创建时间"`
+	ModifyTime                time.Time `description:"修改时间"`
+	FromEdbType               int       `description:"来源指标类型:1:基础指标,2:计算指标"`
+	FromEdbInfoType           int       `description:"来源指标类型: 0-基础指标; 1-预测指标"`
+	FromClassifyId            int       `description:"来源指标分类ID"`
+	FromUniqueCode            string    `description:"来源指标唯一编码"`
+	NoUpdate                  int8      `description:"是否停止更新，0：继续更新；1：停止更新"`
+}
+
 // AddEdbInfoCalculateMappingMulti 批量添加指标关系表
 func AddEdbInfoCalculateMappingMulti(items []*EdbInfoCalculateMapping) (err error) {
 	o := orm.NewOrm()
@@ -119,3 +145,23 @@ func GetEdbInfoCalculateByEdbCode(edbCode, fromEdbCode string) (item *EdbInfoCal
 	err = o.Raw(sql, edbCode, fromEdbCode).QueryRow(&item)
 	return
 }
+
+// GetEdbInfoCalculateMappingListByEdbInfoIds 根据生成的指标id获取来源的指标id列表
+func GetEdbInfoCalculateMappingListByEdbInfoIds(edbInfoIds []int) (items []*EdbInfoCalculateMappingInfo, err error) {
+	o := orm.NewOrm()
+	sql := ` SELECT a.*,b.edb_type as from_edb_type,b.edb_info_type as from_edb_info_type, b.unique_code AS from_unique_code, b.classify_id AS from_classify_id,b.no_update FROM edb_info_calculate_mapping AS a
+			INNER JOIN edb_info AS b ON a.from_edb_info_id=b.edb_info_id
+			WHERE a.edb_info_id in (` + utils.GetOrmInReplace(len(edbInfoIds)) + `) `
+	_, err = o.Raw(sql, edbInfoIds).QueryRows(&items)
+	return
+}
+
+// GetEdbInfoCalculateMappingListByEdbInfoId 根据生成的指标id获取来源的指标id列表
+func GetEdbInfoCalculateMappingListByEdbInfoId(edbInfoId int) (items []*EdbInfoCalculateMappingInfo, err error) {
+	o := orm.NewOrm()
+	sql := ` SELECT a.*,b.edb_type as from_edb_type,b.edb_info_type as from_edb_info_type, b.unique_code AS from_unique_code, b.classify_id AS from_classify_id,b.no_update FROM edb_info_calculate_mapping AS a
+			INNER JOIN edb_info AS b ON a.from_edb_info_id=b.edb_info_id
+			WHERE a.edb_info_id=? `
+	_, err = o.Raw(sql, edbInfoId).QueryRows(&items)
+	return
+}

models/edb_info_record.go

@@ -0,0 +1,28 @@
+package models
+
+import (
+	"time"
+
+	"github.com/beego/beego/v2/client/orm"
+)
+
+type EdbInfoRecord struct {
+	EdbInfoRecordId     int       `orm:"column(edb_info_record_id);pk"`
+	EdbInfoId           int       `orm:"column(edb_info_id)"`
+	OldEdbName          string    `description:"旧的指标名称"`
+	OldFrequency        string    `description:"旧的频率"`
+	OldUnit             string    `description:"旧的单位"`
+	NewEdbName          string    `description:"新的指标名称"`
+	NewFrequency        string    `description:"新的频率"`
+	NewUnit             string    `description:"新的单位"`
+	OperateUserId       int       `description:"执行人id"`
+	OperateUserRealName string    `description:"执行人名称"`
+	CreateTime          time.Time `description:"记录的生成时间"`
+	Timestamp           int64     `description:"时间戳"`
+}
+
+func AddEditEdbInfoRecord(edbRecord *EdbInfoRecord) (e error) {
+	o := orm.NewOrm()
+	_, e = o.Insert(edbRecord)
+	return
+}

models/edb_info_relation.go

@@ -0,0 +1,136 @@
+package models
+
+import (
+	"eta/eta_index_lib/utils"
+	"github.com/beego/beego/v2/client/orm"
+	"time"
+)
+
+type EdbInfoRelation struct {
+	EdbInfoRelationId  int       `orm:"column(edb_info_relation_id);pk"`
+	EdbInfoId          int       `description:"指标id"`
+	Source             int       `description:"来源:1:同花顺,2:wind,3:彭博,4:指标运算,5:累计值转月,6:同比值,7:同差值,8:N数值移动平均计算,9:手工指标,10:隆众"`
+	EdbName            string    `description:"指标名称"`
+	EdbCode            string    `description:"指标编码"`
+	ReferObjectId      int       `description:"引用对象ID（图表ID，ETA逻辑ID等）"`
+	ReferObjectType    int       `description:"引用对象ID类型（1.图表，2.ETA逻辑）"`
+	ReferObjectSubType int       `description:"引用对象子类"`
+	CreateTime         time.Time `description:"创建时间"`
+	ModifyTime         time.Time `description:"修改时间"`
+	RelationTime       time.Time `description:"引用时间"`
+	RelationType       int       `description:"引用类型，0:直接饮用，1间接引用"`
+	RootEdbInfoId      int       `description:"间接引用时，关联的直接引用的指标ID"`
+	ChildEdbInfoId     int       `description:"间接引用时，计算指标直接关联的指标ID"`
+	RelationCode       string    `description:"引用标识"`
+	ParentRelationId   int       `description:"间接引用关联的直接引用的ID"`
+}
+
+func (e *EdbInfoRelation) TableName() string {
+	return "edb_info_relation"
+}
+
+// GetEdbInfoRelationByChildEdbInfoId 查询引用的指标ID
+func GetEdbInfoRelationByChildEdbInfoId(edbInfoId int) (item *EdbInfoRelation, err error) {
+	o := orm.NewOrm()
+	msql := ` SELECT * FROM edb_info_relation WHERE child_edb_info_id = ? or edb_info_id=?`
+	err = o.Raw(msql, edbInfoId, edbInfoId).QueryRow(&item)
+	return
+}
+
+// GetEdbInfoRelationListByChildEdbInfoId 根据间接引用中的的计算指标ID查询引用列表
+func GetEdbInfoRelationListByChildEdbInfoId(edbInfoId int) (items []*EdbInfoRelation, err error) {
+	o := orm.NewOrm()
+	msql := ` SELECT * FROM edb_info_relation WHERE relation_type=1 AND child_edb_info_id=?`
+	_, err = o.Raw(msql, edbInfoId).QueryRows(&items)
+	return
+}
+
+// GetEdbInfoRelationListByParentRelationId 根据间接引用中的的父级ID查询
+func GetEdbInfoRelationEdbIdsByParentRelationId(relationId, edbInfoId int) (items []int, err error) {
+	o := orm.NewOrm()
+	msql := ` SELECT edb_info_id FROM edb_info_relation WHERE parent_relation_id=? AND child_edb_info_id=?`
+	_, err = o.Raw(msql, relationId, edbInfoId).QueryRows(&items)
+	return
+}
+
+// GetEdbInfoRelationByRelationIds 查询引用的指标ID
+func GetEdbInfoRelationByRelationIds(ids []int) (items []*EdbInfoRelation, err error) {
+	o := orm.NewOrm()
+	msql := ` SELECT * FROM edb_info_relation WHERE edb_info_relation_id in (` + utils.GetOrmInReplace(len(ids)) + `) `
+	_, err = o.Raw(msql, ids).QueryRows(&items)
+	return
+}
+
+// UpdateSecondRelationEdbInfoId 更新指标替换后的间接引用记录
+func UpdateSecondRelationEdbInfoId(edbRelationIds []int, relationList []*EdbInfoRelation, refreshEdbInfoIds []int, indexCodeList []string) (err error) {
+	o, err := orm.NewOrm().Begin()
+	if err != nil {
+		return
+	}
+	defer func() {
+		if err != nil {
+			_ = o.Rollback()
+			return
+		}
+		_ = o.Commit()
+	}()
+	// 删除相关的间接引用
+	sql := ` DELETE FROM edb_info_relation WHERE relation_type=1 and parent_relation_id in (` + utils.GetOrmInReplace(len(edbRelationIds)) + `)`
+	_, err = o.Raw(sql, edbRelationIds).Exec()
+	if err != nil {
+		return
+	}
+
+	// 新增间接引用
+	relationCodesMap := make(map[string]struct{}, 0)
+	if len(relationList) > 0 {
+		for _, relation := range relationList {
+			if relation.RelationType == 1 {
+				relationCodesMap[relation.RelationCode] = struct{}{}
+			}
+		}
+		_, err = o.InsertMulti(len(relationList), relationList)
+		if err != nil {
+			return
+		}
+	}
+
+	if len(refreshEdbInfoIds) > 0 {
+		// todo 更新指标的刷新状态
+		sql = ` UPDATE edb_info SET no_update = 0 WHERE  edb_info_id IN (` + utils.GetOrmInReplace(len(refreshEdbInfoIds)) + `) AND no_update = 1`
+		_, err = o.Raw(sql, refreshEdbInfoIds).Exec()
+		if err != nil {
+			return
+		}
+	}
+
+	//更新数据源钢联化工指标
+	if len(indexCodeList) > 0 {
+		// 更改数据源的更新状态
+		sql = ` UPDATE base_from_mysteel_chemical_index SET is_stop = 0 WHERE index_code IN (` + utils.GetOrmInReplace(len(indexCodeList)) + `) and is_stop=1`
+		_, err = o.Raw(sql, indexCodeList).Exec()
+		if err != nil {
+			return
+		}
+	}
+	if len(relationList) > 0 {
+		// 更新间接引用指标的关联ID
+		relationCodes := make([]string, 0)
+		for relationCode := range relationCodesMap {
+			relationCodes = append(relationCodes, relationCode)
+		}
+		if len(relationCodes) > 0 {
+			sql := ` UPDATE edb_info_relation e1  
+JOIN edb_info_relation e2 ON e1.relation_code = e2.relation_code   
+SET e1.parent_relation_id = e2.edb_info_relation_id  
+WHERE  
+    e1.relation_type = 1   
+    AND e2.relation_type = 0 AND e1.parent_relation_id !=e2.edb_info_relation_id AND e1.relation_code in (` + utils.GetOrmInReplace(len(relationCodes)) + `)`
+			_, err = o.Raw(sql, relationCodes).Exec()
+			if err != nil {
+				return
+			}
+		}
+	}
+	return
+}

models/edb_ths_hf.go

@@ -0,0 +1,924 @@
+package models
+
+import (
+	"encoding/json"
+	"eta/eta_index_lib/models/mgo"
+	"eta/eta_index_lib/utils"
+	"fmt"
+	"github.com/beego/beego/v2/client/orm"
+	"github.com/shopspring/decimal"
+	"go.mongodb.org/mongo-driver/bson"
+	"reflect"
+	"sort"
+	"time"
+)
+
+// EdbThsHf 自有数据
+type EdbThsHf struct{}
+
+// GetSource 获取来源编码id
+func (obj EdbThsHf) GetSource() int {
+	return utils.DATA_SOURCE_THS
+}
+
+// GetSubSource 获取子来源编码id
+func (obj EdbThsHf) GetSubSource() int {
+	return utils.DATA_SUB_SOURCE_HIGH_FREQUENCY
+}
+
+// GetSourceName 获取来源名称
+func (obj EdbThsHf) GetSourceName() string {
+	return utils.DATA_SOURCE_NAME_THS
+}
+
+// GetEdbType 获取指标类型
+func (obj EdbThsHf) GetEdbType() int {
+	return utils.DEFAULT_EDB_TYPE
+}
+
+// ThsHfAddBaseParams
+// @Description: 基础指标的添加参数
+type ThsHfAddBaseParams struct {
+	EdbCode         string `description:"指标编码"`
+	EdbName         string `description:"指标名称"`
+	Unit            string `description:"单位"`
+	Frequency       string `description:"频度"`
+	Sort            int    `description:"排序"`
+	ClassifyId      int    `description:"所属分类"`
+	SysUserId       int    `description:"用户id"`
+	SysUserRealName string `description:"用户真实名称"`
+	UniqueCode      string `description:"唯一编码"`
+	ConvertRule     string `description:"转换规则"`
+}
+
+// ThsHfEditBaseParams
+// @Description: 基础指标的修改参数
+type ThsHfEditBaseParams struct {
+	EdbCode         string   `description:"指标编码"`
+	EdbName         string   `description:"指标名称"`
+	EdbNameEn       string   `description:"指标名称（英文）"`
+	Unit            string   `description:"单位"`
+	UnitEn          string   `description:"单位（英文）"`
+	ClassifyId      int      `description:"所属分类"`
+	SysUserId       int      `description:"用户id"`
+	SysUserRealName string   `description:"用户真实名称"`
+	UniqueCode      string   `description:"编码"`
+	Lang            string   `description:"语言版本"`
+	EdbInfo         *EdbInfo `description:"指标信息"`
+}
+
+type ThsHfRefreshBaseParams struct {
+	EdbInfo   *EdbInfo
+	StartDate string
+	EndDate   string
+}
+
+// Add
+// @Description: 添加指标
+func (obj EdbThsHf) Add(params ThsHfAddBaseParams, baseIndex *BaseFromThsHfIndex) (edbInfo *EdbInfo, err error) {
+	o := orm.NewOrm()
+	tx, e := o.Begin()
+	if e != nil {
+		err = fmt.Errorf("orm begin err: %v", e)
+		return
+	}
+	defer func() {
+		if err != nil {
+			_ = tx.Rollback()
+			utils.FileLog.Info(fmt.Sprintf("%s err: %v", reflect.TypeOf(obj).Name(), err))
+			return
+		}
+		_ = tx.Commit()
+	}()
+
+	// 新增指标
+	edbInfo = new(EdbInfo)
+	edbInfo.Source = obj.GetSource()
+	edbInfo.SubSource = obj.GetSubSource()
+	edbInfo.SourceName = obj.GetSourceName()
+	edbInfo.EdbType = obj.GetEdbType()
+	edbInfo.EdbCode = params.EdbCode
+	edbInfo.EdbName = params.EdbName
+	edbInfo.EdbNameEn = params.EdbName
+	edbInfo.EdbNameSource = params.EdbName
+	edbInfo.Frequency = params.Frequency
+	edbInfo.Unit = params.Unit
+	edbInfo.UnitEn = params.Unit
+	edbInfo.StartDate = baseIndex.StartDate.Format(utils.FormatDate) // 默认取源指标的时间, 刷新完成后更新
+	edbInfo.EndDate = baseIndex.EndDate.Format(utils.FormatDate)
+	edbInfo.ClassifyId = params.ClassifyId
+	edbInfo.SysUserId = params.SysUserId
+	edbInfo.SysUserRealName = params.SysUserRealName
+	edbInfo.Sort = params.Sort
+	edbInfo.TerminalCode = baseIndex.TerminalCode
+	edbInfo.UniqueCode = params.UniqueCode
+	edbInfo.CreateTime = time.Now()
+	edbInfo.ModifyTime = time.Now()
+	edbInfoId, e := tx.Insert(edbInfo)
+	if e != nil {
+		err = fmt.Errorf("insert edb err: %v", e)
+		return
+	}
+	edbInfo.EdbInfoId = int(edbInfoId)
+
+	// 新增指标关联
+	edbMapping := new(BaseFromEdbMapping)
+	edbMapping.BaseFromIndexId = baseIndex.BaseFromThsHfIndexId
+	edbMapping.BaseIndexCode = baseIndex.IndexCode
+	edbMapping.EdbInfoId = edbInfo.EdbInfoId
+	edbMapping.EdbCode = edbInfo.EdbCode
+	edbMapping.Source = obj.GetSource()
+	edbMapping.SubSource = obj.GetSubSource()
+	edbMapping.ConvertRule = params.ConvertRule
+	edbMapping.CreateTime = time.Now().Local()
+	edbMapping.ModifyTime = time.Now().Local()
+	edbMappingId, e := tx.Insert(edbMapping)
+	if e != nil {
+		err = fmt.Errorf("insert base edb mapping err: %v", e)
+		return
+	}
+	edbMapping.Id = int(edbMappingId)
+
+	// 刷新数据
+	err = obj.Refresh(edbInfo, edbMapping, "")
+	return
+}
+
+func (obj EdbThsHf) Refresh(edbInfo *EdbInfo, edbBaseMapping *BaseFromEdbMapping, startDate string) (err error) {
+	if utils.UseMongo {
+		return obj.refreshByMongo(edbInfo, edbBaseMapping, startDate)
+	}
+	return obj.refreshByMysql(edbInfo, edbBaseMapping, startDate)
+}
+
+func (obj EdbThsHf) refreshByMysql(edbInfo *EdbInfo, edbBaseMapping *BaseFromEdbMapping, startDate string) (err error) {
+	if edbInfo == nil || edbBaseMapping == nil {
+		err = fmt.Errorf("指标信息/关联信息有误, EdbInfo: %v, EdbBaseMapping: %v", edbInfo, edbBaseMapping)
+		return
+	}
+
+	// 真实数据的最大日期, 插入规则配置的日期
+	var realDataMaxDate, edbDataInsertConfigDate time.Time
+	var edbDataInsertConfig *EdbDataInsertConfig
+	var isFindConfigDateRealData bool
+	{
+		conf, e := GetEdbDataInsertConfigByEdbId(edbInfo.EdbInfoId)
+		if e != nil && e.Error() != utils.ErrNoRow() {
+			err = fmt.Errorf("GetEdbDataInsertConfigByEdbId err: %v", e)
+			return
+		}
+		edbDataInsertConfig = conf
+		if edbDataInsertConfig != nil {
+			edbDataInsertConfigDate = edbDataInsertConfig.Date
+		}
+	}
+
+	// 查询时间为开始时间-3d
+	var queryDate string
+	if startDate != "" {
+		st, e := time.ParseInLocation(utils.FormatDate, startDate, time.Local)
+		if e != nil {
+			err = fmt.Errorf("刷新开始时间有误, %v", e)
+			return
+		}
+		queryDate = st.AddDate(0, 0, -3).Format(utils.FormatDate)
+	}
+
+	// 源指标数据
+	baseDataList := make([]*BaseFromThsHfData, 0)
+	{
+		ob := new(BaseFromThsHfData)
+		cond := fmt.Sprintf(" AND %s = ?", ob.Cols().IndexCode)
+		pars := make([]interface{}, 0)
+		pars = append(pars, edbBaseMapping.BaseIndexCode)
+		if queryDate != "" {
+			cond += fmt.Sprintf(" AND %s >= ?", ob.Cols().DataTime)
+			pars = append(pars, queryDate)
+		}
+		list, e := ob.GetItemsByCondition(cond, pars, []string{}, fmt.Sprintf("%s ASC", ob.Cols().DataTime))
+		if e != nil {
+			err = fmt.Errorf("获取数据源数据失败, %v", e)
+			return
+		}
+		baseDataList = list
+	}
+
+	// 转换数据
+	convertRule := new(ThsHfIndexConvert2EdbRule)
+	if e := json.Unmarshal([]byte(edbBaseMapping.ConvertRule), &convertRule); e != nil {
+		err = fmt.Errorf("转换规则有误, %v", e)
+		return
+	}
+	convertOriginData := make([]*ThsHfConvertOriginData, 0)
+	for _, v := range baseDataList {
+		convertOriginData = append(convertOriginData, &ThsHfConvertOriginData{
+			DataTime: v.DataTime,
+			Value:    v.Value,
+		})
+	}
+	convertData, e := ThsHfConvertData2DayByRule(convertOriginData, convertRule)
+	if e != nil {
+		err = fmt.Errorf("转换数据失败, %v", e)
+		return
+	}
+	if len(convertData) == 0 {
+		utils.FileLog.Info("同花顺高频-转换无数据, EdbCode: %s", edbInfo.EdbCode)
+		return
+	}
+
+	// 获取已有数据
+	dataOb := new(EdbDataThsHf)
+	dataExists := make(map[string]*EdbDataThsHf)
+	searchExistMap := make(map[string]*EdbInfoSearchData)
+	{
+		cond := fmt.Sprintf(" AND %s = ?", dataOb.Cols().EdbInfoId)
+		pars := make([]interface{}, 0)
+		pars = append(pars, edbInfo.EdbInfoId)
+		if queryDate != "" {
+			cond += fmt.Sprintf(" AND %s >= ?", dataOb.Cols().DataTime)
+			pars = append(pars, queryDate)
+		}
+		list, e := dataOb.GetItemsByCondition(cond, pars, []string{}, "")
+		if e != nil {
+			err = fmt.Errorf("获取指标数据失败, %v", e)
+			return
+		}
+		for _, v := range list {
+			dataExists[v.DataTime.Format(utils.FormatDate)] = v
+			searchExistMap[v.DataTime.Format(utils.FormatDate)] = &EdbInfoSearchData{
+				EdbDataId:     v.EdbDataId,
+				EdbInfoId:     v.EdbInfoId,
+				DataTime:      v.DataTime.Format(utils.FormatDate),
+				Value:         v.Value,
+				EdbCode:       v.EdbCode,
+				DataTimestamp: v.DataTimestamp,
+			}
+		}
+	}
+
+	// 比对数据
+	insertExist := make(map[string]bool)
+	insertData := make([]*EdbDataThsHf, 0)
+	updateData := make([]*EdbDataThsHf, 0)
+	for k, v := range convertData {
+		strDate := k.Format(utils.FormatDate)
+
+		// 手动插入数据的判断
+		if realDataMaxDate.IsZero() || k.After(realDataMaxDate) {
+			realDataMaxDate = k
+		}
+		if edbDataInsertConfigDate.IsZero() || k.Equal(edbDataInsertConfigDate) {
+			isFindConfigDateRealData = true
+		}
+
+		// 入库值
+		saveVal := decimal.NewFromFloat(v).Round(4).String()
+		d, e := decimal.NewFromString(saveVal)
+		if e != nil {
+			utils.FileLog.Info(fmt.Sprintf("EdbDataThsHf NewFromString err: %v", e))
+			continue
+		}
+		saveFloat, _ := d.Float64()
+
+		// 更新
+		exists := dataExists[strDate]
+		if exists != nil {
+			existVal := decimal.NewFromFloat(exists.Value).Round(4).String()
+			if saveVal != existVal {
+				exists.Value = saveFloat
+				updateData = append(updateData, exists)
+			}
+			continue
+		}
+
+		// 新增
+		if insertExist[strDate] {
+			continue
+		}
+		insertExist[strDate] = true
+
+		timestamp := k.UnixNano() / 1e6
+		insertData = append(insertData, &EdbDataThsHf{
+			EdbInfoId:     edbInfo.EdbInfoId,
+			EdbCode:       edbInfo.EdbCode,
+			DataTime:      k,
+			Value:         saveFloat,
+			CreateTime:    time.Now(),
+			ModifyTime:    time.Now(),
+			DataTimestamp: timestamp,
+		})
+	}
+
+	// 批量新增/更新
+	if len(insertData) > 0 {
+		if e = dataOb.CreateMulti(insertData); e != nil {
+			err = fmt.Errorf("批量新增指标数据失败, %v", e)
+			return
+		}
+	}
+	if len(updateData) > 0 {
+		if e = dataOb.MultiUpdateValue(updateData); e != nil {
+			err = fmt.Errorf("批量更新指标数据失败, %v", e)
+			return
+		}
+	}
+
+	// 处理手工数据补充的配置
+	HandleConfigInsertEdbData(realDataMaxDate, edbDataInsertConfig, edbInfo.EdbInfoId, obj.GetSource(), obj.GetSubSource(), searchExistMap, isFindConfigDateRealData)
+	return
+}
+
+func (obj EdbThsHf) refreshByMongo(edbInfo *EdbInfo, edbBaseMapping *BaseFromEdbMapping, startDate string) (err error) {
+	defer func() {
+		if err != nil {
+			utils.FileLog.Info(fmt.Sprintf("refreshByMongo, err: %v", err))
+		}
+	}()
+
+	var realDataMaxDate, edbDataInsertConfigDate time.Time
+	var edbDataInsertConfig *EdbDataInsertConfig
+	var isFindConfigDateRealData bool //是否找到配置日期的实际数据的值
+	{
+		insertConfig, e := GetEdbDataInsertConfigByEdbId(edbInfo.EdbInfoId)
+		if e != nil && e.Error() != utils.ErrNoRow() {
+			err = fmt.Errorf("GetEdbDataInsertConfigByEdbId, err: %v", e)
+			return
+		}
+		edbDataInsertConfig = insertConfig
+		if edbDataInsertConfig != nil {
+			edbDataInsertConfigDate = edbDataInsertConfig.Date
+		}
+	}
+
+	// 查询时间为开始时间-3d
+	var queryDate string
+	if startDate != "" {
+		st, e := time.ParseInLocation(utils.FormatDate, startDate, time.Local)
+		if e != nil {
+			err = fmt.Errorf("刷新开始时间有误, %v", e)
+			return
+		}
+		queryDate = st.AddDate(0, 0, -3).Format(utils.FormatDate)
+	}
+
+	// 获取源指标数据
+	baseDataList, e := obj.getBaseIndexDataByMongo(edbBaseMapping.BaseIndexCode, queryDate)
+	if e != nil {
+		err = fmt.Errorf("getBaseIndexDataByMongo, err: %v", e)
+		return
+	}
+
+	// 转换数据
+	convertRule := new(ThsHfIndexConvert2EdbRule)
+	if e := json.Unmarshal([]byte(edbBaseMapping.ConvertRule), &convertRule); e != nil {
+		err = fmt.Errorf("转换规则有误, %v", e)
+		return
+	}
+	convertOriginData := make([]*ThsHfConvertOriginData, 0)
+	for _, v := range baseDataList {
+		convertOriginData = append(convertOriginData, &ThsHfConvertOriginData{
+			DataTime: v.DataTime,
+			Value:    v.Value,
+		})
+	}
+	convertData, e := ThsHfConvertData2DayByRule(convertOriginData, convertRule)
+	if e != nil {
+		err = fmt.Errorf("转换数据失败, %v", e)
+		return
+	}
+	if len(convertData) == 0 {
+		utils.FileLog.Info("同花顺高频-转换无数据, EdbCode: %s", edbInfo.EdbCode)
+		return
+	}
+
+	//获取指标所有数据
+	existDataList := make([]*mgo.EdbDataThsHf, 0)
+	mogDataObj := new(mgo.EdbDataThsHf)
+	{
+		// 构建查询条件
+		queryConditions := bson.M{
+			"edb_code": edbInfo.EdbCode,
+		}
+
+		if queryDate != `` {
+			//获取已存在的所有数据
+			startDateTime, e := time.ParseInLocation(utils.FormatDate, queryDate, time.Local)
+			if e != nil {
+				err = fmt.Errorf("startDateTime parse err: %v", e)
+				return
+			}
+			queryConditions["data_time"] = bson.M{"$gte": startDateTime}
+		}
+		existDataList, e = mogDataObj.GetAllDataList(queryConditions, []string{"data_time"})
+		if e != nil {
+			err = fmt.Errorf("GetAllDataList, err: %v", e)
+			return
+		}
+	}
+
+	existDataMap := make(map[string]*mgo.EdbDataThsHf)
+	removeDataTimeMap := make(map[string]bool) //需要移除的日期数据
+	for _, v := range existDataList {
+		tmpDate := v.DataTime.Format(utils.FormatDate)
+		existDataMap[tmpDate] = v
+		removeDataTimeMap[tmpDate] = true
+	}
+
+	// 待添加的数据集
+	addDataList := make([]interface{}, 0)
+	updateDataList := make([]mgo.EdbDataThsHf, 0)
+
+	insertExist := make(map[string]bool)
+	for k, v := range convertData {
+		strDate := k.Format(utils.FormatDate)
+
+		// 手动插入数据的判断
+		if realDataMaxDate.IsZero() || k.After(realDataMaxDate) {
+			realDataMaxDate = k
+		}
+		if edbDataInsertConfigDate.IsZero() || k.Equal(edbDataInsertConfigDate) {
+			isFindConfigDateRealData = true
+		}
+
+		// 入库值
+		saveVal := decimal.NewFromFloat(v).Round(4).String()
+		d, e := decimal.NewFromString(saveVal)
+		if e != nil {
+			utils.FileLog.Info(fmt.Sprintf("EdbDataThsHf NewFromString err: %v", e))
+			continue
+		}
+		saveFloat, _ := d.Float64()
+
+		// 更新
+		exists := existDataMap[strDate]
+		if exists != nil {
+			existVal := decimal.NewFromFloat(exists.Value).Round(4).String()
+			if saveVal != existVal {
+				exists.Value = saveFloat
+				updateDataList = append(updateDataList, *exists)
+			}
+			continue
+		}
+
+		// 新增
+		if insertExist[strDate] {
+			continue
+		}
+		insertExist[strDate] = true
+
+		timestamp := k.UnixNano() / 1e6
+		addDataList = append(addDataList, &mgo.EdbDataThsHf{
+			EdbInfoId:     edbInfo.EdbInfoId,
+			EdbCode:       edbInfo.EdbCode,
+			DataTime:      k,
+			Value:         saveFloat,
+			CreateTime:    time.Now(),
+			ModifyTime:    time.Now(),
+			DataTimestamp: timestamp,
+		})
+	}
+
+	// 入库
+	{
+		coll := mogDataObj.GetCollection()
+
+		//删除已经不存在的指标数据（由于该指标当日的数据删除了）
+		{
+			removeDateList := make([]time.Time, 0)
+			for dateTime := range removeDataTimeMap {
+				//获取已存在的所有数据
+				tmpDateTime, e := time.ParseInLocation(utils.FormatDate, dateTime, time.Local)
+				if e != nil {
+					err = fmt.Errorf("tmpDateTime parse err: %v", e)
+					return
+				}
+				removeDateList = append(removeDateList, tmpDateTime)
+			}
+			removeNum := len(removeDateList)
+			if removeNum > 0 {
+				if e = mogDataObj.RemoveManyByColl(coll, bson.M{"edb_code": edbInfo.EdbCode, "data_time": bson.M{"$in": removeDateList}}); e != nil {
+					err = fmt.Errorf("RemoveManyByColl, err: %v", e)
+					return
+				}
+			}
+		}
+
+		// 插入新数据
+		if len(addDataList) > 0 {
+			if e = mogDataObj.BatchInsertDataByColl(coll, 500, addDataList); e != nil {
+				err = fmt.Errorf("BatchInsertDataByColl, err: %v", e)
+				return
+			}
+		}
+
+		// 修改历史数据
+		if len(updateDataList) > 0 {
+			for _, v := range updateDataList {
+				if e = mogDataObj.UpdateDataByColl(coll, bson.M{"_id": v.ID}, bson.M{"$set": bson.M{"value": v.Value, "modify_time": v.ModifyTime}}); e != nil {
+					err = fmt.Errorf("UpdateDataByColl, err: %v", e)
+					return
+				}
+			}
+		}
+	}
+
+	// 处理手工数据补充的配置
+	obj.HandleConfigInsertEdbDataByMongo(realDataMaxDate, edbDataInsertConfig, edbInfo.EdbInfoId, edbInfo.Source, edbInfo.SubSource, existDataMap, isFindConfigDateRealData)
+	return
+}
+
+type ThsHfConvertOriginData struct {
+	DataTime time.Time `description:"数据日期（至时分秒）"`
+	Value    float64   `description:"数据值"`
+}
+
+// ThsHfConvertData2DayByRule 原指标数据转换为日度数据
+func ThsHfConvertData2DayByRule(originData []*ThsHfConvertOriginData, convertRule *ThsHfIndexConvert2EdbRule) (timeData map[time.Time]float64, err error) {
+	// PS: originData为期望开始日期前三日(有两天非交易日, 那么周一的前日应当算上周五的)至结束日期的数据
+	timeData = make(map[time.Time]float64)
+	if len(originData) == 0 || convertRule == nil {
+		return
+	}
+	if !utils.InArrayByInt([]int{1, 2}, convertRule.ConvertType) {
+		err = fmt.Errorf("取值类型有误, ConvertType: %d", convertRule.ConvertType)
+		return
+	}
+
+	// 升序排序
+	sort.Slice(originData, func(i, j int) bool {
+		return originData[i].DataTime.Before(originData[j].DataTime)
+	})
+
+	// 将数据根据日期进行分组
+	var sortDates []string
+	groupDateData := make(map[string][]*ThsHfConvertOriginData)
+	for _, v := range originData {
+		d := v.DataTime.Format(utils.FormatDate)
+		if !utils.InArrayByStr(sortDates, d) {
+			sortDates = append(sortDates, d)
+		}
+		if groupDateData[d] == nil {
+			groupDateData[d] = make([]*ThsHfConvertOriginData, 0)
+		}
+		groupDateData[d] = append(groupDateData[d], v)
+	}
+
+	// 取值方式-指定时间的值
+	if convertRule.ConvertType == 1 {
+		for k, v := range sortDates {
+			todayTime, e := time.ParseInLocation(utils.FormatDate, v, time.Local)
+			if e != nil {
+				utils.FileLog.Info("当日日期转换有误, date: %s, err: %v", v, e)
+				continue
+			}
+			var timeTarget time.Time
+			dateData := make([]*ThsHfConvertOriginData, 0)
+
+			// 当日
+			if convertRule.ConvertFixed.FixedDay == 1 {
+				tg, e := time.ParseInLocation(utils.FormatDateTime, fmt.Sprintf("%s %s", v, convertRule.ConvertFixed.FixedTime), time.Local)
+				if e != nil {
+					utils.FileLog.Info(fmt.Sprintf("当日timeTarget转换有误, %v", e))
+					continue
+				}
+				timeTarget = tg
+
+				dt := groupDateData[v]
+				if dt == nil {
+					utils.FileLog.Info(fmt.Sprintf("%s当日无数据", v))
+					continue
+				}
+				if len(dt) == 0 {
+					continue
+				}
+				dateData = dt
+			}
+
+			// 前一日
+			if convertRule.ConvertFixed.FixedDay == 2 {
+				if k < 1 {
+					utils.FileLog.Info(fmt.Sprintf("%s前日无数据", v))
+					continue
+				}
+				preDate := sortDates[k-1]
+
+				tg, e := time.ParseInLocation(utils.FormatDateTime, fmt.Sprintf("%s %s", preDate, convertRule.ConvertFixed.FixedTime), time.Local)
+				if e != nil {
+					utils.FileLog.Info(fmt.Sprintf("前日timeTarget转换有误, %v", e))
+					continue
+				}
+				timeTarget = tg
+
+				dt := groupDateData[preDate]
+				if dt == nil {
+					utils.FileLog.Info(fmt.Sprintf("%s前日无数据", v))
+					continue
+				}
+				if len(dt) == 0 {
+					continue
+				}
+				dateData = dt
+			}
+			if len(dateData) == 0 {
+				utils.FileLog.Info("日期%s无数据序列", v)
+				continue
+			}
+
+			// 重新获取数据序列中, 时间在目标时间点之后的
+			newDateData := make([]*ThsHfConvertOriginData, 0)
+			for kv, dv := range dateData {
+				if dv.DataTime.Before(timeTarget) {
+					continue
+				}
+				// 由于升序排列, 直接取之后所有的数据
+				newDateData = append(newDateData, dateData[kv:]...)
+				break
+			}
+
+			// 取重组后当日数据中的第一个(有可能目标时间点无值, 那么取之后时间最近的值)
+			if len(newDateData) == 0 {
+				utils.FileLog.Info("日期%s无有效数据", v)
+				continue
+			}
+			timeData[todayTime] = newDateData[0].Value
+		}
+		return
+	}
+
+	// 取值方式-区间计算值
+	for k, v := range sortDates {
+		todayTime, e := time.ParseInLocation(utils.FormatDate, v, time.Local)
+		if e != nil {
+			utils.FileLog.Info("当日日期转换有误, date: %s, err: %v", v, e)
+			continue
+		}
+		var thisDate, preDate string
+		thisDate = v
+		if k > 1 {
+			preDate = sortDates[k-1]
+		}
+		var startTimeTarget, endTimeTarget time.Time
+
+		// 起始时间-当日/前一日
+		if convertRule.ConvertArea.StartDay == 1 {
+			tg, e := time.ParseInLocation(utils.FormatDateTime, fmt.Sprintf("%s %s", thisDate, convertRule.ConvertArea.StartTime), time.Local)
+			if e != nil {
+				utils.FileLog.Info(fmt.Sprintf("当日startTimeTarget转换有误, %v", e))
+				continue
+			}
+			startTimeTarget = tg
+		}
+		if convertRule.ConvertArea.StartDay == 2 {
+			if preDate == "" {
+				utils.FileLog.Info(fmt.Sprintf("%s前日无数据", v))
+				continue
+			}
+			tg, e := time.ParseInLocation(utils.FormatDateTime, fmt.Sprintf("%s %s", preDate, convertRule.ConvertArea.StartTime), time.Local)
+			if e != nil {
+				utils.FileLog.Info(fmt.Sprintf("前日startTimeTarget转换有误, %v", e))
+				continue
+			}
+			startTimeTarget = tg
+		}
+
+		// 截止时间-当日/前一日
+		if convertRule.ConvertArea.EndDay == 1 {
+			tg, e := time.ParseInLocation(utils.FormatDateTime, fmt.Sprintf("%s %s", thisDate, convertRule.ConvertArea.EndTime), time.Local)
+			if e != nil {
+				utils.FileLog.Info(fmt.Sprintf("当日endTimeTarget转换有误, %v", e))
+				continue
+			}
+			endTimeTarget = tg
+		}
+		if convertRule.ConvertArea.EndDay == 2 {
+			if preDate == "" {
+				utils.FileLog.Info(fmt.Sprintf("%s前日无数据", v))
+				continue
+			}
+			tg, e := time.ParseInLocation(utils.FormatDateTime, fmt.Sprintf("%s %s", preDate, convertRule.ConvertArea.EndTime), time.Local)
+			if e != nil {
+				utils.FileLog.Info(fmt.Sprintf("前日endTimeTarget转换有误, %v", e))
+				continue
+			}
+			endTimeTarget = tg
+		}
+		if startTimeTarget.IsZero() || endTimeTarget.IsZero() {
+			utils.FileLog.Info(fmt.Sprintf("起始截止时间有误, start: %v, end: %v", startTimeTarget, endTimeTarget))
+			continue
+		}
+
+		// 合并前日当日数据
+		dateData := make([]*ThsHfConvertOriginData, 0)
+		if convertRule.ConvertArea.StartDay == 1 && convertRule.ConvertArea.EndDay == 1 {
+			// 起始截止均为当日
+			dateData = groupDateData[thisDate]
+			if dateData == nil {
+				utils.FileLog.Info(fmt.Sprintf("%s当日无数据", thisDate))
+				continue
+			}
+			if len(dateData) == 0 {
+				utils.FileLog.Info(fmt.Sprintf("%s当日无数据", thisDate))
+				continue
+			}
+		} else {
+			if preDate == "" {
+				continue
+			}
+			// 起始截止时间含前日
+			preData := groupDateData[preDate]
+			if preData == nil {
+				utils.FileLog.Info(fmt.Sprintf("%s前日无数据", thisDate))
+				continue
+			}
+			if len(preData) == 0 {
+				utils.FileLog.Info(fmt.Sprintf("%s前日无数据", thisDate))
+				continue
+			}
+			thisData := groupDateData[thisDate]
+			if thisData == nil {
+				utils.FileLog.Info(fmt.Sprintf("%s当日无数据", thisDate))
+				continue
+			}
+			if len(thisData) == 0 {
+				utils.FileLog.Info(fmt.Sprintf("%s当日无数据", thisDate))
+				continue
+			}
+			dateData = append(dateData, preData...)
+			dateData = append(dateData, thisData...)
+		}
+		if len(dateData) == 0 {
+			utils.FileLog.Info("日期%s无数据序列", v)
+			continue
+		}
+
+		// 重组时间区间内的数据
+		newDateData := make([]*ThsHfConvertOriginData, 0)
+		for _, dv := range dateData {
+			if dv.DataTime.Before(startTimeTarget) || dv.DataTime.After(endTimeTarget) {
+				continue
+			}
+			newDateData = append(newDateData, dv)
+		}
+		if len(newDateData) == 0 {
+			utils.FileLog.Info(fmt.Sprintf("时间区间内无数据, start: %v, end: %v", startTimeTarget, endTimeTarget))
+			continue
+		}
+
+		// 取出区间内的均值/最值
+		var avgVal, minVal, maxVal, sumVal float64
+		minVal, maxVal = newDateData[0].Value, newDateData[0].Value
+		for _, nv := range newDateData {
+			sumVal += nv.Value
+			if nv.Value > maxVal {
+				maxVal = nv.Value
+			}
+			if nv.Value < minVal {
+				minVal = nv.Value
+			}
+		}
+		avgVal = sumVal / float64(len(newDateData))
+
+		switch convertRule.ConvertArea.CalculateType {
+		case 1:
+			timeData[todayTime] = avgVal
+		case 2:
+			timeData[todayTime] = maxVal
+		case 3:
+			timeData[todayTime] = minVal
+		default:
+			utils.FileLog.Info(fmt.Sprintf("计算方式有误, CalculateType: %d", convertRule.ConvertArea.CalculateType))
+		}
+	}
+	return
+}
+
+func (obj EdbThsHf) getBaseIndexDataByMongo(baseIndexCode, startDate string) (newDataList []EdbInfoMgoData, err error) {
+	newDataList = make([]EdbInfoMgoData, 0)
+
+	// 获取数据源的指标数据
+	mogDataObj := new(mgo.BaseFromThsHfData)
+
+	// 构建查询条件
+	queryConditions := bson.M{
+		"index_code": baseIndexCode,
+	}
+
+	if startDate != `` {
+		//获取已存在的所有数据
+		startDateTime, tmpErr := time.ParseInLocation(utils.FormatDate, startDate, time.Local)
+		if tmpErr != nil {
+			err = tmpErr
+			return
+		}
+		queryConditions["data_time"] = bson.M{"$gte": startDateTime}
+	}
+
+	baseDataList, err := mogDataObj.GetAllDataList(queryConditions, []string{"data_time"})
+	if err != nil {
+		fmt.Println("GetAllDataList Err:" + err.Error())
+		return
+	}
+
+	for _, v := range baseDataList {
+		newDataList = append(newDataList, EdbInfoMgoData{
+			//EdbDataId: v.ID,
+			DataTime: v.DataTime,
+			Value:    v.Value,
+			EdbCode:  v.IndexCode,
+		})
+	}
+	return
+}
+
+func (obj EdbThsHf) HandleConfigInsertEdbDataByMongo(realDataMaxDate time.Time, edbDataInsertConfig *EdbDataInsertConfig, edbInfoId, source, subSource int, existMap map[string]*mgo.EdbDataThsHf, isFindConfigDateRealData bool) {
+	if edbDataInsertConfig == nil {
+		return
+	}
+	var err error
+	defer func() {
+		if err != nil {
+			utils.FileLog.Info(fmt.Sprintf("ThsHf-HandleConfigInsertEdbDataByMongo, err: %v", err))
+		}
+	}()
+
+	edbDataInsertConfigDate := edbDataInsertConfig.Date // 配置的日期
+
+	// 如果存在真实数据的最大日期  && 存在配置插入数据的最大日期  && 真实数据的最大日期 晚于/等于 配置插入数据的最大日期
+	if realDataMaxDate.After(edbDataInsertConfigDate) || realDataMaxDate.Equal(edbDataInsertConfigDate) {
+		go DeleteEdbDataInsertConfigByEdbId(edbInfoId)
+
+		mogDataObj := mgo.EdbDataThsHf{}
+		coll := mogDataObj.GetCollection()
+		edbDataInsertConfigDateStr := edbDataInsertConfigDate.Format(utils.FormatDate)
+		// 如果没有找到找到配置日期的实际数据，那么就直接删除
+		if item, ok := existMap[edbDataInsertConfigDateStr]; ok && !isFindConfigDateRealData {
+			mogDataObj.RemoveManyByColl(coll, bson.M{"_id": item.ID})
+		}
+	} else {
+		o := orm.NewOrm()
+		edbDataInsertConfig.RealDate = realDataMaxDate
+		_, err = o.Update(edbDataInsertConfig, "RealDate")
+	}
+	return
+}
+
+func (obj EdbThsHf) UnifiedModifyEdbInfoMaxAndMinInfo(edbInfo *EdbInfo) (err error) {
+	if utils.UseMongo {
+		edbInfoMaxAndMinInfo, tmpErr := obj.getEdbInfoMaxAndMinInfoByMongo(edbInfo.EdbCode)
+		// 如果正常获取到了，那就去修改指标的最大最小值
+		if tmpErr == nil && edbInfoMaxAndMinInfo != nil {
+			err = ModifyEdbInfoMaxAndMinInfo(edbInfo.EdbInfoId, edbInfoMaxAndMinInfo)
+		} else {
+			// 清空的目的是为了避免异常返回
+			err = nil
+		}
+	} else {
+		err, _ = UnifiedModifyEdbInfoMaxAndMinInfo(edbInfo)
+	}
+
+	return
+}
+
+func (obj EdbThsHf) getEdbInfoMaxAndMinInfoByMongo(edbCode string) (item *EdbInfoMaxAndMinInfo, err error) {
+	mogDataObj := new(mgo.EdbDataThsHf)
+	pipeline := []bson.M{
+		{"$match": bson.M{"edb_code": edbCode}},
+		{"$group": bson.M{
+			"_id":       nil,
+			"min_date":  bson.M{"$min": "$data_time"},
+			"max_date":  bson.M{"$max": "$data_time"},
+			"min_value": bson.M{"$min": "$value"},
+			"max_value": bson.M{"$max": "$value"},
+		}},
+		{"$project": bson.M{"_id": 0}}, // 可选，如果不需要_id字段
+	}
+	result, err := mogDataObj.GetEdbInfoMaxAndMinInfo(pipeline)
+	if err != nil {
+		fmt.Println("EdbDataBusiness getEdbDataBusinessList Err:" + err.Error())
+		return
+	}
+
+	if !result.MaxDate.IsZero() {
+		whereQuery := bson.M{"edb_code": edbCode, "data_time": result.MaxDate}
+		selectParam := bson.D{{"value", 1}, {"_id", 0}}
+		latestValue, tmpErr := mogDataObj.GetLatestValue(whereQuery, selectParam)
+		if tmpErr != nil {
+			err = tmpErr
+			return
+		}
+		result.LatestValue = latestValue.Value
+		result.EndValue = latestValue.Value
+	}
+
+	item = &EdbInfoMaxAndMinInfo{
+		MinDate:     result.MinDate.Format(utils.FormatDate),
+		MaxDate:     result.MaxDate.Format(utils.FormatDate),
+		MinValue:    result.MinValue,
+		MaxValue:    result.MaxValue,
+		LatestValue: result.LatestValue,
+		LatestDate:  result.LatestDate.Format(utils.FormatDate),
+		EndValue:    result.EndValue,
+	}
+
+	return
+}

models/factor_edb_series.go

@@ -0,0 +1,363 @@
+package models
+
+import (
+	"encoding/json"
+	"eta/eta_index_lib/utils"
+	"fmt"
+	"github.com/beego/beego/v2/client/orm"
+	"strings"
+	"time"
+)
+
+const (
+	FactorEdbSeriesCalculateNone = 0
+	FactorEdbSeriesCalculating   = 1
+	FactorEdbSeriesCalculated    = 2
+)
+
+// FactorEdbSeries 因子指标系列表
+type FactorEdbSeries struct {
+	FactorEdbSeriesId int       `orm:"column(factor_edb_series_id);pk"`
+	SeriesName        string    `description:"系列名称"`
+	EdbInfoType       int       `description:"关联指标类型：0-普通指标；1-预测指标"`
+	CalculateStep     string    `description:"计算步骤-JSON"`
+	CalculateState    int       `description:"计算状态: 0-无计算; 1-计算中; 2-计算完成"`
+	CreateTime        time.Time `description:"创建时间"`
+	ModifyTime        time.Time `description:"修改时间"`
+}
+
+func (m *FactorEdbSeries) TableName() string {
+	return "factor_edb_series"
+}
+
+type FactorEdbSeriesCols struct {
+	PrimaryId      string
+	SeriesName     string
+	EdbInfoType    string
+	CalculateStep  string
+	CalculateState string
+	CreateTime     string
+	ModifyTime     string
+}
+
+func (m *FactorEdbSeries) Cols() FactorEdbSeriesCols {
+	return FactorEdbSeriesCols{
+		PrimaryId:      "factor_edb_series_id",
+		SeriesName:     "series_name",
+		EdbInfoType:    "edb_info_type",
+		CalculateStep:  "calculate_step",
+		CalculateState: "calculate_state",
+		CreateTime:     "create_time",
+		ModifyTime:     "modify_time",
+	}
+}
+
+func (m *FactorEdbSeries) Create() (err error) {
+	o := orm.NewOrm()
+	id, err := o.Insert(m)
+	if err != nil {
+		return
+	}
+	m.FactorEdbSeriesId = int(id)
+	return
+}
+
+func (m *FactorEdbSeries) CreateMulti(items []*FactorEdbSeries) (err error) {
+	if len(items) == 0 {
+		return
+	}
+	o := orm.NewOrm()
+	_, err = o.InsertMulti(len(items), items)
+	return
+}
+
+func (m *FactorEdbSeries) Update(cols []string) (err error) {
+	o := orm.NewOrm()
+	_, err = o.Update(m, cols...)
+	return
+}
+
+func (m *FactorEdbSeries) Remove() (err error) {
+	o := orm.NewOrm()
+	sql := fmt.Sprintf(`DELETE FROM %s WHERE %s = ? LIMIT 1`, m.TableName(), m.Cols().PrimaryId)
+	_, err = o.Raw(sql, m.FactorEdbSeriesId).Exec()
+	return
+}
+
+func (m *FactorEdbSeries) MultiRemove(ids []int) (err error) {
+	if len(ids) == 0 {
+		return
+	}
+	o := orm.NewOrm()
+	sql := fmt.Sprintf(`DELETE FROM %s WHERE %s IN (%s)`, m.TableName(), m.Cols().PrimaryId, utils.GetOrmInReplace(len(ids)))
+	_, err = o.Raw(sql, ids).Exec()
+	return
+}
+
+func (m *FactorEdbSeries) GetItemById(id int) (item *FactorEdbSeries, err error) {
+	o := orm.NewOrm()
+	sql := fmt.Sprintf(`SELECT * FROM %s WHERE %s = ? LIMIT 1`, m.TableName(), m.Cols().PrimaryId)
+	err = o.Raw(sql, id).QueryRow(&item)
+	return
+}
+
+func (m *FactorEdbSeries) GetItemByCondition(condition string, pars []interface{}, orderRule string) (item *FactorEdbSeries, err error) {
+	o := orm.NewOrm()
+	order := ``
+	if orderRule != "" {
+		order = ` ORDER BY ` + orderRule
+	}
+	sql := fmt.Sprintf(`SELECT * FROM %s WHERE 1=1 %s %s LIMIT 1`, m.TableName(), condition, order)
+	err = o.Raw(sql, pars).QueryRow(&item)
+	return
+}
+
+func (m *FactorEdbSeries) GetCountByCondition(condition string, pars []interface{}) (count int, err error) {
+	o := orm.NewOrm()
+	sql := fmt.Sprintf(`SELECT COUNT(1) FROM %s WHERE 1=1 %s`, m.TableName(), condition)
+	err = o.Raw(sql, pars).QueryRow(&count)
+	return
+}
+
+func (m *FactorEdbSeries) GetItemsByCondition(condition string, pars []interface{}, fieldArr []string, orderRule string) (items []*FactorEdbSeries, err error) {
+	o := orm.NewOrm()
+	fields := strings.Join(fieldArr, ",")
+	if len(fieldArr) == 0 {
+		fields = `*`
+	}
+	order := fmt.Sprintf(`ORDER BY %s DESC`, m.Cols().CreateTime)
+	if orderRule != "" {
+		order = ` ORDER BY ` + orderRule
+	}
+	sql := fmt.Sprintf(`SELECT %s FROM %s WHERE 1=1 %s %s`, fields, m.TableName(), condition, order)
+	_, err = o.Raw(sql, pars).QueryRows(&items)
+	return
+}
+
+func (m *FactorEdbSeries) GetPageItemsByCondition(condition string, pars []interface{}, fieldArr []string, orderRule string, startSize, pageSize int) (items []*FactorEdbSeries, err error) {
+	o := orm.NewOrm()
+	fields := strings.Join(fieldArr, ",")
+	if len(fieldArr) == 0 {
+		fields = `*`
+	}
+	order := fmt.Sprintf(`ORDER BY %s DESC`, m.Cols().CreateTime)
+	if orderRule != "" {
+		order = ` ORDER BY ` + orderRule
+	}
+	sql := fmt.Sprintf(`SELECT %s FROM %s WHERE 1=1 %s %s LIMIT ?,?`, fields, m.TableName(), condition, order)
+	_, err = o.Raw(sql, pars, startSize, pageSize).QueryRows(&items)
+	return
+}
+
+// FactorEdbSeriesItem 多因子系列信息
+type FactorEdbSeriesItem struct {
+	SeriesId      int                            `description:"多因子系列ID"`
+	SeriesName    string                         `description:"系列名称"`
+	EdbInfoType   int                            `description:"关联指标类型：0-普通指标；1-预测指标"`
+	CalculateStep []FactorEdbSeriesCalculatePars `description:"计算步骤-JSON"`
+	CreateTime    string                         `description:"创建时间"`
+	ModifyTime    string                         `description:"修改时间"`
+}
+
+func (m *FactorEdbSeries) Format2Item() (item *FactorEdbSeriesItem) {
+	item = new(FactorEdbSeriesItem)
+	item.SeriesId = m.FactorEdbSeriesId
+	item.SeriesName = m.SeriesName
+	item.EdbInfoType = m.EdbInfoType
+	if m.CalculateStep != "" {
+		_ = json.Unmarshal([]byte(m.CalculateStep), &item.CalculateStep)
+	}
+	item.CreateTime = utils.TimeTransferString(utils.FormatDateTime, m.CreateTime)
+	item.ModifyTime = utils.TimeTransferString(utils.FormatDateTime, m.ModifyTime)
+	return
+}
+
+// FactorEdbSeriesCalculatePars 计算参数
+type FactorEdbSeriesCalculatePars struct {
+	Formula       interface{} `description:"N值/移动天数/指数修匀alpha值/计算公式等"`
+	Calendar      string      `description:"公历/农历"`
+	Frequency     string      `description:"需要转换的频度"`
+	MoveType      int         `description:"移动方式: 1-领先(默认); 2-滞后"`
+	MoveFrequency string      `description:"移动频度"`
+	FromFrequency string      `description:"来源的频度"`
+	Source        int         `description:"计算方式来源(不是指标来源)"`
+	Sort          int         `description:"计算顺序"`
+}
+
+// CreateSeriesAndMapping 新增系列和指标关联
+func (m *FactorEdbSeries) CreateSeriesAndMapping(item *FactorEdbSeries, mappings []*FactorEdbSeriesMapping) (seriesId int, err error) {
+	if item == nil {
+		err = fmt.Errorf("series is nil")
+		return
+	}
+	o := orm.NewOrm()
+	tx, e := o.Begin()
+	if e != nil {
+		err = fmt.Errorf("orm begin err: %v", e)
+		return
+	}
+	defer func() {
+		if err != nil {
+			_ = tx.Rollback()
+			return
+		}
+		_ = tx.Commit()
+	}()
+
+	id, e := tx.Insert(item)
+	if e != nil {
+		err = fmt.Errorf("insert series err: %v", e)
+		return
+	}
+	seriesId = int(id)
+	item.FactorEdbSeriesId = seriesId
+
+	if len(mappings) > 0 {
+		for _, v := range mappings {
+			v.FactorEdbSeriesId = seriesId
+		}
+		_, e = tx.InsertMulti(200, mappings)
+		if e != nil {
+			err = fmt.Errorf("insert multi mapping err: %v", e)
+			return
+		}
+	}
+	return
+}
+
+// EditSeriesAndMapping 编辑系列和指标关联
+func (m *FactorEdbSeries) EditSeriesAndMapping(item *FactorEdbSeries, mappings []*FactorEdbSeriesMapping, updateCols []string) (err error) {
+	if item == nil {
+		err = fmt.Errorf("series is nil")
+		return
+	}
+	o := orm.NewOrm()
+	tx, e := o.Begin()
+	if e != nil {
+		err = fmt.Errorf("orm begin err: %v", e)
+		return
+	}
+	defer func() {
+		if err != nil {
+			_ = tx.Rollback()
+			return
+		}
+		_ = tx.Commit()
+	}()
+
+	_, e = tx.Update(item, updateCols...)
+	if e != nil {
+		err = fmt.Errorf("update series err: %v", e)
+		return
+	}
+
+	// 清除原指标关联
+	mappingOb := new(FactorEdbSeriesMapping)
+	cond := fmt.Sprintf("%s = ?", mappingOb.Cols().FactorEdbSeriesId)
+	pars := make([]interface{}, 0)
+	pars = append(pars, item.FactorEdbSeriesId)
+	sql := fmt.Sprintf(`DELETE FROM %s WHERE %s`, mappingOb.TableName(), cond)
+	_, e = tx.Raw(sql, pars).Exec()
+	if e != nil {
+		err = fmt.Errorf("remove mapping err: %v", e)
+		return
+	}
+
+	if len(mappings) > 0 {
+		for _, v := range mappings {
+			v.FactorEdbSeriesId = item.FactorEdbSeriesId
+		}
+		_, e = tx.InsertMulti(200, mappings)
+		if e != nil {
+			err = fmt.Errorf("insert multi mapping err: %v", e)
+			return
+		}
+	}
+	return
+}
+
+// FactorEdbSeriesStepCalculateResp 批量计算响应
+type FactorEdbSeriesStepCalculateResp struct {
+	SeriesId int                                  `description:"多因子指标系列ID"`
+	Fail     []FactorEdbSeriesStepCalculateResult `description:"计算失败的指标"`
+	Success  []FactorEdbSeriesStepCalculateResult `description:"计算成功的指标"`
+}
+
+// FactorEdbSeriesStepCalculateResult 批量计算结果
+type FactorEdbSeriesStepCalculateResult struct {
+	EdbInfoId int    `description:"指标ID"`
+	EdbCode   string `description:"指标编码"`
+	Msg       string `description:"提示信息"`
+	ErrMsg    string `description:"错误信息"`
+}
+
+// FactorEdbSeriesDetail 因子指标系列-详情
+type FactorEdbSeriesDetail struct {
+	*FactorEdbSeriesItem
+	EdbMappings []*FactorEdbSeriesMappingItem
+}
+
+// FactorEdbSeriesCorrelationMatrixResp 因子指标系列-相关性矩阵响应
+type FactorEdbSeriesCorrelationMatrixResp struct {
+	Fail    []FactorEdbSeriesCorrelationMatrixItem `description:"计算失败的指标"`
+	Success []FactorEdbSeriesCorrelationMatrixItem `description:"计算成功的指标"`
+}
+
+// FactorEdbSeriesCorrelationMatrixItem 因子指标系列-相关性矩阵信息
+type FactorEdbSeriesCorrelationMatrixItem struct {
+	SeriesId   int                                      `description:"因子指标系列ID"`
+	EdbInfoId  int                                      `description:"指标ID"`
+	EdbCode    string                                   `description:"指标编码"`
+	EdbName    string                                   `description:"指标名称"`
+	Values     []FactorEdbSeriesCorrelationMatrixValues `description:"X轴和Y轴数据"`
+	Msg        string                                   `description:"提示信息"`
+	ErrMsg     string                                   `description:"错误信息"`
+	Used       bool                                     `description:"是否选中"`
+	SourceName string                                   `description:"指标来源名称"`
+}
+
+// FactorEdbSeriesCorrelationMatrixValues 因子指标系列-相关性矩阵XY值
+type FactorEdbSeriesCorrelationMatrixValues struct {
+	XData int     `description:"X轴数据"`
+	YData float64 `description:"Y轴数据"`
+}
+
+// FactorEdbSeriesCorrelationMatrixOrder 排序规则[0 1 2 3 -1 -2 -3]
+type FactorEdbSeriesCorrelationMatrixOrder []FactorEdbSeriesCorrelationMatrixValues
+
+func (a FactorEdbSeriesCorrelationMatrixOrder) Len() int {
+	return len(a)
+}
+
+func (a FactorEdbSeriesCorrelationMatrixOrder) Swap(i, j int) {
+	a[i], a[j] = a[j], a[i]
+}
+
+func (a FactorEdbSeriesCorrelationMatrixOrder) Less(i, j int) bool {
+	// 非负数优先
+	if a[i].XData >= 0 && a[j].XData < 0 {
+		return true
+	}
+	if a[i].XData < 0 && a[j].XData >= 0 {
+		return false
+	}
+	// 非负数升序排序
+	if a[i].XData >= 0 {
+		return a[i].XData < a[j].XData
+	}
+	// 负数按绝对值的降序排序（即数值的升序）
+	return a[i].XData > a[j].XData
+}
+
+// FactorEdbRecalculateReq 因子指标重新计算
+type FactorEdbRecalculateReq struct {
+	EdbInfoId int    `description:"指标ID"`
+	EdbCode   string `description:"指标编码"`
+}
+
+// FactorEdbChartRecalculateReq 因子指标关联的图表数据重计算
+type FactorEdbChartRecalculateReq struct {
+	ChartInfoId int `description:"图表ID"`
+	//EdbInfoId   int    `description:"指标ID"`
+	//EdbCode     string `description:"指标编码"`
+}

models/factor_edb_series_calculate_data.go

@@ -0,0 +1,190 @@
+package models
+
+import (
+	"eta/eta_index_lib/utils"
+	"fmt"
+	"github.com/beego/beego/v2/client/orm"
+	"strings"
+	"time"
+)
+
+// FactorEdbSeriesCalculateData 因子指标系列-指标计算数据表
+type FactorEdbSeriesCalculateData struct {
+	FactorEdbSeriesCalculateDataId int       `orm:"column(factor_edb_series_calculate_data_id);pk"`
+	FactorEdbSeriesId              int       `description:"因子指标系列ID"`
+	EdbInfoId                      int       `description:"指标ID"`
+	EdbCode                        string    `description:"指标编码"`
+	DataTime                       time.Time `description:"数据日期"`
+	Value                          float64   `description:"数据值"`
+	CreateTime                     time.Time `description:"创建时间"`
+	ModifyTime                     time.Time `description:"修改时间"`
+	DataTimestamp                  int64     `description:"数据日期时间戳"`
+}
+
+func (m *FactorEdbSeriesCalculateData) TableName() string {
+	return "factor_edb_series_calculate_data"
+}
+
+type FactorEdbSeriesCalculateDataCols struct {
+	PrimaryId         string
+	FactorEdbSeriesId string
+	EdbInfoId         string
+	EdbCode           string
+	DataTime          string
+	Value             string
+	CreateTime        string
+	ModifyTime        string
+	DataTimestamp     string
+}
+
+func (m *FactorEdbSeriesCalculateData) Cols() FactorEdbSeriesCalculateDataCols {
+	return FactorEdbSeriesCalculateDataCols{
+		PrimaryId:         "factor_edb_series_calculate_data_id",
+		FactorEdbSeriesId: "factor_edb_series_id",
+		EdbInfoId:         "edb_info_id",
+		EdbCode:           "edb_code",
+		DataTime:          "data_time",
+		Value:             "value",
+		CreateTime:        "create_time",
+		ModifyTime:        "modify_time",
+		DataTimestamp:     "data_timestamp",
+	}
+}
+
+func (m *FactorEdbSeriesCalculateData) Create() (err error) {
+	o := orm.NewOrm()
+	id, err := o.Insert(m)
+	if err != nil {
+		return
+	}
+	m.FactorEdbSeriesCalculateDataId = int(id)
+	return
+}
+
+func (m *FactorEdbSeriesCalculateData) CreateMulti(items []*FactorEdbSeriesCalculateData) (err error) {
+	if len(items) == 0 {
+		return
+	}
+	o := orm.NewOrm()
+	_, err = o.InsertMulti(500, items)
+	return
+}
+
+func (m *FactorEdbSeriesCalculateData) Update(cols []string) (err error) {
+	o := orm.NewOrm()
+	_, err = o.Update(m, cols...)
+	return
+}
+
+func (m *FactorEdbSeriesCalculateData) Remove() (err error) {
+	o := orm.NewOrm()
+	sql := fmt.Sprintf(`DELETE FROM %s WHERE %s = ? LIMIT 1`, m.TableName(), m.Cols().PrimaryId)
+	_, err = o.Raw(sql, m.FactorEdbSeriesCalculateDataId).Exec()
+	return
+}
+
+func (m *FactorEdbSeriesCalculateData) MultiRemove(ids []int) (err error) {
+	if len(ids) == 0 {
+		return
+	}
+	o := orm.NewOrm()
+	sql := fmt.Sprintf(`DELETE FROM %s WHERE %s IN (%s)`, m.TableName(), m.Cols().PrimaryId, utils.GetOrmInReplace(len(ids)))
+	_, err = o.Raw(sql, ids).Exec()
+	return
+}
+
+func (m *FactorEdbSeriesCalculateData) RemoveByCondition(condition string, pars []interface{}) (err error) {
+	if condition == "" {
+		return
+	}
+	o := orm.NewOrm()
+	sql := fmt.Sprintf(`DELETE FROM %s WHERE %s`, m.TableName(), condition)
+	_, err = o.Raw(sql, pars).Exec()
+	return
+}
+
+func (m *FactorEdbSeriesCalculateData) GetItemById(id int) (item *FactorEdbSeriesCalculateData, err error) {
+	o := orm.NewOrm()
+	sql := fmt.Sprintf(`SELECT * FROM %s WHERE %s = ? LIMIT 1`, m.TableName(), m.Cols().PrimaryId)
+	err = o.Raw(sql, id).QueryRow(&item)
+	return
+}
+
+func (m *FactorEdbSeriesCalculateData) GetItemByCondition(condition string, pars []interface{}, orderRule string) (item *FactorEdbSeriesCalculateData, err error) {
+	o := orm.NewOrm()
+	order := ``
+	if orderRule != "" {
+		order = ` ORDER BY ` + orderRule
+	}
+	sql := fmt.Sprintf(`SELECT * FROM %s WHERE 1=1 %s %s LIMIT 1`, m.TableName(), condition, order)
+	err = o.Raw(sql, pars).QueryRow(&item)
+	return
+}
+
+func (m *FactorEdbSeriesCalculateData) GetCountByCondition(condition string, pars []interface{}) (count int, err error) {
+	o := orm.NewOrm()
+	sql := fmt.Sprintf(`SELECT COUNT(1) FROM %s WHERE 1=1 %s`, m.TableName(), condition)
+	err = o.Raw(sql, pars).QueryRow(&count)
+	return
+}
+
+func (m *FactorEdbSeriesCalculateData) GetItemsByCondition(condition string, pars []interface{}, fieldArr []string, orderRule string) (items []*FactorEdbSeriesCalculateData, err error) {
+	o := orm.NewOrm()
+	fields := strings.Join(fieldArr, ",")
+	if len(fieldArr) == 0 {
+		fields = `*`
+	}
+	order := fmt.Sprintf(`ORDER BY %s DESC`, m.Cols().CreateTime)
+	if orderRule != "" {
+		order = ` ORDER BY ` + orderRule
+	}
+	sql := fmt.Sprintf(`SELECT %s FROM %s WHERE 1=1 %s %s`, fields, m.TableName(), condition, order)
+	_, err = o.Raw(sql, pars).QueryRows(&items)
+	return
+}
+
+func (m *FactorEdbSeriesCalculateData) GetPageItemsByCondition(condition string, pars []interface{}, fieldArr []string, orderRule string, startSize, pageSize int) (items []*FactorEdbSeriesCalculateData, err error) {
+	o := orm.NewOrm()
+	fields := strings.Join(fieldArr, ",")
+	if len(fieldArr) == 0 {
+		fields = `*`
+	}
+	order := fmt.Sprintf(`ORDER BY %s DESC`, m.Cols().CreateTime)
+	if orderRule != "" {
+		order = ` ORDER BY ` + orderRule
+	}
+	sql := fmt.Sprintf(`SELECT %s FROM %s WHERE 1=1 %s %s LIMIT ?,?`, fields, m.TableName(), condition, order)
+	_, err = o.Raw(sql, pars, startSize, pageSize).QueryRows(&items)
+	return
+}
+
+// FactorEdbSeriesCalculateDataItem 因子指标系列-计算数据信息
+type FactorEdbSeriesCalculateDataItem struct {
+	DataId            int     `description:"数据ID"`
+	FactorEdbSeriesId int     `description:"因子指标系列ID"`
+	EdbInfoId         int     `description:"指标ID"`
+	EdbCode           string  `description:"指标编码"`
+	DataTime          string  `description:"数据日期"`
+	Value             float64 `description:"数据值"`
+}
+
+func (m *FactorEdbSeriesCalculateData) Format2Item() (item *FactorEdbSeriesCalculateDataItem) {
+	item = new(FactorEdbSeriesCalculateDataItem)
+	item.DataId = m.FactorEdbSeriesCalculateDataId
+	item.FactorEdbSeriesId = m.FactorEdbSeriesId
+	item.EdbInfoId = m.EdbInfoId
+	item.EdbCode = m.EdbCode
+	return
+}
+
+// TransEdbSeriesCalculateData2EdbDataList 转换数据格式
+func TransEdbSeriesCalculateData2EdbDataList(items []*FactorEdbSeriesCalculateData) (list []*EdbInfoSearchData) {
+	list = make([]*EdbInfoSearchData, 0)
+	for _, v := range items {
+		list = append(list, &EdbInfoSearchData{
+			DataTime: v.DataTime.Format(utils.FormatDate),
+			Value:    v.Value,
+		})
+	}
+	return
+}

models/factor_edb_series_calculate_func.go

@@ -0,0 +1,153 @@
+package models
+
+import (
+	"eta/eta_index_lib/utils"
+	"fmt"
+	"github.com/beego/beego/v2/client/orm"
+	"strings"
+	"time"
+)
+
+// FactorEdbSeriesCalculateFunc 多因子系列-计算方式表
+type FactorEdbSeriesCalculateFunc struct {
+	FactorEdbSeriesCalculateFuncId int       `orm:"column(factor_edb_series_calculate_func_id);pk"`
+	CalculateName                  string    `description:"计算方式名称"`
+	Source                         int       `description:"计算方式来源"`
+	EdbInfoType                    int       `description:"指标计算类型：0-普通指标；1-预测指标"`
+	CreateTime                     time.Time `description:"创建时间"`
+	ModifyTime                     time.Time `description:"修改时间"`
+}
+
+func (m *FactorEdbSeriesCalculateFunc) TableName() string {
+	return "factor_edb_series_calculate_func"
+}
+
+type FactorEdbSeriesCalculateFuncCols struct {
+	PrimaryId     string
+	CalculateName string
+	Source        string
+	EdbInfoType   string
+	CreateTime    string
+	ModifyTime    string
+}
+
+func (m *FactorEdbSeriesCalculateFunc) Cols() FactorEdbSeriesCalculateFuncCols {
+	return FactorEdbSeriesCalculateFuncCols{
+		PrimaryId:     "factor_edb_series_calculate_func_id",
+		CalculateName: "calculate_name",
+		Source:        "source",
+		EdbInfoType:   "edb_info_type",
+		CreateTime:    "create_time",
+		ModifyTime:    "modify_time",
+	}
+}
+
+func (m *FactorEdbSeriesCalculateFunc) Create() (err error) {
+	o := orm.NewOrm()
+	id, err := o.Insert(m)
+	if err != nil {
+		return
+	}
+	m.FactorEdbSeriesCalculateFuncId = int(id)
+	return
+}
+
+func (m *FactorEdbSeriesCalculateFunc) CreateMulti(items []*FactorEdbSeriesCalculateFunc) (err error) {
+	if len(items) == 0 {
+		return
+	}
+	o := orm.NewOrm()
+	_, err = o.InsertMulti(len(items), items)
+	return
+}
+
+func (m *FactorEdbSeriesCalculateFunc) Update(cols []string) (err error) {
+	o := orm.NewOrm()
+	_, err = o.Update(m, cols...)
+	return
+}
+
+func (m *FactorEdbSeriesCalculateFunc) Remove() (err error) {
+	o := orm.NewOrm()
+	sql := fmt.Sprintf(`DELETE FROM %s WHERE %s = ? LIMIT 1`, m.TableName(), m.Cols().PrimaryId)
+	_, err = o.Raw(sql, m.FactorEdbSeriesCalculateFuncId).Exec()
+	return
+}
+
+func (m *FactorEdbSeriesCalculateFunc) MultiRemove(ids []int) (err error) {
+	if len(ids) == 0 {
+		return
+	}
+	o := orm.NewOrm()
+	sql := fmt.Sprintf(`DELETE FROM %s WHERE %s IN (%s)`, m.TableName(), m.Cols().PrimaryId, utils.GetOrmInReplace(len(ids)))
+	_, err = o.Raw(sql, ids).Exec()
+	return
+}
+
+func (m *FactorEdbSeriesCalculateFunc) GetItemById(id int) (item *FactorEdbSeriesCalculateFunc, err error) {
+	o := orm.NewOrm()
+	sql := fmt.Sprintf(`SELECT * FROM %s WHERE %s = ? LIMIT 1`, m.TableName(), m.Cols().PrimaryId)
+	err = o.Raw(sql, id).QueryRow(&item)
+	return
+}
+
+func (m *FactorEdbSeriesCalculateFunc) GetItemByCondition(condition string, pars []interface{}, orderRule string) (item *FactorEdbSeriesCalculateFunc, err error) {
+	o := orm.NewOrm()
+	order := ``
+	if orderRule != "" {
+		order = ` ORDER BY ` + orderRule
+	}
+	sql := fmt.Sprintf(`SELECT * FROM %s WHERE 1=1 %s %s LIMIT 1`, m.TableName(), condition, order)
+	err = o.Raw(sql, pars).QueryRow(&item)
+	return
+}
+
+func (m *FactorEdbSeriesCalculateFunc) GetCountByCondition(condition string, pars []interface{}) (count int, err error) {
+	o := orm.NewOrm()
+	sql := fmt.Sprintf(`SELECT COUNT(1) FROM %s WHERE 1=1 %s`, m.TableName(), condition)
+	err = o.Raw(sql, pars).QueryRow(&count)
+	return
+}
+
+func (m *FactorEdbSeriesCalculateFunc) GetItemsByCondition(condition string, pars []interface{}, fieldArr []string, orderRule string) (items []*FactorEdbSeriesCalculateFunc, err error) {
+	o := orm.NewOrm()
+	fields := strings.Join(fieldArr, ",")
+	if len(fieldArr) == 0 {
+		fields = `*`
+	}
+	order := fmt.Sprintf(`ORDER BY %s DESC`, m.Cols().CreateTime)
+	if orderRule != "" {
+		order = ` ORDER BY ` + orderRule
+	}
+	sql := fmt.Sprintf(`SELECT %s FROM %s WHERE 1=1 %s %s`, fields, m.TableName(), condition, order)
+	_, err = o.Raw(sql, pars).QueryRows(&items)
+	return
+}
+
+func (m *FactorEdbSeriesCalculateFunc) GetPageItemsByCondition(condition string, pars []interface{}, fieldArr []string, orderRule string, startSize, pageSize int) (items []*FactorEdbSeriesCalculateFunc, err error) {
+	o := orm.NewOrm()
+	fields := strings.Join(fieldArr, ",")
+	if len(fieldArr) == 0 {
+		fields = `*`
+	}
+	order := fmt.Sprintf(`ORDER BY %s DESC`, m.Cols().CreateTime)
+	if orderRule != "" {
+		order = ` ORDER BY ` + orderRule
+	}
+	sql := fmt.Sprintf(`SELECT %s FROM %s WHERE 1=1 %s %s LIMIT ?,?`, fields, m.TableName(), condition, order)
+	_, err = o.Raw(sql, pars, startSize, pageSize).QueryRows(&items)
+	return
+}
+
+// FactorEdbSeriesCalculateFuncItem 多因子系列-计算方式
+type FactorEdbSeriesCalculateFuncItem struct {
+	CalculateName string `description:"计算方式名称"`
+	Source        int    `description:"计算方式来源"`
+}
+
+func (m *FactorEdbSeriesCalculateFunc) Format2Item() (item *FactorEdbSeriesCalculateFuncItem) {
+	item = new(FactorEdbSeriesCalculateFuncItem)
+	item.CalculateName = m.CalculateName
+	item.Source = m.Source
+	return
+}

models/factor_edb_series_chart_mapping.go

@@ -0,0 +1,176 @@
+package models
+
+import (
+	"eta/eta_index_lib/utils"
+	"fmt"
+	"github.com/beego/beego/v2/client/orm"
+	"strings"
+	"time"
+)
+
+const (
+	FactorEdbSeriesChartCalculateTypeCorrelation = 1 // 相关性计算
+)
+
+// FactorEdbSeriesChartMapping 因子指标系列-图表关联
+type FactorEdbSeriesChartMapping struct {
+	FactorEdbSeriesChartMappingId int       `orm:"column(factor_edb_series_chart_mapping_id);pk"`
+	ChartInfoId                   int       `description:"图表ID"`
+	Source                        int       `description:"图表来源, 同chart_info表source"`
+	CalculateType                 int       `description:"计算方式: 1-相关性"`
+	CalculatePars                 string    `description:"计算参数-JSON(如计算窗口等)"`
+	CalculateData                 string    `description:"计算数据-JSON(如相关性矩阵等)"`
+	FactorEdbSeriesId             int       `description:"因子指标系列ID"`
+	EdbInfoId                     int       `description:"指标ID"`
+	EdbUsed                       int       `description:"指标是否使用: 0-否; 1-是"`
+	CreateTime                    time.Time `description:"创建时间"`
+	ModifyTime                    time.Time `description:"修改时间"`
+}
+
+func (m *FactorEdbSeriesChartMapping) TableName() string {
+	return "factor_edb_series_chart_mapping"
+}
+
+type MultipleFactorSeriesChartMappingCols struct {
+	PrimaryId         string
+	ChartInfoId       string
+	Source            string
+	CalculateType     string
+	CalculatePars     string
+	CalculateData     string
+	FactorEdbSeriesId string
+	EdbInfoId         string
+	EdbUsed           string
+	CreateTime        string
+	ModifyTime        string
+}
+
+func (m *FactorEdbSeriesChartMapping) Cols() MultipleFactorSeriesChartMappingCols {
+	return MultipleFactorSeriesChartMappingCols{
+		PrimaryId:         "factor_edb_series_chart_mapping_id",
+		ChartInfoId:       "chart_info_id",
+		Source:            "source",
+		CalculateType:     "calculate_type",
+		CalculatePars:     "calculate_pars",
+		CalculateData:     "calculate_data",
+		FactorEdbSeriesId: "factor_edb_series_id",
+		EdbInfoId:         "edb_info_id",
+		EdbUsed:           "edb_used",
+		CreateTime:        "create_time",
+		ModifyTime:        "modify_time",
+	}
+}
+
+func (m *FactorEdbSeriesChartMapping) Create() (err error) {
+	o := orm.NewOrm()
+	id, err := o.Insert(m)
+	if err != nil {
+		return
+	}
+	m.FactorEdbSeriesChartMappingId = int(id)
+	return
+}
+
+func (m *FactorEdbSeriesChartMapping) CreateMulti(items []*FactorEdbSeriesChartMapping) (err error) {
+	if len(items) == 0 {
+		return
+	}
+	o := orm.NewOrm()
+	_, err = o.InsertMulti(len(items), items)
+	return
+}
+
+func (m *FactorEdbSeriesChartMapping) Update(cols []string) (err error) {
+	o := orm.NewOrm()
+	_, err = o.Update(m, cols...)
+	return
+}
+
+func (m *FactorEdbSeriesChartMapping) Remove() (err error) {
+	o := orm.NewOrm()
+	sql := fmt.Sprintf(`DELETE FROM %s WHERE %s = ? LIMIT 1`, m.TableName(), m.Cols().PrimaryId)
+	_, err = o.Raw(sql, m.FactorEdbSeriesChartMappingId).Exec()
+	return
+}
+
+func (m *FactorEdbSeriesChartMapping) MultiRemove(ids []int) (err error) {
+	if len(ids) == 0 {
+		return
+	}
+	o := orm.NewOrm()
+	sql := fmt.Sprintf(`DELETE FROM %s WHERE %s IN (%s)`, m.TableName(), m.Cols().PrimaryId, utils.GetOrmInReplace(len(ids)))
+	_, err = o.Raw(sql, ids).Exec()
+	return
+}
+
+func (m *FactorEdbSeriesChartMapping) GetItemById(id int) (item *FactorEdbSeriesChartMapping, err error) {
+	o := orm.NewOrm()
+	sql := fmt.Sprintf(`SELECT * FROM %s WHERE %s = ? LIMIT 1`, m.TableName(), m.Cols().PrimaryId)
+	err = o.Raw(sql, id).QueryRow(&item)
+	return
+}
+
+func (m *FactorEdbSeriesChartMapping) GetItemByCondition(condition string, pars []interface{}, orderRule string) (item *FactorEdbSeriesChartMapping, err error) {
+	o := orm.NewOrm()
+	order := ``
+	if orderRule != "" {
+		order = ` ORDER BY ` + orderRule
+	}
+	sql := fmt.Sprintf(`SELECT * FROM %s WHERE 1=1 %s %s LIMIT 1`, m.TableName(), condition, order)
+	err = o.Raw(sql, pars).QueryRow(&item)
+	return
+}
+
+func (m *FactorEdbSeriesChartMapping) GetCountByCondition(condition string, pars []interface{}) (count int, err error) {
+	o := orm.NewOrm()
+	sql := fmt.Sprintf(`SELECT COUNT(1) FROM %s WHERE 1=1 %s`, m.TableName(), condition)
+	err = o.Raw(sql, pars).QueryRow(&count)
+	return
+}
+
+func (m *FactorEdbSeriesChartMapping) GetItemsByCondition(condition string, pars []interface{}, fieldArr []string, orderRule string) (items []*FactorEdbSeriesChartMapping, err error) {
+	o := orm.NewOrm()
+	fields := strings.Join(fieldArr, ",")
+	if len(fieldArr) == 0 {
+		fields = `*`
+	}
+	order := fmt.Sprintf(`ORDER BY %s DESC`, m.Cols().CreateTime)
+	if orderRule != "" {
+		order = ` ORDER BY ` + orderRule
+	}
+	sql := fmt.Sprintf(`SELECT %s FROM %s WHERE 1=1 %s %s`, fields, m.TableName(), condition, order)
+	_, err = o.Raw(sql, pars).QueryRows(&items)
+	return
+}
+
+func (m *FactorEdbSeriesChartMapping) GetPageItemsByCondition(condition string, pars []interface{}, fieldArr []string, orderRule string, startSize, pageSize int) (items []*FactorEdbSeriesChartMapping, err error) {
+	o := orm.NewOrm()
+	fields := strings.Join(fieldArr, ",")
+	if len(fieldArr) == 0 {
+		fields = `*`
+	}
+	order := fmt.Sprintf(`ORDER BY %s DESC`, m.Cols().CreateTime)
+	if orderRule != "" {
+		order = ` ORDER BY ` + orderRule
+	}
+	sql := fmt.Sprintf(`SELECT %s FROM %s WHERE 1=1 %s %s LIMIT ?,?`, fields, m.TableName(), condition, order)
+	_, err = o.Raw(sql, pars, startSize, pageSize).QueryRows(&items)
+	return
+}
+
+// GetDistinctSeriesIdByChartId 获取图表关联的系列ID
+func (m *FactorEdbSeriesChartMapping) GetDistinctSeriesIdByChartId(chartId int) (seriesIds []int, err error) {
+	o := orm.NewOrm()
+	sql := fmt.Sprintf(`SELECT DISTINCT %s FROM %s WHERE %s = ?`, m.Cols().FactorEdbSeriesId, m.TableName(), m.Cols().ChartInfoId)
+	_, err = o.Raw(sql, chartId).QueryRows(&seriesIds)
+	return
+}
+
+// FactorEdbSeriesChartCalculateCorrelationReq 图表相关性计算参数
+type FactorEdbSeriesChartCalculateCorrelationReq struct {
+	BaseEdbInfoId  int    `description:"标的指标ID"`
+	LeadValue      int    `description:"领先期数"`
+	LeadUnit       string `description:"频度"`
+	CalculateValue int    `description:"计算窗口"`
+	CalculateUnit  string `description:"计算频度"`
+}

models/factor_edb_series_mapping.go

@@ -0,0 +1,167 @@
+package models
+
+import (
+	"eta/eta_index_lib/utils"
+	"fmt"
+	"github.com/beego/beego/v2/client/orm"
+	"strings"
+	"time"
+)
+
+// FactorEdbSeriesMapping 因子指标系列-指标关联表
+type FactorEdbSeriesMapping struct {
+	FactorEdbSeriesMappingId int       `orm:"column(factor_edb_series_mapping_id);pk"`
+	FactorEdbSeriesId        int       `description:"因子指标系列ID"`
+	EdbInfoId                int       `description:"指标ID"`
+	EdbCode                  string    `description:"指标编码"`
+	CreateTime               time.Time `description:"创建时间"`
+	ModifyTime               time.Time `description:"修改时间"`
+}
+
+func (m *FactorEdbSeriesMapping) TableName() string {
+	return "factor_edb_series_mapping"
+}
+
+type FactorEdbSeriesMappingCols struct {
+	PrimaryId         string
+	FactorEdbSeriesId string
+	EdbInfoId         string
+	EdbCode           string
+	CreateTime        string
+	ModifyTime        string
+}
+
+func (m *FactorEdbSeriesMapping) Cols() FactorEdbSeriesMappingCols {
+	return FactorEdbSeriesMappingCols{
+		PrimaryId:         "factor_edb_series_mapping_id",
+		FactorEdbSeriesId: "factor_edb_series_id",
+		EdbInfoId:         "edb_info_id",
+		EdbCode:           "edb_code",
+		CreateTime:        "create_time",
+		ModifyTime:        "modify_time",
+	}
+}
+
+func (m *FactorEdbSeriesMapping) Create() (err error) {
+	o := orm.NewOrm()
+	id, err := o.Insert(m)
+	if err != nil {
+		return
+	}
+	m.FactorEdbSeriesMappingId = int(id)
+	return
+}
+
+func (m *FactorEdbSeriesMapping) CreateMulti(items []*FactorEdbSeriesMapping) (err error) {
+	if len(items) == 0 {
+		return
+	}
+	o := orm.NewOrm()
+	_, err = o.InsertMulti(len(items), items)
+	return
+}
+
+func (m *FactorEdbSeriesMapping) Update(cols []string) (err error) {
+	o := orm.NewOrm()
+	_, err = o.Update(m, cols...)
+	return
+}
+
+func (m *FactorEdbSeriesMapping) Remove() (err error) {
+	o := orm.NewOrm()
+	sql := fmt.Sprintf(`DELETE FROM %s WHERE %s = ? LIMIT 1`, m.TableName(), m.Cols().PrimaryId)
+	_, err = o.Raw(sql, m.FactorEdbSeriesMappingId).Exec()
+	return
+}
+
+func (m *FactorEdbSeriesMapping) MultiRemove(ids []int) (err error) {
+	if len(ids) == 0 {
+		return
+	}
+	o := orm.NewOrm()
+	sql := fmt.Sprintf(`DELETE FROM %s WHERE %s IN (%s)`, m.TableName(), m.Cols().PrimaryId, utils.GetOrmInReplace(len(ids)))
+	_, err = o.Raw(sql, ids).Exec()
+	return
+}
+
+func (m *FactorEdbSeriesMapping) RemoveByCondition(condition string, pars []interface{}) (err error) {
+	if condition == "" {
+		return
+	}
+	o := orm.NewOrm()
+	sql := fmt.Sprintf(`DELETE FROM %s WHERE %s`, m.TableName(), condition)
+	_, err = o.Raw(sql, pars).Exec()
+	return
+}
+
+func (m *FactorEdbSeriesMapping) GetItemById(id int) (item *FactorEdbSeriesMapping, err error) {
+	o := orm.NewOrm()
+	sql := fmt.Sprintf(`SELECT * FROM %s WHERE %s = ? LIMIT 1`, m.TableName(), m.Cols().PrimaryId)
+	err = o.Raw(sql, id).QueryRow(&item)
+	return
+}
+
+func (m *FactorEdbSeriesMapping) GetItemByCondition(condition string, pars []interface{}, orderRule string) (item *FactorEdbSeriesMapping, err error) {
+	o := orm.NewOrm()
+	order := ``
+	if orderRule != "" {
+		order = ` ORDER BY ` + orderRule
+	}
+	sql := fmt.Sprintf(`SELECT * FROM %s WHERE 1=1 %s %s LIMIT 1`, m.TableName(), condition, order)
+	err = o.Raw(sql, pars).QueryRow(&item)
+	return
+}
+
+func (m *FactorEdbSeriesMapping) GetCountByCondition(condition string, pars []interface{}) (count int, err error) {
+	o := orm.NewOrm()
+	sql := fmt.Sprintf(`SELECT COUNT(1) FROM %s WHERE 1=1 %s`, m.TableName(), condition)
+	err = o.Raw(sql, pars).QueryRow(&count)
+	return
+}
+
+func (m *FactorEdbSeriesMapping) GetItemsByCondition(condition string, pars []interface{}, fieldArr []string, orderRule string) (items []*FactorEdbSeriesMapping, err error) {
+	o := orm.NewOrm()
+	fields := strings.Join(fieldArr, ",")
+	if len(fieldArr) == 0 {
+		fields = `*`
+	}
+	order := fmt.Sprintf(`ORDER BY %s DESC`, m.Cols().CreateTime)
+	if orderRule != "" {
+		order = ` ORDER BY ` + orderRule
+	}
+	sql := fmt.Sprintf(`SELECT %s FROM %s WHERE 1=1 %s %s`, fields, m.TableName(), condition, order)
+	_, err = o.Raw(sql, pars).QueryRows(&items)
+	return
+}
+
+func (m *FactorEdbSeriesMapping) GetPageItemsByCondition(condition string, pars []interface{}, fieldArr []string, orderRule string, startSize, pageSize int) (items []*FactorEdbSeriesMapping, err error) {
+	o := orm.NewOrm()
+	fields := strings.Join(fieldArr, ",")
+	if len(fieldArr) == 0 {
+		fields = `*`
+	}
+	order := fmt.Sprintf(`ORDER BY %s DESC`, m.Cols().CreateTime)
+	if orderRule != "" {
+		order = ` ORDER BY ` + orderRule
+	}
+	sql := fmt.Sprintf(`SELECT %s FROM %s WHERE 1=1 %s %s LIMIT ?,?`, fields, m.TableName(), condition, order)
+	_, err = o.Raw(sql, pars, startSize, pageSize).QueryRows(&items)
+	return
+}
+
+// FactorEdbSeriesMappingItem 因子指标系列-指标关联信息
+type FactorEdbSeriesMappingItem struct {
+	SeriesId  int    `description:"因子指标系列ID"`
+	EdbInfoId int    `description:"指标ID"`
+	EdbCode   string `description:"指标编码"`
+	EdbName   string `description:"指标名称"`
+	EdbNameEn string `description:"指标名称-英文"`
+}
+
+func (m *FactorEdbSeriesMapping) Format2Item() (item *FactorEdbSeriesMappingItem) {
+	item = new(FactorEdbSeriesMappingItem)
+	item.SeriesId = m.FactorEdbSeriesId
+	item.EdbInfoId = m.EdbInfoId
+	item.EdbCode = m.EdbCode
+	return
+}

models/future_good/future_good_edb_data.go

@@ -1,6 +1,7 @@
 package future_good
 
 import (
+	"eta/eta_index_lib/services/alarm_msg"
 	"eta/eta_index_lib/utils"
 	"fmt"
 	"github.com/beego/beego/v2/client/orm"
@@ -84,9 +85,16 @@ func GetFutureGoodEdbDataList(condition string, pars []interface{}) (list []*Fut
 	return
 }
 
-// AddEdbDataFromThs 添加Ths商品指标数据
+// AddEdbDataFromThs 添加同花顺商品指标数据
 func AddEdbDataFromThs(futureGoodEdbInfoId int, edbCode string, item FutureGoodDataFromThs) (err error) {
+	var errMsg string
 	o := orm.NewOrm()
+	defer func() {
+		if err != nil {
+			//go utils.SendEmail(utils.APP_NAME_CN+"【"+utils.RunMode+"】"+"失败提醒", " 同花顺数据获取失败:err:"+errMsg, utils.EmailSendToUsers)
+			go alarm_msg.SendAlarmMsg("wind商品数据获取失败:err:"+errMsg, 3)
+		}
+	}()
 
 	var isAdd bool
 	addSql := ` INSERT INTO future_good_edb_data(future_good_edb_info_id,future_good_edb_code,data_time,trade_code,open,high,low,close,volume,amt,oi,settle,create_time,modify_time,data_timestamp) values `
@@ -97,6 +105,7 @@ func AddEdbDataFromThs(futureGoodEdbInfoId int, edbCode string, item FutureGoodD
 		eDate := table.Time[k]
 		dataTime, err := time.ParseInLocation(utils.FormatDate, eDate, time.Local)
 		if err != nil {
+			errMsg = " time.Parse :" + err.Error()
 			return err
 		}
 		timestamp := dataTime.UnixNano() / 1e6
@@ -121,6 +130,7 @@ func AddEdbDataFromThs(futureGoodEdbInfoId int, edbCode string, item FutureGoodD
 		addSql = strings.TrimRight(addSql, ",")
 		_, err = o.Raw(addSql).Exec()
 		if err != nil {
+			errMsg = " tx.Exec Err :" + err.Error()
 			return
 		}
 	}

models/mgo/base_from_ths_hf_data.go

@@ -0,0 +1,406 @@
+package mgo
+
+import (
+	"context"
+	"errors"
+	"eta/eta_index_lib/utils"
+	"fmt"
+	"github.com/qiniu/qmgo"
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+	"time"
+)
+
+// BaseFromThsHfData
+// @Description: 同花顺高频集合
+type BaseFromThsHfData struct {
+	ID                   primitive.ObjectID `json:"_id" bson:"_id,omitempty"`                                   // 文档id
+	BaseFromThsHfDataId  int64              `json:"base_from_ths_hf_data_id" bson:"base_from_ths_hf_data_id"`   // 指标数据ID
+	BaseFromThsHfIndexId int64              `json:"base_from_ths_hf_index_id" bson:"base_from_ths_hf_index_id"` // 指标ID
+	IndexCode            string             `json:"index_code" bson:"index_code"`                               // 指标编码
+	DataTime             time.Time          `json:"data_time" bson:"data_time"`                                 // 数据日期
+	Value                float64            `json:"value" bson:"value"`                                         // 数据值
+	UniqueCode           string             `json:"unique_code" bson:"unique_code"`                             // 唯一编码
+	CreateTime           time.Time          `json:"create_time" bson:"create_time"`                             // 创建时间
+	ModifyTime           time.Time          `json:"modify_time" bson:"modify_time"`                             // 修改时间
+	DataTimestamp        int64              `json:"data_timestamp" bson:"data_timestamp"`                       // 数据日期时间戳
+}
+
+// CollectionName
+// @Description:  获取集合名称
+func (m *BaseFromThsHfData) CollectionName() string {
+	return "base_from_ths_hf_data"
+}
+
+// DataBaseName
+// @Description: 获取数据库名称
+func (m *BaseFromThsHfData) DataBaseName() string {
+	return utils.MgoDataDbName
+}
+
+// GetCollection
+// @Description: 获取mongodb集合的句柄
+func (m *BaseFromThsHfData) GetCollection() *qmgo.Collection {
+	db := utils.MgoDataCli.Database(m.DataBaseName())
+	return db.Collection(m.CollectionName())
+}
+
+// GetAllDataList
+// @Description: 根据条件获取所有数据
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-26 13:42:19
+// @param sort []string
+// @param whereParams interface{}
+// @return result []BaseFromThsHfData
+// @return err error
+func (m *BaseFromThsHfData) GetAllDataList(whereParams interface{}, sort []string) (result []*BaseFromThsHfData, err error) {
+	if utils.MgoDataCli == nil {
+		err = errors.New("mongodb连接失败")
+		return
+	}
+	db := utils.MgoDataCli.Database(m.DataBaseName())
+	coll := db.Collection(m.CollectionName())
+	ctx := context.TODO()
+	if err != nil {
+		fmt.Println("MgoGetColl Err:", err.Error())
+		return
+	}
+	err = coll.Find(ctx, whereParams).Sort(sort...).All(&result)
+	if err != nil {
+		return
+	}
+
+	for _, v := range result {
+		v.DataTime = v.DataTime.In(time.Local)
+		v.CreateTime = v.CreateTime.In(time.Local)
+		v.ModifyTime = v.ModifyTime.In(time.Local)
+	}
+	return
+}
+
+// GetLimitDataList
+// @Description: 根据条件获取指定数量数据列表
+// @author: Roc
+// @receiver m
+// @datetime 2024-05-06 17:08:32
+// @param whereParams interface{}
+// @param size int64
+// @return result []*BaseFromThsHfData
+// @return err error
+func (m *BaseFromThsHfData) GetLimitDataList(whereParams interface{}, size int64, sort []string) (result []*BaseFromThsHfData, err error) {
+	if utils.MgoDataCli == nil {
+		err = errors.New("mongodb连接失败")
+		return
+	}
+	db := utils.MgoDataCli.Database(m.DataBaseName())
+	coll := db.Collection(m.CollectionName())
+	ctx := context.TODO()
+	if err != nil {
+		fmt.Println("MgoGetColl Err:", err.Error())
+		return
+	}
+	err = coll.Find(ctx, whereParams).Sort(sort...).Limit(size).All(&result)
+	if err != nil {
+		return
+	}
+
+	for _, v := range result {
+		v.DataTime = v.DataTime.In(time.Local)
+		v.CreateTime = v.CreateTime.In(time.Local)
+		v.ModifyTime = v.ModifyTime.In(time.Local)
+	}
+
+	return
+}
+
+// GetPageDataList
+// @Description: 根据条件获取分页数据列表
+// @author: Roc
+// @receiver m
+// @datetime 2024-05-07 10:21:07
+// @param whereParams interface{}
+// @param startSize int64
+// @param size int64
+// @param sort []string
+// @return result []*BaseFromThsHfData
+// @return err error
+func (m *BaseFromThsHfData) GetPageDataList(whereParams interface{}, startSize, size int64, sort []string) (result []*BaseFromThsHfData, err error) {
+	if utils.MgoDataCli == nil {
+		err = errors.New("mongodb连接失败")
+		return
+	}
+	db := utils.MgoDataCli.Database(m.DataBaseName())
+	coll := db.Collection(m.CollectionName())
+	ctx := context.TODO()
+	if err != nil {
+		fmt.Println("MgoGetColl Err:", err.Error())
+		return
+	}
+	err = coll.Find(ctx, whereParams).Sort(sort...).Skip(startSize).Limit(size).All(&result)
+	if err != nil {
+		return
+	}
+
+	for _, v := range result {
+		v.DataTime = v.DataTime.In(time.Local)
+		v.CreateTime = v.CreateTime.In(time.Local)
+		v.ModifyTime = v.ModifyTime.In(time.Local)
+	}
+
+	return
+}
+
+// GetCountDataList
+// @Description:  根据条件获取数据列表总数
+// @author: Roc
+// @receiver m
+// @datetime 2024-05-07 10:29:00
+// @param whereParams interface{}
+// @return count int64
+// @return err error
+func (m *BaseFromThsHfData) GetCountDataList(whereParams interface{}) (count int64, err error) {
+	if utils.MgoDataCli == nil {
+		err = errors.New("mongodb连接失败")
+		return
+	}
+	db := utils.MgoDataCli.Database(m.DataBaseName())
+	coll := db.Collection(m.CollectionName())
+	ctx := context.TODO()
+	if err != nil {
+		fmt.Println("MgoGetColl Err:", err.Error())
+		return
+	}
+	count, err = coll.Find(ctx, whereParams).Count()
+
+	return
+}
+
+// InsertDataByColl
+// @Description: 写入单条数据（外部传入集合）
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-26 14:22:18
+// @param addData interface{}
+// @return err error
+func (m *BaseFromThsHfData) InsertDataByColl(coll *qmgo.Collection, addData interface{}) (err error) {
+	ctx := context.TODO()
+	_, err = coll.InsertOne(ctx, addData)
+	if err != nil {
+		fmt.Println("InsertDataByColl:Err:" + err.Error())
+		return
+	}
+
+	return
+}
+
+// BatchInsertData
+// @Description: 批量写入数据
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-26 14:22:18
+// @param bulk int 每次请求保存的数据量
+// @param dataList []interface{}
+// @return err error
+func (m *BaseFromThsHfData) BatchInsertData(bulk int, dataList []interface{}) (err error) {
+	db := utils.MgoDataCli.Database(m.DataBaseName())
+	coll := db.Collection(m.CollectionName())
+
+	return m.BatchInsertDataByColl(coll, bulk, dataList)
+}
+
+// BatchInsertDataByColl
+// @Description: 批量写入数据（外部传入集合）
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-26 14:22:18
+// @param coll *qmgo.Collection
+// @param bulk int 每次请求保存的数据量
+// @param dataList []interface{}
+// @return err error
+func (m *BaseFromThsHfData) BatchInsertDataByColl(coll *qmgo.Collection, bulk int, dataList []interface{}) (err error) {
+	ctx := context.TODO()
+	dataNum := len(dataList)
+	if dataNum <= 0 {
+		return
+	}
+
+	// 不设置每次保存切片数量大小，或者实际数据量小于设置的切片数量大小，那么就直接保存吧
+	if bulk <= 0 || dataNum <= bulk {
+		_, err = coll.InsertMany(ctx, dataList)
+		if err != nil {
+			fmt.Println("BatchInsertData:Err:" + err.Error())
+			return
+		}
+		return
+	}
+
+	// 分批保存
+	i := 0
+	tmpAddDataList := make([]interface{}, 0)
+	for _, v := range dataList {
+		tmpAddDataList = append(tmpAddDataList, v)
+		i++
+		if i >= bulk {
+			_, err = coll.InsertMany(ctx, tmpAddDataList)
+			if err != nil {
+				fmt.Println("BatchInsertData:Err:" + err.Error())
+				return
+			}
+			i = 0
+			tmpAddDataList = make([]interface{}, 0)
+		}
+	}
+
+	if len(tmpAddDataList) > 0 {
+		_, err = coll.InsertMany(ctx, tmpAddDataList)
+		if err != nil {
+			fmt.Println("BatchInsertData:Err:" + err.Error())
+			return
+		}
+	}
+
+	return
+}
+
+// UpdateDataByColl
+// @Description: 单条数据修改
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-26 15:01:51
+// @param whereParams interface{}
+// @param updateParams interface{}
+// @return err error
+func (m *BaseFromThsHfData) UpdateDataByColl(coll *qmgo.Collection, whereParams, updateParams interface{}) (err error) {
+	ctx := context.TODO()
+	err = coll.UpdateOne(ctx, whereParams, updateParams)
+	if err != nil {
+		fmt.Println("UpdateDataByColl:Err:" + err.Error())
+		return
+	}
+	return
+}
+
+// UpdateData
+// @Description: 单条数据修改
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-26 15:01:51
+// @param whereParams interface{}
+// @param updateParams interface{}
+// @return err error
+func (m *BaseFromThsHfData) UpdateData(whereParams, updateParams interface{}) (err error) {
+	db := utils.MgoDataCli.Database(m.DataBaseName())
+	coll := db.Collection(m.CollectionName())
+	ctx := context.TODO()
+	err = coll.UpdateOne(ctx, whereParams, updateParams)
+	if err != nil {
+		fmt.Println("UpdateData:Err:" + err.Error())
+		return
+	}
+	return
+}
+
+// HandleData
+// @Description: 事务处理数据
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-30 10:40:20
+// @param addDataList []BaseAddFromBusinessData
+// @param updateDataList []BaseFromThsHfData
+// @return result interface{}
+// @return err error
+func (m *BaseFromThsHfData) HandleData(addDataList, updateDataList []BaseFromThsHfData) (result interface{}, err error) {
+
+	ctx := context.TODO()
+
+	callback := func(sessCtx context.Context) (interface{}, error) {
+		// 重要：确保事务中的每一个操作，都使用传入的sessCtx参数
+
+		db := utils.MgoDataCli.Database(m.DataBaseName())
+		coll := db.Collection(m.CollectionName())
+
+		// 插入数据
+		if len(addDataList) > 0 {
+			_, err = coll.InsertMany(sessCtx, addDataList)
+			if err != nil {
+				return nil, err
+			}
+		}
+
+		// 修改
+
+		if len(updateDataList) > 0 {
+			for _, v := range updateDataList {
+				err = coll.UpdateOne(ctx, bson.M{"_id": v.ID}, bson.M{"$set": bson.M{"value": v.Value, "modify_time": v.ModifyTime}})
+				if err != nil {
+					fmt.Println("BatchInsertData:Err:" + err.Error())
+					return nil, err
+				}
+			}
+		}
+
+		return nil, nil
+	}
+	result, err = utils.MgoDataCli.DoTransaction(ctx, callback)
+
+	return
+}
+
+// GetEdbInfoMaxAndMinInfo
+// @Description: 获取当前指标的最大最小值
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-30 17:15:39
+// @param whereParams interface{}
+// @return result EdbInfoMaxAndMinInfo
+// @return err error
+func (m *BaseFromThsHfData) GetEdbInfoMaxAndMinInfo(whereParams interface{}) (result EdbInfoMaxAndMinInfo, err error) {
+	if utils.MgoDataCli == nil {
+		err = errors.New("mongodb连接失败")
+		return
+	}
+	db := utils.MgoDataCli.Database(m.DataBaseName())
+	coll := db.Collection(m.CollectionName())
+	ctx := context.TODO()
+	if err != nil {
+		fmt.Println("MgoGetColl Err:", err.Error())
+		return
+	}
+	err = coll.Aggregate(ctx, whereParams).One(&result)
+	if err != nil {
+		return
+	}
+	result.MinDate = result.MinDate.In(time.Local)
+	result.MaxDate = result.MaxDate.In(time.Local)
+	result.LatestDate = result.LatestDate.In(time.Local)
+	return
+}
+
+// GetLatestValue
+// @Description: 获取当前指标的最新数据记录
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-30 17:16:15
+// @param whereParams interface{}
+// @param selectParam interface{}
+// @return latestValue LatestValue
+// @return err error
+func (m *BaseFromThsHfData) GetLatestValue(whereParams, selectParam interface{}) (latestValue LatestValue, err error) {
+	if utils.MgoDataCli == nil {
+		err = errors.New("mongodb连接失败")
+		return
+	}
+	db := utils.MgoDataCli.Database(m.DataBaseName())
+	coll := db.Collection(m.CollectionName())
+	ctx := context.TODO()
+	if err != nil {
+		fmt.Println("MgoGetColl Err:", err.Error())
+		return
+	}
+
+	//var result interface{}
+	//err = coll.Find(ctx, whereParams).Select(selectParam).One(&result)
+	err = coll.Find(ctx, whereParams).Select(selectParam).One(&latestValue)
+	return
+}

models/mgo/edb_data_ths_hf.go

@@ -0,0 +1,510 @@
+package mgo
+
+import (
+	"context"
+	"errors"
+	"eta/eta_index_lib/utils"
+	"fmt"
+	"github.com/qiniu/qmgo"
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+	"time"
+)
+
+// EdbDataThsHf
+// @Description: 同花顺高频集合（指标库）
+type EdbDataThsHf struct {
+	ID            primitive.ObjectID `json:"_id" bson:"_id,omitempty" `            // 文档id
+	EdbInfoId     int                `json:"edb_info_id" bson:"edb_info_id"`       // 指标ID
+	EdbCode       string             `json:"edb_code" bson:"edb_code"`             // 指标编码
+	DataTime      time.Time          `json:"data_time" bson:"data_time"`           // 数据日期
+	Value         float64            `json:"value" bson:"value"`                   // 数据值
+	CreateTime    time.Time          `json:"create_time" bson:"create_time"`       // 创建时间
+	ModifyTime    time.Time          `json:"modify_time" bson:"modify_time"`       // 修改时间
+	DataTimestamp int64              `json:"data_timestamp" bson:"data_timestamp"` // 数据日期时间戳
+}
+
+// CollectionName
+// @Description:  获取集合名称
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-26 13:41:36
+// @return string
+func (m *EdbDataThsHf) CollectionName() string {
+	return "edb_data_ths_hf"
+}
+
+// DataBaseName
+// @Description: 获取数据库名称
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-26 13:41:33
+// @return string
+func (m *EdbDataThsHf) DataBaseName() string {
+	return utils.MgoDataDbName
+}
+
+// GetCollection
+// @Description: 获取mongodb集合的句柄
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-26 13:41:33
+// @return string
+func (m *EdbDataThsHf) GetCollection() *qmgo.Collection {
+	db := utils.MgoDataCli.Database(m.DataBaseName())
+	return db.Collection(m.CollectionName())
+}
+
+// GetItem
+// @Description: 根据条件获取单条数据
+// @author: Roc
+// @receiver m
+// @datetime 2024-05-09 10:00:49
+// @param whereParams interface{}
+// @return item *EdbDataThsHf
+// @return err error
+func (m *EdbDataThsHf) GetItem(whereParams interface{}) (item *EdbDataThsHf, err error) {
+	if utils.MgoDataCli == nil {
+		err = errors.New("mongodb连接失败")
+		return
+	}
+	db := utils.MgoDataCli.Database(m.DataBaseName())
+	coll := db.Collection(m.CollectionName())
+
+	return m.GetItemByColl(coll, whereParams)
+}
+
+// GetItemByColl
+// @Description: 根据条件获取单条数据
+// @author: Roc
+// @receiver m
+// @datetime 2024-05-09 13:22:06
+// @param coll *qmgo.Collection
+// @param whereParams interface{}
+// @return item *EdbDataThsHf
+// @return err error
+func (m *EdbDataThsHf) GetItemByColl(coll *qmgo.Collection, whereParams interface{}) (item *EdbDataThsHf, err error) {
+	ctx := context.TODO()
+	if err != nil {
+		fmt.Println("MgoGetColl Err:", err.Error())
+		return
+	}
+	err = coll.Find(ctx, whereParams).One(&item)
+	if err != nil {
+		return
+	}
+
+	item.DataTime = item.DataTime.In(time.Local)
+	item.CreateTime = item.CreateTime.In(time.Local)
+	item.ModifyTime = item.ModifyTime.In(time.Local)
+
+	return
+}
+
+// GetAllDataList
+// @Description: 根据条件获取所有数据
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-26 13:42:19
+// @param whereParams interface{}
+// @param sort []string
+// @return result []EdbDataThsHf
+// @return err error
+func (m *EdbDataThsHf) GetAllDataList(whereParams interface{}, sort []string) (result []*EdbDataThsHf, err error) {
+	if utils.MgoDataCli == nil {
+		err = errors.New("mongodb连接失败")
+		return
+	}
+	db := utils.MgoDataCli.Database(m.DataBaseName())
+	coll := db.Collection(m.CollectionName())
+	ctx := context.TODO()
+	if err != nil {
+		fmt.Println("MgoGetColl Err:", err.Error())
+		return
+	}
+	err = coll.Find(ctx, whereParams).Sort(sort...).All(&result)
+	if err != nil {
+		return
+	}
+
+	for _, v := range result {
+		v.DataTime = v.DataTime.In(time.Local)
+		v.CreateTime = v.CreateTime.In(time.Local)
+		v.ModifyTime = v.ModifyTime.In(time.Local)
+	}
+
+	return
+}
+
+// GetLimitDataList
+// @Description: 根据条件获取指定数量数据列表
+// @author: Roc
+// @receiver m
+// @datetime 2024-05-06 17:08:32
+// @param whereParams interface{}
+// @param size int64
+// @param sort []string
+// @return result []*BaseFromBusinessData
+// @return err error
+func (m *EdbDataThsHf) GetLimitDataList(whereParams interface{}, size int64, sort []string) (result []*EdbDataThsHf, err error) {
+	if utils.MgoDataCli == nil {
+		err = errors.New("mongodb连接失败")
+		return
+	}
+	db := utils.MgoDataCli.Database(m.DataBaseName())
+	coll := db.Collection(m.CollectionName())
+	ctx := context.TODO()
+	if err != nil {
+		fmt.Println("MgoGetColl Err:", err.Error())
+		return
+	}
+	err = coll.Find(ctx, whereParams).Sort(sort...).Limit(size).All(&result)
+	if err != nil {
+		return
+	}
+
+	for _, v := range result {
+		v.DataTime = v.DataTime.In(time.Local)
+		v.CreateTime = v.CreateTime.In(time.Local)
+		v.ModifyTime = v.ModifyTime.In(time.Local)
+	}
+
+	return
+}
+
+// GetPageDataList
+// @Description: 根据条件获取分页数据列表
+// @author: Roc
+// @receiver m
+// @datetime 2024-05-07 10:21:07
+// @param whereParams interface{}
+// @param startSize int64
+// @param size int64
+// @param sort []string
+// @return result []*EdbDataThsHf
+// @return err error
+func (m *EdbDataThsHf) GetPageDataList(whereParams interface{}, startSize, size int64, sort []string) (result []*EdbDataThsHf, err error) {
+	if utils.MgoDataCli == nil {
+		err = errors.New("mongodb连接失败")
+		return
+	}
+	db := utils.MgoDataCli.Database(m.DataBaseName())
+	coll := db.Collection(m.CollectionName())
+	ctx := context.TODO()
+	if err != nil {
+		fmt.Println("MgoGetColl Err:", err.Error())
+		return
+	}
+	err = coll.Find(ctx, whereParams).Sort(sort...).Skip(startSize).Limit(size).All(&result)
+	if err != nil {
+		return
+	}
+
+	for _, v := range result {
+		v.DataTime = v.DataTime.In(time.Local)
+		v.CreateTime = v.CreateTime.In(time.Local)
+		v.ModifyTime = v.ModifyTime.In(time.Local)
+	}
+
+	return
+}
+
+// GetCountDataList
+// @Description:  根据条件获取数据列表总数
+// @author: Roc
+// @receiver m
+// @datetime 2024-05-07 10:29:00
+// @param whereParams interface{}
+// @return count int64
+// @return err error
+func (m *EdbDataThsHf) GetCountDataList(whereParams interface{}) (count int64, err error) {
+	if utils.MgoDataCli == nil {
+		err = errors.New("mongodb连接失败")
+		return
+	}
+	db := utils.MgoDataCli.Database(m.DataBaseName())
+	coll := db.Collection(m.CollectionName())
+	ctx := context.TODO()
+	if err != nil {
+		fmt.Println("MgoGetColl Err:", err.Error())
+		return
+	}
+	count, err = coll.Find(ctx, whereParams).Count()
+
+	return
+}
+
+// InsertDataByColl
+// @Description: 写入单条数据（外部传入集合）
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-26 14:22:18
+// @param addData interface{}
+// @return err error
+func (m *EdbDataThsHf) InsertDataByColl(coll *qmgo.Collection, addData interface{}) (err error) {
+	ctx := context.TODO()
+	_, err = coll.InsertOne(ctx, addData)
+	if err != nil {
+		fmt.Println("InsertDataByColl:Err:" + err.Error())
+		return
+	}
+
+	return
+}
+
+// BatchInsertData
+// @Description: 批量写入数据
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-26 14:22:18
+// @param bulk int 每次请求保存的数据量
+// @param dataList []interface{}
+// @return err error
+func (m *EdbDataThsHf) BatchInsertData(bulk int, dataList []interface{}) (err error) {
+	db := utils.MgoDataCli.Database(m.DataBaseName())
+	coll := db.Collection(m.CollectionName())
+
+	return m.BatchInsertDataByColl(coll, bulk, dataList)
+}
+
+// BatchInsertDataByColl
+// @Description: 批量写入数据（外部传入集合）
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-26 14:22:18
+// @param coll *qmgo.Collection
+// @param bulk int 每次请求保存的数据量
+// @param dataList []interface{}
+// @return err error
+func (m *EdbDataThsHf) BatchInsertDataByColl(coll *qmgo.Collection, bulk int, dataList []interface{}) (err error) {
+	ctx := context.TODO()
+	dataNum := len(dataList)
+	if dataNum <= 0 {
+		return
+	}
+
+	// 不设置每次保存切片数量大小，或者实际数据量小于设置的切片数量大小，那么就直接保存吧
+	if bulk <= 0 || dataNum <= bulk {
+		_, err = coll.InsertMany(ctx, dataList)
+		if err != nil {
+			fmt.Println("BatchInsertData:Err:" + err.Error())
+			return
+		}
+		return
+	}
+
+	// 分批保存
+	i := 0
+	tmpAddDataList := make([]interface{}, 0)
+	for _, v := range dataList {
+		tmpAddDataList = append(tmpAddDataList, v)
+		i++
+		if i >= bulk {
+			_, err = coll.InsertMany(ctx, tmpAddDataList)
+			if err != nil {
+				fmt.Println("BatchInsertData:Err:" + err.Error())
+				return
+			}
+			i = 0
+			tmpAddDataList = make([]interface{}, 0)
+		}
+	}
+
+	if len(tmpAddDataList) > 0 {
+		_, err = coll.InsertMany(ctx, tmpAddDataList)
+		if err != nil {
+			fmt.Println("BatchInsertData:Err:" + err.Error())
+			return
+		}
+	}
+
+	return
+}
+
+// UpdateData
+// @Description: 单条数据修改
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-26 15:01:51
+// @param whereParams interface{}
+// @param updateParams interface{}
+// @return err error
+func (m *EdbDataThsHf) UpdateData(whereParams, updateParams interface{}) (err error) {
+	db := utils.MgoDataCli.Database(m.DataBaseName())
+	coll := db.Collection(m.CollectionName())
+
+	return m.UpdateDataByColl(coll, whereParams, updateParams)
+}
+
+// UpdateDataByColl
+// @Description: 单条数据修改（外部传入集合）
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-26 15:01:51
+// @param whereParams interface{}
+// @param updateParams interface{}
+// @return err error
+func (m *EdbDataThsHf) UpdateDataByColl(coll *qmgo.Collection, whereParams, updateParams interface{}) (err error) {
+	ctx := context.TODO()
+	err = coll.UpdateOne(ctx, whereParams, updateParams)
+	if err != nil {
+		fmt.Println("UpdateDataByColl:Err:" + err.Error())
+		return
+	}
+
+	return
+}
+
+// RemoveMany
+// @Description: 根据条件删除多条数据
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-30 13:17:02
+// @param whereParams interface{}
+// @return err error
+func (m *EdbDataThsHf) RemoveMany(whereParams interface{}) (err error) {
+	db := utils.MgoDataCli.Database(m.DataBaseName())
+	coll := db.Collection(m.CollectionName())
+
+	return m.RemoveManyByColl(coll, whereParams)
+}
+
+// RemoveManyByColl
+// @Description: 根据条件删除多条数据（外部传入集合）
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-30 13:18:42
+// @param coll *qmgo.Collection
+// @param whereParams interface{}
+// @return err error
+func (m *EdbDataThsHf) RemoveManyByColl(coll *qmgo.Collection, whereParams interface{}) (err error) {
+	ctx := context.TODO()
+	_, err = coll.RemoveAll(ctx, whereParams)
+	if err != nil {
+		fmt.Println("RemoveManyByColl:Err:" + err.Error())
+		return
+	}
+
+	return
+}
+
+// HandleData
+// @Description: 事务处理数据
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-30 10:39:01
+// @param addDataList []AddEdbDataThsHf
+// @param updateDataList []EdbDataThsHf
+// @return result interface{}
+// @return err error
+func (m *EdbDataThsHf) HandleData(addDataList, updateDataList []EdbDataThsHf) (result interface{}, err error) {
+
+	ctx := context.TODO()
+
+	callback := func(sessCtx context.Context) (interface{}, error) {
+		// 重要：确保事务中的每一个操作，都使用传入的sessCtx参数
+
+		db := utils.MgoDataCli.Database(m.DataBaseName())
+		coll := db.Collection(m.CollectionName())
+
+		// 插入数据
+		if len(addDataList) > 0 {
+			_, err = coll.InsertMany(sessCtx, addDataList)
+			if err != nil {
+				return nil, err
+			}
+		}
+
+		// 修改
+
+		if len(updateDataList) > 0 {
+			for _, v := range updateDataList {
+				err = coll.UpdateOne(ctx, bson.M{"_id": v.ID}, bson.M{"$set": bson.M{"value": v.Value, "modify_time": v.ModifyTime}})
+				if err != nil {
+					fmt.Println("BatchInsertData:Err:" + err.Error())
+					return nil, err
+				}
+			}
+		}
+
+		return nil, nil
+	}
+	result, err = utils.MgoDataCli.DoTransaction(ctx, callback)
+
+	return
+}
+
+// EdbInfoMaxAndMinInfo 指标最新数据记录结构体
+//type EdbInfoMaxAndMinInfo struct {
+//	MinDate     time.Time `description:"最小日期" bson:"min_date"`
+//	MaxDate     time.Time `description:"最大日期" bson:"max_date"`
+//	MinValue    float64   `description:"最小值" bson:"min_value"`
+//	MaxValue    float64   `description:"最大值" bson:"max_value"`
+//	LatestValue float64   `description:"最新值" bson:"latest_value"`
+//	LatestDate  time.Time `description:"实际数据最新日期" bson:"latest_date"`
+//	EndValue    float64   `description:"最新值" bson:"end_value"`
+//}
+
+// GetEdbInfoMaxAndMinInfo
+// @Description: 获取当前指标的最大最小值
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-30 17:15:39
+// @param whereParams interface{}
+// @return result EdbInfoMaxAndMinInfo
+// @return err error
+func (m *EdbDataThsHf) GetEdbInfoMaxAndMinInfo(whereParams interface{}) (result EdbInfoMaxAndMinInfo, err error) {
+	if utils.MgoDataCli == nil {
+		err = errors.New("mongodb连接失败")
+		return
+	}
+	db := utils.MgoDataCli.Database(m.DataBaseName())
+	coll := db.Collection(m.CollectionName())
+	ctx := context.TODO()
+	if err != nil {
+		fmt.Println("MgoGetColl Err:", err.Error())
+		return
+	}
+	err = coll.Aggregate(ctx, whereParams).One(&result)
+	if err != nil {
+		return
+	}
+	result.MinDate = result.MinDate.In(time.Local)
+	result.MaxDate = result.MaxDate.In(time.Local)
+	result.LatestDate = result.LatestDate.In(time.Local)
+
+	return
+}
+
+// LatestValue 指标最新数据记录结构体
+//type LatestValue struct {
+//	Value float64 `description:"值" bson:"value"`
+//}
+
+// GetLatestValue
+// @Description: 获取当前指标的最新数据记录
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-30 17:16:15
+// @param whereParams interface{}
+// @param selectParam interface{}
+// @return latestValue LatestValue
+// @return err error
+func (m *EdbDataThsHf) GetLatestValue(whereParams, selectParam interface{}) (latestValue LatestValue, err error) {
+	if utils.MgoDataCli == nil {
+		err = errors.New("mongodb连接失败")
+		return
+	}
+	db := utils.MgoDataCli.Database(m.DataBaseName())
+	coll := db.Collection(m.CollectionName())
+	ctx := context.TODO()
+	if err != nil {
+		fmt.Println("MgoGetColl Err:", err.Error())
+		return
+	}
+
+	//var result interface{}
+	//err = coll.Find(ctx, whereParams).Select(selectParam).One(&result)
+	err = coll.Find(ctx, whereParams).Select(selectParam).One(&latestValue)
+	return
+}

models/predict_edb_data_calculate_bp.go

@@ -46,6 +46,8 @@ func SavePredictCalculateBp(req *EdbInfoCalculateBatchSaveReq, fromEdbInfo *EdbI
 		edbInfo.UnitEn = req.Unit
 		edbInfo.CalculateFormula = req.Formula
 		edbInfo.EdbType = 2
+		edbInfo.Sort = GetAddEdbMaxSortByClassifyId(req.ClassifyId, utils.PREDICT_EDB_INFO_TYPE)
+		edbInfo.EmptyType = req.EmptyType
 		newEdbInfoId, tmpErr := to.Insert(edbInfo)
 		if tmpErr != nil {
 			err = tmpErr
@@ -90,11 +92,12 @@ func SavePredictCalculateBp(req *EdbInfoCalculateBatchSaveReq, fromEdbInfo *EdbI
 			edbInfo.Unit = req.Unit
 			edbInfo.EdbNameSource = req.EdbName
 		}
-
+		oldEmptyType := edbInfo.EmptyType
 		edbInfo.Frequency = req.Frequency
 		edbInfo.ClassifyId = req.ClassifyId
+		edbInfo.EmptyType = req.EmptyType
 		edbInfo.ModifyTime = time.Now()
-		_, err = to.Update(edbInfo, "EdbName", "EdbNameSource", "Frequency", "Unit", "ClassifyId", "ModifyTime", "EdbNameEn", "UnitEn")
+		_, err = to.Update(edbInfo, "EdbName", "EdbNameSource", "Frequency", "Unit", "ClassifyId", "ModifyTime", "EdbNameEn", "UnitEn", "EmptyType")
 		if err != nil {
 			return
 		}
@@ -110,7 +113,7 @@ func SavePredictCalculateBp(req *EdbInfoCalculateBatchSaveReq, fromEdbInfo *EdbI
 			err = errors.New("判断指标是否改变失败,Err:" + e.Error())
 			return
 		}
-		if count > 0 { // 指标未被替换，无需重新计算
+		if count > 0 && oldEmptyType == edbInfo.EmptyType { // 指标未被替换，无需重新计算
 			return
 		}
 
@@ -153,12 +156,12 @@ func SavePredictCalculateBp(req *EdbInfoCalculateBatchSaveReq, fromEdbInfo *EdbI
 	}
 
 	//计算数据
-	latestDateStr, latestValue, err = refreshAllPredictCalculateBp(to, edbInfo.EdbInfoId, edbInfo.Source, edbInfo.SubSource, fromEdbInfo, edbInfo.EdbCode, "", "", 0)
+	latestDateStr, latestValue, err = refreshAllPredictCalculateBp(to, edbInfo.EdbInfoId, edbInfo.Source, edbInfo.SubSource, fromEdbInfo, edbInfo.EdbCode, "", "", 0, edbInfo.EmptyType)
 
 	return
 }
 
-func RefreshAllPredictCalculateBp(edbInfoId, source, subSource int, fromEdbInfo *EdbInfo, edbCode, startDate, endDate string) (latestDateStr string, latestValue float64, err error) {
+func RefreshAllPredictCalculateBp(edbInfoId, source, subSource int, fromEdbInfo *EdbInfo, edbCode, startDate, endDate string, emptyType int) (latestDateStr string, latestValue float64, err error) {
 	o := orm.NewOrm()
 	to, err := o.Begin()
 	if err != nil {
@@ -174,14 +177,16 @@ func RefreshAllPredictCalculateBp(edbInfoId, source, subSource int, fromEdbInfo
 	}()
 
 	// 计算数据
-	latestDateStr, latestValue, err = refreshAllPredictCalculateBp(to, edbInfoId, source, subSource, fromEdbInfo, edbCode, startDate, endDate, 1)
+	latestDateStr, latestValue, err = refreshAllPredictCalculateBp(to, edbInfoId, source, subSource, fromEdbInfo, edbCode, startDate, endDate, 0, emptyType)
 	return
 }
 
 // refreshAllPredictCalculateBp 刷新变频数据
-func refreshAllPredictCalculateBp(to orm.TxOrmer, edbInfoId, source, subSource int, fromEdbInfo *EdbInfo, edbCode, startDate, endDate string, order int) (latestDateStr string, latestValue float64, err error) {
+func refreshAllPredictCalculateBp(to orm.TxOrmer, edbInfoId, source, subSource int, fromEdbInfo *EdbInfo, edbCode, startDate, endDate string, order, emptyType int) (latestDateStr string, latestValue float64, err error) {
 	edbInfoIdStr := strconv.Itoa(edbInfoId)
-	// 获取关联指标数据
+	//计算数据
+
+	//获取来源指标的数据
 	dataList, err := GetPredictEdbDataListAllByStartDate(fromEdbInfo, order, "")
 	if err != nil {
 		return
@@ -199,189 +204,144 @@ func refreshAllPredictCalculateBp(to orm.TxOrmer, edbInfoId, source, subSource i
 	}
 	fmt.Println("source:", source)
 
-	//获取变频指标所有数据
+	//获取升频指标所有数据
 	existDataList, err := GetAllEdbDataListByTo(to, edbInfoId, source, subSource)
 	if err != nil {
 		return
 	}
 	//计算指标的map
 	existDataMap := make(map[string]*EdbData, 0)
-
+	removeDateMap := make(map[string]struct{})
 	addSql := ` INSERT INTO edb_data_predict_calculate_bp(edb_info_id,edb_code,data_time,value,create_time,modify_time,data_timestamp) values `
-	var isAdd bool
 
-	var lastValue float64   //最后数据的值(float64)
-	var lastValueStr string //最后数据的值(string)
+	var isAdd bool
 	//待删除的日期
 	removeDateList := make([]string, 0)
-	if len(existDataList) > 0 {
-		//第一个已经入库的日期
-		firstExistDataTimeStr := existDataList[0].DataTime //计算指标数据第一条的日期字符串
-		if len(dateArr) > 0 {
-			firstFromDataTimeStr := dateArr[0]                                                                 //来源数据第一条的日期字符串
-			firstExistDataTime, _ := time.ParseInLocation(utils.FormatDate, firstExistDataTimeStr, time.Local) //计算指标数据第一条的日期（time类型）
-			firstFromDataTime, _ := time.ParseInLocation(utils.FormatDate, firstFromDataTimeStr, time.Local)   //来源数据第一条的日期（time类型）
-			nowDateStr := time.Now().Format(utils.FormatDate)                                                  //当天日期字符串
-			nowDate, _ := time.ParseInLocation(utils.FormatDate, nowDateStr, firstFromDataTime.Location())     //当天日期（time类型）
+	if len(existDataList) > 0 && len(dateArr) == 0 {
+		//如果没有来源指标数据，那么已经入库的计算指标数据需要全部删除
+		tableName := GetEdbDataTableName(source, subSource)
+		sql := fmt.Sprintf(` DELETE FROM %s WHERE edb_info_id = ?`, tableName)
+		_, err = to.Raw(sql, edbInfoId).Exec()
+		if err != nil {
+			err = fmt.Errorf("删除所有的升频指标数据失败,Err:" + err.Error())
+			return
+		}
+		return
+	}
 
-			lastValue = fromDataMap[firstFromDataTimeStr]
-			lastValueStr = decimal.NewFromFloat(lastValue).String()
-			//第一步： 判断来源指标的开始时间与计算指标的开始时间是否相等，相等的话，那么就不需要对两个时间之间的数据做处理
-			if firstExistDataTimeStr != firstFromDataTimeStr {
-				if firstExistDataTime.Before(firstFromDataTime) { //如果计算指标第一条数据的开始时间 早于 来源指标的第一条开始时间，那么需要对两个时间之间的计算指标数据做 删除处理
-					for _, v := range existDataList {
-						if v.DataTime == firstFromDataTimeStr {
-							if tmpLastValue, ok := fromDataMap[firstFromDataTimeStr]; ok { //来源指标当天的数据
-								lastValue = tmpLastValue
-								lastValueStr = decimal.NewFromFloat(lastValue).String()
-							}
-							break
-						}
-						removeDateList = append(removeDateList, v.DataTime)
-					}
-				} else {
-					for _, v := range dateArr { //如果计算指标第一条数据的开始时间 晚于 来源指标的第一条开始时间，那么需要对两个时间之间的计算指标数据做 新增处理
-						vDataTime, _ := time.ParseInLocation(utils.FormatDate, v, time.Local) //当前日期（time类型）
-						if firstExistDataTime.Equal(vDataTime) || firstExistDataTime.Before(vDataTime) {
-							if tmpLastValue, ok := fromDataMap[v]; ok { //来源指标当天的数据
-								lastValue = tmpLastValue
-								lastValueStr = decimal.NewFromFloat(lastValue).String()
-							}
-							break
-						}
+	existMap := make(map[string]string)
 
-						currentDate, _ := time.ParseInLocation(utils.FormatDate, v, time.Local)
-						timestamp := currentDate.UnixNano() / 1e6
-						timestampStr := fmt.Sprintf("%d", timestamp)
-						addSql += GetAddSql(edbInfoIdStr, edbCode, v, timestampStr, lastValueStr)
-						// 实际数据的值
-						if fromEdbInfo.LatestDate == v {
-							latestValue = lastValue
-						}
-						isAdd = true
-					}
-				}
+	dataLen := len(dataList)
+	//第三步： 已经入库的数据处理
+	for _, v := range existDataList {
+		existDataMap[v.DataTime] = v
+		removeDateMap[v.DataTime] = struct{}{}
+	}
+	for i := 0; i < dataLen; i++ {
+		//当期
+		currentItem := dataList[i]
+		var prevItem *EdbInfoSearchData
+		if emptyType == 3 { //3后值填充,其余前值填充
+			if i >= 1 {
+				prevItem = dataList[i-1]
 			}
-
-			//第二步 剩余数据每天修改
-
-			day := int(nowDate.Sub(firstExistDataTime).Hours() / float64(24))
-
-			//第三步： 已经入库的数据处理
-			for _, v := range existDataList {
-				existDataMap[v.DataTime] = v
+		}
+		currentDate, _ := time.ParseInLocation(utils.FormatDate, currentItem.DataTime, time.Local)
+		var day int
+		var preItem *EdbInfoSearchData
+		var preDate time.Time
+		if i == 0 {
+			if emptyType == 3 { //后值填充
+				day = 0 //最新的时间就是来源指标的最新日期
+				preDate = currentDate
+			} else {
+				day = int(time.Now().Sub(currentDate).Hours() / float64(24))
+				preDate = time.Date(time.Now().Year(), time.Now().Month(), time.Now().Day(), 0, 0, 0, 0, time.Local)
 			}
-
-			for k := day; k >= 0; k-- {
-				needDay := nowDate.AddDate(0, 0, -k)
-				needDayStr := needDay.Format(utils.FormatDate)
-				tmpExistData, ok := existDataMap[needDayStr]
-				if ok {
-					if tmpLastValue, ok := fromDataMap[tmpExistData.DataTime]; ok { //来源指标当天的数据
-						lastValue = tmpLastValue
-						//lastValueStr = decimal.NewFromFloat(lastValue).String()
-						lastValueStr = fmt.Sprintf("%.4f", lastValue)
-					}
-					if fromEdbInfo.LatestDate == tmpExistData.DataTime {
-						latestValue = lastValue
+		} else {
+			j := i - 1
+			if j < dataLen {
+				preItem = dataList[j]
+				preDate, _ = time.ParseInLocation(utils.FormatDate, preItem.DataTime, time.Local)
+				day = int(preDate.Sub(currentDate).Hours() / float64(24))
+				utils.FileLog.Info("preItem.DataTime:" + preItem.DataTime + ";currentItem.DataTime" + currentItem.DataTime)
+			}
+		}
+		for k := 0; k <= day; k++ {
+			needDay := preDate.AddDate(0, 0, -k)
+			needDayStr := needDay.Format(utils.FormatDate)
+			delete(removeDateMap, needDayStr)
+			existKey := edbCode + needDayStr
+			if _, ok := existMap[existKey]; !ok {
+				timestamp := needDay.UnixNano() / 1e6
+				timestampStr := fmt.Sprintf("%d", timestamp)
+				valStr := decimal.NewFromFloat(currentItem.Value).String()
+				if prevItem != nil && needDayStr != currentItem.DataTime {
+					valStr = decimal.NewFromFloat(prevItem.Value).String()
+				}
+				if fromEdbInfo.LatestDate == needDayStr {
+					if prevItem != nil && needDayStr != currentItem.DataTime {
+						latestValue = prevItem.Value
+					} else {
+						latestValue = currentItem.Value
 					}
+				}
+
+				tmpExistData, ok2 := existDataMap[needDayStr]
+				if !ok2 {
+					addSql += GetAddSql(edbInfoIdStr, edbCode, needDayStr, timestampStr, valStr)
+					isAdd = true
+				} else {
 					//如果对应的值不匹配
-					if tmpExistData.Value != lastValueStr {
-						err = ModifyEdbDataById(source, subSource, tmpExistData.EdbDataId, lastValueStr)
+					if tmpExistData.Value != valStr {
+						err = ModifyEdbDataById(source, subSource, tmpExistData.EdbDataId, valStr)
 						if err != nil {
 							return
 						}
 					}
-				} else {
-					timestamp := needDay.UnixNano() / 1e6
-					timestampStr := fmt.Sprintf("%d", timestamp)
-					addSql += GetAddSql(edbInfoIdStr, edbCode, needDayStr, timestampStr, lastValueStr)
-					if fromEdbInfo.LatestDate == needDayStr {
-						latestValue = lastValue
-					}
-					isAdd = true
 				}
-			}
-		} else {
-			//如果没有来源指标数据，那么已经入库的计算指标数据需要全部删除
-			tableName := GetEdbDataTableName(source, subSource)
-			sql := fmt.Sprintf(` DELETE FROM %s WHERE edb_info_id = ?`, tableName)
-			_, err = to.Raw(sql, edbInfoId).Exec()
-			if err != nil {
-				err = fmt.Errorf("删除所有的变频指标数据失败,Err:" + err.Error())
-				return
-			}
 
-			//for _, v := range existDataList {
-			//	removeDateList = append(removeDateList, v.DataTime)
-			//}
+			}
+			existMap[existKey] = needDayStr
 		}
-	} else {
-		existMap := make(map[string]string)
-		dataLen := len(dataList)
-
-		for i := 0; i < dataLen; i++ {
-			//当期
-			currentItem := dataList[i]
+		existKey := edbCode + currentItem.DataTime
+		if _, ok := existMap[existKey]; !ok {
 			currentDate, _ := time.ParseInLocation(utils.FormatDate, currentItem.DataTime, time.Local)
-			var day int
-			var preItem *EdbInfoSearchData
-			var preDate time.Time
-			if i == 0 {
-				day = int(time.Now().Sub(currentDate).Hours() / float64(24))
-				preDate = time.Now()
-			} else {
-				j := i - 1
-				if j < dataLen {
-					preItem = dataList[j]
-					preDate, _ = time.ParseInLocation(utils.FormatDate, preItem.DataTime, time.Local)
-					day = int(preDate.Sub(currentDate).Hours() / float64(24))
-					utils.FileLog.Info("preItem.DataTime:" + preItem.DataTime + ";currentItem.DataTime" + currentItem.DataTime)
-				}
+			timestamp := currentDate.UnixNano() / 1e6
+			timestampStr := fmt.Sprintf("%d", timestamp)
+			valStr := decimal.NewFromFloat(currentItem.Value).String()
+			if fromEdbInfo.LatestDate == currentItem.DataTime {
+				latestValue = currentItem.Value
 			}
-			for k := 0; k <= day; k++ {
-				needDay := preDate.AddDate(0, 0, -k)
-				needDayStr := needDay.Format(utils.FormatDate)
-				existKey := edbCode + needDayStr
-				if _, ok := existMap[existKey]; !ok {
-					timestamp := needDay.UnixNano() / 1e6
-					timestampStr := fmt.Sprintf("%d", timestamp)
-					valStr := decimal.NewFromFloat(currentItem.Value).String()
-					addSql += GetAddSql(edbInfoIdStr, edbCode, needDayStr, timestampStr, valStr)
-					if fromEdbInfo.LatestDate == needDayStr {
-						latestValue = currentItem.Value
-					}
-					isAdd = true
-				}
-				existMap[existKey] = needDayStr
-			}
-			existKey := edbCode + currentItem.DataTime
-			if _, ok := existMap[existKey]; !ok {
-				currentDate, _ := time.ParseInLocation(utils.FormatDate, currentItem.DataTime, time.Local)
-				timestamp := currentDate.UnixNano() / 1e6
-				timestampStr := fmt.Sprintf("%d", timestamp)
-				valStr := decimal.NewFromFloat(currentItem.Value).String()
+			tmpExistData, ok2 := existDataMap[currentItem.DataTime]
+			if !ok2 {
 				addSql += GetAddSql(edbInfoIdStr, edbCode, currentItem.DataTime, timestampStr, valStr)
-				if fromEdbInfo.LatestDate == currentItem.DataTime {
-					latestValue = currentItem.Value
-				}
 				isAdd = true
+			} else {
+				//如果对应的值不匹配
+				if tmpExistData.Value != valStr {
+					err = ModifyEdbDataById(source, subSource, tmpExistData.EdbDataId, valStr)
+					if err != nil {
+						return
+					}
+				}
 			}
-			existMap[existKey] = currentItem.DataTime
+
 		}
+		existMap[existKey] = currentItem.DataTime
 	}
 
+	for k, _ := range removeDateMap {
+		removeDateList = append(removeDateList, k)
+	}
 	// 删除不需要的指标数据
 	if len(removeDateList) > 0 {
-		removeDateStr := strings.Join(removeDateList, `","`)
-		removeDateStr = `"` + removeDateStr + `"`
 		//如果拼接指标变更了，那么需要删除所有的指标数据
 		tableName := GetEdbDataTableName(source, subSource)
-		sql := fmt.Sprintf(` DELETE FROM %s WHERE edb_info_id = ? and data_time in (%s) `, tableName, removeDateStr)
-
-		_, err = to.Raw(sql, edbInfoId).Exec()
+		sql := fmt.Sprintf(` DELETE FROM %s WHERE edb_info_id = ? and data_time in (`+utils.GetOrmInReplace(len(removeDateList))+`) `, tableName)
+		_, err = to.Raw(sql, edbInfoId, removeDateList).Exec()
 		if err != nil {
-			err = fmt.Errorf("删除不存在的变频指标数据失败,Err:" + err.Error())
+			err = fmt.Errorf("删除不存在的升频指标数据失败,Err:" + err.Error())
 			return
 		}
 	}

models/predict_edb_data_calculate_cjjx.go

@@ -48,6 +48,7 @@ func SavePredictCalculateCjjx(req *EdbInfoCalculateBatchSaveReq, fromEdbInfo *Ed
 		edbInfo.EdbNameEn = req.EdbName
 		edbInfo.UnitEn = req.Unit
 		edbInfo.EdbType = 2
+		edbInfo.Sort = GetAddEdbMaxSortByClassifyId(req.ClassifyId, utils.PREDICT_EDB_INFO_TYPE)
 		newEdbInfoId, tmpErr := to.Insert(edbInfo)
 		if tmpErr != nil {
 			err = tmpErr

models/predict_edb_data_calculate_hbz.go

@@ -46,6 +46,7 @@ func SavePredictCalculateHbz(req *EdbInfoCalculateBatchSaveReq, fromEdbInfo *Edb
 		edbInfo.EdbNameEn = req.EdbName
 		edbInfo.UnitEn = req.Unit
 		edbInfo.EdbType = 2
+		edbInfo.Sort = GetAddEdbMaxSortByClassifyId(req.ClassifyId, utils.PREDICT_EDB_INFO_TYPE)
 		newEdbInfoId, tmpErr := to.Insert(edbInfo)
 		if tmpErr != nil {
 			err = tmpErr

models/predict_edb_data_calculate_hcz.go

@@ -46,6 +46,7 @@ func SavePredictCalculateHcz(req *EdbInfoCalculateBatchSaveReq, fromEdbInfo *Edb
 		edbInfo.EdbNameEn = req.EdbName
 		edbInfo.UnitEn = req.Unit
 		edbInfo.EdbType = 2
+		edbInfo.Sort = GetAddEdbMaxSortByClassifyId(req.ClassifyId, utils.PREDICT_EDB_INFO_TYPE)
 		newEdbInfoId, tmpErr := to.Insert(edbInfo)
 		if tmpErr != nil {
 			err = tmpErr

models/predict_edb_data_calculate_jp.go

@@ -51,7 +51,7 @@ func SavePredictCalculateJp(reqEdbInfoId, classifyId int, edbName, frequency, un
 			MaxValue:         0,
 			CalculateFormula: formula,
 			EdbType:          2,
-			Sort:             0,
+			Sort:             GetAddEdbMaxSortByClassifyId(classifyId, utils.PREDICT_EDB_INFO_TYPE),
 			MoveType:         0,
 			MoveFrequency:    "",
 			NoUpdate:         0,
@@ -238,9 +238,12 @@ func refreshAllPredictCalculateJp(to orm.TxOrmer, edbInfoId, source, subSource i
 		return
 	}
 	//计算指标的map
-	existDataMap := make(map[string]*EdbData, 0)
+	existDataMap := make(map[string]*EdbData)
+	// 已经入库的日期map
+	existDelDateMap := make(map[string]string)
 	for _, v := range existDataList {
 		existDataMap[v.DataTime] = v
+		existDelDateMap[v.DataTime] = v.DataTime
 	}
 
 	latestDateStr = fromEdbInfo.LatestDate
@@ -257,8 +260,7 @@ func refreshAllPredictCalculateJp(to orm.TxOrmer, edbInfoId, source, subSource i
 	startDataTime, _ := time.ParseInLocation(utils.FormatDate, dataList[0].DataTime, time.Local)
 	endDataTime, _ := time.ParseInLocation(utils.FormatDate, dataList[dataLen-1].DataTime, time.Local)
 
-	var lastValue float64     // 最近的值
-	var nextEndDate time.Time // 下一个节点的日期
+	nextEndDate := utils.GetFrequencyEndDay(startDataTime, edbFrequency) // 下一个节点的日期
 	weekDayDataList := make([]float64, 0)
 	for currStartDataTime := startDataTime; !currStartDataTime.After(endDataTime); currStartDataTime = currStartDataTime.AddDate(0, 0, 1) {
 		// 将当前数据加入到 weekDayDataList
@@ -267,10 +269,6 @@ func refreshAllPredictCalculateJp(to orm.TxOrmer, edbInfoId, source, subSource i
 			tmpValueFloat, _ := tmpValue.Round(4).Float64()
 			weekDayDataList = append(weekDayDataList, tmpValueFloat)
 		}
-		// 如果下个节点的日期不存在,那么就先给赋值(兼容时间区间内只有一组数据的情况)
-		if nextEndDate.IsZero() {
-			nextEndDate = utils.GetFrequencyEndDay(currStartDataTime, edbFrequency)
-		}
 
 		// 日期处理过滤
 		switch edbFrequency {
@@ -328,34 +326,35 @@ func refreshAllPredictCalculateJp(to orm.TxOrmer, edbInfoId, source, subSource i
 			return
 		}
 
+		// 本期的数据值
+		lenWeekDayDataList := len(weekDayDataList)
+		if lenWeekDayDataList <= 0 {
+			continue
+		}
+
 		// 当前时间段内的数据计算,得出实际值
 		var currVal float64
-		lenWeekDayDataList := len(weekDayDataList)
-		// 如果这个时间区间内没有数据,那么就采用上一个时间区间的值
-		if len(weekDayDataList) <= 0 {
-			currVal = lastValue
+		if formula == "期末值" { // 期末值，取区间最后一个日期的数据值
+			currVal = weekDayDataList[lenWeekDayDataList-1]
 		} else {
-			if formula == "期末值" {
-				currVal = weekDayDataList[lenWeekDayDataList-1]
-			} else {
-				// 平均值
-				sumValDeci := decimal.NewFromFloat(0)
-				for _, v := range weekDayDataList {
-					tmpValDeci := decimal.NewFromFloat(v)
-					sumValDeci = sumValDeci.Add(tmpValDeci)
-				}
-				lenDeci := decimal.NewFromInt(int64(lenWeekDayDataList))
-				currVal, _ = sumValDeci.Div(lenDeci).Round(4).Float64()
+			// 平均值 取区间平均值
+			sumValDeci := decimal.NewFromFloat(0)
+			for _, v := range weekDayDataList {
+				tmpValDeci := decimal.NewFromFloat(v)
+				sumValDeci = sumValDeci.Add(tmpValDeci)
 			}
+			lenDeci := decimal.NewFromInt(int64(lenWeekDayDataList))
+			currVal, _ = sumValDeci.Div(lenDeci).Round(4).Float64()
 		}
 
 		// 给实际日期数据的值赋值
 		if fromEdbInfo.LatestDate == currStartDataTime.Format(utils.FormatDate) {
 			latestValue = currVal
 		}
+		currStartDataTimeStr := currStartDataTime.Format(utils.FormatDate)
 
 		// 判断降频指标是否存在数据
-		if existData, ok := existDataMap[currStartDataTime.Format(utils.FormatDate)]; ok {
+		if existData, ok := existDataMap[currStartDataTimeStr]; ok {
 			// 处理降频数据的值
 			existValStr := existData.Value
 			existValDeci, tmpErr := decimal.NewFromString(existValStr)
@@ -371,12 +370,18 @@ func refreshAllPredictCalculateJp(to orm.TxOrmer, edbInfoId, source, subSource i
 					return
 				}
 			}
+
+			// 移除待删除的日期
+			delete(existDelDateMap, currStartDataTimeStr)
 		} else {
 			// 直接入库
 			timestamp := currStartDataTime.UnixNano() / 1e6
 			timestampStr := fmt.Sprintf("%d", timestamp)
 			addSql += GetAddSql(edbInfoIdStr, edbCode, currStartDataTime.Format(utils.FormatDate), timestampStr, fmt.Sprint(currVal))
 			isAdd = true
+
+			// 移除待删除的日期
+			delete(existDelDateMap, currStartDataTimeStr)
 		}
 
 		// 一轮结束后,数据清空
@@ -384,30 +389,27 @@ func refreshAllPredictCalculateJp(to orm.TxOrmer, edbInfoId, source, subSource i
 	}
 
 	// 最后已有的日期处理完成后,需要对剩余不在时间段内的数据做处理
-	if len(weekDayDataList) > 0 {
+	lenWeekDayDataList := len(weekDayDataList)
+	if lenWeekDayDataList > 0 {
 		// 当前时间段内的数据计算,得出实际值
 		var currVal float64
-		lenWeekDayDataList := len(weekDayDataList)
-		// 如果这个时间区间内没有数据,那么就采用上一个时间区间的值
-		if len(weekDayDataList) < 0 {
-			currVal = lastValue
+		if formula == "期末值" {
+			currVal = weekDayDataList[lenWeekDayDataList-1]
 		} else {
-			if formula == "期末值" {
-				currVal = weekDayDataList[lenWeekDayDataList-1]
-			} else {
-				// 平均值
-				sumValDeci := decimal.NewFromFloat(0)
-				for _, v := range weekDayDataList {
-					tmpValDeci := decimal.NewFromFloat(v)
-					sumValDeci = sumValDeci.Add(tmpValDeci)
-				}
-				lenDeci := decimal.NewFromInt(int64(lenWeekDayDataList))
-				currVal, _ = sumValDeci.Div(lenDeci).Round(4).Float64()
+			// 平均值
+			sumValDeci := decimal.NewFromFloat(0)
+			for _, v := range weekDayDataList {
+				tmpValDeci := decimal.NewFromFloat(v)
+				sumValDeci = sumValDeci.Add(tmpValDeci)
 			}
+			lenDeci := decimal.NewFromInt(int64(lenWeekDayDataList))
+			currVal, _ = sumValDeci.Div(lenDeci).Round(4).Float64()
 		}
 
+		nextEndDateStr := nextEndDate.Format(utils.FormatDate)
+
 		// 判断降频指标是否存在数据
-		if existData, ok := existDataMap[nextEndDate.Format(utils.FormatDate)]; ok {
+		if existData, ok := existDataMap[nextEndDateStr]; ok {
 			// 处理降频数据的值
 			existValStr := existData.Value
 			existValDeci, tmpErr := decimal.NewFromString(existValStr)
@@ -423,12 +425,18 @@ func refreshAllPredictCalculateJp(to orm.TxOrmer, edbInfoId, source, subSource i
 					return
 				}
 			}
+
+			// 移除待删除的日期
+			delete(existDelDateMap, nextEndDateStr)
 		} else {
 			// 直接入库
 			timestamp := nextEndDate.UnixNano() / 1e6
 			timestampStr := fmt.Sprintf("%d", timestamp)
 			addSql += GetAddSql(edbInfoIdStr, edbCode, nextEndDate.Format(utils.FormatDate), timestampStr, fmt.Sprint(currVal))
 			isAdd = true
+
+			// 移除待删除的日期
+			delete(existDelDateMap, nextEndDateStr)
 		}
 	}
 
@@ -437,5 +445,21 @@ func refreshAllPredictCalculateJp(to orm.TxOrmer, edbInfoId, source, subSource i
 		_, err = to.Raw(addSql).Exec()
 	}
 
+	// 移除不存在的日期数据
+	if len(existDelDateMap) > 0 {
+		removeDateList := make([]string, 0) //需要移除的日期
+		for k := range existDelDateMap {
+			removeDateList = append(removeDateList, k)
+		}
+		removeDateStr := strings.Join(removeDateList, `","`)
+		removeDateStr = `"` + removeDateStr + `"`
+		sql := fmt.Sprintf(` DELETE FROM %s WHERE edb_info_id = ? and data_time in (%s) `, tableName, removeDateStr)
+		_, err = to.Raw(sql, edbInfoId).Exec()
+		if err != nil {
+			err = fmt.Errorf("删除年化指标数据失败,Err:" + err.Error())
+			return
+		}
+	}
+
 	return
 }

models/predict_edb_data_calculate_kszs.go

@@ -54,7 +54,7 @@ func SavePredictCalculateKszs(reqEdbInfoId, classifyId int, edbName, frequency,
 			MaxValue:         0,
 			CalculateFormula: formula,
 			EdbType:          2,
-			Sort:             0,
+			Sort:             GetAddEdbMaxSortByClassifyId(classifyId, utils.PREDICT_EDB_INFO_TYPE),
 			MoveType:         0,
 			MoveFrequency:    "",
 			NoUpdate:         0,

models/predict_edb_data_calculate_ljz.go

@@ -68,7 +68,7 @@ func (obj PredictLjz) Add(params BatchSaveCalculateBatchParams) (edbInfo *EdbInf
 		MaxValue:         0,
 		CalculateFormula: req.Formula,
 		EdbType:          2,
-		Sort:             0,
+		Sort:             GetAddEdbMaxSortByClassifyId(req.ClassifyId, utils.PREDICT_EDB_INFO_TYPE),
 		MoveType:         0,
 		MoveFrequency:    "",
 		NoUpdate:         0,

models/predict_edb_data_calculate_ljznczj.go

@@ -67,7 +67,7 @@ func (obj PredictLjzNczj) Add(params BatchSaveCalculateBatchParams) (edbInfo *Ed
 		MaxValue:         0,
 		CalculateFormula: req.Formula,
 		EdbType:          2,
-		Sort:             0,
+		Sort:             GetAddEdbMaxSortByClassifyId(req.ClassifyId, utils.PREDICT_EDB_INFO_TYPE),
 		MoveType:         0,
 		MoveFrequency:    "",
 		NoUpdate:         0,

models/predict_edb_data_calculate_ljzzj.go

@@ -59,6 +59,7 @@ func (obj PredictLjzzj) Add(params BatchSaveCalculateBatchParams) (edbInfo *EdbI
 	edbInfo.EdbNameEn = req.EdbName
 	edbInfo.UnitEn = req.Unit
 	edbInfo.EdbType = 2
+	edbInfo.Sort = GetAddEdbMaxSortByClassifyId(req.ClassifyId, utils.PREDICT_EDB_INFO_TYPE)
 	newEdbInfoId, tmpErr := to.Insert(edbInfo)
 	if tmpErr != nil {
 		err = tmpErr

models/predict_edb_data_calculate_ljzzy.go

@@ -47,6 +47,7 @@ func SavePredictCalculateLjzzy(req *EdbInfoCalculateBatchSaveReq, fromEdbInfo *E
 		edbInfo.EdbNameEn = req.EdbName
 		edbInfo.UnitEn = req.Unit
 		edbInfo.EdbType = 2
+		edbInfo.Sort = GetAddEdbMaxSortByClassifyId(req.ClassifyId, utils.PREDICT_EDB_INFO_TYPE)
 		newEdbInfoId, tmpErr := to.Insert(edbInfo)
 		if tmpErr != nil {
 			err = tmpErr

models/predict_edb_data_calculate_nh.go

@@ -51,7 +51,7 @@ func SavePredictCalculateNh(reqEdbInfoId, classifyId int, edbName, frequency, un
 			MaxValue:         0,
 			CalculateFormula: formula,
 			EdbType:          2,
-			Sort:             0,
+			Sort:             GetAddEdbMaxSortByClassifyId(classifyId, utils.PREDICT_EDB_INFO_TYPE),
 			MoveType:         0,
 			MoveFrequency:    "",
 			NoUpdate:         0,

models/predict_edb_data_calculate_nhcc.go

@@ -62,6 +62,7 @@ func SavePredictCalculateNhcc(req *EdbInfoCalculateBatchSaveReq, firstEdbInfo, s
 			EdbNameEn:        req.EdbName,
 			UnitEn:           req.Unit,
 			EdbType:          2,
+			Sort:             GetAddEdbMaxSortByClassifyId(req.ClassifyId, utils.PREDICT_EDB_INFO_TYPE),
 		}
 		newEdbInfoId, tmpErr := to.Insert(edbInfo)
 		if tmpErr != nil {

models/predict_edb_data_calculate_nszydbpjjs.go

@@ -57,7 +57,7 @@ func SavePredictCalculateNszydpjjs(reqEdbInfoId, classifyId int, edbName, freque
 			MaxValue:         0,
 			CalculateFormula: formula,
 			EdbType:          2,
-			Sort:             0,
+			Sort:             GetAddEdbMaxSortByClassifyId(classifyId, utils.PREDICT_EDB_INFO_TYPE),
 			MoveType:         0,
 			MoveFrequency:    "",
 			NoUpdate:         0,

models/predict_edb_data_calculate_percentile.go

@@ -58,7 +58,7 @@ func (obj PredictPercentile) Add(params BatchSaveCalculateBatchParams) (edbInfo
 		MaxValue:         0,
 		CalculateFormula: req.Formula,
 		EdbType:          2,
-		Sort:             0,
+		Sort:             GetAddEdbMaxSortByClassifyId(req.ClassifyId, utils.PREDICT_EDB_INFO_TYPE),
 		MoveType:         0,
 		MoveFrequency:    "",
 		NoUpdate:         0,

models/predict_edb_data_calculate_standard_deviation.go

@@ -57,7 +57,7 @@ func (obj PredictStandardDeviation) Add(params BatchSaveCalculateBatchParams) (e
 		MaxValue:         0,
 		CalculateFormula: req.Formula,
 		EdbType:          2,
-		Sort:             0,
+		Sort:             GetAddEdbMaxSortByClassifyId(req.ClassifyId, utils.PREDICT_EDB_INFO_TYPE),
 		MoveType:         0,
 		MoveFrequency:    "",
 		NoUpdate:         0,

models/predict_edb_data_calculate_tbz.go

@@ -57,7 +57,7 @@ func (obj PredictTb) Add(params BatchSaveCalculateBatchParams) (edbInfo *EdbInfo
 		MaxValue:         0,
 		CalculateFormula: req.Formula,
 		EdbType:          2,
-		Sort:             0,
+		Sort:             GetAddEdbMaxSortByClassifyId(req.ClassifyId, utils.PREDICT_EDB_INFO_TYPE),
 		MoveType:         0,
 		MoveFrequency:    "",
 		NoUpdate:         0,

models/predict_edb_data_calculate_tcz.go

@@ -51,7 +51,7 @@ func SavePredictCalculateTcz(reqEdbInfoId, classifyId int, edbName, frequency, u
 			MaxValue:         0,
 			CalculateFormula: formula,
 			EdbType:          2,
-			Sort:             0,
+			Sort:             GetAddEdbMaxSortByClassifyId(classifyId, utils.PREDICT_EDB_INFO_TYPE),
 			MoveType:         0,
 			MoveFrequency:    "",
 			NoUpdate:         0,

models/predict_edb_data_calculate_time_shift.go

@@ -49,6 +49,7 @@ func SavePredictCalculateTimeShift(req *EdbInfoCalculateBatchSaveReq, fromEdbInf
 		edbInfo.MoveFrequency = req.MoveFrequency
 		edbInfo.EdbNameEn = req.EdbName
 		edbInfo.UnitEn = req.Unit
+		edbInfo.Sort = GetAddEdbMaxSortByClassifyId(req.ClassifyId, utils.PREDICT_EDB_INFO_TYPE)
 		newEdbInfoId, tmpErr := to.Insert(edbInfo)
 		if tmpErr != nil {
 			err = tmpErr
@@ -300,7 +301,7 @@ func refreshAllPredictCalculateTimeShift(to orm.TxOrmer, edbInfoId, source, subS
 			return
 		}
 	}
-	
+
 	// 移除不存在的日期数据
 	if len(removeDateMap) > 0 {
 		removeDateList := make([]string, 0) //需要移除的日期

models/predict_edb_data_calculate_zsxy.go

@@ -62,7 +62,7 @@ func (obj PredictExponentialSmoothing) Add(params BatchSaveCalculateBatchParams)
 		MaxValue:         0,
 		CalculateFormula: req.Formula,
 		EdbType:          2,
-		Sort:             0,
+		Sort:             GetAddEdbMaxSortByClassifyId(req.ClassifyId, utils.PREDICT_EDB_INFO_TYPE),
 		MoveType:         0,
 		MoveFrequency:    "",
 		NoUpdate:         0,

routers/commentsRouter.go

@@ -7,6 +7,24 @@ import (
 
 func init() {
 
+    beego.GlobalControllerRouter["eta/eta_index_lib/controllers/factor_edb_series:FactorEdbSeriesController"] = append(beego.GlobalControllerRouter["eta/eta_index_lib/controllers/factor_edb_series:FactorEdbSeriesController"],
+        beego.ControllerComments{
+            Method: "ChartRecalculate",
+            Router: `/chart_recalculate`,
+            AllowHTTPMethods: []string{"post"},
+            MethodParams: param.Make(),
+            Filters: nil,
+            Params: nil})
+
+    beego.GlobalControllerRouter["eta/eta_index_lib/controllers/factor_edb_series:FactorEdbSeriesController"] = append(beego.GlobalControllerRouter["eta/eta_index_lib/controllers/factor_edb_series:FactorEdbSeriesController"],
+        beego.ControllerComments{
+            Method: "Recalculate",
+            Router: `/recalculate`,
+            AllowHTTPMethods: []string{"post"},
+            MethodParams: param.Make(),
+            Filters: nil,
+            Params: nil})
+
     beego.GlobalControllerRouter["eta/eta_index_lib/controllers/fix:CustomAnalysisController"] = append(beego.GlobalControllerRouter["eta/eta_index_lib/controllers/fix:CustomAnalysisController"],
         beego.ControllerComments{
             Method: "FixTableV1",
@@ -232,6 +250,15 @@ func init() {
             Filters: nil,
             Params: nil})
 
+    beego.GlobalControllerRouter["eta/eta_index_lib/controllers:CalculateController"] = append(beego.GlobalControllerRouter["eta/eta_index_lib/controllers:CalculateController"],
+        beego.ControllerComments{
+            Method: "StepCalculate",
+            Router: `/base/step_calculate`,
+            AllowHTTPMethods: []string{"post"},
+            MethodParams: param.Make(),
+            Filters: nil,
+            Params: nil})
+
     beego.GlobalControllerRouter["eta/eta_index_lib/controllers:CalculateController"] = append(beego.GlobalControllerRouter["eta/eta_index_lib/controllers:CalculateController"],
         beego.ControllerComments{
             Method: "BatchEdit",
@@ -943,6 +970,15 @@ func init() {
             Filters: nil,
             Params: nil})
 
+    beego.GlobalControllerRouter["eta/eta_index_lib/controllers:MySteelChemicalController"] = append(beego.GlobalControllerRouter["eta/eta_index_lib/controllers:MySteelChemicalController"],
+        beego.ControllerComments{
+            Method: "HandleApiMysteelIndex",
+            Router: `/handle/api/mysteel/index`,
+            AllowHTTPMethods: []string{"post"},
+            MethodParams: param.Make(),
+            Filters: nil,
+            Params: nil})
+
     beego.GlobalControllerRouter["eta/eta_index_lib/controllers:MySteelChemicalController"] = append(beego.GlobalControllerRouter["eta/eta_index_lib/controllers:MySteelChemicalController"],
         beego.ControllerComments{
             Method: "HandleMysteelIndex",
@@ -1411,6 +1447,51 @@ func init() {
             Filters: nil,
             Params: nil})
 
+    beego.GlobalControllerRouter["eta/eta_index_lib/controllers:ThsHfController"] = append(beego.GlobalControllerRouter["eta/eta_index_lib/controllers:ThsHfController"],
+        beego.ControllerComments{
+            Method: "BaseAdd",
+            Router: `/hf/base/add`,
+            AllowHTTPMethods: []string{"post"},
+            MethodParams: param.Make(),
+            Filters: nil,
+            Params: nil})
+
+    beego.GlobalControllerRouter["eta/eta_index_lib/controllers:ThsHfController"] = append(beego.GlobalControllerRouter["eta/eta_index_lib/controllers:ThsHfController"],
+        beego.ControllerComments{
+            Method: "BaseRefresh",
+            Router: `/hf/base/refresh`,
+            AllowHTTPMethods: []string{"post"},
+            MethodParams: param.Make(),
+            Filters: nil,
+            Params: nil})
+
+    beego.GlobalControllerRouter["eta/eta_index_lib/controllers:ThsHfController"] = append(beego.GlobalControllerRouter["eta/eta_index_lib/controllers:ThsHfController"],
+        beego.ControllerComments{
+            Method: "EdbAdd",
+            Router: `/hf/edb/add`,
+            AllowHTTPMethods: []string{"post"},
+            MethodParams: param.Make(),
+            Filters: nil,
+            Params: nil})
+
+    beego.GlobalControllerRouter["eta/eta_index_lib/controllers:ThsHfController"] = append(beego.GlobalControllerRouter["eta/eta_index_lib/controllers:ThsHfController"],
+        beego.ControllerComments{
+            Method: "EdbRefresh",
+            Router: `/hf/edb/refresh`,
+            AllowHTTPMethods: []string{"post"},
+            MethodParams: param.Make(),
+            Filters: nil,
+            Params: nil})
+
+    beego.GlobalControllerRouter["eta/eta_index_lib/controllers:ThsHfController"] = append(beego.GlobalControllerRouter["eta/eta_index_lib/controllers:ThsHfController"],
+        beego.ControllerComments{
+            Method: "GetData",
+            Router: `/hf/edb_data`,
+            AllowHTTPMethods: []string{"post"},
+            MethodParams: param.Make(),
+            Filters: nil,
+            Params: nil})
+
     beego.GlobalControllerRouter["eta/eta_index_lib/controllers:WindController"] = append(beego.GlobalControllerRouter["eta/eta_index_lib/controllers:WindController"],
         beego.ControllerComments{
             Method: "Add",

routers/router.go

@@ -9,6 +9,7 @@ package routers
 
 import (
 	"eta/eta_index_lib/controllers"
+	"eta/eta_index_lib/controllers/factor_edb_series"
 	"eta/eta_index_lib/controllers/fix"
 	"eta/eta_index_lib/controllers/future_good"
 	"eta/eta_index_lib/controllers/open"
@@ -22,6 +23,7 @@ func init() {
 			beego.NSInclude(
 				&controllers.ThsController{},
 				&controllers.ThsDsController{},
+				&controllers.ThsHfController{},
 			),
 		),
 		beego.NSNamespace("/pb",
@@ -100,14 +102,14 @@ func init() {
 				&controllers.GieController{},
 			),
 		),
-		beego.NSNamespace("/coal",
+		beego.NSNamespace("/python",
 			beego.NSInclude(
-				&controllers.CoalController{},
+				&controllers.PythonController{},
 			),
 		),
-		beego.NSNamespace("/python",
+		beego.NSNamespace("/coal",
 			beego.NSInclude(
-				&controllers.PythonController{},
+				&controllers.CoalController{},
 			),
 		),
 		beego.NSNamespace("/google_travel",
@@ -115,6 +117,16 @@ func init() {
 				&controllers.GoogleTravelController{},
 			),
 		),
+		beego.NSNamespace("/predict_calculate",
+			beego.NSInclude(
+				&controllers.PredictCalculateController{},
+			),
+		),
+		beego.NSNamespace("/mysteel_chemical",
+			beego.NSInclude(
+				&controllers.MySteelChemicalController{},
+			),
+		),
 		beego.NSNamespace("/mysteel_chemical",
 			beego.NSInclude(
 				&controllers.MySteelChemicalController{},
@@ -220,24 +232,24 @@ func init() {
 				&controllers.CoalMineDataController{},
 			),
 		),
-		beego.NSNamespace("/fenwei",
+		beego.NSNamespace("/gz",
 			beego.NSInclude(
-				&controllers.FenweiController{},
+				&controllers.GzController{},
 			),
 		),
-		beego.NSNamespace("/gz",
+		beego.NSNamespace("/icpi",
 			beego.NSInclude(
-				&controllers.GzController{},
+				&controllers.IcpiController{},
 			),
 		),
-		beego.NSNamespace("/mtjh",
+		beego.NSNamespace("/fenwei",
 			beego.NSInclude(
-				&controllers.MtjhDataController{},
+				&controllers.FenweiController{},
 			),
 		),
-		beego.NSNamespace("/icpi",
+		beego.NSNamespace("/mtjh",
 			beego.NSInclude(
-				&controllers.IcpiController{},
+				&controllers.MtjhDataController{},
 			),
 		),
 		beego.NSNamespace("/edb_refresh",
@@ -280,6 +292,11 @@ func init() {
 				&controllers.SciHqController{},
 			),
 		),
+		beego.NSNamespace("/factor_edb_series",
+			beego.NSInclude(
+				&factor_edb_series.FactorEdbSeriesController{},
+			),
+		),
 	)
 	beego.AddNamespace(ns)
 }

services/base_from_business.go

@@ -8,6 +8,7 @@ import (
 	"eta/eta_index_lib/services/alarm_msg"
 	"eta/eta_index_lib/utils"
 	"fmt"
+	"github.com/qiniu/qmgo"
 	"go.mongodb.org/mongo-driver/bson"
 	"strings"
 	"time"
@@ -135,10 +136,44 @@ func HandleBusinessIndex(indexReq *models.AddBusinessIndexReq) (resp models.Base
 	}
 
 	// 数据处理
+	// 当前传入的最小日期
+	var reqMinDate time.Time
+	if utils.UseMongo {
+		reqMinDate, err = handleBusinessDataByMongo(item, indexReq.DataList)
+	} else {
+		reqMinDate, err = handleBusinessDataByMysql(item, indexReq.DataList)
+	}
+	if err != nil {
+		return
+	}
+
+	// 同步刷新指标库的指标
+	go refreshEdbBusiness(item.IndexCode, reqMinDate)
+
+	resp = models.BaseFromBusinessIndexResp{
+		IndexCode:  item.IndexCode,
+		IndexName:  item.IndexName,
+		Unit:       item.Unit,
+		Frequency:  item.Frequency,
+		SourceName: item.SourceName,
+	}
+
+	return
+}
+
+// handleBusinessDataByMongo
+// @Description: 处理外部指标数据（mongo）
+// @author: Roc
+// @datetime 2024-07-01 15:30:41
+// @param item *models.BaseFromBusinessIndex
+// @param reqDataList []*models.AddBusinessDataReq
+// @return reqMinDate time.Time 当前传入的最小日期
+// @return err error
+func handleBusinessDataByMongo(item *models.BaseFromBusinessIndex, reqDataList []models.AddBusinessDataReq) (reqMinDate time.Time, err error) {
 	mogDataObj := new(mgo.BaseFromBusinessData)
 
 	//获取已存在的所有数据
-	exitDataList, err := mogDataObj.GetAllDataList(bson.M{"index_code": indexReq.IndexCode}, []string{"data_time"})
+	exitDataList, err := mogDataObj.GetAllDataList(bson.M{"index_code": item.IndexCode}, []string{"data_time"})
 	if err != nil {
 		fmt.Println("GetIndexDataList Err:" + err.Error())
 		return
@@ -150,14 +185,12 @@ func HandleBusinessIndex(indexReq *models.AddBusinessIndexReq) (resp models.Base
 		exitDataMap[v.DataTime.Format(utils.FormatDate)] = v
 	}
 
-	// 当前传入的最小日期
-	var reqMinDate time.Time
 	// 待添加的数据集
 	addDataList := make([]interface{}, 0)
 	updateDataList := make([]mgo.BaseFromBusinessData, 0)
 	//var hasUpdate bool
 	// 遍历excel数据，然后跟现有的数据做校验，不存在则入库
-	for _, data := range indexReq.DataList {
+	for _, data := range reqDataList {
 		dateTime, tmpErr := utils.DealExcelDate(data.Date)
 		if tmpErr != nil {
 			fmt.Println("time.ParseInLocation Err:" + tmpErr.Error())
@@ -229,27 +262,121 @@ func HandleBusinessIndex(indexReq *models.AddBusinessIndexReq) (resp models.Base
 	//fmt.Println("result", result)
 
 	//修改最大最小日期
-	indexMaxAndMinInfo, err := item.GetEdbInfoMaxAndMinInfo(indexReq.IndexCode)
+	indexMaxAndMinInfo, err := item.GetEdbInfoMaxAndMinInfo(item.IndexCode)
 	if err != nil {
 		return
 	}
 	if err == nil && indexMaxAndMinInfo != nil {
-		e := item.ModifyIndexMaxAndMinInfo(indexReq.IndexCode, indexMaxAndMinInfo, isIndexUpdateOrAdd)
+		e := item.ModifyIndexMaxAndMinInfo(item.IndexCode, indexMaxAndMinInfo, isIndexUpdateOrAdd)
 		if e != nil {
 			fmt.Println("ModifyIndexMaxAndMinInfo Err:" + e.Error())
 		}
 	}
 
-	// 同步刷新指标库的指标
-	go refreshEdbBusiness(item.IndexCode, reqMinDate)
+	return
+}
 
-	resp = models.BaseFromBusinessIndexResp{
-		IndexCode:  item.IndexCode,
-		IndexName:  item.IndexName,
-		Unit:       item.Unit,
-		Frequency:  item.Frequency,
-		SourceName: item.SourceName,
+// handleBusinessDataByMysql
+// @Description: 处理外部指标数据（mysql）
+// @author: Roc
+// @datetime 2024-07-01 15:59:43
+// @param item *models.BaseFromBusinessIndex
+// @param reqDataList []models.AddBusinessDataReq
+// @return reqMinDate time.Time
+// @return err error
+func handleBusinessDataByMysql(item *models.BaseFromBusinessIndex, reqDataList []models.AddBusinessDataReq) (reqMinDate time.Time, err error) {
+	businessDataObj := new(models.BaseFromBusinessData)
+
+	var condition []string
+	var pars []interface{}
+	condition = append(condition, "index_code = ?")
+	pars = append(pars, item.IndexCode)
+	//获取已存在的所有数据
+	exitDataList, err := businessDataObj.GetAllDataList(condition, pars, "data_time ASC")
+	if err != nil {
+		fmt.Println("GetIndexDataList Err:" + err.Error())
+		return
+	}
+
+	// 已经存在的数据集
+	exitDataMap := make(map[string]*models.BaseFromBusinessData)
+	for _, v := range exitDataList {
+		exitDataMap[v.DataTime.Format(utils.FormatDate)] = v
+	}
+
+	// 待添加的数据集
+	addDataList := make([]*models.BaseFromBusinessData, 0)
+	updateDataList := make([]*models.BaseFromBusinessData, 0)
+	//var hasUpdate bool
+	// 遍历excel数据，然后跟现有的数据做校验，不存在则入库
+	for _, data := range reqDataList {
+		dateTime, tmpErr := utils.DealExcelDate(data.Date)
+		if tmpErr != nil {
+			fmt.Println("time.ParseInLocation Err:" + tmpErr.Error())
+			err = tmpErr
+			return
+		}
+
+		// 调整最小日期
+		if reqMinDate.IsZero() || reqMinDate.After(dateTime) {
+			reqMinDate = dateTime
+		}
+
+		date := dateTime.Format(utils.FormatDate)
+
+		findData, ok := exitDataMap[date]
+		if !ok {
+			addDataList = append(addDataList, &models.BaseFromBusinessData{
+				BaseFromBusinessIndexId: int(item.BaseFromBusinessIndexId),
+				IndexCode:               item.IndexCode,
+				DataTime:                dateTime,
+				Value:                   data.Value,
+				CreateTime:              time.Now(),
+				ModifyTime:              time.Now(),
+				//DataTimestamp:           0,
+			})
+			continue
+		}
+
+		// 值不匹配，修改数据
+		if findData.Value != data.Value {
+			findData.Value = data.Value
+			findData.ModifyTime = time.Now()
+			updateDataList = append(updateDataList, findData)
+		}
+	}
+
+	// 指标数据是否新增或修改
+	var isIndexUpdateOrAdd bool
+
+	// 入库
+	{
+		if len(addDataList) > 0 {
+			isIndexUpdateOrAdd = true
+		}
+
+		if len(updateDataList) > 0 {
+			isIndexUpdateOrAdd = true
+		}
+		err = businessDataObj.HandleData(addDataList, updateDataList)
+		if err != nil {
+			fmt.Println("UpdateDataByColl:Err:" + err.Error())
+			return
+		}
 	}
+
+	//修改最大最小日期
+	indexMaxAndMinInfo, err := item.GetEdbInfoMaxAndMinInfo(item.IndexCode)
+	if err != nil {
+		return
+	}
+	if err == nil && indexMaxAndMinInfo != nil {
+		e := item.ModifyIndexMaxAndMinInfo(item.IndexCode, indexMaxAndMinInfo, isIndexUpdateOrAdd)
+		if e != nil {
+			fmt.Println("ModifyIndexMaxAndMinInfo Err:" + e.Error())
+		}
+	}
+
 	return
 }
 
@@ -326,9 +453,22 @@ func DelBusinessIndex(indexCodeList []string) (joinEdbCodeList, needDelEdbCodeLi
 		return
 	}
 
-	// 删除指标明细数据
-	mogDataObj := new(mgo.BaseFromBusinessData)
-	err = mogDataObj.RemoveMany(bson.M{"index_code": bson.M{"$in": needDelEdbCodeList}})
+	if utils.UseMongo {
+		// 删除指标明细数据
+		mogDataObj := new(mgo.BaseFromBusinessData)
+		err = mogDataObj.RemoveMany(bson.M{"index_code": bson.M{"$in": needDelEdbCodeList}})
+	} else {
+		var condition []string
+		var pars []interface{}
+		delNum := len(needDelEdbCodeList)
+		if delNum > 0 {
+			condition = append(condition, "index_code in ("+utils.GetOrmInReplace(delNum)+")")
+			pars = append(pars, needDelEdbCodeList)
+
+			businessDataObj := models.BaseFromBusinessData{}
+			err = businessDataObj.DelDataByCond(condition, pars)
+		}
+	}
 	if err != nil {
 		fmt.Println("删除自有指标明细数据 Err:" + err.Error())
 		return
@@ -348,7 +488,7 @@ func DelBusinessIndex(indexCodeList []string) (joinEdbCodeList, needDelEdbCodeLi
 func DelBusinessIndexData(indexCode string, startDate, endDate string) (err error, errMsg string) {
 	defer func() {
 		if err != nil {
-			fmt.Println("DelBusinessIndex Err:" + err.Error())
+			fmt.Println("DelBusinessIndexData Err:" + err.Error())
 		}
 	}()
 	errMsg = "删除失败"
@@ -377,11 +517,6 @@ func DelBusinessIndexData(indexCode string, startDate, endDate string) (err erro
 		return
 	}
 
-	// 构建查询条件
-	queryConditions := bson.M{
-		"index_code": item.IndexCode,
-	}
-
 	// 当前传入的最小日期
 	var reqMinDate time.Time
 
@@ -410,6 +545,51 @@ func DelBusinessIndexData(indexCode string, startDate, endDate string) (err erro
 		}
 	}
 
+	// 删除具体的数据
+	var indexMaxAndMinInfo *models.EdbInfoMaxAndMinInfo
+	if utils.UseMongo {
+		indexMaxAndMinInfo, err = delBusinessIndexDataByMongo(item, startDateTime, endDateTime)
+	} else {
+		indexMaxAndMinInfo, err = delBusinessIndexDataByMysql(item, startDateTime, endDateTime)
+	}
+	if err != nil {
+		return
+	}
+
+	// 修改指标的最早最晚日期
+	if indexMaxAndMinInfo != nil {
+		e := item.ModifyIndexMaxAndMinInfo(item.IndexCode, indexMaxAndMinInfo, true)
+		if e != nil {
+			fmt.Println("ModifyIndexMaxAndMinInfo Err:" + e.Error())
+		}
+	}
+
+	// 同步刷新指标库的指标
+	go refreshEdbBusiness(item.IndexCode, reqMinDate)
+
+	return
+}
+
+// delBusinessIndexDataByMongo
+// @Description: 删除指标数据（从mongo删除）
+// @author: Roc
+// @datetime 2024-07-01 18:00:07
+// @param item *models.BaseFromBusinessIndex
+// @param startDateTime time.Time
+// @param endDateTime time.Time
+// @return err error
+func delBusinessIndexDataByMongo(item *models.BaseFromBusinessIndex, startDateTime, endDateTime time.Time) (indexMaxAndMinInfo *models.EdbInfoMaxAndMinInfo, err error) {
+	defer func() {
+		if err != nil {
+			utils.FileLog.Error("delBusinessIndexDataByMongo 删除自有指标明细数据 Err:" + err.Error())
+		}
+	}()
+
+	// 构建查询条件
+	queryConditions := bson.M{
+		"index_code": item.IndexCode,
+	}
+
 	dateCondition, err := mgo.BuildDateTimeCondition(startDateTime, endDateTime)
 	if err != nil {
 		return
@@ -422,12 +602,66 @@ func DelBusinessIndexData(indexCode string, startDate, endDate string) (err erro
 	mogDataObj := new(mgo.BaseFromBusinessData)
 	err = mogDataObj.RemoveMany(queryConditions)
 	if err != nil {
-		fmt.Println("删除自有指标明细数据 Err:" + err.Error())
 		return
 	}
 
-	// 同步刷新指标库的指标
-	go refreshEdbBusiness(item.IndexCode, reqMinDate)
+	//修改最大最小日期
+	indexMaxAndMinInfo, err = item.GetEdbInfoMaxAndMinInfo(item.IndexCode)
+	// 如果有错误，且错误信息是取不到文档，那么就不修改了
+	if err != nil && !errors.Is(err, qmgo.ErrNoSuchDocuments) {
+		return
+	}
+	// 清空的目的是为了避免异常返回
+	err = nil
+
+	return
+}
+
+// delBusinessIndexDataByMysql
+// @Description: 删除指标数据（从mysql删除）
+// @author: Roc
+// @datetime 2024-07-02 09:53:13
+// @param item *models.BaseFromBusinessIndex
+// @param startDateTime time.Time
+// @param endDateTime time.Time
+// @return err error
+func delBusinessIndexDataByMysql(item *models.BaseFromBusinessIndex, startDateTime, endDateTime time.Time) (indexMaxAndMinInfo *models.EdbInfoMaxAndMinInfo, err error) {
+	defer func() {
+		if err != nil {
+			utils.FileLog.Error("delBusinessIndexDataByMysql 删除自有指标明细数据 Err:" + err.Error())
+		}
+	}()
+
+	// 构建查询条件
+	var condition []string
+	var pars []interface{}
+	condition = append(condition, "index_code = ? ")
+	pars = append(pars, item.IndexCode)
+
+	if !startDateTime.IsZero() {
+		condition = append(condition, " data_time >= ? ")
+		pars = append(pars, startDateTime.Format(utils.FormatDate))
+	}
+	if !endDateTime.IsZero() {
+		condition = append(condition, " data_time <= ? ")
+		pars = append(pars, endDateTime.Format(utils.FormatDate))
+	}
+
+	// 删除数据源中的指标明细数据
+	businessDataObj := new(models.BaseFromBusinessData)
+	err = businessDataObj.DelDataByCond(condition, pars)
+	if err != nil {
+		return
+	}
+
+	//修改最大最小日期
+	indexMaxAndMinInfo, err = item.GetEdbInfoMaxAndMinInfo(item.IndexCode)
+	// 如果有错误，且错误信息是取不到文档，那么就不修改了
+	if err != nil && err.Error() != utils.ErrNoRow() {
+		return
+	}
+	// 清空的目的是为了避免异常返回
+	err = nil
 
 	return
 }

services/base_from_calculate.go

@@ -5,6 +5,7 @@ import (
 	"eta/eta_index_lib/models"
 	"eta/eta_index_lib/utils"
 	"fmt"
+	"sort"
 	"strconv"
 	"strings"
 	"time"
@@ -159,6 +160,11 @@ func EdbCalculateBatchSave(req models.EdbInfoCalculateBatchSaveReq, lang string)
 		}
 	}
 
+	// 记录旧的指标信息
+	oldEdbName := fromEdbInfo.EdbName
+	oldFrequency := fromEdbInfo.Frequency
+	oldUnit := fromEdbInfo.Unit
+
 	//生成指标编码
 	edbCode, err := utils.GenerateEdbCode(1, "")
 	if err != nil {
@@ -453,8 +459,27 @@ func EdbCalculateBatchSave(req models.EdbInfoCalculateBatchSaveReq, lang string)
 		return
 	}
 
+	// 记录基础信息操作变更日志
+	oldEdbInfo := new(models.EdbInfo)
+	oldEdbInfo.EdbInfoId = fromEdbInfoId
+	oldEdbInfo.EdbName = oldEdbName
+	oldEdbInfo.Frequency = oldFrequency
+	oldEdbInfo.Unit = oldUnit
+	newEdbInfoEditRecord := new(models.EdbInfoEditRecord)
+	newEdbInfoEditRecord.EdbName = req.EdbName
+	newEdbInfoEditRecord.Frequency = req.Frequency
+	newEdbInfoEditRecord.Unit = req.Unit
+	newEdbInfoEditRecord.OperateUserId = req.AdminId
+	newEdbInfoEditRecord.OperateUserRealName = req.AdminName
+	err = AddEditEdbInfoRecord(oldEdbInfo, newEdbInfoEditRecord)
+	if err != nil {
+		errMsg = "记录基础信息操作变更日志失败，Err:" + err.Error()
+		err = fmt.Errorf("操作记录保存失败")
+		return
+	}
 	// 更新ES
 	go logic.UpdateEs(edbInfo.EdbInfoId)
+	go DisableEdbInfoNoUpdate(edbInfo)
 	return
 }
 
@@ -533,6 +558,12 @@ func EdbCalculateBatchEdit(req models.EdbInfoCalculateBatchEditReq) (edbInfo *mo
 
 		return
 	}
+
+	// 记录旧的指标基本信息
+	oldEdbName := edbInfo.EdbName
+	oldFrequency := edbInfo.Frequency
+	oldUnit := edbInfo.Unit
+
 	// 基础指标id
 	fromEdbInfoId := req.FromEdbInfoId
 
@@ -854,8 +885,30 @@ func EdbCalculateBatchEdit(req models.EdbInfoCalculateBatchEditReq) (edbInfo *mo
 		return
 	}
 
+	// 记录基础信息操作变更日志
+	oldEdbInfo := new(models.EdbInfo)
+	oldEdbInfo.EdbInfoId = edbInfoId
+	oldEdbInfo.EdbName = oldEdbName
+	oldEdbInfo.Frequency = oldFrequency
+	oldEdbInfo.Unit = oldUnit
+	newEdbInfoEditRecord := new(models.EdbInfoEditRecord)
+	newEdbInfoEditRecord.EdbName = req.EdbName
+	newEdbInfoEditRecord.Frequency = req.Frequency
+	newEdbInfoEditRecord.Unit = req.Unit
+	newEdbInfoEditRecord.OperateUserId = req.AdminId
+	newEdbInfoEditRecord.OperateUserRealName = req.AdminName
+	err = AddEditEdbInfoRecord(oldEdbInfo, newEdbInfoEditRecord)
+	if err != nil {
+		errMsg = "记录基础信息操作变更日志失败，Err:" + err.Error()
+		err = fmt.Errorf("操作记录保存失败")
+		return
+	}
+
 	// 更新ES
 	go logic.UpdateEs(edbInfo.EdbInfoId)
+
+	// 重置计算指标中的引用关系
+	go ResetEdbRelation(edbInfoId)
 	return
 }
 
@@ -1006,6 +1059,82 @@ func EdbCalculateAdd(req models.EdbInfoCalculateSaveReq, lang string) (edbInfo *
 		err = fmt.Errorf("生成计算指标失败")
 		return
 	}
+	go DisableEdbInfoNoUpdate(edbInfo)
+	return
+}
+
+// StepCalculate 多公式计算
+func StepCalculate(dataList []*models.EdbInfoSearchData, calculates []models.CalculatesReq) (resultData map[string]float64, dates []string, errMsg string, err error) {
+	sort.Slice(calculates, func(i, j int) bool {
+		return calculates[i].Sort < calculates[j].Sort
+	})
+	originData, e := models.EdbInfoSearchDataToData(dataList)
+	if e != nil {
+		err = fmt.Errorf("基础数据转换异常, err: %v", e)
+		return
+	}
 
+	calculateData := originData
+	dateDataMap := make(map[time.Time]float64)
+	for _, v := range calculates {
+		baseCalculate := models.BaseCalculate{
+			DataList:      calculateData,
+			Frequency:     v.Frequency,
+			Formula:       v.Formula,
+			Calendar:      v.Calendar,
+			MoveType:      v.MoveType,
+			MoveFrequency: v.MoveFrequency,
+			FromFrequency: v.FromFrequency,
+			Source:        v.Source,
+		}
+
+		// 计算方式
+		switch baseCalculate.Source {
+		case utils.EdbBaseCalculateLjzzy:
+			dateDataMap, e, errMsg = baseCalculate.Ljzzy()
+		case utils.EdbBaseCalculateLjzzj:
+			dateDataMap, e, errMsg = baseCalculate.Ljzzj()
+		case utils.EdbBaseCalculateTbz:
+			dateDataMap, e, errMsg = baseCalculate.Tbz()
+		case utils.EdbBaseCalculateTcz:
+			dateDataMap, e, errMsg = baseCalculate.Tcz()
+		case utils.EdbBaseCalculateNszydpjjs:
+			dateDataMap, e, errMsg = baseCalculate.Nszydpjjs()
+		case utils.EdbBaseCalculateHbz:
+			dateDataMap, e, errMsg = baseCalculate.Hbz()
+		case utils.EdbBaseCalculateHcz:
+			dateDataMap, e, errMsg = baseCalculate.Hcz()
+		case utils.EdbBaseCalculateUpFrequency:
+			dateDataMap, e, errMsg = baseCalculate.UpFrequency()
+		case utils.EdbBaseCalculateDownFrequency:
+			dateDataMap, e, errMsg = baseCalculate.DownFrequency()
+		case utils.EdbBaseCalculateTimeShift:
+			dateDataMap, e, errMsg = baseCalculate.TimeShift()
+		case utils.EdbBaseCalculateCjjx:
+			dateDataMap, e, errMsg = baseCalculate.Cjjx()
+		case utils.EdbBaseCalculateAnnualized:
+			dateDataMap, e, errMsg = baseCalculate.Annualized()
+		case utils.EdbBaseCalculateLjz:
+			dateDataMap, e, errMsg = baseCalculate.Ljz()
+		case utils.EdbBaseCalculateLjzNczj:
+			dateDataMap, e, errMsg = baseCalculate.LjzNczj()
+		case utils.EdbBaseCalculateExponentialSmoothing:
+			dateDataMap, e, errMsg = baseCalculate.ExponentialSmoothing()
+		case utils.EdbBaseCalculateRjz:
+			dateDataMap, e, errMsg = baseCalculate.Rjz()
+		default:
+			errMsg = "计算方式无效"
+			e = fmt.Errorf("%s:%d", errMsg, baseCalculate.Source)
+		}
+		if e != nil {
+			err = fmt.Errorf("计算失败, err: %v", e)
+			return
+		}
+
+		calculateData = models.TransDateData2EdbData(dateDataMap)
+	}
+	resultData, dates = models.GetDateDataAndDateList(dateDataMap)
+	//resp.DataMap = resultData
+	//resp.DateList = dates
 	return
 }

services/base_from_mysteel_chemical.go

@@ -1,11 +1,16 @@
 package services
 
 import (
+	"encoding/json"
+	"errors"
 	"eta/eta_index_lib/logic"
 	"eta/eta_index_lib/models"
 	"eta/eta_index_lib/services/alarm_msg"
 	"eta/eta_index_lib/utils"
 	"fmt"
+	"io"
+	"net/http"
+	"strconv"
 	"strings"
 	"time"
 )
@@ -27,6 +32,218 @@ func HandleMysteelIndex(req *models.HandleMysteelIndexResp) (err error) {
 	return
 }
 
+func HandleApiMysteelIndex(req *models.HandleMysteelIndexResp) (errMsg string, err error) {
+	addIndexCodeList := make([]string, 0)
+	for _, v := range req.List {
+		if v.IndexCode == "" {
+			continue
+		}
+		addIndexCodeList = append(addIndexCodeList, v.IndexCode)
+	}
+	errMsg, err = HandleApiIndex(addIndexCodeList)
+	if err != nil {
+		return
+	}
+
+	_ = SetMysteelChemicalEdbInfoUpdateStat(false)
+	_ = SetEdbSourceStat(false)
+
+	return
+}
+
+func HandleApiIndex(indexCodes []string) (errMsg string, err error) {
+	if len(indexCodes) == 0 {
+		return
+	}
+	resp, err := GetEdbDataFromMySteelChemical(indexCodes, utils.GetEdbRefreshStartDate(""), utils.BASE_END_DATE, "desc")
+	if err != nil {
+		return
+	}
+	if !resp.Success {
+		errMsg = "获取数据失败"
+		err = errors.New(resp.Message)
+		return
+	}
+	indexInfoMap, err := GetMySteelChemicalIndexNameMap()
+	if err != nil {
+		errMsg = "获取指标数据失败"
+		return
+	}
+
+	indexObj := &models.BaseFromMysteelChemicalIndex{}
+	existIndexs, err := indexObj.GetBatchIndexItem(indexCodes)
+	if err != nil {
+		errMsg = "获取指标数据失败"
+		return
+	}
+
+	//获取已存在的所有数据
+	existDataMap := make(map[string]*models.BaseFromMysteelChemicalData)
+	existIndexMap := make(map[string]*models.BaseFromMysteelChemicalIndex)
+	updateDataObj := new(models.BaseFromMysteelChemicalData)
+	for _, v := range existIndexs {
+		// 更新指标的名称，单位和频度等信息
+		if info, ok := indexInfoMap[v.IndexCode]; ok {
+			v.IndexName = info.IndexName
+			v.Unit = info.UnitName
+			v.Frequency = info.FrequencyName
+			v.ModifyTime = time.Now()
+			err = v.Update([]string{"index_name", "unit", "frequency", "modify_time"})
+			if err != nil {
+				errMsg = "更新指标失败"
+				return
+			}
+		}
+		if err != nil {
+			errMsg = "添加指标失败"
+			return
+		}
+		existIndexMap[v.IndexCode] = v
+		exitDataList, er := updateDataObj.GetIndexDataList(v.IndexCode)
+		if er != nil {
+			errMsg = "获取指标数据失败"
+			err = er
+			return
+		}
+		fmt.Println("exitDataListLen:", len(exitDataList))
+		for _, v := range exitDataList {
+			dateStr := v.DataTime.Format(utils.FormatDate)
+			existDataMap[dateStr] = v
+		}
+	}
+	mysteelChemicalDatas, err := tranformData(resp)
+	if err != nil {
+		errMsg = "转换数据失败"
+		return
+	}
+
+	var indexErr error
+	var lErr error
+	defer func() {
+		if indexErr != nil {
+			tips := fmt.Sprintf("钢联数据刷新-ETA指标刷新异常, 指标编码: %s, err: %s", indexObj.IndexCode, indexErr.Error())
+			alarm_msg.SendAlarmMsg(tips, 3)
+		}
+
+		if lErr != nil {
+			tips := fmt.Sprintf("钢联数据刷新-ETA指标刷新统计异常, 指标编码: %s, err: %s", indexObj.IndexCode, lErr.Error())
+			alarm_msg.SendAlarmMsg(tips, 3)
+		}
+	}()
+	var hasUpdate bool
+	dataObj := new(models.BaseFromMysteelChemicalData)
+	for _, items := range mysteelChemicalDatas {
+		addItems := make([]*models.BaseFromMysteelChemicalData, 0)
+		for _, v := range items {
+			dateStr := v.DataTime.Format(utils.FormatDate)
+			if findData, ok := existDataMap[dateStr]; !ok {
+				index, ok := existIndexMap[v.IndexCode]
+				if !ok {
+					continue
+				}
+				v.BaseFromMysteelChemicalIndexId = index.BaseFromMysteelChemicalIndexId
+				addItems = append(addItems, v)
+			} else {
+				if findData != nil && findData.Value != v.Value {
+					dataObj.BaseFromMysteelChemicalDataId = findData.BaseFromMysteelChemicalDataId
+					dataObj.Value = v.Value
+					dataObj.ModifyTime = time.Now()
+
+					err = dataObj.Update([]string{"value", "modify_time"})
+					if err != nil {
+						errMsg = "更新数据失败"
+						return
+					}
+					hasUpdate = true
+				}
+			}
+		}
+		err = dataObj.AddV2(addItems)
+		if err != nil {
+			return
+		}
+		//修改最大最小日期
+		if len(items) <= 0 {
+			continue
+		}
+		mysteelIndexMaxItem, er := dataObj.GetMysteelIndexInfoMaxAndMinInfo(items[0].IndexCode)
+		if er == nil && mysteelIndexMaxItem != nil {
+			e := dataObj.ModifyMysteelIndexMaxAndMinInfo(items[0].IndexCode, mysteelIndexMaxItem)
+			if e != nil {
+				fmt.Println("ModifyMysteelIndexMaxAndMinInfo Err:" + e.Error())
+				utils.FileLog.Info("修改钢联化工的最大最小日期失败，Err:" + e.Error())
+			}
+		}
+
+		edbInfo, e := models.GetEdbInfoByEdbCode(utils.DATA_SOURCE_MYSTEEL_CHEMICAL, indexObj.IndexCode)
+		if e != nil && e.Error() != utils.ErrNoRow() {
+			indexErr = e
+			return
+		}
+
+		if edbInfo != nil {
+			dataUpdateResult := 2
+			dataUpdateFailedReason := "服务异常"
+			_, logErrMsg, logErr := logic.RefreshBaseEdbInfo(edbInfo, ``)
+			if logErr != nil {
+				lErr = AddEdbInfoUpdateLog(edbInfo.EdbInfoId, 2, logErrMsg+logErr.Error(), dataUpdateResult, dataUpdateFailedReason, 0, 0)
+				return
+			}
+
+			if hasUpdate {
+				dataUpdateResult = 1
+				dataUpdateFailedReason = ""
+			} else {
+				dataUpdateFailedReason = "未刷新到数据"
+			}
+
+			// 添加刷新成功日志
+			lErr = AddEdbInfoUpdateLog(edbInfo.EdbInfoId, 1, "", dataUpdateResult, dataUpdateFailedReason, 0, 0)
+			if lErr != nil {
+				return
+			}
+		}
+	}
+	return
+}
+
+func tranformData(dataResp *models.MySteelChemicalApiResp) (items [][]*models.BaseFromMysteelChemicalData, err error) {
+	for _, v := range dataResp.Data {
+		tmpNewDataMap := make(map[string]int64)
+		tmpDateDataMap := make(map[string]*models.BaseFromMysteelChemicalData)
+		tmpDataItems := make([]*models.BaseFromMysteelChemicalData, 0)
+		for _, vv := range v.DataList {
+			tmpData := new(models.BaseFromMysteelChemicalData)
+			tmpData.IndexCode = vv.IndexCode
+			// 如果存在多条数据，则取发布时间最新的数据
+			pub, ok := tmpNewDataMap[vv.DataDate]
+			if !ok {
+				tmpNewDataMap[vv.DataDate] = vv.PublishTime
+				tmpData.Value = strconv.FormatFloat(vv.DataValue, 'f', -1, 64)
+			} else {
+				if pub < vv.PublishTime {
+					tmpNewDataMap[vv.DataDate] = vv.PublishTime
+					tmpData = tmpDateDataMap[vv.DataDate]
+					tmpData.Value = strconv.FormatFloat(vv.DataValue, 'f', -1, 64)
+				}
+				continue
+			}
+			dataDate, er := time.Parse(utils.FormatDate, vv.DataDate)
+			if er != nil {
+				err = er
+				return
+			}
+			tmpData.DataTime = dataDate
+			tmpData.CreateTime = time.Now()
+			tmpData.ModifyTime = time.Now()
+			tmpDataItems = append(tmpDataItems, tmpData)
+			tmpDateDataMap[vv.DataDate] = tmpData
+		}
+		items = append(items, tmpDataItems)
+	}
+	return
+}
+
 func handleIndex(indexItem *models.HandleMysteelIndex) (err error) {
 	defer func() {
 		if err != nil {
@@ -144,6 +361,20 @@ func handleIndex(indexItem *models.HandleMysteelIndex) (err error) {
 			fmt.Println("Index Update Err:" + e.Error())
 			return
 		}
+		if item.IndexName != indexItem.IndexName {
+			var changeRecord models.BaseFromMysteelChemicalRecord
+			changeRecord.BaseFromMysteelChemicalIndexId = item.BaseFromMysteelChemicalIndexId
+			changeRecord.OldIndexName = item.IndexName
+			changeRecord.NewIndexName = indexItem.IndexName
+			ctime := time.Now()
+			changeRecord.CreateTime = ctime
+			changeRecord.Timestamp = ctime.Unix()
+			e = changeRecord.AddBaseFromMysteelChemicalRecord()
+			if e != nil {
+				fmt.Println("mysteel chemical changeRecord Add Err:" + e.Error())
+				return
+			}
+		}
 
 		dataObj := new(models.BaseFromMysteelChemicalData)
 
@@ -158,6 +389,7 @@ func handleIndex(indexItem *models.HandleMysteelIndex) (err error) {
 			dateStr := v.DataTime.Format(utils.FormatDate)
 			exitDataMap[dateStr] = v
 		}
+
 	}
 
 	dataObj := new(models.BaseFromMysteelChemicalData)
@@ -271,3 +503,218 @@ func handleIndex(indexItem *models.HandleMysteelIndex) (err error) {
 
 	return
 }
+
+type MySteelChemicalApiDataBody struct {
+	IndexCodes []string `json:"indexCodes"`
+	StartTime  string   `json:"startTime"`
+	EndTime    string   `json:"endTime"`
+	Order      string   `json:"order"`
+}
+
+type MySteelChemicalApiInfoBody struct {
+	PageNum     int  `json:"pageNum"`
+	PageSize    int  `json:"pageSize"`
+	IncludeInfo bool `json:"includeInfo"`
+}
+
+// GetEdbDataFromMySteelChemical 批量获得钢联化工的指标数据
+func GetEdbDataFromMySteelChemical(indexCodes []string, startTime, endTime, order string) (item *models.MySteelChemicalApiResp, err error) {
+	if utils.MysteelChemicalApiToken == "" {
+		err = errors.New("钢联接口token未配置")
+		return
+	}
+	m := new(MySteelChemicalApiDataBody)
+	m.IndexCodes = indexCodes
+	m.StartTime = startTime
+	m.EndTime = endTime
+	m.Order = order
+	postData, er := json.Marshal(m)
+	if er != nil {
+		err = er
+		return
+	}
+	postUrl := `https://mds.mysteel.com/dynamic/order/api/fcAbRA`
+	body, err := MySteelChemicalPost(postUrl, "data", postData)
+	if err != nil {
+		return
+	}
+	err = json.Unmarshal(body, &item)
+	if err != nil {
+		return
+	}
+	return
+}
+
+// GetMySteelChemicalIndexNameMap 获取钢联化工的所有指标的信息
+func GetMySteelChemicalIndexNameMap() (indexNameMap map[string]*models.MySteelChemicalApiInfoItem, err error) {
+	if utils.MysteelChemicalApiToken == "" {
+		err = errors.New("钢联接口token未配置")
+		return
+	}
+	item, err := getPageIndexInfoMap(1, 200)
+	if err != nil {
+		return
+	}
+	indexNameMap = make(map[string]*models.MySteelChemicalApiInfoItem)
+	for _, v := range item.Data.List {
+		indexNameMap[v.IndexCode] = v
+	}
+	// 如果总条数大于200，则继续获取
+	if item.Data.Total > 200 || item.Data.Pages > 1 {
+		for i := 2; i <= item.Data.Pages; i++ {
+			item, err = getPageIndexInfoMap(i, 200)
+			if err != nil {
+				return
+			}
+			for _, v := range item.Data.List {
+				indexNameMap[v.IndexCode] = v
+			}
+		}
+		return
+	}
+	return
+}
+
+func getPageIndexInfoMap(pageNum, pageSize int) (item *models.MySteelChemicalApiInfoResp, err error) {
+	m := new(MySteelChemicalApiInfoBody)
+	m.PageNum = pageNum
+	m.PageSize = pageSize
+	m.IncludeInfo = true
+	postData, er := json.Marshal(m)
+	if er != nil {
+		err = er
+		return
+	}
+	postUrl := `https://mds.mysteel.com/dynamic/order/api/fcAbRA`
+	body, er := MySteelChemicalPost(postUrl, "info", postData)
+	if er != nil {
+		err = er
+		return
+	}
+	err = json.Unmarshal(body, &item)
+	if err != nil {
+		return
+	}
+	if !item.Success {
+		err = errors.New(item.Message)
+		utils.FileLog.Info("code:" + item.Code + " message:" + item.Message)
+		return
+	}
+	return
+}
+
+func MySteelChemicalPost(postUrl, hType string, postData []byte) (body []byte, err error) {
+	req, er := http.NewRequest(`POST`, postUrl, strings.NewReader(string(postData)))
+	if er != nil {
+		err = er
+		return
+	}
+	req.Header.Set(`Content-Type`, `application/json`)
+	req.Header.Set(`accessTokenSign`, utils.MysteelChemicalApiToken)
+	req.Header.Set(`infoOrData`, hType)
+
+	client := &http.Client{}
+	resp, er := client.Do(req)
+	if er != nil {
+		err = er
+		return
+	}
+	defer resp.Body.Close()
+	body, err = io.ReadAll(resp.Body)
+	if err != nil {
+		return
+	}
+	return
+}
+
+func RefreshDataFromMysteelChemical(edbCode, startDate, endDate string) (err error) {
+	indexObj := &models.BaseFromMysteelChemicalIndex{}
+	tmpIndex, err := indexObj.GetIndexItem(edbCode)
+	if err != nil {
+		return
+	}
+	alarm_msg.SendAlarmMsg(fmt.Sprintf("开始刷新钢联化工接口数据, %s", edbCode), 1)
+
+	terminal, err := GetTerminal(utils.DATA_SOURCE_MYSTEEL_CHEMICAL, tmpIndex.TerminalCode)
+	if err != nil {
+		err = fmt.Errorf("获取钢联化工接口配置出错 Err: %s", err)
+		return
+	}
+
+	if tmpIndex.TerminalCode == "" {
+		// 设置指标与终端关系的缓存
+		terminalCodeCacheKey := utils.CACHE_EDB_TERMINAL_CODE_URL + edbCode
+		_ = utils.Rc.Put(terminalCodeCacheKey, terminal.TerminalCode, utils.GetTodayLastSecond())
+	}
+
+	// 如果配置了api的token, 那么就走api接口
+	if utils.MysteelChemicalApiToken != "" {
+		resp, er := GetEdbDataFromMySteelChemical([]string{edbCode}, startDate, endDate, "desc")
+		if er != nil {
+			err = er
+			return
+		}
+		if !resp.Success {
+			err = errors.New(resp.Message)
+			return
+		}
+		alarm_msg.SendAlarmMsg("获取钢联化工接口数据", 1)
+
+		dataObj := new(models.BaseFromMysteelChemicalData)
+		exitDataList, er := dataObj.GetIndexDataList(edbCode)
+		if er != nil {
+			err = er
+			return
+		}
+		existDataMap := make(map[string]*models.BaseFromMysteelChemicalData)
+		for _, v := range exitDataList {
+			dateStr := v.DataTime.Format(utils.FormatDate)
+			existDataMap[dateStr] = v
+		}
+		mysteelChemicalDatas, er := tranformData(resp)
+		if er != nil {
+			err = er
+			return
+		}
+
+		addItems := make([]*models.BaseFromMysteelChemicalData, 0)
+		indexObj := &models.BaseFromMysteelChemicalIndex{}
+		existIndex, er := indexObj.GetIndexItem(edbCode)
+		if er != nil {
+			err = er
+			return
+		}
+		if len(mysteelChemicalDatas) == 0 {
+			err = errors.New("没有获取到数据")
+			return
+		}
+		// 因为只有一个指标，所以取第一个就可以了
+		items := mysteelChemicalDatas[0]
+		alarm_msg.SendAlarmMsg("钢联化工接口数据转化成功", 1)
+		for _, v := range items {
+			dateStr := v.DataTime.Format(utils.FormatDate)
+			if findData, ok := existDataMap[dateStr]; !ok {
+				v.BaseFromMysteelChemicalIndexId = existIndex.BaseFromMysteelChemicalIndexId
+				addItems = append(addItems, v)
+			} else {
+				if findData != nil && findData.Value != v.Value {
+					dataObj.BaseFromMysteelChemicalDataId = findData.BaseFromMysteelChemicalDataId
+					dataObj.Value = v.Value
+					dataObj.ModifyTime = time.Now()
+
+					err = dataObj.Update([]string{"value", "modify_time"})
+					if err != nil {
+						return
+					}
+				}
+			}
+		}
+		alarm_msg.SendAlarmMsg("开始添加数据", 1)
+		err = dataObj.AddV2(addItems)
+		if err != nil {
+			return
+		}
+		return
+	}
+	return
+}

services/base_from_python.go

@@ -172,7 +172,7 @@ func getPythonFileAbsolutePath(edbCode string) (pythonFile string, err error) {
 // getPythonFrontStr 获取python前面的代码
 func getPythonFrontStr() string {
 	//return "#!/usr/bin/python\n# -*- coding: UTF-8 -*-\nimport json\n\nimport pymysql\nimport pandas as pd\n\nsql_config = {\n    'host': 'rm-uf67kg347rhjfep5c1o.mysql.rds.aliyuncs.com',\n    'port': 3306,#主机号\n    'user': 'hz_technology',#账户名\n    'passwd': 'hongze@2021',#密码\n    'db': 'test_hz_data',\n    'charset': 'utf8mb4',\n    'cursorclass': pymysql.cursors.DictCursor\n}\n\ndb = pymysql.connect(**sql_config)\ndb.autocommit(1)\ncursor = db.cursor()\npandas_fetch_all = pd.read_sql\n\n# 返回数据\nresult = {}\n\n# 格式化返回数据\ndef format_data(data: pd.DataFrame,\n                index_str: str = \"data_time\",\n                value_str: str = \"value\"\n                ) -> pd.DataFrame:\n    \"\"\"\n        Parameters\n        ----------\n        data : pandas的DataFrame数据结构.\n        index_str : 对象下标字符串，在pandas的DataFrame中的列名.\n        value_str : 对象值字符串，在pandas的DataFrame中的列名\n\n        Returns\n        -------\n        DataFrame or Iterator[DataFrame]\n        例子：{'2007-01-09': 3220.0, '2007-01-10': 3230.0}\n\n        \"\"\"\n    index_list = []  # 空列表\n    value_list = []  # 空列表\n\n    for num in range(1, data.index.size):  # 迭代 所有的指标\n        index_list.append(data[index_str][num])\n        value_list.append(data[value_str][num])\n\n    tmp_data = {\n        \"date\": index_list,\n        \"value\": value_list\n    }\n    pd_data = pd.DataFrame(tmp_data)\n    # print(pd_data)\n    return pd_data\n"
-	str := fmt.Sprintf("#!/usr/bin/python\n# -*- coding: UTF-8 -*-\nimport json\n\nimport pymysql\nimport pandas as pd\nfrom pymongo import MongoClient\nimport pytz\n\nsql_config = {\n    'host': '%s',\n    'port': 3306,#主机号\n    'user': '%s',#账户名\n    'passwd': '%s',#密码\n    'db': '%s',\n    'charset': 'utf8mb4',\n    'cursorclass': pymysql.cursors.DictCursor\n}\n\ndb = pymysql.connect(**sql_config)\ndb.autocommit(1)\ncursor = db.cursor()\npandas_fetch_all = pd.read_sql\n\n", utils.PYTHON_MYSQL_HOST, utils.PYTHON_MYSQL_USER, utils.PYTHON_MYSQL_PASSWD, utils.PYTHON_MYSQL_DB)
+	str := fmt.Sprintf("#!/usr/bin/python\n# -*- coding: UTF-8 -*-\nimport json\n\nimport pymysql\nimport pandas as pd\n\nsql_config = {\n    'host': '%s',\n    'port': 3306,#主机号\n    'user': '%s',#账户名\n    'passwd': '%s',#密码\n    'db': '%s',\n    'charset': 'utf8mb4',\n    'cursorclass': pymysql.cursors.DictCursor\n}\n\ndb = pymysql.connect(**sql_config)\ndb.autocommit(1)\ncursor = db.cursor()\npandas_fetch_all = pd.read_sql\n\n", utils.PYTHON_MYSQL_HOST, utils.PYTHON_MYSQL_USER, utils.PYTHON_MYSQL_PASSWD, utils.PYTHON_MYSQL_DB)
 
 	// mongo部分
 	if utils.PYTHON_MONGO_HOST != `` {
@@ -191,7 +191,7 @@ func getPythonFrontStr() string {
 // @datetime 2024-05-07 18:38:03
 // @return string
 func getPythonFront2Str() string {
-	str := fmt.Sprintf("\nfrom pymongo import MongoClient\nfrom dateutil.tz import tzlocal\n# MongoDB 连接配置\nmongo_config = {\n    'host': '%s',  # 替换为你的 MongoDB 连接字符串\n    'database': '%s',  # 替换为你的数据库名\n    'collection': '%s',  # 替换为你的集合名\n    'auth_mechanism': '%s'  # 替换为你的认证机制\n}\n\n# 创建 MongoClient 并连接到数据库\nclient = MongoClient(mongo_config['host'], authMechanism=mongo_config['auth_mechanism'])\nmgo_db = client[mongo_config['database']]\ncollection = mgo_db[mongo_config['collection']]\n\n# 定义时区\nutc_tz = pytz.utc\nlocal_tz = tzlocal()  # 本地时区", utils.PYTHON_MONGO_HOST, utils.PYTHON_MONGO_DATABASE, "edb_data_business", utils.PYTHON_MONGO_AUTH_MECHANISM)
+	str := fmt.Sprintf("\nfrom pymongo import MongoClient\nimport pytz\nfrom dateutil.tz import tzlocal\n# MongoDB 连接配置\nmongo_config = {\n    'host': '%s',  # 替换为你的 MongoDB 连接字符串\n    'database': '%s',  # 替换为你的数据库名\n    'collection': '%s',  # 替换为你的集合名\n    'auth_mechanism': '%s'  # 替换为你的认证机制\n}\n\n# 创建 MongoClient 并连接到数据库\nclient = MongoClient(mongo_config['host'], authMechanism=mongo_config['auth_mechanism'])\nmgo_db = client[mongo_config['database']]\ncollection = mgo_db[mongo_config['collection']]\n\n# 定义时区\nutc_tz = pytz.utc\nlocal_tz = tzlocal()  # 本地时区", utils.PYTHON_MONGO_HOST, utils.PYTHON_MONGO_DATABASE, "edb_data_business", utils.PYTHON_MONGO_AUTH_MECHANISM)
 	return str
 }
 

services/base_from_smm.go

@@ -114,6 +114,21 @@ func SmmIndexHandle(baseFilePath, renameFilePath, indexName, indexCode, unit, fr
 			err = fmt.Errorf("smm index update err: %s", e.Error())
 			return
 		}
+		if item.IndexName != indexName {
+			baseFromSmmRecord := new(models.BaseFromSmmRecord)
+			baseFromSmmRecord.BaseFromSmmIndexId = item.BaseFromSmmIndexId
+			baseFromSmmRecord.OldIndexName = item.IndexName
+			baseFromSmmRecord.NewIndexName = indexName
+			ctime := time.Now()
+			baseFromSmmRecord.CreateTime = ctime
+			baseFromSmmRecord.Timestamp = ctime.Unix()
+			e = models.AddBaseFromSmmRecord(baseFromSmmRecord)
+			if e != nil {
+				err = fmt.Errorf("add smm record err: %s", e.Error())
+				return
+			}
+		}
+
 	}
 
 	// 遍历excel数据，然后跟现有的数据做校验，不存在则入库
@@ -243,4 +258,4 @@ func getEdbDataFromSmmData(edbCode string) (smmBaseDataList []models.BaseFromSmm
 	smmBaseDataListItem.IndexCode = edbCode
 	smmBaseDataList = append(smmBaseDataList, smmBaseDataListItem)
 	return
-}
+}