Merge branch 'custom' of http://8.136.199.33:3000/eta_server/eta_index_lib into bzq1/sci_hq_custom

controllers/base_from_calculate.go

@@ -118,6 +118,9 @@ func (this *CalculateController) Add() {
 	if err != nil {
 		return
 	}
+
+	// 判断是否需要禁用
+	go services.DisableEdbInfoNoUpdate(edbInfo)
 	resp := models.AddEdbInfoResp{
 		EdbInfoId:  edbInfo.EdbInfoId,
 		UniqueCode: uniqueCode,
@@ -336,6 +339,8 @@ func (this *CalculateController) Edit() {
 		EdbInfoId:  edbInfoDetail.EdbInfoId,
 		UniqueCode: edbInfoDetail.UniqueCode,
 	}
+	// 重置计算指标中的引用关系
+	go services.ResetEdbRelation(edbInfoDetail.EdbInfoId)
 	br.Ret = 200
 	br.Success = true
 	br.Msg = "保存成功"
@@ -777,7 +782,8 @@ func (this *CalculateController) BatchSave() {
 		br.ErrMsg = err.Error()
 		return
 	}
-
+	// 判断是否需要禁用
+	go services.DisableEdbInfoNoUpdate(edbInfo)
 	resp := models.AddEdbInfoResp{
 		EdbInfoId:  edbInfo.EdbInfoId,
 		UniqueCode: edbInfo.UniqueCode,
@@ -1208,6 +1214,9 @@ func (this *CalculateController) BatchEdit() {
 		return
 	}
 
+	// 重置计算指标中的引用关系
+	go services.ResetEdbRelation(edbInfoId)
+
 	resp := models.AddEdbInfoResp{
 		EdbInfoId:  edbInfo.EdbInfoId,
 		UniqueCode: edbInfo.UniqueCode,
@@ -1767,6 +1776,11 @@ func (this *CalculateController) SaveAdjust() {
 		EdbInfoId:  edbInfo.EdbInfoId,
 		UniqueCode: edbInfo.UniqueCode,
 	}
+
+	// 判断是否需要禁用
+	go services.DisableEdbInfoNoUpdate(edbInfo)
+	// 重置计算指标中的引用关系
+	go services.ResetEdbRelation(edbInfo.EdbInfoId)
 	br.Ret = 200
 	br.Success = true
 	br.Msg = "保存成功"
@@ -2302,3 +2316,119 @@ func (this *CalculateController) BatchEditMulti() {
 	br.Data = resp
 	br.IsAddLog = true
 }
+
+// StepCalculate
+// @Title 多步骤计算
+// @Description 多步骤计算
+// @Param request body models.StepCalculateBySearchData true "type json string"
+// @Success Ret=200 返回指标id
+// @router /base/step_calculate [post]
+func (this *CalculateController) StepCalculate() {
+	br := new(models.BaseResponse).Init()
+	defer func() {
+		if br.ErrMsg == "" {
+			br.IsSendEmail = false
+		}
+		this.Data["json"] = br
+		this.ServeJSON()
+	}()
+	var req models.StepCalculateBySearchData
+	if e := json.Unmarshal(this.Ctx.Input.RequestBody, &req); e != nil {
+		br.Msg = "参数解析异常"
+		br.ErrMsg = fmt.Sprintf("参数解析失败, Err: %v", e)
+		return
+	}
+	//sort.Slice(req.Calculates, func(i, j int) bool {
+	//	return req.Calculates[i].Sort < req.Calculates[j].Sort
+	//})
+	//
+	//var errMsg string
+	//originData, e := models.EdbInfoSearchDataToData(req.DataList)
+	//if e != nil {
+	//	br.Msg = "基础数据异常"
+	//	br.ErrMsg = fmt.Sprintf("计算失败, 基础数据异常, Err: %v", e)
+	//	return
+	//}
+	//
+	//calculateData := originData
+	//dateDataMap := make(map[time.Time]float64)
+	//for _, v := range req.Calculates {
+	//	baseCalculate := models.BaseCalculate{
+	//		DataList:      calculateData,
+	//		Frequency:     v.Frequency,
+	//		Formula:       v.Formula,
+	//		Calendar:      v.Calendar,
+	//		MoveType:      v.MoveType,
+	//		MoveFrequency: v.MoveFrequency,
+	//		FromFrequency: v.FromFrequency,
+	//		Source:        v.Source,
+	//	}
+	//
+	//	// 计算方式
+	//	switch baseCalculate.Source {
+	//	case utils.EdbBaseCalculateLjzzy:
+	//		dateDataMap, e, errMsg = baseCalculate.Ljzzy()
+	//	case utils.EdbBaseCalculateLjzzj:
+	//		dateDataMap, e, errMsg = baseCalculate.Ljzzj()
+	//	case utils.EdbBaseCalculateTbz:
+	//		dateDataMap, e, errMsg = baseCalculate.Tbz()
+	//	case utils.EdbBaseCalculateTcz:
+	//		dateDataMap, e, errMsg = baseCalculate.Tcz()
+	//	case utils.EdbBaseCalculateNszydpjjs:
+	//		dateDataMap, e, errMsg = baseCalculate.Nszydpjjs()
+	//	case utils.EdbBaseCalculateHbz:
+	//		dateDataMap, e, errMsg = baseCalculate.Hbz()
+	//	case utils.EdbBaseCalculateHcz:
+	//		dateDataMap, e, errMsg = baseCalculate.Hcz()
+	//	case utils.EdbBaseCalculateUpFrequency:
+	//		dateDataMap, e, errMsg = baseCalculate.UpFrequency()
+	//	case utils.EdbBaseCalculateDownFrequency:
+	//		dateDataMap, e, errMsg = baseCalculate.DownFrequency()
+	//	case utils.EdbBaseCalculateTimeShift:
+	//		dateDataMap, e, errMsg = baseCalculate.TimeShift()
+	//	case utils.EdbBaseCalculateCjjx:
+	//		dateDataMap, e, errMsg = baseCalculate.Cjjx()
+	//	case utils.EdbBaseCalculateAnnualized:
+	//		dateDataMap, e, errMsg = baseCalculate.Annualized()
+	//	case utils.EdbBaseCalculateLjz:
+	//		dateDataMap, e, errMsg = baseCalculate.Ljz()
+	//	case utils.EdbBaseCalculateLjzNczj:
+	//		dateDataMap, e, errMsg = baseCalculate.LjzNczj()
+	//	case utils.EdbBaseCalculateExponentialSmoothing:
+	//		dateDataMap, e, errMsg = baseCalculate.ExponentialSmoothing()
+	//	case utils.EdbBaseCalculateRjz:
+	//		dateDataMap, e, errMsg = baseCalculate.Rjz()
+	//	default:
+	//		errMsg = "计算方式无效"
+	//		e = fmt.Errorf("%s:%d", errMsg, baseCalculate.Source)
+	//	}
+	//	if e != nil {
+	//		br.Msg = "计算失败"
+	//		if errMsg != "" {
+	//			br.Msg = errMsg
+	//		}
+	//		br.ErrMsg = e.Error()
+	//		return
+	//	}
+	//
+	//	calculateData = models.TransDateData2EdbData(dateDataMap)
+	//}
+	resultData, dates, errMsg, e := services.StepCalculate(req.DataList, req.Calculates)
+	if e != nil {
+		br.Msg = "计算失败"
+		if errMsg != "" {
+			br.Msg = errMsg
+		}
+		br.ErrMsg = fmt.Sprintf("分步骤计算失败, Err: %v", e)
+		return
+	}
+
+	br.Ret = 200
+	br.Success = true
+	br.Msg = "计算成功"
+	br.Data = models.BaseCalculateResp{
+		DataMap:  resultData,
+		DateList: dates,
+	}
+	br.IsAddLog = true
+}

controllers/base_from_mysteel_chemical.go

@@ -87,19 +87,12 @@ func (this *MySteelChemicalController) Refresh() {
 		br.ErrMsg = "请输入指标编码，指标编码为空"
 		return
 	}
-	if req.EdbInfoId <= 0 {
+	if req.EdbInfoId < 0 {
 		br.Msg = "请输入指标ID！"
 		br.ErrMsg = "请输入指标ID"
 		return
 	}
 
-	// 获取指标详情
-	edbInfo, err := models.GetEdbInfoByEdbCode(source, req.EdbCode)
-	if err != nil {
-		br.Msg = "指标不存在！"
-		br.ErrMsg = "指标不存在"
-		return
-	}
 	cacheKey = utils.CACHE_EDB_DATA_REFRESH + strconv.Itoa(source) + "_" + req.EdbCode
 	if utils.Rc.IsExist(cacheKey) {
 		br.Ret = 501
@@ -109,30 +102,55 @@ func (this *MySteelChemicalController) Refresh() {
 	}
 	dataUpdateTime := time.Now().Format(utils.FormatDateTime)
 	utils.Rc.SetNX(cacheKey, 1, 1*time.Minute)
+	// 将数据刷新到BaseFromMysteelChemicalData
+	err = services.RefreshDataFromMysteelChemical(req.EdbCode, utils.GetEdbRefreshStartDate(req.StartDate), utils.BASE_END_DATE)
+	if err != nil {
+		br.Msg = "获取指标信息失败！"
+		br.ErrMsg = "获取指标信息失败 RefreshDataFromMysteelChemical，Err:" + err.Error()
+		return
+	}
 	err = models.RefreshEdbDataFromMysteelChemical(req.EdbInfoId, req.EdbCode, req.StartDate)
 	if err != nil && err.Error() != utils.ErrNoRow() {
 		br.Msg = "刷新指标信息失败！"
 		br.ErrMsg = "刷新指标信息失败 RefreshEdbDataFromMysteel，Err:" + err.Error()
 		return
 	}
+	if req.EdbInfoId != 0 {
+		// 获取指标详情
+		edbInfo, err := models.GetEdbInfoByEdbCode(source, req.EdbCode)
+		if err != nil {
+			br.Msg = "指标不存在！"
+			br.ErrMsg = "指标不存在"
+			return
+		}
 
-	// 更新指标最大最小值
-	erDataUpdateDate, err, errMsg := models.UnifiedModifyEdbInfoMaxAndMinInfoDataUpdate(edbInfo, dataUpdateTime)
-	if err != nil {
-		br.Msg = errMsg
-		br.ErrMsg = err.Error()
-		return
-	}
-	// 添加指标刷新成功日志
-	if erDataUpdateDate != "" {
-		_ = services.AddEdbInfoUpdateLog(edbInfo.EdbInfoId, 1, "", 1, "", 0, 0)
-	} else {
-		_ = services.AddEdbInfoUpdateLog(edbInfo.EdbInfoId, 1, "", 2, "未刷新到数据", 0, 0)
-	}
+		// 更新指标最大最小值
+		erDataUpdateDate, err, errMsg := models.UnifiedModifyEdbInfoMaxAndMinInfoDataUpdate(edbInfo, dataUpdateTime)
+		if err != nil {
+			br.Msg = errMsg
+			br.ErrMsg = err.Error()
+			return
+		}
+		// 添加指标刷新成功日志
+		isSourceRefresh := 0
+		if utils.MysteelChemicalApiToken != "" {
+			isSourceRefresh = 1
+		}
+		if erDataUpdateDate != "" {
+			_ = services.AddEdbInfoUpdateLog(edbInfo.EdbInfoId, 1, "", 1, "", isSourceRefresh, 0)
+		} else {
+			_ = services.AddEdbInfoUpdateLog(edbInfo.EdbInfoId, 1, "", 2, "未刷新到数据", isSourceRefresh, 0)
+		}
 
-	// 更新ES
-	go logic.UpdateEs(edbInfo.EdbInfoId)
+		// 更新ES
+		go logic.UpdateEs(edbInfo.EdbInfoId)
+	}
 
+	if utils.MysteelChemicalApiToken != "" {
+		// 钢联走api接口，那就更新数据源明细表
+		_ = services.SetMysteelChemicalEdbInfoUpdateStat(false)
+		_ = services.SetEdbSourceStat(false)
+	}
 	br.Ret = 200
 	br.Success = true
 	br.Msg = "获取成功"
@@ -196,6 +214,41 @@ func (this *MySteelChemicalController) HandleMysteelIndex() {
 	br.Msg = "处理成功"
 }
 
+// @Title 处理钢联指标的接口
+// @Description 处理钢联指标的接口
+// @Success 200 {object} models.HandleMysteelIndexResp
+// @router /handle/api/mysteel/index [post]
+func (this *MySteelChemicalController) HandleApiMysteelIndex() {
+	br := new(models.BaseResponse).Init()
+	defer func() {
+		this.Data["json"] = br
+		this.ServeJSON()
+	}()
+	body := this.Ctx.Input.RequestBody
+	var req models.HandleMysteelIndexResp
+	err := json.Unmarshal(body, &req)
+	if err != nil {
+		br.Msg = "参数解析异常！"
+		br.ErrMsg = "参数解析失败，Err:" + err.Error()
+		return
+	}
+
+	errMsg, err := services.HandleApiMysteelIndex(&req)
+	if err != nil {
+		fmt.Println("HandleMysteelIndex Err:" + err.Error())
+		if errMsg == "" {
+			br.Msg = "数据获取失败"
+		} else {
+			br.Msg = errMsg
+		}
+		br.ErrMsg = "处理失败,Err:" + err.Error()
+		return
+	}
+	br.Ret = 200
+	br.Success = true
+	br.Msg = "处理成功"
+}
+
 // GetMaxFileIndex
 // @Title 获取最大的文件编号下标
 // @Description 获取最大的文件编号下标

controllers/base_from_predict_calculate.go

@@ -506,6 +506,7 @@ func editPredictCalculate(br *models.BaseResponse, req models.EdbInfoCalculateSa
 		br.ErrMsg = err.Error()
 		return
 	}
+
 	resp := models.AddEdbInfoResp{
 		EdbInfoId:  edbInfo.EdbInfoId,
 		UniqueCode: edbInfo.UniqueCode,

controllers/base_from_ths_ds.go

@@ -6,6 +6,7 @@ import (
 	"eta/eta_index_lib/models"
 	"eta/eta_index_lib/services"
 	"eta/eta_index_lib/utils"
+	"fmt"
 	"strconv"
 	"time"
 )
@@ -46,7 +47,7 @@ func (this *ThsDsController) Add() {
 	cacheKey = utils.CACHE_EDB_DATA_ADD + strconv.Itoa(source) + "_" + req.StockCode + req.EdbCode
 	if !utils.Rc.IsExist(cacheKey) {
 		utils.Rc.SetNX(cacheKey, 1, 1*time.Minute)
-		dataItem, err := services.GetEdbDataFromThsDs(req.StockCode, req.EdbCode, utils.BASE_START_DATE, endDate, "")
+		dataItem, err := services.GetEdbDataFromThsDs(req.StockCode, req.EdbCode, utils.BASE_START_DATE, endDate, "", req.ExtraPars)
 		if err != nil {
 			br.Msg = "获取指标信息失败！"
 			br.ErrMsg = "获取指标信息失败 GetEdbDataFromThsDs，Err:" + err.Error()
@@ -118,7 +119,17 @@ func (this *ThsDsController) Refresh() {
 	defer func() {
 		utils.Rc.Delete(cacheKey)
 	}()
-	dataItem, err := services.GetEdbDataFromThsDs(edbInfo.StockCode, edbInfo.IndicatorCode, utils.GetEdbRefreshStartDate(req.StartDate), endDate, edbInfo.TerminalCode)
+
+	var extra models.EdbInfoExtra
+	if edbInfo.Extra != "" {
+		if e := json.Unmarshal([]byte(edbInfo.Extra), &extra); e != nil {
+			br.Msg = "刷新失败"
+			br.ErrMsg = fmt.Sprintf("API额外参数解析失败, %v", e)
+			return
+		}
+	}
+
+	dataItem, err := services.GetEdbDataFromThsDs(edbInfo.StockCode, edbInfo.IndicatorCode, utils.GetEdbRefreshStartDate(req.StartDate), endDate, edbInfo.TerminalCode, extra.ApiExtraPars)
 	if err != nil {
 		br.Msg = "获取指标信息失败！"
 		br.ErrMsg = "获取指标信息失败 GetEdbDataFromThsDs，Err:" + err.Error()

controllers/base_from_ths_hf.go

@@ -0,0 +1,818 @@
+package controllers
+
+import (
+	"encoding/json"
+	"eta/eta_index_lib/logic"
+	"eta/eta_index_lib/models"
+	"eta/eta_index_lib/models/mgo"
+	"eta/eta_index_lib/services"
+	"eta/eta_index_lib/utils"
+	"fmt"
+	"strconv"
+	"strings"
+	"time"
+)
+
+// ThsHfController 同花顺高频数据
+type ThsHfController struct {
+	BaseAuthController
+}
+
+// GetData
+// @Title 同花顺高频数据-获取数据
+// @Description 同花顺高频数据-获取数据
+// @Success 200 {object} models.ThsHfSearchEdbReq
+// @router /hf/edb_data [post]
+func (this *ThsHfController) GetData() {
+	br := new(models.BaseResponse).Init()
+	defer func() {
+		if br.ErrMsg == "" {
+			br.IsSendEmail = false
+		}
+		this.Data["json"] = br
+		this.ServeJSON()
+	}()
+	var params models.ThsHfSearchEdbReq
+	if e := json.Unmarshal(this.Ctx.Input.RequestBody, &params); e != nil {
+		br.Msg = "参数解析异常"
+		br.ErrMsg = fmt.Sprintf("参数解析失败, %v", e)
+		return
+	}
+	params.StockCode = strings.TrimSpace(params.StockCode)
+	if params.StockCode == "" {
+		br.Msg = "请输入证券代码"
+		return
+	}
+	stockCodes := strings.Split(params.StockCode, ",")
+	if len(stockCodes) == 0 {
+		br.Msg = "请输入证券代码"
+		return
+	}
+	if len(stockCodes) > 10 {
+		br.Msg = "最多输入10个证券代码"
+		return
+	}
+	params.EdbCode = strings.TrimSpace(params.EdbCode)
+	if params.EdbCode == "" {
+		br.Msg = "请输入指标代码"
+		return
+	}
+	edbCodes := strings.Split(params.EdbCode, ",")
+	if len(edbCodes) == 0 {
+		br.Msg = "请输入指标代码"
+		return
+	}
+	if len(edbCodes) > 20 {
+		br.Msg = "最多选择/输入20个指标代码"
+		return
+	}
+	if params.StartTime == "" {
+		br.Msg = "请选择起始时间"
+		return
+	}
+	_, e := time.ParseInLocation(utils.FormatDateTime, params.StartTime, time.Local)
+	if e != nil {
+		br.Msg = "起始时间格式有误"
+		br.ErrMsg = fmt.Sprintf("起始时间格式有误, %v", e)
+		return
+	}
+	// 结束时间选填, 不填则为当前时间
+	if params.EndTime != "" {
+		_, e := time.ParseInLocation(utils.FormatDateTime, params.EndTime, time.Local)
+		if e != nil {
+			br.Msg = "截止时间格式有误"
+			br.ErrMsg = fmt.Sprintf("截止时间格式有误, %v", e)
+			return
+		}
+	}
+	if params.EndTime == "" {
+		params.EndTime = time.Now().Local().Format(utils.FormatDateTime)
+	}
+	if !utils.InArrayByInt(models.ThsHfPeriodArr, params.Interval) {
+		br.Msg = "时间周期有误"
+		br.ErrMsg = fmt.Sprintf("时间周期有误, Interval: %d", params.Interval)
+		return
+	}
+	if params.CPS != "" && !utils.InArrayByStr(models.ThsHfCPSArr, params.CPS) {
+		br.Msg = "复权方式有误"
+		br.ErrMsg = fmt.Sprintf("复权方式有误, CPS: %s", params.CPS)
+		return
+	}
+	if params.BaseDate != "" {
+		_, e = time.ParseInLocation(utils.FormatDate, params.BaseDate, time.Local)
+		if e != nil {
+			br.Msg = "复权基点格式有误"
+			br.ErrMsg = fmt.Sprintf("复权基点格式有误, %v", e)
+			return
+		}
+	}
+	if params.Fill != "" && !utils.InArrayByStr(models.ThsHfFillArr, params.Fill) {
+		br.Msg = "非交易间隔处理有误"
+		br.ErrMsg = fmt.Sprintf("非交易间隔处理有误, Fill: %s", params.Fill)
+		return
+	}
+
+	// 根据配置获取指标数据
+	indexes, e := services.GetEdbDataFromThsHf(params, "")
+	if e != nil {
+		br.Msg = "获取失败"
+		br.ErrMsg = fmt.Sprintf("获取同花顺高频指标失败, %v", e)
+		return
+	}
+
+	br.Data = indexes
+	br.Ret = 200
+	br.Success = true
+	br.Msg = "获取成功"
+}
+
+// BaseAdd
+// @Title 同花顺高频数据-新增至数据源
+// @Description 同花顺高频数据-新增至数据源
+// @Success 200 {object} models.ThsHfBaseAddReq
+// @router /hf/base/add [post]
+func (this *ThsHfController) BaseAdd() {
+	br := new(models.BaseResponse).Init()
+	defer func() {
+		if br.ErrMsg == "" {
+			br.IsSendEmail = false
+		}
+		this.Data["json"] = br
+		this.ServeJSON()
+	}()
+	var params models.ThsHfBaseAddReq
+	if e := json.Unmarshal(this.Ctx.Input.RequestBody, &params); e != nil {
+		br.Msg = "参数解析异常"
+		br.ErrMsg = fmt.Sprintf("参数解析失败, %v", e)
+		return
+	}
+	params.StockCode = strings.TrimSpace(params.StockCode)
+	if params.StockCode == "" {
+		br.Msg = "请输入证券代码"
+		return
+	}
+	params.EdbCode = strings.TrimSpace(params.EdbCode)
+	if params.EdbCode == "" {
+		br.Msg = "请输入指标代码"
+		return
+	}
+	if params.StartTime == "" {
+		br.Msg = "请选择起始时间"
+		return
+	}
+	startTime, e := time.ParseInLocation(utils.FormatDateTime, params.StartTime, time.Local)
+	if e != nil {
+		br.Msg = "起始时间格式有误"
+		br.ErrMsg = fmt.Sprintf("起始时间格式有误, %v", e)
+		return
+	}
+	var endTime time.Time
+	if params.EndTime != "" {
+		ed, e := time.ParseInLocation(utils.FormatDateTime, params.EndTime, time.Local)
+		if e != nil {
+			br.Msg = "截止时间格式有误"
+			br.ErrMsg = fmt.Sprintf("截止时间格式有误, %v", e)
+			return
+		}
+		endTime = ed
+	}
+	if !utils.InArrayByInt(models.ThsHfPeriodArr, params.Interval) {
+		br.Msg = "时间周期有误"
+		br.ErrMsg = fmt.Sprintf("时间周期有误, Interval: %d", params.Interval)
+		return
+	}
+	if params.CPS != "" && !utils.InArrayByStr(models.ThsHfCPSArr, params.CPS) {
+		br.Msg = "复权方式有误"
+		br.ErrMsg = fmt.Sprintf("复权方式有误, CPS: %s", params.CPS)
+		return
+	}
+	if params.BaseDate != "" {
+		_, e = time.ParseInLocation(utils.FormatDate, params.BaseDate, time.Local)
+		if e != nil {
+			br.Msg = "复权基点格式有误"
+			br.ErrMsg = fmt.Sprintf("复权基点格式有误, %v", e)
+			return
+		}
+	}
+	if params.Fill != "" && !utils.InArrayByStr(models.ThsHfFillArr, params.Fill) {
+		br.Msg = "非交易间隔处理有误"
+		br.ErrMsg = fmt.Sprintf("非交易间隔处理有误, Fill: %s", params.Fill)
+		return
+	}
+	if params.ClassifyId <= 0 {
+		br.Msg = "请选择分类"
+		return
+	}
+	params.IndexName = strings.TrimSpace(params.IndexName)
+	if params.IndexName == "" {
+		br.Msg = "请输入指标名称"
+		return
+	}
+	if params.Frequency == "" {
+		br.Msg = "请输入频度"
+		return
+	}
+
+	// 缓存
+	source := utils.DATA_SOURCE_THS
+	subSource := utils.DATA_SUB_SOURCE_HIGH_FREQUENCY
+	cacheKey := fmt.Sprintf("%s_%d_%d_%s_%s", utils.CACHE_BASE_EDB_ADD, source, subSource, params.StockCode, params.EdbCode)
+	defer func() {
+		_ = utils.Rc.Delete(cacheKey)
+	}()
+	if utils.Rc.IsExist(cacheKey) {
+		br.Ret = 501
+		br.Success = true
+		br.Msg = "系统处理中，请稍后重试"
+		return
+	}
+	utils.Rc.SetNX(cacheKey, 1, 3*time.Minute)
+
+	// 已添加则忽略
+	indexOb := new(models.BaseFromThsHfIndex)
+	{
+		cond := fmt.Sprintf(" AND %s = ? AND %s = ?", indexOb.Cols().StockCode, indexOb.Cols().Indicator)
+		pars := make([]interface{}, 0)
+		pars = append(pars, params.StockCode, params.EdbCode)
+		item, e := indexOb.GetItemByCondition(cond, pars, "")
+		if e != nil && e.Error() != utils.ErrNoRow() {
+			br.Msg = "操作失败"
+			br.ErrMsg = fmt.Sprintf("获取原始指标失败, %v", e)
+			return
+		}
+		if item != nil {
+			br.Ret = 200
+			br.Success = true
+			br.Msg = "操作成功"
+			return
+		}
+	}
+
+	// 获取指标数据
+	var apiPars models.ThsHfSearchEdbReq
+	apiPars.StockCode = params.StockCode
+	apiPars.EdbCode = params.EdbCode
+	apiPars.StartTime = params.StartTime
+	apiPars.EndTime = params.EndTime
+	apiPars.Interval = params.Interval
+	apiPars.Fill = params.Fill
+	apiPars.CPS = params.CPS
+	apiPars.BaseDate = params.BaseDate
+	indexes, e := services.GetEdbDataFromThsHf(apiPars, "")
+	if e != nil {
+		br.Msg = "操作失败"
+		br.ErrMsg = fmt.Sprintf("获取同花顺高频指标失败, %v", e)
+		return
+	}
+	if len(indexes) == 0 {
+		br.Msg = "未搜索到指标"
+		br.ErrMsg = "未搜索到指标"
+		return
+	}
+	indexWithData := indexes[0]
+
+	indexItem := new(models.BaseFromThsHfIndex)
+	indexItem.BaseFromThsHfClassifyId = params.ClassifyId
+	indexItem.IndexCode = fmt.Sprintf("%s%s%s%s", utils.ThsHf, params.StockCode, params.EdbCode, params.Frequency)
+	indexItem.IndexName = params.IndexName
+	indexItem.Unit = params.Unit
+	indexItem.Frequency = params.Frequency
+	indexItem.StartDate = startTime
+	indexItem.EndDate = endTime
+	indexItem.SysUserId = params.SysAdminId
+	indexItem.SysUserRealName = params.SysAdminName
+	terminal, e := services.GetFirstTerminal(utils.DATA_SOURCE_THS, "")
+	if e != nil {
+		br.Msg = "终端未配置"
+		br.ErrMsg = fmt.Sprintf("终端未配置, %v", e)
+		return
+	}
+	indexItem.TerminalCode = terminal.TerminalCode
+	indexItem.StockCode = params.StockCode
+	indexItem.Indicator = params.EdbCode
+	b, e := json.Marshal(apiPars)
+	if e != nil {
+		br.Msg = "操作失败"
+		br.ErrMsg = fmt.Sprintf("API入参JSON格式化失败, %v", e)
+		return
+	}
+	indexItem.ApiPars = string(b)
+	if len(indexWithData.IndexData) > 0 {
+		indexItem.StartDate = indexWithData.IndexData[0].DataTime
+		indexItem.EndDate = indexWithData.IndexData[len(indexWithData.IndexData)-1].DataTime
+		lastVal, e := utils.FormatFloatPlaces(indexWithData.IndexData[0].Value, 4)
+		if e != nil {
+			br.Msg = "操作失败"
+			br.ErrMsg = fmt.Sprintf("格式化最新值失败, val: %v, err: %v", indexWithData.IndexData[0].Value, e)
+			return
+		}
+		indexItem.LatestValue = lastVal
+	}
+	indexItem.CreateTime = time.Now().Local()
+	indexItem.ModifyTime = time.Now().Local()
+
+	// 新增指标
+	if e := indexItem.Create(); e != nil {
+		br.Msg = "操作失败"
+		br.ErrMsg = fmt.Sprintf("新增指标失败, %v", e)
+		return
+	}
+
+	// 新增数据
+	if utils.UseMongo {
+		dataList := make([]interface{}, 0)
+		for _, v := range indexWithData.IndexData {
+			newVal, e := utils.FormatFloatPlaces(v.Value, 4)
+			if e != nil {
+				utils.FileLog.Info(fmt.Sprintf("FormatFloatPlaces err: %v", e))
+				continue
+			}
+
+			dataList = append(dataList, &mgo.BaseFromThsHfData{
+				BaseFromThsHfIndexId: int64(indexItem.BaseFromThsHfIndexId),
+				IndexCode:            indexItem.IndexCode,
+				DataTime:             v.DataTime,
+				Value:                newVal,
+				UniqueCode:           utils.MD5(fmt.Sprint(indexItem.IndexCode, v.DataTime.Format(utils.FormatDateTimeMinute))),
+				CreateTime:           time.Now().Local(),
+				ModifyTime:           time.Now().Local(),
+				DataTimestamp:        v.DataTime.UnixNano() / 1e6,
+			})
+		}
+		dataOb := new(mgo.BaseFromThsHfData)
+		if e = dataOb.BatchInsertData(500, dataList); e != nil {
+			br.Msg = "操作失败"
+			br.ErrMsg = fmt.Sprintf("批量新增数据失败-Mongo, %v", e)
+			return
+		}
+	} else {
+		dataOb := new(models.BaseFromThsHfData)
+		itemData := make([]*models.BaseFromThsHfData, 0)
+		for _, v := range indexWithData.IndexData {
+			newVal, e := utils.FormatFloatPlaces(v.Value, 4)
+			if e != nil {
+				utils.FileLog.Info(fmt.Sprintf("FormatFloatPlaces err: %v", e))
+				continue
+			}
+
+			t := new(models.BaseFromThsHfData)
+			t.BaseFromThsHfIndexId = indexItem.BaseFromThsHfIndexId
+			t.IndexCode = indexItem.IndexCode
+			t.DataTime = v.DataTime
+			t.Value = newVal
+			t.UniqueCode = utils.MD5(fmt.Sprint(indexItem.IndexCode, v.DataTime.Format(utils.FormatDateTimeMinute)))
+			t.CreateTime = time.Now().Local()
+			t.ModifyTime = time.Now().Local()
+			t.DataTimestamp = v.DataTime.UnixNano() / 1e6
+			itemData = append(itemData, t)
+		}
+		if e = dataOb.CreateMulti(itemData); e != nil {
+			br.Msg = "操作失败"
+			br.ErrMsg = fmt.Sprintf("批量新增数据失败-MySQL, %v", e)
+			return
+		}
+	}
+
+	br.Ret = 200
+	br.Success = true
+	br.Msg = "操作成功"
+}
+
+// BaseRefresh
+// @Title 同花顺高频数据-数据源刷新
+// @Description 同花顺高频数据-数据源刷新
+// @Success 200 {object} models.ThsHfBaseRefreshReq
+// @router /hf/base/refresh [post]
+func (this *ThsHfController) BaseRefresh() {
+	br := new(models.BaseResponse).Init()
+	defer func() {
+		if br.ErrMsg == "" {
+			br.IsSendEmail = false
+		}
+		this.Data["json"] = br
+		this.ServeJSON()
+	}()
+	var params models.ThsHfBaseRefreshReq
+	if e := json.Unmarshal(this.Ctx.Input.RequestBody, &params); e != nil {
+		br.Msg = "参数解析异常"
+		br.ErrMsg = fmt.Sprintf("参数解析失败, %v", e)
+		return
+	}
+	params.BaseIndexCode = strings.TrimSpace(params.BaseIndexCode)
+	if params.BaseIndexCode == "" {
+		br.Msg = "参数异常"
+		br.ErrMsg = fmt.Sprintf("参数异常, BaseIndexCode: %s", params.BaseIndexCode)
+		return
+	}
+	if params.RefreshType <= 0 {
+		params.RefreshType = 1
+	}
+
+	indexItem := new(models.BaseFromThsHfIndex)
+	{
+		ob := new(models.BaseFromThsHfIndex)
+		cond := fmt.Sprintf(" AND %s = ?", ob.Cols().IndexCode)
+		pars := make([]interface{}, 0)
+		pars = append(pars, params.BaseIndexCode)
+		item, e := ob.GetItemByCondition(cond, pars, "")
+		if e != nil {
+			if e.Error() == utils.ErrNoRow() {
+				br.Msg = "指标不存在"
+				return
+			}
+			br.Msg = "操作失败"
+			br.ErrMsg = fmt.Sprintf("获取源指标失败, %v", e)
+			return
+		}
+		indexItem = item
+	}
+
+	source := utils.DATA_SOURCE_THS
+	subSource := utils.DATA_SUB_SOURCE_HIGH_FREQUENCY
+	cacheKey := fmt.Sprintf("%s_%d_%d_%s_%s", utils.CACHE_BASE_EDB_REFRESH, source, subSource, indexItem.StockCode, indexItem.Indicator)
+	defer func() {
+		_ = utils.Rc.Delete(cacheKey)
+	}()
+	if utils.Rc.IsExist(cacheKey) {
+		br.Ret = 501
+		br.Success = true
+		br.Msg = "系统处理中，请稍后重试"
+		return
+	}
+	utils.Rc.SetNX(cacheKey, 1, 3*time.Minute)
+
+	// API参数
+	var apiPars models.ThsHfSearchEdbReq
+	if e := json.Unmarshal([]byte(indexItem.ApiPars), &apiPars); e != nil {
+		br.Msg = "操作失败"
+		br.ErrMsg = fmt.Sprintf("源指标API参数异常, %v", e)
+		return
+	}
+	// 刷新6小时: 指标开始时间前推6小时; 全部: API参数中的开始时间
+	if params.RefreshType == 1 {
+		apiPars.StartTime = indexItem.StartDate.Add(-6 * time.Hour).Format(utils.FormatDateTime)
+	}
+	// 若API参数中的结束时间不为空, 且不在EndDate之后, 那么不再刷新该指标
+	if apiPars.EndTime != "" {
+		apiEnd, e := time.ParseInLocation(utils.FormatDateTime, apiPars.EndTime, time.Local)
+		if e != nil {
+			br.Msg = "操作失败"
+			br.ErrMsg = fmt.Sprintf("API参数结束时间有误, %v", e)
+			return
+		}
+		if !apiEnd.After(indexItem.EndDate) {
+			br.Ret = 200
+			br.Success = true
+			br.Msg = "该指标无需刷新"
+			return
+		}
+	}
+
+	// API-获取指标数据
+	indexes, e := services.GetEdbDataFromThsHf(apiPars, indexItem.TerminalCode)
+	if e != nil {
+		br.Msg = "操作失败"
+		br.ErrMsg = fmt.Sprintf("获取同花顺高频指标失败, %v", e)
+		return
+	}
+	if len(indexes) == 0 {
+		br.Msg = "未搜索到指标"
+		br.ErrMsg = fmt.Sprintf("未搜索到指标, StockCode: %s, Indicator: %s", indexItem.StockCode, indexItem.Indicator)
+		return
+	}
+	indexWithData := indexes[0]
+
+	// 写入指标数据
+	if utils.UseMongo {
+		if e = services.RefreshThsHfBaseIndexMgo(indexItem, indexWithData, apiPars.StartTime); e != nil {
+			br.Msg = "操作失败"
+			br.ErrMsg = fmt.Sprintf("写入源指标数据失败-Mongo, %v", e)
+			return
+		}
+	} else {
+		if e = services.RefreshThsHfBaseIndex(indexItem, indexWithData, apiPars.StartTime); e != nil {
+			br.Msg = "操作失败"
+			br.ErrMsg = fmt.Sprintf("写入源指标数据失败, %v", e)
+			return
+		}
+	}
+
+	br.Ret = 200
+	br.Success = true
+	br.Msg = "操作成功"
+}
+
+// EdbAdd
+// @Title 同花顺高频数据-新增至指标库
+// @Description 同花顺高频数据-新增至指标库
+// @Success 200 {object} models.ThsHfEdbAddReq
+// @router /hf/edb/add [post]
+func (this *ThsHfController) EdbAdd() {
+	br := new(models.BaseResponse).Init()
+	defer func() {
+		if br.ErrMsg == "" {
+			br.IsSendEmail = false
+		}
+		this.Data["json"] = br
+		this.ServeJSON()
+	}()
+	var params models.ThsHfEdbAddReq
+	if e := json.Unmarshal(this.Ctx.Input.RequestBody, &params); e != nil {
+		br.Msg = "参数解析异常"
+		br.ErrMsg = fmt.Sprintf("参数解析失败, %v", e)
+		return
+	}
+	if params.NewIndex == nil {
+		br.Msg = "参数有误"
+		br.ErrMsg = "参数有误, 指标信息有误"
+		return
+	}
+	params.NewIndex.NewIndexName = strings.TrimSpace(params.NewIndex.NewIndexName)
+	if params.NewIndex.NewIndexName == "" {
+		br.Msg = "请输入指标名称"
+		return
+	}
+	if params.NewIndex.ClassifyId <= 0 {
+		br.Msg = "请选择分类"
+		return
+	}
+	if params.NewIndex.Unit == "" {
+		params.NewIndex.Unit = "无"
+	}
+	if params.NewIndex.NewFrequency == "" {
+		br.Msg = "请输入频度"
+		return
+	}
+
+	// 校验转换规则
+	convertRule := params.ConvertRule
+	if convertRule.ConvertType != 1 && convertRule.ConvertType != 2 {
+		br.Msg = "请选择数据转换方式"
+		return
+	}
+	if convertRule.ConvertType == 1 {
+		if convertRule.ConvertFixed.FixedDay != 1 && convertRule.ConvertFixed.FixedDay != 2 {
+			br.Msg = "请选择指定时间"
+			return
+		}
+		if convertRule.ConvertFixed.FixedTime == "" {
+			br.Msg = "请选择指定时间"
+			return
+		}
+		timePrefix := time.Now().Local().Format(utils.FormatDate)
+		st := fmt.Sprintf("%s %s", timePrefix, convertRule.ConvertFixed.FixedTime)
+		_, e := time.Parse(utils.FormatDateTime, st)
+		if e != nil {
+			br.Msg = "指定时间格式有误"
+			return
+		}
+	}
+	if convertRule.ConvertType == 2 {
+		if convertRule.ConvertArea.StartDay != 1 && convertRule.ConvertArea.StartDay != 2 {
+			br.Msg = "请选择起始时间"
+			return
+		}
+		if convertRule.ConvertArea.StartTime == "" {
+			br.Msg = "请选择起始时间"
+			return
+		}
+		var startTimePre string
+		if convertRule.ConvertArea.StartDay == 1 {
+			startTimePre = time.Now().Local().Format(utils.FormatDate)
+		}
+		if convertRule.ConvertArea.StartDay == 2 {
+			startTimePre = time.Now().Local().AddDate(0, 0, -1).Format(utils.FormatDate)
+		}
+		st := fmt.Sprintf("%s %s", startTimePre, convertRule.ConvertArea.StartTime)
+		startTime, e := time.Parse(utils.FormatDateTime, st)
+		if e != nil {
+			br.Msg = "起始时间格式有误"
+			return
+		}
+
+		if convertRule.ConvertArea.EndDay != 1 && convertRule.ConvertArea.EndDay != 2 {
+			br.Msg = "请选择截止时间"
+			return
+		}
+		if convertRule.ConvertArea.EndTime == "" {
+			br.Msg = "请选择截止时间"
+			return
+		}
+		var endTimePre string
+		if convertRule.ConvertArea.EndDay == 1 {
+			endTimePre = time.Now().Local().Format(utils.FormatDate)
+		}
+		if convertRule.ConvertArea.EndDay == 2 {
+			endTimePre = time.Now().Local().AddDate(0, 0, -1).Format(utils.FormatDate)
+		}
+		ed := fmt.Sprintf("%s %s", endTimePre, convertRule.ConvertArea.EndTime)
+		endTime, e := time.Parse(utils.FormatDateTime, ed)
+		if e != nil {
+			br.Msg = "截止时间格式有误"
+			return
+		}
+		if startTime.After(endTime) {
+			br.Msg = "起始日期不可早于截止日期"
+			return
+		}
+	}
+	convertRuleByte, e := json.Marshal(params.ConvertRule)
+	if e != nil {
+		br.Msg = "操作失败"
+		br.ErrMsg = fmt.Sprintf("转换规则JSON格式化失败, %v", e)
+		return
+	}
+
+	// 缓存
+	source := utils.DATA_SOURCE_THS
+	subSource := utils.DATA_SUB_SOURCE_HIGH_FREQUENCY
+	cacheKey := fmt.Sprintf("%s_%d_%d_%s_%s_%s", utils.CACHE_EDB_DATA_ADD, source, subSource, params.NewIndex.StockCode, params.NewIndex.EdbCode, params.NewIndex.NewFrequency)
+	defer func() {
+		_ = utils.Rc.Delete(cacheKey)
+	}()
+	if utils.Rc.IsExist(cacheKey) {
+		br.Ret = 501
+		br.Success = true
+		br.Msg = "系统处理中，请稍后重试"
+		return
+	}
+	utils.Rc.SetNX(cacheKey, 1, 3*time.Minute)
+
+	// 校验指标/分类
+	baseIndexOb := new(models.BaseFromThsHfIndex)
+	baseIndex, e := baseIndexOb.GetItemById(params.NewIndex.IndexId)
+	if e != nil {
+		br.Msg = "原指标不存在"
+		br.ErrMsg = fmt.Sprintf("原指标不存在, %v", e)
+		return
+	}
+	_, e = models.GetEdbClassifyById(params.NewIndex.ClassifyId)
+	if e != nil {
+		br.Msg = "分类信息有误"
+		br.ErrMsg = fmt.Sprintf("获取分类失败, %v", e)
+		return
+	}
+
+	// 判断指标名称是否已存在
+	{
+		var cond string
+		var pars []interface{}
+		if this.Lang == utils.EnLangVersion {
+			cond += " AND edb_name_en = ? "
+		} else {
+			cond += " AND edb_name = ?"
+		}
+		pars = append(pars, params.NewIndex.NewIndexName)
+		count, e := models.GetEdbInfoCountByCondition(cond, pars)
+		if e != nil {
+			br.Msg = "操作失败"
+			br.ErrMsg = fmt.Sprintf("获取重名指标失败, %v", e)
+			return
+		}
+		if count > 0 {
+			br.Msg = "指标名称已存在"
+			return
+		}
+	}
+
+	// 排序/指标编码
+	sortMax, e := models.GetEdbClassifyMaxSort(params.NewIndex.ClassifyId, 0)
+	if e != nil {
+		br.Msg = "操作失败"
+		br.ErrMsg = fmt.Sprintf("获取最大排序失败, %v", e)
+		return
+	}
+	edbCode, e := utils.GenerateEdbCode(1, "")
+	if e != nil {
+		br.Msg = "操作失败"
+		br.ErrMsg = fmt.Sprintf("生成指标编码失败, %v", e)
+		return
+	}
+	timestamp := strconv.FormatInt(time.Now().UnixNano(), 10)
+	uniqueCode := utils.MD5(utils.DATA_PREFIX + "_" + timestamp)
+
+	thsOb := new(models.EdbThsHf)
+	var addPars models.ThsHfAddBaseParams
+	addPars.EdbCode = edbCode
+	addPars.EdbName = params.NewIndex.NewIndexName
+	addPars.Unit = params.NewIndex.Unit
+	addPars.Frequency = params.NewIndex.NewFrequency
+	addPars.Sort = sortMax + 1
+	addPars.ClassifyId = params.NewIndex.ClassifyId
+	addPars.SysUserId = params.NewIndex.SysAdminId
+	addPars.SysUserRealName = params.NewIndex.SysAdminName
+	addPars.UniqueCode = uniqueCode
+	addPars.ConvertRule = string(convertRuleByte)
+	edbInfo, e := thsOb.Add(addPars, baseIndex)
+	if e != nil {
+		br.Msg = "新增失败"
+		br.ErrMsg = fmt.Sprintf("新增指标失败, %v", e)
+		return
+	}
+
+	// 更新指标最值
+	if e = thsOb.UnifiedModifyEdbInfoMaxAndMinInfo(edbInfo); e != nil {
+		br.Msg = "刷新指标失败"
+		br.ErrMsg = fmt.Sprintf("更新指标最值失败, %v", e)
+		return
+	}
+
+	// 添加到es
+	go logic.UpdateEs(edbInfo.EdbInfoId)
+
+	br.Ret = 200
+	br.Success = true
+	br.Msg = "操作成功"
+}
+
+// EdbRefresh
+// @Title 同花顺高频数据-指标库刷新
+// @Description 同花顺高频数据-指标库刷新
+// @Success 200 {object} models.RefreshEdbInfoReq
+// @router /hf/edb/refresh [post]
+func (this *ThsHfController) EdbRefresh() {
+	br := new(models.BaseResponse).Init()
+	defer func() {
+		if br.ErrMsg == "" {
+			br.IsSendEmail = false
+		}
+		this.Data["json"] = br
+		this.ServeJSON()
+	}()
+	var req models.RefreshEdbInfoReq
+	err := json.Unmarshal(this.Ctx.Input.RequestBody, &req)
+	if err != nil {
+		br.Msg = "参数解析异常！"
+		br.ErrMsg = "参数解析失败，Err:" + err.Error()
+		return
+	}
+	if req.EdbCode == "" {
+		br.Msg = "请输入指标编码！"
+		br.ErrMsg = "请输入指标编码，指标编码为空"
+		return
+	}
+	if req.EdbInfoId < 0 {
+		br.Msg = "请输入指标ID！"
+		br.ErrMsg = "请输入指标ID"
+		return
+	}
+	thsOb := new(models.EdbThsHf)
+	source := thsOb.GetSource()
+	subSource := thsOb.GetSubSource()
+	cacheKey := fmt.Sprintf("%s_%d_%d_%s", utils.CACHE_EDB_DATA_REFRESH, source, subSource, req.EdbCode)
+	if utils.Rc.IsExist(cacheKey) {
+		br.Ret = 501
+		br.Success = true
+		br.Msg = "系统处理中，请稍后重试"
+		return
+	}
+	utils.Rc.SetNX(cacheKey, 1, 1*time.Minute)
+	defer func() {
+		_ = utils.Rc.Delete(cacheKey)
+	}()
+
+	// 获取指标详情
+	edbInfo, e := models.GetEdbInfoByEdbCode(source, req.EdbCode)
+	if e != nil {
+		br.Msg = "指标不存在"
+		br.ErrMsg = fmt.Sprintf("指标不存在, %v", e)
+		return
+	}
+
+	// 获取指标关联信息
+	baseMapping := new(models.BaseFromEdbMapping)
+	{
+		ob := new(models.BaseFromEdbMapping)
+		cond := fmt.Sprintf(" AND %s = ? AND %s = ? AND %s = ?", ob.Cols().EdbCode, ob.Cols().Source, ob.Cols().SubSource)
+		pars := make([]interface{}, 0)
+		pars = append(pars, req.EdbCode, thsOb.GetSource(), thsOb.GetSubSource())
+		mapping, e := ob.GetItemByCondition(cond, pars, "")
+		if e != nil {
+			br.Msg = "刷新失败"
+			br.ErrMsg = fmt.Sprintf("指标关联信息有误, %v", e)
+			return
+		}
+		baseMapping = mapping
+	}
+
+	// 刷新指标
+	if e = thsOb.Refresh(edbInfo, baseMapping, req.StartDate); e != nil {
+		br.Msg = "刷新指标失败"
+		br.ErrMsg = fmt.Sprintf("刷新指标失败, %v", e)
+		return
+	}
+
+	// 更新指标最值
+	if e = thsOb.UnifiedModifyEdbInfoMaxAndMinInfo(edbInfo); e != nil {
+		br.Msg = "刷新指标失败"
+		br.ErrMsg = fmt.Sprintf("更新指标最值失败, %v", e)
+		return
+	}
+
+	// 更新ES
+	go logic.UpdateEs(edbInfo.EdbInfoId)
+
+	br.Ret = 200
+	br.Success = true
+	br.Msg = "操作成功"
+}

controllers/commodity_coal.go

@@ -1059,14 +1059,17 @@ func (this *CoalMineDataController) Coastal() {
 	for _, sheet := range req.SheetData {
 		sheetName = sheet.Name
 		//遍历行读取
-		maxCol := sheet.MaxCol
-		for i := 0; i < maxCol; i++ {
+		maxRow := sheet.MaxRow
+		for i := 0; i < maxRow; i++ {
 			if i == 0 {
 				row := sheet.Rows[i]
 				cells := row.Cells
 				for k, cell := range cells {
 					if k > 0 && k < 9 {
 						text := cell.Value
+						if text == "" {
+							continue
+						}
 						groupMap[k] = text
 						var item models.BaseFromCoalmineMapping
 						//合计命名
@@ -1169,6 +1172,7 @@ func (this *CoalMineDataController) Coastal() {
 					codeMap[v.IndexName] = v.IndexCode
 					newId, err := models.AddBaseFromCoalmineMapping(v)
 					if err != nil {
+						utils.FileLog.Info("再次添加公司指标名称错误：", err)
 						fmt.Println("再次添加公司指标名称错误", err)
 						continue
 					} else {
@@ -1181,6 +1185,7 @@ func (this *CoalMineDataController) Coastal() {
 			}
 		}
 	}
+	utils.FileLog.Info("指标操作完成")
 	fmt.Println("指标操作完成")
 
 	//给indexItem中的code赋值并插入index表
@@ -1190,6 +1195,7 @@ func (this *CoalMineDataController) Coastal() {
 			newId, err := models.AddBaseFromCoalCoastalIndex(v)
 			if err != nil {
 				fmt.Println("添加数据错误", err)
+				utils.FileLog.Info("添加数据错误：", err)
 			} else {
 				fmt.Println("新增成功", newId)
 			}
@@ -1197,6 +1203,7 @@ func (this *CoalMineDataController) Coastal() {
 			if indexMap[v.IndexName+v.DataTime] != v.DealValue && v.DealValue != "" {
 				err = models.UpdateBaseFromCoalCoastalIndex(v)
 				if err != nil {
+					utils.FileLog.Info("修改数据错误错误：", err)
 					fmt.Println("修改数据错误错误", err)
 					return
 				}
@@ -1338,6 +1345,7 @@ func (this *CoalMineDataController) Inland() {
 								if err != nil {
 									parsedTime, err = time.Parse("2006/01/02", text)
 									if err != nil {
+										utils.FileLog.Info("解析时间字符串出错：", err)
 										fmt.Println("解析时间字符串出错:", err)
 										return
 									}
@@ -1375,6 +1383,7 @@ func (this *CoalMineDataController) Inland() {
 					codeMap[v.IndexName] = v.IndexCode
 					newId, err := models.AddBaseFromCoalmineMapping(v)
 					if err != nil {
+						utils.FileLog.Info("再次添加公司指标名称错误：", err)
 						fmt.Println("再次添加公司指标名称错误", err)
 						continue
 					} else {
@@ -1387,6 +1396,7 @@ func (this *CoalMineDataController) Inland() {
 			}
 		}
 	}
+	utils.FileLog.Info("指标操作完成：")
 	fmt.Println("指标操作完成")
 
 	//给indexItem中的code赋值并插入index表
@@ -1395,6 +1405,7 @@ func (this *CoalMineDataController) Inland() {
 		if indexMap[v.IndexName+v.DataTime] == "" && v.DealValue != "" {
 			newId, err := models.AddBaseFromCoalInlandIndex(v)
 			if err != nil {
+				utils.FileLog.Info("添加数据错误：", err)
 				fmt.Println("添加数据错误", err)
 			} else {
 				fmt.Println("新增成功", newId)
@@ -1403,6 +1414,7 @@ func (this *CoalMineDataController) Inland() {
 			if indexMap[v.IndexName+v.DataTime] != v.DealValue && v.DealValue != "" {
 				err = models.UpdateBaseFromCoalInlandIndex(v)
 				if err != nil {
+					utils.FileLog.Info("修改数据错误错误：", err)
 					fmt.Println("修改数据错误错误", err)
 					return
 				}

controllers/edb_info.go

@@ -70,15 +70,17 @@ func (this *EdbInfoController) Add() {
 		br.ErrMsg = "请输入指标名称"
 		return
 	}
-	if req.Frequency == "" {
-		br.Msg = "请输入指标频度！"
-		br.ErrMsg = "请输入指标频度"
-		return
-	}
-	if req.Unit == "" {
-		br.Msg = "请输入指标单位！"
-		br.ErrMsg = "请输入指标单位"
-		return
+	if req.Source != utils.DATA_SOURCE_MYSTEEL_CHEMICAL {
+		if req.Frequency == "" {
+			br.Msg = "请输入指标频度！"
+			br.ErrMsg = "请输入指标频度"
+			return
+		}
+		if req.Unit == "" {
+			br.Msg = "请输入指标单位！"
+			br.ErrMsg = "请输入指标单位"
+			return
+		}
 	}
 	var isAdd bool
 	item, err := models.GetEdbInfoByEdbCode(req.Source, req.EdbCode)

controllers/factor_edb_series/factor_edb_series.go

@@ -0,0 +1,416 @@
+package factor_edb_series
+
+import (
+	"encoding/json"
+	"eta/eta_index_lib/controllers"
+	"eta/eta_index_lib/logic"
+	"eta/eta_index_lib/models"
+	"eta/eta_index_lib/services"
+	"eta/eta_index_lib/utils"
+	"fmt"
+	"strconv"
+	"sync"
+	"time"
+)
+
+// FactorEdbSeriesController 因子指标系列
+type FactorEdbSeriesController struct {
+	controllers.BaseAuthController
+}
+
+// Recalculate
+// @Title 因子指标系列-重计算
+// @Description 因子指标系列-重计算
+// @Success 200 {object} models.FactorEdbRecalculateReq
+// @router /recalculate [post]
+func (this *FactorEdbSeriesController) Recalculate() {
+	br := new(models.BaseResponse).Init()
+	defer func() {
+		if br.ErrMsg == "" {
+			br.IsSendEmail = false
+		}
+		this.Data["json"] = br
+		this.ServeJSON()
+	}()
+	var req models.FactorEdbRecalculateReq
+	if e := json.Unmarshal(this.Ctx.Input.RequestBody, &req); e != nil {
+		br.Msg = "参数解析异常！"
+		br.ErrMsg = fmt.Sprintf("参数解析失败, Err: %v", e)
+		return
+	}
+	if req.EdbInfoId <= 0 {
+		br.Msg = "参数有误"
+		br.ErrMsg = fmt.Sprintf("指标ID有误, EdbInfoId: %d", req.EdbInfoId)
+		return
+	}
+
+	cacheKey := utils.CACHE_EDB_DATA_REFRESH + "_factor_edb_recalculate_" + req.EdbCode
+	if utils.Rc.IsExist(cacheKey) {
+		br.Ret = 501
+		br.Success = true
+		br.Msg = fmt.Sprintf("系统处理中, 请稍后重试, 指标编码: %s", req.EdbCode)
+		return
+	}
+	utils.Rc.SetNX(cacheKey, 1, 10*time.Minute)
+	defer func() {
+		_ = utils.Rc.Delete(cacheKey)
+	}()
+
+	// 获取指标信息
+	edbInfo, e := models.GetEdbInfoById(req.EdbInfoId)
+	if e != nil {
+		if e.Error() == utils.ErrNoRow() {
+			br.Msg = "指标不存在"
+			br.ErrMsg = fmt.Sprintf("指标不存在, EdbInfoId: %d, EdbCode: %s", req.EdbInfoId, req.EdbCode)
+			return
+		}
+		br.Msg = "操作失败"
+		br.ErrMsg = fmt.Sprintf("获取指标信息失败, Err: %v", e)
+		return
+	}
+
+	// 查询指标关联的系列
+	mappings := make([]*models.FactorEdbSeriesMapping, 0)
+	{
+		ob := new(models.FactorEdbSeriesMapping)
+		cond := fmt.Sprintf(" AND %s = ?", ob.Cols().EdbInfoId)
+		pars := make([]interface{}, 0)
+		pars = append(pars, req.EdbInfoId)
+		fields := []string{ob.Cols().FactorEdbSeriesId, ob.Cols().EdbInfoId, ob.Cols().EdbCode}
+		list, e := ob.GetItemsByCondition(cond, pars, fields, "")
+		if e != nil {
+			br.Msg = "操作失败"
+			br.ErrMsg = fmt.Sprintf("获取指标系列关联失败, Err: %v", e)
+			return
+		}
+		mappings = list
+	}
+	if len(mappings) == 0 {
+		br.Ret = 200
+		br.Success = true
+		br.Msg = "操作成功"
+		return
+	}
+
+	// 获取系列信息
+	seriesIds := make([]int, 0)
+	seriesExist := make(map[int]bool)
+	for _, v := range mappings {
+		if !seriesExist[v.FactorEdbSeriesId] {
+			seriesExist[v.FactorEdbSeriesId] = true
+			seriesIds = append(seriesIds, v.FactorEdbSeriesId)
+		}
+	}
+	if len(seriesIds) == 0 {
+		br.Ret = 200
+		br.Success = true
+		br.Msg = "操作成功"
+		return
+	}
+	seriesMap := make(map[int]*models.FactorEdbSeries, 0)
+	{
+		ob := new(models.FactorEdbSeries)
+		cond := fmt.Sprintf(" AND %s IN (%s)", ob.Cols().PrimaryId, utils.GetOrmInReplace(len(seriesIds)))
+		pars := make([]interface{}, 0)
+		pars = append(pars, seriesIds)
+		fields := []string{ob.Cols().PrimaryId, ob.Cols().EdbInfoType, ob.Cols().CalculateStep, ob.Cols().CalculateState}
+		list, e := ob.GetItemsByCondition(cond, pars, fields, "")
+		if e != nil {
+			br.Msg = "操作失败"
+			br.ErrMsg = fmt.Sprintf("获取指标系列失败, Err: %v", e)
+			return
+		}
+		for _, v := range list {
+			seriesMap[v.FactorEdbSeriesId] = v
+		}
+	}
+
+	// 获取指标原数据
+	edbData, e := models.GetEdbDataAllByEdbCode(edbInfo.EdbCode, edbInfo.Source, edbInfo.SubSource, 0)
+	if e != nil {
+		br.Msg = "操作失败"
+		br.ErrMsg = fmt.Sprintf("获取指标数据失败, EdbCode: %s, Err: %v", edbInfo.EdbCode, e)
+		return
+	}
+	if len(edbData) == 0 {
+		br.Msg = "指标无数据"
+		br.ErrMsg = fmt.Sprintf("指标无数据, EdbCode: %s", edbInfo.EdbCode)
+		return
+	}
+
+	for _, v := range mappings {
+		series := seriesMap[v.FactorEdbSeriesId]
+		if series == nil {
+			continue
+		}
+		// 系列无计算则忽略
+		if series.CalculateState != models.FactorEdbSeriesCalculateNone || series.CalculateStep == "" {
+			continue
+		}
+		var calculates []models.CalculatesReq
+		if e = json.Unmarshal([]byte(series.CalculateStep), &calculates); e != nil {
+			br.Msg = "计算步骤异常"
+			br.ErrMsg = fmt.Sprintf("计算步骤异常, SeriesId: %d, Err: %v", series.FactorEdbSeriesId, e)
+			return
+		}
+
+		// 重新计算相关数据
+		e = services.FactorEdbStepCalculate(v.FactorEdbSeriesId, v.EdbInfoId, v.EdbCode, edbData, calculates)
+		if e != nil {
+			br.Msg = "因子指标计算失败"
+			br.ErrMsg = fmt.Sprintf("因子指标计算失败, SeriesId: %d, EdbCode: %s, Err: %v", v.FactorEdbSeriesId, v.EdbCode, e)
+			return
+		}
+	}
+
+	br.Ret = 200
+	br.Success = true
+	br.Msg = "操作成功"
+}
+
+// ChartRecalculate
+// @Title 图表数据-重计算
+// @Description 图表数据-重计算
+// @Success 200 {object} models.FactorEdbRecalculateReq
+// @router /chart_recalculate [post]
+func (this *FactorEdbSeriesController) ChartRecalculate() {
+	br := new(models.BaseResponse).Init()
+	defer func() {
+		if br.ErrMsg == "" {
+			br.IsSendEmail = false
+		}
+		this.Data["json"] = br
+		this.ServeJSON()
+	}()
+	var req models.FactorEdbChartRecalculateReq
+	if e := json.Unmarshal(this.Ctx.Input.RequestBody, &req); e != nil {
+		br.Msg = "参数解析异常！"
+		br.ErrMsg = fmt.Sprintf("参数解析失败, Err: %v", e)
+		return
+	}
+	if req.ChartInfoId <= 0 {
+		br.Msg = "参数有误"
+		br.ErrMsg = fmt.Sprintf("指标ID有误, ChartInfoId: %d", req.ChartInfoId)
+		return
+	}
+
+	cacheKey := utils.CACHE_CHART_INFO_DATA + "_factor_chart_recalculate_" + strconv.Itoa(req.ChartInfoId)
+	if utils.Rc.IsExist(cacheKey) {
+		br.Ret = 501
+		br.Success = true
+		br.Msg = fmt.Sprintf("系统处理中, 请稍后重试, 图表ID: %s", req.ChartInfoId)
+		return
+	}
+	utils.Rc.SetNX(cacheKey, 1, 10*time.Minute)
+	defer func() {
+		_ = utils.Rc.Delete(cacheKey)
+	}()
+
+	// 查询图表关联的系列指标
+	mappings := make([]*models.FactorEdbSeriesChartMapping, 0)
+	{
+		ob := new(models.FactorEdbSeriesChartMapping)
+		cond := fmt.Sprintf(" AND %s = ?", ob.Cols().ChartInfoId)
+		pars := make([]interface{}, 0)
+		pars = append(pars, req.ChartInfoId)
+		fields := []string{ob.Cols().PrimaryId, ob.Cols().CalculateType, ob.Cols().CalculatePars, ob.Cols().FactorEdbSeriesId, ob.Cols().EdbInfoId}
+		list, e := ob.GetItemsByCondition(cond, pars, fields, "")
+		if e != nil {
+			br.Msg = "操作失败"
+			br.ErrMsg = fmt.Sprintf("获取指标系列关联失败, Err: %v", e)
+			return
+		}
+		mappings = list
+	}
+	if len(mappings) == 0 {
+		br.Ret = 200
+		br.Success = true
+		br.Msg = "操作成功"
+		return
+	}
+	seriesIds := make([]int, 0)
+	edbInfoIds := make([]int, 0)
+	for _, v := range mappings {
+		seriesIds = append(seriesIds, v.FactorEdbSeriesId)
+		edbInfoIds = append(edbInfoIds, v.EdbInfoId)
+	}
+
+	// 获取因子指标及系列信息
+	seriesIdItem := make(map[int]*models.FactorEdbSeries)
+	{
+		ob := new(models.FactorEdbSeries)
+		cond := fmt.Sprintf(" AND %s IN (%s)", ob.Cols().PrimaryId, utils.GetOrmInReplace(len(seriesIds)))
+		pars := make([]interface{}, 0)
+		pars = append(pars, seriesIds)
+		items, e := ob.GetItemsByCondition(cond, pars, []string{}, fmt.Sprintf("%s ASC", ob.Cols().PrimaryId))
+		if e != nil {
+			br.Msg = "获取失败"
+			br.ErrMsg = "获取因子指标系列失败, Err: " + e.Error()
+			return
+		}
+		for _, v := range items {
+			seriesIdItem[v.FactorEdbSeriesId] = v
+		}
+	}
+	edbIdItem := make(map[int]*models.EdbInfo)
+	edbItems, e := models.GetEdbInfoByIdList(edbInfoIds)
+	if e != nil {
+		br.Msg = "获取失败"
+		br.ErrMsg = "获取因子指标失败, Err: " + e.Error()
+		return
+	}
+	for _, v := range edbItems {
+		edbIdItem[v.EdbInfoId] = v
+	}
+
+	// 重新计算
+	calculateWorkers := make(chan struct{}, 10)
+	wg := sync.WaitGroup{}
+	calculateDataOb := new(models.FactorEdbSeriesCalculateData)
+	for _, v := range mappings {
+		edbItem := edbIdItem[v.EdbInfoId]
+		if edbItem == nil {
+			continue
+		}
+		seriesItem := seriesIdItem[v.FactorEdbSeriesId]
+		if seriesItem == nil {
+			continue
+		}
+
+		wg.Add(1)
+		go func(chartMapping *models.FactorEdbSeriesChartMapping, edb *models.EdbInfo, series *models.FactorEdbSeries) {
+			defer func() {
+				wg.Done()
+				<-calculateWorkers
+			}()
+			calculateWorkers <- struct{}{}
+
+			// 相关性计算
+			if chartMapping.CalculateType == models.FactorEdbSeriesChartCalculateTypeCorrelation {
+				// 解析计算参数
+				if chartMapping.CalculatePars == "" {
+					utils.FileLog.Info(fmt.Sprintf("相关性-计算参数为空, MappingId: %d", chartMapping.FactorEdbSeriesChartMappingId))
+					return
+				}
+				var calculatePars models.FactorEdbSeriesChartCalculateCorrelationReq
+				if e := json.Unmarshal([]byte(chartMapping.CalculatePars), &calculatePars); e != nil {
+					utils.FileLog.Info(fmt.Sprintf("相关性-计算参数解析失败, MappingId: %d, err: %v", chartMapping.FactorEdbSeriesChartMappingId, e))
+					return
+				}
+
+				// 获取标的指标信息及数据
+				baseEdb, e := models.GetEdbInfoById(calculatePars.BaseEdbInfoId)
+				if e != nil {
+					utils.FileLog.Info(fmt.Sprintf("相关性-获取标的指标失败, MappingId: %d, err: %v", chartMapping.FactorEdbSeriesChartMappingId, e))
+					return
+				}
+				calculateUnitDays, ok := utils.FrequencyDaysMap[calculatePars.CalculateUnit]
+				if !ok {
+					utils.FileLog.Info(fmt.Sprintf("相关性-错误的计算窗口频度, MappingId: %d", chartMapping.FactorEdbSeriesChartMappingId))
+					return
+				}
+				calculateDays := calculatePars.CalculateValue * calculateUnitDays
+
+				dataListA := make([]*models.EdbInfoSearchData, 0)
+				{
+					// 标的指标数据日期区间
+					startDate := time.Now().AddDate(0, 0, -calculateDays).Format(utils.FormatDate)
+					endDate := time.Now().Format(utils.FormatDate)
+					startDateTime, _ := time.ParseInLocation(utils.FormatDate, startDate, time.Local)
+					startDate = startDateTime.AddDate(0, 0, 1).Format(utils.FormatDate) // 不包含第一天
+					switch baseEdb.EdbInfoType {
+					case 0:
+						list, e := models.GetEdbDataList(baseEdb.Source, baseEdb.SubSource, baseEdb.EdbInfoId, startDate, endDate)
+						if e != nil {
+							utils.FileLog.Info(fmt.Sprintf("相关性-获取标的指标数据失败, EdbInfoId: %d, err: %v", baseEdb.EdbInfoId, e))
+							return
+						}
+						dataListA = models.TransEdbInfoDataList2SearchData(list)
+					case 1:
+						dataListA, _, e, _ = models.GetPredictDataListByPredictEdbInfo(baseEdb, 1, startDate)
+						if e != nil {
+							utils.FileLog.Info(fmt.Sprintf("相关性-获取标的指标数据失败, EdbInfoId: %d, err: %v", baseEdb.EdbInfoId, e))
+							return
+						}
+					default:
+						utils.FileLog.Info(fmt.Sprintf("相关性-标的指标类型异常, EdbInfoId: %d", baseEdb.EdbInfoId))
+						return
+					}
+				}
+
+				// 获取指标数据
+				dataListB := make([]*models.EdbInfoSearchData, 0)
+				if series.CalculateState == models.FactorEdbSeriesCalculated {
+					cond := fmt.Sprintf(" AND %s = ? AND %s = ?", calculateDataOb.Cols().FactorEdbSeriesId, calculateDataOb.Cols().EdbInfoId)
+					pars := make([]interface{}, 0)
+					pars = append(pars, chartMapping.FactorEdbSeriesId, chartMapping.EdbInfoId)
+					dataItems, e := calculateDataOb.GetItemsByCondition(cond, pars, []string{calculateDataOb.Cols().DataTime, calculateDataOb.Cols().Value}, fmt.Sprintf("%s ASC", calculateDataOb.Cols().DataTime))
+					if e != nil {
+						utils.FileLog.Info(fmt.Sprintf("相关性-获取因子指标数据失败, MappingId: %d, err: %v", chartMapping.FactorEdbSeriesChartMappingId, e))
+						return
+					}
+					dataListB = models.TransEdbSeriesCalculateData2EdbDataList(dataItems)
+				} else {
+					switch edb.EdbInfoType {
+					case 0:
+						list, e := models.GetEdbDataList(edb.Source, edb.SubSource, edb.EdbInfoId, "", "")
+						if e != nil {
+							utils.FileLog.Info(fmt.Sprintf("相关性-获取因子指标数据失败, EdbInfoId: %d, err: %v", edb.EdbInfoId, e))
+							return
+						}
+						dataListB = models.TransEdbInfoDataList2SearchData(list)
+					case 1:
+						dataListB, e = models.GetPredictEdbDataListAll(baseEdb, 1)
+						if e != nil {
+							utils.FileLog.Info(fmt.Sprintf("相关性-获取因子指标数据失败, EdbInfoId: %d, err: %v", edb.EdbInfoId, e))
+							return
+						}
+					default:
+						utils.FileLog.Info(fmt.Sprintf("相关性-因子指标类型异常, EdbInfoId: %d", edb.EdbInfoId))
+						return
+					}
+				}
+
+				// 计算相关性
+				xEdbIdValue, yDataList, e := logic.CalculateCorrelation(calculatePars.LeadValue, calculatePars.LeadUnit, baseEdb.Frequency, edb.Frequency, dataListA, dataListB)
+				if e != nil {
+					utils.FileLog.Info(fmt.Sprintf("相关性-计算失败, EdbInfoId: %d", edb.EdbInfoId))
+					return
+				}
+
+				// X及Y轴数据
+				yData := yDataList[0].Value
+				yLen := len(yData)
+				values := make([]models.FactorEdbSeriesCorrelationMatrixValues, len(xEdbIdValue))
+				for k, x := range xEdbIdValue {
+					var y float64
+					if k >= 0 && k < yLen {
+						y = yData[k]
+					}
+					y = utils.SubFloatToFloat(y, 2)
+					values[k] = models.FactorEdbSeriesCorrelationMatrixValues{
+						XData: x, YData: y,
+					}
+				}
+
+				// 更新计算结果
+				b, e := json.Marshal(values)
+				if e != nil {
+					utils.FileLog.Info(fmt.Sprintf("相关性-计算结果JSON格式化失败, err: %v", e))
+					return
+				}
+				chartMapping.CalculateData = string(b)
+				chartMapping.ModifyTime = time.Now().Local()
+				if e = chartMapping.Update([]string{chartMapping.Cols().CalculateData, chartMapping.Cols().ModifyTime}); e != nil {
+					utils.FileLog.Info(fmt.Sprintf("相关性-更新矩阵数据失败, err: %v", e))
+					return
+				}
+			}
+		}(v, edbItem, seriesItem)
+	}
+	wg.Wait()
+
+	br.Ret = 200
+	br.Success = true
+	br.Msg = "操作成功"
+}

logic/chart_correlation.go

@@ -0,0 +1,195 @@
+package logic
+
+import (
+	"eta/eta_index_lib/models"
+	"eta/eta_index_lib/utils"
+	"fmt"
+)
+
+// CalculateCorrelation 计算相关性-获取x轴和y轴
+func CalculateCorrelation(leadValue int, leadUnit, frequencyA, frequencyB string, dataListA, dataListB []*models.EdbInfoSearchData) (xEdbIdValue []int, yDataList []YData, err error) {
+	xData := make([]int, 0)
+	yData := make([]float64, 0)
+	if leadValue == 0 {
+		xData = append(xData, 0)
+	}
+	if leadValue > 0 {
+		leadMin := 0 - leadValue
+		xLen := 2*leadValue + 1
+		for i := 0; i < xLen; i++ {
+			n := leadMin + i
+			xData = append(xData, n)
+		}
+	}
+
+	// 计算窗口，不包含第一天
+	//startDateTime, _ := time.ParseInLocation(utils.FormatDate, startDate, time.Local)
+	//startDate = startDateTime.AddDate(0, 0, 1).Format(utils.FormatDate)
+
+	//// 2023-03-02 时间序列始终以指标B为基准, 始终是A进行平移
+	//baseEdbInfo := edbInfoMappingB
+	//changeEdbInfo := edbInfoMappingA
+	// 2023-03-17 时间序列始终以指标A为基准, 始终是B进行平移
+	//baseEdbInfo := edbInfoMappingA
+	//changeEdbInfo := edbInfoMappingB
+
+	// 获取时间基准指标在时间区间内的值
+	//aDataList := make([]*EdbDataList, 0)
+	//switch baseEdbInfo.EdbInfoCategoryType {
+	//case 0:
+	//	aDataList, err = GetEdbDataList(baseEdbInfo.Source, baseEdbInfo.SubSource, baseEdbInfo.EdbInfoId, startDate, endDate)
+	//case 1:
+	//	_, aDataList, _, _, err, _ = data.GetPredictDataListByPredictEdbInfoId(baseEdbInfo.EdbInfoId, startDate, endDate, false)
+	//default:
+	//	err = errors.New("指标base类型异常")
+	//	return
+	//}
+	//
+	//// 获取变频指标所有日期的值, 插值法完善数据
+	//bDataList := make([]*EdbDataList, 0)
+	//switch changeEdbInfo.EdbInfoCategoryType {
+	//case 0:
+	//	bDataList, err = GetEdbDataList(changeEdbInfo.Source, changeEdbInfo.SubSource, changeEdbInfo.EdbInfoId, "", "")
+	//case 1:
+	//	_, bDataList, _, _, err, _ = data.GetPredictDataListByPredictEdbInfoId(changeEdbInfo.EdbInfoId, "", "", false)
+	//default:
+	//	err = errors.New("指标change类型异常")
+	//	return
+	//}
+	//changeDataMap := make(map[string]float64)
+	//newChangeDataList, e := HandleDataByLinearRegression(bDataList, changeDataMap)
+	//if e != nil {
+	//	err = fmt.Errorf("获取变频指标插值法Map失败, Err: %s", e.Error())
+	//	return
+	//}
+
+	// 2023-03-17 时间序列始终以指标A为基准, 始终是B进行平移
+	baseDataList := make([]*models.EdbInfoSearchData, 0)
+	baseDataMap := make(map[string]float64)
+	changeDataList := make([]*models.EdbInfoSearchData, 0)
+	changeDataMap := make(map[string]float64)
+
+	// 先把低频指标升频为高频
+	{
+		frequencyIntMap := map[string]int{
+			"日度": 1,
+			"周度": 2,
+			"旬度": 3,
+			"月度": 4,
+			"季度": 5,
+			"年度": 6,
+		}
+
+		// 如果A指标是高频，那么就需要对B指标进行升频
+		if frequencyIntMap[frequencyA] < frequencyIntMap[frequencyB] {
+			tmpNewChangeDataList, e := models.HandleDataByLinearRegression(dataListB, changeDataMap)
+			if e != nil {
+				err = fmt.Errorf("获取变频指标插值法Map失败, Err: %s", e.Error())
+				return
+			}
+			changeDataList = tmpNewChangeDataList
+			baseDataList = dataListA
+			for _, v := range baseDataList {
+				baseDataMap[v.DataTime] = v.Value
+			}
+
+		} else if frequencyIntMap[frequencyA] > frequencyIntMap[frequencyB] {
+			// 如果B指标是高频，那么就需要对A指标进行升频
+			tmpNewChangeDataList, e := models.HandleDataByLinearRegression(dataListA, baseDataMap)
+			if e != nil {
+				err = fmt.Errorf("获取变频指标插值法Map失败, Err: %s", e.Error())
+				return
+			}
+			baseDataList = tmpNewChangeDataList
+
+			changeDataList = dataListB
+			for _, v := range changeDataList {
+				changeDataMap[v.DataTime] = v.Value
+			}
+		} else {
+			baseDataList = dataListA
+			for _, v := range baseDataList {
+				baseDataMap[v.DataTime] = v.Value
+			}
+			changeDataList = dataListB
+			for _, v := range changeDataList {
+				changeDataMap[v.DataTime] = v.Value
+			}
+		}
+
+	}
+
+	// 计算不领先也不滞后时的相关系数
+	baseCalculateData := make([]float64, 0)
+	baseDataTimeArr := make([]string, 0)
+	for i := range baseDataList {
+		baseDataTimeArr = append(baseDataTimeArr, baseDataList[i].DataTime)
+		baseCalculateData = append(baseCalculateData, baseDataList[i].Value)
+	}
+
+	//zeroBaseData := make([]float64, 0)
+	//zeroCalculateData := make([]float64, 0)
+	//for i := range baseDataTimeArr {
+	//	tmpBaseVal, ok1 := baseDataMap[baseDataTimeArr[i]]
+	//	tmpCalculateVal, ok2 := changeDataMap[baseDataTimeArr[i]]
+	//	if ok1 && ok2 {
+	//		zeroBaseData = append(zeroBaseData, tmpBaseVal)
+	//		zeroCalculateData = append(zeroCalculateData, tmpCalculateVal)
+	//	}
+	//}
+	//if len(zeroBaseData) != len(zeroCalculateData) {
+	//	err = fmt.Errorf("相关系数两组序列元素数不一致, %d-%d", len(baseCalculateData), len(zeroCalculateData))
+	//	return
+	//}
+	//zeroRatio := utils.CalculateCorrelationByIntArr(zeroBaseData, zeroCalculateData)
+	//if leadValue == 0 {
+	//	yData = append(yData, zeroRatio)
+	//}
+
+	// 计算领先/滞后N期
+	if leadValue > 0 {
+		// 平移变频指标领先/滞后的日期(单位天)
+		moveUnitDays := utils.FrequencyDaysMap[leadUnit]
+
+		for i := range xData {
+			//if xData[i] == 0 {
+			//	yData = append(yData, zeroRatio)
+			//	continue
+			//}
+			xCalculateData := make([]float64, 0)
+			yCalculateData := make([]float64, 0)
+
+			// 平移指定天数
+			mDays := int(moveUnitDays) * xData[i]
+			_, dMap := models.MoveDataDaysToNewDataList(changeDataList, mDays)
+
+			// 取出对应的基准日期的值
+			for i2 := range baseDataTimeArr {
+				tmpDate := baseDataTimeArr[i2]
+				if yVal, ok := dMap[tmpDate]; ok {
+					xCalculateData = append(xCalculateData, baseCalculateData[i2])
+					yCalculateData = append(yCalculateData, yVal)
+				}
+			}
+			if len(yCalculateData) <= 0 {
+				//err = fmt.Errorf("领先滞后相关系数两组序列元素数不一致, %d-%d", len(baseCalculateData), len(yCalculateData))
+				//return
+				// 领先滞后后，没有可以计算的数据了
+				continue
+			}
+
+			// 公式计算出领先/滞后频度对应点的相关性系数
+			ratio := utils.CalculateCorrelationByIntArr(xCalculateData, yCalculateData)
+			yData = append(yData, ratio)
+		}
+	}
+
+	xEdbIdValue = xData
+	yDataList = make([]YData, 0)
+	yDate := "0000-00-00"
+	yDataList = append(yDataList, YData{
+		Date:  yDate,
+		Value: yData,
+	})
+	return
+}

models/base_calculate.go

@@ -1851,3 +1851,35 @@ func GetDateDataAndDateList(dateDataMap map[time.Time]float64) (dateStrDataMap m
 
 	return
 }
+
+// StepCalculateBySearchData
+// @Description: 分步骤计算
+type StepCalculateBySearchData struct {
+	DataList   []*EdbInfoSearchData `description:"基础数据"`
+	Calculates []CalculatesReq
+}
+
+type CalculatesReq struct {
+	Formula       interface{}
+	Calendar      string `description:"公历/农历"`
+	Frequency     string `description:"需要转换的频度"`
+	MoveType      int    `description:"移动方式:1:领先(默认),2:滞后"`
+	MoveFrequency string `description:"移动频度"`
+	FromFrequency string `description:"来源的频度"`
+	Source        int    `description:"1：累计值转月；2：累计值转季；3：同比值；4：同差值；5：N数值移动平均数计算；6：环比值；7：环差值；8：升频；9：降频；10：时间移位；11：超季节性；12：年化；13：累计值；14：累计值年初至今；15：指数修匀；16：日均值"`
+	Sort          int    `description:"计算顺序"`
+}
+
+func TransDateData2EdbData(dateData map[time.Time]float64) (edbData []*EdbInfoData) {
+	edbData = make([]*EdbInfoData, 0)
+	for d, v := range dateData {
+		edbData = append(edbData, &EdbInfoData{
+			DataTime: d,
+			Value:    v,
+		})
+	}
+	sort.Slice(edbData, func(i, j int) bool {
+		return edbData[i].DataTime.Before(edbData[j].DataTime)
+	})
+	return
+}

models/base_from_business.go

@@ -211,8 +211,33 @@ func (m *EdbBusinessSource) Add() (err error) {
 	return
 }
 
-// GetEdbInfoMaxAndMinInfo 获取指标的最新数据记录信息
+// GetEdbInfoMaxAndMinInfo
+// @Description: 获取指标的最新数据记录信息
+// @author: Roc
+// @receiver m
+// @datetime 2024-07-02 14:50:50
+// @param edbCode string
+// @return item *EdbInfoMaxAndMinInfo
+// @return err error
 func (m BaseFromBusinessIndex) GetEdbInfoMaxAndMinInfo(edbCode string) (item *EdbInfoMaxAndMinInfo, err error) {
+	if utils.UseMongo {
+		return m.getEdbInfoMaxAndMinInfoByMongo(edbCode)
+	} else {
+		return m.getEdbInfoMaxAndMinInfoByMysql(edbCode)
+	}
+
+	return
+}
+
+// getEdbInfoMaxAndMinInfoByMongo
+// @Description: 获取指标的最新数据记录信息（从mongo中获取）
+// @author: Roc
+// @receiver m
+// @datetime 2024-07-02 14:41:20
+// @param edbCode string
+// @return item *EdbInfoMaxAndMinInfo
+// @return err error
+func (m BaseFromBusinessIndex) getEdbInfoMaxAndMinInfoByMongo(edbCode string) (item *EdbInfoMaxAndMinInfo, err error) {
 	mogDataObj := new(mgo.BaseFromBusinessData)
 	pipeline := []bson.M{
 		{"$match": bson.M{"index_code": edbCode}},
@@ -256,6 +281,34 @@ func (m BaseFromBusinessIndex) GetEdbInfoMaxAndMinInfo(edbCode string) (item *Ed
 	return
 }
 
+// getEdbInfoMaxAndMinInfoByMysql
+// @Description: 获取指标的最新数据记录信息（从mysql中获取）
+// @author: Roc
+// @receiver m
+// @datetime 2024-07-02 14:49:58
+// @param edbCode string
+// @return item *EdbInfoMaxAndMinInfo
+// @return err error
+func (m BaseFromBusinessIndex) getEdbInfoMaxAndMinInfoByMysql(edbCode string) (item *EdbInfoMaxAndMinInfo, err error) {
+	dataObj := BaseFromBusinessData{}
+	result, err := dataObj.GetEdbInfoMaxAndMinInfo(edbCode)
+	if err != nil {
+		return
+	}
+
+	item = &EdbInfoMaxAndMinInfo{
+		MinDate:     result.MinDate,
+		MaxDate:     result.MaxDate,
+		MinValue:    result.MinValue,
+		MaxValue:    result.MaxValue,
+		LatestValue: result.LatestValue,
+		LatestDate:  result.LatestDate,
+		EndValue:    result.EndValue,
+	}
+
+	return
+}
+
 // ModifyIndexMaxAndMinInfo
 // @Description: 修改最大值和最小值信息
 // @author: Roc

models/base_from_business_data.go

@@ -0,0 +1,306 @@
+package models
+
+import (
+	"errors"
+	"eta/eta_index_lib/utils"
+	"fmt"
+	"github.com/beego/beego/v2/client/orm"
+	"strings"
+	"time"
+)
+
+// BaseFromBusinessData
+// @Description: 外部指标（商家系统）原始数据表
+type BaseFromBusinessData struct {
+	BusinessDataId          int       `orm:"column(business_data_id);pk" json:"business_data_id"`
+	BaseFromBusinessIndexId int       `json:"base_from_business_index_id"` // 指标id
+	IndexCode               string    `json:"index_code"`                  // 指标编码
+	DataTime                time.Time `json:"data_time"`                   // 数据日期
+	Value                   float64   `json:"value"`                       // 数据值
+	CreateTime              time.Time `json:"create_time"`                 // 创建时间
+	ModifyTime              time.Time `json:"modify_time"`                 // 修改时间
+}
+
+// TableName
+// @Description:  获取表名
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-26 13:41:36
+// @return string
+func (m *BaseFromBusinessData) TableName() string {
+	return "base_from_business_data"
+}
+
+// CollectionName
+// @Description:  获取集合名称
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-26 13:41:36
+// @return string
+func (m *BaseFromBusinessData) CollectionName() string {
+	return "base_from_business_data"
+}
+
+// DataBaseName
+// @Description: 获取数据库名称
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-26 13:41:33
+// @return string
+func (m *BaseFromBusinessData) DataBaseName() string {
+	return utils.MgoDataDbName
+}
+
+type WhereParams struct {
+	Condition string
+	Pars      []interface{}
+	Order     string `description:"排序字段"`
+}
+
+// GetAllDataList
+// @Description: 根据条件获取所有数据
+// @author: Roc
+// @receiver m
+// @datetime 2024-07-01 17:18:57
+// @param condition []string
+// @param pars []interface{}
+// @param order string
+// @return result []*BaseFromBusinessData
+// @return err error
+func (m *BaseFromBusinessData) GetAllDataList(condition []string, pars []interface{}, order string) (result []*BaseFromBusinessData, err error) {
+	o := orm.NewOrm()
+
+	sql := `SELECT * FROM base_from_business_data `
+	if len(condition) > 0 {
+		sql += ` WHERE ` + strings.Join(condition, " AND ")
+	}
+
+	if order != `` {
+		sql += ` ORDER BY ` + order
+	}
+
+	_, err = o.Raw(sql, pars).QueryRows(&result)
+
+	return
+}
+
+// GetLimitDataList
+// @Description: 根据条件获取指定数量数据列表
+// @author: Roc
+// @receiver m
+// @datetime 2024-07-01 17:19:16
+// @param condition []string
+// @param pars []interface{}
+// @param order string
+// @param size int64
+// @return result []*BaseFromBusinessData
+// @return err error
+func (m *BaseFromBusinessData) GetLimitDataList(condition []string, pars []interface{}, order string, size int64) (result []*BaseFromBusinessData, err error) {
+	o := orm.NewOrm()
+	sql := `SELECT * FROM base_from_business_data `
+	if len(condition) > 0 {
+		sql += ` WHERE ` + strings.Join(condition, " AND ")
+	}
+
+	if order != `` {
+		sql += ` ORDER BY ` + order
+	}
+
+	sql += fmt.Sprintf(` LIMIT %d`, size)
+
+	_, err = o.Raw(sql, pars).QueryRows(&result)
+
+	return
+}
+
+// GetPageDataList
+// @Description: 根据条件获取分页数据列表
+// @author: Roc
+// @receiver m
+// @datetime 2024-07-01 17:19:42
+// @param condition []string
+// @param pars []interface{}
+// @param order string
+// @param startSize int64
+// @param size int64
+// @return result []*BaseFromBusinessData
+// @return err error
+func (m *BaseFromBusinessData) GetPageDataList(condition []string, pars []interface{}, order string, startSize, size int64) (result []*BaseFromBusinessData, err error) {
+	o := orm.NewOrm()
+	sql := `SELECT * FROM base_from_business_data `
+	if len(condition) > 0 {
+		sql += ` WHERE ` + strings.Join(condition, " AND ")
+	}
+
+	if order != `` {
+		sql += ` ORDER BY ` + order
+	}
+
+	sql += fmt.Sprintf(` LIMIT %d,%d`, startSize, size)
+
+	_, err = o.Raw(sql, pars).QueryRows(&result)
+
+	return
+}
+
+// GetCountDataList
+// @Description: 根据条件获取数据列表总数
+// @author: Roc
+// @receiver m
+// @datetime 2024-07-01 17:19:50
+// @param condition []string
+// @param pars []interface{}
+// @return count int64
+// @return err error
+func (m *BaseFromBusinessData) GetCountDataList(condition []string, pars []interface{}) (count int64, err error) {
+	o := orm.NewOrm()
+	sql := `SELECT COUNT(1) FROM base_from_business_data `
+	if len(condition) > 0 {
+		sql += ` WHERE ` + strings.Join(condition, " AND ")
+	}
+	err = o.Raw(sql, pars).QueryRow(&count)
+
+	return
+}
+
+// InsertDataByColl
+// @Description: 写入单条数据（外部传入集合）
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-26 14:22:18
+// @param addData interface{}
+// @return err error
+func (m *BaseFromBusinessData) InsertDataByColl(addData interface{}) (err error) {
+	o := orm.NewOrm()
+	_, err = o.Insert(addData)
+
+	return
+}
+
+// BatchInsertData
+// @Description: 批量写入数据
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-26 14:22:18
+// @param bulk int 每次请求保存的数据量
+// @param dataList []*BaseFromBusinessData
+// @return err error
+func (m *BaseFromBusinessData) BatchInsertData(bulk int, dataList []*BaseFromBusinessData) (err error) {
+	o := orm.NewOrm()
+	_, err = o.InsertMulti(bulk, dataList)
+
+	return
+}
+
+// UpdateData
+// @Description: 单条数据修改
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-26 15:01:51
+// @param whereParams interface{}
+// @param updateParams interface{}
+// @return err error
+func (m *BaseFromBusinessData) UpdateData(updateCols []string) (err error) {
+	o := orm.NewOrm()
+	_, err = o.Update(m, updateCols...)
+	if err != nil {
+		fmt.Println("UpdateDataByColl:Err:" + err.Error())
+		return
+	}
+
+	return
+}
+
+// HandleData
+// @Description: 数据处理
+// @author: Roc
+// @receiver m
+// @datetime 2024-07-01 17:35:54
+// @param addDataList []*BaseFromBusinessData
+// @param updateDataList []*BaseFromBusinessData
+// @return err error
+func (m *BaseFromBusinessData) HandleData(addDataList, updateDataList []*BaseFromBusinessData) (err error) {
+	o := orm.NewOrm()
+	to, err := o.Begin()
+	if err != nil {
+		return
+	}
+	defer func() {
+		if err != nil {
+			fmt.Println("BaseFromBusinessData HandleData,Err:" + err.Error())
+			_ = to.Rollback()
+		} else {
+			_ = to.Commit()
+		}
+	}()
+
+	// 插入数据
+	if len(addDataList) > 0 {
+		_, err = to.InsertMulti(500, addDataList)
+		if err != nil {
+			return
+		}
+	}
+
+	// 修改
+
+	if len(updateDataList) > 0 {
+		for _, v := range updateDataList {
+			_, err = to.Update(v, "Value", "ModifyTime")
+			if err != nil {
+				fmt.Println("BaseFromBusinessData HandleData Update:Err:" + err.Error())
+				return
+			}
+		}
+	}
+
+	return
+}
+
+// GetEdbInfoMaxAndMinInfo
+// @Description: 获取当前指标的最大最小值
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-30 17:15:39
+// @param whereParams interface{}
+// @return result EdbInfoMaxAndMinInfo
+// @return err error
+func (m *BaseFromBusinessData) GetEdbInfoMaxAndMinInfo(indexCode string) (result EdbInfoMaxAndMinInfo, err error) {
+	o := orm.NewOrm()
+	sql := ``
+	sql = ` SELECT MIN(data_time) AS min_date,MAX(data_time) AS max_date,MIN(value) AS min_value,MAX(value) AS max_value FROM base_from_business_data WHERE index_code = ? `
+	err = o.Raw(sql, indexCode).QueryRow(&result)
+	if err != nil {
+		return
+	}
+
+	var latestValue float64
+	sql = ` SELECT value AS latest_value FROM base_from_business_data WHERE index_code = ? ORDER BY data_time DESC LIMIT 1 `
+	err = o.Raw(sql, indexCode).QueryRow(&latestValue)
+	result.LatestValue = latestValue
+
+	return
+}
+
+// DelDataByCond
+// @Description: 根据条件删除多条数据
+// @author: Roc
+// @receiver m
+// @datetime 2024-07-01 17:46:56
+// @param condition []string
+// @param pars []interface{}
+// @return err error
+func (m *BaseFromBusinessData) DelDataByCond(condition []string, pars []interface{}) (err error) {
+	if len(condition) <= 0 {
+		err = errors.New("条件不能为空")
+		return
+	}
+	o := orm.NewOrm()
+	sql := `DELETE FROM base_from_business_data `
+
+	sql += ` WHERE ` + strings.Join(condition, " AND ")
+
+	_, err = o.Raw(sql, pars).Exec()
+
+	return
+}

models/base_from_calculate.go

@@ -490,7 +490,7 @@ func refreshAllCalculate(to orm.TxOrmer, edbInfoIdArr []*EdbInfo, edbInfoTag map
 		}
 		// 有计算出来值，那么就从待删除指标中移除
 		delete(removeDateMap, sk)
-		saveValue := decimal.NewFromFloat(calVal).RoundCeil(4).String() //utils.SubFloatToString(calVal, 4)
+		saveValue := decimal.NewFromFloat(calVal).Round(4).String() //utils.SubFloatToString(calVal, 4)
 		if existVal, ok := dataMap[sk]; !ok {
 			dataTime, _ := time.ParseInLocation(utils.FormatDate, sk, time.Local)
 			timestamp := dataTime.UnixNano() / 1e6

models/base_from_edb_mapping.go

@@ -0,0 +1,162 @@
+package models
+
+import (
+	"eta/eta_index_lib/utils"
+	"fmt"
+	"github.com/beego/beego/v2/client/orm"
+	"strings"
+	"time"
+)
+
+// BaseFromEdbMapping 同花顺高频数据
+type BaseFromEdbMapping struct {
+	Id              int       `orm:"column(id);pk"`
+	BaseFromIndexId int       `description:"源指标ID"`
+	BaseIndexCode   string    `description:"源指标编码"`
+	EdbInfoId       int       `description:"指标ID"`
+	EdbCode         string    `description:"指标编码"`
+	Source          int       `description:"指标来源：1-同花顺..."`
+	SubSource       int       `description:"子数据来源：0-经济数据库；1-日期序列；2-高频数据"`
+	ConvertRule     string    `description:"转换规则"`
+	CreateTime      time.Time `description:"创建时间"`
+	ModifyTime      time.Time `description:"修改时间"`
+}
+
+func (m *BaseFromEdbMapping) TableName() string {
+	return "base_from_edb_mapping"
+}
+
+type BaseFromEdbMappingCols struct {
+	PrimaryId       string
+	BaseFromIndexId string
+	BaseIndexCode   string
+	EdbInfoId       string
+	EdbCode         string
+	Source          string
+	SubSource       string
+	ConvertRule     string
+	CreateTime      string
+	ModifyTime      string
+}
+
+func (m *BaseFromEdbMapping) Cols() BaseFromEdbMappingCols {
+	return BaseFromEdbMappingCols{
+		PrimaryId:       "id",
+		BaseFromIndexId: "base_from_index_id",
+		BaseIndexCode:   "base_index_code",
+		EdbInfoId:       "edb_info_id",
+		EdbCode:         "edb_code",
+		Source:          "source",
+		SubSource:       "sub_source",
+		ConvertRule:     "convert_rule",
+		CreateTime:      "create_time",
+		ModifyTime:      "modify_time",
+	}
+}
+
+func (m *BaseFromEdbMapping) Create() (err error) {
+	o := orm.NewOrm()
+	id, err := o.Insert(m)
+	if err != nil {
+		return
+	}
+	m.Id = int(id)
+	return
+}
+
+func (m *BaseFromEdbMapping) CreateMulti(items []*BaseFromEdbMapping) (err error) {
+	if len(items) == 0 {
+		return
+	}
+	o := orm.NewOrm()
+	_, err = o.InsertMulti(len(items), items)
+	return
+}
+
+func (m *BaseFromEdbMapping) Update(cols []string) (err error) {
+	o := orm.NewOrm()
+	_, err = o.Update(m, cols...)
+	return
+}
+
+func (m *BaseFromEdbMapping) Remove() (err error) {
+	o := orm.NewOrm()
+	sql := fmt.Sprintf(`DELETE FROM %s WHERE %s = ? LIMIT 1`, m.TableName(), m.Cols().PrimaryId)
+	_, err = o.Raw(sql, m.Id).Exec()
+	return
+}
+
+func (m *BaseFromEdbMapping) MultiRemove(ids []int) (err error) {
+	if len(ids) == 0 {
+		return
+	}
+	o := orm.NewOrm()
+	sql := fmt.Sprintf(`DELETE FROM %s WHERE %s IN (%s)`, m.TableName(), m.Cols().PrimaryId, utils.GetOrmInReplace(len(ids)))
+	_, err = o.Raw(sql, ids).Exec()
+	return
+}
+
+func (m *BaseFromEdbMapping) RemoveByCondition(condition string, pars []interface{}) (err error) {
+	if condition == "" {
+		return
+	}
+	o := orm.NewOrm()
+	sql := fmt.Sprintf(`DELETE FROM %s WHERE %s`, m.TableName(), condition)
+	_, err = o.Raw(sql, pars).Exec()
+	return
+}
+
+func (m *BaseFromEdbMapping) GetItemById(id int) (item *BaseFromEdbMapping, err error) {
+	o := orm.NewOrm()
+	sql := fmt.Sprintf(`SELECT * FROM %s WHERE %s = ? LIMIT 1`, m.TableName(), m.Cols().PrimaryId)
+	err = o.Raw(sql, id).QueryRow(&item)
+	return
+}
+
+func (m *BaseFromEdbMapping) GetItemByCondition(condition string, pars []interface{}, orderRule string) (item *BaseFromEdbMapping, err error) {
+	o := orm.NewOrm()
+	order := ``
+	if orderRule != "" {
+		order = ` ORDER BY ` + orderRule
+	}
+	sql := fmt.Sprintf(`SELECT * FROM %s WHERE 1=1 %s %s LIMIT 1`, m.TableName(), condition, order)
+	err = o.Raw(sql, pars).QueryRow(&item)
+	return
+}
+
+func (m *BaseFromEdbMapping) GetCountByCondition(condition string, pars []interface{}) (count int, err error) {
+	o := orm.NewOrm()
+	sql := fmt.Sprintf(`SELECT COUNT(1) FROM %s WHERE 1=1 %s`, m.TableName(), condition)
+	err = o.Raw(sql, pars).QueryRow(&count)
+	return
+}
+
+func (m *BaseFromEdbMapping) GetItemsByCondition(condition string, pars []interface{}, fieldArr []string, orderRule string) (items []*BaseFromEdbMapping, err error) {
+	o := orm.NewOrm()
+	fields := strings.Join(fieldArr, ",")
+	if len(fieldArr) == 0 {
+		fields = `*`
+	}
+	order := fmt.Sprintf(`ORDER BY %s DESC`, m.Cols().CreateTime)
+	if orderRule != "" {
+		order = ` ORDER BY ` + orderRule
+	}
+	sql := fmt.Sprintf(`SELECT %s FROM %s WHERE 1=1 %s %s`, fields, m.TableName(), condition, order)
+	_, err = o.Raw(sql, pars).QueryRows(&items)
+	return
+}
+
+func (m *BaseFromEdbMapping) GetPageItemsByCondition(condition string, pars []interface{}, fieldArr []string, orderRule string, startSize, pageSize int) (items []*BaseFromEdbMapping, err error) {
+	o := orm.NewOrm()
+	fields := strings.Join(fieldArr, ",")
+	if len(fieldArr) == 0 {
+		fields = `*`
+	}
+	order := fmt.Sprintf(`ORDER BY %s DESC`, m.Cols().CreateTime)
+	if orderRule != "" {
+		order = ` ORDER BY ` + orderRule
+	}
+	sql := fmt.Sprintf(`SELECT %s FROM %s WHERE 1=1 %s %s LIMIT ?,?`, fields, m.TableName(), condition, order)
+	_, err = o.Raw(sql, pars, startSize, pageSize).QueryRows(&items)
+	return
+}

models/base_from_mysteel_chemical.go

@@ -243,6 +243,21 @@ func (m *BaseFromMysteelChemicalIndex) GetIndexItem(indexCode string) (item *Bas
 	return
 }
 
+func (m *BaseFromMysteelChemicalIndex) GetBatchIndexItem(indexCodes []string) (items []*BaseFromMysteelChemicalIndex, err error) {
+	if len(indexCodes) <= 0 {
+		return
+	}
+	o := orm.NewOrm()
+	sql := `SELECT * FROM base_from_mysteel_chemical_index WHERE index_code IN (%s) `
+	holder := make([]string, 0, len(indexCodes))
+	for range indexCodes {
+		holder = append(holder, "?")
+	}
+	sql = fmt.Sprintf(sql, strings.Join(holder, ","))
+	_, err = o.Raw(sql, indexCodes).QueryRows(&items)
+	return
+}
+
 func (m *BaseFromMysteelChemicalIndex) GetIndexCreate(terminalCode string) (items []*BaseFromMysteelChemicalIndex, err error) {
 	o := orm.NewOrm()
 	sql := `SELECT * FROM base_from_mysteel_chemical_index WHERE index_name = '' AND terminal_code = ? `
@@ -445,6 +460,16 @@ func (r *BaseFromMysteelChemicalData) Add(list []BaseFromMysteelChemicalData) (e
 	return
 }
 
+// AddV2 新增
+func (r *BaseFromMysteelChemicalData) AddV2(list []*BaseFromMysteelChemicalData) (err error) {
+	if len(list) == 0 {
+		return
+	}
+	o := orm.NewOrm()
+	_, err = o.InsertMulti(500, list)
+	return
+}
+
 type AddMysteelIndexResp struct {
 	EdbCode                           string `description:"指标编码"`
 	TerminalCode                      string `description:"指标终端编码"`
@@ -536,3 +561,43 @@ func (m *BaseFromMysteelChemicalIndex) GetIndexByCondition(condition string, par
 	_, err = o.Raw(sql, pars).QueryRows(&items)
 	return
 }
+
+type MySteelChemicalApiResp struct {
+	Code      string                    `json:"code" description:"200成功，其他失败"`
+	Success   bool                      `json:"success" description:"true 成功，false 失败"`
+	Timestamp int64                     `json:"timestamp" description:"时间戳"`
+	Message   string                    `json:"message" description:"显示执行信息"`
+	Data      []*MySteelChemicalApiData `json:"data" description:"数据"`
+}
+type MySteelChemicalApiInfoResp struct {
+	Code      string                  `json:"code" description:"200成功，其他失败"`
+	Success   bool                    `json:"success" description:"true 成功，false 失败"`
+	Timestamp int64                   `json:"timestamp" description:"时间戳"`
+	Message   string                  `json:"message" description:"显示执行信息"`
+	Data      *MySteelChemicalApiInfo `json:"data" description:"数据"`
+}
+
+type MySteelChemicalApiInfo struct {
+	Total int                           `json:"total" description:"总条数"`
+	Pages int                           `json:"pages" description:"总页数"`
+	List  []*MySteelChemicalApiInfoItem `json:"list" description:"数据列表"`
+}
+
+type MySteelChemicalApiInfoItem struct {
+	IndexCode     string `json:"INDEX_CODE"`
+	IndexName     string `json:"INDEX_NAME"`
+	FrequencyName string `json:"FREQUENCY_NAME"`
+	UnitName      string `json:"UNIT_NAME"`
+}
+
+type MySteelChemicalApiData struct {
+	IndexCode string                        `json:"INDEX_CODE"`
+	DataList  []*MySteelChemicalApiDataList `json:"dataList"`
+}
+
+type MySteelChemicalApiDataList struct {
+	PublishTime int64   `json:"PUBLISH_TIME"`
+	IndexCode   string  `json:"INDEX_CODE"`
+	DataDate    string  `json:"DATA_DATE"`
+	DataValue   float64 `json:"DATA_VALUE"`
+}

models/base_from_ths_hf.go

@@ -0,0 +1,476 @@
+package models
+
+import (
+	"eta/eta_index_lib/models/mgo"
+	"eta/eta_index_lib/utils"
+	"fmt"
+	"github.com/beego/beego/v2/client/orm"
+	"go.mongodb.org/mongo-driver/bson"
+	"strings"
+	"time"
+)
+
+var (
+	ThsHfPeriodArr = []int{1, 3, 5, 10, 15, 30, 60}                                                                                     // 时间周期
+	ThsHfCPSArr    = []string{"no", "forward1", "forward2", "forward3", "forward4", "backward1", "backward2", "backward3", "backward4"} // 复权方式
+	ThsHfFillArr   = []string{"Original", "Previous", "Blank"}                                                                          // 非交易间隔处理
+)
+
+// BaseFromThsHfIndex 同花顺高频数据
+type BaseFromThsHfIndex struct {
+	BaseFromThsHfIndexId    int       `orm:"column(base_from_ths_hf_index_id);pk"`
+	BaseFromThsHfClassifyId int       `description:"分类ID"`
+	IndexCode               string    `description:"指标编码"`
+	IndexName               string    `description:"指标名称"`
+	Unit                    string    `description:"单位"`
+	Source                  string    `description:"数据来源"`
+	Frequency               string    `description:"频度"`
+	StartDate               time.Time `description:"开始日期（至时分秒）"`
+	EndDate                 time.Time `description:"结束日期（至时分秒）"`
+	Describe                string    `description:"指标描述"`
+	Sort                    int       `description:"排序"`
+	IsStop                  int       `description:"是否停更：0-否；1-停更"`
+	TerminalCode            string    `description:"所属终端编码"`
+	StockCode               string    `description:"证券代码"`
+	Indicator               string    `description:"同花顺指标代码"`
+	ApiPars                 string    `description:"API请求参数"`
+	LatestValue             float64   `description:"最新值"`
+	SysUserId               int       `description:"创建人ID"`
+	SysUserRealName         string    `description:"创建人姓名"`
+	CreateTime              time.Time `description:"创建时间"`
+	ModifyTime              time.Time `description:"修改时间"`
+}
+
+func (m *BaseFromThsHfIndex) TableName() string {
+	return "base_from_ths_hf_index"
+}
+
+type BaseFromThsHfIndexCols struct {
+	PrimaryId               string
+	BaseFromThsHfClassifyId string
+	IndexCode               string
+	IndexName               string
+	Unit                    string
+	Source                  string
+	Frequency               string
+	StartDate               string
+	EndDate                 string
+	Describe                string
+	Sort                    string
+	IsStop                  string
+	TerminalCode            string
+	StockCode               string
+	Indicator               string
+	ApiPars                 string
+	LatestValue             string
+	SysUserId               string
+	SysUserRealName         string
+	CreateTime              string
+	ModifyTime              string
+}
+
+func (m *BaseFromThsHfIndex) Cols() BaseFromThsHfIndexCols {
+	return BaseFromThsHfIndexCols{
+		PrimaryId:               "base_from_ths_hf_index_id",
+		BaseFromThsHfClassifyId: "base_from_ths_hf_classify_id",
+		IndexCode:               "index_code",
+		IndexName:               "index_name",
+		Unit:                    "unit",
+		Source:                  "source",
+		Frequency:               "frequency",
+		StartDate:               "start_date",
+		EndDate:                 "end_date",
+		Describe:                "describe",
+		Sort:                    "sort",
+		IsStop:                  "is_stop",
+		TerminalCode:            "terminal_code",
+		StockCode:               "stock_code",
+		Indicator:               "indicator",
+		ApiPars:                 "api_pars",
+		LatestValue:             "latest_value",
+		SysUserId:               "sys_user_id",
+		SysUserRealName:         "sys_user_real_name",
+		CreateTime:              "create_time",
+		ModifyTime:              "modify_time",
+	}
+}
+
+func (m *BaseFromThsHfIndex) Create() (err error) {
+	o := orm.NewOrm()
+	id, err := o.Insert(m)
+	if err != nil {
+		return
+	}
+	m.BaseFromThsHfIndexId = int(id)
+	return
+}
+
+func (m *BaseFromThsHfIndex) CreateMulti(items []*BaseFromThsHfIndex) (err error) {
+	if len(items) == 0 {
+		return
+	}
+	o := orm.NewOrm()
+	_, err = o.InsertMulti(len(items), items)
+	return
+}
+
+func (m *BaseFromThsHfIndex) Update(cols []string) (err error) {
+	o := orm.NewOrm()
+	_, err = o.Update(m, cols...)
+	return
+}
+
+func (m *BaseFromThsHfIndex) Remove() (err error) {
+	o := orm.NewOrm()
+	sql := fmt.Sprintf(`DELETE FROM %s WHERE %s = ? LIMIT 1`, m.TableName(), m.Cols().PrimaryId)
+	_, err = o.Raw(sql, m.BaseFromThsHfIndexId).Exec()
+	return
+}
+
+func (m *BaseFromThsHfIndex) MultiRemove(ids []int) (err error) {
+	if len(ids) == 0 {
+		return
+	}
+	o := orm.NewOrm()
+	sql := fmt.Sprintf(`DELETE FROM %s WHERE %s IN (%s)`, m.TableName(), m.Cols().PrimaryId, utils.GetOrmInReplace(len(ids)))
+	_, err = o.Raw(sql, ids).Exec()
+	return
+}
+
+func (m *BaseFromThsHfIndex) RemoveByCondition(condition string, pars []interface{}) (err error) {
+	if condition == "" {
+		return
+	}
+	o := orm.NewOrm()
+	sql := fmt.Sprintf(`DELETE FROM %s WHERE %s`, m.TableName(), condition)
+	_, err = o.Raw(sql, pars).Exec()
+	return
+}
+
+func (m *BaseFromThsHfIndex) GetItemById(id int) (item *BaseFromThsHfIndex, err error) {
+	o := orm.NewOrm()
+	sql := fmt.Sprintf(`SELECT * FROM %s WHERE %s = ? LIMIT 1`, m.TableName(), m.Cols().PrimaryId)
+	err = o.Raw(sql, id).QueryRow(&item)
+	return
+}
+
+func (m *BaseFromThsHfIndex) GetItemByCondition(condition string, pars []interface{}, orderRule string) (item *BaseFromThsHfIndex, err error) {
+	o := orm.NewOrm()
+	order := ``
+	if orderRule != "" {
+		order = ` ORDER BY ` + orderRule
+	}
+	sql := fmt.Sprintf(`SELECT * FROM %s WHERE 1=1 %s %s LIMIT 1`, m.TableName(), condition, order)
+	err = o.Raw(sql, pars).QueryRow(&item)
+	return
+}
+
+func (m *BaseFromThsHfIndex) GetCountByCondition(condition string, pars []interface{}) (count int, err error) {
+	o := orm.NewOrm()
+	sql := fmt.Sprintf(`SELECT COUNT(1) FROM %s WHERE 1=1 %s`, m.TableName(), condition)
+	err = o.Raw(sql, pars).QueryRow(&count)
+	return
+}
+
+func (m *BaseFromThsHfIndex) GetItemsByCondition(condition string, pars []interface{}, fieldArr []string, orderRule string) (items []*BaseFromThsHfIndex, err error) {
+	o := orm.NewOrm()
+	fields := strings.Join(fieldArr, ",")
+	if len(fieldArr) == 0 {
+		fields = `*`
+	}
+	order := fmt.Sprintf(`ORDER BY %s DESC`, m.Cols().CreateTime)
+	if orderRule != "" {
+		order = ` ORDER BY ` + orderRule
+	}
+	sql := fmt.Sprintf(`SELECT %s FROM %s WHERE 1=1 %s %s`, fields, m.TableName(), condition, order)
+	_, err = o.Raw(sql, pars).QueryRows(&items)
+	return
+}
+
+func (m *BaseFromThsHfIndex) GetPageItemsByCondition(condition string, pars []interface{}, fieldArr []string, orderRule string, startSize, pageSize int) (items []*BaseFromThsHfIndex, err error) {
+	o := orm.NewOrm()
+	fields := strings.Join(fieldArr, ",")
+	if len(fieldArr) == 0 {
+		fields = `*`
+	}
+	order := fmt.Sprintf(`ORDER BY %s DESC`, m.Cols().CreateTime)
+	if orderRule != "" {
+		order = ` ORDER BY ` + orderRule
+	}
+	sql := fmt.Sprintf(`SELECT %s FROM %s WHERE 1=1 %s %s LIMIT ?,?`, fields, m.TableName(), condition, order)
+	_, err = o.Raw(sql, pars, startSize, pageSize).QueryRows(&items)
+	return
+}
+
+// BaseFromThsHfIndexItem 同花顺高频数据信息
+type BaseFromThsHfIndexItem struct {
+	IndexId    int    `description:"同花顺高频数据ID"`
+	IndexCode  string `description:"指标编码"`
+	IndexName  string `description:"指标名称"`
+	Unit       string `description:"单位"`
+	Source     string `description:"数据来源"`
+	Frequency  string `description:"频度"`
+	StartDate  string `description:"开始日期（至时分秒）"`
+	EndDate    string `description:"结束日期（至时分秒）"`
+	Describe   string `description:"指标描述"`
+	Sort       int    `description:"排序"`
+	CreateTime string `description:"创建时间"`
+	ModifyTime string `description:"修改时间"`
+}
+
+func (m *BaseFromThsHfIndex) Format2Item() (item *BaseFromThsHfIndexItem) {
+	item = new(BaseFromThsHfIndexItem)
+	item.IndexId = m.BaseFromThsHfIndexId
+	item.IndexCode = m.IndexCode
+	item.IndexName = m.IndexName
+	item.Unit = m.Unit
+	item.Source = m.Source
+	item.Frequency = m.Frequency
+	item.StartDate = utils.TimeTransferString(utils.FormatDateTime, m.StartDate)
+	item.EndDate = utils.TimeTransferString(utils.FormatDateTime, m.EndDate)
+	item.Describe = m.Describe
+	item.Sort = m.Sort
+	item.CreateTime = utils.TimeTransferString(utils.FormatDateTime, m.CreateTime)
+	item.ModifyTime = utils.TimeTransferString(utils.FormatDateTime, m.ModifyTime)
+	return
+}
+
+// ThsHfSearchEdbReq 搜索指标请求体
+type ThsHfSearchEdbReq struct {
+	StockCode string `form:"StockCode" description:"证券代码" `
+	EdbCode   string `form:"EdbCode" description:"指标代码"`
+	StartTime string `form:"StartTime" description:"每日数据开始时间"`
+	EndTime   string `form:"EndTime" description:"每日数据结束时间"`
+	Interval  int    `form:"Interval" description:"时间周期"`
+	Fill      string `form:"Fill" description:"非交易间隔处理"`
+	CPS       string `form:"CPS" description:"复权方式"`
+	BaseDate  string `form:"BaseDate" description:"复权基点"`
+}
+
+// ThsHfApiResp 同花顺接口响应
+type ThsHfApiResp struct {
+	ErrorCode int    `json:"errorcode"`
+	ErrMsg    string `json:"errmsg"`
+	Tables    []struct {
+		ThsCode string               `json:"thscode"`
+		Time    []string             `json:"time"`
+		Table   map[string][]float64 `json:"table"`
+	} `json:"tables"`
+}
+
+// ThsHfAppResp 同花顺公用机响应
+type ThsHfAppResp struct {
+	ErrorCode int    `json:"errorcode"`
+	ErrMsg    string `json:"errmsg"`
+	//Data      []interface{} `json:"data"`
+	Data []struct {
+		Time    string  `json:"time"`
+		ThsCode string  `json:"thscode"`
+		Open    float64 `json:"open"`
+		Close   float64 `json:"close"`
+	} `json:"data"`
+}
+
+// ThsHfIndexWithData 同花顺高频指标
+type ThsHfIndexWithData struct {
+	StockCode string            `description:"证券代码"`
+	EdbCode   string            `description:"指标代码"`
+	IndexData []*ThsHfIndexData `description:"指标数据"`
+}
+
+// ThsHfIndexData 同花顺高频指标数据
+type ThsHfIndexData struct {
+	DataTime time.Time `description:"数据时间(2006-01-02 15:04)"`
+	Value    float64   `description:"数据值"`
+}
+
+type ThsHfBaseAddIndexItem struct {
+	ClassifyId int    `description:"分类ID"`
+	Unit       string `description:"单位"`
+	IndexName  string `description:"指标名称"`
+	Frequency  string `description:"频度"`
+	StockCode  string `description:"证券代码"`
+	EdbCode    string `description:"指标代码"`
+}
+
+// ThsHfBaseAddReq 新增至数据源请求
+type ThsHfBaseAddReq struct {
+	StartTime             string `description:"每日数据开始时间"`
+	EndTime               string `description:"每日数据结束时间"`
+	Interval              int    `description:"时间周期"`
+	Fill                  string `description:"非交易间隔处理"`
+	CPS                   string `description:"复权方式"`
+	BaseDate              string `description:"复权基点"`
+	SysAdminId            int    `description:"创建人ID"`
+	SysAdminName          string `description:"创建人姓名"`
+	ThsHfBaseAddIndexItem `description:"指标信息"`
+}
+
+// CreateIndexAndData 新增指标和数据
+//func (m *BaseFromThsHfIndex) CreateIndexAndData(indexItem *BaseFromThsHfIndex, indexData []*BaseFromThsHfData) (err error) {
+//	o := orm.NewOrm()
+//	tx, e := o.Begin()
+//	if e != nil {
+//		err = fmt.Errorf("tx begin err: %v", e)
+//		return
+//	}
+//	defer func() {
+//		if err != nil {
+//			_ = tx.Rollback()
+//			return
+//		}
+//		_ = tx.Commit()
+//	}()
+//
+//	lastId, e := tx.Insert(indexItem)
+//	if e != nil {
+//		err = fmt.Errorf("insert index err: %v", e)
+//		return
+//	}
+//	indexId := int(lastId)
+//	indexItem.BaseFromThsHfIndexId = indexId
+//
+//	if len(indexData) == 0 {
+//		return
+//	}
+//	for _, v := range indexData {
+//		v.BaseFromThsHfIndexId = indexId
+//	}
+//	if _, e = tx.InsertMulti(200, indexData); e != nil {
+//		err = fmt.Errorf("insert index data err: %v", e)
+//		return
+//	}
+//	return
+//}
+
+// ThsHfBaseRefreshReq 数据源刷新请求
+type ThsHfBaseRefreshReq struct {
+	BaseIndexCode string `description:"源指标编码"`
+	RefreshType   int    `description:"刷新类型: 1-最近6小时; 2-全部刷新"`
+}
+
+// ThsHfEdbAddReq 新增至指标库
+type ThsHfEdbAddReq struct {
+	ConvertRule ThsHfIndexConvert2EdbRule
+	NewIndex    *ThsHfIndexMultiSave2EdbPreItem `description:"新增指标"`
+}
+
+type ThsHfIndexConvert2EdbRule struct {
+	ConvertType  int `description:"转换类型: 1-指定时间值; 2-区间计算值"`
+	ConvertFixed struct {
+		FixedDay  int    `description:"指定时间值日期: 1-当日; 2-前一日"`
+		FixedTime string `description:"指定时间值时点(HH:mm:ss)"`
+	} `description:"指定时间值"`
+	ConvertArea struct {
+		StartDay      int    `description:"起始时间日期: 1-当日; 2-前一日"`
+		StartTime     string `description:"起始时间时点(HH:mm:ss)"`
+		EndDay        int    `description:"截止时间日期: 1-当日; 2-前一日"`
+		EndTime       string `description:"截止时间时点(HH:mm:ss)"`
+		CalculateType int    `description:"计算类型: 1-区间均值; 2-最大值; 3-最小值"`
+	} `description:"区间计算值"`
+}
+
+// ThsHfIndexMultiSave2EdbPreItem 批量新增指标库信息
+type ThsHfIndexMultiSave2EdbPreItem struct {
+	IndexId      int    `description:"指标ID"`
+	IndexCode    string `description:"指标编码"`
+	IndexName    string `description:"原指标名称"`
+	NewIndexName string `description:"新指标名称"`
+	StockCode    string `description:"证券代码"`
+	EdbCode      string `description:"指标代码"`
+	Unit         string `description:"单位"`
+	Frequency    string `description:"原频度"`
+	NewFrequency string `description:"新频度(固定日度)"`
+	ClassifyId   int    `description:"指标库分类ID"`
+	SysAdminId   int    `description:"创建人ID"`
+	SysAdminName string `description:"创建人姓名"`
+	Tips         string `description:"提示信息"`
+	ErrMsg       string `description:"错误信息"`
+}
+
+// GetEdbInfoMaxAndMinInfo
+// @Description: 获取指标的最新数据记录信息
+// @author: Roc
+// @receiver m
+// @datetime 2024-07-02 14:50:50
+// @param edbCode string
+// @return item *EdbInfoMaxAndMinInfo
+// @return err error
+func (m BaseFromThsHfIndex) GetEdbInfoMaxAndMinInfo(edbCode string) (item *EdbInfoMaxAndMinInfo, err error) {
+	if utils.UseMongo {
+		return m.getEdbInfoMaxAndMinInfoByMongo(edbCode)
+	}
+	return m.getEdbInfoMaxAndMinInfoByMysql(edbCode)
+}
+
+// getEdbInfoMaxAndMinInfoByMongo
+// @Description: 获取指标的最新数据记录信息（从mongo中获取）
+// @author: Roc
+// @receiver m
+// @datetime 2024-07-02 14:41:20
+// @param edbCode string
+// @return item *EdbInfoMaxAndMinInfo
+// @return err error
+func (m BaseFromThsHfIndex) getEdbInfoMaxAndMinInfoByMongo(edbCode string) (item *EdbInfoMaxAndMinInfo, err error) {
+	mogDataObj := new(mgo.BaseFromThsHfData)
+	pipeline := []bson.M{
+		{"$match": bson.M{"index_code": edbCode}},
+		{"$group": bson.M{
+			"_id":       nil,
+			"min_date":  bson.M{"$min": "$data_time"},
+			"max_date":  bson.M{"$max": "$data_time"},
+			"min_value": bson.M{"$min": "$value"},
+			"max_value": bson.M{"$max": "$value"},
+		}},
+		{"$project": bson.M{"_id": 0}}, // 可选，如果不需要_id字段
+	}
+	result, err := mogDataObj.GetEdbInfoMaxAndMinInfo(pipeline)
+	if err != nil {
+		fmt.Println("BaseFromThsHfIndex GetEdbInfoMaxAndMinInfo Err:" + err.Error())
+		return
+	}
+
+	if !result.MaxDate.IsZero() {
+		whereQuery := bson.M{"index_code": edbCode, "data_time": result.MaxDate}
+		selectParam := bson.D{{"value", 1}, {"_id", 0}}
+		latestValue, tmpErr := mogDataObj.GetLatestValue(whereQuery, selectParam)
+		if tmpErr != nil {
+			err = tmpErr
+			return
+		}
+		result.LatestValue = latestValue.Value
+		result.EndValue = latestValue.Value
+	}
+
+	item = &EdbInfoMaxAndMinInfo{
+		MinDate:     result.MinDate.Format(utils.FormatDateTime),
+		MaxDate:     result.MaxDate.Format(utils.FormatDateTime),
+		MinValue:    result.MinValue,
+		MaxValue:    result.MaxValue,
+		LatestValue: result.LatestValue,
+		LatestDate:  result.LatestDate.Format(utils.FormatDateTime),
+		EndValue:    result.EndValue,
+	}
+	return
+}
+
+// getEdbInfoMaxAndMinInfoByMysql
+// @Description: 获取指标的最新数据记录信息（从mysql中获取）
+func (m BaseFromThsHfIndex) getEdbInfoMaxAndMinInfoByMysql(edbCode string) (item *EdbInfoMaxAndMinInfo, err error) {
+	dataObj := BaseFromThsHfData{}
+	result, err := dataObj.GetIndexMinMax(edbCode)
+	if err != nil {
+		return
+	}
+
+	item = &EdbInfoMaxAndMinInfo{
+		MinDate:     result.MinDate,
+		MaxDate:     result.MaxDate,
+		MinValue:    result.MinValue,
+		MaxValue:    result.MaxValue,
+		LatestValue: result.LatestValue,
+		LatestDate:  result.LatestDate,
+		EndValue:    result.EndValue,
+	}
+	return
+}

models/base_from_ths_hf_data.go

@@ -0,0 +1,229 @@
+package models
+
+import (
+	"eta/eta_index_lib/utils"
+	"fmt"
+	"github.com/beego/beego/v2/client/orm"
+	"strings"
+	"time"
+)
+
+// BaseFromThsHfData 同花顺高频数据-指标数据
+type BaseFromThsHfData struct {
+	BaseFromThsHfDataId  int       `orm:"column(base_from_ths_hf_data_id);pk"`
+	BaseFromThsHfIndexId int       `description:"指标ID"`
+	IndexCode            string    `description:"指标编码"`
+	DataTime             time.Time `description:"数据日期（至时分秒）"`
+	Value                float64   `description:"数据值"`
+	UniqueCode           string    `description:"唯一编码"`
+	CreateTime           time.Time `description:"创建时间"`
+	ModifyTime           time.Time `description:"修改时间"`
+	DataTimestamp        int64     `description:"数据日期时间戳"`
+}
+
+func (m *BaseFromThsHfData) TableName() string {
+	return "base_from_ths_hf_data"
+}
+
+type BaseFromThsHfDataCols struct {
+	PrimaryId            string
+	BaseFromThsHfIndexId string
+	IndexCode            string
+	DataTime             string
+	Value                string
+	UniqueCode           string
+	CreateTime           string
+	ModifyTime           string
+	DataTimestamp        string
+}
+
+func (m *BaseFromThsHfData) Cols() BaseFromThsHfDataCols {
+	return BaseFromThsHfDataCols{
+		PrimaryId:            "base_from_ths_hf_data_id",
+		BaseFromThsHfIndexId: "base_from_ths_hf_index_id",
+		IndexCode:            "index_code",
+		DataTime:             "data_time",
+		Value:                "value",
+		UniqueCode:           "unique_code",
+		CreateTime:           "create_time",
+		ModifyTime:           "modify_time",
+		DataTimestamp:        "data_timestamp",
+	}
+}
+
+func (m *BaseFromThsHfData) Create() (err error) {
+	o := orm.NewOrm()
+	id, err := o.Insert(m)
+	if err != nil {
+		return
+	}
+	m.BaseFromThsHfDataId = int(id)
+	return
+}
+
+func (m *BaseFromThsHfData) CreateMulti(items []*BaseFromThsHfData) (err error) {
+	if len(items) == 0 {
+		return
+	}
+	o := orm.NewOrm()
+	_, err = o.InsertMulti(500, items)
+	return
+}
+
+func (m *BaseFromThsHfData) Update(cols []string) (err error) {
+	o := orm.NewOrm()
+	_, err = o.Update(m, cols...)
+	return
+}
+
+func (m *BaseFromThsHfData) Remove() (err error) {
+	o := orm.NewOrm()
+	sql := fmt.Sprintf(`DELETE FROM %s WHERE %s = ? LIMIT 1`, m.TableName(), m.Cols().PrimaryId)
+	_, err = o.Raw(sql, m.BaseFromThsHfDataId).Exec()
+	return
+}
+
+func (m *BaseFromThsHfData) MultiRemove(ids []int) (err error) {
+	if len(ids) == 0 {
+		return
+	}
+	o := orm.NewOrm()
+	sql := fmt.Sprintf(`DELETE FROM %s WHERE %s IN (%s)`, m.TableName(), m.Cols().PrimaryId, utils.GetOrmInReplace(len(ids)))
+	_, err = o.Raw(sql, ids).Exec()
+	return
+}
+
+func (m *BaseFromThsHfData) RemoveByCondition(condition string, pars []interface{}) (err error) {
+	if condition == "" {
+		return
+	}
+	o := orm.NewOrm()
+	sql := fmt.Sprintf(`DELETE FROM %s WHERE %s`, m.TableName(), condition)
+	_, err = o.Raw(sql, pars).Exec()
+	return
+}
+
+func (m *BaseFromThsHfData) GetItemById(id int) (item *BaseFromThsHfData, err error) {
+	o := orm.NewOrm()
+	sql := fmt.Sprintf(`SELECT * FROM %s WHERE %s = ? LIMIT 1`, m.TableName(), m.Cols().PrimaryId)
+	err = o.Raw(sql, id).QueryRow(&item)
+	return
+}
+
+func (m *BaseFromThsHfData) GetItemByCondition(condition string, pars []interface{}, orderRule string) (item *BaseFromThsHfData, err error) {
+	o := orm.NewOrm()
+	order := ``
+	if orderRule != "" {
+		order = ` ORDER BY ` + orderRule
+	}
+	sql := fmt.Sprintf(`SELECT * FROM %s WHERE 1=1 %s %s LIMIT 1`, m.TableName(), condition, order)
+	err = o.Raw(sql, pars).QueryRow(&item)
+	return
+}
+
+func (m *BaseFromThsHfData) GetCountByCondition(condition string, pars []interface{}) (count int, err error) {
+	o := orm.NewOrm()
+	sql := fmt.Sprintf(`SELECT COUNT(1) FROM %s WHERE 1=1 %s`, m.TableName(), condition)
+	err = o.Raw(sql, pars).QueryRow(&count)
+	return
+}
+
+func (m *BaseFromThsHfData) GetItemsByCondition(condition string, pars []interface{}, fieldArr []string, orderRule string) (items []*BaseFromThsHfData, err error) {
+	o := orm.NewOrm()
+	fields := strings.Join(fieldArr, ",")
+	if len(fieldArr) == 0 {
+		fields = `*`
+	}
+	order := fmt.Sprintf(`ORDER BY %s DESC`, m.Cols().CreateTime)
+	if orderRule != "" {
+		order = ` ORDER BY ` + orderRule
+	}
+	sql := fmt.Sprintf(`SELECT %s FROM %s WHERE 1=1 %s %s`, fields, m.TableName(), condition, order)
+	_, err = o.Raw(sql, pars).QueryRows(&items)
+	return
+}
+
+func (m *BaseFromThsHfData) GetPageItemsByCondition(condition string, pars []interface{}, fieldArr []string, orderRule string, startSize, pageSize int) (items []*BaseFromThsHfData, err error) {
+	o := orm.NewOrm()
+	fields := strings.Join(fieldArr, ",")
+	if len(fieldArr) == 0 {
+		fields = `*`
+	}
+	order := fmt.Sprintf(`ORDER BY %s DESC`, m.Cols().CreateTime)
+	if orderRule != "" {
+		order = ` ORDER BY ` + orderRule
+	}
+	sql := fmt.Sprintf(`SELECT %s FROM %s WHERE 1=1 %s %s LIMIT ?,?`, fields, m.TableName(), condition, order)
+	_, err = o.Raw(sql, pars, startSize, pageSize).QueryRows(&items)
+	return
+}
+
+// BaseFromThsHfDataItem 同花顺高频数据信息
+type BaseFromThsHfDataItem struct {
+	DataId     int     `description:"数据ID"`
+	IndexId    int     `description:"指标ID"`
+	IndexCode  string  `description:"指标编码"`
+	DataTime   string  `description:"数据日期（至时分秒）"`
+	Value      float64 `description:"数据值"`
+	UniqueCode string  `description:"唯一编码"`
+}
+
+func (m *BaseFromThsHfData) Format2Item() (item *BaseFromThsHfDataItem) {
+	item = new(BaseFromThsHfDataItem)
+	item.DataId = m.BaseFromThsHfDataId
+	item.IndexId = m.BaseFromThsHfIndexId
+	item.IndexCode = m.IndexCode
+	item.DataTime = utils.TimeTransferString(utils.FormatDateTime, m.DataTime)
+	item.Value = m.Value
+	item.UniqueCode = m.UniqueCode
+	return
+}
+
+func (m *BaseFromThsHfData) MultiInsertOrUpdate(inserts, updates []*BaseFromThsHfData) (err error) {
+	o := orm.NewOrm()
+	if len(inserts) > 0 {
+		_, e := o.InsertMulti(600, inserts)
+		if e != nil {
+			err = fmt.Errorf("insert multi err: %s", e.Error())
+			return
+		}
+	}
+	if len(updates) > 0 {
+		sql := fmt.Sprintf("UPDATE %s SET %s = ?, modify_time = NOW() WHERE %s = ?", m.TableName(), m.Cols().Value, m.Cols().UniqueCode)
+		p, e := o.Raw(sql).Prepare()
+		if e != nil {
+			err = fmt.Errorf("prepare err: %s", e.Error())
+			return
+		}
+		defer func() {
+			_ = p.Close()
+		}()
+		for _, v := range updates {
+			_, e = p.Exec(v.Value, v.UniqueCode)
+			if e != nil {
+				err = fmt.Errorf("update err: %s", e.Error())
+				return
+			}
+		}
+	}
+	return
+}
+
+func (m *BaseFromThsHfData) GetIndexMinMax(indexCode string) (item *EdbInfoMaxAndMinInfo, err error) {
+	o := orm.NewOrm()
+	sql := fmt.Sprintf(`SELECT MIN(%s) AS min_date, MAX(%s) AS max_date, MIN(%s) AS min_value,MAX(%s) AS max_value FROM %s WHERE %s = ?`, m.Cols().DataTime, m.Cols().DataTime, m.Cols().Value, m.Cols().Value, m.TableName(), m.Cols().IndexCode)
+	err = o.Raw(sql, indexCode).QueryRow(&item)
+	if err != nil {
+		return
+	}
+
+	// 获取最新值
+	var lastVal float64
+	sql = fmt.Sprintf(`SELECT %s AS latest_value FROM %s WHERE %s = ? ORDER BY %s DESC LIMIT 1`, m.Cols().Value, m.TableName(), m.Cols().IndexCode, m.Cols().DataTime)
+	err = o.Raw(sql, indexCode).QueryRow(&lastVal)
+	if err != nil {
+		return
+	}
+	item.LatestValue = lastVal
+	return
+}

models/business_conf.go

@@ -0,0 +1,238 @@
+package models
+
+import (
+	"eta/eta_index_lib/utils"
+	"fmt"
+	"github.com/beego/beego/v2/client/orm"
+	"html"
+	"strings"
+	"time"
+)
+
+const (
+	BusinessConfUseXf                     = "UseXf"
+	BusinessConfXfAppid                   = "XfAppid"
+	BusinessConfXfApiKey                  = "XfApiKey"
+	BusinessConfXfApiSecret               = "XfApiSecret"
+	BusinessConfXfVcn                     = "XfVcn"
+	BusinessConfEnPptCoverImgs            = "EnPptCoverImgs"
+	BusinessConfIsReportApprove           = "IsReportApprove"
+	BusinessConfReportApproveType         = "ReportApproveType"
+	BusinessConfCompanyName               = "CompanyName"
+	BusinessConfCompanyWatermark          = "CompanyWatermark"
+	BusinessConfWatermarkChart            = "WatermarkChart"
+	BusinessConfLoginSmsTpId              = "LoginSmsTpId"
+	BusinessConfLoginSmsGjTpId            = "LoginSmsGjTpId"
+	BusinessConfSmsJhgnAppKey             = "SmsJhgnAppKey"
+	BusinessConfSmsJhgjAppKey             = "SmsJhgjAppKey"
+	BusinessConfLdapHost                  = "LdapHost"
+	BusinessConfLdapBase                  = "LdapBase"
+	BusinessConfLdapPort                  = "LdapPort"
+	BusinessConfEmailClient               = "EmailClient"
+	BusinessConfEmailServerHost           = "EmailServerHost"
+	BusinessConfEmailServerPort           = "EmailServerPort"
+	BusinessConfEmailSender               = "EmailSender"
+	BusinessConfEmailSenderUserName       = "EmailSenderUserName"
+	BusinessConfEmailSenderPassword       = "EmailSenderPassword"
+	BusinessConfSmsClient                 = "SmsClient"
+	BusinessConfNanHuaSmsAppKey           = "NanHuaSmsAppKey"
+	BusinessConfNanHuaSmsAppSecret        = "NanHuaSmsAppSecret"
+	BusinessConfNanHuaSmsApiHost          = "NanHuaSmsApiHost"
+	BusinessConfLoginSmsTplContent        = "LoginSmsTplContent"
+	BusinessConfLoginEmailTemplateSubject = "LoginEmailTemplateSubject"
+	BusinessConfLoginEmailTemplateContent = "LoginEmailTemplateContent"
+	BusinessConfLdapBindUserSuffix        = "LdapBindUserSuffix"
+	BusinessConfLdapUserFilter            = "LdapUserFilter"
+
+	BusinessConfTencentApiSecretId           = "TencentApiSecretId"           // 腾讯云API-密钥对
+	BusinessConfTencentApiSecretKey          = "TencentApiSecretKey"          // 腾讯云API-密钥对
+	BusinessConfTencentApiRecTaskCallbackUrl = "TencentApiRecTaskCallbackUrl" // 腾讯云API-语音识别回调地址
+	BusinessConfSmsJhgjVariable              = "SmsJhgjVariable"              // 聚合国际短信变量
+)
+
+const (
+	BusinessConfReportApproveTypeEta   = "eta"
+	BusinessConfReportApproveTypeOther = "other"
+	BusinessConfClientFlagNanHua       = "nhqh" // 南华标记
+	BusinessConfEmailClientSmtp        = "smtp" // 普通邮箱标记
+)
+
+// FromSceneMap 数据源名称与数据源ID的对应关系
+var FromSceneMap = map[int]string{
+	1: "SmartReportSheetSize",
+	2: "ReportSheetSize",
+	3: "EnReportSheetSize",
+	4: "CnPptSheetSize",
+	5: "EnPptSheetSize",
+}
+
+// BusinessConf 商户配置表
+type BusinessConf struct {
+	Id         int    `orm:"column(id);pk"`
+	ConfKey    string `description:"配置Key"`
+	ConfVal    string `description:"配置值"`
+	ValType    int    `description:"1-字符串；2-数值；3-字符串数组；4-富文本；"`
+	Necessary  int    `description:"是否必填：0-否；1-是"`
+	Remark     string `description:"备注"`
+	CreateTime time.Time
+}
+
+func (m *BusinessConf) TableName() string {
+	return "business_conf"
+}
+
+func (m *BusinessConf) PrimaryId() string {
+	return "id"
+}
+
+func (m *BusinessConf) Create() (err error) {
+	o := orm.NewOrmUsingDB("master")
+	id, err := o.Insert(m)
+	if err != nil {
+		return
+	}
+	m.Id = int(id)
+	return
+}
+
+func (m *BusinessConf) CreateMulti(items []*BusinessConf) (err error) {
+	if len(items) == 0 {
+		return
+	}
+	o := orm.NewOrmUsingDB("master")
+	_, err = o.InsertMulti(len(items), items)
+	return
+}
+
+func (m *BusinessConf) Update(cols []string) (err error) {
+	o := orm.NewOrmUsingDB("master")
+	_, err = o.Update(m, cols...)
+	return
+}
+
+func (m *BusinessConf) Del() (err error) {
+	o := orm.NewOrmUsingDB("master")
+	sql := fmt.Sprintf(`DELETE FROM %s WHERE %s = ? LIMIT 1`, m.TableName(), m.PrimaryId())
+	_, err = o.Raw(sql, m.Id).Exec()
+	return
+}
+
+func (m *BusinessConf) GetItemById(id int) (item *BusinessConf, err error) {
+	o := orm.NewOrmUsingDB("master")
+	sql := fmt.Sprintf(`SELECT * FROM %s WHERE %s = ? LIMIT 1`, m.TableName(), m.PrimaryId())
+	err = o.Raw(sql, id).QueryRow(&item)
+	return
+}
+
+func (m *BusinessConf) GetItemByCondition(condition string, pars []interface{}) (item *BusinessConf, err error) {
+	o := orm.NewOrmUsingDB("master")
+	sql := fmt.Sprintf(`SELECT * FROM %s WHERE 1=1 %s LIMIT 1`, m.TableName(), condition)
+	err = o.Raw(sql, pars).QueryRow(&item)
+	return
+}
+
+func (m *BusinessConf) GetCountByCondition(condition string, pars []interface{}) (count int, err error) {
+	o := orm.NewOrmUsingDB("master")
+	sql := fmt.Sprintf(`SELECT COUNT(1) FROM %s WHERE 1=1 %s`, m.TableName(), condition)
+	err = o.Raw(sql, pars).QueryRow(&count)
+	return
+}
+
+func (m *BusinessConf) GetItemsByCondition(condition string, pars []interface{}, fieldArr []string, orderRule string) (items []*BusinessConf, err error) {
+	o := orm.NewOrmUsingDB("master")
+	fields := strings.Join(fieldArr, ",")
+	if len(fieldArr) == 0 {
+		fields = `*`
+	}
+	order := `ORDER BY create_time DESC`
+	if orderRule != "" {
+		order = ` ORDER BY ` + orderRule
+	}
+	sql := fmt.Sprintf(`SELECT %s FROM %s WHERE 1=1 %s %s`, fields, m.TableName(), condition, order)
+	_, err = o.Raw(sql, pars).QueryRows(&items)
+	return
+}
+
+func (m *BusinessConf) GetPageItemsByCondition(condition string, pars []interface{}, fieldArr []string, orderRule string, startSize, pageSize int) (items []*BusinessConf, err error) {
+	o := orm.NewOrmUsingDB("master")
+	fields := strings.Join(fieldArr, ",")
+	if len(fieldArr) == 0 {
+		fields = `*`
+	}
+	order := `ORDER BY create_time DESC`
+	if orderRule != "" {
+		order = ` ORDER BY ` + orderRule
+	}
+	sql := fmt.Sprintf(`SELECT %s FROM %s WHERE 1=1 %s %s LIMIT ?,?`, fields, m.TableName(), condition, order)
+	_, err = o.Raw(sql, pars, startSize, pageSize).QueryRows(&items)
+	return
+}
+
+// GetBusinessConf 获取商家配置
+func GetBusinessConf() (list map[string]string, err error) {
+	list = make(map[string]string)
+
+	var items []*BusinessConf
+	o := orm.NewOrmUsingDB("master")
+	sql := `SELECT * FROM business_conf`
+	_, err = o.Raw(sql).QueryRows(&items)
+	if err != nil {
+		return
+	}
+
+	for _, v := range items {
+		if v.ValType == 4 {
+			list[v.ConfKey] = html.UnescapeString(v.ConfVal)
+			continue
+		}
+		list[v.ConfKey] = v.ConfVal
+	}
+	return
+}
+
+// BusinessConfUpdate 更新配置
+type BusinessConfUpdate struct {
+	ConfKey string
+	ConfVal string
+}
+
+// UpdateBusinessConfMulti 批量修改配置
+func UpdateBusinessConfMulti(items []BusinessConfUpdate) (err error) {
+	o := orm.NewOrmUsingDB("master")
+	p, err := o.Raw("UPDATE business_conf SET conf_val = ? WHERE conf_key = ?").Prepare()
+	if err != nil {
+		return
+	}
+	defer func() {
+		_ = p.Close()
+	}()
+	for _, v := range items {
+		_, err = p.Exec(v.ConfVal, v.ConfKey)
+		if err != nil {
+			return
+		}
+	}
+	return
+}
+
+func GetBusinessConfByKey(key string) (item *BusinessConf, err error) {
+	o := orm.NewOrmUsingDB("master")
+	sql := fmt.Sprintf(`SELECT * FROM business_conf WHERE conf_key = ? LIMIT 1`)
+	err = o.Raw(sql, key).QueryRow(&item)
+	return
+}
+
+// InitUseMongoConf
+// @Description:
+// @author: Roc
+// @datetime 2024-07-01 13:49:09
+func InitUseMongoConf() {
+	useMongo, e := GetBusinessConfByKey("UseMongo")
+	if e != nil {
+		return
+	}
+
+	if useMongo.ConfVal == `true` {
+		utils.UseMongo = true
+	}
+}

models/db.go

@@ -35,6 +35,17 @@ func init() {
 	gl, _ := orm.GetDB("gl")
 	gl.SetConnMaxLifetime(10 * time.Minute)
 
+	// master库
+	{
+		_ = orm.RegisterDataBase("master", "mysql", utils.MYSQL_URL_MASTER)
+		orm.SetMaxIdleConns("master", 50)
+		orm.SetMaxOpenConns("master", 100)
+
+		master, _ := orm.GetDB("master")
+		master.SetConnMaxLifetime(10 * time.Minute)
+
+	}
+
 	orm.Debug = true
 	orm.DebugLog = orm.NewLog(utils.Binlog)
 
@@ -58,6 +69,7 @@ func init() {
 		new(EdbDataInsertConfig),
 		new(EdbAdjustConf), // 数据调整的配置
 		new(BaseFromMysteelChemicalClassify),
+		new(EdbInfoRelation), //指标引用记录
 	)
 
 	// 注册期货数据 数据表
@@ -81,8 +93,11 @@ func init() {
 	// 自有数据指标
 	initBusinessEdb()
 
+	// 初始化因子指标系列
+	initFactorEdbSeries()
+
 	// 初始化部分数据表变量(直接init会有顺序问题=_=!)
-	InitEdbSource()
+	afterInitTable()
 }
 
 // initFutureGood 注册期货数据 数据表
@@ -143,6 +158,11 @@ func initBaseIndex() {
 		new(CCFStockExcel),
 		new(BaseFromSciHqData),
 		new(BaseFromSciHqIndex),
+		new(BaseFromThsHfIndex),
+		new(BaseFromThsHfData),
+		new(BaseFromEdbMapping),
+		new(EdbDataThsHf),
+		new(BaseFromBusinessData), // 数据源中自有数据的明细数据表
 	)
 }
 
@@ -181,3 +201,25 @@ func initBusinessEdb() {
 		new(EdbBusinessSource),     // 自有数据（商家）指标来源
 	)
 }
+
+// initFactorEdbSeries 因子指标系列数据表
+func initFactorEdbSeries() {
+	orm.RegisterModel(
+		new(FactorEdbSeries),              // 因子指标系列
+		new(FactorEdbSeriesChartMapping),  // 因子指标系列-图表关联
+		new(FactorEdbSeriesMapping),       // 因子指标系列-指标计算数据
+		new(FactorEdbSeriesCalculateData), // 因子指标系列-指标关联
+	)
+}
+
+// afterInitTable
+// @Description: 初始化表结构的的后置操作
+// @author: Roc
+// @datetime 2024-07-01 13:31:09
+func afterInitTable() {
+	// 初始化指标来源配置
+	InitEdbSource()
+
+	// 初始化是否启用mongo配置
+	InitUseMongoConf()
+}

models/edb_data_base.go

@@ -46,6 +46,7 @@ type AddEdbInfoReq struct {
 	Source    int    `description:"指标来源ID"`
 	StockCode string `description:"证券代码"`
 	Frequency string `description:"频度"`
+	ExtraPars string `description:"额外参数(如同花顺日期序列)"`
 }
 
 // GetEdbInfoCountByCondition 获取指标数量

models/edb_data_business.go

@@ -7,8 +7,8 @@ import (
 	"github.com/beego/beego/v2/client/orm"
 	"github.com/shopspring/decimal"
 	"go.mongodb.org/mongo-driver/bson"
-	"go.mongodb.org/mongo-driver/bson/primitive"
 	"reflect"
+	"strings"
 	"time"
 )
 
@@ -103,7 +103,7 @@ func (obj Business) Add(params AddBaseParams, businessIndexItem *BaseFromBusines
 	edbInfo.EdbInfoId = int(newEdbInfoId)
 
 	// 更新数据
-	err = obj.refresh(edbInfo, "")
+	err = obj.refresh(to, edbInfo, "")
 
 	return
 }
@@ -153,7 +153,7 @@ func (obj Business) Edit(params EditBaseParams, businessIndexItem *BaseFromBusin
 	}
 
 	//计算数据
-	err = obj.refresh(edbInfo, "")
+	err = obj.refresh(to, edbInfo, "")
 
 	return
 }
@@ -174,7 +174,7 @@ func (obj Business) Refresh(params RefreshBaseParams) (err error, errMsg string)
 	}()
 
 	// 计算数据
-	err = obj.refresh(params.EdbInfo, params.StartDate)
+	err = obj.refresh(to, params.EdbInfo, params.StartDate)
 
 	return
 }
@@ -194,7 +194,17 @@ func (obj Business) GetEdbType() int {
 	return utils.DEFAULT_EDB_TYPE
 }
 
-func (obj Business) refresh(edbInfo *EdbInfo, startDate string) (err error) {
+func (obj Business) refresh(to orm.TxOrmer, edbInfo *EdbInfo, startDate string) (err error) {
+	if utils.UseMongo {
+		return obj.refreshByMongo(edbInfo, startDate)
+	} else {
+		return obj.refreshByMysql(to, edbInfo, startDate)
+	}
+
+	return
+}
+
+func (obj Business) refreshByMongo(edbInfo *EdbInfo, startDate string) (err error) {
 	// 真实数据的最大日期  ， 插入规则配置的日期
 	var realDataMaxDate, edbDataInsertConfigDate time.Time
 	var edbDataInsertConfig *EdbDataInsertConfig
@@ -210,7 +220,7 @@ func (obj Business) refresh(edbInfo *EdbInfo, startDate string) (err error) {
 	}
 
 	//获取已存在的所有数据
-	baseDataList, err := obj.getBaseBusinessData(edbInfo, startDate)
+	baseDataList, err := obj.getBaseBusinessDataByMongo(edbInfo, startDate)
 
 	//获取指标所有数据
 	existDataList := make([]*mgo.EdbDataBusiness, 0)
@@ -352,8 +362,144 @@ func (obj Business) refresh(edbInfo *EdbInfo, startDate string) (err error) {
 	return
 }
 
+func (obj Business) refreshByMysql(to orm.TxOrmer, edbInfo *EdbInfo, startDate string) (err error) {
+	dataTableName := GetEdbDataTableName(edbInfo.Source, edbInfo.SubSource)
+	edbInfoIdStr := fmt.Sprint(edbInfo.EdbInfoId)
+
+	// 真实数据的最大日期  ， 插入规则配置的日期
+	var realDataMaxDate, edbDataInsertConfigDate time.Time
+	var edbDataInsertConfig *EdbDataInsertConfig
+	var isFindConfigDateRealData bool //是否找到配置日期的实际数据的值
+	{
+		edbDataInsertConfig, err = GetEdbDataInsertConfigByEdbId(edbInfo.EdbInfoId)
+		if err != nil && err.Error() != utils.ErrNoRow() {
+			return
+		}
+		if edbDataInsertConfig != nil {
+			edbDataInsertConfigDate = edbDataInsertConfig.Date
+		}
+	}
+
+	//获取已存在的所有数据
+	baseDataList, err := obj.getBaseBusinessDataByMysql(edbInfo, startDate)
+
+	//获取指标所有数据
+	var existCondition string
+	var existPars []interface{}
+
+	existCondition += " AND edb_info_id=? "
+	existPars = append(existPars, edbInfo.EdbInfoId)
+	if startDate != "" {
+		existCondition += " AND data_time>=? "
+		existPars = append(existPars, startDate)
+	}
+
+	existList, err := GetEdbDataByCondition(edbInfo.Source, edbInfo.SubSource, existCondition, existPars)
+	if err != nil {
+		fmt.Println(obj.GetSourceName() + ",refreshByMysql err;getEdbDataBusinessList Err:" + err.Error())
+		return err
+	}
+	existDataMap := make(map[string]*EdbInfoSearchData)
+	removeDataTimeMap := make(map[string]bool) //需要移除的日期数据
+	for _, v := range existList {
+		existDataMap[v.DataTime] = v
+		removeDataTimeMap[v.DataTime] = true
+	}
+	needAddDateMap := make(map[time.Time]int)
+
+	// 待添加的数据集
+	addSql := ` INSERT INTO ` + dataTableName + ` (edb_info_id,edb_code,data_time,value,create_time,modify_time,data_timestamp) values `
+	var isAdd bool
+	// 待更新的数据集
+	updateDataList := make([]*EdbInfoSearchData, 0)
+
+	for _, tmpData := range baseDataList {
+		currDate := tmpData.DataTime
+		currDateStr := currDate.Format(utils.FormatDate)
+
+		// 当前的实际值
+		saveValue := decimal.NewFromFloat(tmpData.Value).Round(4).String()
+
+		// 下面代码主要目的是处理掉手动插入的数据判断
+		{
+			if realDataMaxDate.IsZero() || currDate.After(realDataMaxDate) {
+				realDataMaxDate = currDate
+			}
+			if edbDataInsertConfigDate.IsZero() || currDate.Equal(edbDataInsertConfigDate) {
+				isFindConfigDateRealData = true
+			}
+		}
+
+		existData, ok := existDataMap[currDateStr]
+		// 如果库中已经存在该数据的话，那么就进行值的变更操作
+		if ok {
+			// 已经入到指标库的值
+			existValStr := decimal.NewFromFloat(existData.Value).Round(4).String()
+
+			//校验待删除日期数据里面是否存在该元素，如果存在的话，那么移除该日期
+			delete(removeDataTimeMap, currDateStr)
+			if existValStr != saveValue {
+				existData.Value = tmpData.Value
+				updateDataList = append(updateDataList, existData)
+			}
+
+			continue
+		}
+
+		// 库中不存在该日期的数据
+		timestamp := currDate.UnixNano() / 1e6
+		needAddDateMap[currDate] = 1
+		addSql += GetAddSql(edbInfoIdStr, edbInfo.EdbCode, currDateStr, fmt.Sprint(timestamp), saveValue)
+		isAdd = true
+	}
+
+	//删除已经不存在的指标数据（由于该指标当日的数据删除了）
+	{
+		removeDateList := make([]string, 0)
+		for dateTime := range removeDataTimeMap {
+			removeDateList = append(removeDateList, dateTime)
+		}
+		removeNum := len(removeDateList)
+		if removeNum > 0 {
+			//如果拼接指标变更了，那么需要删除所有的指标数据
+			sql := fmt.Sprintf(` DELETE FROM %s WHERE edb_info_id = ? and data_time in (`+utils.GetOrmInReplace(removeNum)+`) `, dataTableName)
+
+			_, err = to.Raw(sql, edbInfo.EdbInfoId, removeDateList).Exec()
+			if err != nil {
+				err = fmt.Errorf("删除自有数据的明细数据失败,Err:" + err.Error())
+				return
+			}
+		}
+	}
+
+	if isAdd {
+		addSql = strings.TrimRight(addSql, ",")
+		_, err = to.Raw(addSql).Exec()
+		if err != nil {
+			fmt.Println("RefreshAllCalculate add Err", err.Error())
+			return
+		}
+	}
+
+	// 修改历史数据
+	if len(updateDataList) > 0 {
+		for _, v := range updateDataList {
+			err = ModifyEdbDataById(edbInfo.Source, edbInfo.SubSource, v.EdbDataId, fmt.Sprint(v.Value))
+			if err != nil {
+				fmt.Println(obj.GetSourceName() + "，refreshByMysql:Err:" + err.Error())
+				return err
+			}
+		}
+	}
+
+	// 处理手工数据补充的配置
+	HandleConfigInsertEdbData(realDataMaxDate, edbDataInsertConfig, edbInfo.EdbInfoId, edbInfo.Source, edbInfo.SubSource, existDataMap, isFindConfigDateRealData)
+
+	return
+}
+
 // GetEdbInfoMaxAndMinInfo 获取指标的最新数据记录信息
-func (obj Business) GetEdbInfoMaxAndMinInfo(edbCode string) (item *EdbInfoMaxAndMinInfo, err error) {
+func (obj Business) getEdbInfoMaxAndMinInfoByMongo(edbCode string) (item *EdbInfoMaxAndMinInfo, err error) {
 	mogDataObj := new(mgo.EdbDataBusiness)
 	pipeline := []bson.M{
 		{"$match": bson.M{"edb_code": edbCode}},
@@ -405,21 +551,29 @@ func (obj Business) GetEdbInfoMaxAndMinInfo(edbCode string) (item *EdbInfoMaxAnd
 // @param edbInfo *EdbInfo
 // @return err error
 func (obj Business) UnifiedModifyEdbInfoMaxAndMinInfo(edbInfo *EdbInfo) (err error) {
-	edbInfoMaxAndMinInfo, err := obj.GetEdbInfoMaxAndMinInfo(edbInfo.EdbCode)
-	if err != nil {
-		return
+	if utils.UseMongo {
+		edbInfoMaxAndMinInfo, tmpErr := obj.getEdbInfoMaxAndMinInfoByMongo(edbInfo.EdbCode)
+		// 如果正常获取到了，那就去修改指标的最大最小值
+		if tmpErr == nil && edbInfoMaxAndMinInfo != nil {
+			err = ModifyEdbInfoMaxAndMinInfo(edbInfo.EdbInfoId, edbInfoMaxAndMinInfo)
+		} else {
+			// 清空的目的是为了避免异常返回
+			err = nil
+		}
+	} else {
+		err, _ = UnifiedModifyEdbInfoMaxAndMinInfo(edbInfo)
 	}
-	err = ModifyEdbInfoMaxAndMinInfo(edbInfo.EdbInfoId, edbInfoMaxAndMinInfo)
+
 	return
 }
 
 // EdbInfoMgoData
 // @Description: mgo里面的数据
 type EdbInfoMgoData struct {
-	EdbDataId primitive.ObjectID `description:"数据ID"`
-	DataTime  time.Time          `description:"数据日期"`
-	Value     float64            `description:"数据"`
-	EdbCode   string             `description:"指标编码"`
+	//EdbDataId primitive.ObjectID `description:"数据ID"`
+	DataTime time.Time `description:"数据日期"`
+	Value    float64   `description:"数据"`
+	EdbCode  string    `description:"指标编码"`
 }
 
 // getBaseBusinessData
@@ -432,6 +586,19 @@ type EdbInfoMgoData struct {
 // @return newDataList []EdbInfoSearchData
 // @return err error
 func (obj Business) getBaseBusinessData(edbInfo *EdbInfo, startDate string) (newDataList []EdbInfoMgoData, err error) {
+	return obj.getBaseBusinessDataByMongo(edbInfo, startDate)
+}
+
+// getBaseBusinessDataByMongo
+// @Description: 从mongo中获取基础的明细数据
+// @author: Roc
+// @receiver obj
+// @datetime 2024-07-02 10:12:02
+// @param edbInfo *EdbInfo
+// @param startDate string
+// @return newDataList []EdbInfoMgoData
+// @return err error
+func (obj Business) getBaseBusinessDataByMongo(edbInfo *EdbInfo, startDate string) (newDataList []EdbInfoMgoData, err error) {
 	newDataList = make([]EdbInfoMgoData, 0)
 
 	// 获取数据源的指标数据
@@ -460,10 +627,53 @@ func (obj Business) getBaseBusinessData(edbInfo *EdbInfo, startDate string) (new
 
 	for _, v := range baseDataList {
 		newDataList = append(newDataList, EdbInfoMgoData{
-			EdbDataId: v.ID,
-			DataTime:  v.DataTime,
-			Value:     v.Value,
-			EdbCode:   v.IndexCode,
+			//EdbDataId: v.ID,
+			DataTime: v.DataTime,
+			Value:    v.Value,
+			EdbCode:  v.IndexCode,
+		})
+	}
+
+	return
+}
+
+// getBaseBusinessDataByMysql
+// @Description: 从mysql中获取基础的明细数据
+// @author: Roc
+// @receiver obj
+// @datetime 2024-07-02 10:12:16
+// @param edbInfo *EdbInfo
+// @param startDate string
+// @return newDataList []EdbInfoMgoData
+// @return err error
+func (obj Business) getBaseBusinessDataByMysql(edbInfo *EdbInfo, startDate string) (newDataList []EdbInfoMgoData, err error) {
+	newDataList = make([]EdbInfoMgoData, 0)
+	// 获取数据源的指标数据
+	baseBusinessDataObj := new(BaseFromBusinessData)
+
+	// 构建查询条件
+	var condition []string
+	var pars []interface{}
+	condition = append(condition, "index_code = ? ")
+	pars = append(pars, edbInfo.EdbCode)
+
+	if startDate != `` {
+		condition = append(condition, " data_time >= ? ")
+		pars = append(pars, startDate)
+	}
+
+	baseDataList, err := baseBusinessDataObj.GetAllDataList(condition, pars, " data_time ASC ")
+	if err != nil {
+		fmt.Println("getBaseBusinessData Err:" + err.Error())
+		return
+	}
+
+	for _, v := range baseDataList {
+		newDataList = append(newDataList, EdbInfoMgoData{
+			//EdbDataId: v.BusinessDataId,
+			DataTime: v.DataTime,
+			Value:    v.Value,
+			EdbCode:  v.IndexCode,
 		})
 	}
 

models/edb_data_table.go

@@ -8,10 +8,12 @@ import (
 func GetEdbDataTableName(source, subSource int) (tableName string) {
 	switch source {
 	case utils.DATA_SOURCE_THS:
-		tableName = "edb_data_ths"
-		if subSource == utils.DATA_SUB_SOURCE_DATE {
+		switch subSource {
+		case utils.DATA_SUB_SOURCE_DATE:
 			tableName = "edb_data_ths_ds"
-		} else {
+		case utils.DATA_SUB_SOURCE_HIGH_FREQUENCY:
+			tableName = "edb_data_ths_hf"
+		default:
 			tableName = "edb_data_ths"
 		}
 	case utils.DATA_SOURCE_WIND:

models/edb_data_ths_hf.go

@@ -0,0 +1,181 @@
+package models
+
+import (
+	"eta/eta_index_lib/utils"
+	"fmt"
+	"github.com/beego/beego/v2/client/orm"
+	"strings"
+	"time"
+)
+
+// EdbDataThsHf 同花顺高频指标数据
+type EdbDataThsHf struct {
+	EdbDataId     int       `orm:"column(edb_data_id);pk"`
+	EdbInfoId     int       `description:"指标ID"`
+	EdbCode       string    `description:"指标编码"`
+	DataTime      time.Time `description:"数据日期"`
+	Value         float64   `description:"数据值"`
+	CreateTime    time.Time `description:"创建时间"`
+	ModifyTime    time.Time `description:"修改时间"`
+	DataTimestamp int64     `description:"数据日期时间戳"`
+}
+
+func (m *EdbDataThsHf) TableName() string {
+	return "edb_data_ths_hf"
+}
+
+type EdbDataThsHfCols struct {
+	PrimaryId     string
+	EdbInfoId     string
+	EdbCode       string
+	DataTime      string
+	Value         string
+	CreateTime    string
+	ModifyTime    string
+	DataTimestamp string
+}
+
+func (m *EdbDataThsHf) Cols() EdbDataThsHfCols {
+	return EdbDataThsHfCols{
+		PrimaryId:     "edb_data_id",
+		EdbInfoId:     "edb_info_id",
+		EdbCode:       "edb_code",
+		DataTime:      "data_time",
+		Value:         "value",
+		CreateTime:    "create_time",
+		ModifyTime:    "modify_time",
+		DataTimestamp: "data_timestamp",
+	}
+}
+
+func (m *EdbDataThsHf) Create() (err error) {
+	o := orm.NewOrm()
+	id, err := o.Insert(m)
+	if err != nil {
+		return
+	}
+	m.EdbDataId = int(id)
+	return
+}
+
+func (m *EdbDataThsHf) CreateMulti(items []*EdbDataThsHf) (err error) {
+	if len(items) == 0 {
+		return
+	}
+	o := orm.NewOrm()
+	_, err = o.InsertMulti(500, items)
+	return
+}
+
+func (m *EdbDataThsHf) Update(cols []string) (err error) {
+	o := orm.NewOrm()
+	_, err = o.Update(m, cols...)
+	return
+}
+
+func (m *EdbDataThsHf) Remove() (err error) {
+	o := orm.NewOrm()
+	sql := fmt.Sprintf(`DELETE FROM %s WHERE %s = ? LIMIT 1`, m.TableName(), m.Cols().PrimaryId)
+	_, err = o.Raw(sql, m.EdbDataId).Exec()
+	return
+}
+
+func (m *EdbDataThsHf) MultiRemove(ids []int) (err error) {
+	if len(ids) == 0 {
+		return
+	}
+	o := orm.NewOrm()
+	sql := fmt.Sprintf(`DELETE FROM %s WHERE %s IN (%s)`, m.TableName(), m.Cols().PrimaryId, utils.GetOrmInReplace(len(ids)))
+	_, err = o.Raw(sql, ids).Exec()
+	return
+}
+
+func (m *EdbDataThsHf) RemoveByCondition(condition string, pars []interface{}) (err error) {
+	if condition == "" {
+		return
+	}
+	o := orm.NewOrm()
+	sql := fmt.Sprintf(`DELETE FROM %s WHERE %s`, m.TableName(), condition)
+	_, err = o.Raw(sql, pars).Exec()
+	return
+}
+
+func (m *EdbDataThsHf) GetItemById(id int) (item *EdbDataThsHf, err error) {
+	o := orm.NewOrm()
+	sql := fmt.Sprintf(`SELECT * FROM %s WHERE %s = ? LIMIT 1`, m.TableName(), m.Cols().PrimaryId)
+	err = o.Raw(sql, id).QueryRow(&item)
+	return
+}
+
+func (m *EdbDataThsHf) GetItemByCondition(condition string, pars []interface{}, orderRule string) (item *EdbDataThsHf, err error) {
+	o := orm.NewOrm()
+	order := ``
+	if orderRule != "" {
+		order = ` ORDER BY ` + orderRule
+	}
+	sql := fmt.Sprintf(`SELECT * FROM %s WHERE 1=1 %s %s LIMIT 1`, m.TableName(), condition, order)
+	err = o.Raw(sql, pars).QueryRow(&item)
+	return
+}
+
+func (m *EdbDataThsHf) GetCountByCondition(condition string, pars []interface{}) (count int, err error) {
+	o := orm.NewOrm()
+	sql := fmt.Sprintf(`SELECT COUNT(1) FROM %s WHERE 1=1 %s`, m.TableName(), condition)
+	err = o.Raw(sql, pars).QueryRow(&count)
+	return
+}
+
+func (m *EdbDataThsHf) GetItemsByCondition(condition string, pars []interface{}, fieldArr []string, orderRule string) (items []*EdbDataThsHf, err error) {
+	o := orm.NewOrm()
+	fields := strings.Join(fieldArr, ",")
+	if len(fieldArr) == 0 {
+		fields = `*`
+	}
+	order := fmt.Sprintf(`ORDER BY %s DESC`, m.Cols().CreateTime)
+	if orderRule != "" {
+		order = ` ORDER BY ` + orderRule
+	}
+	sql := fmt.Sprintf(`SELECT %s FROM %s WHERE 1=1 %s %s`, fields, m.TableName(), condition, order)
+	_, err = o.Raw(sql, pars).QueryRows(&items)
+	return
+}
+
+func (m *EdbDataThsHf) GetPageItemsByCondition(condition string, pars []interface{}, fieldArr []string, orderRule string, startSize, pageSize int) (items []*EdbDataThsHf, err error) {
+	o := orm.NewOrm()
+	fields := strings.Join(fieldArr, ",")
+	if len(fieldArr) == 0 {
+		fields = `*`
+	}
+	order := fmt.Sprintf(`ORDER BY %s DESC`, m.Cols().CreateTime)
+	if orderRule != "" {
+		order = ` ORDER BY ` + orderRule
+	}
+	sql := fmt.Sprintf(`SELECT %s FROM %s WHERE 1=1 %s %s LIMIT ?,?`, fields, m.TableName(), condition, order)
+	_, err = o.Raw(sql, pars, startSize, pageSize).QueryRows(&items)
+	return
+}
+
+// MultiUpdateValue 批量更新数据值
+func (m *EdbDataThsHf) MultiUpdateValue(items []*EdbDataThsHf) (err error) {
+	if len(items) == 0 {
+		return
+	}
+
+	o := orm.NewOrm()
+	sql := fmt.Sprintf("UPDATE %s SET %s = ?, %s = NOW() WHERE %s = ?", m.TableName(), m.Cols().Value, m.Cols().ModifyTime, m.Cols().PrimaryId)
+	p, e := o.Raw(sql).Prepare()
+	if e != nil {
+		err = fmt.Errorf("update sql prepare err: %v", e)
+		return
+	}
+	defer func() {
+		_ = p.Close()
+	}()
+	for _, v := range items {
+		_, err = p.Exec(v.Value, v.EdbDataId)
+		if err != nil {
+			return
+		}
+	}
+	return
+}

models/edb_info.go

@@ -209,7 +209,7 @@ type FindEdbDataListAllCond struct {
 // @return item []*EdbInfoSearchData
 // @return err error
 func GetEdbDataListAll(source, subSource int, findEdbDataListAllCond FindEdbDataListAllCond, order int) (item []*EdbInfoSearchData, err error) {
-	if source == utils.DATA_SOURCE_BUSINESS {
+	if source == utils.DATA_SOURCE_BUSINESS && utils.UseMongo {
 		return GetEdbDataListAllByMongo(source, subSource, findEdbDataListAllCond, order)
 	}
 
@@ -277,7 +277,7 @@ func GetEdbDataListAllByMysql(source, subSource int, findEdbDataListAllCond Find
 // @return item []*EdbInfoSearchData
 // @return err error
 func GetEdbDataListAllByTo(to orm.TxOrmer, source, subSource int, findEdbDataListAllCond FindEdbDataListAllCond, order int) (item []*EdbInfoSearchData, err error) {
-	if source == utils.DATA_SOURCE_BUSINESS {
+	if source == utils.DATA_SOURCE_BUSINESS && utils.UseMongo {
 		return GetEdbDataListAllByMongo(source, subSource, findEdbDataListAllCond, order)
 	}
 
@@ -462,7 +462,7 @@ func ModifyEdbDataUpdateTime(edbInfoId int, dataUpdateTime, erDataUpdateDate str
 // @return count int
 // @return err error
 func GetLteZeroEdbDataCount(source, subSource, edbInfoId int) (count int, err error) {
-	if source == utils.DATA_SOURCE_BUSINESS {
+	if source == utils.DATA_SOURCE_BUSINESS && utils.UseMongo {
 		return GetLteZeroEdbDataCountByMongo(source, subSource, edbInfoId)
 	}
 
@@ -1511,3 +1511,54 @@ func GetEdbInfoByEdbCodeList(source int, edbCodeList []string) (items []*EdbInfo
 
 	return
 }
+
+// EdbInfoExtra 指标额外数据-extra字段
+type EdbInfoExtra struct {
+	ApiExtraPars string `description:"API-额外参数(如同花顺日期序列)"`
+}
+
+// GetEdbInfoNoUpdateTotalByIdList 根据指标id列表获取指标信息
+func GetEdbInfoNoUpdateTotalByIdList(edbInfoIdList []int) (total int, err error) {
+	num := len(edbInfoIdList)
+	if num <= 0 {
+		return
+	}
+	o := orm.NewOrm()
+	sql := ` SELECT count(*) FROM edb_info WHERE edb_info_id in (` + utils.GetOrmInReplace(num) + `)  and no_update=1`
+	err = o.Raw(sql, edbInfoIdList).QueryRow(&total)
+	return
+}
+
+func TransEdbInfoSearchData2DataList(items []*EdbInfoSearchData) (list []*EdbDataList) {
+	if len(items) == 0 {
+		return
+	}
+	list = make([]*EdbDataList, 0)
+	for _, v := range items {
+		list = append(list, &EdbDataList{
+			EdbDataId:     v.EdbDataId,
+			EdbInfoId:     v.EdbInfoId,
+			DataTime:      v.DataTime,
+			DataTimestamp: v.DataTimestamp,
+			Value:         v.Value,
+		})
+	}
+	return
+}
+
+func TransEdbInfoDataList2SearchData(items []*EdbDataList) (list []*EdbInfoSearchData) {
+	if len(items) == 0 {
+		return
+	}
+	list = make([]*EdbInfoSearchData, 0)
+	for _, v := range items {
+		list = append(list, &EdbInfoSearchData{
+			EdbDataId:     v.EdbDataId,
+			EdbInfoId:     v.EdbInfoId,
+			DataTime:      v.DataTime,
+			DataTimestamp: v.DataTimestamp,
+			Value:         v.Value,
+		})
+	}
+	return
+}

models/edb_info_calculate_mapping.go

@@ -1,6 +1,7 @@
 package models
 
 import (
+	"eta/eta_index_lib/utils"
 	"github.com/beego/beego/v2/client/orm"
 	"time"
 )
@@ -25,6 +26,31 @@ type EdbInfoCalculateMapping struct {
 	FromSubSource             int       `description:"渠道子数据库来源"`
 }
 
+// EdbInfoCalculateMappingInfo
+// @Description: 计算指标与基础指标关系表
+type EdbInfoCalculateMappingInfo struct {
+	EdbInfoCalculateMappingId int       `orm:"column(edb_info_calculate_mapping_id);pk"`
+	EdbInfoId                 int       `description:"计算指标id"`
+	Source                    int       `description:"计算指标来源"`
+	SourceName                string    `description:"计算指标来源名称"`
+	EdbCode                   string    `description:"计算指标编码"`
+	FromEdbInfoId             int       `description:"基础指标id"`
+	FromEdbCode               string    `description:"基础指标编码"`
+	FromEdbName               string    `description:"基础指标名称"`
+	FromSource                int       `description:"基础指标来源"`
+	FromSourceName            string    `description:"基础指标来源名称"`
+	MoveValue                 int       `description:"领先值"`
+	FromTag                   string    `description:"来源指标标签"`
+	Sort                      int       `description:"计算指标名称排序"`
+	CreateTime                time.Time `description:"创建时间"`
+	ModifyTime                time.Time `description:"修改时间"`
+	FromEdbType               int       `description:"来源指标类型:1:基础指标,2:计算指标"`
+	FromEdbInfoType           int       `description:"来源指标类型: 0-基础指标; 1-预测指标"`
+	FromClassifyId            int       `description:"来源指标分类ID"`
+	FromUniqueCode            string    `description:"来源指标唯一编码"`
+	NoUpdate                  int8      `description:"是否停止更新，0：继续更新；1：停止更新"`
+}
+
 // AddEdbInfoCalculateMappingMulti 批量添加指标关系表
 func AddEdbInfoCalculateMappingMulti(items []*EdbInfoCalculateMapping) (err error) {
 	o := orm.NewOrm()
@@ -119,3 +145,23 @@ func GetEdbInfoCalculateByEdbCode(edbCode, fromEdbCode string) (item *EdbInfoCal
 	err = o.Raw(sql, edbCode, fromEdbCode).QueryRow(&item)
 	return
 }
+
+// GetEdbInfoCalculateMappingListByEdbInfoIds 根据生成的指标id获取来源的指标id列表
+func GetEdbInfoCalculateMappingListByEdbInfoIds(edbInfoIds []int) (items []*EdbInfoCalculateMappingInfo, err error) {
+	o := orm.NewOrm()
+	sql := ` SELECT a.*,b.edb_type as from_edb_type,b.edb_info_type as from_edb_info_type, b.unique_code AS from_unique_code, b.classify_id AS from_classify_id,b.no_update FROM edb_info_calculate_mapping AS a
+			INNER JOIN edb_info AS b ON a.from_edb_info_id=b.edb_info_id
+			WHERE a.edb_info_id in (` + utils.GetOrmInReplace(len(edbInfoIds)) + `) `
+	_, err = o.Raw(sql, edbInfoIds).QueryRows(&items)
+	return
+}
+
+// GetEdbInfoCalculateMappingListByEdbInfoId 根据生成的指标id获取来源的指标id列表
+func GetEdbInfoCalculateMappingListByEdbInfoId(edbInfoId int) (items []*EdbInfoCalculateMappingInfo, err error) {
+	o := orm.NewOrm()
+	sql := ` SELECT a.*,b.edb_type as from_edb_type,b.edb_info_type as from_edb_info_type, b.unique_code AS from_unique_code, b.classify_id AS from_classify_id,b.no_update FROM edb_info_calculate_mapping AS a
+			INNER JOIN edb_info AS b ON a.from_edb_info_id=b.edb_info_id
+			WHERE a.edb_info_id=? `
+	_, err = o.Raw(sql, edbInfoId).QueryRows(&items)
+	return
+}

models/edb_info_relation.go

@@ -0,0 +1,136 @@
+package models
+
+import (
+	"eta/eta_index_lib/utils"
+	"github.com/beego/beego/v2/client/orm"
+	"time"
+)
+
+type EdbInfoRelation struct {
+	EdbInfoRelationId  int       `orm:"column(edb_info_relation_id);pk"`
+	EdbInfoId          int       `description:"指标id"`
+	Source             int       `description:"来源:1:同花顺,2:wind,3:彭博,4:指标运算,5:累计值转月,6:同比值,7:同差值,8:N数值移动平均计算,9:手工指标,10:隆众"`
+	EdbName            string    `description:"指标名称"`
+	EdbCode            string    `description:"指标编码"`
+	ReferObjectId      int       `description:"引用对象ID（图表ID，ETA逻辑ID等）"`
+	ReferObjectType    int       `description:"引用对象ID类型（1.图表，2.ETA逻辑）"`
+	ReferObjectSubType int       `description:"引用对象子类"`
+	CreateTime         time.Time `description:"创建时间"`
+	ModifyTime         time.Time `description:"修改时间"`
+	RelationTime       time.Time `description:"引用时间"`
+	RelationType       int       `description:"引用类型，0:直接饮用，1间接引用"`
+	RootEdbInfoId      int       `description:"间接引用时，关联的直接引用的指标ID"`
+	ChildEdbInfoId     int       `description:"间接引用时，计算指标直接关联的指标ID"`
+	RelationCode       string    `description:"引用标识"`
+	ParentRelationId   int       `description:"间接引用关联的直接引用的ID"`
+}
+
+func (e *EdbInfoRelation) TableName() string {
+	return "edb_info_relation"
+}
+
+// GetEdbInfoRelationByChildEdbInfoId 查询引用的指标ID
+func GetEdbInfoRelationByChildEdbInfoId(edbInfoId int) (item *EdbInfoRelation, err error) {
+	o := orm.NewOrm()
+	msql := ` SELECT * FROM edb_info_relation WHERE child_edb_info_id = ? or edb_info_id=?`
+	err = o.Raw(msql, edbInfoId, edbInfoId).QueryRow(&item)
+	return
+}
+
+// GetEdbInfoRelationListByChildEdbInfoId 根据间接引用中的的计算指标ID查询引用列表
+func GetEdbInfoRelationListByChildEdbInfoId(edbInfoId int) (items []*EdbInfoRelation, err error) {
+	o := orm.NewOrm()
+	msql := ` SELECT * FROM edb_info_relation WHERE relation_type=1 AND child_edb_info_id=?`
+	_, err = o.Raw(msql, edbInfoId).QueryRows(&items)
+	return
+}
+
+// GetEdbInfoRelationListByParentRelationId 根据间接引用中的的父级ID查询
+func GetEdbInfoRelationEdbIdsByParentRelationId(relationId, edbInfoId int) (items []int, err error) {
+	o := orm.NewOrm()
+	msql := ` SELECT edb_info_id FROM edb_info_relation WHERE parent_relation_id=? AND child_edb_info_id=?`
+	_, err = o.Raw(msql, relationId, edbInfoId).QueryRows(&items)
+	return
+}
+
+// GetEdbInfoRelationByRelationIds 查询引用的指标ID
+func GetEdbInfoRelationByRelationIds(ids []int) (items []*EdbInfoRelation, err error) {
+	o := orm.NewOrm()
+	msql := ` SELECT * FROM edb_info_relation WHERE edb_info_relation_id in (` + utils.GetOrmInReplace(len(ids)) + `) `
+	_, err = o.Raw(msql, ids).QueryRows(&items)
+	return
+}
+
+// UpdateSecondRelationEdbInfoId 更新指标替换后的间接引用记录
+func UpdateSecondRelationEdbInfoId(edbRelationIds []int, relationList []*EdbInfoRelation, refreshEdbInfoIds []int, indexCodeList []string) (err error) {
+	o, err := orm.NewOrm().Begin()
+	if err != nil {
+		return
+	}
+	defer func() {
+		if err != nil {
+			_ = o.Rollback()
+			return
+		}
+		_ = o.Commit()
+	}()
+	// 删除相关的间接引用
+	sql := ` DELETE FROM edb_info_relation WHERE relation_type=1 and parent_relation_id in (` + utils.GetOrmInReplace(len(edbRelationIds)) + `)`
+	_, err = o.Raw(sql, edbRelationIds).Exec()
+	if err != nil {
+		return
+	}
+
+	// 新增间接引用
+	relationCodesMap := make(map[string]struct{}, 0)
+	if len(relationList) > 0 {
+		for _, relation := range relationList {
+			if relation.RelationType == 1 {
+				relationCodesMap[relation.RelationCode] = struct{}{}
+			}
+		}
+		_, err = o.InsertMulti(len(relationList), relationList)
+		if err != nil {
+			return
+		}
+	}
+
+	if len(refreshEdbInfoIds) > 0 {
+		// todo 更新指标的刷新状态
+		sql = ` UPDATE edb_info SET no_update = 0 WHERE  edb_info_id IN (` + utils.GetOrmInReplace(len(refreshEdbInfoIds)) + `) AND no_update = 1`
+		_, err = o.Raw(sql, refreshEdbInfoIds).Exec()
+		if err != nil {
+			return
+		}
+	}
+
+	//更新数据源钢联化工指标
+	if len(indexCodeList) > 0 {
+		// 更改数据源的更新状态
+		sql = ` UPDATE base_from_mysteel_chemical_index SET is_stop = 0 WHERE index_code IN (` + utils.GetOrmInReplace(len(indexCodeList)) + `) and is_stop=1`
+		_, err = o.Raw(sql, indexCodeList).Exec()
+		if err != nil {
+			return
+		}
+	}
+	if len(relationList) > 0 {
+		// 更新间接引用指标的关联ID
+		relationCodes := make([]string, 0)
+		for relationCode := range relationCodesMap {
+			relationCodes = append(relationCodes, relationCode)
+		}
+		if len(relationCodes) > 0 {
+			sql := ` UPDATE edb_info_relation e1  
+JOIN edb_info_relation e2 ON e1.relation_code = e2.relation_code   
+SET e1.parent_relation_id = e2.edb_info_relation_id  
+WHERE  
+    e1.relation_type = 1   
+    AND e2.relation_type = 0 AND e1.parent_relation_id !=e2.edb_info_relation_id AND e1.relation_code in (` + utils.GetOrmInReplace(len(relationCodes)) + `)`
+			_, err = o.Raw(sql, relationCodes).Exec()
+			if err != nil {
+				return
+			}
+		}
+	}
+	return
+}

models/edb_ths_hf.go

@@ -0,0 +1,924 @@
+package models
+
+import (
+	"encoding/json"
+	"eta/eta_index_lib/models/mgo"
+	"eta/eta_index_lib/utils"
+	"fmt"
+	"github.com/beego/beego/v2/client/orm"
+	"github.com/shopspring/decimal"
+	"go.mongodb.org/mongo-driver/bson"
+	"reflect"
+	"sort"
+	"time"
+)
+
+// EdbThsHf 自有数据
+type EdbThsHf struct{}
+
+// GetSource 获取来源编码id
+func (obj EdbThsHf) GetSource() int {
+	return utils.DATA_SOURCE_THS
+}
+
+// GetSubSource 获取子来源编码id
+func (obj EdbThsHf) GetSubSource() int {
+	return utils.DATA_SUB_SOURCE_HIGH_FREQUENCY
+}
+
+// GetSourceName 获取来源名称
+func (obj EdbThsHf) GetSourceName() string {
+	return utils.DATA_SOURCE_NAME_THS
+}
+
+// GetEdbType 获取指标类型
+func (obj EdbThsHf) GetEdbType() int {
+	return utils.DEFAULT_EDB_TYPE
+}
+
+// ThsHfAddBaseParams
+// @Description: 基础指标的添加参数
+type ThsHfAddBaseParams struct {
+	EdbCode         string `description:"指标编码"`
+	EdbName         string `description:"指标名称"`
+	Unit            string `description:"单位"`
+	Frequency       string `description:"频度"`
+	Sort            int    `description:"排序"`
+	ClassifyId      int    `description:"所属分类"`
+	SysUserId       int    `description:"用户id"`
+	SysUserRealName string `description:"用户真实名称"`
+	UniqueCode      string `description:"唯一编码"`
+	ConvertRule     string `description:"转换规则"`
+}
+
+// ThsHfEditBaseParams
+// @Description: 基础指标的修改参数
+type ThsHfEditBaseParams struct {
+	EdbCode         string   `description:"指标编码"`
+	EdbName         string   `description:"指标名称"`
+	EdbNameEn       string   `description:"指标名称（英文）"`
+	Unit            string   `description:"单位"`
+	UnitEn          string   `description:"单位（英文）"`
+	ClassifyId      int      `description:"所属分类"`
+	SysUserId       int      `description:"用户id"`
+	SysUserRealName string   `description:"用户真实名称"`
+	UniqueCode      string   `description:"编码"`
+	Lang            string   `description:"语言版本"`
+	EdbInfo         *EdbInfo `description:"指标信息"`
+}
+
+type ThsHfRefreshBaseParams struct {
+	EdbInfo   *EdbInfo
+	StartDate string
+	EndDate   string
+}
+
+// Add
+// @Description: 添加指标
+func (obj EdbThsHf) Add(params ThsHfAddBaseParams, baseIndex *BaseFromThsHfIndex) (edbInfo *EdbInfo, err error) {
+	o := orm.NewOrm()
+	tx, e := o.Begin()
+	if e != nil {
+		err = fmt.Errorf("orm begin err: %v", e)
+		return
+	}
+	defer func() {
+		if err != nil {
+			_ = tx.Rollback()
+			utils.FileLog.Info(fmt.Sprintf("%s err: %v", reflect.TypeOf(obj).Name(), err))
+			return
+		}
+		_ = tx.Commit()
+	}()
+
+	// 新增指标
+	edbInfo = new(EdbInfo)
+	edbInfo.Source = obj.GetSource()
+	edbInfo.SubSource = obj.GetSubSource()
+	edbInfo.SourceName = obj.GetSourceName()
+	edbInfo.EdbType = obj.GetEdbType()
+	edbInfo.EdbCode = params.EdbCode
+	edbInfo.EdbName = params.EdbName
+	edbInfo.EdbNameEn = params.EdbName
+	edbInfo.EdbNameSource = params.EdbName
+	edbInfo.Frequency = params.Frequency
+	edbInfo.Unit = params.Unit
+	edbInfo.UnitEn = params.Unit
+	edbInfo.StartDate = baseIndex.StartDate.Format(utils.FormatDate) // 默认取源指标的时间, 刷新完成后更新
+	edbInfo.EndDate = baseIndex.EndDate.Format(utils.FormatDate)
+	edbInfo.ClassifyId = params.ClassifyId
+	edbInfo.SysUserId = params.SysUserId
+	edbInfo.SysUserRealName = params.SysUserRealName
+	edbInfo.Sort = params.Sort
+	edbInfo.TerminalCode = baseIndex.TerminalCode
+	edbInfo.UniqueCode = params.UniqueCode
+	edbInfo.CreateTime = time.Now()
+	edbInfo.ModifyTime = time.Now()
+	edbInfoId, e := tx.Insert(edbInfo)
+	if e != nil {
+		err = fmt.Errorf("insert edb err: %v", e)
+		return
+	}
+	edbInfo.EdbInfoId = int(edbInfoId)
+
+	// 新增指标关联
+	edbMapping := new(BaseFromEdbMapping)
+	edbMapping.BaseFromIndexId = baseIndex.BaseFromThsHfIndexId
+	edbMapping.BaseIndexCode = baseIndex.IndexCode
+	edbMapping.EdbInfoId = edbInfo.EdbInfoId
+	edbMapping.EdbCode = edbInfo.EdbCode
+	edbMapping.Source = obj.GetSource()
+	edbMapping.SubSource = obj.GetSubSource()
+	edbMapping.ConvertRule = params.ConvertRule
+	edbMapping.CreateTime = time.Now().Local()
+	edbMapping.ModifyTime = time.Now().Local()
+	edbMappingId, e := tx.Insert(edbMapping)
+	if e != nil {
+		err = fmt.Errorf("insert base edb mapping err: %v", e)
+		return
+	}
+	edbMapping.Id = int(edbMappingId)
+
+	// 刷新数据
+	err = obj.Refresh(edbInfo, edbMapping, "")
+	return
+}
+
+func (obj EdbThsHf) Refresh(edbInfo *EdbInfo, edbBaseMapping *BaseFromEdbMapping, startDate string) (err error) {
+	if utils.UseMongo {
+		return obj.refreshByMongo(edbInfo, edbBaseMapping, startDate)
+	}
+	return obj.refreshByMysql(edbInfo, edbBaseMapping, startDate)
+}
+
+func (obj EdbThsHf) refreshByMysql(edbInfo *EdbInfo, edbBaseMapping *BaseFromEdbMapping, startDate string) (err error) {
+	if edbInfo == nil || edbBaseMapping == nil {
+		err = fmt.Errorf("指标信息/关联信息有误, EdbInfo: %v, EdbBaseMapping: %v", edbInfo, edbBaseMapping)
+		return
+	}
+
+	// 真实数据的最大日期, 插入规则配置的日期
+	var realDataMaxDate, edbDataInsertConfigDate time.Time
+	var edbDataInsertConfig *EdbDataInsertConfig
+	var isFindConfigDateRealData bool
+	{
+		conf, e := GetEdbDataInsertConfigByEdbId(edbInfo.EdbInfoId)
+		if e != nil && e.Error() != utils.ErrNoRow() {
+			err = fmt.Errorf("GetEdbDataInsertConfigByEdbId err: %v", e)
+			return
+		}
+		edbDataInsertConfig = conf
+		if edbDataInsertConfig != nil {
+			edbDataInsertConfigDate = edbDataInsertConfig.Date
+		}
+	}
+
+	// 查询时间为开始时间-3d
+	var queryDate string
+	if startDate != "" {
+		st, e := time.ParseInLocation(utils.FormatDate, startDate, time.Local)
+		if e != nil {
+			err = fmt.Errorf("刷新开始时间有误, %v", e)
+			return
+		}
+		queryDate = st.AddDate(0, 0, -3).Format(utils.FormatDate)
+	}
+
+	// 源指标数据
+	baseDataList := make([]*BaseFromThsHfData, 0)
+	{
+		ob := new(BaseFromThsHfData)
+		cond := fmt.Sprintf(" AND %s = ?", ob.Cols().IndexCode)
+		pars := make([]interface{}, 0)
+		pars = append(pars, edbBaseMapping.BaseIndexCode)
+		if queryDate != "" {
+			cond += fmt.Sprintf(" AND %s >= ?", ob.Cols().DataTime)
+			pars = append(pars, queryDate)
+		}
+		list, e := ob.GetItemsByCondition(cond, pars, []string{}, fmt.Sprintf("%s ASC", ob.Cols().DataTime))
+		if e != nil {
+			err = fmt.Errorf("获取数据源数据失败, %v", e)
+			return
+		}
+		baseDataList = list
+	}
+
+	// 转换数据
+	convertRule := new(ThsHfIndexConvert2EdbRule)
+	if e := json.Unmarshal([]byte(edbBaseMapping.ConvertRule), &convertRule); e != nil {
+		err = fmt.Errorf("转换规则有误, %v", e)
+		return
+	}
+	convertOriginData := make([]*ThsHfConvertOriginData, 0)
+	for _, v := range baseDataList {
+		convertOriginData = append(convertOriginData, &ThsHfConvertOriginData{
+			DataTime: v.DataTime,
+			Value:    v.Value,
+		})
+	}
+	convertData, e := ThsHfConvertData2DayByRule(convertOriginData, convertRule)
+	if e != nil {
+		err = fmt.Errorf("转换数据失败, %v", e)
+		return
+	}
+	if len(convertData) == 0 {
+		utils.FileLog.Info("同花顺高频-转换无数据, EdbCode: %s", edbInfo.EdbCode)
+		return
+	}
+
+	// 获取已有数据
+	dataOb := new(EdbDataThsHf)
+	dataExists := make(map[string]*EdbDataThsHf)
+	searchExistMap := make(map[string]*EdbInfoSearchData)
+	{
+		cond := fmt.Sprintf(" AND %s = ?", dataOb.Cols().EdbInfoId)
+		pars := make([]interface{}, 0)
+		pars = append(pars, edbInfo.EdbInfoId)
+		if queryDate != "" {
+			cond += fmt.Sprintf(" AND %s >= ?", dataOb.Cols().DataTime)
+			pars = append(pars, queryDate)
+		}
+		list, e := dataOb.GetItemsByCondition(cond, pars, []string{}, "")
+		if e != nil {
+			err = fmt.Errorf("获取指标数据失败, %v", e)
+			return
+		}
+		for _, v := range list {
+			dataExists[v.DataTime.Format(utils.FormatDate)] = v
+			searchExistMap[v.DataTime.Format(utils.FormatDate)] = &EdbInfoSearchData{
+				EdbDataId:     v.EdbDataId,
+				EdbInfoId:     v.EdbInfoId,
+				DataTime:      v.DataTime.Format(utils.FormatDate),
+				Value:         v.Value,
+				EdbCode:       v.EdbCode,
+				DataTimestamp: v.DataTimestamp,
+			}
+		}
+	}
+
+	// 比对数据
+	insertExist := make(map[string]bool)
+	insertData := make([]*EdbDataThsHf, 0)
+	updateData := make([]*EdbDataThsHf, 0)
+	for k, v := range convertData {
+		strDate := k.Format(utils.FormatDate)
+
+		// 手动插入数据的判断
+		if realDataMaxDate.IsZero() || k.After(realDataMaxDate) {
+			realDataMaxDate = k
+		}
+		if edbDataInsertConfigDate.IsZero() || k.Equal(edbDataInsertConfigDate) {
+			isFindConfigDateRealData = true
+		}
+
+		// 入库值
+		saveVal := decimal.NewFromFloat(v).Round(4).String()
+		d, e := decimal.NewFromString(saveVal)
+		if e != nil {
+			utils.FileLog.Info(fmt.Sprintf("EdbDataThsHf NewFromString err: %v", e))
+			continue
+		}
+		saveFloat, _ := d.Float64()
+
+		// 更新
+		exists := dataExists[strDate]
+		if exists != nil {
+			existVal := decimal.NewFromFloat(exists.Value).Round(4).String()
+			if saveVal != existVal {
+				exists.Value = saveFloat
+				updateData = append(updateData, exists)
+			}
+			continue
+		}
+
+		// 新增
+		if insertExist[strDate] {
+			continue
+		}
+		insertExist[strDate] = true
+
+		timestamp := k.UnixNano() / 1e6
+		insertData = append(insertData, &EdbDataThsHf{
+			EdbInfoId:     edbInfo.EdbInfoId,
+			EdbCode:       edbInfo.EdbCode,
+			DataTime:      k,
+			Value:         saveFloat,
+			CreateTime:    time.Now(),
+			ModifyTime:    time.Now(),
+			DataTimestamp: timestamp,
+		})
+	}
+
+	// 批量新增/更新
+	if len(insertData) > 0 {
+		if e = dataOb.CreateMulti(insertData); e != nil {
+			err = fmt.Errorf("批量新增指标数据失败, %v", e)
+			return
+		}
+	}
+	if len(updateData) > 0 {
+		if e = dataOb.MultiUpdateValue(updateData); e != nil {
+			err = fmt.Errorf("批量更新指标数据失败, %v", e)
+			return
+		}
+	}
+
+	// 处理手工数据补充的配置
+	HandleConfigInsertEdbData(realDataMaxDate, edbDataInsertConfig, edbInfo.EdbInfoId, obj.GetSource(), obj.GetSubSource(), searchExistMap, isFindConfigDateRealData)
+	return
+}
+
+func (obj EdbThsHf) refreshByMongo(edbInfo *EdbInfo, edbBaseMapping *BaseFromEdbMapping, startDate string) (err error) {
+	defer func() {
+		if err != nil {
+			utils.FileLog.Info(fmt.Sprintf("refreshByMongo, err: %v", err))
+		}
+	}()
+
+	var realDataMaxDate, edbDataInsertConfigDate time.Time
+	var edbDataInsertConfig *EdbDataInsertConfig
+	var isFindConfigDateRealData bool //是否找到配置日期的实际数据的值
+	{
+		insertConfig, e := GetEdbDataInsertConfigByEdbId(edbInfo.EdbInfoId)
+		if e != nil && e.Error() != utils.ErrNoRow() {
+			err = fmt.Errorf("GetEdbDataInsertConfigByEdbId, err: %v", e)
+			return
+		}
+		edbDataInsertConfig = insertConfig
+		if edbDataInsertConfig != nil {
+			edbDataInsertConfigDate = edbDataInsertConfig.Date
+		}
+	}
+
+	// 查询时间为开始时间-3d
+	var queryDate string
+	if startDate != "" {
+		st, e := time.ParseInLocation(utils.FormatDate, startDate, time.Local)
+		if e != nil {
+			err = fmt.Errorf("刷新开始时间有误, %v", e)
+			return
+		}
+		queryDate = st.AddDate(0, 0, -3).Format(utils.FormatDate)
+	}
+
+	// 获取源指标数据
+	baseDataList, e := obj.getBaseIndexDataByMongo(edbBaseMapping.BaseIndexCode, queryDate)
+	if e != nil {
+		err = fmt.Errorf("getBaseIndexDataByMongo, err: %v", e)
+		return
+	}
+
+	// 转换数据
+	convertRule := new(ThsHfIndexConvert2EdbRule)
+	if e := json.Unmarshal([]byte(edbBaseMapping.ConvertRule), &convertRule); e != nil {
+		err = fmt.Errorf("转换规则有误, %v", e)
+		return
+	}
+	convertOriginData := make([]*ThsHfConvertOriginData, 0)
+	for _, v := range baseDataList {
+		convertOriginData = append(convertOriginData, &ThsHfConvertOriginData{
+			DataTime: v.DataTime,
+			Value:    v.Value,
+		})
+	}
+	convertData, e := ThsHfConvertData2DayByRule(convertOriginData, convertRule)
+	if e != nil {
+		err = fmt.Errorf("转换数据失败, %v", e)
+		return
+	}
+	if len(convertData) == 0 {
+		utils.FileLog.Info("同花顺高频-转换无数据, EdbCode: %s", edbInfo.EdbCode)
+		return
+	}
+
+	//获取指标所有数据
+	existDataList := make([]*mgo.EdbDataThsHf, 0)
+	mogDataObj := new(mgo.EdbDataThsHf)
+	{
+		// 构建查询条件
+		queryConditions := bson.M{
+			"edb_code": edbInfo.EdbCode,
+		}
+
+		if queryDate != `` {
+			//获取已存在的所有数据
+			startDateTime, e := time.ParseInLocation(utils.FormatDate, queryDate, time.Local)
+			if e != nil {
+				err = fmt.Errorf("startDateTime parse err: %v", e)
+				return
+			}
+			queryConditions["data_time"] = bson.M{"$gte": startDateTime}
+		}
+		existDataList, e = mogDataObj.GetAllDataList(queryConditions, []string{"data_time"})
+		if e != nil {
+			err = fmt.Errorf("GetAllDataList, err: %v", e)
+			return
+		}
+	}
+
+	existDataMap := make(map[string]*mgo.EdbDataThsHf)
+	removeDataTimeMap := make(map[string]bool) //需要移除的日期数据
+	for _, v := range existDataList {
+		tmpDate := v.DataTime.Format(utils.FormatDate)
+		existDataMap[tmpDate] = v
+		removeDataTimeMap[tmpDate] = true
+	}
+
+	// 待添加的数据集
+	addDataList := make([]interface{}, 0)
+	updateDataList := make([]mgo.EdbDataThsHf, 0)
+
+	insertExist := make(map[string]bool)
+	for k, v := range convertData {
+		strDate := k.Format(utils.FormatDate)
+
+		// 手动插入数据的判断
+		if realDataMaxDate.IsZero() || k.After(realDataMaxDate) {
+			realDataMaxDate = k
+		}
+		if edbDataInsertConfigDate.IsZero() || k.Equal(edbDataInsertConfigDate) {
+			isFindConfigDateRealData = true
+		}
+
+		// 入库值
+		saveVal := decimal.NewFromFloat(v).Round(4).String()
+		d, e := decimal.NewFromString(saveVal)
+		if e != nil {
+			utils.FileLog.Info(fmt.Sprintf("EdbDataThsHf NewFromString err: %v", e))
+			continue
+		}
+		saveFloat, _ := d.Float64()
+
+		// 更新
+		exists := existDataMap[strDate]
+		if exists != nil {
+			existVal := decimal.NewFromFloat(exists.Value).Round(4).String()
+			if saveVal != existVal {
+				exists.Value = saveFloat
+				updateDataList = append(updateDataList, *exists)
+			}
+			continue
+		}
+
+		// 新增
+		if insertExist[strDate] {
+			continue
+		}
+		insertExist[strDate] = true
+
+		timestamp := k.UnixNano() / 1e6
+		addDataList = append(addDataList, &mgo.EdbDataThsHf{
+			EdbInfoId:     edbInfo.EdbInfoId,
+			EdbCode:       edbInfo.EdbCode,
+			DataTime:      k,
+			Value:         saveFloat,
+			CreateTime:    time.Now(),
+			ModifyTime:    time.Now(),
+			DataTimestamp: timestamp,
+		})
+	}
+
+	// 入库
+	{
+		coll := mogDataObj.GetCollection()
+
+		//删除已经不存在的指标数据（由于该指标当日的数据删除了）
+		{
+			removeDateList := make([]time.Time, 0)
+			for dateTime := range removeDataTimeMap {
+				//获取已存在的所有数据
+				tmpDateTime, e := time.ParseInLocation(utils.FormatDate, dateTime, time.Local)
+				if e != nil {
+					err = fmt.Errorf("tmpDateTime parse err: %v", e)
+					return
+				}
+				removeDateList = append(removeDateList, tmpDateTime)
+			}
+			removeNum := len(removeDateList)
+			if removeNum > 0 {
+				if e = mogDataObj.RemoveManyByColl(coll, bson.M{"edb_code": edbInfo.EdbCode, "data_time": bson.M{"$in": removeDateList}}); e != nil {
+					err = fmt.Errorf("RemoveManyByColl, err: %v", e)
+					return
+				}
+			}
+		}
+
+		// 插入新数据
+		if len(addDataList) > 0 {
+			if e = mogDataObj.BatchInsertDataByColl(coll, 500, addDataList); e != nil {
+				err = fmt.Errorf("BatchInsertDataByColl, err: %v", e)
+				return
+			}
+		}
+
+		// 修改历史数据
+		if len(updateDataList) > 0 {
+			for _, v := range updateDataList {
+				if e = mogDataObj.UpdateDataByColl(coll, bson.M{"_id": v.ID}, bson.M{"$set": bson.M{"value": v.Value, "modify_time": v.ModifyTime}}); e != nil {
+					err = fmt.Errorf("UpdateDataByColl, err: %v", e)
+					return
+				}
+			}
+		}
+	}
+
+	// 处理手工数据补充的配置
+	obj.HandleConfigInsertEdbDataByMongo(realDataMaxDate, edbDataInsertConfig, edbInfo.EdbInfoId, edbInfo.Source, edbInfo.SubSource, existDataMap, isFindConfigDateRealData)
+	return
+}
+
+type ThsHfConvertOriginData struct {
+	DataTime time.Time `description:"数据日期（至时分秒）"`
+	Value    float64   `description:"数据值"`
+}
+
+// ThsHfConvertData2DayByRule 原指标数据转换为日度数据
+func ThsHfConvertData2DayByRule(originData []*ThsHfConvertOriginData, convertRule *ThsHfIndexConvert2EdbRule) (timeData map[time.Time]float64, err error) {
+	// PS: originData为期望开始日期前三日(有两天非交易日, 那么周一的前日应当算上周五的)至结束日期的数据
+	timeData = make(map[time.Time]float64)
+	if len(originData) == 0 || convertRule == nil {
+		return
+	}
+	if !utils.InArrayByInt([]int{1, 2}, convertRule.ConvertType) {
+		err = fmt.Errorf("取值类型有误, ConvertType: %d", convertRule.ConvertType)
+		return
+	}
+
+	// 升序排序
+	sort.Slice(originData, func(i, j int) bool {
+		return originData[i].DataTime.Before(originData[j].DataTime)
+	})
+
+	// 将数据根据日期进行分组
+	var sortDates []string
+	groupDateData := make(map[string][]*ThsHfConvertOriginData)
+	for _, v := range originData {
+		d := v.DataTime.Format(utils.FormatDate)
+		if !utils.InArrayByStr(sortDates, d) {
+			sortDates = append(sortDates, d)
+		}
+		if groupDateData[d] == nil {
+			groupDateData[d] = make([]*ThsHfConvertOriginData, 0)
+		}
+		groupDateData[d] = append(groupDateData[d], v)
+	}
+
+	// 取值方式-指定时间的值
+	if convertRule.ConvertType == 1 {
+		for k, v := range sortDates {
+			todayTime, e := time.ParseInLocation(utils.FormatDate, v, time.Local)
+			if e != nil {
+				utils.FileLog.Info("当日日期转换有误, date: %s, err: %v", v, e)
+				continue
+			}
+			var timeTarget time.Time
+			dateData := make([]*ThsHfConvertOriginData, 0)
+
+			// 当日
+			if convertRule.ConvertFixed.FixedDay == 1 {
+				tg, e := time.ParseInLocation(utils.FormatDateTime, fmt.Sprintf("%s %s", v, convertRule.ConvertFixed.FixedTime), time.Local)
+				if e != nil {
+					utils.FileLog.Info(fmt.Sprintf("当日timeTarget转换有误, %v", e))
+					continue
+				}
+				timeTarget = tg
+
+				dt := groupDateData[v]
+				if dt == nil {
+					utils.FileLog.Info(fmt.Sprintf("%s当日无数据", v))
+					continue
+				}
+				if len(dt) == 0 {
+					continue
+				}
+				dateData = dt
+			}
+
+			// 前一日
+			if convertRule.ConvertFixed.FixedDay == 2 {
+				if k < 1 {
+					utils.FileLog.Info(fmt.Sprintf("%s前日无数据", v))
+					continue
+				}
+				preDate := sortDates[k-1]
+
+				tg, e := time.ParseInLocation(utils.FormatDateTime, fmt.Sprintf("%s %s", preDate, convertRule.ConvertFixed.FixedTime), time.Local)
+				if e != nil {
+					utils.FileLog.Info(fmt.Sprintf("前日timeTarget转换有误, %v", e))
+					continue
+				}
+				timeTarget = tg
+
+				dt := groupDateData[preDate]
+				if dt == nil {
+					utils.FileLog.Info(fmt.Sprintf("%s前日无数据", v))
+					continue
+				}
+				if len(dt) == 0 {
+					continue
+				}
+				dateData = dt
+			}
+			if len(dateData) == 0 {
+				utils.FileLog.Info("日期%s无数据序列", v)
+				continue
+			}
+
+			// 重新获取数据序列中, 时间在目标时间点之后的
+			newDateData := make([]*ThsHfConvertOriginData, 0)
+			for kv, dv := range dateData {
+				if dv.DataTime.Before(timeTarget) {
+					continue
+				}
+				// 由于升序排列, 直接取之后所有的数据
+				newDateData = append(newDateData, dateData[kv:]...)
+				break
+			}
+
+			// 取重组后当日数据中的第一个(有可能目标时间点无值, 那么取之后时间最近的值)
+			if len(newDateData) == 0 {
+				utils.FileLog.Info("日期%s无有效数据", v)
+				continue
+			}
+			timeData[todayTime] = newDateData[0].Value
+		}
+		return
+	}
+
+	// 取值方式-区间计算值
+	for k, v := range sortDates {
+		todayTime, e := time.ParseInLocation(utils.FormatDate, v, time.Local)
+		if e != nil {
+			utils.FileLog.Info("当日日期转换有误, date: %s, err: %v", v, e)
+			continue
+		}
+		var thisDate, preDate string
+		thisDate = v
+		if k > 1 {
+			preDate = sortDates[k-1]
+		}
+		var startTimeTarget, endTimeTarget time.Time
+
+		// 起始时间-当日/前一日
+		if convertRule.ConvertArea.StartDay == 1 {
+			tg, e := time.ParseInLocation(utils.FormatDateTime, fmt.Sprintf("%s %s", thisDate, convertRule.ConvertArea.StartTime), time.Local)
+			if e != nil {
+				utils.FileLog.Info(fmt.Sprintf("当日startTimeTarget转换有误, %v", e))
+				continue
+			}
+			startTimeTarget = tg
+		}
+		if convertRule.ConvertArea.StartDay == 2 {
+			if preDate == "" {
+				utils.FileLog.Info(fmt.Sprintf("%s前日无数据", v))
+				continue
+			}
+			tg, e := time.ParseInLocation(utils.FormatDateTime, fmt.Sprintf("%s %s", preDate, convertRule.ConvertArea.StartTime), time.Local)
+			if e != nil {
+				utils.FileLog.Info(fmt.Sprintf("前日startTimeTarget转换有误, %v", e))
+				continue
+			}
+			startTimeTarget = tg
+		}
+
+		// 截止时间-当日/前一日
+		if convertRule.ConvertArea.EndDay == 1 {
+			tg, e := time.ParseInLocation(utils.FormatDateTime, fmt.Sprintf("%s %s", thisDate, convertRule.ConvertArea.EndTime), time.Local)
+			if e != nil {
+				utils.FileLog.Info(fmt.Sprintf("当日endTimeTarget转换有误, %v", e))
+				continue
+			}
+			endTimeTarget = tg
+		}
+		if convertRule.ConvertArea.EndDay == 2 {
+			if preDate == "" {
+				utils.FileLog.Info(fmt.Sprintf("%s前日无数据", v))
+				continue
+			}
+			tg, e := time.ParseInLocation(utils.FormatDateTime, fmt.Sprintf("%s %s", preDate, convertRule.ConvertArea.EndTime), time.Local)
+			if e != nil {
+				utils.FileLog.Info(fmt.Sprintf("前日endTimeTarget转换有误, %v", e))
+				continue
+			}
+			endTimeTarget = tg
+		}
+		if startTimeTarget.IsZero() || endTimeTarget.IsZero() {
+			utils.FileLog.Info(fmt.Sprintf("起始截止时间有误, start: %v, end: %v", startTimeTarget, endTimeTarget))
+			continue
+		}
+
+		// 合并前日当日数据
+		dateData := make([]*ThsHfConvertOriginData, 0)
+		if convertRule.ConvertArea.StartDay == 1 && convertRule.ConvertArea.EndDay == 1 {
+			// 起始截止均为当日
+			dateData = groupDateData[thisDate]
+			if dateData == nil {
+				utils.FileLog.Info(fmt.Sprintf("%s当日无数据", thisDate))
+				continue
+			}
+			if len(dateData) == 0 {
+				utils.FileLog.Info(fmt.Sprintf("%s当日无数据", thisDate))
+				continue
+			}
+		} else {
+			if preDate == "" {
+				continue
+			}
+			// 起始截止时间含前日
+			preData := groupDateData[preDate]
+			if preData == nil {
+				utils.FileLog.Info(fmt.Sprintf("%s前日无数据", thisDate))
+				continue
+			}
+			if len(preData) == 0 {
+				utils.FileLog.Info(fmt.Sprintf("%s前日无数据", thisDate))
+				continue
+			}
+			thisData := groupDateData[thisDate]
+			if thisData == nil {
+				utils.FileLog.Info(fmt.Sprintf("%s当日无数据", thisDate))
+				continue
+			}
+			if len(thisData) == 0 {
+				utils.FileLog.Info(fmt.Sprintf("%s当日无数据", thisDate))
+				continue
+			}
+			dateData = append(dateData, preData...)
+			dateData = append(dateData, thisData...)
+		}
+		if len(dateData) == 0 {
+			utils.FileLog.Info("日期%s无数据序列", v)
+			continue
+		}
+
+		// 重组时间区间内的数据
+		newDateData := make([]*ThsHfConvertOriginData, 0)
+		for _, dv := range dateData {
+			if dv.DataTime.Before(startTimeTarget) || dv.DataTime.After(endTimeTarget) {
+				continue
+			}
+			newDateData = append(newDateData, dv)
+		}
+		if len(newDateData) == 0 {
+			utils.FileLog.Info(fmt.Sprintf("时间区间内无数据, start: %v, end: %v", startTimeTarget, endTimeTarget))
+			continue
+		}
+
+		// 取出区间内的均值/最值
+		var avgVal, minVal, maxVal, sumVal float64
+		minVal, maxVal = newDateData[0].Value, newDateData[0].Value
+		for _, nv := range newDateData {
+			sumVal += nv.Value
+			if nv.Value > maxVal {
+				maxVal = nv.Value
+			}
+			if nv.Value < minVal {
+				minVal = nv.Value
+			}
+		}
+		avgVal = sumVal / float64(len(newDateData))
+
+		switch convertRule.ConvertArea.CalculateType {
+		case 1:
+			timeData[todayTime] = avgVal
+		case 2:
+			timeData[todayTime] = maxVal
+		case 3:
+			timeData[todayTime] = minVal
+		default:
+			utils.FileLog.Info(fmt.Sprintf("计算方式有误, CalculateType: %d", convertRule.ConvertArea.CalculateType))
+		}
+	}
+	return
+}
+
+func (obj EdbThsHf) getBaseIndexDataByMongo(baseIndexCode, startDate string) (newDataList []EdbInfoMgoData, err error) {
+	newDataList = make([]EdbInfoMgoData, 0)
+
+	// 获取数据源的指标数据
+	mogDataObj := new(mgo.BaseFromThsHfData)
+
+	// 构建查询条件
+	queryConditions := bson.M{
+		"index_code": baseIndexCode,
+	}
+
+	if startDate != `` {
+		//获取已存在的所有数据
+		startDateTime, tmpErr := time.ParseInLocation(utils.FormatDate, startDate, time.Local)
+		if tmpErr != nil {
+			err = tmpErr
+			return
+		}
+		queryConditions["data_time"] = bson.M{"$gte": startDateTime}
+	}
+
+	baseDataList, err := mogDataObj.GetAllDataList(queryConditions, []string{"data_time"})
+	if err != nil {
+		fmt.Println("GetAllDataList Err:" + err.Error())
+		return
+	}
+
+	for _, v := range baseDataList {
+		newDataList = append(newDataList, EdbInfoMgoData{
+			//EdbDataId: v.ID,
+			DataTime: v.DataTime,
+			Value:    v.Value,
+			EdbCode:  v.IndexCode,
+		})
+	}
+	return
+}
+
+func (obj EdbThsHf) HandleConfigInsertEdbDataByMongo(realDataMaxDate time.Time, edbDataInsertConfig *EdbDataInsertConfig, edbInfoId, source, subSource int, existMap map[string]*mgo.EdbDataThsHf, isFindConfigDateRealData bool) {
+	if edbDataInsertConfig == nil {
+		return
+	}
+	var err error
+	defer func() {
+		if err != nil {
+			utils.FileLog.Info(fmt.Sprintf("ThsHf-HandleConfigInsertEdbDataByMongo, err: %v", err))
+		}
+	}()
+
+	edbDataInsertConfigDate := edbDataInsertConfig.Date // 配置的日期
+
+	// 如果存在真实数据的最大日期  && 存在配置插入数据的最大日期  && 真实数据的最大日期 晚于/等于 配置插入数据的最大日期
+	if realDataMaxDate.After(edbDataInsertConfigDate) || realDataMaxDate.Equal(edbDataInsertConfigDate) {
+		go DeleteEdbDataInsertConfigByEdbId(edbInfoId)
+
+		mogDataObj := mgo.EdbDataThsHf{}
+		coll := mogDataObj.GetCollection()
+		edbDataInsertConfigDateStr := edbDataInsertConfigDate.Format(utils.FormatDate)
+		// 如果没有找到找到配置日期的实际数据，那么就直接删除
+		if item, ok := existMap[edbDataInsertConfigDateStr]; ok && !isFindConfigDateRealData {
+			mogDataObj.RemoveManyByColl(coll, bson.M{"_id": item.ID})
+		}
+	} else {
+		o := orm.NewOrm()
+		edbDataInsertConfig.RealDate = realDataMaxDate
+		_, err = o.Update(edbDataInsertConfig, "RealDate")
+	}
+	return
+}
+
+func (obj EdbThsHf) UnifiedModifyEdbInfoMaxAndMinInfo(edbInfo *EdbInfo) (err error) {
+	if utils.UseMongo {
+		edbInfoMaxAndMinInfo, tmpErr := obj.getEdbInfoMaxAndMinInfoByMongo(edbInfo.EdbCode)
+		// 如果正常获取到了，那就去修改指标的最大最小值
+		if tmpErr == nil && edbInfoMaxAndMinInfo != nil {
+			err = ModifyEdbInfoMaxAndMinInfo(edbInfo.EdbInfoId, edbInfoMaxAndMinInfo)
+		} else {
+			// 清空的目的是为了避免异常返回
+			err = nil
+		}
+	} else {
+		err, _ = UnifiedModifyEdbInfoMaxAndMinInfo(edbInfo)
+	}
+
+	return
+}
+
+func (obj EdbThsHf) getEdbInfoMaxAndMinInfoByMongo(edbCode string) (item *EdbInfoMaxAndMinInfo, err error) {
+	mogDataObj := new(mgo.EdbDataThsHf)
+	pipeline := []bson.M{
+		{"$match": bson.M{"edb_code": edbCode}},
+		{"$group": bson.M{
+			"_id":       nil,
+			"min_date":  bson.M{"$min": "$data_time"},
+			"max_date":  bson.M{"$max": "$data_time"},
+			"min_value": bson.M{"$min": "$value"},
+			"max_value": bson.M{"$max": "$value"},
+		}},
+		{"$project": bson.M{"_id": 0}}, // 可选，如果不需要_id字段
+	}
+	result, err := mogDataObj.GetEdbInfoMaxAndMinInfo(pipeline)
+	if err != nil {
+		fmt.Println("EdbDataBusiness getEdbDataBusinessList Err:" + err.Error())
+		return
+	}
+
+	if !result.MaxDate.IsZero() {
+		whereQuery := bson.M{"edb_code": edbCode, "data_time": result.MaxDate}
+		selectParam := bson.D{{"value", 1}, {"_id", 0}}
+		latestValue, tmpErr := mogDataObj.GetLatestValue(whereQuery, selectParam)
+		if tmpErr != nil {
+			err = tmpErr
+			return
+		}
+		result.LatestValue = latestValue.Value
+		result.EndValue = latestValue.Value
+	}
+
+	item = &EdbInfoMaxAndMinInfo{
+		MinDate:     result.MinDate.Format(utils.FormatDate),
+		MaxDate:     result.MaxDate.Format(utils.FormatDate),
+		MinValue:    result.MinValue,
+		MaxValue:    result.MaxValue,
+		LatestValue: result.LatestValue,
+		LatestDate:  result.LatestDate.Format(utils.FormatDate),
+		EndValue:    result.EndValue,
+	}
+
+	return
+}

models/factor_edb_series.go

@@ -0,0 +1,363 @@
+package models
+
+import (
+	"encoding/json"
+	"eta/eta_index_lib/utils"
+	"fmt"
+	"github.com/beego/beego/v2/client/orm"
+	"strings"
+	"time"
+)
+
+const (
+	FactorEdbSeriesCalculateNone = 0
+	FactorEdbSeriesCalculating   = 1
+	FactorEdbSeriesCalculated    = 2
+)
+
+// FactorEdbSeries 因子指标系列表
+type FactorEdbSeries struct {
+	FactorEdbSeriesId int       `orm:"column(factor_edb_series_id);pk"`
+	SeriesName        string    `description:"系列名称"`
+	EdbInfoType       int       `description:"关联指标类型：0-普通指标；1-预测指标"`
+	CalculateStep     string    `description:"计算步骤-JSON"`
+	CalculateState    int       `description:"计算状态: 0-无计算; 1-计算中; 2-计算完成"`
+	CreateTime        time.Time `description:"创建时间"`
+	ModifyTime        time.Time `description:"修改时间"`
+}
+
+func (m *FactorEdbSeries) TableName() string {
+	return "factor_edb_series"
+}
+
+type FactorEdbSeriesCols struct {
+	PrimaryId      string
+	SeriesName     string
+	EdbInfoType    string
+	CalculateStep  string
+	CalculateState string
+	CreateTime     string
+	ModifyTime     string
+}
+
+func (m *FactorEdbSeries) Cols() FactorEdbSeriesCols {
+	return FactorEdbSeriesCols{
+		PrimaryId:      "factor_edb_series_id",
+		SeriesName:     "series_name",
+		EdbInfoType:    "edb_info_type",
+		CalculateStep:  "calculate_step",
+		CalculateState: "calculate_state",
+		CreateTime:     "create_time",
+		ModifyTime:     "modify_time",
+	}
+}
+
+func (m *FactorEdbSeries) Create() (err error) {
+	o := orm.NewOrm()
+	id, err := o.Insert(m)
+	if err != nil {
+		return
+	}
+	m.FactorEdbSeriesId = int(id)
+	return
+}
+
+func (m *FactorEdbSeries) CreateMulti(items []*FactorEdbSeries) (err error) {
+	if len(items) == 0 {
+		return
+	}
+	o := orm.NewOrm()
+	_, err = o.InsertMulti(len(items), items)
+	return
+}
+
+func (m *FactorEdbSeries) Update(cols []string) (err error) {
+	o := orm.NewOrm()
+	_, err = o.Update(m, cols...)
+	return
+}
+
+func (m *FactorEdbSeries) Remove() (err error) {
+	o := orm.NewOrm()
+	sql := fmt.Sprintf(`DELETE FROM %s WHERE %s = ? LIMIT 1`, m.TableName(), m.Cols().PrimaryId)
+	_, err = o.Raw(sql, m.FactorEdbSeriesId).Exec()
+	return
+}
+
+func (m *FactorEdbSeries) MultiRemove(ids []int) (err error) {
+	if len(ids) == 0 {
+		return
+	}
+	o := orm.NewOrm()
+	sql := fmt.Sprintf(`DELETE FROM %s WHERE %s IN (%s)`, m.TableName(), m.Cols().PrimaryId, utils.GetOrmInReplace(len(ids)))
+	_, err = o.Raw(sql, ids).Exec()
+	return
+}
+
+func (m *FactorEdbSeries) GetItemById(id int) (item *FactorEdbSeries, err error) {
+	o := orm.NewOrm()
+	sql := fmt.Sprintf(`SELECT * FROM %s WHERE %s = ? LIMIT 1`, m.TableName(), m.Cols().PrimaryId)
+	err = o.Raw(sql, id).QueryRow(&item)
+	return
+}
+
+func (m *FactorEdbSeries) GetItemByCondition(condition string, pars []interface{}, orderRule string) (item *FactorEdbSeries, err error) {
+	o := orm.NewOrm()
+	order := ``
+	if orderRule != "" {
+		order = ` ORDER BY ` + orderRule
+	}
+	sql := fmt.Sprintf(`SELECT * FROM %s WHERE 1=1 %s %s LIMIT 1`, m.TableName(), condition, order)
+	err = o.Raw(sql, pars).QueryRow(&item)
+	return
+}
+
+func (m *FactorEdbSeries) GetCountByCondition(condition string, pars []interface{}) (count int, err error) {
+	o := orm.NewOrm()
+	sql := fmt.Sprintf(`SELECT COUNT(1) FROM %s WHERE 1=1 %s`, m.TableName(), condition)
+	err = o.Raw(sql, pars).QueryRow(&count)
+	return
+}
+
+func (m *FactorEdbSeries) GetItemsByCondition(condition string, pars []interface{}, fieldArr []string, orderRule string) (items []*FactorEdbSeries, err error) {
+	o := orm.NewOrm()
+	fields := strings.Join(fieldArr, ",")
+	if len(fieldArr) == 0 {
+		fields = `*`
+	}
+	order := fmt.Sprintf(`ORDER BY %s DESC`, m.Cols().CreateTime)
+	if orderRule != "" {
+		order = ` ORDER BY ` + orderRule
+	}
+	sql := fmt.Sprintf(`SELECT %s FROM %s WHERE 1=1 %s %s`, fields, m.TableName(), condition, order)
+	_, err = o.Raw(sql, pars).QueryRows(&items)
+	return
+}
+
+func (m *FactorEdbSeries) GetPageItemsByCondition(condition string, pars []interface{}, fieldArr []string, orderRule string, startSize, pageSize int) (items []*FactorEdbSeries, err error) {
+	o := orm.NewOrm()
+	fields := strings.Join(fieldArr, ",")
+	if len(fieldArr) == 0 {
+		fields = `*`
+	}
+	order := fmt.Sprintf(`ORDER BY %s DESC`, m.Cols().CreateTime)
+	if orderRule != "" {
+		order = ` ORDER BY ` + orderRule
+	}
+	sql := fmt.Sprintf(`SELECT %s FROM %s WHERE 1=1 %s %s LIMIT ?,?`, fields, m.TableName(), condition, order)
+	_, err = o.Raw(sql, pars, startSize, pageSize).QueryRows(&items)
+	return
+}
+
+// FactorEdbSeriesItem 多因子系列信息
+type FactorEdbSeriesItem struct {
+	SeriesId      int                            `description:"多因子系列ID"`
+	SeriesName    string                         `description:"系列名称"`
+	EdbInfoType   int                            `description:"关联指标类型：0-普通指标；1-预测指标"`
+	CalculateStep []FactorEdbSeriesCalculatePars `description:"计算步骤-JSON"`
+	CreateTime    string                         `description:"创建时间"`
+	ModifyTime    string                         `description:"修改时间"`
+}
+
+func (m *FactorEdbSeries) Format2Item() (item *FactorEdbSeriesItem) {
+	item = new(FactorEdbSeriesItem)
+	item.SeriesId = m.FactorEdbSeriesId
+	item.SeriesName = m.SeriesName
+	item.EdbInfoType = m.EdbInfoType
+	if m.CalculateStep != "" {
+		_ = json.Unmarshal([]byte(m.CalculateStep), &item.CalculateStep)
+	}
+	item.CreateTime = utils.TimeTransferString(utils.FormatDateTime, m.CreateTime)
+	item.ModifyTime = utils.TimeTransferString(utils.FormatDateTime, m.ModifyTime)
+	return
+}
+
+// FactorEdbSeriesCalculatePars 计算参数
+type FactorEdbSeriesCalculatePars struct {
+	Formula       interface{} `description:"N值/移动天数/指数修匀alpha值/计算公式等"`
+	Calendar      string      `description:"公历/农历"`
+	Frequency     string      `description:"需要转换的频度"`
+	MoveType      int         `description:"移动方式: 1-领先(默认); 2-滞后"`
+	MoveFrequency string      `description:"移动频度"`
+	FromFrequency string      `description:"来源的频度"`
+	Source        int         `description:"计算方式来源(不是指标来源)"`
+	Sort          int         `description:"计算顺序"`
+}
+
+// CreateSeriesAndMapping 新增系列和指标关联
+func (m *FactorEdbSeries) CreateSeriesAndMapping(item *FactorEdbSeries, mappings []*FactorEdbSeriesMapping) (seriesId int, err error) {
+	if item == nil {
+		err = fmt.Errorf("series is nil")
+		return
+	}
+	o := orm.NewOrm()
+	tx, e := o.Begin()
+	if e != nil {
+		err = fmt.Errorf("orm begin err: %v", e)
+		return
+	}
+	defer func() {
+		if err != nil {
+			_ = tx.Rollback()
+			return
+		}
+		_ = tx.Commit()
+	}()
+
+	id, e := tx.Insert(item)
+	if e != nil {
+		err = fmt.Errorf("insert series err: %v", e)
+		return
+	}
+	seriesId = int(id)
+	item.FactorEdbSeriesId = seriesId
+
+	if len(mappings) > 0 {
+		for _, v := range mappings {
+			v.FactorEdbSeriesId = seriesId
+		}
+		_, e = tx.InsertMulti(200, mappings)
+		if e != nil {
+			err = fmt.Errorf("insert multi mapping err: %v", e)
+			return
+		}
+	}
+	return
+}
+
+// EditSeriesAndMapping 编辑系列和指标关联
+func (m *FactorEdbSeries) EditSeriesAndMapping(item *FactorEdbSeries, mappings []*FactorEdbSeriesMapping, updateCols []string) (err error) {
+	if item == nil {
+		err = fmt.Errorf("series is nil")
+		return
+	}
+	o := orm.NewOrm()
+	tx, e := o.Begin()
+	if e != nil {
+		err = fmt.Errorf("orm begin err: %v", e)
+		return
+	}
+	defer func() {
+		if err != nil {
+			_ = tx.Rollback()
+			return
+		}
+		_ = tx.Commit()
+	}()
+
+	_, e = tx.Update(item, updateCols...)
+	if e != nil {
+		err = fmt.Errorf("update series err: %v", e)
+		return
+	}
+
+	// 清除原指标关联
+	mappingOb := new(FactorEdbSeriesMapping)
+	cond := fmt.Sprintf("%s = ?", mappingOb.Cols().FactorEdbSeriesId)
+	pars := make([]interface{}, 0)
+	pars = append(pars, item.FactorEdbSeriesId)
+	sql := fmt.Sprintf(`DELETE FROM %s WHERE %s`, mappingOb.TableName(), cond)
+	_, e = tx.Raw(sql, pars).Exec()
+	if e != nil {
+		err = fmt.Errorf("remove mapping err: %v", e)
+		return
+	}
+
+	if len(mappings) > 0 {
+		for _, v := range mappings {
+			v.FactorEdbSeriesId = item.FactorEdbSeriesId
+		}
+		_, e = tx.InsertMulti(200, mappings)
+		if e != nil {
+			err = fmt.Errorf("insert multi mapping err: %v", e)
+			return
+		}
+	}
+	return
+}
+
+// FactorEdbSeriesStepCalculateResp 批量计算响应
+type FactorEdbSeriesStepCalculateResp struct {
+	SeriesId int                                  `description:"多因子指标系列ID"`
+	Fail     []FactorEdbSeriesStepCalculateResult `description:"计算失败的指标"`
+	Success  []FactorEdbSeriesStepCalculateResult `description:"计算成功的指标"`
+}
+
+// FactorEdbSeriesStepCalculateResult 批量计算结果
+type FactorEdbSeriesStepCalculateResult struct {
+	EdbInfoId int    `description:"指标ID"`
+	EdbCode   string `description:"指标编码"`
+	Msg       string `description:"提示信息"`
+	ErrMsg    string `description:"错误信息"`
+}
+
+// FactorEdbSeriesDetail 因子指标系列-详情
+type FactorEdbSeriesDetail struct {
+	*FactorEdbSeriesItem
+	EdbMappings []*FactorEdbSeriesMappingItem
+}
+
+// FactorEdbSeriesCorrelationMatrixResp 因子指标系列-相关性矩阵响应
+type FactorEdbSeriesCorrelationMatrixResp struct {
+	Fail    []FactorEdbSeriesCorrelationMatrixItem `description:"计算失败的指标"`
+	Success []FactorEdbSeriesCorrelationMatrixItem `description:"计算成功的指标"`
+}
+
+// FactorEdbSeriesCorrelationMatrixItem 因子指标系列-相关性矩阵信息
+type FactorEdbSeriesCorrelationMatrixItem struct {
+	SeriesId   int                                      `description:"因子指标系列ID"`
+	EdbInfoId  int                                      `description:"指标ID"`
+	EdbCode    string                                   `description:"指标编码"`
+	EdbName    string                                   `description:"指标名称"`
+	Values     []FactorEdbSeriesCorrelationMatrixValues `description:"X轴和Y轴数据"`
+	Msg        string                                   `description:"提示信息"`
+	ErrMsg     string                                   `description:"错误信息"`
+	Used       bool                                     `description:"是否选中"`
+	SourceName string                                   `description:"指标来源名称"`
+}
+
+// FactorEdbSeriesCorrelationMatrixValues 因子指标系列-相关性矩阵XY值
+type FactorEdbSeriesCorrelationMatrixValues struct {
+	XData int     `description:"X轴数据"`
+	YData float64 `description:"Y轴数据"`
+}
+
+// FactorEdbSeriesCorrelationMatrixOrder 排序规则[0 1 2 3 -1 -2 -3]
+type FactorEdbSeriesCorrelationMatrixOrder []FactorEdbSeriesCorrelationMatrixValues
+
+func (a FactorEdbSeriesCorrelationMatrixOrder) Len() int {
+	return len(a)
+}
+
+func (a FactorEdbSeriesCorrelationMatrixOrder) Swap(i, j int) {
+	a[i], a[j] = a[j], a[i]
+}
+
+func (a FactorEdbSeriesCorrelationMatrixOrder) Less(i, j int) bool {
+	// 非负数优先
+	if a[i].XData >= 0 && a[j].XData < 0 {
+		return true
+	}
+	if a[i].XData < 0 && a[j].XData >= 0 {
+		return false
+	}
+	// 非负数升序排序
+	if a[i].XData >= 0 {
+		return a[i].XData < a[j].XData
+	}
+	// 负数按绝对值的降序排序（即数值的升序）
+	return a[i].XData > a[j].XData
+}
+
+// FactorEdbRecalculateReq 因子指标重新计算
+type FactorEdbRecalculateReq struct {
+	EdbInfoId int    `description:"指标ID"`
+	EdbCode   string `description:"指标编码"`
+}
+
+// FactorEdbChartRecalculateReq 因子指标关联的图表数据重计算
+type FactorEdbChartRecalculateReq struct {
+	ChartInfoId int `description:"图表ID"`
+	//EdbInfoId   int    `description:"指标ID"`
+	//EdbCode     string `description:"指标编码"`
+}

models/factor_edb_series_calculate_data.go

@@ -0,0 +1,190 @@
+package models
+
+import (
+	"eta/eta_index_lib/utils"
+	"fmt"
+	"github.com/beego/beego/v2/client/orm"
+	"strings"
+	"time"
+)
+
+// FactorEdbSeriesCalculateData 因子指标系列-指标计算数据表
+type FactorEdbSeriesCalculateData struct {
+	FactorEdbSeriesCalculateDataId int       `orm:"column(factor_edb_series_calculate_data_id);pk"`
+	FactorEdbSeriesId              int       `description:"因子指标系列ID"`
+	EdbInfoId                      int       `description:"指标ID"`
+	EdbCode                        string    `description:"指标编码"`
+	DataTime                       time.Time `description:"数据日期"`
+	Value                          float64   `description:"数据值"`
+	CreateTime                     time.Time `description:"创建时间"`
+	ModifyTime                     time.Time `description:"修改时间"`
+	DataTimestamp                  int64     `description:"数据日期时间戳"`
+}
+
+func (m *FactorEdbSeriesCalculateData) TableName() string {
+	return "factor_edb_series_calculate_data"
+}
+
+type FactorEdbSeriesCalculateDataCols struct {
+	PrimaryId         string
+	FactorEdbSeriesId string
+	EdbInfoId         string
+	EdbCode           string
+	DataTime          string
+	Value             string
+	CreateTime        string
+	ModifyTime        string
+	DataTimestamp     string
+}
+
+func (m *FactorEdbSeriesCalculateData) Cols() FactorEdbSeriesCalculateDataCols {
+	return FactorEdbSeriesCalculateDataCols{
+		PrimaryId:         "factor_edb_series_calculate_data_id",
+		FactorEdbSeriesId: "factor_edb_series_id",
+		EdbInfoId:         "edb_info_id",
+		EdbCode:           "edb_code",
+		DataTime:          "data_time",
+		Value:             "value",
+		CreateTime:        "create_time",
+		ModifyTime:        "modify_time",
+		DataTimestamp:     "data_timestamp",
+	}
+}
+
+func (m *FactorEdbSeriesCalculateData) Create() (err error) {
+	o := orm.NewOrm()
+	id, err := o.Insert(m)
+	if err != nil {
+		return
+	}
+	m.FactorEdbSeriesCalculateDataId = int(id)
+	return
+}
+
+func (m *FactorEdbSeriesCalculateData) CreateMulti(items []*FactorEdbSeriesCalculateData) (err error) {
+	if len(items) == 0 {
+		return
+	}
+	o := orm.NewOrm()
+	_, err = o.InsertMulti(500, items)
+	return
+}
+
+func (m *FactorEdbSeriesCalculateData) Update(cols []string) (err error) {
+	o := orm.NewOrm()
+	_, err = o.Update(m, cols...)
+	return
+}
+
+func (m *FactorEdbSeriesCalculateData) Remove() (err error) {
+	o := orm.NewOrm()
+	sql := fmt.Sprintf(`DELETE FROM %s WHERE %s = ? LIMIT 1`, m.TableName(), m.Cols().PrimaryId)
+	_, err = o.Raw(sql, m.FactorEdbSeriesCalculateDataId).Exec()
+	return
+}
+
+func (m *FactorEdbSeriesCalculateData) MultiRemove(ids []int) (err error) {
+	if len(ids) == 0 {
+		return
+	}
+	o := orm.NewOrm()
+	sql := fmt.Sprintf(`DELETE FROM %s WHERE %s IN (%s)`, m.TableName(), m.Cols().PrimaryId, utils.GetOrmInReplace(len(ids)))
+	_, err = o.Raw(sql, ids).Exec()
+	return
+}
+
+func (m *FactorEdbSeriesCalculateData) RemoveByCondition(condition string, pars []interface{}) (err error) {
+	if condition == "" {
+		return
+	}
+	o := orm.NewOrm()
+	sql := fmt.Sprintf(`DELETE FROM %s WHERE %s`, m.TableName(), condition)
+	_, err = o.Raw(sql, pars).Exec()
+	return
+}
+
+func (m *FactorEdbSeriesCalculateData) GetItemById(id int) (item *FactorEdbSeriesCalculateData, err error) {
+	o := orm.NewOrm()
+	sql := fmt.Sprintf(`SELECT * FROM %s WHERE %s = ? LIMIT 1`, m.TableName(), m.Cols().PrimaryId)
+	err = o.Raw(sql, id).QueryRow(&item)
+	return
+}
+
+func (m *FactorEdbSeriesCalculateData) GetItemByCondition(condition string, pars []interface{}, orderRule string) (item *FactorEdbSeriesCalculateData, err error) {
+	o := orm.NewOrm()
+	order := ``
+	if orderRule != "" {
+		order = ` ORDER BY ` + orderRule
+	}
+	sql := fmt.Sprintf(`SELECT * FROM %s WHERE 1=1 %s %s LIMIT 1`, m.TableName(), condition, order)
+	err = o.Raw(sql, pars).QueryRow(&item)
+	return
+}
+
+func (m *FactorEdbSeriesCalculateData) GetCountByCondition(condition string, pars []interface{}) (count int, err error) {
+	o := orm.NewOrm()
+	sql := fmt.Sprintf(`SELECT COUNT(1) FROM %s WHERE 1=1 %s`, m.TableName(), condition)
+	err = o.Raw(sql, pars).QueryRow(&count)
+	return
+}
+
+func (m *FactorEdbSeriesCalculateData) GetItemsByCondition(condition string, pars []interface{}, fieldArr []string, orderRule string) (items []*FactorEdbSeriesCalculateData, err error) {
+	o := orm.NewOrm()
+	fields := strings.Join(fieldArr, ",")
+	if len(fieldArr) == 0 {
+		fields = `*`
+	}
+	order := fmt.Sprintf(`ORDER BY %s DESC`, m.Cols().CreateTime)
+	if orderRule != "" {
+		order = ` ORDER BY ` + orderRule
+	}
+	sql := fmt.Sprintf(`SELECT %s FROM %s WHERE 1=1 %s %s`, fields, m.TableName(), condition, order)
+	_, err = o.Raw(sql, pars).QueryRows(&items)
+	return
+}
+
+func (m *FactorEdbSeriesCalculateData) GetPageItemsByCondition(condition string, pars []interface{}, fieldArr []string, orderRule string, startSize, pageSize int) (items []*FactorEdbSeriesCalculateData, err error) {
+	o := orm.NewOrm()
+	fields := strings.Join(fieldArr, ",")
+	if len(fieldArr) == 0 {
+		fields = `*`
+	}
+	order := fmt.Sprintf(`ORDER BY %s DESC`, m.Cols().CreateTime)
+	if orderRule != "" {
+		order = ` ORDER BY ` + orderRule
+	}
+	sql := fmt.Sprintf(`SELECT %s FROM %s WHERE 1=1 %s %s LIMIT ?,?`, fields, m.TableName(), condition, order)
+	_, err = o.Raw(sql, pars, startSize, pageSize).QueryRows(&items)
+	return
+}
+
+// FactorEdbSeriesCalculateDataItem 因子指标系列-计算数据信息
+type FactorEdbSeriesCalculateDataItem struct {
+	DataId            int     `description:"数据ID"`
+	FactorEdbSeriesId int     `description:"因子指标系列ID"`
+	EdbInfoId         int     `description:"指标ID"`
+	EdbCode           string  `description:"指标编码"`
+	DataTime          string  `description:"数据日期"`
+	Value             float64 `description:"数据值"`
+}
+
+func (m *FactorEdbSeriesCalculateData) Format2Item() (item *FactorEdbSeriesCalculateDataItem) {
+	item = new(FactorEdbSeriesCalculateDataItem)
+	item.DataId = m.FactorEdbSeriesCalculateDataId
+	item.FactorEdbSeriesId = m.FactorEdbSeriesId
+	item.EdbInfoId = m.EdbInfoId
+	item.EdbCode = m.EdbCode
+	return
+}
+
+// TransEdbSeriesCalculateData2EdbDataList 转换数据格式
+func TransEdbSeriesCalculateData2EdbDataList(items []*FactorEdbSeriesCalculateData) (list []*EdbInfoSearchData) {
+	list = make([]*EdbInfoSearchData, 0)
+	for _, v := range items {
+		list = append(list, &EdbInfoSearchData{
+			DataTime: v.DataTime.Format(utils.FormatDate),
+			Value:    v.Value,
+		})
+	}
+	return
+}

models/factor_edb_series_calculate_func.go

@@ -0,0 +1,153 @@
+package models
+
+import (
+	"eta/eta_index_lib/utils"
+	"fmt"
+	"github.com/beego/beego/v2/client/orm"
+	"strings"
+	"time"
+)
+
+// FactorEdbSeriesCalculateFunc 多因子系列-计算方式表
+type FactorEdbSeriesCalculateFunc struct {
+	FactorEdbSeriesCalculateFuncId int       `orm:"column(factor_edb_series_calculate_func_id);pk"`
+	CalculateName                  string    `description:"计算方式名称"`
+	Source                         int       `description:"计算方式来源"`
+	EdbInfoType                    int       `description:"指标计算类型：0-普通指标；1-预测指标"`
+	CreateTime                     time.Time `description:"创建时间"`
+	ModifyTime                     time.Time `description:"修改时间"`
+}
+
+func (m *FactorEdbSeriesCalculateFunc) TableName() string {
+	return "factor_edb_series_calculate_func"
+}
+
+type FactorEdbSeriesCalculateFuncCols struct {
+	PrimaryId     string
+	CalculateName string
+	Source        string
+	EdbInfoType   string
+	CreateTime    string
+	ModifyTime    string
+}
+
+func (m *FactorEdbSeriesCalculateFunc) Cols() FactorEdbSeriesCalculateFuncCols {
+	return FactorEdbSeriesCalculateFuncCols{
+		PrimaryId:     "factor_edb_series_calculate_func_id",
+		CalculateName: "calculate_name",
+		Source:        "source",
+		EdbInfoType:   "edb_info_type",
+		CreateTime:    "create_time",
+		ModifyTime:    "modify_time",
+	}
+}
+
+func (m *FactorEdbSeriesCalculateFunc) Create() (err error) {
+	o := orm.NewOrm()
+	id, err := o.Insert(m)
+	if err != nil {
+		return
+	}
+	m.FactorEdbSeriesCalculateFuncId = int(id)
+	return
+}
+
+func (m *FactorEdbSeriesCalculateFunc) CreateMulti(items []*FactorEdbSeriesCalculateFunc) (err error) {
+	if len(items) == 0 {
+		return
+	}
+	o := orm.NewOrm()
+	_, err = o.InsertMulti(len(items), items)
+	return
+}
+
+func (m *FactorEdbSeriesCalculateFunc) Update(cols []string) (err error) {
+	o := orm.NewOrm()
+	_, err = o.Update(m, cols...)
+	return
+}
+
+func (m *FactorEdbSeriesCalculateFunc) Remove() (err error) {
+	o := orm.NewOrm()
+	sql := fmt.Sprintf(`DELETE FROM %s WHERE %s = ? LIMIT 1`, m.TableName(), m.Cols().PrimaryId)
+	_, err = o.Raw(sql, m.FactorEdbSeriesCalculateFuncId).Exec()
+	return
+}
+
+func (m *FactorEdbSeriesCalculateFunc) MultiRemove(ids []int) (err error) {
+	if len(ids) == 0 {
+		return
+	}
+	o := orm.NewOrm()
+	sql := fmt.Sprintf(`DELETE FROM %s WHERE %s IN (%s)`, m.TableName(), m.Cols().PrimaryId, utils.GetOrmInReplace(len(ids)))
+	_, err = o.Raw(sql, ids).Exec()
+	return
+}
+
+func (m *FactorEdbSeriesCalculateFunc) GetItemById(id int) (item *FactorEdbSeriesCalculateFunc, err error) {
+	o := orm.NewOrm()
+	sql := fmt.Sprintf(`SELECT * FROM %s WHERE %s = ? LIMIT 1`, m.TableName(), m.Cols().PrimaryId)
+	err = o.Raw(sql, id).QueryRow(&item)
+	return
+}
+
+func (m *FactorEdbSeriesCalculateFunc) GetItemByCondition(condition string, pars []interface{}, orderRule string) (item *FactorEdbSeriesCalculateFunc, err error) {
+	o := orm.NewOrm()
+	order := ``
+	if orderRule != "" {
+		order = ` ORDER BY ` + orderRule
+	}
+	sql := fmt.Sprintf(`SELECT * FROM %s WHERE 1=1 %s %s LIMIT 1`, m.TableName(), condition, order)
+	err = o.Raw(sql, pars).QueryRow(&item)
+	return
+}
+
+func (m *FactorEdbSeriesCalculateFunc) GetCountByCondition(condition string, pars []interface{}) (count int, err error) {
+	o := orm.NewOrm()
+	sql := fmt.Sprintf(`SELECT COUNT(1) FROM %s WHERE 1=1 %s`, m.TableName(), condition)
+	err = o.Raw(sql, pars).QueryRow(&count)
+	return
+}
+
+func (m *FactorEdbSeriesCalculateFunc) GetItemsByCondition(condition string, pars []interface{}, fieldArr []string, orderRule string) (items []*FactorEdbSeriesCalculateFunc, err error) {
+	o := orm.NewOrm()
+	fields := strings.Join(fieldArr, ",")
+	if len(fieldArr) == 0 {
+		fields = `*`
+	}
+	order := fmt.Sprintf(`ORDER BY %s DESC`, m.Cols().CreateTime)
+	if orderRule != "" {
+		order = ` ORDER BY ` + orderRule
+	}
+	sql := fmt.Sprintf(`SELECT %s FROM %s WHERE 1=1 %s %s`, fields, m.TableName(), condition, order)
+	_, err = o.Raw(sql, pars).QueryRows(&items)
+	return
+}
+
+func (m *FactorEdbSeriesCalculateFunc) GetPageItemsByCondition(condition string, pars []interface{}, fieldArr []string, orderRule string, startSize, pageSize int) (items []*FactorEdbSeriesCalculateFunc, err error) {
+	o := orm.NewOrm()
+	fields := strings.Join(fieldArr, ",")
+	if len(fieldArr) == 0 {
+		fields = `*`
+	}
+	order := fmt.Sprintf(`ORDER BY %s DESC`, m.Cols().CreateTime)
+	if orderRule != "" {
+		order = ` ORDER BY ` + orderRule
+	}
+	sql := fmt.Sprintf(`SELECT %s FROM %s WHERE 1=1 %s %s LIMIT ?,?`, fields, m.TableName(), condition, order)
+	_, err = o.Raw(sql, pars, startSize, pageSize).QueryRows(&items)
+	return
+}
+
+// FactorEdbSeriesCalculateFuncItem 多因子系列-计算方式
+type FactorEdbSeriesCalculateFuncItem struct {
+	CalculateName string `description:"计算方式名称"`
+	Source        int    `description:"计算方式来源"`
+}
+
+func (m *FactorEdbSeriesCalculateFunc) Format2Item() (item *FactorEdbSeriesCalculateFuncItem) {
+	item = new(FactorEdbSeriesCalculateFuncItem)
+	item.CalculateName = m.CalculateName
+	item.Source = m.Source
+	return
+}

models/factor_edb_series_chart_mapping.go

@@ -0,0 +1,176 @@
+package models
+
+import (
+	"eta/eta_index_lib/utils"
+	"fmt"
+	"github.com/beego/beego/v2/client/orm"
+	"strings"
+	"time"
+)
+
+const (
+	FactorEdbSeriesChartCalculateTypeCorrelation = 1 // 相关性计算
+)
+
+// FactorEdbSeriesChartMapping 因子指标系列-图表关联
+type FactorEdbSeriesChartMapping struct {
+	FactorEdbSeriesChartMappingId int       `orm:"column(factor_edb_series_chart_mapping_id);pk"`
+	ChartInfoId                   int       `description:"图表ID"`
+	Source                        int       `description:"图表来源, 同chart_info表source"`
+	CalculateType                 int       `description:"计算方式: 1-相关性"`
+	CalculatePars                 string    `description:"计算参数-JSON(如计算窗口等)"`
+	CalculateData                 string    `description:"计算数据-JSON(如相关性矩阵等)"`
+	FactorEdbSeriesId             int       `description:"因子指标系列ID"`
+	EdbInfoId                     int       `description:"指标ID"`
+	EdbUsed                       int       `description:"指标是否使用: 0-否; 1-是"`
+	CreateTime                    time.Time `description:"创建时间"`
+	ModifyTime                    time.Time `description:"修改时间"`
+}
+
+func (m *FactorEdbSeriesChartMapping) TableName() string {
+	return "factor_edb_series_chart_mapping"
+}
+
+type MultipleFactorSeriesChartMappingCols struct {
+	PrimaryId         string
+	ChartInfoId       string
+	Source            string
+	CalculateType     string
+	CalculatePars     string
+	CalculateData     string
+	FactorEdbSeriesId string
+	EdbInfoId         string
+	EdbUsed           string
+	CreateTime        string
+	ModifyTime        string
+}
+
+func (m *FactorEdbSeriesChartMapping) Cols() MultipleFactorSeriesChartMappingCols {
+	return MultipleFactorSeriesChartMappingCols{
+		PrimaryId:         "factor_edb_series_chart_mapping_id",
+		ChartInfoId:       "chart_info_id",
+		Source:            "source",
+		CalculateType:     "calculate_type",
+		CalculatePars:     "calculate_pars",
+		CalculateData:     "calculate_data",
+		FactorEdbSeriesId: "factor_edb_series_id",
+		EdbInfoId:         "edb_info_id",
+		EdbUsed:           "edb_used",
+		CreateTime:        "create_time",
+		ModifyTime:        "modify_time",
+	}
+}
+
+func (m *FactorEdbSeriesChartMapping) Create() (err error) {
+	o := orm.NewOrm()
+	id, err := o.Insert(m)
+	if err != nil {
+		return
+	}
+	m.FactorEdbSeriesChartMappingId = int(id)
+	return
+}
+
+func (m *FactorEdbSeriesChartMapping) CreateMulti(items []*FactorEdbSeriesChartMapping) (err error) {
+	if len(items) == 0 {
+		return
+	}
+	o := orm.NewOrm()
+	_, err = o.InsertMulti(len(items), items)
+	return
+}
+
+func (m *FactorEdbSeriesChartMapping) Update(cols []string) (err error) {
+	o := orm.NewOrm()
+	_, err = o.Update(m, cols...)
+	return
+}
+
+func (m *FactorEdbSeriesChartMapping) Remove() (err error) {
+	o := orm.NewOrm()
+	sql := fmt.Sprintf(`DELETE FROM %s WHERE %s = ? LIMIT 1`, m.TableName(), m.Cols().PrimaryId)
+	_, err = o.Raw(sql, m.FactorEdbSeriesChartMappingId).Exec()
+	return
+}
+
+func (m *FactorEdbSeriesChartMapping) MultiRemove(ids []int) (err error) {
+	if len(ids) == 0 {
+		return
+	}
+	o := orm.NewOrm()
+	sql := fmt.Sprintf(`DELETE FROM %s WHERE %s IN (%s)`, m.TableName(), m.Cols().PrimaryId, utils.GetOrmInReplace(len(ids)))
+	_, err = o.Raw(sql, ids).Exec()
+	return
+}
+
+func (m *FactorEdbSeriesChartMapping) GetItemById(id int) (item *FactorEdbSeriesChartMapping, err error) {
+	o := orm.NewOrm()
+	sql := fmt.Sprintf(`SELECT * FROM %s WHERE %s = ? LIMIT 1`, m.TableName(), m.Cols().PrimaryId)
+	err = o.Raw(sql, id).QueryRow(&item)
+	return
+}
+
+func (m *FactorEdbSeriesChartMapping) GetItemByCondition(condition string, pars []interface{}, orderRule string) (item *FactorEdbSeriesChartMapping, err error) {
+	o := orm.NewOrm()
+	order := ``
+	if orderRule != "" {
+		order = ` ORDER BY ` + orderRule
+	}
+	sql := fmt.Sprintf(`SELECT * FROM %s WHERE 1=1 %s %s LIMIT 1`, m.TableName(), condition, order)
+	err = o.Raw(sql, pars).QueryRow(&item)
+	return
+}
+
+func (m *FactorEdbSeriesChartMapping) GetCountByCondition(condition string, pars []interface{}) (count int, err error) {
+	o := orm.NewOrm()
+	sql := fmt.Sprintf(`SELECT COUNT(1) FROM %s WHERE 1=1 %s`, m.TableName(), condition)
+	err = o.Raw(sql, pars).QueryRow(&count)
+	return
+}
+
+func (m *FactorEdbSeriesChartMapping) GetItemsByCondition(condition string, pars []interface{}, fieldArr []string, orderRule string) (items []*FactorEdbSeriesChartMapping, err error) {
+	o := orm.NewOrm()
+	fields := strings.Join(fieldArr, ",")
+	if len(fieldArr) == 0 {
+		fields = `*`
+	}
+	order := fmt.Sprintf(`ORDER BY %s DESC`, m.Cols().CreateTime)
+	if orderRule != "" {
+		order = ` ORDER BY ` + orderRule
+	}
+	sql := fmt.Sprintf(`SELECT %s FROM %s WHERE 1=1 %s %s`, fields, m.TableName(), condition, order)
+	_, err = o.Raw(sql, pars).QueryRows(&items)
+	return
+}
+
+func (m *FactorEdbSeriesChartMapping) GetPageItemsByCondition(condition string, pars []interface{}, fieldArr []string, orderRule string, startSize, pageSize int) (items []*FactorEdbSeriesChartMapping, err error) {
+	o := orm.NewOrm()
+	fields := strings.Join(fieldArr, ",")
+	if len(fieldArr) == 0 {
+		fields = `*`
+	}
+	order := fmt.Sprintf(`ORDER BY %s DESC`, m.Cols().CreateTime)
+	if orderRule != "" {
+		order = ` ORDER BY ` + orderRule
+	}
+	sql := fmt.Sprintf(`SELECT %s FROM %s WHERE 1=1 %s %s LIMIT ?,?`, fields, m.TableName(), condition, order)
+	_, err = o.Raw(sql, pars, startSize, pageSize).QueryRows(&items)
+	return
+}
+
+// GetDistinctSeriesIdByChartId 获取图表关联的系列ID
+func (m *FactorEdbSeriesChartMapping) GetDistinctSeriesIdByChartId(chartId int) (seriesIds []int, err error) {
+	o := orm.NewOrm()
+	sql := fmt.Sprintf(`SELECT DISTINCT %s FROM %s WHERE %s = ?`, m.Cols().FactorEdbSeriesId, m.TableName(), m.Cols().ChartInfoId)
+	_, err = o.Raw(sql, chartId).QueryRows(&seriesIds)
+	return
+}
+
+// FactorEdbSeriesChartCalculateCorrelationReq 图表相关性计算参数
+type FactorEdbSeriesChartCalculateCorrelationReq struct {
+	BaseEdbInfoId  int    `description:"标的指标ID"`
+	LeadValue      int    `description:"领先期数"`
+	LeadUnit       string `description:"频度"`
+	CalculateValue int    `description:"计算窗口"`
+	CalculateUnit  string `description:"计算频度"`
+}

models/factor_edb_series_mapping.go

@@ -0,0 +1,167 @@
+package models
+
+import (
+	"eta/eta_index_lib/utils"
+	"fmt"
+	"github.com/beego/beego/v2/client/orm"
+	"strings"
+	"time"
+)
+
+// FactorEdbSeriesMapping 因子指标系列-指标关联表
+type FactorEdbSeriesMapping struct {
+	FactorEdbSeriesMappingId int       `orm:"column(factor_edb_series_mapping_id);pk"`
+	FactorEdbSeriesId        int       `description:"因子指标系列ID"`
+	EdbInfoId                int       `description:"指标ID"`
+	EdbCode                  string    `description:"指标编码"`
+	CreateTime               time.Time `description:"创建时间"`
+	ModifyTime               time.Time `description:"修改时间"`
+}
+
+func (m *FactorEdbSeriesMapping) TableName() string {
+	return "factor_edb_series_mapping"
+}
+
+type FactorEdbSeriesMappingCols struct {
+	PrimaryId         string
+	FactorEdbSeriesId string
+	EdbInfoId         string
+	EdbCode           string
+	CreateTime        string
+	ModifyTime        string
+}
+
+func (m *FactorEdbSeriesMapping) Cols() FactorEdbSeriesMappingCols {
+	return FactorEdbSeriesMappingCols{
+		PrimaryId:         "factor_edb_series_mapping_id",
+		FactorEdbSeriesId: "factor_edb_series_id",
+		EdbInfoId:         "edb_info_id",
+		EdbCode:           "edb_code",
+		CreateTime:        "create_time",
+		ModifyTime:        "modify_time",
+	}
+}
+
+func (m *FactorEdbSeriesMapping) Create() (err error) {
+	o := orm.NewOrm()
+	id, err := o.Insert(m)
+	if err != nil {
+		return
+	}
+	m.FactorEdbSeriesMappingId = int(id)
+	return
+}
+
+func (m *FactorEdbSeriesMapping) CreateMulti(items []*FactorEdbSeriesMapping) (err error) {
+	if len(items) == 0 {
+		return
+	}
+	o := orm.NewOrm()
+	_, err = o.InsertMulti(len(items), items)
+	return
+}
+
+func (m *FactorEdbSeriesMapping) Update(cols []string) (err error) {
+	o := orm.NewOrm()
+	_, err = o.Update(m, cols...)
+	return
+}
+
+func (m *FactorEdbSeriesMapping) Remove() (err error) {
+	o := orm.NewOrm()
+	sql := fmt.Sprintf(`DELETE FROM %s WHERE %s = ? LIMIT 1`, m.TableName(), m.Cols().PrimaryId)
+	_, err = o.Raw(sql, m.FactorEdbSeriesMappingId).Exec()
+	return
+}
+
+func (m *FactorEdbSeriesMapping) MultiRemove(ids []int) (err error) {
+	if len(ids) == 0 {
+		return
+	}
+	o := orm.NewOrm()
+	sql := fmt.Sprintf(`DELETE FROM %s WHERE %s IN (%s)`, m.TableName(), m.Cols().PrimaryId, utils.GetOrmInReplace(len(ids)))
+	_, err = o.Raw(sql, ids).Exec()
+	return
+}
+
+func (m *FactorEdbSeriesMapping) RemoveByCondition(condition string, pars []interface{}) (err error) {
+	if condition == "" {
+		return
+	}
+	o := orm.NewOrm()
+	sql := fmt.Sprintf(`DELETE FROM %s WHERE %s`, m.TableName(), condition)
+	_, err = o.Raw(sql, pars).Exec()
+	return
+}
+
+func (m *FactorEdbSeriesMapping) GetItemById(id int) (item *FactorEdbSeriesMapping, err error) {
+	o := orm.NewOrm()
+	sql := fmt.Sprintf(`SELECT * FROM %s WHERE %s = ? LIMIT 1`, m.TableName(), m.Cols().PrimaryId)
+	err = o.Raw(sql, id).QueryRow(&item)
+	return
+}
+
+func (m *FactorEdbSeriesMapping) GetItemByCondition(condition string, pars []interface{}, orderRule string) (item *FactorEdbSeriesMapping, err error) {
+	o := orm.NewOrm()
+	order := ``
+	if orderRule != "" {
+		order = ` ORDER BY ` + orderRule
+	}
+	sql := fmt.Sprintf(`SELECT * FROM %s WHERE 1=1 %s %s LIMIT 1`, m.TableName(), condition, order)
+	err = o.Raw(sql, pars).QueryRow(&item)
+	return
+}
+
+func (m *FactorEdbSeriesMapping) GetCountByCondition(condition string, pars []interface{}) (count int, err error) {
+	o := orm.NewOrm()
+	sql := fmt.Sprintf(`SELECT COUNT(1) FROM %s WHERE 1=1 %s`, m.TableName(), condition)
+	err = o.Raw(sql, pars).QueryRow(&count)
+	return
+}
+
+func (m *FactorEdbSeriesMapping) GetItemsByCondition(condition string, pars []interface{}, fieldArr []string, orderRule string) (items []*FactorEdbSeriesMapping, err error) {
+	o := orm.NewOrm()
+	fields := strings.Join(fieldArr, ",")
+	if len(fieldArr) == 0 {
+		fields = `*`
+	}
+	order := fmt.Sprintf(`ORDER BY %s DESC`, m.Cols().CreateTime)
+	if orderRule != "" {
+		order = ` ORDER BY ` + orderRule
+	}
+	sql := fmt.Sprintf(`SELECT %s FROM %s WHERE 1=1 %s %s`, fields, m.TableName(), condition, order)
+	_, err = o.Raw(sql, pars).QueryRows(&items)
+	return
+}
+
+func (m *FactorEdbSeriesMapping) GetPageItemsByCondition(condition string, pars []interface{}, fieldArr []string, orderRule string, startSize, pageSize int) (items []*FactorEdbSeriesMapping, err error) {
+	o := orm.NewOrm()
+	fields := strings.Join(fieldArr, ",")
+	if len(fieldArr) == 0 {
+		fields = `*`
+	}
+	order := fmt.Sprintf(`ORDER BY %s DESC`, m.Cols().CreateTime)
+	if orderRule != "" {
+		order = ` ORDER BY ` + orderRule
+	}
+	sql := fmt.Sprintf(`SELECT %s FROM %s WHERE 1=1 %s %s LIMIT ?,?`, fields, m.TableName(), condition, order)
+	_, err = o.Raw(sql, pars, startSize, pageSize).QueryRows(&items)
+	return
+}
+
+// FactorEdbSeriesMappingItem 因子指标系列-指标关联信息
+type FactorEdbSeriesMappingItem struct {
+	SeriesId  int    `description:"因子指标系列ID"`
+	EdbInfoId int    `description:"指标ID"`
+	EdbCode   string `description:"指标编码"`
+	EdbName   string `description:"指标名称"`
+	EdbNameEn string `description:"指标名称-英文"`
+}
+
+func (m *FactorEdbSeriesMapping) Format2Item() (item *FactorEdbSeriesMappingItem) {
+	item = new(FactorEdbSeriesMappingItem)
+	item.SeriesId = m.FactorEdbSeriesId
+	item.EdbInfoId = m.EdbInfoId
+	item.EdbCode = m.EdbCode
+	return
+}

models/mgo/base_from_ths_hf_data.go

@@ -0,0 +1,406 @@
+package mgo
+
+import (
+	"context"
+	"errors"
+	"eta/eta_index_lib/utils"
+	"fmt"
+	"github.com/qiniu/qmgo"
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+	"time"
+)
+
+// BaseFromThsHfData
+// @Description: 同花顺高频集合
+type BaseFromThsHfData struct {
+	ID                   primitive.ObjectID `json:"_id" bson:"_id,omitempty"`                                   // 文档id
+	BaseFromThsHfDataId  int64              `json:"base_from_ths_hf_data_id" bson:"base_from_ths_hf_data_id"`   // 指标数据ID
+	BaseFromThsHfIndexId int64              `json:"base_from_ths_hf_index_id" bson:"base_from_ths_hf_index_id"` // 指标ID
+	IndexCode            string             `json:"index_code" bson:"index_code"`                               // 指标编码
+	DataTime             time.Time          `json:"data_time" bson:"data_time"`                                 // 数据日期
+	Value                float64            `json:"value" bson:"value"`                                         // 数据值
+	UniqueCode           string             `json:"unique_code" bson:"unique_code"`                             // 唯一编码
+	CreateTime           time.Time          `json:"create_time" bson:"create_time"`                             // 创建时间
+	ModifyTime           time.Time          `json:"modify_time" bson:"modify_time"`                             // 修改时间
+	DataTimestamp        int64              `json:"data_timestamp" bson:"data_timestamp"`                       // 数据日期时间戳
+}
+
+// CollectionName
+// @Description:  获取集合名称
+func (m *BaseFromThsHfData) CollectionName() string {
+	return "base_from_ths_hf_data"
+}
+
+// DataBaseName
+// @Description: 获取数据库名称
+func (m *BaseFromThsHfData) DataBaseName() string {
+	return utils.MgoDataDbName
+}
+
+// GetCollection
+// @Description: 获取mongodb集合的句柄
+func (m *BaseFromThsHfData) GetCollection() *qmgo.Collection {
+	db := utils.MgoDataCli.Database(m.DataBaseName())
+	return db.Collection(m.CollectionName())
+}
+
+// GetAllDataList
+// @Description: 根据条件获取所有数据
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-26 13:42:19
+// @param sort []string
+// @param whereParams interface{}
+// @return result []BaseFromThsHfData
+// @return err error
+func (m *BaseFromThsHfData) GetAllDataList(whereParams interface{}, sort []string) (result []*BaseFromThsHfData, err error) {
+	if utils.MgoDataCli == nil {
+		err = errors.New("mongodb连接失败")
+		return
+	}
+	db := utils.MgoDataCli.Database(m.DataBaseName())
+	coll := db.Collection(m.CollectionName())
+	ctx := context.TODO()
+	if err != nil {
+		fmt.Println("MgoGetColl Err:", err.Error())
+		return
+	}
+	err = coll.Find(ctx, whereParams).Sort(sort...).All(&result)
+	if err != nil {
+		return
+	}
+
+	for _, v := range result {
+		v.DataTime = v.DataTime.In(time.Local)
+		v.CreateTime = v.CreateTime.In(time.Local)
+		v.ModifyTime = v.ModifyTime.In(time.Local)
+	}
+	return
+}
+
+// GetLimitDataList
+// @Description: 根据条件获取指定数量数据列表
+// @author: Roc
+// @receiver m
+// @datetime 2024-05-06 17:08:32
+// @param whereParams interface{}
+// @param size int64
+// @return result []*BaseFromThsHfData
+// @return err error
+func (m *BaseFromThsHfData) GetLimitDataList(whereParams interface{}, size int64, sort []string) (result []*BaseFromThsHfData, err error) {
+	if utils.MgoDataCli == nil {
+		err = errors.New("mongodb连接失败")
+		return
+	}
+	db := utils.MgoDataCli.Database(m.DataBaseName())
+	coll := db.Collection(m.CollectionName())
+	ctx := context.TODO()
+	if err != nil {
+		fmt.Println("MgoGetColl Err:", err.Error())
+		return
+	}
+	err = coll.Find(ctx, whereParams).Sort(sort...).Limit(size).All(&result)
+	if err != nil {
+		return
+	}
+
+	for _, v := range result {
+		v.DataTime = v.DataTime.In(time.Local)
+		v.CreateTime = v.CreateTime.In(time.Local)
+		v.ModifyTime = v.ModifyTime.In(time.Local)
+	}
+
+	return
+}
+
+// GetPageDataList
+// @Description: 根据条件获取分页数据列表
+// @author: Roc
+// @receiver m
+// @datetime 2024-05-07 10:21:07
+// @param whereParams interface{}
+// @param startSize int64
+// @param size int64
+// @param sort []string
+// @return result []*BaseFromThsHfData
+// @return err error
+func (m *BaseFromThsHfData) GetPageDataList(whereParams interface{}, startSize, size int64, sort []string) (result []*BaseFromThsHfData, err error) {
+	if utils.MgoDataCli == nil {
+		err = errors.New("mongodb连接失败")
+		return
+	}
+	db := utils.MgoDataCli.Database(m.DataBaseName())
+	coll := db.Collection(m.CollectionName())
+	ctx := context.TODO()
+	if err != nil {
+		fmt.Println("MgoGetColl Err:", err.Error())
+		return
+	}
+	err = coll.Find(ctx, whereParams).Sort(sort...).Skip(startSize).Limit(size).All(&result)
+	if err != nil {
+		return
+	}
+
+	for _, v := range result {
+		v.DataTime = v.DataTime.In(time.Local)
+		v.CreateTime = v.CreateTime.In(time.Local)
+		v.ModifyTime = v.ModifyTime.In(time.Local)
+	}
+
+	return
+}
+
+// GetCountDataList
+// @Description:  根据条件获取数据列表总数
+// @author: Roc
+// @receiver m
+// @datetime 2024-05-07 10:29:00
+// @param whereParams interface{}
+// @return count int64
+// @return err error
+func (m *BaseFromThsHfData) GetCountDataList(whereParams interface{}) (count int64, err error) {
+	if utils.MgoDataCli == nil {
+		err = errors.New("mongodb连接失败")
+		return
+	}
+	db := utils.MgoDataCli.Database(m.DataBaseName())
+	coll := db.Collection(m.CollectionName())
+	ctx := context.TODO()
+	if err != nil {
+		fmt.Println("MgoGetColl Err:", err.Error())
+		return
+	}
+	count, err = coll.Find(ctx, whereParams).Count()
+
+	return
+}
+
+// InsertDataByColl
+// @Description: 写入单条数据（外部传入集合）
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-26 14:22:18
+// @param addData interface{}
+// @return err error
+func (m *BaseFromThsHfData) InsertDataByColl(coll *qmgo.Collection, addData interface{}) (err error) {
+	ctx := context.TODO()
+	_, err = coll.InsertOne(ctx, addData)
+	if err != nil {
+		fmt.Println("InsertDataByColl:Err:" + err.Error())
+		return
+	}
+
+	return
+}
+
+// BatchInsertData
+// @Description: 批量写入数据
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-26 14:22:18
+// @param bulk int 每次请求保存的数据量
+// @param dataList []interface{}
+// @return err error
+func (m *BaseFromThsHfData) BatchInsertData(bulk int, dataList []interface{}) (err error) {
+	db := utils.MgoDataCli.Database(m.DataBaseName())
+	coll := db.Collection(m.CollectionName())
+
+	return m.BatchInsertDataByColl(coll, bulk, dataList)
+}
+
+// BatchInsertDataByColl
+// @Description: 批量写入数据（外部传入集合）
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-26 14:22:18
+// @param coll *qmgo.Collection
+// @param bulk int 每次请求保存的数据量
+// @param dataList []interface{}
+// @return err error
+func (m *BaseFromThsHfData) BatchInsertDataByColl(coll *qmgo.Collection, bulk int, dataList []interface{}) (err error) {
+	ctx := context.TODO()
+	dataNum := len(dataList)
+	if dataNum <= 0 {
+		return
+	}
+
+	// 不设置每次保存切片数量大小，或者实际数据量小于设置的切片数量大小，那么就直接保存吧
+	if bulk <= 0 || dataNum <= bulk {
+		_, err = coll.InsertMany(ctx, dataList)
+		if err != nil {
+			fmt.Println("BatchInsertData:Err:" + err.Error())
+			return
+		}
+		return
+	}
+
+	// 分批保存
+	i := 0
+	tmpAddDataList := make([]interface{}, 0)
+	for _, v := range dataList {
+		tmpAddDataList = append(tmpAddDataList, v)
+		i++
+		if i >= bulk {
+			_, err = coll.InsertMany(ctx, tmpAddDataList)
+			if err != nil {
+				fmt.Println("BatchInsertData:Err:" + err.Error())
+				return
+			}
+			i = 0
+			tmpAddDataList = make([]interface{}, 0)
+		}
+	}
+
+	if len(tmpAddDataList) > 0 {
+		_, err = coll.InsertMany(ctx, tmpAddDataList)
+		if err != nil {
+			fmt.Println("BatchInsertData:Err:" + err.Error())
+			return
+		}
+	}
+
+	return
+}
+
+// UpdateDataByColl
+// @Description: 单条数据修改
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-26 15:01:51
+// @param whereParams interface{}
+// @param updateParams interface{}
+// @return err error
+func (m *BaseFromThsHfData) UpdateDataByColl(coll *qmgo.Collection, whereParams, updateParams interface{}) (err error) {
+	ctx := context.TODO()
+	err = coll.UpdateOne(ctx, whereParams, updateParams)
+	if err != nil {
+		fmt.Println("UpdateDataByColl:Err:" + err.Error())
+		return
+	}
+	return
+}
+
+// UpdateData
+// @Description: 单条数据修改
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-26 15:01:51
+// @param whereParams interface{}
+// @param updateParams interface{}
+// @return err error
+func (m *BaseFromThsHfData) UpdateData(whereParams, updateParams interface{}) (err error) {
+	db := utils.MgoDataCli.Database(m.DataBaseName())
+	coll := db.Collection(m.CollectionName())
+	ctx := context.TODO()
+	err = coll.UpdateOne(ctx, whereParams, updateParams)
+	if err != nil {
+		fmt.Println("UpdateData:Err:" + err.Error())
+		return
+	}
+	return
+}
+
+// HandleData
+// @Description: 事务处理数据
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-30 10:40:20
+// @param addDataList []BaseAddFromBusinessData
+// @param updateDataList []BaseFromThsHfData
+// @return result interface{}
+// @return err error
+func (m *BaseFromThsHfData) HandleData(addDataList, updateDataList []BaseFromThsHfData) (result interface{}, err error) {
+
+	ctx := context.TODO()
+
+	callback := func(sessCtx context.Context) (interface{}, error) {
+		// 重要：确保事务中的每一个操作，都使用传入的sessCtx参数
+
+		db := utils.MgoDataCli.Database(m.DataBaseName())
+		coll := db.Collection(m.CollectionName())
+
+		// 插入数据
+		if len(addDataList) > 0 {
+			_, err = coll.InsertMany(sessCtx, addDataList)
+			if err != nil {
+				return nil, err
+			}
+		}
+
+		// 修改
+
+		if len(updateDataList) > 0 {
+			for _, v := range updateDataList {
+				err = coll.UpdateOne(ctx, bson.M{"_id": v.ID}, bson.M{"$set": bson.M{"value": v.Value, "modify_time": v.ModifyTime}})
+				if err != nil {
+					fmt.Println("BatchInsertData:Err:" + err.Error())
+					return nil, err
+				}
+			}
+		}
+
+		return nil, nil
+	}
+	result, err = utils.MgoDataCli.DoTransaction(ctx, callback)
+
+	return
+}
+
+// GetEdbInfoMaxAndMinInfo
+// @Description: 获取当前指标的最大最小值
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-30 17:15:39
+// @param whereParams interface{}
+// @return result EdbInfoMaxAndMinInfo
+// @return err error
+func (m *BaseFromThsHfData) GetEdbInfoMaxAndMinInfo(whereParams interface{}) (result EdbInfoMaxAndMinInfo, err error) {
+	if utils.MgoDataCli == nil {
+		err = errors.New("mongodb连接失败")
+		return
+	}
+	db := utils.MgoDataCli.Database(m.DataBaseName())
+	coll := db.Collection(m.CollectionName())
+	ctx := context.TODO()
+	if err != nil {
+		fmt.Println("MgoGetColl Err:", err.Error())
+		return
+	}
+	err = coll.Aggregate(ctx, whereParams).One(&result)
+	if err != nil {
+		return
+	}
+	result.MinDate = result.MinDate.In(time.Local)
+	result.MaxDate = result.MaxDate.In(time.Local)
+	result.LatestDate = result.LatestDate.In(time.Local)
+	return
+}
+
+// GetLatestValue
+// @Description: 获取当前指标的最新数据记录
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-30 17:16:15
+// @param whereParams interface{}
+// @param selectParam interface{}
+// @return latestValue LatestValue
+// @return err error
+func (m *BaseFromThsHfData) GetLatestValue(whereParams, selectParam interface{}) (latestValue LatestValue, err error) {
+	if utils.MgoDataCli == nil {
+		err = errors.New("mongodb连接失败")
+		return
+	}
+	db := utils.MgoDataCli.Database(m.DataBaseName())
+	coll := db.Collection(m.CollectionName())
+	ctx := context.TODO()
+	if err != nil {
+		fmt.Println("MgoGetColl Err:", err.Error())
+		return
+	}
+
+	//var result interface{}
+	//err = coll.Find(ctx, whereParams).Select(selectParam).One(&result)
+	err = coll.Find(ctx, whereParams).Select(selectParam).One(&latestValue)
+	return
+}

models/mgo/edb_data_ths_hf.go

@@ -0,0 +1,510 @@
+package mgo
+
+import (
+	"context"
+	"errors"
+	"eta/eta_index_lib/utils"
+	"fmt"
+	"github.com/qiniu/qmgo"
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+	"time"
+)
+
+// EdbDataThsHf
+// @Description: 同花顺高频集合（指标库）
+type EdbDataThsHf struct {
+	ID            primitive.ObjectID `json:"_id" bson:"_id,omitempty" `            // 文档id
+	EdbInfoId     int                `json:"edb_info_id" bson:"edb_info_id"`       // 指标ID
+	EdbCode       string             `json:"edb_code" bson:"edb_code"`             // 指标编码
+	DataTime      time.Time          `json:"data_time" bson:"data_time"`           // 数据日期
+	Value         float64            `json:"value" bson:"value"`                   // 数据值
+	CreateTime    time.Time          `json:"create_time" bson:"create_time"`       // 创建时间
+	ModifyTime    time.Time          `json:"modify_time" bson:"modify_time"`       // 修改时间
+	DataTimestamp int64              `json:"data_timestamp" bson:"data_timestamp"` // 数据日期时间戳
+}
+
+// CollectionName
+// @Description:  获取集合名称
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-26 13:41:36
+// @return string
+func (m *EdbDataThsHf) CollectionName() string {
+	return "edb_data_ths_hf"
+}
+
+// DataBaseName
+// @Description: 获取数据库名称
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-26 13:41:33
+// @return string
+func (m *EdbDataThsHf) DataBaseName() string {
+	return utils.MgoDataDbName
+}
+
+// GetCollection
+// @Description: 获取mongodb集合的句柄
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-26 13:41:33
+// @return string
+func (m *EdbDataThsHf) GetCollection() *qmgo.Collection {
+	db := utils.MgoDataCli.Database(m.DataBaseName())
+	return db.Collection(m.CollectionName())
+}
+
+// GetItem
+// @Description: 根据条件获取单条数据
+// @author: Roc
+// @receiver m
+// @datetime 2024-05-09 10:00:49
+// @param whereParams interface{}
+// @return item *EdbDataThsHf
+// @return err error
+func (m *EdbDataThsHf) GetItem(whereParams interface{}) (item *EdbDataThsHf, err error) {
+	if utils.MgoDataCli == nil {
+		err = errors.New("mongodb连接失败")
+		return
+	}
+	db := utils.MgoDataCli.Database(m.DataBaseName())
+	coll := db.Collection(m.CollectionName())
+
+	return m.GetItemByColl(coll, whereParams)
+}
+
+// GetItemByColl
+// @Description: 根据条件获取单条数据
+// @author: Roc
+// @receiver m
+// @datetime 2024-05-09 13:22:06
+// @param coll *qmgo.Collection
+// @param whereParams interface{}
+// @return item *EdbDataThsHf
+// @return err error
+func (m *EdbDataThsHf) GetItemByColl(coll *qmgo.Collection, whereParams interface{}) (item *EdbDataThsHf, err error) {
+	ctx := context.TODO()
+	if err != nil {
+		fmt.Println("MgoGetColl Err:", err.Error())
+		return
+	}
+	err = coll.Find(ctx, whereParams).One(&item)
+	if err != nil {
+		return
+	}
+
+	item.DataTime = item.DataTime.In(time.Local)
+	item.CreateTime = item.CreateTime.In(time.Local)
+	item.ModifyTime = item.ModifyTime.In(time.Local)
+
+	return
+}
+
+// GetAllDataList
+// @Description: 根据条件获取所有数据
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-26 13:42:19
+// @param whereParams interface{}
+// @param sort []string
+// @return result []EdbDataThsHf
+// @return err error
+func (m *EdbDataThsHf) GetAllDataList(whereParams interface{}, sort []string) (result []*EdbDataThsHf, err error) {
+	if utils.MgoDataCli == nil {
+		err = errors.New("mongodb连接失败")
+		return
+	}
+	db := utils.MgoDataCli.Database(m.DataBaseName())
+	coll := db.Collection(m.CollectionName())
+	ctx := context.TODO()
+	if err != nil {
+		fmt.Println("MgoGetColl Err:", err.Error())
+		return
+	}
+	err = coll.Find(ctx, whereParams).Sort(sort...).All(&result)
+	if err != nil {
+		return
+	}
+
+	for _, v := range result {
+		v.DataTime = v.DataTime.In(time.Local)
+		v.CreateTime = v.CreateTime.In(time.Local)
+		v.ModifyTime = v.ModifyTime.In(time.Local)
+	}
+
+	return
+}
+
+// GetLimitDataList
+// @Description: 根据条件获取指定数量数据列表
+// @author: Roc
+// @receiver m
+// @datetime 2024-05-06 17:08:32
+// @param whereParams interface{}
+// @param size int64
+// @param sort []string
+// @return result []*BaseFromBusinessData
+// @return err error
+func (m *EdbDataThsHf) GetLimitDataList(whereParams interface{}, size int64, sort []string) (result []*EdbDataThsHf, err error) {
+	if utils.MgoDataCli == nil {
+		err = errors.New("mongodb连接失败")
+		return
+	}
+	db := utils.MgoDataCli.Database(m.DataBaseName())
+	coll := db.Collection(m.CollectionName())
+	ctx := context.TODO()
+	if err != nil {
+		fmt.Println("MgoGetColl Err:", err.Error())
+		return
+	}
+	err = coll.Find(ctx, whereParams).Sort(sort...).Limit(size).All(&result)
+	if err != nil {
+		return
+	}
+
+	for _, v := range result {
+		v.DataTime = v.DataTime.In(time.Local)
+		v.CreateTime = v.CreateTime.In(time.Local)
+		v.ModifyTime = v.ModifyTime.In(time.Local)
+	}
+
+	return
+}
+
+// GetPageDataList
+// @Description: 根据条件获取分页数据列表
+// @author: Roc
+// @receiver m
+// @datetime 2024-05-07 10:21:07
+// @param whereParams interface{}
+// @param startSize int64
+// @param size int64
+// @param sort []string
+// @return result []*EdbDataThsHf
+// @return err error
+func (m *EdbDataThsHf) GetPageDataList(whereParams interface{}, startSize, size int64, sort []string) (result []*EdbDataThsHf, err error) {
+	if utils.MgoDataCli == nil {
+		err = errors.New("mongodb连接失败")
+		return
+	}
+	db := utils.MgoDataCli.Database(m.DataBaseName())
+	coll := db.Collection(m.CollectionName())
+	ctx := context.TODO()
+	if err != nil {
+		fmt.Println("MgoGetColl Err:", err.Error())
+		return
+	}
+	err = coll.Find(ctx, whereParams).Sort(sort...).Skip(startSize).Limit(size).All(&result)
+	if err != nil {
+		return
+	}
+
+	for _, v := range result {
+		v.DataTime = v.DataTime.In(time.Local)
+		v.CreateTime = v.CreateTime.In(time.Local)
+		v.ModifyTime = v.ModifyTime.In(time.Local)
+	}
+
+	return
+}
+
+// GetCountDataList
+// @Description:  根据条件获取数据列表总数
+// @author: Roc
+// @receiver m
+// @datetime 2024-05-07 10:29:00
+// @param whereParams interface{}
+// @return count int64
+// @return err error
+func (m *EdbDataThsHf) GetCountDataList(whereParams interface{}) (count int64, err error) {
+	if utils.MgoDataCli == nil {
+		err = errors.New("mongodb连接失败")
+		return
+	}
+	db := utils.MgoDataCli.Database(m.DataBaseName())
+	coll := db.Collection(m.CollectionName())
+	ctx := context.TODO()
+	if err != nil {
+		fmt.Println("MgoGetColl Err:", err.Error())
+		return
+	}
+	count, err = coll.Find(ctx, whereParams).Count()
+
+	return
+}
+
+// InsertDataByColl
+// @Description: 写入单条数据（外部传入集合）
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-26 14:22:18
+// @param addData interface{}
+// @return err error
+func (m *EdbDataThsHf) InsertDataByColl(coll *qmgo.Collection, addData interface{}) (err error) {
+	ctx := context.TODO()
+	_, err = coll.InsertOne(ctx, addData)
+	if err != nil {
+		fmt.Println("InsertDataByColl:Err:" + err.Error())
+		return
+	}
+
+	return
+}
+
+// BatchInsertData
+// @Description: 批量写入数据
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-26 14:22:18
+// @param bulk int 每次请求保存的数据量
+// @param dataList []interface{}
+// @return err error
+func (m *EdbDataThsHf) BatchInsertData(bulk int, dataList []interface{}) (err error) {
+	db := utils.MgoDataCli.Database(m.DataBaseName())
+	coll := db.Collection(m.CollectionName())
+
+	return m.BatchInsertDataByColl(coll, bulk, dataList)
+}
+
+// BatchInsertDataByColl
+// @Description: 批量写入数据（外部传入集合）
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-26 14:22:18
+// @param coll *qmgo.Collection
+// @param bulk int 每次请求保存的数据量
+// @param dataList []interface{}
+// @return err error
+func (m *EdbDataThsHf) BatchInsertDataByColl(coll *qmgo.Collection, bulk int, dataList []interface{}) (err error) {
+	ctx := context.TODO()
+	dataNum := len(dataList)
+	if dataNum <= 0 {
+		return
+	}
+
+	// 不设置每次保存切片数量大小，或者实际数据量小于设置的切片数量大小，那么就直接保存吧
+	if bulk <= 0 || dataNum <= bulk {
+		_, err = coll.InsertMany(ctx, dataList)
+		if err != nil {
+			fmt.Println("BatchInsertData:Err:" + err.Error())
+			return
+		}
+		return
+	}
+
+	// 分批保存
+	i := 0
+	tmpAddDataList := make([]interface{}, 0)
+	for _, v := range dataList {
+		tmpAddDataList = append(tmpAddDataList, v)
+		i++
+		if i >= bulk {
+			_, err = coll.InsertMany(ctx, tmpAddDataList)
+			if err != nil {
+				fmt.Println("BatchInsertData:Err:" + err.Error())
+				return
+			}
+			i = 0
+			tmpAddDataList = make([]interface{}, 0)
+		}
+	}
+
+	if len(tmpAddDataList) > 0 {
+		_, err = coll.InsertMany(ctx, tmpAddDataList)
+		if err != nil {
+			fmt.Println("BatchInsertData:Err:" + err.Error())
+			return
+		}
+	}
+
+	return
+}
+
+// UpdateData
+// @Description: 单条数据修改
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-26 15:01:51
+// @param whereParams interface{}
+// @param updateParams interface{}
+// @return err error
+func (m *EdbDataThsHf) UpdateData(whereParams, updateParams interface{}) (err error) {
+	db := utils.MgoDataCli.Database(m.DataBaseName())
+	coll := db.Collection(m.CollectionName())
+
+	return m.UpdateDataByColl(coll, whereParams, updateParams)
+}
+
+// UpdateDataByColl
+// @Description: 单条数据修改（外部传入集合）
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-26 15:01:51
+// @param whereParams interface{}
+// @param updateParams interface{}
+// @return err error
+func (m *EdbDataThsHf) UpdateDataByColl(coll *qmgo.Collection, whereParams, updateParams interface{}) (err error) {
+	ctx := context.TODO()
+	err = coll.UpdateOne(ctx, whereParams, updateParams)
+	if err != nil {
+		fmt.Println("UpdateDataByColl:Err:" + err.Error())
+		return
+	}
+
+	return
+}
+
+// RemoveMany
+// @Description: 根据条件删除多条数据
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-30 13:17:02
+// @param whereParams interface{}
+// @return err error
+func (m *EdbDataThsHf) RemoveMany(whereParams interface{}) (err error) {
+	db := utils.MgoDataCli.Database(m.DataBaseName())
+	coll := db.Collection(m.CollectionName())
+
+	return m.RemoveManyByColl(coll, whereParams)
+}
+
+// RemoveManyByColl
+// @Description: 根据条件删除多条数据（外部传入集合）
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-30 13:18:42
+// @param coll *qmgo.Collection
+// @param whereParams interface{}
+// @return err error
+func (m *EdbDataThsHf) RemoveManyByColl(coll *qmgo.Collection, whereParams interface{}) (err error) {
+	ctx := context.TODO()
+	_, err = coll.RemoveAll(ctx, whereParams)
+	if err != nil {
+		fmt.Println("RemoveManyByColl:Err:" + err.Error())
+		return
+	}
+
+	return
+}
+
+// HandleData
+// @Description: 事务处理数据
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-30 10:39:01
+// @param addDataList []AddEdbDataThsHf
+// @param updateDataList []EdbDataThsHf
+// @return result interface{}
+// @return err error
+func (m *EdbDataThsHf) HandleData(addDataList, updateDataList []EdbDataThsHf) (result interface{}, err error) {
+
+	ctx := context.TODO()
+
+	callback := func(sessCtx context.Context) (interface{}, error) {
+		// 重要：确保事务中的每一个操作，都使用传入的sessCtx参数
+
+		db := utils.MgoDataCli.Database(m.DataBaseName())
+		coll := db.Collection(m.CollectionName())
+
+		// 插入数据
+		if len(addDataList) > 0 {
+			_, err = coll.InsertMany(sessCtx, addDataList)
+			if err != nil {
+				return nil, err
+			}
+		}
+
+		// 修改
+
+		if len(updateDataList) > 0 {
+			for _, v := range updateDataList {
+				err = coll.UpdateOne(ctx, bson.M{"_id": v.ID}, bson.M{"$set": bson.M{"value": v.Value, "modify_time": v.ModifyTime}})
+				if err != nil {
+					fmt.Println("BatchInsertData:Err:" + err.Error())
+					return nil, err
+				}
+			}
+		}
+
+		return nil, nil
+	}
+	result, err = utils.MgoDataCli.DoTransaction(ctx, callback)
+
+	return
+}
+
+// EdbInfoMaxAndMinInfo 指标最新数据记录结构体
+//type EdbInfoMaxAndMinInfo struct {
+//	MinDate     time.Time `description:"最小日期" bson:"min_date"`
+//	MaxDate     time.Time `description:"最大日期" bson:"max_date"`
+//	MinValue    float64   `description:"最小值" bson:"min_value"`
+//	MaxValue    float64   `description:"最大值" bson:"max_value"`
+//	LatestValue float64   `description:"最新值" bson:"latest_value"`
+//	LatestDate  time.Time `description:"实际数据最新日期" bson:"latest_date"`
+//	EndValue    float64   `description:"最新值" bson:"end_value"`
+//}
+
+// GetEdbInfoMaxAndMinInfo
+// @Description: 获取当前指标的最大最小值
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-30 17:15:39
+// @param whereParams interface{}
+// @return result EdbInfoMaxAndMinInfo
+// @return err error
+func (m *EdbDataThsHf) GetEdbInfoMaxAndMinInfo(whereParams interface{}) (result EdbInfoMaxAndMinInfo, err error) {
+	if utils.MgoDataCli == nil {
+		err = errors.New("mongodb连接失败")
+		return
+	}
+	db := utils.MgoDataCli.Database(m.DataBaseName())
+	coll := db.Collection(m.CollectionName())
+	ctx := context.TODO()
+	if err != nil {
+		fmt.Println("MgoGetColl Err:", err.Error())
+		return
+	}
+	err = coll.Aggregate(ctx, whereParams).One(&result)
+	if err != nil {
+		return
+	}
+	result.MinDate = result.MinDate.In(time.Local)
+	result.MaxDate = result.MaxDate.In(time.Local)
+	result.LatestDate = result.LatestDate.In(time.Local)
+
+	return
+}
+
+// LatestValue 指标最新数据记录结构体
+//type LatestValue struct {
+//	Value float64 `description:"值" bson:"value"`
+//}
+
+// GetLatestValue
+// @Description: 获取当前指标的最新数据记录
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-30 17:16:15
+// @param whereParams interface{}
+// @param selectParam interface{}
+// @return latestValue LatestValue
+// @return err error
+func (m *EdbDataThsHf) GetLatestValue(whereParams, selectParam interface{}) (latestValue LatestValue, err error) {
+	if utils.MgoDataCli == nil {
+		err = errors.New("mongodb连接失败")
+		return
+	}
+	db := utils.MgoDataCli.Database(m.DataBaseName())
+	coll := db.Collection(m.CollectionName())
+	ctx := context.TODO()
+	if err != nil {
+		fmt.Println("MgoGetColl Err:", err.Error())
+		return
+	}
+
+	//var result interface{}
+	//err = coll.Find(ctx, whereParams).Select(selectParam).One(&result)
+	err = coll.Find(ctx, whereParams).Select(selectParam).One(&latestValue)
+	return
+}

routers/commentsRouter.go

@@ -7,6 +7,24 @@ import (
 
 func init() {
 
+    beego.GlobalControllerRouter["eta/eta_index_lib/controllers/factor_edb_series:FactorEdbSeriesController"] = append(beego.GlobalControllerRouter["eta/eta_index_lib/controllers/factor_edb_series:FactorEdbSeriesController"],
+        beego.ControllerComments{
+            Method: "ChartRecalculate",
+            Router: `/chart_recalculate`,
+            AllowHTTPMethods: []string{"post"},
+            MethodParams: param.Make(),
+            Filters: nil,
+            Params: nil})
+
+    beego.GlobalControllerRouter["eta/eta_index_lib/controllers/factor_edb_series:FactorEdbSeriesController"] = append(beego.GlobalControllerRouter["eta/eta_index_lib/controllers/factor_edb_series:FactorEdbSeriesController"],
+        beego.ControllerComments{
+            Method: "Recalculate",
+            Router: `/recalculate`,
+            AllowHTTPMethods: []string{"post"},
+            MethodParams: param.Make(),
+            Filters: nil,
+            Params: nil})
+
     beego.GlobalControllerRouter["eta/eta_index_lib/controllers/fix:CustomAnalysisController"] = append(beego.GlobalControllerRouter["eta/eta_index_lib/controllers/fix:CustomAnalysisController"],
         beego.ControllerComments{
             Method: "FixTableV1",
@@ -232,6 +250,15 @@ func init() {
             Filters: nil,
             Params: nil})
 
+    beego.GlobalControllerRouter["eta/eta_index_lib/controllers:CalculateController"] = append(beego.GlobalControllerRouter["eta/eta_index_lib/controllers:CalculateController"],
+        beego.ControllerComments{
+            Method: "StepCalculate",
+            Router: `/base/step_calculate`,
+            AllowHTTPMethods: []string{"post"},
+            MethodParams: param.Make(),
+            Filters: nil,
+            Params: nil})
+
     beego.GlobalControllerRouter["eta/eta_index_lib/controllers:CalculateController"] = append(beego.GlobalControllerRouter["eta/eta_index_lib/controllers:CalculateController"],
         beego.ControllerComments{
             Method: "BatchEdit",
@@ -943,6 +970,15 @@ func init() {
             Filters: nil,
             Params: nil})
 
+    beego.GlobalControllerRouter["eta/eta_index_lib/controllers:MySteelChemicalController"] = append(beego.GlobalControllerRouter["eta/eta_index_lib/controllers:MySteelChemicalController"],
+        beego.ControllerComments{
+            Method: "HandleApiMysteelIndex",
+            Router: `/handle/api/mysteel/index`,
+            AllowHTTPMethods: []string{"post"},
+            MethodParams: param.Make(),
+            Filters: nil,
+            Params: nil})
+
     beego.GlobalControllerRouter["eta/eta_index_lib/controllers:MySteelChemicalController"] = append(beego.GlobalControllerRouter["eta/eta_index_lib/controllers:MySteelChemicalController"],
         beego.ControllerComments{
             Method: "HandleMysteelIndex",
@@ -1411,6 +1447,51 @@ func init() {
             Filters: nil,
             Params: nil})
 
+    beego.GlobalControllerRouter["eta/eta_index_lib/controllers:ThsHfController"] = append(beego.GlobalControllerRouter["eta/eta_index_lib/controllers:ThsHfController"],
+        beego.ControllerComments{
+            Method: "BaseAdd",
+            Router: `/hf/base/add`,
+            AllowHTTPMethods: []string{"post"},
+            MethodParams: param.Make(),
+            Filters: nil,
+            Params: nil})
+
+    beego.GlobalControllerRouter["eta/eta_index_lib/controllers:ThsHfController"] = append(beego.GlobalControllerRouter["eta/eta_index_lib/controllers:ThsHfController"],
+        beego.ControllerComments{
+            Method: "BaseRefresh",
+            Router: `/hf/base/refresh`,
+            AllowHTTPMethods: []string{"post"},
+            MethodParams: param.Make(),
+            Filters: nil,
+            Params: nil})
+
+    beego.GlobalControllerRouter["eta/eta_index_lib/controllers:ThsHfController"] = append(beego.GlobalControllerRouter["eta/eta_index_lib/controllers:ThsHfController"],
+        beego.ControllerComments{
+            Method: "EdbAdd",
+            Router: `/hf/edb/add`,
+            AllowHTTPMethods: []string{"post"},
+            MethodParams: param.Make(),
+            Filters: nil,
+            Params: nil})
+
+    beego.GlobalControllerRouter["eta/eta_index_lib/controllers:ThsHfController"] = append(beego.GlobalControllerRouter["eta/eta_index_lib/controllers:ThsHfController"],
+        beego.ControllerComments{
+            Method: "EdbRefresh",
+            Router: `/hf/edb/refresh`,
+            AllowHTTPMethods: []string{"post"},
+            MethodParams: param.Make(),
+            Filters: nil,
+            Params: nil})
+
+    beego.GlobalControllerRouter["eta/eta_index_lib/controllers:ThsHfController"] = append(beego.GlobalControllerRouter["eta/eta_index_lib/controllers:ThsHfController"],
+        beego.ControllerComments{
+            Method: "GetData",
+            Router: `/hf/edb_data`,
+            AllowHTTPMethods: []string{"post"},
+            MethodParams: param.Make(),
+            Filters: nil,
+            Params: nil})
+
     beego.GlobalControllerRouter["eta/eta_index_lib/controllers:WindController"] = append(beego.GlobalControllerRouter["eta/eta_index_lib/controllers:WindController"],
         beego.ControllerComments{
             Method: "Add",

routers/router.go

@@ -9,6 +9,7 @@ package routers
 
 import (
 	"eta/eta_index_lib/controllers"
+	"eta/eta_index_lib/controllers/factor_edb_series"
 	"eta/eta_index_lib/controllers/fix"
 	"eta/eta_index_lib/controllers/future_good"
 	"eta/eta_index_lib/controllers/open"
@@ -22,6 +23,7 @@ func init() {
 			beego.NSInclude(
 				&controllers.ThsController{},
 				&controllers.ThsDsController{},
+				&controllers.ThsHfController{},
 			),
 		),
 		beego.NSNamespace("/pb",
@@ -280,6 +282,11 @@ func init() {
 				&controllers.SciHqController{},
 			),
 		),
+		beego.NSNamespace("/factor_edb_series",
+			beego.NSInclude(
+				&factor_edb_series.FactorEdbSeriesController{},
+			),
+		),
 	)
 	beego.AddNamespace(ns)
 }

services/base_from_business.go

@@ -8,6 +8,7 @@ import (
 	"eta/eta_index_lib/services/alarm_msg"
 	"eta/eta_index_lib/utils"
 	"fmt"
+	"github.com/qiniu/qmgo"
 	"go.mongodb.org/mongo-driver/bson"
 	"strings"
 	"time"
@@ -135,10 +136,44 @@ func HandleBusinessIndex(indexReq *models.AddBusinessIndexReq) (resp models.Base
 	}
 
 	// 数据处理
+	// 当前传入的最小日期
+	var reqMinDate time.Time
+	if utils.UseMongo {
+		reqMinDate, err = handleBusinessDataByMongo(item, indexReq.DataList)
+	} else {
+		reqMinDate, err = handleBusinessDataByMysql(item, indexReq.DataList)
+	}
+	if err != nil {
+		return
+	}
+
+	// 同步刷新指标库的指标
+	go refreshEdbBusiness(item.IndexCode, reqMinDate)
+
+	resp = models.BaseFromBusinessIndexResp{
+		IndexCode:  item.IndexCode,
+		IndexName:  item.IndexName,
+		Unit:       item.Unit,
+		Frequency:  item.Frequency,
+		SourceName: item.SourceName,
+	}
+
+	return
+}
+
+// handleBusinessDataByMongo
+// @Description: 处理外部指标数据（mongo）
+// @author: Roc
+// @datetime 2024-07-01 15:30:41
+// @param item *models.BaseFromBusinessIndex
+// @param reqDataList []*models.AddBusinessDataReq
+// @return reqMinDate time.Time 当前传入的最小日期
+// @return err error
+func handleBusinessDataByMongo(item *models.BaseFromBusinessIndex, reqDataList []models.AddBusinessDataReq) (reqMinDate time.Time, err error) {
 	mogDataObj := new(mgo.BaseFromBusinessData)
 
 	//获取已存在的所有数据
-	exitDataList, err := mogDataObj.GetAllDataList(bson.M{"index_code": indexReq.IndexCode}, []string{"data_time"})
+	exitDataList, err := mogDataObj.GetAllDataList(bson.M{"index_code": item.IndexCode}, []string{"data_time"})
 	if err != nil {
 		fmt.Println("GetIndexDataList Err:" + err.Error())
 		return
@@ -150,14 +185,12 @@ func HandleBusinessIndex(indexReq *models.AddBusinessIndexReq) (resp models.Base
 		exitDataMap[v.DataTime.Format(utils.FormatDate)] = v
 	}
 
-	// 当前传入的最小日期
-	var reqMinDate time.Time
 	// 待添加的数据集
 	addDataList := make([]interface{}, 0)
 	updateDataList := make([]mgo.BaseFromBusinessData, 0)
 	//var hasUpdate bool
 	// 遍历excel数据，然后跟现有的数据做校验，不存在则入库
-	for _, data := range indexReq.DataList {
+	for _, data := range reqDataList {
 		dateTime, tmpErr := utils.DealExcelDate(data.Date)
 		if tmpErr != nil {
 			fmt.Println("time.ParseInLocation Err:" + tmpErr.Error())
@@ -229,27 +262,121 @@ func HandleBusinessIndex(indexReq *models.AddBusinessIndexReq) (resp models.Base
 	//fmt.Println("result", result)
 
 	//修改最大最小日期
-	indexMaxAndMinInfo, err := item.GetEdbInfoMaxAndMinInfo(indexReq.IndexCode)
+	indexMaxAndMinInfo, err := item.GetEdbInfoMaxAndMinInfo(item.IndexCode)
 	if err != nil {
 		return
 	}
 	if err == nil && indexMaxAndMinInfo != nil {
-		e := item.ModifyIndexMaxAndMinInfo(indexReq.IndexCode, indexMaxAndMinInfo, isIndexUpdateOrAdd)
+		e := item.ModifyIndexMaxAndMinInfo(item.IndexCode, indexMaxAndMinInfo, isIndexUpdateOrAdd)
 		if e != nil {
 			fmt.Println("ModifyIndexMaxAndMinInfo Err:" + e.Error())
 		}
 	}
 
-	// 同步刷新指标库的指标
-	go refreshEdbBusiness(item.IndexCode, reqMinDate)
+	return
+}
 
-	resp = models.BaseFromBusinessIndexResp{
-		IndexCode:  item.IndexCode,
-		IndexName:  item.IndexName,
-		Unit:       item.Unit,
-		Frequency:  item.Frequency,
-		SourceName: item.SourceName,
+// handleBusinessDataByMysql
+// @Description: 处理外部指标数据（mysql）
+// @author: Roc
+// @datetime 2024-07-01 15:59:43
+// @param item *models.BaseFromBusinessIndex
+// @param reqDataList []models.AddBusinessDataReq
+// @return reqMinDate time.Time
+// @return err error
+func handleBusinessDataByMysql(item *models.BaseFromBusinessIndex, reqDataList []models.AddBusinessDataReq) (reqMinDate time.Time, err error) {
+	businessDataObj := new(models.BaseFromBusinessData)
+
+	var condition []string
+	var pars []interface{}
+	condition = append(condition, "index_code = ?")
+	pars = append(pars, item.IndexCode)
+	//获取已存在的所有数据
+	exitDataList, err := businessDataObj.GetAllDataList(condition, pars, "data_time ASC")
+	if err != nil {
+		fmt.Println("GetIndexDataList Err:" + err.Error())
+		return
+	}
+
+	// 已经存在的数据集
+	exitDataMap := make(map[string]*models.BaseFromBusinessData)
+	for _, v := range exitDataList {
+		exitDataMap[v.DataTime.Format(utils.FormatDate)] = v
+	}
+
+	// 待添加的数据集
+	addDataList := make([]*models.BaseFromBusinessData, 0)
+	updateDataList := make([]*models.BaseFromBusinessData, 0)
+	//var hasUpdate bool
+	// 遍历excel数据，然后跟现有的数据做校验，不存在则入库
+	for _, data := range reqDataList {
+		dateTime, tmpErr := utils.DealExcelDate(data.Date)
+		if tmpErr != nil {
+			fmt.Println("time.ParseInLocation Err:" + tmpErr.Error())
+			err = tmpErr
+			return
+		}
+
+		// 调整最小日期
+		if reqMinDate.IsZero() || reqMinDate.After(dateTime) {
+			reqMinDate = dateTime
+		}
+
+		date := dateTime.Format(utils.FormatDate)
+
+		findData, ok := exitDataMap[date]
+		if !ok {
+			addDataList = append(addDataList, &models.BaseFromBusinessData{
+				BaseFromBusinessIndexId: int(item.BaseFromBusinessIndexId),
+				IndexCode:               item.IndexCode,
+				DataTime:                dateTime,
+				Value:                   data.Value,
+				CreateTime:              time.Now(),
+				ModifyTime:              time.Now(),
+				//DataTimestamp:           0,
+			})
+			continue
+		}
+
+		// 值不匹配，修改数据
+		if findData.Value != data.Value {
+			findData.Value = data.Value
+			findData.ModifyTime = time.Now()
+			updateDataList = append(updateDataList, findData)
+		}
+	}
+
+	// 指标数据是否新增或修改
+	var isIndexUpdateOrAdd bool
+
+	// 入库
+	{
+		if len(addDataList) > 0 {
+			isIndexUpdateOrAdd = true
+		}
+
+		if len(updateDataList) > 0 {
+			isIndexUpdateOrAdd = true
+		}
+		err = businessDataObj.HandleData(addDataList, updateDataList)
+		if err != nil {
+			fmt.Println("UpdateDataByColl:Err:" + err.Error())
+			return
+		}
 	}
+
+	//修改最大最小日期
+	indexMaxAndMinInfo, err := item.GetEdbInfoMaxAndMinInfo(item.IndexCode)
+	if err != nil {
+		return
+	}
+	if err == nil && indexMaxAndMinInfo != nil {
+		e := item.ModifyIndexMaxAndMinInfo(item.IndexCode, indexMaxAndMinInfo, isIndexUpdateOrAdd)
+		if e != nil {
+			fmt.Println("ModifyIndexMaxAndMinInfo Err:" + e.Error())
+		}
+	}
+
 	return
 }
 
@@ -326,9 +453,22 @@ func DelBusinessIndex(indexCodeList []string) (joinEdbCodeList, needDelEdbCodeLi
 		return
 	}
 
-	// 删除指标明细数据
-	mogDataObj := new(mgo.BaseFromBusinessData)
-	err = mogDataObj.RemoveMany(bson.M{"index_code": bson.M{"$in": needDelEdbCodeList}})
+	if utils.UseMongo {
+		// 删除指标明细数据
+		mogDataObj := new(mgo.BaseFromBusinessData)
+		err = mogDataObj.RemoveMany(bson.M{"index_code": bson.M{"$in": needDelEdbCodeList}})
+	} else {
+		var condition []string
+		var pars []interface{}
+		delNum := len(needDelEdbCodeList)
+		if delNum > 0 {
+			condition = append(condition, "index_code in ("+utils.GetOrmInReplace(delNum)+")")
+			pars = append(pars, needDelEdbCodeList)
+
+			businessDataObj := models.BaseFromBusinessData{}
+			err = businessDataObj.DelDataByCond(condition, pars)
+		}
+	}
 	if err != nil {
 		fmt.Println("删除自有指标明细数据 Err:" + err.Error())
 		return
@@ -348,7 +488,7 @@ func DelBusinessIndex(indexCodeList []string) (joinEdbCodeList, needDelEdbCodeLi
 func DelBusinessIndexData(indexCode string, startDate, endDate string) (err error, errMsg string) {
 	defer func() {
 		if err != nil {
-			fmt.Println("DelBusinessIndex Err:" + err.Error())
+			fmt.Println("DelBusinessIndexData Err:" + err.Error())
 		}
 	}()
 	errMsg = "删除失败"
@@ -377,11 +517,6 @@ func DelBusinessIndexData(indexCode string, startDate, endDate string) (err erro
 		return
 	}
 
-	// 构建查询条件
-	queryConditions := bson.M{
-		"index_code": item.IndexCode,
-	}
-
 	// 当前传入的最小日期
 	var reqMinDate time.Time
 
@@ -410,6 +545,51 @@ func DelBusinessIndexData(indexCode string, startDate, endDate string) (err erro
 		}
 	}
 
+	// 删除具体的数据
+	var indexMaxAndMinInfo *models.EdbInfoMaxAndMinInfo
+	if utils.UseMongo {
+		indexMaxAndMinInfo, err = delBusinessIndexDataByMongo(item, startDateTime, endDateTime)
+	} else {
+		indexMaxAndMinInfo, err = delBusinessIndexDataByMysql(item, startDateTime, endDateTime)
+	}
+	if err != nil {
+		return
+	}
+
+	// 修改指标的最早最晚日期
+	if indexMaxAndMinInfo != nil {
+		e := item.ModifyIndexMaxAndMinInfo(item.IndexCode, indexMaxAndMinInfo, true)
+		if e != nil {
+			fmt.Println("ModifyIndexMaxAndMinInfo Err:" + e.Error())
+		}
+	}
+
+	// 同步刷新指标库的指标
+	go refreshEdbBusiness(item.IndexCode, reqMinDate)
+
+	return
+}
+
+// delBusinessIndexDataByMongo
+// @Description: 删除指标数据（从mongo删除）
+// @author: Roc
+// @datetime 2024-07-01 18:00:07
+// @param item *models.BaseFromBusinessIndex
+// @param startDateTime time.Time
+// @param endDateTime time.Time
+// @return err error
+func delBusinessIndexDataByMongo(item *models.BaseFromBusinessIndex, startDateTime, endDateTime time.Time) (indexMaxAndMinInfo *models.EdbInfoMaxAndMinInfo, err error) {
+	defer func() {
+		if err != nil {
+			utils.FileLog.Error("delBusinessIndexDataByMongo 删除自有指标明细数据 Err:" + err.Error())
+		}
+	}()
+
+	// 构建查询条件
+	queryConditions := bson.M{
+		"index_code": item.IndexCode,
+	}
+
 	dateCondition, err := mgo.BuildDateTimeCondition(startDateTime, endDateTime)
 	if err != nil {
 		return
@@ -422,12 +602,66 @@ func DelBusinessIndexData(indexCode string, startDate, endDate string) (err erro
 	mogDataObj := new(mgo.BaseFromBusinessData)
 	err = mogDataObj.RemoveMany(queryConditions)
 	if err != nil {
-		fmt.Println("删除自有指标明细数据 Err:" + err.Error())
 		return
 	}
 
-	// 同步刷新指标库的指标
-	go refreshEdbBusiness(item.IndexCode, reqMinDate)
+	//修改最大最小日期
+	indexMaxAndMinInfo, err = item.GetEdbInfoMaxAndMinInfo(item.IndexCode)
+	// 如果有错误，且错误信息是取不到文档，那么就不修改了
+	if err != nil && !errors.Is(err, qmgo.ErrNoSuchDocuments) {
+		return
+	}
+	// 清空的目的是为了避免异常返回
+	err = nil
+
+	return
+}
+
+// delBusinessIndexDataByMysql
+// @Description: 删除指标数据（从mysql删除）
+// @author: Roc
+// @datetime 2024-07-02 09:53:13
+// @param item *models.BaseFromBusinessIndex
+// @param startDateTime time.Time
+// @param endDateTime time.Time
+// @return err error
+func delBusinessIndexDataByMysql(item *models.BaseFromBusinessIndex, startDateTime, endDateTime time.Time) (indexMaxAndMinInfo *models.EdbInfoMaxAndMinInfo, err error) {
+	defer func() {
+		if err != nil {
+			utils.FileLog.Error("delBusinessIndexDataByMysql 删除自有指标明细数据 Err:" + err.Error())
+		}
+	}()
+
+	// 构建查询条件
+	var condition []string
+	var pars []interface{}
+	condition = append(condition, "index_code = ? ")
+	pars = append(pars, item.IndexCode)
+
+	if !startDateTime.IsZero() {
+		condition = append(condition, " data_time >= ? ")
+		pars = append(pars, startDateTime.Format(utils.FormatDate))
+	}
+	if !endDateTime.IsZero() {
+		condition = append(condition, " data_time <= ? ")
+		pars = append(pars, endDateTime.Format(utils.FormatDate))
+	}
+
+	// 删除数据源中的指标明细数据
+	businessDataObj := new(models.BaseFromBusinessData)
+	err = businessDataObj.DelDataByCond(condition, pars)
+	if err != nil {
+		return
+	}
+
+	//修改最大最小日期
+	indexMaxAndMinInfo, err = item.GetEdbInfoMaxAndMinInfo(item.IndexCode)
+	// 如果有错误，且错误信息是取不到文档，那么就不修改了
+	if err != nil && err.Error() != utils.ErrNoRow() {
+		return
+	}
+	// 清空的目的是为了避免异常返回
+	err = nil
 
 	return
 }

services/base_from_calculate.go

@@ -5,6 +5,7 @@ import (
 	"eta/eta_index_lib/models"
 	"eta/eta_index_lib/utils"
 	"fmt"
+	"sort"
 	"strconv"
 	"strings"
 	"time"
@@ -455,6 +456,7 @@ func EdbCalculateBatchSave(req models.EdbInfoCalculateBatchSaveReq, lang string)
 
 	// 更新ES
 	go logic.UpdateEs(edbInfo.EdbInfoId)
+	go DisableEdbInfoNoUpdate(edbInfo)
 	return
 }
 
@@ -856,6 +858,9 @@ func EdbCalculateBatchEdit(req models.EdbInfoCalculateBatchEditReq) (edbInfo *mo
 
 	// 更新ES
 	go logic.UpdateEs(edbInfo.EdbInfoId)
+
+	// 重置计算指标中的引用关系
+	go ResetEdbRelation(edbInfoId)
 	return
 }
 
@@ -1006,6 +1011,82 @@ func EdbCalculateAdd(req models.EdbInfoCalculateSaveReq, lang string) (edbInfo *
 		err = fmt.Errorf("生成计算指标失败")
 		return
 	}
+	go DisableEdbInfoNoUpdate(edbInfo)
+	return
+}
+
+// StepCalculate 多公式计算
+func StepCalculate(dataList []*models.EdbInfoSearchData, calculates []models.CalculatesReq) (resultData map[string]float64, dates []string, errMsg string, err error) {
+	sort.Slice(calculates, func(i, j int) bool {
+		return calculates[i].Sort < calculates[j].Sort
+	})
+	originData, e := models.EdbInfoSearchDataToData(dataList)
+	if e != nil {
+		err = fmt.Errorf("基础数据转换异常, err: %v", e)
+		return
+	}
 
+	calculateData := originData
+	dateDataMap := make(map[time.Time]float64)
+	for _, v := range calculates {
+		baseCalculate := models.BaseCalculate{
+			DataList:      calculateData,
+			Frequency:     v.Frequency,
+			Formula:       v.Formula,
+			Calendar:      v.Calendar,
+			MoveType:      v.MoveType,
+			MoveFrequency: v.MoveFrequency,
+			FromFrequency: v.FromFrequency,
+			Source:        v.Source,
+		}
+
+		// 计算方式
+		switch baseCalculate.Source {
+		case utils.EdbBaseCalculateLjzzy:
+			dateDataMap, e, errMsg = baseCalculate.Ljzzy()
+		case utils.EdbBaseCalculateLjzzj:
+			dateDataMap, e, errMsg = baseCalculate.Ljzzj()
+		case utils.EdbBaseCalculateTbz:
+			dateDataMap, e, errMsg = baseCalculate.Tbz()
+		case utils.EdbBaseCalculateTcz:
+			dateDataMap, e, errMsg = baseCalculate.Tcz()
+		case utils.EdbBaseCalculateNszydpjjs:
+			dateDataMap, e, errMsg = baseCalculate.Nszydpjjs()
+		case utils.EdbBaseCalculateHbz:
+			dateDataMap, e, errMsg = baseCalculate.Hbz()
+		case utils.EdbBaseCalculateHcz:
+			dateDataMap, e, errMsg = baseCalculate.Hcz()
+		case utils.EdbBaseCalculateUpFrequency:
+			dateDataMap, e, errMsg = baseCalculate.UpFrequency()
+		case utils.EdbBaseCalculateDownFrequency:
+			dateDataMap, e, errMsg = baseCalculate.DownFrequency()
+		case utils.EdbBaseCalculateTimeShift:
+			dateDataMap, e, errMsg = baseCalculate.TimeShift()
+		case utils.EdbBaseCalculateCjjx:
+			dateDataMap, e, errMsg = baseCalculate.Cjjx()
+		case utils.EdbBaseCalculateAnnualized:
+			dateDataMap, e, errMsg = baseCalculate.Annualized()
+		case utils.EdbBaseCalculateLjz:
+			dateDataMap, e, errMsg = baseCalculate.Ljz()
+		case utils.EdbBaseCalculateLjzNczj:
+			dateDataMap, e, errMsg = baseCalculate.LjzNczj()
+		case utils.EdbBaseCalculateExponentialSmoothing:
+			dateDataMap, e, errMsg = baseCalculate.ExponentialSmoothing()
+		case utils.EdbBaseCalculateRjz:
+			dateDataMap, e, errMsg = baseCalculate.Rjz()
+		default:
+			errMsg = "计算方式无效"
+			e = fmt.Errorf("%s:%d", errMsg, baseCalculate.Source)
+		}
+		if e != nil {
+			err = fmt.Errorf("计算失败, err: %v", e)
+			return
+		}
+
+		calculateData = models.TransDateData2EdbData(dateDataMap)
+	}
+	resultData, dates = models.GetDateDataAndDateList(dateDataMap)
+	//resp.DataMap = resultData
+	//resp.DateList = dates
 	return
 }

services/base_from_mysteel_chemical.go

@@ -1,11 +1,16 @@
 package services
 
 import (
+	"encoding/json"
+	"errors"
 	"eta/eta_index_lib/logic"
 	"eta/eta_index_lib/models"
 	"eta/eta_index_lib/services/alarm_msg"
 	"eta/eta_index_lib/utils"
 	"fmt"
+	"io"
+	"net/http"
+	"strconv"
 	"strings"
 	"time"
 )
@@ -27,6 +32,218 @@ func HandleMysteelIndex(req *models.HandleMysteelIndexResp) (err error) {
 	return
 }
 
+func HandleApiMysteelIndex(req *models.HandleMysteelIndexResp) (errMsg string, err error) {
+	addIndexCodeList := make([]string, 0)
+	for _, v := range req.List {
+		if v.IndexCode == "" {
+			continue
+		}
+		addIndexCodeList = append(addIndexCodeList, v.IndexCode)
+	}
+	errMsg, err = HandleApiIndex(addIndexCodeList)
+	if err != nil {
+		return
+	}
+
+	_ = SetMysteelChemicalEdbInfoUpdateStat(false)
+	_ = SetEdbSourceStat(false)
+
+	return
+}
+
+func HandleApiIndex(indexCodes []string) (errMsg string, err error) {
+	if len(indexCodes) == 0 {
+		return
+	}
+	resp, err := GetEdbDataFromMySteelChemical(indexCodes, utils.GetEdbRefreshStartDate(""), utils.BASE_END_DATE, "desc")
+	if err != nil {
+		return
+	}
+	if !resp.Success {
+		errMsg = "获取数据失败"
+		err = errors.New(resp.Message)
+		return
+	}
+	indexInfoMap, err := GetMySteelChemicalIndexNameMap()
+	if err != nil {
+		errMsg = "获取指标数据失败"
+		return
+	}
+
+	indexObj := &models.BaseFromMysteelChemicalIndex{}
+	existIndexs, err := indexObj.GetBatchIndexItem(indexCodes)
+	if err != nil {
+		errMsg = "获取指标数据失败"
+		return
+	}
+
+	//获取已存在的所有数据
+	existDataMap := make(map[string]*models.BaseFromMysteelChemicalData)
+	existIndexMap := make(map[string]*models.BaseFromMysteelChemicalIndex)
+	updateDataObj := new(models.BaseFromMysteelChemicalData)
+	for _, v := range existIndexs {
+		// 更新指标的名称，单位和频度等信息
+		if info, ok := indexInfoMap[v.IndexCode]; ok {
+			v.IndexName = info.IndexName
+			v.Unit = info.UnitName
+			v.Frequency = info.FrequencyName
+			v.ModifyTime = time.Now()
+			err = v.Update([]string{"index_name", "unit", "frequency", "modify_time"})
+			if err != nil {
+				errMsg = "更新指标失败"
+				return
+			}
+		}
+		if err != nil {
+			errMsg = "添加指标失败"
+			return
+		}
+		existIndexMap[v.IndexCode] = v
+		exitDataList, er := updateDataObj.GetIndexDataList(v.IndexCode)
+		if er != nil {
+			errMsg = "获取指标数据失败"
+			err = er
+			return
+		}
+		fmt.Println("exitDataListLen:", len(exitDataList))
+		for _, v := range exitDataList {
+			dateStr := v.DataTime.Format(utils.FormatDate)
+			existDataMap[dateStr] = v
+		}
+	}
+	mysteelChemicalDatas, err := tranformData(resp)
+	if err != nil {
+		errMsg = "转换数据失败"
+		return
+	}
+
+	var indexErr error
+	var lErr error
+	defer func() {
+		if indexErr != nil {
+			tips := fmt.Sprintf("钢联数据刷新-ETA指标刷新异常, 指标编码: %s, err: %s", indexObj.IndexCode, indexErr.Error())
+			alarm_msg.SendAlarmMsg(tips, 3)
+		}
+
+		if lErr != nil {
+			tips := fmt.Sprintf("钢联数据刷新-ETA指标刷新统计异常, 指标编码: %s, err: %s", indexObj.IndexCode, lErr.Error())
+			alarm_msg.SendAlarmMsg(tips, 3)
+		}
+	}()
+	var hasUpdate bool
+	dataObj := new(models.BaseFromMysteelChemicalData)
+	for _, items := range mysteelChemicalDatas {
+		addItems := make([]*models.BaseFromMysteelChemicalData, 0)
+		for _, v := range items {
+			dateStr := v.DataTime.Format(utils.FormatDate)
+			if findData, ok := existDataMap[dateStr]; !ok {
+				index, ok := existIndexMap[v.IndexCode]
+				if !ok {
+					continue
+				}
+				v.BaseFromMysteelChemicalIndexId = index.BaseFromMysteelChemicalIndexId
+				addItems = append(addItems, v)
+			} else {
+				if findData != nil && findData.Value != v.Value {
+					dataObj.BaseFromMysteelChemicalDataId = findData.BaseFromMysteelChemicalDataId
+					dataObj.Value = v.Value
+					dataObj.ModifyTime = time.Now()
+
+					err = dataObj.Update([]string{"value", "modify_time"})
+					if err != nil {
+						errMsg = "更新数据失败"
+						return
+					}
+					hasUpdate = true
+				}
+			}
+		}
+		err = dataObj.AddV2(addItems)
+		if err != nil {
+			return
+		}
+		//修改最大最小日期
+		if len(items) <= 0 {
+			continue
+		}
+		mysteelIndexMaxItem, er := dataObj.GetMysteelIndexInfoMaxAndMinInfo(items[0].IndexCode)
+		if er == nil && mysteelIndexMaxItem != nil {
+			e := dataObj.ModifyMysteelIndexMaxAndMinInfo(items[0].IndexCode, mysteelIndexMaxItem)
+			if e != nil {
+				fmt.Println("ModifyMysteelIndexMaxAndMinInfo Err:" + e.Error())
+				utils.FileLog.Info("修改钢联化工的最大最小日期失败，Err:" + e.Error())
+			}
+		}
+
+		edbInfo, e := models.GetEdbInfoByEdbCode(utils.DATA_SOURCE_MYSTEEL_CHEMICAL, indexObj.IndexCode)
+		if e != nil && e.Error() != utils.ErrNoRow() {
+			indexErr = e
+			return
+		}
+
+		if edbInfo != nil {
+			dataUpdateResult := 2
+			dataUpdateFailedReason := "服务异常"
+			_, logErrMsg, logErr := logic.RefreshBaseEdbInfo(edbInfo, ``)
+			if logErr != nil {
+				lErr = AddEdbInfoUpdateLog(edbInfo.EdbInfoId, 2, logErrMsg+logErr.Error(), dataUpdateResult, dataUpdateFailedReason, 0, 0)
+				return
+			}
+
+			if hasUpdate {
+				dataUpdateResult = 1
+				dataUpdateFailedReason = ""
+			} else {
+				dataUpdateFailedReason = "未刷新到数据"
+			}
+
+			// 添加刷新成功日志
+			lErr = AddEdbInfoUpdateLog(edbInfo.EdbInfoId, 1, "", dataUpdateResult, dataUpdateFailedReason, 0, 0)
+			if lErr != nil {
+				return
+			}
+		}
+	}
+	return
+}
+
+func tranformData(dataResp *models.MySteelChemicalApiResp) (items [][]*models.BaseFromMysteelChemicalData, err error) {
+	for _, v := range dataResp.Data {
+		tmpNewDataMap := make(map[string]int64)
+		tmpDateDataMap := make(map[string]*models.BaseFromMysteelChemicalData)
+		tmpDataItems := make([]*models.BaseFromMysteelChemicalData, 0)
+		for _, vv := range v.DataList {
+			tmpData := new(models.BaseFromMysteelChemicalData)
+			tmpData.IndexCode = vv.IndexCode
+			// 如果存在多条数据，则取发布时间最新的数据
+			pub, ok := tmpNewDataMap[vv.DataDate]
+			if !ok {
+				tmpNewDataMap[vv.DataDate] = vv.PublishTime
+				tmpData.Value = strconv.FormatFloat(vv.DataValue, 'f', -1, 64)
+			} else {
+				if pub < vv.PublishTime {
+					tmpNewDataMap[vv.DataDate] = vv.PublishTime
+					tmpData = tmpDateDataMap[vv.DataDate]
+					tmpData.Value = strconv.FormatFloat(vv.DataValue, 'f', -1, 64)
+				}
+				continue
+			}
+			dataDate, er := time.Parse(utils.FormatDate, vv.DataDate)
+			if er != nil {
+				err = er
+				return
+			}
+			tmpData.DataTime = dataDate
+			tmpData.CreateTime = time.Now()
+			tmpData.ModifyTime = time.Now()
+			tmpDataItems = append(tmpDataItems, tmpData)
+			tmpDateDataMap[vv.DataDate] = tmpData
+		}
+		items = append(items, tmpDataItems)
+	}
+	return
+}
+
 func handleIndex(indexItem *models.HandleMysteelIndex) (err error) {
 	defer func() {
 		if err != nil {
@@ -271,3 +488,214 @@ func handleIndex(indexItem *models.HandleMysteelIndex) (err error) {
 
 	return
 }
+
+type MySteelChemicalApiDataBody struct {
+	IndexCodes []string `json:"indexCodes"`
+	StartTime  string   `json:"startTime"`
+	EndTime    string   `json:"endTime"`
+	Order      string   `json:"order"`
+}
+
+type MySteelChemicalApiInfoBody struct {
+	PageNum     int  `json:"pageNum"`
+	PageSize    int  `json:"pageSize"`
+	IncludeInfo bool `json:"includeInfo"`
+}
+
+// GetEdbDataFromMySteelChemical 批量获得钢联化工的指标数据
+func GetEdbDataFromMySteelChemical(indexCodes []string, startTime, endTime, order string) (item *models.MySteelChemicalApiResp, err error) {
+	if utils.MysteelChemicalApiToken == "" {
+		err = errors.New("钢联接口token未配置")
+		return
+	}
+	m := new(MySteelChemicalApiDataBody)
+	m.IndexCodes = indexCodes
+	m.StartTime = startTime
+	m.EndTime = endTime
+	m.Order = order
+	postData, er := json.Marshal(m)
+	if er != nil {
+		err = er
+		return
+	}
+	postUrl := `https://mds.mysteel.com/dynamic/order/api/fcAbRA`
+	body, err := MySteelChemicalPost(postUrl, "data", postData)
+	if err != nil {
+		return
+	}
+	err = json.Unmarshal(body, &item)
+	if err != nil {
+		return
+	}
+	return
+}
+
+// GetMySteelChemicalIndexNameMap 获取钢联化工的所有指标的信息
+func GetMySteelChemicalIndexNameMap() (indexNameMap map[string]*models.MySteelChemicalApiInfoItem, err error) {
+	if utils.MysteelChemicalApiToken == "" {
+		err = errors.New("钢联接口token未配置")
+		return
+	}
+	item, err := getPageIndexInfoMap(1, 200)
+	if err != nil {
+		return
+	}
+	indexNameMap = make(map[string]*models.MySteelChemicalApiInfoItem)
+	for _, v := range item.Data.List {
+		indexNameMap[v.IndexCode] = v
+	}
+	// 如果总条数大于200，则继续获取
+	if item.Data.Total > 200 || item.Data.Pages > 1 {
+		for i := 2; i <= item.Data.Pages; i++ {
+			item, err = getPageIndexInfoMap(i, 200)
+			if err != nil {
+				return
+			}
+			for _, v := range item.Data.List {
+				indexNameMap[v.IndexCode] = v
+			}
+		}
+		return
+	}
+	return
+}
+
+func getPageIndexInfoMap(pageNum, pageSize int) (item *models.MySteelChemicalApiInfoResp, err error) {
+	m := new(MySteelChemicalApiInfoBody)
+	m.PageNum = pageNum
+	m.PageSize = pageSize
+	m.IncludeInfo = true
+	postData, er := json.Marshal(m)
+	if er != nil {
+		err = er
+		return
+	}
+	postUrl := `https://mds.mysteel.com/dynamic/order/api/fcAbRA`
+	body, er := MySteelChemicalPost(postUrl, "info", postData)
+	if er != nil {
+		err = er
+		return
+	}
+	err = json.Unmarshal(body, &item)
+	if err != nil {
+		return
+	}
+	if !item.Success {
+		err = errors.New(item.Message)
+		utils.FileLog.Info("code:" + item.Code + " message:" + item.Message)
+		return
+	}
+	return
+}
+
+func MySteelChemicalPost(postUrl, hType string, postData []byte) (body []byte, err error) {
+	req, er := http.NewRequest(`POST`, postUrl, strings.NewReader(string(postData)))
+	if er != nil {
+		err = er
+		return
+	}
+	req.Header.Set(`Content-Type`, `application/json`)
+	req.Header.Set(`accessTokenSign`, utils.MysteelChemicalApiToken)
+	req.Header.Set(`infoOrData`, hType)
+
+	client := &http.Client{}
+	resp, er := client.Do(req)
+	if er != nil {
+		err = er
+		return
+	}
+	defer resp.Body.Close()
+	body, err = io.ReadAll(resp.Body)
+	if err != nil {
+		return
+	}
+	return
+}
+
+func RefreshDataFromMysteelChemical(edbCode, startDate, endDate string) (err error) {
+	indexObj := &models.BaseFromMysteelChemicalIndex{}
+	tmpIndex, err := indexObj.GetIndexItem(edbCode)
+	if err != nil {
+		return
+	}
+
+	terminal, err := GetTerminal(utils.DATA_SOURCE_MYSTEEL_CHEMICAL, tmpIndex.TerminalCode)
+	if err != nil {
+		err = fmt.Errorf("获取钢联化工接口配置出错 Err: %s", err)
+		return
+	}
+
+	if tmpIndex.TerminalCode == "" {
+		// 设置指标与终端关系的缓存
+		terminalCodeCacheKey := utils.CACHE_EDB_TERMINAL_CODE_URL + edbCode
+		_ = utils.Rc.Put(terminalCodeCacheKey, terminal.TerminalCode, utils.GetTodayLastSecond())
+	}
+
+	// 如果配置了api的token, 那么就走api接口
+	if utils.MysteelChemicalApiToken != "" {
+		resp, er := GetEdbDataFromMySteelChemical([]string{edbCode}, startDate, endDate, "desc")
+		if er != nil {
+			err = er
+			return
+		}
+		if !resp.Success {
+			err = errors.New(resp.Message)
+			return
+		}
+
+		dataObj := new(models.BaseFromMysteelChemicalData)
+		exitDataList, er := dataObj.GetIndexDataList(edbCode)
+		if er != nil {
+			err = er
+			return
+		}
+		existDataMap := make(map[string]*models.BaseFromMysteelChemicalData)
+		for _, v := range exitDataList {
+			dateStr := v.DataTime.Format(utils.FormatDate)
+			existDataMap[dateStr] = v
+		}
+		mysteelChemicalDatas, er := tranformData(resp)
+		if er != nil {
+			err = er
+			return
+		}
+
+		addItems := make([]*models.BaseFromMysteelChemicalData, 0)
+		indexObj := &models.BaseFromMysteelChemicalIndex{}
+		existIndex, er := indexObj.GetIndexItem(edbCode)
+		if er != nil {
+			err = er
+			return
+		}
+		if len(mysteelChemicalDatas) == 0 {
+			err = errors.New("没有获取到数据")
+			return
+		}
+		// 因为只有一个指标，所以取第一个就可以了
+		items := mysteelChemicalDatas[0]
+		for _, v := range items {
+			dateStr := v.DataTime.Format(utils.FormatDate)
+			if findData, ok := existDataMap[dateStr]; !ok {
+				v.BaseFromMysteelChemicalIndexId = existIndex.BaseFromMysteelChemicalIndexId
+				addItems = append(addItems, v)
+			} else {
+				if findData != nil && findData.Value != v.Value {
+					dataObj.BaseFromMysteelChemicalDataId = findData.BaseFromMysteelChemicalDataId
+					dataObj.Value = v.Value
+					dataObj.ModifyTime = time.Now()
+
+					err = dataObj.Update([]string{"value", "modify_time"})
+					if err != nil {
+						return
+					}
+				}
+			}
+		}
+		err = dataObj.AddV2(addItems)
+		if err != nil {
+			return
+		}
+		return
+	}
+	return
+}

services/base_from_ths_ds.go

@@ -10,7 +10,7 @@ import (
 	"strings"
 )
 
-func GetEdbDataFromThsDs(stockCode, edbCode, startDate, endDate, edbTerminalCode string) (item models.EdbDataFromThs, err error) {
+func GetEdbDataFromThsDs(stockCode, edbCode, startDate, endDate, edbTerminalCode, extraPars string) (item models.EdbDataFromThs, err error) {
 	terminal, err := GetTerminal(utils.DATA_SOURCE_THS, edbTerminalCode)
 	if err != nil {
 		err = fmt.Errorf("获取同花顺接口配置出错 Err: %s", err)
@@ -34,10 +34,13 @@ func GetEdbDataFromThsDs(stockCode, edbCode, startDate, endDate, edbTerminalCode
 		if err != nil {
 			return
 		}
-		return getEdbDataFromThsDsHttp(stockCode, edbCode, startDate, endDate, terminal.Value, token)
+
+		// TEST
+		//token = "23f339e97fac48d8b99024228fafb6f0128cfbb7.signs_NTc2NjQ4MTA5"
+		return getEdbDataFromThsDsHttp(stockCode, edbCode, startDate, endDate, terminal.Value, token, extraPars)
 	}
 
-	return getEdbDataFromThsDsApp(stockCode, edbCode, startDate, endDate, 0, terminal.ServerUrl)
+	return getEdbDataFromThsDsApp(stockCode, edbCode, startDate, endDate, 0, terminal.ServerUrl, extraPars)
 }
 
 type EdbDataFromThsSdInterface struct {
@@ -94,13 +97,13 @@ type Type struct {
 //}
 
 // getEdbDataFromThsDs 获取同花顺接口数据
-func getEdbDataFromThsDsApp(stockCode, edbCode, startDate, endDate string, num int, serverUrl string) (item models.EdbDataFromThs, err error) {
+func getEdbDataFromThsDsApp(stockCode, edbCode, startDate, endDate string, num int, serverUrl, extraPars string) (item models.EdbDataFromThs, err error) {
 	if serverUrl == `` {
 		err = errors.New("同花顺接口未配置")
 		return
 	}
-	thsUrl := serverUrl + `edbInfo/ths/ds?StockCode=%s&EdbCode=%s&StartDate=%s&EndDate=%s`
-	thsUrl = fmt.Sprintf(thsUrl, stockCode, edbCode, startDate, endDate)
+	thsUrl := serverUrl + `edbInfo/ths/ds?StockCode=%s&EdbCode=%s&StartDate=%s&EndDate=%s&ExtraPars=%s`
+	thsUrl = fmt.Sprintf(thsUrl, stockCode, edbCode, startDate, endDate, extraPars)
 	utils.FileLog.Info("thsUrl:" + thsUrl)
 	body, err := http.Get(thsUrl)
 	utils.FileLog.Info("ths result:" + string(body))
@@ -139,7 +142,7 @@ func getEdbDataFromThsDsApp(stockCode, edbCode, startDate, endDate string, num i
 				//session has expired,please re-login after using the system
 				//如果是同花顺登录session失效了，那么就重新请求获取数据
 				if response.ErrorCode == -1020 && num == 0 {
-					return getEdbDataFromThsDsApp(stockCode, edbCode, startDate, endDate, 1, serverUrl)
+					return getEdbDataFromThsDsApp(stockCode, edbCode, startDate, endDate, 1, serverUrl, extraPars)
 				}
 				err = errors.New(string(body))
 				return
@@ -188,7 +191,7 @@ func getEdbDataFromThsDsApp(stockCode, edbCode, startDate, endDate string, num i
 			//session has expired,please re-login after using the system
 			//如果是同花顺登录session失效了，那么就重新请求获取数据
 			if response.ErrorCode == -1020 && num == 0 {
-				return getEdbDataFromThsDsApp(stockCode, edbCode, startDate, endDate, 1, serverUrl)
+				return getEdbDataFromThsDsApp(stockCode, edbCode, startDate, endDate, 1, serverUrl, extraPars)
 			}
 			err = errors.New(string(body))
 			return

services/base_from_ths_ds_http.go

@@ -10,7 +10,7 @@ import (
 )
 
 // getEdbDataFromThsDsHttp 通过url获取同花顺的日期序列数据
-func getEdbDataFromThsDsHttp(stockCode, edbCode, startDate, endDate, thsRefreshToken, token string) (item models.EdbDataFromThs, err error) {
+func getEdbDataFromThsDsHttp(stockCode, edbCode, startDate, endDate, thsRefreshToken, token, extraPars string) (item models.EdbDataFromThs, err error) {
 	thsUrl := "https://quantapi.51ifind.com/api/v1/date_sequence"
 	//indicators 是 半角逗号分隔的所有指标，宏观指标过多，推荐使用Windows超级命令生成。 "indicators":"M001620326,M002822183"
 	//functionpara 否 key-value格式,省略时不进行更新时间筛选。两个时间控件更新起始时间（startrtime）和更新结束时间（endrtime），不勾选时省略见下方代码块
@@ -20,6 +20,12 @@ func getEdbDataFromThsDsHttp(stockCode, edbCode, startDate, endDate, thsRefreshT
 	// 分割字符串
 	paramList := strings.Split(edbCode, ",")
 
+	var extraArr []string
+	if extraPars != "" {
+		extraPars = strings.TrimSpace(extraPars)
+		extraArr = strings.Split(extraPars, ",")
+	}
+
 	// 创建一个包含映射的切片
 	var indipara []map[string]interface{}
 
@@ -29,6 +35,9 @@ func getEdbDataFromThsDsHttp(stockCode, edbCode, startDate, endDate, thsRefreshT
 		paramMap := map[string]interface{}{
 			"indicator": param,
 		}
+		if len(extraArr) > 0 {
+			paramMap["indiparams"] = extraArr
+		}
 
 		// 将映射添加到切片中
 		indipara = append(indipara, paramMap)

services/base_from_ths_hf.go

@@ -0,0 +1,648 @@
+package services
+
+import (
+	"encoding/json"
+	"eta/eta_index_lib/logic"
+	"eta/eta_index_lib/models"
+	"eta/eta_index_lib/models/mgo"
+	"eta/eta_index_lib/services/alarm_msg"
+	"eta/eta_index_lib/utils"
+	"fmt"
+	"github.com/rdlucklib/rdluck_tools/http"
+	"github.com/shopspring/decimal"
+	"go.mongodb.org/mongo-driver/bson"
+	"net/url"
+	"strings"
+	"time"
+)
+
+const (
+	ThsHfApiUrl = "https://quantapi.51ifind.com/api/v1/high_frequency"
+)
+
+// GetEdbDataFromThsHf 获取高频数据
+func GetEdbDataFromThsHf(thsParams models.ThsHfSearchEdbReq, terminalCode string) (indexes []*models.ThsHfIndexWithData, err error) {
+	terminal, e := GetTerminal(utils.DATA_SOURCE_THS, terminalCode)
+	if e != nil {
+		err = fmt.Errorf("获取同花顺终端配置失败, %v", e)
+		return
+	}
+	if thsParams.EndTime == "" {
+		thsParams.EndTime = time.Now().Local().Format(utils.FormatDateTime)
+	}
+
+	// 走API
+	if utils.ThsDataMethod == "" || utils.ThsDataMethod == "api" {
+		var token string
+		token, e = GetAccessToken(false, terminal.Value)
+		if e != nil {
+			err = fmt.Errorf("获取同花顺API-AccessToken失败, %v", e)
+			return
+		}
+
+		// TEST
+		//token = "9eba1634116ea2aed9a5b12b6e12b0b5fcbe0847.signs_NTc2NjQ4MTA5"
+		return getEdbDataFromThsHfHttp(thsParams, terminal.Value, token)
+	}
+
+	// 走公用机
+	if terminal.ServerUrl == "" {
+		err = fmt.Errorf("同花顺终端地址未配置")
+		return
+	}
+	return getEdbDataFromThsHfApp(thsParams, 0, terminal.ServerUrl)
+}
+
+// getEdbDataFromThsHfHttp API-获取高频指标数据
+func getEdbDataFromThsHfHttp(thsParams models.ThsHfSearchEdbReq, refreshToken, accessToken string) (indexes []*models.ThsHfIndexWithData, err error) {
+	defer func() {
+		if err != nil {
+			tips := fmt.Sprintf("同花顺高频指标API-getEdbDataFromThsHfHttp err: %v", err)
+			utils.FileLog.Info(tips)
+		}
+	}()
+
+	// 请求参数参考
+	//dataMap := map[string]interface{}{
+	//	"codes":      "CU2407.SHF,CU2408.SHF",
+	//	"indicators": "pct_chg",
+	//	"starttime":  "2024-06-06 09:15:00",
+	//	"endtime":    "2024-06-11 15:15:00",
+	//	"functionpara": map[string]interface {
+	//	}{
+	//		"Limitstart": "10:00:00",
+	//		"Limitend":   "14:15:00",
+	//		"Interval":   60,
+	//		"Fill":       "Previous",
+	//		"CPS":        "forward4",
+	//		"Timeformat": "LocalTime",
+	//		"BaseDate":   "2024-01-01",
+	//	},
+	//}
+
+	// 额外参数
+	funcParams := map[string]interface{}{}
+	funcParams["Interval"] = thsParams.Interval
+	if thsParams.Fill != "" {
+		funcParams["Fill"] = thsParams.Fill
+	}
+	if thsParams.CPS != "" {
+		funcParams["CPS"] = thsParams.CPS
+	}
+	if thsParams.BaseDate != "" {
+		funcParams["BaseDate"] = thsParams.BaseDate
+	}
+
+	// TEST
+	//funcParams["Limitstart"] = "10:00:00"
+	//funcParams["Limitend"] = "14:15:00"
+	//funcParams["Fill"] = "Previous"
+	//funcParams["CPS"] = "forward4"
+	//funcParams["Timeformat"] = "LocalTime"
+	//funcParams["BaseDate"] = "2024-01-01"
+
+	dataMap := map[string]interface{}{
+		"codes":        thsParams.StockCode,
+		"indicators":   thsParams.EdbCode,
+		"starttime":    thsParams.StartTime,
+		"endtime":      thsParams.EndTime,
+		"functionpara": funcParams,
+	}
+
+	// 请求接口
+	body, e, _ := postCurl(ThsHfApiUrl, dataMap, 0, refreshToken, accessToken)
+	if e != nil {
+		utils.FileLog.Info(string(body))
+		err = fmt.Errorf("同花顺API-请求失败, %v", e)
+		return
+	}
+
+	apiResp := new(models.ThsHfApiResp)
+	if e = json.Unmarshal(body, &apiResp); e != nil {
+		err = fmt.Errorf("同花顺API-解析响应失败, %v", e)
+		return
+	}
+	if apiResp.ErrorCode != 0 {
+		err = fmt.Errorf("同花顺高频API-状态码: %d, 提示信息: %s", apiResp.ErrorCode, apiResp.ErrMsg)
+		return
+	}
+	indexes = make([]*models.ThsHfIndexWithData, 0)
+	if len(apiResp.Tables) == 0 {
+		utils.FileLog.Info("同花顺高频API-无数据")
+		return
+	}
+
+	// 结果示例
+	// {
+	//	"errorcode": 0,
+	//	"errmsg": "Success!",
+	//	"tables": [{
+	//		"thscode": "CU2407.SHF",
+	//		"time": ["2024-07-01 10:00", "2024-07-01 11:15", "2024-07-01 14:15", "2024-07-01 15:00"],
+	//		"table": {
+	//			"open": [77930.000000, 77980.000000, 77910.000000, 77850.000000],
+	//			"close": [77980.000000, 77920.000000, 77850.000000, 77780.000000]
+	//		}
+	//	}, {
+	//		"thscode": "CU2408.SHF",
+	//		"time": ["2024-07-01 10:00", "2024-07-01 11:15", "2024-07-01 14:15", "2024-07-01 15:00"],
+	//		"table": {
+	//			"open": [78180.000000, 78280.000000, 78220.000000, 78110.000000],
+	//			"close": [78280.000000, 78220.000000, 78110.000000, 78060.000000]
+	//		}
+	//	}]
+	// }
+
+	// Tables中的每一个对应一个证券代码
+	for _, v := range apiResp.Tables {
+		if len(v.Time) == 0 || len(v.Table) == 0 {
+			continue
+		}
+		// Table中的K-V对应指标代码-数据值序列
+		for tk, tv := range v.Table {
+			index := new(models.ThsHfIndexWithData)
+			index.StockCode = v.ThsCode
+			index.EdbCode = tk
+			td := make([]*models.ThsHfIndexData, 0)
+			tvl := len(tv)
+			for k, t := range v.Time {
+				if k >= tvl {
+					continue
+				}
+				dt, e := time.ParseInLocation("2006-01-02 15:04", t, time.Local)
+				if e != nil {
+					utils.FileLog.Info(fmt.Sprintf("同花顺API-time parse t: %s, err: %v", t, e))
+					continue
+				}
+				td = append(td, &models.ThsHfIndexData{
+					DataTime: dt,
+					Value:    tv[k],
+				})
+			}
+			index.IndexData = td
+			indexes = append(indexes, index)
+		}
+	}
+	return
+}
+
+// getEdbDataFromThsHfApp 公用机-获取高频指标数据
+func getEdbDataFromThsHfApp(thsParams models.ThsHfSearchEdbReq, num int, serverUrl string) (indexes []*models.ThsHfIndexWithData, err error) {
+	var requestUrl string
+	defer func() {
+		if err != nil {
+			utils.FileLog.Info(fmt.Sprintf("requestUrl: %s", requestUrl))
+			utils.FileLog.Info(fmt.Sprintf("getEdbDataFromThsHfApp: %v", err))
+		}
+	}()
+	//serverUrl = "http://wxmsgsen1.hzinsights.com:8040/"
+	baseUrl := fmt.Sprintf("%s%s", serverUrl, "edbInfo/ths/hf?")
+
+	// 额外参数
+	var funcParam string
+	if thsParams.Interval > 0 {
+		funcParam += fmt.Sprintf("Interval:%d,", thsParams.Interval)
+	}
+	if thsParams.Fill != "" {
+		funcParam += fmt.Sprintf("Fill:%s,", thsParams.Fill)
+	}
+	if thsParams.CPS != "" {
+		funcParam += fmt.Sprintf("CPS:%s,", thsParams.CPS)
+	}
+	if thsParams.BaseDate != "" {
+		funcParam += fmt.Sprintf("BaseDate:%s,", thsParams.BaseDate)
+	}
+	funcParam = strings.TrimRight(funcParam, ",")
+
+	params := url.Values{}
+	params.Add("codes", thsParams.StockCode)
+	params.Add("indicators", thsParams.EdbCode)
+	params.Add("function_para", funcParam)
+	params.Add("start_time", thsParams.StartTime)
+	params.Add("end_time", thsParams.EndTime)
+
+	// 请求终端
+	requestUrl = baseUrl + params.Encode()
+	body, e := http.Get(requestUrl)
+	if e != nil {
+		err = fmt.Errorf("")
+		return
+	}
+	dataBody := strings.TrimLeft(string(body), `"`)
+	dataBody = strings.TrimRight(dataBody, `"`)
+	dataBody = strings.ReplaceAll(dataBody, `\`, ``)
+	//utils.FileLog.Info(dataBody)
+
+	appResp := new(TerminalResponse)
+	if e = json.Unmarshal([]byte(dataBody), &appResp); e != nil {
+		err = fmt.Errorf("同花顺APP-解析响应失败, %v", e)
+		return
+	}
+	if appResp.ErrorCode != 0 {
+		//如果是同花顺登录session失效了，那么就重新请求获取数据
+		if appResp.ErrorCode == -1020 && num == 0 {
+			return getEdbDataFromThsHfApp(thsParams, 1, serverUrl)
+		}
+		err = fmt.Errorf("同花顺APP-状态码: %d, 提示信息: %s", appResp.ErrorCode, appResp.ErrMsg)
+		return
+	}
+
+	// 响应结果示例
+	// {
+	//	"errorcode": 0,
+	//	"errmsg": "Success!",
+	//	"data": [{
+	//		"time": "2024-06-04 09:30",
+	//		"thscode": "CU2406.SHF",
+	//		"open": 81900.0,
+	//		"close": 81820.0
+	//	}, {
+	//		"time": "2024-06-04 10:00",
+	//		"thscode": "CU2406.SHF",
+	//		"open": 81820.0,
+	//		"close": 81790.0
+	//	}, {
+	//		"time": "2024-06-04 10:45",
+	//		"thscode": "CU2406.SHF",
+	//		"open": 81820.0,
+	//		"close": 81950.0
+	//	}]
+	// }
+
+	indexes = make([]*models.ThsHfIndexWithData, 0)
+	indexMap := make(map[string]*models.ThsHfIndexWithData)
+	for _, stockData := range appResp.Data {
+		strTime := stockData["time"].(string)
+		dataTime, e := time.ParseInLocation("2006-01-02 15:04", strTime, time.Local)
+		if e != nil {
+			utils.FileLog.Info("数据日期格式有误, time: %s, %v", strTime, e)
+			continue
+		}
+		stockCode := stockData["thscode"].(string)
+
+		// 指标代码+数据
+		for k, v := range stockData {
+			if k == "time" || k == "thscode" {
+				continue
+			}
+			if v == nil {
+				continue
+			}
+			val, ok := v.(float64)
+			if !ok {
+				continue
+			}
+			mk := fmt.Sprintf("%s-%s", stockCode, k)
+			if indexMap[mk] == nil {
+				indexMap[mk] = new(models.ThsHfIndexWithData)
+				indexMap[mk].StockCode = stockCode
+				indexMap[mk].EdbCode = k
+				indexMap[mk].IndexData = make([]*models.ThsHfIndexData, 0)
+			}
+			indexMap[mk].IndexData = append(indexMap[mk].IndexData, &models.ThsHfIndexData{
+				DataTime: dataTime,
+				Value:    val,
+			})
+		}
+	}
+	for _, v := range indexMap {
+		indexes = append(indexes, v)
+	}
+	return
+}
+
+// RefreshThsHfBaseIndex 源指标刷新
+func RefreshThsHfBaseIndex(indexItem *models.BaseFromThsHfIndex, codeWithData *models.ThsHfIndexWithData, startTime string) (err error) {
+	defer func() {
+		if err != nil {
+			tips := fmt.Sprintf("RefreshThsHfBaseIndex-更新失败, %v", err)
+			utils.FileLog.Info(tips)
+			go alarm_msg.SendAlarmMsg(tips, 3)
+		}
+	}()
+	if indexItem == nil {
+		err = fmt.Errorf("指标不存在")
+		return
+	}
+	if len(codeWithData.IndexData) == 0 {
+		return
+	}
+
+	// 获取源指标数据
+	dataOb := new(models.BaseFromThsHfData)
+	originData := make([]*models.BaseFromThsHfData, 0)
+	{
+		cond := fmt.Sprintf(" AND %s = ?", dataOb.Cols().IndexCode)
+		pars := make([]interface{}, 0)
+		pars = append(pars, indexItem.IndexCode)
+		if startTime != "" {
+			cond += fmt.Sprintf(" AND %s >= ?", dataOb.Cols().DataTime)
+			pars = append(pars, startTime)
+		}
+		list, e := dataOb.GetItemsByCondition(cond, pars, []string{}, "")
+		if e != nil {
+			err = fmt.Errorf("获取源指标数据失败, %v", e)
+			return
+		}
+		originData = list
+	}
+
+	// 更新指标数据
+	dateExist := make(map[string]*models.BaseFromThsHfData)
+	newValExist := make(map[string]bool)
+	if len(originData) > 0 {
+		// unicode去重
+		for _, d := range originData {
+			dateExist[d.UniqueCode] = d
+		}
+	}
+
+	// 筛选新增/更新数据
+	updateData := make([]*models.BaseFromThsHfData, 0)
+	insertData := make([]*models.BaseFromThsHfData, 0)
+	for _, d := range codeWithData.IndexData {
+		uni := utils.MD5(fmt.Sprint(indexItem.IndexCode, d.DataTime.Format(utils.FormatDateTimeMinute)))
+		origin := dateExist[uni]
+
+		// unicode检验是否存在
+		strNewVal := decimal.NewFromFloat(d.Value).Round(4).String()
+		di, _ := decimal.NewFromString(strNewVal)
+		newVal, _ := di.Float64()
+		if origin != nil {
+			strExistVal := decimal.NewFromFloat(origin.Value).Round(4).String()
+			if strNewVal == strExistVal {
+				continue
+			}
+			origin.Value = newVal
+			origin.ModifyTime = time.Now().Local()
+			updateData = append(updateData, origin)
+			continue
+		}
+
+		// 新增的数据去重
+		if newValExist[uni] {
+			continue
+		}
+		newValExist[uni] = true
+		newData := new(models.BaseFromThsHfData)
+		newData.BaseFromThsHfIndexId = indexItem.BaseFromThsHfIndexId
+		newData.IndexCode = indexItem.IndexCode
+		newData.DataTime = d.DataTime
+		newData.Value = newVal
+		newData.CreateTime = time.Now()
+		newData.ModifyTime = time.Now()
+		newData.UniqueCode = uni
+		newData.DataTimestamp = d.DataTime.UnixNano() / 1e6
+		insertData = append(insertData, newData)
+	}
+	if e := dataOb.MultiInsertOrUpdate(insertData, updateData); e != nil {
+		err = fmt.Errorf("新增/更新源指标数据失败, %v", e)
+		return
+	}
+
+	// 更新指标开始结束时间
+	minMax, e := dataOb.GetIndexMinMax(indexItem.IndexCode)
+	if e == nil && minMax != nil {
+		minDate, e := time.ParseInLocation(utils.FormatDateTime, minMax.MinDate, time.Local)
+		if e != nil {
+			err = fmt.Errorf("源数据最小日期有误, MinDate: %s, %v", minMax.MinDate, e)
+			return
+		}
+		maxDate, e := time.ParseInLocation(utils.FormatDateTime, minMax.MaxDate, time.Local)
+		if e != nil {
+			err = fmt.Errorf("源数据最大日期有误, MaxDate: %s, %v", minMax.MaxDate, e)
+			return
+		}
+		indexItem.StartDate = minDate
+		indexItem.EndDate = maxDate
+		indexItem.ModifyTime = time.Now().Local()
+		updateCols := []string{indexItem.Cols().StartDate, indexItem.Cols().EndDate, indexItem.Cols().ModifyTime}
+		if e = indexItem.Update(updateCols); e != nil {
+			err = fmt.Errorf("更新源指标开始结束时间失败, %v", e)
+			return
+		}
+	}
+
+	// 同步刷新指标库
+	go func() {
+		_ = RefreshEdbFromThsHfBaseIndex(indexItem.IndexCode, startTime)
+	}()
+	return
+}
+
+// RefreshThsHfBaseIndexMgo 源指标刷新-Mongo
+func RefreshThsHfBaseIndexMgo(indexItem *models.BaseFromThsHfIndex, codeWithData *models.ThsHfIndexWithData, startTime string) (err error) {
+	defer func() {
+		if err != nil {
+			tips := fmt.Sprintf("RefreshThsHfBaseIndexMgo-更新失败, %v", err)
+			utils.FileLog.Info(tips)
+			go alarm_msg.SendAlarmMsg(tips, 3)
+		}
+	}()
+	if indexItem == nil {
+		err = fmt.Errorf("指标不存在")
+		return
+	}
+	if len(codeWithData.IndexData) == 0 {
+		return
+	}
+	mogDataObj := new(mgo.BaseFromThsHfData)
+
+	// 获取已存在的所有数据
+	existCond := bson.M{
+		"index_code": indexItem.IndexCode,
+	}
+	if startTime != "" {
+		st, e := time.ParseInLocation(utils.FormatDateTime, startTime, time.Local)
+		if e != nil {
+			err = fmt.Errorf("start time parse err: %v", e)
+			return
+		}
+		existCond["data_time"] = bson.M{
+			"$gte": st,
+		}
+	}
+	exitDataList, e := mogDataObj.GetAllDataList(existCond, []string{"data_time"})
+	if e != nil {
+		err = fmt.Errorf("GetAllDataList err: %v", e)
+		return
+	}
+
+	// 已经存在的数据集
+	exitDataMap := make(map[string]*mgo.BaseFromThsHfData)
+	for _, v := range exitDataList {
+		exitDataMap[v.UniqueCode] = v
+	}
+
+	// 待添加的数据集
+	addDataList := make([]interface{}, 0)
+	updateDataList := make([]mgo.BaseFromThsHfData, 0)
+	for _, data := range codeWithData.IndexData {
+		strNewVal := decimal.NewFromFloat(data.Value).Round(4).String()
+		di, _ := decimal.NewFromString(strNewVal)
+		newVal, _ := di.Float64()
+
+		// unicode检验是否存在
+		uni := utils.MD5(fmt.Sprint(indexItem.IndexCode, data.DataTime.Format(utils.FormatDateTimeMinute)))
+		findData, ok := exitDataMap[uni]
+		if !ok {
+			addDataList = append(addDataList, mgo.BaseFromThsHfData{
+				BaseFromThsHfIndexId: int64(indexItem.BaseFromThsHfIndexId),
+				IndexCode:            indexItem.IndexCode,
+				DataTime:             data.DataTime,
+				Value:                newVal,
+				UniqueCode:           uni,
+				CreateTime:           time.Now(),
+				ModifyTime:           time.Now(),
+				DataTimestamp:        data.DataTime.UnixNano() / 1e6,
+			})
+			continue
+		}
+
+		// 值不匹配，修改数据
+		strExistVal := decimal.NewFromFloat(findData.Value).Round(4).String()
+		if strNewVal == strExistVal {
+			continue
+		}
+		findData.Value = newVal
+		updateDataList = append(updateDataList, *findData)
+	}
+
+	// 入库
+	{
+		coll := mogDataObj.GetCollection()
+		if len(addDataList) > 0 {
+			if e = mogDataObj.BatchInsertDataByColl(coll, 500, addDataList); e != nil {
+				err = fmt.Errorf("BatchInsertDataByColl, err: %v", e)
+				return
+			}
+		}
+
+		if len(updateDataList) > 0 {
+			for _, v := range updateDataList {
+				if e = mogDataObj.UpdateDataByColl(coll, bson.M{"_id": v.ID}, bson.M{"$set": bson.M{"value": v.Value, "modify_time": v.ModifyTime}}); e != nil {
+					err = fmt.Errorf("UpdateDataByColl, err: %v", e)
+					return
+				}
+			}
+		}
+	}
+
+	// 修改最大最小日期
+	minMax, err := indexItem.GetEdbInfoMaxAndMinInfo(indexItem.IndexCode)
+	if err != nil {
+		return
+	}
+	if err == nil && minMax != nil {
+		minDate, e := time.ParseInLocation(utils.FormatDateTime, minMax.MinDate, time.Local)
+		if e != nil {
+			err = fmt.Errorf("源数据最小日期有误, MinDate: %s, %v", minMax.MinDate, e)
+			return
+		}
+		maxDate, e := time.ParseInLocation(utils.FormatDateTime, minMax.MaxDate, time.Local)
+		if e != nil {
+			err = fmt.Errorf("源数据最大日期有误, MaxDate: %s, %v", minMax.MaxDate, e)
+			return
+		}
+		indexItem.StartDate = minDate
+		indexItem.EndDate = maxDate
+		indexItem.ModifyTime = time.Now().Local()
+		updateCols := []string{indexItem.Cols().StartDate, indexItem.Cols().EndDate, indexItem.Cols().ModifyTime}
+		if e = indexItem.Update(updateCols); e != nil {
+			err = fmt.Errorf("更新源指标开始结束时间失败, %v", e)
+			return
+		}
+	}
+
+	// 同步刷新指标库
+	go func() {
+		_ = RefreshEdbFromThsHfBaseIndex(indexItem.IndexCode, startTime)
+	}()
+	return
+}
+
+// RefreshEdbFromThsHfBaseIndex 根据源指标刷新指标库
+func RefreshEdbFromThsHfBaseIndex(baseCode, startTime string) (err error) {
+	defer func() {
+		if err != nil {
+			tips := fmt.Sprintf("RefreshEdbFromThsHfBaseIndex-刷新指标库失败, %v", err)
+			utils.FileLog.Info(tips)
+			go alarm_msg.SendAlarmMsg(tips, 3)
+		}
+	}()
+
+	// 获取指标关联信息
+	mappings := make([]*models.BaseFromEdbMapping, 0)
+	{
+		ob := new(models.BaseFromEdbMapping)
+		cond := fmt.Sprintf(" AND %s = ?", ob.Cols().BaseIndexCode)
+		pars := make([]interface{}, 0)
+		pars = append(pars, baseCode)
+		list, e := ob.GetItemsByCondition(cond, pars, []string{}, "")
+		if e != nil {
+			err = fmt.Errorf("获取源指标关联失败, %v", e)
+			return
+		}
+		mappings = list
+	}
+	if len(mappings) == 0 {
+		return
+	}
+	codeMapping := make(map[string]*models.BaseFromEdbMapping)
+	edbInfoIds := make([]int, 0)
+	for _, v := range mappings {
+		if codeMapping[v.EdbCode] == nil {
+			codeMapping[v.EdbCode] = v
+		}
+		edbInfoIds = append(edbInfoIds, v.EdbInfoId)
+	}
+
+	// 指标信息
+	edbInfoList, e := models.GetEdbInfoByIdList(edbInfoIds)
+	if e != nil {
+		err = fmt.Errorf("获取指标信息列表失败, %v", e)
+		return
+	}
+	codeEdb := make(map[string]*models.EdbInfo)
+	for _, v := range edbInfoList {
+		if codeEdb[v.EdbCode] == nil {
+			codeEdb[v.EdbCode] = v
+		}
+	}
+
+	thsOb := new(models.EdbThsHf)
+	source := thsOb.GetSource()
+	subSource := thsOb.GetSubSource()
+	for _, v := range mappings {
+		cacheKey := fmt.Sprintf("%s_%d_%d_%s", utils.CACHE_EDB_DATA_REFRESH, source, subSource, v.EdbCode)
+		if utils.Rc.IsExist(cacheKey) {
+			continue
+		}
+		utils.Rc.SetNX(cacheKey, 1, 1*time.Minute)
+
+		edb := codeEdb[v.EdbCode]
+		if edb == nil {
+			utils.FileLog.Info(fmt.Sprintf("RefreshEdbFromThsHfBaseIndex-指标信息有误, EdbCode: %s", v.EdbCode))
+			continue
+		}
+
+		// 刷新指标
+		if e = thsOb.Refresh(edb, codeMapping[v.EdbCode], startTime); e != nil {
+			utils.FileLog.Info(fmt.Sprintf("RefreshEdbFromThsHfBaseIndex-刷新指标失败, %v", e))
+			_ = utils.Rc.Delete(cacheKey)
+			continue
+		}
+
+		// 更新指标最值
+		if e = thsOb.UnifiedModifyEdbInfoMaxAndMinInfo(edb); e != nil {
+			utils.FileLog.Info(fmt.Sprintf("RefreshEdbFromThsHfBaseIndex-更新指标最值失败, %v", e))
+			_ = utils.Rc.Delete(cacheKey)
+			return
+		}
+		_ = utils.Rc.Delete(cacheKey)
+
+		// 更新ES
+		go logic.UpdateEs(edb.EdbInfoId)
+	}
+	return
+}

services/edb_info_relation.go

@@ -0,0 +1,381 @@
+package services
+
+import (
+	"eta/eta_index_lib/models"
+	"eta/eta_index_lib/services/alarm_msg"
+	"eta/eta_index_lib/utils"
+	"fmt"
+	"sort"
+	"strconv"
+	"time"
+)
+
+// 重置单个计算指标中的引用关系
+func ResetEdbRelation(edbInfoId int) {
+	var logMsg string
+	var replaceTotal int
+	var err error
+	defer func() {
+		if err != nil {
+			msg := fmt.Sprintf(" 重置单个计算指标中的引用关系失败 ResetEdbRelation  err: %v", err)
+			utils.FileLog.Info(msg)
+			fmt.Println(msg)
+			go alarm_msg.SendAlarmMsg(msg, 3)
+		}
+		if logMsg != `` {
+			utils.FileLog.Info(fmt.Sprintf("重置单个计算指标中的引用关系失败 重置总数%d,涉及到的引用id：%s", replaceTotal, logMsg))
+		}
+	}()
+	//查询与该计算指标相关的间接引用或者间接引用关系，如果记录不存在，则不处理
+	_, err = models.GetEdbInfoRelationByChildEdbInfoId(edbInfoId)
+	if err != nil {
+		if err.Error() == utils.ErrNoRow() {
+			err = nil
+			return
+		}
+		err = fmt.Errorf("查询与该计算指标相关的间接引用或者间接引用关系失败，错误信息：%s", err.Error())
+		return
+	}
+
+	//查询当前计算指标最新的引用指标列表
+	newMappingList, err := models.GetEdbInfoCalculateDetailList(edbInfoId)
+	if err != nil {
+		err = fmt.Errorf("查询当前计算指标最新的指标列表失败，错误信息：%s", err.Error())
+		return
+	}
+	//整理关联的来源指标ID
+	newEdbIdList := make([]string, 0)
+	newMappingListMap := make(map[int]*models.EdbInfoCalculateDetail)
+	for _, v := range newMappingList {
+		newEdbIdList = append(newEdbIdList, strconv.Itoa(v.FromEdbInfoId))
+		newMappingListMap[v.FromEdbInfoId] = v
+	}
+	//对指标ID进行排序
+	sort.Strings(newEdbIdList)
+	newEdbIdStr := ""
+	for _, v := range newEdbIdList {
+		newEdbIdStr += v + ","
+	}
+	//二者匹配一下，如果相同，则不处理，如果不同，先查询所有旧的间接引用记录，整理并分组，则删除旧的间接引用记录，新增新的间接引用记录，
+	tmpList, err := models.GetEdbInfoRelationListByChildEdbInfoId(edbInfoId)
+	if err != nil {
+		err = fmt.Errorf("查询当前计算指标的间接引用关系失败，错误信息：%s", err.Error())
+		return
+	}
+
+	parentRelationIds := make([]int, 0)
+	for _, v := range tmpList {
+		parentRelationIds = append(parentRelationIds, v.ParentRelationId)
+	}
+	if len(parentRelationIds) > 0 {
+		// 查询单个项目的引用列表作为判断依据
+		oldEdbIdList, err := models.GetEdbInfoRelationEdbIdsByParentRelationId(parentRelationIds[0], edbInfoId)
+		if err != nil {
+			err = fmt.Errorf("查询当前计算指标的间接引用关系失败，错误信息：%s", err.Error())
+			return
+		}
+		sort.Ints(oldEdbIdList)
+		oldEdbIdStr := ""
+		for _, v := range oldEdbIdList {
+			oldEdbIdStr += strconv.Itoa(v) + ","
+		}
+		// 把切片转成字符串
+		if newEdbIdStr == oldEdbIdStr {
+			return
+		}
+		list, e := models.GetEdbInfoRelationByRelationIds(parentRelationIds)
+		if e != nil {
+			err = fmt.Errorf("查询图表关联指标列表失败 Err:%s", e)
+			return
+		}
+		//查询直接引用指标关联关系
+		edbInfoListMap := make(map[int]struct{})
+		edbInfoIds := make([]int, 0)
+		for _, v := range list {
+			if _, ok := edbInfoListMap[v.EdbInfoId]; !ok {
+				edbInfoListMap[v.EdbInfoId] = struct{}{}
+				edbInfoIds = append(edbInfoIds, v.EdbInfoId)
+			}
+		}
+		edbInfoList := make([]*models.EdbInfo, 0)
+		if len(edbInfoIds) > 0 {
+			// 查询指标信息
+			edbInfoList, err = models.GetEdbInfoByIdList(edbInfoIds)
+			if err != nil {
+				err = fmt.Errorf("查询指标信息失败 Err:%s", err)
+				return
+			}
+		}
+		calculateEdbMappingListMap, calculateEdbMappingIdsMap, err := GetEdbListByEdbInfoId(edbInfoList)
+		if err != nil {
+			err = fmt.Errorf("查询指标关联指标列表失败 Err:%s", err)
+			return
+		}
+		//如何过滤掉只有间接引用，没有直接引用的
+		replaceTotal1, logMsg1, e := UpdateSecondEdbInRelation(list, calculateEdbMappingListMap, calculateEdbMappingIdsMap, edbInfoList)
+		if e != nil {
+			err = e
+			return
+		}
+		replaceTotal += replaceTotal1
+		logMsg += logMsg1
+	}
+	return
+}
+
+// 更新间接引用
+func UpdateSecondEdbInRelation(list []*models.EdbInfoRelation, calculateEdbMappingListMap map[int]*models.EdbInfoCalculateMapping, calculateEdbMappingIdsMap map[int][]int, edbInfoList []*models.EdbInfo) (replaceTotal int, logMsg string, err error) {
+	nowTime := time.Now()
+	edbInfoRelationIds := make([]int, 0)
+	indexCodeList := make([]string, 0)
+	addList := make([]*models.EdbInfoRelation, 0)
+	refreshIds := make([]int, 0)
+	edbInfoMap := make(map[int]*models.EdbInfo)
+	for _, v := range edbInfoList {
+		edbInfoMap[v.EdbInfoId] = v
+	}
+	// 查询所有的直接引用，删除所有的间接引用，添加所有直接引用的间接引用
+	for _, v := range list {
+		if v.RelationType == 0 {
+			edbInfoRelationIds = append(edbInfoRelationIds, v.EdbInfoRelationId)
+			edbInfo, ok := edbInfoMap[v.EdbInfoId]
+			if !ok {
+				err = fmt.Errorf("查询指标信息失败 EdbInfoId:%d", v.EdbInfoId)
+				return
+			}
+			if edbInfo.EdbType == 2 { //计算指标
+				childEdbMappingIds, ok := calculateEdbMappingIdsMap[edbInfo.EdbInfoId]
+				if !ok {
+					err = fmt.Errorf("查询%d指标关联指标列表为空", edbInfo.EdbInfoId)
+					return
+				}
+				for _, childEdbMappingId := range childEdbMappingIds {
+					childEdbMapping, ok2 := calculateEdbMappingListMap[childEdbMappingId]
+					if !ok2 {
+						continue
+					}
+
+					if childEdbMapping.FromSource == utils.DATA_SOURCE_MYSTEEL_CHEMICAL {
+						indexCodeList = append(indexCodeList, childEdbMapping.FromEdbCode)
+					}
+					tmp1 := &models.EdbInfoRelation{
+						ReferObjectId:      v.ReferObjectId,
+						ReferObjectType:    v.ReferObjectType,
+						ReferObjectSubType: v.ReferObjectSubType,
+						EdbInfoId:          childEdbMapping.FromEdbInfoId,
+						EdbName:            childEdbMapping.FromEdbName,
+						Source:             childEdbMapping.FromSource,
+						EdbCode:            childEdbMapping.FromEdbCode,
+						CreateTime:         nowTime,
+						ModifyTime:         nowTime,
+						RelationTime:       childEdbMapping.CreateTime,
+						RelationType:       1,
+						RootEdbInfoId:      edbInfo.EdbInfoId,
+						ChildEdbInfoId:     childEdbMapping.EdbInfoId,
+					}
+					tmp1.RelationCode = fmt.Sprintf("%d_%d_%d_%d", tmp1.RootEdbInfoId, tmp1.ReferObjectId, tmp1.ReferObjectType, tmp1.ReferObjectSubType)
+					addList = append(addList, tmp1)
+					refreshIds = append(refreshIds, childEdbMapping.FromEdbInfoId)
+				}
+			}
+		}
+	}
+
+	if len(edbInfoRelationIds) > 0 {
+		err = models.UpdateSecondRelationEdbInfoId(edbInfoRelationIds, addList, refreshIds, indexCodeList)
+		if err != nil {
+			logMsg = ""
+			err = fmt.Errorf("替换指标引用表中的指标ID失败 Err:%s", err)
+			return
+		}
+		replaceTotal = len(edbInfoRelationIds)
+	}
+	return
+}
+
+// 设置成禁用状态
+func DisableEdbInfoNoUpdate(edbInfo *models.EdbInfo) (err error) {
+	// 如果一个计算指标里，包涵的基础指标是停用状态，那么计算指标也要是停用状态停用状态
+	newBaseEdbInfoList := make([]int, 0)
+	hasFind := make(map[int]struct{})
+	newBaseEdbInfoIds, err := FindBaseEdbInfo(edbInfo.EdbInfoId, newBaseEdbInfoList, hasFind)
+	if err != nil {
+		err = fmt.Errorf("查找基础指标信息失败,err:%v", err)
+		return
+	}
+	// 查询是否存在停用指标，如果存在，则计算指标也要是停用状态
+	total, err := models.GetEdbInfoNoUpdateTotalByIdList(newBaseEdbInfoIds)
+	if err != nil {
+		err = fmt.Errorf("查询基础指标信息失败,err:%v", err)
+		return
+	}
+	if total > 0 {
+		edbInfo.NoUpdate = 1
+		edbInfo.ModifyTime = time.Now()
+		err = edbInfo.Update([]string{"NoUpdate", "ModifyTime"})
+		if err != nil {
+			err = fmt.Errorf("更新计算指标刷新状态失败,err:%v", err)
+			return
+		}
+	}
+	return
+}
+
+// 找到基础指标的过程
+func FindBaseEdbInfo(edbInfoId int, baseEdbInfoList []int, hasFind map[int]struct{}) (newBaseEdbInfoList []int, err error) {
+	newBaseEdbInfoList = baseEdbInfoList
+	if _, ok := hasFind[edbInfoId]; ok {
+		return
+	}
+	// 先找到所有的引用关系
+	//查询当前计算指标最新的引用指标列表
+	newMappingList, err := models.GetEdbInfoCalculateDetailList(edbInfoId)
+	if err != nil {
+		err = fmt.Errorf("查询当前计算指标最新的指标列表失败，错误信息：%s", err.Error())
+		return
+	}
+	hasFind[edbInfoId] = struct{}{}
+	for _, mapping := range newMappingList {
+		newBaseEdbInfoList = append(newBaseEdbInfoList, mapping.FromEdbInfoId)
+		if mapping.EdbType == 1 { // 如果是基础指标，则加入，否则继续找
+		} else {
+			newBaseEdbInfoList, err = FindBaseEdbInfo(mapping.FromEdbInfoId, newBaseEdbInfoList, hasFind)
+		}
+	}
+	return
+}
+
+// 查找当前计算指标的所有溯源指标
+func GetEdbListByEdbInfoId(edbInfoList []*models.EdbInfo) (edbMappingListMap map[int]*models.EdbInfoCalculateMapping, edbInfoMappingRootIdsMap map[int][]int, err error) {
+	if len(edbInfoList) == 0 {
+		return
+	}
+	edbInfoIds := make([]int, 0)
+	for _, v := range edbInfoList {
+		if v.EdbType == 2 && v.EdbInfoType == 0 { //普通计算指标，排除预算指标
+			edbInfoIds = append(edbInfoIds, v.EdbInfoId)
+		}
+	}
+	if len(edbInfoIds) == 0 {
+		return
+	}
+	//查询指标信息
+	allEdbMappingMap := make(map[int][]*models.EdbInfoCalculateMappingInfo, 0)
+	allMappingList, e := models.GetEdbInfoCalculateMappingListByEdbInfoIds(edbInfoIds)
+	if e != nil {
+		err = fmt.Errorf("GetEdbInfoCalculateMappingListByEdbInfoIds err: %s", e.Error())
+		return
+	}
+	for _, v := range allMappingList {
+		if _, ok := allEdbMappingMap[v.EdbInfoId]; !ok {
+			allEdbMappingMap[v.EdbInfoId] = make([]*models.EdbInfoCalculateMappingInfo, 0)
+		}
+		allEdbMappingMap[v.EdbInfoId] = append(allEdbMappingMap[v.EdbInfoId], v)
+	}
+	//查询指标映射
+	//查询所有指标数据
+	//查询这个指标相关的mapping信息放到数组里，
+	//将得到的指标ID信息放到数组里
+	hasFindMap := make(map[int]struct{})
+	edbInfoIdMap := make(map[int]struct{})
+	edbMappingList := make([]*models.EdbInfoCalculateMapping, 0)
+	edbInfoMappingRootIdsMap = make(map[int][]int, 0)
+	edbMappingMap := make(map[int]struct{})
+	for _, edbInfo := range edbInfoList {
+		if edbInfo.EdbType == 2 && edbInfo.EdbInfoType == 0 {
+			edbInfoId := edbInfo.EdbInfoId
+			edbMappingList, err = getCalculateEdbInfoByEdbInfoId(allEdbMappingMap, edbInfoId, hasFindMap, edbInfoIdMap, edbMappingList, edbMappingMap, edbInfoMappingRootIdsMap, edbInfoId)
+			if err != nil {
+				err = fmt.Errorf(" GetCalculateEdbInfoByEdbInfoId err: %s", err.Error())
+				return
+			}
+		}
+	}
+	if len(edbMappingList) == 0 {
+		return
+	}
+	// 查询指标信息
+	// 指标信息map
+	edbInfoIdList := make([]int, 0)
+	for k, _ := range edbInfoIdMap {
+		edbInfoIdList = append(edbInfoIdList, k)
+	}
+	edbMappingListMap = make(map[int]*models.EdbInfoCalculateMapping)
+
+	if len(edbMappingList) > 0 {
+		for _, v := range edbMappingList {
+			edbMappingListMap[v.EdbInfoCalculateMappingId] = v
+		}
+	}
+	return
+}
+
+// getCalculateEdbInfoByEdbInfoId 计算指标追溯
+func getCalculateEdbInfoByEdbInfoId(allEdbMappingMap map[int][]*models.EdbInfoCalculateMappingInfo, edbInfoId int, hasFindMap map[int]struct{}, edbInfoIdMap map[int]struct{}, edbMappingList []*models.EdbInfoCalculateMapping, edbMappingMap map[int]struct{}, edbInfoMappingRootIdsMap map[int][]int, rootEdbInfoId int) (newEdbMappingList []*models.EdbInfoCalculateMapping, err error) {
+	newEdbMappingList = edbMappingList
+	_, ok := hasFindMap[edbInfoId]
+	if ok {
+		return
+	}
+
+	if _, ok1 := edbInfoIdMap[edbInfoId]; !ok1 {
+		edbInfoIdMap[edbInfoId] = struct{}{}
+	}
+	edbInfoMappingList := make([]*models.EdbInfoCalculateMappingInfo, 0)
+	edbInfoMappingList, ok = allEdbMappingMap[edbInfoId]
+	if !ok {
+		edbInfoMappingList, err = models.GetEdbInfoCalculateMappingListByEdbInfoId(edbInfoId)
+		if err != nil {
+			err = fmt.Errorf("GetEdbInfoCalculateMappingListByEdbInfoId err: %s", err.Error())
+			return
+		}
+	}
+	hasFindMap[edbInfoId] = struct{}{}
+	if len(edbInfoMappingList) > 0 {
+		fromEdbInfoIdList := make([]int, 0)
+		edbInfoMappingIdList := make([]int, 0)
+		for _, v := range edbInfoMappingList {
+			fromEdbInfoIdList = append(fromEdbInfoIdList, v.FromEdbInfoId)
+			edbInfoMappingIdList = append(edbInfoMappingIdList, v.EdbInfoCalculateMappingId)
+			if _, ok1 := edbInfoIdMap[v.FromEdbInfoId]; !ok1 {
+				edbInfoIdMap[v.FromEdbInfoId] = struct{}{}
+			}
+			if _, ok2 := edbMappingMap[v.EdbInfoCalculateMappingId]; !ok2 {
+				edbMappingMap[v.EdbInfoCalculateMappingId] = struct{}{}
+				tmp := &models.EdbInfoCalculateMapping{
+					EdbInfoCalculateMappingId: v.EdbInfoCalculateMappingId,
+					EdbInfoId:                 v.EdbInfoId,
+					Source:                    v.Source,
+					SourceName:                v.SourceName,
+					EdbCode:                   v.EdbCode,
+					FromEdbInfoId:             v.FromEdbInfoId,
+					FromEdbCode:               v.FromEdbCode,
+					FromEdbName:               v.FromEdbName,
+					FromSource:                v.FromSource,
+					FromSourceName:            v.FromSourceName,
+					FromTag:                   v.FromTag,
+					Sort:                      v.Sort,
+					CreateTime:                v.CreateTime,
+					ModifyTime:                v.ModifyTime,
+				}
+				newEdbMappingList = append(newEdbMappingList, tmp)
+
+			}
+
+			if edbInfoId != v.FromEdbInfoId && (v.FromEdbType == 2 || v.FromEdbInfoType == 1) {
+				// 查过了就不查了
+				if _, ok2 := hasFindMap[v.FromEdbInfoId]; !ok2 {
+					newEdbMappingList, err = getCalculateEdbInfoByEdbInfoId(allEdbMappingMap, v.FromEdbInfoId, hasFindMap, edbInfoIdMap, newEdbMappingList, edbMappingMap, edbInfoMappingRootIdsMap, rootEdbInfoId)
+					if err != nil {
+						err = fmt.Errorf("traceEdbInfoByEdbInfoId err: %s", err.Error())
+						return
+					}
+				}
+			}
+			hasFindMap[v.FromEdbInfoId] = struct{}{}
+		}
+		edbInfoMappingRootIdsMap[rootEdbInfoId] = append(edbInfoMappingRootIdsMap[rootEdbInfoId], edbInfoMappingIdList...)
+	}
+
+	return
+}

services/factor_edb_series.go

@@ -0,0 +1,75 @@
+package services
+
+import (
+	"eta/eta_index_lib/models"
+	"eta/eta_index_lib/services/alarm_msg"
+	"eta/eta_index_lib/utils"
+	"fmt"
+	"time"
+)
+
+// FactorEdbStepCalculate 因子指标-多公式计算
+func FactorEdbStepCalculate(seriesId, edbInfoId int, edbCode string, edbData []*models.EdbInfoSearchData, calculates []models.CalculatesReq) (err error) {
+	if len(edbData) == 0 || len(calculates) == 0 {
+		return
+	}
+	defer func() {
+		if err != nil {
+			tips := fmt.Sprintf("FactorEdbStepCalculate计算失败, EdbCode: %s, ErrMsg: %v", edbCode, err)
+			utils.FileLog.Info(tips)
+			go alarm_msg.SendAlarmMsg(tips, 2)
+		}
+	}()
+
+	// 重新计算-先清除原数据
+	calculateDataOb := new(models.FactorEdbSeriesCalculateData)
+	{
+		cond := fmt.Sprintf("%s = ?", calculateDataOb.Cols().FactorEdbSeriesId)
+		pars := make([]interface{}, 0)
+		pars = append(pars, seriesId)
+		if e := calculateDataOb.RemoveByCondition(cond, pars); e != nil {
+			err = fmt.Errorf("清除原数据失败, err: %v", e)
+			return
+		}
+	}
+
+	// 多公式嵌套计算
+	dataMap, dateList, _, e := StepCalculate(edbData, calculates)
+	if e != nil {
+		err = fmt.Errorf("嵌套计算失败, err: %v", e)
+		return
+	}
+
+	// 计算成功的保存结果
+	dataArr := make([]*models.FactorEdbSeriesCalculateData, 0)
+	for _, d := range dateList {
+		val, ok := dataMap[d]
+		if !ok {
+			continue
+		}
+		dataTime, e := time.ParseInLocation(utils.FormatDate, d, time.Local)
+		if e != nil {
+			err = fmt.Errorf("解析计算结果日期失败, err: %v", e)
+			return
+		}
+		dataArr = append(dataArr, &models.FactorEdbSeriesCalculateData{
+			FactorEdbSeriesId: seriesId,
+			EdbInfoId:         edbInfoId,
+			EdbCode:           edbCode,
+			DataTime:          dataTime,
+			Value:             val,
+			CreateTime:        time.Now().Local(),
+			ModifyTime:        time.Now().Local(),
+			DataTimestamp:     dataTime.UnixNano() / 1e6,
+		})
+	}
+	if len(dataArr) == 0 {
+		//err = fmt.Errorf("计算结果无数据")
+		return
+	}
+	if e = calculateDataOb.CreateMulti(dataArr); e != nil {
+		err = fmt.Errorf("保存计算结果失败, err: %v", e)
+		return
+	}
+	return
+}

utils/calculate.go

@@ -50,6 +50,54 @@ func GetLinearResult(s []Coordinate) (gradient, intercept float64) {
 	return
 }
 
+// CalculateCorrelationByIntArr 相关性计算
+// 计算步骤
+// 1.分别计算两个序列的平均值Mx和My
+// 2.分别计算两个序列的标准偏差SDx和SDy	=> √{1/(n-1)*SUM[(Xi-Mx)²]}
+// 3.计算相关系数	=> SUM[(Xi-Mx)*(Yi-My)]/[(N-1)(SDx*SDy)]
+func CalculateCorrelationByIntArr(xArr, yArr []float64) (ratio float64) {
+	// 序列元素数要一致
+	xLen := float64(len(xArr))
+	yLen := float64(len(yArr))
+	if xLen == 0 || xLen != yLen {
+		return
+	}
+
+	// 计算Mx和My
+	var Xa, Ya float64
+	for i := range xArr {
+		Xa += xArr[i]
+	}
+	Mx := Xa / xLen
+	for i := range yArr {
+		Ya += yArr[i]
+	}
+	My := Ya / yLen
+
+	// 计算标准偏差SDx和SDy
+	var Xb, Yb, SDx, SDy float64
+	for i := range xArr {
+		Xb += (xArr[i] - Mx) * (xArr[i] - Mx)
+	}
+	SDx = math.Sqrt(1 / (xLen - 1) * Xb)
+	for i := range yArr {
+		Yb += (yArr[i] - My) * (yArr[i] - My)
+	}
+	SDy = math.Sqrt(1 / (yLen - 1) * Yb)
+
+	// 计算相关系数
+	var Nume, Deno float64
+	for i := 0; i < int(xLen); i++ {
+		Nume += (xArr[i] - Mx) * (yArr[i] - My)
+	}
+	Deno = (xLen - 1) * (SDx * SDy)
+	ratio = Nume / Deno
+	if math.IsNaN(ratio) {
+		ratio = 0
+	}
+	return
+}
+
 // ComputeCorrelation 通过一组数据获取相关系数R
 // 计算步骤
 // 1.分别计算两个序列的平均值Mx和My

utils/common.go

@@ -1310,3 +1310,18 @@ func InsertStr2StrIdx(str, sep string, idx int, value string) string {
 	slice = append(slice[:idx], append([]string{value}, slice[idx:]...)...)
 	return strings.Join(slice, sep)
 }
+
+// FormatFloatPlaces 格式化浮点数位数
+func FormatFloatPlaces(val float64, places int32) (newVal float64, err error) {
+	if places <= 0 {
+		places = 4
+	}
+	strNewVal := decimal.NewFromFloat(val).Round(places).String()
+	di, e := decimal.NewFromString(strNewVal)
+	if e != nil {
+		err = fmt.Errorf("NewFromString err: %v", e)
+		return
+	}
+	newVal, _ = di.Float64()
+	return
+}

utils/config.go

@@ -3,18 +3,20 @@ package utils
 import (
 	"encoding/json"
 	"fmt"
+	"strconv"
+
 	beeLogger "github.com/beego/bee/v2/logger"
 	"github.com/beego/beego/v2/server/web"
 	"github.com/qiniu/qmgo"
-	"strconv"
 )
 
 var (
-	RunMode       string //运行模式
-	MYSQL_URL     string //数据库连接
-	MYSQL_URL_EDB string
-	MYSQL_URL_GL  string
-	MgoUrlData    string // mongodb数据库连接配置
+	RunMode          string //运行模式
+	MYSQL_URL        string //数据库连接
+	MYSQL_URL_EDB    string
+	MYSQL_URL_GL     string
+	MYSQL_URL_MASTER string
+	MgoUrlData       string // mongodb数据库连接配置
 
 	PYTHON_MYSQL_HOST           string // python数据库链接主机地址
 	PYTHON_MYSQL_USER           string // python数据库链接账号
@@ -70,8 +72,9 @@ var (
 	EDB_DATA_LIMIT        = 10
 	Hz_Wind_Data_Url_LIST []WindUrlMap // wind接口服务地址配置（客户本地机器，是个list列表）
 
-	ThsDataMethod   string //同花顺数据获取的方式，app是通过终端；api是通过接口
-	ThsRefreshToken string // 同花顺的刷新token
+	ThsDataMethod           string //同花顺数据获取的方式，app是通过终端；api是通过接口
+	ThsRefreshToken         string // 同花顺的刷新token
+	MysteelChemicalApiToken string // 钢联化工的api数据token
 )
 
 type WindUrlMap struct {
@@ -103,6 +106,10 @@ var (
 	LogMaxDays int //日志最大保留天数
 )
 
+var (
+	UseMongo bool // 是否使用mongo
+)
+
 func init() {
 	tmpRunMode, err := web.AppConfig.String("run_mode")
 	if err != nil {
@@ -132,6 +139,7 @@ func init() {
 	MYSQL_URL = config["mysql_url"]
 	MYSQL_URL_EDB = config["mysql_url_edb"]
 	MYSQL_URL_GL = config["mysql_url_gl"]
+	MYSQL_URL_MASTER = config["mysql_url_master"]
 
 	// mongodb数据库连接配置
 	MgoUrlData = config["mgo_url_data"]
@@ -219,6 +227,7 @@ func init() {
 		if ThsDataMethod == `` {
 			ThsDataMethod = "api"
 		}
+		MysteelChemicalApiToken = config["mysteel_chemical_api_token"]
 	}
 
 	// ES配置

utils/constants.go

@@ -8,6 +8,7 @@ const (
 	FormatDate                 = "2006-01-02"              //日期格式
 	FormatDateUnSpace          = "20060102"                //日期格式
 	FormatDateTime             = "2006-01-02 15:04:05"     //完整时间格式
+	FormatDateTimeMinute       = "2006-01-02 15:04"        //完整时间格式
 	HlbFormatDateTime          = "2006-01-02_15:04:05.999" //完整时间格式
 	FormatDateTimeUnSpace      = "20060102150405"          //完整时间格式
 	FormatShortDateTimeUnSpace = "060102150405"            //省去开头两位年份的时间格式
@@ -228,6 +229,8 @@ const (
 	CACHE_EDB_TERMINAL_CODE_GOOD_URL = "edb:terminal_code:good:edb_code:" // 指标与终端关系的缓存, 商品期货
 	CACHE_EDB_THS_SERVER_TOKEN       = "edb:ths_server_token:"            //同花顺调用凭证
 	CACHE_SELF_EDB_HANDLE            = "CACHE_SELF_EDB_HANDLE:"           // 自定义指标缓存
+	CACHE_BASE_EDB_ADD               = "CACHE_BASE_EDB_ADD_"              // 添加至数据源缓存
+	CACHE_BASE_EDB_REFRESH           = "CACHE_BASE_EDB_REFRESH_"          // 刷新数据源缓存
 )
 
 // 图表类型
@@ -280,8 +283,9 @@ const (
 
 // 子数据来源渠道
 const (
-	DATA_SUB_SOURCE_EDB  = iota //经济数据库
-	DATA_SUB_SOURCE_DATE        //日期序列
+	DATA_SUB_SOURCE_EDB            = iota //经济数据库
+	DATA_SUB_SOURCE_DATE                  //日期序列
+	DATA_SUB_SOURCE_HIGH_FREQUENCY        //高频数据
 )
 
 // 语言版本
@@ -294,3 +298,29 @@ const (
 	PercentCalculateTypeRange = 0 // 百分位算法类型-数据区间
 	PercentCalculateTypeNum   = 1 // 百分位算法类型-数据个数
 )
+
+const (
+	WindDbWsd = "wsd"
+	ThsDs     = "thsds"
+	ThsHf     = "thshf"
+)
+
+// 指标计算方式
+const (
+	EdbBaseCalculateLjzzy                = 1  // 累计值转月->1
+	EdbBaseCalculateLjzzj                = 2  // 累计值转季->2
+	EdbBaseCalculateTbz                  = 3  // 同比值->3
+	EdbBaseCalculateTcz                  = 4  // 同差值->4
+	EdbBaseCalculateNszydpjjs            = 5  // N数值移动平均数计算->5
+	EdbBaseCalculateHbz                  = 6  // 环比值->6
+	EdbBaseCalculateHcz                  = 7  // 环差值->7
+	EdbBaseCalculateUpFrequency          = 8  // 升频->8
+	EdbBaseCalculateDownFrequency        = 9  // 降频->9
+	EdbBaseCalculateTimeShift            = 10 // 时间移位->10
+	EdbBaseCalculateCjjx                 = 11 // 超季节性->11
+	EdbBaseCalculateAnnualized           = 12 // 年化->12
+	EdbBaseCalculateLjz                  = 13 // 累计值->13
+	EdbBaseCalculateLjzNczj              = 14 // 累计值年初至今->14
+	EdbBaseCalculateExponentialSmoothing = 15 // 指数修匀->15
+	EdbBaseCalculateRjz                  = 16 // 日均值->16
+)