Merge branch 'feature/eta_1.9.3' into custom

controllers/base_from_calculate.go

@@ -118,6 +118,9 @@ func (this *CalculateController) Add() {
 	if err != nil {
 		return
 	}
+
+	// 判断是否需要禁用
+	go services.DisableEdbInfoNoUpdate(edbInfo)
 	resp := models.AddEdbInfoResp{
 		EdbInfoId:  edbInfo.EdbInfoId,
 		UniqueCode: uniqueCode,
@@ -336,6 +339,8 @@ func (this *CalculateController) Edit() {
 		EdbInfoId:  edbInfoDetail.EdbInfoId,
 		UniqueCode: edbInfoDetail.UniqueCode,
 	}
+	// 重置计算指标中的引用关系
+	go services.ResetEdbRelation(edbInfoDetail.EdbInfoId)
 	br.Ret = 200
 	br.Success = true
 	br.Msg = "保存成功"
@@ -777,7 +782,8 @@ func (this *CalculateController) BatchSave() {
 		br.ErrMsg = err.Error()
 		return
 	}
-
+	// 判断是否需要禁用
+	go services.DisableEdbInfoNoUpdate(edbInfo)
 	resp := models.AddEdbInfoResp{
 		EdbInfoId:  edbInfo.EdbInfoId,
 		UniqueCode: edbInfo.UniqueCode,
@@ -1208,6 +1214,9 @@ func (this *CalculateController) BatchEdit() {
 		return
 	}
 
+	// 重置计算指标中的引用关系
+	go services.ResetEdbRelation(edbInfoId)
+
 	resp := models.AddEdbInfoResp{
 		EdbInfoId:  edbInfo.EdbInfoId,
 		UniqueCode: edbInfo.UniqueCode,
@@ -1767,6 +1776,11 @@ func (this *CalculateController) SaveAdjust() {
 		EdbInfoId:  edbInfo.EdbInfoId,
 		UniqueCode: edbInfo.UniqueCode,
 	}
+
+	// 判断是否需要禁用
+	go services.DisableEdbInfoNoUpdate(edbInfo)
+	// 重置计算指标中的引用关系
+	go services.ResetEdbRelation(edbInfo.EdbInfoId)
 	br.Ret = 200
 	br.Success = true
 	br.Msg = "保存成功"

controllers/base_from_predict_calculate.go

@@ -506,6 +506,7 @@ func editPredictCalculate(br *models.BaseResponse, req models.EdbInfoCalculateSa
 		br.ErrMsg = err.Error()
 		return
 	}
+
 	resp := models.AddEdbInfoResp{
 		EdbInfoId:  edbInfo.EdbInfoId,
 		UniqueCode: edbInfo.UniqueCode,

controllers/base_from_ths_ds.go

@@ -6,6 +6,7 @@ import (
 	"eta/eta_index_lib/models"
 	"eta/eta_index_lib/services"
 	"eta/eta_index_lib/utils"
+	"fmt"
 	"strconv"
 	"time"
 )
@@ -46,7 +47,7 @@ func (this *ThsDsController) Add() {
 	cacheKey = utils.CACHE_EDB_DATA_ADD + strconv.Itoa(source) + "_" + req.StockCode + req.EdbCode
 	if !utils.Rc.IsExist(cacheKey) {
 		utils.Rc.SetNX(cacheKey, 1, 1*time.Minute)
-		dataItem, err := services.GetEdbDataFromThsDs(req.StockCode, req.EdbCode, utils.BASE_START_DATE, endDate, "")
+		dataItem, err := services.GetEdbDataFromThsDs(req.StockCode, req.EdbCode, utils.BASE_START_DATE, endDate, "", req.ExtraPars)
 		if err != nil {
 			br.Msg = "获取指标信息失败！"
 			br.ErrMsg = "获取指标信息失败 GetEdbDataFromThsDs，Err:" + err.Error()
@@ -118,7 +119,17 @@ func (this *ThsDsController) Refresh() {
 	defer func() {
 		utils.Rc.Delete(cacheKey)
 	}()
-	dataItem, err := services.GetEdbDataFromThsDs(edbInfo.StockCode, edbInfo.IndicatorCode, utils.GetEdbRefreshStartDate(req.StartDate), endDate, edbInfo.TerminalCode)
+
+	var extra models.EdbInfoExtra
+	if edbInfo.Extra != "" {
+		if e := json.Unmarshal([]byte(edbInfo.Extra), &extra); e != nil {
+			br.Msg = "刷新失败"
+			br.ErrMsg = fmt.Sprintf("API额外参数解析失败, %v", e)
+			return
+		}
+	}
+
+	dataItem, err := services.GetEdbDataFromThsDs(edbInfo.StockCode, edbInfo.IndicatorCode, utils.GetEdbRefreshStartDate(req.StartDate), endDate, edbInfo.TerminalCode, extra.ApiExtraPars)
 	if err != nil {
 		br.Msg = "获取指标信息失败！"
 		br.ErrMsg = "获取指标信息失败 GetEdbDataFromThsDs，Err:" + err.Error()

controllers/base_from_ths_hf.go

@@ -0,0 +1,818 @@
+package controllers
+
+import (
+	"encoding/json"
+	"eta/eta_index_lib/logic"
+	"eta/eta_index_lib/models"
+	"eta/eta_index_lib/models/mgo"
+	"eta/eta_index_lib/services"
+	"eta/eta_index_lib/utils"
+	"fmt"
+	"strconv"
+	"strings"
+	"time"
+)
+
+// ThsHfController 同花顺高频数据
+type ThsHfController struct {
+	BaseAuthController
+}
+
+// GetData
+// @Title 同花顺高频数据-获取数据
+// @Description 同花顺高频数据-获取数据
+// @Success 200 {object} models.ThsHfSearchEdbReq
+// @router /hf/edb_data [post]
+func (this *ThsHfController) GetData() {
+	br := new(models.BaseResponse).Init()
+	defer func() {
+		if br.ErrMsg == "" {
+			br.IsSendEmail = false
+		}
+		this.Data["json"] = br
+		this.ServeJSON()
+	}()
+	var params models.ThsHfSearchEdbReq
+	if e := json.Unmarshal(this.Ctx.Input.RequestBody, &params); e != nil {
+		br.Msg = "参数解析异常"
+		br.ErrMsg = fmt.Sprintf("参数解析失败, %v", e)
+		return
+	}
+	params.StockCode = strings.TrimSpace(params.StockCode)
+	if params.StockCode == "" {
+		br.Msg = "请输入证券代码"
+		return
+	}
+	stockCodes := strings.Split(params.StockCode, ",")
+	if len(stockCodes) == 0 {
+		br.Msg = "请输入证券代码"
+		return
+	}
+	if len(stockCodes) > 10 {
+		br.Msg = "最多输入10个证券代码"
+		return
+	}
+	params.EdbCode = strings.TrimSpace(params.EdbCode)
+	if params.EdbCode == "" {
+		br.Msg = "请输入指标代码"
+		return
+	}
+	edbCodes := strings.Split(params.EdbCode, ",")
+	if len(edbCodes) == 0 {
+		br.Msg = "请输入指标代码"
+		return
+	}
+	if len(edbCodes) > 20 {
+		br.Msg = "最多选择/输入20个指标代码"
+		return
+	}
+	if params.StartTime == "" {
+		br.Msg = "请选择起始时间"
+		return
+	}
+	_, e := time.ParseInLocation(utils.FormatDateTime, params.StartTime, time.Local)
+	if e != nil {
+		br.Msg = "起始时间格式有误"
+		br.ErrMsg = fmt.Sprintf("起始时间格式有误, %v", e)
+		return
+	}
+	// 结束时间选填, 不填则为当前时间
+	if params.EndTime != "" {
+		_, e := time.ParseInLocation(utils.FormatDateTime, params.EndTime, time.Local)
+		if e != nil {
+			br.Msg = "截止时间格式有误"
+			br.ErrMsg = fmt.Sprintf("截止时间格式有误, %v", e)
+			return
+		}
+	}
+	if params.EndTime == "" {
+		params.EndTime = time.Now().Local().Format(utils.FormatDateTime)
+	}
+	if !utils.InArrayByInt(models.ThsHfPeriodArr, params.Interval) {
+		br.Msg = "时间周期有误"
+		br.ErrMsg = fmt.Sprintf("时间周期有误, Interval: %d", params.Interval)
+		return
+	}
+	if params.CPS != "" && !utils.InArrayByStr(models.ThsHfCPSArr, params.CPS) {
+		br.Msg = "复权方式有误"
+		br.ErrMsg = fmt.Sprintf("复权方式有误, CPS: %s", params.CPS)
+		return
+	}
+	if params.BaseDate != "" {
+		_, e = time.ParseInLocation(utils.FormatDate, params.BaseDate, time.Local)
+		if e != nil {
+			br.Msg = "复权基点格式有误"
+			br.ErrMsg = fmt.Sprintf("复权基点格式有误, %v", e)
+			return
+		}
+	}
+	if params.Fill != "" && !utils.InArrayByStr(models.ThsHfFillArr, params.Fill) {
+		br.Msg = "非交易间隔处理有误"
+		br.ErrMsg = fmt.Sprintf("非交易间隔处理有误, Fill: %s", params.Fill)
+		return
+	}
+
+	// 根据配置获取指标数据
+	indexes, e := services.GetEdbDataFromThsHf(params, "")
+	if e != nil {
+		br.Msg = "获取失败"
+		br.ErrMsg = fmt.Sprintf("获取同花顺高频指标失败, %v", e)
+		return
+	}
+
+	br.Data = indexes
+	br.Ret = 200
+	br.Success = true
+	br.Msg = "获取成功"
+}
+
+// BaseAdd
+// @Title 同花顺高频数据-新增至数据源
+// @Description 同花顺高频数据-新增至数据源
+// @Success 200 {object} models.ThsHfBaseAddReq
+// @router /hf/base/add [post]
+func (this *ThsHfController) BaseAdd() {
+	br := new(models.BaseResponse).Init()
+	defer func() {
+		if br.ErrMsg == "" {
+			br.IsSendEmail = false
+		}
+		this.Data["json"] = br
+		this.ServeJSON()
+	}()
+	var params models.ThsHfBaseAddReq
+	if e := json.Unmarshal(this.Ctx.Input.RequestBody, &params); e != nil {
+		br.Msg = "参数解析异常"
+		br.ErrMsg = fmt.Sprintf("参数解析失败, %v", e)
+		return
+	}
+	params.StockCode = strings.TrimSpace(params.StockCode)
+	if params.StockCode == "" {
+		br.Msg = "请输入证券代码"
+		return
+	}
+	params.EdbCode = strings.TrimSpace(params.EdbCode)
+	if params.EdbCode == "" {
+		br.Msg = "请输入指标代码"
+		return
+	}
+	if params.StartTime == "" {
+		br.Msg = "请选择起始时间"
+		return
+	}
+	startTime, e := time.ParseInLocation(utils.FormatDateTime, params.StartTime, time.Local)
+	if e != nil {
+		br.Msg = "起始时间格式有误"
+		br.ErrMsg = fmt.Sprintf("起始时间格式有误, %v", e)
+		return
+	}
+	var endTime time.Time
+	if params.EndTime != "" {
+		ed, e := time.ParseInLocation(utils.FormatDateTime, params.EndTime, time.Local)
+		if e != nil {
+			br.Msg = "截止时间格式有误"
+			br.ErrMsg = fmt.Sprintf("截止时间格式有误, %v", e)
+			return
+		}
+		endTime = ed
+	}
+	if !utils.InArrayByInt(models.ThsHfPeriodArr, params.Interval) {
+		br.Msg = "时间周期有误"
+		br.ErrMsg = fmt.Sprintf("时间周期有误, Interval: %d", params.Interval)
+		return
+	}
+	if params.CPS != "" && !utils.InArrayByStr(models.ThsHfCPSArr, params.CPS) {
+		br.Msg = "复权方式有误"
+		br.ErrMsg = fmt.Sprintf("复权方式有误, CPS: %s", params.CPS)
+		return
+	}
+	if params.BaseDate != "" {
+		_, e = time.ParseInLocation(utils.FormatDate, params.BaseDate, time.Local)
+		if e != nil {
+			br.Msg = "复权基点格式有误"
+			br.ErrMsg = fmt.Sprintf("复权基点格式有误, %v", e)
+			return
+		}
+	}
+	if params.Fill != "" && !utils.InArrayByStr(models.ThsHfFillArr, params.Fill) {
+		br.Msg = "非交易间隔处理有误"
+		br.ErrMsg = fmt.Sprintf("非交易间隔处理有误, Fill: %s", params.Fill)
+		return
+	}
+	if params.ClassifyId <= 0 {
+		br.Msg = "请选择分类"
+		return
+	}
+	params.IndexName = strings.TrimSpace(params.IndexName)
+	if params.IndexName == "" {
+		br.Msg = "请输入指标名称"
+		return
+	}
+	if params.Frequency == "" {
+		br.Msg = "请输入频度"
+		return
+	}
+
+	// 缓存
+	source := utils.DATA_SOURCE_THS
+	subSource := utils.DATA_SUB_SOURCE_HIGH_FREQUENCY
+	cacheKey := fmt.Sprintf("%s_%d_%d_%s_%s", utils.CACHE_BASE_EDB_ADD, source, subSource, params.StockCode, params.EdbCode)
+	defer func() {
+		_ = utils.Rc.Delete(cacheKey)
+	}()
+	if utils.Rc.IsExist(cacheKey) {
+		br.Ret = 501
+		br.Success = true
+		br.Msg = "系统处理中，请稍后重试"
+		return
+	}
+	utils.Rc.SetNX(cacheKey, 1, 3*time.Minute)
+
+	// 已添加则忽略
+	indexOb := new(models.BaseFromThsHfIndex)
+	{
+		cond := fmt.Sprintf(" AND %s = ? AND %s = ?", indexOb.Cols().StockCode, indexOb.Cols().Indicator)
+		pars := make([]interface{}, 0)
+		pars = append(pars, params.StockCode, params.EdbCode)
+		item, e := indexOb.GetItemByCondition(cond, pars, "")
+		if e != nil && e.Error() != utils.ErrNoRow() {
+			br.Msg = "操作失败"
+			br.ErrMsg = fmt.Sprintf("获取原始指标失败, %v", e)
+			return
+		}
+		if item != nil {
+			br.Ret = 200
+			br.Success = true
+			br.Msg = "操作成功"
+			return
+		}
+	}
+
+	// 获取指标数据
+	var apiPars models.ThsHfSearchEdbReq
+	apiPars.StockCode = params.StockCode
+	apiPars.EdbCode = params.EdbCode
+	apiPars.StartTime = params.StartTime
+	apiPars.EndTime = params.EndTime
+	apiPars.Interval = params.Interval
+	apiPars.Fill = params.Fill
+	apiPars.CPS = params.CPS
+	apiPars.BaseDate = params.BaseDate
+	indexes, e := services.GetEdbDataFromThsHf(apiPars, "")
+	if e != nil {
+		br.Msg = "操作失败"
+		br.ErrMsg = fmt.Sprintf("获取同花顺高频指标失败, %v", e)
+		return
+	}
+	if len(indexes) == 0 {
+		br.Msg = "未搜索到指标"
+		br.ErrMsg = "未搜索到指标"
+		return
+	}
+	indexWithData := indexes[0]
+
+	indexItem := new(models.BaseFromThsHfIndex)
+	indexItem.BaseFromThsHfClassifyId = params.ClassifyId
+	indexItem.IndexCode = fmt.Sprintf("%s%s%s%s", utils.ThsHf, params.StockCode, params.EdbCode, params.Frequency)
+	indexItem.IndexName = params.IndexName
+	indexItem.Unit = params.Unit
+	indexItem.Frequency = params.Frequency
+	indexItem.StartDate = startTime
+	indexItem.EndDate = endTime
+	indexItem.SysUserId = params.SysAdminId
+	indexItem.SysUserRealName = params.SysAdminName
+	terminal, e := services.GetFirstTerminal(utils.DATA_SOURCE_THS, "")
+	if e != nil {
+		br.Msg = "终端未配置"
+		br.ErrMsg = fmt.Sprintf("终端未配置, %v", e)
+		return
+	}
+	indexItem.TerminalCode = terminal.TerminalCode
+	indexItem.StockCode = params.StockCode
+	indexItem.Indicator = params.EdbCode
+	b, e := json.Marshal(apiPars)
+	if e != nil {
+		br.Msg = "操作失败"
+		br.ErrMsg = fmt.Sprintf("API入参JSON格式化失败, %v", e)
+		return
+	}
+	indexItem.ApiPars = string(b)
+	if len(indexWithData.IndexData) > 0 {
+		indexItem.StartDate = indexWithData.IndexData[0].DataTime
+		indexItem.EndDate = indexWithData.IndexData[len(indexWithData.IndexData)-1].DataTime
+		lastVal, e := utils.FormatFloatPlaces(indexWithData.IndexData[0].Value, 4)
+		if e != nil {
+			br.Msg = "操作失败"
+			br.ErrMsg = fmt.Sprintf("格式化最新值失败, val: %v, err: %v", indexWithData.IndexData[0].Value, e)
+			return
+		}
+		indexItem.LatestValue = lastVal
+	}
+	indexItem.CreateTime = time.Now().Local()
+	indexItem.ModifyTime = time.Now().Local()
+
+	// 新增指标
+	if e := indexItem.Create(); e != nil {
+		br.Msg = "操作失败"
+		br.ErrMsg = fmt.Sprintf("新增指标失败, %v", e)
+		return
+	}
+
+	// 新增数据
+	if utils.UseMongo {
+		dataList := make([]interface{}, 0)
+		for _, v := range indexWithData.IndexData {
+			newVal, e := utils.FormatFloatPlaces(v.Value, 4)
+			if e != nil {
+				utils.FileLog.Info(fmt.Sprintf("FormatFloatPlaces err: %v", e))
+				continue
+			}
+
+			dataList = append(dataList, &mgo.BaseFromThsHfData{
+				BaseFromThsHfIndexId: int64(indexItem.BaseFromThsHfIndexId),
+				IndexCode:            indexItem.IndexCode,
+				DataTime:             v.DataTime,
+				Value:                newVal,
+				UniqueCode:           utils.MD5(fmt.Sprint(indexItem.IndexCode, v.DataTime.Format(utils.FormatDateTimeMinute))),
+				CreateTime:           time.Now().Local(),
+				ModifyTime:           time.Now().Local(),
+				DataTimestamp:        v.DataTime.UnixNano() / 1e6,
+			})
+		}
+		dataOb := new(mgo.BaseFromThsHfData)
+		if e = dataOb.BatchInsertData(500, dataList); e != nil {
+			br.Msg = "操作失败"
+			br.ErrMsg = fmt.Sprintf("批量新增数据失败-Mongo, %v", e)
+			return
+		}
+	} else {
+		dataOb := new(models.BaseFromThsHfData)
+		itemData := make([]*models.BaseFromThsHfData, 0)
+		for _, v := range indexWithData.IndexData {
+			newVal, e := utils.FormatFloatPlaces(v.Value, 4)
+			if e != nil {
+				utils.FileLog.Info(fmt.Sprintf("FormatFloatPlaces err: %v", e))
+				continue
+			}
+
+			t := new(models.BaseFromThsHfData)
+			t.BaseFromThsHfIndexId = indexItem.BaseFromThsHfIndexId
+			t.IndexCode = indexItem.IndexCode
+			t.DataTime = v.DataTime
+			t.Value = newVal
+			t.UniqueCode = utils.MD5(fmt.Sprint(indexItem.IndexCode, v.DataTime.Format(utils.FormatDateTimeMinute)))
+			t.CreateTime = time.Now().Local()
+			t.ModifyTime = time.Now().Local()
+			t.DataTimestamp = v.DataTime.UnixNano() / 1e6
+			itemData = append(itemData, t)
+		}
+		if e = dataOb.CreateMulti(itemData); e != nil {
+			br.Msg = "操作失败"
+			br.ErrMsg = fmt.Sprintf("批量新增数据失败-MySQL, %v", e)
+			return
+		}
+	}
+
+	br.Ret = 200
+	br.Success = true
+	br.Msg = "操作成功"
+}
+
+// BaseRefresh
+// @Title 同花顺高频数据-数据源刷新
+// @Description 同花顺高频数据-数据源刷新
+// @Success 200 {object} models.ThsHfBaseRefreshReq
+// @router /hf/base/refresh [post]
+func (this *ThsHfController) BaseRefresh() {
+	br := new(models.BaseResponse).Init()
+	defer func() {
+		if br.ErrMsg == "" {
+			br.IsSendEmail = false
+		}
+		this.Data["json"] = br
+		this.ServeJSON()
+	}()
+	var params models.ThsHfBaseRefreshReq
+	if e := json.Unmarshal(this.Ctx.Input.RequestBody, &params); e != nil {
+		br.Msg = "参数解析异常"
+		br.ErrMsg = fmt.Sprintf("参数解析失败, %v", e)
+		return
+	}
+	params.BaseIndexCode = strings.TrimSpace(params.BaseIndexCode)
+	if params.BaseIndexCode == "" {
+		br.Msg = "参数异常"
+		br.ErrMsg = fmt.Sprintf("参数异常, BaseIndexCode: %s", params.BaseIndexCode)
+		return
+	}
+	if params.RefreshType <= 0 {
+		params.RefreshType = 1
+	}
+
+	indexItem := new(models.BaseFromThsHfIndex)
+	{
+		ob := new(models.BaseFromThsHfIndex)
+		cond := fmt.Sprintf(" AND %s = ?", ob.Cols().IndexCode)
+		pars := make([]interface{}, 0)
+		pars = append(pars, params.BaseIndexCode)
+		item, e := ob.GetItemByCondition(cond, pars, "")
+		if e != nil {
+			if e.Error() == utils.ErrNoRow() {
+				br.Msg = "指标不存在"
+				return
+			}
+			br.Msg = "操作失败"
+			br.ErrMsg = fmt.Sprintf("获取源指标失败, %v", e)
+			return
+		}
+		indexItem = item
+	}
+
+	source := utils.DATA_SOURCE_THS
+	subSource := utils.DATA_SUB_SOURCE_HIGH_FREQUENCY
+	cacheKey := fmt.Sprintf("%s_%d_%d_%s_%s", utils.CACHE_BASE_EDB_REFRESH, source, subSource, indexItem.StockCode, indexItem.Indicator)
+	defer func() {
+		_ = utils.Rc.Delete(cacheKey)
+	}()
+	if utils.Rc.IsExist(cacheKey) {
+		br.Ret = 501
+		br.Success = true
+		br.Msg = "系统处理中，请稍后重试"
+		return
+	}
+	utils.Rc.SetNX(cacheKey, 1, 3*time.Minute)
+
+	// API参数
+	var apiPars models.ThsHfSearchEdbReq
+	if e := json.Unmarshal([]byte(indexItem.ApiPars), &apiPars); e != nil {
+		br.Msg = "操作失败"
+		br.ErrMsg = fmt.Sprintf("源指标API参数异常, %v", e)
+		return
+	}
+	// 刷新6小时: 指标开始时间前推6小时; 全部: API参数中的开始时间
+	if params.RefreshType == 1 {
+		apiPars.StartTime = indexItem.StartDate.Add(-6 * time.Hour).Format(utils.FormatDateTime)
+	}
+	// 若API参数中的结束时间不为空, 且不在EndDate之后, 那么不再刷新该指标
+	if apiPars.EndTime != "" {
+		apiEnd, e := time.ParseInLocation(utils.FormatDateTime, apiPars.EndTime, time.Local)
+		if e != nil {
+			br.Msg = "操作失败"
+			br.ErrMsg = fmt.Sprintf("API参数结束时间有误, %v", e)
+			return
+		}
+		if !apiEnd.After(indexItem.EndDate) {
+			br.Ret = 200
+			br.Success = true
+			br.Msg = "该指标无需刷新"
+			return
+		}
+	}
+
+	// API-获取指标数据
+	indexes, e := services.GetEdbDataFromThsHf(apiPars, indexItem.TerminalCode)
+	if e != nil {
+		br.Msg = "操作失败"
+		br.ErrMsg = fmt.Sprintf("获取同花顺高频指标失败, %v", e)
+		return
+	}
+	if len(indexes) == 0 {
+		br.Msg = "未搜索到指标"
+		br.ErrMsg = fmt.Sprintf("未搜索到指标, StockCode: %s, Indicator: %s", indexItem.StockCode, indexItem.Indicator)
+		return
+	}
+	indexWithData := indexes[0]
+
+	// 写入指标数据
+	if utils.UseMongo {
+		if e = services.RefreshThsHfBaseIndexMgo(indexItem, indexWithData, apiPars.StartTime); e != nil {
+			br.Msg = "操作失败"
+			br.ErrMsg = fmt.Sprintf("写入源指标数据失败-Mongo, %v", e)
+			return
+		}
+	} else {
+		if e = services.RefreshThsHfBaseIndex(indexItem, indexWithData, apiPars.StartTime); e != nil {
+			br.Msg = "操作失败"
+			br.ErrMsg = fmt.Sprintf("写入源指标数据失败, %v", e)
+			return
+		}
+	}
+
+	br.Ret = 200
+	br.Success = true
+	br.Msg = "操作成功"
+}
+
+// EdbAdd
+// @Title 同花顺高频数据-新增至指标库
+// @Description 同花顺高频数据-新增至指标库
+// @Success 200 {object} models.ThsHfEdbAddReq
+// @router /hf/edb/add [post]
+func (this *ThsHfController) EdbAdd() {
+	br := new(models.BaseResponse).Init()
+	defer func() {
+		if br.ErrMsg == "" {
+			br.IsSendEmail = false
+		}
+		this.Data["json"] = br
+		this.ServeJSON()
+	}()
+	var params models.ThsHfEdbAddReq
+	if e := json.Unmarshal(this.Ctx.Input.RequestBody, &params); e != nil {
+		br.Msg = "参数解析异常"
+		br.ErrMsg = fmt.Sprintf("参数解析失败, %v", e)
+		return
+	}
+	if params.NewIndex == nil {
+		br.Msg = "参数有误"
+		br.ErrMsg = "参数有误, 指标信息有误"
+		return
+	}
+	params.NewIndex.NewIndexName = strings.TrimSpace(params.NewIndex.NewIndexName)
+	if params.NewIndex.NewIndexName == "" {
+		br.Msg = "请输入指标名称"
+		return
+	}
+	if params.NewIndex.ClassifyId <= 0 {
+		br.Msg = "请选择分类"
+		return
+	}
+	if params.NewIndex.Unit == "" {
+		params.NewIndex.Unit = "无"
+	}
+	if params.NewIndex.NewFrequency == "" {
+		br.Msg = "请输入频度"
+		return
+	}
+
+	// 校验转换规则
+	convertRule := params.ConvertRule
+	if convertRule.ConvertType != 1 && convertRule.ConvertType != 2 {
+		br.Msg = "请选择数据转换方式"
+		return
+	}
+	if convertRule.ConvertType == 1 {
+		if convertRule.ConvertFixed.FixedDay != 1 && convertRule.ConvertFixed.FixedDay != 2 {
+			br.Msg = "请选择指定时间"
+			return
+		}
+		if convertRule.ConvertFixed.FixedTime == "" {
+			br.Msg = "请选择指定时间"
+			return
+		}
+		timePrefix := time.Now().Local().Format(utils.FormatDate)
+		st := fmt.Sprintf("%s %s", timePrefix, convertRule.ConvertFixed.FixedTime)
+		_, e := time.Parse(utils.FormatDateTime, st)
+		if e != nil {
+			br.Msg = "指定时间格式有误"
+			return
+		}
+	}
+	if convertRule.ConvertType == 2 {
+		if convertRule.ConvertArea.StartDay != 1 && convertRule.ConvertArea.StartDay != 2 {
+			br.Msg = "请选择起始时间"
+			return
+		}
+		if convertRule.ConvertArea.StartTime == "" {
+			br.Msg = "请选择起始时间"
+			return
+		}
+		var startTimePre string
+		if convertRule.ConvertArea.StartDay == 1 {
+			startTimePre = time.Now().Local().Format(utils.FormatDate)
+		}
+		if convertRule.ConvertArea.StartDay == 2 {
+			startTimePre = time.Now().Local().AddDate(0, 0, -1).Format(utils.FormatDate)
+		}
+		st := fmt.Sprintf("%s %s", startTimePre, convertRule.ConvertArea.StartTime)
+		startTime, e := time.Parse(utils.FormatDateTime, st)
+		if e != nil {
+			br.Msg = "起始时间格式有误"
+			return
+		}
+
+		if convertRule.ConvertArea.EndDay != 1 && convertRule.ConvertArea.EndDay != 2 {
+			br.Msg = "请选择截止时间"
+			return
+		}
+		if convertRule.ConvertArea.EndTime == "" {
+			br.Msg = "请选择截止时间"
+			return
+		}
+		var endTimePre string
+		if convertRule.ConvertArea.EndDay == 1 {
+			endTimePre = time.Now().Local().Format(utils.FormatDate)
+		}
+		if convertRule.ConvertArea.EndDay == 2 {
+			endTimePre = time.Now().Local().AddDate(0, 0, -1).Format(utils.FormatDate)
+		}
+		ed := fmt.Sprintf("%s %s", endTimePre, convertRule.ConvertArea.EndTime)
+		endTime, e := time.Parse(utils.FormatDateTime, ed)
+		if e != nil {
+			br.Msg = "截止时间格式有误"
+			return
+		}
+		if startTime.After(endTime) {
+			br.Msg = "起始日期不可早于截止日期"
+			return
+		}
+	}
+	convertRuleByte, e := json.Marshal(params.ConvertRule)
+	if e != nil {
+		br.Msg = "操作失败"
+		br.ErrMsg = fmt.Sprintf("转换规则JSON格式化失败, %v", e)
+		return
+	}
+
+	// 缓存
+	source := utils.DATA_SOURCE_THS
+	subSource := utils.DATA_SUB_SOURCE_HIGH_FREQUENCY
+	cacheKey := fmt.Sprintf("%s_%d_%d_%s_%s_%s", utils.CACHE_EDB_DATA_ADD, source, subSource, params.NewIndex.StockCode, params.NewIndex.EdbCode, params.NewIndex.NewFrequency)
+	defer func() {
+		_ = utils.Rc.Delete(cacheKey)
+	}()
+	if utils.Rc.IsExist(cacheKey) {
+		br.Ret = 501
+		br.Success = true
+		br.Msg = "系统处理中，请稍后重试"
+		return
+	}
+	utils.Rc.SetNX(cacheKey, 1, 3*time.Minute)
+
+	// 校验指标/分类
+	baseIndexOb := new(models.BaseFromThsHfIndex)
+	baseIndex, e := baseIndexOb.GetItemById(params.NewIndex.IndexId)
+	if e != nil {
+		br.Msg = "原指标不存在"
+		br.ErrMsg = fmt.Sprintf("原指标不存在, %v", e)
+		return
+	}
+	_, e = models.GetEdbClassifyById(params.NewIndex.ClassifyId)
+	if e != nil {
+		br.Msg = "分类信息有误"
+		br.ErrMsg = fmt.Sprintf("获取分类失败, %v", e)
+		return
+	}
+
+	// 判断指标名称是否已存在
+	{
+		var cond string
+		var pars []interface{}
+		if this.Lang == utils.EnLangVersion {
+			cond += " AND edb_name_en = ? "
+		} else {
+			cond += " AND edb_name = ?"
+		}
+		pars = append(pars, params.NewIndex.NewIndexName)
+		count, e := models.GetEdbInfoCountByCondition(cond, pars)
+		if e != nil {
+			br.Msg = "操作失败"
+			br.ErrMsg = fmt.Sprintf("获取重名指标失败, %v", e)
+			return
+		}
+		if count > 0 {
+			br.Msg = "指标名称已存在"
+			return
+		}
+	}
+
+	// 排序/指标编码
+	sortMax, e := models.GetEdbClassifyMaxSort(params.NewIndex.ClassifyId, 0)
+	if e != nil {
+		br.Msg = "操作失败"
+		br.ErrMsg = fmt.Sprintf("获取最大排序失败, %v", e)
+		return
+	}
+	edbCode, e := utils.GenerateEdbCode(1, "")
+	if e != nil {
+		br.Msg = "操作失败"
+		br.ErrMsg = fmt.Sprintf("生成指标编码失败, %v", e)
+		return
+	}
+	timestamp := strconv.FormatInt(time.Now().UnixNano(), 10)
+	uniqueCode := utils.MD5(utils.DATA_PREFIX + "_" + timestamp)
+
+	thsOb := new(models.EdbThsHf)
+	var addPars models.ThsHfAddBaseParams
+	addPars.EdbCode = edbCode
+	addPars.EdbName = params.NewIndex.NewIndexName
+	addPars.Unit = params.NewIndex.Unit
+	addPars.Frequency = params.NewIndex.NewFrequency
+	addPars.Sort = sortMax + 1
+	addPars.ClassifyId = params.NewIndex.ClassifyId
+	addPars.SysUserId = params.NewIndex.SysAdminId
+	addPars.SysUserRealName = params.NewIndex.SysAdminName
+	addPars.UniqueCode = uniqueCode
+	addPars.ConvertRule = string(convertRuleByte)
+	edbInfo, e := thsOb.Add(addPars, baseIndex)
+	if e != nil {
+		br.Msg = "新增失败"
+		br.ErrMsg = fmt.Sprintf("新增指标失败, %v", e)
+		return
+	}
+
+	// 更新指标最值
+	if e = thsOb.UnifiedModifyEdbInfoMaxAndMinInfo(edbInfo); e != nil {
+		br.Msg = "刷新指标失败"
+		br.ErrMsg = fmt.Sprintf("更新指标最值失败, %v", e)
+		return
+	}
+
+	// 添加到es
+	go logic.UpdateEs(edbInfo.EdbInfoId)
+
+	br.Ret = 200
+	br.Success = true
+	br.Msg = "操作成功"
+}
+
+// EdbRefresh
+// @Title 同花顺高频数据-指标库刷新
+// @Description 同花顺高频数据-指标库刷新
+// @Success 200 {object} models.RefreshEdbInfoReq
+// @router /hf/edb/refresh [post]
+func (this *ThsHfController) EdbRefresh() {
+	br := new(models.BaseResponse).Init()
+	defer func() {
+		if br.ErrMsg == "" {
+			br.IsSendEmail = false
+		}
+		this.Data["json"] = br
+		this.ServeJSON()
+	}()
+	var req models.RefreshEdbInfoReq
+	err := json.Unmarshal(this.Ctx.Input.RequestBody, &req)
+	if err != nil {
+		br.Msg = "参数解析异常！"
+		br.ErrMsg = "参数解析失败，Err:" + err.Error()
+		return
+	}
+	if req.EdbCode == "" {
+		br.Msg = "请输入指标编码！"
+		br.ErrMsg = "请输入指标编码，指标编码为空"
+		return
+	}
+	if req.EdbInfoId < 0 {
+		br.Msg = "请输入指标ID！"
+		br.ErrMsg = "请输入指标ID"
+		return
+	}
+	thsOb := new(models.EdbThsHf)
+	source := thsOb.GetSource()
+	subSource := thsOb.GetSubSource()
+	cacheKey := fmt.Sprintf("%s_%d_%d_%s", utils.CACHE_EDB_DATA_REFRESH, source, subSource, req.EdbCode)
+	if utils.Rc.IsExist(cacheKey) {
+		br.Ret = 501
+		br.Success = true
+		br.Msg = "系统处理中，请稍后重试"
+		return
+	}
+	utils.Rc.SetNX(cacheKey, 1, 1*time.Minute)
+	defer func() {
+		_ = utils.Rc.Delete(cacheKey)
+	}()
+
+	// 获取指标详情
+	edbInfo, e := models.GetEdbInfoByEdbCode(source, req.EdbCode)
+	if e != nil {
+		br.Msg = "指标不存在"
+		br.ErrMsg = fmt.Sprintf("指标不存在, %v", e)
+		return
+	}
+
+	// 获取指标关联信息
+	baseMapping := new(models.BaseFromEdbMapping)
+	{
+		ob := new(models.BaseFromEdbMapping)
+		cond := fmt.Sprintf(" AND %s = ? AND %s = ? AND %s = ?", ob.Cols().EdbCode, ob.Cols().Source, ob.Cols().SubSource)
+		pars := make([]interface{}, 0)
+		pars = append(pars, req.EdbCode, thsOb.GetSource(), thsOb.GetSubSource())
+		mapping, e := ob.GetItemByCondition(cond, pars, "")
+		if e != nil {
+			br.Msg = "刷新失败"
+			br.ErrMsg = fmt.Sprintf("指标关联信息有误, %v", e)
+			return
+		}
+		baseMapping = mapping
+	}
+
+	// 刷新指标
+	if e = thsOb.Refresh(edbInfo, baseMapping, req.StartDate); e != nil {
+		br.Msg = "刷新指标失败"
+		br.ErrMsg = fmt.Sprintf("刷新指标失败, %v", e)
+		return
+	}
+
+	// 更新指标最值
+	if e = thsOb.UnifiedModifyEdbInfoMaxAndMinInfo(edbInfo); e != nil {
+		br.Msg = "刷新指标失败"
+		br.ErrMsg = fmt.Sprintf("更新指标最值失败, %v", e)
+		return
+	}
+
+	// 更新ES
+	go logic.UpdateEs(edbInfo.EdbInfoId)
+
+	br.Ret = 200
+	br.Success = true
+	br.Msg = "操作成功"
+}

models/base_from_business.go

@@ -211,8 +211,33 @@ func (m *EdbBusinessSource) Add() (err error) {
 	return
 }
 
-// GetEdbInfoMaxAndMinInfo 获取指标的最新数据记录信息
+// GetEdbInfoMaxAndMinInfo
+// @Description: 获取指标的最新数据记录信息
+// @author: Roc
+// @receiver m
+// @datetime 2024-07-02 14:50:50
+// @param edbCode string
+// @return item *EdbInfoMaxAndMinInfo
+// @return err error
 func (m BaseFromBusinessIndex) GetEdbInfoMaxAndMinInfo(edbCode string) (item *EdbInfoMaxAndMinInfo, err error) {
+	if utils.UseMongo {
+		return m.getEdbInfoMaxAndMinInfoByMongo(edbCode)
+	} else {
+		return m.getEdbInfoMaxAndMinInfoByMysql(edbCode)
+	}
+
+	return
+}
+
+// getEdbInfoMaxAndMinInfoByMongo
+// @Description: 获取指标的最新数据记录信息（从mongo中获取）
+// @author: Roc
+// @receiver m
+// @datetime 2024-07-02 14:41:20
+// @param edbCode string
+// @return item *EdbInfoMaxAndMinInfo
+// @return err error
+func (m BaseFromBusinessIndex) getEdbInfoMaxAndMinInfoByMongo(edbCode string) (item *EdbInfoMaxAndMinInfo, err error) {
 	mogDataObj := new(mgo.BaseFromBusinessData)
 	pipeline := []bson.M{
 		{"$match": bson.M{"index_code": edbCode}},
@@ -256,6 +281,34 @@ func (m BaseFromBusinessIndex) GetEdbInfoMaxAndMinInfo(edbCode string) (item *Ed
 	return
 }
 
+// getEdbInfoMaxAndMinInfoByMysql
+// @Description: 获取指标的最新数据记录信息（从mysql中获取）
+// @author: Roc
+// @receiver m
+// @datetime 2024-07-02 14:49:58
+// @param edbCode string
+// @return item *EdbInfoMaxAndMinInfo
+// @return err error
+func (m BaseFromBusinessIndex) getEdbInfoMaxAndMinInfoByMysql(edbCode string) (item *EdbInfoMaxAndMinInfo, err error) {
+	dataObj := BaseFromBusinessData{}
+	result, err := dataObj.GetEdbInfoMaxAndMinInfo(edbCode)
+	if err != nil {
+		return
+	}
+
+	item = &EdbInfoMaxAndMinInfo{
+		MinDate:     result.MinDate,
+		MaxDate:     result.MaxDate,
+		MinValue:    result.MinValue,
+		MaxValue:    result.MaxValue,
+		LatestValue: result.LatestValue,
+		LatestDate:  result.LatestDate,
+		EndValue:    result.EndValue,
+	}
+
+	return
+}
+
 // ModifyIndexMaxAndMinInfo
 // @Description: 修改最大值和最小值信息
 // @author: Roc

models/base_from_business_data.go

@@ -0,0 +1,306 @@
+package models
+
+import (
+	"errors"
+	"eta/eta_index_lib/utils"
+	"fmt"
+	"github.com/beego/beego/v2/client/orm"
+	"strings"
+	"time"
+)
+
+// BaseFromBusinessData
+// @Description: 外部指标（商家系统）原始数据表
+type BaseFromBusinessData struct {
+	BusinessDataId          int       `orm:"column(business_data_id);pk" json:"business_data_id"`
+	BaseFromBusinessIndexId int       `json:"base_from_business_index_id"` // 指标id
+	IndexCode               string    `json:"index_code"`                  // 指标编码
+	DataTime                time.Time `json:"data_time"`                   // 数据日期
+	Value                   float64   `json:"value"`                       // 数据值
+	CreateTime              time.Time `json:"create_time"`                 // 创建时间
+	ModifyTime              time.Time `json:"modify_time"`                 // 修改时间
+}
+
+// TableName
+// @Description:  获取表名
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-26 13:41:36
+// @return string
+func (m *BaseFromBusinessData) TableName() string {
+	return "base_from_business_data"
+}
+
+// CollectionName
+// @Description:  获取集合名称
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-26 13:41:36
+// @return string
+func (m *BaseFromBusinessData) CollectionName() string {
+	return "base_from_business_data"
+}
+
+// DataBaseName
+// @Description: 获取数据库名称
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-26 13:41:33
+// @return string
+func (m *BaseFromBusinessData) DataBaseName() string {
+	return utils.MgoDataDbName
+}
+
+type WhereParams struct {
+	Condition string
+	Pars      []interface{}
+	Order     string `description:"排序字段"`
+}
+
+// GetAllDataList
+// @Description: 根据条件获取所有数据
+// @author: Roc
+// @receiver m
+// @datetime 2024-07-01 17:18:57
+// @param condition []string
+// @param pars []interface{}
+// @param order string
+// @return result []*BaseFromBusinessData
+// @return err error
+func (m *BaseFromBusinessData) GetAllDataList(condition []string, pars []interface{}, order string) (result []*BaseFromBusinessData, err error) {
+	o := orm.NewOrm()
+
+	sql := `SELECT * FROM base_from_business_data `
+	if len(condition) > 0 {
+		sql += ` WHERE ` + strings.Join(condition, " AND ")
+	}
+
+	if order != `` {
+		sql += ` ORDER BY ` + order
+	}
+
+	_, err = o.Raw(sql, pars).QueryRows(&result)
+
+	return
+}
+
+// GetLimitDataList
+// @Description: 根据条件获取指定数量数据列表
+// @author: Roc
+// @receiver m
+// @datetime 2024-07-01 17:19:16
+// @param condition []string
+// @param pars []interface{}
+// @param order string
+// @param size int64
+// @return result []*BaseFromBusinessData
+// @return err error
+func (m *BaseFromBusinessData) GetLimitDataList(condition []string, pars []interface{}, order string, size int64) (result []*BaseFromBusinessData, err error) {
+	o := orm.NewOrm()
+	sql := `SELECT * FROM base_from_business_data `
+	if len(condition) > 0 {
+		sql += ` WHERE ` + strings.Join(condition, " AND ")
+	}
+
+	if order != `` {
+		sql += ` ORDER BY ` + order
+	}
+
+	sql += fmt.Sprintf(` LIMIT %d`, size)
+
+	_, err = o.Raw(sql, pars).QueryRows(&result)
+
+	return
+}
+
+// GetPageDataList
+// @Description: 根据条件获取分页数据列表
+// @author: Roc
+// @receiver m
+// @datetime 2024-07-01 17:19:42
+// @param condition []string
+// @param pars []interface{}
+// @param order string
+// @param startSize int64
+// @param size int64
+// @return result []*BaseFromBusinessData
+// @return err error
+func (m *BaseFromBusinessData) GetPageDataList(condition []string, pars []interface{}, order string, startSize, size int64) (result []*BaseFromBusinessData, err error) {
+	o := orm.NewOrm()
+	sql := `SELECT * FROM base_from_business_data `
+	if len(condition) > 0 {
+		sql += ` WHERE ` + strings.Join(condition, " AND ")
+	}
+
+	if order != `` {
+		sql += ` ORDER BY ` + order
+	}
+
+	sql += fmt.Sprintf(` LIMIT %d,%d`, startSize, size)
+
+	_, err = o.Raw(sql, pars).QueryRows(&result)
+
+	return
+}
+
+// GetCountDataList
+// @Description: 根据条件获取数据列表总数
+// @author: Roc
+// @receiver m
+// @datetime 2024-07-01 17:19:50
+// @param condition []string
+// @param pars []interface{}
+// @return count int64
+// @return err error
+func (m *BaseFromBusinessData) GetCountDataList(condition []string, pars []interface{}) (count int64, err error) {
+	o := orm.NewOrm()
+	sql := `SELECT COUNT(1) FROM base_from_business_data `
+	if len(condition) > 0 {
+		sql += ` WHERE ` + strings.Join(condition, " AND ")
+	}
+	err = o.Raw(sql, pars).QueryRow(&count)
+
+	return
+}
+
+// InsertDataByColl
+// @Description: 写入单条数据（外部传入集合）
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-26 14:22:18
+// @param addData interface{}
+// @return err error
+func (m *BaseFromBusinessData) InsertDataByColl(addData interface{}) (err error) {
+	o := orm.NewOrm()
+	_, err = o.Insert(addData)
+
+	return
+}
+
+// BatchInsertData
+// @Description: 批量写入数据
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-26 14:22:18
+// @param bulk int 每次请求保存的数据量
+// @param dataList []*BaseFromBusinessData
+// @return err error
+func (m *BaseFromBusinessData) BatchInsertData(bulk int, dataList []*BaseFromBusinessData) (err error) {
+	o := orm.NewOrm()
+	_, err = o.InsertMulti(bulk, dataList)
+
+	return
+}
+
+// UpdateData
+// @Description: 单条数据修改
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-26 15:01:51
+// @param whereParams interface{}
+// @param updateParams interface{}
+// @return err error
+func (m *BaseFromBusinessData) UpdateData(updateCols []string) (err error) {
+	o := orm.NewOrm()
+	_, err = o.Update(m, updateCols...)
+	if err != nil {
+		fmt.Println("UpdateDataByColl:Err:" + err.Error())
+		return
+	}
+
+	return
+}
+
+// HandleData
+// @Description: 数据处理
+// @author: Roc
+// @receiver m
+// @datetime 2024-07-01 17:35:54
+// @param addDataList []*BaseFromBusinessData
+// @param updateDataList []*BaseFromBusinessData
+// @return err error
+func (m *BaseFromBusinessData) HandleData(addDataList, updateDataList []*BaseFromBusinessData) (err error) {
+	o := orm.NewOrm()
+	to, err := o.Begin()
+	if err != nil {
+		return
+	}
+	defer func() {
+		if err != nil {
+			fmt.Println("BaseFromBusinessData HandleData,Err:" + err.Error())
+			_ = to.Rollback()
+		} else {
+			_ = to.Commit()
+		}
+	}()
+
+	// 插入数据
+	if len(addDataList) > 0 {
+		_, err = to.InsertMulti(500, addDataList)
+		if err != nil {
+			return
+		}
+	}
+
+	// 修改
+
+	if len(updateDataList) > 0 {
+		for _, v := range updateDataList {
+			_, err = to.Update(v, "Value", "ModifyTime")
+			if err != nil {
+				fmt.Println("BaseFromBusinessData HandleData Update:Err:" + err.Error())
+				return
+			}
+		}
+	}
+
+	return
+}
+
+// GetEdbInfoMaxAndMinInfo
+// @Description: 获取当前指标的最大最小值
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-30 17:15:39
+// @param whereParams interface{}
+// @return result EdbInfoMaxAndMinInfo
+// @return err error
+func (m *BaseFromBusinessData) GetEdbInfoMaxAndMinInfo(indexCode string) (result EdbInfoMaxAndMinInfo, err error) {
+	o := orm.NewOrm()
+	sql := ``
+	sql = ` SELECT MIN(data_time) AS min_date,MAX(data_time) AS max_date,MIN(value) AS min_value,MAX(value) AS max_value FROM base_from_business_data WHERE index_code = ? `
+	err = o.Raw(sql, indexCode).QueryRow(&result)
+	if err != nil {
+		return
+	}
+
+	var latestValue float64
+	sql = ` SELECT value AS latest_value FROM base_from_business_data WHERE index_code = ? ORDER BY data_time DESC LIMIT 1 `
+	err = o.Raw(sql, indexCode).QueryRow(&latestValue)
+	result.LatestValue = latestValue
+
+	return
+}
+
+// DelDataByCond
+// @Description: 根据条件删除多条数据
+// @author: Roc
+// @receiver m
+// @datetime 2024-07-01 17:46:56
+// @param condition []string
+// @param pars []interface{}
+// @return err error
+func (m *BaseFromBusinessData) DelDataByCond(condition []string, pars []interface{}) (err error) {
+	if len(condition) <= 0 {
+		err = errors.New("条件不能为空")
+		return
+	}
+	o := orm.NewOrm()
+	sql := `DELETE FROM base_from_business_data `
+
+	sql += ` WHERE ` + strings.Join(condition, " AND ")
+
+	_, err = o.Raw(sql, pars).Exec()
+
+	return
+}

models/base_from_edb_mapping.go

@@ -0,0 +1,162 @@
+package models
+
+import (
+	"eta/eta_index_lib/utils"
+	"fmt"
+	"github.com/beego/beego/v2/client/orm"
+	"strings"
+	"time"
+)
+
+// BaseFromEdbMapping 同花顺高频数据
+type BaseFromEdbMapping struct {
+	Id              int       `orm:"column(id);pk"`
+	BaseFromIndexId int       `description:"源指标ID"`
+	BaseIndexCode   string    `description:"源指标编码"`
+	EdbInfoId       int       `description:"指标ID"`
+	EdbCode         string    `description:"指标编码"`
+	Source          int       `description:"指标来源：1-同花顺..."`
+	SubSource       int       `description:"子数据来源：0-经济数据库；1-日期序列；2-高频数据"`
+	ConvertRule     string    `description:"转换规则"`
+	CreateTime      time.Time `description:"创建时间"`
+	ModifyTime      time.Time `description:"修改时间"`
+}
+
+func (m *BaseFromEdbMapping) TableName() string {
+	return "base_from_edb_mapping"
+}
+
+type BaseFromEdbMappingCols struct {
+	PrimaryId       string
+	BaseFromIndexId string
+	BaseIndexCode   string
+	EdbInfoId       string
+	EdbCode         string
+	Source          string
+	SubSource       string
+	ConvertRule     string
+	CreateTime      string
+	ModifyTime      string
+}
+
+func (m *BaseFromEdbMapping) Cols() BaseFromEdbMappingCols {
+	return BaseFromEdbMappingCols{
+		PrimaryId:       "id",
+		BaseFromIndexId: "base_from_index_id",
+		BaseIndexCode:   "base_index_code",
+		EdbInfoId:       "edb_info_id",
+		EdbCode:         "edb_code",
+		Source:          "source",
+		SubSource:       "sub_source",
+		ConvertRule:     "convert_rule",
+		CreateTime:      "create_time",
+		ModifyTime:      "modify_time",
+	}
+}
+
+func (m *BaseFromEdbMapping) Create() (err error) {
+	o := orm.NewOrm()
+	id, err := o.Insert(m)
+	if err != nil {
+		return
+	}
+	m.Id = int(id)
+	return
+}
+
+func (m *BaseFromEdbMapping) CreateMulti(items []*BaseFromEdbMapping) (err error) {
+	if len(items) == 0 {
+		return
+	}
+	o := orm.NewOrm()
+	_, err = o.InsertMulti(len(items), items)
+	return
+}
+
+func (m *BaseFromEdbMapping) Update(cols []string) (err error) {
+	o := orm.NewOrm()
+	_, err = o.Update(m, cols...)
+	return
+}
+
+func (m *BaseFromEdbMapping) Remove() (err error) {
+	o := orm.NewOrm()
+	sql := fmt.Sprintf(`DELETE FROM %s WHERE %s = ? LIMIT 1`, m.TableName(), m.Cols().PrimaryId)
+	_, err = o.Raw(sql, m.Id).Exec()
+	return
+}
+
+func (m *BaseFromEdbMapping) MultiRemove(ids []int) (err error) {
+	if len(ids) == 0 {
+		return
+	}
+	o := orm.NewOrm()
+	sql := fmt.Sprintf(`DELETE FROM %s WHERE %s IN (%s)`, m.TableName(), m.Cols().PrimaryId, utils.GetOrmInReplace(len(ids)))
+	_, err = o.Raw(sql, ids).Exec()
+	return
+}
+
+func (m *BaseFromEdbMapping) RemoveByCondition(condition string, pars []interface{}) (err error) {
+	if condition == "" {
+		return
+	}
+	o := orm.NewOrm()
+	sql := fmt.Sprintf(`DELETE FROM %s WHERE %s`, m.TableName(), condition)
+	_, err = o.Raw(sql, pars).Exec()
+	return
+}
+
+func (m *BaseFromEdbMapping) GetItemById(id int) (item *BaseFromEdbMapping, err error) {
+	o := orm.NewOrm()
+	sql := fmt.Sprintf(`SELECT * FROM %s WHERE %s = ? LIMIT 1`, m.TableName(), m.Cols().PrimaryId)
+	err = o.Raw(sql, id).QueryRow(&item)
+	return
+}
+
+func (m *BaseFromEdbMapping) GetItemByCondition(condition string, pars []interface{}, orderRule string) (item *BaseFromEdbMapping, err error) {
+	o := orm.NewOrm()
+	order := ``
+	if orderRule != "" {
+		order = ` ORDER BY ` + orderRule
+	}
+	sql := fmt.Sprintf(`SELECT * FROM %s WHERE 1=1 %s %s LIMIT 1`, m.TableName(), condition, order)
+	err = o.Raw(sql, pars).QueryRow(&item)
+	return
+}
+
+func (m *BaseFromEdbMapping) GetCountByCondition(condition string, pars []interface{}) (count int, err error) {
+	o := orm.NewOrm()
+	sql := fmt.Sprintf(`SELECT COUNT(1) FROM %s WHERE 1=1 %s`, m.TableName(), condition)
+	err = o.Raw(sql, pars).QueryRow(&count)
+	return
+}
+
+func (m *BaseFromEdbMapping) GetItemsByCondition(condition string, pars []interface{}, fieldArr []string, orderRule string) (items []*BaseFromEdbMapping, err error) {
+	o := orm.NewOrm()
+	fields := strings.Join(fieldArr, ",")
+	if len(fieldArr) == 0 {
+		fields = `*`
+	}
+	order := fmt.Sprintf(`ORDER BY %s DESC`, m.Cols().CreateTime)
+	if orderRule != "" {
+		order = ` ORDER BY ` + orderRule
+	}
+	sql := fmt.Sprintf(`SELECT %s FROM %s WHERE 1=1 %s %s`, fields, m.TableName(), condition, order)
+	_, err = o.Raw(sql, pars).QueryRows(&items)
+	return
+}
+
+func (m *BaseFromEdbMapping) GetPageItemsByCondition(condition string, pars []interface{}, fieldArr []string, orderRule string, startSize, pageSize int) (items []*BaseFromEdbMapping, err error) {
+	o := orm.NewOrm()
+	fields := strings.Join(fieldArr, ",")
+	if len(fieldArr) == 0 {
+		fields = `*`
+	}
+	order := fmt.Sprintf(`ORDER BY %s DESC`, m.Cols().CreateTime)
+	if orderRule != "" {
+		order = ` ORDER BY ` + orderRule
+	}
+	sql := fmt.Sprintf(`SELECT %s FROM %s WHERE 1=1 %s %s LIMIT ?,?`, fields, m.TableName(), condition, order)
+	_, err = o.Raw(sql, pars, startSize, pageSize).QueryRows(&items)
+	return
+}

models/base_from_ths_hf.go

@@ -0,0 +1,476 @@
+package models
+
+import (
+	"eta/eta_index_lib/models/mgo"
+	"eta/eta_index_lib/utils"
+	"fmt"
+	"github.com/beego/beego/v2/client/orm"
+	"go.mongodb.org/mongo-driver/bson"
+	"strings"
+	"time"
+)
+
+var (
+	ThsHfPeriodArr = []int{1, 3, 5, 10, 15, 30, 60}                                                                                     // 时间周期
+	ThsHfCPSArr    = []string{"no", "forward1", "forward2", "forward3", "forward4", "backward1", "backward2", "backward3", "backward4"} // 复权方式
+	ThsHfFillArr   = []string{"Original", "Previous", "Blank"}                                                                          // 非交易间隔处理
+)
+
+// BaseFromThsHfIndex 同花顺高频数据
+type BaseFromThsHfIndex struct {
+	BaseFromThsHfIndexId    int       `orm:"column(base_from_ths_hf_index_id);pk"`
+	BaseFromThsHfClassifyId int       `description:"分类ID"`
+	IndexCode               string    `description:"指标编码"`
+	IndexName               string    `description:"指标名称"`
+	Unit                    string    `description:"单位"`
+	Source                  string    `description:"数据来源"`
+	Frequency               string    `description:"频度"`
+	StartDate               time.Time `description:"开始日期（至时分秒）"`
+	EndDate                 time.Time `description:"结束日期（至时分秒）"`
+	Describe                string    `description:"指标描述"`
+	Sort                    int       `description:"排序"`
+	IsStop                  int       `description:"是否停更：0-否；1-停更"`
+	TerminalCode            string    `description:"所属终端编码"`
+	StockCode               string    `description:"证券代码"`
+	Indicator               string    `description:"同花顺指标代码"`
+	ApiPars                 string    `description:"API请求参数"`
+	LatestValue             float64   `description:"最新值"`
+	SysUserId               int       `description:"创建人ID"`
+	SysUserRealName         string    `description:"创建人姓名"`
+	CreateTime              time.Time `description:"创建时间"`
+	ModifyTime              time.Time `description:"修改时间"`
+}
+
+func (m *BaseFromThsHfIndex) TableName() string {
+	return "base_from_ths_hf_index"
+}
+
+type BaseFromThsHfIndexCols struct {
+	PrimaryId               string
+	BaseFromThsHfClassifyId string
+	IndexCode               string
+	IndexName               string
+	Unit                    string
+	Source                  string
+	Frequency               string
+	StartDate               string
+	EndDate                 string
+	Describe                string
+	Sort                    string
+	IsStop                  string
+	TerminalCode            string
+	StockCode               string
+	Indicator               string
+	ApiPars                 string
+	LatestValue             string
+	SysUserId               string
+	SysUserRealName         string
+	CreateTime              string
+	ModifyTime              string
+}
+
+func (m *BaseFromThsHfIndex) Cols() BaseFromThsHfIndexCols {
+	return BaseFromThsHfIndexCols{
+		PrimaryId:               "base_from_ths_hf_index_id",
+		BaseFromThsHfClassifyId: "base_from_ths_hf_classify_id",
+		IndexCode:               "index_code",
+		IndexName:               "index_name",
+		Unit:                    "unit",
+		Source:                  "source",
+		Frequency:               "frequency",
+		StartDate:               "start_date",
+		EndDate:                 "end_date",
+		Describe:                "describe",
+		Sort:                    "sort",
+		IsStop:                  "is_stop",
+		TerminalCode:            "terminal_code",
+		StockCode:               "stock_code",
+		Indicator:               "indicator",
+		ApiPars:                 "api_pars",
+		LatestValue:             "latest_value",
+		SysUserId:               "sys_user_id",
+		SysUserRealName:         "sys_user_real_name",
+		CreateTime:              "create_time",
+		ModifyTime:              "modify_time",
+	}
+}
+
+func (m *BaseFromThsHfIndex) Create() (err error) {
+	o := orm.NewOrm()
+	id, err := o.Insert(m)
+	if err != nil {
+		return
+	}
+	m.BaseFromThsHfIndexId = int(id)
+	return
+}
+
+func (m *BaseFromThsHfIndex) CreateMulti(items []*BaseFromThsHfIndex) (err error) {
+	if len(items) == 0 {
+		return
+	}
+	o := orm.NewOrm()
+	_, err = o.InsertMulti(len(items), items)
+	return
+}
+
+func (m *BaseFromThsHfIndex) Update(cols []string) (err error) {
+	o := orm.NewOrm()
+	_, err = o.Update(m, cols...)
+	return
+}
+
+func (m *BaseFromThsHfIndex) Remove() (err error) {
+	o := orm.NewOrm()
+	sql := fmt.Sprintf(`DELETE FROM %s WHERE %s = ? LIMIT 1`, m.TableName(), m.Cols().PrimaryId)
+	_, err = o.Raw(sql, m.BaseFromThsHfIndexId).Exec()
+	return
+}
+
+func (m *BaseFromThsHfIndex) MultiRemove(ids []int) (err error) {
+	if len(ids) == 0 {
+		return
+	}
+	o := orm.NewOrm()
+	sql := fmt.Sprintf(`DELETE FROM %s WHERE %s IN (%s)`, m.TableName(), m.Cols().PrimaryId, utils.GetOrmInReplace(len(ids)))
+	_, err = o.Raw(sql, ids).Exec()
+	return
+}
+
+func (m *BaseFromThsHfIndex) RemoveByCondition(condition string, pars []interface{}) (err error) {
+	if condition == "" {
+		return
+	}
+	o := orm.NewOrm()
+	sql := fmt.Sprintf(`DELETE FROM %s WHERE %s`, m.TableName(), condition)
+	_, err = o.Raw(sql, pars).Exec()
+	return
+}
+
+func (m *BaseFromThsHfIndex) GetItemById(id int) (item *BaseFromThsHfIndex, err error) {
+	o := orm.NewOrm()
+	sql := fmt.Sprintf(`SELECT * FROM %s WHERE %s = ? LIMIT 1`, m.TableName(), m.Cols().PrimaryId)
+	err = o.Raw(sql, id).QueryRow(&item)
+	return
+}
+
+func (m *BaseFromThsHfIndex) GetItemByCondition(condition string, pars []interface{}, orderRule string) (item *BaseFromThsHfIndex, err error) {
+	o := orm.NewOrm()
+	order := ``
+	if orderRule != "" {
+		order = ` ORDER BY ` + orderRule
+	}
+	sql := fmt.Sprintf(`SELECT * FROM %s WHERE 1=1 %s %s LIMIT 1`, m.TableName(), condition, order)
+	err = o.Raw(sql, pars).QueryRow(&item)
+	return
+}
+
+func (m *BaseFromThsHfIndex) GetCountByCondition(condition string, pars []interface{}) (count int, err error) {
+	o := orm.NewOrm()
+	sql := fmt.Sprintf(`SELECT COUNT(1) FROM %s WHERE 1=1 %s`, m.TableName(), condition)
+	err = o.Raw(sql, pars).QueryRow(&count)
+	return
+}
+
+func (m *BaseFromThsHfIndex) GetItemsByCondition(condition string, pars []interface{}, fieldArr []string, orderRule string) (items []*BaseFromThsHfIndex, err error) {
+	o := orm.NewOrm()
+	fields := strings.Join(fieldArr, ",")
+	if len(fieldArr) == 0 {
+		fields = `*`
+	}
+	order := fmt.Sprintf(`ORDER BY %s DESC`, m.Cols().CreateTime)
+	if orderRule != "" {
+		order = ` ORDER BY ` + orderRule
+	}
+	sql := fmt.Sprintf(`SELECT %s FROM %s WHERE 1=1 %s %s`, fields, m.TableName(), condition, order)
+	_, err = o.Raw(sql, pars).QueryRows(&items)
+	return
+}
+
+func (m *BaseFromThsHfIndex) GetPageItemsByCondition(condition string, pars []interface{}, fieldArr []string, orderRule string, startSize, pageSize int) (items []*BaseFromThsHfIndex, err error) {
+	o := orm.NewOrm()
+	fields := strings.Join(fieldArr, ",")
+	if len(fieldArr) == 0 {
+		fields = `*`
+	}
+	order := fmt.Sprintf(`ORDER BY %s DESC`, m.Cols().CreateTime)
+	if orderRule != "" {
+		order = ` ORDER BY ` + orderRule
+	}
+	sql := fmt.Sprintf(`SELECT %s FROM %s WHERE 1=1 %s %s LIMIT ?,?`, fields, m.TableName(), condition, order)
+	_, err = o.Raw(sql, pars, startSize, pageSize).QueryRows(&items)
+	return
+}
+
+// BaseFromThsHfIndexItem 同花顺高频数据信息
+type BaseFromThsHfIndexItem struct {
+	IndexId    int    `description:"同花顺高频数据ID"`
+	IndexCode  string `description:"指标编码"`
+	IndexName  string `description:"指标名称"`
+	Unit       string `description:"单位"`
+	Source     string `description:"数据来源"`
+	Frequency  string `description:"频度"`
+	StartDate  string `description:"开始日期（至时分秒）"`
+	EndDate    string `description:"结束日期（至时分秒）"`
+	Describe   string `description:"指标描述"`
+	Sort       int    `description:"排序"`
+	CreateTime string `description:"创建时间"`
+	ModifyTime string `description:"修改时间"`
+}
+
+func (m *BaseFromThsHfIndex) Format2Item() (item *BaseFromThsHfIndexItem) {
+	item = new(BaseFromThsHfIndexItem)
+	item.IndexId = m.BaseFromThsHfIndexId
+	item.IndexCode = m.IndexCode
+	item.IndexName = m.IndexName
+	item.Unit = m.Unit
+	item.Source = m.Source
+	item.Frequency = m.Frequency
+	item.StartDate = utils.TimeTransferString(utils.FormatDateTime, m.StartDate)
+	item.EndDate = utils.TimeTransferString(utils.FormatDateTime, m.EndDate)
+	item.Describe = m.Describe
+	item.Sort = m.Sort
+	item.CreateTime = utils.TimeTransferString(utils.FormatDateTime, m.CreateTime)
+	item.ModifyTime = utils.TimeTransferString(utils.FormatDateTime, m.ModifyTime)
+	return
+}
+
+// ThsHfSearchEdbReq 搜索指标请求体
+type ThsHfSearchEdbReq struct {
+	StockCode string `form:"StockCode" description:"证券代码" `
+	EdbCode   string `form:"EdbCode" description:"指标代码"`
+	StartTime string `form:"StartTime" description:"每日数据开始时间"`
+	EndTime   string `form:"EndTime" description:"每日数据结束时间"`
+	Interval  int    `form:"Interval" description:"时间周期"`
+	Fill      string `form:"Fill" description:"非交易间隔处理"`
+	CPS       string `form:"CPS" description:"复权方式"`
+	BaseDate  string `form:"BaseDate" description:"复权基点"`
+}
+
+// ThsHfApiResp 同花顺接口响应
+type ThsHfApiResp struct {
+	ErrorCode int    `json:"errorcode"`
+	ErrMsg    string `json:"errmsg"`
+	Tables    []struct {
+		ThsCode string               `json:"thscode"`
+		Time    []string             `json:"time"`
+		Table   map[string][]float64 `json:"table"`
+	} `json:"tables"`
+}
+
+// ThsHfAppResp 同花顺公用机响应
+type ThsHfAppResp struct {
+	ErrorCode int    `json:"errorcode"`
+	ErrMsg    string `json:"errmsg"`
+	//Data      []interface{} `json:"data"`
+	Data []struct {
+		Time    string  `json:"time"`
+		ThsCode string  `json:"thscode"`
+		Open    float64 `json:"open"`
+		Close   float64 `json:"close"`
+	} `json:"data"`
+}
+
+// ThsHfIndexWithData 同花顺高频指标
+type ThsHfIndexWithData struct {
+	StockCode string            `description:"证券代码"`
+	EdbCode   string            `description:"指标代码"`
+	IndexData []*ThsHfIndexData `description:"指标数据"`
+}
+
+// ThsHfIndexData 同花顺高频指标数据
+type ThsHfIndexData struct {
+	DataTime time.Time `description:"数据时间(2006-01-02 15:04)"`
+	Value    float64   `description:"数据值"`
+}
+
+type ThsHfBaseAddIndexItem struct {
+	ClassifyId int    `description:"分类ID"`
+	Unit       string `description:"单位"`
+	IndexName  string `description:"指标名称"`
+	Frequency  string `description:"频度"`
+	StockCode  string `description:"证券代码"`
+	EdbCode    string `description:"指标代码"`
+}
+
+// ThsHfBaseAddReq 新增至数据源请求
+type ThsHfBaseAddReq struct {
+	StartTime             string `description:"每日数据开始时间"`
+	EndTime               string `description:"每日数据结束时间"`
+	Interval              int    `description:"时间周期"`
+	Fill                  string `description:"非交易间隔处理"`
+	CPS                   string `description:"复权方式"`
+	BaseDate              string `description:"复权基点"`
+	SysAdminId            int    `description:"创建人ID"`
+	SysAdminName          string `description:"创建人姓名"`
+	ThsHfBaseAddIndexItem `description:"指标信息"`
+}
+
+// CreateIndexAndData 新增指标和数据
+//func (m *BaseFromThsHfIndex) CreateIndexAndData(indexItem *BaseFromThsHfIndex, indexData []*BaseFromThsHfData) (err error) {
+//	o := orm.NewOrm()
+//	tx, e := o.Begin()
+//	if e != nil {
+//		err = fmt.Errorf("tx begin err: %v", e)
+//		return
+//	}
+//	defer func() {
+//		if err != nil {
+//			_ = tx.Rollback()
+//			return
+//		}
+//		_ = tx.Commit()
+//	}()
+//
+//	lastId, e := tx.Insert(indexItem)
+//	if e != nil {
+//		err = fmt.Errorf("insert index err: %v", e)
+//		return
+//	}
+//	indexId := int(lastId)
+//	indexItem.BaseFromThsHfIndexId = indexId
+//
+//	if len(indexData) == 0 {
+//		return
+//	}
+//	for _, v := range indexData {
+//		v.BaseFromThsHfIndexId = indexId
+//	}
+//	if _, e = tx.InsertMulti(200, indexData); e != nil {
+//		err = fmt.Errorf("insert index data err: %v", e)
+//		return
+//	}
+//	return
+//}
+
+// ThsHfBaseRefreshReq 数据源刷新请求
+type ThsHfBaseRefreshReq struct {
+	BaseIndexCode string `description:"源指标编码"`
+	RefreshType   int    `description:"刷新类型: 1-最近6小时; 2-全部刷新"`
+}
+
+// ThsHfEdbAddReq 新增至指标库
+type ThsHfEdbAddReq struct {
+	ConvertRule ThsHfIndexConvert2EdbRule
+	NewIndex    *ThsHfIndexMultiSave2EdbPreItem `description:"新增指标"`
+}
+
+type ThsHfIndexConvert2EdbRule struct {
+	ConvertType  int `description:"转换类型: 1-指定时间值; 2-区间计算值"`
+	ConvertFixed struct {
+		FixedDay  int    `description:"指定时间值日期: 1-当日; 2-前一日"`
+		FixedTime string `description:"指定时间值时点(HH:mm:ss)"`
+	} `description:"指定时间值"`
+	ConvertArea struct {
+		StartDay      int    `description:"起始时间日期: 1-当日; 2-前一日"`
+		StartTime     string `description:"起始时间时点(HH:mm:ss)"`
+		EndDay        int    `description:"截止时间日期: 1-当日; 2-前一日"`
+		EndTime       string `description:"截止时间时点(HH:mm:ss)"`
+		CalculateType int    `description:"计算类型: 1-区间均值; 2-最大值; 3-最小值"`
+	} `description:"区间计算值"`
+}
+
+// ThsHfIndexMultiSave2EdbPreItem 批量新增指标库信息
+type ThsHfIndexMultiSave2EdbPreItem struct {
+	IndexId      int    `description:"指标ID"`
+	IndexCode    string `description:"指标编码"`
+	IndexName    string `description:"原指标名称"`
+	NewIndexName string `description:"新指标名称"`
+	StockCode    string `description:"证券代码"`
+	EdbCode      string `description:"指标代码"`
+	Unit         string `description:"单位"`
+	Frequency    string `description:"原频度"`
+	NewFrequency string `description:"新频度(固定日度)"`
+	ClassifyId   int    `description:"指标库分类ID"`
+	SysAdminId   int    `description:"创建人ID"`
+	SysAdminName string `description:"创建人姓名"`
+	Tips         string `description:"提示信息"`
+	ErrMsg       string `description:"错误信息"`
+}
+
+// GetEdbInfoMaxAndMinInfo
+// @Description: 获取指标的最新数据记录信息
+// @author: Roc
+// @receiver m
+// @datetime 2024-07-02 14:50:50
+// @param edbCode string
+// @return item *EdbInfoMaxAndMinInfo
+// @return err error
+func (m BaseFromThsHfIndex) GetEdbInfoMaxAndMinInfo(edbCode string) (item *EdbInfoMaxAndMinInfo, err error) {
+	if utils.UseMongo {
+		return m.getEdbInfoMaxAndMinInfoByMongo(edbCode)
+	}
+	return m.getEdbInfoMaxAndMinInfoByMysql(edbCode)
+}
+
+// getEdbInfoMaxAndMinInfoByMongo
+// @Description: 获取指标的最新数据记录信息（从mongo中获取）
+// @author: Roc
+// @receiver m
+// @datetime 2024-07-02 14:41:20
+// @param edbCode string
+// @return item *EdbInfoMaxAndMinInfo
+// @return err error
+func (m BaseFromThsHfIndex) getEdbInfoMaxAndMinInfoByMongo(edbCode string) (item *EdbInfoMaxAndMinInfo, err error) {
+	mogDataObj := new(mgo.BaseFromThsHfData)
+	pipeline := []bson.M{
+		{"$match": bson.M{"index_code": edbCode}},
+		{"$group": bson.M{
+			"_id":       nil,
+			"min_date":  bson.M{"$min": "$data_time"},
+			"max_date":  bson.M{"$max": "$data_time"},
+			"min_value": bson.M{"$min": "$value"},
+			"max_value": bson.M{"$max": "$value"},
+		}},
+		{"$project": bson.M{"_id": 0}}, // 可选，如果不需要_id字段
+	}
+	result, err := mogDataObj.GetEdbInfoMaxAndMinInfo(pipeline)
+	if err != nil {
+		fmt.Println("BaseFromThsHfIndex GetEdbInfoMaxAndMinInfo Err:" + err.Error())
+		return
+	}
+
+	if !result.MaxDate.IsZero() {
+		whereQuery := bson.M{"index_code": edbCode, "data_time": result.MaxDate}
+		selectParam := bson.D{{"value", 1}, {"_id", 0}}
+		latestValue, tmpErr := mogDataObj.GetLatestValue(whereQuery, selectParam)
+		if tmpErr != nil {
+			err = tmpErr
+			return
+		}
+		result.LatestValue = latestValue.Value
+		result.EndValue = latestValue.Value
+	}
+
+	item = &EdbInfoMaxAndMinInfo{
+		MinDate:     result.MinDate.Format(utils.FormatDateTime),
+		MaxDate:     result.MaxDate.Format(utils.FormatDateTime),
+		MinValue:    result.MinValue,
+		MaxValue:    result.MaxValue,
+		LatestValue: result.LatestValue,
+		LatestDate:  result.LatestDate.Format(utils.FormatDateTime),
+		EndValue:    result.EndValue,
+	}
+	return
+}
+
+// getEdbInfoMaxAndMinInfoByMysql
+// @Description: 获取指标的最新数据记录信息（从mysql中获取）
+func (m BaseFromThsHfIndex) getEdbInfoMaxAndMinInfoByMysql(edbCode string) (item *EdbInfoMaxAndMinInfo, err error) {
+	dataObj := BaseFromThsHfData{}
+	result, err := dataObj.GetIndexMinMax(edbCode)
+	if err != nil {
+		return
+	}
+
+	item = &EdbInfoMaxAndMinInfo{
+		MinDate:     result.MinDate,
+		MaxDate:     result.MaxDate,
+		MinValue:    result.MinValue,
+		MaxValue:    result.MaxValue,
+		LatestValue: result.LatestValue,
+		LatestDate:  result.LatestDate,
+		EndValue:    result.EndValue,
+	}
+	return
+}

models/base_from_ths_hf_data.go

@@ -0,0 +1,229 @@
+package models
+
+import (
+	"eta/eta_index_lib/utils"
+	"fmt"
+	"github.com/beego/beego/v2/client/orm"
+	"strings"
+	"time"
+)
+
+// BaseFromThsHfData 同花顺高频数据-指标数据
+type BaseFromThsHfData struct {
+	BaseFromThsHfDataId  int       `orm:"column(base_from_ths_hf_data_id);pk"`
+	BaseFromThsHfIndexId int       `description:"指标ID"`
+	IndexCode            string    `description:"指标编码"`
+	DataTime             time.Time `description:"数据日期（至时分秒）"`
+	Value                float64   `description:"数据值"`
+	UniqueCode           string    `description:"唯一编码"`
+	CreateTime           time.Time `description:"创建时间"`
+	ModifyTime           time.Time `description:"修改时间"`
+	DataTimestamp        int64     `description:"数据日期时间戳"`
+}
+
+func (m *BaseFromThsHfData) TableName() string {
+	return "base_from_ths_hf_data"
+}
+
+type BaseFromThsHfDataCols struct {
+	PrimaryId            string
+	BaseFromThsHfIndexId string
+	IndexCode            string
+	DataTime             string
+	Value                string
+	UniqueCode           string
+	CreateTime           string
+	ModifyTime           string
+	DataTimestamp        string
+}
+
+func (m *BaseFromThsHfData) Cols() BaseFromThsHfDataCols {
+	return BaseFromThsHfDataCols{
+		PrimaryId:            "base_from_ths_hf_data_id",
+		BaseFromThsHfIndexId: "base_from_ths_hf_index_id",
+		IndexCode:            "index_code",
+		DataTime:             "data_time",
+		Value:                "value",
+		UniqueCode:           "unique_code",
+		CreateTime:           "create_time",
+		ModifyTime:           "modify_time",
+		DataTimestamp:        "data_timestamp",
+	}
+}
+
+func (m *BaseFromThsHfData) Create() (err error) {
+	o := orm.NewOrm()
+	id, err := o.Insert(m)
+	if err != nil {
+		return
+	}
+	m.BaseFromThsHfDataId = int(id)
+	return
+}
+
+func (m *BaseFromThsHfData) CreateMulti(items []*BaseFromThsHfData) (err error) {
+	if len(items) == 0 {
+		return
+	}
+	o := orm.NewOrm()
+	_, err = o.InsertMulti(500, items)
+	return
+}
+
+func (m *BaseFromThsHfData) Update(cols []string) (err error) {
+	o := orm.NewOrm()
+	_, err = o.Update(m, cols...)
+	return
+}
+
+func (m *BaseFromThsHfData) Remove() (err error) {
+	o := orm.NewOrm()
+	sql := fmt.Sprintf(`DELETE FROM %s WHERE %s = ? LIMIT 1`, m.TableName(), m.Cols().PrimaryId)
+	_, err = o.Raw(sql, m.BaseFromThsHfDataId).Exec()
+	return
+}
+
+func (m *BaseFromThsHfData) MultiRemove(ids []int) (err error) {
+	if len(ids) == 0 {
+		return
+	}
+	o := orm.NewOrm()
+	sql := fmt.Sprintf(`DELETE FROM %s WHERE %s IN (%s)`, m.TableName(), m.Cols().PrimaryId, utils.GetOrmInReplace(len(ids)))
+	_, err = o.Raw(sql, ids).Exec()
+	return
+}
+
+func (m *BaseFromThsHfData) RemoveByCondition(condition string, pars []interface{}) (err error) {
+	if condition == "" {
+		return
+	}
+	o := orm.NewOrm()
+	sql := fmt.Sprintf(`DELETE FROM %s WHERE %s`, m.TableName(), condition)
+	_, err = o.Raw(sql, pars).Exec()
+	return
+}
+
+func (m *BaseFromThsHfData) GetItemById(id int) (item *BaseFromThsHfData, err error) {
+	o := orm.NewOrm()
+	sql := fmt.Sprintf(`SELECT * FROM %s WHERE %s = ? LIMIT 1`, m.TableName(), m.Cols().PrimaryId)
+	err = o.Raw(sql, id).QueryRow(&item)
+	return
+}
+
+func (m *BaseFromThsHfData) GetItemByCondition(condition string, pars []interface{}, orderRule string) (item *BaseFromThsHfData, err error) {
+	o := orm.NewOrm()
+	order := ``
+	if orderRule != "" {
+		order = ` ORDER BY ` + orderRule
+	}
+	sql := fmt.Sprintf(`SELECT * FROM %s WHERE 1=1 %s %s LIMIT 1`, m.TableName(), condition, order)
+	err = o.Raw(sql, pars).QueryRow(&item)
+	return
+}
+
+func (m *BaseFromThsHfData) GetCountByCondition(condition string, pars []interface{}) (count int, err error) {
+	o := orm.NewOrm()
+	sql := fmt.Sprintf(`SELECT COUNT(1) FROM %s WHERE 1=1 %s`, m.TableName(), condition)
+	err = o.Raw(sql, pars).QueryRow(&count)
+	return
+}
+
+func (m *BaseFromThsHfData) GetItemsByCondition(condition string, pars []interface{}, fieldArr []string, orderRule string) (items []*BaseFromThsHfData, err error) {
+	o := orm.NewOrm()
+	fields := strings.Join(fieldArr, ",")
+	if len(fieldArr) == 0 {
+		fields = `*`
+	}
+	order := fmt.Sprintf(`ORDER BY %s DESC`, m.Cols().CreateTime)
+	if orderRule != "" {
+		order = ` ORDER BY ` + orderRule
+	}
+	sql := fmt.Sprintf(`SELECT %s FROM %s WHERE 1=1 %s %s`, fields, m.TableName(), condition, order)
+	_, err = o.Raw(sql, pars).QueryRows(&items)
+	return
+}
+
+func (m *BaseFromThsHfData) GetPageItemsByCondition(condition string, pars []interface{}, fieldArr []string, orderRule string, startSize, pageSize int) (items []*BaseFromThsHfData, err error) {
+	o := orm.NewOrm()
+	fields := strings.Join(fieldArr, ",")
+	if len(fieldArr) == 0 {
+		fields = `*`
+	}
+	order := fmt.Sprintf(`ORDER BY %s DESC`, m.Cols().CreateTime)
+	if orderRule != "" {
+		order = ` ORDER BY ` + orderRule
+	}
+	sql := fmt.Sprintf(`SELECT %s FROM %s WHERE 1=1 %s %s LIMIT ?,?`, fields, m.TableName(), condition, order)
+	_, err = o.Raw(sql, pars, startSize, pageSize).QueryRows(&items)
+	return
+}
+
+// BaseFromThsHfDataItem 同花顺高频数据信息
+type BaseFromThsHfDataItem struct {
+	DataId     int     `description:"数据ID"`
+	IndexId    int     `description:"指标ID"`
+	IndexCode  string  `description:"指标编码"`
+	DataTime   string  `description:"数据日期（至时分秒）"`
+	Value      float64 `description:"数据值"`
+	UniqueCode string  `description:"唯一编码"`
+}
+
+func (m *BaseFromThsHfData) Format2Item() (item *BaseFromThsHfDataItem) {
+	item = new(BaseFromThsHfDataItem)
+	item.DataId = m.BaseFromThsHfDataId
+	item.IndexId = m.BaseFromThsHfIndexId
+	item.IndexCode = m.IndexCode
+	item.DataTime = utils.TimeTransferString(utils.FormatDateTime, m.DataTime)
+	item.Value = m.Value
+	item.UniqueCode = m.UniqueCode
+	return
+}
+
+func (m *BaseFromThsHfData) MultiInsertOrUpdate(inserts, updates []*BaseFromThsHfData) (err error) {
+	o := orm.NewOrm()
+	if len(inserts) > 0 {
+		_, e := o.InsertMulti(600, inserts)
+		if e != nil {
+			err = fmt.Errorf("insert multi err: %s", e.Error())
+			return
+		}
+	}
+	if len(updates) > 0 {
+		sql := fmt.Sprintf("UPDATE %s SET %s = ?, modify_time = NOW() WHERE %s = ?", m.TableName(), m.Cols().Value, m.Cols().UniqueCode)
+		p, e := o.Raw(sql).Prepare()
+		if e != nil {
+			err = fmt.Errorf("prepare err: %s", e.Error())
+			return
+		}
+		defer func() {
+			_ = p.Close()
+		}()
+		for _, v := range updates {
+			_, e = p.Exec(v.Value, v.UniqueCode)
+			if e != nil {
+				err = fmt.Errorf("update err: %s", e.Error())
+				return
+			}
+		}
+	}
+	return
+}
+
+func (m *BaseFromThsHfData) GetIndexMinMax(indexCode string) (item *EdbInfoMaxAndMinInfo, err error) {
+	o := orm.NewOrm()
+	sql := fmt.Sprintf(`SELECT MIN(%s) AS min_date, MAX(%s) AS max_date, MIN(%s) AS min_value,MAX(%s) AS max_value FROM %s WHERE %s = ?`, m.Cols().DataTime, m.Cols().DataTime, m.Cols().Value, m.Cols().Value, m.TableName(), m.Cols().IndexCode)
+	err = o.Raw(sql, indexCode).QueryRow(&item)
+	if err != nil {
+		return
+	}
+
+	// 获取最新值
+	var lastVal float64
+	sql = fmt.Sprintf(`SELECT %s AS latest_value FROM %s WHERE %s = ? ORDER BY %s DESC LIMIT 1`, m.Cols().Value, m.TableName(), m.Cols().IndexCode, m.Cols().DataTime)
+	err = o.Raw(sql, indexCode).QueryRow(&lastVal)
+	if err != nil {
+		return
+	}
+	item.LatestValue = lastVal
+	return
+}

models/business_conf.go

@@ -0,0 +1,238 @@
+package models
+
+import (
+	"eta/eta_index_lib/utils"
+	"fmt"
+	"github.com/beego/beego/v2/client/orm"
+	"html"
+	"strings"
+	"time"
+)
+
+const (
+	BusinessConfUseXf                     = "UseXf"
+	BusinessConfXfAppid                   = "XfAppid"
+	BusinessConfXfApiKey                  = "XfApiKey"
+	BusinessConfXfApiSecret               = "XfApiSecret"
+	BusinessConfXfVcn                     = "XfVcn"
+	BusinessConfEnPptCoverImgs            = "EnPptCoverImgs"
+	BusinessConfIsReportApprove           = "IsReportApprove"
+	BusinessConfReportApproveType         = "ReportApproveType"
+	BusinessConfCompanyName               = "CompanyName"
+	BusinessConfCompanyWatermark          = "CompanyWatermark"
+	BusinessConfWatermarkChart            = "WatermarkChart"
+	BusinessConfLoginSmsTpId              = "LoginSmsTpId"
+	BusinessConfLoginSmsGjTpId            = "LoginSmsGjTpId"
+	BusinessConfSmsJhgnAppKey             = "SmsJhgnAppKey"
+	BusinessConfSmsJhgjAppKey             = "SmsJhgjAppKey"
+	BusinessConfLdapHost                  = "LdapHost"
+	BusinessConfLdapBase                  = "LdapBase"
+	BusinessConfLdapPort                  = "LdapPort"
+	BusinessConfEmailClient               = "EmailClient"
+	BusinessConfEmailServerHost           = "EmailServerHost"
+	BusinessConfEmailServerPort           = "EmailServerPort"
+	BusinessConfEmailSender               = "EmailSender"
+	BusinessConfEmailSenderUserName       = "EmailSenderUserName"
+	BusinessConfEmailSenderPassword       = "EmailSenderPassword"
+	BusinessConfSmsClient                 = "SmsClient"
+	BusinessConfNanHuaSmsAppKey           = "NanHuaSmsAppKey"
+	BusinessConfNanHuaSmsAppSecret        = "NanHuaSmsAppSecret"
+	BusinessConfNanHuaSmsApiHost          = "NanHuaSmsApiHost"
+	BusinessConfLoginSmsTplContent        = "LoginSmsTplContent"
+	BusinessConfLoginEmailTemplateSubject = "LoginEmailTemplateSubject"
+	BusinessConfLoginEmailTemplateContent = "LoginEmailTemplateContent"
+	BusinessConfLdapBindUserSuffix        = "LdapBindUserSuffix"
+	BusinessConfLdapUserFilter            = "LdapUserFilter"
+
+	BusinessConfTencentApiSecretId           = "TencentApiSecretId"           // 腾讯云API-密钥对
+	BusinessConfTencentApiSecretKey          = "TencentApiSecretKey"          // 腾讯云API-密钥对
+	BusinessConfTencentApiRecTaskCallbackUrl = "TencentApiRecTaskCallbackUrl" // 腾讯云API-语音识别回调地址
+	BusinessConfSmsJhgjVariable              = "SmsJhgjVariable"              // 聚合国际短信变量
+)
+
+const (
+	BusinessConfReportApproveTypeEta   = "eta"
+	BusinessConfReportApproveTypeOther = "other"
+	BusinessConfClientFlagNanHua       = "nhqh" // 南华标记
+	BusinessConfEmailClientSmtp        = "smtp" // 普通邮箱标记
+)
+
+// FromSceneMap 数据源名称与数据源ID的对应关系
+var FromSceneMap = map[int]string{
+	1: "SmartReportSheetSize",
+	2: "ReportSheetSize",
+	3: "EnReportSheetSize",
+	4: "CnPptSheetSize",
+	5: "EnPptSheetSize",
+}
+
+// BusinessConf 商户配置表
+type BusinessConf struct {
+	Id         int    `orm:"column(id);pk"`
+	ConfKey    string `description:"配置Key"`
+	ConfVal    string `description:"配置值"`
+	ValType    int    `description:"1-字符串；2-数值；3-字符串数组；4-富文本；"`
+	Necessary  int    `description:"是否必填：0-否；1-是"`
+	Remark     string `description:"备注"`
+	CreateTime time.Time
+}
+
+func (m *BusinessConf) TableName() string {
+	return "business_conf"
+}
+
+func (m *BusinessConf) PrimaryId() string {
+	return "id"
+}
+
+func (m *BusinessConf) Create() (err error) {
+	o := orm.NewOrmUsingDB("master")
+	id, err := o.Insert(m)
+	if err != nil {
+		return
+	}
+	m.Id = int(id)
+	return
+}
+
+func (m *BusinessConf) CreateMulti(items []*BusinessConf) (err error) {
+	if len(items) == 0 {
+		return
+	}
+	o := orm.NewOrmUsingDB("master")
+	_, err = o.InsertMulti(len(items), items)
+	return
+}
+
+func (m *BusinessConf) Update(cols []string) (err error) {
+	o := orm.NewOrmUsingDB("master")
+	_, err = o.Update(m, cols...)
+	return
+}
+
+func (m *BusinessConf) Del() (err error) {
+	o := orm.NewOrmUsingDB("master")
+	sql := fmt.Sprintf(`DELETE FROM %s WHERE %s = ? LIMIT 1`, m.TableName(), m.PrimaryId())
+	_, err = o.Raw(sql, m.Id).Exec()
+	return
+}
+
+func (m *BusinessConf) GetItemById(id int) (item *BusinessConf, err error) {
+	o := orm.NewOrmUsingDB("master")
+	sql := fmt.Sprintf(`SELECT * FROM %s WHERE %s = ? LIMIT 1`, m.TableName(), m.PrimaryId())
+	err = o.Raw(sql, id).QueryRow(&item)
+	return
+}
+
+func (m *BusinessConf) GetItemByCondition(condition string, pars []interface{}) (item *BusinessConf, err error) {
+	o := orm.NewOrmUsingDB("master")
+	sql := fmt.Sprintf(`SELECT * FROM %s WHERE 1=1 %s LIMIT 1`, m.TableName(), condition)
+	err = o.Raw(sql, pars).QueryRow(&item)
+	return
+}
+
+func (m *BusinessConf) GetCountByCondition(condition string, pars []interface{}) (count int, err error) {
+	o := orm.NewOrmUsingDB("master")
+	sql := fmt.Sprintf(`SELECT COUNT(1) FROM %s WHERE 1=1 %s`, m.TableName(), condition)
+	err = o.Raw(sql, pars).QueryRow(&count)
+	return
+}
+
+func (m *BusinessConf) GetItemsByCondition(condition string, pars []interface{}, fieldArr []string, orderRule string) (items []*BusinessConf, err error) {
+	o := orm.NewOrmUsingDB("master")
+	fields := strings.Join(fieldArr, ",")
+	if len(fieldArr) == 0 {
+		fields = `*`
+	}
+	order := `ORDER BY create_time DESC`
+	if orderRule != "" {
+		order = ` ORDER BY ` + orderRule
+	}
+	sql := fmt.Sprintf(`SELECT %s FROM %s WHERE 1=1 %s %s`, fields, m.TableName(), condition, order)
+	_, err = o.Raw(sql, pars).QueryRows(&items)
+	return
+}
+
+func (m *BusinessConf) GetPageItemsByCondition(condition string, pars []interface{}, fieldArr []string, orderRule string, startSize, pageSize int) (items []*BusinessConf, err error) {
+	o := orm.NewOrmUsingDB("master")
+	fields := strings.Join(fieldArr, ",")
+	if len(fieldArr) == 0 {
+		fields = `*`
+	}
+	order := `ORDER BY create_time DESC`
+	if orderRule != "" {
+		order = ` ORDER BY ` + orderRule
+	}
+	sql := fmt.Sprintf(`SELECT %s FROM %s WHERE 1=1 %s %s LIMIT ?,?`, fields, m.TableName(), condition, order)
+	_, err = o.Raw(sql, pars, startSize, pageSize).QueryRows(&items)
+	return
+}
+
+// GetBusinessConf 获取商家配置
+func GetBusinessConf() (list map[string]string, err error) {
+	list = make(map[string]string)
+
+	var items []*BusinessConf
+	o := orm.NewOrmUsingDB("master")
+	sql := `SELECT * FROM business_conf`
+	_, err = o.Raw(sql).QueryRows(&items)
+	if err != nil {
+		return
+	}
+
+	for _, v := range items {
+		if v.ValType == 4 {
+			list[v.ConfKey] = html.UnescapeString(v.ConfVal)
+			continue
+		}
+		list[v.ConfKey] = v.ConfVal
+	}
+	return
+}
+
+// BusinessConfUpdate 更新配置
+type BusinessConfUpdate struct {
+	ConfKey string
+	ConfVal string
+}
+
+// UpdateBusinessConfMulti 批量修改配置
+func UpdateBusinessConfMulti(items []BusinessConfUpdate) (err error) {
+	o := orm.NewOrmUsingDB("master")
+	p, err := o.Raw("UPDATE business_conf SET conf_val = ? WHERE conf_key = ?").Prepare()
+	if err != nil {
+		return
+	}
+	defer func() {
+		_ = p.Close()
+	}()
+	for _, v := range items {
+		_, err = p.Exec(v.ConfVal, v.ConfKey)
+		if err != nil {
+			return
+		}
+	}
+	return
+}
+
+func GetBusinessConfByKey(key string) (item *BusinessConf, err error) {
+	o := orm.NewOrmUsingDB("master")
+	sql := fmt.Sprintf(`SELECT * FROM business_conf WHERE conf_key = ? LIMIT 1`)
+	err = o.Raw(sql, key).QueryRow(&item)
+	return
+}
+
+// InitUseMongoConf
+// @Description:
+// @author: Roc
+// @datetime 2024-07-01 13:49:09
+func InitUseMongoConf() {
+	useMongo, e := GetBusinessConfByKey("UseMongo")
+	if e != nil {
+		return
+	}
+
+	if useMongo.ConfVal == `true` {
+		utils.UseMongo = true
+	}
+}

models/db.go

@@ -34,6 +34,17 @@ func init() {
 	gl, _ := orm.GetDB("gl")
 	gl.SetConnMaxLifetime(10 * time.Minute)
 
+	// master库
+	{
+		_ = orm.RegisterDataBase("master", "mysql", utils.MYSQL_URL_MASTER)
+		orm.SetMaxIdleConns("master", 50)
+		orm.SetMaxOpenConns("master", 100)
+
+		master, _ := orm.GetDB("master")
+		master.SetConnMaxLifetime(10 * time.Minute)
+
+	}
+
 	orm.Debug = true
 	orm.DebugLog = orm.NewLog(utils.Binlog)
 
@@ -57,6 +68,7 @@ func init() {
 		new(EdbDataInsertConfig),
 		new(EdbAdjustConf), // 数据调整的配置
 		new(BaseFromMysteelChemicalClassify),
+		new(EdbInfoRelation), //指标引用记录
 	)
 
 	// 注册期货数据 数据表
@@ -84,7 +96,7 @@ func init() {
 	initFactorEdbSeries()
 
 	// 初始化部分数据表变量(直接init会有顺序问题=_=!)
-	InitEdbSource()
+	afterInitTable()
 }
 
 // initFutureGood 注册期货数据 数据表
@@ -143,6 +155,11 @@ func initBaseIndex() {
 		new(BaseFromCCFIndex),
 		new(BaseFromCCFData),
 		new(CCFStockExcel),
+		new(BaseFromThsHfIndex),
+		new(BaseFromThsHfData),
+		new(BaseFromEdbMapping),
+		new(EdbDataThsHf),
+		new(BaseFromBusinessData), // 数据源中自有数据的明细数据表
 	)
 }
 
@@ -191,3 +208,15 @@ func initFactorEdbSeries() {
 		new(FactorEdbSeriesCalculateData), // 因子指标系列-指标关联
 	)
 }
+
+// afterInitTable
+// @Description: 初始化表结构的的后置操作
+// @author: Roc
+// @datetime 2024-07-01 13:31:09
+func afterInitTable() {
+	// 初始化指标来源配置
+	InitEdbSource()
+
+	// 初始化是否启用mongo配置
+	InitUseMongoConf()
+}

models/edb_data_base.go

@@ -46,6 +46,7 @@ type AddEdbInfoReq struct {
 	Source    int    `description:"指标来源ID"`
 	StockCode string `description:"证券代码"`
 	Frequency string `description:"频度"`
+	ExtraPars string `description:"额外参数(如同花顺日期序列)"`
 }
 
 // GetEdbInfoCountByCondition 获取指标数量

models/edb_data_business.go

@@ -7,8 +7,8 @@ import (
 	"github.com/beego/beego/v2/client/orm"
 	"github.com/shopspring/decimal"
 	"go.mongodb.org/mongo-driver/bson"
-	"go.mongodb.org/mongo-driver/bson/primitive"
 	"reflect"
+	"strings"
 	"time"
 )
 
@@ -103,7 +103,7 @@ func (obj Business) Add(params AddBaseParams, businessIndexItem *BaseFromBusines
 	edbInfo.EdbInfoId = int(newEdbInfoId)
 
 	// 更新数据
-	err = obj.refresh(edbInfo, "")
+	err = obj.refresh(to, edbInfo, "")
 
 	return
 }
@@ -153,7 +153,7 @@ func (obj Business) Edit(params EditBaseParams, businessIndexItem *BaseFromBusin
 	}
 
 	//计算数据
-	err = obj.refresh(edbInfo, "")
+	err = obj.refresh(to, edbInfo, "")
 
 	return
 }
@@ -174,7 +174,7 @@ func (obj Business) Refresh(params RefreshBaseParams) (err error, errMsg string)
 	}()
 
 	// 计算数据
-	err = obj.refresh(params.EdbInfo, params.StartDate)
+	err = obj.refresh(to, params.EdbInfo, params.StartDate)
 
 	return
 }
@@ -194,7 +194,17 @@ func (obj Business) GetEdbType() int {
 	return utils.DEFAULT_EDB_TYPE
 }
 
-func (obj Business) refresh(edbInfo *EdbInfo, startDate string) (err error) {
+func (obj Business) refresh(to orm.TxOrmer, edbInfo *EdbInfo, startDate string) (err error) {
+	if utils.UseMongo {
+		return obj.refreshByMongo(edbInfo, startDate)
+	} else {
+		return obj.refreshByMysql(to, edbInfo, startDate)
+	}
+
+	return
+}
+
+func (obj Business) refreshByMongo(edbInfo *EdbInfo, startDate string) (err error) {
 	// 真实数据的最大日期  ， 插入规则配置的日期
 	var realDataMaxDate, edbDataInsertConfigDate time.Time
 	var edbDataInsertConfig *EdbDataInsertConfig
@@ -210,7 +220,7 @@ func (obj Business) refresh(edbInfo *EdbInfo, startDate string) (err error) {
 	}
 
 	//获取已存在的所有数据
-	baseDataList, err := obj.getBaseBusinessData(edbInfo, startDate)
+	baseDataList, err := obj.getBaseBusinessDataByMongo(edbInfo, startDate)
 
 	//获取指标所有数据
 	existDataList := make([]*mgo.EdbDataBusiness, 0)
@@ -352,8 +362,144 @@ func (obj Business) refresh(edbInfo *EdbInfo, startDate string) (err error) {
 	return
 }
 
+func (obj Business) refreshByMysql(to orm.TxOrmer, edbInfo *EdbInfo, startDate string) (err error) {
+	dataTableName := GetEdbDataTableName(edbInfo.Source, edbInfo.SubSource)
+	edbInfoIdStr := fmt.Sprint(edbInfo.EdbInfoId)
+
+	// 真实数据的最大日期  ， 插入规则配置的日期
+	var realDataMaxDate, edbDataInsertConfigDate time.Time
+	var edbDataInsertConfig *EdbDataInsertConfig
+	var isFindConfigDateRealData bool //是否找到配置日期的实际数据的值
+	{
+		edbDataInsertConfig, err = GetEdbDataInsertConfigByEdbId(edbInfo.EdbInfoId)
+		if err != nil && err.Error() != utils.ErrNoRow() {
+			return
+		}
+		if edbDataInsertConfig != nil {
+			edbDataInsertConfigDate = edbDataInsertConfig.Date
+		}
+	}
+
+	//获取已存在的所有数据
+	baseDataList, err := obj.getBaseBusinessDataByMysql(edbInfo, startDate)
+
+	//获取指标所有数据
+	var existCondition string
+	var existPars []interface{}
+
+	existCondition += " AND edb_info_id=? "
+	existPars = append(existPars, edbInfo.EdbInfoId)
+	if startDate != "" {
+		existCondition += " AND data_time>=? "
+		existPars = append(existPars, startDate)
+	}
+
+	existList, err := GetEdbDataByCondition(edbInfo.Source, edbInfo.SubSource, existCondition, existPars)
+	if err != nil {
+		fmt.Println(obj.GetSourceName() + ",refreshByMysql err;getEdbDataBusinessList Err:" + err.Error())
+		return err
+	}
+	existDataMap := make(map[string]*EdbInfoSearchData)
+	removeDataTimeMap := make(map[string]bool) //需要移除的日期数据
+	for _, v := range existList {
+		existDataMap[v.DataTime] = v
+		removeDataTimeMap[v.DataTime] = true
+	}
+	needAddDateMap := make(map[time.Time]int)
+
+	// 待添加的数据集
+	addSql := ` INSERT INTO ` + dataTableName + ` (edb_info_id,edb_code,data_time,value,create_time,modify_time,data_timestamp) values `
+	var isAdd bool
+	// 待更新的数据集
+	updateDataList := make([]*EdbInfoSearchData, 0)
+
+	for _, tmpData := range baseDataList {
+		currDate := tmpData.DataTime
+		currDateStr := currDate.Format(utils.FormatDate)
+
+		// 当前的实际值
+		saveValue := decimal.NewFromFloat(tmpData.Value).Round(4).String()
+
+		// 下面代码主要目的是处理掉手动插入的数据判断
+		{
+			if realDataMaxDate.IsZero() || currDate.After(realDataMaxDate) {
+				realDataMaxDate = currDate
+			}
+			if edbDataInsertConfigDate.IsZero() || currDate.Equal(edbDataInsertConfigDate) {
+				isFindConfigDateRealData = true
+			}
+		}
+
+		existData, ok := existDataMap[currDateStr]
+		// 如果库中已经存在该数据的话，那么就进行值的变更操作
+		if ok {
+			// 已经入到指标库的值
+			existValStr := decimal.NewFromFloat(existData.Value).Round(4).String()
+
+			//校验待删除日期数据里面是否存在该元素，如果存在的话，那么移除该日期
+			delete(removeDataTimeMap, currDateStr)
+			if existValStr != saveValue {
+				existData.Value = tmpData.Value
+				updateDataList = append(updateDataList, existData)
+			}
+
+			continue
+		}
+
+		// 库中不存在该日期的数据
+		timestamp := currDate.UnixNano() / 1e6
+		needAddDateMap[currDate] = 1
+		addSql += GetAddSql(edbInfoIdStr, edbInfo.EdbCode, currDateStr, fmt.Sprint(timestamp), saveValue)
+		isAdd = true
+	}
+
+	//删除已经不存在的指标数据（由于该指标当日的数据删除了）
+	{
+		removeDateList := make([]string, 0)
+		for dateTime := range removeDataTimeMap {
+			removeDateList = append(removeDateList, dateTime)
+		}
+		removeNum := len(removeDateList)
+		if removeNum > 0 {
+			//如果拼接指标变更了，那么需要删除所有的指标数据
+			sql := fmt.Sprintf(` DELETE FROM %s WHERE edb_info_id = ? and data_time in (`+utils.GetOrmInReplace(removeNum)+`) `, dataTableName)
+
+			_, err = to.Raw(sql, edbInfo.EdbInfoId, removeDateList).Exec()
+			if err != nil {
+				err = fmt.Errorf("删除自有数据的明细数据失败,Err:" + err.Error())
+				return
+			}
+		}
+	}
+
+	if isAdd {
+		addSql = strings.TrimRight(addSql, ",")
+		_, err = to.Raw(addSql).Exec()
+		if err != nil {
+			fmt.Println("RefreshAllCalculate add Err", err.Error())
+			return
+		}
+	}
+
+	// 修改历史数据
+	if len(updateDataList) > 0 {
+		for _, v := range updateDataList {
+			err = ModifyEdbDataById(edbInfo.Source, edbInfo.SubSource, v.EdbDataId, fmt.Sprint(v.Value))
+			if err != nil {
+				fmt.Println(obj.GetSourceName() + "，refreshByMysql:Err:" + err.Error())
+				return err
+			}
+		}
+	}
+
+	// 处理手工数据补充的配置
+	HandleConfigInsertEdbData(realDataMaxDate, edbDataInsertConfig, edbInfo.EdbInfoId, edbInfo.Source, edbInfo.SubSource, existDataMap, isFindConfigDateRealData)
+
+	return
+}
+
 // GetEdbInfoMaxAndMinInfo 获取指标的最新数据记录信息
-func (obj Business) GetEdbInfoMaxAndMinInfo(edbCode string) (item *EdbInfoMaxAndMinInfo, err error) {
+func (obj Business) getEdbInfoMaxAndMinInfoByMongo(edbCode string) (item *EdbInfoMaxAndMinInfo, err error) {
 	mogDataObj := new(mgo.EdbDataBusiness)
 	pipeline := []bson.M{
 		{"$match": bson.M{"edb_code": edbCode}},
@@ -405,21 +551,29 @@ func (obj Business) GetEdbInfoMaxAndMinInfo(edbCode string) (item *EdbInfoMaxAnd
 // @param edbInfo *EdbInfo
 // @return err error
 func (obj Business) UnifiedModifyEdbInfoMaxAndMinInfo(edbInfo *EdbInfo) (err error) {
-	edbInfoMaxAndMinInfo, err := obj.GetEdbInfoMaxAndMinInfo(edbInfo.EdbCode)
-	if err != nil {
-		return
+	if utils.UseMongo {
+		edbInfoMaxAndMinInfo, tmpErr := obj.getEdbInfoMaxAndMinInfoByMongo(edbInfo.EdbCode)
+		// 如果正常获取到了，那就去修改指标的最大最小值
+		if tmpErr == nil && edbInfoMaxAndMinInfo != nil {
+			err = ModifyEdbInfoMaxAndMinInfo(edbInfo.EdbInfoId, edbInfoMaxAndMinInfo)
+		} else {
+			// 清空的目的是为了避免异常返回
+			err = nil
+		}
+	} else {
+		err, _ = UnifiedModifyEdbInfoMaxAndMinInfo(edbInfo)
 	}
-	err = ModifyEdbInfoMaxAndMinInfo(edbInfo.EdbInfoId, edbInfoMaxAndMinInfo)
+
 	return
 }
 
 // EdbInfoMgoData
 // @Description: mgo里面的数据
 type EdbInfoMgoData struct {
-	EdbDataId primitive.ObjectID `description:"数据ID"`
-	DataTime  time.Time          `description:"数据日期"`
-	Value     float64            `description:"数据"`
-	EdbCode   string             `description:"指标编码"`
+	//EdbDataId primitive.ObjectID `description:"数据ID"`
+	DataTime time.Time `description:"数据日期"`
+	Value    float64   `description:"数据"`
+	EdbCode  string    `description:"指标编码"`
 }
 
 // getBaseBusinessData
@@ -432,6 +586,19 @@ type EdbInfoMgoData struct {
 // @return newDataList []EdbInfoSearchData
 // @return err error
 func (obj Business) getBaseBusinessData(edbInfo *EdbInfo, startDate string) (newDataList []EdbInfoMgoData, err error) {
+	return obj.getBaseBusinessDataByMongo(edbInfo, startDate)
+}
+
+// getBaseBusinessDataByMongo
+// @Description: 从mongo中获取基础的明细数据
+// @author: Roc
+// @receiver obj
+// @datetime 2024-07-02 10:12:02
+// @param edbInfo *EdbInfo
+// @param startDate string
+// @return newDataList []EdbInfoMgoData
+// @return err error
+func (obj Business) getBaseBusinessDataByMongo(edbInfo *EdbInfo, startDate string) (newDataList []EdbInfoMgoData, err error) {
 	newDataList = make([]EdbInfoMgoData, 0)
 
 	// 获取数据源的指标数据
@@ -460,10 +627,53 @@ func (obj Business) getBaseBusinessData(edbInfo *EdbInfo, startDate string) (new
 
 	for _, v := range baseDataList {
 		newDataList = append(newDataList, EdbInfoMgoData{
-			EdbDataId: v.ID,
-			DataTime:  v.DataTime,
-			Value:     v.Value,
-			EdbCode:   v.IndexCode,
+			//EdbDataId: v.ID,
+			DataTime: v.DataTime,
+			Value:    v.Value,
+			EdbCode:  v.IndexCode,
+		})
+	}
+
+	return
+}
+
+// getBaseBusinessDataByMysql
+// @Description: 从mysql中获取基础的明细数据
+// @author: Roc
+// @receiver obj
+// @datetime 2024-07-02 10:12:16
+// @param edbInfo *EdbInfo
+// @param startDate string
+// @return newDataList []EdbInfoMgoData
+// @return err error
+func (obj Business) getBaseBusinessDataByMysql(edbInfo *EdbInfo, startDate string) (newDataList []EdbInfoMgoData, err error) {
+	newDataList = make([]EdbInfoMgoData, 0)
+	// 获取数据源的指标数据
+	baseBusinessDataObj := new(BaseFromBusinessData)
+
+	// 构建查询条件
+	var condition []string
+	var pars []interface{}
+	condition = append(condition, "index_code = ? ")
+	pars = append(pars, edbInfo.EdbCode)
+
+	if startDate != `` {
+		condition = append(condition, " data_time >= ? ")
+		pars = append(pars, startDate)
+	}
+
+	baseDataList, err := baseBusinessDataObj.GetAllDataList(condition, pars, " data_time ASC ")
+	if err != nil {
+		fmt.Println("getBaseBusinessData Err:" + err.Error())
+		return
+	}
+
+	for _, v := range baseDataList {
+		newDataList = append(newDataList, EdbInfoMgoData{
+			//EdbDataId: v.BusinessDataId,
+			DataTime: v.DataTime,
+			Value:    v.Value,
+			EdbCode:  v.IndexCode,
 		})
 	}
 

models/edb_data_table.go

@@ -8,10 +8,12 @@ import (
 func GetEdbDataTableName(source, subSource int) (tableName string) {
 	switch source {
 	case utils.DATA_SOURCE_THS:
-		tableName = "edb_data_ths"
-		if subSource == utils.DATA_SUB_SOURCE_DATE {
+		switch subSource {
+		case utils.DATA_SUB_SOURCE_DATE:
 			tableName = "edb_data_ths_ds"
-		} else {
+		case utils.DATA_SUB_SOURCE_HIGH_FREQUENCY:
+			tableName = "edb_data_ths_hf"
+		default:
 			tableName = "edb_data_ths"
 		}
 	case utils.DATA_SOURCE_WIND:

models/edb_data_ths_hf.go

@@ -0,0 +1,181 @@
+package models
+
+import (
+	"eta/eta_index_lib/utils"
+	"fmt"
+	"github.com/beego/beego/v2/client/orm"
+	"strings"
+	"time"
+)
+
+// EdbDataThsHf 同花顺高频指标数据
+type EdbDataThsHf struct {
+	EdbDataId     int       `orm:"column(edb_data_id);pk"`
+	EdbInfoId     int       `description:"指标ID"`
+	EdbCode       string    `description:"指标编码"`
+	DataTime      time.Time `description:"数据日期"`
+	Value         float64   `description:"数据值"`
+	CreateTime    time.Time `description:"创建时间"`
+	ModifyTime    time.Time `description:"修改时间"`
+	DataTimestamp int64     `description:"数据日期时间戳"`
+}
+
+func (m *EdbDataThsHf) TableName() string {
+	return "edb_data_ths_hf"
+}
+
+type EdbDataThsHfCols struct {
+	PrimaryId     string
+	EdbInfoId     string
+	EdbCode       string
+	DataTime      string
+	Value         string
+	CreateTime    string
+	ModifyTime    string
+	DataTimestamp string
+}
+
+func (m *EdbDataThsHf) Cols() EdbDataThsHfCols {
+	return EdbDataThsHfCols{
+		PrimaryId:     "edb_data_id",
+		EdbInfoId:     "edb_info_id",
+		EdbCode:       "edb_code",
+		DataTime:      "data_time",
+		Value:         "value",
+		CreateTime:    "create_time",
+		ModifyTime:    "modify_time",
+		DataTimestamp: "data_timestamp",
+	}
+}
+
+func (m *EdbDataThsHf) Create() (err error) {
+	o := orm.NewOrm()
+	id, err := o.Insert(m)
+	if err != nil {
+		return
+	}
+	m.EdbDataId = int(id)
+	return
+}
+
+func (m *EdbDataThsHf) CreateMulti(items []*EdbDataThsHf) (err error) {
+	if len(items) == 0 {
+		return
+	}
+	o := orm.NewOrm()
+	_, err = o.InsertMulti(500, items)
+	return
+}
+
+func (m *EdbDataThsHf) Update(cols []string) (err error) {
+	o := orm.NewOrm()
+	_, err = o.Update(m, cols...)
+	return
+}
+
+func (m *EdbDataThsHf) Remove() (err error) {
+	o := orm.NewOrm()
+	sql := fmt.Sprintf(`DELETE FROM %s WHERE %s = ? LIMIT 1`, m.TableName(), m.Cols().PrimaryId)
+	_, err = o.Raw(sql, m.EdbDataId).Exec()
+	return
+}
+
+func (m *EdbDataThsHf) MultiRemove(ids []int) (err error) {
+	if len(ids) == 0 {
+		return
+	}
+	o := orm.NewOrm()
+	sql := fmt.Sprintf(`DELETE FROM %s WHERE %s IN (%s)`, m.TableName(), m.Cols().PrimaryId, utils.GetOrmInReplace(len(ids)))
+	_, err = o.Raw(sql, ids).Exec()
+	return
+}
+
+func (m *EdbDataThsHf) RemoveByCondition(condition string, pars []interface{}) (err error) {
+	if condition == "" {
+		return
+	}
+	o := orm.NewOrm()
+	sql := fmt.Sprintf(`DELETE FROM %s WHERE %s`, m.TableName(), condition)
+	_, err = o.Raw(sql, pars).Exec()
+	return
+}
+
+func (m *EdbDataThsHf) GetItemById(id int) (item *EdbDataThsHf, err error) {
+	o := orm.NewOrm()
+	sql := fmt.Sprintf(`SELECT * FROM %s WHERE %s = ? LIMIT 1`, m.TableName(), m.Cols().PrimaryId)
+	err = o.Raw(sql, id).QueryRow(&item)
+	return
+}
+
+func (m *EdbDataThsHf) GetItemByCondition(condition string, pars []interface{}, orderRule string) (item *EdbDataThsHf, err error) {
+	o := orm.NewOrm()
+	order := ``
+	if orderRule != "" {
+		order = ` ORDER BY ` + orderRule
+	}
+	sql := fmt.Sprintf(`SELECT * FROM %s WHERE 1=1 %s %s LIMIT 1`, m.TableName(), condition, order)
+	err = o.Raw(sql, pars).QueryRow(&item)
+	return
+}
+
+func (m *EdbDataThsHf) GetCountByCondition(condition string, pars []interface{}) (count int, err error) {
+	o := orm.NewOrm()
+	sql := fmt.Sprintf(`SELECT COUNT(1) FROM %s WHERE 1=1 %s`, m.TableName(), condition)
+	err = o.Raw(sql, pars).QueryRow(&count)
+	return
+}
+
+func (m *EdbDataThsHf) GetItemsByCondition(condition string, pars []interface{}, fieldArr []string, orderRule string) (items []*EdbDataThsHf, err error) {
+	o := orm.NewOrm()
+	fields := strings.Join(fieldArr, ",")
+	if len(fieldArr) == 0 {
+		fields = `*`
+	}
+	order := fmt.Sprintf(`ORDER BY %s DESC`, m.Cols().CreateTime)
+	if orderRule != "" {
+		order = ` ORDER BY ` + orderRule
+	}
+	sql := fmt.Sprintf(`SELECT %s FROM %s WHERE 1=1 %s %s`, fields, m.TableName(), condition, order)
+	_, err = o.Raw(sql, pars).QueryRows(&items)
+	return
+}
+
+func (m *EdbDataThsHf) GetPageItemsByCondition(condition string, pars []interface{}, fieldArr []string, orderRule string, startSize, pageSize int) (items []*EdbDataThsHf, err error) {
+	o := orm.NewOrm()
+	fields := strings.Join(fieldArr, ",")
+	if len(fieldArr) == 0 {
+		fields = `*`
+	}
+	order := fmt.Sprintf(`ORDER BY %s DESC`, m.Cols().CreateTime)
+	if orderRule != "" {
+		order = ` ORDER BY ` + orderRule
+	}
+	sql := fmt.Sprintf(`SELECT %s FROM %s WHERE 1=1 %s %s LIMIT ?,?`, fields, m.TableName(), condition, order)
+	_, err = o.Raw(sql, pars, startSize, pageSize).QueryRows(&items)
+	return
+}
+
+// MultiUpdateValue 批量更新数据值
+func (m *EdbDataThsHf) MultiUpdateValue(items []*EdbDataThsHf) (err error) {
+	if len(items) == 0 {
+		return
+	}
+
+	o := orm.NewOrm()
+	sql := fmt.Sprintf("UPDATE %s SET %s = ?, %s = NOW() WHERE %s = ?", m.TableName(), m.Cols().Value, m.Cols().ModifyTime, m.Cols().PrimaryId)
+	p, e := o.Raw(sql).Prepare()
+	if e != nil {
+		err = fmt.Errorf("update sql prepare err: %v", e)
+		return
+	}
+	defer func() {
+		_ = p.Close()
+	}()
+	for _, v := range items {
+		_, err = p.Exec(v.Value, v.EdbDataId)
+		if err != nil {
+			return
+		}
+	}
+	return
+}

models/edb_info.go

@@ -209,7 +209,7 @@ type FindEdbDataListAllCond struct {
 // @return item []*EdbInfoSearchData
 // @return err error
 func GetEdbDataListAll(source, subSource int, findEdbDataListAllCond FindEdbDataListAllCond, order int) (item []*EdbInfoSearchData, err error) {
-	if source == utils.DATA_SOURCE_BUSINESS {
+	if source == utils.DATA_SOURCE_BUSINESS && utils.UseMongo {
 		return GetEdbDataListAllByMongo(source, subSource, findEdbDataListAllCond, order)
 	}
 
@@ -277,7 +277,7 @@ func GetEdbDataListAllByMysql(source, subSource int, findEdbDataListAllCond Find
 // @return item []*EdbInfoSearchData
 // @return err error
 func GetEdbDataListAllByTo(to orm.TxOrmer, source, subSource int, findEdbDataListAllCond FindEdbDataListAllCond, order int) (item []*EdbInfoSearchData, err error) {
-	if source == utils.DATA_SOURCE_BUSINESS {
+	if source == utils.DATA_SOURCE_BUSINESS && utils.UseMongo {
 		return GetEdbDataListAllByMongo(source, subSource, findEdbDataListAllCond, order)
 	}
 
@@ -462,7 +462,7 @@ func ModifyEdbDataUpdateTime(edbInfoId int, dataUpdateTime, erDataUpdateDate str
 // @return count int
 // @return err error
 func GetLteZeroEdbDataCount(source, subSource, edbInfoId int) (count int, err error) {
-	if source == utils.DATA_SOURCE_BUSINESS {
+	if source == utils.DATA_SOURCE_BUSINESS && utils.UseMongo {
 		return GetLteZeroEdbDataCountByMongo(source, subSource, edbInfoId)
 	}
 
@@ -1512,6 +1512,23 @@ func GetEdbInfoByEdbCodeList(source int, edbCodeList []string) (items []*EdbInfo
 	return
 }
 
+// EdbInfoExtra 指标额外数据-extra字段
+type EdbInfoExtra struct {
+	ApiExtraPars string `description:"API-额外参数(如同花顺日期序列)"`
+}
+
+// GetEdbInfoNoUpdateTotalByIdList 根据指标id列表获取指标信息
+func GetEdbInfoNoUpdateTotalByIdList(edbInfoIdList []int) (total int, err error) {
+	num := len(edbInfoIdList)
+	if num <= 0 {
+		return
+	}
+	o := orm.NewOrm()
+	sql := ` SELECT count(*) FROM edb_info WHERE edb_info_id in (` + utils.GetOrmInReplace(num) + `)  and no_update=1`
+	err = o.Raw(sql, edbInfoIdList).QueryRow(&total)
+	return
+}
+
 func TransEdbInfoSearchData2DataList(items []*EdbInfoSearchData) (list []*EdbDataList) {
 	if len(items) == 0 {
 		return

models/edb_info_calculate_mapping.go

@@ -1,6 +1,7 @@
 package models
 
 import (
+	"eta/eta_index_lib/utils"
 	"github.com/beego/beego/v2/client/orm"
 	"time"
 )
@@ -25,6 +26,31 @@ type EdbInfoCalculateMapping struct {
 	FromSubSource             int       `description:"渠道子数据库来源"`
 }
 
+// EdbInfoCalculateMappingInfo
+// @Description: 计算指标与基础指标关系表
+type EdbInfoCalculateMappingInfo struct {
+	EdbInfoCalculateMappingId int       `orm:"column(edb_info_calculate_mapping_id);pk"`
+	EdbInfoId                 int       `description:"计算指标id"`
+	Source                    int       `description:"计算指标来源"`
+	SourceName                string    `description:"计算指标来源名称"`
+	EdbCode                   string    `description:"计算指标编码"`
+	FromEdbInfoId             int       `description:"基础指标id"`
+	FromEdbCode               string    `description:"基础指标编码"`
+	FromEdbName               string    `description:"基础指标名称"`
+	FromSource                int       `description:"基础指标来源"`
+	FromSourceName            string    `description:"基础指标来源名称"`
+	MoveValue                 int       `description:"领先值"`
+	FromTag                   string    `description:"来源指标标签"`
+	Sort                      int       `description:"计算指标名称排序"`
+	CreateTime                time.Time `description:"创建时间"`
+	ModifyTime                time.Time `description:"修改时间"`
+	FromEdbType               int       `description:"来源指标类型:1:基础指标,2:计算指标"`
+	FromEdbInfoType           int       `description:"来源指标类型: 0-基础指标; 1-预测指标"`
+	FromClassifyId            int       `description:"来源指标分类ID"`
+	FromUniqueCode            string    `description:"来源指标唯一编码"`
+	NoUpdate                  int8      `description:"是否停止更新，0：继续更新；1：停止更新"`
+}
+
 // AddEdbInfoCalculateMappingMulti 批量添加指标关系表
 func AddEdbInfoCalculateMappingMulti(items []*EdbInfoCalculateMapping) (err error) {
 	o := orm.NewOrm()
@@ -119,3 +145,23 @@ func GetEdbInfoCalculateByEdbCode(edbCode, fromEdbCode string) (item *EdbInfoCal
 	err = o.Raw(sql, edbCode, fromEdbCode).QueryRow(&item)
 	return
 }
+
+// GetEdbInfoCalculateMappingListByEdbInfoIds 根据生成的指标id获取来源的指标id列表
+func GetEdbInfoCalculateMappingListByEdbInfoIds(edbInfoIds []int) (items []*EdbInfoCalculateMappingInfo, err error) {
+	o := orm.NewOrm()
+	sql := ` SELECT a.*,b.edb_type as from_edb_type,b.edb_info_type as from_edb_info_type, b.unique_code AS from_unique_code, b.classify_id AS from_classify_id,b.no_update FROM edb_info_calculate_mapping AS a
+			INNER JOIN edb_info AS b ON a.from_edb_info_id=b.edb_info_id
+			WHERE a.edb_info_id in (` + utils.GetOrmInReplace(len(edbInfoIds)) + `) `
+	_, err = o.Raw(sql, edbInfoIds).QueryRows(&items)
+	return
+}
+
+// GetEdbInfoCalculateMappingListByEdbInfoId 根据生成的指标id获取来源的指标id列表
+func GetEdbInfoCalculateMappingListByEdbInfoId(edbInfoId int) (items []*EdbInfoCalculateMappingInfo, err error) {
+	o := orm.NewOrm()
+	sql := ` SELECT a.*,b.edb_type as from_edb_type,b.edb_info_type as from_edb_info_type, b.unique_code AS from_unique_code, b.classify_id AS from_classify_id,b.no_update FROM edb_info_calculate_mapping AS a
+			INNER JOIN edb_info AS b ON a.from_edb_info_id=b.edb_info_id
+			WHERE a.edb_info_id=? `
+	_, err = o.Raw(sql, edbInfoId).QueryRows(&items)
+	return
+}

models/edb_info_relation.go

@@ -0,0 +1,136 @@
+package models
+
+import (
+	"eta/eta_index_lib/utils"
+	"github.com/beego/beego/v2/client/orm"
+	"time"
+)
+
+type EdbInfoRelation struct {
+	EdbInfoRelationId  int       `orm:"column(edb_info_relation_id);pk"`
+	EdbInfoId          int       `description:"指标id"`
+	Source             int       `description:"来源:1:同花顺,2:wind,3:彭博,4:指标运算,5:累计值转月,6:同比值,7:同差值,8:N数值移动平均计算,9:手工指标,10:隆众"`
+	EdbName            string    `description:"指标名称"`
+	EdbCode            string    `description:"指标编码"`
+	ReferObjectId      int       `description:"引用对象ID（图表ID，ETA逻辑ID等）"`
+	ReferObjectType    int       `description:"引用对象ID类型（1.图表，2.ETA逻辑）"`
+	ReferObjectSubType int       `description:"引用对象子类"`
+	CreateTime         time.Time `description:"创建时间"`
+	ModifyTime         time.Time `description:"修改时间"`
+	RelationTime       time.Time `description:"引用时间"`
+	RelationType       int       `description:"引用类型，0:直接饮用，1间接引用"`
+	RootEdbInfoId      int       `description:"间接引用时，关联的直接引用的指标ID"`
+	ChildEdbInfoId     int       `description:"间接引用时，计算指标直接关联的指标ID"`
+	RelationCode       string    `description:"引用标识"`
+	ParentRelationId   int       `description:"间接引用关联的直接引用的ID"`
+}
+
+func (e *EdbInfoRelation) TableName() string {
+	return "edb_info_relation"
+}
+
+// GetEdbInfoRelationByChildEdbInfoId 查询引用的指标ID
+func GetEdbInfoRelationByChildEdbInfoId(edbInfoId int) (item *EdbInfoRelation, err error) {
+	o := orm.NewOrm()
+	msql := ` SELECT * FROM edb_info_relation WHERE child_edb_info_id = ? or edb_info_id=?`
+	err = o.Raw(msql, edbInfoId, edbInfoId).QueryRow(&item)
+	return
+}
+
+// GetEdbInfoRelationListByChildEdbInfoId 根据间接引用中的的计算指标ID查询引用列表
+func GetEdbInfoRelationListByChildEdbInfoId(edbInfoId int) (items []*EdbInfoRelation, err error) {
+	o := orm.NewOrm()
+	msql := ` SELECT * FROM edb_info_relation WHERE relation_type=1 AND child_edb_info_id=?`
+	_, err = o.Raw(msql, edbInfoId).QueryRows(&items)
+	return
+}
+
+// GetEdbInfoRelationListByParentRelationId 根据间接引用中的的父级ID查询
+func GetEdbInfoRelationEdbIdsByParentRelationId(relationId, edbInfoId int) (items []int, err error) {
+	o := orm.NewOrm()
+	msql := ` SELECT edb_info_id FROM edb_info_relation WHERE parent_relation_id=? AND child_edb_info_id=?`
+	_, err = o.Raw(msql, relationId, edbInfoId).QueryRows(&items)
+	return
+}
+
+// GetEdbInfoRelationByRelationIds 查询引用的指标ID
+func GetEdbInfoRelationByRelationIds(ids []int) (items []*EdbInfoRelation, err error) {
+	o := orm.NewOrm()
+	msql := ` SELECT * FROM edb_info_relation WHERE edb_info_relation_id in (` + utils.GetOrmInReplace(len(ids)) + `) `
+	_, err = o.Raw(msql, ids).QueryRows(&items)
+	return
+}
+
+// UpdateSecondRelationEdbInfoId 更新指标替换后的间接引用记录
+func UpdateSecondRelationEdbInfoId(edbRelationIds []int, relationList []*EdbInfoRelation, refreshEdbInfoIds []int, indexCodeList []string) (err error) {
+	o, err := orm.NewOrm().Begin()
+	if err != nil {
+		return
+	}
+	defer func() {
+		if err != nil {
+			_ = o.Rollback()
+			return
+		}
+		_ = o.Commit()
+	}()
+	// 删除相关的间接引用
+	sql := ` DELETE FROM edb_info_relation WHERE relation_type=1 and parent_relation_id in (` + utils.GetOrmInReplace(len(edbRelationIds)) + `)`
+	_, err = o.Raw(sql, edbRelationIds).Exec()
+	if err != nil {
+		return
+	}
+
+	// 新增间接引用
+	relationCodesMap := make(map[string]struct{}, 0)
+	if len(relationList) > 0 {
+		for _, relation := range relationList {
+			if relation.RelationType == 1 {
+				relationCodesMap[relation.RelationCode] = struct{}{}
+			}
+		}
+		_, err = o.InsertMulti(len(relationList), relationList)
+		if err != nil {
+			return
+		}
+	}
+
+	if len(refreshEdbInfoIds) > 0 {
+		// todo 更新指标的刷新状态
+		sql = ` UPDATE edb_info SET no_update = 0 WHERE  edb_info_id IN (` + utils.GetOrmInReplace(len(refreshEdbInfoIds)) + `) AND no_update = 1`
+		_, err = o.Raw(sql, refreshEdbInfoIds).Exec()
+		if err != nil {
+			return
+		}
+	}
+
+	//更新数据源钢联化工指标
+	if len(indexCodeList) > 0 {
+		// 更改数据源的更新状态
+		sql = ` UPDATE base_from_mysteel_chemical_index SET is_stop = 0 WHERE index_code IN (` + utils.GetOrmInReplace(len(indexCodeList)) + `) and is_stop=1`
+		_, err = o.Raw(sql, indexCodeList).Exec()
+		if err != nil {
+			return
+		}
+	}
+	if len(relationList) > 0 {
+		// 更新间接引用指标的关联ID
+		relationCodes := make([]string, 0)
+		for relationCode := range relationCodesMap {
+			relationCodes = append(relationCodes, relationCode)
+		}
+		if len(relationCodes) > 0 {
+			sql := ` UPDATE edb_info_relation e1  
+JOIN edb_info_relation e2 ON e1.relation_code = e2.relation_code   
+SET e1.parent_relation_id = e2.edb_info_relation_id  
+WHERE  
+    e1.relation_type = 1   
+    AND e2.relation_type = 0 AND e1.parent_relation_id !=e2.edb_info_relation_id AND e1.relation_code in (` + utils.GetOrmInReplace(len(relationCodes)) + `)`
+			_, err = o.Raw(sql, relationCodes).Exec()
+			if err != nil {
+				return
+			}
+		}
+	}
+	return
+}

models/edb_ths_hf.go

@@ -0,0 +1,924 @@
+package models
+
+import (
+	"encoding/json"
+	"eta/eta_index_lib/models/mgo"
+	"eta/eta_index_lib/utils"
+	"fmt"
+	"github.com/beego/beego/v2/client/orm"
+	"github.com/shopspring/decimal"
+	"go.mongodb.org/mongo-driver/bson"
+	"reflect"
+	"sort"
+	"time"
+)
+
+// EdbThsHf 自有数据
+type EdbThsHf struct{}
+
+// GetSource 获取来源编码id
+func (obj EdbThsHf) GetSource() int {
+	return utils.DATA_SOURCE_THS
+}
+
+// GetSubSource 获取子来源编码id
+func (obj EdbThsHf) GetSubSource() int {
+	return utils.DATA_SUB_SOURCE_HIGH_FREQUENCY
+}
+
+// GetSourceName 获取来源名称
+func (obj EdbThsHf) GetSourceName() string {
+	return utils.DATA_SOURCE_NAME_THS
+}
+
+// GetEdbType 获取指标类型
+func (obj EdbThsHf) GetEdbType() int {
+	return utils.DEFAULT_EDB_TYPE
+}
+
+// ThsHfAddBaseParams
+// @Description: 基础指标的添加参数
+type ThsHfAddBaseParams struct {
+	EdbCode         string `description:"指标编码"`
+	EdbName         string `description:"指标名称"`
+	Unit            string `description:"单位"`
+	Frequency       string `description:"频度"`
+	Sort            int    `description:"排序"`
+	ClassifyId      int    `description:"所属分类"`
+	SysUserId       int    `description:"用户id"`
+	SysUserRealName string `description:"用户真实名称"`
+	UniqueCode      string `description:"唯一编码"`
+	ConvertRule     string `description:"转换规则"`
+}
+
+// ThsHfEditBaseParams
+// @Description: 基础指标的修改参数
+type ThsHfEditBaseParams struct {
+	EdbCode         string   `description:"指标编码"`
+	EdbName         string   `description:"指标名称"`
+	EdbNameEn       string   `description:"指标名称（英文）"`
+	Unit            string   `description:"单位"`
+	UnitEn          string   `description:"单位（英文）"`
+	ClassifyId      int      `description:"所属分类"`
+	SysUserId       int      `description:"用户id"`
+	SysUserRealName string   `description:"用户真实名称"`
+	UniqueCode      string   `description:"编码"`
+	Lang            string   `description:"语言版本"`
+	EdbInfo         *EdbInfo `description:"指标信息"`
+}
+
+type ThsHfRefreshBaseParams struct {
+	EdbInfo   *EdbInfo
+	StartDate string
+	EndDate   string
+}
+
+// Add
+// @Description: 添加指标
+func (obj EdbThsHf) Add(params ThsHfAddBaseParams, baseIndex *BaseFromThsHfIndex) (edbInfo *EdbInfo, err error) {
+	o := orm.NewOrm()
+	tx, e := o.Begin()
+	if e != nil {
+		err = fmt.Errorf("orm begin err: %v", e)
+		return
+	}
+	defer func() {
+		if err != nil {
+			_ = tx.Rollback()
+			utils.FileLog.Info(fmt.Sprintf("%s err: %v", reflect.TypeOf(obj).Name(), err))
+			return
+		}
+		_ = tx.Commit()
+	}()
+
+	// 新增指标
+	edbInfo = new(EdbInfo)
+	edbInfo.Source = obj.GetSource()
+	edbInfo.SubSource = obj.GetSubSource()
+	edbInfo.SourceName = obj.GetSourceName()
+	edbInfo.EdbType = obj.GetEdbType()
+	edbInfo.EdbCode = params.EdbCode
+	edbInfo.EdbName = params.EdbName
+	edbInfo.EdbNameEn = params.EdbName
+	edbInfo.EdbNameSource = params.EdbName
+	edbInfo.Frequency = params.Frequency
+	edbInfo.Unit = params.Unit
+	edbInfo.UnitEn = params.Unit
+	edbInfo.StartDate = baseIndex.StartDate.Format(utils.FormatDate) // 默认取源指标的时间, 刷新完成后更新
+	edbInfo.EndDate = baseIndex.EndDate.Format(utils.FormatDate)
+	edbInfo.ClassifyId = params.ClassifyId
+	edbInfo.SysUserId = params.SysUserId
+	edbInfo.SysUserRealName = params.SysUserRealName
+	edbInfo.Sort = params.Sort
+	edbInfo.TerminalCode = baseIndex.TerminalCode
+	edbInfo.UniqueCode = params.UniqueCode
+	edbInfo.CreateTime = time.Now()
+	edbInfo.ModifyTime = time.Now()
+	edbInfoId, e := tx.Insert(edbInfo)
+	if e != nil {
+		err = fmt.Errorf("insert edb err: %v", e)
+		return
+	}
+	edbInfo.EdbInfoId = int(edbInfoId)
+
+	// 新增指标关联
+	edbMapping := new(BaseFromEdbMapping)
+	edbMapping.BaseFromIndexId = baseIndex.BaseFromThsHfIndexId
+	edbMapping.BaseIndexCode = baseIndex.IndexCode
+	edbMapping.EdbInfoId = edbInfo.EdbInfoId
+	edbMapping.EdbCode = edbInfo.EdbCode
+	edbMapping.Source = obj.GetSource()
+	edbMapping.SubSource = obj.GetSubSource()
+	edbMapping.ConvertRule = params.ConvertRule
+	edbMapping.CreateTime = time.Now().Local()
+	edbMapping.ModifyTime = time.Now().Local()
+	edbMappingId, e := tx.Insert(edbMapping)
+	if e != nil {
+		err = fmt.Errorf("insert base edb mapping err: %v", e)
+		return
+	}
+	edbMapping.Id = int(edbMappingId)
+
+	// 刷新数据
+	err = obj.Refresh(edbInfo, edbMapping, "")
+	return
+}
+
+func (obj EdbThsHf) Refresh(edbInfo *EdbInfo, edbBaseMapping *BaseFromEdbMapping, startDate string) (err error) {
+	if utils.UseMongo {
+		return obj.refreshByMongo(edbInfo, edbBaseMapping, startDate)
+	}
+	return obj.refreshByMysql(edbInfo, edbBaseMapping, startDate)
+}
+
+func (obj EdbThsHf) refreshByMysql(edbInfo *EdbInfo, edbBaseMapping *BaseFromEdbMapping, startDate string) (err error) {
+	if edbInfo == nil || edbBaseMapping == nil {
+		err = fmt.Errorf("指标信息/关联信息有误, EdbInfo: %v, EdbBaseMapping: %v", edbInfo, edbBaseMapping)
+		return
+	}
+
+	// 真实数据的最大日期, 插入规则配置的日期
+	var realDataMaxDate, edbDataInsertConfigDate time.Time
+	var edbDataInsertConfig *EdbDataInsertConfig
+	var isFindConfigDateRealData bool
+	{
+		conf, e := GetEdbDataInsertConfigByEdbId(edbInfo.EdbInfoId)
+		if e != nil && e.Error() != utils.ErrNoRow() {
+			err = fmt.Errorf("GetEdbDataInsertConfigByEdbId err: %v", e)
+			return
+		}
+		edbDataInsertConfig = conf
+		if edbDataInsertConfig != nil {
+			edbDataInsertConfigDate = edbDataInsertConfig.Date
+		}
+	}
+
+	// 查询时间为开始时间-3d
+	var queryDate string
+	if startDate != "" {
+		st, e := time.ParseInLocation(utils.FormatDate, startDate, time.Local)
+		if e != nil {
+			err = fmt.Errorf("刷新开始时间有误, %v", e)
+			return
+		}
+		queryDate = st.AddDate(0, 0, -3).Format(utils.FormatDate)
+	}
+
+	// 源指标数据
+	baseDataList := make([]*BaseFromThsHfData, 0)
+	{
+		ob := new(BaseFromThsHfData)
+		cond := fmt.Sprintf(" AND %s = ?", ob.Cols().IndexCode)
+		pars := make([]interface{}, 0)
+		pars = append(pars, edbBaseMapping.BaseIndexCode)
+		if queryDate != "" {
+			cond += fmt.Sprintf(" AND %s >= ?", ob.Cols().DataTime)
+			pars = append(pars, queryDate)
+		}
+		list, e := ob.GetItemsByCondition(cond, pars, []string{}, fmt.Sprintf("%s ASC", ob.Cols().DataTime))
+		if e != nil {
+			err = fmt.Errorf("获取数据源数据失败, %v", e)
+			return
+		}
+		baseDataList = list
+	}
+
+	// 转换数据
+	convertRule := new(ThsHfIndexConvert2EdbRule)
+	if e := json.Unmarshal([]byte(edbBaseMapping.ConvertRule), &convertRule); e != nil {
+		err = fmt.Errorf("转换规则有误, %v", e)
+		return
+	}
+	convertOriginData := make([]*ThsHfConvertOriginData, 0)
+	for _, v := range baseDataList {
+		convertOriginData = append(convertOriginData, &ThsHfConvertOriginData{
+			DataTime: v.DataTime,
+			Value:    v.Value,
+		})
+	}
+	convertData, e := ThsHfConvertData2DayByRule(convertOriginData, convertRule)
+	if e != nil {
+		err = fmt.Errorf("转换数据失败, %v", e)
+		return
+	}
+	if len(convertData) == 0 {
+		utils.FileLog.Info("同花顺高频-转换无数据, EdbCode: %s", edbInfo.EdbCode)
+		return
+	}
+
+	// 获取已有数据
+	dataOb := new(EdbDataThsHf)
+	dataExists := make(map[string]*EdbDataThsHf)
+	searchExistMap := make(map[string]*EdbInfoSearchData)
+	{
+		cond := fmt.Sprintf(" AND %s = ?", dataOb.Cols().EdbInfoId)
+		pars := make([]interface{}, 0)
+		pars = append(pars, edbInfo.EdbInfoId)
+		if queryDate != "" {
+			cond += fmt.Sprintf(" AND %s >= ?", dataOb.Cols().DataTime)
+			pars = append(pars, queryDate)
+		}
+		list, e := dataOb.GetItemsByCondition(cond, pars, []string{}, "")
+		if e != nil {
+			err = fmt.Errorf("获取指标数据失败, %v", e)
+			return
+		}
+		for _, v := range list {
+			dataExists[v.DataTime.Format(utils.FormatDate)] = v
+			searchExistMap[v.DataTime.Format(utils.FormatDate)] = &EdbInfoSearchData{
+				EdbDataId:     v.EdbDataId,
+				EdbInfoId:     v.EdbInfoId,
+				DataTime:      v.DataTime.Format(utils.FormatDate),
+				Value:         v.Value,
+				EdbCode:       v.EdbCode,
+				DataTimestamp: v.DataTimestamp,
+			}
+		}
+	}
+
+	// 比对数据
+	insertExist := make(map[string]bool)
+	insertData := make([]*EdbDataThsHf, 0)
+	updateData := make([]*EdbDataThsHf, 0)
+	for k, v := range convertData {
+		strDate := k.Format(utils.FormatDate)
+
+		// 手动插入数据的判断
+		if realDataMaxDate.IsZero() || k.After(realDataMaxDate) {
+			realDataMaxDate = k
+		}
+		if edbDataInsertConfigDate.IsZero() || k.Equal(edbDataInsertConfigDate) {
+			isFindConfigDateRealData = true
+		}
+
+		// 入库值
+		saveVal := decimal.NewFromFloat(v).Round(4).String()
+		d, e := decimal.NewFromString(saveVal)
+		if e != nil {
+			utils.FileLog.Info(fmt.Sprintf("EdbDataThsHf NewFromString err: %v", e))
+			continue
+		}
+		saveFloat, _ := d.Float64()
+
+		// 更新
+		exists := dataExists[strDate]
+		if exists != nil {
+			existVal := decimal.NewFromFloat(exists.Value).Round(4).String()
+			if saveVal != existVal {
+				exists.Value = saveFloat
+				updateData = append(updateData, exists)
+			}
+			continue
+		}
+
+		// 新增
+		if insertExist[strDate] {
+			continue
+		}
+		insertExist[strDate] = true
+
+		timestamp := k.UnixNano() / 1e6
+		insertData = append(insertData, &EdbDataThsHf{
+			EdbInfoId:     edbInfo.EdbInfoId,
+			EdbCode:       edbInfo.EdbCode,
+			DataTime:      k,
+			Value:         saveFloat,
+			CreateTime:    time.Now(),
+			ModifyTime:    time.Now(),
+			DataTimestamp: timestamp,
+		})
+	}
+
+	// 批量新增/更新
+	if len(insertData) > 0 {
+		if e = dataOb.CreateMulti(insertData); e != nil {
+			err = fmt.Errorf("批量新增指标数据失败, %v", e)
+			return
+		}
+	}
+	if len(updateData) > 0 {
+		if e = dataOb.MultiUpdateValue(updateData); e != nil {
+			err = fmt.Errorf("批量更新指标数据失败, %v", e)
+			return
+		}
+	}
+
+	// 处理手工数据补充的配置
+	HandleConfigInsertEdbData(realDataMaxDate, edbDataInsertConfig, edbInfo.EdbInfoId, obj.GetSource(), obj.GetSubSource(), searchExistMap, isFindConfigDateRealData)
+	return
+}
+
+func (obj EdbThsHf) refreshByMongo(edbInfo *EdbInfo, edbBaseMapping *BaseFromEdbMapping, startDate string) (err error) {
+	defer func() {
+		if err != nil {
+			utils.FileLog.Info(fmt.Sprintf("refreshByMongo, err: %v", err))
+		}
+	}()
+
+	var realDataMaxDate, edbDataInsertConfigDate time.Time
+	var edbDataInsertConfig *EdbDataInsertConfig
+	var isFindConfigDateRealData bool //是否找到配置日期的实际数据的值
+	{
+		insertConfig, e := GetEdbDataInsertConfigByEdbId(edbInfo.EdbInfoId)
+		if e != nil && e.Error() != utils.ErrNoRow() {
+			err = fmt.Errorf("GetEdbDataInsertConfigByEdbId, err: %v", e)
+			return
+		}
+		edbDataInsertConfig = insertConfig
+		if edbDataInsertConfig != nil {
+			edbDataInsertConfigDate = edbDataInsertConfig.Date
+		}
+	}
+
+	// 查询时间为开始时间-3d
+	var queryDate string
+	if startDate != "" {
+		st, e := time.ParseInLocation(utils.FormatDate, startDate, time.Local)
+		if e != nil {
+			err = fmt.Errorf("刷新开始时间有误, %v", e)
+			return
+		}
+		queryDate = st.AddDate(0, 0, -3).Format(utils.FormatDate)
+	}
+
+	// 获取源指标数据
+	baseDataList, e := obj.getBaseIndexDataByMongo(edbBaseMapping.BaseIndexCode, queryDate)
+	if e != nil {
+		err = fmt.Errorf("getBaseIndexDataByMongo, err: %v", e)
+		return
+	}
+
+	// 转换数据
+	convertRule := new(ThsHfIndexConvert2EdbRule)
+	if e := json.Unmarshal([]byte(edbBaseMapping.ConvertRule), &convertRule); e != nil {
+		err = fmt.Errorf("转换规则有误, %v", e)
+		return
+	}
+	convertOriginData := make([]*ThsHfConvertOriginData, 0)
+	for _, v := range baseDataList {
+		convertOriginData = append(convertOriginData, &ThsHfConvertOriginData{
+			DataTime: v.DataTime,
+			Value:    v.Value,
+		})
+	}
+	convertData, e := ThsHfConvertData2DayByRule(convertOriginData, convertRule)
+	if e != nil {
+		err = fmt.Errorf("转换数据失败, %v", e)
+		return
+	}
+	if len(convertData) == 0 {
+		utils.FileLog.Info("同花顺高频-转换无数据, EdbCode: %s", edbInfo.EdbCode)
+		return
+	}
+
+	//获取指标所有数据
+	existDataList := make([]*mgo.EdbDataThsHf, 0)
+	mogDataObj := new(mgo.EdbDataThsHf)
+	{
+		// 构建查询条件
+		queryConditions := bson.M{
+			"edb_code": edbInfo.EdbCode,
+		}
+
+		if queryDate != `` {
+			//获取已存在的所有数据
+			startDateTime, e := time.ParseInLocation(utils.FormatDate, queryDate, time.Local)
+			if e != nil {
+				err = fmt.Errorf("startDateTime parse err: %v", e)
+				return
+			}
+			queryConditions["data_time"] = bson.M{"$gte": startDateTime}
+		}
+		existDataList, e = mogDataObj.GetAllDataList(queryConditions, []string{"data_time"})
+		if e != nil {
+			err = fmt.Errorf("GetAllDataList, err: %v", e)
+			return
+		}
+	}
+
+	existDataMap := make(map[string]*mgo.EdbDataThsHf)
+	removeDataTimeMap := make(map[string]bool) //需要移除的日期数据
+	for _, v := range existDataList {
+		tmpDate := v.DataTime.Format(utils.FormatDate)
+		existDataMap[tmpDate] = v
+		removeDataTimeMap[tmpDate] = true
+	}
+
+	// 待添加的数据集
+	addDataList := make([]interface{}, 0)
+	updateDataList := make([]mgo.EdbDataThsHf, 0)
+
+	insertExist := make(map[string]bool)
+	for k, v := range convertData {
+		strDate := k.Format(utils.FormatDate)
+
+		// 手动插入数据的判断
+		if realDataMaxDate.IsZero() || k.After(realDataMaxDate) {
+			realDataMaxDate = k
+		}
+		if edbDataInsertConfigDate.IsZero() || k.Equal(edbDataInsertConfigDate) {
+			isFindConfigDateRealData = true
+		}
+
+		// 入库值
+		saveVal := decimal.NewFromFloat(v).Round(4).String()
+		d, e := decimal.NewFromString(saveVal)
+		if e != nil {
+			utils.FileLog.Info(fmt.Sprintf("EdbDataThsHf NewFromString err: %v", e))
+			continue
+		}
+		saveFloat, _ := d.Float64()
+
+		// 更新
+		exists := existDataMap[strDate]
+		if exists != nil {
+			existVal := decimal.NewFromFloat(exists.Value).Round(4).String()
+			if saveVal != existVal {
+				exists.Value = saveFloat
+				updateDataList = append(updateDataList, *exists)
+			}
+			continue
+		}
+
+		// 新增
+		if insertExist[strDate] {
+			continue
+		}
+		insertExist[strDate] = true
+
+		timestamp := k.UnixNano() / 1e6
+		addDataList = append(addDataList, &mgo.EdbDataThsHf{
+			EdbInfoId:     edbInfo.EdbInfoId,
+			EdbCode:       edbInfo.EdbCode,
+			DataTime:      k,
+			Value:         saveFloat,
+			CreateTime:    time.Now(),
+			ModifyTime:    time.Now(),
+			DataTimestamp: timestamp,
+		})
+	}
+
+	// 入库
+	{
+		coll := mogDataObj.GetCollection()
+
+		//删除已经不存在的指标数据（由于该指标当日的数据删除了）
+		{
+			removeDateList := make([]time.Time, 0)
+			for dateTime := range removeDataTimeMap {
+				//获取已存在的所有数据
+				tmpDateTime, e := time.ParseInLocation(utils.FormatDate, dateTime, time.Local)
+				if e != nil {
+					err = fmt.Errorf("tmpDateTime parse err: %v", e)
+					return
+				}
+				removeDateList = append(removeDateList, tmpDateTime)
+			}
+			removeNum := len(removeDateList)
+			if removeNum > 0 {
+				if e = mogDataObj.RemoveManyByColl(coll, bson.M{"edb_code": edbInfo.EdbCode, "data_time": bson.M{"$in": removeDateList}}); e != nil {
+					err = fmt.Errorf("RemoveManyByColl, err: %v", e)
+					return
+				}
+			}
+		}
+
+		// 插入新数据
+		if len(addDataList) > 0 {
+			if e = mogDataObj.BatchInsertDataByColl(coll, 500, addDataList); e != nil {
+				err = fmt.Errorf("BatchInsertDataByColl, err: %v", e)
+				return
+			}
+		}
+
+		// 修改历史数据
+		if len(updateDataList) > 0 {
+			for _, v := range updateDataList {
+				if e = mogDataObj.UpdateDataByColl(coll, bson.M{"_id": v.ID}, bson.M{"$set": bson.M{"value": v.Value, "modify_time": v.ModifyTime}}); e != nil {
+					err = fmt.Errorf("UpdateDataByColl, err: %v", e)
+					return
+				}
+			}
+		}
+	}
+
+	// 处理手工数据补充的配置
+	obj.HandleConfigInsertEdbDataByMongo(realDataMaxDate, edbDataInsertConfig, edbInfo.EdbInfoId, edbInfo.Source, edbInfo.SubSource, existDataMap, isFindConfigDateRealData)
+	return
+}
+
+type ThsHfConvertOriginData struct {
+	DataTime time.Time `description:"数据日期（至时分秒）"`
+	Value    float64   `description:"数据值"`
+}
+
+// ThsHfConvertData2DayByRule 原指标数据转换为日度数据
+func ThsHfConvertData2DayByRule(originData []*ThsHfConvertOriginData, convertRule *ThsHfIndexConvert2EdbRule) (timeData map[time.Time]float64, err error) {
+	// PS: originData为期望开始日期前三日(有两天非交易日, 那么周一的前日应当算上周五的)至结束日期的数据
+	timeData = make(map[time.Time]float64)
+	if len(originData) == 0 || convertRule == nil {
+		return
+	}
+	if !utils.InArrayByInt([]int{1, 2}, convertRule.ConvertType) {
+		err = fmt.Errorf("取值类型有误, ConvertType: %d", convertRule.ConvertType)
+		return
+	}
+
+	// 升序排序
+	sort.Slice(originData, func(i, j int) bool {
+		return originData[i].DataTime.Before(originData[j].DataTime)
+	})
+
+	// 将数据根据日期进行分组
+	var sortDates []string
+	groupDateData := make(map[string][]*ThsHfConvertOriginData)
+	for _, v := range originData {
+		d := v.DataTime.Format(utils.FormatDate)
+		if !utils.InArrayByStr(sortDates, d) {
+			sortDates = append(sortDates, d)
+		}
+		if groupDateData[d] == nil {
+			groupDateData[d] = make([]*ThsHfConvertOriginData, 0)
+		}
+		groupDateData[d] = append(groupDateData[d], v)
+	}
+
+	// 取值方式-指定时间的值
+	if convertRule.ConvertType == 1 {
+		for k, v := range sortDates {
+			todayTime, e := time.ParseInLocation(utils.FormatDate, v, time.Local)
+			if e != nil {
+				utils.FileLog.Info("当日日期转换有误, date: %s, err: %v", v, e)
+				continue
+			}
+			var timeTarget time.Time
+			dateData := make([]*ThsHfConvertOriginData, 0)
+
+			// 当日
+			if convertRule.ConvertFixed.FixedDay == 1 {
+				tg, e := time.ParseInLocation(utils.FormatDateTime, fmt.Sprintf("%s %s", v, convertRule.ConvertFixed.FixedTime), time.Local)
+				if e != nil {
+					utils.FileLog.Info(fmt.Sprintf("当日timeTarget转换有误, %v", e))
+					continue
+				}
+				timeTarget = tg
+
+				dt := groupDateData[v]
+				if dt == nil {
+					utils.FileLog.Info(fmt.Sprintf("%s当日无数据", v))
+					continue
+				}
+				if len(dt) == 0 {
+					continue
+				}
+				dateData = dt
+			}
+
+			// 前一日
+			if convertRule.ConvertFixed.FixedDay == 2 {
+				if k < 1 {
+					utils.FileLog.Info(fmt.Sprintf("%s前日无数据", v))
+					continue
+				}
+				preDate := sortDates[k-1]
+
+				tg, e := time.ParseInLocation(utils.FormatDateTime, fmt.Sprintf("%s %s", preDate, convertRule.ConvertFixed.FixedTime), time.Local)
+				if e != nil {
+					utils.FileLog.Info(fmt.Sprintf("前日timeTarget转换有误, %v", e))
+					continue
+				}
+				timeTarget = tg
+
+				dt := groupDateData[preDate]
+				if dt == nil {
+					utils.FileLog.Info(fmt.Sprintf("%s前日无数据", v))
+					continue
+				}
+				if len(dt) == 0 {
+					continue
+				}
+				dateData = dt
+			}
+			if len(dateData) == 0 {
+				utils.FileLog.Info("日期%s无数据序列", v)
+				continue
+			}
+
+			// 重新获取数据序列中, 时间在目标时间点之后的
+			newDateData := make([]*ThsHfConvertOriginData, 0)
+			for kv, dv := range dateData {
+				if dv.DataTime.Before(timeTarget) {
+					continue
+				}
+				// 由于升序排列, 直接取之后所有的数据
+				newDateData = append(newDateData, dateData[kv:]...)
+				break
+			}
+
+			// 取重组后当日数据中的第一个(有可能目标时间点无值, 那么取之后时间最近的值)
+			if len(newDateData) == 0 {
+				utils.FileLog.Info("日期%s无有效数据", v)
+				continue
+			}
+			timeData[todayTime] = newDateData[0].Value
+		}
+		return
+	}
+
+	// 取值方式-区间计算值
+	for k, v := range sortDates {
+		todayTime, e := time.ParseInLocation(utils.FormatDate, v, time.Local)
+		if e != nil {
+			utils.FileLog.Info("当日日期转换有误, date: %s, err: %v", v, e)
+			continue
+		}
+		var thisDate, preDate string
+		thisDate = v
+		if k > 1 {
+			preDate = sortDates[k-1]
+		}
+		var startTimeTarget, endTimeTarget time.Time
+
+		// 起始时间-当日/前一日
+		if convertRule.ConvertArea.StartDay == 1 {
+			tg, e := time.ParseInLocation(utils.FormatDateTime, fmt.Sprintf("%s %s", thisDate, convertRule.ConvertArea.StartTime), time.Local)
+			if e != nil {
+				utils.FileLog.Info(fmt.Sprintf("当日startTimeTarget转换有误, %v", e))
+				continue
+			}
+			startTimeTarget = tg
+		}
+		if convertRule.ConvertArea.StartDay == 2 {
+			if preDate == "" {
+				utils.FileLog.Info(fmt.Sprintf("%s前日无数据", v))
+				continue
+			}
+			tg, e := time.ParseInLocation(utils.FormatDateTime, fmt.Sprintf("%s %s", preDate, convertRule.ConvertArea.StartTime), time.Local)
+			if e != nil {
+				utils.FileLog.Info(fmt.Sprintf("前日startTimeTarget转换有误, %v", e))
+				continue
+			}
+			startTimeTarget = tg
+		}
+
+		// 截止时间-当日/前一日
+		if convertRule.ConvertArea.EndDay == 1 {
+			tg, e := time.ParseInLocation(utils.FormatDateTime, fmt.Sprintf("%s %s", thisDate, convertRule.ConvertArea.EndTime), time.Local)
+			if e != nil {
+				utils.FileLog.Info(fmt.Sprintf("当日endTimeTarget转换有误, %v", e))
+				continue
+			}
+			endTimeTarget = tg
+		}
+		if convertRule.ConvertArea.EndDay == 2 {
+			if preDate == "" {
+				utils.FileLog.Info(fmt.Sprintf("%s前日无数据", v))
+				continue
+			}
+			tg, e := time.ParseInLocation(utils.FormatDateTime, fmt.Sprintf("%s %s", preDate, convertRule.ConvertArea.EndTime), time.Local)
+			if e != nil {
+				utils.FileLog.Info(fmt.Sprintf("前日endTimeTarget转换有误, %v", e))
+				continue
+			}
+			endTimeTarget = tg
+		}
+		if startTimeTarget.IsZero() || endTimeTarget.IsZero() {
+			utils.FileLog.Info(fmt.Sprintf("起始截止时间有误, start: %v, end: %v", startTimeTarget, endTimeTarget))
+			continue
+		}
+
+		// 合并前日当日数据
+		dateData := make([]*ThsHfConvertOriginData, 0)
+		if convertRule.ConvertArea.StartDay == 1 && convertRule.ConvertArea.EndDay == 1 {
+			// 起始截止均为当日
+			dateData = groupDateData[thisDate]
+			if dateData == nil {
+				utils.FileLog.Info(fmt.Sprintf("%s当日无数据", thisDate))
+				continue
+			}
+			if len(dateData) == 0 {
+				utils.FileLog.Info(fmt.Sprintf("%s当日无数据", thisDate))
+				continue
+			}
+		} else {
+			if preDate == "" {
+				continue
+			}
+			// 起始截止时间含前日
+			preData := groupDateData[preDate]
+			if preData == nil {
+				utils.FileLog.Info(fmt.Sprintf("%s前日无数据", thisDate))
+				continue
+			}
+			if len(preData) == 0 {
+				utils.FileLog.Info(fmt.Sprintf("%s前日无数据", thisDate))
+				continue
+			}
+			thisData := groupDateData[thisDate]
+			if thisData == nil {
+				utils.FileLog.Info(fmt.Sprintf("%s当日无数据", thisDate))
+				continue
+			}
+			if len(thisData) == 0 {
+				utils.FileLog.Info(fmt.Sprintf("%s当日无数据", thisDate))
+				continue
+			}
+			dateData = append(dateData, preData...)
+			dateData = append(dateData, thisData...)
+		}
+		if len(dateData) == 0 {
+			utils.FileLog.Info("日期%s无数据序列", v)
+			continue
+		}
+
+		// 重组时间区间内的数据
+		newDateData := make([]*ThsHfConvertOriginData, 0)
+		for _, dv := range dateData {
+			if dv.DataTime.Before(startTimeTarget) || dv.DataTime.After(endTimeTarget) {
+				continue
+			}
+			newDateData = append(newDateData, dv)
+		}
+		if len(newDateData) == 0 {
+			utils.FileLog.Info(fmt.Sprintf("时间区间内无数据, start: %v, end: %v", startTimeTarget, endTimeTarget))
+			continue
+		}
+
+		// 取出区间内的均值/最值
+		var avgVal, minVal, maxVal, sumVal float64
+		minVal, maxVal = newDateData[0].Value, newDateData[0].Value
+		for _, nv := range newDateData {
+			sumVal += nv.Value
+			if nv.Value > maxVal {
+				maxVal = nv.Value
+			}
+			if nv.Value < minVal {
+				minVal = nv.Value
+			}
+		}
+		avgVal = sumVal / float64(len(newDateData))
+
+		switch convertRule.ConvertArea.CalculateType {
+		case 1:
+			timeData[todayTime] = avgVal
+		case 2:
+			timeData[todayTime] = maxVal
+		case 3:
+			timeData[todayTime] = minVal
+		default:
+			utils.FileLog.Info(fmt.Sprintf("计算方式有误, CalculateType: %d", convertRule.ConvertArea.CalculateType))
+		}
+	}
+	return
+}
+
+func (obj EdbThsHf) getBaseIndexDataByMongo(baseIndexCode, startDate string) (newDataList []EdbInfoMgoData, err error) {
+	newDataList = make([]EdbInfoMgoData, 0)
+
+	// 获取数据源的指标数据
+	mogDataObj := new(mgo.BaseFromThsHfData)
+
+	// 构建查询条件
+	queryConditions := bson.M{
+		"index_code": baseIndexCode,
+	}
+
+	if startDate != `` {
+		//获取已存在的所有数据
+		startDateTime, tmpErr := time.ParseInLocation(utils.FormatDate, startDate, time.Local)
+		if tmpErr != nil {
+			err = tmpErr
+			return
+		}
+		queryConditions["data_time"] = bson.M{"$gte": startDateTime}
+	}
+
+	baseDataList, err := mogDataObj.GetAllDataList(queryConditions, []string{"data_time"})
+	if err != nil {
+		fmt.Println("GetAllDataList Err:" + err.Error())
+		return
+	}
+
+	for _, v := range baseDataList {
+		newDataList = append(newDataList, EdbInfoMgoData{
+			//EdbDataId: v.ID,
+			DataTime: v.DataTime,
+			Value:    v.Value,
+			EdbCode:  v.IndexCode,
+		})
+	}
+	return
+}
+
+func (obj EdbThsHf) HandleConfigInsertEdbDataByMongo(realDataMaxDate time.Time, edbDataInsertConfig *EdbDataInsertConfig, edbInfoId, source, subSource int, existMap map[string]*mgo.EdbDataThsHf, isFindConfigDateRealData bool) {
+	if edbDataInsertConfig == nil {
+		return
+	}
+	var err error
+	defer func() {
+		if err != nil {
+			utils.FileLog.Info(fmt.Sprintf("ThsHf-HandleConfigInsertEdbDataByMongo, err: %v", err))
+		}
+	}()
+
+	edbDataInsertConfigDate := edbDataInsertConfig.Date // 配置的日期
+
+	// 如果存在真实数据的最大日期  && 存在配置插入数据的最大日期  && 真实数据的最大日期 晚于/等于 配置插入数据的最大日期
+	if realDataMaxDate.After(edbDataInsertConfigDate) || realDataMaxDate.Equal(edbDataInsertConfigDate) {
+		go DeleteEdbDataInsertConfigByEdbId(edbInfoId)
+
+		mogDataObj := mgo.EdbDataThsHf{}
+		coll := mogDataObj.GetCollection()
+		edbDataInsertConfigDateStr := edbDataInsertConfigDate.Format(utils.FormatDate)
+		// 如果没有找到找到配置日期的实际数据，那么就直接删除
+		if item, ok := existMap[edbDataInsertConfigDateStr]; ok && !isFindConfigDateRealData {
+			mogDataObj.RemoveManyByColl(coll, bson.M{"_id": item.ID})
+		}
+	} else {
+		o := orm.NewOrm()
+		edbDataInsertConfig.RealDate = realDataMaxDate
+		_, err = o.Update(edbDataInsertConfig, "RealDate")
+	}
+	return
+}
+
+func (obj EdbThsHf) UnifiedModifyEdbInfoMaxAndMinInfo(edbInfo *EdbInfo) (err error) {
+	if utils.UseMongo {
+		edbInfoMaxAndMinInfo, tmpErr := obj.getEdbInfoMaxAndMinInfoByMongo(edbInfo.EdbCode)
+		// 如果正常获取到了，那就去修改指标的最大最小值
+		if tmpErr == nil && edbInfoMaxAndMinInfo != nil {
+			err = ModifyEdbInfoMaxAndMinInfo(edbInfo.EdbInfoId, edbInfoMaxAndMinInfo)
+		} else {
+			// 清空的目的是为了避免异常返回
+			err = nil
+		}
+	} else {
+		err, _ = UnifiedModifyEdbInfoMaxAndMinInfo(edbInfo)
+	}
+
+	return
+}
+
+func (obj EdbThsHf) getEdbInfoMaxAndMinInfoByMongo(edbCode string) (item *EdbInfoMaxAndMinInfo, err error) {
+	mogDataObj := new(mgo.EdbDataThsHf)
+	pipeline := []bson.M{
+		{"$match": bson.M{"edb_code": edbCode}},
+		{"$group": bson.M{
+			"_id":       nil,
+			"min_date":  bson.M{"$min": "$data_time"},
+			"max_date":  bson.M{"$max": "$data_time"},
+			"min_value": bson.M{"$min": "$value"},
+			"max_value": bson.M{"$max": "$value"},
+		}},
+		{"$project": bson.M{"_id": 0}}, // 可选，如果不需要_id字段
+	}
+	result, err := mogDataObj.GetEdbInfoMaxAndMinInfo(pipeline)
+	if err != nil {
+		fmt.Println("EdbDataBusiness getEdbDataBusinessList Err:" + err.Error())
+		return
+	}
+
+	if !result.MaxDate.IsZero() {
+		whereQuery := bson.M{"edb_code": edbCode, "data_time": result.MaxDate}
+		selectParam := bson.D{{"value", 1}, {"_id", 0}}
+		latestValue, tmpErr := mogDataObj.GetLatestValue(whereQuery, selectParam)
+		if tmpErr != nil {
+			err = tmpErr
+			return
+		}
+		result.LatestValue = latestValue.Value
+		result.EndValue = latestValue.Value
+	}
+
+	item = &EdbInfoMaxAndMinInfo{
+		MinDate:     result.MinDate.Format(utils.FormatDate),
+		MaxDate:     result.MaxDate.Format(utils.FormatDate),
+		MinValue:    result.MinValue,
+		MaxValue:    result.MaxValue,
+		LatestValue: result.LatestValue,
+		LatestDate:  result.LatestDate.Format(utils.FormatDate),
+		EndValue:    result.EndValue,
+	}
+
+	return
+}

models/mgo/base_from_ths_hf_data.go

@@ -0,0 +1,406 @@
+package mgo
+
+import (
+	"context"
+	"errors"
+	"eta/eta_index_lib/utils"
+	"fmt"
+	"github.com/qiniu/qmgo"
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+	"time"
+)
+
+// BaseFromThsHfData
+// @Description: 同花顺高频集合
+type BaseFromThsHfData struct {
+	ID                   primitive.ObjectID `json:"_id" bson:"_id,omitempty"`                                   // 文档id
+	BaseFromThsHfDataId  int64              `json:"base_from_ths_hf_data_id" bson:"base_from_ths_hf_data_id"`   // 指标数据ID
+	BaseFromThsHfIndexId int64              `json:"base_from_ths_hf_index_id" bson:"base_from_ths_hf_index_id"` // 指标ID
+	IndexCode            string             `json:"index_code" bson:"index_code"`                               // 指标编码
+	DataTime             time.Time          `json:"data_time" bson:"data_time"`                                 // 数据日期
+	Value                float64            `json:"value" bson:"value"`                                         // 数据值
+	UniqueCode           string             `json:"unique_code" bson:"unique_code"`                             // 唯一编码
+	CreateTime           time.Time          `json:"create_time" bson:"create_time"`                             // 创建时间
+	ModifyTime           time.Time          `json:"modify_time" bson:"modify_time"`                             // 修改时间
+	DataTimestamp        int64              `json:"data_timestamp" bson:"data_timestamp"`                       // 数据日期时间戳
+}
+
+// CollectionName
+// @Description:  获取集合名称
+func (m *BaseFromThsHfData) CollectionName() string {
+	return "base_from_ths_hf_data"
+}
+
+// DataBaseName
+// @Description: 获取数据库名称
+func (m *BaseFromThsHfData) DataBaseName() string {
+	return utils.MgoDataDbName
+}
+
+// GetCollection
+// @Description: 获取mongodb集合的句柄
+func (m *BaseFromThsHfData) GetCollection() *qmgo.Collection {
+	db := utils.MgoDataCli.Database(m.DataBaseName())
+	return db.Collection(m.CollectionName())
+}
+
+// GetAllDataList
+// @Description: 根据条件获取所有数据
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-26 13:42:19
+// @param sort []string
+// @param whereParams interface{}
+// @return result []BaseFromThsHfData
+// @return err error
+func (m *BaseFromThsHfData) GetAllDataList(whereParams interface{}, sort []string) (result []*BaseFromThsHfData, err error) {
+	if utils.MgoDataCli == nil {
+		err = errors.New("mongodb连接失败")
+		return
+	}
+	db := utils.MgoDataCli.Database(m.DataBaseName())
+	coll := db.Collection(m.CollectionName())
+	ctx := context.TODO()
+	if err != nil {
+		fmt.Println("MgoGetColl Err:", err.Error())
+		return
+	}
+	err = coll.Find(ctx, whereParams).Sort(sort...).All(&result)
+	if err != nil {
+		return
+	}
+
+	for _, v := range result {
+		v.DataTime = v.DataTime.In(time.Local)
+		v.CreateTime = v.CreateTime.In(time.Local)
+		v.ModifyTime = v.ModifyTime.In(time.Local)
+	}
+	return
+}
+
+// GetLimitDataList
+// @Description: 根据条件获取指定数量数据列表
+// @author: Roc
+// @receiver m
+// @datetime 2024-05-06 17:08:32
+// @param whereParams interface{}
+// @param size int64
+// @return result []*BaseFromThsHfData
+// @return err error
+func (m *BaseFromThsHfData) GetLimitDataList(whereParams interface{}, size int64, sort []string) (result []*BaseFromThsHfData, err error) {
+	if utils.MgoDataCli == nil {
+		err = errors.New("mongodb连接失败")
+		return
+	}
+	db := utils.MgoDataCli.Database(m.DataBaseName())
+	coll := db.Collection(m.CollectionName())
+	ctx := context.TODO()
+	if err != nil {
+		fmt.Println("MgoGetColl Err:", err.Error())
+		return
+	}
+	err = coll.Find(ctx, whereParams).Sort(sort...).Limit(size).All(&result)
+	if err != nil {
+		return
+	}
+
+	for _, v := range result {
+		v.DataTime = v.DataTime.In(time.Local)
+		v.CreateTime = v.CreateTime.In(time.Local)
+		v.ModifyTime = v.ModifyTime.In(time.Local)
+	}
+
+	return
+}
+
+// GetPageDataList
+// @Description: 根据条件获取分页数据列表
+// @author: Roc
+// @receiver m
+// @datetime 2024-05-07 10:21:07
+// @param whereParams interface{}
+// @param startSize int64
+// @param size int64
+// @param sort []string
+// @return result []*BaseFromThsHfData
+// @return err error
+func (m *BaseFromThsHfData) GetPageDataList(whereParams interface{}, startSize, size int64, sort []string) (result []*BaseFromThsHfData, err error) {
+	if utils.MgoDataCli == nil {
+		err = errors.New("mongodb连接失败")
+		return
+	}
+	db := utils.MgoDataCli.Database(m.DataBaseName())
+	coll := db.Collection(m.CollectionName())
+	ctx := context.TODO()
+	if err != nil {
+		fmt.Println("MgoGetColl Err:", err.Error())
+		return
+	}
+	err = coll.Find(ctx, whereParams).Sort(sort...).Skip(startSize).Limit(size).All(&result)
+	if err != nil {
+		return
+	}
+
+	for _, v := range result {
+		v.DataTime = v.DataTime.In(time.Local)
+		v.CreateTime = v.CreateTime.In(time.Local)
+		v.ModifyTime = v.ModifyTime.In(time.Local)
+	}
+
+	return
+}
+
+// GetCountDataList
+// @Description:  根据条件获取数据列表总数
+// @author: Roc
+// @receiver m
+// @datetime 2024-05-07 10:29:00
+// @param whereParams interface{}
+// @return count int64
+// @return err error
+func (m *BaseFromThsHfData) GetCountDataList(whereParams interface{}) (count int64, err error) {
+	if utils.MgoDataCli == nil {
+		err = errors.New("mongodb连接失败")
+		return
+	}
+	db := utils.MgoDataCli.Database(m.DataBaseName())
+	coll := db.Collection(m.CollectionName())
+	ctx := context.TODO()
+	if err != nil {
+		fmt.Println("MgoGetColl Err:", err.Error())
+		return
+	}
+	count, err = coll.Find(ctx, whereParams).Count()
+
+	return
+}
+
+// InsertDataByColl
+// @Description: 写入单条数据（外部传入集合）
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-26 14:22:18
+// @param addData interface{}
+// @return err error
+func (m *BaseFromThsHfData) InsertDataByColl(coll *qmgo.Collection, addData interface{}) (err error) {
+	ctx := context.TODO()
+	_, err = coll.InsertOne(ctx, addData)
+	if err != nil {
+		fmt.Println("InsertDataByColl:Err:" + err.Error())
+		return
+	}
+
+	return
+}
+
+// BatchInsertData
+// @Description: 批量写入数据
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-26 14:22:18
+// @param bulk int 每次请求保存的数据量
+// @param dataList []interface{}
+// @return err error
+func (m *BaseFromThsHfData) BatchInsertData(bulk int, dataList []interface{}) (err error) {
+	db := utils.MgoDataCli.Database(m.DataBaseName())
+	coll := db.Collection(m.CollectionName())
+
+	return m.BatchInsertDataByColl(coll, bulk, dataList)
+}
+
+// BatchInsertDataByColl
+// @Description: 批量写入数据（外部传入集合）
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-26 14:22:18
+// @param coll *qmgo.Collection
+// @param bulk int 每次请求保存的数据量
+// @param dataList []interface{}
+// @return err error
+func (m *BaseFromThsHfData) BatchInsertDataByColl(coll *qmgo.Collection, bulk int, dataList []interface{}) (err error) {
+	ctx := context.TODO()
+	dataNum := len(dataList)
+	if dataNum <= 0 {
+		return
+	}
+
+	// 不设置每次保存切片数量大小，或者实际数据量小于设置的切片数量大小，那么就直接保存吧
+	if bulk <= 0 || dataNum <= bulk {
+		_, err = coll.InsertMany(ctx, dataList)
+		if err != nil {
+			fmt.Println("BatchInsertData:Err:" + err.Error())
+			return
+		}
+		return
+	}
+
+	// 分批保存
+	i := 0
+	tmpAddDataList := make([]interface{}, 0)
+	for _, v := range dataList {
+		tmpAddDataList = append(tmpAddDataList, v)
+		i++
+		if i >= bulk {
+			_, err = coll.InsertMany(ctx, tmpAddDataList)
+			if err != nil {
+				fmt.Println("BatchInsertData:Err:" + err.Error())
+				return
+			}
+			i = 0
+			tmpAddDataList = make([]interface{}, 0)
+		}
+	}
+
+	if len(tmpAddDataList) > 0 {
+		_, err = coll.InsertMany(ctx, tmpAddDataList)
+		if err != nil {
+			fmt.Println("BatchInsertData:Err:" + err.Error())
+			return
+		}
+	}
+
+	return
+}
+
+// UpdateDataByColl
+// @Description: 单条数据修改
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-26 15:01:51
+// @param whereParams interface{}
+// @param updateParams interface{}
+// @return err error
+func (m *BaseFromThsHfData) UpdateDataByColl(coll *qmgo.Collection, whereParams, updateParams interface{}) (err error) {
+	ctx := context.TODO()
+	err = coll.UpdateOne(ctx, whereParams, updateParams)
+	if err != nil {
+		fmt.Println("UpdateDataByColl:Err:" + err.Error())
+		return
+	}
+	return
+}
+
+// UpdateData
+// @Description: 单条数据修改
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-26 15:01:51
+// @param whereParams interface{}
+// @param updateParams interface{}
+// @return err error
+func (m *BaseFromThsHfData) UpdateData(whereParams, updateParams interface{}) (err error) {
+	db := utils.MgoDataCli.Database(m.DataBaseName())
+	coll := db.Collection(m.CollectionName())
+	ctx := context.TODO()
+	err = coll.UpdateOne(ctx, whereParams, updateParams)
+	if err != nil {
+		fmt.Println("UpdateData:Err:" + err.Error())
+		return
+	}
+	return
+}
+
+// HandleData
+// @Description: 事务处理数据
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-30 10:40:20
+// @param addDataList []BaseAddFromBusinessData
+// @param updateDataList []BaseFromThsHfData
+// @return result interface{}
+// @return err error
+func (m *BaseFromThsHfData) HandleData(addDataList, updateDataList []BaseFromThsHfData) (result interface{}, err error) {
+
+	ctx := context.TODO()
+
+	callback := func(sessCtx context.Context) (interface{}, error) {
+		// 重要：确保事务中的每一个操作，都使用传入的sessCtx参数
+
+		db := utils.MgoDataCli.Database(m.DataBaseName())
+		coll := db.Collection(m.CollectionName())
+
+		// 插入数据
+		if len(addDataList) > 0 {
+			_, err = coll.InsertMany(sessCtx, addDataList)
+			if err != nil {
+				return nil, err
+			}
+		}
+
+		// 修改
+
+		if len(updateDataList) > 0 {
+			for _, v := range updateDataList {
+				err = coll.UpdateOne(ctx, bson.M{"_id": v.ID}, bson.M{"$set": bson.M{"value": v.Value, "modify_time": v.ModifyTime}})
+				if err != nil {
+					fmt.Println("BatchInsertData:Err:" + err.Error())
+					return nil, err
+				}
+			}
+		}
+
+		return nil, nil
+	}
+	result, err = utils.MgoDataCli.DoTransaction(ctx, callback)
+
+	return
+}
+
+// GetEdbInfoMaxAndMinInfo
+// @Description: 获取当前指标的最大最小值
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-30 17:15:39
+// @param whereParams interface{}
+// @return result EdbInfoMaxAndMinInfo
+// @return err error
+func (m *BaseFromThsHfData) GetEdbInfoMaxAndMinInfo(whereParams interface{}) (result EdbInfoMaxAndMinInfo, err error) {
+	if utils.MgoDataCli == nil {
+		err = errors.New("mongodb连接失败")
+		return
+	}
+	db := utils.MgoDataCli.Database(m.DataBaseName())
+	coll := db.Collection(m.CollectionName())
+	ctx := context.TODO()
+	if err != nil {
+		fmt.Println("MgoGetColl Err:", err.Error())
+		return
+	}
+	err = coll.Aggregate(ctx, whereParams).One(&result)
+	if err != nil {
+		return
+	}
+	result.MinDate = result.MinDate.In(time.Local)
+	result.MaxDate = result.MaxDate.In(time.Local)
+	result.LatestDate = result.LatestDate.In(time.Local)
+	return
+}
+
+// GetLatestValue
+// @Description: 获取当前指标的最新数据记录
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-30 17:16:15
+// @param whereParams interface{}
+// @param selectParam interface{}
+// @return latestValue LatestValue
+// @return err error
+func (m *BaseFromThsHfData) GetLatestValue(whereParams, selectParam interface{}) (latestValue LatestValue, err error) {
+	if utils.MgoDataCli == nil {
+		err = errors.New("mongodb连接失败")
+		return
+	}
+	db := utils.MgoDataCli.Database(m.DataBaseName())
+	coll := db.Collection(m.CollectionName())
+	ctx := context.TODO()
+	if err != nil {
+		fmt.Println("MgoGetColl Err:", err.Error())
+		return
+	}
+
+	//var result interface{}
+	//err = coll.Find(ctx, whereParams).Select(selectParam).One(&result)
+	err = coll.Find(ctx, whereParams).Select(selectParam).One(&latestValue)
+	return
+}

models/mgo/edb_data_ths_hf.go

@@ -0,0 +1,510 @@
+package mgo
+
+import (
+	"context"
+	"errors"
+	"eta/eta_index_lib/utils"
+	"fmt"
+	"github.com/qiniu/qmgo"
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+	"time"
+)
+
+// EdbDataThsHf
+// @Description: 同花顺高频集合（指标库）
+type EdbDataThsHf struct {
+	ID            primitive.ObjectID `json:"_id" bson:"_id,omitempty" `            // 文档id
+	EdbInfoId     int                `json:"edb_info_id" bson:"edb_info_id"`       // 指标ID
+	EdbCode       string             `json:"edb_code" bson:"edb_code"`             // 指标编码
+	DataTime      time.Time          `json:"data_time" bson:"data_time"`           // 数据日期
+	Value         float64            `json:"value" bson:"value"`                   // 数据值
+	CreateTime    time.Time          `json:"create_time" bson:"create_time"`       // 创建时间
+	ModifyTime    time.Time          `json:"modify_time" bson:"modify_time"`       // 修改时间
+	DataTimestamp int64              `json:"data_timestamp" bson:"data_timestamp"` // 数据日期时间戳
+}
+
+// CollectionName
+// @Description:  获取集合名称
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-26 13:41:36
+// @return string
+func (m *EdbDataThsHf) CollectionName() string {
+	return "edb_data_ths_hf"
+}
+
+// DataBaseName
+// @Description: 获取数据库名称
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-26 13:41:33
+// @return string
+func (m *EdbDataThsHf) DataBaseName() string {
+	return utils.MgoDataDbName
+}
+
+// GetCollection
+// @Description: 获取mongodb集合的句柄
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-26 13:41:33
+// @return string
+func (m *EdbDataThsHf) GetCollection() *qmgo.Collection {
+	db := utils.MgoDataCli.Database(m.DataBaseName())
+	return db.Collection(m.CollectionName())
+}
+
+// GetItem
+// @Description: 根据条件获取单条数据
+// @author: Roc
+// @receiver m
+// @datetime 2024-05-09 10:00:49
+// @param whereParams interface{}
+// @return item *EdbDataThsHf
+// @return err error
+func (m *EdbDataThsHf) GetItem(whereParams interface{}) (item *EdbDataThsHf, err error) {
+	if utils.MgoDataCli == nil {
+		err = errors.New("mongodb连接失败")
+		return
+	}
+	db := utils.MgoDataCli.Database(m.DataBaseName())
+	coll := db.Collection(m.CollectionName())
+
+	return m.GetItemByColl(coll, whereParams)
+}
+
+// GetItemByColl
+// @Description: 根据条件获取单条数据
+// @author: Roc
+// @receiver m
+// @datetime 2024-05-09 13:22:06
+// @param coll *qmgo.Collection
+// @param whereParams interface{}
+// @return item *EdbDataThsHf
+// @return err error
+func (m *EdbDataThsHf) GetItemByColl(coll *qmgo.Collection, whereParams interface{}) (item *EdbDataThsHf, err error) {
+	ctx := context.TODO()
+	if err != nil {
+		fmt.Println("MgoGetColl Err:", err.Error())
+		return
+	}
+	err = coll.Find(ctx, whereParams).One(&item)
+	if err != nil {
+		return
+	}
+
+	item.DataTime = item.DataTime.In(time.Local)
+	item.CreateTime = item.CreateTime.In(time.Local)
+	item.ModifyTime = item.ModifyTime.In(time.Local)
+
+	return
+}
+
+// GetAllDataList
+// @Description: 根据条件获取所有数据
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-26 13:42:19
+// @param whereParams interface{}
+// @param sort []string
+// @return result []EdbDataThsHf
+// @return err error
+func (m *EdbDataThsHf) GetAllDataList(whereParams interface{}, sort []string) (result []*EdbDataThsHf, err error) {
+	if utils.MgoDataCli == nil {
+		err = errors.New("mongodb连接失败")
+		return
+	}
+	db := utils.MgoDataCli.Database(m.DataBaseName())
+	coll := db.Collection(m.CollectionName())
+	ctx := context.TODO()
+	if err != nil {
+		fmt.Println("MgoGetColl Err:", err.Error())
+		return
+	}
+	err = coll.Find(ctx, whereParams).Sort(sort...).All(&result)
+	if err != nil {
+		return
+	}
+
+	for _, v := range result {
+		v.DataTime = v.DataTime.In(time.Local)
+		v.CreateTime = v.CreateTime.In(time.Local)
+		v.ModifyTime = v.ModifyTime.In(time.Local)
+	}
+
+	return
+}
+
+// GetLimitDataList
+// @Description: 根据条件获取指定数量数据列表
+// @author: Roc
+// @receiver m
+// @datetime 2024-05-06 17:08:32
+// @param whereParams interface{}
+// @param size int64
+// @param sort []string
+// @return result []*BaseFromBusinessData
+// @return err error
+func (m *EdbDataThsHf) GetLimitDataList(whereParams interface{}, size int64, sort []string) (result []*EdbDataThsHf, err error) {
+	if utils.MgoDataCli == nil {
+		err = errors.New("mongodb连接失败")
+		return
+	}
+	db := utils.MgoDataCli.Database(m.DataBaseName())
+	coll := db.Collection(m.CollectionName())
+	ctx := context.TODO()
+	if err != nil {
+		fmt.Println("MgoGetColl Err:", err.Error())
+		return
+	}
+	err = coll.Find(ctx, whereParams).Sort(sort...).Limit(size).All(&result)
+	if err != nil {
+		return
+	}
+
+	for _, v := range result {
+		v.DataTime = v.DataTime.In(time.Local)
+		v.CreateTime = v.CreateTime.In(time.Local)
+		v.ModifyTime = v.ModifyTime.In(time.Local)
+	}
+
+	return
+}
+
+// GetPageDataList
+// @Description: 根据条件获取分页数据列表
+// @author: Roc
+// @receiver m
+// @datetime 2024-05-07 10:21:07
+// @param whereParams interface{}
+// @param startSize int64
+// @param size int64
+// @param sort []string
+// @return result []*EdbDataThsHf
+// @return err error
+func (m *EdbDataThsHf) GetPageDataList(whereParams interface{}, startSize, size int64, sort []string) (result []*EdbDataThsHf, err error) {
+	if utils.MgoDataCli == nil {
+		err = errors.New("mongodb连接失败")
+		return
+	}
+	db := utils.MgoDataCli.Database(m.DataBaseName())
+	coll := db.Collection(m.CollectionName())
+	ctx := context.TODO()
+	if err != nil {
+		fmt.Println("MgoGetColl Err:", err.Error())
+		return
+	}
+	err = coll.Find(ctx, whereParams).Sort(sort...).Skip(startSize).Limit(size).All(&result)
+	if err != nil {
+		return
+	}
+
+	for _, v := range result {
+		v.DataTime = v.DataTime.In(time.Local)
+		v.CreateTime = v.CreateTime.In(time.Local)
+		v.ModifyTime = v.ModifyTime.In(time.Local)
+	}
+
+	return
+}
+
+// GetCountDataList
+// @Description:  根据条件获取数据列表总数
+// @author: Roc
+// @receiver m
+// @datetime 2024-05-07 10:29:00
+// @param whereParams interface{}
+// @return count int64
+// @return err error
+func (m *EdbDataThsHf) GetCountDataList(whereParams interface{}) (count int64, err error) {
+	if utils.MgoDataCli == nil {
+		err = errors.New("mongodb连接失败")
+		return
+	}
+	db := utils.MgoDataCli.Database(m.DataBaseName())
+	coll := db.Collection(m.CollectionName())
+	ctx := context.TODO()
+	if err != nil {
+		fmt.Println("MgoGetColl Err:", err.Error())
+		return
+	}
+	count, err = coll.Find(ctx, whereParams).Count()
+
+	return
+}
+
+// InsertDataByColl
+// @Description: 写入单条数据（外部传入集合）
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-26 14:22:18
+// @param addData interface{}
+// @return err error
+func (m *EdbDataThsHf) InsertDataByColl(coll *qmgo.Collection, addData interface{}) (err error) {
+	ctx := context.TODO()
+	_, err = coll.InsertOne(ctx, addData)
+	if err != nil {
+		fmt.Println("InsertDataByColl:Err:" + err.Error())
+		return
+	}
+
+	return
+}
+
+// BatchInsertData
+// @Description: 批量写入数据
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-26 14:22:18
+// @param bulk int 每次请求保存的数据量
+// @param dataList []interface{}
+// @return err error
+func (m *EdbDataThsHf) BatchInsertData(bulk int, dataList []interface{}) (err error) {
+	db := utils.MgoDataCli.Database(m.DataBaseName())
+	coll := db.Collection(m.CollectionName())
+
+	return m.BatchInsertDataByColl(coll, bulk, dataList)
+}
+
+// BatchInsertDataByColl
+// @Description: 批量写入数据（外部传入集合）
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-26 14:22:18
+// @param coll *qmgo.Collection
+// @param bulk int 每次请求保存的数据量
+// @param dataList []interface{}
+// @return err error
+func (m *EdbDataThsHf) BatchInsertDataByColl(coll *qmgo.Collection, bulk int, dataList []interface{}) (err error) {
+	ctx := context.TODO()
+	dataNum := len(dataList)
+	if dataNum <= 0 {
+		return
+	}
+
+	// 不设置每次保存切片数量大小，或者实际数据量小于设置的切片数量大小，那么就直接保存吧
+	if bulk <= 0 || dataNum <= bulk {
+		_, err = coll.InsertMany(ctx, dataList)
+		if err != nil {
+			fmt.Println("BatchInsertData:Err:" + err.Error())
+			return
+		}
+		return
+	}
+
+	// 分批保存
+	i := 0
+	tmpAddDataList := make([]interface{}, 0)
+	for _, v := range dataList {
+		tmpAddDataList = append(tmpAddDataList, v)
+		i++
+		if i >= bulk {
+			_, err = coll.InsertMany(ctx, tmpAddDataList)
+			if err != nil {
+				fmt.Println("BatchInsertData:Err:" + err.Error())
+				return
+			}
+			i = 0
+			tmpAddDataList = make([]interface{}, 0)
+		}
+	}
+
+	if len(tmpAddDataList) > 0 {
+		_, err = coll.InsertMany(ctx, tmpAddDataList)
+		if err != nil {
+			fmt.Println("BatchInsertData:Err:" + err.Error())
+			return
+		}
+	}
+
+	return
+}
+
+// UpdateData
+// @Description: 单条数据修改
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-26 15:01:51
+// @param whereParams interface{}
+// @param updateParams interface{}
+// @return err error
+func (m *EdbDataThsHf) UpdateData(whereParams, updateParams interface{}) (err error) {
+	db := utils.MgoDataCli.Database(m.DataBaseName())
+	coll := db.Collection(m.CollectionName())
+
+	return m.UpdateDataByColl(coll, whereParams, updateParams)
+}
+
+// UpdateDataByColl
+// @Description: 单条数据修改（外部传入集合）
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-26 15:01:51
+// @param whereParams interface{}
+// @param updateParams interface{}
+// @return err error
+func (m *EdbDataThsHf) UpdateDataByColl(coll *qmgo.Collection, whereParams, updateParams interface{}) (err error) {
+	ctx := context.TODO()
+	err = coll.UpdateOne(ctx, whereParams, updateParams)
+	if err != nil {
+		fmt.Println("UpdateDataByColl:Err:" + err.Error())
+		return
+	}
+
+	return
+}
+
+// RemoveMany
+// @Description: 根据条件删除多条数据
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-30 13:17:02
+// @param whereParams interface{}
+// @return err error
+func (m *EdbDataThsHf) RemoveMany(whereParams interface{}) (err error) {
+	db := utils.MgoDataCli.Database(m.DataBaseName())
+	coll := db.Collection(m.CollectionName())
+
+	return m.RemoveManyByColl(coll, whereParams)
+}
+
+// RemoveManyByColl
+// @Description: 根据条件删除多条数据（外部传入集合）
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-30 13:18:42
+// @param coll *qmgo.Collection
+// @param whereParams interface{}
+// @return err error
+func (m *EdbDataThsHf) RemoveManyByColl(coll *qmgo.Collection, whereParams interface{}) (err error) {
+	ctx := context.TODO()
+	_, err = coll.RemoveAll(ctx, whereParams)
+	if err != nil {
+		fmt.Println("RemoveManyByColl:Err:" + err.Error())
+		return
+	}
+
+	return
+}
+
+// HandleData
+// @Description: 事务处理数据
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-30 10:39:01
+// @param addDataList []AddEdbDataThsHf
+// @param updateDataList []EdbDataThsHf
+// @return result interface{}
+// @return err error
+func (m *EdbDataThsHf) HandleData(addDataList, updateDataList []EdbDataThsHf) (result interface{}, err error) {
+
+	ctx := context.TODO()
+
+	callback := func(sessCtx context.Context) (interface{}, error) {
+		// 重要：确保事务中的每一个操作，都使用传入的sessCtx参数
+
+		db := utils.MgoDataCli.Database(m.DataBaseName())
+		coll := db.Collection(m.CollectionName())
+
+		// 插入数据
+		if len(addDataList) > 0 {
+			_, err = coll.InsertMany(sessCtx, addDataList)
+			if err != nil {
+				return nil, err
+			}
+		}
+
+		// 修改
+
+		if len(updateDataList) > 0 {
+			for _, v := range updateDataList {
+				err = coll.UpdateOne(ctx, bson.M{"_id": v.ID}, bson.M{"$set": bson.M{"value": v.Value, "modify_time": v.ModifyTime}})
+				if err != nil {
+					fmt.Println("BatchInsertData:Err:" + err.Error())
+					return nil, err
+				}
+			}
+		}
+
+		return nil, nil
+	}
+	result, err = utils.MgoDataCli.DoTransaction(ctx, callback)
+
+	return
+}
+
+// EdbInfoMaxAndMinInfo 指标最新数据记录结构体
+//type EdbInfoMaxAndMinInfo struct {
+//	MinDate     time.Time `description:"最小日期" bson:"min_date"`
+//	MaxDate     time.Time `description:"最大日期" bson:"max_date"`
+//	MinValue    float64   `description:"最小值" bson:"min_value"`
+//	MaxValue    float64   `description:"最大值" bson:"max_value"`
+//	LatestValue float64   `description:"最新值" bson:"latest_value"`
+//	LatestDate  time.Time `description:"实际数据最新日期" bson:"latest_date"`
+//	EndValue    float64   `description:"最新值" bson:"end_value"`
+//}
+
+// GetEdbInfoMaxAndMinInfo
+// @Description: 获取当前指标的最大最小值
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-30 17:15:39
+// @param whereParams interface{}
+// @return result EdbInfoMaxAndMinInfo
+// @return err error
+func (m *EdbDataThsHf) GetEdbInfoMaxAndMinInfo(whereParams interface{}) (result EdbInfoMaxAndMinInfo, err error) {
+	if utils.MgoDataCli == nil {
+		err = errors.New("mongodb连接失败")
+		return
+	}
+	db := utils.MgoDataCli.Database(m.DataBaseName())
+	coll := db.Collection(m.CollectionName())
+	ctx := context.TODO()
+	if err != nil {
+		fmt.Println("MgoGetColl Err:", err.Error())
+		return
+	}
+	err = coll.Aggregate(ctx, whereParams).One(&result)
+	if err != nil {
+		return
+	}
+	result.MinDate = result.MinDate.In(time.Local)
+	result.MaxDate = result.MaxDate.In(time.Local)
+	result.LatestDate = result.LatestDate.In(time.Local)
+
+	return
+}
+
+// LatestValue 指标最新数据记录结构体
+//type LatestValue struct {
+//	Value float64 `description:"值" bson:"value"`
+//}
+
+// GetLatestValue
+// @Description: 获取当前指标的最新数据记录
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-30 17:16:15
+// @param whereParams interface{}
+// @param selectParam interface{}
+// @return latestValue LatestValue
+// @return err error
+func (m *EdbDataThsHf) GetLatestValue(whereParams, selectParam interface{}) (latestValue LatestValue, err error) {
+	if utils.MgoDataCli == nil {
+		err = errors.New("mongodb连接失败")
+		return
+	}
+	db := utils.MgoDataCli.Database(m.DataBaseName())
+	coll := db.Collection(m.CollectionName())
+	ctx := context.TODO()
+	if err != nil {
+		fmt.Println("MgoGetColl Err:", err.Error())
+		return
+	}
+
+	//var result interface{}
+	//err = coll.Find(ctx, whereParams).Select(selectParam).One(&result)
+	err = coll.Find(ctx, whereParams).Select(selectParam).One(&latestValue)
+	return
+}

routers/commentsRouter.go

@@ -1420,6 +1420,51 @@ func init() {
             Filters: nil,
             Params: nil})
 
+    beego.GlobalControllerRouter["eta/eta_index_lib/controllers:ThsHfController"] = append(beego.GlobalControllerRouter["eta/eta_index_lib/controllers:ThsHfController"],
+        beego.ControllerComments{
+            Method: "BaseAdd",
+            Router: `/hf/base/add`,
+            AllowHTTPMethods: []string{"post"},
+            MethodParams: param.Make(),
+            Filters: nil,
+            Params: nil})
+
+    beego.GlobalControllerRouter["eta/eta_index_lib/controllers:ThsHfController"] = append(beego.GlobalControllerRouter["eta/eta_index_lib/controllers:ThsHfController"],
+        beego.ControllerComments{
+            Method: "BaseRefresh",
+            Router: `/hf/base/refresh`,
+            AllowHTTPMethods: []string{"post"},
+            MethodParams: param.Make(),
+            Filters: nil,
+            Params: nil})
+
+    beego.GlobalControllerRouter["eta/eta_index_lib/controllers:ThsHfController"] = append(beego.GlobalControllerRouter["eta/eta_index_lib/controllers:ThsHfController"],
+        beego.ControllerComments{
+            Method: "EdbAdd",
+            Router: `/hf/edb/add`,
+            AllowHTTPMethods: []string{"post"},
+            MethodParams: param.Make(),
+            Filters: nil,
+            Params: nil})
+
+    beego.GlobalControllerRouter["eta/eta_index_lib/controllers:ThsHfController"] = append(beego.GlobalControllerRouter["eta/eta_index_lib/controllers:ThsHfController"],
+        beego.ControllerComments{
+            Method: "EdbRefresh",
+            Router: `/hf/edb/refresh`,
+            AllowHTTPMethods: []string{"post"},
+            MethodParams: param.Make(),
+            Filters: nil,
+            Params: nil})
+
+    beego.GlobalControllerRouter["eta/eta_index_lib/controllers:ThsHfController"] = append(beego.GlobalControllerRouter["eta/eta_index_lib/controllers:ThsHfController"],
+        beego.ControllerComments{
+            Method: "GetData",
+            Router: `/hf/edb_data`,
+            AllowHTTPMethods: []string{"post"},
+            MethodParams: param.Make(),
+            Filters: nil,
+            Params: nil})
+
     beego.GlobalControllerRouter["eta/eta_index_lib/controllers:WindController"] = append(beego.GlobalControllerRouter["eta/eta_index_lib/controllers:WindController"],
         beego.ControllerComments{
             Method: "Add",

routers/router.go

@@ -23,6 +23,7 @@ func init() {
 			beego.NSInclude(
 				&controllers.ThsController{},
 				&controllers.ThsDsController{},
+				&controllers.ThsHfController{},
 			),
 		),
 		beego.NSNamespace("/pb",

services/base_from_business.go

@@ -8,6 +8,7 @@ import (
 	"eta/eta_index_lib/services/alarm_msg"
 	"eta/eta_index_lib/utils"
 	"fmt"
+	"github.com/qiniu/qmgo"
 	"go.mongodb.org/mongo-driver/bson"
 	"strings"
 	"time"
@@ -135,10 +136,44 @@ func HandleBusinessIndex(indexReq *models.AddBusinessIndexReq) (resp models.Base
 	}
 
 	// 数据处理
+	// 当前传入的最小日期
+	var reqMinDate time.Time
+	if utils.UseMongo {
+		reqMinDate, err = handleBusinessDataByMongo(item, indexReq.DataList)
+	} else {
+		reqMinDate, err = handleBusinessDataByMysql(item, indexReq.DataList)
+	}
+	if err != nil {
+		return
+	}
+
+	// 同步刷新指标库的指标
+	go refreshEdbBusiness(item.IndexCode, reqMinDate)
+
+	resp = models.BaseFromBusinessIndexResp{
+		IndexCode:  item.IndexCode,
+		IndexName:  item.IndexName,
+		Unit:       item.Unit,
+		Frequency:  item.Frequency,
+		SourceName: item.SourceName,
+	}
+
+	return
+}
+
+// handleBusinessDataByMongo
+// @Description: 处理外部指标数据（mongo）
+// @author: Roc
+// @datetime 2024-07-01 15:30:41
+// @param item *models.BaseFromBusinessIndex
+// @param reqDataList []*models.AddBusinessDataReq
+// @return reqMinDate time.Time 当前传入的最小日期
+// @return err error
+func handleBusinessDataByMongo(item *models.BaseFromBusinessIndex, reqDataList []models.AddBusinessDataReq) (reqMinDate time.Time, err error) {
 	mogDataObj := new(mgo.BaseFromBusinessData)
 
 	//获取已存在的所有数据
-	exitDataList, err := mogDataObj.GetAllDataList(bson.M{"index_code": indexReq.IndexCode}, []string{"data_time"})
+	exitDataList, err := mogDataObj.GetAllDataList(bson.M{"index_code": item.IndexCode}, []string{"data_time"})
 	if err != nil {
 		fmt.Println("GetIndexDataList Err:" + err.Error())
 		return
@@ -150,14 +185,12 @@ func HandleBusinessIndex(indexReq *models.AddBusinessIndexReq) (resp models.Base
 		exitDataMap[v.DataTime.Format(utils.FormatDate)] = v
 	}
 
-	// 当前传入的最小日期
-	var reqMinDate time.Time
 	// 待添加的数据集
 	addDataList := make([]interface{}, 0)
 	updateDataList := make([]mgo.BaseFromBusinessData, 0)
 	//var hasUpdate bool
 	// 遍历excel数据，然后跟现有的数据做校验，不存在则入库
-	for _, data := range indexReq.DataList {
+	for _, data := range reqDataList {
 		dateTime, tmpErr := utils.DealExcelDate(data.Date)
 		if tmpErr != nil {
 			fmt.Println("time.ParseInLocation Err:" + tmpErr.Error())
@@ -229,27 +262,121 @@ func HandleBusinessIndex(indexReq *models.AddBusinessIndexReq) (resp models.Base
 	//fmt.Println("result", result)
 
 	//修改最大最小日期
-	indexMaxAndMinInfo, err := item.GetEdbInfoMaxAndMinInfo(indexReq.IndexCode)
+	indexMaxAndMinInfo, err := item.GetEdbInfoMaxAndMinInfo(item.IndexCode)
 	if err != nil {
 		return
 	}
 	if err == nil && indexMaxAndMinInfo != nil {
-		e := item.ModifyIndexMaxAndMinInfo(indexReq.IndexCode, indexMaxAndMinInfo, isIndexUpdateOrAdd)
+		e := item.ModifyIndexMaxAndMinInfo(item.IndexCode, indexMaxAndMinInfo, isIndexUpdateOrAdd)
 		if e != nil {
 			fmt.Println("ModifyIndexMaxAndMinInfo Err:" + e.Error())
 		}
 	}
 
-	// 同步刷新指标库的指标
-	go refreshEdbBusiness(item.IndexCode, reqMinDate)
+	return
+}
 
-	resp = models.BaseFromBusinessIndexResp{
-		IndexCode:  item.IndexCode,
-		IndexName:  item.IndexName,
-		Unit:       item.Unit,
-		Frequency:  item.Frequency,
-		SourceName: item.SourceName,
+// handleBusinessDataByMysql
+// @Description: 处理外部指标数据（mysql）
+// @author: Roc
+// @datetime 2024-07-01 15:59:43
+// @param item *models.BaseFromBusinessIndex
+// @param reqDataList []models.AddBusinessDataReq
+// @return reqMinDate time.Time
+// @return err error
+func handleBusinessDataByMysql(item *models.BaseFromBusinessIndex, reqDataList []models.AddBusinessDataReq) (reqMinDate time.Time, err error) {
+	businessDataObj := new(models.BaseFromBusinessData)
+
+	var condition []string
+	var pars []interface{}
+	condition = append(condition, "index_code = ?")
+	pars = append(pars, item.IndexCode)
+	//获取已存在的所有数据
+	exitDataList, err := businessDataObj.GetAllDataList(condition, pars, "data_time ASC")
+	if err != nil {
+		fmt.Println("GetIndexDataList Err:" + err.Error())
+		return
+	}
+
+	// 已经存在的数据集
+	exitDataMap := make(map[string]*models.BaseFromBusinessData)
+	for _, v := range exitDataList {
+		exitDataMap[v.DataTime.Format(utils.FormatDate)] = v
+	}
+
+	// 待添加的数据集
+	addDataList := make([]*models.BaseFromBusinessData, 0)
+	updateDataList := make([]*models.BaseFromBusinessData, 0)
+	//var hasUpdate bool
+	// 遍历excel数据，然后跟现有的数据做校验，不存在则入库
+	for _, data := range reqDataList {
+		dateTime, tmpErr := utils.DealExcelDate(data.Date)
+		if tmpErr != nil {
+			fmt.Println("time.ParseInLocation Err:" + tmpErr.Error())
+			err = tmpErr
+			return
+		}
+
+		// 调整最小日期
+		if reqMinDate.IsZero() || reqMinDate.After(dateTime) {
+			reqMinDate = dateTime
+		}
+
+		date := dateTime.Format(utils.FormatDate)
+
+		findData, ok := exitDataMap[date]
+		if !ok {
+			addDataList = append(addDataList, &models.BaseFromBusinessData{
+				BaseFromBusinessIndexId: int(item.BaseFromBusinessIndexId),
+				IndexCode:               item.IndexCode,
+				DataTime:                dateTime,
+				Value:                   data.Value,
+				CreateTime:              time.Now(),
+				ModifyTime:              time.Now(),
+				//DataTimestamp:           0,
+			})
+			continue
+		}
+
+		// 值不匹配，修改数据
+		if findData.Value != data.Value {
+			findData.Value = data.Value
+			findData.ModifyTime = time.Now()
+			updateDataList = append(updateDataList, findData)
+		}
+	}
+
+	// 指标数据是否新增或修改
+	var isIndexUpdateOrAdd bool
+
+	// 入库
+	{
+		if len(addDataList) > 0 {
+			isIndexUpdateOrAdd = true
+		}
+
+		if len(updateDataList) > 0 {
+			isIndexUpdateOrAdd = true
+		}
+		err = businessDataObj.HandleData(addDataList, updateDataList)
+		if err != nil {
+			fmt.Println("UpdateDataByColl:Err:" + err.Error())
+			return
+		}
 	}
+
+	//修改最大最小日期
+	indexMaxAndMinInfo, err := item.GetEdbInfoMaxAndMinInfo(item.IndexCode)
+	if err != nil {
+		return
+	}
+	if err == nil && indexMaxAndMinInfo != nil {
+		e := item.ModifyIndexMaxAndMinInfo(item.IndexCode, indexMaxAndMinInfo, isIndexUpdateOrAdd)
+		if e != nil {
+			fmt.Println("ModifyIndexMaxAndMinInfo Err:" + e.Error())
+		}
+	}
+
 	return
 }
 
@@ -326,9 +453,22 @@ func DelBusinessIndex(indexCodeList []string) (joinEdbCodeList, needDelEdbCodeLi
 		return
 	}
 
-	// 删除指标明细数据
-	mogDataObj := new(mgo.BaseFromBusinessData)
-	err = mogDataObj.RemoveMany(bson.M{"index_code": bson.M{"$in": needDelEdbCodeList}})
+	if utils.UseMongo {
+		// 删除指标明细数据
+		mogDataObj := new(mgo.BaseFromBusinessData)
+		err = mogDataObj.RemoveMany(bson.M{"index_code": bson.M{"$in": needDelEdbCodeList}})
+	} else {
+		var condition []string
+		var pars []interface{}
+		delNum := len(needDelEdbCodeList)
+		if delNum > 0 {
+			condition = append(condition, "index_code in ("+utils.GetOrmInReplace(delNum)+")")
+			pars = append(pars, needDelEdbCodeList)
+
+			businessDataObj := models.BaseFromBusinessData{}
+			err = businessDataObj.DelDataByCond(condition, pars)
+		}
+	}
 	if err != nil {
 		fmt.Println("删除自有指标明细数据 Err:" + err.Error())
 		return
@@ -348,7 +488,7 @@ func DelBusinessIndex(indexCodeList []string) (joinEdbCodeList, needDelEdbCodeLi
 func DelBusinessIndexData(indexCode string, startDate, endDate string) (err error, errMsg string) {
 	defer func() {
 		if err != nil {
-			fmt.Println("DelBusinessIndex Err:" + err.Error())
+			fmt.Println("DelBusinessIndexData Err:" + err.Error())
 		}
 	}()
 	errMsg = "删除失败"
@@ -377,11 +517,6 @@ func DelBusinessIndexData(indexCode string, startDate, endDate string) (err erro
 		return
 	}
 
-	// 构建查询条件
-	queryConditions := bson.M{
-		"index_code": item.IndexCode,
-	}
-
 	// 当前传入的最小日期
 	var reqMinDate time.Time
 
@@ -410,6 +545,51 @@ func DelBusinessIndexData(indexCode string, startDate, endDate string) (err erro
 		}
 	}
 
+	// 删除具体的数据
+	var indexMaxAndMinInfo *models.EdbInfoMaxAndMinInfo
+	if utils.UseMongo {
+		indexMaxAndMinInfo, err = delBusinessIndexDataByMongo(item, startDateTime, endDateTime)
+	} else {
+		indexMaxAndMinInfo, err = delBusinessIndexDataByMysql(item, startDateTime, endDateTime)
+	}
+	if err != nil {
+		return
+	}
+
+	// 修改指标的最早最晚日期
+	if indexMaxAndMinInfo != nil {
+		e := item.ModifyIndexMaxAndMinInfo(item.IndexCode, indexMaxAndMinInfo, true)
+		if e != nil {
+			fmt.Println("ModifyIndexMaxAndMinInfo Err:" + e.Error())
+		}
+	}
+
+	// 同步刷新指标库的指标
+	go refreshEdbBusiness(item.IndexCode, reqMinDate)
+
+	return
+}
+
+// delBusinessIndexDataByMongo
+// @Description: 删除指标数据（从mongo删除）
+// @author: Roc
+// @datetime 2024-07-01 18:00:07
+// @param item *models.BaseFromBusinessIndex
+// @param startDateTime time.Time
+// @param endDateTime time.Time
+// @return err error
+func delBusinessIndexDataByMongo(item *models.BaseFromBusinessIndex, startDateTime, endDateTime time.Time) (indexMaxAndMinInfo *models.EdbInfoMaxAndMinInfo, err error) {
+	defer func() {
+		if err != nil {
+			utils.FileLog.Error("delBusinessIndexDataByMongo 删除自有指标明细数据 Err:" + err.Error())
+		}
+	}()
+
+	// 构建查询条件
+	queryConditions := bson.M{
+		"index_code": item.IndexCode,
+	}
+
 	dateCondition, err := mgo.BuildDateTimeCondition(startDateTime, endDateTime)
 	if err != nil {
 		return
@@ -422,12 +602,66 @@ func DelBusinessIndexData(indexCode string, startDate, endDate string) (err erro
 	mogDataObj := new(mgo.BaseFromBusinessData)
 	err = mogDataObj.RemoveMany(queryConditions)
 	if err != nil {
-		fmt.Println("删除自有指标明细数据 Err:" + err.Error())
 		return
 	}
 
-	// 同步刷新指标库的指标
-	go refreshEdbBusiness(item.IndexCode, reqMinDate)
+	//修改最大最小日期
+	indexMaxAndMinInfo, err = item.GetEdbInfoMaxAndMinInfo(item.IndexCode)
+	// 如果有错误，且错误信息是取不到文档，那么就不修改了
+	if err != nil && !errors.Is(err, qmgo.ErrNoSuchDocuments) {
+		return
+	}
+	// 清空的目的是为了避免异常返回
+	err = nil
+
+	return
+}
+
+// delBusinessIndexDataByMysql
+// @Description: 删除指标数据（从mysql删除）
+// @author: Roc
+// @datetime 2024-07-02 09:53:13
+// @param item *models.BaseFromBusinessIndex
+// @param startDateTime time.Time
+// @param endDateTime time.Time
+// @return err error
+func delBusinessIndexDataByMysql(item *models.BaseFromBusinessIndex, startDateTime, endDateTime time.Time) (indexMaxAndMinInfo *models.EdbInfoMaxAndMinInfo, err error) {
+	defer func() {
+		if err != nil {
+			utils.FileLog.Error("delBusinessIndexDataByMysql 删除自有指标明细数据 Err:" + err.Error())
+		}
+	}()
+
+	// 构建查询条件
+	var condition []string
+	var pars []interface{}
+	condition = append(condition, "index_code = ? ")
+	pars = append(pars, item.IndexCode)
+
+	if !startDateTime.IsZero() {
+		condition = append(condition, " data_time >= ? ")
+		pars = append(pars, startDateTime.Format(utils.FormatDate))
+	}
+	if !endDateTime.IsZero() {
+		condition = append(condition, " data_time <= ? ")
+		pars = append(pars, endDateTime.Format(utils.FormatDate))
+	}
+
+	// 删除数据源中的指标明细数据
+	businessDataObj := new(models.BaseFromBusinessData)
+	err = businessDataObj.DelDataByCond(condition, pars)
+	if err != nil {
+		return
+	}
+
+	//修改最大最小日期
+	indexMaxAndMinInfo, err = item.GetEdbInfoMaxAndMinInfo(item.IndexCode)
+	// 如果有错误，且错误信息是取不到文档，那么就不修改了
+	if err != nil && err.Error() != utils.ErrNoRow() {
+		return
+	}
+	// 清空的目的是为了避免异常返回
+	err = nil
 
 	return
 }

services/base_from_calculate.go

@@ -456,6 +456,7 @@ func EdbCalculateBatchSave(req models.EdbInfoCalculateBatchSaveReq, lang string)
 
 	// 更新ES
 	go logic.UpdateEs(edbInfo.EdbInfoId)
+	go DisableEdbInfoNoUpdate(edbInfo)
 	return
 }
 
@@ -857,6 +858,9 @@ func EdbCalculateBatchEdit(req models.EdbInfoCalculateBatchEditReq) (edbInfo *mo
 
 	// 更新ES
 	go logic.UpdateEs(edbInfo.EdbInfoId)
+
+	// 重置计算指标中的引用关系
+	go ResetEdbRelation(edbInfoId)
 	return
 }
 
@@ -1007,7 +1011,7 @@ func EdbCalculateAdd(req models.EdbInfoCalculateSaveReq, lang string) (edbInfo *
 		err = fmt.Errorf("生成计算指标失败")
 		return
 	}
-
+	go DisableEdbInfoNoUpdate(edbInfo)
 	return
 }
 

services/base_from_ths_ds.go

@@ -10,7 +10,7 @@ import (
 	"strings"
 )
 
-func GetEdbDataFromThsDs(stockCode, edbCode, startDate, endDate, edbTerminalCode string) (item models.EdbDataFromThs, err error) {
+func GetEdbDataFromThsDs(stockCode, edbCode, startDate, endDate, edbTerminalCode, extraPars string) (item models.EdbDataFromThs, err error) {
 	terminal, err := GetTerminal(utils.DATA_SOURCE_THS, edbTerminalCode)
 	if err != nil {
 		err = fmt.Errorf("获取同花顺接口配置出错 Err: %s", err)
@@ -34,10 +34,13 @@ func GetEdbDataFromThsDs(stockCode, edbCode, startDate, endDate, edbTerminalCode
 		if err != nil {
 			return
 		}
-		return getEdbDataFromThsDsHttp(stockCode, edbCode, startDate, endDate, terminal.Value, token)
+
+		// TEST
+		//token = "23f339e97fac48d8b99024228fafb6f0128cfbb7.signs_NTc2NjQ4MTA5"
+		return getEdbDataFromThsDsHttp(stockCode, edbCode, startDate, endDate, terminal.Value, token, extraPars)
 	}
 
-	return getEdbDataFromThsDsApp(stockCode, edbCode, startDate, endDate, 0, terminal.ServerUrl)
+	return getEdbDataFromThsDsApp(stockCode, edbCode, startDate, endDate, 0, terminal.ServerUrl, extraPars)
 }
 
 type EdbDataFromThsSdInterface struct {
@@ -94,13 +97,13 @@ type Type struct {
 //}
 
 // getEdbDataFromThsDs 获取同花顺接口数据
-func getEdbDataFromThsDsApp(stockCode, edbCode, startDate, endDate string, num int, serverUrl string) (item models.EdbDataFromThs, err error) {
+func getEdbDataFromThsDsApp(stockCode, edbCode, startDate, endDate string, num int, serverUrl, extraPars string) (item models.EdbDataFromThs, err error) {
 	if serverUrl == `` {
 		err = errors.New("同花顺接口未配置")
 		return
 	}
-	thsUrl := serverUrl + `edbInfo/ths/ds?StockCode=%s&EdbCode=%s&StartDate=%s&EndDate=%s`
-	thsUrl = fmt.Sprintf(thsUrl, stockCode, edbCode, startDate, endDate)
+	thsUrl := serverUrl + `edbInfo/ths/ds?StockCode=%s&EdbCode=%s&StartDate=%s&EndDate=%s&ExtraPars=%s`
+	thsUrl = fmt.Sprintf(thsUrl, stockCode, edbCode, startDate, endDate, extraPars)
 	utils.FileLog.Info("thsUrl:" + thsUrl)
 	body, err := http.Get(thsUrl)
 	utils.FileLog.Info("ths result:" + string(body))
@@ -139,7 +142,7 @@ func getEdbDataFromThsDsApp(stockCode, edbCode, startDate, endDate string, num i
 				//session has expired,please re-login after using the system
 				//如果是同花顺登录session失效了，那么就重新请求获取数据
 				if response.ErrorCode == -1020 && num == 0 {
-					return getEdbDataFromThsDsApp(stockCode, edbCode, startDate, endDate, 1, serverUrl)
+					return getEdbDataFromThsDsApp(stockCode, edbCode, startDate, endDate, 1, serverUrl, extraPars)
 				}
 				err = errors.New(string(body))
 				return
@@ -188,7 +191,7 @@ func getEdbDataFromThsDsApp(stockCode, edbCode, startDate, endDate string, num i
 			//session has expired,please re-login after using the system
 			//如果是同花顺登录session失效了，那么就重新请求获取数据
 			if response.ErrorCode == -1020 && num == 0 {
-				return getEdbDataFromThsDsApp(stockCode, edbCode, startDate, endDate, 1, serverUrl)
+				return getEdbDataFromThsDsApp(stockCode, edbCode, startDate, endDate, 1, serverUrl, extraPars)
 			}
 			err = errors.New(string(body))
 			return

services/base_from_ths_ds_http.go

@@ -10,7 +10,7 @@ import (
 )
 
 // getEdbDataFromThsDsHttp 通过url获取同花顺的日期序列数据
-func getEdbDataFromThsDsHttp(stockCode, edbCode, startDate, endDate, thsRefreshToken, token string) (item models.EdbDataFromThs, err error) {
+func getEdbDataFromThsDsHttp(stockCode, edbCode, startDate, endDate, thsRefreshToken, token, extraPars string) (item models.EdbDataFromThs, err error) {
 	thsUrl := "https://quantapi.51ifind.com/api/v1/date_sequence"
 	//indicators 是 半角逗号分隔的所有指标，宏观指标过多，推荐使用Windows超级命令生成。 "indicators":"M001620326,M002822183"
 	//functionpara 否 key-value格式,省略时不进行更新时间筛选。两个时间控件更新起始时间（startrtime）和更新结束时间（endrtime），不勾选时省略见下方代码块
@@ -20,6 +20,12 @@ func getEdbDataFromThsDsHttp(stockCode, edbCode, startDate, endDate, thsRefreshT
 	// 分割字符串
 	paramList := strings.Split(edbCode, ",")
 
+	var extraArr []string
+	if extraPars != "" {
+		extraPars = strings.TrimSpace(extraPars)
+		extraArr = strings.Split(extraPars, ",")
+	}
+
 	// 创建一个包含映射的切片
 	var indipara []map[string]interface{}
 
@@ -29,6 +35,9 @@ func getEdbDataFromThsDsHttp(stockCode, edbCode, startDate, endDate, thsRefreshT
 		paramMap := map[string]interface{}{
 			"indicator": param,
 		}
+		if len(extraArr) > 0 {
+			paramMap["indiparams"] = extraArr
+		}
 
 		// 将映射添加到切片中
 		indipara = append(indipara, paramMap)

services/base_from_ths_hf.go

@@ -0,0 +1,647 @@
+package services
+
+import (
+	"encoding/json"
+	"eta/eta_index_lib/logic"
+	"eta/eta_index_lib/models"
+	"eta/eta_index_lib/models/mgo"
+	"eta/eta_index_lib/services/alarm_msg"
+	"eta/eta_index_lib/utils"
+	"fmt"
+	"github.com/rdlucklib/rdluck_tools/http"
+	"github.com/shopspring/decimal"
+	"go.mongodb.org/mongo-driver/bson"
+	"net/url"
+	"strings"
+	"time"
+)
+
+const (
+	ThsHfApiUrl = "https://quantapi.51ifind.com/api/v1/high_frequency"
+)
+
+// GetEdbDataFromThsHf 获取高频数据
+func GetEdbDataFromThsHf(thsParams models.ThsHfSearchEdbReq, terminalCode string) (indexes []*models.ThsHfIndexWithData, err error) {
+	terminal, e := GetTerminal(utils.DATA_SOURCE_THS, terminalCode)
+	if e != nil {
+		err = fmt.Errorf("获取同花顺终端配置失败, %v", e)
+		return
+	}
+	if thsParams.EndTime == "" {
+		thsParams.EndTime = time.Now().Local().Format(utils.FormatDateTime)
+	}
+
+	// 走API
+	if utils.ThsDataMethod == "" || utils.ThsDataMethod == "api" {
+		var token string
+		token, e = GetAccessToken(false, terminal.Value)
+		if e != nil {
+			err = fmt.Errorf("获取同花顺API-AccessToken失败, %v", e)
+			return
+		}
+
+		// TEST
+		//token = "9eba1634116ea2aed9a5b12b6e12b0b5fcbe0847.signs_NTc2NjQ4MTA5"
+		return getEdbDataFromThsHfHttp(thsParams, terminal.Value, token)
+	}
+
+	// 走公用机
+	if terminal.ServerUrl == "" {
+		err = fmt.Errorf("同花顺终端地址未配置")
+		return
+	}
+	return getEdbDataFromThsHfApp(thsParams, 0, terminal.ServerUrl)
+}
+
+// getEdbDataFromThsHfHttp API-获取高频指标数据
+func getEdbDataFromThsHfHttp(thsParams models.ThsHfSearchEdbReq, refreshToken, accessToken string) (indexes []*models.ThsHfIndexWithData, err error) {
+	defer func() {
+		if err != nil {
+			tips := fmt.Sprintf("同花顺高频指标API-getEdbDataFromThsHfHttp err: %v", err)
+			utils.FileLog.Info(tips)
+		}
+	}()
+
+	// 请求参数参考
+	//dataMap := map[string]interface{}{
+	//	"codes":      "CU2407.SHF,CU2408.SHF",
+	//	"indicators": "pct_chg",
+	//	"starttime":  "2024-06-06 09:15:00",
+	//	"endtime":    "2024-06-11 15:15:00",
+	//	"functionpara": map[string]interface {
+	//	}{
+	//		"Limitstart": "10:00:00",
+	//		"Limitend":   "14:15:00",
+	//		"Interval":   60,
+	//		"Fill":       "Previous",
+	//		"CPS":        "forward4",
+	//		"Timeformat": "LocalTime",
+	//		"BaseDate":   "2024-01-01",
+	//	},
+	//}
+
+	// 额外参数
+	funcParams := map[string]interface{}{}
+	funcParams["Interval"] = thsParams.Interval
+	if thsParams.Fill != "" {
+		funcParams["Fill"] = thsParams.Fill
+	}
+	if thsParams.CPS != "" {
+		funcParams["CPS"] = thsParams.CPS
+	}
+	if thsParams.BaseDate != "" {
+		funcParams["BaseDate"] = thsParams.BaseDate
+	}
+
+	// TEST
+	//funcParams["Limitstart"] = "10:00:00"
+	//funcParams["Limitend"] = "14:15:00"
+	//funcParams["Fill"] = "Previous"
+	//funcParams["CPS"] = "forward4"
+	//funcParams["Timeformat"] = "LocalTime"
+	//funcParams["BaseDate"] = "2024-01-01"
+
+	dataMap := map[string]interface{}{
+		"codes":        thsParams.StockCode,
+		"indicators":   thsParams.EdbCode,
+		"starttime":    thsParams.StartTime,
+		"endtime":      thsParams.EndTime,
+		"functionpara": funcParams,
+	}
+
+	// 请求接口
+	body, e, _ := postCurl(ThsHfApiUrl, dataMap, 0, refreshToken, accessToken)
+	if e != nil {
+		utils.FileLog.Info(string(body))
+		err = fmt.Errorf("同花顺API-请求失败, %v", e)
+		return
+	}
+
+	apiResp := new(models.ThsHfApiResp)
+	if e = json.Unmarshal(body, &apiResp); e != nil {
+		err = fmt.Errorf("同花顺API-解析响应失败, %v", e)
+		return
+	}
+	if apiResp.ErrorCode != 0 {
+		err = fmt.Errorf("同花顺高频API-状态码: %d, 提示信息: %s", apiResp.ErrorCode, apiResp.ErrMsg)
+		return
+	}
+	indexes = make([]*models.ThsHfIndexWithData, 0)
+	if len(apiResp.Tables) == 0 {
+		utils.FileLog.Info("同花顺高频API-无数据")
+		return
+	}
+
+	// 结果示例
+	// {
+	//	"errorcode": 0,
+	//	"errmsg": "Success!",
+	//	"tables": [{
+	//		"thscode": "CU2407.SHF",
+	//		"time": ["2024-07-01 10:00", "2024-07-01 11:15", "2024-07-01 14:15", "2024-07-01 15:00"],
+	//		"table": {
+	//			"open": [77930.000000, 77980.000000, 77910.000000, 77850.000000],
+	//			"close": [77980.000000, 77920.000000, 77850.000000, 77780.000000]
+	//		}
+	//	}, {
+	//		"thscode": "CU2408.SHF",
+	//		"time": ["2024-07-01 10:00", "2024-07-01 11:15", "2024-07-01 14:15", "2024-07-01 15:00"],
+	//		"table": {
+	//			"open": [78180.000000, 78280.000000, 78220.000000, 78110.000000],
+	//			"close": [78280.000000, 78220.000000, 78110.000000, 78060.000000]
+	//		}
+	//	}]
+	// }
+
+	// Tables中的每一个对应一个证券代码
+	for _, v := range apiResp.Tables {
+		if len(v.Time) == 0 || len(v.Table) == 0 {
+			continue
+		}
+		// Table中的K-V对应指标代码-数据值序列
+		for tk, tv := range v.Table {
+			index := new(models.ThsHfIndexWithData)
+			index.StockCode = v.ThsCode
+			index.EdbCode = tk
+			td := make([]*models.ThsHfIndexData, 0)
+			tvl := len(tv)
+			for k, t := range v.Time {
+				if k >= tvl {
+					continue
+				}
+				dt, e := time.ParseInLocation("2006-01-02 15:04", t, time.Local)
+				if e != nil {
+					utils.FileLog.Info(fmt.Sprintf("同花顺API-time parse t: %s, err: %v", t, e))
+					continue
+				}
+				td = append(td, &models.ThsHfIndexData{
+					DataTime: dt,
+					Value:    tv[k],
+				})
+			}
+			index.IndexData = td
+			indexes = append(indexes, index)
+		}
+	}
+	return
+}
+
+// getEdbDataFromThsHfApp 公用机-获取高频指标数据
+func getEdbDataFromThsHfApp(thsParams models.ThsHfSearchEdbReq, num int, serverUrl string) (indexes []*models.ThsHfIndexWithData, err error) {
+	var requestUrl string
+	defer func() {
+		if err != nil {
+			utils.FileLog.Info(fmt.Sprintf("requestUrl: %s", requestUrl))
+			utils.FileLog.Info(fmt.Sprintf("getEdbDataFromThsHfApp: %v", err))
+		}
+	}()
+	//serverUrl = "http://wxmsgsen1.hzinsights.com:8040/"
+	baseUrl := fmt.Sprintf("%s%s", serverUrl, "edbInfo/ths/hf?")
+
+	// 额外参数
+	var funcParam string
+	if thsParams.Interval > 0 {
+		funcParam += fmt.Sprintf("Interval:%d,", thsParams.Interval)
+	}
+	if thsParams.Fill != "" {
+		funcParam += fmt.Sprintf("Fill:%s,", thsParams.Fill)
+	}
+	if thsParams.CPS != "" {
+		funcParam += fmt.Sprintf("CPS:%s,", thsParams.CPS)
+	}
+	if thsParams.BaseDate != "" {
+		funcParam += fmt.Sprintf("BaseDate:%s,", thsParams.BaseDate)
+	}
+	funcParam = strings.TrimRight(funcParam, ",")
+
+	params := url.Values{}
+	params.Add("codes", thsParams.StockCode)
+	params.Add("indicators", thsParams.EdbCode)
+	params.Add("function_para", funcParam)
+	params.Add("start_time", thsParams.StartTime)
+	params.Add("end_time", thsParams.EndTime)
+
+	// 请求终端
+	requestUrl = baseUrl + params.Encode()
+	body, e := http.Get(requestUrl)
+	if e != nil {
+		err = fmt.Errorf("")
+		return
+	}
+	dataBody := strings.TrimLeft(string(body), `"`)
+	dataBody = strings.TrimRight(dataBody, `"`)
+	dataBody = strings.ReplaceAll(dataBody, `\`, ``)
+	//utils.FileLog.Info(dataBody)
+
+	appResp := new(TerminalResponse)
+	if e = json.Unmarshal([]byte(dataBody), &appResp); e != nil {
+		err = fmt.Errorf("同花顺APP-解析响应失败, %v", e)
+		return
+	}
+	if appResp.ErrorCode != 0 {
+		//如果是同花顺登录session失效了，那么就重新请求获取数据
+		if appResp.ErrorCode == -1020 && num == 0 {
+			return getEdbDataFromThsHfApp(thsParams, 1, serverUrl)
+		}
+		err = fmt.Errorf("同花顺APP-状态码: %d, 提示信息: %s", appResp.ErrorCode, appResp.ErrMsg)
+		return
+	}
+
+	// 响应结果示例
+	// {
+	//	"errorcode": 0,
+	//	"errmsg": "Success!",
+	//	"data": [{
+	//		"time": "2024-06-04 09:30",
+	//		"thscode": "CU2406.SHF",
+	//		"open": 81900.0,
+	//		"close": 81820.0
+	//	}, {
+	//		"time": "2024-06-04 10:00",
+	//		"thscode": "CU2406.SHF",
+	//		"open": 81820.0,
+	//		"close": 81790.0
+	//	}, {
+	//		"time": "2024-06-04 10:45",
+	//		"thscode": "CU2406.SHF",
+	//		"open": 81820.0,
+	//		"close": 81950.0
+	//	}]
+	// }
+
+	indexes = make([]*models.ThsHfIndexWithData, 0)
+	indexMap := make(map[string]*models.ThsHfIndexWithData)
+	for _, stockData := range appResp.Data {
+		strTime := stockData["time"].(string)
+		dataTime, e := time.ParseInLocation("2006-01-02 15:04", strTime, time.Local)
+		if e != nil {
+			utils.FileLog.Info("数据日期格式有误, time: %s, %v", strTime, e)
+			continue
+		}
+		stockCode := stockData["thscode"].(string)
+
+		// 指标代码+数据
+		for k, v := range stockData {
+			if k == "time" || k == "thscode" {
+				continue
+			}
+			if v == nil {
+				continue
+			}
+			val, ok := v.(float64)
+			if !ok {
+				continue
+			}
+			mk := fmt.Sprintf("%s-%s", stockCode, k)
+			if indexMap[mk] == nil {
+				indexMap[mk] = new(models.ThsHfIndexWithData)
+				indexMap[mk].StockCode = stockCode
+				indexMap[mk].EdbCode = k
+				indexMap[mk].IndexData = make([]*models.ThsHfIndexData, 0)
+			}
+			indexMap[mk].IndexData = append(indexMap[mk].IndexData, &models.ThsHfIndexData{
+				DataTime: dataTime,
+				Value:    val,
+			})
+		}
+	}
+	for _, v := range indexMap {
+		indexes = append(indexes, v)
+	}
+	return
+}
+
+// RefreshThsHfBaseIndex 源指标刷新
+func RefreshThsHfBaseIndex(indexItem *models.BaseFromThsHfIndex, codeWithData *models.ThsHfIndexWithData, startTime string) (err error) {
+	defer func() {
+		if err != nil {
+			tips := fmt.Sprintf("RefreshThsHfBaseIndex-更新失败, %v", err)
+			utils.FileLog.Info(tips)
+			go alarm_msg.SendAlarmMsg(tips, 3)
+		}
+	}()
+	if indexItem == nil {
+		err = fmt.Errorf("指标不存在")
+		return
+	}
+	if len(codeWithData.IndexData) == 0 {
+		return
+	}
+
+	// 获取源指标数据
+	dataOb := new(models.BaseFromThsHfData)
+	originData := make([]*models.BaseFromThsHfData, 0)
+	{
+		cond := fmt.Sprintf(" AND %s = ?", dataOb.Cols().IndexCode)
+		pars := make([]interface{}, 0)
+		pars = append(pars, indexItem.IndexCode)
+		if startTime != "" {
+			cond += fmt.Sprintf(" AND %s >= ?", dataOb.Cols().DataTime)
+			pars = append(pars, startTime)
+		}
+		list, e := dataOb.GetItemsByCondition(cond, pars, []string{}, "")
+		if e != nil {
+			err = fmt.Errorf("获取源指标数据失败, %v", e)
+			return
+		}
+		originData = list
+	}
+
+	// 更新指标数据
+	dateExist := make(map[string]*models.BaseFromThsHfData)
+	newValExist := make(map[string]bool)
+	if len(originData) > 0 {
+		// unicode去重
+		for _, d := range originData {
+			dateExist[d.UniqueCode] = d
+		}
+	}
+
+	// 筛选新增/更新数据
+	updateData := make([]*models.BaseFromThsHfData, 0)
+	insertData := make([]*models.BaseFromThsHfData, 0)
+	for _, d := range codeWithData.IndexData {
+		uni := utils.MD5(fmt.Sprint(indexItem.IndexCode, d.DataTime.Format(utils.FormatDateTimeMinute)))
+		origin := dateExist[uni]
+
+		// unicode检验是否存在
+		strNewVal := decimal.NewFromFloat(d.Value).Round(4).String()
+		di, _ := decimal.NewFromString(strNewVal)
+		newVal, _ := di.Float64()
+		if origin != nil {
+			strExistVal := decimal.NewFromFloat(origin.Value).Round(4).String()
+			if strNewVal == strExistVal {
+				continue
+			}
+			origin.Value = newVal
+			origin.ModifyTime = time.Now().Local()
+			updateData = append(updateData, origin)
+		}
+
+		// 新增的数据去重
+		if newValExist[uni] {
+			continue
+		}
+		newValExist[uni] = true
+		newData := new(models.BaseFromThsHfData)
+		newData.BaseFromThsHfIndexId = indexItem.BaseFromThsHfIndexId
+		newData.IndexCode = indexItem.IndexCode
+		newData.DataTime = d.DataTime
+		newData.Value = newVal
+		newData.CreateTime = time.Now()
+		newData.ModifyTime = time.Now()
+		newData.UniqueCode = uni
+		newData.DataTimestamp = d.DataTime.UnixNano() / 1e6
+		insertData = append(insertData, newData)
+	}
+	if e := dataOb.MultiInsertOrUpdate(insertData, updateData); e != nil {
+		err = fmt.Errorf("新增/更新源指标数据失败, %v", e)
+		return
+	}
+
+	// 更新指标开始结束时间
+	minMax, e := dataOb.GetIndexMinMax(indexItem.IndexCode)
+	if e == nil && minMax != nil {
+		minDate, e := time.ParseInLocation(utils.FormatDateTime, minMax.MinDate, time.Local)
+		if e != nil {
+			err = fmt.Errorf("源数据最小日期有误, MinDate: %s, %v", minMax.MinDate, e)
+			return
+		}
+		maxDate, e := time.ParseInLocation(utils.FormatDateTime, minMax.MaxDate, time.Local)
+		if e != nil {
+			err = fmt.Errorf("源数据最大日期有误, MaxDate: %s, %v", minMax.MaxDate, e)
+			return
+		}
+		indexItem.StartDate = minDate
+		indexItem.EndDate = maxDate
+		indexItem.ModifyTime = time.Now().Local()
+		updateCols := []string{indexItem.Cols().StartDate, indexItem.Cols().EndDate, indexItem.Cols().ModifyTime}
+		if e = indexItem.Update(updateCols); e != nil {
+			err = fmt.Errorf("更新源指标开始结束时间失败, %v", e)
+			return
+		}
+	}
+
+	// 同步刷新指标库
+	go func() {
+		_ = RefreshEdbFromThsHfBaseIndex(indexItem.IndexCode, startTime)
+	}()
+	return
+}
+
+// RefreshThsHfBaseIndexMgo 源指标刷新-Mongo
+func RefreshThsHfBaseIndexMgo(indexItem *models.BaseFromThsHfIndex, codeWithData *models.ThsHfIndexWithData, startTime string) (err error) {
+	defer func() {
+		if err != nil {
+			tips := fmt.Sprintf("RefreshThsHfBaseIndexMgo-更新失败, %v", err)
+			utils.FileLog.Info(tips)
+			go alarm_msg.SendAlarmMsg(tips, 3)
+		}
+	}()
+	if indexItem == nil {
+		err = fmt.Errorf("指标不存在")
+		return
+	}
+	if len(codeWithData.IndexData) == 0 {
+		return
+	}
+	mogDataObj := new(mgo.BaseFromThsHfData)
+
+	// 获取已存在的所有数据
+	existCond := bson.M{
+		"index_code": indexItem.IndexCode,
+	}
+	if startTime != "" {
+		st, e := time.ParseInLocation(utils.FormatDateTime, startTime, time.Local)
+		if e != nil {
+			err = fmt.Errorf("start time parse err: %v", e)
+			return
+		}
+		existCond["data_time"] = bson.M{
+			"$gte": st,
+		}
+	}
+	exitDataList, e := mogDataObj.GetAllDataList(existCond, []string{"data_time"})
+	if e != nil {
+		err = fmt.Errorf("GetAllDataList err: %v", e)
+		return
+	}
+
+	// 已经存在的数据集
+	exitDataMap := make(map[string]*mgo.BaseFromThsHfData)
+	for _, v := range exitDataList {
+		exitDataMap[v.UniqueCode] = v
+	}
+
+	// 待添加的数据集
+	addDataList := make([]interface{}, 0)
+	updateDataList := make([]mgo.BaseFromThsHfData, 0)
+	for _, data := range codeWithData.IndexData {
+		strNewVal := decimal.NewFromFloat(data.Value).Round(4).String()
+		di, _ := decimal.NewFromString(strNewVal)
+		newVal, _ := di.Float64()
+
+		// unicode检验是否存在
+		uni := utils.MD5(fmt.Sprint(indexItem.IndexCode, data.DataTime.Format(utils.FormatDateTimeMinute)))
+		findData, ok := exitDataMap[uni]
+		if !ok {
+			addDataList = append(addDataList, mgo.BaseFromThsHfData{
+				BaseFromThsHfIndexId: int64(indexItem.BaseFromThsHfIndexId),
+				IndexCode:            indexItem.IndexCode,
+				DataTime:             data.DataTime,
+				Value:                newVal,
+				UniqueCode:           uni,
+				CreateTime:           time.Now(),
+				ModifyTime:           time.Now(),
+				DataTimestamp:        data.DataTime.UnixNano() / 1e6,
+			})
+			continue
+		}
+
+		// 值不匹配，修改数据
+		strExistVal := decimal.NewFromFloat(findData.Value).Round(4).String()
+		if strNewVal == strExistVal {
+			continue
+		}
+		findData.Value = newVal
+		updateDataList = append(updateDataList, *findData)
+	}
+
+	// 入库
+	{
+		coll := mogDataObj.GetCollection()
+		if len(addDataList) > 0 {
+			if e = mogDataObj.BatchInsertDataByColl(coll, 500, addDataList); e != nil {
+				err = fmt.Errorf("BatchInsertDataByColl, err: %v", e)
+				return
+			}
+		}
+
+		if len(updateDataList) > 0 {
+			for _, v := range updateDataList {
+				if e = mogDataObj.UpdateDataByColl(coll, bson.M{"_id": v.ID}, bson.M{"$set": bson.M{"value": v.Value, "modify_time": v.ModifyTime}}); e != nil {
+					err = fmt.Errorf("UpdateDataByColl, err: %v", e)
+					return
+				}
+			}
+		}
+	}
+
+	// 修改最大最小日期
+	minMax, err := indexItem.GetEdbInfoMaxAndMinInfo(indexItem.IndexCode)
+	if err != nil {
+		return
+	}
+	if err == nil && minMax != nil {
+		minDate, e := time.ParseInLocation(utils.FormatDateTime, minMax.MinDate, time.Local)
+		if e != nil {
+			err = fmt.Errorf("源数据最小日期有误, MinDate: %s, %v", minMax.MinDate, e)
+			return
+		}
+		maxDate, e := time.ParseInLocation(utils.FormatDateTime, minMax.MaxDate, time.Local)
+		if e != nil {
+			err = fmt.Errorf("源数据最大日期有误, MaxDate: %s, %v", minMax.MaxDate, e)
+			return
+		}
+		indexItem.StartDate = minDate
+		indexItem.EndDate = maxDate
+		indexItem.ModifyTime = time.Now().Local()
+		updateCols := []string{indexItem.Cols().StartDate, indexItem.Cols().EndDate, indexItem.Cols().ModifyTime}
+		if e = indexItem.Update(updateCols); e != nil {
+			err = fmt.Errorf("更新源指标开始结束时间失败, %v", e)
+			return
+		}
+	}
+
+	// 同步刷新指标库
+	go func() {
+		_ = RefreshEdbFromThsHfBaseIndex(indexItem.IndexCode, startTime)
+	}()
+	return
+}
+
+// RefreshEdbFromThsHfBaseIndex 根据源指标刷新指标库
+func RefreshEdbFromThsHfBaseIndex(baseCode, startTime string) (err error) {
+	defer func() {
+		if err != nil {
+			tips := fmt.Sprintf("RefreshEdbFromThsHfBaseIndex-刷新指标库失败, %v", err)
+			utils.FileLog.Info(tips)
+			go alarm_msg.SendAlarmMsg(tips, 3)
+		}
+	}()
+
+	// 获取指标关联信息
+	mappings := make([]*models.BaseFromEdbMapping, 0)
+	{
+		ob := new(models.BaseFromEdbMapping)
+		cond := fmt.Sprintf(" AND %s = ?", ob.Cols().BaseIndexCode)
+		pars := make([]interface{}, 0)
+		pars = append(pars, baseCode)
+		list, e := ob.GetItemsByCondition(cond, pars, []string{}, "")
+		if e != nil {
+			err = fmt.Errorf("获取源指标关联失败, %v", e)
+			return
+		}
+		mappings = list
+	}
+	if len(mappings) == 0 {
+		return
+	}
+	codeMapping := make(map[string]*models.BaseFromEdbMapping)
+	edbInfoIds := make([]int, 0)
+	for _, v := range mappings {
+		if codeMapping[v.EdbCode] == nil {
+			codeMapping[v.EdbCode] = v
+		}
+		edbInfoIds = append(edbInfoIds, v.EdbInfoId)
+	}
+
+	// 指标信息
+	edbInfoList, e := models.GetEdbInfoByIdList(edbInfoIds)
+	if e != nil {
+		err = fmt.Errorf("获取指标信息列表失败, %v", e)
+		return
+	}
+	codeEdb := make(map[string]*models.EdbInfo)
+	for _, v := range edbInfoList {
+		if codeEdb[v.EdbCode] == nil {
+			codeEdb[v.EdbCode] = v
+		}
+	}
+
+	thsOb := new(models.EdbThsHf)
+	source := thsOb.GetSource()
+	subSource := thsOb.GetSubSource()
+	for _, v := range mappings {
+		cacheKey := fmt.Sprintf("%s_%d_%d_%s", utils.CACHE_EDB_DATA_REFRESH, source, subSource, v.EdbCode)
+		if utils.Rc.IsExist(cacheKey) {
+			continue
+		}
+		utils.Rc.SetNX(cacheKey, 1, 1*time.Minute)
+
+		edb := codeEdb[v.EdbCode]
+		if edb == nil {
+			utils.FileLog.Info(fmt.Sprintf("RefreshEdbFromThsHfBaseIndex-指标信息有误, EdbCode: %s", v.EdbCode))
+			continue
+		}
+
+		// 刷新指标
+		if e = thsOb.Refresh(edb, codeMapping[v.EdbCode], startTime); e != nil {
+			utils.FileLog.Info(fmt.Sprintf("RefreshEdbFromThsHfBaseIndex-刷新指标失败, %v", e))
+			_ = utils.Rc.Delete(cacheKey)
+			continue
+		}
+
+		// 更新指标最值
+		if e = thsOb.UnifiedModifyEdbInfoMaxAndMinInfo(edb); e != nil {
+			utils.FileLog.Info(fmt.Sprintf("RefreshEdbFromThsHfBaseIndex-更新指标最值失败, %v", e))
+			_ = utils.Rc.Delete(cacheKey)
+			return
+		}
+		_ = utils.Rc.Delete(cacheKey)
+
+		// 更新ES
+		go logic.UpdateEs(edb.EdbInfoId)
+	}
+	return
+}

services/edb_info_relation.go

@@ -0,0 +1,381 @@
+package services
+
+import (
+	"eta/eta_index_lib/models"
+	"eta/eta_index_lib/services/alarm_msg"
+	"eta/eta_index_lib/utils"
+	"fmt"
+	"sort"
+	"strconv"
+	"time"
+)
+
+// 重置单个计算指标中的引用关系
+func ResetEdbRelation(edbInfoId int) {
+	var logMsg string
+	var replaceTotal int
+	var err error
+	defer func() {
+		if err != nil {
+			msg := fmt.Sprintf(" 重置单个计算指标中的引用关系失败 ResetEdbRelation  err: %v", err)
+			utils.FileLog.Info(msg)
+			fmt.Println(msg)
+			go alarm_msg.SendAlarmMsg(msg, 3)
+		}
+		if logMsg != `` {
+			utils.FileLog.Info(fmt.Sprintf("重置单个计算指标中的引用关系失败 重置总数%d,涉及到的引用id：%s", replaceTotal, logMsg))
+		}
+	}()
+	//查询与该计算指标相关的间接引用或者间接引用关系，如果记录不存在，则不处理
+	_, err = models.GetEdbInfoRelationByChildEdbInfoId(edbInfoId)
+	if err != nil {
+		if err.Error() == utils.ErrNoRow() {
+			err = nil
+			return
+		}
+		err = fmt.Errorf("查询与该计算指标相关的间接引用或者间接引用关系失败，错误信息：%s", err.Error())
+		return
+	}
+
+	//查询当前计算指标最新的引用指标列表
+	newMappingList, err := models.GetEdbInfoCalculateDetailList(edbInfoId)
+	if err != nil {
+		err = fmt.Errorf("查询当前计算指标最新的指标列表失败，错误信息：%s", err.Error())
+		return
+	}
+	//整理关联的来源指标ID
+	newEdbIdList := make([]string, 0)
+	newMappingListMap := make(map[int]*models.EdbInfoCalculateDetail)
+	for _, v := range newMappingList {
+		newEdbIdList = append(newEdbIdList, strconv.Itoa(v.FromEdbInfoId))
+		newMappingListMap[v.FromEdbInfoId] = v
+	}
+	//对指标ID进行排序
+	sort.Strings(newEdbIdList)
+	newEdbIdStr := ""
+	for _, v := range newEdbIdList {
+		newEdbIdStr += v + ","
+	}
+	//二者匹配一下，如果相同，则不处理，如果不同，先查询所有旧的间接引用记录，整理并分组，则删除旧的间接引用记录，新增新的间接引用记录，
+	tmpList, err := models.GetEdbInfoRelationListByChildEdbInfoId(edbInfoId)
+	if err != nil {
+		err = fmt.Errorf("查询当前计算指标的间接引用关系失败，错误信息：%s", err.Error())
+		return
+	}
+
+	parentRelationIds := make([]int, 0)
+	for _, v := range tmpList {
+		parentRelationIds = append(parentRelationIds, v.ParentRelationId)
+	}
+	if len(parentRelationIds) > 0 {
+		// 查询单个项目的引用列表作为判断依据
+		oldEdbIdList, err := models.GetEdbInfoRelationEdbIdsByParentRelationId(parentRelationIds[0], edbInfoId)
+		if err != nil {
+			err = fmt.Errorf("查询当前计算指标的间接引用关系失败，错误信息：%s", err.Error())
+			return
+		}
+		sort.Ints(oldEdbIdList)
+		oldEdbIdStr := ""
+		for _, v := range oldEdbIdList {
+			oldEdbIdStr += strconv.Itoa(v) + ","
+		}
+		// 把切片转成字符串
+		if newEdbIdStr == oldEdbIdStr {
+			return
+		}
+		list, e := models.GetEdbInfoRelationByRelationIds(parentRelationIds)
+		if e != nil {
+			err = fmt.Errorf("查询图表关联指标列表失败 Err:%s", e)
+			return
+		}
+		//查询直接引用指标关联关系
+		edbInfoListMap := make(map[int]struct{})
+		edbInfoIds := make([]int, 0)
+		for _, v := range list {
+			if _, ok := edbInfoListMap[v.EdbInfoId]; !ok {
+				edbInfoListMap[v.EdbInfoId] = struct{}{}
+				edbInfoIds = append(edbInfoIds, v.EdbInfoId)
+			}
+		}
+		edbInfoList := make([]*models.EdbInfo, 0)
+		if len(edbInfoIds) > 0 {
+			// 查询指标信息
+			edbInfoList, err = models.GetEdbInfoByIdList(edbInfoIds)
+			if err != nil {
+				err = fmt.Errorf("查询指标信息失败 Err:%s", err)
+				return
+			}
+		}
+		calculateEdbMappingListMap, calculateEdbMappingIdsMap, err := GetEdbListByEdbInfoId(edbInfoList)
+		if err != nil {
+			err = fmt.Errorf("查询指标关联指标列表失败 Err:%s", err)
+			return
+		}
+		//如何过滤掉只有间接引用，没有直接引用的
+		replaceTotal1, logMsg1, e := UpdateSecondEdbInRelation(list, calculateEdbMappingListMap, calculateEdbMappingIdsMap, edbInfoList)
+		if e != nil {
+			err = e
+			return
+		}
+		replaceTotal += replaceTotal1
+		logMsg += logMsg1
+	}
+	return
+}
+
+// 更新间接引用
+func UpdateSecondEdbInRelation(list []*models.EdbInfoRelation, calculateEdbMappingListMap map[int]*models.EdbInfoCalculateMapping, calculateEdbMappingIdsMap map[int][]int, edbInfoList []*models.EdbInfo) (replaceTotal int, logMsg string, err error) {
+	nowTime := time.Now()
+	edbInfoRelationIds := make([]int, 0)
+	indexCodeList := make([]string, 0)
+	addList := make([]*models.EdbInfoRelation, 0)
+	refreshIds := make([]int, 0)
+	edbInfoMap := make(map[int]*models.EdbInfo)
+	for _, v := range edbInfoList {
+		edbInfoMap[v.EdbInfoId] = v
+	}
+	// 查询所有的直接引用，删除所有的间接引用，添加所有直接引用的间接引用
+	for _, v := range list {
+		if v.RelationType == 0 {
+			edbInfoRelationIds = append(edbInfoRelationIds, v.EdbInfoRelationId)
+			edbInfo, ok := edbInfoMap[v.EdbInfoId]
+			if !ok {
+				err = fmt.Errorf("查询指标信息失败 EdbInfoId:%d", v.EdbInfoId)
+				return
+			}
+			if edbInfo.EdbType == 2 { //计算指标
+				childEdbMappingIds, ok := calculateEdbMappingIdsMap[edbInfo.EdbInfoId]
+				if !ok {
+					err = fmt.Errorf("查询%d指标关联指标列表为空", edbInfo.EdbInfoId)
+					return
+				}
+				for _, childEdbMappingId := range childEdbMappingIds {
+					childEdbMapping, ok2 := calculateEdbMappingListMap[childEdbMappingId]
+					if !ok2 {
+						continue
+					}
+
+					if childEdbMapping.FromSource == utils.DATA_SOURCE_MYSTEEL_CHEMICAL {
+						indexCodeList = append(indexCodeList, childEdbMapping.FromEdbCode)
+					}
+					tmp1 := &models.EdbInfoRelation{
+						ReferObjectId:      v.ReferObjectId,
+						ReferObjectType:    v.ReferObjectType,
+						ReferObjectSubType: v.ReferObjectSubType,
+						EdbInfoId:          childEdbMapping.FromEdbInfoId,
+						EdbName:            childEdbMapping.FromEdbName,
+						Source:             childEdbMapping.FromSource,
+						EdbCode:            childEdbMapping.FromEdbCode,
+						CreateTime:         nowTime,
+						ModifyTime:         nowTime,
+						RelationTime:       childEdbMapping.CreateTime,
+						RelationType:       1,
+						RootEdbInfoId:      edbInfo.EdbInfoId,
+						ChildEdbInfoId:     childEdbMapping.EdbInfoId,
+					}
+					tmp1.RelationCode = fmt.Sprintf("%d_%d_%d_%d", tmp1.RootEdbInfoId, tmp1.ReferObjectId, tmp1.ReferObjectType, tmp1.ReferObjectSubType)
+					addList = append(addList, tmp1)
+					refreshIds = append(refreshIds, childEdbMapping.FromEdbInfoId)
+				}
+			}
+		}
+	}
+
+	if len(edbInfoRelationIds) > 0 {
+		err = models.UpdateSecondRelationEdbInfoId(edbInfoRelationIds, addList, refreshIds, indexCodeList)
+		if err != nil {
+			logMsg = ""
+			err = fmt.Errorf("替换指标引用表中的指标ID失败 Err:%s", err)
+			return
+		}
+		replaceTotal = len(edbInfoRelationIds)
+	}
+	return
+}
+
+// 设置成禁用状态
+func DisableEdbInfoNoUpdate(edbInfo *models.EdbInfo) (err error) {
+	// 如果一个计算指标里，包涵的基础指标是停用状态，那么计算指标也要是停用状态停用状态
+	newBaseEdbInfoList := make([]int, 0)
+	hasFind := make(map[int]struct{})
+	newBaseEdbInfoIds, err := FindBaseEdbInfo(edbInfo.EdbInfoId, newBaseEdbInfoList, hasFind)
+	if err != nil {
+		err = fmt.Errorf("查找基础指标信息失败,err:%v", err)
+		return
+	}
+	// 查询是否存在停用指标，如果存在，则计算指标也要是停用状态
+	total, err := models.GetEdbInfoNoUpdateTotalByIdList(newBaseEdbInfoIds)
+	if err != nil {
+		err = fmt.Errorf("查询基础指标信息失败,err:%v", err)
+		return
+	}
+	if total > 0 {
+		edbInfo.NoUpdate = 1
+		edbInfo.ModifyTime = time.Now()
+		err = edbInfo.Update([]string{"NoUpdate", "ModifyTime"})
+		if err != nil {
+			err = fmt.Errorf("更新计算指标刷新状态失败,err:%v", err)
+			return
+		}
+	}
+	return
+}
+
+// 找到基础指标的过程
+func FindBaseEdbInfo(edbInfoId int, baseEdbInfoList []int, hasFind map[int]struct{}) (newBaseEdbInfoList []int, err error) {
+	newBaseEdbInfoList = baseEdbInfoList
+	if _, ok := hasFind[edbInfoId]; ok {
+		return
+	}
+	// 先找到所有的引用关系
+	//查询当前计算指标最新的引用指标列表
+	newMappingList, err := models.GetEdbInfoCalculateDetailList(edbInfoId)
+	if err != nil {
+		err = fmt.Errorf("查询当前计算指标最新的指标列表失败，错误信息：%s", err.Error())
+		return
+	}
+	hasFind[edbInfoId] = struct{}{}
+	for _, mapping := range newMappingList {
+		newBaseEdbInfoList = append(newBaseEdbInfoList, mapping.FromEdbInfoId)
+		if mapping.EdbType == 1 { // 如果是基础指标，则加入，否则继续找
+		} else {
+			newBaseEdbInfoList, err = FindBaseEdbInfo(mapping.FromEdbInfoId, newBaseEdbInfoList, hasFind)
+		}
+	}
+	return
+}
+
+// 查找当前计算指标的所有溯源指标
+func GetEdbListByEdbInfoId(edbInfoList []*models.EdbInfo) (edbMappingListMap map[int]*models.EdbInfoCalculateMapping, edbInfoMappingRootIdsMap map[int][]int, err error) {
+	if len(edbInfoList) == 0 {
+		return
+	}
+	edbInfoIds := make([]int, 0)
+	for _, v := range edbInfoList {
+		if v.EdbType == 2 && v.EdbInfoType == 0 { //普通计算指标，排除预算指标
+			edbInfoIds = append(edbInfoIds, v.EdbInfoId)
+		}
+	}
+	if len(edbInfoIds) == 0 {
+		return
+	}
+	//查询指标信息
+	allEdbMappingMap := make(map[int][]*models.EdbInfoCalculateMappingInfo, 0)
+	allMappingList, e := models.GetEdbInfoCalculateMappingListByEdbInfoIds(edbInfoIds)
+	if e != nil {
+		err = fmt.Errorf("GetEdbInfoCalculateMappingListByEdbInfoIds err: %s", e.Error())
+		return
+	}
+	for _, v := range allMappingList {
+		if _, ok := allEdbMappingMap[v.EdbInfoId]; !ok {
+			allEdbMappingMap[v.EdbInfoId] = make([]*models.EdbInfoCalculateMappingInfo, 0)
+		}
+		allEdbMappingMap[v.EdbInfoId] = append(allEdbMappingMap[v.EdbInfoId], v)
+	}
+	//查询指标映射
+	//查询所有指标数据
+	//查询这个指标相关的mapping信息放到数组里，
+	//将得到的指标ID信息放到数组里
+	hasFindMap := make(map[int]struct{})
+	edbInfoIdMap := make(map[int]struct{})
+	edbMappingList := make([]*models.EdbInfoCalculateMapping, 0)
+	edbInfoMappingRootIdsMap = make(map[int][]int, 0)
+	edbMappingMap := make(map[int]struct{})
+	for _, edbInfo := range edbInfoList {
+		if edbInfo.EdbType == 2 && edbInfo.EdbInfoType == 0 {
+			edbInfoId := edbInfo.EdbInfoId
+			edbMappingList, err = getCalculateEdbInfoByEdbInfoId(allEdbMappingMap, edbInfoId, hasFindMap, edbInfoIdMap, edbMappingList, edbMappingMap, edbInfoMappingRootIdsMap, edbInfoId)
+			if err != nil {
+				err = fmt.Errorf(" GetCalculateEdbInfoByEdbInfoId err: %s", err.Error())
+				return
+			}
+		}
+	}
+	if len(edbMappingList) == 0 {
+		return
+	}
+	// 查询指标信息
+	// 指标信息map
+	edbInfoIdList := make([]int, 0)
+	for k, _ := range edbInfoIdMap {
+		edbInfoIdList = append(edbInfoIdList, k)
+	}
+	edbMappingListMap = make(map[int]*models.EdbInfoCalculateMapping)
+
+	if len(edbMappingList) > 0 {
+		for _, v := range edbMappingList {
+			edbMappingListMap[v.EdbInfoCalculateMappingId] = v
+		}
+	}
+	return
+}
+
+// getCalculateEdbInfoByEdbInfoId 计算指标追溯
+func getCalculateEdbInfoByEdbInfoId(allEdbMappingMap map[int][]*models.EdbInfoCalculateMappingInfo, edbInfoId int, hasFindMap map[int]struct{}, edbInfoIdMap map[int]struct{}, edbMappingList []*models.EdbInfoCalculateMapping, edbMappingMap map[int]struct{}, edbInfoMappingRootIdsMap map[int][]int, rootEdbInfoId int) (newEdbMappingList []*models.EdbInfoCalculateMapping, err error) {
+	newEdbMappingList = edbMappingList
+	_, ok := hasFindMap[edbInfoId]
+	if ok {
+		return
+	}
+
+	if _, ok1 := edbInfoIdMap[edbInfoId]; !ok1 {
+		edbInfoIdMap[edbInfoId] = struct{}{}
+	}
+	edbInfoMappingList := make([]*models.EdbInfoCalculateMappingInfo, 0)
+	edbInfoMappingList, ok = allEdbMappingMap[edbInfoId]
+	if !ok {
+		edbInfoMappingList, err = models.GetEdbInfoCalculateMappingListByEdbInfoId(edbInfoId)
+		if err != nil {
+			err = fmt.Errorf("GetEdbInfoCalculateMappingListByEdbInfoId err: %s", err.Error())
+			return
+		}
+	}
+	hasFindMap[edbInfoId] = struct{}{}
+	if len(edbInfoMappingList) > 0 {
+		fromEdbInfoIdList := make([]int, 0)
+		edbInfoMappingIdList := make([]int, 0)
+		for _, v := range edbInfoMappingList {
+			fromEdbInfoIdList = append(fromEdbInfoIdList, v.FromEdbInfoId)
+			edbInfoMappingIdList = append(edbInfoMappingIdList, v.EdbInfoCalculateMappingId)
+			if _, ok1 := edbInfoIdMap[v.FromEdbInfoId]; !ok1 {
+				edbInfoIdMap[v.FromEdbInfoId] = struct{}{}
+			}
+			if _, ok2 := edbMappingMap[v.EdbInfoCalculateMappingId]; !ok2 {
+				edbMappingMap[v.EdbInfoCalculateMappingId] = struct{}{}
+				tmp := &models.EdbInfoCalculateMapping{
+					EdbInfoCalculateMappingId: v.EdbInfoCalculateMappingId,
+					EdbInfoId:                 v.EdbInfoId,
+					Source:                    v.Source,
+					SourceName:                v.SourceName,
+					EdbCode:                   v.EdbCode,
+					FromEdbInfoId:             v.FromEdbInfoId,
+					FromEdbCode:               v.FromEdbCode,
+					FromEdbName:               v.FromEdbName,
+					FromSource:                v.FromSource,
+					FromSourceName:            v.FromSourceName,
+					FromTag:                   v.FromTag,
+					Sort:                      v.Sort,
+					CreateTime:                v.CreateTime,
+					ModifyTime:                v.ModifyTime,
+				}
+				newEdbMappingList = append(newEdbMappingList, tmp)
+
+			}
+
+			if edbInfoId != v.FromEdbInfoId && (v.FromEdbType == 2 || v.FromEdbInfoType == 1) {
+				// 查过了就不查了
+				if _, ok2 := hasFindMap[v.FromEdbInfoId]; !ok2 {
+					newEdbMappingList, err = getCalculateEdbInfoByEdbInfoId(allEdbMappingMap, v.FromEdbInfoId, hasFindMap, edbInfoIdMap, newEdbMappingList, edbMappingMap, edbInfoMappingRootIdsMap, rootEdbInfoId)
+					if err != nil {
+						err = fmt.Errorf("traceEdbInfoByEdbInfoId err: %s", err.Error())
+						return
+					}
+				}
+			}
+			hasFindMap[v.FromEdbInfoId] = struct{}{}
+		}
+		edbInfoMappingRootIdsMap[rootEdbInfoId] = append(edbInfoMappingRootIdsMap[rootEdbInfoId], edbInfoMappingIdList...)
+	}
+
+	return
+}

utils/common.go

@@ -1310,3 +1310,18 @@ func InsertStr2StrIdx(str, sep string, idx int, value string) string {
 	slice = append(slice[:idx], append([]string{value}, slice[idx:]...)...)
 	return strings.Join(slice, sep)
 }
+
+// FormatFloatPlaces 格式化浮点数位数
+func FormatFloatPlaces(val float64, places int32) (newVal float64, err error) {
+	if places <= 0 {
+		places = 4
+	}
+	strNewVal := decimal.NewFromFloat(val).Round(places).String()
+	di, e := decimal.NewFromString(strNewVal)
+	if e != nil {
+		err = fmt.Errorf("NewFromString err: %v", e)
+		return
+	}
+	newVal, _ = di.Float64()
+	return
+}

utils/config.go

@@ -11,11 +11,12 @@ import (
 )
 
 var (
-	RunMode       string //运行模式
-	MYSQL_URL     string //数据库连接
-	MYSQL_URL_EDB string
-	MYSQL_URL_GL  string
-	MgoUrlData    string // mongodb数据库连接配置
+	RunMode          string //运行模式
+	MYSQL_URL        string //数据库连接
+	MYSQL_URL_EDB    string
+	MYSQL_URL_GL     string
+	MYSQL_URL_MASTER string
+	MgoUrlData       string // mongodb数据库连接配置
 
 	PYTHON_MYSQL_HOST           string // python数据库链接主机地址
 	PYTHON_MYSQL_USER           string // python数据库链接账号
@@ -105,6 +106,10 @@ var (
 	LogMaxDays int //日志最大保留天数
 )
 
+var (
+	UseMongo bool // 是否使用mongo
+)
+
 func init() {
 	tmpRunMode, err := web.AppConfig.String("run_mode")
 	if err != nil {
@@ -134,6 +139,7 @@ func init() {
 	MYSQL_URL = config["mysql_url"]
 	MYSQL_URL_EDB = config["mysql_url_edb"]
 	MYSQL_URL_GL = config["mysql_url_gl"]
+	MYSQL_URL_MASTER = config["mysql_url_master"]
 
 	// mongodb数据库连接配置
 	MgoUrlData = config["mgo_url_data"]

utils/constants.go

@@ -8,6 +8,7 @@ const (
 	FormatDate                 = "2006-01-02"              //日期格式
 	FormatDateUnSpace          = "20060102"                //日期格式
 	FormatDateTime             = "2006-01-02 15:04:05"     //完整时间格式
+	FormatDateTimeMinute       = "2006-01-02 15:04"        //完整时间格式
 	HlbFormatDateTime          = "2006-01-02_15:04:05.999" //完整时间格式
 	FormatDateTimeUnSpace      = "20060102150405"          //完整时间格式
 	FormatShortDateTimeUnSpace = "060102150405"            //省去开头两位年份的时间格式
@@ -227,6 +228,8 @@ const (
 	CACHE_EDB_TERMINAL_CODE_GOOD_URL = "edb:terminal_code:good:edb_code:" // 指标与终端关系的缓存, 商品期货
 	CACHE_EDB_THS_SERVER_TOKEN       = "edb:ths_server_token:"            //同花顺调用凭证
 	CACHE_SELF_EDB_HANDLE            = "CACHE_SELF_EDB_HANDLE:"           // 自定义指标缓存
+	CACHE_BASE_EDB_ADD               = "CACHE_BASE_EDB_ADD_"              // 添加至数据源缓存
+	CACHE_BASE_EDB_REFRESH           = "CACHE_BASE_EDB_REFRESH_"          // 刷新数据源缓存
 )
 
 // 图表类型
@@ -279,8 +282,9 @@ const (
 
 // 子数据来源渠道
 const (
-	DATA_SUB_SOURCE_EDB  = iota //经济数据库
-	DATA_SUB_SOURCE_DATE        //日期序列
+	DATA_SUB_SOURCE_EDB            = iota //经济数据库
+	DATA_SUB_SOURCE_DATE                  //日期序列
+	DATA_SUB_SOURCE_HIGH_FREQUENCY        //高频数据
 )
 
 // 语言版本
@@ -294,6 +298,12 @@ const (
 	PercentCalculateTypeNum   = 1 // 百分位算法类型-数据个数
 )
 
+const (
+	WindDbWsd = "wsd"
+	ThsDs     = "thsds"
+	ThsHf     = "thshf"
+)
+
 // 指标计算方式
 const (
 	EdbBaseCalculateLjzzy                = 1  // 累计值转月->1