Merge branch 'master' of http://8.136.199.33:3000/eta_server/eta_index_lib into bzq1/lib_record

.gitignore

@@ -13,4 +13,5 @@ eta_index_lib
 /etalogs
 *.gz
 *.exe
-/.vscode/
+test/
+/.vscode

controllers/base_from_bloomberg.go

@@ -149,12 +149,12 @@ func (this *BloombergController) Refresh() {
 	br.Msg = "获取成功"
 }
 
-// PCSGRefreshDaily
-// @Title 中石油新加坡-刷新日度指标
-// @Description  中石油新加坡-刷新日度指标
-// @Success 200 {object} models.AddEdbInfoReq
-// @router /pcsg/refresh_daily [post]
-func (this *BloombergController) PCSGRefreshDaily() {
+// PCSGImportHistoryData
+// @Title 中石油新加坡-导入历史数据
+// @Description  中石油新加坡-导入历史数据
+// @Success 200 {object} models.PCSGImportHistoryDataReq
+// @router /pcsg/import_history_data [post]
+func (this *BloombergController) PCSGImportHistoryData() {
 	br := new(models.BaseResponse).Init()
 	defer func() {
 		if br.ErrMsg == "" {
@@ -163,23 +163,28 @@ func (this *BloombergController) PCSGRefreshDaily() {
 		this.Data["json"] = br
 		this.ServeJSON()
 	}()
-
-	// 获取数据
-	indexes, e := services.GetPCSGBloombergDailyFromBridge()
-	if e != nil {
-		br.Msg = "刷新失败"
-		br.ErrMsg = "Bridge-获取PCSG彭博日度指标失败, Err: " + e.Error()
+	var req models.PCSGImportHistoryDataReq
+	if e := json.Unmarshal(this.Ctx.Input.RequestBody, &req); e != nil {
+		br.Msg = "参数解析异常！"
+		br.ErrMsg = "参数解析失败，Err:" + e.Error()
 		return
 	}
-	if len(indexes) == 0 {
-		br.Ret = 200
-		br.Success = true
-		br.Msg = "操作成功"
-		return
+
+	var indexes []models.BaseFromBloombergApiIndexAndData
+	var index models.BaseFromBloombergApiIndexAndData
+	var indexData []models.BaseFromBloombergApiIndexData
+	index.IndexCode = req.IndexCode
+	for k, v := range req.DataMap {
+		indexData = append(indexData, models.BaseFromBloombergApiIndexData{
+			DataTime: k,
+			Value:    v,
+		})
 	}
+	index.Data = indexData
+	indexes = append(indexes, index)
 
 	// 写入数据
-	if e = services.PCSGWrite2BaseBloomberg(indexes); e != nil {
+	if e := services.PCSGWrite2BaseBloomberg(indexes, req.IsVCode, req.ExtraLetter, ""); e != nil {
 		br.Msg = "刷新失败"
 		br.ErrMsg = "PCSG-写入Bloomberg数据源失败, Err: " + e.Error()
 		return
@@ -190,12 +195,12 @@ func (this *BloombergController) PCSGRefreshDaily() {
 	br.Msg = "操作成功"
 }
 
-// PCSGRefreshWeekly
-// @Title 中石油新加坡-刷新周度指标
-// @Description  中石油新加坡-刷新周度指标
+// PCSGRefreshTask
+// @Title 中石油新加坡-刷新任务
+// @Description  中石油新加坡-刷新任务
 // @Success 200 {object} models.AddEdbInfoReq
-// @router /pcsg/refresh_weekly [post]
-func (this *BloombergController) PCSGRefreshWeekly() {
+// @router /pcsg/refresh_task [post]
+func (this *BloombergController) PCSGRefreshTask() {
 	br := new(models.BaseResponse).Init()
 	defer func() {
 		if br.ErrMsg == "" {
@@ -204,67 +209,72 @@ func (this *BloombergController) PCSGRefreshWeekly() {
 		this.Data["json"] = br
 		this.ServeJSON()
 	}()
-
-	// 获取数据
-	indexes, e := services.GetPCSGBloombergWeeklyFromBridge()
-	if e != nil {
-		br.Msg = "刷新失败"
-		br.ErrMsg = "Bridge-获取PCSG彭博周度指标失败, Err: " + e.Error()
-		return
-	}
-	if len(indexes) == 0 {
-		br.Ret = 200
-		br.Success = true
-		br.Msg = "操作成功"
-		return
+	// 非必传, 只是手动请求的时候用
+	var req struct {
+		TaskKey string
 	}
+	_ = json.Unmarshal(this.Ctx.Input.RequestBody, &req)
 
-	// 写入数据
-	if e = services.PCSGWrite2BaseBloomberg(indexes); e != nil {
-		br.Msg = "刷新失败"
-		br.ErrMsg = "PCSG-写入Bloomberg数据源失败, Err: " + e.Error()
+	tasks, e := services.LoadPCSGBloombergTask()
+	if e != nil {
+		br.Msg = "加载配置失败"
+		br.ErrMsg = fmt.Sprintf("加载配置失败, Err: %v", e)
 		return
 	}
 
-	br.Ret = 200
-	br.Success = true
-	br.Msg = "操作成功"
-}
+	for _, v := range tasks {
+		if req.TaskKey != "" && v.TaskKey != req.TaskKey {
+			continue
+		}
+		fmt.Println(v)
+		time.Sleep(5 * time.Second)
 
-// PCSGRefreshMonthly
-// @Title 中石油新加坡-刷新月度指标
-// @Description  中石油新加坡-刷新周度指标
-// @Success 200 {object} models.AddEdbInfoReq
-// @router /pcsg/refresh_monthly [post]
-func (this *BloombergController) PCSGRefreshMonthly() {
-	br := new(models.BaseResponse).Init()
-	defer func() {
-		if br.ErrMsg == "" {
-			br.IsSendEmail = false
+		// 获取数据
+		var r services.PCSGBloombergApiReq
+		r.TaskKey = v.TaskKey
+		r.Frequency = v.Frequency
+		indexes, e := services.GetPCSGBloombergGeneralIndexFromBridge(r)
+		if e != nil {
+			br.Msg = "刷新失败"
+			br.ErrMsg = "Bridge-GetPCSGBloombergGeneralIndexFromBridge, Err: " + e.Error()
+			return
+		}
+		if len(indexes) == 0 {
+			br.Ret = 200
+			br.Success = true
+			br.Msg = "操作成功"
+			return
 		}
-		this.Data["json"] = br
-		this.ServeJSON()
-	}()
 
-	// 获取数据
-	indexes, e := services.GetPCSGBloombergMonthlyFromBridge()
-	if e != nil {
-		br.Msg = "刷新失败"
-		br.ErrMsg = "Bridge-获取PCSG彭博月度指标失败, Err: " + e.Error()
-		return
-	}
-	if len(indexes) == 0 {
-		br.Ret = 200
-		br.Success = true
-		br.Msg = "操作成功"
-		return
-	}
+		// 此处重新整理一下, 将同一指标的数据合并, 否则会使刷新时间变得很长
+		var newIndexes []models.BaseFromBloombergApiIndexAndData
+		codeMap := make(map[string]models.BaseFromBloombergApiIndexAndData)
+		indexCodeData := make(map[string][]models.BaseFromBloombergApiIndexData)
+		for _, iv := range indexes {
+			if codeMap[iv.IndexCode].IndexCode == "" {
+				var indexItem models.BaseFromBloombergApiIndexAndData
+				indexItem.IndexCode = iv.IndexCode
+				indexItem.IndexName = iv.IndexName
+				indexItem.Unit = iv.Unit
+				indexItem.Source = iv.Source
+				indexItem.Frequency = iv.Frequency
+				indexItem.CreateTime = iv.CreateTime
+				indexItem.ModifyTime = iv.ModifyTime
+				codeMap[iv.IndexCode] = indexItem
+			}
+			indexCodeData[iv.IndexCode] = append(indexCodeData[iv.IndexCode], iv.Data...)
+		}
+		for _, mv := range codeMap {
+			mv.Data = indexCodeData[mv.IndexCode]
+			newIndexes = append(newIndexes, mv)
+		}
 
-	// 写入数据
-	if e = services.PCSGWrite2BaseBloomberg(indexes); e != nil {
-		br.Msg = "刷新失败"
-		br.ErrMsg = "PCSG-写入Bloomberg数据源失败, Err: " + e.Error()
-		return
+		// 写入数据
+		if e = services.PCSGWrite2BaseBloomberg(newIndexes, v.VCode, v.ExtraLetter, v.IndexNamePrefix); e != nil {
+			br.Msg = "刷新失败"
+			br.ErrMsg = "PCSG-写入Bloomberg数据源失败, Err: " + e.Error()
+			return
+		}
 	}
 
 	br.Ret = 200

controllers/base_from_business.go

@@ -0,0 +1,493 @@
+package controllers
+
+import (
+	"encoding/json"
+	"eta/eta_index_lib/logic"
+	"eta/eta_index_lib/models"
+	"eta/eta_index_lib/services"
+	"eta/eta_index_lib/utils"
+	"fmt"
+	"strconv"
+	"strings"
+	"time"
+)
+
+// BusinessIndexController 自有数据（商家）
+type BusinessIndexController struct {
+	BaseAuthController
+}
+
+// HandleBusinessIndexData
+// @Title 处理外部指标（商家）的接口
+// @Description 处理外部指标（商家）的接口
+// @Success 200 {object} models.BaseFromBusinessIndexResp
+// @router /handle [post]
+func (c *BusinessIndexController) HandleBusinessIndexData() {
+	br := new(models.BaseResponse).Init()
+	defer func() {
+		c.Data["json"] = br
+		c.ServeJSON()
+	}()
+	body := c.Ctx.Input.RequestBody
+	var req models.AddBusinessIndexReq
+	err := json.Unmarshal(body, &req)
+	if err != nil {
+		br.Msg = "参数解析异常！"
+		br.ErrMsg = "参数解析失败，Err:" + err.Error()
+		return
+	}
+
+	// 如果没有传入指标编码的话，那么自动生成指标编码
+	if req.IndexCode == `` {
+		cacheKey := utils.CACHE_SELF_EDB_HANDLE
+		if utils.Rc.IsExist(cacheKey) {
+			br.Ret = 501
+			br.Success = true
+			br.Msg = "系统处理中，请稍后重试"
+			return
+		}
+
+		// 刷新指标
+		utils.Rc.SetNX(cacheKey, 1, 1*time.Minute)
+		defer func() {
+			_ = utils.Rc.Delete(cacheKey)
+		}()
+	}
+
+	businessIndex, err := services.HandleBusinessIndex(&req)
+	if err != nil {
+		fmt.Println("HandleBusinessIndexData Err:" + err.Error())
+		br.Msg = "处理失败"
+		br.ErrMsg = "处理失败,Err:" + err.Error()
+		return
+	}
+	br.Ret = 200
+	br.Success = true
+	br.Msg = "处理成功"
+	br.Data = businessIndex
+}
+
+// DelIndex
+// @Title 删除外部指标（商家）的接口
+// @Description 删除外部指标（商家）的接口
+// @Success 200 string "删除成功"
+// @router /index/del [post]
+func (c *BusinessIndexController) DelIndex() {
+	br := new(models.BaseResponse).Init()
+	defer func() {
+		c.Data["json"] = br
+		c.ServeJSON()
+	}()
+	body := c.Ctx.Input.RequestBody
+	var req models.DelBusinessIndexReq
+	err := json.Unmarshal(body, &req)
+	if err != nil {
+		br.Msg = "参数解析异常！"
+		br.ErrMsg = "参数解析失败，Err:" + err.Error()
+		return
+	}
+
+	// 如果没有传入指标编码的话，那么自动生成指标编码
+	if len(req.IndexCodeList) < 0 {
+		br.Msg = "指标编码不允许为空"
+		return
+	}
+
+	joinEdbCodeList, needDelEdbCodeList, err, errMsg := services.DelBusinessIndex(req.IndexCodeList)
+	if err != nil {
+		br.Msg = errMsg
+		br.ErrMsg = "删除失败,Err:" + err.Error()
+		return
+	}
+
+	resp := services.DelBusinessIndexResp{
+		IsDeleteEdbCodeList: needDelEdbCodeList,
+		NoDeleteEdbCodeList: joinEdbCodeList,
+	}
+
+	br.Ret = 200
+	br.Success = true
+	br.Msg = "删除成功"
+	br.Data = resp
+}
+
+// DelIndexData
+// @Title 删除外部指标（商家）的指定日期数据接口
+// @Description 删除外部指标（商家）的指定日期数据接口
+// @Success 200 string "删除成功"
+// @router /data/del [post]
+func (c *BusinessIndexController) DelIndexData() {
+	br := new(models.BaseResponse).Init()
+	defer func() {
+		c.Data["json"] = br
+		c.ServeJSON()
+	}()
+	body := c.Ctx.Input.RequestBody
+	var req models.DelBusinessIndexDataReq
+	err := json.Unmarshal(body, &req)
+	if err != nil {
+		br.Msg = "参数解析异常！"
+		br.ErrMsg = "参数解析失败，Err:" + err.Error()
+		return
+	}
+
+	// 如果没有传入指标编码的话，那么自动生成指标编码
+	if req.IndexCode == `` {
+		br.Msg = "指标编码不允许为空"
+		return
+	}
+	if req.StartDate == `` && req.EndDate == `` {
+		br.Msg = "开始日期和结束日期不允许同时为空"
+		return
+	}
+
+	err, errMsg := services.DelBusinessIndexData(req.IndexCode, req.StartDate, req.EndDate)
+	if err != nil {
+		br.Msg = errMsg
+		br.ErrMsg = "删除失败,Err:" + err.Error()
+		return
+	}
+	br.Ret = 200
+	br.Success = true
+	br.Msg = "删除成功"
+}
+
+// Add
+// @Title 新增自有数据（商家）指标接口
+// @Description 新增自有数据（商家）指标接口
+// @Success 200 {object} models.AddEdbBaseInfoReq
+// @router /add [post]
+func (c *BusinessIndexController) Add() {
+	br := new(models.BaseResponse).Init()
+	var cacheKey string
+	defer func() {
+		c.Data["json"] = br
+		c.ServeJSON()
+	}()
+	var req models.AddEdbBaseInfoReq
+	err := json.Unmarshal(c.Ctx.Input.RequestBody, &req)
+	if err != nil {
+		br.Msg = "参数解析异常！"
+		br.ErrMsg = "参数解析失败，Err:" + err.Error()
+		return
+	}
+	if req.EdbCode == "" {
+		br.Msg = "请输入指标编码！"
+		br.ErrMsg = "请输入指标编码，指标编码为空"
+		return
+	}
+	cacheKey = utils.CACHE_EDB_DATA_ADD + "_BusinessIndexController_" + req.EdbCode
+	if utils.Rc.IsExist(cacheKey) {
+		br.Ret = 501
+		br.Success = true
+		br.Msg = "系统处理中，请稍后重试"
+		return
+	}
+
+	utils.Rc.SetNX(cacheKey, 1, 1*time.Minute)
+	defer func() {
+		_ = utils.Rc.Delete(cacheKey)
+	}()
+
+	baseFromBusinessIndexObj := models.BaseFromBusinessIndex{}
+
+	businessIndexItem, err := baseFromBusinessIndexObj.GetIndexItem(req.EdbCode)
+	if err != nil {
+		br.Msg = "获取数据源中指标信息失败！"
+		br.ErrMsg = "获取数据源中指标信息失败 baseFromBusinessIndexObj.GetIndexItem，Err:" + err.Error()
+		return
+	}
+
+	// 添加指标
+	timestamp := strconv.FormatInt(time.Now().UnixNano(), 10)
+	uniqueCode := utils.MD5(utils.DATA_PREFIX + "_" + timestamp)
+	params := models.AddBaseParams{
+		EdbCode:         req.EdbCode,
+		EdbName:         req.EdbName,
+		Unit:            req.Unit,
+		ClassifyId:      req.ClassifyId,
+		SysUserId:       req.SysUserId,
+		SysUserRealName: req.SysUserRealName,
+		UniqueCode:      uniqueCode,
+	}
+	obj := models.Business{}
+	edbInfo, err, errMsg := obj.Add(params, businessIndexItem)
+	if err != nil {
+		br.Msg = "生成" + obj.GetSourceName() + "失败"
+		if errMsg != `` {
+			br.Msg = errMsg
+		}
+		br.ErrMsg = "生成" + obj.GetSourceName() + "失败 Err:" + err.Error()
+		return
+	}
+
+	if edbInfo == nil {
+		br.Msg = "生成" + obj.GetSourceName() + "失败2"
+		br.ErrMsg = "生成" + obj.GetSourceName() + "失败"
+		return
+	}
+
+	// 更新指标最大最小值
+	err = obj.UnifiedModifyEdbInfoMaxAndMinInfo(edbInfo)
+	if err != nil {
+		br.Msg = "更新指标最大最小值失败"
+		br.ErrMsg = err.Error()
+		return
+	}
+
+	// 添加到es
+	go logic.UpdateEs(edbInfo.EdbInfoId)
+
+	resp := models.AddEdbInfoResp{
+		EdbInfoId:  edbInfo.EdbInfoId,
+		UniqueCode: edbInfo.UniqueCode,
+	}
+	br.Ret = 200
+	br.Success = true
+	br.Msg = "保存成功"
+	br.Data = resp
+	br.IsAddLog = true
+}
+
+// Edit
+// @Title 编辑自有数据（商家）指标接口
+// @Description 编辑自有数据（商家）指标接口
+// @Success 200 {object} models.EditEdbBaseInfoReq
+// @router /edit [post]
+func (c *BusinessIndexController) Edit() {
+	br := new(models.BaseResponse).Init()
+	defer func() {
+		c.Data["json"] = br
+		c.ServeJSON()
+	}()
+
+	var req models.EditEdbBaseInfoReq
+	err := json.Unmarshal(c.Ctx.Input.RequestBody, &req)
+	if err != nil {
+		br.Msg = "参数解析异常！"
+		br.ErrMsg = "参数解析失败，Err:" + err.Error()
+		return
+	}
+	cacheKey := utils.CACHE_EDB_DATA_EDIT + "_BusinessIndexController_" + fmt.Sprint(req.EdbInfoId)
+	if utils.Rc.IsExist(cacheKey) {
+		br.Ret = 501
+		br.Success = true
+		br.Msg = "系统处理中，请稍后重试"
+		return
+	}
+
+	utils.Rc.SetNX(cacheKey, 1, 1*time.Minute)
+	defer func() {
+		_ = utils.Rc.Delete(cacheKey)
+	}()
+
+	if req.EdbInfoId <= 0 {
+		br.Msg = "参数错误"
+		br.ErrMsg = "指标ID：" + strconv.Itoa(req.EdbInfoId)
+		br.IsSendEmail = false
+		return
+	}
+
+	req.EdbName = strings.Trim(req.EdbName, " ")
+	if req.EdbName == "" {
+		br.Msg = "指标名称不能为空"
+		br.IsSendEmail = false
+		return
+	}
+
+	//if req.Frequency == "" {
+	//	br.Msg = "频率不能为空"
+	//	return
+	//}
+
+	if req.Unit == "" {
+		br.Msg = "单位不能为空"
+		br.IsSendEmail = false
+		return
+	}
+
+	if req.ClassifyId <= 0 {
+		br.Msg = "请选择分类"
+		br.IsSendEmail = false
+		return
+	}
+
+	// 根据指标名称和指标ID校验库中是否还存在其他同名指标
+	existEdbName, err := logic.CheckExistByEdbNameAndEdbInfoId(0, req.EdbInfoId, req.EdbName, c.Lang)
+	if err != nil {
+		br.Msg = "判断指标名称是否存在失败"
+		br.ErrMsg = "判断指标名称是否存在失败,Err:" + err.Error()
+		return
+	}
+	if existEdbName {
+		br.Msg = "指标名称已存在，请重新填写"
+		br.ErrMsg = "指标名称已存在，请重新填写"
+		br.IsSendEmail = false
+		return
+	}
+
+	//判断公式，指标是否有改动
+	edbInfo, err := models.GetEdbInfoById(req.EdbInfoId)
+	if err != nil {
+		br.Msg = "修改失败"
+		br.Msg = "获取指标信息失败,GetEdbInfoById Err:" + err.Error()
+		return
+	}
+	if edbInfo == nil {
+		br.Msg = "修改失败"
+		br.Msg = "指标信息不存在，EdbInfoId:" + strconv.Itoa(req.EdbInfoId)
+		return
+	}
+
+	baseFromBusinessIndexObj := models.BaseFromBusinessIndex{}
+
+	businessIndexItem, err := baseFromBusinessIndexObj.GetIndexItem(edbInfo.EdbCode)
+	if err != nil {
+		br.Msg = "获取数据源中指标信息失败！"
+		br.ErrMsg = "获取数据源中指标信息失败 baseFromBusinessIndexObj.GetIndexItem，Err:" + err.Error()
+		return
+	}
+
+	// 额外赋值
+	switch c.Lang {
+	case utils.EnLangVersion:
+		req.EdbNameEn = req.EdbName
+		req.UnitEn = req.Unit
+
+		req.EdbName = edbInfo.EdbName
+		req.Unit = edbInfo.Unit
+	default:
+		req.EdbNameEn = edbInfo.EdbNameEn
+		req.UnitEn = edbInfo.UnitEn
+	}
+
+	// 编辑指标
+	params := models.EditBaseParams{
+		EdbCode:         edbInfo.EdbCode,
+		EdbName:         req.EdbName,
+		Unit:            req.Unit,
+		ClassifyId:      req.ClassifyId,
+		SysUserId:       req.SysUserId,
+		SysUserRealName: req.SysUserRealName,
+		Lang:            c.Lang,
+		EdbInfo:         edbInfo,
+		EdbNameEn:       req.EdbNameEn,
+		UnitEn:          req.UnitEn,
+	}
+	obj := models.Business{}
+	err, errMsg := obj.Edit(params, businessIndexItem)
+	if err != nil {
+		br.Msg = "修改" + obj.GetSourceName() + "失败"
+		if errMsg != `` {
+			br.Msg = errMsg
+		}
+		br.ErrMsg = "修改" + obj.GetSourceName() + "失败 Err:" + err.Error()
+		return
+	}
+
+	// 更新指标最大最小值
+	err = obj.UnifiedModifyEdbInfoMaxAndMinInfo(edbInfo)
+	if err != nil {
+		br.Msg = "更新指标最大最小值失败"
+		br.ErrMsg = err.Error()
+		return
+	}
+
+	// 添加到es
+	go logic.UpdateEs(edbInfo.EdbInfoId)
+
+	resp := models.AddEdbInfoResp{
+		EdbInfoId:  edbInfo.EdbInfoId,
+		UniqueCode: edbInfo.UniqueCode,
+	}
+	br.Ret = 200
+	br.Success = true
+	br.Msg = "保存成功"
+	br.Data = resp
+	br.IsAddLog = true
+}
+
+// Refresh
+// @Title 刷新计算指标接口
+// @Description 刷新计算指标接口
+// @Success 200 {object} models.RefreshEdbInfoReq
+// @router /refresh [post]
+func (c *BusinessIndexController) Refresh() {
+	br := new(models.BaseResponse).Init()
+	var cacheKey string
+	defer func() {
+		c.Data["json"] = br
+		c.ServeJSON()
+	}()
+	var req models.RefreshEdbInfoReq
+	err := json.Unmarshal(c.Ctx.Input.RequestBody, &req)
+	if err != nil {
+		br.Msg = "参数解析异常！"
+		br.ErrMsg = "参数解析失败，Err:" + err.Error()
+		return
+	}
+	if req.EdbCode == "" {
+		br.Msg = "请输入指标编码！"
+		br.ErrMsg = "请输入指标编码，指标编码为空"
+		return
+	}
+	if req.EdbInfoId <= 0 {
+		br.Msg = "请输入指标ID！"
+		br.ErrMsg = "请输入指标ID"
+		return
+	}
+	edbInfo, err := models.GetEdbInfoById(req.EdbInfoId)
+	if err != nil {
+		br.Msg = "指标不存在！"
+		br.ErrMsg = "指标不存在"
+		return
+	}
+
+	cacheKey = utils.CACHE_EDB_DATA_REFRESH + strconv.Itoa(edbInfo.Source) + "_" + req.EdbCode
+
+	if utils.Rc.IsExist(cacheKey) {
+		br.Ret = 501
+		br.Success = true
+		br.Msg = "系统处理中，请稍后重试"
+		return
+	}
+
+	// 刷新指标
+	utils.Rc.SetNX(cacheKey, 1, 1*time.Minute)
+	defer func() {
+		_ = utils.Rc.Delete(cacheKey)
+	}()
+
+	params := models.RefreshBaseParams{
+		EdbInfo:   edbInfo,
+		StartDate: req.StartDate,
+	}
+	obj := models.Business{}
+
+	err, errMsg := obj.Refresh(params)
+	if errMsg != `` {
+		br.Msg = "刷新指标失败！"
+		br.ErrMsg = "刷新指标失败，err:" + errMsg
+		return
+	}
+	if err != nil && err.Error() != utils.ErrNoRow() {
+		br.Msg = "刷新指标信息失败！"
+		br.ErrMsg = "刷新指标信息失败 BusinessIndexController，Err:" + err.Error()
+		return
+	}
+
+	// 更新指标最大最小值
+	err = obj.UnifiedModifyEdbInfoMaxAndMinInfo(edbInfo)
+	if err != nil {
+		br.Msg = "更新指标最大最小值失败"
+		br.ErrMsg = err.Error()
+		return
+	}
+
+	// 更新ES
+	go logic.UpdateEs(edbInfo.EdbInfoId)
+
+	br.Ret = 200
+	br.Success = true
+	br.Msg = "刷新成功"
+}

controllers/base_from_calculate.go

@@ -118,6 +118,9 @@ func (this *CalculateController) Add() {
 	if err != nil {
 		return
 	}
+
+	// 判断是否需要禁用
+	go services.DisableEdbInfoNoUpdate(edbInfo)
 	resp := models.AddEdbInfoResp{
 		EdbInfoId:  edbInfo.EdbInfoId,
 		UniqueCode: uniqueCode,
@@ -360,6 +363,8 @@ func (this *CalculateController) Edit() {
 		EdbInfoId:  edbInfoDetail.EdbInfoId,
 		UniqueCode: edbInfoDetail.UniqueCode,
 	}
+	// 重置计算指标中的引用关系
+	go services.ResetEdbRelation(edbInfoDetail.EdbInfoId)
 	br.Ret = 200
 	br.Success = true
 	br.Msg = "保存成功"
@@ -535,8 +540,12 @@ func (this *CalculateController) BatchSave() {
 	}
 
 	//生成指标编码
-	randStr := utils.GetRandDigit(4)
-	edbCode := `C` + time.Now().Format("060102") + randStr
+	edbCode, err := utils.GenerateEdbCode(1, "")
+	if err != nil {
+		br.Msg = "指标生成失败"
+		br.ErrMsg = "指标编码生成失败，Err：" + err.Error()
+		return
+	}
 	timestamp := strconv.FormatInt(time.Now().UnixNano(), 10)
 	uniqueCode := utils.MD5(utils.DATA_PREFIX + "_" + timestamp)
 
@@ -592,12 +601,7 @@ func (this *CalculateController) BatchSave() {
 		sourName = "N数值移动平均计算"
 		edbInfo, err = models.AddCalculateNszydpjjs(&req, fromEdbInfo, edbCode, uniqueCode, sysUserId, sysUserName, formulaInt)
 	case utils.DATA_SOURCE_CALCULATE_HBZ:
-		var condition string
-		var pars []interface{}
-		condition += " AND edb_info_id =? "
-		pars = append(pars, fromEdbInfoId)
-		condition += " AND value <=0 "
-		checkCount, tmpErr := models.GetEdbDataCount(condition, pars, fromEdbInfo.Source, fromEdbInfo.SubSource)
+		checkCount, tmpErr := models.GetLteZeroEdbDataCount(fromEdbInfo.Source, fromEdbInfo.SubSource, fromEdbInfoId)
 		if tmpErr != nil && tmpErr.Error() != utils.ErrNoRow() {
 			br.Msg = "判断环比值是否可计算失败"
 			br.ErrMsg = "判断环比值是否可计算失败,Err:" + tmpErr.Error()
@@ -815,7 +819,8 @@ func (this *CalculateController) BatchSave() {
 		br.ErrMsg = "保存失败,Err:" + err.Error()
 		return
 	}
-
+	// 判断是否需要禁用
+	go services.DisableEdbInfoNoUpdate(edbInfo)
 	resp := models.AddEdbInfoResp{
 		EdbInfoId:  edbInfo.EdbInfoId,
 		UniqueCode: edbInfo.UniqueCode,
@@ -1046,12 +1051,7 @@ func (this *CalculateController) BatchEdit() {
 		sourName = "N数值移动平均计算"
 		err = models.EditCalculateNszydpjjs(edbInfo, &req, fromEdbInfo, formulaInt, edbInfo.CalculateFormula)
 	case utils.DATA_SOURCE_CALCULATE_HBZ:
-		var condition string
-		var pars []interface{}
-		condition += " AND edb_info_id =? "
-		pars = append(pars, req.FromEdbInfoId)
-		condition += " AND value <=0 "
-		checkCount, tmpErr := models.GetEdbDataCount(condition, pars, fromEdbInfo.Source, fromEdbInfo.SubSource)
+		checkCount, tmpErr := models.GetLteZeroEdbDataCount(fromEdbInfo.Source, fromEdbInfo.SubSource, req.FromEdbInfoId)
 		if tmpErr != nil && tmpErr.Error() != utils.ErrNoRow() {
 			br.Msg = "判断环比值是否可计算失败"
 			br.ErrMsg = "判断环比值是否可计算失败,Err:" + tmpErr.Error()
@@ -1272,6 +1272,9 @@ func (this *CalculateController) BatchEdit() {
 		br.ErrMsg = "记录指标操作记录失败，Err:" + err.Error()
 	}
 
+	// 重置计算指标中的引用关系
+	go services.ResetEdbRelation(edbInfoId)
+
 	resp := models.AddEdbInfoResp{
 		EdbInfoId:  edbInfo.EdbInfoId,
 		UniqueCode: edbInfo.UniqueCode,
@@ -1831,6 +1834,11 @@ func (this *CalculateController) SaveAdjust() {
 		EdbInfoId:  edbInfo.EdbInfoId,
 		UniqueCode: edbInfo.UniqueCode,
 	}
+
+	// 判断是否需要禁用
+	go services.DisableEdbInfoNoUpdate(edbInfo)
+	// 重置计算指标中的引用关系
+	go services.ResetEdbRelation(edbInfo.EdbInfoId)
 	br.Ret = 200
 	br.Success = true
 	br.Msg = "保存成功"
@@ -2368,3 +2376,119 @@ func (this *CalculateController) BatchEditMulti() {
 	br.Data = resp
 	br.IsAddLog = true
 }
+
+// StepCalculate
+// @Title 多步骤计算
+// @Description 多步骤计算
+// @Param request body models.StepCalculateBySearchData true "type json string"
+// @Success Ret=200 返回指标id
+// @router /base/step_calculate [post]
+func (this *CalculateController) StepCalculate() {
+	br := new(models.BaseResponse).Init()
+	defer func() {
+		if br.ErrMsg == "" {
+			br.IsSendEmail = false
+		}
+		this.Data["json"] = br
+		this.ServeJSON()
+	}()
+	var req models.StepCalculateBySearchData
+	if e := json.Unmarshal(this.Ctx.Input.RequestBody, &req); e != nil {
+		br.Msg = "参数解析异常"
+		br.ErrMsg = fmt.Sprintf("参数解析失败, Err: %v", e)
+		return
+	}
+	//sort.Slice(req.Calculates, func(i, j int) bool {
+	//	return req.Calculates[i].Sort < req.Calculates[j].Sort
+	//})
+	//
+	//var errMsg string
+	//originData, e := models.EdbInfoSearchDataToData(req.DataList)
+	//if e != nil {
+	//	br.Msg = "基础数据异常"
+	//	br.ErrMsg = fmt.Sprintf("计算失败, 基础数据异常, Err: %v", e)
+	//	return
+	//}
+	//
+	//calculateData := originData
+	//dateDataMap := make(map[time.Time]float64)
+	//for _, v := range req.Calculates {
+	//	baseCalculate := models.BaseCalculate{
+	//		DataList:      calculateData,
+	//		Frequency:     v.Frequency,
+	//		Formula:       v.Formula,
+	//		Calendar:      v.Calendar,
+	//		MoveType:      v.MoveType,
+	//		MoveFrequency: v.MoveFrequency,
+	//		FromFrequency: v.FromFrequency,
+	//		Source:        v.Source,
+	//	}
+	//
+	//	// 计算方式
+	//	switch baseCalculate.Source {
+	//	case utils.EdbBaseCalculateLjzzy:
+	//		dateDataMap, e, errMsg = baseCalculate.Ljzzy()
+	//	case utils.EdbBaseCalculateLjzzj:
+	//		dateDataMap, e, errMsg = baseCalculate.Ljzzj()
+	//	case utils.EdbBaseCalculateTbz:
+	//		dateDataMap, e, errMsg = baseCalculate.Tbz()
+	//	case utils.EdbBaseCalculateTcz:
+	//		dateDataMap, e, errMsg = baseCalculate.Tcz()
+	//	case utils.EdbBaseCalculateNszydpjjs:
+	//		dateDataMap, e, errMsg = baseCalculate.Nszydpjjs()
+	//	case utils.EdbBaseCalculateHbz:
+	//		dateDataMap, e, errMsg = baseCalculate.Hbz()
+	//	case utils.EdbBaseCalculateHcz:
+	//		dateDataMap, e, errMsg = baseCalculate.Hcz()
+	//	case utils.EdbBaseCalculateUpFrequency:
+	//		dateDataMap, e, errMsg = baseCalculate.UpFrequency()
+	//	case utils.EdbBaseCalculateDownFrequency:
+	//		dateDataMap, e, errMsg = baseCalculate.DownFrequency()
+	//	case utils.EdbBaseCalculateTimeShift:
+	//		dateDataMap, e, errMsg = baseCalculate.TimeShift()
+	//	case utils.EdbBaseCalculateCjjx:
+	//		dateDataMap, e, errMsg = baseCalculate.Cjjx()
+	//	case utils.EdbBaseCalculateAnnualized:
+	//		dateDataMap, e, errMsg = baseCalculate.Annualized()
+	//	case utils.EdbBaseCalculateLjz:
+	//		dateDataMap, e, errMsg = baseCalculate.Ljz()
+	//	case utils.EdbBaseCalculateLjzNczj:
+	//		dateDataMap, e, errMsg = baseCalculate.LjzNczj()
+	//	case utils.EdbBaseCalculateExponentialSmoothing:
+	//		dateDataMap, e, errMsg = baseCalculate.ExponentialSmoothing()
+	//	case utils.EdbBaseCalculateRjz:
+	//		dateDataMap, e, errMsg = baseCalculate.Rjz()
+	//	default:
+	//		errMsg = "计算方式无效"
+	//		e = fmt.Errorf("%s:%d", errMsg, baseCalculate.Source)
+	//	}
+	//	if e != nil {
+	//		br.Msg = "计算失败"
+	//		if errMsg != "" {
+	//			br.Msg = errMsg
+	//		}
+	//		br.ErrMsg = e.Error()
+	//		return
+	//	}
+	//
+	//	calculateData = models.TransDateData2EdbData(dateDataMap)
+	//}
+	resultData, dates, errMsg, e := services.StepCalculate(req.DataList, req.Calculates)
+	if e != nil {
+		br.Msg = "计算失败"
+		if errMsg != "" {
+			br.Msg = errMsg
+		}
+		br.ErrMsg = fmt.Sprintf("分步骤计算失败, Err: %v", e)
+		return
+	}
+
+	br.Ret = 200
+	br.Success = true
+	br.Msg = "计算成功"
+	br.Data = models.BaseCalculateResp{
+		DataMap:  resultData,
+		DateList: dates,
+	}
+	br.IsAddLog = true
+}

controllers/base_from_ccf.go

@@ -0,0 +1,217 @@
+package controllers
+
+import (
+	"encoding/json"
+	"eta/eta_index_lib/logic"
+	"eta/eta_index_lib/models"
+	"eta/eta_index_lib/services"
+	"eta/eta_index_lib/utils"
+	"strconv"
+	"time"
+)
+
+// CCFController CCF化纤信息
+type CCFController struct {
+	BaseAuthController
+}
+
+// Add
+// @Title 新增CCF指标接口
+// @Description 新增CCF指标接口
+// @Success 200 {object} models.AddEdbInfoReq
+// @router /add [post]
+func (this *CCFController) Add() {
+	br := new(models.BaseResponse).Init()
+	var cacheKey string
+	defer func() {
+		if br.ErrMsg == "" {
+			br.IsSendEmail = false
+		}
+		this.Data["json"] = br
+		this.ServeJSON()
+	}()
+	var req models.AddEdbInfoReq
+	err := json.Unmarshal(this.Ctx.Input.RequestBody, &req)
+	if err != nil {
+		br.Msg = "参数解析异常！"
+		br.ErrMsg = "参数解析失败，Err:" + err.Error()
+		return
+	}
+	if req.EdbCode == "" {
+		br.Msg = "请输入指标编码！"
+		br.ErrMsg = "请输入指标编码，指标编码为空"
+		return
+	}
+
+	ob := new(models.BaseFromCCF)
+	cacheKey = utils.CACHE_EDB_DATA_ADD + strconv.Itoa(ob.GetSource()) + "_" + req.EdbCode
+	if utils.Rc.IsExist(cacheKey) {
+		br.Ret = 501
+		br.Success = true
+		br.Msg = "系统处理中，请稍后重试"
+		return
+	}
+	utils.Rc.SetNX(cacheKey, 1, 1*time.Minute)
+	defer func() {
+		_ = utils.Rc.Delete(cacheKey)
+	}()
+
+	if err = ob.Add(req.EdbCode); err != nil {
+		br.Msg = "获取指标信息失败！"
+		br.ErrMsg = "获取指标信息失败 AddEdbDataFromCCF，Err:" + err.Error()
+		return
+	}
+	br.Ret = 200
+	br.Success = true
+	br.Msg = "获取成功"
+}
+
+// Refresh
+// @Title 刷新CCF指标接口
+// @Description 刷新CCF指标接口
+// @Success 200 {object} models.RefreshEdbInfoReq
+// @router /refresh [post]
+func (this *CCFController) Refresh() {
+	br := new(models.BaseResponse).Init()
+	var cacheKey string
+	defer func() {
+		if br.ErrMsg == "" {
+			br.IsSendEmail = false
+		}
+		this.Data["json"] = br
+		this.ServeJSON()
+	}()
+	var req models.RefreshEdbInfoReq
+	err := json.Unmarshal(this.Ctx.Input.RequestBody, &req)
+	if err != nil {
+		br.Msg = "参数解析异常！"
+		br.ErrMsg = "参数解析失败，Err:" + err.Error()
+		return
+	}
+	if req.EdbCode == "" {
+		br.Msg = "请输入指标编码！"
+		br.ErrMsg = "请输入指标编码，指标编码为空"
+		return
+	}
+	if req.EdbInfoId < 0 {
+		br.Msg = "请输入指标ID！"
+		br.ErrMsg = "请输入指标ID"
+		return
+	}
+
+	ob := new(models.BaseFromCCF)
+	source := ob.GetSource()
+	// 获取指标详情
+	edbInfo, err := models.GetEdbInfoByEdbCode(source, req.EdbCode)
+	if err != nil {
+		br.Msg = "指标不存在！"
+		br.ErrMsg = "指标不存在"
+		return
+	}
+	cacheKey = utils.CACHE_EDB_DATA_REFRESH + strconv.Itoa(source) + "_" + req.EdbCode
+	if utils.Rc.IsExist(cacheKey) {
+		br.Ret = 501
+		br.Success = true
+		br.Msg = "系统处理中，请稍后重试"
+		return
+	}
+	utils.Rc.SetNX(cacheKey, 1, 1*time.Minute)
+	defer func() {
+		_ = utils.Rc.Delete(cacheKey)
+	}()
+	if req.EdbInfoId <= 0 {
+		req.EdbInfoId = edbInfo.EdbInfoId
+	}
+	err = ob.Refresh(req.EdbInfoId, req.EdbCode, req.StartDate)
+	if err != nil && err.Error() != utils.ErrNoRow() {
+		br.Msg = "刷新指标信息失败！"
+		br.ErrMsg = "刷新指标信息失败 RefreshEdbDataFromCCF，Err:" + err.Error()
+		return
+	}
+	// 更新指标最大最小值
+	err, errMsg := models.UnifiedModifyEdbInfoMaxAndMinInfo(edbInfo)
+	if err != nil {
+		br.Msg = errMsg
+		br.ErrMsg = err.Error()
+		return
+	}
+
+	// 更新ES
+	go logic.UpdateEs(edbInfo.EdbInfoId)
+
+	br.Ret = 200
+	br.Success = true
+	br.Msg = "获取成功"
+}
+
+// HandleEdbData
+// @Title 处理CCF指标的接口
+// @Description 处理CCF指标的接口
+// @Success 200 string "操作成功"
+// @router /handle/edb_data [post]
+func (this *CCFController) HandleEdbData() {
+	br := new(models.BaseResponse).Init()
+	defer func() {
+		if br.ErrMsg == "" {
+			br.IsSendEmail = false
+		}
+		this.Data["json"] = br
+		this.ServeJSON()
+	}()
+	var req models.HandleCCFEdbDataReq
+	err := json.Unmarshal(this.Ctx.Input.RequestBody, &req)
+	if err != nil {
+		br.Msg = "参数解析异常！"
+		br.ErrMsg = "参数解析失败，Err:" + err.Error()
+		return
+	}
+
+	// 处理excel表数据
+	if err = services.HandleCCFIndex(&req); err != nil {
+		br.Msg = "处理失败"
+		br.ErrMsg = "处理失败,Err:" + err.Error()
+		return
+	}
+
+	br.Ret = 200
+	br.Success = true
+	br.Msg = "处理成功"
+}
+
+// HandleTableData
+// @Title 处理CCF装置表格的接口
+// @Description 处理CCF装置表格的接口
+// @Success 200 string "操作成功"
+// @router /handle/table_data [post]
+func (this *CCFController) HandleTableData() {
+	br := new(models.BaseResponse).Init()
+	defer func() {
+		if br.ErrMsg == "" {
+			br.IsSendEmail = false
+		}
+		this.Data["json"] = br
+		this.ServeJSON()
+	}()
+	var req models.HandleCCFStockTableReq
+	err := json.Unmarshal(this.Ctx.Input.RequestBody, &req)
+	if err != nil {
+		br.Msg = "参数解析异常！"
+		br.ErrMsg = "参数解析失败，Err:" + err.Error()
+		return
+	}
+	if req.Table == nil {
+		br.Msg = "表格信息为空"
+		br.ErrMsg = "表格信息为空"
+		return
+	}
+
+	if err = services.HandleCCFStockTable(&req); err != nil {
+		br.Msg = "处理失败"
+		br.ErrMsg = "处理失败,Err:" + err.Error()
+		return
+	}
+
+	br.Ret = 200
+	br.Success = true
+	br.Msg = "处理成功"
+}

controllers/base_from_predict_calculate.go

@@ -194,8 +194,13 @@ func addPredictCalculate(br *models.BaseResponse, req models.EdbInfoCalculateSav
 	}
 
 	// 指标入库
-	randStr := utils.GetRandDigit(4)
-	edbCode := `C2` + time.Now().Format("060102") + randStr
+	//生成指标编码
+	edbCode, err := utils.GenerateEdbCode(1, "2")
+	if err != nil {
+		br.Msg = "指标生成失败"
+		br.ErrMsg = "指标编码生成失败，Err：" + err.Error()
+		return
+	}
 	timestamp := strconv.FormatInt(time.Now().UnixNano(), 10)
 	uniqueCode := utils.MD5(utils.DATA_PREFIX + "_" + timestamp)
 	edbInfo := &models.EdbInfo{
@@ -617,8 +622,12 @@ func (this *PredictCalculateController) CalculateBatchSave() {
 	}
 
 	//生成指标编码
-	randStr := utils.GetRandDigit(4)
-	edbCode := `C` + time.Now().Format("060102") + randStr
+	edbCode, err := utils.GenerateEdbCode(1, "2")
+	if err != nil {
+		br.Msg = "指标生成失败"
+		br.ErrMsg = "指标编码生成失败，Err：" + err.Error()
+		return
+	}
 	timestamp := strconv.FormatInt(time.Now().UnixNano(), 10)
 	uniqueCode := utils.MD5(utils.DATA_PREFIX + "_" + timestamp)
 

controllers/base_from_python.go

@@ -52,7 +52,7 @@ func (this *PythonController) ExcePython() {
 		br.Msg = "获取数据失败"
 		br.ErrMsg = "python代码获取数据失败，err:" + err.Error()
 		if errMsg != "" {
-			br.ErrMsg = errMsg
+			br.Msg = errMsg
 		}
 		return
 	}
@@ -82,7 +82,8 @@ func (this *PythonController) Add() {
 		br.ErrMsg = "参数解析失败，Err:" + err.Error()
 		return
 	}
-	req.EdbName = strings.Trim(req.EdbName, " ")
+	// 移除首尾空格
+	req.EdbName = strings.TrimSpace(req.EdbName)
 	if req.EdbName == "" {
 		br.Msg = "指标名称不能为空"
 		return
@@ -154,8 +155,13 @@ func (this *PythonController) Add() {
 		return
 	}
 	//指标code生成
-	randStr := utils.GetRandDigit(4)
-	edbCode := `C` + time.Now().Format("060102") + randStr
+	//生成指标编码
+	edbCode, err := utils.GenerateEdbCode(1, "")
+	if err != nil {
+		br.Msg = "指标生成失败"
+		br.ErrMsg = "指标编码生成失败，Err：" + err.Error()
+		return
+	}
 
 	timestamp := strconv.FormatInt(time.Now().UnixNano(), 10)
 	uniqueCode := utils.MD5(utils.DATA_PREFIX + "_" + timestamp)
@@ -268,7 +274,9 @@ func (this *PythonController) Edit() {
 		br.ErrMsg = "参数解析失败，Err:" + err.Error()
 		return
 	}
-	req.EdbName = strings.Trim(req.EdbName, " ")
+
+	// 移除首尾空格
+	req.EdbName = strings.TrimSpace(req.EdbName)
 
 	if req.EdbInfoId <= 0 {
 		br.Msg = "指标id不能为空"
@@ -358,8 +366,8 @@ func (this *PythonController) Edit() {
 			return
 		}
 	}
-	edbInfo.EdbName = utils.TrimStr(req.EdbName)
-	edbInfo.EdbNameSource = utils.TrimStr(req.EdbName)
+	edbInfo.EdbName = req.EdbName
+	edbInfo.EdbNameSource = req.EdbName
 	edbInfo.Unit = req.Unit
 	edbInfo.ClassifyId = req.ClassifyId
 	edbInfo.Frequency = req.Frequency

controllers/base_from_sci99.go

@@ -0,0 +1,128 @@
+package controllers
+
+import (
+	"encoding/json"
+	"eta/eta_index_lib/logic"
+	"eta/eta_index_lib/models"
+	"eta/eta_index_lib/utils"
+	"strconv"
+	"time"
+)
+
+// 卓创资讯价格指数
+type Sci99Controller struct {
+	BaseAuthController
+}
+
+// @Title 新增卓创资讯指标接口
+// @Description 新增卓创资讯指标接口
+// @Success 200 {object} models.AddEdbInfoReq
+// @router /add [post]
+func (this *Sci99Controller) Add() {
+	br := new(models.BaseResponse).Init()
+	var cacheKey string
+	defer func() {
+		utils.Rc.Delete(cacheKey)
+		this.Data["json"] = br
+		this.ServeJSON()
+	}()
+	source := utils.DATA_SOURCE_SCI99
+	var req models.AddEdbInfoReq
+	err := json.Unmarshal(this.Ctx.Input.RequestBody, &req)
+	if err != nil {
+		br.Msg = "参数解析异常！"
+		br.ErrMsg = "参数解析失败，Err:" + err.Error()
+		return
+	}
+	if req.EdbCode == "" {
+		br.Msg = "请输入指标编码！"
+		br.ErrMsg = "请输入指标编码，指标编码为空"
+		return
+	}
+	cacheKey = utils.CACHE_EDB_DATA_ADD + strconv.Itoa(source) + "_" + req.EdbCode
+	if !utils.Rc.IsExist(cacheKey) {
+		utils.Rc.SetNX(cacheKey, 1, 1*time.Minute)
+		err = models.AddEdbDataFromSci99(req.EdbCode)
+		if err != nil {
+			br.Msg = "获取指标信息失败！"
+			br.ErrMsg = "获取指标信息失败 AddEdbDataFromSci99，Err:" + err.Error()
+			return
+		}
+		br.Ret = 200
+		br.Success = true
+		br.Msg = "获取成功"
+	} else {
+		br.Ret = 501
+		br.Success = true
+		br.Msg = "系统处理中，请稍后重试"
+	}
+}
+
+// @Title 刷新卓创资讯指标接口
+// @Description 刷新卓创资讯指标接口
+// @Success 200 {object} models.RefreshEdbInfoReq
+// @router /refresh [post]
+func (this *Sci99Controller) Refresh() {
+	br := new(models.BaseResponse).Init()
+	var cacheKey string
+	defer func() {
+		utils.Rc.Delete(cacheKey)
+		this.Data["json"] = br
+		this.ServeJSON()
+	}()
+	source := utils.DATA_SOURCE_SCI99
+	var req models.RefreshEdbInfoReq
+	err := json.Unmarshal(this.Ctx.Input.RequestBody, &req)
+	if err != nil {
+		br.Msg = "参数解析异常！"
+		br.ErrMsg = "参数解析失败，Err:" + err.Error()
+		return
+	}
+	if req.EdbCode == "" {
+		br.Msg = "请输入指标编码！"
+		br.ErrMsg = "请输入指标编码，指标编码为空"
+		return
+	}
+	if req.EdbInfoId <= 0 {
+		br.Msg = "请输入指标ID！"
+		br.ErrMsg = "请输入指标ID"
+		return
+	}
+
+	// 获取指标详情
+	edbInfo, err := models.GetEdbInfoByEdbCode(source, req.EdbCode)
+	if err != nil {
+		br.Msg = "指标不存在！"
+		br.ErrMsg = "指标不存在"
+		return
+	}
+	cacheKey = utils.CACHE_EDB_DATA_REFRESH + strconv.Itoa(source) + "_" + req.EdbCode
+	if utils.Rc.IsExist(cacheKey) {
+		br.Ret = 501
+		br.Success = true
+		br.Msg = "系统处理中，请稍后重试"
+		return
+	}
+
+	utils.Rc.SetNX(cacheKey, 1, 1*time.Minute)
+	err = models.RefreshEdbDataFromSci99(req.EdbInfoId, req.EdbCode, req.StartDate)
+	if err != nil && err.Error() != utils.ErrNoRow() {
+		br.Msg = "刷新指标信息失败！"
+		br.ErrMsg = "刷新指标信息失败 RefreshEdbDataFromIcpi，Err:" + err.Error()
+		return
+	}
+	// 更新指标最大最小值
+	err, errMsg := models.UnifiedModifyEdbInfoMaxAndMinInfo(edbInfo)
+	if err != nil {
+		br.Msg = errMsg
+		br.ErrMsg = err.Error()
+		return
+	}
+
+	// 更新ES
+	go logic.UpdateEs(edbInfo.EdbInfoId)
+
+	br.Ret = 200
+	br.Success = true
+	br.Msg = "获取成功"
+}

controllers/factor_edb_series/factor_edb_series.go

@@ -0,0 +1,416 @@
+package factor_edb_series
+
+import (
+	"encoding/json"
+	"eta/eta_index_lib/controllers"
+	"eta/eta_index_lib/logic"
+	"eta/eta_index_lib/models"
+	"eta/eta_index_lib/services"
+	"eta/eta_index_lib/utils"
+	"fmt"
+	"strconv"
+	"sync"
+	"time"
+)
+
+// FactorEdbSeriesController 因子指标系列
+type FactorEdbSeriesController struct {
+	controllers.BaseAuthController
+}
+
+// Recalculate
+// @Title 因子指标系列-重计算
+// @Description 因子指标系列-重计算
+// @Success 200 {object} models.FactorEdbRecalculateReq
+// @router /recalculate [post]
+func (this *FactorEdbSeriesController) Recalculate() {
+	br := new(models.BaseResponse).Init()
+	defer func() {
+		if br.ErrMsg == "" {
+			br.IsSendEmail = false
+		}
+		this.Data["json"] = br
+		this.ServeJSON()
+	}()
+	var req models.FactorEdbRecalculateReq
+	if e := json.Unmarshal(this.Ctx.Input.RequestBody, &req); e != nil {
+		br.Msg = "参数解析异常！"
+		br.ErrMsg = fmt.Sprintf("参数解析失败, Err: %v", e)
+		return
+	}
+	if req.EdbInfoId <= 0 {
+		br.Msg = "参数有误"
+		br.ErrMsg = fmt.Sprintf("指标ID有误, EdbInfoId: %d", req.EdbInfoId)
+		return
+	}
+
+	cacheKey := utils.CACHE_EDB_DATA_REFRESH + "_factor_edb_recalculate_" + req.EdbCode
+	if utils.Rc.IsExist(cacheKey) {
+		br.Ret = 501
+		br.Success = true
+		br.Msg = fmt.Sprintf("系统处理中, 请稍后重试, 指标编码: %s", req.EdbCode)
+		return
+	}
+	utils.Rc.SetNX(cacheKey, 1, 10*time.Minute)
+	defer func() {
+		_ = utils.Rc.Delete(cacheKey)
+	}()
+
+	// 获取指标信息
+	edbInfo, e := models.GetEdbInfoById(req.EdbInfoId)
+	if e != nil {
+		if e.Error() == utils.ErrNoRow() {
+			br.Msg = "指标不存在"
+			br.ErrMsg = fmt.Sprintf("指标不存在, EdbInfoId: %d, EdbCode: %s", req.EdbInfoId, req.EdbCode)
+			return
+		}
+		br.Msg = "操作失败"
+		br.ErrMsg = fmt.Sprintf("获取指标信息失败, Err: %v", e)
+		return
+	}
+
+	// 查询指标关联的系列
+	mappings := make([]*models.FactorEdbSeriesMapping, 0)
+	{
+		ob := new(models.FactorEdbSeriesMapping)
+		cond := fmt.Sprintf(" AND %s = ?", ob.Cols().EdbInfoId)
+		pars := make([]interface{}, 0)
+		pars = append(pars, req.EdbInfoId)
+		fields := []string{ob.Cols().FactorEdbSeriesId, ob.Cols().EdbInfoId, ob.Cols().EdbCode}
+		list, e := ob.GetItemsByCondition(cond, pars, fields, "")
+		if e != nil {
+			br.Msg = "操作失败"
+			br.ErrMsg = fmt.Sprintf("获取指标系列关联失败, Err: %v", e)
+			return
+		}
+		mappings = list
+	}
+	if len(mappings) == 0 {
+		br.Ret = 200
+		br.Success = true
+		br.Msg = "操作成功"
+		return
+	}
+
+	// 获取系列信息
+	seriesIds := make([]int, 0)
+	seriesExist := make(map[int]bool)
+	for _, v := range mappings {
+		if !seriesExist[v.FactorEdbSeriesId] {
+			seriesExist[v.FactorEdbSeriesId] = true
+			seriesIds = append(seriesIds, v.FactorEdbSeriesId)
+		}
+	}
+	if len(seriesIds) == 0 {
+		br.Ret = 200
+		br.Success = true
+		br.Msg = "操作成功"
+		return
+	}
+	seriesMap := make(map[int]*models.FactorEdbSeries, 0)
+	{
+		ob := new(models.FactorEdbSeries)
+		cond := fmt.Sprintf(" AND %s IN (%s)", ob.Cols().PrimaryId, utils.GetOrmInReplace(len(seriesIds)))
+		pars := make([]interface{}, 0)
+		pars = append(pars, seriesIds)
+		fields := []string{ob.Cols().PrimaryId, ob.Cols().EdbInfoType, ob.Cols().CalculateStep, ob.Cols().CalculateState}
+		list, e := ob.GetItemsByCondition(cond, pars, fields, "")
+		if e != nil {
+			br.Msg = "操作失败"
+			br.ErrMsg = fmt.Sprintf("获取指标系列失败, Err: %v", e)
+			return
+		}
+		for _, v := range list {
+			seriesMap[v.FactorEdbSeriesId] = v
+		}
+	}
+
+	// 获取指标原数据
+	edbData, e := models.GetEdbDataAllByEdbCode(edbInfo.EdbCode, edbInfo.Source, edbInfo.SubSource, 0)
+	if e != nil {
+		br.Msg = "操作失败"
+		br.ErrMsg = fmt.Sprintf("获取指标数据失败, EdbCode: %s, Err: %v", edbInfo.EdbCode, e)
+		return
+	}
+	if len(edbData) == 0 {
+		br.Msg = "指标无数据"
+		br.ErrMsg = fmt.Sprintf("指标无数据, EdbCode: %s", edbInfo.EdbCode)
+		return
+	}
+
+	for _, v := range mappings {
+		series := seriesMap[v.FactorEdbSeriesId]
+		if series == nil {
+			continue
+		}
+		// 系列无计算则忽略
+		if series.CalculateState != models.FactorEdbSeriesCalculateNone || series.CalculateStep == "" {
+			continue
+		}
+		var calculates []models.CalculatesReq
+		if e = json.Unmarshal([]byte(series.CalculateStep), &calculates); e != nil {
+			br.Msg = "计算步骤异常"
+			br.ErrMsg = fmt.Sprintf("计算步骤异常, SeriesId: %d, Err: %v", series.FactorEdbSeriesId, e)
+			return
+		}
+
+		// 重新计算相关数据
+		e = services.FactorEdbStepCalculate(v.FactorEdbSeriesId, v.EdbInfoId, v.EdbCode, edbData, calculates)
+		if e != nil {
+			br.Msg = "因子指标计算失败"
+			br.ErrMsg = fmt.Sprintf("因子指标计算失败, SeriesId: %d, EdbCode: %s, Err: %v", v.FactorEdbSeriesId, v.EdbCode, e)
+			return
+		}
+	}
+
+	br.Ret = 200
+	br.Success = true
+	br.Msg = "操作成功"
+}
+
+// ChartRecalculate
+// @Title 图表数据-重计算
+// @Description 图表数据-重计算
+// @Success 200 {object} models.FactorEdbRecalculateReq
+// @router /chart_recalculate [post]
+func (this *FactorEdbSeriesController) ChartRecalculate() {
+	br := new(models.BaseResponse).Init()
+	defer func() {
+		if br.ErrMsg == "" {
+			br.IsSendEmail = false
+		}
+		this.Data["json"] = br
+		this.ServeJSON()
+	}()
+	var req models.FactorEdbChartRecalculateReq
+	if e := json.Unmarshal(this.Ctx.Input.RequestBody, &req); e != nil {
+		br.Msg = "参数解析异常！"
+		br.ErrMsg = fmt.Sprintf("参数解析失败, Err: %v", e)
+		return
+	}
+	if req.ChartInfoId <= 0 {
+		br.Msg = "参数有误"
+		br.ErrMsg = fmt.Sprintf("指标ID有误, ChartInfoId: %d", req.ChartInfoId)
+		return
+	}
+
+	cacheKey := utils.CACHE_CHART_INFO_DATA + "_factor_chart_recalculate_" + strconv.Itoa(req.ChartInfoId)
+	if utils.Rc.IsExist(cacheKey) {
+		br.Ret = 501
+		br.Success = true
+		br.Msg = fmt.Sprintf("系统处理中, 请稍后重试, 图表ID: %s", req.ChartInfoId)
+		return
+	}
+	utils.Rc.SetNX(cacheKey, 1, 10*time.Minute)
+	defer func() {
+		_ = utils.Rc.Delete(cacheKey)
+	}()
+
+	// 查询图表关联的系列指标
+	mappings := make([]*models.FactorEdbSeriesChartMapping, 0)
+	{
+		ob := new(models.FactorEdbSeriesChartMapping)
+		cond := fmt.Sprintf(" AND %s = ?", ob.Cols().ChartInfoId)
+		pars := make([]interface{}, 0)
+		pars = append(pars, req.ChartInfoId)
+		fields := []string{ob.Cols().PrimaryId, ob.Cols().CalculateType, ob.Cols().CalculatePars, ob.Cols().FactorEdbSeriesId, ob.Cols().EdbInfoId}
+		list, e := ob.GetItemsByCondition(cond, pars, fields, "")
+		if e != nil {
+			br.Msg = "操作失败"
+			br.ErrMsg = fmt.Sprintf("获取指标系列关联失败, Err: %v", e)
+			return
+		}
+		mappings = list
+	}
+	if len(mappings) == 0 {
+		br.Ret = 200
+		br.Success = true
+		br.Msg = "操作成功"
+		return
+	}
+	seriesIds := make([]int, 0)
+	edbInfoIds := make([]int, 0)
+	for _, v := range mappings {
+		seriesIds = append(seriesIds, v.FactorEdbSeriesId)
+		edbInfoIds = append(edbInfoIds, v.EdbInfoId)
+	}
+
+	// 获取因子指标及系列信息
+	seriesIdItem := make(map[int]*models.FactorEdbSeries)
+	{
+		ob := new(models.FactorEdbSeries)
+		cond := fmt.Sprintf(" AND %s IN (%s)", ob.Cols().PrimaryId, utils.GetOrmInReplace(len(seriesIds)))
+		pars := make([]interface{}, 0)
+		pars = append(pars, seriesIds)
+		items, e := ob.GetItemsByCondition(cond, pars, []string{}, fmt.Sprintf("%s ASC", ob.Cols().PrimaryId))
+		if e != nil {
+			br.Msg = "获取失败"
+			br.ErrMsg = "获取因子指标系列失败, Err: " + e.Error()
+			return
+		}
+		for _, v := range items {
+			seriesIdItem[v.FactorEdbSeriesId] = v
+		}
+	}
+	edbIdItem := make(map[int]*models.EdbInfo)
+	edbItems, e := models.GetEdbInfoByIdList(edbInfoIds)
+	if e != nil {
+		br.Msg = "获取失败"
+		br.ErrMsg = "获取因子指标失败, Err: " + e.Error()
+		return
+	}
+	for _, v := range edbItems {
+		edbIdItem[v.EdbInfoId] = v
+	}
+
+	// 重新计算
+	calculateWorkers := make(chan struct{}, 10)
+	wg := sync.WaitGroup{}
+	calculateDataOb := new(models.FactorEdbSeriesCalculateData)
+	for _, v := range mappings {
+		edbItem := edbIdItem[v.EdbInfoId]
+		if edbItem == nil {
+			continue
+		}
+		seriesItem := seriesIdItem[v.FactorEdbSeriesId]
+		if seriesItem == nil {
+			continue
+		}
+
+		wg.Add(1)
+		go func(chartMapping *models.FactorEdbSeriesChartMapping, edb *models.EdbInfo, series *models.FactorEdbSeries) {
+			defer func() {
+				wg.Done()
+				<-calculateWorkers
+			}()
+			calculateWorkers <- struct{}{}
+
+			// 相关性计算
+			if chartMapping.CalculateType == models.FactorEdbSeriesChartCalculateTypeCorrelation {
+				// 解析计算参数
+				if chartMapping.CalculatePars == "" {
+					utils.FileLog.Info(fmt.Sprintf("相关性-计算参数为空, MappingId: %d", chartMapping.FactorEdbSeriesChartMappingId))
+					return
+				}
+				var calculatePars models.FactorEdbSeriesChartCalculateCorrelationReq
+				if e := json.Unmarshal([]byte(chartMapping.CalculatePars), &calculatePars); e != nil {
+					utils.FileLog.Info(fmt.Sprintf("相关性-计算参数解析失败, MappingId: %d, err: %v", chartMapping.FactorEdbSeriesChartMappingId, e))
+					return
+				}
+
+				// 获取标的指标信息及数据
+				baseEdb, e := models.GetEdbInfoById(calculatePars.BaseEdbInfoId)
+				if e != nil {
+					utils.FileLog.Info(fmt.Sprintf("相关性-获取标的指标失败, MappingId: %d, err: %v", chartMapping.FactorEdbSeriesChartMappingId, e))
+					return
+				}
+				calculateUnitDays, ok := utils.FrequencyDaysMap[calculatePars.CalculateUnit]
+				if !ok {
+					utils.FileLog.Info(fmt.Sprintf("相关性-错误的计算窗口频度, MappingId: %d", chartMapping.FactorEdbSeriesChartMappingId))
+					return
+				}
+				calculateDays := calculatePars.CalculateValue * calculateUnitDays
+
+				dataListA := make([]*models.EdbInfoSearchData, 0)
+				{
+					// 标的指标数据日期区间
+					startDate := time.Now().AddDate(0, 0, -calculateDays).Format(utils.FormatDate)
+					endDate := time.Now().Format(utils.FormatDate)
+					startDateTime, _ := time.ParseInLocation(utils.FormatDate, startDate, time.Local)
+					startDate = startDateTime.AddDate(0, 0, 1).Format(utils.FormatDate) // 不包含第一天
+					switch baseEdb.EdbInfoType {
+					case 0:
+						list, e := models.GetEdbDataList(baseEdb.Source, baseEdb.SubSource, baseEdb.EdbInfoId, startDate, endDate)
+						if e != nil {
+							utils.FileLog.Info(fmt.Sprintf("相关性-获取标的指标数据失败, EdbInfoId: %d, err: %v", baseEdb.EdbInfoId, e))
+							return
+						}
+						dataListA = models.TransEdbInfoDataList2SearchData(list)
+					case 1:
+						dataListA, _, e, _ = models.GetPredictDataListByPredictEdbInfo(baseEdb, 1, startDate)
+						if e != nil {
+							utils.FileLog.Info(fmt.Sprintf("相关性-获取标的指标数据失败, EdbInfoId: %d, err: %v", baseEdb.EdbInfoId, e))
+							return
+						}
+					default:
+						utils.FileLog.Info(fmt.Sprintf("相关性-标的指标类型异常, EdbInfoId: %d", baseEdb.EdbInfoId))
+						return
+					}
+				}
+
+				// 获取指标数据
+				dataListB := make([]*models.EdbInfoSearchData, 0)
+				if series.CalculateState == models.FactorEdbSeriesCalculated {
+					cond := fmt.Sprintf(" AND %s = ? AND %s = ?", calculateDataOb.Cols().FactorEdbSeriesId, calculateDataOb.Cols().EdbInfoId)
+					pars := make([]interface{}, 0)
+					pars = append(pars, chartMapping.FactorEdbSeriesId, chartMapping.EdbInfoId)
+					dataItems, e := calculateDataOb.GetItemsByCondition(cond, pars, []string{calculateDataOb.Cols().DataTime, calculateDataOb.Cols().Value}, fmt.Sprintf("%s ASC", calculateDataOb.Cols().DataTime))
+					if e != nil {
+						utils.FileLog.Info(fmt.Sprintf("相关性-获取因子指标数据失败, MappingId: %d, err: %v", chartMapping.FactorEdbSeriesChartMappingId, e))
+						return
+					}
+					dataListB = models.TransEdbSeriesCalculateData2EdbDataList(dataItems)
+				} else {
+					switch edb.EdbInfoType {
+					case 0:
+						list, e := models.GetEdbDataList(edb.Source, edb.SubSource, edb.EdbInfoId, "", "")
+						if e != nil {
+							utils.FileLog.Info(fmt.Sprintf("相关性-获取因子指标数据失败, EdbInfoId: %d, err: %v", edb.EdbInfoId, e))
+							return
+						}
+						dataListB = models.TransEdbInfoDataList2SearchData(list)
+					case 1:
+						dataListB, e = models.GetPredictEdbDataListAll(baseEdb, 1)
+						if e != nil {
+							utils.FileLog.Info(fmt.Sprintf("相关性-获取因子指标数据失败, EdbInfoId: %d, err: %v", edb.EdbInfoId, e))
+							return
+						}
+					default:
+						utils.FileLog.Info(fmt.Sprintf("相关性-因子指标类型异常, EdbInfoId: %d", edb.EdbInfoId))
+						return
+					}
+				}
+
+				// 计算相关性
+				xEdbIdValue, yDataList, e := logic.CalculateCorrelation(calculatePars.LeadValue, calculatePars.LeadUnit, baseEdb.Frequency, edb.Frequency, dataListA, dataListB)
+				if e != nil {
+					utils.FileLog.Info(fmt.Sprintf("相关性-计算失败, EdbInfoId: %d", edb.EdbInfoId))
+					return
+				}
+
+				// X及Y轴数据
+				yData := yDataList[0].Value
+				yLen := len(yData)
+				values := make([]models.FactorEdbSeriesCorrelationMatrixValues, len(xEdbIdValue))
+				for k, x := range xEdbIdValue {
+					var y float64
+					if k >= 0 && k < yLen {
+						y = yData[k]
+					}
+					y = utils.SubFloatToFloat(y, 2)
+					values[k] = models.FactorEdbSeriesCorrelationMatrixValues{
+						XData: x, YData: y,
+					}
+				}
+
+				// 更新计算结果
+				b, e := json.Marshal(values)
+				if e != nil {
+					utils.FileLog.Info(fmt.Sprintf("相关性-计算结果JSON格式化失败, err: %v", e))
+					return
+				}
+				chartMapping.CalculateData = string(b)
+				chartMapping.ModifyTime = time.Now().Local()
+				if e = chartMapping.Update([]string{chartMapping.Cols().CalculateData, chartMapping.Cols().ModifyTime}); e != nil {
+					utils.FileLog.Info(fmt.Sprintf("相关性-更新矩阵数据失败, err: %v", e))
+					return
+				}
+			}
+		}(v, edbItem, seriesItem)
+	}
+	wg.Wait()
+
+	br.Ret = 200
+	br.Success = true
+	br.Msg = "操作成功"
+}

controllers/open/edb_info.go

@@ -49,13 +49,10 @@ func (this *EdbInfoController) Detail() {
 	dataList := make([]*models.EdbInfoSearchData, 0)
 	switch edbInfo.EdbInfoType {
 	case 0:
-		var condition string
-		var pars []interface{}
-		condition += " AND edb_info_id=? "
-		pars = append(pars, edbInfo.EdbInfoId)
-
 		//获取来源指标的数据
-		dataList, err = models.GetEdbDataListAll(condition, pars, edbInfo.Source, edbInfo.SubSource, 1)
+		dataList, err = models.GetEdbDataListAll(edbInfo.Source, edbInfo.SubSource, models.FindEdbDataListAllCond{
+			EdbInfoId: edbInfo.EdbInfoId,
+		}, 1)
 	case 1:
 		dataList, err = models.GetPredictEdbDataListAllByStartDate(edbInfo, 1, "")
 	default:

controllers/sci99_crawler.go

@@ -0,0 +1,304 @@
+package controllers
+
+import (
+	"encoding/json"
+	"eta/eta_index_lib/models"
+	"eta/eta_index_lib/services/sci99"
+	"eta/eta_index_lib/utils"
+	"fmt"
+	"github.com/mozillazg/go-pinyin"
+	"strings"
+	"time"
+)
+
+// 卓创资讯爬虫
+type Sci99Crawler struct {
+	BaseAuthController
+}
+
+var IndexCodeMap = make(map[string]string)
+var IndexMap = make(map[string]*models.BaseFromSci99Index)
+var ClassifyMap = make(map[string]*models.BaseFromSci99Classify)
+
+// @Title 刷新数据
+// @Description 刷新数据接口
+// @Param	request	body models.AddEdbClassifyReq true "type json string"
+// @Success 200 {object} models.EdbClassify
+// @router /refresh/list [post]
+func (this *Sci99Crawler) RefreshSci99Classify() {
+	br := new(models.BaseResponse).Init()
+	defer func() {
+		this.Data["json"] = br
+		this.ServeJSON()
+	}()
+	var req sci99.RefreshSci99CrawlerReq
+	err := json.Unmarshal(this.Ctx.Input.RequestBody, &req)
+	if err != nil {
+		br.Msg = "参数解析异常！"
+		br.ErrMsg = "参数解析失败，Err:" + err.Error()
+		return
+	}
+	allClassify, err := models.GetBaseFromSci99Classify()
+	if err != nil {
+		fmt.Println("select Code err:", err)
+		utils.FileLog.Info("GetBaseFromSci99Index err:", err)
+		return
+	}
+	for _, item := range allClassify {
+		ClassifyMap[item.ClassifyName] = item
+	}
+
+	if _, ok := ClassifyMap[req.ClassifyName]; !ok {
+		classifyItem := models.BaseFromSci99Classify{
+			ClassifyName: req.ClassifyName,
+			CreateTime:   time.Now(),
+			ModifyTime:   time.Now(),
+		}
+
+		id, e := models.AddBaseFromSci99Classify(&classifyItem)
+		if e != nil {
+			err = e
+			fmt.Println("Error inserting into database:", err)
+			utils.FileLog.Info("Error inserting into database:", err)
+			return
+		}
+		fmt.Println("新增分类：", classifyItem.ClassifyName)
+		classifyItem.BaseFromSciClassifyId = int(id)
+		ClassifyMap[req.ClassifyName] = &classifyItem
+	}
+
+	allCode, e := models.GetBaseFromSci99Index()
+	if e != nil {
+		err = e
+		fmt.Println("select Code err:", err)
+		utils.FileLog.Info("GetBaseFromSci99Index err:", err)
+		return
+	}
+
+	for _, item := range allCode {
+		IndexCodeMap[item.IndexName] = item.IndexCode
+		IndexMap[item.IndexName] = item
+	}
+
+	v := req.ListData
+	// 组成指标
+	indexName := req.IndexName
+	fmt.Println("indexName:", indexName)
+	indexCodeStr := "卓创资讯" + v.ProductName + v.PriceType
+	indexCode, needAdd := Sci99IndexCodeGenerator(indexName, indexCodeStr, v.MarketSampleName, v.FactorySampleName+v.Model)
+	if needAdd {
+		index := models.BaseFromSci99Index{
+			IndexCode:  indexCode,
+			IndexName:  indexName,
+			ClassifyId: ClassifyMap[req.ClassifyName].BaseFromSciClassifyId,
+			Unit:       v.Unit,
+			Frequency:  "日度",
+			Describe:   v.DataItemName,
+			CreateTime: time.Now(),
+			ModifyTime: time.Now(),
+		}
+		id, e := models.AddBaseFromSci99Index(&index)
+		if e != nil {
+			err = e
+			fmt.Println("Error inserting into database:", err)
+			utils.FileLog.Info("Error inserting into database:", err)
+			return
+		}
+		fmt.Println("新增指标：", index.IndexName)
+		index.BaseFromSciIndexId = int(id)
+		IndexMap[indexName] = &index
+	}
+
+	detailResponse := req.DetailData
+
+	existDataMap := make(map[string]*models.BaseFromSci99Data)
+
+	//获取所有指标信息
+	allData, e := models.GetBaseFromTradeSci99DataAll(indexCode)
+	if e != nil {
+		err = e
+		fmt.Println("select err:", err)
+		utils.FileLog.Info("GetBaseFromTradeSci99IndexAll err:", err)
+	}
+	for _, vv := range allData {
+		indexKey := vv.IndexCode + vv.DataTime
+		existDataMap[indexKey] = vv
+	}
+
+	// 新增data数据
+	addList := make([]*models.BaseFromSci99Data, 0)
+	for _, v := range detailResponse.Data.List {
+		if dataItem, ok := existDataMap[indexCode+v.RealDate]; ok {
+			if dataItem.Value != v.MDataValue {
+				// 更新
+				fmt.Println("更新指标：", indexCode+v.RealDate)
+				e = models.UpdateBaseFromSci99Data(dataItem.Value, indexCode, v.RealDate)
+				if e != nil {
+					err = e
+					fmt.Println("Error update into database:", err)
+					utils.FileLog.Info("Error update into database:", err)
+					return
+				}
+			}
+		} else {
+			// 新增
+			dataItem := models.BaseFromSci99Data{
+				BaseFromSciIndexId: IndexMap[indexName].BaseFromSciIndexId,
+				IndexCode:          indexCode,
+				DataTime:           v.RealDate,
+				Value:              v.MDataValue,
+				CreateTime:         time.Now(),
+				ModifyTime:         time.Now(),
+			}
+			addList = append(addList, &dataItem)
+			fmt.Println("新增数据：", indexCode+v.RealDate)
+		}
+	}
+	if len(addList) > 0 {
+		e = models.AddBaseFromSci99DataMulti(addList)
+		if e != nil {
+			err = e
+			fmt.Println("Error inserting into database:", err)
+			utils.FileLog.Info("Error inserting into database:", err)
+			return
+		}
+	}
+	time.Sleep(1 * time.Second)
+
+	br.Ret = 200
+	br.Msg = "获取成功"
+	br.Success = true
+}
+
+func Sci99IndexCodeGenerator(indexName, indexCodeStr, marketSampleName, model string) (indexCode string, needAdd bool) {
+	strResult := ""
+	indexCode, _ = IndexCodeMap[indexName]
+	if indexCode == "" {
+		//首字母
+		a := pinyin.NewArgs()
+		a.Fallback = func(r rune, a pinyin.Args) []string {
+			return []string{string(r)}
+		}
+		rows := pinyin.Pinyin(indexCodeStr, a)
+		for i := 0; i < len(rows); i++ {
+			//strResult += rows[i][0]
+			if len(rows[i]) != 0 {
+				str := rows[i][0]
+				pi := str[0:1]
+				strResult += pi
+			}
+		}
+
+		// 处理市场名称
+		if province, ok := ProvinceMap[marketSampleName]; ok {
+			strResult += province
+		} else {
+			a := pinyin.NewArgs()
+			rows := pinyin.LazyPinyin(marketSampleName, a)
+			for i := 0; i < len(rows); i++ {
+				strResult += rows[i]
+			}
+		}
+
+		// 去除特殊符号
+		model = strings.Replace(model, " ", "", -1)
+		model = strings.Replace(model, "-", "", -1)
+		model = strings.Replace(model, "/", "", -1)
+		model = strings.Replace(model, "#", "", -1)
+		model = strings.Replace(model, "：", "", -1)
+		model = strings.Replace(model, "(", "", -1)
+		model = strings.Replace(model, ")", "", -1)
+
+		// 拼接型号
+		modelRows := pinyin.Pinyin(model, a)
+		for i := 0; i < len(modelRows); i++ {
+			if len(modelRows[i]) != 0 {
+				str := modelRows[i][0]
+				pi := str[0:1]
+				strResult += pi
+			}
+		}
+
+		needAdd = true
+		indexCode = strings.Replace(strResult, " ", "", -1)
+		IndexCodeMap[indexName] = indexCode
+	}
+	return
+}
+
+var ProvinceMap = map[string]string{
+	"上海":  "shanghai",
+	"云南":  "yunnan",
+	"内蒙古": "innermongolia",
+	"北京":  "beijing",
+	"台湾":  "taiwan",
+	"吉林":  "jilin",
+	"四川":  "sichuan",
+	"天津":  "tianjin",
+	"宁夏":  "ningxia",
+	"安徽":  "anhui",
+	"山东":  "shandong",
+	"山西":  "shanxi",
+	"广东":  "guangdong",
+	"广西":  "guangxi",
+	"新疆":  "xinjiang",
+	"江苏":  "jiangsu",
+	"江西":  "jiangxi",
+	"河北":  "hebei",
+	"河南":  "henan",
+	"浙江":  "zhejiang",
+	"海南":  "hainan",
+	"湖北":  "hubei",
+	"湖南":  "hunan",
+	"澳门":  "macao",
+	"甘肃":  "gansu",
+	"福建":  "fujian",
+	"西藏":  "tibet",
+	"贵州":  "guizhou",
+	"辽宁":  "liaoning",
+	"重庆":  "chongqing",
+	"陕西":  "shaanxi",
+	"青海":  "qinhai",
+	"香港":  "hongkong",
+	"黑龙江": "heilongjiang",
+}
+
+var requestList = []sci99.ListRequestBody{
+	{
+		Ppname: "天然橡胶原料",
+		PPIDs:  "13676",
+		Navid:  "593",
+	}, {
+		Ppname: "混合胶",
+		PPIDs:  "12973",
+		Navid:  "591",
+	}, {
+		Ppname: "标准胶",
+		PPIDs:  "12960",
+		Navid:  "590",
+	}, {
+		Ppname: "烟片胶",
+		PPIDs:  "12985",
+		Navid:  "592",
+	}, {
+		Ppname: "天然乳胶",
+		PPIDs:  "12947",
+		Navid:  "595",
+	}, {
+		Ppname: "丁苯橡胶",
+		PPIDs:  "12951",
+		Navid:  "596",
+	}, {
+		Ppname: "顺丁橡胶",
+		PPIDs:  "12964",
+		Navid:  "597",
+	}, {
+		Ppname: "SBS",
+		PPIDs:  "12948",
+		Navid:  "598",
+	}, {
+		Ppname: "丁腈橡胶",
+		PPIDs:  "12945",
+		Navid:  "605",
+	},
+}

controllers/shanghai_smm.go

@@ -0,0 +1,379 @@
+package controllers
+
+import (
+	"encoding/json"
+	"eta/eta_index_lib/models"
+	shanghaismm "eta/eta_index_lib/services/shanghai_smm"
+	"eta/eta_index_lib/utils"
+	"fmt"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/mozillazg/go-pinyin"
+)
+
+type ShanghaiSmmController struct {
+	BaseAuthController
+}
+
+var ShanghaiSmmNameToIndexMap = make(map[string]*models.BaseFromSmmIndex)
+var ShanghaiSmmCodeToNameMap = make(map[string]string)
+var ShanghaiSmmNameToCodeMap = make(map[string]string)
+
+// @Title 刷新数据
+// @Description 刷新数据接口
+// @Param	request	body models.AddEdbClassifyReq true "type json string"
+// @Success 200 {object} models.EdbClassify
+// @router /refresh/list [post]
+func (this *ShanghaiSmmController) RefreshData() {
+	br := new(models.BaseResponse).Init()
+	defer func() {
+		this.Data["json"] = br
+		this.ServeJSON()
+	}()
+
+	var req []*shanghaismm.EdbInfoData
+	if err := json.Unmarshal(this.Ctx.Input.RequestBody, &req); err != nil {
+		br.Msg = "参数解析异常！"
+		br.ErrMsg = "参数解析失败，Err:" + err.Error()
+		return
+	}
+
+	allIndex, err := models.GetBaseFromSmmIndex()
+	if err != nil {
+		fmt.Println("select Code err:", err)
+		utils.FileLog.Info("GetBaseFromSci99Index err:", err)
+		return
+	}
+	for _, item := range allIndex {
+		ShanghaiSmmNameToIndexMap[item.IndexName] = item
+		ShanghaiSmmNameToCodeMap[item.IndexName] = item.IndexCode
+	}
+
+	// 检查指标是否存在，如果不存在添加指标
+	addIndexList := make([]*models.BaseFromSmmIndex, 0)
+	updateIndexList := make([]*models.BaseFromSmmIndex, 0)
+	addDateList := make([]*models.BaseFromSmmData, 0)
+	readyDateList := make([]*models.BaseFromSmmData, 0)
+	for _, v := range req {
+		indexInfo := ShanghaiSmmNameToIndexMap[v.IndexName]
+		if indexInfo == nil {
+			// 指标不存在，需要添加指标索引
+			indexCodeStr := "smm" + v.IndexName
+			indexCodeStr, needAdd := ShanghaiSmmIndexCodeGenerator(v.IndexName, indexCodeStr)
+			// 添加指标
+			if needAdd {
+				fmt.Printf("Name:%s, indexCode:%s \n", v.IndexName, indexCodeStr)
+				// 添加指标
+				item := new(models.BaseFromSmmIndex)
+				item.IndexCode = indexCodeStr
+				item.IndexName = v.IndexName
+				item.Frequency = v.Frequency
+				item.StartDate = v.RenewDate
+				item.EndDate = v.RenewDate
+				item.CreateTime = time.Now()
+				item.Unit = v.Unit
+				item.ModifyTime = time.Now()
+				addIndexList = append(addIndexList, item)
+
+				// 添加指标数据
+				itemDate := new(models.BaseFromSmmData)
+				itemDate.IndexCode = indexCodeStr
+				itemDate.DataTime = v.RenewDate
+				if v.Value == "highs" {
+					itemDate.Value = strconv.FormatFloat(v.Highs, 'f', -1, 64)
+				} else if v.Value == "low" {
+					itemDate.Value = strconv.FormatFloat(v.Low, 'f', -1, 64)
+				} else {
+					itemDate.Value = strconv.FormatFloat(v.Average, 'f', -1, 64)
+				}
+				itemDate.CreateTime = time.Now()
+				itemDate.ModifyTime = time.Now()
+				itemDate.DataTimestamp = time.Now().UnixMilli()
+				readyDateList = append(readyDateList, itemDate)
+			} else {
+				fmt.Printf("有重复的指标或指标名称 Name:%s, indexCode:%s \n", v.IndexName, indexCodeStr)
+				return
+			}
+		} else {
+			//存在指标，检查是否需要更新指标索引，及指标数据
+			if indexInfo.EndDate != v.RenewDate {
+				// 指标日期更新
+				t := new(models.BaseFromSmmIndex)
+				t.BaseFromSmmIndexId = indexInfo.BaseFromSmmIndexId
+				t.EndDate = v.RenewDate
+				updateIndexList = append(updateIndexList, t)
+
+				// 指标数据更新
+				item := new(models.BaseFromSmmData)
+				item.BaseFromSmmIndexId = int(indexInfo.BaseFromSmmIndexId)
+				item.IndexCode = indexInfo.IndexCode
+				item.DataTime = v.RenewDate
+				if v.Value == "highs" {
+					item.Value = strconv.FormatFloat(v.Highs, 'f', -1, 64)
+				} else if v.Value == "low" {
+					item.Value = strconv.FormatFloat(v.Low, 'f', -1, 64)
+				} else {
+					item.Value = strconv.FormatFloat(v.Average, 'f', -1, 64)
+				}
+				item.CreateTime = time.Now()
+				item.ModifyTime = time.Now()
+				item.DataTimestamp = time.Now().UnixMilli()
+				addDateList = append(addDateList, item)
+			}
+		}
+	}
+	if len(addIndexList) > 0 {
+		_, err = models.AddBaseFromSmmIndex(addIndexList)
+		if err != nil {
+			utils.FileLog.Info("Error 插入指标库失败:", err)
+			br.Msg = "插入指标库失败"
+			br.ErrMsg = "插入指标库失败,Err:" + err.Error()
+			return
+		}
+	}
+	if len(updateIndexList) > 0 {
+		for _, v := range updateIndexList {
+			err = models.ModifyBaseFromSmmIndexDate(v)
+			if err != nil {
+				br.Msg = "更新指标数据失败"
+				br.ErrMsg = "更新指标失败,Err:" + err.Error()
+				return
+			}
+		}
+	}
+	if len(addDateList) > 0 {
+		err = models.AddBaseFromSmmData(addDateList)
+		if err != nil {
+			utils.FileLog.Info("Error 插入指标数据失败:", err)
+			br.Msg = "插入指标数据失败"
+			br.ErrMsg = "插入指标数据失败,Err:" + err.Error()
+			return
+		}
+	}
+	allIndex, err = models.GetBaseFromSmmIndex()
+	if err != nil {
+		fmt.Println("select Code err:", err)
+		utils.FileLog.Info("GetBaseFromSci99Index err:", err)
+		return
+	}
+	// 获得更新后的数据
+	for _, item := range allIndex {
+		ShanghaiSmmNameToIndexMap[item.IndexName] = item
+		ShanghaiSmmNameToCodeMap[item.IndexName] = item.IndexCode
+	}
+	for _, v := range readyDateList {
+		indexName := ShanghaiSmmCodeToNameMap[v.IndexCode]
+		baseSmmIndex := ShanghaiSmmNameToIndexMap[indexName]
+		v.BaseFromSmmIndexId = int(baseSmmIndex.BaseFromSmmIndexId)
+	}
+	if len(readyDateList) > 0 {
+		err = models.AddBaseFromSmmData(readyDateList)
+		if err != nil {
+			utils.FileLog.Info("Error 插入指标数据失败:", err)
+			br.Msg = "插入指标数据失败"
+			br.ErrMsg = "插入指标数据失败,Err:" + err.Error()
+			return
+		}
+	}
+	br.Msg = "数据刷新成功"
+	br.Success = true
+	br.Ret = 200
+}
+
+// @Title 刷新数据
+// @Description 刷新数据接口
+// @Param	request	body models.AddEdbClassifyReq true "type json string"
+// @Success 200 {object} models.EdbClassify
+// @router /refresh/excel [post]
+func (this *ShanghaiSmmController) RefreshExcel() {
+	br := new(models.BaseResponse).Init()
+	defer func() {
+		this.Data["json"] = br
+		this.ServeJSON()
+	}()
+
+	var req []*shanghaismm.EdbInfoData
+	if err := json.Unmarshal(this.Ctx.Input.RequestBody, &req); err != nil {
+		br.Msg = "参数解析异常！"
+		br.ErrMsg = "参数解析失败，Err:" + err.Error()
+		return
+	}
+
+	allIndex, err := models.GetBaseFromSmmIndex()
+	if err != nil {
+		fmt.Println("select Code err:", err)
+		utils.FileLog.Info("GetBaseFromSci99Index err:", err)
+		return
+	}
+	ShanghaiSmmCodeToIndexMap := make(map[string]*models.BaseFromSmmIndex)
+	ShanghaiSmmCodeToIndexCount := make(map[string]int)
+	for _, item := range allIndex {
+		ShanghaiSmmCodeToIndexMap[item.IndexCode] = item
+		ShanghaiSmmCodeToIndexCount[item.IndexCode] = 0
+		ShanghaiSmmNameToCodeMap[item.IndexCode] = item.IndexName
+	}
+	updateIndexList := make([]*models.BaseFromSmmIndex, 0)
+	addDataList := make([]*models.BaseFromSmmData, 0)
+	updateDataList := make([]*models.BaseFromSmmData, 0)
+
+	queryIndexIds := make([]int, 0)
+	for _, v := range req {
+		indexCode := strings.Replace(v.IndexCode, " ", "", -1)
+		indexInfo := ShanghaiSmmCodeToIndexMap[indexCode]
+		if indexInfo == nil {
+			fmt.Printf("指标名称或指标id有误v.IndexName:%s, v.IndexCode:%s", v.IndexName, v.IndexCode)
+			return
+		}
+		ShanghaiSmmCodeToIndexCount[v.IndexCode] += 1
+		if ShanghaiSmmCodeToIndexCount[v.IndexCode] > 1 {
+			fmt.Printf("指标名称或指标id有误v.IndexName:%s, v.IndexCode:%s", v.IndexName, v.IndexCode)
+			return
+		}
+		queryIndexIds = append(queryIndexIds, int(indexInfo.BaseFromSmmIndexId))
+	}
+	list, err := models.GetBaseFromSmmDataByIds(queryIndexIds)
+	if err != nil {
+		br.Msg = "查询指标数据失败"
+		br.ErrMsg = "查询指标数据失败,Err:" + err.Error()
+		return
+	}
+	dateDataMap := make(map[int]map[string]string)
+	for _, v := range list {
+		dataMap, ok := dateDataMap[v.BaseFromSmmIndexId]
+		if ok {
+			dataMap[v.DataTime] = v.Value
+		} else {
+			dateDataMap[v.BaseFromSmmIndexId] = map[string]string{
+				v.DataTime: v.Value,
+			}
+		}
+	}
+	for _, v := range req {
+		indexCode := strings.Replace(v.IndexCode, " ", "", -1)
+		indexInfo := ShanghaiSmmCodeToIndexMap[indexCode]
+
+		isAdd := false
+		indexItem := new(models.BaseFromSmmIndex)
+		indexItem.BaseFromSmmIndexId = indexInfo.BaseFromSmmIndexId
+		startDate, _ := time.Parse(utils.FormatDate, indexInfo.StartDate)
+		endDate, _ := time.Parse(utils.FormatDate, indexInfo.EndDate)
+		if v.LastDate.After(endDate) {
+			isAdd = true
+			indexItem.EndDate = v.LastDate.Format(utils.FormatDate)
+		} else {
+			indexItem.EndDate = indexInfo.EndDate
+		}
+		if v.OldDate.Before(startDate) {
+			isAdd = true
+			indexItem.StartDate = v.OldDate.Format(utils.FormatDate)
+		} else {
+			indexItem.StartDate = indexInfo.StartDate
+		}
+		if isAdd {
+			indexItem.IndexName = v.IndexName
+			indexItem.ModifyTime = time.Now()
+			updateIndexList = append(updateIndexList, indexItem)
+		}
+		for k, dv := range v.Data {
+			valueStr := strconv.FormatFloat(dv, 'f', -1, 64)
+			valueRaw := dateDataMap[int(indexInfo.BaseFromSmmIndexId)][k]
+			if valueRaw == "" {
+				dataItem := new(models.BaseFromSmmData)
+				dataItem.BaseFromSmmIndexId = int(indexInfo.BaseFromSmmIndexId)
+				dataItem.IndexCode = indexInfo.IndexCode
+				dataItem.DataTime = k
+				dataItem.Value = valueStr
+				dataItem.CreateTime = time.Now()
+				dataItem.ModifyTime = time.Now()
+				dataItem.DataTimestamp = time.Now().UnixMilli()
+				addDataList = append(addDataList, dataItem)
+			} else if valueStr != "" && valueRaw != valueStr {
+				dataItem := new(models.BaseFromSmmData)
+				dataItem.BaseFromSmmIndexId = int(indexInfo.BaseFromSmmIndexId)
+				dataItem.IndexCode = indexInfo.IndexCode
+				dataItem.DataTime = k
+				dataItem.Value = valueStr
+				dataItem.ModifyTime = time.Now()
+				dataItem.DataTimestamp = time.Now().UnixMilli()
+				updateDataList = append(updateDataList, dataItem)
+			}
+		}
+	}
+	if len(addDataList) > 0 {
+		err = models.AddBaseFromSmmData(addDataList)
+		if err != nil {
+			br.Msg = "插入指标数据失败"
+			br.ErrMsg = "插入指标数据失败,Err:" + err.Error()
+			return
+		}
+	}
+	if len(updateIndexList) > 0 {
+		for _, v := range updateIndexList {
+			err = v.UpdateCols([]string{"end_date", "start_date", "modify_time"})
+			if err != nil {
+				br.Msg = "更新指标索引失败"
+				br.ErrMsg = "更新指标索引失败,Err:" + err.Error()
+				return
+			}
+		}
+	}
+	// 不修改
+	fmt.Println(len(updateDataList))
+	// if len(updateDataList) > 0 {
+	// 	err = models.UpdateBaseFromSmmDataListByIndexCode(updateDataList)
+	// 	if err != nil {
+	// 		br.Msg = "更新指标数据失败"
+	// 		br.ErrMsg = "更新指标失败,Err:" + err.Error()
+	// 		return
+	// 	}
+	// }
+	br.Ret = 200
+	br.Success = true
+	br.Msg = "数据刷新成功"
+}
+
+func ShanghaiSmmIndexCodeGenerator(indexName, indexCodeStr string) (indexCode string, needAdd bool) {
+	strResult := ""
+	//首字母
+	a := pinyin.NewArgs()
+	a.Fallback = func(r rune, a pinyin.Args) []string {
+		return []string{string(r)}
+	}
+	rows := pinyin.Pinyin(indexCodeStr, a)
+	for i := 0; i < len(rows); i++ {
+		//strResult += rows[i][0]
+		if len(rows[i]) != 0 {
+			str := rows[i][0]
+			pi := str[0:1]
+			strResult += pi
+		}
+	}
+
+	// 去除特殊符号
+	strResult = strings.Replace(strResult, " ", "", -1)
+	strResult = strings.Replace(strResult, "-", "", -1)
+	strResult = strings.Replace(strResult, "/", "", -1)
+	strResult = strings.Replace(strResult, "#", "", -1)
+	strResult = strings.Replace(strResult, "：", "", -1)
+	strResult = strings.Replace(strResult, "(", "", -1)
+	strResult = strings.Replace(strResult, ")", "", -1)
+	strResult = strings.Replace(strResult, "%", "", -1)
+	strResult = strings.Replace(strResult, "<", "1", -1)
+	strResult = strings.Replace(strResult, "\xe2", "2", -1) // ≥
+	strResult = strings.Replace(strResult, ".", "", -1)
+	strResult = strings.Replace(strResult, ",", "", -1)
+	strResult = strings.Replace(strResult, ":", "", -1)
+
+	needAdd = true
+	indexCode = strings.Replace(strResult, " ", "", -1)
+	indexCode = strings.ToLower(indexCode)
+	if _, ok := ShanghaiSmmCodeToNameMap[indexCode]; !ok {
+		ShanghaiSmmCodeToNameMap[indexCode] = indexName
+		ShanghaiSmmNameToCodeMap[indexName] = indexCode
+	} else {
+		needAdd = false
+	}
+	return
+}

go.mod

@@ -12,9 +12,11 @@ require (
 	github.com/mozillazg/go-pinyin v0.20.0
 	github.com/nosixtools/solarlunar v0.0.0-20211112060703-1b6dea7b4a19
 	github.com/olivere/elastic/v7 v7.0.32
+	github.com/qiniu/qmgo v1.1.8
 	github.com/rdlucklib/rdluck_tools v1.0.3
 	github.com/shopspring/decimal v1.3.1
 	github.com/yidane/formula v0.0.0-20210902154546-0782e1736717
+	go.mongodb.org/mongo-driver v1.15.0
 	gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df
 )
 
@@ -23,7 +25,11 @@ require (
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/cespare/xxhash/v2 v2.1.2 // indirect
 	github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
+	github.com/go-playground/locales v0.13.0 // indirect
+	github.com/go-playground/universal-translator v0.17.0 // indirect
+	github.com/go-playground/validator/v10 v10.4.1 // indirect
 	github.com/golang/protobuf v1.5.2 // indirect
+	github.com/golang/snappy v0.0.1 // indirect
 	github.com/gonum/blas v0.0.0-20181208220705-f22b278b28ac // indirect
 	github.com/gonum/floats v0.0.0-20181209220543-c233463c7e82 // indirect
 	github.com/gonum/integrate v0.0.0-20181209220457-a422b5c0fdf2 // indirect
@@ -32,21 +38,29 @@ require (
 	github.com/gonum/matrix v0.0.0-20181209220409-c518dec07be9 // indirect
 	github.com/hashicorp/golang-lru v0.5.4 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
+	github.com/klauspost/compress v1.13.6 // indirect
+	github.com/leodido/go-urn v1.2.0 // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
 	github.com/mitchellh/mapstructure v1.4.1 // indirect
+	github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/prometheus/client_golang v1.11.0 // indirect
 	github.com/prometheus/client_model v0.2.0 // indirect
 	github.com/prometheus/common v0.26.0 // indirect
 	github.com/prometheus/procfs v0.6.0 // indirect
 	github.com/shiena/ansicolor v0.0.0-20200904210342-c7312218db18 // indirect
-	golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a // indirect
-	golang.org/x/net v0.0.0-20210428140749-89ef3d95e781 // indirect
-	golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e // indirect
-	golang.org/x/text v0.3.6 // indirect
+	github.com/xdg-go/pbkdf2 v1.0.0 // indirect
+	github.com/xdg-go/scram v1.1.2 // indirect
+	github.com/xdg-go/stringprep v1.0.4 // indirect
+	github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect
+	golang.org/x/crypto v0.17.0 // indirect
+	golang.org/x/net v0.10.0 // indirect
+	golang.org/x/sync v0.1.0 // indirect
+	golang.org/x/sys v0.15.0 // indirect
+	golang.org/x/text v0.14.0 // indirect
 	google.golang.org/protobuf v1.26.0 // indirect
 	gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
-	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

go.sum

@@ -142,6 +142,14 @@ github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vb
 github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
 github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A=
+github.com/go-playground/assert/v2 v2.0.1 h1:MsBgLAaY856+nPRTKrp3/OZK38U/wa0CcBYNjji3q3A=
+github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
+github.com/go-playground/locales v0.13.0 h1:HyWk6mgj5qFqCT5fjGBuRArbVDfE4hi8+e8ceBS/t7Q=
+github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8=
+github.com/go-playground/universal-translator v0.17.0 h1:icxd5fm+REJzpZx7ZfpaD876Lmtgy7VtROAbHHXk8no=
+github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA=
+github.com/go-playground/validator/v10 v10.4.1 h1:pH2c5ADXtd66mxoE0Zm9SUhxE20r7aM3F26W0hOn+GE=
+github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn9GlaMV7XkbRSipzJ0Ii4=
 github.com/go-redis/redis v6.14.2+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA=
 github.com/go-redis/redis/v7 v7.4.0/go.mod h1:JDNMw23GTyLNC4GZu9njt15ctBQVn7xjRfnwdHj/Dcg=
 github.com/go-redis/redis/v8 v8.11.5 h1:AcZZR7igkdvfVmQTPnu9WE37LRrO/YrBH5zWyjDC0oI=
@@ -182,6 +190,8 @@ github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw
 github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
 github.com/golang/snappy v0.0.0-20170215233205-553a64147049/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
+github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/gomodule/redigo v2.0.0+incompatible/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4=
 github.com/gonum/blas v0.0.0-20181208220705-f22b278b28ac h1:Q0Jsdxl5jbxouNs1TQYt0gxesYMU4VXRbsTlgDloZ50=
 github.com/gonum/blas v0.0.0-20181208220705-f22b278b28ac/go.mod h1:P32wAyui1PQ58Oce/KYkOqQv8cVw1zAapXOl+dRFGbc=
@@ -204,6 +214,7 @@ github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw
 github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.7 h1:81/ik6ipDQS2aGcBfIN5dHDB36BwrStyeAQquSYCV4o=
@@ -282,6 +293,8 @@ github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvW
 github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
 github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
+github.com/klauspost/compress v1.13.6 h1:P76CopJELS0TiO2mebmnzgWaajssP/EszplttgQxcgc=
+github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
@@ -290,6 +303,8 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/ledisdb/ledisdb v0.0.0-20200510135210-d35789ec47e6/go.mod h1:n931TsDuKuq+uX4v1fulaMbA/7ZLLhjc85h7chZGBCQ=
+github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y=
+github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII=
 github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
 github.com/lib/pq v1.7.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
 github.com/lib/pq v1.10.2/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
@@ -327,6 +342,8 @@ github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJ
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe h1:iruDEfMl2E6fbMZ9s0scYfZQ84/6SPL6zC8ACM2oIL0=
+github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc=
 github.com/mozillazg/go-pinyin v0.20.0 h1:BtR3DsxpApHfKReaPO1fCqF4pThRwH9uwvXzm+GnMFQ=
 github.com/mozillazg/go-pinyin v0.20.0/go.mod h1:iR4EnMMRXkfpFVV5FMi4FNB6wGq9NV6uDWbUuPhP4Yc=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
@@ -421,6 +438,8 @@ github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4O
 github.com/prometheus/procfs v0.6.0 h1:mxy4L2jP6qMonqmq+aTtOx1ifVWUgG/TAmntgbh3xv4=
 github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
 github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
+github.com/qiniu/qmgo v1.1.8 h1:E64M+P59aqQpXKI24ClVtluYkLaJLkkeD2hTVhrdMks=
+github.com/qiniu/qmgo v1.1.8/go.mod h1:QvZkzWNEv0buWPx0kdZsSs6URhESVubacxFPlITmvB8=
 github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
 github.com/rdlucklib/rdluck_tools v1.0.3 h1:iOtK2QPlPQ6CL6c1htCk5VnFCHzyG6DCfJtunrMswK0=
 github.com/rdlucklib/rdluck_tools v1.0.3/go.mod h1:9Onw9o4w19C8KE5lxb8GyxgRBbZweRVkQSc79v38EaA=
@@ -464,29 +483,43 @@ github.com/streadway/amqp v0.0.0-20190404075320-75d898a42a94/go.mod h1:AZpEONHx3
 github.com/streadway/amqp v0.0.0-20190827072141-edfb9018d271/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw=
 github.com/streadway/handy v0.0.0-20190108123426-d5acb3125c2a/go.mod h1:qNTQ5P5JnDBl6z3cMAg/SywNDC5ABu5ApDIw6lUbRmI=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.1.1 h1:2vfRuCMp5sSVIDSqO8oNnWJq7mPa6KVP3iPIwFBuy8A=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
+github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
 github.com/syndtr/goleveldb v0.0.0-20160425020131-cfa635847112/go.mod h1:Z4AUp2Km+PwemOoO/VB5AOx9XSsIItzFjoJlOSiYmn0=
 github.com/syndtr/goleveldb v0.0.0-20181127023241-353a9fca669c/go.mod h1:Z4AUp2Km+PwemOoO/VB5AOx9XSsIItzFjoJlOSiYmn0=
+github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
 github.com/ugorji/go v0.0.0-20171122102828-84cb69a8af83/go.mod h1:hnLbHMwcvSihnDhEfx2/BzKp2xb0Y+ErdfYcrs9tkJQ=
 github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
 github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
 github.com/wendal/errors v0.0.0-20130201093226-f66c77a7882b/go.mod h1:Q12BUT7DqIlHRmgv3RskH+UCM/4eqVMgI0EMmlSpAXc=
+github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c=
+github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
+github.com/xdg-go/scram v1.1.1/go.mod h1:RaEWvsqvNKKvBPvcKeFjrG2cJqOkHTiyTpzz23ni57g=
+github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY=
+github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4=
+github.com/xdg-go/stringprep v1.0.3/go.mod h1:W3f5j4i+9rC0kuIEJL0ky1VpHXQU3ocBgklLGvcBnW8=
+github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8=
+github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM=
 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
 github.com/yidane/formula v0.0.0-20210902154546-0782e1736717 h1:9CTJJpdISGxMAELfVlprj5kZEsJEaNAWiobv8ZAd72U=
 github.com/yidane/formula v0.0.0-20210902154546-0782e1736717/go.mod h1:9/dQiKiN04yPMdgsuFmKGuI2Hdp6OmFV9gSWS1col6g=
 github.com/ylywyn/jpush-api-go-client v0.0.0-20190906031852-8c4466c6e369/go.mod h1:Nv7wKD2/bCdKUFNKcJRa99a+1+aSLlCRJFriFYdjz/I=
+github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d h1:splanxYIlg+5LfHAM6xpdFEAYOk8iySO56hMFq6uLyA=
+github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
+github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
 github.com/yuin/gopher-lua v0.0.0-20171031051903-609c9cd26973/go.mod h1:aEV29XrmTYFr3CiRxZeGHpkvbwq+prZduBqMaascyCU=
 go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
 go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
@@ -495,6 +528,9 @@ go.etcd.io/etcd v3.3.25+incompatible/go.mod h1:yaeTdrJi5lOmYerz05bd8+V7KubZs8YSF
 go.etcd.io/etcd/api/v3 v3.5.0/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs=
 go.etcd.io/etcd/client/pkg/v3 v3.5.0/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g=
 go.etcd.io/etcd/client/v3 v3.5.0/go.mod h1:AIKXXVX/DQXtfTEqBryiLTUXwON+GuvO6Z7lLS/oTh0=
+go.mongodb.org/mongo-driver v1.11.6/go.mod h1:G9TgswdsWjX4tmDA5zfs2+6AEPpYJwqblyjsfuh8oXY=
+go.mongodb.org/mongo-driver v1.15.0 h1:rJCKC8eEliewXjZGf0ddURtl7tTVy1TK3bfl0gkUSLc=
+go.mongodb.org/mongo-driver v1.15.0/go.mod h1:Vzb0Mk/pa7e6cWw85R4F/endUC3u0U9jGcNU603k65c=
 go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk=
 go.opencensus.io v0.20.2/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk=
 go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
@@ -524,8 +560,11 @@ golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8U
 golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
-golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a h1:kr2P4QFmQr29mSLA43kwrOcgcReGTfbE9N577tCTuBc=
 golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8=
+golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
+golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
+golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k=
+golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@@ -550,6 +589,7 @@ golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzB
 golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -575,8 +615,10 @@ golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81R
 golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
-golang.org/x/net v0.0.0-20210428140749-89ef3d95e781 h1:DzZ89McO9/gWPsQXS/FVKAlG02ZjaQ6AlZRBimEYOd0=
-golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk=
+golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
+golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M=
+golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -590,6 +632,9 @@ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o=
+golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -625,16 +670,23 @@ golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e h1:fLOSk5Q00efkSvAm+4xcoXD+RRmLmmulPn5I3Y9F2EM=
-golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc=
+golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
+golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
+golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
 golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
@@ -671,6 +723,7 @@ golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roY
 golang.org/x/tools v0.0.0-20201211185031-d93e913c1a58/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
+golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -758,8 +811,9 @@ gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=

logic/base_edb_info.go

@@ -29,6 +29,11 @@ func RefreshBaseEdbInfo(edbInfo *models.EdbInfo, startDate string) (isHandling b
 		err = models.RefreshEdbDataFromMysteelChemical(edbInfo.EdbInfoId, edbInfo.EdbCode, startDate)
 	case utils.DATA_SOURCE_YS:
 		err = models.RefreshEdbDataFromSmm(edbInfo.EdbInfoId, edbInfo.EdbCode, startDate)
+	case utils.DATA_SOURCE_BLOOMBERG:
+		err = models.RefreshEdbDataFromBloomberg(edbInfo.EdbInfoId, edbInfo.EdbCode, startDate)
+	case utils.DATA_SOURCE_CCF:
+		ccfOb := new(models.BaseFromCCF)
+		err = ccfOb.Refresh(edbInfo.EdbInfoId, edbInfo.EdbCode, startDate)
 	default:
 		return
 	}

logic/chart_correlation.go

@@ -0,0 +1,195 @@
+package logic
+
+import (
+	"eta/eta_index_lib/models"
+	"eta/eta_index_lib/utils"
+	"fmt"
+)
+
+// CalculateCorrelation 计算相关性-获取x轴和y轴
+func CalculateCorrelation(leadValue int, leadUnit, frequencyA, frequencyB string, dataListA, dataListB []*models.EdbInfoSearchData) (xEdbIdValue []int, yDataList []YData, err error) {
+	xData := make([]int, 0)
+	yData := make([]float64, 0)
+	if leadValue == 0 {
+		xData = append(xData, 0)
+	}
+	if leadValue > 0 {
+		leadMin := 0 - leadValue
+		xLen := 2*leadValue + 1
+		for i := 0; i < xLen; i++ {
+			n := leadMin + i
+			xData = append(xData, n)
+		}
+	}
+
+	// 计算窗口，不包含第一天
+	//startDateTime, _ := time.ParseInLocation(utils.FormatDate, startDate, time.Local)
+	//startDate = startDateTime.AddDate(0, 0, 1).Format(utils.FormatDate)
+
+	//// 2023-03-02 时间序列始终以指标B为基准, 始终是A进行平移
+	//baseEdbInfo := edbInfoMappingB
+	//changeEdbInfo := edbInfoMappingA
+	// 2023-03-17 时间序列始终以指标A为基准, 始终是B进行平移
+	//baseEdbInfo := edbInfoMappingA
+	//changeEdbInfo := edbInfoMappingB
+
+	// 获取时间基准指标在时间区间内的值
+	//aDataList := make([]*EdbDataList, 0)
+	//switch baseEdbInfo.EdbInfoCategoryType {
+	//case 0:
+	//	aDataList, err = GetEdbDataList(baseEdbInfo.Source, baseEdbInfo.SubSource, baseEdbInfo.EdbInfoId, startDate, endDate)
+	//case 1:
+	//	_, aDataList, _, _, err, _ = data.GetPredictDataListByPredictEdbInfoId(baseEdbInfo.EdbInfoId, startDate, endDate, false)
+	//default:
+	//	err = errors.New("指标base类型异常")
+	//	return
+	//}
+	//
+	//// 获取变频指标所有日期的值, 插值法完善数据
+	//bDataList := make([]*EdbDataList, 0)
+	//switch changeEdbInfo.EdbInfoCategoryType {
+	//case 0:
+	//	bDataList, err = GetEdbDataList(changeEdbInfo.Source, changeEdbInfo.SubSource, changeEdbInfo.EdbInfoId, "", "")
+	//case 1:
+	//	_, bDataList, _, _, err, _ = data.GetPredictDataListByPredictEdbInfoId(changeEdbInfo.EdbInfoId, "", "", false)
+	//default:
+	//	err = errors.New("指标change类型异常")
+	//	return
+	//}
+	//changeDataMap := make(map[string]float64)
+	//newChangeDataList, e := HandleDataByLinearRegression(bDataList, changeDataMap)
+	//if e != nil {
+	//	err = fmt.Errorf("获取变频指标插值法Map失败, Err: %s", e.Error())
+	//	return
+	//}
+
+	// 2023-03-17 时间序列始终以指标A为基准, 始终是B进行平移
+	baseDataList := make([]*models.EdbInfoSearchData, 0)
+	baseDataMap := make(map[string]float64)
+	changeDataList := make([]*models.EdbInfoSearchData, 0)
+	changeDataMap := make(map[string]float64)
+
+	// 先把低频指标升频为高频
+	{
+		frequencyIntMap := map[string]int{
+			"日度": 1,
+			"周度": 2,
+			"旬度": 3,
+			"月度": 4,
+			"季度": 5,
+			"年度": 6,
+		}
+
+		// 如果A指标是高频，那么就需要对B指标进行升频
+		if frequencyIntMap[frequencyA] < frequencyIntMap[frequencyB] {
+			tmpNewChangeDataList, e := models.HandleDataByLinearRegression(dataListB, changeDataMap)
+			if e != nil {
+				err = fmt.Errorf("获取变频指标插值法Map失败, Err: %s", e.Error())
+				return
+			}
+			changeDataList = tmpNewChangeDataList
+			baseDataList = dataListA
+			for _, v := range baseDataList {
+				baseDataMap[v.DataTime] = v.Value
+			}
+
+		} else if frequencyIntMap[frequencyA] > frequencyIntMap[frequencyB] {
+			// 如果B指标是高频，那么就需要对A指标进行升频
+			tmpNewChangeDataList, e := models.HandleDataByLinearRegression(dataListA, baseDataMap)
+			if e != nil {
+				err = fmt.Errorf("获取变频指标插值法Map失败, Err: %s", e.Error())
+				return
+			}
+			baseDataList = tmpNewChangeDataList
+
+			changeDataList = dataListB
+			for _, v := range changeDataList {
+				changeDataMap[v.DataTime] = v.Value
+			}
+		} else {
+			baseDataList = dataListA
+			for _, v := range baseDataList {
+				baseDataMap[v.DataTime] = v.Value
+			}
+			changeDataList = dataListB
+			for _, v := range changeDataList {
+				changeDataMap[v.DataTime] = v.Value
+			}
+		}
+
+	}
+
+	// 计算不领先也不滞后时的相关系数
+	baseCalculateData := make([]float64, 0)
+	baseDataTimeArr := make([]string, 0)
+	for i := range baseDataList {
+		baseDataTimeArr = append(baseDataTimeArr, baseDataList[i].DataTime)
+		baseCalculateData = append(baseCalculateData, baseDataList[i].Value)
+	}
+
+	//zeroBaseData := make([]float64, 0)
+	//zeroCalculateData := make([]float64, 0)
+	//for i := range baseDataTimeArr {
+	//	tmpBaseVal, ok1 := baseDataMap[baseDataTimeArr[i]]
+	//	tmpCalculateVal, ok2 := changeDataMap[baseDataTimeArr[i]]
+	//	if ok1 && ok2 {
+	//		zeroBaseData = append(zeroBaseData, tmpBaseVal)
+	//		zeroCalculateData = append(zeroCalculateData, tmpCalculateVal)
+	//	}
+	//}
+	//if len(zeroBaseData) != len(zeroCalculateData) {
+	//	err = fmt.Errorf("相关系数两组序列元素数不一致, %d-%d", len(baseCalculateData), len(zeroCalculateData))
+	//	return
+	//}
+	//zeroRatio := utils.CalculateCorrelationByIntArr(zeroBaseData, zeroCalculateData)
+	//if leadValue == 0 {
+	//	yData = append(yData, zeroRatio)
+	//}
+
+	// 计算领先/滞后N期
+	if leadValue > 0 {
+		// 平移变频指标领先/滞后的日期(单位天)
+		moveUnitDays := utils.FrequencyDaysMap[leadUnit]
+
+		for i := range xData {
+			//if xData[i] == 0 {
+			//	yData = append(yData, zeroRatio)
+			//	continue
+			//}
+			xCalculateData := make([]float64, 0)
+			yCalculateData := make([]float64, 0)
+
+			// 平移指定天数
+			mDays := int(moveUnitDays) * xData[i]
+			_, dMap := models.MoveDataDaysToNewDataList(changeDataList, mDays)
+
+			// 取出对应的基准日期的值
+			for i2 := range baseDataTimeArr {
+				tmpDate := baseDataTimeArr[i2]
+				if yVal, ok := dMap[tmpDate]; ok {
+					xCalculateData = append(xCalculateData, baseCalculateData[i2])
+					yCalculateData = append(yCalculateData, yVal)
+				}
+			}
+			if len(yCalculateData) <= 0 {
+				//err = fmt.Errorf("领先滞后相关系数两组序列元素数不一致, %d-%d", len(baseCalculateData), len(yCalculateData))
+				//return
+				// 领先滞后后，没有可以计算的数据了
+				continue
+			}
+
+			// 公式计算出领先/滞后频度对应点的相关性系数
+			ratio := utils.CalculateCorrelationByIntArr(xCalculateData, yCalculateData)
+			yData = append(yData, ratio)
+		}
+	}
+
+	xEdbIdValue = xData
+	yDataList = make([]YData, 0)
+	yDate := "0000-00-00"
+	yDataList = append(yDataList, YData{
+		Date:  yDate,
+		Value: yData,
+	})
+	return
+}

logic/profit_chart_info.go

@@ -411,7 +411,7 @@ func ProfitChartChartData(baseDataList []*models.EdbDataList, futureGoodEdbInfoM
 		xEdbInfoIdList := make([]int, 0)   // 当前数据的指标id列表
 
 		// 现货指标
-		realDateTime, findDataValue, isFind, tmpErr := GetNeedDateData(findDateTime, baseDataList, baseEdbDateData)
+		realDateTime, findDataValue, isFind, tmpErr := GetNeedDateData(findDateTime, baseDataList, baseEdbDateData, edbDataMap)
 		if tmpErr != nil {
 			err = tmpErr
 			return
@@ -437,7 +437,9 @@ func ProfitChartChartData(baseDataList []*models.EdbDataList, futureGoodEdbInfoM
 			//findDateTime
 
 			// 获取当前日期相对开始日期的期数
-			tmpN := (currDate.Year()-findDateTime.Year())*12 + int(currDate.Month()-findDateTime.Month())
+			//tmpN := (currDate.Year()-findDateTime.Year())*12 + int(currDate.Month()-findDateTime.Month())
+			// 用实际日期的月份作为基准，往前推12个月（2024-5-13 16:26:43修改）
+			tmpN := (currDate.Year()-realDateTime.Year())*12 + int(currDate.Month()-realDateTime.Month())
 			if tmpN <= 0 {
 				continue
 			}
@@ -834,7 +836,7 @@ func GetFormulaMap() map[string]string {
 }
 
 // GetNeedDateData 获取合约内需要的日期数据
-func GetNeedDateData(needDateTime time.Time, dataList []*models.EdbDataList, edbDataMap map[string]float64) (findDateTime time.Time, findDataValue float64, isFind bool, err error) {
+func GetNeedDateData(needDateTime time.Time, dataList []*models.EdbDataList, edbDataMap map[string]float64, allEdbDataMap map[int]map[string]float64) (findDateTime time.Time, findDataValue float64, isFind bool, err error) {
 	//dataList := edbDataListMap[edbInfoId] //指标的所有数据值
 	if len(dataList) <= 0 {
 		// 没有数据的指标id
@@ -847,18 +849,43 @@ func GetNeedDateData(needDateTime time.Time, dataList []*models.EdbDataList, edb
 		return
 	}
 
+	// 该日期存在数据的期货指标的最小数量，目前是现货和期货各1个，总共2个
+	maxCount := 1
+
 	for tmpDateTime := needDateTime; tmpDateTime.After(minDateTime) || tmpDateTime.Equal(minDateTime); tmpDateTime = tmpDateTime.AddDate(0, 0, -1) {
 		tmpDate := tmpDateTime.Format(utils.FormatDate)
-		if tmpValue, ok := edbDataMap[tmpDate]; ok { //如果能找到数据，那么就返回
-			// 数据为0,也直接返回,做无值处理
-			if tmpValue == 0 {
-				return
+		tmpValue, ok := edbDataMap[tmpDate]
+		if !ok {
+			continue
+		}
+
+		// 该日期存在数据的指标数量
+		count := 0
+
+		for _, currEdbDataMap := range allEdbDataMap {
+			_, tmpIsFind := currEdbDataMap[tmpDate]
+			if tmpIsFind {
+				count++
+				if count >= maxCount {
+					continue
+				}
 			}
-			findDateTime, _ = time.ParseInLocation(utils.FormatDate, tmpDate, time.Local)
-			findDataValue = tmpValue
-			isFind = true
+		}
+
+		// 该日期存在数据的期货指标数量小于2个，那么要继续往前找
+		if count < maxCount {
+			continue
+		}
+
+		//如果能找到数据，那么就返回
+		// 数据为0,也直接返回,做无值处理
+		if tmpValue == 0 {
 			return
 		}
+		findDateTime, _ = time.ParseInLocation(utils.FormatDate, tmpDate, time.Local)
+		findDataValue = tmpValue
+		isFind = true
+		return
 	}
 
 	return

models/base_calculate.go

@@ -304,6 +304,7 @@ func (obj BaseCalculate) Tbz() (dateDataMap map[time.Time]float64, err error, er
 	var dateArr []time.Time
 	dataMap := make(map[string]*EdbInfoData) // 避免因为时间戳导致的日期不对，还是用string来表示比较合适
 	for _, v := range dataList {
+		v.DataTime = time.Date(v.DataTime.Year(), v.DataTime.Month(), v.DataTime.Day(), 0, 0, 0, 0, v.DataTime.Location())
 		dateArr = append(dateArr, v.DataTime)
 		dataMap[v.DataTime.Format(utils.FormatDate)] = v
 	}
@@ -416,6 +417,7 @@ func (obj BaseCalculate) Tcz() (dateDataMap map[time.Time]float64, err error, er
 	var dateArr []time.Time
 	dataMap := make(map[string]*EdbInfoData)
 	for _, v := range dataList {
+		v.DataTime = time.Date(v.DataTime.Year(), v.DataTime.Month(), v.DataTime.Day(), 0, 0, 0, 0, v.DataTime.Location())
 		dateArr = append(dateArr, v.DataTime)
 		dataMap[v.DataTime.Format(utils.FormatDate)] = v
 	}
@@ -587,6 +589,7 @@ func (obj BaseCalculate) Hbz() (dateDataMap map[time.Time]float64, err error, er
 
 	var dateArr []time.Time
 	for _, v := range dataList {
+		v.DataTime = time.Date(v.DataTime.Year(), v.DataTime.Month(), v.DataTime.Day(), 0, 0, 0, 0, v.DataTime.Location())
 		dateArr = append(dateArr, v.DataTime)
 	}
 
@@ -662,11 +665,12 @@ func (obj BaseCalculate) Hcz() (dateDataMap map[time.Time]float64, err error, er
 
 	var dateArr []time.Time
 	for _, v := range dataList {
+		v.DataTime = time.Date(v.DataTime.Year(), v.DataTime.Month(), v.DataTime.Day(), 0, 0, 0, 0, v.DataTime.Location())
 		dateArr = append(dateArr, v.DataTime)
 	}
 
 	dataLen := len(dataList)
-	fmt.Println("dataLen:", dataLen)
+	//fmt.Println("dataLen:", dataLen)
 	for i := 0; i < dataLen; i++ {
 		j := i + formulaInt
 		if j < dataLen {
@@ -722,6 +726,7 @@ func (obj BaseCalculate) UpFrequency() (dateDataMap map[time.Time]float64, err e
 	fromDataMap := make(map[time.Time]float64)
 	//来源指指标数据
 	for _, v := range dataList {
+		v.DataTime = time.Date(v.DataTime.Year(), v.DataTime.Month(), v.DataTime.Day(), 0, 0, 0, 0, v.DataTime.Location())
 		dateArr = append(dateArr, v.DataTime)
 		dataMap[v.DataTime] = v
 		fromDataMap[v.DataTime] = v.Value
@@ -733,8 +738,12 @@ func (obj BaseCalculate) UpFrequency() (dateDataMap map[time.Time]float64, err e
 		return
 	}
 
+	nowDay := time.Now()
+	nowDay = time.Date(nowDay.Year(), nowDay.Month(), nowDay.Day(), 0, 0, 0, 0, nowDay.Location())
+
 	// 数据计算
 	dataLen := len(dataList)
+	lastI := dataLen - 1
 	for i := 0; i < dataLen; i++ {
 		//当期
 		currentItem := dataList[i]
@@ -742,22 +751,39 @@ func (obj BaseCalculate) UpFrequency() (dateDataMap map[time.Time]float64, err e
 		var day int
 		var preItem *EdbInfoData
 		var preDate time.Time
+
+		// 如果是第一期的数据，那么直接赋值
 		if i == 0 {
-			day = int(time.Now().Sub(currentDate).Hours() / float64(24))
-			preDate = time.Now()
-		} else {
-			j := i - 1
-			if j < dataLen {
-				preItem = dataList[j]
-				day = int(preItem.DataTime.Sub(currentDate).Hours() / float64(24))
-				utils.FileLog.Info("preItem.DataTime:" + preItem.DataTime.Format(utils.FormatDate) + ";currentItem.DataTime" + currentItem.DataTime.Format(utils.FormatDate))
-			}
+			dateDataMap[currentItem.DataTime] = currentItem.Value
+			continue
 		}
-		for k := 0; k <= day; k++ {
-			needDay := preDate.AddDate(0, 0, -k)
-			dateDataMap[needDay] = currentItem.Value
+
+		// 上一期
+		j := i - 1
+
+		preItem = dataList[j]
+		preDate = preItem.DataTime
+		day = int(currentDate.Sub(preItem.DataTime).Hours() / float64(24))
+		//utils.FileLog.Info("preItem.DataTime:" + preItem.DataTime.Format(utils.FormatDate) + ";currentItem.DataTime" + currentItem.DataTime.Format(utils.FormatDate))
+		//fmt.Println("preItem.DataTime:" + preItem.DataTime.Format(utils.FormatDate) + ";currentItem.DataTime:" + currentItem.DataTime.Format(utils.FormatDate))
+
+		for k := 1; k < day; k++ {
+			needDay := preDate.AddDate(0, 0, k)
+			dateDataMap[needDay] = preItem.Value
 		}
 		dateDataMap[currentItem.DataTime] = currentItem.Value
+
+		// 如果是最后一期的数据
+		if i == lastI {
+			day = int(nowDay.Sub(currentDate).Hours() / float64(24))
+			//utils.FileLog.Info("preItem.DataTime:" + currentDate.Format(utils.FormatDate) + ";currentItem.DataTime" + nowDay.Format(utils.FormatDate))
+
+			for k := 1; k <= day; k++ {
+				needDay := currentDate.AddDate(0, 0, k)
+				dateDataMap[needDay] = currentItem.Value
+			}
+		}
+
 	}
 
 	return
@@ -788,6 +814,7 @@ func (obj BaseCalculate) DownFrequency() (dateDataMap map[time.Time]float64, err
 	fromDataMap := make(map[time.Time]float64)
 	//来源指指标数据
 	for _, v := range dataList {
+		v.DataTime = time.Date(v.DataTime.Year(), v.DataTime.Month(), v.DataTime.Day(), 0, 0, 0, 0, v.DataTime.Location())
 		dateArr = append(dateArr, v.DataTime)
 		dataMap[v.DataTime] = v
 		fromDataMap[v.DataTime] = v.Value
@@ -954,6 +981,7 @@ func (obj BaseCalculate) TimeShift() (dateDataMap map[time.Time]float64, err err
 	var dateArr []time.Time
 	dataMap := make(map[time.Time]*EdbInfoData)
 	for _, v := range dataList {
+		v.DataTime = time.Date(v.DataTime.Year(), v.DataTime.Month(), v.DataTime.Day(), 0, 0, 0, 0, v.DataTime.Location())
 		dateArr = append(dateArr, v.DataTime)
 		dataMap[v.DataTime] = v
 	}
@@ -1017,6 +1045,7 @@ func (obj BaseCalculate) Cjjx() (dateDataMap map[time.Time]float64, err error, e
 	var dateArr []time.Time
 	dataMap := make(map[time.Time]*EdbInfoData)
 	for _, v := range dataList {
+		v.DataTime = time.Date(v.DataTime.Year(), v.DataTime.Month(), v.DataTime.Day(), 0, 0, 0, 0, v.DataTime.Location())
 		dateArr = append(dateArr, v.DataTime)
 		dataMap[v.DataTime] = v
 	}
@@ -1260,6 +1289,7 @@ func (obj BaseCalculate) Ljz() (dateDataMap map[time.Time]float64, err error, er
 		yearMap := make(map[int]float64)
 		yearList := make([]int, 0)
 		for _, item := range dataList {
+			item.DataTime = time.Date(item.DataTime.Year(), item.DataTime.Month(), item.DataTime.Day(), 0, 0, 0, 0, item.DataTime.Location())
 			year := item.DataTime.Year()
 			yearVal, ok := yearMap[year]
 			if ok {
@@ -1278,6 +1308,7 @@ func (obj BaseCalculate) Ljz() (dateDataMap map[time.Time]float64, err error, er
 		yearMonthMap := make(map[string]float64)
 		yearMonthList := make([]string, 0)
 		for _, item := range dataList {
+			item.DataTime = time.Date(item.DataTime.Year(), item.DataTime.Month(), item.DataTime.Day(), 0, 0, 0, 0, item.DataTime.Location())
 			itemDate := item.DataTime
 			year := itemDate.Year()
 			var tmpK string
@@ -1309,6 +1340,7 @@ func (obj BaseCalculate) Ljz() (dateDataMap map[time.Time]float64, err error, er
 		yearMonthMap := make(map[string]float64)
 		yearMonthList := make([]string, 0)
 		for _, item := range dataList {
+			item.DataTime = time.Date(item.DataTime.Year(), item.DataTime.Month(), item.DataTime.Day(), 0, 0, 0, 0, item.DataTime.Location())
 			itemDate := item.DataTime
 			year := itemDate.Year()
 			var tmpK string
@@ -1344,6 +1376,7 @@ func (obj BaseCalculate) Ljz() (dateDataMap map[time.Time]float64, err error, er
 		yearMonthMap := make(map[string]float64)
 		yearMonthList := make([]string, 0)
 		for _, item := range dataList {
+			item.DataTime = time.Date(item.DataTime.Year(), item.DataTime.Month(), item.DataTime.Day(), 0, 0, 0, 0, item.DataTime.Location())
 			itemDate := item.DataTime
 			year := itemDate.Year()
 			var tmpK string
@@ -1371,6 +1404,7 @@ func (obj BaseCalculate) Ljz() (dateDataMap map[time.Time]float64, err error, er
 		tmpDateDataMap := make(map[time.Time]float64)
 		tmpDateList := make([]time.Time, 0)
 		for _, item := range dataList {
+			item.DataTime = time.Date(item.DataTime.Year(), item.DataTime.Month(), item.DataTime.Day(), 0, 0, 0, 0, item.DataTime.Location())
 			itemDate := item.DataTime
 			dayInt := itemDate.Year()*100 + int(itemDate.Month())
 			var currTime time.Time
@@ -1414,6 +1448,7 @@ func (obj BaseCalculate) Ljz() (dateDataMap map[time.Time]float64, err error, er
 		tmpDateDataMap := make(map[time.Time]float64)
 		tmpDateList := make([]time.Time, 0)
 		for _, item := range dataList {
+			item.DataTime = time.Date(item.DataTime.Year(), item.DataTime.Month(), item.DataTime.Day(), 0, 0, 0, 0, item.DataTime.Location())
 			itemDate := item.DataTime
 			var currTime time.Time
 			// 周六周日，这是下一个周五的数据
@@ -1504,6 +1539,7 @@ func (obj BaseCalculate) LjzNczj() (dateDataMap map[time.Time]float64, err error
 		tmpDateDataMap := make(map[time.Time]float64)
 		tmpDateList := make([]time.Time, 0)
 		for _, item := range dataList {
+			item.DataTime = time.Date(item.DataTime.Year(), item.DataTime.Month(), item.DataTime.Day(), 0, 0, 0, 0, item.DataTime.Location())
 			itemDate := item.DataTime
 			var currTime time.Time
 			// 周六周日，这是下一个周五的数据
@@ -1534,6 +1570,7 @@ func (obj BaseCalculate) LjzNczj() (dateDataMap map[time.Time]float64, err error
 		}
 	default:
 		for _, item := range dataList {
+			item.DataTime = time.Date(item.DataTime.Year(), item.DataTime.Month(), item.DataTime.Day(), 0, 0, 0, 0, item.DataTime.Location())
 			itemDate := item.DataTime
 			year := itemDate.Year()
 			yearVal, ok := yearMap[year]
@@ -1624,6 +1661,7 @@ func calculateExponentialSmoothingData(dataList []*EdbInfoData, alpha float64) (
 	alphaDecimal := decimal.NewFromFloat(alpha)
 	subAlpha := decimal.NewFromFloat(1).Sub(alphaDecimal)
 	for k, d := range dataList {
+		d.DataTime = time.Date(d.DataTime.Year(), d.DataTime.Month(), d.DataTime.Day(), 0, 0, 0, 0, d.DataTime.Location())
 		// 首期的值以原始值作为指数修匀的计算值
 		if k == 0 {
 			newDataList = append(newDataList, EdbInfoData{
@@ -1684,6 +1722,7 @@ func (obj BaseCalculate) Rjz() (dateDataMap map[time.Time]float64, err error, er
 	var dateArr []time.Time
 	dataMap := make(map[time.Time]*EdbInfoData)
 	for _, v := range dataList {
+		v.DataTime = time.Date(v.DataTime.Year(), v.DataTime.Month(), v.DataTime.Day(), 0, 0, 0, 0, v.DataTime.Location())
 		dateArr = append(dateArr, v.DataTime)
 		dataMap[v.DataTime] = v
 	}
@@ -1812,3 +1851,35 @@ func GetDateDataAndDateList(dateDataMap map[time.Time]float64) (dateStrDataMap m
 
 	return
 }
+
+// StepCalculateBySearchData
+// @Description: 分步骤计算
+type StepCalculateBySearchData struct {
+	DataList   []*EdbInfoSearchData `description:"基础数据"`
+	Calculates []CalculatesReq
+}
+
+type CalculatesReq struct {
+	Formula       interface{}
+	Calendar      string `description:"公历/农历"`
+	Frequency     string `description:"需要转换的频度"`
+	MoveType      int    `description:"移动方式:1:领先(默认),2:滞后"`
+	MoveFrequency string `description:"移动频度"`
+	FromFrequency string `description:"来源的频度"`
+	Source        int    `description:"1：累计值转月；2：累计值转季；3：同比值；4：同差值；5：N数值移动平均数计算；6：环比值；7：环差值；8：升频；9：降频；10：时间移位；11：超季节性；12：年化；13：累计值；14：累计值年初至今；15：指数修匀；16：日均值"`
+	Sort          int    `description:"计算顺序"`
+}
+
+func TransDateData2EdbData(dateData map[time.Time]float64) (edbData []*EdbInfoData) {
+	edbData = make([]*EdbInfoData, 0)
+	for d, v := range dateData {
+		edbData = append(edbData, &EdbInfoData{
+			DataTime: d,
+			Value:    v,
+		})
+	}
+	sort.Slice(edbData, func(i, j int) bool {
+		return edbData[i].DataTime.Before(edbData[j].DataTime)
+	})
+	return
+}

models/base_from_adjust.go

@@ -48,8 +48,12 @@ func SaveAdjustEdb(req SaveAdjustEdbReq) (edbInfo *EdbInfo, err error, errMsg st
 	var edbCode string
 	if req.EdbInfoId <= 0 {
 		//指标code生成
-		randStr := utils.GetRandDigit(4)
-		edbCode = `C` + time.Now().Format("060102") + randStr
+		//生成指标编码
+		edbCode, err = utils.GenerateEdbCode(1, "")
+		if err != nil {
+			err = errors.New("指标编码生成失败，Err：" + err.Error())
+			return
+		}
 
 		timestamp := strconv.FormatInt(time.Now().UnixNano(), 10)
 		uniqueCode := utils.MD5(utils.DATA_PREFIX + "_" + timestamp)
@@ -267,11 +271,11 @@ func RefreshAllAdjustEdb(edbInfo *EdbInfo, fromEdbInfo *EdbInfo) (err error) {
 	}
 
 	// 查询关联指标的数据
-	var condition string
-	var pars []interface{}
-	condition += " AND edb_info_id=? AND data_time > ? "
-	pars = append(pars, fromEdbInfo.EdbInfoId, edbAdjustConf.SourceEndDate)
-	dataList, err := GetEdbDataListAllByTo(to, condition, pars, fromEdbInfo.Source, fromEdbInfo.SubSource, 1)
+	dataList, err := GetEdbDataListAllByTo(to, fromEdbInfo.Source, fromEdbInfo.SubSource, FindEdbDataListAllCond{
+		EdbInfoId:         fromEdbInfo.EdbInfoId,
+		StartDataTime:     edbAdjustConf.SourceEndDate.Format(utils.FormatDate),
+		StartDataTimeCond: ">",
+	}, 1)
 	if err != nil {
 		return err
 	}
@@ -279,11 +283,11 @@ func RefreshAllAdjustEdb(edbInfo *EdbInfo, fromEdbInfo *EdbInfo) (err error) {
 	// 已经入库的日期数据
 	existDataMap := make(map[string]float64)
 	{
-		var tmpCondition string
-		var tmpPars []interface{}
-		tmpCondition += " AND edb_info_id=? AND data_time > ? "
-		tmpPars = append(tmpPars, edbInfo.EdbInfoId, edbAdjustConf.SourceEndDate)
-		existDataList, tmpErr := GetEdbDataListAllByTo(to, tmpCondition, tmpPars, edbInfo.Source, edbInfo.SubSource, 1)
+		existDataList, tmpErr := GetEdbDataListAllByTo(to, edbInfo.Source, edbInfo.SubSource, FindEdbDataListAllCond{
+			EdbInfoId:         edbInfo.EdbInfoId,
+			StartDataTime:     edbAdjustConf.SourceEndDate.Format(utils.FormatDate),
+			StartDataTimeCond: ">",
+		}, 1)
 		if tmpErr != nil {
 			err = tmpErr
 			return

models/base_from_bloomberg.go

@@ -43,6 +43,12 @@ func (m *BaseFromBloombergIndex) Create() (err error) {
 	return
 }
 
+func (m *BaseFromBloombergIndex) Update(cols []string) (err error) {
+	o := orm.NewOrm()
+	_, err = o.Update(m, cols...)
+	return
+}
+
 func GetBaseFromBloombergIndexByCode(indexCode string) (item *BaseFromBloombergIndex, err error) {
 	o := orm.NewOrm()
 	sql := `SELECT * FROM base_from_bloomberg_index WHERE index_code = ? LIMIT 1`
@@ -80,7 +86,7 @@ func GetBaseFromBloombergDataByCondition(condition string, pars []interface{}) (
 func MultiInsertOrUpdateBaseFromBloombergData(inserts, updates []*BaseFromBloombergData) (err error) {
 	o := orm.NewOrm()
 	if len(inserts) > 0 {
-		_, e := o.InsertMulti(len(inserts), inserts)
+		_, e := o.InsertMulti(1000, inserts)
 		if e != nil {
 			err = fmt.Errorf("insert multi err: %s", e.Error())
 			return
@@ -345,3 +351,12 @@ type BaseFromBloombergApiIndexData struct {
 	DataTime time.Time `description:"数据日期"`
 	Value    float64   `description:"数据值"`
 }
+
+// PCSGImportHistoryDataReq 导入历史数据
+type PCSGImportHistoryDataReq struct {
+	IndexCode       string                `description:"指标编码"`
+	DataMap         map[time.Time]float64 `description:"数据日期/值"`
+	IsVCode         bool                  `description:"是否指标编码中间加V"`
+	ExtraLetter     string                `description:"指标编码中间额外加的字母...比如V"`
+	IndexNamePrefix string                `description:"指标名称前缀"`
+}

models/base_from_business.go

@@ -0,0 +1,330 @@
+package models
+
+import (
+	"eta/eta_index_lib/models/mgo"
+	"eta/eta_index_lib/utils"
+	"fmt"
+	"go.mongodb.org/mongo-driver/bson"
+	"time"
+
+	"github.com/beego/beego/v2/client/orm"
+)
+
+// BaseFromBusinessIndex
+// @Description: 外部指标（商家系统）表
+type BaseFromBusinessIndex struct {
+	BaseFromBusinessIndexId int64     `orm:"column(base_from_business_index_id);pk"`
+	IndexCode               string    `description:"指标编码"`
+	IndexName               string    `description:"指标名称"`
+	Unit                    string    `description:"单位"`
+	Frequency               string    `description:"频度"`
+	Source                  int       `description:"数据来源"`
+	SourceName              string    `description:"数据来源名称"`
+	StartDate               time.Time `description:"开始日期"`
+	EndDate                 time.Time `description:"结束日期"`
+	Remark                  string    `description:"备注字段"`
+	BaseModifyTime          time.Time `description:"基础信息(名称，单位，频度)变更时间"`
+	DataUpdateTime          time.Time `description:"最近一次数据发生变化的时间"`
+	CreateTime              time.Time `description:"创建时间"`
+	ModifyTime              time.Time `description:"修改时间"`
+}
+
+// BaseFromBusinessIndexResp
+// @Description: 外部指标（商家系统）表
+type BaseFromBusinessIndexResp struct {
+	IndexCode  string `description:"指标编码"`
+	IndexName  string `description:"指标名称"`
+	Unit       string `description:"单位"`
+	Frequency  string `description:"频度"`
+	SourceName string `description:"数据来源名称"`
+}
+
+// EdbBusinessSource
+// @Description: 自有数据（商家）指标来源
+type EdbBusinessSource struct {
+	EdbBusinessSourceId int64     `orm:"column(edb_business_source_id);pk"`
+	SourceName          string    `description:"来源名称"` // 来源名称
+	CreateTime          time.Time `description:"创建时间"` // 创建时间
+}
+
+// AddBusinessIndexReq
+// @Description:  添加外部指标（商家）请求
+type AddBusinessIndexReq struct {
+	IndexCode  string               `description:"指标编码"`
+	IndexName  string               `description:"指标名称"`
+	Unit       string               `description:"单位"`
+	Frequency  string               `description:"频度"`
+	SourceName string               `description:"数据来源名称"`
+	Remark     string               `description:"备注字段"`
+	DataList   []AddBusinessDataReq `description:"指标数据"`
+}
+
+// AddBusinessDataReq
+// @Description: 外部指标（商家系统）数据
+type AddBusinessDataReq struct {
+	Value float64 `description:"值"`
+	Date  string  `description:"日期"`
+}
+
+// DelBusinessIndexReq
+// @Description:  删除外部指标（商家）请求
+type DelBusinessIndexReq struct {
+	IndexCodeList []string `description:"指标编码"`
+}
+
+// DelBusinessIndexDataReq
+// @Description:  删除外部指标（商家）明细数据请求
+type DelBusinessIndexDataReq struct {
+	IndexCode string `description:"指标编码"`
+	StartDate string `description:"开始日期"`
+	EndDate   string `description:"结束日期"`
+}
+
+// GetIndexItem
+// @Description: 根据指标编码获取自有数据指标
+// @author: Roc
+// @receiver m
+// @datetime 2024-05-31 16:29:30
+// @param indexCode string
+// @return item *BaseFromBusinessIndex
+// @return err error
+func (m *BaseFromBusinessIndex) GetIndexItem(indexCode string) (item *BaseFromBusinessIndex, err error) {
+	o := orm.NewOrm()
+	sql := `SELECT * FROM base_from_business_index WHERE index_code = ? `
+	err = o.Raw(sql, indexCode).QueryRow(&item)
+	return
+}
+
+// GetIndexItemList
+// @Description: 根据指标编码列表获取自有数据指标列表
+// @author: Roc
+// @receiver m
+// @datetime 2024-05-31 16:29:12
+// @param indexCodeList []string
+// @return items []*BaseFromBusinessIndex
+// @return err error
+func (m *BaseFromBusinessIndex) GetIndexItemList(indexCodeList []string) (items []*BaseFromBusinessIndex, err error) {
+	num := len(indexCodeList)
+	if num <= 0 {
+		return
+	}
+	o := orm.NewOrm()
+	sql := `SELECT * FROM base_from_business_index WHERE index_code in (` + utils.GetOrmInReplace(num) + `) `
+	_, err = o.Raw(sql, indexCodeList).QueryRows(&items)
+
+	return
+}
+
+// DelIndexItemList
+// @Description: 根据指标编码列表删除自有数据指标
+// @author: Roc
+// @receiver m
+// @datetime 2024-05-31 16:36:52
+// @param indexCodeList []string
+// @return err error
+func (m *BaseFromBusinessIndex) DelIndexItemList(indexCodeList []string) (err error) {
+	num := len(indexCodeList)
+	if num <= 0 {
+		return
+	}
+	o := orm.NewOrm()
+	sql := `DELETE FROM base_from_business_index WHERE index_code in (` + utils.GetOrmInReplace(num) + `) `
+	_, err = o.Raw(sql, indexCodeList).Exec()
+
+	return
+}
+
+// GetMaxId
+// @Description: 获取自有数据库中的最大id
+// @author: Roc
+// @receiver m
+// @datetime 2024-05-31 13:11:34
+// @return maxId int
+// @return err error
+func (m *BaseFromBusinessIndex) GetMaxId() (maxId int, err error) {
+	o := orm.NewOrm()
+	sql := `SELECT max(base_from_business_index_id) id FROM base_from_business_index limit 1`
+	err = o.Raw(sql).QueryRow(&maxId)
+	return
+}
+
+// Add 新增
+func (m *BaseFromBusinessIndex) Add() (err error) {
+	o := orm.NewOrm()
+	lastId, err := o.Insert(m)
+	if err != nil {
+		return
+	}
+	m.BaseFromBusinessIndexId = lastId
+
+	return
+}
+
+func (m *BaseFromBusinessIndex) Update(cols []string) (err error) {
+	o := orm.NewOrm()
+	_, err = o.Update(m, cols...)
+	return
+}
+
+func (m *BaseFromBusinessIndex) Del() (err error) {
+	o := orm.NewOrm()
+	_, err = o.Delete(m)
+	return
+}
+
+func (m *BaseFromBusinessIndex) UpdateIndex(item *BaseFromBusinessIndex, updateCols []string) (err error) {
+	if item == nil {
+		return
+	}
+	if len(updateCols) == 0 {
+		return
+	}
+	o := orm.NewOrm()
+	_, err = o.Update(item, updateCols...)
+	return
+}
+
+// GetEdbBusinessSourceItem
+// @Description: 根据来源名称获取来源信息
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-25 18:09:03
+// @param sourceName string
+// @return item *EdbBusinessSource
+// @return err error
+func (m *EdbBusinessSource) GetEdbBusinessSourceItem(sourceName string) (item *EdbBusinessSource, err error) {
+	o := orm.NewOrm()
+	sql := `SELECT * FROM edb_business_source WHERE source_name = ? `
+	err = o.Raw(sql, sourceName).QueryRow(&item)
+	return
+}
+
+// Add 新增
+func (m *EdbBusinessSource) Add() (err error) {
+	o := orm.NewOrm()
+	lastId, err := o.Insert(m)
+	if err != nil {
+		return
+	}
+	m.EdbBusinessSourceId = lastId
+
+	return
+}
+
+// GetEdbInfoMaxAndMinInfo
+// @Description: 获取指标的最新数据记录信息
+// @author: Roc
+// @receiver m
+// @datetime 2024-07-02 14:50:50
+// @param edbCode string
+// @return item *EdbInfoMaxAndMinInfo
+// @return err error
+func (m BaseFromBusinessIndex) GetEdbInfoMaxAndMinInfo(edbCode string) (item *EdbInfoMaxAndMinInfo, err error) {
+	if utils.UseMongo {
+		return m.getEdbInfoMaxAndMinInfoByMongo(edbCode)
+	} else {
+		return m.getEdbInfoMaxAndMinInfoByMysql(edbCode)
+	}
+
+	return
+}
+
+// getEdbInfoMaxAndMinInfoByMongo
+// @Description: 获取指标的最新数据记录信息（从mongo中获取）
+// @author: Roc
+// @receiver m
+// @datetime 2024-07-02 14:41:20
+// @param edbCode string
+// @return item *EdbInfoMaxAndMinInfo
+// @return err error
+func (m BaseFromBusinessIndex) getEdbInfoMaxAndMinInfoByMongo(edbCode string) (item *EdbInfoMaxAndMinInfo, err error) {
+	mogDataObj := new(mgo.BaseFromBusinessData)
+	pipeline := []bson.M{
+		{"$match": bson.M{"index_code": edbCode}},
+		{"$group": bson.M{
+			"_id":       nil,
+			"min_date":  bson.M{"$min": "$data_time"},
+			"max_date":  bson.M{"$max": "$data_time"},
+			"min_value": bson.M{"$min": "$value"},
+			"max_value": bson.M{"$max": "$value"},
+		}},
+		{"$project": bson.M{"_id": 0}}, // 可选，如果不需要_id字段
+	}
+	result, err := mogDataObj.GetEdbInfoMaxAndMinInfo(pipeline)
+	if err != nil {
+		fmt.Println("BaseFromBusinessIndex GetEdbInfoMaxAndMinInfo Err:" + err.Error())
+		return
+	}
+
+	if !result.MaxDate.IsZero() {
+		whereQuery := bson.M{"index_code": edbCode, "data_time": result.MaxDate}
+		selectParam := bson.D{{"value", 1}, {"_id", 0}}
+		latestValue, tmpErr := mogDataObj.GetLatestValue(whereQuery, selectParam)
+		if tmpErr != nil {
+			err = tmpErr
+			return
+		}
+		result.LatestValue = latestValue.Value
+		result.EndValue = latestValue.Value
+	}
+
+	item = &EdbInfoMaxAndMinInfo{
+		MinDate:     result.MinDate.Format(utils.FormatDate),
+		MaxDate:     result.MaxDate.Format(utils.FormatDate),
+		MinValue:    result.MinValue,
+		MaxValue:    result.MaxValue,
+		LatestValue: result.LatestValue,
+		LatestDate:  result.LatestDate.Format(utils.FormatDate),
+		EndValue:    result.EndValue,
+	}
+
+	return
+}
+
+// getEdbInfoMaxAndMinInfoByMysql
+// @Description: 获取指标的最新数据记录信息（从mysql中获取）
+// @author: Roc
+// @receiver m
+// @datetime 2024-07-02 14:49:58
+// @param edbCode string
+// @return item *EdbInfoMaxAndMinInfo
+// @return err error
+func (m BaseFromBusinessIndex) getEdbInfoMaxAndMinInfoByMysql(edbCode string) (item *EdbInfoMaxAndMinInfo, err error) {
+	dataObj := BaseFromBusinessData{}
+	result, err := dataObj.GetEdbInfoMaxAndMinInfo(edbCode)
+	if err != nil {
+		return
+	}
+
+	item = &EdbInfoMaxAndMinInfo{
+		MinDate:     result.MinDate,
+		MaxDate:     result.MaxDate,
+		MinValue:    result.MinValue,
+		MaxValue:    result.MaxValue,
+		LatestValue: result.LatestValue,
+		LatestDate:  result.LatestDate,
+		EndValue:    result.EndValue,
+	}
+
+	return
+}
+
+// ModifyIndexMaxAndMinInfo
+// @Description: 修改最大值和最小值信息
+// @author: Roc
+// @receiver m
+// @datetime 2024-05-06 14:07:46
+// @param indexCode string
+// @param item *EdbInfoMaxAndMinInfo
+// @param isIndexUpdateOrAdd bool
+// @return err error
+func (m *BaseFromBusinessIndex) ModifyIndexMaxAndMinInfo(indexCode string, item *EdbInfoMaxAndMinInfo, isIndexUpdateOrAdd bool) (err error) {
+	o := orm.NewOrm()
+	sql := ` UPDATE base_from_business_index SET start_date=?,end_date=?,modify_time=NOW() `
+	if isIndexUpdateOrAdd {
+		sql += `,data_update_time=NOW() `
+	}
+	sql += ` WHERE index_code=?`
+	_, err = o.Raw(sql, item.MinDate, item.MaxDate, indexCode).Exec()
+	return
+}

models/base_from_business_data.go

@@ -0,0 +1,306 @@
+package models
+
+import (
+	"errors"
+	"eta/eta_index_lib/utils"
+	"fmt"
+	"github.com/beego/beego/v2/client/orm"
+	"strings"
+	"time"
+)
+
+// BaseFromBusinessData
+// @Description: 外部指标（商家系统）原始数据表
+type BaseFromBusinessData struct {
+	BusinessDataId          int       `orm:"column(business_data_id);pk" json:"business_data_id"`
+	BaseFromBusinessIndexId int       `json:"base_from_business_index_id"` // 指标id
+	IndexCode               string    `json:"index_code"`                  // 指标编码
+	DataTime                time.Time `json:"data_time"`                   // 数据日期
+	Value                   float64   `json:"value"`                       // 数据值
+	CreateTime              time.Time `json:"create_time"`                 // 创建时间
+	ModifyTime              time.Time `json:"modify_time"`                 // 修改时间
+}
+
+// TableName
+// @Description:  获取表名
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-26 13:41:36
+// @return string
+func (m *BaseFromBusinessData) TableName() string {
+	return "base_from_business_data"
+}
+
+// CollectionName
+// @Description:  获取集合名称
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-26 13:41:36
+// @return string
+func (m *BaseFromBusinessData) CollectionName() string {
+	return "base_from_business_data"
+}
+
+// DataBaseName
+// @Description: 获取数据库名称
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-26 13:41:33
+// @return string
+func (m *BaseFromBusinessData) DataBaseName() string {
+	return utils.MgoDataDbName
+}
+
+type WhereParams struct {
+	Condition string
+	Pars      []interface{}
+	Order     string `description:"排序字段"`
+}
+
+// GetAllDataList
+// @Description: 根据条件获取所有数据
+// @author: Roc
+// @receiver m
+// @datetime 2024-07-01 17:18:57
+// @param condition []string
+// @param pars []interface{}
+// @param order string
+// @return result []*BaseFromBusinessData
+// @return err error
+func (m *BaseFromBusinessData) GetAllDataList(condition []string, pars []interface{}, order string) (result []*BaseFromBusinessData, err error) {
+	o := orm.NewOrm()
+
+	sql := `SELECT * FROM base_from_business_data `
+	if len(condition) > 0 {
+		sql += ` WHERE ` + strings.Join(condition, " AND ")
+	}
+
+	if order != `` {
+		sql += ` ORDER BY ` + order
+	}
+
+	_, err = o.Raw(sql, pars).QueryRows(&result)
+
+	return
+}
+
+// GetLimitDataList
+// @Description: 根据条件获取指定数量数据列表
+// @author: Roc
+// @receiver m
+// @datetime 2024-07-01 17:19:16
+// @param condition []string
+// @param pars []interface{}
+// @param order string
+// @param size int64
+// @return result []*BaseFromBusinessData
+// @return err error
+func (m *BaseFromBusinessData) GetLimitDataList(condition []string, pars []interface{}, order string, size int64) (result []*BaseFromBusinessData, err error) {
+	o := orm.NewOrm()
+	sql := `SELECT * FROM base_from_business_data `
+	if len(condition) > 0 {
+		sql += ` WHERE ` + strings.Join(condition, " AND ")
+	}
+
+	if order != `` {
+		sql += ` ORDER BY ` + order
+	}
+
+	sql += fmt.Sprintf(` LIMIT %d`, size)
+
+	_, err = o.Raw(sql, pars).QueryRows(&result)
+
+	return
+}
+
+// GetPageDataList
+// @Description: 根据条件获取分页数据列表
+// @author: Roc
+// @receiver m
+// @datetime 2024-07-01 17:19:42
+// @param condition []string
+// @param pars []interface{}
+// @param order string
+// @param startSize int64
+// @param size int64
+// @return result []*BaseFromBusinessData
+// @return err error
+func (m *BaseFromBusinessData) GetPageDataList(condition []string, pars []interface{}, order string, startSize, size int64) (result []*BaseFromBusinessData, err error) {
+	o := orm.NewOrm()
+	sql := `SELECT * FROM base_from_business_data `
+	if len(condition) > 0 {
+		sql += ` WHERE ` + strings.Join(condition, " AND ")
+	}
+
+	if order != `` {
+		sql += ` ORDER BY ` + order
+	}
+
+	sql += fmt.Sprintf(` LIMIT %d,%d`, startSize, size)
+
+	_, err = o.Raw(sql, pars).QueryRows(&result)
+
+	return
+}
+
+// GetCountDataList
+// @Description: 根据条件获取数据列表总数
+// @author: Roc
+// @receiver m
+// @datetime 2024-07-01 17:19:50
+// @param condition []string
+// @param pars []interface{}
+// @return count int64
+// @return err error
+func (m *BaseFromBusinessData) GetCountDataList(condition []string, pars []interface{}) (count int64, err error) {
+	o := orm.NewOrm()
+	sql := `SELECT COUNT(1) FROM base_from_business_data `
+	if len(condition) > 0 {
+		sql += ` WHERE ` + strings.Join(condition, " AND ")
+	}
+	err = o.Raw(sql, pars).QueryRow(&count)
+
+	return
+}
+
+// InsertDataByColl
+// @Description: 写入单条数据（外部传入集合）
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-26 14:22:18
+// @param addData interface{}
+// @return err error
+func (m *BaseFromBusinessData) InsertDataByColl(addData interface{}) (err error) {
+	o := orm.NewOrm()
+	_, err = o.Insert(addData)
+
+	return
+}
+
+// BatchInsertData
+// @Description: 批量写入数据
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-26 14:22:18
+// @param bulk int 每次请求保存的数据量
+// @param dataList []*BaseFromBusinessData
+// @return err error
+func (m *BaseFromBusinessData) BatchInsertData(bulk int, dataList []*BaseFromBusinessData) (err error) {
+	o := orm.NewOrm()
+	_, err = o.InsertMulti(bulk, dataList)
+
+	return
+}
+
+// UpdateData
+// @Description: 单条数据修改
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-26 15:01:51
+// @param whereParams interface{}
+// @param updateParams interface{}
+// @return err error
+func (m *BaseFromBusinessData) UpdateData(updateCols []string) (err error) {
+	o := orm.NewOrm()
+	_, err = o.Update(m, updateCols...)
+	if err != nil {
+		fmt.Println("UpdateDataByColl:Err:" + err.Error())
+		return
+	}
+
+	return
+}
+
+// HandleData
+// @Description: 数据处理
+// @author: Roc
+// @receiver m
+// @datetime 2024-07-01 17:35:54
+// @param addDataList []*BaseFromBusinessData
+// @param updateDataList []*BaseFromBusinessData
+// @return err error
+func (m *BaseFromBusinessData) HandleData(addDataList, updateDataList []*BaseFromBusinessData) (err error) {
+	o := orm.NewOrm()
+	to, err := o.Begin()
+	if err != nil {
+		return
+	}
+	defer func() {
+		if err != nil {
+			fmt.Println("BaseFromBusinessData HandleData,Err:" + err.Error())
+			_ = to.Rollback()
+		} else {
+			_ = to.Commit()
+		}
+	}()
+
+	// 插入数据
+	if len(addDataList) > 0 {
+		_, err = to.InsertMulti(500, addDataList)
+		if err != nil {
+			return
+		}
+	}
+
+	// 修改
+
+	if len(updateDataList) > 0 {
+		for _, v := range updateDataList {
+			_, err = to.Update(v, "Value", "ModifyTime")
+			if err != nil {
+				fmt.Println("BaseFromBusinessData HandleData Update:Err:" + err.Error())
+				return
+			}
+		}
+	}
+
+	return
+}
+
+// GetEdbInfoMaxAndMinInfo
+// @Description: 获取当前指标的最大最小值
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-30 17:15:39
+// @param whereParams interface{}
+// @return result EdbInfoMaxAndMinInfo
+// @return err error
+func (m *BaseFromBusinessData) GetEdbInfoMaxAndMinInfo(indexCode string) (result EdbInfoMaxAndMinInfo, err error) {
+	o := orm.NewOrm()
+	sql := ``
+	sql = ` SELECT MIN(data_time) AS min_date,MAX(data_time) AS max_date,MIN(value) AS min_value,MAX(value) AS max_value FROM base_from_business_data WHERE index_code = ? `
+	err = o.Raw(sql, indexCode).QueryRow(&result)
+	if err != nil {
+		return
+	}
+
+	var latestValue float64
+	sql = ` SELECT value AS latest_value FROM base_from_business_data WHERE index_code = ? ORDER BY data_time DESC LIMIT 1 `
+	err = o.Raw(sql, indexCode).QueryRow(&latestValue)
+	result.LatestValue = latestValue
+
+	return
+}
+
+// DelDataByCond
+// @Description: 根据条件删除多条数据
+// @author: Roc
+// @receiver m
+// @datetime 2024-07-01 17:46:56
+// @param condition []string
+// @param pars []interface{}
+// @return err error
+func (m *BaseFromBusinessData) DelDataByCond(condition []string, pars []interface{}) (err error) {
+	if len(condition) <= 0 {
+		err = errors.New("条件不能为空")
+		return
+	}
+	o := orm.NewOrm()
+	sql := `DELETE FROM base_from_business_data `
+
+	sql += ` WHERE ` + strings.Join(condition, " AND ")
+
+	_, err = o.Raw(sql, pars).Exec()
+
+	return
+}

models/base_from_calculate.go

@@ -318,20 +318,11 @@ func refreshAllCalculate(to orm.TxOrmer, edbInfoIdArr []*EdbInfo, edbInfoTag map
 	}
 
 	for edbInfoIndex, v := range edbInfoIdArr {
-		var condition string
-		var pars []interface{}
-		condition += " AND edb_info_id=? "
-		pars = append(pars, v.EdbInfoId)
-		if startDate != "" {
-			condition += " AND data_time>=? "
-			pars = append(pars, startDate)
-		}
-		//if endDate != "" {
-		//	condition += " AND data_time<=? "
-		//	pars = append(pars, endDate)
-		//}
-		//fmt.Println("v.Source:", v.Source)
-		dataList, err := GetEdbDataListAllByTo(to, condition, pars, v.Source, v.SubSource, 1)
+		dataList, err := GetEdbDataListAllByTo(to, v.Source, v.SubSource, FindEdbDataListAllCond{
+			EdbInfoId:         v.EdbInfoId,
+			StartDataTime:     startDate,
+			StartDataTimeCond: ">=",
+		}, 1)
 		if err != nil {
 			return err
 		}

models/base_from_ccf.go

@@ -0,0 +1,426 @@
+package models
+
+import (
+	"eta/eta_index_lib/utils"
+	"fmt"
+	"github.com/beego/beego/v2/client/orm"
+	"strconv"
+	"strings"
+	"time"
+)
+
+// BaseFromCCF CCF化纤信息
+type BaseFromCCF struct{}
+
+type BaseFromCCFData struct {
+	BaseFromCcfDataId  int `orm:"column(base_from_ccf_data_id);pk"`
+	BaseFromCcfIndexId int
+	IndexCode          string
+	DataTime           string
+	Value              string
+	CreateTime         time.Time
+	ModifyTime         time.Time
+	DataTimestamp      int64
+}
+
+func (m *BaseFromCCFData) TableName() string {
+	return "base_from_ccf_data"
+}
+
+func GetBaseFromCCFDataByCondition(condition string, pars []interface{}) (list []*BaseFromCCFData, err error) {
+	o := orm.NewOrm()
+	sql := `SELECT * FROM base_from_ccf_data WHERE 1=1 `
+	if condition != "" {
+		sql += condition
+	}
+	_, err = o.Raw(sql, pars).QueryRows(&list)
+	return
+}
+
+// Add 添加
+func (obj BaseFromCCF) Add(edbCode string) (err error) {
+	o := orm.NewOrm()
+
+	var condition string
+	var pars []interface{}
+	if edbCode != "" {
+		condition += " AND index_code = ? "
+		pars = append(pars, edbCode)
+	}
+	ccfBaseDataAll, err := GetBaseFromCCFDataByCondition(condition, pars)
+	if err != nil && err.Error() != utils.ErrNoRow() {
+		return
+	}
+	var isAdd bool
+	addSql := ` INSERT INTO edb_data_ccf(edb_info_id,edb_code,data_time,value,create_time,modify_time,data_timestamp) values `
+	existMap := make(map[string]string)
+	for _, sv := range ccfBaseDataAll {
+		eDate := sv.DataTime
+		dataTime, err := time.Parse(utils.FormatDate, eDate)
+		if err != nil {
+			return err
+		}
+		timestamp := dataTime.UnixNano() / 1e6
+		timeStr := fmt.Sprintf("%d", timestamp)
+		if _, ok := existMap[eDate]; !ok {
+			addSql += GetAddSql("0", edbCode, eDate, timeStr, sv.Value)
+			isAdd = true
+		}
+		existMap[eDate] = sv.Value
+	}
+	if isAdd {
+		addSql = strings.TrimRight(addSql, ",")
+		utils.FileLog.Info("addSql:" + addSql)
+		_, err = o.Raw(addSql).Exec()
+		if err != nil {
+			return err
+		}
+	}
+	return
+}
+
+// Refresh 刷新CCF化纤指标数据
+func (obj BaseFromCCF) Refresh(edbInfoId int, edbCode, startDate string) (err error) {
+	source := obj.GetSource()
+	o := orm.NewOrm()
+	if err != nil {
+		return
+	}
+	edbInfoIdStr := strconv.Itoa(edbInfoId)
+	//计算数据
+	var condition string
+	var pars []interface{}
+
+	if edbCode != "" {
+		condition += " AND index_code=? "
+		pars = append(pars, edbCode)
+	}
+
+	if startDate != "" {
+		condition += " AND data_time>=? "
+		pars = append(pars, startDate)
+	}
+
+	ccfDataList, err := GetBaseFromCCFDataByCondition(condition, pars)
+	if err != nil {
+		return
+	}
+
+	// 真实数据的最大日期  ， 插入规则配置的日期
+	var realDataMaxDate, edbDataInsertConfigDate time.Time
+	var edbDataInsertConfig *EdbDataInsertConfig
+	var isFindConfigDateRealData bool //是否找到配置日期的实际数据的值
+	{
+		edbDataInsertConfig, err = GetEdbDataInsertConfigByEdbId(edbInfoId)
+		if err != nil && err.Error() != utils.ErrNoRow() {
+			return
+		}
+		if edbDataInsertConfig != nil {
+			edbDataInsertConfigDate = edbDataInsertConfig.Date
+		}
+	}
+
+	var existCondition string
+	var existPars []interface{}
+
+	existCondition += " AND edb_info_id=? "
+	existPars = append(existPars, edbInfoId)
+	if startDate != "" {
+		existCondition += " AND data_time>=? "
+		existPars = append(existPars, startDate)
+	}
+
+	existList, err := GetEdbDataByCondition(source, 0, existCondition, existPars)
+	if err != nil {
+		return err
+	}
+	existMap := make(map[string]*EdbInfoSearchData)
+	for _, v := range existList {
+		existMap[v.DataTime] = v
+	}
+	addSql := ` INSERT INTO edb_data_ccf(edb_info_id,edb_code,data_time,value,create_time,modify_time,data_timestamp) values `
+	var isAdd bool
+	for _, v := range ccfDataList {
+		item := v
+		eDate := item.DataTime
+		dataTime, err := time.ParseInLocation(utils.FormatDate, eDate, time.Local)
+		if err != nil {
+			return err
+		}
+		if findItem, ok := existMap[v.DataTime]; !ok {
+			sValue := item.Value
+
+			timestamp := dataTime.UnixNano() / 1e6
+			timeStr := fmt.Sprintf("%d", timestamp)
+
+			addSql += GetAddSql(edbInfoIdStr, edbCode, eDate, timeStr, sValue)
+			isAdd = true
+		} else {
+			if findItem != nil && utils.SubFloatToString(findItem.Value, 30) != item.Value {
+				err = ModifyEdbDataById(source, 0, findItem.EdbDataId, item.Value)
+				if err != nil {
+					return err
+				}
+			}
+		}
+
+		// 下面代码主要目的是处理掉手动插入的数据判断
+		{
+			if realDataMaxDate.IsZero() || dataTime.After(realDataMaxDate) {
+				realDataMaxDate = dataTime
+			}
+			if edbDataInsertConfigDate.IsZero() || dataTime.Equal(edbDataInsertConfigDate) {
+				isFindConfigDateRealData = true
+			}
+		}
+	}
+
+	// 处理手工数据补充的配置
+	HandleConfigInsertEdbData(realDataMaxDate, edbDataInsertConfig, edbInfoId, source, 0, existMap, isFindConfigDateRealData)
+
+	if isAdd {
+		addSql = strings.TrimRight(addSql, ",")
+		_, err = o.Raw(addSql).Exec()
+		if err != nil {
+			fmt.Println("RefreshEdbDataFromBaiinfo add Err", err.Error())
+			return
+		}
+	}
+	return
+}
+
+// GetSource 获取来源编码id
+func (obj BaseFromCCF) GetSource() int {
+	return utils.DATA_SOURCE_CCF
+}
+
+// GetSourceName 获取来源名称
+func (obj BaseFromCCF) GetSourceName() string {
+	return utils.DATA_SOURCE_NAME_CCF
+}
+
+type BaseFromCCFIndex struct {
+	BaseFromCcfIndexId int64 `orm:"column(base_from_ccf_index_id);pk"`
+	IndexCode          string
+	IndexName          string
+	Frequency          string
+	Unit               string
+	ClassifyId         int
+	StartDate          string
+	EndDate            string
+	Sort               int
+	TerminalCode       string
+	CreateTime         time.Time
+	ModifyTime         time.Time
+}
+
+func (m *BaseFromCCFIndex) TableName() string {
+	return "base_from_ccf_index"
+}
+
+func (m *BaseFromCCFIndex) Add() (lastId int64, err error) {
+	o := orm.NewOrm()
+	lastId, err = o.Insert(m)
+	return
+}
+
+func (m *BaseFromCCFIndex) Update(updateCols []string) (err error) {
+	o := orm.NewOrm()
+	_, err = o.Update(m, updateCols...)
+
+	return
+}
+
+func (m *BaseFromCCFIndex) ModifyIndexMaxAndMinDate(indexCode string, item *EdbInfoMaxAndMinInfo) (err error) {
+	o := orm.NewOrm()
+	sql := ` UPDATE base_from_ccf_index SET start_date=?,end_date=?,modify_time=NOW() WHERE index_code=? `
+	_, err = o.Raw(sql, item.MinDate, item.MaxDate, indexCode).Exec()
+	return
+}
+
+func (m *BaseFromCCFIndex) GetByIndexCode(indexCode string) (item *BaseFromCCFIndex, err error) {
+	o := orm.NewOrm()
+	sql := ` SELECT * FROM base_from_ccf_index WHERE index_code=? `
+	err = o.Raw(sql, indexCode).QueryRow(&item)
+	return
+}
+
+func (m *BaseFromCCFIndex) GetItemsByCondition(condition string, pars []interface{}, fieldArr []string, orderRule string) (items []*BaseFromCCFIndex, err error) {
+	o := orm.NewOrm()
+	fields := strings.Join(fieldArr, ",")
+	if len(fieldArr) == 0 {
+		fields = `*`
+	}
+	order := `ORDER BY create_time DESC`
+	if orderRule != "" {
+		order = ` ORDER BY ` + orderRule
+	}
+	sql := fmt.Sprintf(`SELECT %s FROM base_from_ccf_index WHERE 1=1 %s %s`, fields, condition, order)
+	_, err = o.Raw(sql, pars).QueryRows(&items)
+	return
+}
+
+type BaseFromCCFIndexList struct {
+	CcfIndexId   int64 `orm:"column(ccf_index_id);pk"`
+	IndexCode    string
+	IndexName    string
+	Frequency    string
+	Unit         string
+	ClassifyId   int
+	StartDate    string
+	EndDate      string
+	TerminalCode string
+	CreateTime   string
+	ModifyTime   string
+}
+
+func (m *BaseFromCCFData) GetByIndexCode(indexCode string) (list []*BaseFromCCFData, err error) {
+	o := orm.NewOrm()
+	sql := ` SELECT * FROM base_from_ccf_data WHERE index_code=? `
+	_, err = o.Raw(sql, indexCode).QueryRows(&list)
+	return
+}
+
+func (m *BaseFromCCFData) AddMulti(item []*BaseFromCCFData) (err error) {
+	o := orm.NewOrm()
+	_, err = o.InsertMulti(100, item)
+	return
+}
+
+// HandleCCFEdbData CCF化纤的指标数据
+type HandleCCFEdbData struct {
+	IndexName    string            `description:"指标名称"`
+	IndexCode    string            `description:"指标编码"`
+	ClassifyId   int               `description:"分类ID"`
+	Unit         string            `description:"单位"`
+	Sort         int               `description:"排序"`
+	Frequency    string            `description:"频度"`
+	TerminalCode string            `description:"终端编码"`
+	DateData     map[string]string `description:"日期数据"`
+}
+
+type HandleCCFEdbDataReq struct {
+	List         []*HandleCCFEdbData
+	TerminalCode string `description:"编码"`
+}
+
+type HandleCCFTableData struct {
+	ClassifyId   int       `description:"分类ID"`
+	FromPage     string    `description:"表格来源"`
+	TableDate    time.Time `description:"表格日期"`
+	TableContent string    `description:"表格HTML"`
+}
+
+type HandleCCFStockTableReq struct {
+	Table        *HandleCCFTableData
+	TerminalCode string `description:"编码"`
+}
+
+func (m *BaseFromCCFData) GetMaxAndMinDateByIndexCode(indexCode string) (item *EdbInfoMaxAndMinInfo, err error) {
+	o := orm.NewOrm()
+	sql := ` SELECT MIN(data_time) AS min_date,MAX(data_time) AS max_date,MIN(value) AS min_value,MAX(value) AS max_value FROM base_from_ccf_data WHERE index_code=? `
+	err = o.Raw(sql, indexCode).QueryRow(&item)
+	return
+}
+
+// BaseFromCCFClassify CCF化纤数据分类表
+type BaseFromCCFClassify struct {
+	ClassifyId      int       `orm:"column(classify_id);pk"`
+	ClassifyName    string    `description:"分类名称"`
+	ParentId        int       `description:"父级id"`
+	SysUserId       int       `description:"创建人id"`
+	SysUserRealName string    `description:"创建人姓名"`
+	Level           int       `description:"层级"`
+	Sort            int       `description:"排序字段，越小越靠前，默认值：10"`
+	ModifyTime      time.Time `description:"修改时间"`
+	CreateTime      time.Time `description:"创建时间"`
+}
+
+func (m *BaseFromCCFClassify) Add() (lastId int64, err error) {
+	o := orm.NewOrm()
+	lastId, err = o.Insert(m)
+	return
+}
+
+func (m *BaseFromCCFClassify) Update(updateCols []string) (err error) {
+	o := orm.NewOrm()
+	_, err = o.Update(m, updateCols...)
+
+	return
+}
+
+func (m *BaseFromCCFClassify) GetByClassifyName(classifyName string) (item *BaseFromCCFClassify, err error) {
+	o := orm.NewOrm()
+	sql := ` SELECT * FROM base_from_ccf_classify WHERE classify_name=? `
+	err = o.Raw(sql, classifyName).QueryRow(&item)
+	return
+}
+
+func (m *BaseFromCCFClassify) GetItemsByCondition(condition string, pars []interface{}, fieldArr []string, orderRule string) (items []*BaseFromCCFClassify, err error) {
+	o := orm.NewOrm()
+	fields := strings.Join(fieldArr, ",")
+	if len(fieldArr) == 0 {
+		fields = `*`
+	}
+	order := `ORDER BY create_time DESC`
+	if orderRule != "" {
+		order = ` ORDER BY ` + orderRule
+	}
+	sql := fmt.Sprintf(`SELECT %s FROM base_from_ccf_classify WHERE 1=1 %s %s`, fields, condition, order)
+	_, err = o.Raw(sql, pars).QueryRows(&items)
+	return
+}
+
+// BaseFromCCFClassifyItem CCF化纤数据分类信息
+type BaseFromCCFClassifyItem struct {
+	ClassifyId   int                        `description:"分类ID"`
+	ClassifyName string                     `description:"分类名称"`
+	ParentId     int                        `description:"父级id"`
+	Level        int                        `description:"层级"`
+	Sort         int                        `description:"排序字段"`
+	CreateTime   string                     `description:"创建时间"`
+	ModifyTime   string                     `description:"修改时间"`
+	Child        []*BaseFromCCFClassifyItem `description:"子分类"`
+}
+
+func (m *BaseFromCCFClassify) Format2Item(origin *BaseFromCCFClassify) (item *BaseFromCCFClassifyItem) {
+	if origin == nil {
+		return
+	}
+	item = new(BaseFromCCFClassifyItem)
+	item.ClassifyId = origin.ClassifyId
+	item.ClassifyName = origin.ClassifyName
+	item.ParentId = origin.ParentId
+	item.Level = origin.Level
+	item.Sort = origin.Sort
+	item.CreateTime = utils.TimeTransferString(utils.FormatDateTime, origin.CreateTime)
+	item.ModifyTime = utils.TimeTransferString(utils.FormatDateTime, origin.ModifyTime)
+	return
+}
+
+// MultiUpdateBaseFromCCFDataValue 批量更新CCF化纤指标数据
+func MultiUpdateBaseFromCCFDataValue(items []*BaseFromCCFData) (err error) {
+	if len(items) == 0 {
+		return
+	}
+
+	o := orm.NewOrm()
+	sql := `UPDATE base_from_ccf_data SET value = ?, modify_time = NOW() WHERE index_code = ? AND data_time = ? LIMIT 1`
+	p, err := o.Raw(sql).Prepare()
+	if err != nil {
+		return
+	}
+	defer func() {
+		_ = p.Close()
+	}()
+	for _, v := range items {
+		if v.IndexCode == "" || v.DataTime == "" {
+			continue
+		}
+		_, err = p.Exec(v.Value, v.IndexCode, v.DataTime)
+		if err != nil {
+			return
+		}
+	}
+	return
+}

models/base_from_manual.go

@@ -1,10 +1,12 @@
 package models
 
 import (
+	"eta/eta_index_lib/models/mgo"
 	"eta/eta_index_lib/utils"
 	"fmt"
 	"github.com/beego/beego/v2/client/orm"
 	"github.com/shopspring/decimal"
+	"go.mongodb.org/mongo-driver/bson"
 	"strconv"
 	"strings"
 	"time"
@@ -219,7 +221,17 @@ func RefreshEdbDataFromManual(edbInfoId int, edbCode, startDate string) (err err
 	return
 }
 
-// HandleConfigInsertEdbData 处理手工数据补充的配置
+// HandleConfigInsertEdbData
+// @Description: 处理人工数据补充的配置（mysql)
+// @author: Roc
+// @datetime 2024-05-09 13:07:32
+// @param realDataMaxDate time.Time
+// @param edbDataInsertConfig *EdbDataInsertConfig
+// @param edbInfoId int
+// @param source int
+// @param subSource int
+// @param existMap map[string]*EdbInfoSearchData
+// @param isFindConfigDateRealData bool
 func HandleConfigInsertEdbData(realDataMaxDate time.Time, edbDataInsertConfig *EdbDataInsertConfig, edbInfoId, source, subSource int, existMap map[string]*EdbInfoSearchData, isFindConfigDateRealData bool) {
 	if edbDataInsertConfig == nil {
 		return
@@ -249,3 +261,47 @@ func HandleConfigInsertEdbData(realDataMaxDate time.Time, edbDataInsertConfig *E
 
 	return
 }
+
+// HandleConfigInsertEdbDataByMongo
+// @Description: 处理人工数据补充的配置（mongo)
+// @author: Roc
+// @datetime 2024-05-09 13:07:12
+// @param realDataMaxDate time.Time
+// @param edbDataInsertConfig *EdbDataInsertConfig
+// @param edbInfoId int
+// @param source int
+// @param subSource int
+// @param existMap map[string]*mgo.EdbDataBusiness
+// @param isFindConfigDateRealData bool
+func HandleConfigInsertEdbDataByMongo(realDataMaxDate time.Time, edbDataInsertConfig *EdbDataInsertConfig, edbInfoId, source, subSource int, existMap map[string]*mgo.EdbDataBusiness, isFindConfigDateRealData bool) {
+	if edbDataInsertConfig == nil {
+		return
+	}
+	var err error
+	defer func() {
+		if err != nil {
+			fmt.Println("处理手工数据补充的配置失败,err:", err)
+		}
+	}()
+
+	edbDataInsertConfigDate := edbDataInsertConfig.Date // 配置的日期
+
+	// 如果存在真实数据的最大日期  && 存在配置插入数据的最大日期  && 真实数据的最大日期 晚于/等于 配置插入数据的最大日期
+	if realDataMaxDate.After(edbDataInsertConfigDate) || realDataMaxDate.Equal(edbDataInsertConfigDate) {
+		go DeleteEdbDataInsertConfigByEdbId(edbInfoId)
+
+		mogDataObj := mgo.EdbDataBusiness{}
+		coll := mogDataObj.GetCollection()
+		edbDataInsertConfigDateStr := edbDataInsertConfigDate.Format(utils.FormatDate)
+		// 如果没有找到找到配置日期的实际数据，那么就直接删除
+		if item, ok := existMap[edbDataInsertConfigDateStr]; ok && !isFindConfigDateRealData {
+			mogDataObj.RemoveManyByColl(coll, bson.M{"_id": item.ID})
+		}
+	} else {
+		o := orm.NewOrm()
+		edbDataInsertConfig.RealDate = realDataMaxDate
+		_, err = o.Update(edbDataInsertConfig, "RealDate")
+	}
+
+	return
+}

models/base_from_mysteel_chemical.go

@@ -245,9 +245,8 @@ func (m *BaseFromMysteelChemicalIndex) GetIndexItem(indexCode string) (item *Bas
 
 func (m *BaseFromMysteelChemicalIndex) GetIndexCreate(terminalCode string) (items []*BaseFromMysteelChemicalIndex, err error) {
 	o := orm.NewOrm()
-	endTime := time.Now().Add(-2 * time.Minute).Format(utils.FormatDateTime)
-	sql := `SELECT * FROM base_from_mysteel_chemical_index WHERE index_name = '' AND create_time <= ? AND terminal_code = ? `
-	_, err = o.Raw(sql, endTime, terminalCode).QueryRows(&items)
+	sql := `SELECT * FROM base_from_mysteel_chemical_index WHERE index_name = '' AND terminal_code = ? `
+	_, err = o.Raw(sql, terminalCode).QueryRows(&items)
 	return
 }
 

models/base_from_sci99.go

@@ -0,0 +1,281 @@
+package models
+
+import (
+	"eta/eta_index_lib/utils"
+	"fmt"
+	"github.com/beego/beego/v2/client/orm"
+	"strconv"
+	"strings"
+	"time"
+)
+
+// BaseFromSci99Index 代表卓创资讯-原始指标表的结构
+type BaseFromSci99Index struct {
+	BaseFromSciIndexId int       `orm:"column(base_from_sci_index_id);pk"` // 主键，自动递增
+	IndexCode          string    // 指标编码
+	IndexName          string    // 指标名称
+	ClassifyId         int       // 分类Id
+	Unit               string    // 单位
+	Frequency          string    // 频度
+	Describe           string    // 指标描述
+	CreateTime         time.Time // 创建时间
+	ModifyTime         time.Time // 修改时间
+}
+
+// BaseFromSci99Data 代表卓创资讯-原始指标数据表的结构
+type BaseFromSci99DataItem struct {
+	BaseFromSciDataId  int       `orm:"column(base_from_sci_data_id);pk"` // 主键，自动递增
+	BaseFromSciIndexId int       // 指标id
+	IndexCode          string    // 指标编码
+	DataTime           string    // 数据日期
+	Value              string    // 数据值
+	CreateTime         time.Time // 创建时间
+	ModifyTime         time.Time // 修改时间
+}
+
+// BaseFromSci99Data 代表卓创资讯-原始指标数据表的结构
+type BaseFromSci99Data struct {
+	BaseFromSciDataId  int       `orm:"column(base_from_sci_data_id);pk"` // 主键，自动递增
+	BaseFromSciIndexId int       // 指标id
+	IndexCode          string    // 指标编码
+	DataTime           string    // 数据日期
+	Value              float64    // 数据值
+	CreateTime         time.Time // 创建时间
+	ModifyTime         time.Time // 修改时间
+}
+
+// BaseFromSci99Classify 代表卓创资讯-原始指标分类表的结构
+type BaseFromSci99Classify struct {
+	BaseFromSciClassifyId int       `orm:"column(base_from_sci_classify_id);pk"` // 主键，自动递增
+	ClassifyName          string    // 分类名称
+	Sort                  int       // 排序
+	CreateTime            time.Time // 创建时间
+	ModifyTime            time.Time // 修改时间
+}
+
+func GetBaseFromSci99DataDataByCondition(condition string, pars []interface{}) (item []*BaseFromIcpiData, err error) {
+	sql := ` SELECT * FROM base_from_sci99_data WHERE 1=1 `
+	o := orm.NewOrm()
+	if condition != "" {
+		sql += condition
+	}
+	sql += ` ORDER BY data_time DESC `
+	_, err = o.Raw(sql, pars).QueryRows(&item)
+	return
+}
+
+// 添加数据
+func AddBaseFromSci99Index(item *BaseFromSci99Index) (lastId int64, err error) {
+	o := orm.NewOrm()
+	lastId, err = o.Insert(item)
+	return
+}
+
+func AddBaseFromSci99Classify(item *BaseFromSci99Classify) (lastId int64, err error) {
+	o := orm.NewOrm()
+	lastId, err = o.Insert(item)
+	return
+}
+
+func AddBaseFromSci99DataMulti(item []*BaseFromSci99Data) (err error) {
+	o := orm.NewOrm()
+	_, err = o.InsertMulti(1000, item)
+	return
+}
+
+func GetBaseFromSci99Index() (list []*BaseFromSci99Index, err error) {
+	o := orm.NewOrm()
+	sql := `SELECT * FROM base_from_sci99_index `
+	_, err = o.Raw(sql).QueryRows(&list)
+	return
+}
+
+func GetBaseFromSci99DataByIndexCode(indexCode string) (items []*BaseFromSci99DataItem, err error) {
+	o := orm.NewOrm()
+	sql := `SELECT * FROM base_from_sci99_data WHERE index_code=? `
+	_, err = o.Raw(sql, indexCode).QueryRows(&items)
+	return
+}
+
+// 新增卓创资讯指标数据
+func AddEdbDataFromSci99(edbCode string) (err error) {
+	o := orm.NewOrm()
+	dataAll, err := GetBaseFromSci99DataByIndexCode(edbCode)
+	if err != nil && err.Error() != utils.ErrNoRow() {
+		return
+	}
+
+	var isAdd bool
+	addSql := ` INSERT INTO edb_data_sci99(edb_info_id,edb_code,data_time,value,create_time,modify_time,data_timestamp) values `
+	existMap := make(map[string]string)
+
+	for _, sv := range dataAll {
+		eDate := sv.DataTime
+		dataTime, err := time.ParseInLocation(utils.FormatDate, eDate, time.Local)
+		if err != nil {
+			fmt.Println("time.Parse Err:" + eDate)
+			return err
+		}
+		timestamp := dataTime.UnixNano() / 1e6
+		timeStr := fmt.Sprintf("%d", timestamp)
+		if _, ok := existMap[eDate]; !ok {
+			addSql += GetAddSql("0", edbCode, eDate, timeStr, sv.Value)
+			isAdd = true
+		}
+	}
+	if isAdd {
+		addSql = strings.TrimRight(addSql, ",")
+		utils.FileLog.Info("addSql:" + addSql)
+		_, err = o.Raw(addSql).Exec()
+		if err != nil {
+			return err
+		}
+	}
+	return
+}
+
+// 刷新卓创资讯指标数据
+func RefreshEdbDataFromSci99(edbInfoId int, edbCode, startDate string) (err error) {
+	source := utils.DATA_SOURCE_SCI99
+	subSource := utils.DATA_SUB_SOURCE_EDB
+
+	o := orm.NewOrm()
+	if err != nil {
+		return
+	}
+	edbInfoIdStr := strconv.Itoa(edbInfoId)
+	//计算数据
+	var condition string
+	var pars []interface{}
+
+	if edbCode != "" {
+		condition += " AND index_code=? "
+		pars = append(pars, edbCode)
+	}
+
+	if startDate != "" {
+		condition += " AND data_time>=? "
+		pars = append(pars, startDate)
+	}
+
+	dataList, err := GetBaseFromSci99DataDataByCondition(condition, pars)
+	if err != nil {
+		return
+	}
+
+	// 真实数据的最大日期  ， 插入规则配置的日期
+	var realDataMaxDate, edbDataInsertConfigDate time.Time
+	var edbDataInsertConfig *EdbDataInsertConfig
+	var isFindConfigDateRealData bool //是否找到配置日期的实际数据的值
+	{
+		edbDataInsertConfig, err = GetEdbDataInsertConfigByEdbId(edbInfoId)
+		if err != nil && err.Error() != utils.ErrNoRow() {
+			return
+		}
+		if edbDataInsertConfig != nil {
+			edbDataInsertConfigDate = edbDataInsertConfig.Date
+		}
+	}
+
+	//获取指标所有数据
+	var existCondition string
+	var existPars []interface{}
+
+	existCondition += " AND edb_info_id=? "
+	existPars = append(existPars, edbInfoId)
+	if startDate != "" {
+		existCondition += " AND data_time>=? "
+		existPars = append(existPars, startDate)
+	}
+
+	existList, err := GetEdbDataByCondition(source, subSource, existCondition, existPars)
+	if err != nil {
+		return err
+	}
+	existMap := make(map[string]*EdbInfoSearchData)
+	for _, v := range existList {
+		existMap[v.DataTime] = v
+	}
+
+	addSql := ` INSERT INTO edb_data_sci99(edb_info_id,edb_code,data_time,value,create_time,modify_time,data_timestamp) values `
+	var isAdd bool
+	for _, v := range dataList {
+		item := v
+		itemValue := v.Value
+		eDate := item.DataTime
+		dataTime, err := time.ParseInLocation(utils.FormatDate, eDate, time.Local)
+		if err != nil {
+			return err
+		}
+		if _, ok := existMap[v.DataTime]; !ok {
+			sValue := itemValue
+			if sValue != "" {
+				timestamp := dataTime.UnixNano() / 1e6
+				timeStr := fmt.Sprintf("%d", timestamp)
+				saveValue := sValue
+
+				if findItem, ok := existMap[eDate]; !ok {
+					addSql += GetAddSql(edbInfoIdStr, edbCode, eDate, timeStr, saveValue)
+					isAdd = true
+				} else {
+					if findItem != nil && utils.SubFloatToString(findItem.Value, 30) != sValue {
+						err = ModifyEdbDataById(source, subSource, findItem.EdbDataId, sValue)
+						if err != nil {
+							return err
+						}
+					}
+				}
+			}
+		}
+
+		// 下面代码主要目的是处理掉手动插入的数据判断
+		{
+			if realDataMaxDate.IsZero() || dataTime.After(realDataMaxDate) {
+				realDataMaxDate = dataTime
+			}
+			if edbDataInsertConfigDate.IsZero() || dataTime.Equal(edbDataInsertConfigDate) {
+				isFindConfigDateRealData = true
+			}
+		}
+	}
+
+	// 处理手工数据补充的配置
+	HandleConfigInsertEdbData(realDataMaxDate, edbDataInsertConfig, edbInfoId, source, subSource, existMap, isFindConfigDateRealData)
+
+	if isAdd {
+		addSql = strings.TrimRight(addSql, ",")
+		_, err = o.Raw(addSql).Exec()
+		if err != nil {
+			return err
+		}
+	}
+	return
+}
+
+func GetBaseFromSci99Classify() (list []*BaseFromSci99Classify, err error) {
+	o := orm.NewOrm()
+	sql := `SELECT * FROM base_from_sci99_classify `
+	_, err = o.Raw(sql).QueryRows(&list)
+	return
+}
+
+func GetBaseFromTradeSci99DataAll(indexCode string) (list []*BaseFromSci99Data, err error) {
+	o := orm.NewOrm()
+	sql := `SELECT * FROM base_from_sci99_data where index_code=?`
+	_, err = o.Raw(sql, indexCode).QueryRows(&list)
+	return
+}
+func GetBaseFromTradeSci99LatestData(indexCode string) (date time.Time, err error) {
+	o := orm.NewOrm()
+	sql := `SELECT data_time FROM base_from_sci99_data where index_code=? ORDER BY data_time DESC limit 1 `
+	err = o.Raw(sql, indexCode).QueryRow(&date)
+	return
+}
+
+// UpdateBaseFromSci99Data
+func UpdateBaseFromSci99Data(value float64, indexCode, dataTime string) (err error) {
+	o := orm.NewOrm()
+	sql := `UPDATE base_from_sci99_data SET value=?,modify_time=NOW() WHERE index_code = ? AND data_time = ? `
+	_, err = o.Raw(sql, value, indexCode, dataTime).Exec()
+	return
+}

models/base_from_smm.go

@@ -23,7 +23,7 @@ type BaseFromSmmData struct {
 
 func AddBaseFromSmmData(item []*BaseFromSmmData) (err error) {
 	o := orm.NewOrm()
-	_, err = o.InsertMulti(len(item), item)
+	_, err = o.InsertMulti(500, item)
 	return
 }
 
@@ -37,6 +37,33 @@ func GetBaseFromSmmDataByCondition(condition string, pars []interface{}) (list [
 	return
 }
 
+func UpdateBaseFromSmmDataListByIndexCode(items []*BaseFromSmmData) (err error) {
+	o := orm.NewOrm()
+	sql := `UPDATE base_from_smm_data SET value=? WHERE index_code=? AND data_time=? `
+	stmt, err := o.Raw(sql).Prepare()
+	if err != nil {
+		return
+	}
+	defer stmt.Close()
+	for _, item := range items {
+		_, err = stmt.Exec(item.Value, item.IndexCode, item.DataTime)
+		if err != nil {
+			return
+		}
+	}
+	return
+}
+
+func GetBaseFromSmmDataByIds(smmDataIds []int) (list []*BaseFromSmmData, err error) {
+	if len(smmDataIds) == 0 {
+		return
+	}
+	o := orm.NewOrm()
+	sql := `SELECT * FROM base_from_smm_data WHERE 1=1 AND base_from_smm_index_id in (` + utils.GetOrmInReplace(len(smmDataIds)) + `)`
+	_, err = o.Raw(sql, smmDataIds).QueryRows(&list)
+	return
+}
+
 // 新增有色指标数据
 func AddEdbDataFromSmm(edbCode string, smmBaseDataAll []*BaseFromSmmData) (err error) {
 	o := orm.NewOrm()
@@ -440,6 +467,19 @@ func ModifyBaseFromSmmIndex(item *BaseFromSmmIndex) (err error) {
 	return
 }
 
+func (m *BaseFromSmmIndex) UpdateCols(cols []string) (err error) {
+	o := orm.NewOrm()
+	_, err = o.Update(m, cols...)
+	return
+}
+
+func ModifyBaseFromSmmIndexDate(item *BaseFromSmmIndex) (err error) {
+	o := orm.NewOrm()
+	sql := ` UPDATE base_from_smm_index SET end_date = ?, modify_time=NOW(), data_state= ? WHERE base_from_smm_index_id=?`
+	_, err = o.Raw(sql, item.EndDate, item.DataState, item.BaseFromSmmIndexId).Exec()
+	return
+}
+
 type SmmLatestDataResponse struct {
 	Code int           `json:"code"`
 	Msg  string        `json:"msg"`

models/base_from_wind_wsd.go

@@ -1,6 +1,7 @@
 package models
 
 import (
+	"errors"
 	"eta/eta_index_lib/utils"
 	"fmt"
 	"github.com/beego/beego/v2/client/orm"
@@ -31,6 +32,12 @@ func AddEdbDataFromWindWsd(stockCode string, item map[string]map[string]interfac
 				return err
 			}
 
+			if vk == "OUTMESSAGE" {
+				utils.FileLog.Info("OUTMESSAGE:" + vv.(string))
+				err = errors.New("OUTMESSAGE:" + vv.(string))
+				return err
+			}
+			
 			vk = strings.ToLower(vk)
 			indexCode = windWsd + stockCode + vk
 

models/business_conf.go

@@ -0,0 +1,238 @@
+package models
+
+import (
+	"eta/eta_index_lib/utils"
+	"fmt"
+	"github.com/beego/beego/v2/client/orm"
+	"html"
+	"strings"
+	"time"
+)
+
+const (
+	BusinessConfUseXf                     = "UseXf"
+	BusinessConfXfAppid                   = "XfAppid"
+	BusinessConfXfApiKey                  = "XfApiKey"
+	BusinessConfXfApiSecret               = "XfApiSecret"
+	BusinessConfXfVcn                     = "XfVcn"
+	BusinessConfEnPptCoverImgs            = "EnPptCoverImgs"
+	BusinessConfIsReportApprove           = "IsReportApprove"
+	BusinessConfReportApproveType         = "ReportApproveType"
+	BusinessConfCompanyName               = "CompanyName"
+	BusinessConfCompanyWatermark          = "CompanyWatermark"
+	BusinessConfWatermarkChart            = "WatermarkChart"
+	BusinessConfLoginSmsTpId              = "LoginSmsTpId"
+	BusinessConfLoginSmsGjTpId            = "LoginSmsGjTpId"
+	BusinessConfSmsJhgnAppKey             = "SmsJhgnAppKey"
+	BusinessConfSmsJhgjAppKey             = "SmsJhgjAppKey"
+	BusinessConfLdapHost                  = "LdapHost"
+	BusinessConfLdapBase                  = "LdapBase"
+	BusinessConfLdapPort                  = "LdapPort"
+	BusinessConfEmailClient               = "EmailClient"
+	BusinessConfEmailServerHost           = "EmailServerHost"
+	BusinessConfEmailServerPort           = "EmailServerPort"
+	BusinessConfEmailSender               = "EmailSender"
+	BusinessConfEmailSenderUserName       = "EmailSenderUserName"
+	BusinessConfEmailSenderPassword       = "EmailSenderPassword"
+	BusinessConfSmsClient                 = "SmsClient"
+	BusinessConfNanHuaSmsAppKey           = "NanHuaSmsAppKey"
+	BusinessConfNanHuaSmsAppSecret        = "NanHuaSmsAppSecret"
+	BusinessConfNanHuaSmsApiHost          = "NanHuaSmsApiHost"
+	BusinessConfLoginSmsTplContent        = "LoginSmsTplContent"
+	BusinessConfLoginEmailTemplateSubject = "LoginEmailTemplateSubject"
+	BusinessConfLoginEmailTemplateContent = "LoginEmailTemplateContent"
+	BusinessConfLdapBindUserSuffix        = "LdapBindUserSuffix"
+	BusinessConfLdapUserFilter            = "LdapUserFilter"
+
+	BusinessConfTencentApiSecretId           = "TencentApiSecretId"           // 腾讯云API-密钥对
+	BusinessConfTencentApiSecretKey          = "TencentApiSecretKey"          // 腾讯云API-密钥对
+	BusinessConfTencentApiRecTaskCallbackUrl = "TencentApiRecTaskCallbackUrl" // 腾讯云API-语音识别回调地址
+	BusinessConfSmsJhgjVariable              = "SmsJhgjVariable"              // 聚合国际短信变量
+)
+
+const (
+	BusinessConfReportApproveTypeEta   = "eta"
+	BusinessConfReportApproveTypeOther = "other"
+	BusinessConfClientFlagNanHua       = "nhqh" // 南华标记
+	BusinessConfEmailClientSmtp        = "smtp" // 普通邮箱标记
+)
+
+// FromSceneMap 数据源名称与数据源ID的对应关系
+var FromSceneMap = map[int]string{
+	1: "SmartReportSheetSize",
+	2: "ReportSheetSize",
+	3: "EnReportSheetSize",
+	4: "CnPptSheetSize",
+	5: "EnPptSheetSize",
+}
+
+// BusinessConf 商户配置表
+type BusinessConf struct {
+	Id         int    `orm:"column(id);pk"`
+	ConfKey    string `description:"配置Key"`
+	ConfVal    string `description:"配置值"`
+	ValType    int    `description:"1-字符串；2-数值；3-字符串数组；4-富文本；"`
+	Necessary  int    `description:"是否必填：0-否；1-是"`
+	Remark     string `description:"备注"`
+	CreateTime time.Time
+}
+
+func (m *BusinessConf) TableName() string {
+	return "business_conf"
+}
+
+func (m *BusinessConf) PrimaryId() string {
+	return "id"
+}
+
+func (m *BusinessConf) Create() (err error) {
+	o := orm.NewOrmUsingDB("master")
+	id, err := o.Insert(m)
+	if err != nil {
+		return
+	}
+	m.Id = int(id)
+	return
+}
+
+func (m *BusinessConf) CreateMulti(items []*BusinessConf) (err error) {
+	if len(items) == 0 {
+		return
+	}
+	o := orm.NewOrmUsingDB("master")
+	_, err = o.InsertMulti(len(items), items)
+	return
+}
+
+func (m *BusinessConf) Update(cols []string) (err error) {
+	o := orm.NewOrmUsingDB("master")
+	_, err = o.Update(m, cols...)
+	return
+}
+
+func (m *BusinessConf) Del() (err error) {
+	o := orm.NewOrmUsingDB("master")
+	sql := fmt.Sprintf(`DELETE FROM %s WHERE %s = ? LIMIT 1`, m.TableName(), m.PrimaryId())
+	_, err = o.Raw(sql, m.Id).Exec()
+	return
+}
+
+func (m *BusinessConf) GetItemById(id int) (item *BusinessConf, err error) {
+	o := orm.NewOrmUsingDB("master")
+	sql := fmt.Sprintf(`SELECT * FROM %s WHERE %s = ? LIMIT 1`, m.TableName(), m.PrimaryId())
+	err = o.Raw(sql, id).QueryRow(&item)
+	return
+}
+
+func (m *BusinessConf) GetItemByCondition(condition string, pars []interface{}) (item *BusinessConf, err error) {
+	o := orm.NewOrmUsingDB("master")
+	sql := fmt.Sprintf(`SELECT * FROM %s WHERE 1=1 %s LIMIT 1`, m.TableName(), condition)
+	err = o.Raw(sql, pars).QueryRow(&item)
+	return
+}
+
+func (m *BusinessConf) GetCountByCondition(condition string, pars []interface{}) (count int, err error) {
+	o := orm.NewOrmUsingDB("master")
+	sql := fmt.Sprintf(`SELECT COUNT(1) FROM %s WHERE 1=1 %s`, m.TableName(), condition)
+	err = o.Raw(sql, pars).QueryRow(&count)
+	return
+}
+
+func (m *BusinessConf) GetItemsByCondition(condition string, pars []interface{}, fieldArr []string, orderRule string) (items []*BusinessConf, err error) {
+	o := orm.NewOrmUsingDB("master")
+	fields := strings.Join(fieldArr, ",")
+	if len(fieldArr) == 0 {
+		fields = `*`
+	}
+	order := `ORDER BY create_time DESC`
+	if orderRule != "" {
+		order = ` ORDER BY ` + orderRule
+	}
+	sql := fmt.Sprintf(`SELECT %s FROM %s WHERE 1=1 %s %s`, fields, m.TableName(), condition, order)
+	_, err = o.Raw(sql, pars).QueryRows(&items)
+	return
+}
+
+func (m *BusinessConf) GetPageItemsByCondition(condition string, pars []interface{}, fieldArr []string, orderRule string, startSize, pageSize int) (items []*BusinessConf, err error) {
+	o := orm.NewOrmUsingDB("master")
+	fields := strings.Join(fieldArr, ",")
+	if len(fieldArr) == 0 {
+		fields = `*`
+	}
+	order := `ORDER BY create_time DESC`
+	if orderRule != "" {
+		order = ` ORDER BY ` + orderRule
+	}
+	sql := fmt.Sprintf(`SELECT %s FROM %s WHERE 1=1 %s %s LIMIT ?,?`, fields, m.TableName(), condition, order)
+	_, err = o.Raw(sql, pars, startSize, pageSize).QueryRows(&items)
+	return
+}
+
+// GetBusinessConf 获取商家配置
+func GetBusinessConf() (list map[string]string, err error) {
+	list = make(map[string]string)
+
+	var items []*BusinessConf
+	o := orm.NewOrmUsingDB("master")
+	sql := `SELECT * FROM business_conf`
+	_, err = o.Raw(sql).QueryRows(&items)
+	if err != nil {
+		return
+	}
+
+	for _, v := range items {
+		if v.ValType == 4 {
+			list[v.ConfKey] = html.UnescapeString(v.ConfVal)
+			continue
+		}
+		list[v.ConfKey] = v.ConfVal
+	}
+	return
+}
+
+// BusinessConfUpdate 更新配置
+type BusinessConfUpdate struct {
+	ConfKey string
+	ConfVal string
+}
+
+// UpdateBusinessConfMulti 批量修改配置
+func UpdateBusinessConfMulti(items []BusinessConfUpdate) (err error) {
+	o := orm.NewOrmUsingDB("master")
+	p, err := o.Raw("UPDATE business_conf SET conf_val = ? WHERE conf_key = ?").Prepare()
+	if err != nil {
+		return
+	}
+	defer func() {
+		_ = p.Close()
+	}()
+	for _, v := range items {
+		_, err = p.Exec(v.ConfVal, v.ConfKey)
+		if err != nil {
+			return
+		}
+	}
+	return
+}
+
+func GetBusinessConfByKey(key string) (item *BusinessConf, err error) {
+	o := orm.NewOrmUsingDB("master")
+	sql := fmt.Sprintf(`SELECT * FROM business_conf WHERE conf_key = ? LIMIT 1`)
+	err = o.Raw(sql, key).QueryRow(&item)
+	return
+}
+
+// InitUseMongoConf
+// @Description:
+// @author: Roc
+// @datetime 2024-07-01 13:49:09
+func InitUseMongoConf() {
+	useMongo, e := GetBusinessConfByKey("UseMongo")
+	if e != nil {
+		return
+	}
+
+	if useMongo.ConfVal == `true` {
+		utils.UseMongo = true
+	}
+}

models/ccf_stock_excel.go

@@ -0,0 +1,147 @@
+package models
+
+import (
+	"eta/eta_index_lib/utils"
+	"fmt"
+	"github.com/beego/beego/v2/client/orm"
+	"strings"
+	"time"
+)
+
+// CCFStockExcel CCF化纤装置表格
+type CCFStockExcel struct {
+	CcfStockExcelId int       `orm:"column(ccf_stock_excel_id);pk"`
+	ClassifyId      int       `description:"分类ID"`
+	ExcelDate       time.Time `description:"表格日期"`
+	ExcelContent    string    `description:"表格HTML"`
+	FromPage        string    `description:"表格来源"`
+	CreateTime      time.Time `description:"创建时间"`
+	ModifyTime      time.Time `description:"修改时间"`
+}
+
+func (m *CCFStockExcel) TableName() string {
+	return "ccf_stock_excel"
+}
+
+type CCFStockExcelCols struct {
+	CcfStockExcelId string
+	ClassifyId      string
+	ExcelDate       string
+	ExcelContent    string
+	FromPage        string
+	CreateTime      string
+	ModifyTime      string
+}
+
+func (m *CCFStockExcel) Cols() CCFStockExcelCols {
+	return CCFStockExcelCols{
+		CcfStockExcelId: "ccf_stock_excel_id",
+		ClassifyId:      "classify_id",
+		ExcelDate:       "excel_date",
+		ExcelContent:    "excel_content",
+		FromPage:        "from_page",
+		CreateTime:      "create_time",
+		ModifyTime:      "modify_time",
+	}
+}
+
+func (m *CCFStockExcel) PrimaryId() string {
+	return m.Cols().CcfStockExcelId
+}
+
+func (m *CCFStockExcel) Create() (err error) {
+	o := orm.NewOrm()
+	id, err := o.Insert(m)
+	if err != nil {
+		return
+	}
+	m.CcfStockExcelId = int(id)
+	return
+}
+
+func (m *CCFStockExcel) CreateMulti(items []*CCFStockExcel) (err error) {
+	if len(items) == 0 {
+		return
+	}
+	o := orm.NewOrm()
+	_, err = o.InsertMulti(len(items), items)
+	return
+}
+
+func (m *CCFStockExcel) Update(cols []string) (err error) {
+	o := orm.NewOrm()
+	_, err = o.Update(m, cols...)
+	return
+}
+
+func (m *CCFStockExcel) Del() (err error) {
+	o := orm.NewOrm()
+	sql := fmt.Sprintf(`DELETE FROM %s WHERE %s = ? LIMIT 1`, m.TableName(), m.PrimaryId())
+	_, err = o.Raw(sql, m.CcfStockExcelId).Exec()
+	return
+}
+
+func (m *CCFStockExcel) MultiDel(menuIds []int) (err error) {
+	if len(menuIds) == 0 {
+		return
+	}
+	o := orm.NewOrm()
+	sql := fmt.Sprintf(`DELETE FROM %s WHERE %s IN (%s)`, m.TableName(), m.PrimaryId(), utils.GetOrmInReplace(len(menuIds)))
+	_, err = o.Raw(sql, menuIds).Exec()
+	return
+}
+
+func (m *CCFStockExcel) GetItemById(id int) (item *CCFStockExcel, err error) {
+	o := orm.NewOrm()
+	sql := fmt.Sprintf(`SELECT * FROM %s WHERE %s = ? LIMIT 1`, m.TableName(), m.PrimaryId())
+	err = o.Raw(sql, id).QueryRow(&item)
+	return
+}
+
+func (m *CCFStockExcel) GetItemByCondition(condition string, pars []interface{}, orderRule string) (item *CCFStockExcel, err error) {
+	o := orm.NewOrm()
+	order := ``
+	if orderRule != "" {
+		order = ` ORDER BY ` + orderRule
+	}
+	sql := fmt.Sprintf(`SELECT * FROM %s WHERE 1=1 %s %s LIMIT 1`, m.TableName(), condition, order)
+	err = o.Raw(sql, pars).QueryRow(&item)
+	return
+}
+
+func (m *CCFStockExcel) GetCountByCondition(condition string, pars []interface{}) (count int, err error) {
+	o := orm.NewOrm()
+	sql := fmt.Sprintf(`SELECT COUNT(1) FROM %s WHERE 1=1 %s`, m.TableName(), condition)
+	err = o.Raw(sql, pars).QueryRow(&count)
+	return
+}
+
+func (m *CCFStockExcel) GetItemsByCondition(condition string, pars []interface{}, fieldArr []string, orderRule string) (items []*CCFStockExcel, err error) {
+	o := orm.NewOrm()
+	fields := strings.Join(fieldArr, ",")
+	if len(fieldArr) == 0 {
+		fields = `*`
+	}
+	order := fmt.Sprintf(`ORDER BY %s DESC`, m.Cols().CreateTime)
+	if orderRule != "" {
+		order = ` ORDER BY ` + orderRule
+	}
+	sql := fmt.Sprintf(`SELECT %s FROM %s WHERE 1=1 %s %s`, fields, m.TableName(), condition, order)
+	_, err = o.Raw(sql, pars).QueryRows(&items)
+	return
+}
+
+func (m *CCFStockExcel) GetPageItemsByCondition(condition string, pars []interface{}, fieldArr []string, orderRule string, startSize, pageSize int) (items []*CCFStockExcel, err error) {
+	o := orm.NewOrm()
+	fields := strings.Join(fieldArr, ",")
+	if len(fieldArr) == 0 {
+		fields = `*`
+	}
+	order := fmt.Sprintf(`ORDER BY %s DESC`, m.Cols().CreateTime)
+	if orderRule != "" {
+		order = ` ORDER BY ` + orderRule
+	}
+	sql := fmt.Sprintf(`SELECT %s FROM %s WHERE 1=1 %s %s LIMIT ?,?`, fields, m.TableName(), condition, order)
+	_, err = o.Raw(sql, pars, startSize, pageSize).QueryRows(&items)
+	return
+}

models/db.go

@@ -35,6 +35,17 @@ func init() {
 	gl, _ := orm.GetDB("gl")
 	gl.SetConnMaxLifetime(10 * time.Minute)
 
+	// master库
+	{
+		_ = orm.RegisterDataBase("master", "mysql", utils.MYSQL_URL_MASTER)
+		orm.SetMaxIdleConns("master", 50)
+		orm.SetMaxOpenConns("master", 100)
+
+		master, _ := orm.GetDB("master")
+		master.SetConnMaxLifetime(10 * time.Minute)
+
+	}
+
 	orm.Debug = true
 	orm.DebugLog = orm.NewLog(utils.Binlog)
 
@@ -61,6 +72,7 @@ func init() {
 		new(EdbDataInsertConfig),
 		new(EdbAdjustConf), // 数据调整的配置
 		new(BaseFromMysteelChemicalClassify),
+		new(EdbInfoRelation), //指标引用记录
 	)
 
 	// 注册期货数据 数据表
@@ -81,8 +93,14 @@ func init() {
 	// 初始化指标刷新
 	initEdbRefresh()
 
+	// 自有数据指标
+	initBusinessEdb()
+
+	// 初始化因子指标系列
+	initFactorEdbSeries()
+
 	// 初始化部分数据表变量(直接init会有顺序问题=_=!)
-	InitEdbSource()
+	afterInitTable()
 }
 
 // initFutureGood 注册期货数据 数据表
@@ -135,6 +153,13 @@ func initBaseIndex() {
 		new(BaseFromFenweiData),
 		new(BaseFromBloombergIndex),
 		new(BaseFromBloombergData),
+		new(BaseFromSci99Data),
+		new(BaseFromSci99Index),
+		new(BaseFromSci99Classify),
+		new(BaseFromCCFIndex),
+		new(BaseFromCCFData),
+		new(CCFStockExcel),
+		new(BaseFromBusinessData), // 数据源中自有数据的明细数据表
 	)
 }
 
@@ -165,3 +190,33 @@ func initEdbRefresh() {
 		new(edb_refresh.EdbRefreshMapping),       // 指标刷新时间配置关系表
 	)
 }
+
+// initBusinessEdb 初始化指标刷新
+func initBusinessEdb() {
+	orm.RegisterModel(
+		new(BaseFromBusinessIndex), // 外部指标（商家系统）表
+		new(EdbBusinessSource),     // 自有数据（商家）指标来源
+	)
+}
+
+// initFactorEdbSeries 因子指标系列数据表
+func initFactorEdbSeries() {
+	orm.RegisterModel(
+		new(FactorEdbSeries),              // 因子指标系列
+		new(FactorEdbSeriesChartMapping),  // 因子指标系列-图表关联
+		new(FactorEdbSeriesMapping),       // 因子指标系列-指标计算数据
+		new(FactorEdbSeriesCalculateData), // 因子指标系列-指标关联
+	)
+}
+
+// afterInitTable
+// @Description: 初始化表结构的的后置操作
+// @author: Roc
+// @datetime 2024-07-01 13:31:09
+func afterInitTable() {
+	// 初始化指标来源配置
+	InitEdbSource()
+
+	// 初始化是否启用mongo配置
+	InitUseMongoConf()
+}

models/edb_data_base.go

@@ -1,11 +1,10 @@
 package models
 
 import (
-	"errors"
 	"eta/eta_index_lib/utils"
 	"fmt"
 	"github.com/beego/beego/v2/client/orm"
-	"strconv"
+	"strings"
 	"time"
 )
 
@@ -179,27 +178,74 @@ type EdbDataList struct {
 	Value         float64 `description:"数据值"`
 }
 
-func GetEdbDataList(source, subSource, endInfoId int, startDate, endDate string) (list []*EdbDataList, err error) {
-	tableName := GetEdbDataTableName(source, subSource)
-	if tableName == "" {
-		err = errors.New("无效的渠道：" + strconv.Itoa(source))
-		list = make([]*EdbDataList, 0)
-		return list, err
+func DelEdbDataByMysql(to orm.TxOrmer, edbInfoId int, tableName string, removeDateList []string) (err error) {
+	if len(removeDateList) <= 0 {
+		return
 	}
-	var pars []interface{}
-	sql := `SELECT edb_data_id,edb_info_id,data_time,value,data_timestamp FROM %s WHERE edb_info_id=? `
-	if startDate != "" {
-		sql += ` AND data_time>=? `
-		pars = append(pars, startDate)
+	// 移除不存在的日期数据
+	removeDateStr := strings.Join(removeDateList, `","`)
+	removeDateStr = `"` + removeDateStr + `"`
+	sql := fmt.Sprintf(` DELETE FROM %s WHERE edb_info_id = ? and data_time in (%s) `, tableName, removeDateStr)
+	_, err = to.Raw(sql, edbInfoId).Exec()
+	return
+}
+
+// GetEdbDataList
+// @Description:
+// @author: Roc
+// @datetime 2024-05-08 16:10:49
+// @param source int
+// @param subSource int
+// @param endInfoId int
+// @param startDate string
+// @param endDate string
+// @return list []*EdbDataList
+// @return err error
+func GetEdbDataList(source, subSource, endInfoId int, startDate, endDate string) (list []*EdbDataList, err error) {
+	list = make([]*EdbDataList, 0)
+	tmpDataList, err := GetEdbDataListAll(source, subSource, FindEdbDataListAllCond{
+		EdbInfoId:         endInfoId,
+		StartDataTime:     startDate,
+		StartDataTimeCond: ">=",
+		EndDataTime:       endDate,
+		EndDataTimeCond:   "<=",
+	}, 1)
+	if err != nil {
+		return
 	}
-	if endDate != "" {
-		sql += ` AND data_time<=? `
-		pars = append(pars, endDate)
+
+	// 数据拼接
+	for _, v := range tmpDataList {
+		list = append(list, &EdbDataList{
+			EdbDataId:     v.EdbDataId,
+			EdbInfoId:     v.EdbInfoId,
+			DataTime:      v.DataTime,
+			DataTimestamp: v.DataTimestamp,
+			Value:         v.Value,
+		})
 	}
 
-	sql += ` ORDER BY data_time ASC `
-	sql = fmt.Sprintf(sql, tableName)
-	o := orm.NewOrm()
-	_, err = o.Raw(sql, endInfoId, pars).QueryRows(&list)
 	return
 }
+
+type AddEdbBaseInfoReq struct {
+	EdbCode         string `description:"指标编码"`
+	EdbName         string `description:"指标名称"`
+	Unit            string `description:"单位"`
+	ClassifyId      int    `description:"所属分类"`
+	SysUserId       int    `description:"用户id"`
+	SysUserRealName string `description:"用户真实名称"`
+}
+
+// EditEdbBaseInfoReq 编辑基础指标请求参数
+type EditEdbBaseInfoReq struct {
+	SysUserId       int    `description:"用户id"`
+	SysUserRealName string `description:"用户真实名称"`
+	EdbInfoId       int    `description:"指标id"`
+	EdbName         string `description:"指标名称"`
+	EdbNameEn       string `description:"英文指标名称"`
+	Frequency       string `description:"频率"`
+	Unit            string `description:"单位"`
+	UnitEn          string `description:"英文单位"`
+	ClassifyId      int    `description:"分类id"`
+}

models/edb_data_business.go

@@ -0,0 +1,681 @@
+package models
+
+import (
+	"eta/eta_index_lib/models/mgo"
+	"eta/eta_index_lib/utils"
+	"fmt"
+	"github.com/beego/beego/v2/client/orm"
+	"github.com/shopspring/decimal"
+	"go.mongodb.org/mongo-driver/bson"
+	"reflect"
+	"strings"
+	"time"
+)
+
+// Business 自有数据
+type Business struct {
+}
+
+// AddBaseParams
+// @Description: 基础指标的添加参数
+type AddBaseParams struct {
+	EdbCode         string `description:"指标编码"`
+	EdbName         string `description:"指标名称"`
+	Unit            string `description:"单位"`
+	ClassifyId      int    `description:"所属分类"`
+	SysUserId       int    `description:"用户id"`
+	SysUserRealName string `description:"用户真实名称"`
+	UniqueCode      string `description:"编码"`
+}
+
+// EditBaseParams
+// @Description: 基础指标的修改参数
+type EditBaseParams struct {
+	EdbCode         string   `description:"指标编码"`
+	EdbName         string   `description:"指标名称"`
+	EdbNameEn       string   `description:"指标名称（英文）"`
+	Unit            string   `description:"单位"`
+	UnitEn          string   `description:"单位（英文）"`
+	ClassifyId      int      `description:"所属分类"`
+	SysUserId       int      `description:"用户id"`
+	SysUserRealName string   `description:"用户真实名称"`
+	UniqueCode      string   `description:"编码"`
+	Lang            string   `description:"语言版本"`
+	EdbInfo         *EdbInfo `description:"指标信息"`
+}
+
+type RefreshBaseParams struct {
+	EdbInfo   *EdbInfo
+	StartDate string
+	EndDate   string
+}
+
+// Add
+// @Description: 添加指标
+// @author: Roc
+// @receiver obj
+// @datetime 2024-04-30 17:35:14
+// @param params AddBaseParams
+// @param businessIndexItem *BaseFromBusinessIndex
+// @return edbInfo *EdbInfo
+// @return err error
+// @return errMsg string
+func (obj Business) Add(params AddBaseParams, businessIndexItem *BaseFromBusinessIndex) (edbInfo *EdbInfo, err error, errMsg string) {
+	o := orm.NewOrm()
+	to, err := o.Begin()
+	if err != nil {
+		return
+	}
+	defer func() {
+		if err != nil {
+			_ = to.Rollback()
+			fmt.Println(reflect.TypeOf(obj).Name(), "；Add,Err:"+err.Error())
+		} else {
+			_ = to.Commit()
+		}
+	}()
+
+	edbInfo = new(EdbInfo)
+	edbInfo.Source = obj.GetSource()
+	edbInfo.SourceName = obj.GetSourceName()
+	edbInfo.EdbCode = params.EdbCode
+	edbInfo.EdbName = params.EdbName
+	edbInfo.EdbNameSource = params.EdbName
+	edbInfo.Frequency = businessIndexItem.Frequency
+	edbInfo.Unit = params.Unit
+	edbInfo.ClassifyId = params.ClassifyId
+	edbInfo.SysUserId = params.SysUserId
+	edbInfo.SysUserRealName = params.SysUserRealName
+	edbInfo.CreateTime = time.Now()
+	edbInfo.ModifyTime = time.Now()
+	edbInfo.UniqueCode = params.UniqueCode
+	edbInfo.CalculateFormula = ``
+	edbInfo.EdbNameEn = params.EdbName
+	edbInfo.UnitEn = params.Unit
+	edbInfo.EdbType = obj.GetEdbType()
+	edbInfo.SubSource = businessIndexItem.Source
+	edbInfo.SubSourceName = businessIndexItem.SourceName
+	newEdbInfoId, tmpErr := to.Insert(edbInfo)
+	if tmpErr != nil {
+		err = tmpErr
+		return
+	}
+	edbInfo.EdbInfoId = int(newEdbInfoId)
+
+	// 更新数据
+	err = obj.refresh(to, edbInfo, "")
+
+	return
+}
+
+// Edit
+// @Description: 编辑指标
+// @author: Roc
+// @receiver obj
+// @datetime 2024-04-30 17:35:05
+// @param params EditBaseParams
+// @param businessIndexItem *BaseFromBusinessIndex
+// @return err error
+// @return errMsg string
+func (obj Business) Edit(params EditBaseParams, businessIndexItem *BaseFromBusinessIndex) (err error, errMsg string) {
+	edbInfo := params.EdbInfo
+
+	o := orm.NewOrm()
+	to, err := o.Begin()
+	if err != nil {
+		return
+	}
+	defer func() {
+		if err != nil {
+			_ = to.Rollback()
+			fmt.Println(reflect.TypeOf(obj).Name(), "；Edit,Err:"+err.Error())
+		} else {
+			_ = to.Commit()
+		}
+	}()
+
+	//oldEdbInfo := *edbInfo
+
+	//修改指标信息
+	edbInfo.EdbName = params.EdbName
+	edbInfo.EdbNameSource = params.EdbName
+	edbInfo.Frequency = businessIndexItem.Frequency
+	edbInfo.Unit = params.Unit
+	edbInfo.ClassifyId = params.ClassifyId
+	edbInfo.EdbNameEn = params.EdbNameEn
+	edbInfo.UnitEn = params.UnitEn
+	edbInfo.SubSource = businessIndexItem.Source
+	edbInfo.SubSourceName = businessIndexItem.SourceName
+	edbInfo.ModifyTime = time.Now()
+	_, err = to.Update(edbInfo, "EdbName", "EdbNameSource", "Frequency", "Unit", "ClassifyId", "CalculateFormula", "ModifyTime", "EdbNameEn", "UnitEn", "SubSource", "SubSourceName")
+	if err != nil {
+		return
+	}
+
+	//计算数据
+	err = obj.refresh(to, edbInfo, "")
+
+	return
+}
+
+// Refresh 刷新
+func (obj Business) Refresh(params RefreshBaseParams) (err error, errMsg string) {
+	to, err := orm.NewOrm().Begin()
+	if err != nil {
+		return
+	}
+	defer func() {
+		if err != nil {
+			_ = to.Rollback()
+			fmt.Println(reflect.TypeOf(obj).Name(), "；Refresh,Err:"+err.Error())
+		} else {
+			_ = to.Commit()
+		}
+	}()
+
+	// 计算数据
+	err = obj.refresh(to, params.EdbInfo, params.StartDate)
+
+	return
+}
+
+// GetSource 获取来源编码id
+func (obj Business) GetSource() int {
+	return utils.DATA_SOURCE_BUSINESS
+}
+
+// GetSourceName 获取来源名称
+func (obj Business) GetSourceName() string {
+	return utils.DATA_SOURCE_NAME_BUSINESS
+}
+
+// GetEdbType 获取指标类型
+func (obj Business) GetEdbType() int {
+	return utils.DEFAULT_EDB_TYPE
+}
+
+func (obj Business) refresh(to orm.TxOrmer, edbInfo *EdbInfo, startDate string) (err error) {
+	if utils.UseMongo {
+		return obj.refreshByMongo(edbInfo, startDate)
+	} else {
+		return obj.refreshByMysql(to, edbInfo, startDate)
+	}
+
+	return
+}
+
+func (obj Business) refreshByMongo(edbInfo *EdbInfo, startDate string) (err error) {
+	// 真实数据的最大日期  ， 插入规则配置的日期
+	var realDataMaxDate, edbDataInsertConfigDate time.Time
+	var edbDataInsertConfig *EdbDataInsertConfig
+	var isFindConfigDateRealData bool //是否找到配置日期的实际数据的值
+	{
+		edbDataInsertConfig, err = GetEdbDataInsertConfigByEdbId(edbInfo.EdbInfoId)
+		if err != nil && err.Error() != utils.ErrNoRow() {
+			return
+		}
+		if edbDataInsertConfig != nil {
+			edbDataInsertConfigDate = edbDataInsertConfig.Date
+		}
+	}
+
+	//获取已存在的所有数据
+	baseDataList, err := obj.getBaseBusinessDataByMongo(edbInfo, startDate)
+
+	//获取指标所有数据
+	existDataList := make([]*mgo.EdbDataBusiness, 0)
+	mogDataObj := new(mgo.EdbDataBusiness)
+	{
+		// 构建查询条件
+		queryConditions := bson.M{
+			"edb_code": edbInfo.EdbCode,
+		}
+
+		if startDate != `` {
+			//获取已存在的所有数据
+			startDateTime, tmpErr := time.ParseInLocation(utils.FormatDate, startDate, time.Local)
+			if tmpErr != nil {
+				err = tmpErr
+				return
+			}
+			queryConditions["data_time"] = bson.M{"$gte": startDateTime}
+		}
+
+		existDataList, err = mogDataObj.GetAllDataList(queryConditions, []string{"data_time"})
+		if err != nil {
+			fmt.Println(obj.GetSourceName() + ",refresh err;getEdbDataBusinessList Err:" + err.Error())
+			return
+		}
+	}
+
+	existDataMap := make(map[string]*mgo.EdbDataBusiness)
+	removeDataTimeMap := make(map[string]bool) //需要移除的日期数据
+	for _, v := range existDataList {
+		tmpDate := v.DataTime.Format(utils.FormatDate)
+		existDataMap[tmpDate] = v
+		removeDataTimeMap[tmpDate] = true
+	}
+	needAddDateMap := make(map[time.Time]int)
+
+	// 待添加的数据集
+	addDataList := make([]interface{}, 0)
+	// 待更新的数据集
+	updateDataList := make([]mgo.EdbDataBusiness, 0)
+
+	for _, tmpData := range baseDataList {
+		currDate := tmpData.DataTime
+		currDateStr := currDate.Format(utils.FormatDate)
+
+		// 当前的实际值
+		saveValue := decimal.NewFromFloat(tmpData.Value).Round(4).String()
+
+		// 下面代码主要目的是处理掉手动插入的数据判断
+		{
+			if realDataMaxDate.IsZero() || currDate.After(realDataMaxDate) {
+				realDataMaxDate = currDate
+			}
+			if edbDataInsertConfigDate.IsZero() || currDate.Equal(edbDataInsertConfigDate) {
+				isFindConfigDateRealData = true
+			}
+		}
+
+		existData, ok := existDataMap[currDateStr]
+		// 如果库中已经存在该数据的话，那么就进行值的变更操作
+		if ok {
+			// 已经入到指标库的值
+			existValStr := decimal.NewFromFloat(existData.Value).Round(4).String()
+
+			//校验待删除日期数据里面是否存在该元素，如果存在的话，那么移除该日期
+			delete(removeDataTimeMap, currDateStr)
+			if existValStr != saveValue {
+				existData.Value = tmpData.Value
+				updateDataList = append(updateDataList, *existData)
+			}
+
+			continue
+		}
+
+		// 库中不存在该日期的数据
+		timestamp := currDate.UnixNano() / 1e6
+		addDataList = append(addDataList, mgo.EdbDataBusiness{
+			EdbInfoId:     edbInfo.EdbInfoId,
+			EdbCode:       edbInfo.EdbCode,
+			DataTime:      currDate,
+			Value:         tmpData.Value,
+			CreateTime:    time.Now(),
+			ModifyTime:    time.Now(),
+			DataTimestamp: timestamp,
+		})
+		needAddDateMap[currDate] = 1
+	}
+
+	// 入库
+	{
+		coll := mogDataObj.GetCollection()
+
+		//删除已经不存在的指标数据（由于该指标当日的数据删除了）
+		{
+			removeDateList := make([]time.Time, 0)
+			for dateTime := range removeDataTimeMap {
+				//获取已存在的所有数据
+				tmpDateTime, tmpErr := time.ParseInLocation(utils.FormatDate, dateTime, time.Local)
+				if tmpErr != nil {
+					err = tmpErr
+					return
+				}
+				removeDateList = append(removeDateList, tmpDateTime)
+			}
+			removeNum := len(removeDateList)
+			if removeNum > 0 {
+				err = mogDataObj.RemoveManyByColl(coll, bson.M{"edb_code": edbInfo.EdbCode, "data_time": bson.M{"$in": removeDateList}})
+				if err != nil {
+					fmt.Println("mogDataObj.RemoveMany() Err:" + err.Error())
+					return
+				}
+			}
+		}
+
+		// 插入新数据
+		if len(addDataList) > 0 {
+			err = mogDataObj.BatchInsertDataByColl(coll, 500, addDataList)
+			if err != nil {
+				fmt.Println("mogDataObj.BatchInsertData() Err:" + err.Error())
+				return
+			}
+		}
+
+		// 修改历史数据
+		if len(updateDataList) > 0 {
+			for _, v := range updateDataList {
+				err = mogDataObj.UpdateDataByColl(coll, bson.M{"_id": v.ID}, bson.M{"$set": bson.M{"value": v.Value, "modify_time": v.ModifyTime}})
+				if err != nil {
+					fmt.Println("mogDataObj.UpdateDataByColl:Err:" + err.Error())
+					return
+				}
+			}
+		}
+	}
+
+	// 处理手工数据补充的配置
+	HandleConfigInsertEdbDataByMongo(realDataMaxDate, edbDataInsertConfig, edbInfo.EdbInfoId, edbInfo.Source, edbInfo.SubSource, existDataMap, isFindConfigDateRealData)
+
+	return
+}
+
+func (obj Business) refreshByMysql(to orm.TxOrmer, edbInfo *EdbInfo, startDate string) (err error) {
+	dataTableName := GetEdbDataTableName(edbInfo.Source, edbInfo.SubSource)
+	edbInfoIdStr := fmt.Sprint(edbInfo.EdbInfoId)
+
+	// 真实数据的最大日期  ， 插入规则配置的日期
+	var realDataMaxDate, edbDataInsertConfigDate time.Time
+	var edbDataInsertConfig *EdbDataInsertConfig
+	var isFindConfigDateRealData bool //是否找到配置日期的实际数据的值
+	{
+		edbDataInsertConfig, err = GetEdbDataInsertConfigByEdbId(edbInfo.EdbInfoId)
+		if err != nil && err.Error() != utils.ErrNoRow() {
+			return
+		}
+		if edbDataInsertConfig != nil {
+			edbDataInsertConfigDate = edbDataInsertConfig.Date
+		}
+	}
+
+	//获取已存在的所有数据
+	baseDataList, err := obj.getBaseBusinessDataByMysql(edbInfo, startDate)
+
+	//获取指标所有数据
+	var existCondition string
+	var existPars []interface{}
+
+	existCondition += " AND edb_info_id=? "
+	existPars = append(existPars, edbInfo.EdbInfoId)
+	if startDate != "" {
+		existCondition += " AND data_time>=? "
+		existPars = append(existPars, startDate)
+	}
+
+	existList, err := GetEdbDataByCondition(edbInfo.Source, edbInfo.SubSource, existCondition, existPars)
+	if err != nil {
+		fmt.Println(obj.GetSourceName() + ",refreshByMysql err;getEdbDataBusinessList Err:" + err.Error())
+		return err
+	}
+	existDataMap := make(map[string]*EdbInfoSearchData)
+	removeDataTimeMap := make(map[string]bool) //需要移除的日期数据
+	for _, v := range existList {
+		existDataMap[v.DataTime] = v
+		removeDataTimeMap[v.DataTime] = true
+	}
+	needAddDateMap := make(map[time.Time]int)
+
+	// 待添加的数据集
+	addSql := ` INSERT INTO ` + dataTableName + ` (edb_info_id,edb_code,data_time,value,create_time,modify_time,data_timestamp) values `
+	var isAdd bool
+	// 待更新的数据集
+	updateDataList := make([]*EdbInfoSearchData, 0)
+
+	for _, tmpData := range baseDataList {
+		currDate := tmpData.DataTime
+		currDateStr := currDate.Format(utils.FormatDate)
+
+		// 当前的实际值
+		saveValue := decimal.NewFromFloat(tmpData.Value).Round(4).String()
+
+		// 下面代码主要目的是处理掉手动插入的数据判断
+		{
+			if realDataMaxDate.IsZero() || currDate.After(realDataMaxDate) {
+				realDataMaxDate = currDate
+			}
+			if edbDataInsertConfigDate.IsZero() || currDate.Equal(edbDataInsertConfigDate) {
+				isFindConfigDateRealData = true
+			}
+		}
+
+		existData, ok := existDataMap[currDateStr]
+		// 如果库中已经存在该数据的话，那么就进行值的变更操作
+		if ok {
+			// 已经入到指标库的值
+			existValStr := decimal.NewFromFloat(existData.Value).Round(4).String()
+
+			//校验待删除日期数据里面是否存在该元素，如果存在的话，那么移除该日期
+			delete(removeDataTimeMap, currDateStr)
+			if existValStr != saveValue {
+				existData.Value = tmpData.Value
+				updateDataList = append(updateDataList, existData)
+			}
+
+			continue
+		}
+
+		// 库中不存在该日期的数据
+		timestamp := currDate.UnixNano() / 1e6
+		needAddDateMap[currDate] = 1
+		addSql += GetAddSql(edbInfoIdStr, edbInfo.EdbCode, currDateStr, fmt.Sprint(timestamp), saveValue)
+		isAdd = true
+	}
+
+	//删除已经不存在的指标数据（由于该指标当日的数据删除了）
+	{
+		removeDateList := make([]string, 0)
+		for dateTime := range removeDataTimeMap {
+			removeDateList = append(removeDateList, dateTime)
+		}
+		removeNum := len(removeDateList)
+		if removeNum > 0 {
+			//如果拼接指标变更了，那么需要删除所有的指标数据
+			sql := fmt.Sprintf(` DELETE FROM %s WHERE edb_info_id = ? and data_time in (`+utils.GetOrmInReplace(removeNum)+`) `, dataTableName)
+
+			_, err = to.Raw(sql, edbInfo.EdbInfoId, removeDateList).Exec()
+			if err != nil {
+				err = fmt.Errorf("删除自有数据的明细数据失败,Err:" + err.Error())
+				return
+			}
+		}
+	}
+
+	if isAdd {
+		addSql = strings.TrimRight(addSql, ",")
+		_, err = to.Raw(addSql).Exec()
+		if err != nil {
+			fmt.Println("RefreshAllCalculate add Err", err.Error())
+			return
+		}
+	}
+
+	// 修改历史数据
+	if len(updateDataList) > 0 {
+		for _, v := range updateDataList {
+			err = ModifyEdbDataById(edbInfo.Source, edbInfo.SubSource, v.EdbDataId, fmt.Sprint(v.Value))
+			if err != nil {
+				fmt.Println(obj.GetSourceName() + "，refreshByMysql:Err:" + err.Error())
+				return err
+			}
+		}
+	}
+
+	// 处理手工数据补充的配置
+	HandleConfigInsertEdbData(realDataMaxDate, edbDataInsertConfig, edbInfo.EdbInfoId, edbInfo.Source, edbInfo.SubSource, existDataMap, isFindConfigDateRealData)
+
+	return
+}
+
+// GetEdbInfoMaxAndMinInfo 获取指标的最新数据记录信息
+func (obj Business) getEdbInfoMaxAndMinInfoByMongo(edbCode string) (item *EdbInfoMaxAndMinInfo, err error) {
+	mogDataObj := new(mgo.EdbDataBusiness)
+	pipeline := []bson.M{
+		{"$match": bson.M{"edb_code": edbCode}},
+		{"$group": bson.M{
+			"_id":       nil,
+			"min_date":  bson.M{"$min": "$data_time"},
+			"max_date":  bson.M{"$max": "$data_time"},
+			"min_value": bson.M{"$min": "$value"},
+			"max_value": bson.M{"$max": "$value"},
+		}},
+		{"$project": bson.M{"_id": 0}}, // 可选，如果不需要_id字段
+	}
+	result, err := mogDataObj.GetEdbInfoMaxAndMinInfo(pipeline)
+	if err != nil {
+		fmt.Println("EdbDataBusiness getEdbDataBusinessList Err:" + err.Error())
+		return
+	}
+
+	if !result.MaxDate.IsZero() {
+		whereQuery := bson.M{"edb_code": edbCode, "data_time": result.MaxDate}
+		selectParam := bson.D{{"value", 1}, {"_id", 0}}
+		latestValue, tmpErr := mogDataObj.GetLatestValue(whereQuery, selectParam)
+		if tmpErr != nil {
+			err = tmpErr
+			return
+		}
+		result.LatestValue = latestValue.Value
+		result.EndValue = latestValue.Value
+	}
+
+	item = &EdbInfoMaxAndMinInfo{
+		MinDate:     result.MinDate.Format(utils.FormatDate),
+		MaxDate:     result.MaxDate.Format(utils.FormatDate),
+		MinValue:    result.MinValue,
+		MaxValue:    result.MaxValue,
+		LatestValue: result.LatestValue,
+		LatestDate:  result.LatestDate.Format(utils.FormatDate),
+		EndValue:    result.EndValue,
+	}
+
+	return
+}
+
+// UnifiedModifyEdbInfoMaxAndMinInfo
+// @Description: 修改指标的最大最小值和最新值
+// @author: Roc
+// @receiver obj
+// @datetime 2024-04-30 17:07:35
+// @param edbInfo *EdbInfo
+// @return err error
+func (obj Business) UnifiedModifyEdbInfoMaxAndMinInfo(edbInfo *EdbInfo) (err error) {
+	if utils.UseMongo {
+		edbInfoMaxAndMinInfo, tmpErr := obj.getEdbInfoMaxAndMinInfoByMongo(edbInfo.EdbCode)
+		// 如果正常获取到了，那就去修改指标的最大最小值
+		if tmpErr == nil && edbInfoMaxAndMinInfo != nil {
+			err = ModifyEdbInfoMaxAndMinInfo(edbInfo.EdbInfoId, edbInfoMaxAndMinInfo)
+		} else {
+			// 清空的目的是为了避免异常返回
+			err = nil
+		}
+	} else {
+		err, _ = UnifiedModifyEdbInfoMaxAndMinInfo(edbInfo)
+	}
+
+	return
+}
+
+// EdbInfoMgoData
+// @Description: mgo里面的数据
+type EdbInfoMgoData struct {
+	//EdbDataId primitive.ObjectID `description:"数据ID"`
+	DataTime time.Time `description:"数据日期"`
+	Value    float64   `description:"数据"`
+	EdbCode  string    `description:"指标编码"`
+}
+
+// getBaseBusinessData
+// @Description: 获取基础数据
+// @author: Roc
+// @receiver obj
+// @datetime 2024-04-30 11:10:46
+// @param edbInfo *EdbInfo
+// @param startDate string
+// @return newDataList []EdbInfoSearchData
+// @return err error
+func (obj Business) getBaseBusinessData(edbInfo *EdbInfo, startDate string) (newDataList []EdbInfoMgoData, err error) {
+	return obj.getBaseBusinessDataByMongo(edbInfo, startDate)
+}
+
+// getBaseBusinessDataByMongo
+// @Description: 从mongo中获取基础的明细数据
+// @author: Roc
+// @receiver obj
+// @datetime 2024-07-02 10:12:02
+// @param edbInfo *EdbInfo
+// @param startDate string
+// @return newDataList []EdbInfoMgoData
+// @return err error
+func (obj Business) getBaseBusinessDataByMongo(edbInfo *EdbInfo, startDate string) (newDataList []EdbInfoMgoData, err error) {
+	newDataList = make([]EdbInfoMgoData, 0)
+
+	// 获取数据源的指标数据
+	mogDataObj := new(mgo.BaseFromBusinessData)
+
+	// 构建查询条件
+	queryConditions := bson.M{
+		"index_code": edbInfo.EdbCode,
+	}
+
+	if startDate != `` {
+		//获取已存在的所有数据
+		startDateTime, tmpErr := time.ParseInLocation(utils.FormatDate, startDate, time.Local)
+		if tmpErr != nil {
+			err = tmpErr
+			return
+		}
+		queryConditions["data_time"] = bson.M{"$gte": startDateTime}
+	}
+
+	baseDataList, err := mogDataObj.GetAllDataList(queryConditions, []string{"data_time"})
+	if err != nil {
+		fmt.Println("getBaseBusinessData Err:" + err.Error())
+		return
+	}
+
+	for _, v := range baseDataList {
+		newDataList = append(newDataList, EdbInfoMgoData{
+			//EdbDataId: v.ID,
+			DataTime: v.DataTime,
+			Value:    v.Value,
+			EdbCode:  v.IndexCode,
+		})
+	}
+
+	return
+}
+
+// getBaseBusinessDataByMysql
+// @Description: 从mysql中获取基础的明细数据
+// @author: Roc
+// @receiver obj
+// @datetime 2024-07-02 10:12:16
+// @param edbInfo *EdbInfo
+// @param startDate string
+// @return newDataList []EdbInfoMgoData
+// @return err error
+func (obj Business) getBaseBusinessDataByMysql(edbInfo *EdbInfo, startDate string) (newDataList []EdbInfoMgoData, err error) {
+	newDataList = make([]EdbInfoMgoData, 0)
+	// 获取数据源的指标数据
+	baseBusinessDataObj := new(BaseFromBusinessData)
+
+	// 构建查询条件
+	var condition []string
+	var pars []interface{}
+	condition = append(condition, "index_code = ? ")
+	pars = append(pars, edbInfo.EdbCode)
+
+	if startDate != `` {
+		condition = append(condition, " data_time >= ? ")
+		pars = append(pars, startDate)
+	}
+
+	baseDataList, err := baseBusinessDataObj.GetAllDataList(condition, pars, " data_time ASC ")
+	if err != nil {
+		fmt.Println("getBaseBusinessData Err:" + err.Error())
+		return
+	}
+
+	for _, v := range baseDataList {
+		newDataList = append(newDataList, EdbInfoMgoData{
+			//EdbDataId: v.BusinessDataId,
+			DataTime: v.DataTime,
+			Value:    v.Value,
+			EdbCode:  v.IndexCode,
+		})
+	}
+
+	return
+}

models/edb_data_calculate_avg.go

@@ -298,11 +298,9 @@ func (obj CalculateAvg) refresh(to orm.TxOrmer, edbInfoId, source, subSource int
 	}
 
 	for edbInfoIndex, v := range edbInfoIdArr {
-		var condition string
-		var pars []interface{}
-		condition += " AND edb_info_id=? "
-		pars = append(pars, v.EdbInfoId)
-		dataList, err := GetEdbDataListAllByTo(to, condition, pars, v.Source, v.SubSource, 1)
+		dataList, err := GetEdbDataListAllByTo(to, v.Source, v.SubSource, FindEdbDataListAllCond{
+			EdbInfoId: v.EdbInfoId,
+		}, 1)
 		if err != nil {
 			return err
 		}

models/edb_data_calculate_bp.go

@@ -205,22 +205,13 @@ func RefreshAllCalculateBpBak(edbInfoId, source, subSource int, fromEdbInfo *Edb
 	edbInfoIdStr := strconv.Itoa(edbInfoId)
 	//计算数据
 
-	//计算数据
-	var condition string
-	var pars []interface{}
-	condition += " AND edb_info_id=? "
-	pars = append(pars, fromEdbInfo.EdbInfoId)
-
-	if startDate != "" {
-		condition += " AND data_time>=? "
-		pars = append(pars, startDate)
-	}
-	if endDate != "" {
-		condition += " AND data_time<=? "
-		pars = append(pars, endDate)
-	}
-
-	dataList, err := GetEdbDataListAllByTo(to, condition, pars, fromEdbInfo.Source, fromEdbInfo.SubSource, 0)
+	dataList, err := GetEdbDataListAllByTo(to, fromEdbInfo.Source, fromEdbInfo.SubSource, FindEdbDataListAllCond{
+		EdbInfoId:         fromEdbInfo.EdbInfoId,
+		StartDataTime:     startDate,
+		StartDataTimeCond: ">=",
+		EndDataTime:       endDate,
+		EndDataTimeCond:   "<=",
+	}, 0)
 	if err != nil {
 		return err
 	}
@@ -348,197 +339,116 @@ func RefreshAllCalculateBp(edbInfoId, source, subSource int, fromEdbInfo *EdbInf
 // refreshAllCalculateBp 刷新升频数据
 func refreshAllCalculateBp(to orm.TxOrmer, edbInfoId, source, subSource int, fromEdbInfo *EdbInfo, edbCode, startDate, endDate string, order int) (err error) {
 	edbInfoIdStr := strconv.Itoa(edbInfoId)
-
 	//计算数据
-	var condition string
-	var pars []interface{}
-	condition += " AND edb_info_id=? "
-	pars = append(pars, fromEdbInfo.EdbInfoId)
-
-	//if startDate != "" {
-	//	condition += " AND data_time>=? "
-	//	pars = append(pars, startDate)
-	//}
-	//if endDate != "" {
-	//	condition += " AND data_time<=? "
-	//	pars = append(pars, endDate)
-	//}
+
 	//获取来源指标的数据
-	dataList, err := GetEdbDataListAllByTo(to, condition, pars, fromEdbInfo.Source, fromEdbInfo.SubSource, order)
+	dataList, err := GetEdbDataListAllByTo(to, fromEdbInfo.Source, fromEdbInfo.SubSource, FindEdbDataListAllCond{
+		EdbInfoId: fromEdbInfo.EdbInfoId,
+	}, order)
 	if err != nil {
 		return err
 	}
-	var dateArr []string
-	dataMap := make(map[string]*EdbInfoSearchData)
-	fromDataMap := make(map[string]float64)
-	//来源指指标数据
-	for _, v := range dataList {
-		dateArr = append(dateArr, v.DataTime)
-		dataMap[v.DataTime] = v
-		fromDataMap[v.DataTime] = v.Value
+
+	// 来源指标没有数据，那么需要删除所有的计算指标数据
+	if len(dataList) <= 0 {
+		// todo 删除所有的计算指标数据
+		return
 	}
-	fmt.Println("source:", source)
+	// 来源指标的第一个日期
+	fromFirstDate, err := time.ParseInLocation(utils.FormatDate, dataList[0].DataTime, time.Local)
+	if err != nil {
+		return
+	}
+	fromFirstDate = time.Date(fromFirstDate.Year(), fromFirstDate.Month(), fromFirstDate.Day(), 0, 0, 0, 0, time.Local)
 
-	//获取升频指标所有数据
+	// 变频计算
+	newDataList, err := EdbInfoSearchDataToData(dataList)
+	if err != nil {
+		return
+	}
+
+	baseCalculate := BaseCalculate{
+		DataList:      newDataList,
+		Frequency:     "",
+		Formula:       nil,
+		Calendar:      "",
+		MoveType:      0,
+		MoveFrequency: "",
+		FromFrequency: "",
+		Source:        source,
+	}
+	dateDataMap, err, _ := baseCalculate.UpFrequency()
+	if err != nil {
+		return
+	}
+
+	// 获取升频指所有已经存在的计算指标数据
 	existDataList, err := GetAllEdbDataListByTo(to, edbInfoId, source, subSource)
 	if err != nil {
 		return
 	}
 	//计算指标的map
 	existDataMap := make(map[string]*EdbData, 0)
+	for _, v := range existDataList {
+		existDataMap[v.DataTime] = v
+	}
 
 	addSql := ` INSERT INTO edb_data_calculate_bp(edb_info_id,edb_code,data_time,value,create_time,modify_time,data_timestamp) values `
 	var isAdd bool
 
-	var lastValue float64   //最后数据的值(float64)
-	var lastValueStr string //最后数据的值(string)
-	//待删除的日期
-	removeDateList := make([]string, 0)
-	if len(existDataList) > 0 {
-		//第一个已经入库的日期
-		firstExistDataTimeStr := existDataList[0].DataTime //计算指标数据第一条的日期字符串
-		if len(dateArr) > 0 {
-			firstFromDataTimeStr := dateArr[0]                                                                 //来源数据第一条的日期字符串
-			firstExistDataTime, _ := time.ParseInLocation(utils.FormatDate, firstExistDataTimeStr, time.Local) //计算指标数据第一条的日期（time类型）
-			firstFromDataTime, _ := time.ParseInLocation(utils.FormatDate, firstFromDataTimeStr, time.Local)   //来源数据第一条的日期（time类型）
-			nowDateStr := time.Now().Format(utils.FormatDate)                                                  //当天日期字符串
-			nowDate, _ := time.ParseInLocation(utils.FormatDate, nowDateStr, firstFromDataTime.Location())     //当天日期（time类型）
-
-			lastValue = fromDataMap[firstFromDataTimeStr]
-			lastValueStr = decimal.NewFromFloat(lastValue).String()
-			//第一步： 判断来源指标的开始时间与计算指标的开始时间是否相等，相等的话，那么就不需要对两个时间之间的数据做处理
-			if firstExistDataTimeStr != firstFromDataTimeStr {
-				if firstExistDataTime.Before(firstFromDataTime) { //如果计算指标第一条数据的开始时间 早于 来源指标的第一条开始时间，那么需要对两个时间之间的计算指标数据做 删除处理
-					for _, v := range existDataList {
-						if v.DataTime == firstFromDataTimeStr {
-							if tmpLastValue, ok := fromDataMap[firstFromDataTimeStr]; ok { //来源指标当天的数据
-								lastValue = tmpLastValue
-								lastValueStr = decimal.NewFromFloat(lastValue).String()
-							}
-							break
-						}
-						removeDateList = append(removeDateList, v.DataTime)
-					}
-				} else {
-					for _, v := range dateArr { //如果计算指标第一条数据的开始时间 晚于 来源指标的第一条开始时间，那么需要对两个时间之间的计算指标数据做 新增处理
-						vDataTime, _ := time.ParseInLocation(utils.FormatDate, v, time.Local) //当前日期（time类型）
-						if firstExistDataTime.Equal(vDataTime) || firstExistDataTime.Before(vDataTime) {
-							if tmpLastValue, ok := fromDataMap[v]; ok { //来源指标当天的数据
-								lastValue = tmpLastValue
-								lastValueStr = decimal.NewFromFloat(lastValue).String()
-							}
-							break
-						}
-
-						currentDate, _ := time.ParseInLocation(utils.FormatDate, v, time.Local)
-						timestamp := currentDate.UnixNano() / 1e6
-						timestampStr := fmt.Sprintf("%d", timestamp)
-						addSql += GetAddSql(edbInfoIdStr, edbCode, v, timestampStr, lastValueStr)
-
-						isAdd = true
-					}
-				}
-			}
-
-			//第二步 剩余数据每天修改
+	now := time.Date(time.Now().Year(), time.Now().Month(), time.Now().Day(), 0, 0, 0, 0, time.Local)
 
-			day := int(nowDate.Sub(firstExistDataTime).Hours() / float64(24))
+	for currDate := fromFirstDate; !currDate.After(now); currDate = currDate.AddDate(0, 0, 1) {
+		currDateStr := currDate.Format(utils.FormatDate)
+		timestamp := currDate.UnixNano() / 1e6
+		timestampStr := fmt.Sprintf("%d", timestamp)
 
-			//第三步： 已经入库的数据处理
-			for _, v := range existDataList {
-				existDataMap[v.DataTime] = v
-			}
+		// 当前计算的值
+		currValue, ok := dateDataMap[currDate]
+		if !ok {
+			// 没有计算成功就过滤
+			continue
+		}
+		lastValueStr := decimal.NewFromFloat(currValue).Round(4).String()
+
+		// 已经入库的值
+		existData, ok := existDataMap[currDateStr]
+		if !ok {
+			// 没有入库那么就插入添加
+			isAdd = true
+			addSql += GetAddSql(edbInfoIdStr, edbCode, currDateStr, timestampStr, lastValueStr)
+			continue
+		}
 
-			for k := day; k >= 0; k-- {
-				needDay := nowDate.AddDate(0, 0, -k)
-				needDayStr := needDay.Format(utils.FormatDate)
-				tmpExistData, ok := existDataMap[needDayStr]
-				if ok {
-					if tmpLastValue, ok := fromDataMap[tmpExistData.DataTime]; ok { //来源指标当天的数据
-						lastValue = tmpLastValue
-						//lastValueStr = decimal.NewFromFloat(lastValue).String()
-						lastValueStr = fmt.Sprintf("%.4f", lastValue)
-					}
-					//如果对应的值不匹配
-					if tmpExistData.Value != lastValueStr {
-						err = ModifyEdbDataById(source, subSource, tmpExistData.EdbDataId, lastValueStr)
-						if err != nil {
-							return err
-						}
-					}
-				} else {
-					timestamp := needDay.UnixNano() / 1e6
-					timestampStr := fmt.Sprintf("%d", timestamp)
-					addSql += GetAddSql(edbInfoIdStr, edbCode, needDayStr, timestampStr, lastValueStr)
+		// 将已经入库的值转换为decimal类型，然后再保留4位小数，目的是为了做匹配，要不然取出来的数据与计算的数据不一致
+		existDataValueDec, tmpErr := decimal.NewFromString(existData.Value)
+		if tmpErr != nil {
+			err = tmpErr
+			return
+		}
+		existDataValueStr := existDataValueDec.Round(4).String()
 
-					isAdd = true
-				}
-			}
-		} else {
-			//如果没有来源指标数据，那么已经入库的计算指标数据需要全部删除
-			tableName := GetEdbDataTableName(source, subSource)
-			sql := fmt.Sprintf(` DELETE FROM %s WHERE edb_info_id = ?`, tableName)
-			_, err = to.Raw(sql, edbInfoId).Exec()
+		// 如果该日期已经入库了，且两个值不匹配，那么就更新
+		if lastValueStr != existDataValueStr {
+			err = ModifyEdbDataById(source, subSource, existData.EdbDataId, lastValueStr)
 			if err != nil {
-				err = fmt.Errorf("删除所有的升频指标数据失败,Err:" + err.Error())
-				return
+				return err
 			}
-
-			//for _, v := range existDataList {
-			//	removeDateList = append(removeDateList, v.DataTime)
-			//}
 		}
-	} else {
-		existMap := make(map[string]string)
-		dataLen := len(dataList)
 
-		for i := 0; i < dataLen; i++ {
-			//当期
-			currentItem := dataList[i]
-			currentDate, _ := time.ParseInLocation(utils.FormatDate, currentItem.DataTime, time.Local)
-			var day int
-			var preItem *EdbInfoSearchData
-			var preDate time.Time
-			if i == 0 {
-				day = int(time.Now().Sub(currentDate).Hours() / float64(24))
-				preDate = time.Now()
-			} else {
-				j := i - 1
-				if j < dataLen {
-					preItem = dataList[j]
-					preDate, _ = time.ParseInLocation(utils.FormatDate, preItem.DataTime, time.Local)
-					day = int(preDate.Sub(currentDate).Hours() / float64(24))
-					utils.FileLog.Info("preItem.DataTime:" + preItem.DataTime + ";currentItem.DataTime" + currentItem.DataTime)
-				}
-			}
-			for k := 0; k <= day; k++ {
-				needDay := preDate.AddDate(0, 0, -k)
-				needDayStr := needDay.Format(utils.FormatDate)
-				existKey := edbCode + needDayStr
-				if _, ok := existMap[existKey]; !ok {
-					timestamp := needDay.UnixNano() / 1e6
-					timestampStr := fmt.Sprintf("%d", timestamp)
-					valStr := decimal.NewFromFloat(currentItem.Value).String()
-					addSql += GetAddSql(edbInfoIdStr, edbCode, needDayStr, timestampStr, valStr)
-					isAdd = true
-				}
-				existMap[existKey] = needDayStr
-			}
-			existKey := edbCode + currentItem.DataTime
-			if _, ok := existMap[existKey]; !ok {
-				currentDate, _ := time.ParseInLocation(utils.FormatDate, currentItem.DataTime, time.Local)
-				timestamp := currentDate.UnixNano() / 1e6
-				timestampStr := fmt.Sprintf("%d", timestamp)
-				valStr := decimal.NewFromFloat(currentItem.Value).String()
-				addSql += GetAddSql(edbInfoIdStr, edbCode, currentItem.DataTime, timestampStr, valStr)
-				isAdd = true
-			}
-			existMap[existKey] = currentItem.DataTime
-		}
+		// 该日期已经处理过了，所以需要移除，如果后面该map还有数据，那么需要删除该map里面的日期数据
+		delete(existDataMap, currDateStr)
+
 	}
 
 	// 删除不需要的指标数据
-	if len(removeDateList) > 0 {
+	if len(existDataMap) > 0 {
+		//待删除的日期
+		removeDateList := make([]string, 0)
+		for date := range existDataMap {
+			removeDateList = append(removeDateList, date)
+		}
+
 		removeDateStr := strings.Join(removeDateList, `","`)
 		removeDateStr = `"` + removeDateStr + `"`
 		//如果拼接指标变更了，那么需要删除所有的指标数据
@@ -552,9 +462,11 @@ func refreshAllCalculateBp(to orm.TxOrmer, edbInfoId, source, subSource int, fro
 		}
 	}
 
+	// 新增的数据值
 	if isAdd {
 		addSql = strings.TrimRight(addSql, ",")
 		_, err = to.Raw(addSql).Exec()
 	}
+
 	return
 }

models/edb_data_calculate_cjjx.go

@@ -232,22 +232,9 @@ func refreshAllCalculateCjjx(to orm.TxOrmer, edbInfoId, source, subSource int, f
 	edbInfoIdStr := strconv.Itoa(edbInfoId)
 	//计算数据
 
-	var condition string
-	var pars []interface{}
-	condition += " AND edb_info_id=? "
-	pars = append(pars, fromEdbInfo.EdbInfoId)
-
-	//if startDate != "" {
-	//	condition += " AND data_time>=? "
-	//	pars = append(pars, startDate)
-	//}
-
-	//if endDate != "" {
-	//	condition += " AND data_time<=? "
-	//	pars = append(pars, endDate)
-	//}
-
-	dataList, err := GetEdbDataListAllByTo(to, condition, pars, fromEdbInfo.Source, fromEdbInfo.SubSource, 1)
+	dataList, err := GetEdbDataListAllByTo(to, fromEdbInfo.Source, fromEdbInfo.SubSource, FindEdbDataListAllCond{
+		EdbInfoId: fromEdbInfo.EdbInfoId,
+	}, 1)
 	if err != nil {
 		return err
 	}

models/edb_data_calculate_correlation.go

@@ -396,13 +396,10 @@ func GetRollingCorrelationChartDataByEdbInfo(edbInfoMappingA, edbInfoMappingB *E
 	aDataList := make([]*EdbInfoSearchData, 0)
 	switch baseEdbInfo.EdbInfoType {
 	case 0:
-		var condition string
-		var pars []interface{}
-		condition += " AND edb_info_id=? "
-		pars = append(pars, baseEdbInfo.EdbInfoId)
-
 		//获取来源指标的数据
-		aDataList, err = GetEdbDataListAll(condition, pars, baseEdbInfo.Source, baseEdbInfo.SubSource, 1)
+		aDataList, err = GetEdbDataListAll(baseEdbInfo.Source, baseEdbInfo.SubSource, FindEdbDataListAllCond{
+			EdbInfoId: baseEdbInfo.EdbInfoId,
+		}, 1)
 	case 1:
 		aDataList, err = GetPredictEdbDataListAllByStartDate(baseEdbInfo, 1, "")
 	default:
@@ -414,13 +411,10 @@ func GetRollingCorrelationChartDataByEdbInfo(edbInfoMappingA, edbInfoMappingB *E
 	bDataList := make([]*EdbInfoSearchData, 0)
 	switch changeEdbInfo.EdbInfoType {
 	case 0:
-		var condition string
-		var pars []interface{}
-		condition += " AND edb_info_id=? "
-		pars = append(pars, changeEdbInfo.EdbInfoId)
-
 		//获取来源指标的数据
-		bDataList, err = GetEdbDataListAll(condition, pars, changeEdbInfo.Source, changeEdbInfo.SubSource, 1)
+		bDataList, err = GetEdbDataListAll(changeEdbInfo.Source, changeEdbInfo.SubSource, FindEdbDataListAllCond{
+			EdbInfoId: changeEdbInfo.EdbInfoId,
+		}, 1)
 	case 1:
 		bDataList, err = GetPredictEdbDataListAllByStartDate(changeEdbInfo, 1, "")
 	default:

models/edb_data_calculate_hbz.go

@@ -209,22 +209,10 @@ func RefreshAllCalculateHbz(edbInfoId, source, subSource int, fromEdbInfo *EdbIn
 func refreshAllCalculateHbz(to orm.TxOrmer, edbInfoId, source, subSource int, fromEdbInfo *EdbInfo, edbCode, startDate, endDate string, formulaInt int) (err error) {
 	edbInfoIdStr := strconv.Itoa(edbInfoId)
 	//计算数据
-	var condition string
-	var pars []interface{}
-	condition += " AND edb_info_id=? "
-	pars = append(pars, fromEdbInfo.EdbInfoId)
 
-	//if startDate != "" {
-	//	condition += " AND data_time>=? "
-	//	pars = append(pars, startDate)
-	//}
-
-	//if endDate != "" {
-	//	condition += " AND data_time<=? "
-	//	pars = append(pars, endDate)
-	//}
-
-	dataList, err := GetEdbDataListAllByTo(to, condition, pars, fromEdbInfo.Source, fromEdbInfo.SubSource, 0)
+	dataList, err := GetEdbDataListAllByTo(to, fromEdbInfo.Source, fromEdbInfo.SubSource, FindEdbDataListAllCond{
+		EdbInfoId: fromEdbInfo.EdbInfoId,
+	}, 0)
 	if err != nil {
 		return err
 	}

models/edb_data_calculate_hcz.go

@@ -211,22 +211,9 @@ func refreshAllCalculateHcz(to orm.TxOrmer, edbInfoId, source, subSource int, fr
 	edbInfoIdStr := strconv.Itoa(edbInfoId)
 
 	//计算数据
-	var condition string
-	var pars []interface{}
-	condition += " AND edb_info_id=? "
-	pars = append(pars, fromEdbInfo.EdbInfoId)
-
-	//if startDate != "" {
-	//	condition += " AND data_time>=? "
-	//	pars = append(pars, startDate)
-	//}
-
-	//if endDate != "" {
-	//	condition += " AND data_time<=? "
-	//	pars = append(pars, endDate)
-	//}
-
-	dataList, err := GetEdbDataListAllByTo(to, condition, pars, fromEdbInfo.Source, fromEdbInfo.SubSource, 0)
+	dataList, err := GetEdbDataListAllByTo(to, fromEdbInfo.Source, fromEdbInfo.SubSource, FindEdbDataListAllCond{
+		EdbInfoId: fromEdbInfo.EdbInfoId,
+	}, 0)
 	if err != nil {
 		return err
 	}

models/edb_data_calculate_jp.go

@@ -238,21 +238,10 @@ func refreshAllCalculateJp(to orm.TxOrmer, edbInfoId, source, subSource int, fro
 	edbInfoIdStr := strconv.Itoa(edbInfoId)
 
 	//计算数据
-	var condition string
-	var pars []interface{}
-	condition += " AND edb_info_id=? "
-	pars = append(pars, fromEdbInfo.EdbInfoId)
-
-	//if startDate != "" {
-	//	condition += " AND data_time>=? "
-	//	pars = append(pars, startDate)
-	//}
-	//if endDate != "" {
-	//	condition += " AND data_time<=? "
-	//	pars = append(pars, endDate)
-	//}
 	//获取来源指标的数据
-	dataList, err := GetEdbDataListAllByTo(to, condition, pars, fromEdbInfo.Source, fromEdbInfo.SubSource, 1)
+	dataList, err := GetEdbDataListAllByTo(to, fromEdbInfo.Source, fromEdbInfo.SubSource, FindEdbDataListAllCond{
+		EdbInfoId: fromEdbInfo.EdbInfoId,
+	}, 1)
 	if err != nil {
 		return err
 	}
@@ -465,12 +454,16 @@ func refreshAllCalculateJp(to orm.TxOrmer, edbInfoId, source, subSource int, fro
 					return err
 				}
 			}
+			// 移除待删除的日期
+			delete(existDelDateMap, nextEndDate.Format(utils.FormatDate))
 		} else {
 			// 直接入库
 			timestamp := nextEndDate.UnixNano() / 1e6
 			timestampStr := fmt.Sprintf("%d", timestamp)
 			addSql += GetAddSql(edbInfoIdStr, edbCode, nextEndDate.Format(utils.FormatDate), timestampStr, fmt.Sprint(currVal))
 			isAdd = true
+			// 移除待删除的日期
+			delete(existDelDateMap, nextEndDate.Format(utils.FormatDate))
 		}
 	}
 

models/edb_data_calculate_kszs.go

@@ -269,11 +269,9 @@ func refreshAllCalculateKszs(to orm.TxOrmer, edbInfoId, source, subSource int, r
 	// 获取选择指标的 需要数据的 开始日期和结束日期
 	var startDate, endDate time.Time
 	for _, v := range relationEdbInfoList {
-		var condition string
-		var pars []interface{}
-		condition += " AND edb_info_id=? "
-		pars = append(pars, v.EdbInfoId)
-		tmpDataList, tmpErr := GetEdbDataListAllByTo(to, condition, pars, v.Source, v.SubSource, 1)
+		tmpDataList, tmpErr := GetEdbDataListAllByTo(to, v.Source, v.SubSource, FindEdbDataListAllCond{
+			EdbInfoId: v.EdbInfoId,
+		}, 1)
 		if tmpErr != nil {
 			err = tmpErr
 			return err

models/edb_data_calculate_ljz.go

@@ -290,15 +290,6 @@ func (obj Ljz) refresh(to orm.TxOrmer, edbInfoId, source, subSource int, edbInfo
 	dataTableName := GetEdbDataTableName(source, subSource)
 	edbInfoIdStr := strconv.Itoa(edbInfoId)
 	//计算数据
-	var condition string
-	var pars []interface{}
-	condition += " AND edb_info_id=? "
-	pars = append(pars, fromEdbInfo.EdbInfoId)
-
-	//if startDate != "" {
-	//	condition += " AND data_time>=? "
-	//	pars = append(pars, startDate)
-	//}
 
 	var isWeekData bool // 是否周度数据，如果是周度数据的话，是需要变频的，最后结果还需要除以7
 	// 周度数据需要先变成日度的
@@ -306,7 +297,9 @@ func (obj Ljz) refresh(to orm.TxOrmer, edbInfoId, source, subSource int, edbInfo
 		isWeekData = true
 	}
 
-	dataList, err := GetEdbDataListAllByTo(to, condition, pars, fromEdbInfo.Source, fromEdbInfo.SubSource, 1)
+	dataList, err := GetEdbDataListAllByTo(to, fromEdbInfo.Source, fromEdbInfo.SubSource, FindEdbDataListAllCond{
+		EdbInfoId: fromEdbInfo.EdbInfoId,
+	}, 1)
 	if err != nil {
 		return err
 	}

models/edb_data_calculate_ljznczj.go

@@ -287,15 +287,6 @@ func (obj LjzNczj) refresh(to orm.TxOrmer, edbInfoId, source, subSource int, edb
 	dataTableName := GetEdbDataTableName(source, subSource)
 	edbInfoIdStr := strconv.Itoa(edbInfoId)
 	//计算数据
-	var condition string
-	var pars []interface{}
-	condition += " AND edb_info_id=? "
-	pars = append(pars, fromEdbInfo.EdbInfoId)
-
-	//if startDate != "" {
-	//	condition += " AND data_time>=? "
-	//	pars = append(pars, startDate)
-	//}
 
 	var isWeekData bool // 是否周度数据，如果是周度数据的话，是需要变频的，最后结果还需要除以7
 	// 周度数据需要先变成日度的
@@ -303,7 +294,9 @@ func (obj LjzNczj) refresh(to orm.TxOrmer, edbInfoId, source, subSource int, edb
 		isWeekData = true
 	}
 
-	dataList, err := GetEdbDataListAllByTo(to, condition, pars, fromEdbInfo.Source, fromEdbInfo.SubSource, 1)
+	dataList, err := GetEdbDataListAllByTo(to, fromEdbInfo.Source, fromEdbInfo.SubSource, FindEdbDataListAllCond{
+		EdbInfoId: fromEdbInfo.EdbInfoId,
+	}, 1)
 	if err != nil {
 		return err
 	}

models/edb_data_calculate_ljztbpj.go

@@ -64,12 +64,11 @@ func AddCalculateLjztbpj(req *EdbInfoCalculateBatchSaveReq, firstEdbInfo, second
 		lastDateTime, _ = time.ParseInLocation(utils.FormatDate, lastEdbData.DataTime, time.Local)
 
 		//获取待拼接指标的数据列表
-		condition = ``
-		pars = make([]interface{}, 0)
-		condition += " AND data_time <= ? AND edb_info_id=? "
-		pars = append(pars, lastEdbData.DataTime, firstEdbInfo.EdbInfoId)
-
-		firstDataList, tmpErr := GetEdbDataListAllByTo(to, condition, pars, firstEdbInfo.Source, firstEdbInfo.SubSource, 0)
+		firstDataList, tmpErr := GetEdbDataListAllByTo(to, firstEdbInfo.Source, firstEdbInfo.SubSource, FindEdbDataListAllCond{
+			EdbInfoId:       firstEdbInfo.EdbInfoId,
+			EndDataTime:     lastEdbData.DataTime,
+			EndDataTimeCond: "<=",
+		}, 0)
 		if tmpErr != nil {
 			err = tmpErr
 			return
@@ -475,14 +474,12 @@ func refreshAllCalculateLjztbpj(to orm.TxOrmer, edbInfo *EdbInfo, existItemA, ex
 
 	//待拼接指标
 	{
-		var condition string
-		var pars []interface{}
-
-		condition += " AND data_time <= ? AND edb_info_id=? "
-		pars = append(pars, startCalculationDate, existItemA.FromEdbInfoId)
-
 		//第一个指标的数据列表
-		firstDataList, tmpErr := GetEdbDataListAllByTo(to, condition, pars, existItemA.FromSource, existItemA.FromSubSource, 0)
+		firstDataList, tmpErr := GetEdbDataListAllByTo(to, existItemA.FromSource, existItemA.FromSubSource, FindEdbDataListAllCond{
+			EdbInfoId:       existItemA.FromEdbInfoId,
+			EndDataTime:     startCalculationDate.Format(utils.FormatDate),
+			EndDataTimeCond: "<=",
+		}, 0)
 		if tmpErr != nil {
 			return tmpErr
 		}
@@ -513,14 +510,12 @@ func refreshAllCalculateLjztbpj(to orm.TxOrmer, edbInfo *EdbInfo, existItemA, ex
 
 	//同比值指标
 	{
-		var condition string
-		var pars []interface{}
-
-		condition += " AND data_time > ? AND edb_info_id = ? "
-		pars = append(pars, startCalculationDate, existItemB.FromEdbInfoId)
-
 		//第二个指标的数据列表
-		secondDataList, tmpErr := GetEdbDataListAllByTo(to, condition, pars, existItemB.FromSource, existItemB.FromSubSource, 0)
+		secondDataList, tmpErr := GetEdbDataListAllByTo(to, existItemB.FromSource, existItemB.FromSubSource, FindEdbDataListAllCond{
+			EdbInfoId:         existItemB.FromEdbInfoId,
+			StartDataTime:     startCalculationDate.Format(utils.FormatDate),
+			StartDataTimeCond: ">",
+		}, 0)
 		if tmpErr != nil {
 			return tmpErr
 		}

models/edb_data_calculate_ljzzj.go

@@ -247,11 +247,10 @@ func (obj Ljzzj) refresh(to orm.TxOrmer, edbInfoId, source, subSource int, fromE
 	edbInfoIdStr := strconv.Itoa(edbInfoId)
 
 	// 获取来源指标的数据
-	var condition string
-	var pars []interface{}
-	condition += " AND edb_info_id=? "
-	pars = append(pars, fromEdbInfo.EdbInfoId)
-	dataList, err := GetEdbDataListAllByTo(to, condition, pars, fromEdbInfo.Source, fromEdbInfo.SubSource, 1)
+	dataList, err := GetEdbDataListAllByTo(to, fromEdbInfo.Source, fromEdbInfo.SubSource,
+		FindEdbDataListAllCond{
+			EdbInfoId: fromEdbInfo.EdbInfoId,
+		}, 1)
 	if err != nil {
 		return err
 	}

models/edb_data_calculate_ljzzy.go

@@ -212,21 +212,13 @@ func RefreshAllCalculateLjzzy(edbInfoId, source, subSource int, fromEdbInfo *Edb
 func refreshAllCalculateLjzzy(to orm.TxOrmer, edbInfoId, source, subSource int, fromEdbInfo *EdbInfo, edbCode, startDate, endDate string) (err error) {
 	edbInfoIdStr := strconv.Itoa(edbInfoId)
 	//计算数据
-	var condition string
-	var pars []interface{}
-	condition += " AND edb_info_id=? "
-	pars = append(pars, fromEdbInfo.EdbInfoId)
 
-	if startDate != "" {
-		condition += " AND data_time>=? "
-		pars = append(pars, startDate)
-	}
-	//if endDate != "" {
-	//	condition += " AND data_time<=? "
-	//	pars = append(pars, endDate)
-	//}
-
-	dataList, err := GetEdbDataListAllByTo(to, condition, pars, fromEdbInfo.Source, fromEdbInfo.SubSource, 1)
+	dataList, err := GetEdbDataListAllByTo(to, fromEdbInfo.Source, fromEdbInfo.SubSource,
+		FindEdbDataListAllCond{
+			EdbInfoId:         fromEdbInfo.EdbInfoId,
+			StartDataTime:     startDate,
+			StartDataTimeCond: ">=",
+		}, 1)
 	if err != nil {
 		return err
 	}

models/edb_data_calculate_nh.go

@@ -225,13 +225,11 @@ func refreshAllCalculateNh(to orm.TxOrmer, edbInfoId, source, subSource int, fro
 	}
 
 	//计算数据
-	var condition string
-	var pars []interface{}
-	condition += " AND edb_info_id=? "
-	pars = append(pars, fromEdbInfo.EdbInfoId)
 
 	//获取来源指标的数据
-	fromDataList, err := GetEdbDataListAllByTo(to, condition, pars, fromEdbInfo.Source, fromEdbInfo.SubSource, 1)
+	fromDataList, err := GetEdbDataListAllByTo(to, fromEdbInfo.Source, fromEdbInfo.SubSource, FindEdbDataListAllCond{
+		EdbInfoId: fromEdbInfo.EdbInfoId,
+	}, 1)
 	if err != nil {
 		return err
 	}

models/edb_data_calculate_nhcc.go

@@ -751,14 +751,11 @@ func getNhccData(to orm.TxOrmer, existItemA, existItemB *EdbInfoCalculateMapping
 	aDataList = make([]EdbInfoSearchData, 0)
 	aDataMap = make(map[string]float64)
 	{
-		var condition string
-		var pars []interface{}
-
-		condition += " AND edb_info_id=? "
-		pars = append(pars, existItemA.FromEdbInfoId)
 
 		//第一个指标的数据列表
-		firstDataList, tmpErr := GetEdbDataListAllByTo(to, condition, pars, existItemA.FromSource, existItemA.FromSubSource, 0)
+		firstDataList, tmpErr := GetEdbDataListAllByTo(to, existItemA.FromSource, existItemA.FromSubSource, FindEdbDataListAllCond{
+			EdbInfoId: existItemA.FromEdbInfoId,
+		}, 0)
 		if tmpErr != nil {
 			err = tmpErr
 			return
@@ -772,14 +769,11 @@ func getNhccData(to orm.TxOrmer, existItemA, existItemB *EdbInfoCalculateMapping
 	secondDataList = make([]*EdbInfoSearchData, 0)
 	bDataMap = make(map[string]float64)
 	{
-		condition := ``
-		pars := make([]interface{}, 0)
-
-		condition += "  AND edb_info_id = ? "
-		pars = append(pars, existItemB.FromEdbInfoId)
 
 		//第二个指标的数据列表
-		secondDataList, err = GetEdbDataListAllByTo(to, condition, pars, existItemB.FromSource, existItemB.FromSubSource, 0)
+		secondDataList, err = GetEdbDataListAllByTo(to, existItemB.FromSource, existItemB.FromSubSource, FindEdbDataListAllCond{
+			EdbInfoId: existItemB.FromEdbInfoId,
+		}, 0)
 		if err != nil {
 			return
 		}

models/edb_data_calculate_nszydbpjjs.go

@@ -225,25 +225,19 @@ func refreshAllCalculateNszydpjjs(to orm.TxOrmer, edbInfoId, source, subSource,
 		removeDataTimeMap[v.DataTime] = 1
 	}
 
-	//计算来源数据
-	var fromCondition string
-	var fromPars []interface{}
-	fromCondition += " AND edb_info_id=? "
-	fromPars = append(fromPars, fromEdbInfo.EdbInfoId)
-	//if startDate != `` {
-	//	fromCondition += " AND data_time>=? "
-	//	fromPars = append(fromPars, startDate)
-	//}
-
-	//fmt.Println("fromPars:", fromPars)
-	fromDataList, err := GetEdbDataListAllV1ByTo(to, fromCondition, fromPars, fromEdbInfo.Source, fromEdbInfo.SubSource, 0)
+	// 获取来源数据
+	fromDataList, err := GetEdbDataListAll(fromEdbInfo.Source, fromEdbInfo.SubSource, FindEdbDataListAllCond{
+		EdbInfoId: fromEdbInfo.EdbInfoId,
+	}, 0)
 	if err != nil {
 		fmt.Println("from GetEdbDataListAll Err:" + err.Error())
 		return err
 	}
+	//fmt.Println("fromPars:", fromPars)
+	//fromDataList, err := GetEdbDataListAllV1ByTo(to, fromCondition, fromPars, fromEdbInfo.Source, fromEdbInfo.SubSource, 0)
 
 	var fromDateArr []string
-	fromDataMap := make(map[string]*EdbInfoSearchDataV1)
+	fromDataMap := make(map[string]*EdbInfoSearchData)
 	for _, v := range fromDataList {
 		fromDateArr = append(fromDateArr, v.DataTime)
 		fromDataMap[v.DataTime] = v
@@ -256,7 +250,7 @@ func refreshAllCalculateNszydpjjs(to orm.TxOrmer, edbInfoId, source, subSource,
 	existAddDataMap := make(map[string]string)
 	for ak, av := range fromDateArr {
 		//处理第一个值
-		var valArr []string
+		var valArr []float64
 		if findItem, ok := fromDataMap[av]; ok {
 			valArr = append(valArr, findItem.Value)
 		} else {
@@ -281,7 +275,7 @@ func refreshAllCalculateNszydpjjs(to orm.TxOrmer, edbInfoId, source, subSource,
 		//var totalVal float64
 		totalVal := decimal.NewFromFloat(0.00)
 		for _, v := range valArr {
-			newDecimal, _ := decimal.NewFromString(v)
+			newDecimal := decimal.NewFromFloat(v)
 			totalVal = totalVal.Add(newDecimal)
 		}
 		af := totalVal //decimal.NewFromFloat(totalVal)

models/edb_data_calculate_percentile.go

@@ -244,6 +244,7 @@ func (obj Percentile) GetEdbType() int {
 type PercentileConfig struct {
 	CalculateValue int    `description:"时间长度期数"`
 	CalculateUnit  string `description:"时间长度频度"`
+	PercentType    int    `description:"百分位：0-数据区间(兼容历史数据); 1-数据个数；"`
 }
 
 // refresh 刷新
@@ -259,7 +260,7 @@ func (obj Percentile) refresh(to orm.TxOrmer, edbInfo, fromEdbInfo *EdbInfo, edb
 	}
 
 	// 获取百分位的指标数据
-	fromDataList, err, errMsg := obj.getPercentileData(fromEdbInfo, percentileConfig.CalculateValue, percentileConfig.CalculateUnit)
+	fromDataList, err, errMsg := obj.getPercentileData(fromEdbInfo, percentileConfig.CalculateValue, percentileConfig.CalculateUnit, percentileConfig.PercentType)
 	if err != nil {
 		return
 	}
@@ -374,18 +375,15 @@ func (obj Percentile) calculate(edbInfoId int, date, edbInfoIdStr, edbCode, data
 }
 
 // GetPercentileData 获取百分位图表的指标数据
-func (obj Percentile) getPercentileData(fromEdbInfo *EdbInfo, calculateValue int, calculateUnit string) (newDataList []EdbInfoSearchData, err error, errMsg string) {
+func (obj Percentile) getPercentileData(fromEdbInfo *EdbInfo, calculateValue int, calculateUnit string, percentType int) (newDataList []EdbInfoSearchData, err error, errMsg string) {
 	// 获取时间基准指标在时间区间内的值
 	dataList := make([]*EdbInfoSearchData, 0)
 	switch fromEdbInfo.EdbInfoType {
 	case 0:
-		var condition string
-		var pars []interface{}
-		condition += " AND edb_info_id=? "
-		pars = append(pars, fromEdbInfo.EdbInfoId)
-
 		//获取来源指标的数据
-		dataList, err = GetEdbDataListAll(condition, pars, fromEdbInfo.Source, fromEdbInfo.SubSource, 1)
+		dataList, err = GetEdbDataListAll(fromEdbInfo.Source, fromEdbInfo.SubSource, FindEdbDataListAllCond{
+			EdbInfoId: fromEdbInfo.EdbInfoId,
+		}, 1)
 	case 1:
 		dataList, err = GetPredictEdbDataListAllByStartDate(fromEdbInfo, 1, "")
 	default:
@@ -411,33 +409,79 @@ func (obj Percentile) getPercentileData(fromEdbInfo *EdbInfo, calculateValue int
 	}
 
 	//百分位：对所选指标滚动地取对应时间长度的数据值，取最大值Max，最小值Min，计算Max-Min，百分位=(现值-Min)/(Max-Min)，Max=Min时不予计算。
-	for i, tmpData := range dataList {
-		currDateTime, _ := time.ParseInLocation(utils.FormatDate, tmpData.DataTime, time.Local)
-		maxVal := tmpData.Value
-		minVal := tmpData.Value
-		for k := 0; k < calculateDay; k++ {
-			preVal, ok2 := dataMap[currDateTime.AddDate(0, 0, -k)]
-			if ok2 {
-				if preVal > maxVal {
-					maxVal = preVal
-				}
-				if preVal < minVal {
-					minVal = preVal
+	if percentType == utils.PercentCalculateTypeRange {
+		for i, tmpData := range dataList {
+			currDateTime, _ := time.ParseInLocation(utils.FormatDate, tmpData.DataTime, time.Local)
+			maxVal := tmpData.Value
+			minVal := tmpData.Value
+			for k := 0; k < calculateDay; k++ {
+				preVal, ok2 := dataMap[currDateTime.AddDate(0, 0, -k)]
+				if ok2 {
+					if preVal > maxVal {
+						maxVal = preVal
+					}
+					if preVal < minVal {
+						minVal = preVal
+					}
 				}
 			}
+
+			if maxVal == minVal {
+				continue
+			}
+			tmpV := (tmpData.Value - minVal) / (maxVal - minVal) * 100
+			tmpV, _ = decimal.NewFromFloat(tmpV).Round(4).Float64()
+			//百分位=(现值-Min)/(Max-Min)
+			newDataList = append(newDataList, EdbInfoSearchData{
+				EdbDataId: i,
+				DataTime:  dataList[i].DataTime,
+				Value:     tmpV,
+			})
 		}
+	}
+	// 百分位数据个数算法
+	// 数据区间第一个和最后一个数据点的时间和数据分别为(T1,S1)(T2,S2); N=T1到T2指标数据个数, n=小于等于S2的数据个数
+	// 个数百分位=(n-1)/(N-1)
+	maxDay := len(dataList) // 往前找数据的边界
+	if percentType == utils.PercentCalculateTypeNum {
+		for i, d := range dataList {
+			// T2为当前日期
+			s2 := decimal.NewFromFloat(d.Value)
+			t2, _ := time.ParseInLocation(utils.FormatDate, d.DataTime, time.Local)
+
+			// 计算N和n
+			var bigN, tinyN int
+			for k := 0; k < maxDay; k++ {
+				// 往前找(时间长度)个有数据的, N理论上只有最前面几个日期<calculateDay, 后面的N=calculateDay
+				if bigN >= calculateDay {
+					break
+				}
+				preVal, preOk := dataMap[t2.AddDate(0, 0, -k)]
+				if !preOk {
+					continue
+				}
+				bigN += 1
+				if decimal.NewFromFloat(preVal).LessThanOrEqual(s2) {
+					tinyN += 1
+				}
+			}
 
-		if maxVal == minVal {
-			continue
+			// N<=1时说明计算无效
+			if bigN <= 1 {
+				continue
+			}
+			numerator := decimal.NewFromInt(int64(tinyN - 1))
+			denominator := decimal.NewFromInt(int64(bigN - 1))
+			// 因为是百分位所以这里是要*100, 跟之前的算法保持同步
+			percentVal, _ := numerator.Div(denominator).Mul(decimal.NewFromFloat(100)).Round(4).Float64()
+
+			// 写进数组并判断指标最大最小值
+			newDataList = append(newDataList, EdbInfoSearchData{
+				EdbDataId: i,
+				DataTime:  dataList[i].DataTime,
+				Value:     percentVal,
+			})
 		}
-		tmpV := (tmpData.Value - minVal) / (maxVal - minVal) * 100
-		tmpV, _ = decimal.NewFromFloat(tmpV).Round(4).Float64()
-		//百分位=(现值-Min)/(Max-Min)
-		newDataList = append(newDataList, EdbInfoSearchData{
-			EdbDataId: i,
-			DataTime:  dataList[i].DataTime,
-			Value:     tmpV,
-		})
 	}
 
 	return

models/edb_data_calculate_rjz.go

@@ -209,22 +209,11 @@ func refreshAllCalculateRjz(to orm.TxOrmer, edbInfoId, source int, fromEdbInfo *
 	edbInfoIdStr := strconv.Itoa(edbInfoId)
 	//计算数据
 
-	//计算数据
-	var condition string
-	var pars []interface{}
-	condition += " AND edb_info_id=? "
-	pars = append(pars, fromEdbInfo.EdbInfoId)
-
-	if startDate != "" {
-		condition += " AND data_time>=? "
-		pars = append(pars, startDate)
-	}
-	//if endDate != "" {
-	//	condition += " AND data_time<=? "
-	//	pars = append(pars, endDate)
-	//}
-
-	dataList, err := GetEdbDataListAll(condition, pars, fromEdbInfo.Source, fromEdbInfo.SubSource, 0)
+	dataList, err := GetEdbDataListAll(fromEdbInfo.Source, fromEdbInfo.SubSource, FindEdbDataListAllCond{
+		EdbInfoId:         fromEdbInfo.EdbInfoId,
+		StartDataTime:     startDate,
+		StartDataTimeCond: ">=",
+	}, 0)
 	if err != nil {
 		return err
 	}

models/edb_data_calculate_standard_deviation.go

@@ -346,13 +346,10 @@ func (obj StandardDeviation) getStandardDeviationData(fromEdbInfo *EdbInfo, calc
 	dataList := make([]*EdbInfoSearchData, 0)
 	switch fromEdbInfo.EdbInfoType {
 	case 0:
-		var condition string
-		var pars []interface{}
-		condition += " AND edb_info_id=? "
-		pars = append(pars, fromEdbInfo.EdbInfoId)
-
 		//获取来源指标的数据
-		dataList, err = GetEdbDataListAll(condition, pars, fromEdbInfo.Source, fromEdbInfo.SubSource, 1)
+		dataList, err = GetEdbDataListAll(fromEdbInfo.Source, fromEdbInfo.SubSource, FindEdbDataListAllCond{
+			EdbInfoId: fromEdbInfo.EdbInfoId,
+		}, 1)
 	case 1:
 		dataList, err = GetPredictEdbDataListAllByStartDate(fromEdbInfo, 1, "")
 	default:

models/edb_data_calculate_sum.go

@@ -304,11 +304,9 @@ func (obj CalculateSum) refresh(to orm.TxOrmer, edbInfoId, source, subSource int
 	}
 
 	for edbInfoIndex, v := range edbInfoIdArr {
-		var condition string
-		var pars []interface{}
-		condition += " AND edb_info_id=? "
-		pars = append(pars, v.EdbInfoId)
-		dataList, err := GetEdbDataListAllByTo(to, condition, pars, v.Source, v.SubSource, 1)
+		dataList, err := GetEdbDataListAllByTo(to, v.Source, v.SubSource, FindEdbDataListAllCond{
+			EdbInfoId: v.EdbInfoId,
+		}, 1)
 		if err != nil {
 			return err
 		}

models/edb_data_calculate_tbz.go

@@ -209,22 +209,11 @@ func refreshAllCalculateTbz(to orm.TxOrmer, edbInfoId, source, subSource int, fr
 	edbInfoIdStr := strconv.Itoa(edbInfoId)
 	//计算数据
 
-	//计算数据
-	var condition string
-	var pars []interface{}
-	condition += " AND edb_info_id=? "
-	pars = append(pars, fromEdbInfo.EdbInfoId)
-
-	if startDate != "" {
-		condition += " AND data_time>=? "
-		pars = append(pars, startDate)
-	}
-	//if endDate != "" {
-	//	condition += " AND data_time<=? "
-	//	pars = append(pars, endDate)
-	//}
-
-	dataList, err := GetEdbDataListAll(condition, pars, fromEdbInfo.Source, fromEdbInfo.SubSource, 0)
+	dataList, err := GetEdbDataListAll(fromEdbInfo.Source, fromEdbInfo.SubSource, FindEdbDataListAllCond{
+		EdbInfoId:         fromEdbInfo.EdbInfoId,
+		StartDataTime:     startDate,
+		StartDataTimeCond: ">=",
+	}, 0)
 	if err != nil {
 		return err
 	}

models/edb_data_calculate_tcz.go

@@ -211,22 +211,11 @@ func refreshAllCalculateTcz(to orm.TxOrmer, edbInfoId, source, subSource int, fr
 	edbInfoIdStr := strconv.Itoa(edbInfoId)
 	//计算数据
 
-	//计算数据
-	var condition string
-	var pars []interface{}
-	condition += " AND edb_info_id=? "
-	pars = append(pars, fromEdbInfo.EdbInfoId)
-
-	if startDate != "" {
-		condition += " AND data_time>=? "
-		pars = append(pars, startDate)
-	}
-	//if endDate != "" {
-	//	condition += " AND data_time<=? "
-	//	pars = append(pars, endDate)
-	//}
-
-	dataList, err := GetEdbDataListAllByTo(to, condition, pars, fromEdbInfo.Source, fromEdbInfo.SubSource, 0)
+	dataList, err := GetEdbDataListAllByTo(to, fromEdbInfo.Source, fromEdbInfo.SubSource, FindEdbDataListAllCond{
+		EdbInfoId:         fromEdbInfo.EdbInfoId,
+		StartDataTime:     startDate,
+		StartDataTimeCond: ">=",
+	}, 0)
 	if err != nil {
 		return err
 	}

models/edb_data_calculate_time_shift.go

@@ -223,19 +223,6 @@ func refreshAllCalculateTimeShift(to orm.TxOrmer, edbInfoId, source, subSource,
 	edbInfoIdStr := strconv.Itoa(edbInfoId)
 
 	//计算数据
-	var condition string
-	var pars []interface{}
-	condition += " AND edb_info_id=? "
-	pars = append(pars, fromEdbInfo.EdbInfoId)
-
-	if startDate != "" {
-		condition += " AND data_time>=? "
-		pars = append(pars, startDate)
-	}
-	//if endDate != "" {
-	//	condition += " AND data_time<=? "
-	//	pars = append(pars, endDate)
-	//}
 
 	var shiftDay int
 	switch moveFrequency {
@@ -257,7 +244,11 @@ func refreshAllCalculateTimeShift(to orm.TxOrmer, edbInfoId, source, subSource,
 		shiftDay = -shiftDay
 	}
 
-	dataList, err := GetEdbDataListAllByTo(to, condition, pars, fromEdbInfo.Source, fromEdbInfo.SubSource, 0)
+	dataList, err := GetEdbDataListAllByTo(to, fromEdbInfo.Source, fromEdbInfo.SubSource, FindEdbDataListAllCond{
+		EdbInfoId:         fromEdbInfo.EdbInfoId,
+		StartDataTime:     startDate,
+		StartDataTimeCond: ">=",
+	}, 0)
 	if err != nil {
 		return err
 	}
@@ -278,9 +269,12 @@ func refreshAllCalculateTimeShift(to orm.TxOrmer, edbInfoId, source, subSource,
 	if err != nil {
 		return err
 	}
+
 	existDataMap := make(map[string]string)
+	removeDateMap := make(map[string]string)
 	for _, v := range existDataList {
 		existDataMap[v.DataTime] = v.Value
+		removeDateMap[v.DataTime] = ``
 	}
 	fmt.Println("existDataMap:", existDataMap)
 	addSql := ` INSERT INTO edb_data_calculate_time_shift(edb_info_id,edb_code,data_time,value,create_time,modify_time,data_timestamp) values `
@@ -314,6 +308,9 @@ func refreshAllCalculateTimeShift(to orm.TxOrmer, edbInfoId, source, subSource,
 					}
 				}
 			}
+
+			// 指标已存在，那么就移除该日期
+			delete(removeDateMap, newDate.Format(utils.FormatDate))
 		}
 		existMap[existKey] = currentItem.DataTime
 	}
@@ -325,5 +322,19 @@ func refreshAllCalculateTimeShift(to orm.TxOrmer, edbInfoId, source, subSource,
 			return err
 		}
 	}
+
+	// 移除不存在的日期数据
+	if len(removeDateMap) > 0 {
+		removeDateList := make([]string, 0) //需要移除的日期
+		for k := range removeDateMap {
+			removeDateList = append(removeDateList, k)
+		}
+		err = DelEdbDataByMysql(to, edbInfoId, dataTableName, removeDateList)
+		if err != nil {
+			err = fmt.Errorf("删除年化指标数据失败,Err:" + err.Error())
+			return
+		}
+	}
+
 	return
 }

models/edb_data_calculate_zjpj.go

@@ -395,14 +395,12 @@ func refreshAllCalculateZjpj(to orm.TxOrmer, edbInfo *EdbInfo, existItemA, exist
 	addDataList := make([]*EdbDataCalculateZjpj, 0)
 	//第一个指标
 	{
-		var condition string
-		var pars []interface{}
-
-		condition += " AND data_time < ? AND edb_info_id=? "
-		pars = append(pars, edbInfo.CalculateFormula, existItemA.FromEdbInfoId)
-
 		//第一个指标的数据列表
-		firstDataList, tmpErr := GetEdbDataListAllByTo(to, condition, pars, existItemA.FromSource, existItemA.FromSubSource, 0)
+		firstDataList, tmpErr := GetEdbDataListAllByTo(to, existItemA.FromSource, existItemA.FromSubSource, FindEdbDataListAllCond{
+			EdbInfoId:       existItemA.FromEdbInfoId,
+			EndDataTime:     edbInfo.CalculateFormula,
+			EndDataTimeCond: "<",
+		}, 0)
 		if tmpErr != nil {
 			return tmpErr
 		}
@@ -441,14 +439,12 @@ func refreshAllCalculateZjpj(to orm.TxOrmer, edbInfo *EdbInfo, existItemA, exist
 
 	//第二个指标
 	{
-		condition = ``
-		pars = make([]interface{}, 0)
-
-		condition += " AND data_time >= ? AND edb_info_id = ? "
-		pars = append(pars, edbInfo.CalculateFormula, existItemB.FromEdbInfoId)
-
 		//第二个指标的数据列表
-		secondDataList, tmpErr := GetEdbDataListAllByTo(to, condition, pars, existItemB.FromSource, existItemB.FromSubSource, 0)
+		secondDataList, tmpErr := GetEdbDataListAllByTo(to, existItemB.FromSource, existItemB.FromSubSource, FindEdbDataListAllCond{
+			EdbInfoId:         existItemB.FromEdbInfoId,
+			StartDataTime:     edbInfo.CalculateFormula,
+			StartDataTimeCond: ">=",
+		}, 0)
 		if tmpErr != nil {
 			return tmpErr
 		}

models/edb_data_calculate_zsxy.go

@@ -345,13 +345,11 @@ func CalculateExponentialSmoothingData(fromEdbInfo *EdbInfo, strAlpha string) (n
 	dataList := make([]*EdbInfoSearchData, 0)
 	switch fromEdbInfo.EdbInfoType {
 	case 0:
-		var condition string
-		var pars []interface{}
-		condition += " AND edb_info_id=? "
-		pars = append(pars, fromEdbInfo.EdbInfoId)
-
 		//获取来源指标的数据
-		dataList, err = GetEdbDataListAll(condition, pars, fromEdbInfo.Source, fromEdbInfo.SubSource, 1)
+		dataList, err = GetEdbDataListAll(fromEdbInfo.Source, fromEdbInfo.SubSource,
+			FindEdbDataListAllCond{
+				EdbInfoId: fromEdbInfo.EdbInfoId,
+			}, 1)
 	case 1:
 		dataList, err = GetPredictEdbDataListAllByStartDate(fromEdbInfo, 1, "")
 	default:

models/edb_info.go

@@ -3,11 +3,14 @@ package models
 import (
 	"encoding/json"
 	"errors"
+	"eta/eta_index_lib/models/mgo"
 	"eta/eta_index_lib/utils"
 	"fmt"
 	"strconv"
 	"time"
 
+	"go.mongodb.org/mongo-driver/bson"
+
 	"github.com/beego/beego/v2/client/orm"
 	"github.com/shopspring/decimal"
 )
@@ -182,20 +185,77 @@ func (edbInfo *EdbInfo) Update(cols []string) (err error) {
 
 // EdbInfoSearchData
 type EdbInfoSearchData struct {
-	EdbDataId int     `description:"数据ID"`
-	DataTime  string  `description:"数据日期"`
-	Value     float64 `description:"数据"`
-	EdbCode   string  `description:"指标编码"`
+	EdbDataId     int     `description:"数据ID"`
+	EdbInfoId     int     `description:"指标ID"`
+	DataTime      string  `description:"数据日期"`
+	Value         float64 `description:"数据"`
+	EdbCode       string  `description:"指标编码"`
+	DataTimestamp int64   `description:"时间戳"`
 }
 
-// GetEdbDataListAll 获取指标数据列表 order:1升序，其余值为降序
-func GetEdbDataListAll(condition string, pars []interface{}, source, subSource, order int) (item []*EdbInfoSearchData, err error) {
-	o := orm.NewOrm()
+type FindEdbDataListAllCond struct {
+	EdbInfoId         int
+	StartDataTime     string
+	StartDataTimeCond string
+	EndDataTime       string
+	EndDataTimeCond   string
+}
+
+// GetEdbDataListAll
+// @Description: 获取指标数据列表 order:1升序，其余值为降序
+// @author: Roc
+// @datetime 2024-05-08 15:34:01
+// @param source int
+// @param subSource int
+// @param findEdbDataListAllCond FindEdbDataListAllCond
+// @param order int  order:1升序，其余值为降序
+// @return item []*EdbInfoSearchData
+// @return err error
+func GetEdbDataListAll(source, subSource int, findEdbDataListAllCond FindEdbDataListAllCond, order int) (item []*EdbInfoSearchData, err error) {
+	if source == utils.DATA_SOURCE_BUSINESS && utils.UseMongo {
+		return GetEdbDataListAllByMongo(source, subSource, findEdbDataListAllCond, order)
+	}
+
+	return GetEdbDataListAllByMysql(source, subSource, findEdbDataListAllCond, order)
+}
+
+// GetEdbDataListAllByMysql
+// @Description: 从mysql数据库中获取指标数据列表 order:1升序，其余值为降序
+// @author: Roc
+// @datetime 2024-05-08 15:32:55
+// @param source int
+// @param subSource int
+// @param findEdbDataListAllCond FindEdbDataListAllCond
+// @param order int
+// @return item []*EdbInfoSearchData
+// @return err error
+func GetEdbDataListAllByMysql(source, subSource int, findEdbDataListAllCond FindEdbDataListAllCond, order int) (item []*EdbInfoSearchData, err error) {
+	if findEdbDataListAllCond.EdbInfoId <= 0 {
+		return
+	}
+	to := orm.NewOrm()
+
 	sql := ``
 	tableName := GetEdbDataTableName(source, subSource)
 	sql = ` SELECT * FROM %s WHERE 1=1 `
 	sql = fmt.Sprintf(sql, tableName)
 
+	var condition string
+	var pars []interface{}
+	condition += " AND edb_info_id=? "
+	pars = append(pars, findEdbDataListAllCond.EdbInfoId)
+
+	// 开始日期
+	if findEdbDataListAllCond.StartDataTime != "" && findEdbDataListAllCond.StartDataTimeCond != `` {
+		condition += fmt.Sprintf(" AND data_time %s ? ", findEdbDataListAllCond.StartDataTimeCond)
+		pars = append(pars, findEdbDataListAllCond.StartDataTime)
+	}
+	// 结束日期
+	if findEdbDataListAllCond.EndDataTime != "" && findEdbDataListAllCond.EndDataTimeCond != `` {
+		condition += fmt.Sprintf(" AND data_time %s ? ", findEdbDataListAllCond.EndDataTimeCond)
+		pars = append(pars, findEdbDataListAllCond.EndDataTime)
+	}
+
 	if condition != "" {
 		sql += condition
 	}
@@ -204,17 +264,65 @@ func GetEdbDataListAll(condition string, pars []interface{}, source, subSource,
 	} else {
 		sql += ` ORDER BY data_time DESC `
 	}
-	_, err = o.Raw(sql, pars).QueryRows(&item)
+	_, err = to.Raw(sql, pars).QueryRows(&item)
 	return
 }
 
-// GetEdbDataListAllByTo 根据事务链接获取指标数据列表 order:1升序，其余值为降序
-func GetEdbDataListAllByTo(to orm.TxOrmer, condition string, pars []interface{}, source, subSource, order int) (item []*EdbInfoSearchData, err error) {
+// GetEdbDataListAllByTo
+// @Description: 根据事务链接获取指标数据列表 order:1升序，其余值为降序
+// @author: Roc
+// @datetime 2024-05-08 15:34:06
+// @param to orm.TxOrmer
+// @param source int
+// @param subSource int
+// @param findEdbDataListAllCond FindEdbDataListAllCond
+// @param order int
+// @return item []*EdbInfoSearchData
+// @return err error
+func GetEdbDataListAllByTo(to orm.TxOrmer, source, subSource int, findEdbDataListAllCond FindEdbDataListAllCond, order int) (item []*EdbInfoSearchData, err error) {
+	if source == utils.DATA_SOURCE_BUSINESS && utils.UseMongo {
+		return GetEdbDataListAllByMongo(source, subSource, findEdbDataListAllCond, order)
+	}
+
+	return GetEdbDataListAllByMysqlTo(to, source, subSource, findEdbDataListAllCond, order)
+}
+
+// GetEdbDataListAllByMysqlTo
+// @Description: 根据事务链接获取指标数据列表 order:1升序，其余值为降序（Mysql）
+// @author: Roc
+// @datetime 2024-05-08 15:34:13
+// @param to orm.TxOrmer
+// @param source int
+// @param subSource int
+// @param findEdbDataListAllCond FindEdbDataListAllCond
+// @param order int
+// @return item []*EdbInfoSearchData
+// @return err error
+func GetEdbDataListAllByMysqlTo(to orm.TxOrmer, source, subSource int, findEdbDataListAllCond FindEdbDataListAllCond, order int) (item []*EdbInfoSearchData, err error) {
+	if findEdbDataListAllCond.EdbInfoId <= 0 {
+		return
+	}
 	sql := ``
 	tableName := GetEdbDataTableName(source, subSource)
 	sql = ` SELECT * FROM %s WHERE 1=1 `
 	sql = fmt.Sprintf(sql, tableName)
 
+	var condition string
+	var pars []interface{}
+	condition += " AND edb_info_id=? "
+	pars = append(pars, findEdbDataListAllCond.EdbInfoId)
+
+	// 开始日期
+	if findEdbDataListAllCond.StartDataTime != "" && findEdbDataListAllCond.StartDataTimeCond != `` {
+		condition += fmt.Sprintf(" AND data_time %s ? ", findEdbDataListAllCond.StartDataTimeCond)
+		pars = append(pars, findEdbDataListAllCond.StartDataTime)
+	}
+	// 结束日期
+	if findEdbDataListAllCond.EndDataTime != "" && findEdbDataListAllCond.EndDataTimeCond != `` {
+		condition += fmt.Sprintf(" AND data_time %s ? ", findEdbDataListAllCond.EndDataTimeCond)
+		pars = append(pars, findEdbDataListAllCond.EndDataTime)
+	}
+
 	if condition != "" {
 		sql += condition
 	}
@@ -227,15 +335,68 @@ func GetEdbDataListAllByTo(to orm.TxOrmer, condition string, pars []interface{},
 	return
 }
 
+// GetEdbDataListAllByMongo
+// @Description: 根据事务链接获取指标数据列表 order:1升序，其余值为降序（Mongo）
+// @author: Roc
+// @datetime 2024-05-08 15:34:24
+// @param source int
+// @param subSource int
+// @param findEdbDataListAllCond FindEdbDataListAllCond
+// @param order int
+// @return dataList []*EdbInfoSearchData
+// @return err error
+func GetEdbDataListAllByMongo(source, subSource int, findEdbDataListAllCond FindEdbDataListAllCond, order int) (dataList []*EdbInfoSearchData, err error) {
+	dataList = make([]*EdbInfoSearchData, 0)
+	if findEdbDataListAllCond.EdbInfoId <= 0 {
+		return
+	}
+	mogDataObj := mgo.EdbDataBusiness{}
+	// 构建查询条件
+	queryConditions := bson.M{
+		"edb_info_id": findEdbDataListAllCond.EdbInfoId,
+	}
+	dateCondition, err := mgo.BuildDateCondition(findEdbDataListAllCond.StartDataTime, findEdbDataListAllCond.EndDataTime)
+	if err != nil {
+		return
+	}
+	if len(dateCondition) > 0 {
+		queryConditions["data_time"] = dateCondition
+	}
+
+	sortList := make([]string, 0)
+	if order == 1 {
+		sortList = append(sortList, "data_time")
+	} else {
+		sortList = append(sortList, "-data_time")
+	}
+	// 获取列表数据
+	tmpDataList, tmpErr := mogDataObj.GetAllDataList(queryConditions, sortList)
+	if tmpErr != nil {
+		err = tmpErr
+		return
+	}
+	for k, v := range tmpDataList {
+		dataList = append(dataList, &EdbInfoSearchData{
+			EdbDataId:     k + 1,
+			EdbInfoId:     v.EdbInfoId,
+			EdbCode:       v.EdbCode,
+			DataTime:      v.DataTime.Format(utils.FormatDate),
+			Value:         v.Value,
+			DataTimestamp: v.DataTimestamp,
+		})
+	}
+	return
+}
+
 // EdbInfoMaxAndMinInfo 指标最新数据记录结构体
 type EdbInfoMaxAndMinInfo struct {
-	MinDate     string  `description:"最小日期"`
-	MaxDate     string  `description:"最大日期"`
-	MinValue    float64 `description:"最小值"`
-	MaxValue    float64 `description:"最大值"`
-	LatestValue float64 `description:"最新值"`
-	LatestDate  string  `description:"实际数据最新日期"`
-	EndValue    float64 `description:"最新值"`
+	MinDate     string  `description:"最小日期" bson:"min_date"`
+	MaxDate     string  `description:"最大日期" bson:"max_date"`
+	MinValue    float64 `description:"最小值" bson:"min_value"`
+	MaxValue    float64 `description:"最大值" bson:"max_value"`
+	LatestValue float64 `description:"最新值" bson:"latest_value"`
+	LatestDate  string  `description:"实际数据最新日期" bson:"latest_date"`
+	EndValue    float64 `description:"最新值" bson:"end_value"`
 }
 
 // GetEdbInfoMaxAndMinInfo 获取指标的最新数据记录信息
@@ -294,18 +455,66 @@ func ModifyEdbDataUpdateTime(edbInfoId int, dataUpdateTime, erDataUpdateDate str
 	return
 }
 
-// GetEdbDataCount 获取edb指标数据的数量； order:1升序，其余值为降序
-func GetEdbDataCount(condition string, pars []interface{}, source, subSource int) (count int, err error) {
+// GetLteZeroEdbDataCount
+// @Description: 获取小于等于0的数据数量
+// @author: Roc
+// @datetime 2024-05-31 10:44:39
+// @param source int
+// @param subSource int
+// @param edbInfoId int
+// @return count int
+// @return err error
+func GetLteZeroEdbDataCount(source, subSource, edbInfoId int) (count int, err error) {
+	if source == utils.DATA_SOURCE_BUSINESS && utils.UseMongo {
+		return GetLteZeroEdbDataCountByMongo(source, subSource, edbInfoId)
+	}
+
+	return GetLteZeroEdbDataCountByMysql(source, subSource, edbInfoId)
+}
+
+// GetLteZeroEdbDataCountByMongo
+// @Description: 获取小于等于0的数据数量（从mongo）
+// @author: Roc
+// @datetime 2024-05-31 10:41:04
+// @param source int
+// @param subSource int
+// @param edbInfoId int
+// @return count int
+// @return err error
+func GetLteZeroEdbDataCountByMongo(source, subSource, edbInfoId int) (count int, err error) {
+	mogDataObj := mgo.EdbDataBusiness{}
+	// 构建查询条件
+	whereQuery := bson.M{
+		"edb_info_id": edbInfoId,
+		"value":       bson.M{"$lte": 0},
+	}
+	// 获数量数据
+	tmpCount, err := mogDataObj.GetCountDataList(whereQuery)
+	if err != nil {
+		return
+	}
+
+	count = int(tmpCount)
+
+	return
+}
+
+// GetLteZeroEdbDataCountByMysql
+// @Description: 获取小于等于0的数据数量（从mysql）
+// @author: Roc
+// @datetime 2024-05-31 10:29:19
+// @param source int
+// @param subSource int
+// @param edbInfoId int
+// @return count int
+// @return err error
+func GetLteZeroEdbDataCountByMysql(source, subSource, edbInfoId int) (count int, err error) {
 	o := orm.NewOrm()
 	sql := ``
 	tableName := GetEdbDataTableName(source, subSource)
-	sql = ` SELECT COUNT(1) AS count FROM %s WHERE 1=1 `
+	sql = ` SELECT COUNT(1) AS count FROM %s WHERE 1=1 AND edb_info_id =?  AND value <=0 `
 	sql = fmt.Sprintf(sql, tableName)
-
-	if condition != "" {
-		sql += condition
-	}
-	err = o.Raw(sql, pars).QueryRow(&count)
+	err = o.Raw(sql, edbInfoId).QueryRow(&count)
 	return
 }
 
@@ -566,14 +775,14 @@ func GetChartPredictEdbInfoDataListByConfList(predictEdbConfList []*PredictEdbCo
 	//预测规则，1：最新，2：固定值，3：同比，4：同差，5：环比，6：环差，7：N期移动均值，8：N期段线性外推值
 
 	for _, predictEdbConf := range predictEdbConfList {
-		dataEndTime := endDate
-		if predictEdbConf.EndDate.Before(dataEndTime) {
-			dataEndTime = predictEdbConf.EndDate
-		}
-
+		//dataEndTime := endDate
+		//if predictEdbConf.EndDate.Before(dataEndTime) {
+		//	dataEndTime = predictEdbConf.EndDate
+		//}
+		endDate = predictEdbConf.EndDate
 		var tmpMinValue, tmpMaxValue float64 // 当前预测结果中的最大/最小值
 
-		dayList := getPredictEdbDayList(startDate, dataEndTime, frequency, dataDateType)
+		dayList := getPredictEdbDayList(startDate, endDate, frequency, dataDateType)
 		if len(dayList) <= 0 { // 如果未来没有日期的话，那么就退出当前循环，进入下一个循环
 			continue
 		}
@@ -906,29 +1115,29 @@ func GetPredictDataListByPredictEdbInfo(edbInfo *EdbInfo, order int, startDate s
 
 // GetPredictDataListByPredictEdbConfList 根据预测指标信息获取预测指标的数据，order:1升序，其余值为降序
 func GetPredictDataListByPredictEdbConfList(edbInfo, sourceEdbInfoItem *EdbInfo, predictEdbConfList []*PredictEdbConfAndData, order int, startDate string) (dataList []*EdbInfoSearchData, err error, errMsg string) {
-
 	allDataList := make([]*EdbInfoSearchData, 0)
 	//获取指标数据（实际已生成）
-	var condition string
-	var pars []interface{}
-	condition += " AND edb_info_id=? "
-	pars = append(pars, sourceEdbInfoItem.EdbInfoId)
-	if startDate != "" {
-		condition += " AND data_time>=? "
-		pars = append(pars, startDate)
-	}
-	tmpDataList, err := GetEdbDataListAll(condition, pars, sourceEdbInfoItem.Source, sourceEdbInfoItem.SubSource, 1)
+	tmpDataList, err := GetEdbDataListAll(sourceEdbInfoItem.Source, sourceEdbInfoItem.SubSource, FindEdbDataListAllCond{
+		EdbInfoId:         sourceEdbInfoItem.EdbInfoId,
+		StartDataTime:     startDate,
+		StartDataTimeCond: ">=",
+	}, 1)
 	if err != nil {
 		return
 	}
-	// 如果选择了日期，那么需要筛选所有的数据，用于未来指标的生成
+
+	// 默认认为上一个查询是把所有的数据查出来了
+	allDataList = tmpDataList
+
+	// 如果条件中有选择了日期，那么上次查询实际并不是把所有的数据查出来了
+	// 那么需要把日期给过滤掉，然后筛选所有的数据，用于未来指标的生成
 	if startDate != `` {
-		allDataList, err = GetEdbDataListAll(" AND edb_info_id=? ", []interface{}{sourceEdbInfoItem.EdbInfoId}, sourceEdbInfoItem.Source, sourceEdbInfoItem.SubSource, 1)
+		allDataList, err = GetEdbDataListAll(sourceEdbInfoItem.Source, sourceEdbInfoItem.SubSource, FindEdbDataListAllCond{
+			EdbInfoId: sourceEdbInfoItem.EdbInfoId,
+		}, 1)
 		if err != nil {
 			return
 		}
-	} else {
-		allDataList = tmpDataList
 	}
 
 	// 获取预测指标未来的数据
@@ -982,11 +1191,9 @@ func GetPredictEdbDataListAll(edbInfo *EdbInfo, order int) (items []*EdbInfoSear
 	if edbInfo.Source == utils.DATA_SOURCE_PREDICT { //普通的预测指标是没有入库数据的，直接往配置里面获取
 		items, _, err, _ = GetPredictDataListByPredictEdbInfo(edbInfo, 1, "")
 	} else {
-		var condition string
-		var pars []interface{}
-		condition += " AND edb_info_id=? "
-		pars = append(pars, edbInfo.EdbInfoId)
-		items, err = GetEdbDataListAll(condition, pars, edbInfo.Source, edbInfo.SubSource, order)
+		items, err = GetEdbDataListAll(edbInfo.Source, edbInfo.SubSource, FindEdbDataListAllCond{
+			EdbInfoId: edbInfo.EdbInfoId,
+		}, order)
 	}
 	return
 }
@@ -996,15 +1203,11 @@ func GetPredictEdbDataListAllByStartDate(edbInfo *EdbInfo, order int, startDate
 	if edbInfo.Source == utils.DATA_SOURCE_PREDICT { //普通的预测指标是没有入库数据的，直接往配置里面获取
 		items, _, err, _ = GetPredictDataListByPredictEdbInfo(edbInfo, order, startDate)
 	} else {
-		var condition string
-		var pars []interface{}
-		condition += " AND edb_info_id=? "
-		pars = append(pars, edbInfo.EdbInfoId)
-		if startDate != "" {
-			condition += " AND data_time>=? "
-			pars = append(pars, startDate)
-		}
-		items, err = GetEdbDataListAll(condition, pars, edbInfo.Source, edbInfo.SubSource, order)
+		items, err = GetEdbDataListAll(edbInfo.Source, edbInfo.SubSource, FindEdbDataListAllCond{
+			EdbInfoId:         edbInfo.EdbInfoId,
+			StartDataTime:     startDate,
+			StartDataTimeCond: ">=",
+		}, order)
 	}
 	return
 }
@@ -1195,8 +1398,10 @@ func EdbInfoAdd(req *AddEdbInfoParams, serverUrl string, sysUserId int, sysUserR
 	edbInfo.EdbCode = req.EdbCode
 	edbInfo.EdbName = req.EdbName
 	edbInfo.EdbNameSource = req.EdbName
+	edbInfo.EdbNameEn = req.EdbName
 	edbInfo.Frequency = req.Frequency
 	edbInfo.Unit = req.Unit
+	edbInfo.UnitEn = req.Unit
 	edbInfo.ClassifyId = req.ClassifyId
 	edbInfo.SysUserId = sysUserId
 	edbInfo.SysUserRealName = sysUserRealName
@@ -1298,3 +1503,69 @@ type EdbInfoEditRecord struct {
 	OperateUserId       int    `description:"操作人id"`
 	OperateUserRealName string `description:"操作人姓名"`
 }
+
+// GetEdbInfoByEdbCodeList
+// @Description: 根据来源和指标编码列表获取指标信息列表
+// @author: Roc
+// @datetime 2024-05-31 16:31:52
+// @param source int
+// @param edbCodeList []string
+// @return items []*EdbInfo
+// @return err error
+func GetEdbInfoByEdbCodeList(source int, edbCodeList []string) (items []*EdbInfo, err error) {
+	num := len(edbCodeList)
+	if num <= 0 {
+		return
+	}
+	o := orm.NewOrm()
+	sql := ` SELECT * FROM edb_info WHERE source=? AND edb_code IN (` + utils.GetOrmInReplace(num) + `) `
+	_, err = o.Raw(sql, source, edbCodeList).QueryRows(&items)
+
+	return
+}
+
+// GetEdbInfoNoUpdateTotalByIdList 根据指标id列表获取指标信息
+func GetEdbInfoNoUpdateTotalByIdList(edbInfoIdList []int) (total int, err error) {
+	num := len(edbInfoIdList)
+	if num <= 0 {
+		return
+	}
+	o := orm.NewOrm()
+	sql := ` SELECT count(*) FROM edb_info WHERE edb_info_id in (` + utils.GetOrmInReplace(num) + `)  and no_update=1`
+	err = o.Raw(sql, edbInfoIdList).QueryRow(&total)
+	return
+}
+
+func TransEdbInfoSearchData2DataList(items []*EdbInfoSearchData) (list []*EdbDataList) {
+	if len(items) == 0 {
+		return
+	}
+	list = make([]*EdbDataList, 0)
+	for _, v := range items {
+		list = append(list, &EdbDataList{
+			EdbDataId:     v.EdbDataId,
+			EdbInfoId:     v.EdbInfoId,
+			DataTime:      v.DataTime,
+			DataTimestamp: v.DataTimestamp,
+			Value:         v.Value,
+		})
+	}
+	return
+}
+
+func TransEdbInfoDataList2SearchData(items []*EdbDataList) (list []*EdbInfoSearchData) {
+	if len(items) == 0 {
+		return
+	}
+	list = make([]*EdbInfoSearchData, 0)
+	for _, v := range items {
+		list = append(list, &EdbInfoSearchData{
+			EdbDataId:     v.EdbDataId,
+			EdbInfoId:     v.EdbInfoId,
+			DataTime:      v.DataTime,
+			DataTimestamp: v.DataTimestamp,
+			Value:         v.Value,
+		})
+	}
+	return
+}

models/edb_info_calculate_mapping.go

@@ -1,6 +1,7 @@
 package models
 
 import (
+	"eta/eta_index_lib/utils"
 	"github.com/beego/beego/v2/client/orm"
 	"time"
 )
@@ -25,6 +26,31 @@ type EdbInfoCalculateMapping struct {
 	FromSubSource             int       `description:"渠道子数据库来源"`
 }
 
+// EdbInfoCalculateMappingInfo
+// @Description: 计算指标与基础指标关系表
+type EdbInfoCalculateMappingInfo struct {
+	EdbInfoCalculateMappingId int       `orm:"column(edb_info_calculate_mapping_id);pk"`
+	EdbInfoId                 int       `description:"计算指标id"`
+	Source                    int       `description:"计算指标来源"`
+	SourceName                string    `description:"计算指标来源名称"`
+	EdbCode                   string    `description:"计算指标编码"`
+	FromEdbInfoId             int       `description:"基础指标id"`
+	FromEdbCode               string    `description:"基础指标编码"`
+	FromEdbName               string    `description:"基础指标名称"`
+	FromSource                int       `description:"基础指标来源"`
+	FromSourceName            string    `description:"基础指标来源名称"`
+	MoveValue                 int       `description:"领先值"`
+	FromTag                   string    `description:"来源指标标签"`
+	Sort                      int       `description:"计算指标名称排序"`
+	CreateTime                time.Time `description:"创建时间"`
+	ModifyTime                time.Time `description:"修改时间"`
+	FromEdbType               int       `description:"来源指标类型:1:基础指标,2:计算指标"`
+	FromEdbInfoType           int       `description:"来源指标类型: 0-基础指标; 1-预测指标"`
+	FromClassifyId            int       `description:"来源指标分类ID"`
+	FromUniqueCode            string    `description:"来源指标唯一编码"`
+	NoUpdate                  int8      `description:"是否停止更新，0：继续更新；1：停止更新"`
+}
+
 // AddEdbInfoCalculateMappingMulti 批量添加指标关系表
 func AddEdbInfoCalculateMappingMulti(items []*EdbInfoCalculateMapping) (err error) {
 	o := orm.NewOrm()
@@ -119,3 +145,23 @@ func GetEdbInfoCalculateByEdbCode(edbCode, fromEdbCode string) (item *EdbInfoCal
 	err = o.Raw(sql, edbCode, fromEdbCode).QueryRow(&item)
 	return
 }
+
+// GetEdbInfoCalculateMappingListByEdbInfoIds 根据生成的指标id获取来源的指标id列表
+func GetEdbInfoCalculateMappingListByEdbInfoIds(edbInfoIds []int) (items []*EdbInfoCalculateMappingInfo, err error) {
+	o := orm.NewOrm()
+	sql := ` SELECT a.*,b.edb_type as from_edb_type,b.edb_info_type as from_edb_info_type, b.unique_code AS from_unique_code, b.classify_id AS from_classify_id,b.no_update FROM edb_info_calculate_mapping AS a
+			INNER JOIN edb_info AS b ON a.from_edb_info_id=b.edb_info_id
+			WHERE a.edb_info_id in (` + utils.GetOrmInReplace(len(edbInfoIds)) + `) `
+	_, err = o.Raw(sql, edbInfoIds).QueryRows(&items)
+	return
+}
+
+// GetEdbInfoCalculateMappingListByEdbInfoId 根据生成的指标id获取来源的指标id列表
+func GetEdbInfoCalculateMappingListByEdbInfoId(edbInfoId int) (items []*EdbInfoCalculateMappingInfo, err error) {
+	o := orm.NewOrm()
+	sql := ` SELECT a.*,b.edb_type as from_edb_type,b.edb_info_type as from_edb_info_type, b.unique_code AS from_unique_code, b.classify_id AS from_classify_id,b.no_update FROM edb_info_calculate_mapping AS a
+			INNER JOIN edb_info AS b ON a.from_edb_info_id=b.edb_info_id
+			WHERE a.edb_info_id=? `
+	_, err = o.Raw(sql, edbInfoId).QueryRows(&items)
+	return
+}

models/edb_info_relation.go

@@ -0,0 +1,136 @@
+package models
+
+import (
+	"eta/eta_index_lib/utils"
+	"github.com/beego/beego/v2/client/orm"
+	"time"
+)
+
+type EdbInfoRelation struct {
+	EdbInfoRelationId  int       `orm:"column(edb_info_relation_id);pk"`
+	EdbInfoId          int       `description:"指标id"`
+	Source             int       `description:"来源:1:同花顺,2:wind,3:彭博,4:指标运算,5:累计值转月,6:同比值,7:同差值,8:N数值移动平均计算,9:手工指标,10:隆众"`
+	EdbName            string    `description:"指标名称"`
+	EdbCode            string    `description:"指标编码"`
+	ReferObjectId      int       `description:"引用对象ID（图表ID，ETA逻辑ID等）"`
+	ReferObjectType    int       `description:"引用对象ID类型（1.图表，2.ETA逻辑）"`
+	ReferObjectSubType int       `description:"引用对象子类"`
+	CreateTime         time.Time `description:"创建时间"`
+	ModifyTime         time.Time `description:"修改时间"`
+	RelationTime       time.Time `description:"引用时间"`
+	RelationType       int       `description:"引用类型，0:直接饮用，1间接引用"`
+	RootEdbInfoId      int       `description:"间接引用时，关联的直接引用的指标ID"`
+	ChildEdbInfoId     int       `description:"间接引用时，计算指标直接关联的指标ID"`
+	RelationCode       string    `description:"引用标识"`
+	ParentRelationId   int       `description:"间接引用关联的直接引用的ID"`
+}
+
+func (e *EdbInfoRelation) TableName() string {
+	return "edb_info_relation"
+}
+
+// GetEdbInfoRelationByChildEdbInfoId 查询引用的指标ID
+func GetEdbInfoRelationByChildEdbInfoId(edbInfoId int) (item *EdbInfoRelation, err error) {
+	o := orm.NewOrm()
+	msql := ` SELECT * FROM edb_info_relation WHERE child_edb_info_id = ? or edb_info_id=?`
+	err = o.Raw(msql, edbInfoId, edbInfoId).QueryRow(&item)
+	return
+}
+
+// GetEdbInfoRelationListByChildEdbInfoId 根据间接引用中的的计算指标ID查询引用列表
+func GetEdbInfoRelationListByChildEdbInfoId(edbInfoId int) (items []*EdbInfoRelation, err error) {
+	o := orm.NewOrm()
+	msql := ` SELECT * FROM edb_info_relation WHERE relation_type=1 AND child_edb_info_id=?`
+	_, err = o.Raw(msql, edbInfoId).QueryRows(&items)
+	return
+}
+
+// GetEdbInfoRelationListByParentRelationId 根据间接引用中的的父级ID查询
+func GetEdbInfoRelationEdbIdsByParentRelationId(relationId, edbInfoId int) (items []int, err error) {
+	o := orm.NewOrm()
+	msql := ` SELECT edb_info_id FROM edb_info_relation WHERE parent_relation_id=? AND child_edb_info_id=?`
+	_, err = o.Raw(msql, relationId, edbInfoId).QueryRows(&items)
+	return
+}
+
+// GetEdbInfoRelationByRelationIds 查询引用的指标ID
+func GetEdbInfoRelationByRelationIds(ids []int) (items []*EdbInfoRelation, err error) {
+	o := orm.NewOrm()
+	msql := ` SELECT * FROM edb_info_relation WHERE edb_info_relation_id in (` + utils.GetOrmInReplace(len(ids)) + `) `
+	_, err = o.Raw(msql, ids).QueryRows(&items)
+	return
+}
+
+// UpdateSecondRelationEdbInfoId 更新指标替换后的间接引用记录
+func UpdateSecondRelationEdbInfoId(edbRelationIds []int, relationList []*EdbInfoRelation, refreshEdbInfoIds []int, indexCodeList []string) (err error) {
+	o, err := orm.NewOrm().Begin()
+	if err != nil {
+		return
+	}
+	defer func() {
+		if err != nil {
+			_ = o.Rollback()
+			return
+		}
+		_ = o.Commit()
+	}()
+	// 删除相关的间接引用
+	sql := ` DELETE FROM edb_info_relation WHERE relation_type=1 and parent_relation_id in (` + utils.GetOrmInReplace(len(edbRelationIds)) + `)`
+	_, err = o.Raw(sql, edbRelationIds).Exec()
+	if err != nil {
+		return
+	}
+
+	// 新增间接引用
+	relationCodesMap := make(map[string]struct{}, 0)
+	if len(relationList) > 0 {
+		for _, relation := range relationList {
+			if relation.RelationType == 1 {
+				relationCodesMap[relation.RelationCode] = struct{}{}
+			}
+		}
+		_, err = o.InsertMulti(len(relationList), relationList)
+		if err != nil {
+			return
+		}
+	}
+
+	if len(refreshEdbInfoIds) > 0 {
+		// todo 更新指标的刷新状态
+		sql = ` UPDATE edb_info SET no_update = 0 WHERE  edb_info_id IN (` + utils.GetOrmInReplace(len(refreshEdbInfoIds)) + `) AND no_update = 1`
+		_, err = o.Raw(sql, refreshEdbInfoIds).Exec()
+		if err != nil {
+			return
+		}
+	}
+
+	//更新数据源钢联化工指标
+	if len(indexCodeList) > 0 {
+		// 更改数据源的更新状态
+		sql = ` UPDATE base_from_mysteel_chemical_index SET is_stop = 0 WHERE index_code IN (` + utils.GetOrmInReplace(len(indexCodeList)) + `) and is_stop=1`
+		_, err = o.Raw(sql, indexCodeList).Exec()
+		if err != nil {
+			return
+		}
+	}
+	if len(relationList) > 0 {
+		// 更新间接引用指标的关联ID
+		relationCodes := make([]string, 0)
+		for relationCode := range relationCodesMap {
+			relationCodes = append(relationCodes, relationCode)
+		}
+		if len(relationCodes) > 0 {
+			sql := ` UPDATE edb_info_relation e1  
+JOIN edb_info_relation e2 ON e1.relation_code = e2.relation_code   
+SET e1.parent_relation_id = e2.edb_info_relation_id  
+WHERE  
+    e1.relation_type = 1   
+    AND e2.relation_type = 0 AND e1.parent_relation_id !=e2.edb_info_relation_id AND e1.relation_code in (` + utils.GetOrmInReplace(len(relationCodes)) + `)`
+			_, err = o.Raw(sql, relationCodes).Exec()
+			if err != nil {
+				return
+			}
+		}
+	}
+	return
+}

models/factor_edb_series.go

@@ -0,0 +1,363 @@
+package models
+
+import (
+	"encoding/json"
+	"eta/eta_index_lib/utils"
+	"fmt"
+	"github.com/beego/beego/v2/client/orm"
+	"strings"
+	"time"
+)
+
+const (
+	FactorEdbSeriesCalculateNone = 0
+	FactorEdbSeriesCalculating   = 1
+	FactorEdbSeriesCalculated    = 2
+)
+
+// FactorEdbSeries 因子指标系列表
+type FactorEdbSeries struct {
+	FactorEdbSeriesId int       `orm:"column(factor_edb_series_id);pk"`
+	SeriesName        string    `description:"系列名称"`
+	EdbInfoType       int       `description:"关联指标类型：0-普通指标；1-预测指标"`
+	CalculateStep     string    `description:"计算步骤-JSON"`
+	CalculateState    int       `description:"计算状态: 0-无计算; 1-计算中; 2-计算完成"`
+	CreateTime        time.Time `description:"创建时间"`
+	ModifyTime        time.Time `description:"修改时间"`
+}
+
+func (m *FactorEdbSeries) TableName() string {
+	return "factor_edb_series"
+}
+
+type FactorEdbSeriesCols struct {
+	PrimaryId      string
+	SeriesName     string
+	EdbInfoType    string
+	CalculateStep  string
+	CalculateState string
+	CreateTime     string
+	ModifyTime     string
+}
+
+func (m *FactorEdbSeries) Cols() FactorEdbSeriesCols {
+	return FactorEdbSeriesCols{
+		PrimaryId:      "factor_edb_series_id",
+		SeriesName:     "series_name",
+		EdbInfoType:    "edb_info_type",
+		CalculateStep:  "calculate_step",
+		CalculateState: "calculate_state",
+		CreateTime:     "create_time",
+		ModifyTime:     "modify_time",
+	}
+}
+
+func (m *FactorEdbSeries) Create() (err error) {
+	o := orm.NewOrm()
+	id, err := o.Insert(m)
+	if err != nil {
+		return
+	}
+	m.FactorEdbSeriesId = int(id)
+	return
+}
+
+func (m *FactorEdbSeries) CreateMulti(items []*FactorEdbSeries) (err error) {
+	if len(items) == 0 {
+		return
+	}
+	o := orm.NewOrm()
+	_, err = o.InsertMulti(len(items), items)
+	return
+}
+
+func (m *FactorEdbSeries) Update(cols []string) (err error) {
+	o := orm.NewOrm()
+	_, err = o.Update(m, cols...)
+	return
+}
+
+func (m *FactorEdbSeries) Remove() (err error) {
+	o := orm.NewOrm()
+	sql := fmt.Sprintf(`DELETE FROM %s WHERE %s = ? LIMIT 1`, m.TableName(), m.Cols().PrimaryId)
+	_, err = o.Raw(sql, m.FactorEdbSeriesId).Exec()
+	return
+}
+
+func (m *FactorEdbSeries) MultiRemove(ids []int) (err error) {
+	if len(ids) == 0 {
+		return
+	}
+	o := orm.NewOrm()
+	sql := fmt.Sprintf(`DELETE FROM %s WHERE %s IN (%s)`, m.TableName(), m.Cols().PrimaryId, utils.GetOrmInReplace(len(ids)))
+	_, err = o.Raw(sql, ids).Exec()
+	return
+}
+
+func (m *FactorEdbSeries) GetItemById(id int) (item *FactorEdbSeries, err error) {
+	o := orm.NewOrm()
+	sql := fmt.Sprintf(`SELECT * FROM %s WHERE %s = ? LIMIT 1`, m.TableName(), m.Cols().PrimaryId)
+	err = o.Raw(sql, id).QueryRow(&item)
+	return
+}
+
+func (m *FactorEdbSeries) GetItemByCondition(condition string, pars []interface{}, orderRule string) (item *FactorEdbSeries, err error) {
+	o := orm.NewOrm()
+	order := ``
+	if orderRule != "" {
+		order = ` ORDER BY ` + orderRule
+	}
+	sql := fmt.Sprintf(`SELECT * FROM %s WHERE 1=1 %s %s LIMIT 1`, m.TableName(), condition, order)
+	err = o.Raw(sql, pars).QueryRow(&item)
+	return
+}
+
+func (m *FactorEdbSeries) GetCountByCondition(condition string, pars []interface{}) (count int, err error) {
+	o := orm.NewOrm()
+	sql := fmt.Sprintf(`SELECT COUNT(1) FROM %s WHERE 1=1 %s`, m.TableName(), condition)
+	err = o.Raw(sql, pars).QueryRow(&count)
+	return
+}
+
+func (m *FactorEdbSeries) GetItemsByCondition(condition string, pars []interface{}, fieldArr []string, orderRule string) (items []*FactorEdbSeries, err error) {
+	o := orm.NewOrm()
+	fields := strings.Join(fieldArr, ",")
+	if len(fieldArr) == 0 {
+		fields = `*`
+	}
+	order := fmt.Sprintf(`ORDER BY %s DESC`, m.Cols().CreateTime)
+	if orderRule != "" {
+		order = ` ORDER BY ` + orderRule
+	}
+	sql := fmt.Sprintf(`SELECT %s FROM %s WHERE 1=1 %s %s`, fields, m.TableName(), condition, order)
+	_, err = o.Raw(sql, pars).QueryRows(&items)
+	return
+}
+
+func (m *FactorEdbSeries) GetPageItemsByCondition(condition string, pars []interface{}, fieldArr []string, orderRule string, startSize, pageSize int) (items []*FactorEdbSeries, err error) {
+	o := orm.NewOrm()
+	fields := strings.Join(fieldArr, ",")
+	if len(fieldArr) == 0 {
+		fields = `*`
+	}
+	order := fmt.Sprintf(`ORDER BY %s DESC`, m.Cols().CreateTime)
+	if orderRule != "" {
+		order = ` ORDER BY ` + orderRule
+	}
+	sql := fmt.Sprintf(`SELECT %s FROM %s WHERE 1=1 %s %s LIMIT ?,?`, fields, m.TableName(), condition, order)
+	_, err = o.Raw(sql, pars, startSize, pageSize).QueryRows(&items)
+	return
+}
+
+// FactorEdbSeriesItem 多因子系列信息
+type FactorEdbSeriesItem struct {
+	SeriesId      int                            `description:"多因子系列ID"`
+	SeriesName    string                         `description:"系列名称"`
+	EdbInfoType   int                            `description:"关联指标类型：0-普通指标；1-预测指标"`
+	CalculateStep []FactorEdbSeriesCalculatePars `description:"计算步骤-JSON"`
+	CreateTime    string                         `description:"创建时间"`
+	ModifyTime    string                         `description:"修改时间"`
+}
+
+func (m *FactorEdbSeries) Format2Item() (item *FactorEdbSeriesItem) {
+	item = new(FactorEdbSeriesItem)
+	item.SeriesId = m.FactorEdbSeriesId
+	item.SeriesName = m.SeriesName
+	item.EdbInfoType = m.EdbInfoType
+	if m.CalculateStep != "" {
+		_ = json.Unmarshal([]byte(m.CalculateStep), &item.CalculateStep)
+	}
+	item.CreateTime = utils.TimeTransferString(utils.FormatDateTime, m.CreateTime)
+	item.ModifyTime = utils.TimeTransferString(utils.FormatDateTime, m.ModifyTime)
+	return
+}
+
+// FactorEdbSeriesCalculatePars 计算参数
+type FactorEdbSeriesCalculatePars struct {
+	Formula       interface{} `description:"N值/移动天数/指数修匀alpha值/计算公式等"`
+	Calendar      string      `description:"公历/农历"`
+	Frequency     string      `description:"需要转换的频度"`
+	MoveType      int         `description:"移动方式: 1-领先(默认); 2-滞后"`
+	MoveFrequency string      `description:"移动频度"`
+	FromFrequency string      `description:"来源的频度"`
+	Source        int         `description:"计算方式来源(不是指标来源)"`
+	Sort          int         `description:"计算顺序"`
+}
+
+// CreateSeriesAndMapping 新增系列和指标关联
+func (m *FactorEdbSeries) CreateSeriesAndMapping(item *FactorEdbSeries, mappings []*FactorEdbSeriesMapping) (seriesId int, err error) {
+	if item == nil {
+		err = fmt.Errorf("series is nil")
+		return
+	}
+	o := orm.NewOrm()
+	tx, e := o.Begin()
+	if e != nil {
+		err = fmt.Errorf("orm begin err: %v", e)
+		return
+	}
+	defer func() {
+		if err != nil {
+			_ = tx.Rollback()
+			return
+		}
+		_ = tx.Commit()
+	}()
+
+	id, e := tx.Insert(item)
+	if e != nil {
+		err = fmt.Errorf("insert series err: %v", e)
+		return
+	}
+	seriesId = int(id)
+	item.FactorEdbSeriesId = seriesId
+
+	if len(mappings) > 0 {
+		for _, v := range mappings {
+			v.FactorEdbSeriesId = seriesId
+		}
+		_, e = tx.InsertMulti(200, mappings)
+		if e != nil {
+			err = fmt.Errorf("insert multi mapping err: %v", e)
+			return
+		}
+	}
+	return
+}
+
+// EditSeriesAndMapping 编辑系列和指标关联
+func (m *FactorEdbSeries) EditSeriesAndMapping(item *FactorEdbSeries, mappings []*FactorEdbSeriesMapping, updateCols []string) (err error) {
+	if item == nil {
+		err = fmt.Errorf("series is nil")
+		return
+	}
+	o := orm.NewOrm()
+	tx, e := o.Begin()
+	if e != nil {
+		err = fmt.Errorf("orm begin err: %v", e)
+		return
+	}
+	defer func() {
+		if err != nil {
+			_ = tx.Rollback()
+			return
+		}
+		_ = tx.Commit()
+	}()
+
+	_, e = tx.Update(item, updateCols...)
+	if e != nil {
+		err = fmt.Errorf("update series err: %v", e)
+		return
+	}
+
+	// 清除原指标关联
+	mappingOb := new(FactorEdbSeriesMapping)
+	cond := fmt.Sprintf("%s = ?", mappingOb.Cols().FactorEdbSeriesId)
+	pars := make([]interface{}, 0)
+	pars = append(pars, item.FactorEdbSeriesId)
+	sql := fmt.Sprintf(`DELETE FROM %s WHERE %s`, mappingOb.TableName(), cond)
+	_, e = tx.Raw(sql, pars).Exec()
+	if e != nil {
+		err = fmt.Errorf("remove mapping err: %v", e)
+		return
+	}
+
+	if len(mappings) > 0 {
+		for _, v := range mappings {
+			v.FactorEdbSeriesId = item.FactorEdbSeriesId
+		}
+		_, e = tx.InsertMulti(200, mappings)
+		if e != nil {
+			err = fmt.Errorf("insert multi mapping err: %v", e)
+			return
+		}
+	}
+	return
+}
+
+// FactorEdbSeriesStepCalculateResp 批量计算响应
+type FactorEdbSeriesStepCalculateResp struct {
+	SeriesId int                                  `description:"多因子指标系列ID"`
+	Fail     []FactorEdbSeriesStepCalculateResult `description:"计算失败的指标"`
+	Success  []FactorEdbSeriesStepCalculateResult `description:"计算成功的指标"`
+}
+
+// FactorEdbSeriesStepCalculateResult 批量计算结果
+type FactorEdbSeriesStepCalculateResult struct {
+	EdbInfoId int    `description:"指标ID"`
+	EdbCode   string `description:"指标编码"`
+	Msg       string `description:"提示信息"`
+	ErrMsg    string `description:"错误信息"`
+}
+
+// FactorEdbSeriesDetail 因子指标系列-详情
+type FactorEdbSeriesDetail struct {
+	*FactorEdbSeriesItem
+	EdbMappings []*FactorEdbSeriesMappingItem
+}
+
+// FactorEdbSeriesCorrelationMatrixResp 因子指标系列-相关性矩阵响应
+type FactorEdbSeriesCorrelationMatrixResp struct {
+	Fail    []FactorEdbSeriesCorrelationMatrixItem `description:"计算失败的指标"`
+	Success []FactorEdbSeriesCorrelationMatrixItem `description:"计算成功的指标"`
+}
+
+// FactorEdbSeriesCorrelationMatrixItem 因子指标系列-相关性矩阵信息
+type FactorEdbSeriesCorrelationMatrixItem struct {
+	SeriesId   int                                      `description:"因子指标系列ID"`
+	EdbInfoId  int                                      `description:"指标ID"`
+	EdbCode    string                                   `description:"指标编码"`
+	EdbName    string                                   `description:"指标名称"`
+	Values     []FactorEdbSeriesCorrelationMatrixValues `description:"X轴和Y轴数据"`
+	Msg        string                                   `description:"提示信息"`
+	ErrMsg     string                                   `description:"错误信息"`
+	Used       bool                                     `description:"是否选中"`
+	SourceName string                                   `description:"指标来源名称"`
+}
+
+// FactorEdbSeriesCorrelationMatrixValues 因子指标系列-相关性矩阵XY值
+type FactorEdbSeriesCorrelationMatrixValues struct {
+	XData int     `description:"X轴数据"`
+	YData float64 `description:"Y轴数据"`
+}
+
+// FactorEdbSeriesCorrelationMatrixOrder 排序规则[0 1 2 3 -1 -2 -3]
+type FactorEdbSeriesCorrelationMatrixOrder []FactorEdbSeriesCorrelationMatrixValues
+
+func (a FactorEdbSeriesCorrelationMatrixOrder) Len() int {
+	return len(a)
+}
+
+func (a FactorEdbSeriesCorrelationMatrixOrder) Swap(i, j int) {
+	a[i], a[j] = a[j], a[i]
+}
+
+func (a FactorEdbSeriesCorrelationMatrixOrder) Less(i, j int) bool {
+	// 非负数优先
+	if a[i].XData >= 0 && a[j].XData < 0 {
+		return true
+	}
+	if a[i].XData < 0 && a[j].XData >= 0 {
+		return false
+	}
+	// 非负数升序排序
+	if a[i].XData >= 0 {
+		return a[i].XData < a[j].XData
+	}
+	// 负数按绝对值的降序排序（即数值的升序）
+	return a[i].XData > a[j].XData
+}
+
+// FactorEdbRecalculateReq 因子指标重新计算
+type FactorEdbRecalculateReq struct {
+	EdbInfoId int    `description:"指标ID"`
+	EdbCode   string `description:"指标编码"`
+}
+
+// FactorEdbChartRecalculateReq 因子指标关联的图表数据重计算
+type FactorEdbChartRecalculateReq struct {
+	ChartInfoId int `description:"图表ID"`
+	//EdbInfoId   int    `description:"指标ID"`
+	//EdbCode     string `description:"指标编码"`
+}

models/factor_edb_series_calculate_data.go

@@ -0,0 +1,190 @@
+package models
+
+import (
+	"eta/eta_index_lib/utils"
+	"fmt"
+	"github.com/beego/beego/v2/client/orm"
+	"strings"
+	"time"
+)
+
+// FactorEdbSeriesCalculateData 因子指标系列-指标计算数据表
+type FactorEdbSeriesCalculateData struct {
+	FactorEdbSeriesCalculateDataId int       `orm:"column(factor_edb_series_calculate_data_id);pk"`
+	FactorEdbSeriesId              int       `description:"因子指标系列ID"`
+	EdbInfoId                      int       `description:"指标ID"`
+	EdbCode                        string    `description:"指标编码"`
+	DataTime                       time.Time `description:"数据日期"`
+	Value                          float64   `description:"数据值"`
+	CreateTime                     time.Time `description:"创建时间"`
+	ModifyTime                     time.Time `description:"修改时间"`
+	DataTimestamp                  int64     `description:"数据日期时间戳"`
+}
+
+func (m *FactorEdbSeriesCalculateData) TableName() string {
+	return "factor_edb_series_calculate_data"
+}
+
+type FactorEdbSeriesCalculateDataCols struct {
+	PrimaryId         string
+	FactorEdbSeriesId string
+	EdbInfoId         string
+	EdbCode           string
+	DataTime          string
+	Value             string
+	CreateTime        string
+	ModifyTime        string
+	DataTimestamp     string
+}
+
+func (m *FactorEdbSeriesCalculateData) Cols() FactorEdbSeriesCalculateDataCols {
+	return FactorEdbSeriesCalculateDataCols{
+		PrimaryId:         "factor_edb_series_calculate_data_id",
+		FactorEdbSeriesId: "factor_edb_series_id",
+		EdbInfoId:         "edb_info_id",
+		EdbCode:           "edb_code",
+		DataTime:          "data_time",
+		Value:             "value",
+		CreateTime:        "create_time",
+		ModifyTime:        "modify_time",
+		DataTimestamp:     "data_timestamp",
+	}
+}
+
+func (m *FactorEdbSeriesCalculateData) Create() (err error) {
+	o := orm.NewOrm()
+	id, err := o.Insert(m)
+	if err != nil {
+		return
+	}
+	m.FactorEdbSeriesCalculateDataId = int(id)
+	return
+}
+
+func (m *FactorEdbSeriesCalculateData) CreateMulti(items []*FactorEdbSeriesCalculateData) (err error) {
+	if len(items) == 0 {
+		return
+	}
+	o := orm.NewOrm()
+	_, err = o.InsertMulti(500, items)
+	return
+}
+
+func (m *FactorEdbSeriesCalculateData) Update(cols []string) (err error) {
+	o := orm.NewOrm()
+	_, err = o.Update(m, cols...)
+	return
+}
+
+func (m *FactorEdbSeriesCalculateData) Remove() (err error) {
+	o := orm.NewOrm()
+	sql := fmt.Sprintf(`DELETE FROM %s WHERE %s = ? LIMIT 1`, m.TableName(), m.Cols().PrimaryId)
+	_, err = o.Raw(sql, m.FactorEdbSeriesCalculateDataId).Exec()
+	return
+}
+
+func (m *FactorEdbSeriesCalculateData) MultiRemove(ids []int) (err error) {
+	if len(ids) == 0 {
+		return
+	}
+	o := orm.NewOrm()
+	sql := fmt.Sprintf(`DELETE FROM %s WHERE %s IN (%s)`, m.TableName(), m.Cols().PrimaryId, utils.GetOrmInReplace(len(ids)))
+	_, err = o.Raw(sql, ids).Exec()
+	return
+}
+
+func (m *FactorEdbSeriesCalculateData) RemoveByCondition(condition string, pars []interface{}) (err error) {
+	if condition == "" {
+		return
+	}
+	o := orm.NewOrm()
+	sql := fmt.Sprintf(`DELETE FROM %s WHERE %s`, m.TableName(), condition)
+	_, err = o.Raw(sql, pars).Exec()
+	return
+}
+
+func (m *FactorEdbSeriesCalculateData) GetItemById(id int) (item *FactorEdbSeriesCalculateData, err error) {
+	o := orm.NewOrm()
+	sql := fmt.Sprintf(`SELECT * FROM %s WHERE %s = ? LIMIT 1`, m.TableName(), m.Cols().PrimaryId)
+	err = o.Raw(sql, id).QueryRow(&item)
+	return
+}
+
+func (m *FactorEdbSeriesCalculateData) GetItemByCondition(condition string, pars []interface{}, orderRule string) (item *FactorEdbSeriesCalculateData, err error) {
+	o := orm.NewOrm()
+	order := ``
+	if orderRule != "" {
+		order = ` ORDER BY ` + orderRule
+	}
+	sql := fmt.Sprintf(`SELECT * FROM %s WHERE 1=1 %s %s LIMIT 1`, m.TableName(), condition, order)
+	err = o.Raw(sql, pars).QueryRow(&item)
+	return
+}
+
+func (m *FactorEdbSeriesCalculateData) GetCountByCondition(condition string, pars []interface{}) (count int, err error) {
+	o := orm.NewOrm()
+	sql := fmt.Sprintf(`SELECT COUNT(1) FROM %s WHERE 1=1 %s`, m.TableName(), condition)
+	err = o.Raw(sql, pars).QueryRow(&count)
+	return
+}
+
+func (m *FactorEdbSeriesCalculateData) GetItemsByCondition(condition string, pars []interface{}, fieldArr []string, orderRule string) (items []*FactorEdbSeriesCalculateData, err error) {
+	o := orm.NewOrm()
+	fields := strings.Join(fieldArr, ",")
+	if len(fieldArr) == 0 {
+		fields = `*`
+	}
+	order := fmt.Sprintf(`ORDER BY %s DESC`, m.Cols().CreateTime)
+	if orderRule != "" {
+		order = ` ORDER BY ` + orderRule
+	}
+	sql := fmt.Sprintf(`SELECT %s FROM %s WHERE 1=1 %s %s`, fields, m.TableName(), condition, order)
+	_, err = o.Raw(sql, pars).QueryRows(&items)
+	return
+}
+
+func (m *FactorEdbSeriesCalculateData) GetPageItemsByCondition(condition string, pars []interface{}, fieldArr []string, orderRule string, startSize, pageSize int) (items []*FactorEdbSeriesCalculateData, err error) {
+	o := orm.NewOrm()
+	fields := strings.Join(fieldArr, ",")
+	if len(fieldArr) == 0 {
+		fields = `*`
+	}
+	order := fmt.Sprintf(`ORDER BY %s DESC`, m.Cols().CreateTime)
+	if orderRule != "" {
+		order = ` ORDER BY ` + orderRule
+	}
+	sql := fmt.Sprintf(`SELECT %s FROM %s WHERE 1=1 %s %s LIMIT ?,?`, fields, m.TableName(), condition, order)
+	_, err = o.Raw(sql, pars, startSize, pageSize).QueryRows(&items)
+	return
+}
+
+// FactorEdbSeriesCalculateDataItem 因子指标系列-计算数据信息
+type FactorEdbSeriesCalculateDataItem struct {
+	DataId            int     `description:"数据ID"`
+	FactorEdbSeriesId int     `description:"因子指标系列ID"`
+	EdbInfoId         int     `description:"指标ID"`
+	EdbCode           string  `description:"指标编码"`
+	DataTime          string  `description:"数据日期"`
+	Value             float64 `description:"数据值"`
+}
+
+func (m *FactorEdbSeriesCalculateData) Format2Item() (item *FactorEdbSeriesCalculateDataItem) {
+	item = new(FactorEdbSeriesCalculateDataItem)
+	item.DataId = m.FactorEdbSeriesCalculateDataId
+	item.FactorEdbSeriesId = m.FactorEdbSeriesId
+	item.EdbInfoId = m.EdbInfoId
+	item.EdbCode = m.EdbCode
+	return
+}
+
+// TransEdbSeriesCalculateData2EdbDataList 转换数据格式
+func TransEdbSeriesCalculateData2EdbDataList(items []*FactorEdbSeriesCalculateData) (list []*EdbInfoSearchData) {
+	list = make([]*EdbInfoSearchData, 0)
+	for _, v := range items {
+		list = append(list, &EdbInfoSearchData{
+			DataTime: v.DataTime.Format(utils.FormatDate),
+			Value:    v.Value,
+		})
+	}
+	return
+}

models/factor_edb_series_calculate_func.go

@@ -0,0 +1,153 @@
+package models
+
+import (
+	"eta/eta_index_lib/utils"
+	"fmt"
+	"github.com/beego/beego/v2/client/orm"
+	"strings"
+	"time"
+)
+
+// FactorEdbSeriesCalculateFunc 多因子系列-计算方式表
+type FactorEdbSeriesCalculateFunc struct {
+	FactorEdbSeriesCalculateFuncId int       `orm:"column(factor_edb_series_calculate_func_id);pk"`
+	CalculateName                  string    `description:"计算方式名称"`
+	Source                         int       `description:"计算方式来源"`
+	EdbInfoType                    int       `description:"指标计算类型：0-普通指标；1-预测指标"`
+	CreateTime                     time.Time `description:"创建时间"`
+	ModifyTime                     time.Time `description:"修改时间"`
+}
+
+func (m *FactorEdbSeriesCalculateFunc) TableName() string {
+	return "factor_edb_series_calculate_func"
+}
+
+type FactorEdbSeriesCalculateFuncCols struct {
+	PrimaryId     string
+	CalculateName string
+	Source        string
+	EdbInfoType   string
+	CreateTime    string
+	ModifyTime    string
+}
+
+func (m *FactorEdbSeriesCalculateFunc) Cols() FactorEdbSeriesCalculateFuncCols {
+	return FactorEdbSeriesCalculateFuncCols{
+		PrimaryId:     "factor_edb_series_calculate_func_id",
+		CalculateName: "calculate_name",
+		Source:        "source",
+		EdbInfoType:   "edb_info_type",
+		CreateTime:    "create_time",
+		ModifyTime:    "modify_time",
+	}
+}
+
+func (m *FactorEdbSeriesCalculateFunc) Create() (err error) {
+	o := orm.NewOrm()
+	id, err := o.Insert(m)
+	if err != nil {
+		return
+	}
+	m.FactorEdbSeriesCalculateFuncId = int(id)
+	return
+}
+
+func (m *FactorEdbSeriesCalculateFunc) CreateMulti(items []*FactorEdbSeriesCalculateFunc) (err error) {
+	if len(items) == 0 {
+		return
+	}
+	o := orm.NewOrm()
+	_, err = o.InsertMulti(len(items), items)
+	return
+}
+
+func (m *FactorEdbSeriesCalculateFunc) Update(cols []string) (err error) {
+	o := orm.NewOrm()
+	_, err = o.Update(m, cols...)
+	return
+}
+
+func (m *FactorEdbSeriesCalculateFunc) Remove() (err error) {
+	o := orm.NewOrm()
+	sql := fmt.Sprintf(`DELETE FROM %s WHERE %s = ? LIMIT 1`, m.TableName(), m.Cols().PrimaryId)
+	_, err = o.Raw(sql, m.FactorEdbSeriesCalculateFuncId).Exec()
+	return
+}
+
+func (m *FactorEdbSeriesCalculateFunc) MultiRemove(ids []int) (err error) {
+	if len(ids) == 0 {
+		return
+	}
+	o := orm.NewOrm()
+	sql := fmt.Sprintf(`DELETE FROM %s WHERE %s IN (%s)`, m.TableName(), m.Cols().PrimaryId, utils.GetOrmInReplace(len(ids)))
+	_, err = o.Raw(sql, ids).Exec()
+	return
+}
+
+func (m *FactorEdbSeriesCalculateFunc) GetItemById(id int) (item *FactorEdbSeriesCalculateFunc, err error) {
+	o := orm.NewOrm()
+	sql := fmt.Sprintf(`SELECT * FROM %s WHERE %s = ? LIMIT 1`, m.TableName(), m.Cols().PrimaryId)
+	err = o.Raw(sql, id).QueryRow(&item)
+	return
+}
+
+func (m *FactorEdbSeriesCalculateFunc) GetItemByCondition(condition string, pars []interface{}, orderRule string) (item *FactorEdbSeriesCalculateFunc, err error) {
+	o := orm.NewOrm()
+	order := ``
+	if orderRule != "" {
+		order = ` ORDER BY ` + orderRule
+	}
+	sql := fmt.Sprintf(`SELECT * FROM %s WHERE 1=1 %s %s LIMIT 1`, m.TableName(), condition, order)
+	err = o.Raw(sql, pars).QueryRow(&item)
+	return
+}
+
+func (m *FactorEdbSeriesCalculateFunc) GetCountByCondition(condition string, pars []interface{}) (count int, err error) {
+	o := orm.NewOrm()
+	sql := fmt.Sprintf(`SELECT COUNT(1) FROM %s WHERE 1=1 %s`, m.TableName(), condition)
+	err = o.Raw(sql, pars).QueryRow(&count)
+	return
+}
+
+func (m *FactorEdbSeriesCalculateFunc) GetItemsByCondition(condition string, pars []interface{}, fieldArr []string, orderRule string) (items []*FactorEdbSeriesCalculateFunc, err error) {
+	o := orm.NewOrm()
+	fields := strings.Join(fieldArr, ",")
+	if len(fieldArr) == 0 {
+		fields = `*`
+	}
+	order := fmt.Sprintf(`ORDER BY %s DESC`, m.Cols().CreateTime)
+	if orderRule != "" {
+		order = ` ORDER BY ` + orderRule
+	}
+	sql := fmt.Sprintf(`SELECT %s FROM %s WHERE 1=1 %s %s`, fields, m.TableName(), condition, order)
+	_, err = o.Raw(sql, pars).QueryRows(&items)
+	return
+}
+
+func (m *FactorEdbSeriesCalculateFunc) GetPageItemsByCondition(condition string, pars []interface{}, fieldArr []string, orderRule string, startSize, pageSize int) (items []*FactorEdbSeriesCalculateFunc, err error) {
+	o := orm.NewOrm()
+	fields := strings.Join(fieldArr, ",")
+	if len(fieldArr) == 0 {
+		fields = `*`
+	}
+	order := fmt.Sprintf(`ORDER BY %s DESC`, m.Cols().CreateTime)
+	if orderRule != "" {
+		order = ` ORDER BY ` + orderRule
+	}
+	sql := fmt.Sprintf(`SELECT %s FROM %s WHERE 1=1 %s %s LIMIT ?,?`, fields, m.TableName(), condition, order)
+	_, err = o.Raw(sql, pars, startSize, pageSize).QueryRows(&items)
+	return
+}
+
+// FactorEdbSeriesCalculateFuncItem 多因子系列-计算方式
+type FactorEdbSeriesCalculateFuncItem struct {
+	CalculateName string `description:"计算方式名称"`
+	Source        int    `description:"计算方式来源"`
+}
+
+func (m *FactorEdbSeriesCalculateFunc) Format2Item() (item *FactorEdbSeriesCalculateFuncItem) {
+	item = new(FactorEdbSeriesCalculateFuncItem)
+	item.CalculateName = m.CalculateName
+	item.Source = m.Source
+	return
+}

models/factor_edb_series_chart_mapping.go

@@ -0,0 +1,176 @@
+package models
+
+import (
+	"eta/eta_index_lib/utils"
+	"fmt"
+	"github.com/beego/beego/v2/client/orm"
+	"strings"
+	"time"
+)
+
+const (
+	FactorEdbSeriesChartCalculateTypeCorrelation = 1 // 相关性计算
+)
+
+// FactorEdbSeriesChartMapping 因子指标系列-图表关联
+type FactorEdbSeriesChartMapping struct {
+	FactorEdbSeriesChartMappingId int       `orm:"column(factor_edb_series_chart_mapping_id);pk"`
+	ChartInfoId                   int       `description:"图表ID"`
+	Source                        int       `description:"图表来源, 同chart_info表source"`
+	CalculateType                 int       `description:"计算方式: 1-相关性"`
+	CalculatePars                 string    `description:"计算参数-JSON(如计算窗口等)"`
+	CalculateData                 string    `description:"计算数据-JSON(如相关性矩阵等)"`
+	FactorEdbSeriesId             int       `description:"因子指标系列ID"`
+	EdbInfoId                     int       `description:"指标ID"`
+	EdbUsed                       int       `description:"指标是否使用: 0-否; 1-是"`
+	CreateTime                    time.Time `description:"创建时间"`
+	ModifyTime                    time.Time `description:"修改时间"`
+}
+
+func (m *FactorEdbSeriesChartMapping) TableName() string {
+	return "factor_edb_series_chart_mapping"
+}
+
+type MultipleFactorSeriesChartMappingCols struct {
+	PrimaryId         string
+	ChartInfoId       string
+	Source            string
+	CalculateType     string
+	CalculatePars     string
+	CalculateData     string
+	FactorEdbSeriesId string
+	EdbInfoId         string
+	EdbUsed           string
+	CreateTime        string
+	ModifyTime        string
+}
+
+func (m *FactorEdbSeriesChartMapping) Cols() MultipleFactorSeriesChartMappingCols {
+	return MultipleFactorSeriesChartMappingCols{
+		PrimaryId:         "factor_edb_series_chart_mapping_id",
+		ChartInfoId:       "chart_info_id",
+		Source:            "source",
+		CalculateType:     "calculate_type",
+		CalculatePars:     "calculate_pars",
+		CalculateData:     "calculate_data",
+		FactorEdbSeriesId: "factor_edb_series_id",
+		EdbInfoId:         "edb_info_id",
+		EdbUsed:           "edb_used",
+		CreateTime:        "create_time",
+		ModifyTime:        "modify_time",
+	}
+}
+
+func (m *FactorEdbSeriesChartMapping) Create() (err error) {
+	o := orm.NewOrm()
+	id, err := o.Insert(m)
+	if err != nil {
+		return
+	}
+	m.FactorEdbSeriesChartMappingId = int(id)
+	return
+}
+
+func (m *FactorEdbSeriesChartMapping) CreateMulti(items []*FactorEdbSeriesChartMapping) (err error) {
+	if len(items) == 0 {
+		return
+	}
+	o := orm.NewOrm()
+	_, err = o.InsertMulti(len(items), items)
+	return
+}
+
+func (m *FactorEdbSeriesChartMapping) Update(cols []string) (err error) {
+	o := orm.NewOrm()
+	_, err = o.Update(m, cols...)
+	return
+}
+
+func (m *FactorEdbSeriesChartMapping) Remove() (err error) {
+	o := orm.NewOrm()
+	sql := fmt.Sprintf(`DELETE FROM %s WHERE %s = ? LIMIT 1`, m.TableName(), m.Cols().PrimaryId)
+	_, err = o.Raw(sql, m.FactorEdbSeriesChartMappingId).Exec()
+	return
+}
+
+func (m *FactorEdbSeriesChartMapping) MultiRemove(ids []int) (err error) {
+	if len(ids) == 0 {
+		return
+	}
+	o := orm.NewOrm()
+	sql := fmt.Sprintf(`DELETE FROM %s WHERE %s IN (%s)`, m.TableName(), m.Cols().PrimaryId, utils.GetOrmInReplace(len(ids)))
+	_, err = o.Raw(sql, ids).Exec()
+	return
+}
+
+func (m *FactorEdbSeriesChartMapping) GetItemById(id int) (item *FactorEdbSeriesChartMapping, err error) {
+	o := orm.NewOrm()
+	sql := fmt.Sprintf(`SELECT * FROM %s WHERE %s = ? LIMIT 1`, m.TableName(), m.Cols().PrimaryId)
+	err = o.Raw(sql, id).QueryRow(&item)
+	return
+}
+
+func (m *FactorEdbSeriesChartMapping) GetItemByCondition(condition string, pars []interface{}, orderRule string) (item *FactorEdbSeriesChartMapping, err error) {
+	o := orm.NewOrm()
+	order := ``
+	if orderRule != "" {
+		order = ` ORDER BY ` + orderRule
+	}
+	sql := fmt.Sprintf(`SELECT * FROM %s WHERE 1=1 %s %s LIMIT 1`, m.TableName(), condition, order)
+	err = o.Raw(sql, pars).QueryRow(&item)
+	return
+}
+
+func (m *FactorEdbSeriesChartMapping) GetCountByCondition(condition string, pars []interface{}) (count int, err error) {
+	o := orm.NewOrm()
+	sql := fmt.Sprintf(`SELECT COUNT(1) FROM %s WHERE 1=1 %s`, m.TableName(), condition)
+	err = o.Raw(sql, pars).QueryRow(&count)
+	return
+}
+
+func (m *FactorEdbSeriesChartMapping) GetItemsByCondition(condition string, pars []interface{}, fieldArr []string, orderRule string) (items []*FactorEdbSeriesChartMapping, err error) {
+	o := orm.NewOrm()
+	fields := strings.Join(fieldArr, ",")
+	if len(fieldArr) == 0 {
+		fields = `*`
+	}
+	order := fmt.Sprintf(`ORDER BY %s DESC`, m.Cols().CreateTime)
+	if orderRule != "" {
+		order = ` ORDER BY ` + orderRule
+	}
+	sql := fmt.Sprintf(`SELECT %s FROM %s WHERE 1=1 %s %s`, fields, m.TableName(), condition, order)
+	_, err = o.Raw(sql, pars).QueryRows(&items)
+	return
+}
+
+func (m *FactorEdbSeriesChartMapping) GetPageItemsByCondition(condition string, pars []interface{}, fieldArr []string, orderRule string, startSize, pageSize int) (items []*FactorEdbSeriesChartMapping, err error) {
+	o := orm.NewOrm()
+	fields := strings.Join(fieldArr, ",")
+	if len(fieldArr) == 0 {
+		fields = `*`
+	}
+	order := fmt.Sprintf(`ORDER BY %s DESC`, m.Cols().CreateTime)
+	if orderRule != "" {
+		order = ` ORDER BY ` + orderRule
+	}
+	sql := fmt.Sprintf(`SELECT %s FROM %s WHERE 1=1 %s %s LIMIT ?,?`, fields, m.TableName(), condition, order)
+	_, err = o.Raw(sql, pars, startSize, pageSize).QueryRows(&items)
+	return
+}
+
+// GetDistinctSeriesIdByChartId 获取图表关联的系列ID
+func (m *FactorEdbSeriesChartMapping) GetDistinctSeriesIdByChartId(chartId int) (seriesIds []int, err error) {
+	o := orm.NewOrm()
+	sql := fmt.Sprintf(`SELECT DISTINCT %s FROM %s WHERE %s = ?`, m.Cols().FactorEdbSeriesId, m.TableName(), m.Cols().ChartInfoId)
+	_, err = o.Raw(sql, chartId).QueryRows(&seriesIds)
+	return
+}
+
+// FactorEdbSeriesChartCalculateCorrelationReq 图表相关性计算参数
+type FactorEdbSeriesChartCalculateCorrelationReq struct {
+	BaseEdbInfoId  int    `description:"标的指标ID"`
+	LeadValue      int    `description:"领先期数"`
+	LeadUnit       string `description:"频度"`
+	CalculateValue int    `description:"计算窗口"`
+	CalculateUnit  string `description:"计算频度"`
+}

models/factor_edb_series_mapping.go

@@ -0,0 +1,167 @@
+package models
+
+import (
+	"eta/eta_index_lib/utils"
+	"fmt"
+	"github.com/beego/beego/v2/client/orm"
+	"strings"
+	"time"
+)
+
+// FactorEdbSeriesMapping 因子指标系列-指标关联表
+type FactorEdbSeriesMapping struct {
+	FactorEdbSeriesMappingId int       `orm:"column(factor_edb_series_mapping_id);pk"`
+	FactorEdbSeriesId        int       `description:"因子指标系列ID"`
+	EdbInfoId                int       `description:"指标ID"`
+	EdbCode                  string    `description:"指标编码"`
+	CreateTime               time.Time `description:"创建时间"`
+	ModifyTime               time.Time `description:"修改时间"`
+}
+
+func (m *FactorEdbSeriesMapping) TableName() string {
+	return "factor_edb_series_mapping"
+}
+
+type FactorEdbSeriesMappingCols struct {
+	PrimaryId         string
+	FactorEdbSeriesId string
+	EdbInfoId         string
+	EdbCode           string
+	CreateTime        string
+	ModifyTime        string
+}
+
+func (m *FactorEdbSeriesMapping) Cols() FactorEdbSeriesMappingCols {
+	return FactorEdbSeriesMappingCols{
+		PrimaryId:         "factor_edb_series_mapping_id",
+		FactorEdbSeriesId: "factor_edb_series_id",
+		EdbInfoId:         "edb_info_id",
+		EdbCode:           "edb_code",
+		CreateTime:        "create_time",
+		ModifyTime:        "modify_time",
+	}
+}
+
+func (m *FactorEdbSeriesMapping) Create() (err error) {
+	o := orm.NewOrm()
+	id, err := o.Insert(m)
+	if err != nil {
+		return
+	}
+	m.FactorEdbSeriesMappingId = int(id)
+	return
+}
+
+func (m *FactorEdbSeriesMapping) CreateMulti(items []*FactorEdbSeriesMapping) (err error) {
+	if len(items) == 0 {
+		return
+	}
+	o := orm.NewOrm()
+	_, err = o.InsertMulti(len(items), items)
+	return
+}
+
+func (m *FactorEdbSeriesMapping) Update(cols []string) (err error) {
+	o := orm.NewOrm()
+	_, err = o.Update(m, cols...)
+	return
+}
+
+func (m *FactorEdbSeriesMapping) Remove() (err error) {
+	o := orm.NewOrm()
+	sql := fmt.Sprintf(`DELETE FROM %s WHERE %s = ? LIMIT 1`, m.TableName(), m.Cols().PrimaryId)
+	_, err = o.Raw(sql, m.FactorEdbSeriesMappingId).Exec()
+	return
+}
+
+func (m *FactorEdbSeriesMapping) MultiRemove(ids []int) (err error) {
+	if len(ids) == 0 {
+		return
+	}
+	o := orm.NewOrm()
+	sql := fmt.Sprintf(`DELETE FROM %s WHERE %s IN (%s)`, m.TableName(), m.Cols().PrimaryId, utils.GetOrmInReplace(len(ids)))
+	_, err = o.Raw(sql, ids).Exec()
+	return
+}
+
+func (m *FactorEdbSeriesMapping) RemoveByCondition(condition string, pars []interface{}) (err error) {
+	if condition == "" {
+		return
+	}
+	o := orm.NewOrm()
+	sql := fmt.Sprintf(`DELETE FROM %s WHERE %s`, m.TableName(), condition)
+	_, err = o.Raw(sql, pars).Exec()
+	return
+}
+
+func (m *FactorEdbSeriesMapping) GetItemById(id int) (item *FactorEdbSeriesMapping, err error) {
+	o := orm.NewOrm()
+	sql := fmt.Sprintf(`SELECT * FROM %s WHERE %s = ? LIMIT 1`, m.TableName(), m.Cols().PrimaryId)
+	err = o.Raw(sql, id).QueryRow(&item)
+	return
+}
+
+func (m *FactorEdbSeriesMapping) GetItemByCondition(condition string, pars []interface{}, orderRule string) (item *FactorEdbSeriesMapping, err error) {
+	o := orm.NewOrm()
+	order := ``
+	if orderRule != "" {
+		order = ` ORDER BY ` + orderRule
+	}
+	sql := fmt.Sprintf(`SELECT * FROM %s WHERE 1=1 %s %s LIMIT 1`, m.TableName(), condition, order)
+	err = o.Raw(sql, pars).QueryRow(&item)
+	return
+}
+
+func (m *FactorEdbSeriesMapping) GetCountByCondition(condition string, pars []interface{}) (count int, err error) {
+	o := orm.NewOrm()
+	sql := fmt.Sprintf(`SELECT COUNT(1) FROM %s WHERE 1=1 %s`, m.TableName(), condition)
+	err = o.Raw(sql, pars).QueryRow(&count)
+	return
+}
+
+func (m *FactorEdbSeriesMapping) GetItemsByCondition(condition string, pars []interface{}, fieldArr []string, orderRule string) (items []*FactorEdbSeriesMapping, err error) {
+	o := orm.NewOrm()
+	fields := strings.Join(fieldArr, ",")
+	if len(fieldArr) == 0 {
+		fields = `*`
+	}
+	order := fmt.Sprintf(`ORDER BY %s DESC`, m.Cols().CreateTime)
+	if orderRule != "" {
+		order = ` ORDER BY ` + orderRule
+	}
+	sql := fmt.Sprintf(`SELECT %s FROM %s WHERE 1=1 %s %s`, fields, m.TableName(), condition, order)
+	_, err = o.Raw(sql, pars).QueryRows(&items)
+	return
+}
+
+func (m *FactorEdbSeriesMapping) GetPageItemsByCondition(condition string, pars []interface{}, fieldArr []string, orderRule string, startSize, pageSize int) (items []*FactorEdbSeriesMapping, err error) {
+	o := orm.NewOrm()
+	fields := strings.Join(fieldArr, ",")
+	if len(fieldArr) == 0 {
+		fields = `*`
+	}
+	order := fmt.Sprintf(`ORDER BY %s DESC`, m.Cols().CreateTime)
+	if orderRule != "" {
+		order = ` ORDER BY ` + orderRule
+	}
+	sql := fmt.Sprintf(`SELECT %s FROM %s WHERE 1=1 %s %s LIMIT ?,?`, fields, m.TableName(), condition, order)
+	_, err = o.Raw(sql, pars, startSize, pageSize).QueryRows(&items)
+	return
+}
+
+// FactorEdbSeriesMappingItem 因子指标系列-指标关联信息
+type FactorEdbSeriesMappingItem struct {
+	SeriesId  int    `description:"因子指标系列ID"`
+	EdbInfoId int    `description:"指标ID"`
+	EdbCode   string `description:"指标编码"`
+	EdbName   string `description:"指标名称"`
+	EdbNameEn string `description:"指标名称-英文"`
+}
+
+func (m *FactorEdbSeriesMapping) Format2Item() (item *FactorEdbSeriesMappingItem) {
+	item = new(FactorEdbSeriesMappingItem)
+	item.SeriesId = m.FactorEdbSeriesId
+	item.EdbInfoId = m.EdbInfoId
+	item.EdbCode = m.EdbCode
+	return
+}

models/mgo/base_from_business_data.go

@@ -0,0 +1,452 @@
+package mgo
+
+import (
+	"context"
+	"errors"
+	"eta/eta_index_lib/utils"
+	"fmt"
+	"github.com/qiniu/qmgo"
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+	"time"
+)
+
+// BaseFromBusinessData
+// @Description: 自有数据集合
+type BaseFromBusinessData struct {
+	ID primitive.ObjectID `json:"_id" bson:"_id,omitempty"` // 文档id
+	//ID string `json:"_id" bson:"_id" ` // 文档id
+	BaseFromBusinessIndexId int64     `json:"base_from_business_index_id" bson:"base_from_business_index_id"` // 指标id
+	IndexCode               string    `json:"index_code" bson:"index_code"`                                   // 指标编码
+	DataTime                time.Time `json:"data_time" bson:"data_time"`                                     // 数据日期
+	Value                   float64   `json:"value" bson:"value"`                                             // 数据值
+	CreateTime              time.Time `json:"create_time" bson:"create_time"`                                 // 创建时间
+	ModifyTime              time.Time `json:"modify_time" bson:"modify_time"`                                 // 修改时间
+}
+
+// CollectionName
+// @Description:  获取集合名称
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-26 13:41:36
+// @return string
+func (m *BaseFromBusinessData) CollectionName() string {
+	return "base_from_business_data"
+}
+
+// DataBaseName
+// @Description: 获取数据库名称
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-26 13:41:33
+// @return string
+func (m *BaseFromBusinessData) DataBaseName() string {
+	return utils.MgoDataDbName
+}
+
+// GetCollection
+// @Description: 获取mongodb集合的句柄
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-26 13:41:33
+// @return string
+func (m *BaseFromBusinessData) GetCollection() *qmgo.Collection {
+	db := utils.MgoDataCli.Database(m.DataBaseName())
+	return db.Collection(m.CollectionName())
+}
+
+// GetAllDataList
+// @Description: 根据条件获取所有数据
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-26 13:42:19
+// @param sort []string
+// @param whereParams interface{}
+// @return result []BaseFromBusinessData
+// @return err error
+func (m *BaseFromBusinessData) GetAllDataList(whereParams interface{}, sort []string) (result []*BaseFromBusinessData, err error) {
+	if utils.MgoDataCli == nil {
+		err = errors.New("mongodb连接失败")
+		return
+	}
+	db := utils.MgoDataCli.Database(m.DataBaseName())
+	coll := db.Collection(m.CollectionName())
+	ctx := context.TODO()
+	if err != nil {
+		fmt.Println("MgoGetColl Err:", err.Error())
+		return
+	}
+	err = coll.Find(ctx, whereParams).Sort(sort...).All(&result)
+	if err != nil {
+		return
+	}
+
+	for _, v := range result {
+		v.DataTime = v.DataTime.In(time.Local)
+		v.CreateTime = v.CreateTime.In(time.Local)
+		v.ModifyTime = v.ModifyTime.In(time.Local)
+	}
+
+	return
+}
+
+// GetLimitDataList
+// @Description: 根据条件获取指定数量数据列表
+// @author: Roc
+// @receiver m
+// @datetime 2024-05-06 17:08:32
+// @param whereParams interface{}
+// @param size int64
+// @return result []*BaseFromBusinessData
+// @return err error
+func (m *BaseFromBusinessData) GetLimitDataList(whereParams interface{}, size int64, sort []string) (result []*BaseFromBusinessData, err error) {
+	if utils.MgoDataCli == nil {
+		err = errors.New("mongodb连接失败")
+		return
+	}
+	db := utils.MgoDataCli.Database(m.DataBaseName())
+	coll := db.Collection(m.CollectionName())
+	ctx := context.TODO()
+	if err != nil {
+		fmt.Println("MgoGetColl Err:", err.Error())
+		return
+	}
+	err = coll.Find(ctx, whereParams).Sort(sort...).Limit(size).All(&result)
+	if err != nil {
+		return
+	}
+
+	for _, v := range result {
+		v.DataTime = v.DataTime.In(time.Local)
+		v.CreateTime = v.CreateTime.In(time.Local)
+		v.ModifyTime = v.ModifyTime.In(time.Local)
+	}
+
+	return
+}
+
+// GetPageDataList
+// @Description: 根据条件获取分页数据列表
+// @author: Roc
+// @receiver m
+// @datetime 2024-05-07 10:21:07
+// @param whereParams interface{}
+// @param startSize int64
+// @param size int64
+// @param sort []string
+// @return result []*BaseFromBusinessData
+// @return err error
+func (m *BaseFromBusinessData) GetPageDataList(whereParams interface{}, startSize, size int64, sort []string) (result []*BaseFromBusinessData, err error) {
+	if utils.MgoDataCli == nil {
+		err = errors.New("mongodb连接失败")
+		return
+	}
+	db := utils.MgoDataCli.Database(m.DataBaseName())
+	coll := db.Collection(m.CollectionName())
+	ctx := context.TODO()
+	if err != nil {
+		fmt.Println("MgoGetColl Err:", err.Error())
+		return
+	}
+	err = coll.Find(ctx, whereParams).Sort(sort...).Skip(startSize).Limit(size).All(&result)
+	if err != nil {
+		return
+	}
+
+	for _, v := range result {
+		v.DataTime = v.DataTime.In(time.Local)
+		v.CreateTime = v.CreateTime.In(time.Local)
+		v.ModifyTime = v.ModifyTime.In(time.Local)
+	}
+
+	return
+}
+
+// GetCountDataList
+// @Description:  根据条件获取数据列表总数
+// @author: Roc
+// @receiver m
+// @datetime 2024-05-07 10:29:00
+// @param whereParams interface{}
+// @return count int64
+// @return err error
+func (m *BaseFromBusinessData) GetCountDataList(whereParams interface{}) (count int64, err error) {
+	if utils.MgoDataCli == nil {
+		err = errors.New("mongodb连接失败")
+		return
+	}
+	db := utils.MgoDataCli.Database(m.DataBaseName())
+	coll := db.Collection(m.CollectionName())
+	ctx := context.TODO()
+	if err != nil {
+		fmt.Println("MgoGetColl Err:", err.Error())
+		return
+	}
+	count, err = coll.Find(ctx, whereParams).Count()
+
+	return
+}
+
+// InsertDataByColl
+// @Description: 写入单条数据（外部传入集合）
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-26 14:22:18
+// @param addData interface{}
+// @return err error
+func (m *BaseFromBusinessData) InsertDataByColl(coll *qmgo.Collection, addData interface{}) (err error) {
+	ctx := context.TODO()
+	_, err = coll.InsertOne(ctx, addData)
+	if err != nil {
+		fmt.Println("InsertDataByColl:Err:" + err.Error())
+		return
+	}
+
+	return
+}
+
+// BatchInsertData
+// @Description: 批量写入数据
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-26 14:22:18
+// @param bulk int 每次请求保存的数据量
+// @param dataList []interface{}
+// @return err error
+func (m *BaseFromBusinessData) BatchInsertData(bulk int, dataList []interface{}) (err error) {
+	db := utils.MgoDataCli.Database(m.DataBaseName())
+	coll := db.Collection(m.CollectionName())
+
+	return m.BatchInsertDataByColl(coll, bulk, dataList)
+}
+
+// BatchInsertDataByColl
+// @Description: 批量写入数据（外部传入集合）
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-26 14:22:18
+// @param coll *qmgo.Collection
+// @param bulk int 每次请求保存的数据量
+// @param dataList []interface{}
+// @return err error
+func (m *BaseFromBusinessData) BatchInsertDataByColl(coll *qmgo.Collection, bulk int, dataList []interface{}) (err error) {
+	ctx := context.TODO()
+	dataNum := len(dataList)
+	if dataNum <= 0 {
+		return
+	}
+
+	// 不设置每次保存切片数量大小，或者实际数据量小于设置的切片数量大小，那么就直接保存吧
+	if bulk <= 0 || dataNum <= bulk {
+		_, err = coll.InsertMany(ctx, dataList)
+		if err != nil {
+			fmt.Println("BatchInsertData:Err:" + err.Error())
+			return
+		}
+		return
+	}
+
+	// 分批保存
+	i := 0
+	tmpAddDataList := make([]interface{}, 0)
+	for _, v := range dataList {
+		tmpAddDataList = append(tmpAddDataList, v)
+		i++
+		if i >= bulk {
+			_, err = coll.InsertMany(ctx, tmpAddDataList)
+			if err != nil {
+				fmt.Println("BatchInsertData:Err:" + err.Error())
+				return
+			}
+			i = 0
+			tmpAddDataList = make([]interface{}, 0)
+		}
+	}
+
+	if len(tmpAddDataList) > 0 {
+		_, err = coll.InsertMany(ctx, tmpAddDataList)
+		if err != nil {
+			fmt.Println("BatchInsertData:Err:" + err.Error())
+			return
+		}
+	}
+
+	return
+}
+
+// UpdateDataByColl
+// @Description: 单条数据修改
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-26 15:01:51
+// @param whereParams interface{}
+// @param updateParams interface{}
+// @return err error
+func (m *BaseFromBusinessData) UpdateDataByColl(coll *qmgo.Collection, whereParams, updateParams interface{}) (err error) {
+	ctx := context.TODO()
+	err = coll.UpdateOne(ctx, whereParams, updateParams)
+	if err != nil {
+		fmt.Println("UpdateDataByColl:Err:" + err.Error())
+		return
+	}
+	return
+}
+
+// UpdateData
+// @Description: 单条数据修改
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-26 15:01:51
+// @param whereParams interface{}
+// @param updateParams interface{}
+// @return err error
+func (m *BaseFromBusinessData) UpdateData(whereParams, updateParams interface{}) (err error) {
+	db := utils.MgoDataCli.Database(m.DataBaseName())
+	coll := db.Collection(m.CollectionName())
+	ctx := context.TODO()
+	err = coll.UpdateOne(ctx, whereParams, updateParams)
+	if err != nil {
+		fmt.Println("UpdateData:Err:" + err.Error())
+		return
+	}
+	return
+}
+
+// HandleData
+// @Description: 事务处理数据
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-30 10:40:20
+// @param addDataList []BaseAddFromBusinessData
+// @param updateDataList []BaseFromBusinessData
+// @return result interface{}
+// @return err error
+func (m *BaseFromBusinessData) HandleData(addDataList, updateDataList []BaseFromBusinessData) (result interface{}, err error) {
+
+	ctx := context.TODO()
+
+	callback := func(sessCtx context.Context) (interface{}, error) {
+		// 重要：确保事务中的每一个操作，都使用传入的sessCtx参数
+
+		db := utils.MgoDataCli.Database(m.DataBaseName())
+		coll := db.Collection(m.CollectionName())
+
+		// 插入数据
+		if len(addDataList) > 0 {
+			_, err = coll.InsertMany(sessCtx, addDataList)
+			if err != nil {
+				return nil, err
+			}
+		}
+
+		// 修改
+
+		if len(updateDataList) > 0 {
+			for _, v := range updateDataList {
+				err = coll.UpdateOne(ctx, bson.M{"_id": v.ID}, bson.M{"$set": bson.M{"value": v.Value, "modify_time": v.ModifyTime}})
+				if err != nil {
+					fmt.Println("BatchInsertData:Err:" + err.Error())
+					return nil, err
+				}
+			}
+		}
+
+		return nil, nil
+	}
+	result, err = utils.MgoDataCli.DoTransaction(ctx, callback)
+
+	return
+}
+
+// GetEdbInfoMaxAndMinInfo
+// @Description: 获取当前指标的最大最小值
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-30 17:15:39
+// @param whereParams interface{}
+// @return result EdbInfoMaxAndMinInfo
+// @return err error
+func (m *BaseFromBusinessData) GetEdbInfoMaxAndMinInfo(whereParams interface{}) (result EdbInfoMaxAndMinInfo, err error) {
+	if utils.MgoDataCli == nil {
+		err = errors.New("mongodb连接失败")
+		return
+	}
+	db := utils.MgoDataCli.Database(m.DataBaseName())
+	coll := db.Collection(m.CollectionName())
+	ctx := context.TODO()
+	if err != nil {
+		fmt.Println("MgoGetColl Err:", err.Error())
+		return
+	}
+	err = coll.Aggregate(ctx, whereParams).One(&result)
+	if err != nil {
+		return
+	}
+	result.MinDate = result.MinDate.In(time.Local)
+	result.MaxDate = result.MaxDate.In(time.Local)
+	result.LatestDate = result.LatestDate.In(time.Local)
+
+	return
+}
+
+// GetLatestValue
+// @Description: 获取当前指标的最新数据记录
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-30 17:16:15
+// @param whereParams interface{}
+// @param selectParam interface{}
+// @return latestValue LatestValue
+// @return err error
+func (m *BaseFromBusinessData) GetLatestValue(whereParams, selectParam interface{}) (latestValue LatestValue, err error) {
+	if utils.MgoDataCli == nil {
+		err = errors.New("mongodb连接失败")
+		return
+	}
+	db := utils.MgoDataCli.Database(m.DataBaseName())
+	coll := db.Collection(m.CollectionName())
+	ctx := context.TODO()
+	if err != nil {
+		fmt.Println("MgoGetColl Err:", err.Error())
+		return
+	}
+
+	//var result interface{}
+	//err = coll.Find(ctx, whereParams).Select(selectParam).One(&result)
+	err = coll.Find(ctx, whereParams).Select(selectParam).One(&latestValue)
+
+	return
+}
+
+// RemoveMany
+// @Description: 根据条件删除多条数据
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-30 13:17:02
+// @param whereParams interface{}
+// @return err error
+func (m *BaseFromBusinessData) RemoveMany(whereParams interface{}) (err error) {
+	db := utils.MgoDataCli.Database(m.DataBaseName())
+	coll := db.Collection(m.CollectionName())
+
+	return m.RemoveManyByColl(coll, whereParams)
+}
+
+// RemoveManyByColl
+// @Description: 根据条件删除多条数据（外部传入集合）
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-30 13:18:42
+// @param coll *qmgo.Collection
+// @param whereParams interface{}
+// @return err error
+func (m *BaseFromBusinessData) RemoveManyByColl(coll *qmgo.Collection, whereParams interface{}) (err error) {
+	ctx := context.TODO()
+	_, err = coll.RemoveAll(ctx, whereParams)
+	if err != nil {
+		fmt.Println("RemoveManyByColl:Err:" + err.Error())
+		return
+	}
+
+	return
+}

models/mgo/common.go

@@ -0,0 +1,58 @@
+package mgo
+
+import (
+	"eta/eta_index_lib/utils"
+	"go.mongodb.org/mongo-driver/bson"
+	"time"
+)
+
+// BuildDateCondition
+// @Description:  构建日期查询条件
+// @author: Roc
+// @datetime 2024-06-03 09:41:19
+// @param start string
+// @param end string
+// @return condition bson.M
+// @return err error
+func BuildDateCondition(start, end string) (condition bson.M, err error) {
+	var startDateTime, endDateTime time.Time
+	if start != "" {
+		// 使用开始日期条件
+		startDateTime, err = time.ParseInLocation(utils.FormatDate, start, time.Local)
+		if err != nil {
+			return
+		}
+	}
+	if end != "" {
+		// 使用结束日期条件
+		endDateTime, err = time.ParseInLocation(utils.FormatDate, end, time.Local)
+		if err != nil {
+			return
+		}
+	}
+
+	return BuildDateTimeCondition(startDateTime, endDateTime)
+
+}
+
+// BuildDateTimeCondition
+// @Description: 构建日期查询条件
+// @author: Roc
+// @datetime 2024-06-03 09:47:32
+// @param startDateTime time.Time
+// @param endDateTime time.Time
+// @return condition bson.M
+// @return err error
+func BuildDateTimeCondition(startDateTime, endDateTime time.Time) (condition bson.M, err error) {
+	if !startDateTime.IsZero() && !endDateTime.IsZero() {
+		condition = bson.M{utils.DateConvMysqlConvMongo(">="): startDateTime, utils.DateConvMysqlConvMongo("<="): endDateTime}
+	} else if !startDateTime.IsZero() {
+		cond := utils.DateConvMysqlConvMongo(">=")
+		condition = bson.M{cond: startDateTime}
+	} else if !endDateTime.IsZero() {
+		cond := utils.DateConvMysqlConvMongo("<=")
+		condition = bson.M{cond: endDateTime}
+	}
+
+	return
+}

models/mgo/edb_data_business.go

@@ -0,0 +1,511 @@
+package mgo
+
+import (
+	"context"
+	"errors"
+	"eta/eta_index_lib/utils"
+	"fmt"
+	"github.com/qiniu/qmgo"
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+	"time"
+)
+
+// EdbDataBusiness
+// @Description: 自有数据集合（指标库）
+type EdbDataBusiness struct {
+	ID            primitive.ObjectID `json:"_id" bson:"_id,omitempty" `            // 文档id
+	EdbInfoId     int                `json:"edb_info_id" bson:"edb_info_id"`       // 指标编码
+	EdbCode       string             `json:"edb_code" bson:"edb_code"`             // 指标编码
+	DataTime      time.Time          `json:"data_time" bson:"data_time"`           // 数据日期
+	Value         float64            `json:"value" bson:"value"`                   // 数据值
+	CreateTime    time.Time          `json:"create_time" bson:"create_time"`       // 创建时间
+	ModifyTime    time.Time          `json:"modify_time" bson:"modify_time"`       // 修改时间
+	DataTimestamp int64              `json:"data_timestamp" bson:"data_timestamp"` // 数据日期时间戳
+}
+
+// CollectionName
+// @Description:  获取集合名称
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-26 13:41:36
+// @return string
+func (m *EdbDataBusiness) CollectionName() string {
+	return "edb_data_business"
+}
+
+// DataBaseName
+// @Description: 获取数据库名称
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-26 13:41:33
+// @return string
+func (m *EdbDataBusiness) DataBaseName() string {
+	return utils.MgoDataDbName
+}
+
+// GetCollection
+// @Description: 获取mongodb集合的句柄
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-26 13:41:33
+// @return string
+func (m *EdbDataBusiness) GetCollection() *qmgo.Collection {
+	db := utils.MgoDataCli.Database(m.DataBaseName())
+	return db.Collection(m.CollectionName())
+}
+
+// GetItem
+// @Description: 根据条件获取单条数据
+// @author: Roc
+// @receiver m
+// @datetime 2024-05-09 10:00:49
+// @param whereParams interface{}
+// @return item *EdbDataBusiness
+// @return err error
+func (m *EdbDataBusiness) GetItem(whereParams interface{}) (item *EdbDataBusiness, err error) {
+	if utils.MgoDataCli == nil {
+		err = errors.New("mongodb连接失败")
+		return
+	}
+	db := utils.MgoDataCli.Database(m.DataBaseName())
+	coll := db.Collection(m.CollectionName())
+
+	return m.GetItemByColl(coll, whereParams)
+}
+
+// GetItemByColl
+// @Description: 根据条件获取单条数据
+// @author: Roc
+// @receiver m
+// @datetime 2024-05-09 13:22:06
+// @param coll *qmgo.Collection
+// @param whereParams interface{}
+// @return item *EdbDataBusiness
+// @return err error
+func (m *EdbDataBusiness) GetItemByColl(coll *qmgo.Collection, whereParams interface{}) (item *EdbDataBusiness, err error) {
+	ctx := context.TODO()
+	if err != nil {
+		fmt.Println("MgoGetColl Err:", err.Error())
+		return
+	}
+	err = coll.Find(ctx, whereParams).One(&item)
+	if err != nil {
+		return
+	}
+
+	item.DataTime = item.DataTime.In(time.Local)
+	item.CreateTime = item.CreateTime.In(time.Local)
+	item.ModifyTime = item.ModifyTime.In(time.Local)
+
+	return
+}
+
+// GetAllDataList
+// @Description: 根据条件获取所有数据
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-26 13:42:19
+// @param whereParams interface{}
+// @param sort []string
+// @return result []EdbDataBusiness
+// @return err error
+func (m *EdbDataBusiness) GetAllDataList(whereParams interface{}, sort []string) (result []*EdbDataBusiness, err error) {
+	if utils.MgoDataCli == nil {
+		err = errors.New("mongodb连接失败")
+		return
+	}
+	db := utils.MgoDataCli.Database(m.DataBaseName())
+	coll := db.Collection(m.CollectionName())
+	ctx := context.TODO()
+	if err != nil {
+		fmt.Println("MgoGetColl Err:", err.Error())
+		return
+	}
+	err = coll.Find(ctx, whereParams).Sort(sort...).All(&result)
+	if err != nil {
+		return
+	}
+
+	for _, v := range result {
+		v.DataTime = v.DataTime.In(time.Local)
+		v.CreateTime = v.CreateTime.In(time.Local)
+		v.ModifyTime = v.ModifyTime.In(time.Local)
+	}
+
+	return
+}
+
+// GetLimitDataList
+// @Description: 根据条件获取指定数量数据列表
+// @author: Roc
+// @receiver m
+// @datetime 2024-05-06 17:08:32
+// @param whereParams interface{}
+// @param size int64
+// @param sort []string
+// @return result []*BaseFromBusinessData
+// @return err error
+func (m *EdbDataBusiness) GetLimitDataList(whereParams interface{}, size int64, sort []string) (result []*EdbDataBusiness, err error) {
+	if utils.MgoDataCli == nil {
+		err = errors.New("mongodb连接失败")
+		return
+	}
+	db := utils.MgoDataCli.Database(m.DataBaseName())
+	coll := db.Collection(m.CollectionName())
+	ctx := context.TODO()
+	if err != nil {
+		fmt.Println("MgoGetColl Err:", err.Error())
+		return
+	}
+	err = coll.Find(ctx, whereParams).Sort(sort...).Limit(size).All(&result)
+	if err != nil {
+		return
+	}
+
+	for _, v := range result {
+		v.DataTime = v.DataTime.In(time.Local)
+		v.CreateTime = v.CreateTime.In(time.Local)
+		v.ModifyTime = v.ModifyTime.In(time.Local)
+	}
+
+	return
+}
+
+// GetPageDataList
+// @Description: 根据条件获取分页数据列表
+// @author: Roc
+// @receiver m
+// @datetime 2024-05-07 10:21:07
+// @param whereParams interface{}
+// @param startSize int64
+// @param size int64
+// @param sort []string
+// @return result []*EdbDataBusiness
+// @return err error
+func (m *EdbDataBusiness) GetPageDataList(whereParams interface{}, startSize, size int64, sort []string) (result []*EdbDataBusiness, err error) {
+	if utils.MgoDataCli == nil {
+		err = errors.New("mongodb连接失败")
+		return
+	}
+	db := utils.MgoDataCli.Database(m.DataBaseName())
+	coll := db.Collection(m.CollectionName())
+	ctx := context.TODO()
+	if err != nil {
+		fmt.Println("MgoGetColl Err:", err.Error())
+		return
+	}
+	err = coll.Find(ctx, whereParams).Sort(sort...).Skip(startSize).Limit(size).All(&result)
+	if err != nil {
+		return
+	}
+
+	for _, v := range result {
+		v.DataTime = v.DataTime.In(time.Local)
+		v.CreateTime = v.CreateTime.In(time.Local)
+		v.ModifyTime = v.ModifyTime.In(time.Local)
+	}
+
+	return
+}
+
+// GetCountDataList
+// @Description:  根据条件获取数据列表总数
+// @author: Roc
+// @receiver m
+// @datetime 2024-05-07 10:29:00
+// @param whereParams interface{}
+// @return count int64
+// @return err error
+func (m *EdbDataBusiness) GetCountDataList(whereParams interface{}) (count int64, err error) {
+	if utils.MgoDataCli == nil {
+		err = errors.New("mongodb连接失败")
+		return
+	}
+	db := utils.MgoDataCli.Database(m.DataBaseName())
+	coll := db.Collection(m.CollectionName())
+	ctx := context.TODO()
+	if err != nil {
+		fmt.Println("MgoGetColl Err:", err.Error())
+		return
+	}
+	count, err = coll.Find(ctx, whereParams).Count()
+
+	return
+}
+
+// InsertDataByColl
+// @Description: 写入单条数据（外部传入集合）
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-26 14:22:18
+// @param addData interface{}
+// @return err error
+func (m *EdbDataBusiness) InsertDataByColl(coll *qmgo.Collection, addData interface{}) (err error) {
+	ctx := context.TODO()
+	_, err = coll.InsertOne(ctx, addData)
+	if err != nil {
+		fmt.Println("InsertDataByColl:Err:" + err.Error())
+		return
+	}
+
+	return
+}
+
+// BatchInsertData
+// @Description: 批量写入数据
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-26 14:22:18
+// @param bulk int 每次请求保存的数据量
+// @param dataList []interface{}
+// @return err error
+func (m *EdbDataBusiness) BatchInsertData(bulk int, dataList []interface{}) (err error) {
+	db := utils.MgoDataCli.Database(m.DataBaseName())
+	coll := db.Collection(m.CollectionName())
+
+	return m.BatchInsertDataByColl(coll, bulk, dataList)
+}
+
+// BatchInsertDataByColl
+// @Description: 批量写入数据（外部传入集合）
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-26 14:22:18
+// @param coll *qmgo.Collection
+// @param bulk int 每次请求保存的数据量
+// @param dataList []interface{}
+// @return err error
+func (m *EdbDataBusiness) BatchInsertDataByColl(coll *qmgo.Collection, bulk int, dataList []interface{}) (err error) {
+	ctx := context.TODO()
+	dataNum := len(dataList)
+	if dataNum <= 0 {
+		return
+	}
+
+	// 不设置每次保存切片数量大小，或者实际数据量小于设置的切片数量大小，那么就直接保存吧
+	if bulk <= 0 || dataNum <= bulk {
+		_, err = coll.InsertMany(ctx, dataList)
+		if err != nil {
+			fmt.Println("BatchInsertData:Err:" + err.Error())
+			return
+		}
+		return
+	}
+
+	// 分批保存
+	i := 0
+	tmpAddDataList := make([]interface{}, 0)
+	for _, v := range dataList {
+		tmpAddDataList = append(tmpAddDataList, v)
+		i++
+		if i >= bulk {
+			_, err = coll.InsertMany(ctx, tmpAddDataList)
+			if err != nil {
+				fmt.Println("BatchInsertData:Err:" + err.Error())
+				return
+			}
+			i = 0
+			tmpAddDataList = make([]interface{}, 0)
+		}
+	}
+
+	if len(tmpAddDataList) > 0 {
+		_, err = coll.InsertMany(ctx, tmpAddDataList)
+		if err != nil {
+			fmt.Println("BatchInsertData:Err:" + err.Error())
+			return
+		}
+	}
+
+	return
+}
+
+// UpdateData
+// @Description: 单条数据修改
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-26 15:01:51
+// @param whereParams interface{}
+// @param updateParams interface{}
+// @return err error
+func (m *EdbDataBusiness) UpdateData(whereParams, updateParams interface{}) (err error) {
+	db := utils.MgoDataCli.Database(m.DataBaseName())
+	coll := db.Collection(m.CollectionName())
+
+	return m.UpdateDataByColl(coll, whereParams, updateParams)
+}
+
+// UpdateDataByColl
+// @Description: 单条数据修改（外部传入集合）
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-26 15:01:51
+// @param whereParams interface{}
+// @param updateParams interface{}
+// @return err error
+func (m *EdbDataBusiness) UpdateDataByColl(coll *qmgo.Collection, whereParams, updateParams interface{}) (err error) {
+	ctx := context.TODO()
+	err = coll.UpdateOne(ctx, whereParams, updateParams)
+	if err != nil {
+		fmt.Println("UpdateDataByColl:Err:" + err.Error())
+		return
+	}
+
+	return
+}
+
+// RemoveMany
+// @Description: 根据条件删除多条数据
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-30 13:17:02
+// @param whereParams interface{}
+// @return err error
+func (m *EdbDataBusiness) RemoveMany(whereParams interface{}) (err error) {
+	db := utils.MgoDataCli.Database(m.DataBaseName())
+	coll := db.Collection(m.CollectionName())
+
+	return m.RemoveManyByColl(coll, whereParams)
+}
+
+// RemoveManyByColl
+// @Description: 根据条件删除多条数据（外部传入集合）
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-30 13:18:42
+// @param coll *qmgo.Collection
+// @param whereParams interface{}
+// @return err error
+func (m *EdbDataBusiness) RemoveManyByColl(coll *qmgo.Collection, whereParams interface{}) (err error) {
+	ctx := context.TODO()
+	_, err = coll.RemoveAll(ctx, whereParams)
+	if err != nil {
+		fmt.Println("RemoveManyByColl:Err:" + err.Error())
+		return
+	}
+
+	return
+}
+
+// HandleData
+// @Description: 事务处理数据
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-30 10:39:01
+// @param addDataList []AddEdbDataBusiness
+// @param updateDataList []EdbDataBusiness
+// @return result interface{}
+// @return err error
+func (m *EdbDataBusiness) HandleData(addDataList, updateDataList []EdbDataBusiness) (result interface{}, err error) {
+
+	ctx := context.TODO()
+
+	callback := func(sessCtx context.Context) (interface{}, error) {
+		// 重要：确保事务中的每一个操作，都使用传入的sessCtx参数
+
+		db := utils.MgoDataCli.Database(m.DataBaseName())
+		coll := db.Collection(m.CollectionName())
+
+		// 插入数据
+		if len(addDataList) > 0 {
+			_, err = coll.InsertMany(sessCtx, addDataList)
+			if err != nil {
+				return nil, err
+			}
+		}
+
+		// 修改
+
+		if len(updateDataList) > 0 {
+			for _, v := range updateDataList {
+				err = coll.UpdateOne(ctx, bson.M{"_id": v.ID}, bson.M{"$set": bson.M{"value": v.Value, "modify_time": v.ModifyTime}})
+				if err != nil {
+					fmt.Println("BatchInsertData:Err:" + err.Error())
+					return nil, err
+				}
+			}
+		}
+
+		return nil, nil
+	}
+	result, err = utils.MgoDataCli.DoTransaction(ctx, callback)
+
+	return
+}
+
+// EdbInfoMaxAndMinInfo 指标最新数据记录结构体
+type EdbInfoMaxAndMinInfo struct {
+	MinDate     time.Time `description:"最小日期" bson:"min_date"`
+	MaxDate     time.Time `description:"最大日期" bson:"max_date"`
+	MinValue    float64   `description:"最小值" bson:"min_value"`
+	MaxValue    float64   `description:"最大值" bson:"max_value"`
+	LatestValue float64   `description:"最新值" bson:"latest_value"`
+	LatestDate  time.Time `description:"实际数据最新日期" bson:"latest_date"`
+	EndValue    float64   `description:"最新值" bson:"end_value"`
+}
+
+// GetEdbInfoMaxAndMinInfo
+// @Description: 获取当前指标的最大最小值
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-30 17:15:39
+// @param whereParams interface{}
+// @return result EdbInfoMaxAndMinInfo
+// @return err error
+func (m *EdbDataBusiness) GetEdbInfoMaxAndMinInfo(whereParams interface{}) (result EdbInfoMaxAndMinInfo, err error) {
+	if utils.MgoDataCli == nil {
+		err = errors.New("mongodb连接失败")
+		return
+	}
+	db := utils.MgoDataCli.Database(m.DataBaseName())
+	coll := db.Collection(m.CollectionName())
+	ctx := context.TODO()
+	if err != nil {
+		fmt.Println("MgoGetColl Err:", err.Error())
+		return
+	}
+	err = coll.Aggregate(ctx, whereParams).One(&result)
+	if err != nil {
+		return
+	}
+	result.MinDate = result.MinDate.In(time.Local)
+	result.MaxDate = result.MaxDate.In(time.Local)
+	result.LatestDate = result.LatestDate.In(time.Local)
+
+	return
+}
+
+// LatestValue 指标最新数据记录结构体
+type LatestValue struct {
+	Value float64 `description:"值" bson:"value"`
+}
+
+// GetLatestValue
+// @Description: 获取当前指标的最新数据记录
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-30 17:16:15
+// @param whereParams interface{}
+// @param selectParam interface{}
+// @return latestValue LatestValue
+// @return err error
+func (m *EdbDataBusiness) GetLatestValue(whereParams, selectParam interface{}) (latestValue LatestValue, err error) {
+	if utils.MgoDataCli == nil {
+		err = errors.New("mongodb连接失败")
+		return
+	}
+	db := utils.MgoDataCli.Database(m.DataBaseName())
+	coll := db.Collection(m.CollectionName())
+	ctx := context.TODO()
+	if err != nil {
+		fmt.Println("MgoGetColl Err:", err.Error())
+		return
+	}
+
+	//var result interface{}
+	//err = coll.Find(ctx, whereParams).Select(selectParam).One(&result)
+	err = coll.Find(ctx, whereParams).Select(selectParam).One(&latestValue)
+
+	return
+}

models/mgo_base.go

@@ -0,0 +1,274 @@
+package models
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"eta/eta_index_lib/utils"
+	"eta/eta_index_lib/utils/mgodb"
+	"fmt"
+)
+
+func init() {
+	if utils.MgoUrlData != `` {
+		var mgoConfig mgodb.MgoConfig
+		if e := json.Unmarshal([]byte(utils.MgoUrlData), &mgoConfig); e != nil {
+			panic("mongodb链接失败，Err:" + e.Error())
+			return
+		}
+
+		mgoCli := mgodb.MgoNewClient(mgoConfig)
+		utils.MgoDataCli = mgoCli
+		utils.MgoDataDbName = mgoConfig.Database
+		//result, err := TestMgoFindOne("data_ths", bson.M{"edbcode": "s0033227", "datatime": "2015-12-04"})
+		//fmt.Println(time.Now())
+		//result, err := TestMgoFindOne("data_wind", bson.M{"edbcode": "s0033227", "datatime": "2007-12-29"})
+		//fmt.Println(time.Now())
+		//
+		//fmt.Println(err)
+		//fmt.Println(result)
+	}
+
+}
+
+// TestMgoFindOne
+// @Description: 获取单条数据
+// @author: Roc
+// @datetime 2024-04-25 15:44:07
+// @param colName string 集合名词：`data_ths`
+// @param whereParams interface{} bson.M{"edbcode": "s0033227", "datatime": "2015-12-04"}
+// @return result interface{}
+// @return err error
+func TestMgoFindOne(colName string, whereParams interface{}) (result interface{}, err error) {
+	if colName == "" {
+		err = errors.New("集合名称不可为空")
+		return
+	}
+	if utils.MgoDataCli == nil {
+		err = errors.New("mongodb连接失败")
+		return
+	}
+	db := utils.MgoDataCli.Database("hz_data")
+	coll := db.Collection("data_ths")
+	ctx := context.TODO()
+	if err != nil {
+		fmt.Println("MgoGetColl Err:", err.Error())
+		return
+	}
+	//err = coll.Find(ctx, whereParams).One(&result)
+	err = coll.Find(ctx, whereParams).All(&result)
+	return
+}
+
+// 创建集合
+//func MgoCreateCollection(collectionName string) (err error) {
+//	ctx := context.TODO()
+//
+//	mgoClient := mgodb.MgoNewClient()
+//
+//	defer func() {
+//		mgoClient.Close(ctx)
+//	}()
+//
+//	colName := collectionName
+//	err = mgoClient.Database("hz_data").CreateCollection(ctx, colName)
+//	return
+//}
+
+// 批量新增多条数据
+func MgoBatchInsertData(colName string, dataList interface{}) (err error) {
+	if colName == "" {
+		err = errors.New("集合名称不可为空")
+		return
+	}
+	ctx := context.TODO()
+	coll, err := mgodb.MgoGetColl(colName)
+	if err != nil {
+		fmt.Println("MgoGetColl Err:", err.Error())
+		return
+	}
+	defer func() {
+		coll.Close(ctx)
+	}()
+	_, err = coll.Collection.InsertMany(ctx, dataList)
+	if err != nil {
+		fmt.Println("InsertMany:Err:" + err.Error())
+		return
+	}
+	return
+}
+
+// 插入单条数据
+func MgoInsertData(colName string, dataItem interface{}) (insertedID interface{}, err error) {
+	if colName == "" {
+		err = errors.New("集合名称不可为空")
+		return
+	}
+	ctx := context.TODO()
+	coll, err := mgodb.MgoGetColl(colName)
+	if err != nil {
+		fmt.Println("MgoGetColl Err:", err.Error())
+		return
+	}
+	defer func() {
+		coll.Close(ctx)
+	}()
+	result, err := coll.Collection.InsertOne(ctx, dataItem)
+	if err != nil {
+		return
+	}
+	insertedID = result.InsertedID
+	return
+}
+
+/*
+删除数据
+colName := `data_gl`
+whereParams := bson.M{"edbcode": "s0033227"}
+err := MgoRemove(colName, whereParams)
+*/
+func MgoRemove(colName string, whereParams interface{}) (err error) {
+	if colName == "" {
+		err = errors.New("集合名称不可为空")
+		return
+	}
+	ctx := context.TODO()
+	coll, err := mgodb.MgoGetColl(colName)
+	if err != nil {
+		fmt.Println("MgoGetColl Err:", err.Error())
+		return
+	}
+	defer func() {
+		coll.Close(ctx)
+	}()
+	err = coll.Remove(ctx, whereParams)
+	return
+}
+
+/*
+删除多条数据
+colName := `data_gl`
+whereParams := bson.M{"edbcode": "s0033227"}
+err := MgoRemoveAll(colName, whereParams)
+*/
+func MgoRemoveAll(colName string, whereParams interface{}) (count int64, err error) {
+	if colName == "" {
+		err = errors.New("集合名称不可为空")
+		return
+	}
+	ctx := context.TODO()
+	coll, err := mgodb.MgoGetColl(colName)
+	if err != nil {
+		fmt.Println("MgoGetColl Err:", err.Error())
+		return
+	}
+	defer func() {
+		coll.Close(ctx)
+	}()
+	res, err := coll.RemoveAll(ctx, whereParams)
+	if err != nil {
+		fmt.Println("RemoveAll Err:", err.Error())
+		return
+	}
+	count = res.DeletedCount
+	return
+}
+
+/*
+修改单条数据
+colName := `data_ths`
+whereParams := bson.M{"edbcode": "s0033227", "datatime": "2015-12-04"}
+params := bson.M{"$set": bson.M{"value": "1000"}}
+*/
+func MgoUpdateData(colName string, params, whereParams interface{}) (err error) {
+	if colName == "" {
+		err = errors.New("集合名称不可为空")
+		return
+	}
+	ctx := context.TODO()
+	coll, err := mgodb.MgoGetColl(colName)
+	if err != nil {
+		fmt.Println("MgoGetColl Err:", err.Error())
+		return
+	}
+	defer func() {
+		coll.Close(ctx)
+	}()
+	err = coll.UpdateOne(ctx, whereParams, params)
+	return
+}
+
+/*
+获取单条数据
+colName := `data_ths`
+whereParams := bson.M{"edbcode": "s0033227", "datatime": "2015-12-04"}
+*/
+func MgoFindOne(colName string, whereParams interface{}) (result interface{}, err error) {
+	if colName == "" {
+		err = errors.New("集合名称不可为空")
+		return
+	}
+	ctx := context.TODO()
+	coll, err := mgodb.MgoGetColl(colName)
+	if err != nil {
+		fmt.Println("MgoGetColl Err:", err.Error())
+		return
+	}
+	defer func() {
+		coll.Close(ctx)
+	}()
+	err = coll.Find(ctx, whereParams).One(&result)
+	return
+}
+
+/*
+分页获取数据
+colName := `data_ths`
+whereParams := bson.M{"edbcode": "s0033227", "datatime": "2015-12-04"}
+
+sort:="-weight"
+skip:0
+limit:10
+*/
+func MgoBatchFind(colName, sort string, whereParams interface{}, skip, limit int64) (list []interface{}, err error) {
+	if colName == "" {
+		err = errors.New("集合名称不可为空")
+		return
+	}
+	ctx := context.TODO()
+	coll, err := mgodb.MgoGetColl(colName)
+	if err != nil {
+		fmt.Println("MgoGetColl Err:", err.Error())
+		return
+	}
+	defer func() {
+		coll.Close(ctx)
+	}()
+	err = coll.Find(ctx, whereParams).Sort(sort).Skip(skip).Limit(limit).All(&list)
+	return
+}
+
+/*
+MgoUpsertData
+更新插入数据-数据存在即更新, 否则新增
+colName := `data_ths`
+filter := bson.M{"edbcode": "s0033227", "datatime": "2015-12-04"}
+replacement := bson.M{"edb_info_id":0,"edb_code":m.EdbCode,"data_time":d.DataTime,"value":f,"create_time":now,"modify_time":now,"data_timestamp": stamp}
+*/
+func MgoUpsertData(colName string, filter, replacement interface{}) (err error) {
+	if colName == "" {
+		err = errors.New("集合名称不可为空")
+		return
+	}
+	ctx := context.TODO()
+	coll, err := mgodb.MgoGetColl(colName)
+	if err != nil {
+		fmt.Println("MgoGetColl Err:", err.Error())
+		return
+	}
+	defer func() {
+		_ = coll.Close(ctx)
+	}()
+	_, err = coll.Upsert(ctx, filter, replacement)
+	return
+}

models/predict_edb.go

@@ -134,12 +134,13 @@ func CalculateByRuleBy9(to orm.TxOrmer, rule CalculateRule) (resultDataList []*E
 		return
 	}
 	dataMap := make(map[string]*PredictEdbRuleData)
+	removeDateMap := make(map[string]*PredictEdbRuleData) //需要移除的日期
 	for _, v := range dataList {
 		dataMap[v.DataTime] = v
+		removeDateMap[v.DataTime] = v
 	}
 	existDataMap := make(map[string]string)
 
-	removeDateList := make([]string, 0) //需要移除的日期
 	// 判断是否特殊处理max和min函数
 	maxDealFlag := false
 	if rule.EmptyType == 4 && rule.MaxEmptyType == 2 {
@@ -184,7 +185,6 @@ func CalculateByRuleBy9(to orm.TxOrmer, rule CalculateRule) (resultDataList []*E
 		formulaFormStr := ReplaceFormula(rule.EdbInfoList, sv, svMax, formulaMap, formulaStr, rule.EdbInfoIdBytes, maxDealFlag)
 		//计算公式异常，那么就移除该指标
 		if formulaFormStr == "" {
-			removeDateList = append(removeDateList, sk)
 			continue
 		}
 
@@ -194,7 +194,6 @@ func CalculateByRuleBy9(to orm.TxOrmer, rule CalculateRule) (resultDataList []*E
 		if tmpErr != nil {
 			// 分母为0的报错
 			if strings.Contains(tmpErr.Error(), "divide by zero") {
-				removeDateList = append(removeDateList, sk)
 				continue
 			}
 			err = errors.New("计算失败：Err:" + tmpErr.Error() + ";formulaStr:" + formulaFormStr)
@@ -209,9 +208,12 @@ func CalculateByRuleBy9(to orm.TxOrmer, rule CalculateRule) (resultDataList []*E
 		}
 		nanCheck := fmt.Sprintf("%0.f", calVal)
 		if nanCheck == "NaN" || nanCheck == "+Inf" || nanCheck == "-Inf" {
-			removeDateList = append(removeDateList, sk)
 			continue
 		}
+
+		// 移除不存在的日期
+		delete(removeDateMap, sk)
+
 		saveValue := decimal.NewFromFloat(calVal).RoundCeil(4).String() //utils.SubFloatToString(calVal, 4)
 		existPredictEdbRuleData, ok := dataMap[sk]
 		if !ok {
@@ -267,8 +269,12 @@ func CalculateByRuleBy9(to orm.TxOrmer, rule CalculateRule) (resultDataList []*E
 	}
 
 	//删除多余的值
-	lenRemoveDateList := len(removeDateList)
+	lenRemoveDateList := len(removeDateMap)
 	if lenRemoveDateList > 0 {
+		removeDateList := make([]string, 0) //需要移除的日期
+		for date, _ := range removeDateMap {
+			removeDateList = append(removeDateList, date)
+		}
 		//如果拼接指标变更了，那么需要删除所有的指标数据
 		sql := ` DELETE FROM predict_edb_rule_data WHERE config_id = ? and data_time in (` + utils.GetOrmInReplace(lenRemoveDateList) + `) `
 

models/predict_edb_data_base.go

@@ -408,6 +408,14 @@ func (obj PredictStandardBase) refresh(to orm.TxOrmer, edbInfo *EdbInfo, predict
 			//校验待删除日期数据里面是否存在该元素，如果存在的话，那么移除该日期
 			delete(removeDataTimeMap, currDateStr)
 
+			// 格式化库中的指标数据，并保留4位小数
+			existValDec, tmpErr := decimal.NewFromString(existVal)
+			if tmpErr != nil {
+				err = tmpErr
+				return
+			}
+			existVal = existValDec.Round(4).String()
+
 			if existVal != saveValue {
 				sql := ` UPDATE %s SET value=?,modify_time=NOW() WHERE edb_info_id=? AND data_time=? `
 				sql = fmt.Sprintf(sql, dataTableName)

models/predict_edb_data_calculate_ljztbpj.go

@@ -444,14 +444,12 @@ func refreshAllPredictCalculateLjztbpj(to orm.TxOrmer, edbInfo, firstEdbInfo, se
 
 	//同比值指标
 	{
-		var condition string
-		pars := make([]interface{}, 0)
-
-		condition += " AND data_time > ? AND edb_info_id = ? "
-		pars = append(pars, startCalculationDate, existItemB.FromEdbInfoId)
-
 		//第二个指标的数据列表
-		secondDataList, tmpErr := GetEdbDataListAllByTo(to, condition, pars, existItemB.FromSource, existItemB.FromSubSource, 0)
+		secondDataList, tmpErr := GetEdbDataListAllByTo(to, existItemB.FromSource, existItemB.FromSubSource, FindEdbDataListAllCond{
+			EdbInfoId:         existItemB.FromEdbInfoId,
+			StartDataTime:     startCalculationDate.Format(utils.FormatDate),
+			StartDataTimeCond: ">",
+		}, 0)
 		if tmpErr != nil {
 			err = tmpErr
 			return

models/predict_edb_data_calculate_percentile.go

@@ -382,13 +382,10 @@ func (obj PredictPercentile) getPercentileData(fromEdbInfo *EdbInfo, calculateVa
 	dataList := make([]*EdbInfoSearchData, 0)
 	switch fromEdbInfo.EdbInfoType {
 	case 0:
-		var condition string
-		var pars []interface{}
-		condition += " AND edb_info_id=? "
-		pars = append(pars, fromEdbInfo.EdbInfoId)
-
 		//获取来源指标的数据
-		dataList, err = GetEdbDataListAll(condition, pars, fromEdbInfo.Source, fromEdbInfo.SubSource, 1)
+		dataList, err = GetEdbDataListAll(fromEdbInfo.Source, fromEdbInfo.SubSource, FindEdbDataListAllCond{
+			EdbInfoId: fromEdbInfo.EdbInfoId,
+		}, 1)
 	case 1:
 		dataList, err = GetPredictEdbDataListAllByStartDate(fromEdbInfo, 1, "")
 	default:

models/predict_edb_data_calculate_standard_deviation.go

@@ -393,13 +393,10 @@ func (obj PredictStandardDeviation) getStandardDeviationData(fromEdbInfo *EdbInf
 	dataList := make([]*EdbInfoSearchData, 0)
 	switch fromEdbInfo.EdbInfoType {
 	case 0:
-		var condition string
-		var pars []interface{}
-		condition += " AND edb_info_id=? "
-		pars = append(pars, fromEdbInfo.EdbInfoId)
-
 		//获取来源指标的数据
-		dataList, err = GetEdbDataListAll(condition, pars, fromEdbInfo.Source, fromEdbInfo.SubSource, 1)
+		dataList, err = GetEdbDataListAll(fromEdbInfo.Source, fromEdbInfo.SubSource, FindEdbDataListAllCond{
+			EdbInfoId: fromEdbInfo.EdbInfoId,
+		}, 1)
 	case 1:
 		dataList, err = GetPredictEdbDataListAllByStartDate(fromEdbInfo, 1, "")
 	default:

models/predict_edb_data_calculate_time_shift.go

@@ -241,7 +241,9 @@ func refreshAllPredictCalculateTimeShift(to orm.TxOrmer, edbInfoId, source, subS
 	if err != nil {
 		return
 	}
+
 	existDataMap := make(map[string]string)
+	removeDateMap := make(map[string]string)
 	for _, v := range existDataList {
 		existDataMap[v.DataTime] = v.Value
 	}
@@ -284,6 +286,9 @@ func refreshAllPredictCalculateTimeShift(to orm.TxOrmer, edbInfoId, source, subS
 					}
 				}
 			}
+
+			// 指标已存在，那么就移除该日期
+			delete(removeDateMap, newDate.Format(utils.FormatDate))
 		}
 		existMap[existKey] = currentItem.DataTime
 	}
@@ -295,5 +300,19 @@ func refreshAllPredictCalculateTimeShift(to orm.TxOrmer, edbInfoId, source, subS
 			return
 		}
 	}
+	
+	// 移除不存在的日期数据
+	if len(removeDateMap) > 0 {
+		removeDateList := make([]string, 0) //需要移除的日期
+		for k := range removeDateMap {
+			removeDateList = append(removeDateList, k)
+		}
+		err = DelEdbDataByMysql(to, edbInfoId, dataTableName, removeDateList)
+		if err != nil {
+			err = fmt.Errorf("删除年化指标数据失败,Err:" + err.Error())
+			return
+		}
+	}
+
 	return
 }

models/predict_edb_info_rule.go

@@ -1265,13 +1265,10 @@ func getCalculateNhccData(secondDataList []*EdbInfoSearchData, ruleConf RuleLine
 		var firstDataList []*EdbInfoSearchData
 		switch edbInfo.EdbInfoType {
 		case 0:
-			var condition string
-			var pars []interface{}
-			condition += " AND edb_info_id=? "
-			pars = append(pars, edbInfo.EdbInfoId)
-
 			//获取来源指标的数据
-			firstDataList, err = GetEdbDataListAll(condition, pars, edbInfo.Source, edbInfo.SubSource, 1)
+			firstDataList, err = GetEdbDataListAll(edbInfo.Source, edbInfo.SubSource, FindEdbDataListAllCond{
+				EdbInfoId: edbInfo.EdbInfoId,
+			}, 1)
 		case 1:
 			firstDataList, err = GetPredictEdbDataListAllByStartDate(edbInfo, 1, "")
 		default:

routers/commentsRouter.go

@@ -7,6 +7,24 @@ import (
 
 func init() {
 
+    beego.GlobalControllerRouter["eta/eta_index_lib/controllers/factor_edb_series:FactorEdbSeriesController"] = append(beego.GlobalControllerRouter["eta/eta_index_lib/controllers/factor_edb_series:FactorEdbSeriesController"],
+        beego.ControllerComments{
+            Method: "ChartRecalculate",
+            Router: `/chart_recalculate`,
+            AllowHTTPMethods: []string{"post"},
+            MethodParams: param.Make(),
+            Filters: nil,
+            Params: nil})
+
+    beego.GlobalControllerRouter["eta/eta_index_lib/controllers/factor_edb_series:FactorEdbSeriesController"] = append(beego.GlobalControllerRouter["eta/eta_index_lib/controllers/factor_edb_series:FactorEdbSeriesController"],
+        beego.ControllerComments{
+            Method: "Recalculate",
+            Router: `/recalculate`,
+            AllowHTTPMethods: []string{"post"},
+            MethodParams: param.Make(),
+            Filters: nil,
+            Params: nil})
+
     beego.GlobalControllerRouter["eta/eta_index_lib/controllers/fix:CustomAnalysisController"] = append(beego.GlobalControllerRouter["eta/eta_index_lib/controllers/fix:CustomAnalysisController"],
         beego.ControllerComments{
             Method: "FixTableV1",
@@ -90,8 +108,8 @@ func init() {
 
     beego.GlobalControllerRouter["eta/eta_index_lib/controllers:BloombergController"] = append(beego.GlobalControllerRouter["eta/eta_index_lib/controllers:BloombergController"],
         beego.ControllerComments{
-            Method: "PCSGRefreshDaily",
-            Router: `/pcsg/refresh_daily`,
+            Method: "PCSGImportHistoryData",
+            Router: `/pcsg/import_history_data`,
             AllowHTTPMethods: []string{"post"},
             MethodParams: param.Make(),
             Filters: nil,
@@ -99,8 +117,8 @@ func init() {
 
     beego.GlobalControllerRouter["eta/eta_index_lib/controllers:BloombergController"] = append(beego.GlobalControllerRouter["eta/eta_index_lib/controllers:BloombergController"],
         beego.ControllerComments{
-            Method: "PCSGRefreshMonthly",
-            Router: `/pcsg/refresh_monthly`,
+            Method: "PCSGRefreshTask",
+            Router: `/pcsg/refresh_task`,
             AllowHTTPMethods: []string{"post"},
             MethodParams: param.Make(),
             Filters: nil,
@@ -108,14 +126,95 @@ func init() {
 
     beego.GlobalControllerRouter["eta/eta_index_lib/controllers:BloombergController"] = append(beego.GlobalControllerRouter["eta/eta_index_lib/controllers:BloombergController"],
         beego.ControllerComments{
-            Method: "PCSGRefreshWeekly",
-            Router: `/pcsg/refresh_weekly`,
+            Method: "Refresh",
+            Router: `/refresh`,
             AllowHTTPMethods: []string{"post"},
             MethodParams: param.Make(),
             Filters: nil,
             Params: nil})
 
-    beego.GlobalControllerRouter["eta/eta_index_lib/controllers:BloombergController"] = append(beego.GlobalControllerRouter["eta/eta_index_lib/controllers:BloombergController"],
+    beego.GlobalControllerRouter["eta/eta_index_lib/controllers:BusinessIndexController"] = append(beego.GlobalControllerRouter["eta/eta_index_lib/controllers:BusinessIndexController"],
+        beego.ControllerComments{
+            Method: "Add",
+            Router: `/add`,
+            AllowHTTPMethods: []string{"post"},
+            MethodParams: param.Make(),
+            Filters: nil,
+            Params: nil})
+
+    beego.GlobalControllerRouter["eta/eta_index_lib/controllers:BusinessIndexController"] = append(beego.GlobalControllerRouter["eta/eta_index_lib/controllers:BusinessIndexController"],
+        beego.ControllerComments{
+            Method: "DelIndexData",
+            Router: `/data/del`,
+            AllowHTTPMethods: []string{"post"},
+            MethodParams: param.Make(),
+            Filters: nil,
+            Params: nil})
+
+    beego.GlobalControllerRouter["eta/eta_index_lib/controllers:BusinessIndexController"] = append(beego.GlobalControllerRouter["eta/eta_index_lib/controllers:BusinessIndexController"],
+        beego.ControllerComments{
+            Method: "Edit",
+            Router: `/edit`,
+            AllowHTTPMethods: []string{"post"},
+            MethodParams: param.Make(),
+            Filters: nil,
+            Params: nil})
+
+    beego.GlobalControllerRouter["eta/eta_index_lib/controllers:BusinessIndexController"] = append(beego.GlobalControllerRouter["eta/eta_index_lib/controllers:BusinessIndexController"],
+        beego.ControllerComments{
+            Method: "HandleBusinessIndexData",
+            Router: `/handle`,
+            AllowHTTPMethods: []string{"post"},
+            MethodParams: param.Make(),
+            Filters: nil,
+            Params: nil})
+
+    beego.GlobalControllerRouter["eta/eta_index_lib/controllers:BusinessIndexController"] = append(beego.GlobalControllerRouter["eta/eta_index_lib/controllers:BusinessIndexController"],
+        beego.ControllerComments{
+            Method: "DelIndex",
+            Router: `/index/del`,
+            AllowHTTPMethods: []string{"post"},
+            MethodParams: param.Make(),
+            Filters: nil,
+            Params: nil})
+
+    beego.GlobalControllerRouter["eta/eta_index_lib/controllers:BusinessIndexController"] = append(beego.GlobalControllerRouter["eta/eta_index_lib/controllers:BusinessIndexController"],
+        beego.ControllerComments{
+            Method: "Refresh",
+            Router: `/refresh`,
+            AllowHTTPMethods: []string{"post"},
+            MethodParams: param.Make(),
+            Filters: nil,
+            Params: nil})
+
+    beego.GlobalControllerRouter["eta/eta_index_lib/controllers:CCFController"] = append(beego.GlobalControllerRouter["eta/eta_index_lib/controllers:CCFController"],
+        beego.ControllerComments{
+            Method: "Add",
+            Router: `/add`,
+            AllowHTTPMethods: []string{"post"},
+            MethodParams: param.Make(),
+            Filters: nil,
+            Params: nil})
+
+    beego.GlobalControllerRouter["eta/eta_index_lib/controllers:CCFController"] = append(beego.GlobalControllerRouter["eta/eta_index_lib/controllers:CCFController"],
+        beego.ControllerComments{
+            Method: "HandleEdbData",
+            Router: `/handle/edb_data`,
+            AllowHTTPMethods: []string{"post"},
+            MethodParams: param.Make(),
+            Filters: nil,
+            Params: nil})
+
+    beego.GlobalControllerRouter["eta/eta_index_lib/controllers:CCFController"] = append(beego.GlobalControllerRouter["eta/eta_index_lib/controllers:CCFController"],
+        beego.ControllerComments{
+            Method: "HandleTableData",
+            Router: `/handle/table_data`,
+            AllowHTTPMethods: []string{"post"},
+            MethodParams: param.Make(),
+            Filters: nil,
+            Params: nil})
+
+    beego.GlobalControllerRouter["eta/eta_index_lib/controllers:CCFController"] = append(beego.GlobalControllerRouter["eta/eta_index_lib/controllers:CCFController"],
         beego.ControllerComments{
             Method: "Refresh",
             Router: `/refresh`,
@@ -151,6 +250,15 @@ func init() {
             Filters: nil,
             Params: nil})
 
+    beego.GlobalControllerRouter["eta/eta_index_lib/controllers:CalculateController"] = append(beego.GlobalControllerRouter["eta/eta_index_lib/controllers:CalculateController"],
+        beego.ControllerComments{
+            Method: "StepCalculate",
+            Router: `/base/step_calculate`,
+            AllowHTTPMethods: []string{"post"},
+            MethodParams: param.Make(),
+            Filters: nil,
+            Params: nil})
+
     beego.GlobalControllerRouter["eta/eta_index_lib/controllers:CalculateController"] = append(beego.GlobalControllerRouter["eta/eta_index_lib/controllers:CalculateController"],
         beego.ControllerComments{
             Method: "BatchEdit",
@@ -1078,6 +1186,33 @@ func init() {
             Filters: nil,
             Params: nil})
 
+    beego.GlobalControllerRouter["eta/eta_index_lib/controllers:Sci99Controller"] = append(beego.GlobalControllerRouter["eta/eta_index_lib/controllers:Sci99Controller"],
+        beego.ControllerComments{
+            Method: "Add",
+            Router: `/add`,
+            AllowHTTPMethods: []string{"post"},
+            MethodParams: param.Make(),
+            Filters: nil,
+            Params: nil})
+
+    beego.GlobalControllerRouter["eta/eta_index_lib/controllers:Sci99Controller"] = append(beego.GlobalControllerRouter["eta/eta_index_lib/controllers:Sci99Controller"],
+        beego.ControllerComments{
+            Method: "Refresh",
+            Router: `/refresh`,
+            AllowHTTPMethods: []string{"post"},
+            MethodParams: param.Make(),
+            Filters: nil,
+            Params: nil})
+
+    beego.GlobalControllerRouter["eta/eta_index_lib/controllers:Sci99Crawler"] = append(beego.GlobalControllerRouter["eta/eta_index_lib/controllers:Sci99Crawler"],
+        beego.ControllerComments{
+            Method: "RefreshSci99Classify",
+            Router: `/refresh/list`,
+            AllowHTTPMethods: []string{"post"},
+            MethodParams: param.Make(),
+            Filters: nil,
+            Params: nil})
+
     beego.GlobalControllerRouter["eta/eta_index_lib/controllers:SciController"] = append(beego.GlobalControllerRouter["eta/eta_index_lib/controllers:SciController"],
         beego.ControllerComments{
             Method: "Add",
@@ -1123,6 +1258,24 @@ func init() {
             Filters: nil,
             Params: nil})
 
+    beego.GlobalControllerRouter["eta/eta_index_lib/controllers:ShanghaiSmmController"] = append(beego.GlobalControllerRouter["eta/eta_index_lib/controllers:ShanghaiSmmController"],
+        beego.ControllerComments{
+            Method: "RefreshExcel",
+            Router: `/refresh/excel`,
+            AllowHTTPMethods: []string{"post"},
+            MethodParams: param.Make(),
+            Filters: nil,
+            Params: nil})
+
+    beego.GlobalControllerRouter["eta/eta_index_lib/controllers:ShanghaiSmmController"] = append(beego.GlobalControllerRouter["eta/eta_index_lib/controllers:ShanghaiSmmController"],
+        beego.ControllerComments{
+            Method: "RefreshData",
+            Router: `/refresh/list`,
+            AllowHTTPMethods: []string{"post"},
+            MethodParams: param.Make(),
+            Filters: nil,
+            Params: nil})
+
     beego.GlobalControllerRouter["eta/eta_index_lib/controllers:ShfeController"] = append(beego.GlobalControllerRouter["eta/eta_index_lib/controllers:ShfeController"],
         beego.ControllerComments{
             Method: "Add",

routers/router.go

@@ -9,9 +9,11 @@ package routers
 
 import (
 	"eta/eta_index_lib/controllers"
+	"eta/eta_index_lib/controllers/factor_edb_series"
 	"eta/eta_index_lib/controllers/fix"
 	"eta/eta_index_lib/controllers/future_good"
 	"eta/eta_index_lib/controllers/open"
+
 	beego "github.com/beego/beego/v2/server/web"
 )
 
@@ -249,6 +251,36 @@ func init() {
 				&controllers.BloombergController{},
 			),
 		),
+		beego.NSNamespace("/business_index",
+			beego.NSInclude(
+				&controllers.BusinessIndexController{},
+			),
+		),
+		beego.NSNamespace("/sci99",
+			beego.NSInclude(
+				&controllers.Sci99Controller{},
+			),
+		),
+		beego.NSNamespace("/sci99_crawler",
+			beego.NSInclude(
+				&controllers.Sci99Crawler{},
+			),
+		),
+		beego.NSNamespace("/ccf",
+			beego.NSInclude(
+				&controllers.CCFController{},
+			),
+		),
+		beego.NSNamespace("/shanghai_smm",
+			beego.NSInclude(
+				&controllers.ShanghaiSmmController{},
+			),
+		),
+		beego.NSNamespace("/factor_edb_series",
+			beego.NSInclude(
+				&factor_edb_series.FactorEdbSeriesController{},
+			),
+		),
 	)
 	beego.AddNamespace(ns)
 }

services/base_from_baiinfo.go

@@ -91,12 +91,13 @@ func HandleBaiinfoIndex(baseFilePath, terminalCode, renameFilePath, indexName, i
 
 	// 遍历excel数据，然后跟现有的数据做校验，不存在则入库
 	for date, value := range excelDataMap {
+		dateT, tmpErr := utils.DealExcelDate(date)
+		if tmpErr != nil {
+			fmt.Println("time.ParseInLocation Err:" + tmpErr.Error())
+			return
+		}
+		date = dateT.Format(utils.FormatDate)
 		if findData, ok := exitDataMap[date]; !ok {
-			_, err := time.ParseInLocation(utils.FormatDate, date, time.Local)
-			if err != nil {
-				fmt.Println("time.ParseInLocation Err:" + err.Error())
-				return
-			}
 			if !strings.Contains(value, "#N/A") {
 				var saveDataTime time.Time
 				if strings.Contains(date, "00:00:00") {

services/base_from_business.go

@@ -0,0 +1,712 @@
+package services
+
+import (
+	"errors"
+	"eta/eta_index_lib/logic"
+	"eta/eta_index_lib/models"
+	"eta/eta_index_lib/models/mgo"
+	"eta/eta_index_lib/services/alarm_msg"
+	"eta/eta_index_lib/utils"
+	"fmt"
+	"github.com/qiniu/qmgo"
+	"go.mongodb.org/mongo-driver/bson"
+	"strings"
+	"time"
+)
+
+// HandleBusinessIndex
+// @Description:  处理外部指标
+// @author: Roc
+// @datetime 2024-04-26 14:23:42
+// @param indexReq *models.AddBusinessIndexReq
+// @return err error
+func HandleBusinessIndex(indexReq *models.AddBusinessIndexReq) (resp models.BaseFromBusinessIndexResp, err error) {
+	defer func() {
+		if err != nil {
+			// 添加刷新失败日志
+			dataUpdateResult := 2
+			dataUpdateFailedReason := "服务异常"
+			edbInfo, e := models.GetEdbInfoByEdbCode(utils.DATA_SOURCE_MYSTEEL_CHEMICAL, indexReq.IndexCode)
+			if e == nil {
+				//查询指标存在，才添加刷新日志
+				_ = AddEdbInfoUpdateLog(edbInfo.EdbInfoId, 2, err.Error(), dataUpdateResult, dataUpdateFailedReason, 1, 0)
+			}
+		}
+	}()
+
+	// 没有数据就返回
+	if indexReq.DataList == nil || len(indexReq.DataList) <= 0 {
+		return
+	}
+	// 兼容频度缺少度的字段
+	if !strings.Contains(indexReq.Frequency, "度") {
+		indexReq.Frequency = indexReq.Frequency + "度"
+	}
+	if !utils.VerifyFrequency(indexReq.Frequency) {
+		err = errors.New("指标频度不合法:" + indexReq.Frequency)
+		return
+	}
+
+	// 判断来源，如果来源不存在的话，则创建
+	sourceObj := new(models.EdbBusinessSource)
+	sourceItem, err := sourceObj.GetEdbBusinessSourceItem(indexReq.SourceName)
+	if err != nil {
+		if err.Error() == utils.ErrNoRow() {
+			sourceItem = &models.EdbBusinessSource{
+				EdbBusinessSourceId: 0,
+				SourceName:          indexReq.SourceName,
+				CreateTime:          time.Now(),
+			}
+			err = sourceItem.Add()
+		} else {
+			return
+		}
+	}
+
+	// 指标
+	indexObj := new(models.BaseFromBusinessIndex)
+	if indexReq.IndexCode == `` {
+		// 如果指标编码为空，那么自动生成
+		currId, tmpErr := indexObj.GetMaxId()
+		if tmpErr != nil {
+			err = tmpErr
+			return
+		}
+		indexReq.IndexCode = fmt.Sprintf("SELF%07d", currId+1)
+	}
+	//判断指标是否存在
+	item, err := indexObj.GetIndexItem(indexReq.IndexCode)
+	if err != nil {
+		if err.Error() != utils.ErrNoRow() {
+			return
+		}
+
+		// 添加指标
+		item = &models.BaseFromBusinessIndex{
+			BaseFromBusinessIndexId: 0,
+			IndexCode:               indexReq.IndexCode,
+			IndexName:               indexReq.IndexName,
+			Unit:                    indexReq.Unit,
+			Frequency:               indexReq.Frequency,
+			Source:                  int(sourceItem.EdbBusinessSourceId),
+			SourceName:              sourceItem.SourceName,
+			//StartDate:               time.Time{},
+			//EndDate:                 time.Time{},
+			Remark:         indexReq.Remark,
+			BaseModifyTime: time.Now(),
+			DataUpdateTime: time.Now(),
+			CreateTime:     time.Now(),
+			ModifyTime:     time.Now(),
+		}
+		err = item.Add()
+		if err != nil {
+			fmt.Println("add err:" + err.Error())
+			return
+		}
+	} else {
+		updateCols := make([]string, 0)
+		if item.IndexName != indexReq.IndexName {
+			item.IndexName = indexReq.IndexName
+			updateCols = append(updateCols, "IndexName")
+		}
+		if item.Unit != indexReq.Unit {
+			item.Unit = indexReq.Unit
+			updateCols = append(updateCols, "Unit")
+		}
+		if item.Frequency != indexReq.Frequency {
+			item.Frequency = indexReq.Frequency
+			updateCols = append(updateCols, "Frequency")
+		}
+		if item.Source != int(sourceItem.EdbBusinessSourceId) {
+			item.Source = int(sourceItem.EdbBusinessSourceId)
+			item.SourceName = sourceItem.SourceName
+			updateCols = append(updateCols, "Source", "SourceName")
+		}
+
+		if len(updateCols) > 0 {
+			item.BaseModifyTime = time.Now()
+			item.ModifyTime = time.Now()
+			updateCols = append(updateCols, "BaseModifyTime", "ModifyTime")
+			err = item.Update(updateCols)
+			if err != nil {
+				fmt.Println("update index err:" + err.Error())
+				return
+			}
+		}
+	}
+
+	// 数据处理
+	// 当前传入的最小日期
+	var reqMinDate time.Time
+	if utils.UseMongo {
+		reqMinDate, err = handleBusinessDataByMongo(item, indexReq.DataList)
+	} else {
+		reqMinDate, err = handleBusinessDataByMysql(item, indexReq.DataList)
+	}
+	if err != nil {
+		return
+	}
+
+	// 同步刷新指标库的指标
+	go refreshEdbBusiness(item.IndexCode, reqMinDate)
+
+	resp = models.BaseFromBusinessIndexResp{
+		IndexCode:  item.IndexCode,
+		IndexName:  item.IndexName,
+		Unit:       item.Unit,
+		Frequency:  item.Frequency,
+		SourceName: item.SourceName,
+	}
+
+	return
+}
+
+// handleBusinessDataByMongo
+// @Description: 处理外部指标数据（mongo）
+// @author: Roc
+// @datetime 2024-07-01 15:30:41
+// @param item *models.BaseFromBusinessIndex
+// @param reqDataList []*models.AddBusinessDataReq
+// @return reqMinDate time.Time 当前传入的最小日期
+// @return err error
+func handleBusinessDataByMongo(item *models.BaseFromBusinessIndex, reqDataList []models.AddBusinessDataReq) (reqMinDate time.Time, err error) {
+	mogDataObj := new(mgo.BaseFromBusinessData)
+
+	//获取已存在的所有数据
+	exitDataList, err := mogDataObj.GetAllDataList(bson.M{"index_code": item.IndexCode}, []string{"data_time"})
+	if err != nil {
+		fmt.Println("GetIndexDataList Err:" + err.Error())
+		return
+	}
+
+	// 已经存在的数据集
+	exitDataMap := make(map[string]*mgo.BaseFromBusinessData)
+	for _, v := range exitDataList {
+		exitDataMap[v.DataTime.Format(utils.FormatDate)] = v
+	}
+
+	// 待添加的数据集
+	addDataList := make([]interface{}, 0)
+	updateDataList := make([]mgo.BaseFromBusinessData, 0)
+	//var hasUpdate bool
+	// 遍历excel数据，然后跟现有的数据做校验，不存在则入库
+	for _, data := range reqDataList {
+		dateTime, tmpErr := utils.DealExcelDate(data.Date)
+		if tmpErr != nil {
+			fmt.Println("time.ParseInLocation Err:" + tmpErr.Error())
+			err = tmpErr
+			return
+		}
+
+		// 调整最小日期
+		if reqMinDate.IsZero() || reqMinDate.After(dateTime) {
+			reqMinDate = dateTime
+		}
+
+		date := dateTime.Format(utils.FormatDate)
+
+		findData, ok := exitDataMap[date]
+		if !ok {
+			addDataList = append(addDataList, mgo.BaseFromBusinessData{
+				BaseFromBusinessIndexId: item.BaseFromBusinessIndexId,
+				IndexCode:               item.IndexCode,
+				DataTime:                dateTime,
+				Value:                   data.Value,
+				CreateTime:              time.Now(),
+				ModifyTime:              time.Now(),
+				//DataTimestamp:           0,
+			})
+			continue
+		}
+
+		// 值不匹配，修改数据
+		if findData.Value != data.Value {
+			findData.Value = data.Value
+			updateDataList = append(updateDataList, *findData)
+		}
+	}
+
+	// 指标数据是否新增或修改
+	var isIndexUpdateOrAdd bool
+
+	// 入库
+	{
+		coll := mogDataObj.GetCollection()
+		if len(addDataList) > 0 {
+			isIndexUpdateOrAdd = true
+			err = mogDataObj.BatchInsertDataByColl(coll, 500, addDataList)
+			if err != nil {
+				fmt.Println("mogDataObj.HandleData() Err:" + err.Error())
+				return
+			}
+		}
+
+		if len(updateDataList) > 0 {
+			isIndexUpdateOrAdd = true
+			for _, v := range updateDataList {
+				err = mogDataObj.UpdateDataByColl(coll, bson.M{"_id": v.ID}, bson.M{"$set": bson.M{"value": v.Value, "modify_time": v.ModifyTime}})
+				if err != nil {
+					fmt.Println("UpdateDataByColl:Err:" + err.Error())
+					return
+				}
+			}
+		}
+	}
+
+	// 支持事务的话，下面操作
+	//result, err := mogDataObj.HandleData(addDataList, updateDataList)
+	//if err != nil {
+	//	fmt.Println("mogDataObj.HandleData() Err:" + err.Error())
+	//	return
+	//}
+	//fmt.Println("result", result)
+
+	//修改最大最小日期
+	indexMaxAndMinInfo, err := item.GetEdbInfoMaxAndMinInfo(item.IndexCode)
+	if err != nil {
+		return
+	}
+	if err == nil && indexMaxAndMinInfo != nil {
+		e := item.ModifyIndexMaxAndMinInfo(item.IndexCode, indexMaxAndMinInfo, isIndexUpdateOrAdd)
+		if e != nil {
+			fmt.Println("ModifyIndexMaxAndMinInfo Err:" + e.Error())
+		}
+	}
+
+	return
+}
+
+// handleBusinessDataByMysql
+// @Description: 处理外部指标数据（mysql）
+// @author: Roc
+// @datetime 2024-07-01 15:59:43
+// @param item *models.BaseFromBusinessIndex
+// @param reqDataList []models.AddBusinessDataReq
+// @return reqMinDate time.Time
+// @return err error
+func handleBusinessDataByMysql(item *models.BaseFromBusinessIndex, reqDataList []models.AddBusinessDataReq) (reqMinDate time.Time, err error) {
+	businessDataObj := new(models.BaseFromBusinessData)
+
+	var condition []string
+	var pars []interface{}
+	condition = append(condition, "index_code = ?")
+	pars = append(pars, item.IndexCode)
+	//获取已存在的所有数据
+	exitDataList, err := businessDataObj.GetAllDataList(condition, pars, "data_time ASC")
+	if err != nil {
+		fmt.Println("GetIndexDataList Err:" + err.Error())
+		return
+	}
+
+	// 已经存在的数据集
+	exitDataMap := make(map[string]*models.BaseFromBusinessData)
+	for _, v := range exitDataList {
+		exitDataMap[v.DataTime.Format(utils.FormatDate)] = v
+	}
+
+	// 待添加的数据集
+	addDataList := make([]*models.BaseFromBusinessData, 0)
+	updateDataList := make([]*models.BaseFromBusinessData, 0)
+	//var hasUpdate bool
+	// 遍历excel数据，然后跟现有的数据做校验，不存在则入库
+	for _, data := range reqDataList {
+		dateTime, tmpErr := utils.DealExcelDate(data.Date)
+		if tmpErr != nil {
+			fmt.Println("time.ParseInLocation Err:" + tmpErr.Error())
+			err = tmpErr
+			return
+		}
+
+		// 调整最小日期
+		if reqMinDate.IsZero() || reqMinDate.After(dateTime) {
+			reqMinDate = dateTime
+		}
+
+		date := dateTime.Format(utils.FormatDate)
+
+		findData, ok := exitDataMap[date]
+		if !ok {
+			addDataList = append(addDataList, &models.BaseFromBusinessData{
+				BaseFromBusinessIndexId: int(item.BaseFromBusinessIndexId),
+				IndexCode:               item.IndexCode,
+				DataTime:                dateTime,
+				Value:                   data.Value,
+				CreateTime:              time.Now(),
+				ModifyTime:              time.Now(),
+				//DataTimestamp:           0,
+			})
+			continue
+		}
+
+		// 值不匹配，修改数据
+		if findData.Value != data.Value {
+			findData.Value = data.Value
+			findData.ModifyTime = time.Now()
+			updateDataList = append(updateDataList, findData)
+		}
+	}
+
+	// 指标数据是否新增或修改
+	var isIndexUpdateOrAdd bool
+
+	// 入库
+	{
+		if len(addDataList) > 0 {
+			isIndexUpdateOrAdd = true
+		}
+
+		if len(updateDataList) > 0 {
+			isIndexUpdateOrAdd = true
+		}
+		err = businessDataObj.HandleData(addDataList, updateDataList)
+		if err != nil {
+			fmt.Println("UpdateDataByColl:Err:" + err.Error())
+			return
+		}
+	}
+
+	//修改最大最小日期
+	indexMaxAndMinInfo, err := item.GetEdbInfoMaxAndMinInfo(item.IndexCode)
+	if err != nil {
+		return
+	}
+	if err == nil && indexMaxAndMinInfo != nil {
+		e := item.ModifyIndexMaxAndMinInfo(item.IndexCode, indexMaxAndMinInfo, isIndexUpdateOrAdd)
+		if e != nil {
+			fmt.Println("ModifyIndexMaxAndMinInfo Err:" + e.Error())
+		}
+	}
+
+	return
+}
+
+// DelBusinessIndexResp
+// @Description: 删除外部指标的返回
+type DelBusinessIndexResp struct {
+	IsDeleteEdbCodeList []string `description:"已经删除了的指标编码"`
+	NoDeleteEdbCodeList []string `description:"未删除的指标编码"`
+}
+
+// DelBusinessIndex
+// @Description: 删除外部指标
+// @author: Roc
+// @datetime 2024-05-31 16:27:37
+// @param indexCodeList []string
+// @return err error
+// @return errMsg string
+func DelBusinessIndex(indexCodeList []string) (joinEdbCodeList, needDelEdbCodeList []string, err error, errMsg string) {
+	defer func() {
+		if err != nil {
+			fmt.Println("DelBusinessIndex Err:" + err.Error())
+		}
+	}()
+	errMsg = "删除失败"
+
+	if len(indexCodeList) < 0 {
+		errMsg = "指标编码不允许为空"
+		err = errors.New(errMsg)
+		return
+	}
+
+	// 指标
+	indexObj := new(models.BaseFromBusinessIndex)
+
+	//判断指标是否存在
+	baseIndexList, err := indexObj.GetIndexItemList(indexCodeList)
+	if err != nil {
+		return
+	}
+
+	// 已加入到eta指标库的编码map
+	joinEdbCodeList = make([]string, 0)
+	joinEdbMap := make(map[string]string)
+
+	// 未加入到eta指标库的编码
+	needDelEdbCodeList = make([]string, 0)
+
+	// 判断指标是否加入到指标库
+	tmpEdbInfoList, err := models.GetEdbInfoByEdbCodeList(utils.DATA_SOURCE_BUSINESS, indexCodeList)
+	if err != nil {
+		return
+	}
+	for _, v := range tmpEdbInfoList {
+		joinEdbCodeList = append(joinEdbCodeList, v.EdbCode)
+		joinEdbMap[v.EdbCode] = v.EdbCode
+	}
+
+	for _, v := range baseIndexList {
+		_, ok := joinEdbMap[v.IndexCode]
+		if !ok {
+			needDelEdbCodeList = append(needDelEdbCodeList, v.IndexCode)
+		}
+	}
+
+	// 如果需要删除的指标，则直接返回
+	if len(needDelEdbCodeList) <= 0 {
+		return
+	}
+
+	// 删除指标
+	err = indexObj.DelIndexItemList(needDelEdbCodeList)
+	if err != nil {
+		fmt.Println("删除自有指标失败， Err:" + err.Error())
+		return
+	}
+
+	if utils.UseMongo {
+		// 删除指标明细数据
+		mogDataObj := new(mgo.BaseFromBusinessData)
+		err = mogDataObj.RemoveMany(bson.M{"index_code": bson.M{"$in": needDelEdbCodeList}})
+	} else {
+		var condition []string
+		var pars []interface{}
+		delNum := len(needDelEdbCodeList)
+		if delNum > 0 {
+			condition = append(condition, "index_code in ("+utils.GetOrmInReplace(delNum)+")")
+			pars = append(pars, needDelEdbCodeList)
+
+			businessDataObj := models.BaseFromBusinessData{}
+			err = businessDataObj.DelDataByCond(condition, pars)
+		}
+	}
+	if err != nil {
+		fmt.Println("删除自有指标明细数据 Err:" + err.Error())
+		return
+	}
+
+	return
+}
+
+// DelBusinessIndexData
+// @Description: 删除指标数据
+// @author: Roc
+// @datetime 2024-05-31 15:43:32
+// @param indexCode string
+// @param dateList []string
+// @return err error
+// @return errMsg string
+func DelBusinessIndexData(indexCode string, startDate, endDate string) (err error, errMsg string) {
+	defer func() {
+		if err != nil {
+			fmt.Println("DelBusinessIndexData Err:" + err.Error())
+		}
+	}()
+	errMsg = "删除失败"
+
+	if indexCode == `` {
+		errMsg = "指标编码不允许为空"
+		err = errors.New(errMsg)
+		return
+	}
+	if startDate == `` && endDate == `` {
+		errMsg = "开始日期和结束日期不允许同时为空"
+		err = errors.New(errMsg)
+		return
+	}
+
+	// 指标
+	indexObj := new(models.BaseFromBusinessIndex)
+
+	//判断指标是否存在
+	item, err := indexObj.GetIndexItem(indexCode)
+	if err != nil {
+		if err.Error() != utils.ErrNoRow() {
+			return
+		}
+		err = nil
+		return
+	}
+
+	// 当前传入的最小日期
+	var reqMinDate time.Time
+
+	var startDateTime, endDateTime time.Time
+	if startDate != `` {
+		//获取已存在的所有数据
+		startDateTime, err = time.ParseInLocation(utils.FormatDate, startDate, time.Local)
+		if err != nil {
+			return
+		}
+		// 调整最小日期
+		if reqMinDate.IsZero() || reqMinDate.After(startDateTime) {
+			reqMinDate = startDateTime
+		}
+	}
+
+	if endDate != `` {
+		//获取已存在的所有数据
+		endDateTime, err = time.ParseInLocation(utils.FormatDate, endDate, time.Local)
+		if err != nil {
+			return
+		}
+		// 调整最小日期
+		if reqMinDate.IsZero() || reqMinDate.After(endDateTime) {
+			reqMinDate = endDateTime
+		}
+	}
+
+	// 删除具体的数据
+	var indexMaxAndMinInfo *models.EdbInfoMaxAndMinInfo
+	if utils.UseMongo {
+		indexMaxAndMinInfo, err = delBusinessIndexDataByMongo(item, startDateTime, endDateTime)
+	} else {
+		indexMaxAndMinInfo, err = delBusinessIndexDataByMysql(item, startDateTime, endDateTime)
+	}
+	if err != nil {
+		return
+	}
+
+	// 修改指标的最早最晚日期
+	if indexMaxAndMinInfo != nil {
+		e := item.ModifyIndexMaxAndMinInfo(item.IndexCode, indexMaxAndMinInfo, true)
+		if e != nil {
+			fmt.Println("ModifyIndexMaxAndMinInfo Err:" + e.Error())
+		}
+	}
+
+	// 同步刷新指标库的指标
+	go refreshEdbBusiness(item.IndexCode, reqMinDate)
+
+	return
+}
+
+// delBusinessIndexDataByMongo
+// @Description: 删除指标数据（从mongo删除）
+// @author: Roc
+// @datetime 2024-07-01 18:00:07
+// @param item *models.BaseFromBusinessIndex
+// @param startDateTime time.Time
+// @param endDateTime time.Time
+// @return err error
+func delBusinessIndexDataByMongo(item *models.BaseFromBusinessIndex, startDateTime, endDateTime time.Time) (indexMaxAndMinInfo *models.EdbInfoMaxAndMinInfo, err error) {
+	defer func() {
+		if err != nil {
+			utils.FileLog.Error("delBusinessIndexDataByMongo 删除自有指标明细数据 Err:" + err.Error())
+		}
+	}()
+
+	// 构建查询条件
+	queryConditions := bson.M{
+		"index_code": item.IndexCode,
+	}
+
+	dateCondition, err := mgo.BuildDateTimeCondition(startDateTime, endDateTime)
+	if err != nil {
+		return
+	}
+	if len(dateCondition) > 0 {
+		queryConditions["data_time"] = dateCondition
+	}
+
+	// 删除数据源中的指标明细数据
+	mogDataObj := new(mgo.BaseFromBusinessData)
+	err = mogDataObj.RemoveMany(queryConditions)
+	if err != nil {
+		return
+	}
+
+	//修改最大最小日期
+	indexMaxAndMinInfo, err = item.GetEdbInfoMaxAndMinInfo(item.IndexCode)
+	// 如果有错误，且错误信息是取不到文档，那么就不修改了
+	if err != nil && !errors.Is(err, qmgo.ErrNoSuchDocuments) {
+		return
+	}
+	// 清空的目的是为了避免异常返回
+	err = nil
+
+	return
+}
+
+// delBusinessIndexDataByMysql
+// @Description: 删除指标数据（从mysql删除）
+// @author: Roc
+// @datetime 2024-07-02 09:53:13
+// @param item *models.BaseFromBusinessIndex
+// @param startDateTime time.Time
+// @param endDateTime time.Time
+// @return err error
+func delBusinessIndexDataByMysql(item *models.BaseFromBusinessIndex, startDateTime, endDateTime time.Time) (indexMaxAndMinInfo *models.EdbInfoMaxAndMinInfo, err error) {
+	defer func() {
+		if err != nil {
+			utils.FileLog.Error("delBusinessIndexDataByMysql 删除自有指标明细数据 Err:" + err.Error())
+		}
+	}()
+
+	// 构建查询条件
+	var condition []string
+	var pars []interface{}
+	condition = append(condition, "index_code = ? ")
+	pars = append(pars, item.IndexCode)
+
+	if !startDateTime.IsZero() {
+		condition = append(condition, " data_time >= ? ")
+		pars = append(pars, startDateTime.Format(utils.FormatDate))
+	}
+	if !endDateTime.IsZero() {
+		condition = append(condition, " data_time <= ? ")
+		pars = append(pars, endDateTime.Format(utils.FormatDate))
+	}
+
+	// 删除数据源中的指标明细数据
+	businessDataObj := new(models.BaseFromBusinessData)
+	err = businessDataObj.DelDataByCond(condition, pars)
+	if err != nil {
+		return
+	}
+
+	//修改最大最小日期
+	indexMaxAndMinInfo, err = item.GetEdbInfoMaxAndMinInfo(item.IndexCode)
+	// 如果有错误，且错误信息是取不到文档，那么就不修改了
+	if err != nil && err.Error() != utils.ErrNoRow() {
+		return
+	}
+	// 清空的目的是为了避免异常返回
+	err = nil
+
+	return
+}
+
+func refreshEdbBusiness(indexCode string, reqMinDate time.Time) {
+	var indexErr error
+	var errMsg string
+	defer func() {
+		if indexErr != nil {
+			tips := fmt.Sprintf("自有数据刷新-ETA指标刷新异常, 指标编码: %s, err: %s, 错误信息：%s", indexCode, indexErr.Error(), errMsg)
+			alarm_msg.SendAlarmMsg(tips, 3)
+		}
+	}()
+
+	edbInfo, e := models.GetEdbInfoByEdbCode(utils.DATA_SOURCE_BUSINESS, indexCode)
+	if e != nil && e.Error() != utils.ErrNoRow() {
+		indexErr = e
+		return
+	}
+
+	if edbInfo != nil {
+		startDate := ``
+		if reqMinDate.IsZero() {
+			startDate = edbInfo.EndDate
+		} else {
+			startDate = reqMinDate.Format(utils.FormatDate)
+		}
+		params := models.RefreshBaseParams{
+			EdbInfo:   edbInfo,
+			StartDate: startDate,
+		}
+		obj := models.Business{}
+
+		indexErr, errMsg = obj.Refresh(params)
+		if indexErr != nil {
+			return
+		}
+
+		// 更新指标最大最小值
+		indexErr = obj.UnifiedModifyEdbInfoMaxAndMinInfo(edbInfo)
+		if indexErr != nil {
+			return
+		}
+
+		// 更新ES
+		go logic.UpdateEs(edbInfo.EdbInfoId)
+	}
+}

services/base_from_calculate.go

@@ -5,6 +5,7 @@ import (
 	"eta/eta_index_lib/models"
 	"eta/eta_index_lib/utils"
 	"fmt"
+	"sort"
 	"strconv"
 	"strings"
 	"time"
@@ -165,8 +166,13 @@ func EdbCalculateBatchSave(req models.EdbInfoCalculateBatchSaveReq, lang string)
 	oldUnit := fromEdbInfo.Unit
 
 	//生成指标编码
-	randStr := utils.GetRandDigit(4)
-	edbCode := `C` + time.Now().Format("060102") + randStr
+	edbCode, err := utils.GenerateEdbCode(1, "")
+	if err != nil {
+		errMsg = "指标编码生成失败:Err:" + err.Error()
+		err = fmt.Errorf("指标生成失败")
+		return
+	}
+
 	timestamp := strconv.FormatInt(time.Now().UnixNano(), 10)
 	uniqueCode = utils.MD5(utils.DATA_PREFIX + "_" + timestamp)
 
@@ -252,12 +258,7 @@ func EdbCalculateBatchSave(req models.EdbInfoCalculateBatchSaveReq, lang string)
 		sourName = "N数值移动平均计算"
 		edbInfo, err = models.AddCalculateNszydpjjs(&req, fromEdbInfo, edbCode, uniqueCode, sysUserId, sysUserName, formulaInt)
 	case utils.DATA_SOURCE_CALCULATE_HBZ:
-		var condition string
-		var pars []interface{}
-		condition += " AND edb_info_id =? "
-		pars = append(pars, fromEdbInfoId)
-		condition += " AND value <=0 "
-		checkCount, tmpErr := models.GetEdbDataCount(condition, pars, fromEdbInfo.Source, fromEdbInfo.SubSource)
+		checkCount, tmpErr := models.GetLteZeroEdbDataCount(fromEdbInfo.Source, fromEdbInfo.SubSource, fromEdbInfoId)
 		if tmpErr != nil && tmpErr.Error() != utils.ErrNoRow() {
 			err = fmt.Errorf("判断环比值是否可计算失败")
 			errMsg = "判断环比值是否可计算失败,Err:" + tmpErr.Error()
@@ -478,6 +479,7 @@ func EdbCalculateBatchSave(req models.EdbInfoCalculateBatchSaveReq, lang string)
 	}
 	// 更新ES
 	go logic.UpdateEs(edbInfo.EdbInfoId)
+	go DisableEdbInfoNoUpdate(edbInfo)
 	return
 }
 
@@ -683,12 +685,7 @@ func EdbCalculateBatchEdit(req models.EdbInfoCalculateBatchEditReq) (edbInfo *mo
 		sourName = "N数值移动平均计算"
 		err = models.EditCalculateNszydpjjs(edbInfo, &req, fromEdbInfo, formulaInt, edbInfo.CalculateFormula)
 	case utils.DATA_SOURCE_CALCULATE_HBZ:
-		var condition string
-		var pars []interface{}
-		condition += " AND edb_info_id =? "
-		pars = append(pars, req.FromEdbInfoId)
-		condition += " AND value <=0 "
-		checkCount, tmpErr := models.GetEdbDataCount(condition, pars, fromEdbInfo.Source, fromEdbInfo.SubSource)
+		checkCount, tmpErr := models.GetLteZeroEdbDataCount(fromEdbInfo.Source, fromEdbInfo.SubSource, req.FromEdbInfoId)
 		if tmpErr != nil && tmpErr.Error() != utils.ErrNoRow() {
 			errMsg = "判断环比值是否可计算失败,Err:" + tmpErr.Error()
 			err = fmt.Errorf("判断环比值是否可计算失败")
@@ -909,6 +906,9 @@ func EdbCalculateBatchEdit(req models.EdbInfoCalculateBatchEditReq) (edbInfo *mo
 
 	// 更新ES
 	go logic.UpdateEs(edbInfo.EdbInfoId)
+
+	// 重置计算指标中的引用关系
+	go ResetEdbRelation(edbInfoId)
 	return
 }
 
@@ -1042,9 +1042,13 @@ func EdbCalculateAdd(req models.EdbInfoCalculateSaveReq, lang string) (edbInfo *
 			return
 		}
 	}
-
-	randStr := utils.GetRandDigit(4)
-	edbCode := `C` + time.Now().Format("060102") + randStr
+	//生成指标编码
+	edbCode, err := utils.GenerateEdbCode(1, "")
+	if err != nil {
+		errMsg = "生成计算指标失败,AddEdbInfo Err:" + err.Error()
+		err = fmt.Errorf("指标编码生成失败，Err：" + err.Error())
+		return
+	}
 	timestamp := strconv.FormatInt(time.Now().UnixNano(), 10)
 	uniqueCode = utils.MD5(utils.DATA_PREFIX + "_" + timestamp)
 
@@ -1055,6 +1059,82 @@ func EdbCalculateAdd(req models.EdbInfoCalculateSaveReq, lang string) (edbInfo *
 		err = fmt.Errorf("生成计算指标失败")
 		return
 	}
+	go DisableEdbInfoNoUpdate(edbInfo)
+	return
+}
+
+// StepCalculate 多公式计算
+func StepCalculate(dataList []*models.EdbInfoSearchData, calculates []models.CalculatesReq) (resultData map[string]float64, dates []string, errMsg string, err error) {
+	sort.Slice(calculates, func(i, j int) bool {
+		return calculates[i].Sort < calculates[j].Sort
+	})
+	originData, e := models.EdbInfoSearchDataToData(dataList)
+	if e != nil {
+		err = fmt.Errorf("基础数据转换异常, err: %v", e)
+		return
+	}
 
+	calculateData := originData
+	dateDataMap := make(map[time.Time]float64)
+	for _, v := range calculates {
+		baseCalculate := models.BaseCalculate{
+			DataList:      calculateData,
+			Frequency:     v.Frequency,
+			Formula:       v.Formula,
+			Calendar:      v.Calendar,
+			MoveType:      v.MoveType,
+			MoveFrequency: v.MoveFrequency,
+			FromFrequency: v.FromFrequency,
+			Source:        v.Source,
+		}
+
+		// 计算方式
+		switch baseCalculate.Source {
+		case utils.EdbBaseCalculateLjzzy:
+			dateDataMap, e, errMsg = baseCalculate.Ljzzy()
+		case utils.EdbBaseCalculateLjzzj:
+			dateDataMap, e, errMsg = baseCalculate.Ljzzj()
+		case utils.EdbBaseCalculateTbz:
+			dateDataMap, e, errMsg = baseCalculate.Tbz()
+		case utils.EdbBaseCalculateTcz:
+			dateDataMap, e, errMsg = baseCalculate.Tcz()
+		case utils.EdbBaseCalculateNszydpjjs:
+			dateDataMap, e, errMsg = baseCalculate.Nszydpjjs()
+		case utils.EdbBaseCalculateHbz:
+			dateDataMap, e, errMsg = baseCalculate.Hbz()
+		case utils.EdbBaseCalculateHcz:
+			dateDataMap, e, errMsg = baseCalculate.Hcz()
+		case utils.EdbBaseCalculateUpFrequency:
+			dateDataMap, e, errMsg = baseCalculate.UpFrequency()
+		case utils.EdbBaseCalculateDownFrequency:
+			dateDataMap, e, errMsg = baseCalculate.DownFrequency()
+		case utils.EdbBaseCalculateTimeShift:
+			dateDataMap, e, errMsg = baseCalculate.TimeShift()
+		case utils.EdbBaseCalculateCjjx:
+			dateDataMap, e, errMsg = baseCalculate.Cjjx()
+		case utils.EdbBaseCalculateAnnualized:
+			dateDataMap, e, errMsg = baseCalculate.Annualized()
+		case utils.EdbBaseCalculateLjz:
+			dateDataMap, e, errMsg = baseCalculate.Ljz()
+		case utils.EdbBaseCalculateLjzNczj:
+			dateDataMap, e, errMsg = baseCalculate.LjzNczj()
+		case utils.EdbBaseCalculateExponentialSmoothing:
+			dateDataMap, e, errMsg = baseCalculate.ExponentialSmoothing()
+		case utils.EdbBaseCalculateRjz:
+			dateDataMap, e, errMsg = baseCalculate.Rjz()
+		default:
+			errMsg = "计算方式无效"
+			e = fmt.Errorf("%s:%d", errMsg, baseCalculate.Source)
+		}
+		if e != nil {
+			err = fmt.Errorf("计算失败, err: %v", e)
+			return
+		}
+
+		calculateData = models.TransDateData2EdbData(dateDataMap)
+	}
+	resultData, dates = models.GetDateDataAndDateList(dateDataMap)
+	//resp.DataMap = resultData
+	//resp.DateList = dates
 	return
 }

services/base_from_ccf.go

@@ -0,0 +1,270 @@
+package services
+
+import (
+	"eta/eta_index_lib/logic"
+	"eta/eta_index_lib/models"
+	"eta/eta_index_lib/services/alarm_msg"
+	"eta/eta_index_lib/utils"
+	"fmt"
+	"html"
+	"strings"
+	"time"
+)
+
+// HandleCCFIndex 处理CCF的excel数据
+func HandleCCFIndex(req *models.HandleCCFEdbDataReq) (err error) {
+	errMsgList := make([]string, 0)
+	defer func() {
+		if len(errMsgList) > 0 {
+			msg := fmt.Sprint("数据源-CCF数据处理失败, err:", strings.Join(errMsgList, "\n"))
+			utils.FileLog.Info(msg)
+			go alarm_msg.SendAlarmMsg(msg, 3)
+		}
+	}()
+	for _, v := range req.List {
+		if v.IndexName == "" || v.IndexCode == "" {
+			errMsgList = append(errMsgList, fmt.Sprintf("新增指标异常，指标编码%s或者指标ID%s为空：", v.IndexCode, v.IndexName))
+			continue
+		}
+		err = handleCCFIndex(v, req.TerminalCode)
+		if err != nil {
+			errMsgList = append(errMsgList, fmt.Sprintf("新增指标异常，指标编码：%s, Err: %s", v.IndexCode, err))
+			return
+		}
+	}
+	return
+}
+
+func handleCCFIndex(req *models.HandleCCFEdbData, terminalCode string) (err error) {
+	indexName := req.IndexName
+	indexCode := req.IndexCode
+	excelDataMap := req.DateData
+	errMsgList := make([]string, 0)
+	defer func() {
+		if len(errMsgList) > 0 {
+			msg := fmt.Sprint("数据源-CCF数据处理失败,err:", strings.Join(errMsgList, "\n"))
+			utils.FileLog.Info(msg)
+			go alarm_msg.SendAlarmMsg(msg, 3)
+		}
+	}()
+	indexObj := new(models.BaseFromCCFIndex)
+	dataObj := new(models.BaseFromCCFData)
+	//classifyObj := new(models.BaseFromCCFClassify)
+
+	var indexId int64
+
+	addDataList := make([]*models.BaseFromCCFData, 0)
+	updateDataList := make([]*models.BaseFromCCFData, 0)
+
+	exitDataMap := make(map[string]*models.BaseFromCCFData)
+
+	// 修改指标信息
+	if indexName == "" {
+		utils.FileLog.Info("未刷新到指标数据:indexName:" + indexName)
+		return
+	}
+
+	//判断指标是否存在
+	var isAdd int
+	item, err := indexObj.GetByIndexCode(indexCode)
+	if err != nil {
+		if err.Error() == utils.ErrNoRow() {
+			isAdd = 1
+			err = nil
+		} else {
+			isAdd = -1
+			err = fmt.Errorf("查询数据源指标库失败 GetByIndexCode Err：%s", err)
+			return
+		}
+	}
+	if item != nil && item.BaseFromCcfIndexId > 0 {
+		fmt.Println("item:", item)
+		isAdd = 2
+	} else {
+		isAdd = 1
+	}
+
+	if isAdd == 1 {
+		indexObj.IndexCode = indexCode
+		indexObj.IndexName = indexName
+		indexObj.Frequency = req.Frequency
+		indexObj.ClassifyId = req.ClassifyId
+		if req.Unit == "" {
+			req.Unit = "无"
+		}
+		indexObj.Unit = req.Unit
+		indexObj.Sort = req.Sort
+		indexObj.ModifyTime = time.Now()
+		indexObj.CreateTime = time.Now()
+		indexObj.TerminalCode = terminalCode
+		indexId, err = indexObj.Add()
+		if err != nil {
+			err = fmt.Errorf("数据源新增CCF指标失败 Err：%s", err)
+			return
+		}
+		indexObj.BaseFromCcfIndexId = indexId
+	} else if isAdd == 2 {
+		if item.TerminalCode == `` && terminalCode != `` {
+			item.TerminalCode = terminalCode
+			err = item.Update([]string{"TerminalCode"})
+			if err != nil {
+				err = fmt.Errorf("数据源更新CCF指标失败 Err：%s", err)
+				return
+			}
+		}
+
+		//获取已存在的所有数据
+		var exitDataList []*models.BaseFromCCFData
+		exitDataList, err = dataObj.GetByIndexCode(indexCode)
+		if err != nil {
+			err = fmt.Errorf("数据源查询CCF指标数据失败 Err：%s", err)
+			return
+		}
+		fmt.Println("exitDataListLen:", len(exitDataList))
+		for _, v := range exitDataList {
+			dateStr := v.DataTime
+			exitDataMap[dateStr] = v
+		}
+		indexId = item.BaseFromCcfIndexId
+	}
+
+	// 遍历excel数据，然后跟现有的数据做校验，不存在则入库
+	for date, value := range excelDataMap {
+		if findData, ok := exitDataMap[date]; !ok {
+			_, err = time.ParseInLocation(utils.FormatDate, date, time.Local)
+			if err != nil {
+				err = fmt.Errorf("%s 转换日期格式失败 Err：%s", date, err)
+				return
+			}
+			if !strings.Contains(value, "#N/A") {
+				var saveDataTime time.Time
+				if strings.Contains(date, "00:00:00") {
+					saveDataTime, err = time.Parse(utils.FormatDateTime, date)
+				} else {
+					saveDataTime, err = time.Parse(utils.FormatDate, date)
+				}
+				if err != nil {
+					err = fmt.Errorf("%s 转换日期格式失败 Err：%s", date, err)
+					continue
+				}
+				timestamp := saveDataTime.UnixNano() / 1e6
+
+				dataItem := new(models.BaseFromCCFData)
+				dataItem.BaseFromCcfIndexId = int(indexId)
+				dataItem.IndexCode = indexCode
+				dataItem.DataTime = date
+				dataItem.Value = value
+				dataItem.CreateTime = time.Now()
+				dataItem.ModifyTime = time.Now()
+				dataItem.DataTimestamp = timestamp
+				addDataList = append(addDataList, dataItem)
+			}
+		} else {
+			if findData != nil && findData.Value != value && !strings.Contains(value, "#N/A") { //修改数据
+				findData.Value = value
+				findData.ModifyTime = time.Now().Local()
+				updateDataList = append(updateDataList, findData)
+			}
+		}
+	}
+
+	if len(addDataList) > 0 {
+		err = dataObj.AddMulti(addDataList)
+		if err != nil {
+			err = fmt.Errorf("批量新增指标失败 Err：%s", err)
+			return
+		}
+
+		var dateItem *models.EdbInfoMaxAndMinInfo
+		dateItem, err = dataObj.GetMaxAndMinDateByIndexCode(indexCode)
+		if err != nil {
+			err = fmt.Errorf("查询指标最新日期失败 Err：%s", err)
+			return
+		}
+
+		go func() {
+			indexObj.ModifyIndexMaxAndMinDate(indexCode, dateItem)
+		}()
+	}
+
+	// 批量更新数据
+	if len(updateDataList) > 0 {
+		e := models.MultiUpdateBaseFromCCFDataValue(updateDataList)
+		if e != nil {
+			err = fmt.Errorf("MultiUpdateBaseFromSmmDataValue err: %s", e.Error())
+			return
+		}
+	}
+
+	// 同步刷新ETA指标库的指标
+	{
+		// 获取指标详情
+		baseObj := new(models.BaseFromCCF)
+		var edbInfo *models.EdbInfo
+		edbInfo, err = models.GetEdbInfoByEdbCode(baseObj.GetSource(), indexCode)
+		if err != nil {
+			if err.Error() != utils.ErrNoRow() {
+				errMsgList = append(errMsgList, fmt.Sprint("刷新ETA指标异常，指标编码：", indexCode, err.Error()))
+				return
+			} else {
+				err = nil
+			}
+		}
+
+		// 已经加入到指标库的话，那么就去更新ETA指标库吧
+		if edbInfo != nil {
+			go logic.RefreshBaseEdbInfo(edbInfo, ``)
+		}
+	}
+	return
+}
+
+// HandleCCFStockTable 处理CCF的表格数据
+func HandleCCFStockTable(req *models.HandleCCFStockTableReq) (err error) {
+	defer func() {
+		if err != nil {
+			tips := fmt.Sprintf("数据源-CCF数据装置处理失败, err: %v", err)
+			utils.FileLog.Info(tips)
+			go alarm_msg.SendAlarmMsg(tips, 3)
+		}
+	}()
+	if req.Table == nil {
+		err = fmt.Errorf("表格信息为空")
+		return
+	}
+
+	// 已存在则更新表格
+	excelOb := new(models.CCFStockExcel)
+	cond := fmt.Sprintf(` AND %s = ? AND %s = ?`, excelOb.Cols().ClassifyId, excelOb.Cols().ExcelDate)
+	pars := make([]interface{}, 0)
+	pars = append(pars, req.Table.ClassifyId, req.Table.TableDate.Format(utils.FormatDate))
+	exists, e := excelOb.GetItemByCondition(cond, pars, "")
+	if e != nil && e.Error() != utils.ErrNoRow() {
+		err = fmt.Errorf("获取已存在表格失败, err: %v", e)
+		return
+	}
+	if exists != nil {
+		exists.ExcelContent = html.EscapeString(req.Table.TableContent)
+		exists.FromPage = req.Table.FromPage
+		exists.ModifyTime = time.Now().Local()
+		cols := []string{excelOb.Cols().ExcelContent, excelOb.Cols().FromPage, excelOb.Cols().ModifyTime}
+		if e = exists.Update(cols); e != nil {
+			err = fmt.Errorf("更新已存在表格失败, err: %v", e)
+			return
+		}
+		return
+	}
+
+	// 新增表格
+	excelOb.ClassifyId = req.Table.ClassifyId
+	excelOb.ExcelDate = req.Table.TableDate
+	excelOb.ExcelContent = html.EscapeString(req.Table.TableContent)
+	excelOb.FromPage = req.Table.FromPage
+	excelOb.CreateTime = time.Now().Local()
+	excelOb.ModifyTime = time.Now().Local()
+	if e = excelOb.Create(); e != nil {
+		err = fmt.Errorf("新增表格失败, err: %v", e)
+		return
+	}
+	return
+}

services/base_from_pcsg.go

@@ -14,143 +14,55 @@ import (
 )
 
 var (
-	BridgeApiPCSGBloombergDailyUrl   = "/api/pcsg/bloomberg/daily_index"   // 日度指标API
-	BridgeApiPCSGBloombergWeeklyUrl  = "/api/pcsg/bloomberg/weekly_index"  // 周度指标API
-	BridgeApiPCSGBloombergMonthlyUrl = "/api/pcsg/bloomberg/monthly_index" // 月度指标API
+	PCSGBloombergGeneralIndexDataUrl = "/api/pcsg/bloomberg/index_data/general" // 通用指标API
 )
 
-// GetPCSGBloombergDailyFromBridge 获取彭博日度指标
-func GetPCSGBloombergDailyFromBridge() (indexes []models.BaseFromBloombergApiIndexAndData, err error) {
-	defer func() {
-		if err != nil {
-			tips := fmt.Sprintf("GetPCSGBloombergDailyFromBridge-获取彭博日度指标失败, err: %s", err.Error())
-			utils.FileLog.Info(tips)
-			go alarm_msg.SendAlarmMsg(tips, 3)
-		}
-	}()
+type PCSGBloombergApiReq struct {
+	TaskKey   string `description:"任务key"`
+	Frequency string `description:"指标频度"`
+}
 
-	url := fmt.Sprint(utils.EtaBridgeUrl, BridgeApiPCSGBloombergDailyUrl)
-	body := ioutil.NopCloser(strings.NewReader(""))
-	client := &http.Client{}
-	req, e := http.NewRequest("POST", url, body)
-	if e != nil {
-		err = fmt.Errorf("http create request err: %s", e.Error())
-		return
-	}
+type PCSGBloombergTask struct {
+	TaskKey         string `json:"TaskKey"`
+	Frequency       string `json:"Frequency"`
+	VCode           bool   `json:"VCode"`
+	ExtraLetter     string `json:"ExtraLetter"`
+	IndexNamePrefix string `json:"IndexNamePrefix"`
+}
 
-	checkToken := utils.MD5(utils.EtaBridgeAppNameEn + utils.EtaBridgeMd5Key)
-	contentType := "application/json;charset=utf-8"
-	req.Header.Set("Content-Type", contentType)
-	req.Header.Set("Authorization", checkToken)
-	resp, e := client.Do(req)
+// LoadPCSGBloombergTask 加载配置
+func LoadPCSGBloombergTask() (tasks []*PCSGBloombergTask, err error) {
+	filePath := "./static/pcsg_task.json"
+	b, e := ioutil.ReadFile(filePath)
 	if e != nil {
-		err = fmt.Errorf("http client do err: %s", e.Error())
+		err = fmt.Errorf("读取配置失败, err: %v", e)
 		return
 	}
-	defer func() {
-		_ = resp.Body.Close()
-	}()
-	b, e := ioutil.ReadAll(resp.Body)
-	if e != nil {
-		err = fmt.Errorf("resp body read err: %s", e.Error())
+	if e = json.Unmarshal(b, &tasks); e != nil {
+		err = fmt.Errorf("解析配置失败, err: %v", e)
 		return
 	}
-	if len(b) == 0 {
-		err = fmt.Errorf("resp body is empty")
-		return
-	}
-	// 生产环境解密
-	if utils.RunMode == "release" {
-		str := string(b)
-		str = strings.Trim(str, `"`)
-		b = utils.DesBase64Decrypt([]byte(str), utils.EtaBridgeDesKey)
-	}
-
-	result := new(models.BridgePCSGBloombergResultData)
-	if e = json.Unmarshal(b, &result); e != nil {
-		err = fmt.Errorf("result unmarshal err: %s\nresult: %s", e.Error(), string(b))
-		return
-	}
-	if result.Code != 200 {
-		err = fmt.Errorf("result: %s", string(b))
-		return
-	}
-	indexes = result.Data
 	return
 }
 
-// GetPCSGBloombergWeeklyFromBridge 获取彭博周度指标
-func GetPCSGBloombergWeeklyFromBridge() (indexes []models.BaseFromBloombergApiIndexAndData, err error) {
+// GetPCSGBloombergGeneralIndexFromBridge 获取通用数据类型指标
+func GetPCSGBloombergGeneralIndexFromBridge(params PCSGBloombergApiReq) (indexes []models.BaseFromBloombergApiIndexAndData, err error) {
 	defer func() {
 		if err != nil {
-			tips := fmt.Sprintf("GetPCSGBloombergWeeklyFromBridge-获取彭博周度指标失败, err: %s", err.Error())
+			tips := fmt.Sprintf("GetPCSGBloombergGeneralIndexFromBridge-获取指标数据失败, err: %s", err.Error())
 			utils.FileLog.Info(tips)
 			go alarm_msg.SendAlarmMsg(tips, 3)
 		}
 	}()
 
-	url := fmt.Sprint(utils.EtaBridgeUrl, BridgeApiPCSGBloombergWeeklyUrl)
-	body := ioutil.NopCloser(strings.NewReader(""))
-	client := &http.Client{}
-	req, e := http.NewRequest("POST", url, body)
-	if e != nil {
-		err = fmt.Errorf("http create request err: %s", e.Error())
-		return
-	}
-
-	checkToken := utils.MD5(utils.EtaBridgeAppNameEn + utils.EtaBridgeMd5Key)
-	contentType := "application/json;charset=utf-8"
-	req.Header.Set("Content-Type", contentType)
-	req.Header.Set("Authorization", checkToken)
-	resp, e := client.Do(req)
-	if e != nil {
-		err = fmt.Errorf("http client do err: %s", e.Error())
-		return
-	}
-	defer func() {
-		_ = resp.Body.Close()
-	}()
-	b, e := ioutil.ReadAll(resp.Body)
+	p, e := json.Marshal(params)
 	if e != nil {
-		err = fmt.Errorf("resp body read err: %s", e.Error())
-		return
-	}
-	if len(b) == 0 {
-		err = fmt.Errorf("resp body is empty")
+		err = fmt.Errorf("params json marshal err: %v", e)
 		return
 	}
-	// 生产环境解密
-	if utils.RunMode == "release" {
-		str := string(b)
-		str = strings.Trim(str, `"`)
-		b = utils.DesBase64Decrypt([]byte(str), utils.EtaBridgeDesKey)
-	}
 
-	result := new(models.BridgePCSGBloombergResultData)
-	if e = json.Unmarshal(b, &result); e != nil {
-		err = fmt.Errorf("result unmarshal err: %s\nresult: %s", e.Error(), string(b))
-		return
-	}
-	if result.Code != 200 {
-		err = fmt.Errorf("result: %s", string(b))
-		return
-	}
-	indexes = result.Data
-	return
-}
-
-// GetPCSGBloombergMonthlyFromBridge 获取彭博月度指标
-func GetPCSGBloombergMonthlyFromBridge() (indexes []models.BaseFromBloombergApiIndexAndData, err error) {
-	defer func() {
-		if err != nil {
-			tips := fmt.Sprintf("GetPCSGBloombergMonthlyFromBridge-获取彭博月度指标失败, err: %s", err.Error())
-			utils.FileLog.Info(tips)
-			go alarm_msg.SendAlarmMsg(tips, 3)
-		}
-	}()
-
-	url := fmt.Sprint(utils.EtaBridgeUrl, BridgeApiPCSGBloombergMonthlyUrl)
-	body := ioutil.NopCloser(strings.NewReader(""))
+	url := fmt.Sprint(utils.EtaBridgeUrl, PCSGBloombergGeneralIndexDataUrl)
+	body := ioutil.NopCloser(strings.NewReader(string(p)))
 	client := &http.Client{}
 	req, e := http.NewRequest("POST", url, body)
 	if e != nil {
@@ -200,7 +112,7 @@ func GetPCSGBloombergMonthlyFromBridge() (indexes []models.BaseFromBloombergApiI
 }
 
 // PCSGWrite2BaseBloomberg 写入彭博数据源
-func PCSGWrite2BaseBloomberg(indexes []models.BaseFromBloombergApiIndexAndData) (err error) {
+func PCSGWrite2BaseBloomberg(indexes []models.BaseFromBloombergApiIndexAndData, isVCode bool, extraLetter, namePrefix string) (err error) {
 	defer func() {
 		if err != nil {
 			tips := fmt.Sprintf("PCSGWrite2BaseBloomberg-写入彭博数据源失败, err: %s", err.Error())
@@ -208,11 +120,23 @@ func PCSGWrite2BaseBloomberg(indexes []models.BaseFromBloombergApiIndexAndData)
 			go alarm_msg.SendAlarmMsg(tips, 3)
 		}
 	}()
+	// 这里挡一下...万一没限制加进库了不好删...
+	if isVCode && extraLetter == "" {
+		err = fmt.Errorf("中间字母有误")
+		return
+	}
 
 	for _, v := range indexes {
 		if v.IndexCode == "" {
 			continue
 		}
+		// 无数据的情况不处理
+		if len(v.Data) == 0 {
+			continue
+		}
+		if isVCode {
+			v.IndexCode = utils.InsertStr2StrIdx(v.IndexCode, " ", 1, extraLetter)
+		}
 
 		// 指标是否存在
 		index, e := models.GetBaseFromBloombergIndexByCode(v.IndexCode)
@@ -221,11 +145,17 @@ func PCSGWrite2BaseBloomberg(indexes []models.BaseFromBloombergApiIndexAndData)
 			return
 		}
 
+		// 指标名称+前缀
+		indexName := v.IndexName
+		if indexName != "" && namePrefix != "" {
+			indexName = fmt.Sprint(namePrefix, indexName)
+		}
+
 		// 新增指标
 		if index == nil {
 			newIndex := new(models.BaseFromBloombergIndex)
 			newIndex.IndexCode = v.IndexCode
-			newIndex.IndexName = v.IndexName
+			newIndex.IndexName = indexName
 			newIndex.Unit = v.Unit
 			newIndex.Source = utils.DATA_SOURCE_BLOOMBERG
 			newIndex.Frequency = v.Frequency
@@ -236,6 +166,18 @@ func PCSGWrite2BaseBloomberg(indexes []models.BaseFromBloombergApiIndexAndData)
 				return
 			}
 			index = newIndex
+		} else {
+			// 无指标名称的情况下更新指标基础信息
+			if index.IndexName == "" {
+				index.IndexName = indexName
+				index.Unit = v.Unit
+				index.Frequency = v.Frequency
+				index.ModifyTime = time.Now().Local()
+				if e = index.Update([]string{"IndexName", "Unit", "Frequency", "ModifyTime"}); e != nil {
+					err = fmt.Errorf("更新Bloomberg原始指标失败, err: %s", e.Error())
+					return
+				}
+			}
 		}
 
 		// 更新指标数据
@@ -249,6 +191,7 @@ func PCSGWrite2BaseBloomberg(indexes []models.BaseFromBloombergApiIndexAndData)
 			return
 		}
 		dateExist := make(map[string]*models.BaseFromBloombergData)
+		newValExist := make(map[string]bool)
 		if len(indexData) > 0 {
 			for _, d := range indexData {
 				strDate := d.DataTime.Format(utils.FormatDate)
@@ -270,6 +213,11 @@ func PCSGWrite2BaseBloomberg(indexes []models.BaseFromBloombergApiIndexAndData)
 				originData.ModifyTime = time.Now().Local()
 				updateData = append(updateData, originData)
 			} else {
+				// 新增的数据去重
+				if newValExist[strDate] {
+					continue
+				}
+				newValExist[strDate] = true
 				newData := new(models.BaseFromBloombergData)
 				newData.BaseFromBloombergIndexId = index.BaseFromBloombergIndexId
 				newData.IndexCode = index.IndexCode
@@ -305,7 +253,11 @@ func PCSGWrite2BaseBloomberg(indexes []models.BaseFromBloombergApiIndexAndData)
 				return
 			}
 			if edb != nil {
-				logic.RefreshBaseEdbInfo(edb, ``)
+				_, _, e = logic.RefreshBaseEdbInfo(edb, ``)
+				if e != nil {
+					utils.FileLog.Info(fmt.Sprintf("Bloomberg RefreshBaseEdbInfo, edbCode: %s, err: %v", index.IndexCode, e))
+					return
+				}
 			}
 		}()
 	}

services/base_from_python.go

@@ -81,7 +81,7 @@ func ExecPythonCode(edbCode, reqCode string) (dataMap models.EdbDataFromPython,
 	if err != nil {
 		return
 	}
-	pthonCodeStr := getPythonFrontStr() + reqCode + getPythonLaterStr()
+	pythonCodeStr := getPythonFrontStr() + reqCode + getPythonLaterStr()
 	fileHandle, err := os.OpenFile(pythonFile, os.O_RDWR|os.O_CREATE|os.O_EXCL, 0766)
 	if err != nil {
 		return
@@ -89,7 +89,7 @@ func ExecPythonCode(edbCode, reqCode string) (dataMap models.EdbDataFromPython,
 	defer func() {
 		os.Remove(pythonFile)
 	}()
-	_, err = fileHandle.Write([]byte(pthonCodeStr))
+	_, err = fileHandle.Write([]byte(pythonCodeStr))
 	if err != nil {
 		return
 	}
@@ -172,7 +172,36 @@ func getPythonFileAbsolutePath(edbCode string) (pythonFile string, err error) {
 // getPythonFrontStr 获取python前面的代码
 func getPythonFrontStr() string {
 	//return "#!/usr/bin/python\n# -*- coding: UTF-8 -*-\nimport json\n\nimport pymysql\nimport pandas as pd\n\nsql_config = {\n    'host': 'rm-uf67kg347rhjfep5c1o.mysql.rds.aliyuncs.com',\n    'port': 3306,#主机号\n    'user': 'hz_technology',#账户名\n    'passwd': 'hongze@2021',#密码\n    'db': 'test_hz_data',\n    'charset': 'utf8mb4',\n    'cursorclass': pymysql.cursors.DictCursor\n}\n\ndb = pymysql.connect(**sql_config)\ndb.autocommit(1)\ncursor = db.cursor()\npandas_fetch_all = pd.read_sql\n\n# 返回数据\nresult = {}\n\n# 格式化返回数据\ndef format_data(data: pd.DataFrame,\n                index_str: str = \"data_time\",\n                value_str: str = \"value\"\n                ) -> pd.DataFrame:\n    \"\"\"\n        Parameters\n        ----------\n        data : pandas的DataFrame数据结构.\n        index_str : 对象下标字符串，在pandas的DataFrame中的列名.\n        value_str : 对象值字符串，在pandas的DataFrame中的列名\n\n        Returns\n        -------\n        DataFrame or Iterator[DataFrame]\n        例子：{'2007-01-09': 3220.0, '2007-01-10': 3230.0}\n\n        \"\"\"\n    index_list = []  # 空列表\n    value_list = []  # 空列表\n\n    for num in range(1, data.index.size):  # 迭代 所有的指标\n        index_list.append(data[index_str][num])\n        value_list.append(data[value_str][num])\n\n    tmp_data = {\n        \"date\": index_list,\n        \"value\": value_list\n    }\n    pd_data = pd.DataFrame(tmp_data)\n    # print(pd_data)\n    return pd_data\n"
-	str := fmt.Sprintf("#!/usr/bin/python\n# -*- coding: UTF-8 -*-\nimport json\n\nimport pymysql\nimport pandas as pd\n\nsql_config = {\n    'host': '%s',\n    'port': 3306,#主机号\n    'user': '%s',#账户名\n    'passwd': '%s',#密码\n    'db': '%s',\n    'charset': 'utf8mb4',\n    'cursorclass': pymysql.cursors.DictCursor\n}\n\ndb = pymysql.connect(**sql_config)\ndb.autocommit(1)\ncursor = db.cursor()\npandas_fetch_all = pd.read_sql\n\n# 返回数据\nresult = {}\n\n# 格式化返回数据\ndef format_data(data: pd.DataFrame,\n                index_str: str = \"data_time\",\n                value_str: str = \"value\"\n                ) -> pd.DataFrame:\n    \"\"\"\n        Parameters\n        ----------\n        data : pandas的DataFrame数据结构.\n        index_str : 对象下标字符串，在pandas的DataFrame中的列名.\n        value_str : 对象值字符串，在pandas的DataFrame中的列名\n\n        Returns\n        -------\n        DataFrame or Iterator[DataFrame]\n        例子：{'2007-01-09': 3220.0, '2007-01-10': 3230.0}\n\n        \"\"\"\n    index_list = []  # 空列表\n    value_list = []  # 空列表\n\n    for num in range(0, data.index.size):  # 迭代 所有的指标\n        date=data[index_str][num]\n        if isinstance(date,str) is False:\n            index_list.append(data[index_str][num].strftime(\"%%Y-%%m-%%d\"))\n        else:\n            index_list.append(data[index_str][num])\n        value_list.append(data[value_str][num])\n\n    tmp_data = {\n        \"date\": index_list,\n        \"value\": value_list\n    }\n    pd_data = pd.DataFrame(tmp_data)\n    return pd_data\n\n", utils.PYTHON_MYSQL_HOST, utils.PYTHON_MYSQL_USER, utils.PYTHON_MYSQL_PASSWD, utils.PYTHON_MYSQL_DB)
+	str := fmt.Sprintf("#!/usr/bin/python\n# -*- coding: UTF-8 -*-\nimport json\n\nimport pymysql\nimport pandas as pd\nfrom pymongo import MongoClient\nimport pytz\n\nsql_config = {\n    'host': '%s',\n    'port': 3306,#主机号\n    'user': '%s',#账户名\n    'passwd': '%s',#密码\n    'db': '%s',\n    'charset': 'utf8mb4',\n    'cursorclass': pymysql.cursors.DictCursor\n}\n\ndb = pymysql.connect(**sql_config)\ndb.autocommit(1)\ncursor = db.cursor()\npandas_fetch_all = pd.read_sql\n\n", utils.PYTHON_MYSQL_HOST, utils.PYTHON_MYSQL_USER, utils.PYTHON_MYSQL_PASSWD, utils.PYTHON_MYSQL_DB)
+
+	// mongo部分
+	if utils.PYTHON_MONGO_HOST != `` {
+		str += getPythonFront2Str()
+	}
+
+	// 格式化数据部分
+	str += getPythonFront3Str()
+
+	return str
+}
+
+// getPythonFront2Str
+// @Description: mongo部分
+// @author: Roc
+// @datetime 2024-05-07 18:38:03
+// @return string
+func getPythonFront2Str() string {
+	str := fmt.Sprintf("\nfrom pymongo import MongoClient\nfrom dateutil.tz import tzlocal\n# MongoDB 连接配置\nmongo_config = {\n    'host': '%s',  # 替换为你的 MongoDB 连接字符串\n    'database': '%s',  # 替换为你的数据库名\n    'collection': '%s',  # 替换为你的集合名\n    'auth_mechanism': '%s'  # 替换为你的认证机制\n}\n\n# 创建 MongoClient 并连接到数据库\nclient = MongoClient(mongo_config['host'], authMechanism=mongo_config['auth_mechanism'])\nmgo_db = client[mongo_config['database']]\ncollection = mgo_db[mongo_config['collection']]\n\n# 定义时区\nutc_tz = pytz.utc\nlocal_tz = tzlocal()  # 本地时区", utils.PYTHON_MONGO_HOST, utils.PYTHON_MONGO_DATABASE, "edb_data_business", utils.PYTHON_MONGO_AUTH_MECHANISM)
+	return str
+}
+
+// getPythonFront3Str
+// @Description: 第三部分
+// @author: Roc
+// @datetime 2024-05-07 18:37:56
+// @return string
+func getPythonFront3Str() string {
+	str := fmt.Sprintf("\n# 返回数据\nresult = {}\n\n# 格式化返回数据\ndef format_data(data: pd.DataFrame,\n                index_str: str = \"data_time\",\n                value_str: str = \"value\"\n                ) -> pd.DataFrame:\n    \"\"\"\n        Parameters\n        ----------\n        data : pandas的DataFrame数据结构.\n        index_str : 对象下标字符串，在pandas的DataFrame中的列名.\n        value_str : 对象值字符串，在pandas的DataFrame中的列名\n\n        Returns\n        -------\n        DataFrame or Iterator[DataFrame]\n        例子：{'2007-01-09': 3220.0, '2007-01-10': 3230.0}\n\n        \"\"\"\n    index_list = []  # 空列表\n    value_list = []  # 空列表\n\n    for num in range(0, data.index.size):  # 迭代 所有的指标\n        date=data[index_str][num]\n        if isinstance(date,str) is False:\n            index_list.append(data[index_str][num].strftime(\"%%Y-%%m-%%d\"))\n        else:\n            index_list.append(data[index_str][num])\n        value_list.append(data[value_str][num])\n\n    tmp_data = {\n        \"date\": index_list,\n        \"value\": value_list\n    }\n    pd_data = pd.DataFrame(tmp_data)\n    return pd_data\n\n")
 	return str
 }
 

services/base_from_ths_ds_http.go

@@ -80,17 +80,15 @@ func getEdbDataFromThsDsHttp(stockCode, edbCode, startDate, endDate, thsRefreshT
 					value := reflect.ValueOf(sliceValue.Index(i).Interface())
 					if value.Kind() == reflect.Float64 {
 						tableValue = value.Float()
-						if tableValue != 0 {
-							tableTimeList = append(tableTimeList, tmpItems.Tables[0].Time[i])
-							tableValueList = append(tableValueList, tableValue)
-						}
+						tableTimeList = append(tableTimeList, tmpItems.Tables[0].Time[i])
+						tableValueList = append(tableValueList, tableValue)
 					}
 				}
 			} else {
 				fmt.Printf("Field '%s' not found\n", field)
 			}
 			tmpTable := models.Tables{
-				ID:  []string{},
+				ID:    []string{},
 				Time:  tableTimeList,
 				Value: tableValueList,
 			}

services/edb_info_relation.go

@@ -0,0 +1,381 @@
+package services
+
+import (
+	"eta/eta_index_lib/models"
+	"eta/eta_index_lib/services/alarm_msg"
+	"eta/eta_index_lib/utils"
+	"fmt"
+	"sort"
+	"strconv"
+	"time"
+)
+
+// 重置单个计算指标中的引用关系
+func ResetEdbRelation(edbInfoId int) {
+	var logMsg string
+	var replaceTotal int
+	var err error
+	defer func() {
+		if err != nil {
+			msg := fmt.Sprintf(" 重置单个计算指标中的引用关系失败 ResetEdbRelation  err: %v", err)
+			utils.FileLog.Info(msg)
+			fmt.Println(msg)
+			go alarm_msg.SendAlarmMsg(msg, 3)
+		}
+		if logMsg != `` {
+			utils.FileLog.Info(fmt.Sprintf("重置单个计算指标中的引用关系失败 重置总数%d,涉及到的引用id：%s", replaceTotal, logMsg))
+		}
+	}()
+	//查询与该计算指标相关的间接引用或者间接引用关系，如果记录不存在，则不处理
+	_, err = models.GetEdbInfoRelationByChildEdbInfoId(edbInfoId)
+	if err != nil {
+		if err.Error() == utils.ErrNoRow() {
+			err = nil
+			return
+		}
+		err = fmt.Errorf("查询与该计算指标相关的间接引用或者间接引用关系失败，错误信息：%s", err.Error())
+		return
+	}
+
+	//查询当前计算指标最新的引用指标列表
+	newMappingList, err := models.GetEdbInfoCalculateDetailList(edbInfoId)
+	if err != nil {
+		err = fmt.Errorf("查询当前计算指标最新的指标列表失败，错误信息：%s", err.Error())
+		return
+	}
+	//整理关联的来源指标ID
+	newEdbIdList := make([]string, 0)
+	newMappingListMap := make(map[int]*models.EdbInfoCalculateDetail)
+	for _, v := range newMappingList {
+		newEdbIdList = append(newEdbIdList, strconv.Itoa(v.FromEdbInfoId))
+		newMappingListMap[v.FromEdbInfoId] = v
+	}
+	//对指标ID进行排序
+	sort.Strings(newEdbIdList)
+	newEdbIdStr := ""
+	for _, v := range newEdbIdList {
+		newEdbIdStr += v + ","
+	}
+	//二者匹配一下，如果相同，则不处理，如果不同，先查询所有旧的间接引用记录，整理并分组，则删除旧的间接引用记录，新增新的间接引用记录，
+	tmpList, err := models.GetEdbInfoRelationListByChildEdbInfoId(edbInfoId)
+	if err != nil {
+		err = fmt.Errorf("查询当前计算指标的间接引用关系失败，错误信息：%s", err.Error())
+		return
+	}
+
+	parentRelationIds := make([]int, 0)
+	for _, v := range tmpList {
+		parentRelationIds = append(parentRelationIds, v.ParentRelationId)
+	}
+	if len(parentRelationIds) > 0 {
+		// 查询单个项目的引用列表作为判断依据
+		oldEdbIdList, err := models.GetEdbInfoRelationEdbIdsByParentRelationId(parentRelationIds[0], edbInfoId)
+		if err != nil {
+			err = fmt.Errorf("查询当前计算指标的间接引用关系失败，错误信息：%s", err.Error())
+			return
+		}
+		sort.Ints(oldEdbIdList)
+		oldEdbIdStr := ""
+		for _, v := range oldEdbIdList {
+			oldEdbIdStr += strconv.Itoa(v) + ","
+		}
+		// 把切片转成字符串
+		if newEdbIdStr == oldEdbIdStr {
+			return
+		}
+		list, e := models.GetEdbInfoRelationByRelationIds(parentRelationIds)
+		if e != nil {
+			err = fmt.Errorf("查询图表关联指标列表失败 Err:%s", e)
+			return
+		}
+		//查询直接引用指标关联关系
+		edbInfoListMap := make(map[int]struct{})
+		edbInfoIds := make([]int, 0)
+		for _, v := range list {
+			if _, ok := edbInfoListMap[v.EdbInfoId]; !ok {
+				edbInfoListMap[v.EdbInfoId] = struct{}{}
+				edbInfoIds = append(edbInfoIds, v.EdbInfoId)
+			}
+		}
+		edbInfoList := make([]*models.EdbInfo, 0)
+		if len(edbInfoIds) > 0 {
+			// 查询指标信息
+			edbInfoList, err = models.GetEdbInfoByIdList(edbInfoIds)
+			if err != nil {
+				err = fmt.Errorf("查询指标信息失败 Err:%s", err)
+				return
+			}
+		}
+		calculateEdbMappingListMap, calculateEdbMappingIdsMap, err := GetEdbListByEdbInfoId(edbInfoList)
+		if err != nil {
+			err = fmt.Errorf("查询指标关联指标列表失败 Err:%s", err)
+			return
+		}
+		//如何过滤掉只有间接引用，没有直接引用的
+		replaceTotal1, logMsg1, e := UpdateSecondEdbInRelation(list, calculateEdbMappingListMap, calculateEdbMappingIdsMap, edbInfoList)
+		if e != nil {
+			err = e
+			return
+		}
+		replaceTotal += replaceTotal1
+		logMsg += logMsg1
+	}
+	return
+}
+
+// 更新间接引用
+func UpdateSecondEdbInRelation(list []*models.EdbInfoRelation, calculateEdbMappingListMap map[int]*models.EdbInfoCalculateMapping, calculateEdbMappingIdsMap map[int][]int, edbInfoList []*models.EdbInfo) (replaceTotal int, logMsg string, err error) {
+	nowTime := time.Now()
+	edbInfoRelationIds := make([]int, 0)
+	indexCodeList := make([]string, 0)
+	addList := make([]*models.EdbInfoRelation, 0)
+	refreshIds := make([]int, 0)
+	edbInfoMap := make(map[int]*models.EdbInfo)
+	for _, v := range edbInfoList {
+		edbInfoMap[v.EdbInfoId] = v
+	}
+	// 查询所有的直接引用，删除所有的间接引用，添加所有直接引用的间接引用
+	for _, v := range list {
+		if v.RelationType == 0 {
+			edbInfoRelationIds = append(edbInfoRelationIds, v.EdbInfoRelationId)
+			edbInfo, ok := edbInfoMap[v.EdbInfoId]
+			if !ok {
+				err = fmt.Errorf("查询指标信息失败 EdbInfoId:%d", v.EdbInfoId)
+				return
+			}
+			if edbInfo.EdbType == 2 { //计算指标
+				childEdbMappingIds, ok := calculateEdbMappingIdsMap[edbInfo.EdbInfoId]
+				if !ok {
+					err = fmt.Errorf("查询%d指标关联指标列表为空", edbInfo.EdbInfoId)
+					return
+				}
+				for _, childEdbMappingId := range childEdbMappingIds {
+					childEdbMapping, ok2 := calculateEdbMappingListMap[childEdbMappingId]
+					if !ok2 {
+						continue
+					}
+
+					if childEdbMapping.FromSource == utils.DATA_SOURCE_MYSTEEL_CHEMICAL {
+						indexCodeList = append(indexCodeList, childEdbMapping.FromEdbCode)
+					}
+					tmp1 := &models.EdbInfoRelation{
+						ReferObjectId:      v.ReferObjectId,
+						ReferObjectType:    v.ReferObjectType,
+						ReferObjectSubType: v.ReferObjectSubType,
+						EdbInfoId:          childEdbMapping.FromEdbInfoId,
+						EdbName:            childEdbMapping.FromEdbName,
+						Source:             childEdbMapping.FromSource,
+						EdbCode:            childEdbMapping.FromEdbCode,
+						CreateTime:         nowTime,
+						ModifyTime:         nowTime,
+						RelationTime:       childEdbMapping.CreateTime,
+						RelationType:       1,
+						RootEdbInfoId:      edbInfo.EdbInfoId,
+						ChildEdbInfoId:     childEdbMapping.EdbInfoId,
+					}
+					tmp1.RelationCode = fmt.Sprintf("%d_%d_%d_%d", tmp1.RootEdbInfoId, tmp1.ReferObjectId, tmp1.ReferObjectType, tmp1.ReferObjectSubType)
+					addList = append(addList, tmp1)
+					refreshIds = append(refreshIds, childEdbMapping.FromEdbInfoId)
+				}
+			}
+		}
+	}
+
+	if len(edbInfoRelationIds) > 0 {
+		err = models.UpdateSecondRelationEdbInfoId(edbInfoRelationIds, addList, refreshIds, indexCodeList)
+		if err != nil {
+			logMsg = ""
+			err = fmt.Errorf("替换指标引用表中的指标ID失败 Err:%s", err)
+			return
+		}
+		replaceTotal = len(edbInfoRelationIds)
+	}
+	return
+}
+
+// 设置成禁用状态
+func DisableEdbInfoNoUpdate(edbInfo *models.EdbInfo) (err error) {
+	// 如果一个计算指标里，包涵的基础指标是停用状态，那么计算指标也要是停用状态停用状态
+	newBaseEdbInfoList := make([]int, 0)
+	hasFind := make(map[int]struct{})
+	newBaseEdbInfoIds, err := FindBaseEdbInfo(edbInfo.EdbInfoId, newBaseEdbInfoList, hasFind)
+	if err != nil {
+		err = fmt.Errorf("查找基础指标信息失败,err:%v", err)
+		return
+	}
+	// 查询是否存在停用指标，如果存在，则计算指标也要是停用状态
+	total, err := models.GetEdbInfoNoUpdateTotalByIdList(newBaseEdbInfoIds)
+	if err != nil {
+		err = fmt.Errorf("查询基础指标信息失败,err:%v", err)
+		return
+	}
+	if total > 0 {
+		edbInfo.NoUpdate = 1
+		edbInfo.ModifyTime = time.Now()
+		err = edbInfo.Update([]string{"NoUpdate", "ModifyTime"})
+		if err != nil {
+			err = fmt.Errorf("更新计算指标刷新状态失败,err:%v", err)
+			return
+		}
+	}
+	return
+}
+
+// 找到基础指标的过程
+func FindBaseEdbInfo(edbInfoId int, baseEdbInfoList []int, hasFind map[int]struct{}) (newBaseEdbInfoList []int, err error) {
+	newBaseEdbInfoList = baseEdbInfoList
+	if _, ok := hasFind[edbInfoId]; ok {
+		return
+	}
+	// 先找到所有的引用关系
+	//查询当前计算指标最新的引用指标列表
+	newMappingList, err := models.GetEdbInfoCalculateDetailList(edbInfoId)
+	if err != nil {
+		err = fmt.Errorf("查询当前计算指标最新的指标列表失败，错误信息：%s", err.Error())
+		return
+	}
+	hasFind[edbInfoId] = struct{}{}
+	for _, mapping := range newMappingList {
+		newBaseEdbInfoList = append(newBaseEdbInfoList, mapping.FromEdbInfoId)
+		if mapping.EdbType == 1 { // 如果是基础指标，则加入，否则继续找
+		} else {
+			newBaseEdbInfoList, err = FindBaseEdbInfo(mapping.FromEdbInfoId, newBaseEdbInfoList, hasFind)
+		}
+	}
+	return
+}
+
+// 查找当前计算指标的所有溯源指标
+func GetEdbListByEdbInfoId(edbInfoList []*models.EdbInfo) (edbMappingListMap map[int]*models.EdbInfoCalculateMapping, edbInfoMappingRootIdsMap map[int][]int, err error) {
+	if len(edbInfoList) == 0 {
+		return
+	}
+	edbInfoIds := make([]int, 0)
+	for _, v := range edbInfoList {
+		if v.EdbType == 2 && v.EdbInfoType == 0 { //普通计算指标，排除预算指标
+			edbInfoIds = append(edbInfoIds, v.EdbInfoId)
+		}
+	}
+	if len(edbInfoIds) == 0 {
+		return
+	}
+	//查询指标信息
+	allEdbMappingMap := make(map[int][]*models.EdbInfoCalculateMappingInfo, 0)
+	allMappingList, e := models.GetEdbInfoCalculateMappingListByEdbInfoIds(edbInfoIds)
+	if e != nil {
+		err = fmt.Errorf("GetEdbInfoCalculateMappingListByEdbInfoIds err: %s", e.Error())
+		return
+	}
+	for _, v := range allMappingList {
+		if _, ok := allEdbMappingMap[v.EdbInfoId]; !ok {
+			allEdbMappingMap[v.EdbInfoId] = make([]*models.EdbInfoCalculateMappingInfo, 0)
+		}
+		allEdbMappingMap[v.EdbInfoId] = append(allEdbMappingMap[v.EdbInfoId], v)
+	}
+	//查询指标映射
+	//查询所有指标数据
+	//查询这个指标相关的mapping信息放到数组里，
+	//将得到的指标ID信息放到数组里
+	hasFindMap := make(map[int]struct{})
+	edbInfoIdMap := make(map[int]struct{})
+	edbMappingList := make([]*models.EdbInfoCalculateMapping, 0)
+	edbInfoMappingRootIdsMap = make(map[int][]int, 0)
+	edbMappingMap := make(map[int]struct{})
+	for _, edbInfo := range edbInfoList {
+		if edbInfo.EdbType == 2 && edbInfo.EdbInfoType == 0 {
+			edbInfoId := edbInfo.EdbInfoId
+			edbMappingList, err = getCalculateEdbInfoByEdbInfoId(allEdbMappingMap, edbInfoId, hasFindMap, edbInfoIdMap, edbMappingList, edbMappingMap, edbInfoMappingRootIdsMap, edbInfoId)
+			if err != nil {
+				err = fmt.Errorf(" GetCalculateEdbInfoByEdbInfoId err: %s", err.Error())
+				return
+			}
+		}
+	}
+	if len(edbMappingList) == 0 {
+		return
+	}
+	// 查询指标信息
+	// 指标信息map
+	edbInfoIdList := make([]int, 0)
+	for k, _ := range edbInfoIdMap {
+		edbInfoIdList = append(edbInfoIdList, k)
+	}
+	edbMappingListMap = make(map[int]*models.EdbInfoCalculateMapping)
+
+	if len(edbMappingList) > 0 {
+		for _, v := range edbMappingList {
+			edbMappingListMap[v.EdbInfoCalculateMappingId] = v
+		}
+	}
+	return
+}
+
+// getCalculateEdbInfoByEdbInfoId 计算指标追溯
+func getCalculateEdbInfoByEdbInfoId(allEdbMappingMap map[int][]*models.EdbInfoCalculateMappingInfo, edbInfoId int, hasFindMap map[int]struct{}, edbInfoIdMap map[int]struct{}, edbMappingList []*models.EdbInfoCalculateMapping, edbMappingMap map[int]struct{}, edbInfoMappingRootIdsMap map[int][]int, rootEdbInfoId int) (newEdbMappingList []*models.EdbInfoCalculateMapping, err error) {
+	newEdbMappingList = edbMappingList
+	_, ok := hasFindMap[edbInfoId]
+	if ok {
+		return
+	}
+
+	if _, ok1 := edbInfoIdMap[edbInfoId]; !ok1 {
+		edbInfoIdMap[edbInfoId] = struct{}{}
+	}
+	edbInfoMappingList := make([]*models.EdbInfoCalculateMappingInfo, 0)
+	edbInfoMappingList, ok = allEdbMappingMap[edbInfoId]
+	if !ok {
+		edbInfoMappingList, err = models.GetEdbInfoCalculateMappingListByEdbInfoId(edbInfoId)
+		if err != nil {
+			err = fmt.Errorf("GetEdbInfoCalculateMappingListByEdbInfoId err: %s", err.Error())
+			return
+		}
+	}
+	hasFindMap[edbInfoId] = struct{}{}
+	if len(edbInfoMappingList) > 0 {
+		fromEdbInfoIdList := make([]int, 0)
+		edbInfoMappingIdList := make([]int, 0)
+		for _, v := range edbInfoMappingList {
+			fromEdbInfoIdList = append(fromEdbInfoIdList, v.FromEdbInfoId)
+			edbInfoMappingIdList = append(edbInfoMappingIdList, v.EdbInfoCalculateMappingId)
+			if _, ok1 := edbInfoIdMap[v.FromEdbInfoId]; !ok1 {
+				edbInfoIdMap[v.FromEdbInfoId] = struct{}{}
+			}
+			if _, ok2 := edbMappingMap[v.EdbInfoCalculateMappingId]; !ok2 {
+				edbMappingMap[v.EdbInfoCalculateMappingId] = struct{}{}
+				tmp := &models.EdbInfoCalculateMapping{
+					EdbInfoCalculateMappingId: v.EdbInfoCalculateMappingId,
+					EdbInfoId:                 v.EdbInfoId,
+					Source:                    v.Source,
+					SourceName:                v.SourceName,
+					EdbCode:                   v.EdbCode,
+					FromEdbInfoId:             v.FromEdbInfoId,
+					FromEdbCode:               v.FromEdbCode,
+					FromEdbName:               v.FromEdbName,
+					FromSource:                v.FromSource,
+					FromSourceName:            v.FromSourceName,
+					FromTag:                   v.FromTag,
+					Sort:                      v.Sort,
+					CreateTime:                v.CreateTime,
+					ModifyTime:                v.ModifyTime,
+				}
+				newEdbMappingList = append(newEdbMappingList, tmp)
+
+			}
+
+			if edbInfoId != v.FromEdbInfoId && (v.FromEdbType == 2 || v.FromEdbInfoType == 1) {
+				// 查过了就不查了
+				if _, ok2 := hasFindMap[v.FromEdbInfoId]; !ok2 {
+					newEdbMappingList, err = getCalculateEdbInfoByEdbInfoId(allEdbMappingMap, v.FromEdbInfoId, hasFindMap, edbInfoIdMap, newEdbMappingList, edbMappingMap, edbInfoMappingRootIdsMap, rootEdbInfoId)
+					if err != nil {
+						err = fmt.Errorf("traceEdbInfoByEdbInfoId err: %s", err.Error())
+						return
+					}
+				}
+			}
+			hasFindMap[v.FromEdbInfoId] = struct{}{}
+		}
+		edbInfoMappingRootIdsMap[rootEdbInfoId] = append(edbInfoMappingRootIdsMap[rootEdbInfoId], edbInfoMappingIdList...)
+	}
+
+	return
+}

services/factor_edb_series.go

@@ -0,0 +1,75 @@
+package services
+
+import (
+	"eta/eta_index_lib/models"
+	"eta/eta_index_lib/services/alarm_msg"
+	"eta/eta_index_lib/utils"
+	"fmt"
+	"time"
+)
+
+// FactorEdbStepCalculate 因子指标-多公式计算
+func FactorEdbStepCalculate(seriesId, edbInfoId int, edbCode string, edbData []*models.EdbInfoSearchData, calculates []models.CalculatesReq) (err error) {
+	if len(edbData) == 0 || len(calculates) == 0 {
+		return
+	}
+	defer func() {
+		if err != nil {
+			tips := fmt.Sprintf("FactorEdbStepCalculate计算失败, EdbCode: %s, ErrMsg: %v", edbCode, err)
+			utils.FileLog.Info(tips)
+			go alarm_msg.SendAlarmMsg(tips, 2)
+		}
+	}()
+
+	// 重新计算-先清除原数据
+	calculateDataOb := new(models.FactorEdbSeriesCalculateData)
+	{
+		cond := fmt.Sprintf("%s = ?", calculateDataOb.Cols().FactorEdbSeriesId)
+		pars := make([]interface{}, 0)
+		pars = append(pars, seriesId)
+		if e := calculateDataOb.RemoveByCondition(cond, pars); e != nil {
+			err = fmt.Errorf("清除原数据失败, err: %v", e)
+			return
+		}
+	}
+
+	// 多公式嵌套计算
+	dataMap, dateList, _, e := StepCalculate(edbData, calculates)
+	if e != nil {
+		err = fmt.Errorf("嵌套计算失败, err: %v", e)
+		return
+	}
+
+	// 计算成功的保存结果
+	dataArr := make([]*models.FactorEdbSeriesCalculateData, 0)
+	for _, d := range dateList {
+		val, ok := dataMap[d]
+		if !ok {
+			continue
+		}
+		dataTime, e := time.ParseInLocation(utils.FormatDate, d, time.Local)
+		if e != nil {
+			err = fmt.Errorf("解析计算结果日期失败, err: %v", e)
+			return
+		}
+		dataArr = append(dataArr, &models.FactorEdbSeriesCalculateData{
+			FactorEdbSeriesId: seriesId,
+			EdbInfoId:         edbInfoId,
+			EdbCode:           edbCode,
+			DataTime:          dataTime,
+			Value:             val,
+			CreateTime:        time.Now().Local(),
+			ModifyTime:        time.Now().Local(),
+			DataTimestamp:     dataTime.UnixNano() / 1e6,
+		})
+	}
+	if len(dataArr) == 0 {
+		//err = fmt.Errorf("计算结果无数据")
+		return
+	}
+	if e = calculateDataOb.CreateMulti(dataArr); e != nil {
+		err = fmt.Errorf("保存计算结果失败, err: %v", e)
+		return
+	}
+	return
+}

services/sci99/detail_struct.go

@@ -0,0 +1,140 @@
+package sci99
+
+// DataItem 定义了数据项的结构
+type DataItem struct {
+	DataTemplate        string          `json:"DataTemplate"`
+	ProductName         string          `json:"ProductName"`
+	ProductID           int             `json:"ProductID"`
+	Region              string          `json:"Region"`
+	Area                string          `json:"Area"`
+	MarketSampleName    string          `json:"MarketSampleName"`
+	Model               string          `json:"Model"`
+	FactorySampleName   string          `json:"FactorySampleName"`
+	DataTypeName        string          `json:"DataTypeName"`
+	DataTypeID          int             `json:"DataTypeID"`
+	Order               int             `json:"Order"`
+	Unit                string          `json:"Unit"`
+	DataName            string          `json:"DataName"`
+	DIID                int             `json:"DIID"`
+	Gid                 string          `json:"Gid"`
+	PriceProduct        *string         `json:"PriceProduct"` // 使用指针以处理null值
+	PriceProductID      int             `json:"PriceProductID"`
+	PriceType           string          `json:"PriceType"`
+	PriceTypeID         int             `json:"PriceTypeID"`
+	PriceCondition      string          `json:"PriceCondition"`
+	TradeTerms          string          `json:"TradeTerms"`
+	DataStatus          int             `json:"DataStatus"`
+	Cycle               string          `json:"Cycle"`
+	DataModelName       string          `json:"DataModelName"`
+	TimeMarkID          int             `json:"TimeMarkID"`
+	Digits              int             `json:"Digits"`
+	Model_zh            string          `json:"Model_zh"`
+	Province            string          `json:"Province"`
+	Payment             string          `json:"Payment"`
+	PickModel           string          `json:"PickModel"`
+	Tax                 string          `json:"Tax"`
+	PackModel           string          `json:"PackModel"`
+	TransModel          string          `json:"TransModel"`
+	BalanceModel        string          `json:"BalanceModel"`
+	EName               string          `json:"EName"`
+	GasRate             string          `json:"GasRate"`
+	FromArea            string          `json:"FromArea"`
+	DIHistory           []DIHistoryItem `json:"DIHistory"`
+	Purpose             string          `json:"Purpose"`
+	FuturesMark         string          `json:"FuturesMark"`
+	City                string          `json:"City"`
+	MainShippers        string          `json:"MainShippers"`
+	PriceClassification string          `json:"PriceClassification"`
+	FactoryID           int             `json:"FactoryID"`
+	Stand               string          `json:"Stand"`
+	Type                string          `json:"Type"`
+	DItemDTypeID        int             `json:"DItemDTypeID"`
+	Classify            string          `json:"Classify"`
+	LockDateName        string          `json:"LockDateName"`
+	DataItemName        string          `json:"DataItemName"`
+}
+
+// DIHistoryItem 定义了DIHistory中的每个项目的结构
+type DIHistoryItem struct {
+	TransModel          map[string]string `json:"TransModel"`
+	SpecialData         map[string]string `json:"SpecialData"`
+	LockTimeMarkID      int               `json:"LockTimeMarkID"`
+	DataModelName       string            `json:"DataModelName"`
+	Factory             map[string]string `json:"Factory"`
+	Tax                 map[string]string `json:"Tax"`
+	LockTimeMarkName    map[string]string `json:"LockTimeMarkName"`
+	StartDate           string            `json:"StartDate"`
+	NumeratorUnitID     int               `json:"NumeratorUnitID"`
+	DenominatorUnitID   int               `json:"DenominatorUnitID"`
+	FromArea            map[string]string `json:"FromArea"`
+	PushTimeMarkID      int               `json:"PushTimeMarkID"`
+	TradeTerms          map[string]string `json:"TradeTerms"`
+	TimeMarkName        map[string]string `json:"TimeMarkName"`
+	NumeratorUnitName   map[string]string `json:"NumeratorUnitName"`
+	PushTimeMarkName    map[string]string `json:"PushTimeMarkName"`
+	TimeMarkID          int               `json:"TimeMarkID"`
+	PickModel           map[string]string `json:"PickModel"`
+	EndDate             string            `json:"EndDate"`
+	Market              map[string]string `json:"Market"`
+	DenominatorUnitName map[string]string `json::"DenominatorUnitName"`
+	Area                map[string]string `json:"Area"`
+	Payment             map[string]string `json:"Payment"`
+	FactorySampleName   map[string]string `json:"FactorySampleName"`
+	PackModel           map[string]string `json:"PackModel"`
+	DataModel           int               `json:"DataModel"`
+	Model               map[string]string `json:"Model"`
+	DecimalPoint        int               `json:"DecimalPoint"`
+	BalanceModel        map[string]string `json:"BalanceModel"`
+}
+
+// List 定义了Data数组中的每个项目的结构
+type List struct {
+	DIID              int     `json:"DIID"`
+	DataTypeID        int     `json:"DataTypeID"`
+	InitialValue      float64 `json:"InitialValue"`
+	EndingValue       float64 `json:"EndingValue"`
+	LDataValue        float64 `json:"LDataValue"`
+	HDataValue        float64 `json:"HDataValue"`
+	MDataValue        float64 `json:"MDataValue"`
+	Change            float64 `json:"Change"`
+	ChangeRate        float64 `json:"ChangeRate"`
+	AmplitudeValue    float64 `json:"AmplitudeValue"`
+	Remark            *string `json:"Remark"` // 使用指针以处理null值
+	DataDate          string  `json:"DataDate"`
+	RealDate          string  `json:"realDate"`
+	MarketSampleName  string  `json:"MarketSampleName"`
+	FactorySampleName *string `json:"FactorySampleName"` // 使用指针以处理null值
+	Model             string  `json:"Model"`
+	Unit              string  `json:"Unit"`
+	PriceCondition    string  `json:"PriceCondition"`
+	Area              string  `json:"Area"`
+	Province          string  `json:"Province"`
+	City              string  `json:"City"`
+	Region            string  `json:"Region"`
+	LockState         string  `json:"LockState"`
+}
+
+// DetailResponse
+type DetailResponse struct {
+	Status int    `json:"status"`
+	Msg    string `json:"msg"`
+	Data   struct {
+		DataItem   DataItem  `json:"DataItem"`
+		UserPower  UserPower `json:"UserPower"`
+		List       []List    `json:"List"`
+		Privileged bool      `json:"Privileged"`
+		Choice     string    `json:"choice"`
+		LastDate   string    `json:"LastDate"`
+		IsCollect  bool      `json:"IsCollect"`
+		CollectId  string    `json:"collectId"`
+	} `json:"data"`
+}
+
+// UserPower 定义了用户权限的结构
+type UserPower struct {
+	Start     string `json:"Start"`
+	End       string `json:"End"`
+	PowerType int    `json:"PowerType"`
+	ProductID int    `json:"ProductID"`
+	Ppid      int    `json:"Ppid"`
+}

services/sci99/list_struct.go

@@ -0,0 +1,127 @@
+package sci99
+
+// Header 代表表头信息
+type Header struct {
+	Code        string  `json:"Code"`
+	DisplayName string  `json:"DisplayName"`
+	OrderNO     float64 `json:"OrderNO"`
+	DataType    int     `json:"DataType"`
+	DisplayType int     `json:"DisplayType"`
+}
+
+// Item 代表数据项
+type Item struct {
+	Area                string `json:"Area"`
+	DataModelName       string `json:"DataModelName"`
+	DataTypeID          int    `json:"DataTypeID"`
+	Digits              int    `json:"Digits"`
+	DIID                int    `json:"DIID"`
+	FactorySampleName   string `json:"FactorySampleName"`
+	MarketSampleName    string `json:"MarketSampleName"`
+	Model               string `json:"Model"`
+	TradeTerms          string `json:"TradeTerms"`
+	PriceType           string `json:"PriceType"`
+	PriceTypeID         int    `json:"PriceTypeID"`
+	ProductName         string `json:"ProductName"`
+	ProductID           int    `json:"ProductID"`
+	Province            string `json:"Province"`
+	Region              string `json:"Region"`
+	TimeMarkID          int    `json:"TimeMarkID"`
+	Unit                string `json:"Unit"`
+	Order               int    `json:"Order"`
+	DataName            string `json:"DataName"`
+	DataItemName        string `json:"DataItemName"`
+	DataTypeName        string `json:"DataTypeName"`
+	IsWarn              bool   `json:"IsWarn"`
+	EName               string `json:"EName"`
+	GasRate             string `json:"GasRate"`
+	FromArea            string `json:"FromArea"`
+	DataStatus          int    `json:"DataStatus"`
+	Purpose             string `json:"Purpose"`
+	FuturesMark         string `json:"FuturesMark"`
+	MainShippers        string `json:"MainShippers"`
+	PriceClassification string `json:"PriceClassification"`
+	Stand               string `json:"Stand"`
+	Type                string `json:"Type"`
+	DItemDTypeID        int    `json:"DItemDTypeID"`
+	Classification      string `json:"Classification"`
+	LockDateName        string `json:"LockDateName"`
+	PriceCondition      string `json:"PriceCondition"`
+	// 动态日期字段，需要特殊处理
+	//DateFields        map[string]string `json:"date_fields,inline"`
+	Change      string `json:"Change"`
+	ChangeRate  string `json:"ChangeRate"`
+	Remark      string `json:"Remark"`
+	LockState   string `json:"LockState"`
+	PowerStatus int    `json:"PowerStatus"`
+}
+
+// Data 代表数据部分
+type FirstData struct {
+	Headers    []Header   `json:"Headers"`
+	SecondData SecondData `json:"data"`
+}
+
+type SecondData struct {
+	LockTimeRange string  `json:"LockTimeRange"`
+	PageNO        int     `json:"PageNO"`
+	PageSize      int     `json:"PageSize"`
+	TotalItems    int     `json:"TotalItems"`
+	TotalPages    int     `json:"TotalPages"`
+	Items         []Item  `json:"Items"`
+	DiidDtypeids  *string `json:"DiidDtypeids"` // 使用指针以处理null值
+	TableHeader   *string `json:"TableHeader"`  // 使用指针以处理null值
+}
+
+// ListResponse 代表整个API响应
+type ListResponse struct {
+	Status    int       `json:"status"`
+	Msg       string    `json:"msg"`
+	FirstData FirstData `json:"data"`
+}
+
+// 创建请求体的数据结构
+type ListRequestBody struct {
+	Region           string `json:"region"`
+	Market           string `json:"market"`
+	Factory          string `json:"factory"`
+	Model            string `json:"model"`
+	Pname            string `json:"pname"`
+	CycleType        string `json:"cycletype"`
+	PriceCycle       string `json:"pricecycle"`
+	SpecialPriceType string `json:"specialpricetype"`
+	Groupname        string `json:"groupname"`
+	Ppname           string `json:"ppname"`
+	Province         string `json:"province"`
+	PriceTypeID      int    `json:"pricetypeid"`
+	PPIDs            string `json:"ppids"`
+	Navid            string `json:"navid"`
+	SiteType         int    `json:"sitetype"`
+	PageNo           int    `json:"pageno"`
+	PageSize         string `json:"pagesize"`
+	Purpose          string `json:"purpose"`
+	Stand            string `json:"stand"`
+	Type             string `json:"type"`
+	FromArea         string `json:"fromarea"`
+	Classification   string `json:"classification"`
+}
+
+// DetailRequest
+type DetailRequest struct {
+	Start        string `json:"start"`
+	End          string `json:"end"`
+	APIStart     string `json:"apiStart"`
+	APIEnd       string `json:"apiEnd"`
+	DIIDD        string `json:"diid"`
+	DataTypeID   string `json:"datatypeid"`
+	PPID         string `json:"ppid"`
+	CycleType    string `json:"cycletype"`
+	SelectConfig int    `json:"selectconfig"`
+}
+
+type RefreshSci99CrawlerReq struct {
+	ClassifyName string
+	IndexName    string
+	ListData     Item
+	DetailData   DetailResponse
+}

services/sci99/sci99.go

@@ -0,0 +1,146 @@
+package sci99
+
+import (
+	"eta/eta_index_lib/models"
+	"github.com/mozillazg/go-pinyin"
+	"strings"
+)
+
+var IndexCodeMap = make(map[string]string)
+var IndexMap = make(map[string]*models.BaseFromSci99Index)
+var ClassifyMap = make(map[string]*models.BaseFromSci99Classify)
+
+var PriceTypeIDs = []int{34320, 34318}
+var requestList = []ListRequestBody{
+	{
+		Ppname: "天然橡胶原料",
+		PPIDs:  "13676",
+		Navid:  "593",
+	}, {
+		Ppname: "混合胶",
+		PPIDs:  "12973",
+		Navid:  "591",
+	}, {
+		Ppname: "标准胶",
+		PPIDs:  "12960",
+		Navid:  "590",
+	}, {
+		Ppname: "烟片胶",
+		PPIDs:  "12985",
+		Navid:  "592",
+	}, {
+		Ppname: "天然乳胶",
+		PPIDs:  "12947",
+		Navid:  "595",
+	}, {
+		Ppname: "丁苯橡胶",
+		PPIDs:  "12951",
+		Navid:  "596",
+	}, {
+		Ppname: "顺丁橡胶",
+		PPIDs:  "12964",
+		Navid:  "597",
+	}, {
+		Ppname: "SBS",
+		PPIDs:  "12948",
+		Navid:  "598",
+	}, {
+		Ppname: "丁腈橡胶",
+		PPIDs:  "12945",
+		Navid:  "605",
+	},
+}
+
+
+func Sci99IndexCodeGenerator(indexName, indexCodeStr, marketSampleName, model string) (indexCode string, needAdd bool) {
+	strResult := ""
+	indexCode, _ = IndexCodeMap[indexName]
+	if indexCode == "" {
+		//首字母
+		a := pinyin.NewArgs()
+		a.Fallback = func(r rune, a pinyin.Args) []string {
+			return []string{string(r)}
+		}
+		rows := pinyin.Pinyin(indexCodeStr, a)
+		for i := 0; i < len(rows); i++ {
+			//strResult += rows[i][0]
+			if len(rows[i]) != 0 {
+				str := rows[i][0]
+				pi := str[0:1]
+				strResult += pi
+			}
+		}
+
+		// 处理市场名称
+		if province, ok := ProvinceMap[marketSampleName]; ok {
+			strResult += province
+		} else {
+			a := pinyin.NewArgs()
+			rows := pinyin.LazyPinyin(marketSampleName, a)
+			for i := 0; i < len(rows); i++ {
+				strResult += rows[i]
+			}
+		}
+
+		// 去除特殊符号
+		model = strings.Replace(model, " ", "", -1)
+		model = strings.Replace(model, "-", "", -1)
+		model = strings.Replace(model, "/", "", -1)
+		model = strings.Replace(model, "#", "", -1)
+		model = strings.Replace(model, "：", "", -1)
+		model = strings.Replace(model, "(", "", -1)
+		model = strings.Replace(model, ")", "", -1)
+
+		// 拼接型号
+		modelRows := pinyin.Pinyin(model, a)
+		for i := 0; i < len(modelRows); i++ {
+			if len(modelRows[i]) != 0 {
+				str := modelRows[i][0]
+				pi := str[0:1]
+				strResult += pi
+			}
+		}
+
+		needAdd = true
+		indexCode = strings.Replace(strResult, " ", "", -1)
+		IndexCodeMap[indexName] = indexCode
+	}
+	return
+}
+
+var ProvinceMap = map[string]string{
+	"上海":  "shanghai",
+	"云南":  "yunnan",
+	"内蒙古": "innermongolia",
+	"北京":  "beijing",
+	"台湾":  "taiwan",
+	"吉林":  "jilin",
+	"四川":  "sichuan",
+	"天津":  "tianjin",
+	"宁夏":  "ningxia",
+	"安徽":  "anhui",
+	"山东":  "shandong",
+	"山西":  "shanxi",
+	"广东":  "guangdong",
+	"广西":  "guangxi",
+	"新疆":  "xinjiang",
+	"江苏":  "jiangsu",
+	"江西":  "jiangxi",
+	"河北":  "hebei",
+	"河南":  "henan",
+	"浙江":  "zhejiang",
+	"海南":  "hainan",
+	"湖北":  "hubei",
+	"湖南":  "hunan",
+	"澳门":  "macao",
+	"甘肃":  "gansu",
+	"福建":  "fujian",
+	"西藏":  "tibet",
+	"贵州":  "guizhou",
+	"辽宁":  "liaoning",
+	"重庆":  "chongqing",
+	"陕西":  "shaanxi",
+	"青海":  "qinhai",
+	"香港":  "hongkong",
+	"黑龙江": "heilongjiang",
+}

services/shanghai_smm/shanghai_smm.go

@@ -0,0 +1,31 @@
+package shanghaismm
+
+import "time"
+
+type SmmData struct {
+	Highs           float64 `json:"highs"`
+	Low             float64 `json:"low"`
+	Average         float64 `json:"average"`
+	VchangeRate     float64 `json:"vchange_rate"`
+	LowShow         string  `json:"low_show"`
+	HighShow        string  `json:"high_show"`
+	AverageShow     string  `json:"average_show"`
+	ProductId       string  `json:"product_id"`
+	Vchange         float64 `json:"vchange"`
+	RenewDate       string  `json:"renew_date"`
+	ChangeValueShow string  `json:"change_value_show"`
+	ChangeRateShow  string  `json:"change_rate_show"`
+}
+
+type EdbInfoData struct {
+	ClassifyName string
+	IndexName    string
+	IndexCode    string
+	Frequency    string
+	Unit         string
+	Value        string
+	LastDate     time.Time
+	OldDate      time.Time
+	Data         map[string]float64
+	SmmData
+}

static/pcsg_task.json

@@ -0,0 +1,65 @@
+[
+  {
+    "TaskKey": "IDpcsgDailyRunHistU2",
+    "Frequency": "日度",
+    "VCode": false,
+    "ExtraLetter": "",
+    "IndexNamePrefix": ""
+  },
+  {
+    "TaskKey": "IDpcsgDailyRunHist4",
+    "Frequency": "日度",
+    "VCode": false,
+    "ExtraLetter": "",
+    "IndexNamePrefix": ""
+  },
+  {
+    "TaskKey": "IDpcsgDailyRunHist1",
+    "Frequency": "日度",
+    "VCode": false,
+    "ExtraLetter": "",
+    "IndexNamePrefix": ""
+  },
+  {
+    "TaskKey": "IDpcsgDailyRunHist2",
+    "Frequency": "日度",
+    "VCode": false,
+    "ExtraLetter": "",
+    "IndexNamePrefix": ""
+  },
+  {
+    "TaskKey": "IDpcsgDailyRunHistV1",
+    "Frequency": "日度",
+    "VCode": false,
+    "ExtraLetter": "",
+    "IndexNamePrefix": ""
+  },
+  {
+    "TaskKey": "IDpcsgDailyRun4",
+    "Frequency": "日度",
+    "VCode": false,
+    "ExtraLetter": "",
+    "IndexNamePrefix": ""
+  },
+  {
+    "TaskKey": "IDpcsgDailyRun6",
+    "Frequency": "日度",
+    "VCode": false,
+    "ExtraLetter": "",
+    "IndexNamePrefix": ""
+  },
+  {
+    "TaskKey": "IDpcsgDailyRun7",
+    "Frequency": "日度",
+    "VCode": true,
+    "ExtraLetter": "O",
+    "IndexNamePrefix": "Open Interest -"
+  },
+  {
+    "TaskKey": "IDpcsgMonthRun2",
+    "Frequency": "月度",
+    "VCode": false,
+    "ExtraLetter": "",
+    "IndexNamePrefix": ""
+  }
+]

utils/calculate.go

@@ -50,6 +50,54 @@ func GetLinearResult(s []Coordinate) (gradient, intercept float64) {
 	return
 }
 
+// CalculateCorrelationByIntArr 相关性计算
+// 计算步骤
+// 1.分别计算两个序列的平均值Mx和My
+// 2.分别计算两个序列的标准偏差SDx和SDy	=> √{1/(n-1)*SUM[(Xi-Mx)²]}
+// 3.计算相关系数	=> SUM[(Xi-Mx)*(Yi-My)]/[(N-1)(SDx*SDy)]
+func CalculateCorrelationByIntArr(xArr, yArr []float64) (ratio float64) {
+	// 序列元素数要一致
+	xLen := float64(len(xArr))
+	yLen := float64(len(yArr))
+	if xLen == 0 || xLen != yLen {
+		return
+	}
+
+	// 计算Mx和My
+	var Xa, Ya float64
+	for i := range xArr {
+		Xa += xArr[i]
+	}
+	Mx := Xa / xLen
+	for i := range yArr {
+		Ya += yArr[i]
+	}
+	My := Ya / yLen
+
+	// 计算标准偏差SDx和SDy
+	var Xb, Yb, SDx, SDy float64
+	for i := range xArr {
+		Xb += (xArr[i] - Mx) * (xArr[i] - Mx)
+	}
+	SDx = math.Sqrt(1 / (xLen - 1) * Xb)
+	for i := range yArr {
+		Yb += (yArr[i] - My) * (yArr[i] - My)
+	}
+	SDy = math.Sqrt(1 / (yLen - 1) * Yb)
+
+	// 计算相关系数
+	var Nume, Deno float64
+	for i := 0; i < int(xLen); i++ {
+		Nume += (xArr[i] - Mx) * (yArr[i] - My)
+	}
+	Deno = (xLen - 1) * (SDx * SDy)
+	ratio = Nume / Deno
+	if math.IsNaN(ratio) {
+		ratio = 0
+	}
+	return
+}
+
 // ComputeCorrelation 通过一组数据获取相关系数R
 // 计算步骤
 // 1.分别计算两个序列的平均值Mx和My

utils/common.go

@@ -7,6 +7,7 @@ import (
 	"encoding/base64"
 	"encoding/hex"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"github.com/shopspring/decimal"
 	"image"
@@ -1224,3 +1225,88 @@ func FloatAlmostEqual(a, b float64) bool {
 	epsilon := 1e-9 // 容差值
 	return math.Abs(a-b) <= epsilon
 }
+
+// VerifyFrequency
+// @Description: 校验频度是否合规
+// @author: Roc
+// @datetime 2024-04-26 13:30:22
+// @param frequency string 待校验的频度
+// @return bool
+func VerifyFrequency(frequency string) bool {
+	return InArrayByStr([]string{"年度", "半年度", "季度", "月度", "旬度", "周度", "日度"}, frequency)
+}
+
+// DateConvMysqlConvMongo
+// @Description: 将mysql中的日期比较符转换成mongo中的日期比较符
+// @author: Roc
+// @datetime 2024-05-08 11:03:26
+// @param dateCon string
+func DateConvMysqlConvMongo(dateCon string) string {
+	cond := ""
+	switch dateCon {
+	case "=":
+		cond = "$eq"
+	case "<":
+		cond = "$lt"
+	case "<=":
+		cond = "$lte"
+	case ">":
+		cond = "$gt"
+	case ">=":
+		cond = "$gte"
+	}
+	return cond
+}
+
+// GenerateEdbCodeMap 当前已经生成的指标编码map（暂时不做定时数据清理了，因为数据不大，我们至少每个月会重启一次，所以暂时不做定时数据清理）
+var GenerateEdbCodeMap = map[string]bool{}
+
+// GenerateEdbCode
+// @Description:  生成指标编码
+// @author: Roc
+// @datetime 2024-06-05 09:49:53
+// @param num int
+// @param pre string 前缀
+// @return edbCode string
+// @return err error
+func GenerateEdbCode(num int, pre string) (edbCode string, err error) {
+	if num >= 10 {
+		err = errors.New("指标编码生成失败，请重新生成")
+		return
+	}
+
+	// 4位随机数
+	randStr := GetRandDigit(4)
+	// 年月日时分秒+4位随机数
+	edbCode = `C` + pre + time.Now().Format(FormatShortDateTimeUnSpace) + randStr
+
+	if _, ok := GenerateEdbCodeMap[edbCode]; ok {
+		num++
+		edbCode, err = GenerateEdbCode(num, pre)
+	}
+
+	GenerateEdbCodeMap[edbCode] = true
+
+	return
+}
+
+// InsertStr2StrIdx 可分隔的字符串中插入指定字符串, 例如: CO1 Comdty插入V => CO1 V Comdty
+func InsertStr2StrIdx(str, sep string, idx int, value string) string {
+	str = strings.TrimSpace(str)
+	// 默认以空格作为分隔符
+	if sep == "" {
+		sep = " "
+	}
+	slice := strings.Split(str, sep)
+	if len(slice) < 2 {
+		return str
+	}
+
+	// 如果idx不在切片的有效范围内，直接返回原字符串
+	if idx < 0 || idx > len(slice) {
+		return str
+	}
+
+	slice = append(slice[:idx], append([]string{value}, slice[idx:]...)...)
+	return strings.Join(slice, sep)
+}

utils/config.go

@@ -5,25 +5,33 @@ import (
 	"fmt"
 	beeLogger "github.com/beego/bee/v2/logger"
 	"github.com/beego/beego/v2/server/web"
+	"github.com/qiniu/qmgo"
 	"strconv"
 )
 
 var (
-	RunMode       string //运行模式
-	MYSQL_URL     string //数据库连接
-	MYSQL_URL_EDB string
-	MYSQL_URL_GL  string
-
-	PYTHON_MYSQL_HOST   string //python数据库链接主机地址
-	PYTHON_MYSQL_USER   string //python数据库链接账号
-	PYTHON_MYSQL_PASSWD string //python数据库链接密码
-	PYTHON_MYSQL_DB     string //python数据库链接数据库名
-	PYTHON_PATH         string //python可执行文件地址
+	RunMode          string //运行模式
+	MYSQL_URL        string //数据库连接
+	MYSQL_URL_EDB    string
+	MYSQL_URL_GL     string
+	MYSQL_URL_MASTER string
+	MgoUrlData       string // mongodb数据库连接配置
+
+	PYTHON_MYSQL_HOST           string // python数据库链接主机地址
+	PYTHON_MYSQL_USER           string // python数据库链接账号
+	PYTHON_MYSQL_PASSWD         string // python数据库链接密码
+	PYTHON_MYSQL_DB             string // python数据库链接数据库名
+	PYTHON_MONGO_HOST           string // python链接mongo主机地址
+	PYTHON_MONGO_DATABASE       string // python链接mongo的库名
+	PYTHON_MONGO_AUTH_MECHANISM string // python认证机制
+	PYTHON_PATH                 string // python可执行文件地址
 
 	REDIS_CACHE string //缓存地址
 	//Rc          *cache.Cache //redis缓存
-	Re error       //redis错误
-	Rc RedisClient //redis缓存
+	Re            error        //redis错误
+	Rc            RedisClient  //redis缓存
+	MgoDataCli    *qmgo.Client // mongodb客户端连接
+	MgoDataDbName string       // mongodb指标数据的库名
 )
 
 // DATA_INDEX_NAME 数据指标库ES索引名称
@@ -96,6 +104,10 @@ var (
 	LogMaxDays int //日志最大保留天数
 )
 
+var (
+	UseMongo bool // 是否使用mongo
+)
+
 func init() {
 	tmpRunMode, err := web.AppConfig.String("run_mode")
 	if err != nil {
@@ -125,12 +137,21 @@ func init() {
 	MYSQL_URL = config["mysql_url"]
 	MYSQL_URL_EDB = config["mysql_url_edb"]
 	MYSQL_URL_GL = config["mysql_url_gl"]
+	MYSQL_URL_MASTER = config["mysql_url_master"]
+
+	// mongodb数据库连接配置
+	MgoUrlData = config["mgo_url_data"]
 
 	//python链接
 	PYTHON_MYSQL_HOST = config["python_mysql_host"]
 	PYTHON_MYSQL_USER = config["python_mysql_user"]
 	PYTHON_MYSQL_PASSWD = config["python_mysql_passwd"]
 	PYTHON_MYSQL_DB = config["python_mysql_db"]
+
+	PYTHON_MONGO_HOST = config["python_mongo_host"]                     // python链接mongo主机地址
+	PYTHON_MONGO_DATABASE = config["python_mongo_db"]                   // python链接mongo的库名
+	PYTHON_MONGO_AUTH_MECHANISM = config["python_mongo_auth_mechanism"] // python链接认证机制
+
 	PYTHON_PATH = config["python_path"]
 
 	REDIS_CACHE = config["beego_cache"]

utils/constants.go

@@ -20,11 +20,6 @@ const (
 	PageSize30                 = 30
 )
 
-const (
-	DATA_SOURCE_YONYI  = 76 //涌益咨询
-	DATA_SOURCE_FENWEI = 77 //汾渭煤炭
-)
-
 // 数据来源渠道
 const (
 	DATA_SOURCE_THS                                  = iota + 1 //同花顺
@@ -103,105 +98,111 @@ const (
 	DATA_SOURCE_CALCULATE_ZDYFX                                 // 自定义分析->74
 	DATA_SOURCE_CALCULATE_RJZ                                   // 日均值计算->75
 
+	DATA_SOURCE_YONYI         = 76 //涌益咨询
+	DATA_SOURCE_FENWEI        = 77 //汾渭煤炭
 	DATA_SOURCE_GFEX          = 78 // 广州期货交易所->78
 	DATA_SOURCE_ICPI          = 79 // ICPI消费价格指数->79
 	DATA_SOURCE_MTJH          = 80 // 煤炭江湖->80
 	DATA_SOURCE_CALCULATE_SUM = 81
 	DATA_SOURCE_CALCULATE_AVG = 82
 	DATA_SOURCE_BLOOMBERG     = 83 // bloomberg彭博数据
+	DATA_SOURCE_BUSINESS      = 84 // 来源于自有数据
+	DATA_SOURCE_SCI99         = 85 // 卓创资讯
+	DATA_SOURCE_CCF           = 86 // CCF化纤信息
 )
 
 // 指标来源的中文展示
 const (
-	DATA_SOURCE_NAME_THS                                  = `同花顺`               //同花顺
-	DATA_SOURCE_NAME_WIND                                 = `wind`              //wind
-	DATA_SOURCE_NAME_PB                                   = `彭博`                //彭博
+	DATA_SOURCE_NAME_THS                                  = `同花顺`                //同花顺
+	DATA_SOURCE_NAME_WIND                                 = `wind`                  //wind
+	DATA_SOURCE_NAME_PB                                   = `彭博`                  //彭博
 	DATA_SOURCE_NAME_CALCULATE                            = `指标运算`              //指标运算
-	DATA_SOURCE_NAME_CALCULATE_LJZZY                      = `累计值转月值`            //累计值转月
-	DATA_SOURCE_NAME_CALCULATE_TBZ                        = `同比值`               //同比值
-	DATA_SOURCE_NAME_CALCULATE_TCZ                        = `同差值`               //同差值
-	DATA_SOURCE_NAME_CALCULATE_NSZYDPJJS                  = `N数值移动平均计算`         //N数值移动平均计算
+	DATA_SOURCE_NAME_CALCULATE_LJZZY                      = `累计值转月值`          //累计值转月
+	DATA_SOURCE_NAME_CALCULATE_TBZ                        = `同比值`                //同比值
+	DATA_SOURCE_NAME_CALCULATE_TCZ                        = `同差值`                //同差值
+	DATA_SOURCE_NAME_CALCULATE_NSZYDPJJS                  = `N数值移动平均计算`     //N数值移动平均计算
 	DATA_SOURCE_NAME_MANUAL                               = `手工数据`              //手工指标
-	DATA_SOURCE_NAME_LZ                                   = `隆众`                //隆众
-	DATA_SOURCE_NAME_YS                                   = `SMM`               //有色
-	DATA_SOURCE_NAME_CALCULATE_HBZ                        = `环比值`               //环比值->12
-	DATA_SOURCE_NAME_CALCULATE_HCZ                        = `环差值`               //环差值->13
-	DATA_SOURCE_NAME_CALCULATE_BP                         = `升频`                //变频,2023-2-10 13:56:01调整为"升频"->14
-	DATA_SOURCE_NAME_GL                                   = `钢联`                //钢联->15
-	DATA_SOURCE_NAME_ZZ                                   = `郑商所`               //郑商所->16
-	DATA_SOURCE_NAME_DL                                   = `大商所`               //大商所->17
-	DATA_SOURCE_NAME_SH                                   = `上期所`               //上期所->18
-	DATA_SOURCE_NAME_CFFEX                                = `中金所`               //中金所->19
+	DATA_SOURCE_NAME_LZ                                   = `隆众`                  //隆众
+	DATA_SOURCE_NAME_YS                                   = `SMM`                   //有色
+	DATA_SOURCE_NAME_CALCULATE_HBZ                        = `环比值`                //环比值->12
+	DATA_SOURCE_NAME_CALCULATE_HCZ                        = `环差值`                //环差值->13
+	DATA_SOURCE_NAME_CALCULATE_BP                         = `升频`                  //变频,2023-2-10 13:56:01调整为"升频"->14
+	DATA_SOURCE_NAME_GL                                   = `钢联`                  //钢联->15
+	DATA_SOURCE_NAME_ZZ                                   = `郑商所`                //郑商所->16
+	DATA_SOURCE_NAME_DL                                   = `大商所`                //大商所->17
+	DATA_SOURCE_NAME_SH                                   = `上期所`                //上期所->18
+	DATA_SOURCE_NAME_CFFEX                                = `中金所`                //中金所->19
 	DATA_SOURCE_NAME_SHFE                                 = `上期能源`              //上期能源->20
-	DATA_SOURCE_NAME_GIE                                  = `欧洲天然气`             //欧洲天然气->21
+	DATA_SOURCE_NAME_GIE                                  = `欧洲天然气`            //欧洲天然气->21
 	DATA_SOURCE_NAME_CALCULATE_TIME_SHIFT                 = `时间移位`              //时间移位->22
 	DATA_SOURCE_NAME_CALCULATE_ZJPJ                       = `直接拼接`              //直接拼接->23
-	DATA_SOURCE_NAME_CALCULATE_LJZTBPJ                    = `累计值同比拼接`           //累计值同比拼接->24
-	DATA_SOURCE_NAME_LT                                   = `路透`                //路透->25
-	DATA_SOURCE_NAME_COAL                                 = `中国煤炭市场网`           //中国煤炭市场网->26
+	DATA_SOURCE_NAME_CALCULATE_LJZTBPJ                    = `累计值同比拼接`        //累计值同比拼接->24
+	DATA_SOURCE_NAME_LT                                   = `路透`                  //路透->25
+	DATA_SOURCE_NAME_COAL                                 = `中国煤炭市场网`        //中国煤炭市场网->26
 	DATA_SOURCE_NAME_PYTHON                               = `代码运算`              //python代码->27
 	DATA_SOURCE_NAME_PB_FINANCE                           = `彭博财务`              //彭博财务数据->28
-	DATA_SOURCE_NAME_GOOGLE_TRAVEL                        = `our world in data` //谷歌出行数据->29
+	DATA_SOURCE_NAME_GOOGLE_TRAVEL                        = `our world in data`     //谷歌出行数据->29
 	DATA_SOURCE_NAME_PREDICT                              = `预测指标`              //普通预测指标->30
-	DATA_SOURCE_NAME_PREDICT_CALCULATE                    = `预测指标运算`            //预测指标运算->31
+	DATA_SOURCE_NAME_PREDICT_CALCULATE                    = `预测指标运算`          //预测指标运算->31
 	DATA_SOURCE_NAME_PREDICT_CALCULATE_TBZ                = `预测同比`              //预测指标 - 同比值->32
 	DATA_SOURCE_NAME_PREDICT_CALCULATE_TCZ                = `预测同差`              //预测指标 - 同差值->33
 	DATA_SOURCE_NAME_MYSTEEL_CHEMICAL                     = `钢联化工`              //钢联化工->34
 	DATA_SOURCE_NAME_CALCULATE_CJJX                       = `超季节性`              //超季节性->35
-	DATA_SOURCE_NAME_EIA_STEO                             = `EIA STERO报告`       //eia stero报告->36
+	DATA_SOURCE_NAME_EIA_STEO                             = `EIA STERO报告`         //eia stero报告->36
 	DATA_SOURCE_NAME_CALCULATE_NHCC                       = `拟合残差`              //计算指标（拟合残差）->37
-	DATA_SOURCE_NAME_COM_TRADE                            = `UN`                //联合国商品贸易数据->38
-	DATA_SOURCE_NAME_PREDICT_CALCULATE_NSZYDPJJS          = `预测N数值移动平均计算`       //预测指标 - N数值移动平均计算 -> 39
+	DATA_SOURCE_NAME_COM_TRADE                            = `UN`                    //联合国商品贸易数据->38
+	DATA_SOURCE_NAME_PREDICT_CALCULATE_NSZYDPJJS          = `预测N数值移动平均计算` //预测指标 - N数值移动平均计算 -> 39
 	DATA_SOURCE_NAME_CALCULATE_ADJUST                     = `数据调整`              //数据调整->40
-	DATA_SOURCE_NAME_SCI                                  = `SCI`               //卓创数据（红桃三）->41
-	DATA_SOURCE_NAME_PREDICT_CALCULATE_LJZZY              = `预测累计值转月值`          //预测指标 - 累计值转月->42
-	DATA_SOURCE_NAME_PREDICT_CALCULATE_HBZ                = `预测环比值`             //预测指标 - 环比值->43
-	DATA_SOURCE_NAME_PREDICT_CALCULATE_HCZ                = `预测环差值`             //预测指标 - 环差值->44
+	DATA_SOURCE_NAME_SCI                                  = `SCI`                   //卓创数据（红桃三）->41
+	DATA_SOURCE_NAME_PREDICT_CALCULATE_LJZZY              = `预测累计值转月值`      //预测指标 - 累计值转月->42
+	DATA_SOURCE_NAME_PREDICT_CALCULATE_HBZ                = `预测环比值`            //预测指标 - 环比值->43
+	DATA_SOURCE_NAME_PREDICT_CALCULATE_HCZ                = `预测环差值`            //预测指标 - 环差值->44
 	DATA_SOURCE_NAME_PREDICT_CALCULATE_BP                 = `预测升频`              //预测指标 - 升频->45
-	DATA_SOURCE_NAME_PREDICT_CALCULATE_TIME_SHIFT         = `预测时间移位`            //预测指标 - 时间移位->46
-	DATA_SOURCE_NAME_PREDICT_CALCULATE_ZJPJ               = `预测直接拼接`            //预测指标 - 直接拼接->47
-	DATA_SOURCE_NAME_PREDICT_CALCULATE_LJZTBPJ            = `预测累计值同比拼接`         //预测指标 - 累计值同比拼接->48
-	DATA_SOURCE_NAME_PREDICT_CALCULATE_CJJX               = `预测超季节性`            //预测指标 - 超季节性->49
-	DATA_SOURCE_NAME_PREDICT_CALCULATE_NHCC               = `预测拟合残差`            //预测指标 - 计算指标（拟合残差）->50
-	DATA_SOURCE_NAME_CALCULATE_JP                         = `降频`                //降频->51
-	DATA_SOURCE_NAME_CALCULATE_NH                         = `年化`                //年化->52
+	DATA_SOURCE_NAME_PREDICT_CALCULATE_TIME_SHIFT         = `预测时间移位`          //预测指标 - 时间移位->46
+	DATA_SOURCE_NAME_PREDICT_CALCULATE_ZJPJ               = `预测直接拼接`          //预测指标 - 直接拼接->47
+	DATA_SOURCE_NAME_PREDICT_CALCULATE_LJZTBPJ            = `预测累计值同比拼接`    //预测指标 - 累计值同比拼接->48
+	DATA_SOURCE_NAME_PREDICT_CALCULATE_CJJX               = `预测超季节性`          //预测指标 - 超季节性->49
+	DATA_SOURCE_NAME_PREDICT_CALCULATE_NHCC               = `预测拟合残差`          //预测指标 - 计算指标（拟合残差）->50
+	DATA_SOURCE_NAME_CALCULATE_JP                         = `降频`                  //降频->51
+	DATA_SOURCE_NAME_CALCULATE_NH                         = `年化`                  //年化->52
 	DATA_SOURCE_NAME_CALCULATE_KSZS                       = `扩散指数`              //扩散指数->53
 	DATA_SOURCE_NAME_PREDICT_CALCULATE_JP                 = `预测降频`              //预测指标 - 计算指标(降频)->54
 	DATA_SOURCE_NAME_PREDICT_CALCULATE_NH                 = `预测年化`              //预测指标 - 计算指标(年化)->55
-	DATA_SOURCE_NAME_PREDICT_CALCULATE_KSZS               = `预测扩散指数`            //预测指标 - 计算指标(扩散指数)->56
+	DATA_SOURCE_NAME_PREDICT_CALCULATE_KSZS               = `预测扩散指数`          //预测指标 - 计算指标(扩散指数)->56
 	DATA_SOURCE_NAME_BAIINFO                              = `百川盈孚`              //百川盈孚 ->57
 	DATA_SOURCE_NAME_STOCK_PLANT                          = `存量装置`              //存量装置 ->58
-	DATA_SOURCE_NAME_CALCULATE_CORRELATION                = `相关性计算`             //相关性计算->59
-	DATA_SOURCE_NAME_NATIONAL_STATISTICS                  = `国家统计局`             //国家统计局->60
-	DATA_SOURCE_NAME_CALCULATE_LJZZJ                      = `累计值转季值`            //累计值转季 -> 61
-	DATA_SOURCE_NAME_CALCULATE_LJZ                        = `累计值`               //累计值 -> 62
-	DATA_SOURCE_NAME_CALCULATE_LJZNCZJ                    = `年初至今累计值`           //累计值(年初至今） -> 63
-	DATA_SOURCE_NAME_PREDICT_CALCULATE_LJZZJ              = `预测累计值转季值`          //预测指标 - 累计值转季->64
-	DATA_SOURCE_NAME_PREDICT_CALCULATE_LJZ                = `预测累计值`             //预测指标 - 累计值 -> 65
-	DATA_SOURCE_NAME_PREDICT_CALCULATE_LJZNCZJ            = `预测年初至今累计值`         //预测指标 - 累计值(年初至今） -> 66
-	DATA_SOURCE_NAME_CALCULATE_STANDARD_DEVIATION         = `标准差`               //标准差->67
-	DATA_SOURCE_NAME_CALCULATE_PERCENTILE                 = `百分位`               //百分位->68
-	DATA_SOURCE_NAME_PREDICT_CALCULATE_STANDARD_DEVIATION = `预测标准差`             //预测标准差->69
-	DATA_SOURCE_NAME_PREDICT_CALCULATE_PERCENTILE         = `预测百分位`             //预测百分位->70
+	DATA_SOURCE_NAME_CALCULATE_CORRELATION                = `相关性计算`            //相关性计算->59
+	DATA_SOURCE_NAME_NATIONAL_STATISTICS                  = `国家统计局`            //国家统计局->60
+	DATA_SOURCE_NAME_CALCULATE_LJZZJ                      = `累计值转季值`          //累计值转季 -> 61
+	DATA_SOURCE_NAME_CALCULATE_LJZ                        = `累计值`                //累计值 -> 62
+	DATA_SOURCE_NAME_CALCULATE_LJZNCZJ                    = `年初至今累计值`        //累计值(年初至今） -> 63
+	DATA_SOURCE_NAME_PREDICT_CALCULATE_LJZZJ              = `预测累计值转季值`      //预测指标 - 累计值转季->64
+	DATA_SOURCE_NAME_PREDICT_CALCULATE_LJZ                = `预测累计值`            //预测指标 - 累计值 -> 65
+	DATA_SOURCE_NAME_PREDICT_CALCULATE_LJZNCZJ            = `预测年初至今累计值`    //预测指标 - 累计值(年初至今） -> 66
+	DATA_SOURCE_NAME_CALCULATE_STANDARD_DEVIATION         = `标准差`                //标准差->67
+	DATA_SOURCE_NAME_CALCULATE_PERCENTILE                 = `百分位`                //百分位->68
+	DATA_SOURCE_NAME_PREDICT_CALCULATE_STANDARD_DEVIATION = `预测标准差`            //预测标准差->69
+	DATA_SOURCE_NAME_PREDICT_CALCULATE_PERCENTILE         = `预测百分位`            //预测百分位->70
 	DATA_SOURCE_NAME_FUBAO                                = `富宝数据`              //富宝数据->71
 	DATA_SOURCE_NAME_CALCULATE_ZSXY                       = `指数修匀`              //指数修匀->72
-	DATA_SOURCE_NAME_PREDICT_CALCULATE_ZSXY               = `预测指数修匀`            //预测指数修匀->73
-	DATA_SOURCE_NAME_CALCULATE_ZDYFX                      = `自定义分析`             //自定义分析->74
+	DATA_SOURCE_NAME_PREDICT_CALCULATE_ZSXY               = `预测指数修匀`          //预测指数修匀->73
+	DATA_SOURCE_NAME_CALCULATE_ZDYFX                      = `自定义分析`            //自定义分析->74
 	DATA_SOURCE_NAME_YONYI                                = `涌益咨询`              // 涌益咨询
-
-	DATA_SOURCE_NAME_FENWEI        = `汾渭数据` // 汾渭煤炭
-	DATA_SOURCE_NAME_MTJH          = `煤炭江湖` // 煤炭江湖->80
-	DATA_SOURCE_NAME_ICPI          = "ICPI消费价格指数"
-	DATA_SOURCE_NAME_CALCULATE_SUM = `多指标求和`
-	DATA_SOURCE_NAME_CALCULATE_AVG = `多指标求平均`
+	DATA_SOURCE_NAME_FENWEI                               = `汾渭数据`              // 汾渭煤炭
+	DATA_SOURCE_NAME_MTJH                                 = `煤炭江湖`              // 煤炭江湖->80
+	DATA_SOURCE_NAME_ICPI                                 = "ICPI消费价格指数"
+	DATA_SOURCE_NAME_CALCULATE_SUM                        = `多指标求和`
+	DATA_SOURCE_NAME_CALCULATE_AVG                        = `多指标求平均`
+	DATA_SOURCE_NAME_BUSINESS                             = `自有数据`
+	DATA_SOURCE_NAME_CCF                                  = `CCF` // CCF化纤信息
 )
 
 // 基础数据初始化日期
 var (
-	BASE_START_DATE         = time.Now().AddDate(-30, 0, 0).Format(FormatDate)        //基础数据开始日期
-	BASE_END_DATE           = time.Now().AddDate(4, 0, 0).Format(FormatDate)          //基础数据结束日期
-	BASE_START_DATE_UnSpace = time.Now().AddDate(-30, 0, 0).Format(FormatDateUnSpace) //基础数据开始日期
-	BASE_END_DATE_UnSpace   = time.Now().AddDate(4, 0, 0).Format(FormatDateUnSpace)   //基础数据结束日期
+	BASE_START_DATE         = `1900-01-01`                                          //基础数据开始日期
+	BASE_END_DATE           = time.Now().AddDate(4, 0, 0).Format(FormatDate)        //基础数据结束日期
+	BASE_START_DATE_UnSpace = "19000101"                                            //基础数据开始日期
+	BASE_END_DATE_UnSpace   = time.Now().AddDate(4, 0, 0).Format(FormatDateUnSpace) //基础数据结束日期
 )
 
 const (
@@ -217,6 +218,7 @@ const (
 
 const (
 	CACHE_EDB_DATA_ADD               = "CACHE_EDB_DATA_ADD_"
+	CACHE_EDB_DATA_EDIT              = "CACHE_EDB_DATA_EDIT_"
 	CACHE_EDB_DATA_REFRESH           = "CACHE_EDB_DATA_REFRESH_"
 	CACHE_WIND_URL                   = "CACHE_WIND_URL"
 	CACHE_CHART_INFO_DATA            = "chart:info:data:"                 //图表数据
@@ -224,6 +226,7 @@ const (
 	CACHE_EDB_TERMINAL_CODE_URL      = "edb:terminal_code:edb_code:"      // 指标与终端关系的缓存
 	CACHE_EDB_TERMINAL_CODE_GOOD_URL = "edb:terminal_code:good:edb_code:" // 指标与终端关系的缓存, 商品期货
 	CACHE_EDB_THS_SERVER_TOKEN       = "edb:ths_server_token:"            //同花顺调用凭证
+	CACHE_SELF_EDB_HANDLE            = "CACHE_SELF_EDB_HANDLE:"           // 自定义指标缓存
 )
 
 // 图表类型
@@ -285,3 +288,28 @@ const (
 	ZhLangVersion = "zh" // 中文语言版本
 	EnLangVersion = "en" // 英文语言版本
 )
+
+const (
+	PercentCalculateTypeRange = 0 // 百分位算法类型-数据区间
+	PercentCalculateTypeNum   = 1 // 百分位算法类型-数据个数
+)
+
+// 指标计算方式
+const (
+	EdbBaseCalculateLjzzy                = 1  // 累计值转月->1
+	EdbBaseCalculateLjzzj                = 2  // 累计值转季->2
+	EdbBaseCalculateTbz                  = 3  // 同比值->3
+	EdbBaseCalculateTcz                  = 4  // 同差值->4
+	EdbBaseCalculateNszydpjjs            = 5  // N数值移动平均数计算->5
+	EdbBaseCalculateHbz                  = 6  // 环比值->6
+	EdbBaseCalculateHcz                  = 7  // 环差值->7
+	EdbBaseCalculateUpFrequency          = 8  // 升频->8
+	EdbBaseCalculateDownFrequency        = 9  // 降频->9
+	EdbBaseCalculateTimeShift            = 10 // 时间移位->10
+	EdbBaseCalculateCjjx                 = 11 // 超季节性->11
+	EdbBaseCalculateAnnualized           = 12 // 年化->12
+	EdbBaseCalculateLjz                  = 13 // 累计值->13
+	EdbBaseCalculateLjzNczj              = 14 // 累计值年初至今->14
+	EdbBaseCalculateExponentialSmoothing = 15 // 指数修匀->15
+	EdbBaseCalculateRjz                  = 16 // 日均值->16
+)

utils/logs.go

@@ -16,6 +16,7 @@ const (
 var FileLog *logs.BeeLogger
 var ApiLog *logs.BeeLogger
 var Binlog *logs.BeeLogger
+var MongoLog *logs.BeeLogger
 
 func init() {
 	if LogMaxDays == 0 {
@@ -43,6 +44,7 @@ func init() {
 
 	initBinlog()
 	initApiLog()
+	initMgoLog()
 }
 
 type logConfig struct {
@@ -103,6 +105,41 @@ func initApiLog() {
 	ApiLog.EnableFuncCallDepth(true)
 }
 
+// initMgoLog
+// @Description: mongo日志
+// @author: Roc
+// @datetime 2024-05-06 16:42:47
+func initMgoLog() {
+	//mgo日志
+
+	var logPath string
+	{
+		binlogPath := BinLogPath
+		if binlogPath == "" {
+			binlogPath = DefaultBinlogPath
+		}
+		logPath = path.Dir(binlogPath)
+	}
+	if logPath == `` {
+		logPath = "./etalogs"
+	}
+
+	mongoLogPath := logPath + "/mongolog"
+	mongoLogFile := "mongolog.log"
+
+	os.MkdirAll(mongoLogPath, os.ModePerm)
+	logFileName := path.Join(mongoLogPath, mongoLogFile)
+	MongoLog = logs.NewLogger(1000000)
+	logConf := getDefaultLogConfig()
+
+	logConf.FileName = logFileName
+	//logConf.MaxLines = 10000000
+	//logConf.Rotate = true
+	b, _ := json.Marshal(logConf)
+	MongoLog.SetLogger(logs.AdapterFile, string(b))
+	MongoLog.EnableFuncCallDepth(true)
+}
+
 func getDefaultLogConfig() logConfig {
 	return logConfig{
 		FileName: "",

utils/mgodb/mgo_config.go

@@ -0,0 +1,137 @@
+package mgodb
+
+import (
+	"context"
+	"eta/eta_index_lib/utils"
+	"github.com/qiniu/qmgo"
+	"github.com/qiniu/qmgo/options"
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/event"
+	mgoptions "go.mongodb.org/mongo-driver/mongo/options"
+	"sync"
+)
+
+var (
+	ConnectTimeoutMS = int64(3000)
+	SocketTimeoutMS  = int64(500000)
+	MaxPoolSize      = uint64(50)
+	MinPoolSize      = uint64(0)
+)
+
+//func init() {
+//	fmt.Println("start MgoNewClient")
+//	MgoNewClient()
+//	fmt.Println("end MgoNewClient")
+//}
+
+type MgoConfig struct {
+	Host          string `json:"host"`
+	Url           string `json:"url"`
+	Port          string `json:"port"`
+	Username      string `json:"username"`
+	Password      string `json:"password"`
+	AuthSource    string `json:"authSource"`
+	AuthMechanism string `json:"auth_mechanism"`
+	Database      string `json:"database"`
+}
+
+// MgoNewClient
+// @Description: 创建一个mongodb客户端链接
+// @author: Roc
+// @datetime 2024-04-25 14:01:14
+// @return *qmgo.Client
+func MgoNewClient(mgoConfig MgoConfig) *qmgo.Client {
+	// 创建cmdMonitor，用于打印SQL
+	//startedCommands := make(map[int64]bson.Raw)
+	startedCommands := sync.Map{} // map[int64]bson.Raw
+
+	cmdMonitor := &event.CommandMonitor{
+		Started: func(_ context.Context, evt *event.CommandStartedEvent) {
+			startedCommands.Store(evt.RequestID, evt.Command)
+			//startedCommands[evt.RequestID] = evt.Command
+		},
+		Succeeded: func(_ context.Context, evt *event.CommandSucceededEvent) {
+			//log.Printf("Command: %v Reply: %v\n",
+			//	startedCommands[evt.RequestID],
+			//	evt.Reply,
+			//)
+			var commands bson.Raw
+			v, ok := startedCommands.Load(evt.RequestID)
+			if ok {
+				commands = v.(bson.Raw)
+			}
+			//utils.MongoLog.Info("\n【MongoDB】[%.3fms] [%v] %v \n", float64(evt.Duration)/1e6, commands, evt.Reply)
+			utils.MongoLog.Info("\n【MongoDB】[%.3fms] [%v] \n", float64(evt.Duration)/1e6, commands)
+		},
+		Failed: func(_ context.Context, evt *event.CommandFailedEvent) {
+			//log.Printf("Command: %v Failure: %v\n",
+			//	startedCommands[evt.RequestID],
+			//	evt.Failure,
+			//)
+			var commands bson.Raw
+			v, ok := startedCommands.Load(evt.RequestID)
+			if ok {
+				commands = v.(bson.Raw)
+			}
+			utils.MongoLog.Info("\n【MongoDB】[%.3fms] [%v] \n %v \n", float64(evt.Duration)/1e6, commands, evt.Failure)
+		},
+	}
+
+	// 创建options
+	ops := options.ClientOptions{ClientOptions: &mgoptions.ClientOptions{}}
+	ops.SetMonitor(cmdMonitor)
+
+	ctx := context.Background()
+
+	//mongodb+srv://myDatabaseUser:D1fficultP%40ssw0rd@mongodb0.example.com/?authSource=admin&replicaSet=myRepl
+
+	//var mongoUrl string
+	//if mgoConfig.Password != "" {
+	//	mongoUrl = "mongodb://" + mgoConfig.Username + ":" + mgoConfig.Password + "@" +
+	//		mgoConfig.Host + ":" + mgoConfig.Port
+	//
+	//	if mgoConfig.AuthSource != `` {
+	//		mongoUrl += mongoUrl + "?&authSource=" + mgoConfig.AuthSource
+	//	}
+	//} else {
+	//	mongoUrl = "mongodb://" + mgoConfig.Host + ":" + mgoConfig.Port
+	//}
+
+	// 创建一个数据库链接
+	client, err := qmgo.NewClient(ctx, &qmgo.Config{
+		Uri: mgoConfig.Url,
+		Auth: &qmgo.Credential{
+			AuthMechanism: mgoConfig.AuthMechanism,
+			AuthSource:    mgoConfig.AuthSource,
+			Username:      mgoConfig.Username,
+			Password:      mgoConfig.Password,
+			//PasswordSet:   false,
+		},
+		Database:         mgoConfig.Database,
+		ConnectTimeoutMS: &ConnectTimeoutMS,
+		SocketTimeoutMS:  &SocketTimeoutMS,
+		MaxPoolSize:      &MaxPoolSize,
+		MinPoolSize:      &MinPoolSize,
+	}, ops)
+
+	if err != nil {
+		panic("MongoDB连接异常：" + err.Error())
+	}
+
+	return client
+}
+
+func MgoGetColl(collName string) (cli *qmgo.QmgoClient, err error) {
+	ctx := context.Background()
+	config := &qmgo.Config{
+		//Uri:              MGO_URL,
+		//Database:         MGO_DB,
+		Coll:             collName,
+		ConnectTimeoutMS: &ConnectTimeoutMS,
+		SocketTimeoutMS:  &SocketTimeoutMS,
+		MaxPoolSize:      &MaxPoolSize,
+		MinPoolSize:      &MinPoolSize,
+	}
+	cli, err = qmgo.Open(ctx, config)
+	return
+}