Merge branch 'debug' of http://8.136.199.33:3000/eta_server/eta_index_lib into bzq1/fix_conflict_lib_record

controllers/base_from_bloomberg.go

@@ -271,3 +271,49 @@ func (this *BloombergController) PCSGRefreshMonthly() {
 	br.Success = true
 	br.Msg = "操作成功"
 }
+
+// PCSGImportHistoryData
+// @Title 中石油新加坡-导入历史数据
+// @Description  中石油新加坡-导入历史数据
+// @Success 200 {object} models.PCSGImportHistoryDataReq
+// @router /pcsg/import_history_data [post]
+func (this *BloombergController) PCSGImportHistoryData() {
+	br := new(models.BaseResponse).Init()
+	defer func() {
+		if br.ErrMsg == "" {
+			br.IsSendEmail = false
+		}
+		this.Data["json"] = br
+		this.ServeJSON()
+	}()
+	var req models.PCSGImportHistoryDataReq
+	if e := json.Unmarshal(this.Ctx.Input.RequestBody, &req); e != nil {
+		br.Msg = "参数解析异常！"
+		br.ErrMsg = "参数解析失败，Err:" + e.Error()
+		return
+	}
+
+	var indexes []models.BaseFromBloombergApiIndexAndData
+	var index models.BaseFromBloombergApiIndexAndData
+	var indexData []models.BaseFromBloombergApiIndexData
+	index.IndexCode = req.IndexCode
+	for k, v := range req.DataMap {
+		indexData = append(indexData, models.BaseFromBloombergApiIndexData{
+			DataTime: k,
+			Value:    v,
+		})
+	}
+	index.Data = indexData
+	indexes = append(indexes, index)
+
+	// 写入数据
+	if e := services.PCSGWrite2BaseBloomberg(indexes); e != nil {
+		br.Msg = "刷新失败"
+		br.ErrMsg = "PCSG-写入Bloomberg数据源失败, Err: " + e.Error()
+		return
+	}
+
+	br.Ret = 200
+	br.Success = true
+	br.Msg = "操作成功"
+}

controllers/base_from_business.go

@@ -0,0 +1,390 @@
+package controllers
+
+import (
+	"encoding/json"
+	"eta/eta_index_lib/logic"
+	"eta/eta_index_lib/models"
+	"eta/eta_index_lib/services"
+	"eta/eta_index_lib/utils"
+	"fmt"
+	"strconv"
+	"strings"
+	"time"
+)
+
+// BusinessIndexController 自有数据（商家）
+type BusinessIndexController struct {
+	BaseAuthController
+}
+
+// HandleBusinessIndexData
+// @Title 处理外部指标（商家）的接口
+// @Description 处理外部指标（商家）的接口
+// @Success 200 {object} models.AddBusinessIndexReq
+// @router /handle [post]
+func (c *BusinessIndexController) HandleBusinessIndexData() {
+	br := new(models.BaseResponse).Init()
+	defer func() {
+		c.Data["json"] = br
+		c.ServeJSON()
+	}()
+	body := c.Ctx.Input.RequestBody
+	var req models.AddBusinessIndexReq
+	err := json.Unmarshal(body, &req)
+	if err != nil {
+		br.Msg = "参数解析异常！"
+		br.ErrMsg = "参数解析失败，Err:" + err.Error()
+		return
+	}
+
+	err = services.HandleBusinessIndex(&req)
+	if err != nil {
+		fmt.Println("HandleBusinessIndexData Err:" + err.Error())
+		br.Msg = "处理失败"
+		br.ErrMsg = "处理失败,Err:" + err.Error()
+		return
+	}
+	br.Ret = 200
+	br.Success = true
+	br.Msg = "处理成功"
+}
+
+// Add
+// @Title 新增自有数据（商家）指标接口
+// @Description 新增自有数据（商家）指标接口
+// @Success 200 {object} models.AddEdbBaseInfoReq
+// @router /add [post]
+func (c *BusinessIndexController) Add() {
+	br := new(models.BaseResponse).Init()
+	var cacheKey string
+	defer func() {
+		c.Data["json"] = br
+		c.ServeJSON()
+	}()
+	var req models.AddEdbBaseInfoReq
+	err := json.Unmarshal(c.Ctx.Input.RequestBody, &req)
+	if err != nil {
+		br.Msg = "参数解析异常！"
+		br.ErrMsg = "参数解析失败，Err:" + err.Error()
+		return
+	}
+	if req.EdbCode == "" {
+		br.Msg = "请输入指标编码！"
+		br.ErrMsg = "请输入指标编码，指标编码为空"
+		return
+	}
+	cacheKey = utils.CACHE_EDB_DATA_ADD + "_BusinessIndexController_" + req.EdbCode
+	if utils.Rc.IsExist(cacheKey) {
+		br.Ret = 501
+		br.Success = true
+		br.Msg = "系统处理中，请稍后重试"
+		return
+	}
+
+	utils.Rc.SetNX(cacheKey, 1, 1*time.Minute)
+	defer func() {
+		_ = utils.Rc.Delete(cacheKey)
+	}()
+
+	baseFromBusinessIndexObj := models.BaseFromBusinessIndex{}
+
+	businessIndexItem, err := baseFromBusinessIndexObj.GetIndexItem(req.EdbCode)
+	if err != nil {
+		br.Msg = "获取数据源中指标信息失败！"
+		br.ErrMsg = "获取数据源中指标信息失败 baseFromBusinessIndexObj.GetIndexItem，Err:" + err.Error()
+		return
+	}
+
+	// 添加指标
+	timestamp := strconv.FormatInt(time.Now().UnixNano(), 10)
+	uniqueCode := utils.MD5(utils.DATA_PREFIX + "_" + timestamp)
+	params := models.AddBaseParams{
+		EdbCode:         req.EdbCode,
+		EdbName:         req.EdbName,
+		Unit:            req.Unit,
+		ClassifyId:      req.ClassifyId,
+		SysUserId:       req.SysUserId,
+		SysUserRealName: req.SysUserRealName,
+		UniqueCode:      uniqueCode,
+	}
+	obj := models.Business{}
+	edbInfo, err, errMsg := obj.Add(params, businessIndexItem)
+	if err != nil {
+		br.Msg = "生成" + obj.GetSourceName() + "失败"
+		if errMsg != `` {
+			br.Msg = errMsg
+		}
+		br.ErrMsg = "生成" + obj.GetSourceName() + "失败 Err:" + err.Error()
+		return
+	}
+
+	if edbInfo == nil {
+		br.Msg = "生成" + obj.GetSourceName() + "失败2"
+		br.ErrMsg = "生成" + obj.GetSourceName() + "失败"
+		return
+	}
+
+	// 更新指标最大最小值
+	err = obj.UnifiedModifyEdbInfoMaxAndMinInfo(edbInfo)
+	if err != nil {
+		br.Msg = "更新指标最大最小值失败"
+		br.ErrMsg = err.Error()
+		return
+	}
+
+	// 添加到es
+	go logic.UpdateEs(edbInfo.EdbInfoId)
+
+	resp := models.AddEdbInfoResp{
+		EdbInfoId:  edbInfo.EdbInfoId,
+		UniqueCode: edbInfo.UniqueCode,
+	}
+	br.Ret = 200
+	br.Success = true
+	br.Msg = "保存成功"
+	br.Data = resp
+	br.IsAddLog = true
+}
+
+// Edit
+// @Title 编辑自有数据（商家）指标接口
+// @Description 编辑自有数据（商家）指标接口
+// @Success 200 {object} models.EditEdbBaseInfoReq
+// @router /edit [post]
+func (c *BusinessIndexController) Edit() {
+	br := new(models.BaseResponse).Init()
+	defer func() {
+		c.Data["json"] = br
+		c.ServeJSON()
+	}()
+
+	var req models.EditEdbBaseInfoReq
+	err := json.Unmarshal(c.Ctx.Input.RequestBody, &req)
+	if err != nil {
+		br.Msg = "参数解析异常！"
+		br.ErrMsg = "参数解析失败，Err:" + err.Error()
+		return
+	}
+	cacheKey := utils.CACHE_EDB_DATA_EDIT + "_BusinessIndexController_" + fmt.Sprint(req.EdbInfoId)
+	if utils.Rc.IsExist(cacheKey) {
+		br.Ret = 501
+		br.Success = true
+		br.Msg = "系统处理中，请稍后重试"
+		return
+	}
+
+	utils.Rc.SetNX(cacheKey, 1, 1*time.Minute)
+	defer func() {
+		_ = utils.Rc.Delete(cacheKey)
+	}()
+
+	if req.EdbInfoId <= 0 {
+		br.Msg = "参数错误"
+		br.ErrMsg = "指标ID：" + strconv.Itoa(req.EdbInfoId)
+		br.IsSendEmail = false
+		return
+	}
+
+	req.EdbName = strings.Trim(req.EdbName, " ")
+	if req.EdbName == "" {
+		br.Msg = "指标名称不能为空"
+		br.IsSendEmail = false
+		return
+	}
+
+	//if req.Frequency == "" {
+	//	br.Msg = "频率不能为空"
+	//	return
+	//}
+
+	if req.Unit == "" {
+		br.Msg = "单位不能为空"
+		br.IsSendEmail = false
+		return
+	}
+
+	if req.ClassifyId <= 0 {
+		br.Msg = "请选择分类"
+		br.IsSendEmail = false
+		return
+	}
+
+	// 根据指标名称和指标ID校验库中是否还存在其他同名指标
+	existEdbName, err := logic.CheckExistByEdbNameAndEdbInfoId(0, req.EdbInfoId, req.EdbName, c.Lang)
+	if err != nil {
+		br.Msg = "判断指标名称是否存在失败"
+		br.ErrMsg = "判断指标名称是否存在失败,Err:" + err.Error()
+		return
+	}
+	if existEdbName {
+		br.Msg = "指标名称已存在，请重新填写"
+		br.ErrMsg = "指标名称已存在，请重新填写"
+		br.IsSendEmail = false
+		return
+	}
+
+	//判断公式，指标是否有改动
+	edbInfo, err := models.GetEdbInfoById(req.EdbInfoId)
+	if err != nil {
+		br.Msg = "修改失败"
+		br.Msg = "获取指标信息失败,GetEdbInfoById Err:" + err.Error()
+		return
+	}
+	if edbInfo == nil {
+		br.Msg = "修改失败"
+		br.Msg = "指标信息不存在，EdbInfoId:" + strconv.Itoa(req.EdbInfoId)
+		return
+	}
+
+	baseFromBusinessIndexObj := models.BaseFromBusinessIndex{}
+
+	businessIndexItem, err := baseFromBusinessIndexObj.GetIndexItem(edbInfo.EdbCode)
+	if err != nil {
+		br.Msg = "获取数据源中指标信息失败！"
+		br.ErrMsg = "获取数据源中指标信息失败 baseFromBusinessIndexObj.GetIndexItem，Err:" + err.Error()
+		return
+	}
+
+	// 额外赋值
+	switch c.Lang {
+	case utils.EnLangVersion:
+		req.EdbNameEn = req.EdbName
+		req.UnitEn = req.Unit
+
+		req.EdbName = edbInfo.EdbName
+		req.Unit = edbInfo.Unit
+	default:
+		req.EdbNameEn = edbInfo.EdbNameEn
+		req.UnitEn = edbInfo.UnitEn
+	}
+
+	// 编辑指标
+	params := models.EditBaseParams{
+		EdbCode:         edbInfo.EdbCode,
+		EdbName:         req.EdbName,
+		Unit:            req.Unit,
+		ClassifyId:      req.ClassifyId,
+		SysUserId:       req.SysUserId,
+		SysUserRealName: req.SysUserRealName,
+		Lang:            c.Lang,
+		EdbInfo:         edbInfo,
+		EdbNameEn:       req.EdbNameEn,
+		UnitEn:          req.UnitEn,
+	}
+	obj := models.Business{}
+	err, errMsg := obj.Edit(params, businessIndexItem)
+	if err != nil {
+		br.Msg = "修改" + obj.GetSourceName() + "失败"
+		if errMsg != `` {
+			br.Msg = errMsg
+		}
+		br.ErrMsg = "修改" + obj.GetSourceName() + "失败 Err:" + err.Error()
+		return
+	}
+
+	// 更新指标最大最小值
+	err = obj.UnifiedModifyEdbInfoMaxAndMinInfo(edbInfo)
+	if err != nil {
+		br.Msg = "更新指标最大最小值失败"
+		br.ErrMsg = err.Error()
+		return
+	}
+
+	// 添加到es
+	go logic.UpdateEs(edbInfo.EdbInfoId)
+
+	resp := models.AddEdbInfoResp{
+		EdbInfoId:  edbInfo.EdbInfoId,
+		UniqueCode: edbInfo.UniqueCode,
+	}
+	br.Ret = 200
+	br.Success = true
+	br.Msg = "保存成功"
+	br.Data = resp
+	br.IsAddLog = true
+}
+
+// Refresh
+// @Title 刷新计算指标接口
+// @Description 刷新计算指标接口
+// @Success 200 {object} models.RefreshEdbInfoReq
+// @router /refresh [post]
+func (c *BusinessIndexController) Refresh() {
+	br := new(models.BaseResponse).Init()
+	var cacheKey string
+	defer func() {
+		c.Data["json"] = br
+		c.ServeJSON()
+	}()
+	var req models.RefreshEdbInfoReq
+	err := json.Unmarshal(c.Ctx.Input.RequestBody, &req)
+	if err != nil {
+		br.Msg = "参数解析异常！"
+		br.ErrMsg = "参数解析失败，Err:" + err.Error()
+		return
+	}
+	if req.EdbCode == "" {
+		br.Msg = "请输入指标编码！"
+		br.ErrMsg = "请输入指标编码，指标编码为空"
+		return
+	}
+	if req.EdbInfoId <= 0 {
+		br.Msg = "请输入指标ID！"
+		br.ErrMsg = "请输入指标ID"
+		return
+	}
+	edbInfo, err := models.GetEdbInfoById(req.EdbInfoId)
+	if err != nil {
+		br.Msg = "指标不存在！"
+		br.ErrMsg = "指标不存在"
+		return
+	}
+
+	cacheKey = utils.CACHE_EDB_DATA_REFRESH + strconv.Itoa(edbInfo.Source) + "_" + req.EdbCode
+
+	if utils.Rc.IsExist(cacheKey) {
+		br.Ret = 501
+		br.Success = true
+		br.Msg = "系统处理中，请稍后重试"
+		return
+	}
+
+	// 刷新指标
+	utils.Rc.SetNX(cacheKey, 1, 1*time.Minute)
+	defer func() {
+		_ = utils.Rc.Delete(cacheKey)
+	}()
+
+	params := models.RefreshBaseParams{
+		EdbInfo:   edbInfo,
+		StartDate: req.StartDate,
+	}
+	obj := models.Business{}
+
+	err, errMsg := obj.Refresh(params)
+	if errMsg != `` {
+		br.Msg = "刷新指标失败！"
+		br.ErrMsg = "刷新指标失败，err:" + errMsg
+		return
+	}
+	if err != nil && err.Error() != utils.ErrNoRow() {
+		br.Msg = "刷新指标信息失败！"
+		br.ErrMsg = "刷新指标信息失败 BusinessIndexController，Err:" + err.Error()
+		return
+	}
+
+	// 更新指标最大最小值
+	err = obj.UnifiedModifyEdbInfoMaxAndMinInfo(edbInfo)
+	if err != nil {
+		br.Msg = "更新指标最大最小值失败"
+		br.ErrMsg = err.Error()
+		return
+	}
+
+	// 更新ES
+	go logic.UpdateEs(edbInfo.EdbInfoId)
+
+	br.Ret = 200
+	br.Success = true
+	br.Msg = "刷新成功"
+}

controllers/base_from_predict_calculate.go

@@ -603,6 +603,13 @@ func (this *PredictCalculateController) CalculateBatchSave() {
 	}
 	//加入缓存机制，避免创建同一个名称的指标 end
 
+	oldEdbInfo, err := models.GetEdbInfoById(req.EdbInfoId)
+	if err != nil {
+		br.Msg = "获取指标信息失败"
+		br.ErrMsg = "获取指标信息失败,Err:" + err.Error()
+		return
+	}
+
 	// 根据指标名称和指标ID校验库中是否还存在其他同名指标
 	existEdbName, err := logic.CheckExistByEdbNameAndEdbInfoId(1, req.EdbInfoId, req.EdbName, this.Lang)
 	if err != nil {

controllers/base_from_python.go

@@ -52,7 +52,7 @@ func (this *PythonController) ExcePython() {
 		br.Msg = "获取数据失败"
 		br.ErrMsg = "python代码获取数据失败，err:" + err.Error()
 		if errMsg != "" {
-			br.ErrMsg = errMsg
+			br.Msg = errMsg
 		}
 		return
 	}

controllers/base_from_wind.go

@@ -57,6 +57,7 @@ func (this *WindController) Add() {
 
 		// 试用平台的话，需要额外从弘则这边获取下地址
 		if utils.BusinessCode == utils.HZ_TRIAL_BUSSINESS_CODE {
+			// todo 从体验版中获取对应的终端信息
 			tmpWindUrl, tmpTerminalCode, err, errMsg := services.GetHzWindUrl(req.EdbCode, source)
 			if err != nil {
 				br.Msg = "添加失败！"

controllers/base_from_wind_wsd.go

@@ -53,7 +53,6 @@ func (this *WindWsdController) Add() {
 
 		windUrl := ``
 		terminalCode := ""
-
 		// 试用平台的话，需要额外从弘则这边获取下地址
 		if utils.BusinessCode == utils.HZ_TRIAL_BUSSINESS_CODE {
 			tmpWindUrl, tmpTerminalCode, err, errMsg := services.GetHzWindUrl(req.EdbCode, source)

controllers/open/edb_info.go

@@ -49,13 +49,10 @@ func (this *EdbInfoController) Detail() {
 	dataList := make([]*models.EdbInfoSearchData, 0)
 	switch edbInfo.EdbInfoType {
 	case 0:
-		var condition string
-		var pars []interface{}
-		condition += " AND edb_info_id=? "
-		pars = append(pars, edbInfo.EdbInfoId)
-
 		//获取来源指标的数据
-		dataList, err = models.GetEdbDataListAll(condition, pars, edbInfo.Source, edbInfo.SubSource, 1)
+		dataList, err = models.GetEdbDataListAll(edbInfo.Source, edbInfo.SubSource, models.FindEdbDataListAllCond{
+			EdbInfoId: edbInfo.EdbInfoId,
+		}, 1)
 	case 1:
 		dataList, err = models.GetPredictEdbDataListAllByStartDate(edbInfo, 1, "")
 	default:

go.mod

@@ -12,9 +12,11 @@ require (
 	github.com/mozillazg/go-pinyin v0.20.0
 	github.com/nosixtools/solarlunar v0.0.0-20211112060703-1b6dea7b4a19
 	github.com/olivere/elastic/v7 v7.0.32
+	github.com/qiniu/qmgo v1.1.8
 	github.com/rdlucklib/rdluck_tools v1.0.3
 	github.com/shopspring/decimal v1.3.1
 	github.com/yidane/formula v0.0.0-20210902154546-0782e1736717
+	go.mongodb.org/mongo-driver v1.15.0
 	gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df
 )
 
@@ -23,7 +25,11 @@ require (
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/cespare/xxhash/v2 v2.1.2 // indirect
 	github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
+	github.com/go-playground/locales v0.13.0 // indirect
+	github.com/go-playground/universal-translator v0.17.0 // indirect
+	github.com/go-playground/validator/v10 v10.4.1 // indirect
 	github.com/golang/protobuf v1.5.2 // indirect
+	github.com/golang/snappy v0.0.1 // indirect
 	github.com/gonum/blas v0.0.0-20181208220705-f22b278b28ac // indirect
 	github.com/gonum/floats v0.0.0-20181209220543-c233463c7e82 // indirect
 	github.com/gonum/integrate v0.0.0-20181209220457-a422b5c0fdf2 // indirect
@@ -32,21 +38,29 @@ require (
 	github.com/gonum/matrix v0.0.0-20181209220409-c518dec07be9 // indirect
 	github.com/hashicorp/golang-lru v0.5.4 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
+	github.com/klauspost/compress v1.13.6 // indirect
+	github.com/leodido/go-urn v1.2.0 // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
 	github.com/mitchellh/mapstructure v1.4.1 // indirect
+	github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/prometheus/client_golang v1.11.0 // indirect
 	github.com/prometheus/client_model v0.2.0 // indirect
 	github.com/prometheus/common v0.26.0 // indirect
 	github.com/prometheus/procfs v0.6.0 // indirect
 	github.com/shiena/ansicolor v0.0.0-20200904210342-c7312218db18 // indirect
-	golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a // indirect
-	golang.org/x/net v0.0.0-20210428140749-89ef3d95e781 // indirect
-	golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e // indirect
-	golang.org/x/text v0.3.6 // indirect
+	github.com/xdg-go/pbkdf2 v1.0.0 // indirect
+	github.com/xdg-go/scram v1.1.2 // indirect
+	github.com/xdg-go/stringprep v1.0.4 // indirect
+	github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect
+	golang.org/x/crypto v0.17.0 // indirect
+	golang.org/x/net v0.10.0 // indirect
+	golang.org/x/sync v0.1.0 // indirect
+	golang.org/x/sys v0.15.0 // indirect
+	golang.org/x/text v0.14.0 // indirect
 	google.golang.org/protobuf v1.26.0 // indirect
 	gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
-	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

go.sum

@@ -142,6 +142,14 @@ github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vb
 github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
 github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A=
+github.com/go-playground/assert/v2 v2.0.1 h1:MsBgLAaY856+nPRTKrp3/OZK38U/wa0CcBYNjji3q3A=
+github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
+github.com/go-playground/locales v0.13.0 h1:HyWk6mgj5qFqCT5fjGBuRArbVDfE4hi8+e8ceBS/t7Q=
+github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8=
+github.com/go-playground/universal-translator v0.17.0 h1:icxd5fm+REJzpZx7ZfpaD876Lmtgy7VtROAbHHXk8no=
+github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA=
+github.com/go-playground/validator/v10 v10.4.1 h1:pH2c5ADXtd66mxoE0Zm9SUhxE20r7aM3F26W0hOn+GE=
+github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn9GlaMV7XkbRSipzJ0Ii4=
 github.com/go-redis/redis v6.14.2+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA=
 github.com/go-redis/redis/v7 v7.4.0/go.mod h1:JDNMw23GTyLNC4GZu9njt15ctBQVn7xjRfnwdHj/Dcg=
 github.com/go-redis/redis/v8 v8.11.5 h1:AcZZR7igkdvfVmQTPnu9WE37LRrO/YrBH5zWyjDC0oI=
@@ -182,6 +190,8 @@ github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw
 github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
 github.com/golang/snappy v0.0.0-20170215233205-553a64147049/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
+github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/gomodule/redigo v2.0.0+incompatible/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4=
 github.com/gonum/blas v0.0.0-20181208220705-f22b278b28ac h1:Q0Jsdxl5jbxouNs1TQYt0gxesYMU4VXRbsTlgDloZ50=
 github.com/gonum/blas v0.0.0-20181208220705-f22b278b28ac/go.mod h1:P32wAyui1PQ58Oce/KYkOqQv8cVw1zAapXOl+dRFGbc=
@@ -204,6 +214,7 @@ github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw
 github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.7 h1:81/ik6ipDQS2aGcBfIN5dHDB36BwrStyeAQquSYCV4o=
@@ -282,6 +293,8 @@ github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvW
 github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
 github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
+github.com/klauspost/compress v1.13.6 h1:P76CopJELS0TiO2mebmnzgWaajssP/EszplttgQxcgc=
+github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
@@ -290,6 +303,8 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/ledisdb/ledisdb v0.0.0-20200510135210-d35789ec47e6/go.mod h1:n931TsDuKuq+uX4v1fulaMbA/7ZLLhjc85h7chZGBCQ=
+github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y=
+github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII=
 github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
 github.com/lib/pq v1.7.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
 github.com/lib/pq v1.10.2/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
@@ -327,6 +342,8 @@ github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJ
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe h1:iruDEfMl2E6fbMZ9s0scYfZQ84/6SPL6zC8ACM2oIL0=
+github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc=
 github.com/mozillazg/go-pinyin v0.20.0 h1:BtR3DsxpApHfKReaPO1fCqF4pThRwH9uwvXzm+GnMFQ=
 github.com/mozillazg/go-pinyin v0.20.0/go.mod h1:iR4EnMMRXkfpFVV5FMi4FNB6wGq9NV6uDWbUuPhP4Yc=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
@@ -421,6 +438,8 @@ github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4O
 github.com/prometheus/procfs v0.6.0 h1:mxy4L2jP6qMonqmq+aTtOx1ifVWUgG/TAmntgbh3xv4=
 github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
 github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
+github.com/qiniu/qmgo v1.1.8 h1:E64M+P59aqQpXKI24ClVtluYkLaJLkkeD2hTVhrdMks=
+github.com/qiniu/qmgo v1.1.8/go.mod h1:QvZkzWNEv0buWPx0kdZsSs6URhESVubacxFPlITmvB8=
 github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
 github.com/rdlucklib/rdluck_tools v1.0.3 h1:iOtK2QPlPQ6CL6c1htCk5VnFCHzyG6DCfJtunrMswK0=
 github.com/rdlucklib/rdluck_tools v1.0.3/go.mod h1:9Onw9o4w19C8KE5lxb8GyxgRBbZweRVkQSc79v38EaA=
@@ -464,29 +483,43 @@ github.com/streadway/amqp v0.0.0-20190404075320-75d898a42a94/go.mod h1:AZpEONHx3
 github.com/streadway/amqp v0.0.0-20190827072141-edfb9018d271/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw=
 github.com/streadway/handy v0.0.0-20190108123426-d5acb3125c2a/go.mod h1:qNTQ5P5JnDBl6z3cMAg/SywNDC5ABu5ApDIw6lUbRmI=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.1.1 h1:2vfRuCMp5sSVIDSqO8oNnWJq7mPa6KVP3iPIwFBuy8A=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
+github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
 github.com/syndtr/goleveldb v0.0.0-20160425020131-cfa635847112/go.mod h1:Z4AUp2Km+PwemOoO/VB5AOx9XSsIItzFjoJlOSiYmn0=
 github.com/syndtr/goleveldb v0.0.0-20181127023241-353a9fca669c/go.mod h1:Z4AUp2Km+PwemOoO/VB5AOx9XSsIItzFjoJlOSiYmn0=
+github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
 github.com/ugorji/go v0.0.0-20171122102828-84cb69a8af83/go.mod h1:hnLbHMwcvSihnDhEfx2/BzKp2xb0Y+ErdfYcrs9tkJQ=
 github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
 github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
 github.com/wendal/errors v0.0.0-20130201093226-f66c77a7882b/go.mod h1:Q12BUT7DqIlHRmgv3RskH+UCM/4eqVMgI0EMmlSpAXc=
+github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c=
+github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
+github.com/xdg-go/scram v1.1.1/go.mod h1:RaEWvsqvNKKvBPvcKeFjrG2cJqOkHTiyTpzz23ni57g=
+github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY=
+github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4=
+github.com/xdg-go/stringprep v1.0.3/go.mod h1:W3f5j4i+9rC0kuIEJL0ky1VpHXQU3ocBgklLGvcBnW8=
+github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8=
+github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM=
 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
 github.com/yidane/formula v0.0.0-20210902154546-0782e1736717 h1:9CTJJpdISGxMAELfVlprj5kZEsJEaNAWiobv8ZAd72U=
 github.com/yidane/formula v0.0.0-20210902154546-0782e1736717/go.mod h1:9/dQiKiN04yPMdgsuFmKGuI2Hdp6OmFV9gSWS1col6g=
 github.com/ylywyn/jpush-api-go-client v0.0.0-20190906031852-8c4466c6e369/go.mod h1:Nv7wKD2/bCdKUFNKcJRa99a+1+aSLlCRJFriFYdjz/I=
+github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d h1:splanxYIlg+5LfHAM6xpdFEAYOk8iySO56hMFq6uLyA=
+github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
+github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
 github.com/yuin/gopher-lua v0.0.0-20171031051903-609c9cd26973/go.mod h1:aEV29XrmTYFr3CiRxZeGHpkvbwq+prZduBqMaascyCU=
 go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
 go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
@@ -495,6 +528,9 @@ go.etcd.io/etcd v3.3.25+incompatible/go.mod h1:yaeTdrJi5lOmYerz05bd8+V7KubZs8YSF
 go.etcd.io/etcd/api/v3 v3.5.0/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs=
 go.etcd.io/etcd/client/pkg/v3 v3.5.0/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g=
 go.etcd.io/etcd/client/v3 v3.5.0/go.mod h1:AIKXXVX/DQXtfTEqBryiLTUXwON+GuvO6Z7lLS/oTh0=
+go.mongodb.org/mongo-driver v1.11.6/go.mod h1:G9TgswdsWjX4tmDA5zfs2+6AEPpYJwqblyjsfuh8oXY=
+go.mongodb.org/mongo-driver v1.15.0 h1:rJCKC8eEliewXjZGf0ddURtl7tTVy1TK3bfl0gkUSLc=
+go.mongodb.org/mongo-driver v1.15.0/go.mod h1:Vzb0Mk/pa7e6cWw85R4F/endUC3u0U9jGcNU603k65c=
 go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk=
 go.opencensus.io v0.20.2/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk=
 go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
@@ -524,8 +560,11 @@ golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8U
 golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
-golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a h1:kr2P4QFmQr29mSLA43kwrOcgcReGTfbE9N577tCTuBc=
 golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8=
+golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
+golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
+golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k=
+golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@@ -550,6 +589,7 @@ golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzB
 golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -575,8 +615,10 @@ golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81R
 golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
-golang.org/x/net v0.0.0-20210428140749-89ef3d95e781 h1:DzZ89McO9/gWPsQXS/FVKAlG02ZjaQ6AlZRBimEYOd0=
-golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk=
+golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
+golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M=
+golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -590,6 +632,9 @@ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o=
+golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -625,16 +670,23 @@ golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e h1:fLOSk5Q00efkSvAm+4xcoXD+RRmLmmulPn5I3Y9F2EM=
-golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc=
+golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
+golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
+golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
 golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
@@ -671,6 +723,7 @@ golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roY
 golang.org/x/tools v0.0.0-20201211185031-d93e913c1a58/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
+golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -758,8 +811,9 @@ gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=

logic/profit_chart_info.go

@@ -261,10 +261,10 @@ func GetProfitChartEdbData(baseEdbInfo *models.EdbInfo, zlFutureGoodEdbInfoList
 	// 特殊的商品期货合约（只有M+N的合约，没有固定日期的合约）
 	specialFutureGoodEdbInfoMap := make(map[int]map[int]*future_good.FutureGoodEdbInfo, 0)
 
-	isAllChina := true // 是否都是国内的期货合约
+	hasChina := false // 是否包含国内的期货合约
 	for _, v := range zlFutureGoodEdbInfoList {
-		if v.RegionType != "国内" {
-			isAllChina = false
+		if v.RegionType == "国内" {
+			hasChina = true
 			break
 		}
 	}
@@ -283,7 +283,7 @@ func GetProfitChartEdbData(baseEdbInfo *models.EdbInfo, zlFutureGoodEdbInfoList
 			return
 		}
 
-		childFutureGoodEdbInfoMap, tmpMaxN, tmpErr := getProfitFutureGoodEdbInfoList(earliestDateTime, v, tmpFutureGoodEdbInfoList, isAllChina, monthNum)
+		childFutureGoodEdbInfoMap, tmpMaxN, tmpErr := getProfitFutureGoodEdbInfoList(earliestDateTime, v, tmpFutureGoodEdbInfoList, monthNum)
 		if tmpErr != nil {
 			err = tmpErr
 			return
@@ -322,6 +322,11 @@ func GetProfitChartEdbData(baseEdbInfo *models.EdbInfo, zlFutureGoodEdbInfoList
 		}
 	}
 
+	// 需求池604，只要包含了国内合约，最大必须是12期
+	if hasChina {
+		maxN = 12
+	}
+
 	// 找出所有的N值，并进行正序排列
 	dateList := make([]string, 0)
 	for _, n := range nMap {
@@ -411,7 +416,7 @@ func ProfitChartChartData(baseDataList []*models.EdbDataList, futureGoodEdbInfoM
 		xEdbInfoIdList := make([]int, 0)   // 当前数据的指标id列表
 
 		// 现货指标
-		realDateTime, findDataValue, isFind, tmpErr := GetNeedDateData(findDateTime, baseDataList, baseEdbDateData)
+		realDateTime, findDataValue, isFind, tmpErr := GetNeedDateData(findDateTime, baseDataList, baseEdbDateData, edbDataMap)
 		if tmpErr != nil {
 			err = tmpErr
 			return
@@ -437,7 +442,9 @@ func ProfitChartChartData(baseDataList []*models.EdbDataList, futureGoodEdbInfoM
 			//findDateTime
 
 			// 获取当前日期相对开始日期的期数
-			tmpN := (currDate.Year()-findDateTime.Year())*12 + int(currDate.Month()-findDateTime.Month())
+			//tmpN := (currDate.Year()-findDateTime.Year())*12 + int(currDate.Month()-findDateTime.Month())
+			// 用实际日期的月份作为基准，往前推12个月（2024-5-13 16:26:43修改）
+			tmpN := (currDate.Year()-realDateTime.Year())*12 + int(currDate.Month()-realDateTime.Month())
 			if tmpN <= 0 {
 				continue
 			}
@@ -568,7 +575,7 @@ func ProfitChartChartData(baseDataList []*models.EdbDataList, futureGoodEdbInfoM
 }
 
 // getFutureGoodEdbInfoList 获取适用的指标列表
-func getProfitFutureGoodEdbInfoList(earliestDateTime time.Time, zlFutureGoodEdbInfo *future_good.FutureGoodEdbInfo, tmpFutureGoodEdbInfoList []*future_good.FutureGoodEdbInfo, isAllChina bool, monthNum int) (futureGoodEdbInfoDateMap map[string]*future_good.FutureGoodEdbInfo, newMaxN int, err error) {
+func getProfitFutureGoodEdbInfoList(earliestDateTime time.Time, zlFutureGoodEdbInfo *future_good.FutureGoodEdbInfo, tmpFutureGoodEdbInfoList []*future_good.FutureGoodEdbInfo, monthNum int) (futureGoodEdbInfoDateMap map[string]*future_good.FutureGoodEdbInfo, newMaxN int, err error) {
 	maxN := 36 //最大36期合约
 	futureGoodEdbInfoList := make([]*future_good.FutureGoodEdbInfo, 0)
 	futureGoodEdbInfoDateMap = make(map[string]*future_good.FutureGoodEdbInfo)
@@ -633,7 +640,7 @@ func getProfitFutureGoodEdbInfoList(earliestDateTime time.Time, zlFutureGoodEdbI
 		// 如果（当前年-最新日期的年份） * 12个月 + (当前月-最新日期的月份) 小于总月份
 		tmpN := subYear*12 + subMonth
 		if tmpN < newMaxN {
-			tmpDateTime := time.Date(v.Year, time.Month(v.Month), 0, 0, 0, 0, 0, time.Local)
+			tmpDateTime := time.Date(v.Year, time.Month(v.Month), 1, 0, 0, 0, 0, time.Local)
 			futureGoodEdbInfoDateMap[tmpDateTime.Format(utils.FormatYearMonthDate)] = v
 			if tmpN > newMaxN {
 				newMaxN = tmpN
@@ -834,7 +841,7 @@ func GetFormulaMap() map[string]string {
 }
 
 // GetNeedDateData 获取合约内需要的日期数据
-func GetNeedDateData(needDateTime time.Time, dataList []*models.EdbDataList, edbDataMap map[string]float64) (findDateTime time.Time, findDataValue float64, isFind bool, err error) {
+func GetNeedDateData(needDateTime time.Time, dataList []*models.EdbDataList, edbDataMap map[string]float64, allEdbDataMap map[int]map[string]float64) (findDateTime time.Time, findDataValue float64, isFind bool, err error) {
 	//dataList := edbDataListMap[edbInfoId] //指标的所有数据值
 	if len(dataList) <= 0 {
 		// 没有数据的指标id
@@ -847,18 +854,43 @@ func GetNeedDateData(needDateTime time.Time, dataList []*models.EdbDataList, edb
 		return
 	}
 
+	// 该日期存在数据的期货指标的最小数量，目前是现货和期货各1个，总共2个
+	maxCount := 1
+
 	for tmpDateTime := needDateTime; tmpDateTime.After(minDateTime) || tmpDateTime.Equal(minDateTime); tmpDateTime = tmpDateTime.AddDate(0, 0, -1) {
 		tmpDate := tmpDateTime.Format(utils.FormatDate)
-		if tmpValue, ok := edbDataMap[tmpDate]; ok { //如果能找到数据，那么就返回
-			// 数据为0,也直接返回,做无值处理
-			if tmpValue == 0 {
-				return
+		tmpValue, ok := edbDataMap[tmpDate]
+		if !ok {
+			continue
+		}
+
+		// 该日期存在数据的指标数量
+		count := 0
+
+		for _, currEdbDataMap := range allEdbDataMap {
+			_, tmpIsFind := currEdbDataMap[tmpDate]
+			if tmpIsFind {
+				count++
+				if count >= maxCount {
+					continue
+				}
 			}
-			findDateTime, _ = time.ParseInLocation(utils.FormatDate, tmpDate, time.Local)
-			findDataValue = tmpValue
-			isFind = true
+		}
+
+		// 该日期存在数据的期货指标数量小于2个，那么要继续往前找
+		if count < maxCount {
+			continue
+		}
+
+		//如果能找到数据，那么就返回
+		// 数据为0,也直接返回,做无值处理
+		if tmpValue == 0 {
 			return
 		}
+		findDateTime, _ = time.ParseInLocation(utils.FormatDate, tmpDate, time.Local)
+		findDataValue = tmpValue
+		isFind = true
+		return
 	}
 
 	return

models/base_calculate.go

@@ -304,6 +304,7 @@ func (obj BaseCalculate) Tbz() (dateDataMap map[time.Time]float64, err error, er
 	var dateArr []time.Time
 	dataMap := make(map[string]*EdbInfoData) // 避免因为时间戳导致的日期不对，还是用string来表示比较合适
 	for _, v := range dataList {
+		v.DataTime = time.Date(v.DataTime.Year(), v.DataTime.Month(), v.DataTime.Day(), 0, 0, 0, 0, v.DataTime.Location())
 		dateArr = append(dateArr, v.DataTime)
 		dataMap[v.DataTime.Format(utils.FormatDate)] = v
 	}
@@ -416,6 +417,7 @@ func (obj BaseCalculate) Tcz() (dateDataMap map[time.Time]float64, err error, er
 	var dateArr []time.Time
 	dataMap := make(map[string]*EdbInfoData)
 	for _, v := range dataList {
+		v.DataTime = time.Date(v.DataTime.Year(), v.DataTime.Month(), v.DataTime.Day(), 0, 0, 0, 0, v.DataTime.Location())
 		dateArr = append(dateArr, v.DataTime)
 		dataMap[v.DataTime.Format(utils.FormatDate)] = v
 	}
@@ -587,6 +589,7 @@ func (obj BaseCalculate) Hbz() (dateDataMap map[time.Time]float64, err error, er
 
 	var dateArr []time.Time
 	for _, v := range dataList {
+		v.DataTime = time.Date(v.DataTime.Year(), v.DataTime.Month(), v.DataTime.Day(), 0, 0, 0, 0, v.DataTime.Location())
 		dateArr = append(dateArr, v.DataTime)
 	}
 
@@ -662,6 +665,7 @@ func (obj BaseCalculate) Hcz() (dateDataMap map[time.Time]float64, err error, er
 
 	var dateArr []time.Time
 	for _, v := range dataList {
+		v.DataTime = time.Date(v.DataTime.Year(), v.DataTime.Month(), v.DataTime.Day(), 0, 0, 0, 0, v.DataTime.Location())
 		dateArr = append(dateArr, v.DataTime)
 	}
 
@@ -722,6 +726,7 @@ func (obj BaseCalculate) UpFrequency() (dateDataMap map[time.Time]float64, err e
 	fromDataMap := make(map[time.Time]float64)
 	//来源指指标数据
 	for _, v := range dataList {
+		v.DataTime = time.Date(v.DataTime.Year(), v.DataTime.Month(), v.DataTime.Day(), 0, 0, 0, 0, v.DataTime.Location())
 		dateArr = append(dateArr, v.DataTime)
 		dataMap[v.DataTime] = v
 		fromDataMap[v.DataTime] = v.Value
@@ -733,8 +738,12 @@ func (obj BaseCalculate) UpFrequency() (dateDataMap map[time.Time]float64, err e
 		return
 	}
 
+	nowDay := time.Now()
+	nowDay = time.Date(nowDay.Year(), nowDay.Month(), nowDay.Day(), 0, 0, 0, 0, nowDay.Location())
+
 	// 数据计算
 	dataLen := len(dataList)
+	lastI := dataLen - 1
 	for i := 0; i < dataLen; i++ {
 		//当期
 		currentItem := dataList[i]
@@ -742,22 +751,39 @@ func (obj BaseCalculate) UpFrequency() (dateDataMap map[time.Time]float64, err e
 		var day int
 		var preItem *EdbInfoData
 		var preDate time.Time
+
+		// 如果是第一期的数据，那么直接赋值
 		if i == 0 {
-			day = int(time.Now().Sub(currentDate).Hours() / float64(24))
-			preDate = time.Now()
-		} else {
-			j := i - 1
-			if j < dataLen {
-				preItem = dataList[j]
-				day = int(preItem.DataTime.Sub(currentDate).Hours() / float64(24))
-				utils.FileLog.Info("preItem.DataTime:" + preItem.DataTime.Format(utils.FormatDate) + ";currentItem.DataTime" + currentItem.DataTime.Format(utils.FormatDate))
-			}
+			dateDataMap[currentItem.DataTime] = currentItem.Value
+			continue
 		}
-		for k := 0; k <= day; k++ {
-			needDay := preDate.AddDate(0, 0, -k)
-			dateDataMap[needDay] = currentItem.Value
+
+		// 上一期
+		j := i - 1
+
+		preItem = dataList[j]
+		preDate = preItem.DataTime
+		day = int(currentDate.Sub(preItem.DataTime).Hours() / float64(24))
+		//utils.FileLog.Info("preItem.DataTime:" + preItem.DataTime.Format(utils.FormatDate) + ";currentItem.DataTime" + currentItem.DataTime.Format(utils.FormatDate))
+		fmt.Println("preItem.DataTime:" + preItem.DataTime.Format(utils.FormatDate) + ";currentItem.DataTime:" + currentItem.DataTime.Format(utils.FormatDate))
+
+		for k := 1; k < day; k++ {
+			needDay := preDate.AddDate(0, 0, k)
+			dateDataMap[needDay] = preItem.Value
 		}
 		dateDataMap[currentItem.DataTime] = currentItem.Value
+
+		// 如果是最后一期的数据
+		if i == lastI {
+			day = int(nowDay.Sub(currentDate).Hours() / float64(24))
+			//utils.FileLog.Info("preItem.DataTime:" + currentDate.Format(utils.FormatDate) + ";currentItem.DataTime" + nowDay.Format(utils.FormatDate))
+
+			for k := 1; k <= day; k++ {
+				needDay := currentDate.AddDate(0, 0, k)
+				dateDataMap[needDay] = currentItem.Value
+			}
+		}
+
 	}
 
 	return
@@ -788,6 +814,7 @@ func (obj BaseCalculate) DownFrequency() (dateDataMap map[time.Time]float64, err
 	fromDataMap := make(map[time.Time]float64)
 	//来源指指标数据
 	for _, v := range dataList {
+		v.DataTime = time.Date(v.DataTime.Year(), v.DataTime.Month(), v.DataTime.Day(), 0, 0, 0, 0, v.DataTime.Location())
 		dateArr = append(dateArr, v.DataTime)
 		dataMap[v.DataTime] = v
 		fromDataMap[v.DataTime] = v.Value
@@ -954,6 +981,7 @@ func (obj BaseCalculate) TimeShift() (dateDataMap map[time.Time]float64, err err
 	var dateArr []time.Time
 	dataMap := make(map[time.Time]*EdbInfoData)
 	for _, v := range dataList {
+		v.DataTime = time.Date(v.DataTime.Year(), v.DataTime.Month(), v.DataTime.Day(), 0, 0, 0, 0, v.DataTime.Location())
 		dateArr = append(dateArr, v.DataTime)
 		dataMap[v.DataTime] = v
 	}
@@ -1017,6 +1045,7 @@ func (obj BaseCalculate) Cjjx() (dateDataMap map[time.Time]float64, err error, e
 	var dateArr []time.Time
 	dataMap := make(map[time.Time]*EdbInfoData)
 	for _, v := range dataList {
+		v.DataTime = time.Date(v.DataTime.Year(), v.DataTime.Month(), v.DataTime.Day(), 0, 0, 0, 0, v.DataTime.Location())
 		dateArr = append(dateArr, v.DataTime)
 		dataMap[v.DataTime] = v
 	}
@@ -1260,6 +1289,7 @@ func (obj BaseCalculate) Ljz() (dateDataMap map[time.Time]float64, err error, er
 		yearMap := make(map[int]float64)
 		yearList := make([]int, 0)
 		for _, item := range dataList {
+			item.DataTime = time.Date(item.DataTime.Year(), item.DataTime.Month(), item.DataTime.Day(), 0, 0, 0, 0, item.DataTime.Location())
 			year := item.DataTime.Year()
 			yearVal, ok := yearMap[year]
 			if ok {
@@ -1278,6 +1308,7 @@ func (obj BaseCalculate) Ljz() (dateDataMap map[time.Time]float64, err error, er
 		yearMonthMap := make(map[string]float64)
 		yearMonthList := make([]string, 0)
 		for _, item := range dataList {
+			item.DataTime = time.Date(item.DataTime.Year(), item.DataTime.Month(), item.DataTime.Day(), 0, 0, 0, 0, item.DataTime.Location())
 			itemDate := item.DataTime
 			year := itemDate.Year()
 			var tmpK string
@@ -1309,6 +1340,7 @@ func (obj BaseCalculate) Ljz() (dateDataMap map[time.Time]float64, err error, er
 		yearMonthMap := make(map[string]float64)
 		yearMonthList := make([]string, 0)
 		for _, item := range dataList {
+			item.DataTime = time.Date(item.DataTime.Year(), item.DataTime.Month(), item.DataTime.Day(), 0, 0, 0, 0, item.DataTime.Location())
 			itemDate := item.DataTime
 			year := itemDate.Year()
 			var tmpK string
@@ -1344,6 +1376,7 @@ func (obj BaseCalculate) Ljz() (dateDataMap map[time.Time]float64, err error, er
 		yearMonthMap := make(map[string]float64)
 		yearMonthList := make([]string, 0)
 		for _, item := range dataList {
+			item.DataTime = time.Date(item.DataTime.Year(), item.DataTime.Month(), item.DataTime.Day(), 0, 0, 0, 0, item.DataTime.Location())
 			itemDate := item.DataTime
 			year := itemDate.Year()
 			var tmpK string
@@ -1371,6 +1404,7 @@ func (obj BaseCalculate) Ljz() (dateDataMap map[time.Time]float64, err error, er
 		tmpDateDataMap := make(map[time.Time]float64)
 		tmpDateList := make([]time.Time, 0)
 		for _, item := range dataList {
+			item.DataTime = time.Date(item.DataTime.Year(), item.DataTime.Month(), item.DataTime.Day(), 0, 0, 0, 0, item.DataTime.Location())
 			itemDate := item.DataTime
 			dayInt := itemDate.Year()*100 + int(itemDate.Month())
 			var currTime time.Time
@@ -1414,6 +1448,7 @@ func (obj BaseCalculate) Ljz() (dateDataMap map[time.Time]float64, err error, er
 		tmpDateDataMap := make(map[time.Time]float64)
 		tmpDateList := make([]time.Time, 0)
 		for _, item := range dataList {
+			item.DataTime = time.Date(item.DataTime.Year(), item.DataTime.Month(), item.DataTime.Day(), 0, 0, 0, 0, item.DataTime.Location())
 			itemDate := item.DataTime
 			var currTime time.Time
 			// 周六周日，这是下一个周五的数据
@@ -1504,6 +1539,7 @@ func (obj BaseCalculate) LjzNczj() (dateDataMap map[time.Time]float64, err error
 		tmpDateDataMap := make(map[time.Time]float64)
 		tmpDateList := make([]time.Time, 0)
 		for _, item := range dataList {
+			item.DataTime = time.Date(item.DataTime.Year(), item.DataTime.Month(), item.DataTime.Day(), 0, 0, 0, 0, item.DataTime.Location())
 			itemDate := item.DataTime
 			var currTime time.Time
 			// 周六周日，这是下一个周五的数据
@@ -1534,6 +1570,7 @@ func (obj BaseCalculate) LjzNczj() (dateDataMap map[time.Time]float64, err error
 		}
 	default:
 		for _, item := range dataList {
+			item.DataTime = time.Date(item.DataTime.Year(), item.DataTime.Month(), item.DataTime.Day(), 0, 0, 0, 0, item.DataTime.Location())
 			itemDate := item.DataTime
 			year := itemDate.Year()
 			yearVal, ok := yearMap[year]
@@ -1624,6 +1661,7 @@ func calculateExponentialSmoothingData(dataList []*EdbInfoData, alpha float64) (
 	alphaDecimal := decimal.NewFromFloat(alpha)
 	subAlpha := decimal.NewFromFloat(1).Sub(alphaDecimal)
 	for k, d := range dataList {
+		d.DataTime = time.Date(d.DataTime.Year(), d.DataTime.Month(), d.DataTime.Day(), 0, 0, 0, 0, d.DataTime.Location())
 		// 首期的值以原始值作为指数修匀的计算值
 		if k == 0 {
 			newDataList = append(newDataList, EdbInfoData{
@@ -1684,6 +1722,7 @@ func (obj BaseCalculate) Rjz() (dateDataMap map[time.Time]float64, err error, er
 	var dateArr []time.Time
 	dataMap := make(map[time.Time]*EdbInfoData)
 	for _, v := range dataList {
+		v.DataTime = time.Date(v.DataTime.Year(), v.DataTime.Month(), v.DataTime.Day(), 0, 0, 0, 0, v.DataTime.Location())
 		dateArr = append(dateArr, v.DataTime)
 		dataMap[v.DataTime] = v
 	}

models/base_from_adjust.go

@@ -267,11 +267,11 @@ func RefreshAllAdjustEdb(edbInfo *EdbInfo, fromEdbInfo *EdbInfo) (err error) {
 	}
 
 	// 查询关联指标的数据
-	var condition string
-	var pars []interface{}
-	condition += " AND edb_info_id=? AND data_time > ? "
-	pars = append(pars, fromEdbInfo.EdbInfoId, edbAdjustConf.SourceEndDate)
-	dataList, err := GetEdbDataListAllByTo(to, condition, pars, fromEdbInfo.Source, fromEdbInfo.SubSource, 1)
+	dataList, err := GetEdbDataListAllByTo(to, fromEdbInfo.Source, fromEdbInfo.SubSource, FindEdbDataListAllCond{
+		EdbInfoId:         fromEdbInfo.EdbInfoId,
+		StartDataTime:     edbAdjustConf.SourceEndDate.Format(utils.FormatDate),
+		StartDataTimeCond: ">",
+	}, 1)
 	if err != nil {
 		return err
 	}
@@ -279,11 +279,11 @@ func RefreshAllAdjustEdb(edbInfo *EdbInfo, fromEdbInfo *EdbInfo) (err error) {
 	// 已经入库的日期数据
 	existDataMap := make(map[string]float64)
 	{
-		var tmpCondition string
-		var tmpPars []interface{}
-		tmpCondition += " AND edb_info_id=? AND data_time > ? "
-		tmpPars = append(tmpPars, edbInfo.EdbInfoId, edbAdjustConf.SourceEndDate)
-		existDataList, tmpErr := GetEdbDataListAllByTo(to, tmpCondition, tmpPars, edbInfo.Source, edbInfo.SubSource, 1)
+		existDataList, tmpErr := GetEdbDataListAllByTo(to, edbInfo.Source, edbInfo.SubSource, FindEdbDataListAllCond{
+			EdbInfoId:         edbInfo.EdbInfoId,
+			StartDataTime:     edbAdjustConf.SourceEndDate.Format(utils.FormatDate),
+			StartDataTimeCond: ">",
+		}, 1)
 		if tmpErr != nil {
 			err = tmpErr
 			return
@@ -302,7 +302,7 @@ func RefreshAllAdjustEdb(edbInfo *EdbInfo, fromEdbInfo *EdbInfo) (err error) {
 		saveVal := utils.SubFloatToString(val, 20)
 		if existVal, ok := existDataMap[currDay]; !ok {
 			//格式化时间
-			currentDate, tmpErr := time.Parse(utils.FormatDate, item.DataTime)
+			currentDate, tmpErr := time.ParseInLocation(utils.FormatDate, item.DataTime, time.Local)
 			if tmpErr != nil {
 				err = tmpErr
 				return

models/base_from_bloomberg.go

@@ -43,6 +43,12 @@ func (m *BaseFromBloombergIndex) Create() (err error) {
 	return
 }
 
+func (m *BaseFromBloombergIndex) Update(cols []string) (err error) {
+	o := orm.NewOrm()
+	_, err = o.Update(m, cols...)
+	return
+}
+
 func GetBaseFromBloombergIndexByCode(indexCode string) (item *BaseFromBloombergIndex, err error) {
 	o := orm.NewOrm()
 	sql := `SELECT * FROM base_from_bloomberg_index WHERE index_code = ? LIMIT 1`
@@ -80,7 +86,7 @@ func GetBaseFromBloombergDataByCondition(condition string, pars []interface{}) (
 func MultiInsertOrUpdateBaseFromBloombergData(inserts, updates []*BaseFromBloombergData) (err error) {
 	o := orm.NewOrm()
 	if len(inserts) > 0 {
-		_, e := o.InsertMulti(len(inserts), inserts)
+		_, e := o.InsertMulti(1000, inserts)
 		if e != nil {
 			err = fmt.Errorf("insert multi err: %s", e.Error())
 			return
@@ -345,3 +351,9 @@ type BaseFromBloombergApiIndexData struct {
 	DataTime time.Time `description:"数据日期"`
 	Value    float64   `description:"数据值"`
 }
+
+// PCSGImportHistoryDataReq 导入历史数据
+type PCSGImportHistoryDataReq struct {
+	IndexCode string                `description:"指标编码"`
+	DataMap   map[time.Time]float64 `description:"数据日期/值"`
+}

models/base_from_business.go

@@ -0,0 +1,194 @@
+package models
+
+import (
+	"eta/eta_index_lib/models/mgo"
+	"eta/eta_index_lib/utils"
+	"fmt"
+	"go.mongodb.org/mongo-driver/bson"
+	"time"
+
+	"github.com/beego/beego/v2/client/orm"
+)
+
+// BaseFromBusinessIndex
+// @Description: 外部指标（商家系统）表
+type BaseFromBusinessIndex struct {
+	BaseFromBusinessIndexId int64     `orm:"column(base_from_business_index_id);pk"`
+	IndexCode               string    `description:"指标编码"`
+	IndexName               string    `description:"指标名称"`
+	Unit                    string    `description:"单位"`
+	Frequency               string    `description:"频度"`
+	Source                  int       `description:"数据来源"`
+	SourceName              string    `description:"数据来源名称"`
+	StartDate               time.Time `description:"开始日期"`
+	EndDate                 time.Time `description:"结束日期"`
+	Remark                  string    `description:"备注字段"`
+	BaseModifyTime          time.Time `description:"基础信息(名称，单位，频度)变更时间"`
+	DataUpdateTime          time.Time `description:"最近一次数据发生变化的时间"`
+	CreateTime              time.Time `description:"创建时间"`
+	ModifyTime              time.Time `description:"修改时间"`
+}
+
+// EdbBusinessSource
+// @Description: 自有数据（商家）指标来源
+type EdbBusinessSource struct {
+	EdbBusinessSourceId int64     `orm:"column(edb_business_source_id);pk"`
+	SourceName          string    `description:"来源名称"` // 来源名称
+	CreateTime          time.Time `description:"创建时间"` // 创建时间
+}
+
+// AddBusinessIndexReq
+// @Description:  添加外部指标（商家）请求
+type AddBusinessIndexReq struct {
+	IndexCode  string               `description:"指标编码"`
+	IndexName  string               `description:"指标名称"`
+	Unit       string               `description:"单位"`
+	Frequency  string               `description:"频度"`
+	SourceName string               `description:"数据来源名称"`
+	Remark     string               `description:"备注字段"`
+	DataList   []AddBusinessDataReq `description:"指标数据"`
+}
+
+// AddBusinessDataReq
+// @Description: 外部指标（商家系统）数据
+type AddBusinessDataReq struct {
+	Value float64 `description:"值"`
+	Date  string  `description:"日期"`
+}
+
+func (m *BaseFromBusinessIndex) GetIndexItem(indexCode string) (item *BaseFromBusinessIndex, err error) {
+	o := orm.NewOrm()
+	sql := `SELECT * FROM base_from_business_index WHERE index_code = ? `
+	err = o.Raw(sql, indexCode).QueryRow(&item)
+	return
+}
+
+func (m *BaseFromBusinessIndex) GetIndexCreate(terminalCode string) (items []*BaseFromBusinessIndex, err error) {
+	o := orm.NewOrm()
+	endTime := time.Now().Add(-2 * time.Minute).Format(utils.FormatDateTime)
+	sql := `SELECT * FROM base_from_business_index WHERE index_name = '' AND create_time <= ? AND terminal_code = ? `
+	_, err = o.Raw(sql, endTime, terminalCode).QueryRows(&items)
+	return
+}
+
+// Add 新增
+func (m *BaseFromBusinessIndex) Add() (err error) {
+	o := orm.NewOrm()
+	lastId, err := o.Insert(m)
+	if err != nil {
+		return
+	}
+	m.BaseFromBusinessIndexId = lastId
+
+	return
+}
+
+func (m *BaseFromBusinessIndex) Update(cols []string) (err error) {
+	o := orm.NewOrm()
+	_, err = o.Update(m, cols...)
+	return
+}
+
+func (m *BaseFromBusinessIndex) UpdateIndex(item *BaseFromBusinessIndex, updateCols []string) (err error) {
+	if item == nil {
+		return
+	}
+	if len(updateCols) == 0 {
+		return
+	}
+	o := orm.NewOrm()
+	_, err = o.Update(item, updateCols...)
+	return
+}
+
+// GetEdbBusinessSourceItem
+// @Description: 根据来源名称获取来源信息
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-25 18:09:03
+// @param sourceName string
+// @return item *EdbBusinessSource
+// @return err error
+func (m *EdbBusinessSource) GetEdbBusinessSourceItem(sourceName string) (item *EdbBusinessSource, err error) {
+	o := orm.NewOrm()
+	sql := `SELECT * FROM edb_business_source WHERE source_name = ? `
+	err = o.Raw(sql, sourceName).QueryRow(&item)
+	return
+}
+
+// Add 新增
+func (m *EdbBusinessSource) Add() (err error) {
+	o := orm.NewOrm()
+	lastId, err := o.Insert(m)
+	if err != nil {
+		return
+	}
+	m.EdbBusinessSourceId = lastId
+
+	return
+}
+
+// GetEdbInfoMaxAndMinInfo 获取指标的最新数据记录信息
+func (m BaseFromBusinessIndex) GetEdbInfoMaxAndMinInfo(edbCode string) (item *EdbInfoMaxAndMinInfo, err error) {
+	mogDataObj := new(mgo.BaseFromBusinessData)
+	pipeline := []bson.M{
+		{"$match": bson.M{"index_code": edbCode}},
+		{"$group": bson.M{
+			"_id":       nil,
+			"min_date":  bson.M{"$min": "$data_time"},
+			"max_date":  bson.M{"$max": "$data_time"},
+			"min_value": bson.M{"$min": "$value"},
+			"max_value": bson.M{"$max": "$value"},
+		}},
+		{"$project": bson.M{"_id": 0}}, // 可选，如果不需要_id字段
+	}
+	result, err := mogDataObj.GetEdbInfoMaxAndMinInfo(pipeline)
+	if err != nil {
+		fmt.Println("BaseFromBusinessIndex GetEdbInfoMaxAndMinInfo Err:" + err.Error())
+		return
+	}
+
+	if !result.MaxDate.IsZero() {
+		whereQuery := bson.M{"index_code": edbCode, "data_time": result.MaxDate}
+		selectParam := bson.D{{"value", 1}, {"_id", 0}}
+		latestValue, tmpErr := mogDataObj.GetLatestValue(whereQuery, selectParam)
+		if tmpErr != nil {
+			err = tmpErr
+			return
+		}
+		result.LatestValue = latestValue.Value
+		result.EndValue = latestValue.Value
+	}
+
+	item = &EdbInfoMaxAndMinInfo{
+		MinDate:     result.MinDate.Format(utils.FormatDate),
+		MaxDate:     result.MaxDate.Format(utils.FormatDate),
+		MinValue:    result.MinValue,
+		MaxValue:    result.MaxValue,
+		LatestValue: result.LatestValue,
+		LatestDate:  result.LatestDate.Format(utils.FormatDate),
+		EndValue:    result.EndValue,
+	}
+
+	return
+}
+
+// ModifyIndexMaxAndMinInfo
+// @Description: 修改最大值和最小值信息
+// @author: Roc
+// @receiver m
+// @datetime 2024-05-06 14:07:46
+// @param indexCode string
+// @param item *EdbInfoMaxAndMinInfo
+// @param isIndexUpdateOrAdd bool
+// @return err error
+func (m *BaseFromBusinessIndex) ModifyIndexMaxAndMinInfo(indexCode string, item *EdbInfoMaxAndMinInfo, isIndexUpdateOrAdd bool) (err error) {
+	o := orm.NewOrm()
+	sql := ` UPDATE base_from_business_index SET start_date=?,end_date=?,modify_time=NOW() `
+	if isIndexUpdateOrAdd {
+		sql += `,data_update_time=NOW() `
+	}
+	sql += ` WHERE index_code=?`
+	_, err = o.Raw(sql, item.MinDate, item.MaxDate, indexCode).Exec()
+	return
+}

models/base_from_calculate.go

@@ -230,7 +230,6 @@ func EditCalculateInfo(edbInfo *EdbInfo, req EdbInfoCalculateSaveReq, formulaSli
 				}
 			}
 		}
-
 		//计算数据
 		err = refreshAllCalculate(to, edbInfoList, edbInfoTag, edbInfo.EdbInfoId, edbInfo.Source, edbInfo.SubSource, edbInfo.EdbCode, edbInfo.CalculateFormula, "", "", edbInfoIdBytes, edbInfo.EmptyType, edbInfo.MaxEmptyType, edbInfo.Extra)
 	}
@@ -318,20 +317,11 @@ func refreshAllCalculate(to orm.TxOrmer, edbInfoIdArr []*EdbInfo, edbInfoTag map
 	}
 
 	for edbInfoIndex, v := range edbInfoIdArr {
-		var condition string
-		var pars []interface{}
-		condition += " AND edb_info_id=? "
-		pars = append(pars, v.EdbInfoId)
-		if startDate != "" {
-			condition += " AND data_time>=? "
-			pars = append(pars, startDate)
-		}
-		//if endDate != "" {
-		//	condition += " AND data_time<=? "
-		//	pars = append(pars, endDate)
-		//}
-		//fmt.Println("v.Source:", v.Source)
-		dataList, err := GetEdbDataListAllByTo(to, condition, pars, v.Source, v.SubSource, 1)
+		dataList, err := GetEdbDataListAllByTo(to, v.Source, v.SubSource, FindEdbDataListAllCond{
+			EdbInfoId:         v.EdbInfoId,
+			StartDataTime:     startDate,
+			StartDataTimeCond: ">=",
+		}, 1)
 		if err != nil {
 			return err
 		}
@@ -687,6 +677,8 @@ type EdbInfoCalculateEdbInfoIdReq struct {
 
 // EdbInfoCalculateBatchEditReq 编辑计算指标的请求参数
 type EdbInfoCalculateBatchEditReq struct {
+	AdminId          int                            `description:"添加人id"`
+	AdminName        string                         `description:"添加人名称"`
 	EdbName          string                         `description:"指标名称"`
 	EdbNameEn        string                         `description:"英文指标名称"`
 	Frequency        string                         `description:"频度"`
@@ -710,6 +702,14 @@ type EdbInfoCalculateBatchEditReq struct {
 	CalculateFormula string                         `description:"计算公式"`
 }
 
+// DeleteCalculateData 删除计算数据
+func DeleteCalculateData(edbInfoId int) (err error) {
+	o := orm.NewOrm()
+	sql := `DELETE FROM edb_data_calculate WHERE edb_info_id=?`
+	_, err = o.Raw(sql, edbInfoId).Exec()
+	return
+}
+
 // CheckFormula2 校验公式是否正常（比如说除法的分母不能为0之类的，实际上就是用预设的字段数据做一次计算）
 func CheckFormula2(edbInfoArr []*EdbInfo, formulaMap map[string]string, formulaStr string, edbInfoIdBytes []string) (ok bool, err error) {
 	valArr := make(map[int]float64)

models/base_from_manual.go

@@ -1,10 +1,12 @@
 package models
 
 import (
+	"eta/eta_index_lib/models/mgo"
 	"eta/eta_index_lib/utils"
 	"fmt"
 	"github.com/beego/beego/v2/client/orm"
 	"github.com/shopspring/decimal"
+	"go.mongodb.org/mongo-driver/bson"
 	"strconv"
 	"strings"
 	"time"
@@ -219,7 +221,17 @@ func RefreshEdbDataFromManual(edbInfoId int, edbCode, startDate string) (err err
 	return
 }
 
-// HandleConfigInsertEdbData 处理手工数据补充的配置
+// HandleConfigInsertEdbData
+// @Description: 处理人工数据补充的配置（mysql)
+// @author: Roc
+// @datetime 2024-05-09 13:07:32
+// @param realDataMaxDate time.Time
+// @param edbDataInsertConfig *EdbDataInsertConfig
+// @param edbInfoId int
+// @param source int
+// @param subSource int
+// @param existMap map[string]*EdbInfoSearchData
+// @param isFindConfigDateRealData bool
 func HandleConfigInsertEdbData(realDataMaxDate time.Time, edbDataInsertConfig *EdbDataInsertConfig, edbInfoId, source, subSource int, existMap map[string]*EdbInfoSearchData, isFindConfigDateRealData bool) {
 	if edbDataInsertConfig == nil {
 		return
@@ -249,3 +261,47 @@ func HandleConfigInsertEdbData(realDataMaxDate time.Time, edbDataInsertConfig *E
 
 	return
 }
+
+// HandleConfigInsertEdbDataByMongo
+// @Description: 处理人工数据补充的配置（mongo)
+// @author: Roc
+// @datetime 2024-05-09 13:07:12
+// @param realDataMaxDate time.Time
+// @param edbDataInsertConfig *EdbDataInsertConfig
+// @param edbInfoId int
+// @param source int
+// @param subSource int
+// @param existMap map[string]*mgo.EdbDataBusiness
+// @param isFindConfigDateRealData bool
+func HandleConfigInsertEdbDataByMongo(realDataMaxDate time.Time, edbDataInsertConfig *EdbDataInsertConfig, edbInfoId, source, subSource int, existMap map[string]*mgo.EdbDataBusiness, isFindConfigDateRealData bool) {
+	if edbDataInsertConfig == nil {
+		return
+	}
+	var err error
+	defer func() {
+		if err != nil {
+			fmt.Println("处理手工数据补充的配置失败,err:", err)
+		}
+	}()
+
+	edbDataInsertConfigDate := edbDataInsertConfig.Date // 配置的日期
+
+	// 如果存在真实数据的最大日期  && 存在配置插入数据的最大日期  && 真实数据的最大日期 晚于/等于 配置插入数据的最大日期
+	if realDataMaxDate.After(edbDataInsertConfigDate) || realDataMaxDate.Equal(edbDataInsertConfigDate) {
+		go DeleteEdbDataInsertConfigByEdbId(edbInfoId)
+
+		mogDataObj := mgo.EdbDataBusiness{}
+		coll := mogDataObj.GetCollection()
+		edbDataInsertConfigDateStr := edbDataInsertConfigDate.Format(utils.FormatDate)
+		// 如果没有找到找到配置日期的实际数据，那么就直接删除
+		if item, ok := existMap[edbDataInsertConfigDateStr]; ok && !isFindConfigDateRealData {
+			mogDataObj.RemoveManyByColl(coll, bson.M{"_id": item.ID})
+		}
+	} else {
+		o := orm.NewOrm()
+		edbDataInsertConfig.RealDate = realDataMaxDate
+		_, err = o.Update(edbDataInsertConfig, "RealDate")
+	}
+
+	return
+}

models/base_from_ths.go

@@ -1,6 +1,7 @@
 package models
 
 import (
+	"eta/eta_index_lib/services/alarm_msg"
 	"eta/eta_index_lib/utils"
 	"fmt"
 	"github.com/beego/beego/v2/client/orm"
@@ -26,7 +27,14 @@ type Tables struct {
 
 // 新增同花顺指标数据
 func AddEdbDataFromThs(edbCode string, item EdbDataFromThs) (err error) {
+	var errMsg string
 	o := orm.NewOrm()
+	defer func() {
+		if err != nil {
+			//go utils.SendEmail(utils.APP_NAME_CN+"【"+utils.RunMode+"】"+"失败提醒", " 同花顺数据获取失败:err:"+errMsg, utils.EmailSendToUsers)
+			go alarm_msg.SendAlarmMsg("同花顺数据获取失败:err:"+errMsg, 3)
+		}
+	}()
 
 	if len(item.Tables) > 0 {
 		table := item.Tables[0]
@@ -42,6 +50,7 @@ func AddEdbDataFromThs(edbCode string, item EdbDataFromThs) (err error) {
 			sValue := table.Value[i]
 			dataTime, err := time.ParseInLocation(utils.FormatDate, eDate, time.Local)
 			if err != nil {
+				errMsg = " time.Parse :" + err.Error()
 				return err
 			}
 			timestamp := dataTime.UnixNano() / 1e6
@@ -53,6 +62,7 @@ func AddEdbDataFromThs(edbCode string, item EdbDataFromThs) (err error) {
 			addSql = strings.TrimRight(addSql, ",")
 			_, err = o.Raw(addSql).Exec()
 			if err != nil {
+				errMsg = " tx.Exec Err :" + err.Error()
 				return
 			}
 		}

models/base_predict_from_calculate.go

@@ -303,6 +303,15 @@ func refreshAllPredictCalculate(to orm.TxOrmer, edbInfoIdList []*EdbInfo, edbInf
 		}
 		if maxStartDate.IsZero() || maxStartDate.Before(tmpStartDate) {
 			maxStartDate = tmpStartDate
+		}
+		tmpEndDate, tmpErr := time.ParseInLocation(utils.FormatDate, v.EndDate, time.Local)
+		if tmpErr != nil {
+			err = errors.New(`最近的日期格式化失败;日期：` + v.EndDate + `;err:` + tmpErr.Error())
+			return
+		}
+		if minEndDate.IsZero() || tmpEndDate.Before(minEndDate) {
+			minEndDate = tmpEndDate
+		}
 		}*/
 
 		// 获取关联指标数据

models/db.go

@@ -58,8 +58,8 @@ func init() {
 		new(EdbDataPredictCalculateLjztbpj),
 		new(EdbDataPredictCalculateNhcc),
 		new(EdbDataPredictCalculateZjpj),
-		new(EdbDataInsertConfig),
 		new(EdbAdjustConf), // 数据调整的配置
+		new(EdbDataInsertConfig),
 		new(BaseFromMysteelChemicalClassify),
 	)
 
@@ -81,6 +81,9 @@ func init() {
 	// 初始化指标刷新
 	initEdbRefresh()
 
+	// 自有数据指标
+	initBusinessEdb()
+
 	// 初始化部分数据表变量(直接init会有顺序问题=_=!)
 	InitEdbSource()
 }
@@ -129,10 +132,10 @@ func initBaseIndex() {
 		new(BaseFromCoalmineInlandIndex),
 		new(BaseFromCoalmineCompanyIndex),
 		new(BaseFromCoalmineFirmIndex),
-		new(BaseFromMtjhMapping),
-		new(BaseFromMtjhIndex),
 		new(BaseFromFenweiIndex),
 		new(BaseFromFenweiData),
+		new(BaseFromMtjhMapping),
+		new(BaseFromMtjhIndex),
 		new(BaseFromBloombergIndex),
 		new(BaseFromBloombergData),
 	)
@@ -165,3 +168,11 @@ func initEdbRefresh() {
 		new(edb_refresh.EdbRefreshMapping),       // 指标刷新时间配置关系表
 	)
 }
+
+// initBusinessEdb 初始化指标刷新
+func initBusinessEdb() {
+	orm.RegisterModel(
+		new(BaseFromBusinessIndex), // 外部指标（商家系统）表
+		new(EdbBusinessSource),     // 自有数据（商家）指标来源
+	)
+}

models/edb_data_base.go

@@ -1,11 +1,10 @@
 package models
 
 import (
-	"errors"
 	"eta/eta_index_lib/utils"
 	"fmt"
 	"github.com/beego/beego/v2/client/orm"
-	"strconv"
+	"strings"
 	"time"
 )
 
@@ -179,27 +178,74 @@ type EdbDataList struct {
 	Value         float64 `description:"数据值"`
 }
 
+// GetEdbDataList
+// @Description:
+// @author: Roc
+// @datetime 2024-05-08 16:10:49
+// @param source int
+// @param subSource int
+// @param endInfoId int
+// @param startDate string
+// @param endDate string
+// @return list []*EdbDataList
+// @return err error
 func GetEdbDataList(source, subSource, endInfoId int, startDate, endDate string) (list []*EdbDataList, err error) {
-	tableName := GetEdbDataTableName(source, subSource)
-	if tableName == "" {
-		err = errors.New("无效的渠道：" + strconv.Itoa(source))
-		list = make([]*EdbDataList, 0)
-		return list, err
-	}
-	var pars []interface{}
-	sql := `SELECT edb_data_id,edb_info_id,data_time,value,data_timestamp FROM %s WHERE edb_info_id=? `
-	if startDate != "" {
-		sql += ` AND data_time>=? `
-		pars = append(pars, startDate)
+	list = make([]*EdbDataList, 0)
+	tmpDataList, err := GetEdbDataListAll(source, subSource, FindEdbDataListAllCond{
+		EdbInfoId:         endInfoId,
+		StartDataTime:     startDate,
+		StartDataTimeCond: ">=",
+		EndDataTime:       endDate,
+		EndDataTimeCond:   "<=",
+	}, 1)
+	if err != nil {
+		return
 	}
-	if endDate != "" {
-		sql += ` AND data_time<=? `
-		pars = append(pars, endDate)
+
+	// 数据拼接
+	for _, v := range tmpDataList {
+		list = append(list, &EdbDataList{
+			EdbDataId:     v.EdbDataId,
+			EdbInfoId:     v.EdbInfoId,
+			DataTime:      v.DataTime,
+			DataTimestamp: v.DataTimestamp,
+			Value:         v.Value,
+		})
 	}
 
-	sql += ` ORDER BY data_time ASC `
-	sql = fmt.Sprintf(sql, tableName)
-	o := orm.NewOrm()
-	_, err = o.Raw(sql, endInfoId, pars).QueryRows(&list)
+	return
+}
+
+type AddEdbBaseInfoReq struct {
+	EdbCode         string `description:"指标编码"`
+	EdbName         string `description:"指标名称"`
+	Unit            string `description:"单位"`
+	ClassifyId      int    `description:"所属分类"`
+	SysUserId       int    `description:"用户id"`
+	SysUserRealName string `description:"用户真实名称"`
+}
+
+// EditEdbBaseInfoReq 编辑基础指标请求参数
+type EditEdbBaseInfoReq struct {
+	SysUserId       int    `description:"用户id"`
+	SysUserRealName string `description:"用户真实名称"`
+	EdbInfoId       int    `description:"指标id"`
+	EdbName         string `description:"指标名称"`
+	EdbNameEn       string `description:"英文指标名称"`
+	Frequency       string `description:"频率"`
+	Unit            string `description:"单位"`
+	UnitEn          string `description:"英文单位"`
+	ClassifyId      int    `description:"分类id"`
+}
+
+func DelEdbDataByMysql(to orm.TxOrmer, edbInfoId int, tableName string, removeDateList []string) (err error) {
+	if len(removeDateList) <= 0 {
+		return
+	}
+	// 移除不存在的日期数据
+	removeDateStr := strings.Join(removeDateList, `","`)
+	removeDateStr = `"` + removeDateStr + `"`
+	sql := fmt.Sprintf(` DELETE FROM %s WHERE edb_info_id = ? and data_time in (%s) `, tableName, removeDateStr)
+	_, err = to.Raw(sql, edbInfoId).Exec()
 	return
 }

models/edb_data_business.go

@@ -0,0 +1,478 @@
+package models
+
+import (
+	"eta/eta_index_lib/models/mgo"
+	"eta/eta_index_lib/utils"
+	"fmt"
+	"github.com/beego/beego/v2/client/orm"
+	"github.com/shopspring/decimal"
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+	"reflect"
+	"time"
+)
+
+// Business 自有数据
+type Business struct {
+}
+
+// AddBaseParams
+// @Description: 基础指标的添加参数
+type AddBaseParams struct {
+	EdbCode         string `description:"指标编码"`
+	EdbName         string `description:"指标名称"`
+	Unit            string `description:"单位"`
+	ClassifyId      int    `description:"所属分类"`
+	SysUserId       int    `description:"用户id"`
+	SysUserRealName string `description:"用户真实名称"`
+	UniqueCode      string `description:"编码"`
+}
+
+// EditBaseParams
+// @Description: 基础指标的修改参数
+type EditBaseParams struct {
+	EdbCode         string   `description:"指标编码"`
+	EdbName         string   `description:"指标名称"`
+	EdbNameEn       string   `description:"指标名称（英文）"`
+	Unit            string   `description:"单位"`
+	UnitEn          string   `description:"单位（英文）"`
+	ClassifyId      int      `description:"所属分类"`
+	SysUserId       int      `description:"用户id"`
+	SysUserRealName string   `description:"用户真实名称"`
+	UniqueCode      string   `description:"编码"`
+	Lang            string   `description:"语言版本"`
+	EdbInfo         *EdbInfo `description:"指标信息"`
+}
+
+type RefreshBaseParams struct {
+	EdbInfo   *EdbInfo
+	StartDate string
+	EndDate   string
+}
+
+// Add
+// @Description: 添加指标
+// @author: Roc
+// @receiver obj
+// @datetime 2024-04-30 17:35:14
+// @param params AddBaseParams
+// @param businessIndexItem *BaseFromBusinessIndex
+// @return edbInfo *EdbInfo
+// @return err error
+// @return errMsg string
+func (obj Business) Add(params AddBaseParams, businessIndexItem *BaseFromBusinessIndex) (edbInfo *EdbInfo, err error, errMsg string) {
+	o := orm.NewOrm()
+	to, err := o.Begin()
+	if err != nil {
+		return
+	}
+	defer func() {
+		if err != nil {
+			_ = to.Rollback()
+			fmt.Println(reflect.TypeOf(obj).Name(), "；Add,Err:"+err.Error())
+		} else {
+			_ = to.Commit()
+		}
+	}()
+
+	edbInfo = new(EdbInfo)
+	edbInfo.Source = obj.GetSource()
+	edbInfo.SourceName = obj.GetSourceName()
+	edbInfo.EdbCode = params.EdbCode
+	edbInfo.EdbName = params.EdbName
+	edbInfo.EdbNameSource = params.EdbName
+	edbInfo.Frequency = businessIndexItem.Frequency
+	edbInfo.Unit = params.Unit
+	edbInfo.ClassifyId = params.ClassifyId
+	edbInfo.SysUserId = params.SysUserId
+	edbInfo.SysUserRealName = params.SysUserRealName
+	edbInfo.CreateTime = time.Now()
+	edbInfo.ModifyTime = time.Now()
+	edbInfo.UniqueCode = params.UniqueCode
+	edbInfo.CalculateFormula = ``
+	edbInfo.EdbNameEn = params.EdbName
+	edbInfo.UnitEn = params.Unit
+	edbInfo.EdbType = obj.GetEdbType()
+	edbInfo.SubSource = businessIndexItem.Source
+	edbInfo.SubSourceName = businessIndexItem.SourceName
+	newEdbInfoId, tmpErr := to.Insert(edbInfo)
+	if tmpErr != nil {
+		err = tmpErr
+		return
+	}
+	edbInfo.EdbInfoId = int(newEdbInfoId)
+
+	// 更新数据
+	err = obj.refresh(edbInfo, "")
+
+	return
+}
+
+// Edit
+// @Description: 编辑指标
+// @author: Roc
+// @receiver obj
+// @datetime 2024-04-30 17:35:05
+// @param params EditBaseParams
+// @param businessIndexItem *BaseFromBusinessIndex
+// @return err error
+// @return errMsg string
+func (obj Business) Edit(params EditBaseParams, businessIndexItem *BaseFromBusinessIndex) (err error, errMsg string) {
+	edbInfo := params.EdbInfo
+
+	o := orm.NewOrm()
+	to, err := o.Begin()
+	if err != nil {
+		return
+	}
+	defer func() {
+		if err != nil {
+			_ = to.Rollback()
+			fmt.Println(reflect.TypeOf(obj).Name(), "；Edit,Err:"+err.Error())
+		} else {
+			_ = to.Commit()
+		}
+	}()
+
+	//oldEdbInfo := *edbInfo
+
+	//修改指标信息
+	edbInfo.EdbName = params.EdbName
+	edbInfo.EdbNameSource = params.EdbName
+	edbInfo.Frequency = businessIndexItem.Frequency
+	edbInfo.Unit = params.Unit
+	edbInfo.ClassifyId = params.ClassifyId
+	edbInfo.EdbNameEn = params.EdbNameEn
+	edbInfo.UnitEn = params.UnitEn
+	edbInfo.SubSource = businessIndexItem.Source
+	edbInfo.SubSourceName = businessIndexItem.SourceName
+	edbInfo.ModifyTime = time.Now()
+	_, err = to.Update(edbInfo, "EdbName", "EdbNameSource", "Frequency", "Unit", "ClassifyId", "CalculateFormula", "ModifyTime", "EdbNameEn", "UnitEn", "SubSource", "SubSourceName")
+	if err != nil {
+		return
+	}
+
+	//计算数据
+	err = obj.refresh(edbInfo, "")
+
+	return
+}
+
+// Refresh 刷新
+func (obj Business) Refresh(params RefreshBaseParams) (err error, errMsg string) {
+	to, err := orm.NewOrm().Begin()
+	if err != nil {
+		return
+	}
+	defer func() {
+		if err != nil {
+			_ = to.Rollback()
+			fmt.Println(reflect.TypeOf(obj).Name(), "；Refresh,Err:"+err.Error())
+		} else {
+			_ = to.Commit()
+		}
+	}()
+
+	// 计算数据
+	err = obj.refresh(params.EdbInfo, params.StartDate)
+
+	return
+}
+
+// GetSource 获取来源编码id
+func (obj Business) GetSource() int {
+	return utils.DATA_SOURCE_BUSINESS
+}
+
+// GetSourceName 获取来源名称
+func (obj Business) GetSourceName() string {
+	return utils.DATA_SOURCE_NAME_BUSINESS
+}
+
+// GetEdbType 获取指标类型
+func (obj Business) GetEdbType() int {
+	return utils.DEFAULT_EDB_TYPE
+}
+
+func (obj Business) refresh(edbInfo *EdbInfo, startDate string) (err error) {
+	// 真实数据的最大日期  ， 插入规则配置的日期
+	var realDataMaxDate, edbDataInsertConfigDate time.Time
+	var edbDataInsertConfig *EdbDataInsertConfig
+	var isFindConfigDateRealData bool //是否找到配置日期的实际数据的值
+	{
+		edbDataInsertConfig, err = GetEdbDataInsertConfigByEdbId(edbInfo.EdbInfoId)
+		if err != nil && err.Error() != utils.ErrNoRow() {
+			return
+		}
+		if edbDataInsertConfig != nil {
+			edbDataInsertConfigDate = edbDataInsertConfig.Date
+		}
+	}
+
+	//获取已存在的所有数据
+	baseDataList, err := obj.getBaseBusinessData(edbInfo, startDate)
+
+	//获取指标所有数据
+	existDataList := make([]*mgo.EdbDataBusiness, 0)
+	mogDataObj := new(mgo.EdbDataBusiness)
+	{
+		// 构建查询条件
+		queryConditions := bson.M{
+			"edb_code": edbInfo.EdbCode,
+		}
+
+		if startDate != `` {
+			//获取已存在的所有数据
+			startDateTime, tmpErr := time.ParseInLocation(utils.FormatDate, startDate, time.Local)
+			if tmpErr != nil {
+				err = tmpErr
+				return
+			}
+			queryConditions["data_time"] = bson.M{"$gte": startDateTime}
+		}
+
+		existDataList, err = mogDataObj.GetAllDataList(queryConditions, []string{"data_time"})
+		if err != nil {
+			fmt.Println(obj.GetSourceName() + ",refresh err;getEdbDataBusinessList Err:" + err.Error())
+			return
+		}
+	}
+
+	existDataMap := make(map[string]*mgo.EdbDataBusiness)
+	removeDataTimeMap := make(map[string]bool) //需要移除的日期数据
+	for _, v := range existDataList {
+		tmpDate := v.DataTime.Format(utils.FormatDate)
+		existDataMap[tmpDate] = v
+		removeDataTimeMap[tmpDate] = true
+	}
+	needAddDateMap := make(map[time.Time]int)
+
+	// 待添加的数据集
+	addDataList := make([]interface{}, 0)
+	// 待更新的数据集
+	updateDataList := make([]mgo.EdbDataBusiness, 0)
+
+	for _, tmpData := range baseDataList {
+		currDate := tmpData.DataTime
+		currDateStr := currDate.Format(utils.FormatDate)
+
+		// 当前的实际值
+		saveValue := decimal.NewFromFloat(tmpData.Value).Round(4).String()
+
+		// 下面代码主要目的是处理掉手动插入的数据判断
+		{
+			if realDataMaxDate.IsZero() || currDate.After(realDataMaxDate) {
+				realDataMaxDate = currDate
+			}
+			if edbDataInsertConfigDate.IsZero() || currDate.Equal(edbDataInsertConfigDate) {
+				isFindConfigDateRealData = true
+			}
+		}
+
+		existData, ok := existDataMap[currDateStr]
+		// 如果库中已经存在该数据的话，那么就进行值的变更操作
+		if ok {
+			// 已经入到指标库的值
+			existValStr := decimal.NewFromFloat(existData.Value).Round(4).String()
+
+			//校验待删除日期数据里面是否存在该元素，如果存在的话，那么移除该日期
+			delete(removeDataTimeMap, currDateStr)
+			if existValStr != saveValue {
+				existData.Value = tmpData.Value
+				updateDataList = append(updateDataList, *existData)
+			}
+
+			continue
+		}
+
+		// 库中不存在该日期的数据
+		timestamp := currDate.UnixNano() / 1e6
+		addDataList = append(addDataList, mgo.EdbDataBusiness{
+			EdbInfoId:     edbInfo.EdbInfoId,
+			EdbCode:       edbInfo.EdbCode,
+			DataTime:      currDate,
+			Value:         tmpData.Value,
+			CreateTime:    time.Now(),
+			ModifyTime:    time.Now(),
+			DataTimestamp: timestamp,
+		})
+		needAddDateMap[currDate] = 1
+	}
+
+	//删除已经不存在的指标数据（由于该指标当日的数据删除了）
+	removeDateList := make([]string, 0)
+	{
+		for dateTime := range removeDataTimeMap {
+			removeDateList = append(removeDateList, dateTime)
+		}
+		removeNum := len(removeDateList)
+		if removeNum > 0 {
+			// todo 删除指定日期的数据
+
+		}
+	}
+
+	// 入库
+	{
+
+		coll := mogDataObj.GetCollection()
+
+		//删除已经不存在的指标数据（由于该指标当日的数据删除了）
+		//{
+		//	for dateTime := range removeDataTimeMap {
+		//		removeDateList = append(removeDateList, dateTime)
+		//	}
+		//	removeNum := len(removeDateList)
+		//	if removeNum > 0 {
+		//		err = mogDataObj.RemoveManyByColl(coll, addDataList)
+		//		if err != nil {
+		//			fmt.Println("mogDataObj.RemoveMany() Err:" + err.Error())
+		//			return
+		//		}
+		//	}
+		//}
+
+		// 插入新数据
+		if len(addDataList) > 0 {
+			err = mogDataObj.BatchInsertDataByColl(coll, 500, addDataList)
+			if err != nil {
+				fmt.Println("mogDataObj.BatchInsertData() Err:" + err.Error())
+				return
+			}
+		}
+
+		// 修改历史数据
+		if len(updateDataList) > 0 {
+			for _, v := range updateDataList {
+				err = mogDataObj.UpdateDataByColl(coll, bson.M{"_id": v.ID}, bson.M{"$set": bson.M{"value": v.Value, "modify_time": v.ModifyTime}})
+				if err != nil {
+					fmt.Println("mogDataObj.UpdateDataByColl:Err:" + err.Error())
+					return
+				}
+			}
+		}
+	}
+
+	// 处理手工数据补充的配置
+	HandleConfigInsertEdbDataByMongo(realDataMaxDate, edbDataInsertConfig, edbInfo.EdbInfoId, edbInfo.Source, edbInfo.SubSource, existDataMap, isFindConfigDateRealData)
+
+	return
+}
+
+// GetEdbInfoMaxAndMinInfo 获取指标的最新数据记录信息
+func (obj Business) GetEdbInfoMaxAndMinInfo(edbCode string) (item *EdbInfoMaxAndMinInfo, err error) {
+	mogDataObj := new(mgo.EdbDataBusiness)
+	pipeline := []bson.M{
+		{"$match": bson.M{"edb_code": edbCode}},
+		{"$group": bson.M{
+			"_id":       nil,
+			"min_date":  bson.M{"$min": "$data_time"},
+			"max_date":  bson.M{"$max": "$data_time"},
+			"min_value": bson.M{"$min": "$value"},
+			"max_value": bson.M{"$max": "$value"},
+		}},
+		{"$project": bson.M{"_id": 0}}, // 可选，如果不需要_id字段
+	}
+	result, err := mogDataObj.GetEdbInfoMaxAndMinInfo(pipeline)
+	if err != nil {
+		fmt.Println("EdbDataBusiness getEdbDataBusinessList Err:" + err.Error())
+		return
+	}
+
+	if !result.MaxDate.IsZero() {
+		whereQuery := bson.M{"edb_code": edbCode, "data_time": result.MaxDate}
+		selectParam := bson.D{{"value", 1}, {"_id", 0}}
+		latestValue, tmpErr := mogDataObj.GetLatestValue(whereQuery, selectParam)
+		if tmpErr != nil {
+			err = tmpErr
+			return
+		}
+		result.LatestValue = latestValue.Value
+		result.EndValue = latestValue.Value
+	}
+
+	item = &EdbInfoMaxAndMinInfo{
+		MinDate:     result.MinDate.Format(utils.FormatDate),
+		MaxDate:     result.MaxDate.Format(utils.FormatDate),
+		MinValue:    result.MinValue,
+		MaxValue:    result.MaxValue,
+		LatestValue: result.LatestValue,
+		LatestDate:  result.LatestDate.Format(utils.FormatDate),
+		EndValue:    result.EndValue,
+	}
+
+	return
+}
+
+// UnifiedModifyEdbInfoMaxAndMinInfo
+// @Description: 修改指标的最大最小值和最新值
+// @author: Roc
+// @receiver obj
+// @datetime 2024-04-30 17:07:35
+// @param edbInfo *EdbInfo
+// @return err error
+func (obj Business) UnifiedModifyEdbInfoMaxAndMinInfo(edbInfo *EdbInfo) (err error) {
+	edbInfoMaxAndMinInfo, err := obj.GetEdbInfoMaxAndMinInfo(edbInfo.EdbCode)
+	if err != nil {
+		return
+	}
+	err = ModifyEdbInfoMaxAndMinInfo(edbInfo.EdbInfoId, edbInfoMaxAndMinInfo)
+	return
+}
+
+// EdbInfoMgoData
+// @Description: mgo里面的数据
+type EdbInfoMgoData struct {
+	EdbDataId primitive.ObjectID `description:"数据ID"`
+	DataTime  time.Time          `description:"数据日期"`
+	Value     float64            `description:"数据"`
+	EdbCode   string             `description:"指标编码"`
+}
+
+// getBaseBusinessData
+// @Description: 获取基础数据
+// @author: Roc
+// @receiver obj
+// @datetime 2024-04-30 11:10:46
+// @param edbInfo *EdbInfo
+// @param startDate string
+// @return newDataList []EdbInfoSearchData
+// @return err error
+func (obj Business) getBaseBusinessData(edbInfo *EdbInfo, startDate string) (newDataList []EdbInfoMgoData, err error) {
+	newDataList = make([]EdbInfoMgoData, 0)
+
+	// 获取数据源的指标数据
+	mogDataObj := new(mgo.BaseFromBusinessData)
+
+	// 构建查询条件
+	queryConditions := bson.M{
+		"index_code": edbInfo.EdbCode,
+	}
+
+	if startDate != `` {
+		//获取已存在的所有数据
+		startDateTime, tmpErr := time.ParseInLocation(utils.FormatDate, startDate, time.Local)
+		if tmpErr != nil {
+			err = tmpErr
+			return
+		}
+		queryConditions["data_time"] = bson.M{"$gte": startDateTime}
+	}
+
+	baseDataList, err := mogDataObj.GetAllDataList(queryConditions, []string{"data_time"})
+	if err != nil {
+		fmt.Println("getBaseBusinessData Err:" + err.Error())
+		return
+	}
+
+	for _, v := range baseDataList {
+		newDataList = append(newDataList, EdbInfoMgoData{
+			EdbDataId: v.ID,
+			DataTime:  v.DataTime,
+			Value:     v.Value,
+			EdbCode:   v.IndexCode,
+		})
+	}
+
+	return
+}

models/edb_data_calculate_avg.go

@@ -298,11 +298,9 @@ func (obj CalculateAvg) refresh(to orm.TxOrmer, edbInfoId, source, subSource int
 	}
 
 	for edbInfoIndex, v := range edbInfoIdArr {
-		var condition string
-		var pars []interface{}
-		condition += " AND edb_info_id=? "
-		pars = append(pars, v.EdbInfoId)
-		dataList, err := GetEdbDataListAllByTo(to, condition, pars, v.Source, v.SubSource, 1)
+		dataList, err := GetEdbDataListAllByTo(to, v.Source, v.SubSource, FindEdbDataListAllCond{
+			EdbInfoId: v.EdbInfoId,
+		}, 1)
 		if err != nil {
 			return err
 		}

models/edb_data_calculate_bp.go

@@ -205,22 +205,13 @@ func RefreshAllCalculateBpBak(edbInfoId, source, subSource int, fromEdbInfo *Edb
 	edbInfoIdStr := strconv.Itoa(edbInfoId)
 	//计算数据
 
-	//计算数据
-	var condition string
-	var pars []interface{}
-	condition += " AND edb_info_id=? "
-	pars = append(pars, fromEdbInfo.EdbInfoId)
-
-	if startDate != "" {
-		condition += " AND data_time>=? "
-		pars = append(pars, startDate)
-	}
-	if endDate != "" {
-		condition += " AND data_time<=? "
-		pars = append(pars, endDate)
-	}
-
-	dataList, err := GetEdbDataListAllByTo(to, condition, pars, fromEdbInfo.Source, fromEdbInfo.SubSource, 0)
+	dataList, err := GetEdbDataListAllByTo(to, fromEdbInfo.Source, fromEdbInfo.SubSource, FindEdbDataListAllCond{
+		EdbInfoId:         fromEdbInfo.EdbInfoId,
+		StartDataTime:     startDate,
+		StartDataTimeCond: ">=",
+		EndDataTime:       endDate,
+		EndDataTimeCond:   "<=",
+	}, 0)
 	if err != nil {
 		return err
 	}
@@ -348,197 +339,116 @@ func RefreshAllCalculateBp(edbInfoId, source, subSource int, fromEdbInfo *EdbInf
 // refreshAllCalculateBp 刷新升频数据
 func refreshAllCalculateBp(to orm.TxOrmer, edbInfoId, source, subSource int, fromEdbInfo *EdbInfo, edbCode, startDate, endDate string, order int) (err error) {
 	edbInfoIdStr := strconv.Itoa(edbInfoId)
-
 	//计算数据
-	var condition string
-	var pars []interface{}
-	condition += " AND edb_info_id=? "
-	pars = append(pars, fromEdbInfo.EdbInfoId)
-
-	//if startDate != "" {
-	//	condition += " AND data_time>=? "
-	//	pars = append(pars, startDate)
-	//}
-	//if endDate != "" {
-	//	condition += " AND data_time<=? "
-	//	pars = append(pars, endDate)
-	//}
+
 	//获取来源指标的数据
-	dataList, err := GetEdbDataListAllByTo(to, condition, pars, fromEdbInfo.Source, fromEdbInfo.SubSource, order)
+	dataList, err := GetEdbDataListAllByTo(to, fromEdbInfo.Source, fromEdbInfo.SubSource, FindEdbDataListAllCond{
+		EdbInfoId: fromEdbInfo.EdbInfoId,
+	}, order)
 	if err != nil {
 		return err
 	}
-	var dateArr []string
-	dataMap := make(map[string]*EdbInfoSearchData)
-	fromDataMap := make(map[string]float64)
-	//来源指指标数据
-	for _, v := range dataList {
-		dateArr = append(dateArr, v.DataTime)
-		dataMap[v.DataTime] = v
-		fromDataMap[v.DataTime] = v.Value
+
+	// 来源指标没有数据，那么需要删除所有的计算指标数据
+	if len(dataList) <= 0 {
+		// todo 删除所有的计算指标数据
+		return
 	}
-	fmt.Println("source:", source)
+	// 来源指标的第一个日期
+	fromFirstDate, err := time.ParseInLocation(utils.FormatDate, dataList[0].DataTime, time.Local)
+	if err != nil {
+		return
+	}
+	fromFirstDate = time.Date(fromFirstDate.Year(), fromFirstDate.Month(), fromFirstDate.Day(), 0, 0, 0, 0, time.Local)
 
-	//获取升频指标所有数据
+	// 变频计算
+	newDataList, err := EdbInfoSearchDataToData(dataList)
+	if err != nil {
+		return
+	}
+
+	baseCalculate := BaseCalculate{
+		DataList:      newDataList,
+		Frequency:     "",
+		Formula:       nil,
+		Calendar:      "",
+		MoveType:      0,
+		MoveFrequency: "",
+		FromFrequency: "",
+		Source:        source,
+	}
+	dateDataMap, err, _ := baseCalculate.UpFrequency()
+	if err != nil {
+		return
+	}
+
+	// 获取升频指所有已经存在的计算指标数据
 	existDataList, err := GetAllEdbDataListByTo(to, edbInfoId, source, subSource)
 	if err != nil {
 		return
 	}
 	//计算指标的map
 	existDataMap := make(map[string]*EdbData, 0)
+	for _, v := range existDataList {
+		existDataMap[v.DataTime] = v
+	}
 
 	addSql := ` INSERT INTO edb_data_calculate_bp(edb_info_id,edb_code,data_time,value,create_time,modify_time,data_timestamp) values `
 	var isAdd bool
 
-	var lastValue float64   //最后数据的值(float64)
-	var lastValueStr string //最后数据的值(string)
-	//待删除的日期
-	removeDateList := make([]string, 0)
-	if len(existDataList) > 0 {
-		//第一个已经入库的日期
-		firstExistDataTimeStr := existDataList[0].DataTime //计算指标数据第一条的日期字符串
-		if len(dateArr) > 0 {
-			firstFromDataTimeStr := dateArr[0]                                                                 //来源数据第一条的日期字符串
-			firstExistDataTime, _ := time.ParseInLocation(utils.FormatDate, firstExistDataTimeStr, time.Local) //计算指标数据第一条的日期（time类型）
-			firstFromDataTime, _ := time.ParseInLocation(utils.FormatDate, firstFromDataTimeStr, time.Local)   //来源数据第一条的日期（time类型）
-			nowDateStr := time.Now().Format(utils.FormatDate)                                                  //当天日期字符串
-			nowDate, _ := time.ParseInLocation(utils.FormatDate, nowDateStr, firstFromDataTime.Location())     //当天日期（time类型）
-
-			lastValue = fromDataMap[firstFromDataTimeStr]
-			lastValueStr = decimal.NewFromFloat(lastValue).String()
-			//第一步： 判断来源指标的开始时间与计算指标的开始时间是否相等，相等的话，那么就不需要对两个时间之间的数据做处理
-			if firstExistDataTimeStr != firstFromDataTimeStr {
-				if firstExistDataTime.Before(firstFromDataTime) { //如果计算指标第一条数据的开始时间 早于 来源指标的第一条开始时间，那么需要对两个时间之间的计算指标数据做 删除处理
-					for _, v := range existDataList {
-						if v.DataTime == firstFromDataTimeStr {
-							if tmpLastValue, ok := fromDataMap[firstFromDataTimeStr]; ok { //来源指标当天的数据
-								lastValue = tmpLastValue
-								lastValueStr = decimal.NewFromFloat(lastValue).String()
-							}
-							break
-						}
-						removeDateList = append(removeDateList, v.DataTime)
-					}
-				} else {
-					for _, v := range dateArr { //如果计算指标第一条数据的开始时间 晚于 来源指标的第一条开始时间，那么需要对两个时间之间的计算指标数据做 新增处理
-						vDataTime, _ := time.ParseInLocation(utils.FormatDate, v, time.Local) //当前日期（time类型）
-						if firstExistDataTime.Equal(vDataTime) || firstExistDataTime.Before(vDataTime) {
-							if tmpLastValue, ok := fromDataMap[v]; ok { //来源指标当天的数据
-								lastValue = tmpLastValue
-								lastValueStr = decimal.NewFromFloat(lastValue).String()
-							}
-							break
-						}
-
-						currentDate, _ := time.ParseInLocation(utils.FormatDate, v, time.Local)
-						timestamp := currentDate.UnixNano() / 1e6
-						timestampStr := fmt.Sprintf("%d", timestamp)
-						addSql += GetAddSql(edbInfoIdStr, edbCode, v, timestampStr, lastValueStr)
-
-						isAdd = true
-					}
-				}
-			}
-
-			//第二步 剩余数据每天修改
+	now := time.Date(time.Now().Year(), time.Now().Month(), time.Now().Day(), 0, 0, 0, 0, time.Local)
 
-			day := int(nowDate.Sub(firstExistDataTime).Hours() / float64(24))
+	for currDate := fromFirstDate; !currDate.After(now); currDate = currDate.AddDate(0, 0, 1) {
+		currDateStr := currDate.Format(utils.FormatDate)
+		timestamp := currDate.UnixNano() / 1e6
+		timestampStr := fmt.Sprintf("%d", timestamp)
 
-			//第三步： 已经入库的数据处理
-			for _, v := range existDataList {
-				existDataMap[v.DataTime] = v
-			}
+		// 当前计算的值
+		currValue, ok := dateDataMap[currDate]
+		if !ok {
+			// 没有计算成功就过滤
+			continue
+		}
+		lastValueStr := decimal.NewFromFloat(currValue).Round(4).String()
+
+		// 已经入库的值
+		existData, ok := existDataMap[currDateStr]
+		if !ok {
+			// 没有入库那么就插入添加
+			isAdd = true
+			addSql += GetAddSql(edbInfoIdStr, edbCode, currDateStr, timestampStr, lastValueStr)
+			continue
+		}
 
-			for k := day; k >= 0; k-- {
-				needDay := nowDate.AddDate(0, 0, -k)
-				needDayStr := needDay.Format(utils.FormatDate)
-				tmpExistData, ok := existDataMap[needDayStr]
-				if ok {
-					if tmpLastValue, ok := fromDataMap[tmpExistData.DataTime]; ok { //来源指标当天的数据
-						lastValue = tmpLastValue
-						//lastValueStr = decimal.NewFromFloat(lastValue).String()
-						lastValueStr = fmt.Sprintf("%.4f", lastValue)
-					}
-					//如果对应的值不匹配
-					if tmpExistData.Value != lastValueStr {
-						err = ModifyEdbDataById(source, subSource, tmpExistData.EdbDataId, lastValueStr)
-						if err != nil {
-							return err
-						}
-					}
-				} else {
-					timestamp := needDay.UnixNano() / 1e6
-					timestampStr := fmt.Sprintf("%d", timestamp)
-					addSql += GetAddSql(edbInfoIdStr, edbCode, needDayStr, timestampStr, lastValueStr)
+		// 将已经入库的值转换为decimal类型，然后再保留4位小数，目的是为了做匹配，要不然取出来的数据与计算的数据不一致
+		existDataValueDec, tmpErr := decimal.NewFromString(existData.Value)
+		if tmpErr != nil {
+			err = tmpErr
+			return
+		}
+		existDataValueStr := existDataValueDec.Round(4).String()
 
-					isAdd = true
-				}
-			}
-		} else {
-			//如果没有来源指标数据，那么已经入库的计算指标数据需要全部删除
-			tableName := GetEdbDataTableName(source, subSource)
-			sql := fmt.Sprintf(` DELETE FROM %s WHERE edb_info_id = ?`, tableName)
-			_, err = to.Raw(sql, edbInfoId).Exec()
+		// 如果该日期已经入库了，且两个值不匹配，那么就更新
+		if lastValueStr != existDataValueStr {
+			err = ModifyEdbDataById(source, subSource, existData.EdbDataId, lastValueStr)
 			if err != nil {
-				err = fmt.Errorf("删除所有的升频指标数据失败,Err:" + err.Error())
-				return
+				return err
 			}
-
-			//for _, v := range existDataList {
-			//	removeDateList = append(removeDateList, v.DataTime)
-			//}
 		}
-	} else {
-		existMap := make(map[string]string)
-		dataLen := len(dataList)
 
-		for i := 0; i < dataLen; i++ {
-			//当期
-			currentItem := dataList[i]
-			currentDate, _ := time.ParseInLocation(utils.FormatDate, currentItem.DataTime, time.Local)
-			var day int
-			var preItem *EdbInfoSearchData
-			var preDate time.Time
-			if i == 0 {
-				day = int(time.Now().Sub(currentDate).Hours() / float64(24))
-				preDate = time.Now()
-			} else {
-				j := i - 1
-				if j < dataLen {
-					preItem = dataList[j]
-					preDate, _ = time.ParseInLocation(utils.FormatDate, preItem.DataTime, time.Local)
-					day = int(preDate.Sub(currentDate).Hours() / float64(24))
-					utils.FileLog.Info("preItem.DataTime:" + preItem.DataTime + ";currentItem.DataTime" + currentItem.DataTime)
-				}
-			}
-			for k := 0; k <= day; k++ {
-				needDay := preDate.AddDate(0, 0, -k)
-				needDayStr := needDay.Format(utils.FormatDate)
-				existKey := edbCode + needDayStr
-				if _, ok := existMap[existKey]; !ok {
-					timestamp := needDay.UnixNano() / 1e6
-					timestampStr := fmt.Sprintf("%d", timestamp)
-					valStr := decimal.NewFromFloat(currentItem.Value).String()
-					addSql += GetAddSql(edbInfoIdStr, edbCode, needDayStr, timestampStr, valStr)
-					isAdd = true
-				}
-				existMap[existKey] = needDayStr
-			}
-			existKey := edbCode + currentItem.DataTime
-			if _, ok := existMap[existKey]; !ok {
-				currentDate, _ := time.ParseInLocation(utils.FormatDate, currentItem.DataTime, time.Local)
-				timestamp := currentDate.UnixNano() / 1e6
-				timestampStr := fmt.Sprintf("%d", timestamp)
-				valStr := decimal.NewFromFloat(currentItem.Value).String()
-				addSql += GetAddSql(edbInfoIdStr, edbCode, currentItem.DataTime, timestampStr, valStr)
-				isAdd = true
-			}
-			existMap[existKey] = currentItem.DataTime
-		}
+		// 该日期已经处理过了，所以需要移除，如果后面该map还有数据，那么需要删除该map里面的日期数据
+		delete(existDataMap, currDateStr)
+
 	}
 
 	// 删除不需要的指标数据
-	if len(removeDateList) > 0 {
+	if len(existDataMap) > 0 {
+		//待删除的日期
+		removeDateList := make([]string, 0)
+		for date := range existDataMap {
+			removeDateList = append(removeDateList, date)
+		}
+
 		removeDateStr := strings.Join(removeDateList, `","`)
 		removeDateStr = `"` + removeDateStr + `"`
 		//如果拼接指标变更了，那么需要删除所有的指标数据
@@ -552,9 +462,11 @@ func refreshAllCalculateBp(to orm.TxOrmer, edbInfoId, source, subSource int, fro
 		}
 	}
 
+	// 新增的数据值
 	if isAdd {
 		addSql = strings.TrimRight(addSql, ",")
 		_, err = to.Raw(addSql).Exec()
 	}
+
 	return
 }

models/edb_data_calculate_cjjx.go

@@ -232,22 +232,9 @@ func refreshAllCalculateCjjx(to orm.TxOrmer, edbInfoId, source, subSource int, f
 	edbInfoIdStr := strconv.Itoa(edbInfoId)
 	//计算数据
 
-	var condition string
-	var pars []interface{}
-	condition += " AND edb_info_id=? "
-	pars = append(pars, fromEdbInfo.EdbInfoId)
-
-	//if startDate != "" {
-	//	condition += " AND data_time>=? "
-	//	pars = append(pars, startDate)
-	//}
-
-	//if endDate != "" {
-	//	condition += " AND data_time<=? "
-	//	pars = append(pars, endDate)
-	//}
-
-	dataList, err := GetEdbDataListAllByTo(to, condition, pars, fromEdbInfo.Source, fromEdbInfo.SubSource, 1)
+	dataList, err := GetEdbDataListAllByTo(to, fromEdbInfo.Source, fromEdbInfo.SubSource, FindEdbDataListAllCond{
+		EdbInfoId: fromEdbInfo.EdbInfoId,
+	}, 1)
 	if err != nil {
 		return err
 	}

models/edb_data_calculate_correlation.go

@@ -396,13 +396,10 @@ func GetRollingCorrelationChartDataByEdbInfo(edbInfoMappingA, edbInfoMappingB *E
 	aDataList := make([]*EdbInfoSearchData, 0)
 	switch baseEdbInfo.EdbInfoType {
 	case 0:
-		var condition string
-		var pars []interface{}
-		condition += " AND edb_info_id=? "
-		pars = append(pars, baseEdbInfo.EdbInfoId)
-
 		//获取来源指标的数据
-		aDataList, err = GetEdbDataListAll(condition, pars, baseEdbInfo.Source, baseEdbInfo.SubSource, 1)
+		aDataList, err = GetEdbDataListAll(baseEdbInfo.Source, baseEdbInfo.SubSource, FindEdbDataListAllCond{
+			EdbInfoId: baseEdbInfo.EdbInfoId,
+		}, 1)
 	case 1:
 		aDataList, err = GetPredictEdbDataListAllByStartDate(baseEdbInfo, 1, "")
 	default:
@@ -414,13 +411,10 @@ func GetRollingCorrelationChartDataByEdbInfo(edbInfoMappingA, edbInfoMappingB *E
 	bDataList := make([]*EdbInfoSearchData, 0)
 	switch changeEdbInfo.EdbInfoType {
 	case 0:
-		var condition string
-		var pars []interface{}
-		condition += " AND edb_info_id=? "
-		pars = append(pars, changeEdbInfo.EdbInfoId)
-
 		//获取来源指标的数据
-		bDataList, err = GetEdbDataListAll(condition, pars, changeEdbInfo.Source, changeEdbInfo.SubSource, 1)
+		bDataList, err = GetEdbDataListAll(changeEdbInfo.Source, changeEdbInfo.SubSource, FindEdbDataListAllCond{
+			EdbInfoId: changeEdbInfo.EdbInfoId,
+		}, 1)
 	case 1:
 		bDataList, err = GetPredictEdbDataListAllByStartDate(changeEdbInfo, 1, "")
 	default:

models/edb_data_calculate_hbz.go

@@ -209,22 +209,10 @@ func RefreshAllCalculateHbz(edbInfoId, source, subSource int, fromEdbInfo *EdbIn
 func refreshAllCalculateHbz(to orm.TxOrmer, edbInfoId, source, subSource int, fromEdbInfo *EdbInfo, edbCode, startDate, endDate string, formulaInt int) (err error) {
 	edbInfoIdStr := strconv.Itoa(edbInfoId)
 	//计算数据
-	var condition string
-	var pars []interface{}
-	condition += " AND edb_info_id=? "
-	pars = append(pars, fromEdbInfo.EdbInfoId)
 
-	//if startDate != "" {
-	//	condition += " AND data_time>=? "
-	//	pars = append(pars, startDate)
-	//}
-
-	//if endDate != "" {
-	//	condition += " AND data_time<=? "
-	//	pars = append(pars, endDate)
-	//}
-
-	dataList, err := GetEdbDataListAllByTo(to, condition, pars, fromEdbInfo.Source, fromEdbInfo.SubSource, 0)
+	dataList, err := GetEdbDataListAllByTo(to, fromEdbInfo.Source, fromEdbInfo.SubSource, FindEdbDataListAllCond{
+		EdbInfoId: fromEdbInfo.EdbInfoId,
+	}, 0)
 	if err != nil {
 		return err
 	}

models/edb_data_calculate_hcz.go

@@ -211,22 +211,9 @@ func refreshAllCalculateHcz(to orm.TxOrmer, edbInfoId, source, subSource int, fr
 	edbInfoIdStr := strconv.Itoa(edbInfoId)
 
 	//计算数据
-	var condition string
-	var pars []interface{}
-	condition += " AND edb_info_id=? "
-	pars = append(pars, fromEdbInfo.EdbInfoId)
-
-	//if startDate != "" {
-	//	condition += " AND data_time>=? "
-	//	pars = append(pars, startDate)
-	//}
-
-	//if endDate != "" {
-	//	condition += " AND data_time<=? "
-	//	pars = append(pars, endDate)
-	//}
-
-	dataList, err := GetEdbDataListAllByTo(to, condition, pars, fromEdbInfo.Source, fromEdbInfo.SubSource, 0)
+	dataList, err := GetEdbDataListAllByTo(to, fromEdbInfo.Source, fromEdbInfo.SubSource, FindEdbDataListAllCond{
+		EdbInfoId: fromEdbInfo.EdbInfoId,
+	}, 0)
 	if err != nil {
 		return err
 	}

models/edb_data_calculate_jp.go

@@ -238,21 +238,10 @@ func refreshAllCalculateJp(to orm.TxOrmer, edbInfoId, source, subSource int, fro
 	edbInfoIdStr := strconv.Itoa(edbInfoId)
 
 	//计算数据
-	var condition string
-	var pars []interface{}
-	condition += " AND edb_info_id=? "
-	pars = append(pars, fromEdbInfo.EdbInfoId)
-
-	//if startDate != "" {
-	//	condition += " AND data_time>=? "
-	//	pars = append(pars, startDate)
-	//}
-	//if endDate != "" {
-	//	condition += " AND data_time<=? "
-	//	pars = append(pars, endDate)
-	//}
 	//获取来源指标的数据
-	dataList, err := GetEdbDataListAllByTo(to, condition, pars, fromEdbInfo.Source, fromEdbInfo.SubSource, 1)
+	dataList, err := GetEdbDataListAllByTo(to, fromEdbInfo.Source, fromEdbInfo.SubSource, FindEdbDataListAllCond{
+		EdbInfoId: fromEdbInfo.EdbInfoId,
+	}, 1)
 	if err != nil {
 		return err
 	}
@@ -465,12 +454,16 @@ func refreshAllCalculateJp(to orm.TxOrmer, edbInfoId, source, subSource int, fro
 					return err
 				}
 			}
+			// 移除待删除的日期
+			delete(existDelDateMap, nextEndDate.Format(utils.FormatDate))
 		} else {
 			// 直接入库
 			timestamp := nextEndDate.UnixNano() / 1e6
 			timestampStr := fmt.Sprintf("%d", timestamp)
 			addSql += GetAddSql(edbInfoIdStr, edbCode, nextEndDate.Format(utils.FormatDate), timestampStr, fmt.Sprint(currVal))
 			isAdd = true
+			// 移除待删除的日期
+			delete(existDelDateMap, nextEndDate.Format(utils.FormatDate))
 		}
 	}
 

models/edb_data_calculate_kszs.go

@@ -269,11 +269,9 @@ func refreshAllCalculateKszs(to orm.TxOrmer, edbInfoId, source, subSource int, r
 	// 获取选择指标的 需要数据的 开始日期和结束日期
 	var startDate, endDate time.Time
 	for _, v := range relationEdbInfoList {
-		var condition string
-		var pars []interface{}
-		condition += " AND edb_info_id=? "
-		pars = append(pars, v.EdbInfoId)
-		tmpDataList, tmpErr := GetEdbDataListAllByTo(to, condition, pars, v.Source, v.SubSource, 1)
+		tmpDataList, tmpErr := GetEdbDataListAllByTo(to, v.Source, v.SubSource, FindEdbDataListAllCond{
+			EdbInfoId: v.EdbInfoId,
+		}, 1)
 		if tmpErr != nil {
 			err = tmpErr
 			return err

models/edb_data_calculate_ljz.go

@@ -290,15 +290,6 @@ func (obj Ljz) refresh(to orm.TxOrmer, edbInfoId, source, subSource int, edbInfo
 	dataTableName := GetEdbDataTableName(source, subSource)
 	edbInfoIdStr := strconv.Itoa(edbInfoId)
 	//计算数据
-	var condition string
-	var pars []interface{}
-	condition += " AND edb_info_id=? "
-	pars = append(pars, fromEdbInfo.EdbInfoId)
-
-	//if startDate != "" {
-	//	condition += " AND data_time>=? "
-	//	pars = append(pars, startDate)
-	//}
 
 	var isWeekData bool // 是否周度数据，如果是周度数据的话，是需要变频的，最后结果还需要除以7
 	// 周度数据需要先变成日度的
@@ -306,7 +297,9 @@ func (obj Ljz) refresh(to orm.TxOrmer, edbInfoId, source, subSource int, edbInfo
 		isWeekData = true
 	}
 
-	dataList, err := GetEdbDataListAllByTo(to, condition, pars, fromEdbInfo.Source, fromEdbInfo.SubSource, 1)
+	dataList, err := GetEdbDataListAllByTo(to, fromEdbInfo.Source, fromEdbInfo.SubSource, FindEdbDataListAllCond{
+		EdbInfoId: fromEdbInfo.EdbInfoId,
+	}, 1)
 	if err != nil {
 		return err
 	}

models/edb_data_calculate_ljznczj.go

@@ -287,15 +287,6 @@ func (obj LjzNczj) refresh(to orm.TxOrmer, edbInfoId, source, subSource int, edb
 	dataTableName := GetEdbDataTableName(source, subSource)
 	edbInfoIdStr := strconv.Itoa(edbInfoId)
 	//计算数据
-	var condition string
-	var pars []interface{}
-	condition += " AND edb_info_id=? "
-	pars = append(pars, fromEdbInfo.EdbInfoId)
-
-	//if startDate != "" {
-	//	condition += " AND data_time>=? "
-	//	pars = append(pars, startDate)
-	//}
 
 	var isWeekData bool // 是否周度数据，如果是周度数据的话，是需要变频的，最后结果还需要除以7
 	// 周度数据需要先变成日度的
@@ -303,7 +294,9 @@ func (obj LjzNczj) refresh(to orm.TxOrmer, edbInfoId, source, subSource int, edb
 		isWeekData = true
 	}
 
-	dataList, err := GetEdbDataListAllByTo(to, condition, pars, fromEdbInfo.Source, fromEdbInfo.SubSource, 1)
+	dataList, err := GetEdbDataListAllByTo(to, fromEdbInfo.Source, fromEdbInfo.SubSource, FindEdbDataListAllCond{
+		EdbInfoId: fromEdbInfo.EdbInfoId,
+	}, 1)
 	if err != nil {
 		return err
 	}

models/edb_data_calculate_ljztbpj.go

@@ -64,12 +64,11 @@ func AddCalculateLjztbpj(req *EdbInfoCalculateBatchSaveReq, firstEdbInfo, second
 		lastDateTime, _ = time.ParseInLocation(utils.FormatDate, lastEdbData.DataTime, time.Local)
 
 		//获取待拼接指标的数据列表
-		condition = ``
-		pars = make([]interface{}, 0)
-		condition += " AND data_time <= ? AND edb_info_id=? "
-		pars = append(pars, lastEdbData.DataTime, firstEdbInfo.EdbInfoId)
-
-		firstDataList, tmpErr := GetEdbDataListAllByTo(to, condition, pars, firstEdbInfo.Source, firstEdbInfo.SubSource, 0)
+		firstDataList, tmpErr := GetEdbDataListAllByTo(to, firstEdbInfo.Source, firstEdbInfo.SubSource, FindEdbDataListAllCond{
+			EdbInfoId:       firstEdbInfo.EdbInfoId,
+			EndDataTime:     lastEdbData.DataTime,
+			EndDataTimeCond: "<=",
+		}, 0)
 		if tmpErr != nil {
 			err = tmpErr
 			return
@@ -475,14 +474,12 @@ func refreshAllCalculateLjztbpj(to orm.TxOrmer, edbInfo *EdbInfo, existItemA, ex
 
 	//待拼接指标
 	{
-		var condition string
-		var pars []interface{}
-
-		condition += " AND data_time <= ? AND edb_info_id=? "
-		pars = append(pars, startCalculationDate, existItemA.FromEdbInfoId)
-
 		//第一个指标的数据列表
-		firstDataList, tmpErr := GetEdbDataListAllByTo(to, condition, pars, existItemA.FromSource, existItemA.FromSubSource, 0)
+		firstDataList, tmpErr := GetEdbDataListAllByTo(to, existItemA.FromSource, existItemA.FromSubSource, FindEdbDataListAllCond{
+			EdbInfoId:       existItemA.FromEdbInfoId,
+			EndDataTime:     startCalculationDate.Format(utils.FormatDate),
+			EndDataTimeCond: "<=",
+		}, 0)
 		if tmpErr != nil {
 			return tmpErr
 		}
@@ -513,14 +510,12 @@ func refreshAllCalculateLjztbpj(to orm.TxOrmer, edbInfo *EdbInfo, existItemA, ex
 
 	//同比值指标
 	{
-		var condition string
-		var pars []interface{}
-
-		condition += " AND data_time > ? AND edb_info_id = ? "
-		pars = append(pars, startCalculationDate, existItemB.FromEdbInfoId)
-
 		//第二个指标的数据列表
-		secondDataList, tmpErr := GetEdbDataListAllByTo(to, condition, pars, existItemB.FromSource, existItemB.FromSubSource, 0)
+		secondDataList, tmpErr := GetEdbDataListAllByTo(to, existItemB.FromSource, existItemB.FromSubSource, FindEdbDataListAllCond{
+			EdbInfoId:         existItemB.FromEdbInfoId,
+			StartDataTime:     startCalculationDate.Format(utils.FormatDate),
+			StartDataTimeCond: ">",
+		}, 0)
 		if tmpErr != nil {
 			return tmpErr
 		}

models/edb_data_calculate_ljzzj.go

@@ -247,11 +247,10 @@ func (obj Ljzzj) refresh(to orm.TxOrmer, edbInfoId, source, subSource int, fromE
 	edbInfoIdStr := strconv.Itoa(edbInfoId)
 
 	// 获取来源指标的数据
-	var condition string
-	var pars []interface{}
-	condition += " AND edb_info_id=? "
-	pars = append(pars, fromEdbInfo.EdbInfoId)
-	dataList, err := GetEdbDataListAllByTo(to, condition, pars, fromEdbInfo.Source, fromEdbInfo.SubSource, 1)
+	dataList, err := GetEdbDataListAllByTo(to, fromEdbInfo.Source, fromEdbInfo.SubSource,
+		FindEdbDataListAllCond{
+			EdbInfoId: fromEdbInfo.EdbInfoId,
+		}, 1)
 	if err != nil {
 		return err
 	}

models/edb_data_calculate_ljzzy.go

@@ -212,21 +212,13 @@ func RefreshAllCalculateLjzzy(edbInfoId, source, subSource int, fromEdbInfo *Edb
 func refreshAllCalculateLjzzy(to orm.TxOrmer, edbInfoId, source, subSource int, fromEdbInfo *EdbInfo, edbCode, startDate, endDate string) (err error) {
 	edbInfoIdStr := strconv.Itoa(edbInfoId)
 	//计算数据
-	var condition string
-	var pars []interface{}
-	condition += " AND edb_info_id=? "
-	pars = append(pars, fromEdbInfo.EdbInfoId)
 
-	if startDate != "" {
-		condition += " AND data_time>=? "
-		pars = append(pars, startDate)
-	}
-	//if endDate != "" {
-	//	condition += " AND data_time<=? "
-	//	pars = append(pars, endDate)
-	//}
-
-	dataList, err := GetEdbDataListAllByTo(to, condition, pars, fromEdbInfo.Source, fromEdbInfo.SubSource, 1)
+	dataList, err := GetEdbDataListAllByTo(to, fromEdbInfo.Source, fromEdbInfo.SubSource,
+		FindEdbDataListAllCond{
+			EdbInfoId:         fromEdbInfo.EdbInfoId,
+			StartDataTime:     startDate,
+			StartDataTimeCond: ">=",
+		}, 1)
 	if err != nil {
 		return err
 	}

models/edb_data_calculate_nh.go

@@ -225,13 +225,11 @@ func refreshAllCalculateNh(to orm.TxOrmer, edbInfoId, source, subSource int, fro
 	}
 
 	//计算数据
-	var condition string
-	var pars []interface{}
-	condition += " AND edb_info_id=? "
-	pars = append(pars, fromEdbInfo.EdbInfoId)
 
 	//获取来源指标的数据
-	fromDataList, err := GetEdbDataListAllByTo(to, condition, pars, fromEdbInfo.Source, fromEdbInfo.SubSource, 1)
+	fromDataList, err := GetEdbDataListAllByTo(to, fromEdbInfo.Source, fromEdbInfo.SubSource, FindEdbDataListAllCond{
+		EdbInfoId: fromEdbInfo.EdbInfoId,
+	}, 1)
 	if err != nil {
 		return err
 	}

models/edb_data_calculate_nhcc.go

@@ -751,14 +751,11 @@ func getNhccData(to orm.TxOrmer, existItemA, existItemB *EdbInfoCalculateMapping
 	aDataList = make([]EdbInfoSearchData, 0)
 	aDataMap = make(map[string]float64)
 	{
-		var condition string
-		var pars []interface{}
-
-		condition += " AND edb_info_id=? "
-		pars = append(pars, existItemA.FromEdbInfoId)
 
 		//第一个指标的数据列表
-		firstDataList, tmpErr := GetEdbDataListAllByTo(to, condition, pars, existItemA.FromSource, existItemA.FromSubSource, 0)
+		firstDataList, tmpErr := GetEdbDataListAllByTo(to, existItemA.FromSource, existItemA.FromSubSource, FindEdbDataListAllCond{
+			EdbInfoId: existItemA.FromEdbInfoId,
+		}, 0)
 		if tmpErr != nil {
 			err = tmpErr
 			return
@@ -772,14 +769,11 @@ func getNhccData(to orm.TxOrmer, existItemA, existItemB *EdbInfoCalculateMapping
 	secondDataList = make([]*EdbInfoSearchData, 0)
 	bDataMap = make(map[string]float64)
 	{
-		condition := ``
-		pars := make([]interface{}, 0)
-
-		condition += "  AND edb_info_id = ? "
-		pars = append(pars, existItemB.FromEdbInfoId)
 
 		//第二个指标的数据列表
-		secondDataList, err = GetEdbDataListAllByTo(to, condition, pars, existItemB.FromSource, existItemB.FromSubSource, 0)
+		secondDataList, err = GetEdbDataListAllByTo(to, existItemB.FromSource, existItemB.FromSubSource, FindEdbDataListAllCond{
+			EdbInfoId: existItemB.FromEdbInfoId,
+		}, 0)
 		if err != nil {
 			return
 		}

models/edb_data_calculate_percentile.go

@@ -379,13 +379,10 @@ func (obj Percentile) getPercentileData(fromEdbInfo *EdbInfo, calculateValue int
 	dataList := make([]*EdbInfoSearchData, 0)
 	switch fromEdbInfo.EdbInfoType {
 	case 0:
-		var condition string
-		var pars []interface{}
-		condition += " AND edb_info_id=? "
-		pars = append(pars, fromEdbInfo.EdbInfoId)
-
 		//获取来源指标的数据
-		dataList, err = GetEdbDataListAll(condition, pars, fromEdbInfo.Source, fromEdbInfo.SubSource, 1)
+		dataList, err = GetEdbDataListAll(fromEdbInfo.Source, fromEdbInfo.SubSource, FindEdbDataListAllCond{
+			EdbInfoId: fromEdbInfo.EdbInfoId,
+		}, 1)
 	case 1:
 		dataList, err = GetPredictEdbDataListAllByStartDate(fromEdbInfo, 1, "")
 	default:

models/edb_data_calculate_rjz.go

@@ -209,22 +209,11 @@ func refreshAllCalculateRjz(to orm.TxOrmer, edbInfoId, source int, fromEdbInfo *
 	edbInfoIdStr := strconv.Itoa(edbInfoId)
 	//计算数据
 
-	//计算数据
-	var condition string
-	var pars []interface{}
-	condition += " AND edb_info_id=? "
-	pars = append(pars, fromEdbInfo.EdbInfoId)
-
-	if startDate != "" {
-		condition += " AND data_time>=? "
-		pars = append(pars, startDate)
-	}
-	//if endDate != "" {
-	//	condition += " AND data_time<=? "
-	//	pars = append(pars, endDate)
-	//}
-
-	dataList, err := GetEdbDataListAll(condition, pars, fromEdbInfo.Source, fromEdbInfo.SubSource, 0)
+	dataList, err := GetEdbDataListAll(fromEdbInfo.Source, fromEdbInfo.SubSource, FindEdbDataListAllCond{
+		EdbInfoId:         fromEdbInfo.EdbInfoId,
+		StartDataTime:     startDate,
+		StartDataTimeCond: ">=",
+	}, 0)
 	if err != nil {
 		return err
 	}

models/edb_data_calculate_standard_deviation.go

@@ -346,13 +346,10 @@ func (obj StandardDeviation) getStandardDeviationData(fromEdbInfo *EdbInfo, calc
 	dataList := make([]*EdbInfoSearchData, 0)
 	switch fromEdbInfo.EdbInfoType {
 	case 0:
-		var condition string
-		var pars []interface{}
-		condition += " AND edb_info_id=? "
-		pars = append(pars, fromEdbInfo.EdbInfoId)
-
 		//获取来源指标的数据
-		dataList, err = GetEdbDataListAll(condition, pars, fromEdbInfo.Source, fromEdbInfo.SubSource, 1)
+		dataList, err = GetEdbDataListAll(fromEdbInfo.Source, fromEdbInfo.SubSource, FindEdbDataListAllCond{
+			EdbInfoId: fromEdbInfo.EdbInfoId,
+		}, 1)
 	case 1:
 		dataList, err = GetPredictEdbDataListAllByStartDate(fromEdbInfo, 1, "")
 	default:

models/edb_data_calculate_sum.go

@@ -304,11 +304,9 @@ func (obj CalculateSum) refresh(to orm.TxOrmer, edbInfoId, source, subSource int
 	}
 
 	for edbInfoIndex, v := range edbInfoIdArr {
-		var condition string
-		var pars []interface{}
-		condition += " AND edb_info_id=? "
-		pars = append(pars, v.EdbInfoId)
-		dataList, err := GetEdbDataListAllByTo(to, condition, pars, v.Source, v.SubSource, 1)
+		dataList, err := GetEdbDataListAllByTo(to, v.Source, v.SubSource, FindEdbDataListAllCond{
+			EdbInfoId: v.EdbInfoId,
+		}, 1)
 		if err != nil {
 			return err
 		}

models/edb_data_calculate_tbz.go

@@ -209,22 +209,11 @@ func refreshAllCalculateTbz(to orm.TxOrmer, edbInfoId, source, subSource int, fr
 	edbInfoIdStr := strconv.Itoa(edbInfoId)
 	//计算数据
 
-	//计算数据
-	var condition string
-	var pars []interface{}
-	condition += " AND edb_info_id=? "
-	pars = append(pars, fromEdbInfo.EdbInfoId)
-
-	if startDate != "" {
-		condition += " AND data_time>=? "
-		pars = append(pars, startDate)
-	}
-	//if endDate != "" {
-	//	condition += " AND data_time<=? "
-	//	pars = append(pars, endDate)
-	//}
-
-	dataList, err := GetEdbDataListAll(condition, pars, fromEdbInfo.Source, fromEdbInfo.SubSource, 0)
+	dataList, err := GetEdbDataListAll(fromEdbInfo.Source, fromEdbInfo.SubSource, FindEdbDataListAllCond{
+		EdbInfoId:         fromEdbInfo.EdbInfoId,
+		StartDataTime:     startDate,
+		StartDataTimeCond: ">=",
+	}, 0)
 	if err != nil {
 		return err
 	}

models/edb_data_calculate_tcz.go

@@ -211,22 +211,11 @@ func refreshAllCalculateTcz(to orm.TxOrmer, edbInfoId, source, subSource int, fr
 	edbInfoIdStr := strconv.Itoa(edbInfoId)
 	//计算数据
 
-	//计算数据
-	var condition string
-	var pars []interface{}
-	condition += " AND edb_info_id=? "
-	pars = append(pars, fromEdbInfo.EdbInfoId)
-
-	if startDate != "" {
-		condition += " AND data_time>=? "
-		pars = append(pars, startDate)
-	}
-	//if endDate != "" {
-	//	condition += " AND data_time<=? "
-	//	pars = append(pars, endDate)
-	//}
-
-	dataList, err := GetEdbDataListAllByTo(to, condition, pars, fromEdbInfo.Source, fromEdbInfo.SubSource, 0)
+	dataList, err := GetEdbDataListAllByTo(to, fromEdbInfo.Source, fromEdbInfo.SubSource, FindEdbDataListAllCond{
+		EdbInfoId:         fromEdbInfo.EdbInfoId,
+		StartDataTime:     startDate,
+		StartDataTimeCond: ">=",
+	}, 0)
 	if err != nil {
 		return err
 	}

models/edb_data_calculate_time_shift.go

@@ -223,19 +223,6 @@ func refreshAllCalculateTimeShift(to orm.TxOrmer, edbInfoId, source, subSource,
 	edbInfoIdStr := strconv.Itoa(edbInfoId)
 
 	//计算数据
-	var condition string
-	var pars []interface{}
-	condition += " AND edb_info_id=? "
-	pars = append(pars, fromEdbInfo.EdbInfoId)
-
-	if startDate != "" {
-		condition += " AND data_time>=? "
-		pars = append(pars, startDate)
-	}
-	//if endDate != "" {
-	//	condition += " AND data_time<=? "
-	//	pars = append(pars, endDate)
-	//}
 
 	var shiftDay int
 	switch moveFrequency {
@@ -257,7 +244,11 @@ func refreshAllCalculateTimeShift(to orm.TxOrmer, edbInfoId, source, subSource,
 		shiftDay = -shiftDay
 	}
 
-	dataList, err := GetEdbDataListAllByTo(to, condition, pars, fromEdbInfo.Source, fromEdbInfo.SubSource, 0)
+	dataList, err := GetEdbDataListAllByTo(to, fromEdbInfo.Source, fromEdbInfo.SubSource, FindEdbDataListAllCond{
+		EdbInfoId:         fromEdbInfo.EdbInfoId,
+		StartDataTime:     startDate,
+		StartDataTimeCond: ">=",
+	}, 0)
 	if err != nil {
 		return err
 	}
@@ -278,9 +269,12 @@ func refreshAllCalculateTimeShift(to orm.TxOrmer, edbInfoId, source, subSource,
 	if err != nil {
 		return err
 	}
+
 	existDataMap := make(map[string]string)
+	removeDateMap := make(map[string]string)
 	for _, v := range existDataList {
 		existDataMap[v.DataTime] = v.Value
+		removeDateMap[v.DataTime] = ``
 	}
 	fmt.Println("existDataMap:", existDataMap)
 	addSql := ` INSERT INTO edb_data_calculate_time_shift(edb_info_id,edb_code,data_time,value,create_time,modify_time,data_timestamp) values `
@@ -314,6 +308,9 @@ func refreshAllCalculateTimeShift(to orm.TxOrmer, edbInfoId, source, subSource,
 					}
 				}
 			}
+
+			// 指标已存在，那么就移除该日期
+			delete(removeDateMap, newDate.Format(utils.FormatDate))
 		}
 		existMap[existKey] = currentItem.DataTime
 	}
@@ -325,5 +322,19 @@ func refreshAllCalculateTimeShift(to orm.TxOrmer, edbInfoId, source, subSource,
 			return err
 		}
 	}
+
+	// 移除不存在的日期数据
+	if len(removeDateMap) > 0 {
+		removeDateList := make([]string, 0) //需要移除的日期
+		for k := range removeDateMap {
+			removeDateList = append(removeDateList, k)
+		}
+		err = DelEdbDataByMysql(to, edbInfoId, dataTableName, removeDateList)
+		if err != nil {
+			err = fmt.Errorf("删除年化指标数据失败,Err:" + err.Error())
+			return
+		}
+	}
+
 	return
 }

models/edb_data_calculate_zjpj.go

@@ -395,14 +395,12 @@ func refreshAllCalculateZjpj(to orm.TxOrmer, edbInfo *EdbInfo, existItemA, exist
 	addDataList := make([]*EdbDataCalculateZjpj, 0)
 	//第一个指标
 	{
-		var condition string
-		var pars []interface{}
-
-		condition += " AND data_time < ? AND edb_info_id=? "
-		pars = append(pars, edbInfo.CalculateFormula, existItemA.FromEdbInfoId)
-
 		//第一个指标的数据列表
-		firstDataList, tmpErr := GetEdbDataListAllByTo(to, condition, pars, existItemA.FromSource, existItemA.FromSubSource, 0)
+		firstDataList, tmpErr := GetEdbDataListAllByTo(to, existItemA.FromSource, existItemA.FromSubSource, FindEdbDataListAllCond{
+			EdbInfoId:       existItemA.FromEdbInfoId,
+			EndDataTime:     edbInfo.CalculateFormula,
+			EndDataTimeCond: "<",
+		}, 0)
 		if tmpErr != nil {
 			return tmpErr
 		}
@@ -441,14 +439,12 @@ func refreshAllCalculateZjpj(to orm.TxOrmer, edbInfo *EdbInfo, existItemA, exist
 
 	//第二个指标
 	{
-		condition = ``
-		pars = make([]interface{}, 0)
-
-		condition += " AND data_time >= ? AND edb_info_id = ? "
-		pars = append(pars, edbInfo.CalculateFormula, existItemB.FromEdbInfoId)
-
 		//第二个指标的数据列表
-		secondDataList, tmpErr := GetEdbDataListAllByTo(to, condition, pars, existItemB.FromSource, existItemB.FromSubSource, 0)
+		secondDataList, tmpErr := GetEdbDataListAllByTo(to, existItemB.FromSource, existItemB.FromSubSource, FindEdbDataListAllCond{
+			EdbInfoId:         existItemB.FromEdbInfoId,
+			StartDataTime:     edbInfo.CalculateFormula,
+			StartDataTimeCond: ">=",
+		}, 0)
 		if tmpErr != nil {
 			return tmpErr
 		}

models/edb_data_calculate_zsxy.go

@@ -345,13 +345,11 @@ func CalculateExponentialSmoothingData(fromEdbInfo *EdbInfo, strAlpha string) (n
 	dataList := make([]*EdbInfoSearchData, 0)
 	switch fromEdbInfo.EdbInfoType {
 	case 0:
-		var condition string
-		var pars []interface{}
-		condition += " AND edb_info_id=? "
-		pars = append(pars, fromEdbInfo.EdbInfoId)
-
 		//获取来源指标的数据
-		dataList, err = GetEdbDataListAll(condition, pars, fromEdbInfo.Source, fromEdbInfo.SubSource, 1)
+		dataList, err = GetEdbDataListAll(fromEdbInfo.Source, fromEdbInfo.SubSource,
+			FindEdbDataListAllCond{
+				EdbInfoId: fromEdbInfo.EdbInfoId,
+			}, 1)
 	case 1:
 		dataList, err = GetPredictEdbDataListAllByStartDate(fromEdbInfo, 1, "")
 	default:

models/edb_data_table.go

@@ -152,7 +152,7 @@ func GetEdbDataTableName(source, subSource int) (tableName string) {
 		tableName = "edb_data_predict_ccalculate_standard_deviation"
 	case utils.DATA_SOURCE_PREDICT_CALCULATE_PERCENTILE: //预测百分位->70
 		tableName = "edb_data_predict_ccalculate_percentile"
-	case utils.DATA_SOURCE_FUBAO:
+	case utils.DATA_SOURCE_FUBAO: //数宝->71
 		tableName = "edb_data_fubao"
 	case utils.DATA_SOURCE_CALCULATE_ZSXY:
 		tableName = "edb_data_calculate_zsxy" // 指数修匀->72

models/edb_info.go

@@ -3,11 +3,14 @@ package models
 import (
 	"encoding/json"
 	"errors"
+	"eta/eta_index_lib/models/mgo"
 	"eta/eta_index_lib/utils"
 	"fmt"
 	"strconv"
 	"time"
 
+	"go.mongodb.org/mongo-driver/bson"
+
 	"github.com/beego/beego/v2/client/orm"
 	"github.com/shopspring/decimal"
 )
@@ -182,20 +185,77 @@ func (edbInfo *EdbInfo) Update(cols []string) (err error) {
 
 // EdbInfoSearchData
 type EdbInfoSearchData struct {
-	EdbDataId int     `description:"数据ID"`
-	DataTime  string  `description:"数据日期"`
-	Value     float64 `description:"数据"`
-	EdbCode   string  `description:"指标编码"`
+	EdbDataId     int     `description:"数据ID"`
+	EdbInfoId     int     `description:"指标ID"`
+	DataTime      string  `description:"数据日期"`
+	Value         float64 `description:"数据"`
+	EdbCode       string  `description:"指标编码"`
+	DataTimestamp int64   `description:"时间戳"`
 }
 
-// GetEdbDataListAll 获取指标数据列表 order:1升序，其余值为降序
-func GetEdbDataListAll(condition string, pars []interface{}, source, subSource, order int) (item []*EdbInfoSearchData, err error) {
-	o := orm.NewOrm()
+type FindEdbDataListAllCond struct {
+	EdbInfoId         int
+	StartDataTime     string
+	StartDataTimeCond string
+	EndDataTime       string
+	EndDataTimeCond   string
+}
+
+// GetEdbDataListAll
+// @Description: 获取指标数据列表 order:1升序，其余值为降序
+// @author: Roc
+// @datetime 2024-05-08 15:34:01
+// @param source int
+// @param subSource int
+// @param findEdbDataListAllCond FindEdbDataListAllCond
+// @param order int  order:1升序，其余值为降序
+// @return item []*EdbInfoSearchData
+// @return err error
+func GetEdbDataListAll(source, subSource int, findEdbDataListAllCond FindEdbDataListAllCond, order int) (item []*EdbInfoSearchData, err error) {
+	if source == utils.DATA_SOURCE_BUSINESS {
+		return GetEdbDataListAllByMongo(source, subSource, findEdbDataListAllCond, order)
+	}
+
+	return GetEdbDataListAllByMysql(source, subSource, findEdbDataListAllCond, order)
+}
+
+// GetEdbDataListAllByMysql
+// @Description: 从mysql数据库中获取指标数据列表 order:1升序，其余值为降序
+// @author: Roc
+// @datetime 2024-05-08 15:32:55
+// @param source int
+// @param subSource int
+// @param findEdbDataListAllCond FindEdbDataListAllCond
+// @param order int
+// @return item []*EdbInfoSearchData
+// @return err error
+func GetEdbDataListAllByMysql(source, subSource int, findEdbDataListAllCond FindEdbDataListAllCond, order int) (item []*EdbInfoSearchData, err error) {
+	if findEdbDataListAllCond.EdbInfoId <= 0 {
+		return
+	}
+	to := orm.NewOrm()
+
 	sql := ``
 	tableName := GetEdbDataTableName(source, subSource)
 	sql = ` SELECT * FROM %s WHERE 1=1 `
 	sql = fmt.Sprintf(sql, tableName)
 
+	var condition string
+	var pars []interface{}
+	condition += " AND edb_info_id=? "
+	pars = append(pars, findEdbDataListAllCond.EdbInfoId)
+
+	// 开始日期
+	if findEdbDataListAllCond.StartDataTime != "" && findEdbDataListAllCond.StartDataTimeCond != `` {
+		condition += fmt.Sprintf(" AND data_time %s ? ", findEdbDataListAllCond.StartDataTimeCond)
+		pars = append(pars, findEdbDataListAllCond.StartDataTime)
+	}
+	// 结束日期
+	if findEdbDataListAllCond.EndDataTime != "" && findEdbDataListAllCond.EndDataTimeCond != `` {
+		condition += fmt.Sprintf(" AND data_time %s ? ", findEdbDataListAllCond.EndDataTimeCond)
+		pars = append(pars, findEdbDataListAllCond.EndDataTime)
+	}
+
 	if condition != "" {
 		sql += condition
 	}
@@ -204,17 +264,65 @@ func GetEdbDataListAll(condition string, pars []interface{}, source, subSource,
 	} else {
 		sql += ` ORDER BY data_time DESC `
 	}
-	_, err = o.Raw(sql, pars).QueryRows(&item)
+	_, err = to.Raw(sql, pars).QueryRows(&item)
 	return
 }
 
-// GetEdbDataListAllByTo 根据事务链接获取指标数据列表 order:1升序，其余值为降序
-func GetEdbDataListAllByTo(to orm.TxOrmer, condition string, pars []interface{}, source, subSource, order int) (item []*EdbInfoSearchData, err error) {
+// GetEdbDataListAllByTo
+// @Description: 根据事务链接获取指标数据列表 order:1升序，其余值为降序
+// @author: Roc
+// @datetime 2024-05-08 15:34:06
+// @param to orm.TxOrmer
+// @param source int
+// @param subSource int
+// @param findEdbDataListAllCond FindEdbDataListAllCond
+// @param order int
+// @return item []*EdbInfoSearchData
+// @return err error
+func GetEdbDataListAllByTo(to orm.TxOrmer, source, subSource int, findEdbDataListAllCond FindEdbDataListAllCond, order int) (item []*EdbInfoSearchData, err error) {
+	if source == utils.DATA_SOURCE_BUSINESS {
+		return GetEdbDataListAllByMongo(source, subSource, findEdbDataListAllCond, order)
+	}
+
+	return GetEdbDataListAllByMysqlTo(to, source, subSource, findEdbDataListAllCond, order)
+}
+
+// GetEdbDataListAllByMysqlTo
+// @Description: 根据事务链接获取指标数据列表 order:1升序，其余值为降序（Mysql）
+// @author: Roc
+// @datetime 2024-05-08 15:34:13
+// @param to orm.TxOrmer
+// @param source int
+// @param subSource int
+// @param findEdbDataListAllCond FindEdbDataListAllCond
+// @param order int
+// @return item []*EdbInfoSearchData
+// @return err error
+func GetEdbDataListAllByMysqlTo(to orm.TxOrmer, source, subSource int, findEdbDataListAllCond FindEdbDataListAllCond, order int) (item []*EdbInfoSearchData, err error) {
+	if findEdbDataListAllCond.EdbInfoId <= 0 {
+		return
+	}
 	sql := ``
 	tableName := GetEdbDataTableName(source, subSource)
 	sql = ` SELECT * FROM %s WHERE 1=1 `
 	sql = fmt.Sprintf(sql, tableName)
 
+	var condition string
+	var pars []interface{}
+	condition += " AND edb_info_id=? "
+	pars = append(pars, findEdbDataListAllCond.EdbInfoId)
+
+	// 开始日期
+	if findEdbDataListAllCond.StartDataTime != "" && findEdbDataListAllCond.StartDataTimeCond != `` {
+		condition += fmt.Sprintf(" AND data_time %s ? ", findEdbDataListAllCond.StartDataTimeCond)
+		pars = append(pars, findEdbDataListAllCond.StartDataTime)
+	}
+	// 结束日期
+	if findEdbDataListAllCond.EndDataTime != "" && findEdbDataListAllCond.EndDataTimeCond != `` {
+		condition += fmt.Sprintf(" AND data_time %s ? ", findEdbDataListAllCond.EndDataTimeCond)
+		pars = append(pars, findEdbDataListAllCond.EndDataTime)
+	}
+
 	if condition != "" {
 		sql += condition
 	}
@@ -227,15 +335,89 @@ func GetEdbDataListAllByTo(to orm.TxOrmer, condition string, pars []interface{},
 	return
 }
 
+// GetEdbDataListAllByMongo
+// @Description: 根据事务链接获取指标数据列表 order:1升序，其余值为降序（Mongo）
+// @author: Roc
+// @datetime 2024-05-08 15:34:24
+// @param source int
+// @param subSource int
+// @param findEdbDataListAllCond FindEdbDataListAllCond
+// @param order int
+// @return dataList []*EdbInfoSearchData
+// @return err error
+func GetEdbDataListAllByMongo(source, subSource int, findEdbDataListAllCond FindEdbDataListAllCond, order int) (dataList []*EdbInfoSearchData, err error) {
+	dataList = make([]*EdbInfoSearchData, 0)
+	if findEdbDataListAllCond.EdbInfoId <= 0 {
+		return
+	}
+	mogDataObj := mgo.EdbDataBusiness{}
+	// 构建查询条件
+	queryConditions := bson.M{
+		"edb_info_id": findEdbDataListAllCond.EdbInfoId,
+	}
+	// 开始日期
+	if findEdbDataListAllCond.StartDataTime != "" {
+		startDateTime, tmpErr := time.ParseInLocation(utils.FormatDate, findEdbDataListAllCond.StartDataTime, time.Local)
+		if tmpErr != nil {
+			err = tmpErr
+			return
+		}
+
+		// 日期比较符
+		cond := utils.DateConvMysqlConvMongo(findEdbDataListAllCond.StartDataTimeCond)
+		if cond != `` {
+			queryConditions["data_time"] = bson.M{cond: startDateTime}
+		}
+	}
+	// 结束日期
+	if findEdbDataListAllCond.EndDataTime != "" {
+		endDateTime, tmpErr := time.ParseInLocation(utils.FormatDate, findEdbDataListAllCond.EndDataTime, time.Local)
+		if tmpErr != nil {
+			err = tmpErr
+			return
+		}
+
+		// 日期比较符
+		cond := utils.DateConvMysqlConvMongo(findEdbDataListAllCond.EndDataTimeCond)
+		if cond != `` {
+			queryConditions["data_time"] = bson.M{cond: endDateTime}
+		}
+	}
+
+	sortList := make([]string, 0)
+	if order == 1 {
+		sortList = append(sortList, "-data_time")
+	} else {
+		sortList = append(sortList, "data_time")
+	}
+	// 获取列表数据
+	tmpDataList, tmpErr := mogDataObj.GetAllDataList(queryConditions, sortList)
+	if tmpErr != nil {
+		err = tmpErr
+		return
+	}
+	for k, v := range tmpDataList {
+		dataList = append(dataList, &EdbInfoSearchData{
+			EdbDataId:     k + 1,
+			EdbInfoId:     v.EdbInfoId,
+			EdbCode:       v.EdbCode,
+			DataTime:      v.DataTime.Format(utils.FormatDate),
+			Value:         v.Value,
+			DataTimestamp: v.DataTimestamp,
+		})
+	}
+	return
+}
+
 // EdbInfoMaxAndMinInfo 指标最新数据记录结构体
 type EdbInfoMaxAndMinInfo struct {
-	MinDate     string  `description:"最小日期"`
-	MaxDate     string  `description:"最大日期"`
-	MinValue    float64 `description:"最小值"`
-	MaxValue    float64 `description:"最大值"`
-	LatestValue float64 `description:"最新值"`
-	LatestDate  string  `description:"实际数据最新日期"`
-	EndValue    float64 `description:"最新值"`
+	MinDate     string  `description:"最小日期" bson:"min_date"`
+	MaxDate     string  `description:"最大日期" bson:"max_date"`
+	MinValue    float64 `description:"最小值" bson:"min_value"`
+	MaxValue    float64 `description:"最大值" bson:"max_value"`
+	LatestValue float64 `description:"最新值" bson:"latest_value"`
+	LatestDate  string  `description:"实际数据最新日期" bson:"latest_date"`
+	EndValue    float64 `description:"最新值" bson:"end_value"`
 }
 
 // GetEdbInfoMaxAndMinInfo 获取指标的最新数据记录信息
@@ -906,29 +1088,29 @@ func GetPredictDataListByPredictEdbInfo(edbInfo *EdbInfo, order int, startDate s
 
 // GetPredictDataListByPredictEdbConfList 根据预测指标信息获取预测指标的数据，order:1升序，其余值为降序
 func GetPredictDataListByPredictEdbConfList(edbInfo, sourceEdbInfoItem *EdbInfo, predictEdbConfList []*PredictEdbConfAndData, order int, startDate string) (dataList []*EdbInfoSearchData, err error, errMsg string) {
-
 	allDataList := make([]*EdbInfoSearchData, 0)
 	//获取指标数据（实际已生成）
-	var condition string
-	var pars []interface{}
-	condition += " AND edb_info_id=? "
-	pars = append(pars, sourceEdbInfoItem.EdbInfoId)
-	if startDate != "" {
-		condition += " AND data_time>=? "
-		pars = append(pars, startDate)
-	}
-	tmpDataList, err := GetEdbDataListAll(condition, pars, sourceEdbInfoItem.Source, sourceEdbInfoItem.SubSource, 1)
+	tmpDataList, err := GetEdbDataListAll(sourceEdbInfoItem.Source, sourceEdbInfoItem.SubSource, FindEdbDataListAllCond{
+		EdbInfoId:         sourceEdbInfoItem.EdbInfoId,
+		StartDataTime:     startDate,
+		StartDataTimeCond: ">=",
+	}, 1)
 	if err != nil {
 		return
 	}
-	// 如果选择了日期，那么需要筛选所有的数据，用于未来指标的生成
+
+	// 默认认为上一个查询是把所有的数据查出来了
+	allDataList = tmpDataList
+
+	// 如果条件中有选择了日期，那么上次查询实际并不是把所有的数据查出来了
+	// 那么需要把日期给过滤掉，然后筛选所有的数据，用于未来指标的生成
 	if startDate != `` {
-		allDataList, err = GetEdbDataListAll(" AND edb_info_id=? ", []interface{}{sourceEdbInfoItem.EdbInfoId}, sourceEdbInfoItem.Source, sourceEdbInfoItem.SubSource, 1)
+		allDataList, err = GetEdbDataListAll(sourceEdbInfoItem.Source, sourceEdbInfoItem.SubSource, FindEdbDataListAllCond{
+			EdbInfoId: sourceEdbInfoItem.EdbInfoId,
+		}, 1)
 		if err != nil {
 			return
 		}
-	} else {
-		allDataList = tmpDataList
 	}
 
 	// 获取预测指标未来的数据
@@ -982,11 +1164,9 @@ func GetPredictEdbDataListAll(edbInfo *EdbInfo, order int) (items []*EdbInfoSear
 	if edbInfo.Source == utils.DATA_SOURCE_PREDICT { //普通的预测指标是没有入库数据的，直接往配置里面获取
 		items, _, err, _ = GetPredictDataListByPredictEdbInfo(edbInfo, 1, "")
 	} else {
-		var condition string
-		var pars []interface{}
-		condition += " AND edb_info_id=? "
-		pars = append(pars, edbInfo.EdbInfoId)
-		items, err = GetEdbDataListAll(condition, pars, edbInfo.Source, edbInfo.SubSource, order)
+		items, err = GetEdbDataListAll(edbInfo.Source, edbInfo.SubSource, FindEdbDataListAllCond{
+			EdbInfoId: edbInfo.EdbInfoId,
+		}, order)
 	}
 	return
 }
@@ -996,15 +1176,11 @@ func GetPredictEdbDataListAllByStartDate(edbInfo *EdbInfo, order int, startDate
 	if edbInfo.Source == utils.DATA_SOURCE_PREDICT { //普通的预测指标是没有入库数据的，直接往配置里面获取
 		items, _, err, _ = GetPredictDataListByPredictEdbInfo(edbInfo, order, startDate)
 	} else {
-		var condition string
-		var pars []interface{}
-		condition += " AND edb_info_id=? "
-		pars = append(pars, edbInfo.EdbInfoId)
-		if startDate != "" {
-			condition += " AND data_time>=? "
-			pars = append(pars, startDate)
-		}
-		items, err = GetEdbDataListAll(condition, pars, edbInfo.Source, edbInfo.SubSource, order)
+		items, err = GetEdbDataListAll(edbInfo.Source, edbInfo.SubSource, FindEdbDataListAllCond{
+			EdbInfoId:         edbInfo.EdbInfoId,
+			StartDataTime:     startDate,
+			StartDataTimeCond: ">=",
+		}, order)
 	}
 	return
 }

models/future_good/future_good_edb_data.go

@@ -1,6 +1,7 @@
 package future_good
 
 import (
+	"eta/eta_index_lib/services/alarm_msg"
 	"eta/eta_index_lib/utils"
 	"fmt"
 	"github.com/beego/beego/v2/client/orm"
@@ -84,9 +85,16 @@ func GetFutureGoodEdbDataList(condition string, pars []interface{}) (list []*Fut
 	return
 }
 
-// AddEdbDataFromThs 添加Ths商品指标数据
+// AddEdbDataFromThs 添加同花顺商品指标数据
 func AddEdbDataFromThs(futureGoodEdbInfoId int, edbCode string, item FutureGoodDataFromThs) (err error) {
+	var errMsg string
 	o := orm.NewOrm()
+	defer func() {
+		if err != nil {
+			//go utils.SendEmail(utils.APP_NAME_CN+"【"+utils.RunMode+"】"+"失败提醒", " 同花顺数据获取失败:err:"+errMsg, utils.EmailSendToUsers)
+			go alarm_msg.SendAlarmMsg("wind商品数据获取失败:err:"+errMsg, 3)
+		}
+	}()
 
 	var isAdd bool
 	addSql := ` INSERT INTO future_good_edb_data(future_good_edb_info_id,future_good_edb_code,data_time,trade_code,open,high,low,close,volume,amt,oi,settle,create_time,modify_time,data_timestamp) values `
@@ -97,6 +105,7 @@ func AddEdbDataFromThs(futureGoodEdbInfoId int, edbCode string, item FutureGoodD
 		eDate := table.Time[k]
 		dataTime, err := time.ParseInLocation(utils.FormatDate, eDate, time.Local)
 		if err != nil {
+			errMsg = " time.Parse :" + err.Error()
 			return err
 		}
 		timestamp := dataTime.UnixNano() / 1e6
@@ -121,6 +130,7 @@ func AddEdbDataFromThs(futureGoodEdbInfoId int, edbCode string, item FutureGoodD
 		addSql = strings.TrimRight(addSql, ",")
 		_, err = o.Raw(addSql).Exec()
 		if err != nil {
+			errMsg = " tx.Exec Err :" + err.Error()
 			return
 		}
 	}

models/mgo/base_from_business_data.go

@@ -0,0 +1,419 @@
+package mgo
+
+import (
+	"context"
+	"errors"
+	"eta/eta_index_lib/utils"
+	"fmt"
+	"github.com/qiniu/qmgo"
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+	"time"
+)
+
+// BaseFromBusinessData
+// @Description: 自有数据集合
+type BaseFromBusinessData struct {
+	ID primitive.ObjectID `json:"_id" bson:"_id,omitempty"` // 文档id
+	//ID string `json:"_id" bson:"_id" ` // 文档id
+	BaseFromBusinessIndexId int64     `json:"base_from_business_index_id" bson:"base_from_business_index_id"` // 指标id
+	IndexCode               string    `json:"index_code" bson:"index_code"`                                   // 指标编码
+	DataTime                time.Time `json:"data_time" bson:"data_time"`                                     // 数据日期
+	Value                   float64   `json:"value" bson:"value"`                                             // 数据值
+	CreateTime              time.Time `json:"create_time" bson:"create_time"`                                 // 创建时间
+	ModifyTime              time.Time `json:"modify_time" bson:"modify_time"`                                 // 修改时间
+}
+
+// CollectionName
+// @Description:  获取集合名称
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-26 13:41:36
+// @return string
+func (m *BaseFromBusinessData) CollectionName() string {
+	return "base_from_business_data"
+}
+
+// DataBaseName
+// @Description: 获取数据库名称
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-26 13:41:33
+// @return string
+func (m *BaseFromBusinessData) DataBaseName() string {
+	return utils.MgoDataDbName
+}
+
+// GetCollection
+// @Description: 获取mongodb集合的句柄
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-26 13:41:33
+// @return string
+func (m *BaseFromBusinessData) GetCollection() *qmgo.Collection {
+	db := utils.MgoDataCli.Database(m.DataBaseName())
+	return db.Collection(m.CollectionName())
+}
+
+// GetAllDataList
+// @Description: 根据条件获取所有数据
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-26 13:42:19
+// @param sort []string
+// @param whereParams interface{}
+// @return result []BaseFromBusinessData
+// @return err error
+func (m *BaseFromBusinessData) GetAllDataList(whereParams interface{}, sort []string) (result []*BaseFromBusinessData, err error) {
+	if utils.MgoDataCli == nil {
+		err = errors.New("mongodb连接失败")
+		return
+	}
+	db := utils.MgoDataCli.Database(m.DataBaseName())
+	coll := db.Collection(m.CollectionName())
+	ctx := context.TODO()
+	if err != nil {
+		fmt.Println("MgoGetColl Err:", err.Error())
+		return
+	}
+	err = coll.Find(ctx, whereParams).Sort(sort...).All(&result)
+	if err != nil {
+		return
+	}
+
+	for _, v := range result {
+		v.DataTime = v.DataTime.In(time.Local)
+		v.CreateTime = v.CreateTime.In(time.Local)
+		v.ModifyTime = v.ModifyTime.In(time.Local)
+	}
+
+	return
+}
+
+// GetLimitDataList
+// @Description: 根据条件获取指定数量数据列表
+// @author: Roc
+// @receiver m
+// @datetime 2024-05-06 17:08:32
+// @param whereParams interface{}
+// @param size int64
+// @return result []*BaseFromBusinessData
+// @return err error
+func (m *BaseFromBusinessData) GetLimitDataList(whereParams interface{}, size int64, sort []string) (result []*BaseFromBusinessData, err error) {
+	if utils.MgoDataCli == nil {
+		err = errors.New("mongodb连接失败")
+		return
+	}
+	db := utils.MgoDataCli.Database(m.DataBaseName())
+	coll := db.Collection(m.CollectionName())
+	ctx := context.TODO()
+	if err != nil {
+		fmt.Println("MgoGetColl Err:", err.Error())
+		return
+	}
+	err = coll.Find(ctx, whereParams).Sort(sort...).Limit(size).All(&result)
+	if err != nil {
+		return
+	}
+
+	for _, v := range result {
+		v.DataTime = v.DataTime.In(time.Local)
+		v.CreateTime = v.CreateTime.In(time.Local)
+		v.ModifyTime = v.ModifyTime.In(time.Local)
+	}
+
+	return
+}
+
+// GetPageDataList
+// @Description: 根据条件获取分页数据列表
+// @author: Roc
+// @receiver m
+// @datetime 2024-05-07 10:21:07
+// @param whereParams interface{}
+// @param startSize int64
+// @param size int64
+// @param sort []string
+// @return result []*BaseFromBusinessData
+// @return err error
+func (m *BaseFromBusinessData) GetPageDataList(whereParams interface{}, startSize, size int64, sort []string) (result []*BaseFromBusinessData, err error) {
+	if utils.MgoDataCli == nil {
+		err = errors.New("mongodb连接失败")
+		return
+	}
+	db := utils.MgoDataCli.Database(m.DataBaseName())
+	coll := db.Collection(m.CollectionName())
+	ctx := context.TODO()
+	if err != nil {
+		fmt.Println("MgoGetColl Err:", err.Error())
+		return
+	}
+	err = coll.Find(ctx, whereParams).Sort(sort...).Skip(startSize).Limit(size).All(&result)
+	if err != nil {
+		return
+	}
+
+	for _, v := range result {
+		v.DataTime = v.DataTime.In(time.Local)
+		v.CreateTime = v.CreateTime.In(time.Local)
+		v.ModifyTime = v.ModifyTime.In(time.Local)
+	}
+
+	return
+}
+
+// GetCountDataList
+// @Description:  根据条件获取数据列表总数
+// @author: Roc
+// @receiver m
+// @datetime 2024-05-07 10:29:00
+// @param whereParams interface{}
+// @return count int64
+// @return err error
+func (m *BaseFromBusinessData) GetCountDataList(whereParams interface{}) (count int64, err error) {
+	if utils.MgoDataCli == nil {
+		err = errors.New("mongodb连接失败")
+		return
+	}
+	db := utils.MgoDataCli.Database(m.DataBaseName())
+	coll := db.Collection(m.CollectionName())
+	ctx := context.TODO()
+	if err != nil {
+		fmt.Println("MgoGetColl Err:", err.Error())
+		return
+	}
+	count, err = coll.Find(ctx, whereParams).Count()
+
+	return
+}
+
+// InsertDataByColl
+// @Description: 写入单条数据（外部传入集合）
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-26 14:22:18
+// @param addData interface{}
+// @return err error
+func (m *BaseFromBusinessData) InsertDataByColl(coll *qmgo.Collection, addData interface{}) (err error) {
+	ctx := context.TODO()
+	_, err = coll.InsertOne(ctx, addData)
+	if err != nil {
+		fmt.Println("InsertDataByColl:Err:" + err.Error())
+		return
+	}
+
+	return
+}
+
+// BatchInsertData
+// @Description: 批量写入数据
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-26 14:22:18
+// @param bulk int 每次请求保存的数据量
+// @param dataList []interface{}
+// @return err error
+func (m *BaseFromBusinessData) BatchInsertData(bulk int, dataList []interface{}) (err error) {
+	db := utils.MgoDataCli.Database(m.DataBaseName())
+	coll := db.Collection(m.CollectionName())
+
+	return m.BatchInsertDataByColl(coll, bulk, dataList)
+}
+
+// BatchInsertDataByColl
+// @Description: 批量写入数据（外部传入集合）
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-26 14:22:18
+// @param coll *qmgo.Collection
+// @param bulk int 每次请求保存的数据量
+// @param dataList []interface{}
+// @return err error
+func (m *BaseFromBusinessData) BatchInsertDataByColl(coll *qmgo.Collection, bulk int, dataList []interface{}) (err error) {
+	ctx := context.TODO()
+	dataNum := len(dataList)
+	if dataNum <= 0 {
+		return
+	}
+
+	// 不设置每次保存切片数量大小，或者实际数据量小于设置的切片数量大小，那么就直接保存吧
+	if bulk <= 0 || dataNum <= bulk {
+		_, err = coll.InsertMany(ctx, dataList)
+		if err != nil {
+			fmt.Println("BatchInsertData:Err:" + err.Error())
+			return
+		}
+		return
+	}
+
+	// 分批保存
+	i := 0
+	tmpAddDataList := make([]interface{}, 0)
+	for _, v := range dataList {
+		tmpAddDataList = append(tmpAddDataList, v)
+		i++
+		if i >= bulk {
+			_, err = coll.InsertMany(ctx, tmpAddDataList)
+			if err != nil {
+				fmt.Println("BatchInsertData:Err:" + err.Error())
+				return
+			}
+			i = 0
+			tmpAddDataList = make([]interface{}, 0)
+		}
+	}
+
+	if len(tmpAddDataList) > 0 {
+		_, err = coll.InsertMany(ctx, tmpAddDataList)
+		if err != nil {
+			fmt.Println("BatchInsertData:Err:" + err.Error())
+			return
+		}
+	}
+
+	return
+}
+
+// UpdateDataByColl
+// @Description: 单条数据修改
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-26 15:01:51
+// @param whereParams interface{}
+// @param updateParams interface{}
+// @return err error
+func (m *BaseFromBusinessData) UpdateDataByColl(coll *qmgo.Collection, whereParams, updateParams interface{}) (err error) {
+	ctx := context.TODO()
+	err = coll.UpdateOne(ctx, whereParams, updateParams)
+	if err != nil {
+		fmt.Println("UpdateDataByColl:Err:" + err.Error())
+		return
+	}
+	return
+}
+
+// UpdateData
+// @Description: 单条数据修改
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-26 15:01:51
+// @param whereParams interface{}
+// @param updateParams interface{}
+// @return err error
+func (m *BaseFromBusinessData) UpdateData(whereParams, updateParams interface{}) (err error) {
+	db := utils.MgoDataCli.Database(m.DataBaseName())
+	coll := db.Collection(m.CollectionName())
+	ctx := context.TODO()
+	err = coll.UpdateOne(ctx, whereParams, updateParams)
+	if err != nil {
+		fmt.Println("UpdateData:Err:" + err.Error())
+		return
+	}
+	return
+}
+
+// HandleData
+// @Description: 事务处理数据
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-30 10:40:20
+// @param addDataList []BaseAddFromBusinessData
+// @param updateDataList []BaseFromBusinessData
+// @return result interface{}
+// @return err error
+func (m *BaseFromBusinessData) HandleData(addDataList, updateDataList []BaseFromBusinessData) (result interface{}, err error) {
+
+	ctx := context.TODO()
+
+	callback := func(sessCtx context.Context) (interface{}, error) {
+		// 重要：确保事务中的每一个操作，都使用传入的sessCtx参数
+
+		db := utils.MgoDataCli.Database(m.DataBaseName())
+		coll := db.Collection(m.CollectionName())
+
+		// 插入数据
+		if len(addDataList) > 0 {
+			_, err = coll.InsertMany(sessCtx, addDataList)
+			if err != nil {
+				return nil, err
+			}
+		}
+
+		// 修改
+
+		if len(updateDataList) > 0 {
+			for _, v := range updateDataList {
+				err = coll.UpdateOne(ctx, bson.M{"_id": v.ID}, bson.M{"$set": bson.M{"value": v.Value, "modify_time": v.ModifyTime}})
+				if err != nil {
+					fmt.Println("BatchInsertData:Err:" + err.Error())
+					return nil, err
+				}
+			}
+		}
+
+		return nil, nil
+	}
+	result, err = utils.MgoDataCli.DoTransaction(ctx, callback)
+
+	return
+}
+
+// GetEdbInfoMaxAndMinInfo
+// @Description: 获取当前指标的最大最小值
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-30 17:15:39
+// @param whereParams interface{}
+// @return result EdbInfoMaxAndMinInfo
+// @return err error
+func (m *BaseFromBusinessData) GetEdbInfoMaxAndMinInfo(whereParams interface{}) (result EdbInfoMaxAndMinInfo, err error) {
+	if utils.MgoDataCli == nil {
+		err = errors.New("mongodb连接失败")
+		return
+	}
+	db := utils.MgoDataCli.Database(m.DataBaseName())
+	coll := db.Collection(m.CollectionName())
+	ctx := context.TODO()
+	if err != nil {
+		fmt.Println("MgoGetColl Err:", err.Error())
+		return
+	}
+	err = coll.Aggregate(ctx, whereParams).One(&result)
+	if err != nil {
+		return
+	}
+	result.MinDate = result.MinDate.In(time.Local)
+	result.MaxDate = result.MaxDate.In(time.Local)
+	result.LatestDate = result.LatestDate.In(time.Local)
+
+	return
+}
+
+// GetLatestValue
+// @Description: 获取当前指标的最新数据记录
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-30 17:16:15
+// @param whereParams interface{}
+// @param selectParam interface{}
+// @return latestValue LatestValue
+// @return err error
+func (m *BaseFromBusinessData) GetLatestValue(whereParams, selectParam interface{}) (latestValue LatestValue, err error) {
+	if utils.MgoDataCli == nil {
+		err = errors.New("mongodb连接失败")
+		return
+	}
+	db := utils.MgoDataCli.Database(m.DataBaseName())
+	coll := db.Collection(m.CollectionName())
+	ctx := context.TODO()
+	if err != nil {
+		fmt.Println("MgoGetColl Err:", err.Error())
+		return
+	}
+
+	//var result interface{}
+	//err = coll.Find(ctx, whereParams).Select(selectParam).One(&result)
+	err = coll.Find(ctx, whereParams).Select(selectParam).One(&latestValue)
+
+	return
+}

models/mgo/edb_data_business.go

@@ -0,0 +1,511 @@
+package mgo
+
+import (
+	"context"
+	"errors"
+	"eta/eta_index_lib/utils"
+	"fmt"
+	"github.com/qiniu/qmgo"
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+	"time"
+)
+
+// EdbDataBusiness
+// @Description: 自有数据集合（指标库）
+type EdbDataBusiness struct {
+	ID            primitive.ObjectID `json:"_id" bson:"_id,omitempty" `            // 文档id
+	EdbInfoId     int                `json:"edb_info_id" bson:"edb_info_id"`       // 指标编码
+	EdbCode       string             `json:"edb_code" bson:"edb_code"`             // 指标编码
+	DataTime      time.Time          `json:"data_time" bson:"data_time"`           // 数据日期
+	Value         float64            `json:"value" bson:"value"`                   // 数据值
+	CreateTime    time.Time          `json:"create_time" bson:"create_time"`       // 创建时间
+	ModifyTime    time.Time          `json:"modify_time" bson:"modify_time"`       // 修改时间
+	DataTimestamp int64              `json:"data_timestamp" bson:"data_timestamp"` // 数据日期时间戳
+}
+
+// CollectionName
+// @Description:  获取集合名称
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-26 13:41:36
+// @return string
+func (m *EdbDataBusiness) CollectionName() string {
+	return "edb_data_business"
+}
+
+// DataBaseName
+// @Description: 获取数据库名称
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-26 13:41:33
+// @return string
+func (m *EdbDataBusiness) DataBaseName() string {
+	return utils.MgoDataDbName
+}
+
+// GetCollection
+// @Description: 获取mongodb集合的句柄
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-26 13:41:33
+// @return string
+func (m *EdbDataBusiness) GetCollection() *qmgo.Collection {
+	db := utils.MgoDataCli.Database(m.DataBaseName())
+	return db.Collection(m.CollectionName())
+}
+
+// GetItem
+// @Description: 根据条件获取单条数据
+// @author: Roc
+// @receiver m
+// @datetime 2024-05-09 10:00:49
+// @param whereParams interface{}
+// @return item *EdbDataBusiness
+// @return err error
+func (m *EdbDataBusiness) GetItem(whereParams interface{}) (item *EdbDataBusiness, err error) {
+	if utils.MgoDataCli == nil {
+		err = errors.New("mongodb连接失败")
+		return
+	}
+	db := utils.MgoDataCli.Database(m.DataBaseName())
+	coll := db.Collection(m.CollectionName())
+
+	return m.GetItemByColl(coll, whereParams)
+}
+
+// GetItemByColl
+// @Description: 根据条件获取单条数据
+// @author: Roc
+// @receiver m
+// @datetime 2024-05-09 13:22:06
+// @param coll *qmgo.Collection
+// @param whereParams interface{}
+// @return item *EdbDataBusiness
+// @return err error
+func (m *EdbDataBusiness) GetItemByColl(coll *qmgo.Collection, whereParams interface{}) (item *EdbDataBusiness, err error) {
+	ctx := context.TODO()
+	if err != nil {
+		fmt.Println("MgoGetColl Err:", err.Error())
+		return
+	}
+	err = coll.Find(ctx, whereParams).One(&item)
+	if err != nil {
+		return
+	}
+
+	item.DataTime = item.DataTime.In(time.Local)
+	item.CreateTime = item.CreateTime.In(time.Local)
+	item.ModifyTime = item.ModifyTime.In(time.Local)
+
+	return
+}
+
+// GetAllDataList
+// @Description: 根据条件获取所有数据
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-26 13:42:19
+// @param whereParams interface{}
+// @param sort []string
+// @return result []EdbDataBusiness
+// @return err error
+func (m *EdbDataBusiness) GetAllDataList(whereParams interface{}, sort []string) (result []*EdbDataBusiness, err error) {
+	if utils.MgoDataCli == nil {
+		err = errors.New("mongodb连接失败")
+		return
+	}
+	db := utils.MgoDataCli.Database(m.DataBaseName())
+	coll := db.Collection(m.CollectionName())
+	ctx := context.TODO()
+	if err != nil {
+		fmt.Println("MgoGetColl Err:", err.Error())
+		return
+	}
+	err = coll.Find(ctx, whereParams).Sort(sort...).All(&result)
+	if err != nil {
+		return
+	}
+
+	for _, v := range result {
+		v.DataTime = v.DataTime.In(time.Local)
+		v.CreateTime = v.CreateTime.In(time.Local)
+		v.ModifyTime = v.ModifyTime.In(time.Local)
+	}
+
+	return
+}
+
+// GetLimitDataList
+// @Description: 根据条件获取指定数量数据列表
+// @author: Roc
+// @receiver m
+// @datetime 2024-05-06 17:08:32
+// @param whereParams interface{}
+// @param size int64
+// @param sort []string
+// @return result []*BaseFromBusinessData
+// @return err error
+func (m *EdbDataBusiness) GetLimitDataList(whereParams interface{}, size int64, sort []string) (result []*EdbDataBusiness, err error) {
+	if utils.MgoDataCli == nil {
+		err = errors.New("mongodb连接失败")
+		return
+	}
+	db := utils.MgoDataCli.Database(m.DataBaseName())
+	coll := db.Collection(m.CollectionName())
+	ctx := context.TODO()
+	if err != nil {
+		fmt.Println("MgoGetColl Err:", err.Error())
+		return
+	}
+	err = coll.Find(ctx, whereParams).Sort(sort...).Limit(size).All(&result)
+	if err != nil {
+		return
+	}
+
+	for _, v := range result {
+		v.DataTime = v.DataTime.In(time.Local)
+		v.CreateTime = v.CreateTime.In(time.Local)
+		v.ModifyTime = v.ModifyTime.In(time.Local)
+	}
+
+	return
+}
+
+// GetPageDataList
+// @Description: 根据条件获取分页数据列表
+// @author: Roc
+// @receiver m
+// @datetime 2024-05-07 10:21:07
+// @param whereParams interface{}
+// @param startSize int64
+// @param size int64
+// @param sort []string
+// @return result []*EdbDataBusiness
+// @return err error
+func (m *EdbDataBusiness) GetPageDataList(whereParams interface{}, startSize, size int64, sort []string) (result []*EdbDataBusiness, err error) {
+	if utils.MgoDataCli == nil {
+		err = errors.New("mongodb连接失败")
+		return
+	}
+	db := utils.MgoDataCli.Database(m.DataBaseName())
+	coll := db.Collection(m.CollectionName())
+	ctx := context.TODO()
+	if err != nil {
+		fmt.Println("MgoGetColl Err:", err.Error())
+		return
+	}
+	err = coll.Find(ctx, whereParams).Sort(sort...).Skip(startSize).Limit(size).All(&result)
+	if err != nil {
+		return
+	}
+
+	for _, v := range result {
+		v.DataTime = v.DataTime.In(time.Local)
+		v.CreateTime = v.CreateTime.In(time.Local)
+		v.ModifyTime = v.ModifyTime.In(time.Local)
+	}
+
+	return
+}
+
+// GetCountDataList
+// @Description:  根据条件获取数据列表总数
+// @author: Roc
+// @receiver m
+// @datetime 2024-05-07 10:29:00
+// @param whereParams interface{}
+// @return count int64
+// @return err error
+func (m *EdbDataBusiness) GetCountDataList(whereParams interface{}) (count int64, err error) {
+	if utils.MgoDataCli == nil {
+		err = errors.New("mongodb连接失败")
+		return
+	}
+	db := utils.MgoDataCli.Database(m.DataBaseName())
+	coll := db.Collection(m.CollectionName())
+	ctx := context.TODO()
+	if err != nil {
+		fmt.Println("MgoGetColl Err:", err.Error())
+		return
+	}
+	count, err = coll.Find(ctx, whereParams).Count()
+
+	return
+}
+
+// InsertDataByColl
+// @Description: 写入单条数据（外部传入集合）
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-26 14:22:18
+// @param addData interface{}
+// @return err error
+func (m *EdbDataBusiness) InsertDataByColl(coll *qmgo.Collection, addData interface{}) (err error) {
+	ctx := context.TODO()
+	_, err = coll.InsertOne(ctx, addData)
+	if err != nil {
+		fmt.Println("InsertDataByColl:Err:" + err.Error())
+		return
+	}
+
+	return
+}
+
+// BatchInsertData
+// @Description: 批量写入数据
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-26 14:22:18
+// @param bulk int 每次请求保存的数据量
+// @param dataList []interface{}
+// @return err error
+func (m *EdbDataBusiness) BatchInsertData(bulk int, dataList []interface{}) (err error) {
+	db := utils.MgoDataCli.Database(m.DataBaseName())
+	coll := db.Collection(m.CollectionName())
+
+	return m.BatchInsertDataByColl(coll, bulk, dataList)
+}
+
+// BatchInsertDataByColl
+// @Description: 批量写入数据（外部传入集合）
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-26 14:22:18
+// @param coll *qmgo.Collection
+// @param bulk int 每次请求保存的数据量
+// @param dataList []interface{}
+// @return err error
+func (m *EdbDataBusiness) BatchInsertDataByColl(coll *qmgo.Collection, bulk int, dataList []interface{}) (err error) {
+	ctx := context.TODO()
+	dataNum := len(dataList)
+	if dataNum <= 0 {
+		return
+	}
+
+	// 不设置每次保存切片数量大小，或者实际数据量小于设置的切片数量大小，那么就直接保存吧
+	if bulk <= 0 || dataNum <= bulk {
+		_, err = coll.InsertMany(ctx, dataList)
+		if err != nil {
+			fmt.Println("BatchInsertData:Err:" + err.Error())
+			return
+		}
+		return
+	}
+
+	// 分批保存
+	i := 0
+	tmpAddDataList := make([]interface{}, 0)
+	for _, v := range dataList {
+		tmpAddDataList = append(tmpAddDataList, v)
+		i++
+		if i >= bulk {
+			_, err = coll.InsertMany(ctx, tmpAddDataList)
+			if err != nil {
+				fmt.Println("BatchInsertData:Err:" + err.Error())
+				return
+			}
+			i = 0
+			tmpAddDataList = make([]interface{}, 0)
+		}
+	}
+
+	if len(tmpAddDataList) > 0 {
+		_, err = coll.InsertMany(ctx, tmpAddDataList)
+		if err != nil {
+			fmt.Println("BatchInsertData:Err:" + err.Error())
+			return
+		}
+	}
+
+	return
+}
+
+// UpdateData
+// @Description: 单条数据修改
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-26 15:01:51
+// @param whereParams interface{}
+// @param updateParams interface{}
+// @return err error
+func (m *EdbDataBusiness) UpdateData(whereParams, updateParams interface{}) (err error) {
+	db := utils.MgoDataCli.Database(m.DataBaseName())
+	coll := db.Collection(m.CollectionName())
+
+	return m.UpdateDataByColl(coll, whereParams, updateParams)
+}
+
+// UpdateDataByColl
+// @Description: 单条数据修改（外部传入集合）
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-26 15:01:51
+// @param whereParams interface{}
+// @param updateParams interface{}
+// @return err error
+func (m *EdbDataBusiness) UpdateDataByColl(coll *qmgo.Collection, whereParams, updateParams interface{}) (err error) {
+	ctx := context.TODO()
+	err = coll.UpdateOne(ctx, whereParams, updateParams)
+	if err != nil {
+		fmt.Println("UpdateDataByColl:Err:" + err.Error())
+		return
+	}
+
+	return
+}
+
+// RemoveMany
+// @Description: 根据条件删除多条数据
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-30 13:17:02
+// @param whereParams interface{}
+// @return err error
+func (m *EdbDataBusiness) RemoveMany(whereParams interface{}) (err error) {
+	db := utils.MgoDataCli.Database(m.DataBaseName())
+	coll := db.Collection(m.CollectionName())
+
+	return m.RemoveManyByColl(coll, whereParams)
+}
+
+// RemoveManyByColl
+// @Description: 根据条件删除多条数据（外部传入集合）
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-30 13:18:42
+// @param coll *qmgo.Collection
+// @param whereParams interface{}
+// @return err error
+func (m *EdbDataBusiness) RemoveManyByColl(coll *qmgo.Collection, whereParams interface{}) (err error) {
+	ctx := context.TODO()
+	_, err = coll.RemoveAll(ctx, whereParams)
+	if err != nil {
+		fmt.Println("RemoveManyByColl:Err:" + err.Error())
+		return
+	}
+
+	return
+}
+
+// HandleData
+// @Description: 事务处理数据
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-30 10:39:01
+// @param addDataList []AddEdbDataBusiness
+// @param updateDataList []EdbDataBusiness
+// @return result interface{}
+// @return err error
+func (m *EdbDataBusiness) HandleData(addDataList, updateDataList []EdbDataBusiness) (result interface{}, err error) {
+
+	ctx := context.TODO()
+
+	callback := func(sessCtx context.Context) (interface{}, error) {
+		// 重要：确保事务中的每一个操作，都使用传入的sessCtx参数
+
+		db := utils.MgoDataCli.Database(m.DataBaseName())
+		coll := db.Collection(m.CollectionName())
+
+		// 插入数据
+		if len(addDataList) > 0 {
+			_, err = coll.InsertMany(sessCtx, addDataList)
+			if err != nil {
+				return nil, err
+			}
+		}
+
+		// 修改
+
+		if len(updateDataList) > 0 {
+			for _, v := range updateDataList {
+				err = coll.UpdateOne(ctx, bson.M{"_id": v.ID}, bson.M{"$set": bson.M{"value": v.Value, "modify_time": v.ModifyTime}})
+				if err != nil {
+					fmt.Println("BatchInsertData:Err:" + err.Error())
+					return nil, err
+				}
+			}
+		}
+
+		return nil, nil
+	}
+	result, err = utils.MgoDataCli.DoTransaction(ctx, callback)
+
+	return
+}
+
+// EdbInfoMaxAndMinInfo 指标最新数据记录结构体
+type EdbInfoMaxAndMinInfo struct {
+	MinDate     time.Time `description:"最小日期" bson:"min_date"`
+	MaxDate     time.Time `description:"最大日期" bson:"max_date"`
+	MinValue    float64   `description:"最小值" bson:"min_value"`
+	MaxValue    float64   `description:"最大值" bson:"max_value"`
+	LatestValue float64   `description:"最新值" bson:"latest_value"`
+	LatestDate  time.Time `description:"实际数据最新日期" bson:"latest_date"`
+	EndValue    float64   `description:"最新值" bson:"end_value"`
+}
+
+// GetEdbInfoMaxAndMinInfo
+// @Description: 获取当前指标的最大最小值
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-30 17:15:39
+// @param whereParams interface{}
+// @return result EdbInfoMaxAndMinInfo
+// @return err error
+func (m *EdbDataBusiness) GetEdbInfoMaxAndMinInfo(whereParams interface{}) (result EdbInfoMaxAndMinInfo, err error) {
+	if utils.MgoDataCli == nil {
+		err = errors.New("mongodb连接失败")
+		return
+	}
+	db := utils.MgoDataCli.Database(m.DataBaseName())
+	coll := db.Collection(m.CollectionName())
+	ctx := context.TODO()
+	if err != nil {
+		fmt.Println("MgoGetColl Err:", err.Error())
+		return
+	}
+	err = coll.Aggregate(ctx, whereParams).One(&result)
+	if err != nil {
+		return
+	}
+	result.MinDate = result.MinDate.In(time.Local)
+	result.MaxDate = result.MaxDate.In(time.Local)
+	result.LatestDate = result.LatestDate.In(time.Local)
+
+	return
+}
+
+// LatestValue 指标最新数据记录结构体
+type LatestValue struct {
+	Value float64 `description:"值" bson:"value"`
+}
+
+// GetLatestValue
+// @Description: 获取当前指标的最新数据记录
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-30 17:16:15
+// @param whereParams interface{}
+// @param selectParam interface{}
+// @return latestValue LatestValue
+// @return err error
+func (m *EdbDataBusiness) GetLatestValue(whereParams, selectParam interface{}) (latestValue LatestValue, err error) {
+	if utils.MgoDataCli == nil {
+		err = errors.New("mongodb连接失败")
+		return
+	}
+	db := utils.MgoDataCli.Database(m.DataBaseName())
+	coll := db.Collection(m.CollectionName())
+	ctx := context.TODO()
+	if err != nil {
+		fmt.Println("MgoGetColl Err:", err.Error())
+		return
+	}
+
+	//var result interface{}
+	//err = coll.Find(ctx, whereParams).Select(selectParam).One(&result)
+	err = coll.Find(ctx, whereParams).Select(selectParam).One(&latestValue)
+
+	return
+}

models/mgo_base.go

@@ -0,0 +1,274 @@
+package models
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"eta/eta_index_lib/utils"
+	"eta/eta_index_lib/utils/mgodb"
+	"fmt"
+)
+
+func init() {
+	if utils.MgoUrlData != `` {
+		var mgoConfig mgodb.MgoConfig
+		if e := json.Unmarshal([]byte(utils.MgoUrlData), &mgoConfig); e != nil {
+			panic("mongodb链接失败，Err:" + e.Error())
+			return
+		}
+
+		mgoCli := mgodb.MgoNewClient(mgoConfig)
+		utils.MgoDataCli = mgoCli
+		utils.MgoDataDbName = mgoConfig.Database
+		//result, err := TestMgoFindOne("data_ths", bson.M{"edbcode": "s0033227", "datatime": "2015-12-04"})
+		//fmt.Println(time.Now())
+		//result, err := TestMgoFindOne("data_wind", bson.M{"edbcode": "s0033227", "datatime": "2007-12-29"})
+		//fmt.Println(time.Now())
+		//
+		//fmt.Println(err)
+		//fmt.Println(result)
+	}
+
+}
+
+// TestMgoFindOne
+// @Description: 获取单条数据
+// @author: Roc
+// @datetime 2024-04-25 15:44:07
+// @param colName string 集合名词：`data_ths`
+// @param whereParams interface{} bson.M{"edbcode": "s0033227", "datatime": "2015-12-04"}
+// @return result interface{}
+// @return err error
+func TestMgoFindOne(colName string, whereParams interface{}) (result interface{}, err error) {
+	if colName == "" {
+		err = errors.New("集合名称不可为空")
+		return
+	}
+	if utils.MgoDataCli == nil {
+		err = errors.New("mongodb连接失败")
+		return
+	}
+	db := utils.MgoDataCli.Database("hz_data")
+	coll := db.Collection("data_ths")
+	ctx := context.TODO()
+	if err != nil {
+		fmt.Println("MgoGetColl Err:", err.Error())
+		return
+	}
+	//err = coll.Find(ctx, whereParams).One(&result)
+	err = coll.Find(ctx, whereParams).All(&result)
+	return
+}
+
+// 创建集合
+//func MgoCreateCollection(collectionName string) (err error) {
+//	ctx := context.TODO()
+//
+//	mgoClient := mgodb.MgoNewClient()
+//
+//	defer func() {
+//		mgoClient.Close(ctx)
+//	}()
+//
+//	colName := collectionName
+//	err = mgoClient.Database("hz_data").CreateCollection(ctx, colName)
+//	return
+//}
+
+// 批量新增多条数据
+func MgoBatchInsertData(colName string, dataList interface{}) (err error) {
+	if colName == "" {
+		err = errors.New("集合名称不可为空")
+		return
+	}
+	ctx := context.TODO()
+	coll, err := mgodb.MgoGetColl(colName)
+	if err != nil {
+		fmt.Println("MgoGetColl Err:", err.Error())
+		return
+	}
+	defer func() {
+		coll.Close(ctx)
+	}()
+	_, err = coll.Collection.InsertMany(ctx, dataList)
+	if err != nil {
+		fmt.Println("InsertMany:Err:" + err.Error())
+		return
+	}
+	return
+}
+
+// 插入单条数据
+func MgoInsertData(colName string, dataItem interface{}) (insertedID interface{}, err error) {
+	if colName == "" {
+		err = errors.New("集合名称不可为空")
+		return
+	}
+	ctx := context.TODO()
+	coll, err := mgodb.MgoGetColl(colName)
+	if err != nil {
+		fmt.Println("MgoGetColl Err:", err.Error())
+		return
+	}
+	defer func() {
+		coll.Close(ctx)
+	}()
+	result, err := coll.Collection.InsertOne(ctx, dataItem)
+	if err != nil {
+		return
+	}
+	insertedID = result.InsertedID
+	return
+}
+
+/*
+删除数据
+colName := `data_gl`
+whereParams := bson.M{"edbcode": "s0033227"}
+err := MgoRemove(colName, whereParams)
+*/
+func MgoRemove(colName string, whereParams interface{}) (err error) {
+	if colName == "" {
+		err = errors.New("集合名称不可为空")
+		return
+	}
+	ctx := context.TODO()
+	coll, err := mgodb.MgoGetColl(colName)
+	if err != nil {
+		fmt.Println("MgoGetColl Err:", err.Error())
+		return
+	}
+	defer func() {
+		coll.Close(ctx)
+	}()
+	err = coll.Remove(ctx, whereParams)
+	return
+}
+
+/*
+删除多条数据
+colName := `data_gl`
+whereParams := bson.M{"edbcode": "s0033227"}
+err := MgoRemoveAll(colName, whereParams)
+*/
+func MgoRemoveAll(colName string, whereParams interface{}) (count int64, err error) {
+	if colName == "" {
+		err = errors.New("集合名称不可为空")
+		return
+	}
+	ctx := context.TODO()
+	coll, err := mgodb.MgoGetColl(colName)
+	if err != nil {
+		fmt.Println("MgoGetColl Err:", err.Error())
+		return
+	}
+	defer func() {
+		coll.Close(ctx)
+	}()
+	res, err := coll.RemoveAll(ctx, whereParams)
+	if err != nil {
+		fmt.Println("RemoveAll Err:", err.Error())
+		return
+	}
+	count = res.DeletedCount
+	return
+}
+
+/*
+修改单条数据
+colName := `data_ths`
+whereParams := bson.M{"edbcode": "s0033227", "datatime": "2015-12-04"}
+params := bson.M{"$set": bson.M{"value": "1000"}}
+*/
+func MgoUpdateData(colName string, params, whereParams interface{}) (err error) {
+	if colName == "" {
+		err = errors.New("集合名称不可为空")
+		return
+	}
+	ctx := context.TODO()
+	coll, err := mgodb.MgoGetColl(colName)
+	if err != nil {
+		fmt.Println("MgoGetColl Err:", err.Error())
+		return
+	}
+	defer func() {
+		coll.Close(ctx)
+	}()
+	err = coll.UpdateOne(ctx, whereParams, params)
+	return
+}
+
+/*
+获取单条数据
+colName := `data_ths`
+whereParams := bson.M{"edbcode": "s0033227", "datatime": "2015-12-04"}
+*/
+func MgoFindOne(colName string, whereParams interface{}) (result interface{}, err error) {
+	if colName == "" {
+		err = errors.New("集合名称不可为空")
+		return
+	}
+	ctx := context.TODO()
+	coll, err := mgodb.MgoGetColl(colName)
+	if err != nil {
+		fmt.Println("MgoGetColl Err:", err.Error())
+		return
+	}
+	defer func() {
+		coll.Close(ctx)
+	}()
+	err = coll.Find(ctx, whereParams).One(&result)
+	return
+}
+
+/*
+分页获取数据
+colName := `data_ths`
+whereParams := bson.M{"edbcode": "s0033227", "datatime": "2015-12-04"}
+
+sort:="-weight"
+skip:0
+limit:10
+*/
+func MgoBatchFind(colName, sort string, whereParams interface{}, skip, limit int64) (list []interface{}, err error) {
+	if colName == "" {
+		err = errors.New("集合名称不可为空")
+		return
+	}
+	ctx := context.TODO()
+	coll, err := mgodb.MgoGetColl(colName)
+	if err != nil {
+		fmt.Println("MgoGetColl Err:", err.Error())
+		return
+	}
+	defer func() {
+		coll.Close(ctx)
+	}()
+	err = coll.Find(ctx, whereParams).Sort(sort).Skip(skip).Limit(limit).All(&list)
+	return
+}
+
+/*
+MgoUpsertData
+更新插入数据-数据存在即更新, 否则新增
+colName := `data_ths`
+filter := bson.M{"edbcode": "s0033227", "datatime": "2015-12-04"}
+replacement := bson.M{"edb_info_id":0,"edb_code":m.EdbCode,"data_time":d.DataTime,"value":f,"create_time":now,"modify_time":now,"data_timestamp": stamp}
+*/
+func MgoUpsertData(colName string, filter, replacement interface{}) (err error) {
+	if colName == "" {
+		err = errors.New("集合名称不可为空")
+		return
+	}
+	ctx := context.TODO()
+	coll, err := mgodb.MgoGetColl(colName)
+	if err != nil {
+		fmt.Println("MgoGetColl Err:", err.Error())
+		return
+	}
+	defer func() {
+		_ = coll.Close(ctx)
+	}()
+	_, err = coll.Upsert(ctx, filter, replacement)
+	return
+}

models/predict_edb_data_calculate_bp.go

@@ -217,12 +217,13 @@ func refreshAllPredictCalculateBp(to orm.TxOrmer, edbInfoId, source, subSource i
 	if len(existDataList) > 0 {
 		//第一个已经入库的日期
 		firstExistDataTimeStr := existDataList[0].DataTime //计算指标数据第一条的日期字符串
-		if len(dateArr) > 0 {
-			firstFromDataTimeStr := dateArr[0]                                                                 //来源数据第一条的日期字符串
-			firstExistDataTime, _ := time.ParseInLocation(utils.FormatDate, firstExistDataTimeStr, time.Local) //计算指标数据第一条的日期（time类型）
-			firstFromDataTime, _ := time.ParseInLocation(utils.FormatDate, firstFromDataTimeStr, time.Local)   //来源数据第一条的日期（time类型）
-			nowDateStr := time.Now().Format(utils.FormatDate)                                                  //当天日期字符串
-			nowDate, _ := time.ParseInLocation(utils.FormatDate, nowDateStr, firstFromDataTime.Location())     //当天日期（time类型）
+		lenDateArr := len(dateArr)
+		if lenDateArr > 0 {
+			firstFromDataTimeStr := dateArr[0]                                                                     //来源数据第一条的日期字符串
+			firstExistDataTime, _ := time.ParseInLocation(utils.FormatDate, firstExistDataTimeStr, time.Local)     //计算指标数据第一条的日期（time类型）
+			firstFromDataTime, _ := time.ParseInLocation(utils.FormatDate, firstFromDataTimeStr, time.Local)       //来源数据第一条的日期（time类型）
+			fromEndDateStr := dateArr[lenDateArr-1]                                                                //当天日期字符串
+			fromEndDate, _ := time.ParseInLocation(utils.FormatDate, fromEndDateStr, firstFromDataTime.Location()) //当天日期（time类型）
 
 			lastValue = fromDataMap[firstFromDataTimeStr]
 			lastValueStr = decimal.NewFromFloat(lastValue).String()
@@ -265,7 +266,7 @@ func refreshAllPredictCalculateBp(to orm.TxOrmer, edbInfoId, source, subSource i
 
 			//第二步 剩余数据每天修改
 
-			day := int(nowDate.Sub(firstExistDataTime).Hours() / float64(24))
+			day := int(fromEndDate.Sub(firstExistDataTime).Hours() / float64(24))
 
 			//第三步： 已经入库的数据处理
 			for _, v := range existDataList {
@@ -273,7 +274,7 @@ func refreshAllPredictCalculateBp(to orm.TxOrmer, edbInfoId, source, subSource i
 			}
 
 			for k := day; k >= 0; k-- {
-				needDay := nowDate.AddDate(0, 0, -k)
+				needDay := fromEndDate.AddDate(0, 0, -k)
 				needDayStr := needDay.Format(utils.FormatDate)
 				tmpExistData, ok := existDataMap[needDayStr]
 				if ok {

models/predict_edb_data_calculate_ljztbpj.go

@@ -444,14 +444,12 @@ func refreshAllPredictCalculateLjztbpj(to orm.TxOrmer, edbInfo, firstEdbInfo, se
 
 	//同比值指标
 	{
-		var condition string
-		pars := make([]interface{}, 0)
-
-		condition += " AND data_time > ? AND edb_info_id = ? "
-		pars = append(pars, startCalculationDate, existItemB.FromEdbInfoId)
-
 		//第二个指标的数据列表
-		secondDataList, tmpErr := GetEdbDataListAllByTo(to, condition, pars, existItemB.FromSource, existItemB.FromSubSource, 0)
+		secondDataList, tmpErr := GetEdbDataListAllByTo(to, existItemB.FromSource, existItemB.FromSubSource, FindEdbDataListAllCond{
+			EdbInfoId:         existItemB.FromEdbInfoId,
+			StartDataTime:     startCalculationDate.Format(utils.FormatDate),
+			StartDataTimeCond: ">",
+		}, 0)
 		if tmpErr != nil {
 			err = tmpErr
 			return

models/predict_edb_data_calculate_percentile.go

@@ -382,13 +382,10 @@ func (obj PredictPercentile) getPercentileData(fromEdbInfo *EdbInfo, calculateVa
 	dataList := make([]*EdbInfoSearchData, 0)
 	switch fromEdbInfo.EdbInfoType {
 	case 0:
-		var condition string
-		var pars []interface{}
-		condition += " AND edb_info_id=? "
-		pars = append(pars, fromEdbInfo.EdbInfoId)
-
 		//获取来源指标的数据
-		dataList, err = GetEdbDataListAll(condition, pars, fromEdbInfo.Source, fromEdbInfo.SubSource, 1)
+		dataList, err = GetEdbDataListAll(fromEdbInfo.Source, fromEdbInfo.SubSource, FindEdbDataListAllCond{
+			EdbInfoId: fromEdbInfo.EdbInfoId,
+		}, 1)
 	case 1:
 		dataList, err = GetPredictEdbDataListAllByStartDate(fromEdbInfo, 1, "")
 	default:

models/predict_edb_data_calculate_standard_deviation.go

@@ -393,13 +393,10 @@ func (obj PredictStandardDeviation) getStandardDeviationData(fromEdbInfo *EdbInf
 	dataList := make([]*EdbInfoSearchData, 0)
 	switch fromEdbInfo.EdbInfoType {
 	case 0:
-		var condition string
-		var pars []interface{}
-		condition += " AND edb_info_id=? "
-		pars = append(pars, fromEdbInfo.EdbInfoId)
-
 		//获取来源指标的数据
-		dataList, err = GetEdbDataListAll(condition, pars, fromEdbInfo.Source, fromEdbInfo.SubSource, 1)
+		dataList, err = GetEdbDataListAll(fromEdbInfo.Source, fromEdbInfo.SubSource, FindEdbDataListAllCond{
+			EdbInfoId: fromEdbInfo.EdbInfoId,
+		}, 1)
 	case 1:
 		dataList, err = GetPredictEdbDataListAllByStartDate(fromEdbInfo, 1, "")
 	default:

models/predict_edb_data_calculate_time_shift.go

@@ -241,7 +241,9 @@ func refreshAllPredictCalculateTimeShift(to orm.TxOrmer, edbInfoId, source, subS
 	if err != nil {
 		return
 	}
+
 	existDataMap := make(map[string]string)
+	removeDateMap := make(map[string]string)
 	for _, v := range existDataList {
 		existDataMap[v.DataTime] = v.Value
 	}
@@ -284,6 +286,9 @@ func refreshAllPredictCalculateTimeShift(to orm.TxOrmer, edbInfoId, source, subS
 					}
 				}
 			}
+
+			// 指标已存在，那么就移除该日期
+			delete(removeDateMap, newDate.Format(utils.FormatDate))
 		}
 		existMap[existKey] = currentItem.DataTime
 	}
@@ -295,5 +300,19 @@ func refreshAllPredictCalculateTimeShift(to orm.TxOrmer, edbInfoId, source, subS
 			return
 		}
 	}
+	
+	// 移除不存在的日期数据
+	if len(removeDateMap) > 0 {
+		removeDateList := make([]string, 0) //需要移除的日期
+		for k := range removeDateMap {
+			removeDateList = append(removeDateList, k)
+		}
+		err = DelEdbDataByMysql(to, edbInfoId, dataTableName, removeDateList)
+		if err != nil {
+			err = fmt.Errorf("删除年化指标数据失败,Err:" + err.Error())
+			return
+		}
+	}
+
 	return
 }

models/predict_edb_info_rule.go

@@ -1265,13 +1265,10 @@ func getCalculateNhccData(secondDataList []*EdbInfoSearchData, ruleConf RuleLine
 		var firstDataList []*EdbInfoSearchData
 		switch edbInfo.EdbInfoType {
 		case 0:
-			var condition string
-			var pars []interface{}
-			condition += " AND edb_info_id=? "
-			pars = append(pars, edbInfo.EdbInfoId)
-
 			//获取来源指标的数据
-			firstDataList, err = GetEdbDataListAll(condition, pars, edbInfo.Source, edbInfo.SubSource, 1)
+			firstDataList, err = GetEdbDataListAll(edbInfo.Source, edbInfo.SubSource, FindEdbDataListAllCond{
+				EdbInfoId: edbInfo.EdbInfoId,
+			}, 1)
 		case 1:
 			firstDataList, err = GetPredictEdbDataListAllByStartDate(edbInfo, 1, "")
 		default:

routers/commentsRouter.go

@@ -88,6 +88,15 @@ func init() {
             Filters: nil,
             Params: nil})
 
+    beego.GlobalControllerRouter["eta/eta_index_lib/controllers:BloombergController"] = append(beego.GlobalControllerRouter["eta/eta_index_lib/controllers:BloombergController"],
+        beego.ControllerComments{
+            Method: "PCSGImportHistoryData",
+            Router: `/pcsg/import_history_data`,
+            AllowHTTPMethods: []string{"post"},
+            MethodParams: param.Make(),
+            Filters: nil,
+            Params: nil})
+
     beego.GlobalControllerRouter["eta/eta_index_lib/controllers:BloombergController"] = append(beego.GlobalControllerRouter["eta/eta_index_lib/controllers:BloombergController"],
         beego.ControllerComments{
             Method: "PCSGRefreshDaily",
@@ -124,6 +133,42 @@ func init() {
             Filters: nil,
             Params: nil})
 
+    beego.GlobalControllerRouter["eta/eta_index_lib/controllers:BusinessIndexController"] = append(beego.GlobalControllerRouter["eta/eta_index_lib/controllers:BusinessIndexController"],
+        beego.ControllerComments{
+            Method: "Add",
+            Router: `/add`,
+            AllowHTTPMethods: []string{"post"},
+            MethodParams: param.Make(),
+            Filters: nil,
+            Params: nil})
+
+    beego.GlobalControllerRouter["eta/eta_index_lib/controllers:BusinessIndexController"] = append(beego.GlobalControllerRouter["eta/eta_index_lib/controllers:BusinessIndexController"],
+        beego.ControllerComments{
+            Method: "Edit",
+            Router: `/edit`,
+            AllowHTTPMethods: []string{"post"},
+            MethodParams: param.Make(),
+            Filters: nil,
+            Params: nil})
+
+    beego.GlobalControllerRouter["eta/eta_index_lib/controllers:BusinessIndexController"] = append(beego.GlobalControllerRouter["eta/eta_index_lib/controllers:BusinessIndexController"],
+        beego.ControllerComments{
+            Method: "HandleBusinessIndexData",
+            Router: `/handle`,
+            AllowHTTPMethods: []string{"post"},
+            MethodParams: param.Make(),
+            Filters: nil,
+            Params: nil})
+
+    beego.GlobalControllerRouter["eta/eta_index_lib/controllers:BusinessIndexController"] = append(beego.GlobalControllerRouter["eta/eta_index_lib/controllers:BusinessIndexController"],
+        beego.ControllerComments{
+            Method: "Refresh",
+            Router: `/refresh`,
+            AllowHTTPMethods: []string{"post"},
+            MethodParams: param.Make(),
+            Filters: nil,
+            Params: nil})
+
     beego.GlobalControllerRouter["eta/eta_index_lib/controllers:CalculateController"] = append(beego.GlobalControllerRouter["eta/eta_index_lib/controllers:CalculateController"],
         beego.ControllerComments{
             Method: "Add",

routers/router.go

@@ -99,14 +99,14 @@ func init() {
 				&controllers.GieController{},
 			),
 		),
-		beego.NSNamespace("/coal",
+		beego.NSNamespace("/python",
 			beego.NSInclude(
-				&controllers.CoalController{},
+				&controllers.PythonController{},
 			),
 		),
-		beego.NSNamespace("/python",
+		beego.NSNamespace("/coal",
 			beego.NSInclude(
-				&controllers.PythonController{},
+				&controllers.CoalController{},
 			),
 		),
 		beego.NSNamespace("/google_travel",
@@ -114,6 +114,16 @@ func init() {
 				&controllers.GoogleTravelController{},
 			),
 		),
+		beego.NSNamespace("/predict_calculate",
+			beego.NSInclude(
+				&controllers.PredictCalculateController{},
+			),
+		),
+		beego.NSNamespace("/mysteel_chemical",
+			beego.NSInclude(
+				&controllers.MySteelChemicalController{},
+			),
+		),
 		beego.NSNamespace("/mysteel_chemical",
 			beego.NSInclude(
 				&controllers.MySteelChemicalController{},
@@ -219,24 +229,24 @@ func init() {
 				&controllers.CoalMineDataController{},
 			),
 		),
-		beego.NSNamespace("/fenwei",
+		beego.NSNamespace("/gz",
 			beego.NSInclude(
-				&controllers.FenweiController{},
+				&controllers.GzController{},
 			),
 		),
-		beego.NSNamespace("/gz",
+		beego.NSNamespace("/icpi",
 			beego.NSInclude(
-				&controllers.GzController{},
+				&controllers.IcpiController{},
 			),
 		),
-		beego.NSNamespace("/mtjh",
+		beego.NSNamespace("/fenwei",
 			beego.NSInclude(
-				&controllers.MtjhDataController{},
+				&controllers.FenweiController{},
 			),
 		),
-		beego.NSNamespace("/icpi",
+		beego.NSNamespace("/mtjh",
 			beego.NSInclude(
-				&controllers.IcpiController{},
+				&controllers.MtjhDataController{},
 			),
 		),
 		beego.NSNamespace("/edb_refresh",
@@ -249,6 +259,11 @@ func init() {
 				&controllers.BloombergController{},
 			),
 		),
+		beego.NSNamespace("/business_index",
+			beego.NSInclude(
+				&controllers.BusinessIndexController{},
+			),
+		),
 	)
 	beego.AddNamespace(ns)
 }

services/base_from_business.go

@@ -0,0 +1,283 @@
+package services
+
+import (
+	"errors"
+	"eta/eta_index_lib/logic"
+	"eta/eta_index_lib/models"
+	"eta/eta_index_lib/models/mgo"
+	"eta/eta_index_lib/services/alarm_msg"
+	"eta/eta_index_lib/utils"
+	"fmt"
+	"go.mongodb.org/mongo-driver/bson"
+	"strings"
+	"time"
+)
+
+// HandleBusinessIndex
+// @Description:  处理处理外部指标
+// @author: Roc
+// @datetime 2024-04-26 14:23:42
+// @param indexItem *models.AddBusinessIndexReq
+// @return err error
+func HandleBusinessIndex(indexItem *models.AddBusinessIndexReq) (err error) {
+	defer func() {
+		if err != nil {
+			// 添加刷新失败日志
+			dataUpdateResult := 2
+			dataUpdateFailedReason := "服务异常"
+			edbInfo, e := models.GetEdbInfoByEdbCode(utils.DATA_SOURCE_MYSTEEL_CHEMICAL, indexItem.IndexCode)
+			if e == nil {
+				//查询指标存在，才添加刷新日志
+				_ = AddEdbInfoUpdateLog(edbInfo.EdbInfoId, 2, err.Error(), dataUpdateResult, dataUpdateFailedReason, 1, 0)
+			}
+		}
+	}()
+
+	// 没有数据就返回
+	if indexItem.DataList == nil || len(indexItem.DataList) <= 0 {
+		return
+	}
+	// 兼容频度缺少度的字段
+	if !strings.Contains(indexItem.Frequency, "度") {
+		indexItem.Frequency = indexItem.Frequency + "度"
+	}
+	if !utils.VerifyFrequency(indexItem.Frequency) {
+		err = errors.New("指标频度不合法:" + indexItem.Frequency)
+		return
+	}
+
+	// 判断来源，如果来源不存在的话，则创建
+	sourceObj := new(models.EdbBusinessSource)
+	sourceItem, err := sourceObj.GetEdbBusinessSourceItem(indexItem.SourceName)
+	if err != nil {
+		if err.Error() == utils.ErrNoRow() {
+			sourceItem = &models.EdbBusinessSource{
+				EdbBusinessSourceId: 0,
+				SourceName:          indexItem.SourceName,
+				CreateTime:          time.Now(),
+			}
+			err = sourceItem.Add()
+		} else {
+			return
+		}
+	}
+
+	// 指标
+
+	indexObj := new(models.BaseFromBusinessIndex)
+	//判断指标是否存在
+	item, err := indexObj.GetIndexItem(indexItem.IndexCode)
+	if err != nil {
+		if err.Error() != utils.ErrNoRow() {
+			return
+		}
+
+		// 添加指标
+		item = &models.BaseFromBusinessIndex{
+			BaseFromBusinessIndexId: 0,
+			IndexCode:               indexItem.IndexCode,
+			IndexName:               indexItem.IndexName,
+			Unit:                    indexItem.Unit,
+			Frequency:               indexItem.Frequency,
+			Source:                  int(sourceItem.EdbBusinessSourceId),
+			SourceName:              sourceItem.SourceName,
+			//StartDate:               time.Time{},
+			//EndDate:                 time.Time{},
+			Remark:         indexItem.Remark,
+			BaseModifyTime: time.Now(),
+			DataUpdateTime: time.Now(),
+			CreateTime:     time.Now(),
+			ModifyTime:     time.Now(),
+		}
+		err = item.Add()
+		if err != nil {
+			fmt.Println("add err:" + err.Error())
+			return
+		}
+	} else {
+		updateCols := make([]string, 0)
+		if item.IndexName != indexItem.IndexName {
+			item.IndexName = indexItem.IndexName
+			updateCols = append(updateCols, "IndexName")
+		}
+		if item.Unit != indexItem.Unit {
+			item.Unit = indexItem.Unit
+			updateCols = append(updateCols, "Unit")
+		}
+		if item.Frequency != indexItem.Frequency {
+			item.Frequency = indexItem.Frequency
+			updateCols = append(updateCols, "Frequency")
+		}
+		if item.Source != int(sourceItem.EdbBusinessSourceId) {
+			item.Source = int(sourceItem.EdbBusinessSourceId)
+			item.SourceName = sourceItem.SourceName
+			updateCols = append(updateCols, "Source", "SourceName")
+		}
+
+		if len(updateCols) > 0 {
+			item.BaseModifyTime = time.Now()
+			item.ModifyTime = time.Now()
+			updateCols = append(updateCols, "BaseModifyTime", "ModifyTime")
+			err = item.Update(updateCols)
+			if err != nil {
+				fmt.Println("update index err:" + err.Error())
+				return
+			}
+		}
+	}
+
+	// 数据处理
+	mogDataObj := new(mgo.BaseFromBusinessData)
+
+	//获取已存在的所有数据
+	exitDataList, err := mogDataObj.GetAllDataList(bson.M{"index_code": indexItem.IndexCode}, []string{"data_time"})
+	if err != nil {
+		fmt.Println("GetIndexDataList Err:" + err.Error())
+		return err
+	}
+
+	// 已经存在的数据集
+	exitDataMap := make(map[string]*mgo.BaseFromBusinessData)
+	for _, v := range exitDataList {
+		exitDataMap[v.DataTime.Format(utils.FormatDate)] = v
+	}
+
+	// 当前传入的最小日期
+	var reqMinDate time.Time
+	// 待添加的数据集
+	addDataList := make([]interface{}, 0)
+	updateDataList := make([]mgo.BaseFromBusinessData, 0)
+	//var hasUpdate bool
+	// 遍历excel数据，然后跟现有的数据做校验，不存在则入库
+	for _, data := range indexItem.DataList {
+		dateTime, err := utils.DealExcelDate(data.Date)
+		if err != nil {
+			fmt.Println("time.ParseInLocation Err:" + err.Error())
+			return err
+		}
+
+		// 调整最小日期
+		if reqMinDate.IsZero() || reqMinDate.After(dateTime) {
+			reqMinDate = dateTime
+		}
+
+		date := dateTime.Format(utils.FormatDate)
+
+		findData, ok := exitDataMap[date]
+		if !ok {
+			addDataList = append(addDataList, mgo.BaseFromBusinessData{
+				BaseFromBusinessIndexId: item.BaseFromBusinessIndexId,
+				IndexCode:               item.IndexCode,
+				DataTime:                dateTime,
+				Value:                   data.Value,
+				CreateTime:              time.Now(),
+				ModifyTime:              time.Now(),
+				//DataTimestamp:           0,
+			})
+			continue
+		}
+
+		// 值不匹配，修改数据
+		if findData.Value != data.Value {
+			findData.Value = data.Value
+			updateDataList = append(updateDataList, *findData)
+		}
+	}
+
+	// 指标数据是否新增或修改
+	var isIndexUpdateOrAdd bool
+
+	// 入库
+	{
+		coll := mogDataObj.GetCollection()
+		if len(addDataList) > 0 {
+			isIndexUpdateOrAdd = true
+			err = mogDataObj.BatchInsertDataByColl(coll, 500, addDataList)
+			if err != nil {
+				fmt.Println("mogDataObj.HandleData() Err:" + err.Error())
+				return
+			}
+		}
+
+		if len(updateDataList) > 0 {
+			isIndexUpdateOrAdd = true
+			for _, v := range updateDataList {
+				err = mogDataObj.UpdateDataByColl(coll, bson.M{"_id": v.ID}, bson.M{"$set": bson.M{"value": v.Value, "modify_time": v.ModifyTime}})
+				if err != nil {
+					fmt.Println("UpdateDataByColl:Err:" + err.Error())
+					return
+				}
+			}
+		}
+	}
+
+	// 支持事务的话，下面操作
+	//result, err := mogDataObj.HandleData(addDataList, updateDataList)
+	//if err != nil {
+	//	fmt.Println("mogDataObj.HandleData() Err:" + err.Error())
+	//	return
+	//}
+	//fmt.Println("result", result)
+
+	//修改最大最小日期
+	indexMaxAndMinInfo, err := item.GetEdbInfoMaxAndMinInfo(indexItem.IndexCode)
+	if err != nil {
+		return
+	}
+	if err == nil && indexMaxAndMinInfo != nil {
+		e := item.ModifyIndexMaxAndMinInfo(indexItem.IndexCode, indexMaxAndMinInfo, isIndexUpdateOrAdd)
+		if e != nil {
+			fmt.Println("ModifyIndexMaxAndMinInfo Err:" + e.Error())
+		}
+	}
+
+	// 同步刷新指标库的指标
+	go refreshEdbBusiness(item.IndexCode, reqMinDate)
+
+	return
+}
+
+func refreshEdbBusiness(indexCode string, reqMinDate time.Time) {
+	var indexErr error
+	var errMsg string
+	defer func() {
+		if indexErr != nil {
+			tips := fmt.Sprintf("自有数据刷新-ETA指标刷新异常, 指标编码: %s, err: %s, 错误信息：%s", indexCode, indexErr.Error(), errMsg)
+			alarm_msg.SendAlarmMsg(tips, 3)
+		}
+	}()
+
+	edbInfo, e := models.GetEdbInfoByEdbCode(utils.DATA_SOURCE_BUSINESS, indexCode)
+	if e != nil && e.Error() != utils.ErrNoRow() {
+		indexErr = e
+		return
+	}
+
+	if edbInfo != nil {
+		startDate := ``
+		if reqMinDate.IsZero() {
+			startDate = edbInfo.EndDate
+		} else {
+			startDate = reqMinDate.Format(utils.FormatDate)
+		}
+		params := models.RefreshBaseParams{
+			EdbInfo:   edbInfo,
+			StartDate: startDate,
+		}
+		obj := models.Business{}
+
+		indexErr, errMsg = obj.Refresh(params)
+		if indexErr != nil {
+			return
+		}
+
+		// 更新指标最大最小值
+		indexErr = obj.UnifiedModifyEdbInfoMaxAndMinInfo(edbInfo)
+		if indexErr != nil {
+			return
+		}
+
+		// 更新ES
+		go logic.UpdateEs(edbInfo.EdbInfoId)
+	}
+}

services/base_from_pcsg.go

@@ -213,6 +213,10 @@ func PCSGWrite2BaseBloomberg(indexes []models.BaseFromBloombergApiIndexAndData)
 		if v.IndexCode == "" {
 			continue
 		}
+		// 无数据的情况不处理
+		if len(v.Data) == 0 {
+			continue
+		}
 
 		// 指标是否存在
 		index, e := models.GetBaseFromBloombergIndexByCode(v.IndexCode)
@@ -236,6 +240,18 @@ func PCSGWrite2BaseBloomberg(indexes []models.BaseFromBloombergApiIndexAndData)
 				return
 			}
 			index = newIndex
+		} else {
+			// 无指标名称的情况下更新指标基础信息
+			if index.IndexName == "" {
+				index.IndexName = v.IndexName
+				index.Unit = v.Unit
+				index.Frequency = v.Frequency
+				index.ModifyTime = time.Now().Local()
+				if e = index.Update([]string{"IndexName", "Unit", "Frequency", "ModifyTime"}); e != nil {
+					err = fmt.Errorf("更新Bloomberg原始指标失败, err: %s", e.Error())
+					return
+				}
+			}
 		}
 
 		// 更新指标数据
@@ -249,6 +265,7 @@ func PCSGWrite2BaseBloomberg(indexes []models.BaseFromBloombergApiIndexAndData)
 			return
 		}
 		dateExist := make(map[string]*models.BaseFromBloombergData)
+		newValExist := make(map[string]bool)
 		if len(indexData) > 0 {
 			for _, d := range indexData {
 				strDate := d.DataTime.Format(utils.FormatDate)
@@ -270,6 +287,11 @@ func PCSGWrite2BaseBloomberg(indexes []models.BaseFromBloombergApiIndexAndData)
 				originData.ModifyTime = time.Now().Local()
 				updateData = append(updateData, originData)
 			} else {
+				// 新增的数据去重
+				if newValExist[strDate] {
+					continue
+				}
+				newValExist[strDate] = true
 				newData := new(models.BaseFromBloombergData)
 				newData.BaseFromBloombergIndexId = index.BaseFromBloombergIndexId
 				newData.IndexCode = index.IndexCode

services/base_from_python.go

@@ -81,7 +81,7 @@ func ExecPythonCode(edbCode, reqCode string) (dataMap models.EdbDataFromPython,
 	if err != nil {
 		return
 	}
-	pthonCodeStr := getPythonFrontStr() + reqCode + getPythonLaterStr()
+	pythonCodeStr := getPythonFrontStr() + reqCode + getPythonLaterStr()
 	fileHandle, err := os.OpenFile(pythonFile, os.O_RDWR|os.O_CREATE|os.O_EXCL, 0766)
 	if err != nil {
 		return
@@ -89,7 +89,7 @@ func ExecPythonCode(edbCode, reqCode string) (dataMap models.EdbDataFromPython,
 	defer func() {
 		os.Remove(pythonFile)
 	}()
-	_, err = fileHandle.Write([]byte(pthonCodeStr))
+	_, err = fileHandle.Write([]byte(pythonCodeStr))
 	if err != nil {
 		return
 	}
@@ -172,7 +172,36 @@ func getPythonFileAbsolutePath(edbCode string) (pythonFile string, err error) {
 // getPythonFrontStr 获取python前面的代码
 func getPythonFrontStr() string {
 	//return "#!/usr/bin/python\n# -*- coding: UTF-8 -*-\nimport json\n\nimport pymysql\nimport pandas as pd\n\nsql_config = {\n    'host': 'rm-uf67kg347rhjfep5c1o.mysql.rds.aliyuncs.com',\n    'port': 3306,#主机号\n    'user': 'hz_technology',#账户名\n    'passwd': 'hongze@2021',#密码\n    'db': 'test_hz_data',\n    'charset': 'utf8mb4',\n    'cursorclass': pymysql.cursors.DictCursor\n}\n\ndb = pymysql.connect(**sql_config)\ndb.autocommit(1)\ncursor = db.cursor()\npandas_fetch_all = pd.read_sql\n\n# 返回数据\nresult = {}\n\n# 格式化返回数据\ndef format_data(data: pd.DataFrame,\n                index_str: str = \"data_time\",\n                value_str: str = \"value\"\n                ) -> pd.DataFrame:\n    \"\"\"\n        Parameters\n        ----------\n        data : pandas的DataFrame数据结构.\n        index_str : 对象下标字符串，在pandas的DataFrame中的列名.\n        value_str : 对象值字符串，在pandas的DataFrame中的列名\n\n        Returns\n        -------\n        DataFrame or Iterator[DataFrame]\n        例子：{'2007-01-09': 3220.0, '2007-01-10': 3230.0}\n\n        \"\"\"\n    index_list = []  # 空列表\n    value_list = []  # 空列表\n\n    for num in range(1, data.index.size):  # 迭代 所有的指标\n        index_list.append(data[index_str][num])\n        value_list.append(data[value_str][num])\n\n    tmp_data = {\n        \"date\": index_list,\n        \"value\": value_list\n    }\n    pd_data = pd.DataFrame(tmp_data)\n    # print(pd_data)\n    return pd_data\n"
-	str := fmt.Sprintf("#!/usr/bin/python\n# -*- coding: UTF-8 -*-\nimport json\n\nimport pymysql\nimport pandas as pd\n\nsql_config = {\n    'host': '%s',\n    'port': 3306,#主机号\n    'user': '%s',#账户名\n    'passwd': '%s',#密码\n    'db': '%s',\n    'charset': 'utf8mb4',\n    'cursorclass': pymysql.cursors.DictCursor\n}\n\ndb = pymysql.connect(**sql_config)\ndb.autocommit(1)\ncursor = db.cursor()\npandas_fetch_all = pd.read_sql\n\n# 返回数据\nresult = {}\n\n# 格式化返回数据\ndef format_data(data: pd.DataFrame,\n                index_str: str = \"data_time\",\n                value_str: str = \"value\"\n                ) -> pd.DataFrame:\n    \"\"\"\n        Parameters\n        ----------\n        data : pandas的DataFrame数据结构.\n        index_str : 对象下标字符串，在pandas的DataFrame中的列名.\n        value_str : 对象值字符串，在pandas的DataFrame中的列名\n\n        Returns\n        -------\n        DataFrame or Iterator[DataFrame]\n        例子：{'2007-01-09': 3220.0, '2007-01-10': 3230.0}\n\n        \"\"\"\n    index_list = []  # 空列表\n    value_list = []  # 空列表\n\n    for num in range(0, data.index.size):  # 迭代 所有的指标\n        date=data[index_str][num]\n        if isinstance(date,str) is False:\n            index_list.append(data[index_str][num].strftime(\"%%Y-%%m-%%d\"))\n        else:\n            index_list.append(data[index_str][num])\n        value_list.append(data[value_str][num])\n\n    tmp_data = {\n        \"date\": index_list,\n        \"value\": value_list\n    }\n    pd_data = pd.DataFrame(tmp_data)\n    return pd_data\n\n", utils.PYTHON_MYSQL_HOST, utils.PYTHON_MYSQL_USER, utils.PYTHON_MYSQL_PASSWD, utils.PYTHON_MYSQL_DB)
+	str := fmt.Sprintf("#!/usr/bin/python\n# -*- coding: UTF-8 -*-\nimport json\n\nimport pymysql\nimport pandas as pd\nfrom pymongo import MongoClient\nimport pytz\n\nsql_config = {\n    'host': '%s',\n    'port': 3306,#主机号\n    'user': '%s',#账户名\n    'passwd': '%s',#密码\n    'db': '%s',\n    'charset': 'utf8mb4',\n    'cursorclass': pymysql.cursors.DictCursor\n}\n\ndb = pymysql.connect(**sql_config)\ndb.autocommit(1)\ncursor = db.cursor()\npandas_fetch_all = pd.read_sql\n\n", utils.PYTHON_MYSQL_HOST, utils.PYTHON_MYSQL_USER, utils.PYTHON_MYSQL_PASSWD, utils.PYTHON_MYSQL_DB)
+
+	// mongo部分
+	if utils.PYTHON_MONGO_HOST != `` {
+		str += getPythonFront2Str()
+	}
+
+	// 格式化数据部分
+	str += getPythonFront3Str()
+
+	return str
+}
+
+// getPythonFront2Str
+// @Description: mongo部分
+// @author: Roc
+// @datetime 2024-05-07 18:38:03
+// @return string
+func getPythonFront2Str() string {
+	str := fmt.Sprintf("\nfrom pymongo import MongoClient\nfrom dateutil.tz import tzlocal\n# MongoDB 连接配置\nmongo_config = {\n    'host': '%s',  # 替换为你的 MongoDB 连接字符串\n    'database': '%s',  # 替换为你的数据库名\n    'collection': '%s'  # 替换为你的集合名\n}\n\n# 创建 MongoClient 并连接到数据库\nclient = MongoClient(mongo_config['host'])\nmgo_db = client[mongo_config['database']]\ncollection = mgo_db[mongo_config['collection']]\n\n# 定义时区\nutc_tz = pytz.utc\nlocal_tz = tzlocal()  # 本地时区", utils.PYTHON_MONGO_HOST, utils.PYTHON_MONGO_DATABASE, "edb_data_business")
+	return str
+}
+
+// getPythonFront3Str
+// @Description: 第三部分
+// @author: Roc
+// @datetime 2024-05-07 18:37:56
+// @return string
+func getPythonFront3Str() string {
+	str := fmt.Sprintf("\n# 返回数据\nresult = {}\n\n# 格式化返回数据\ndef format_data(data: pd.DataFrame,\n                index_str: str = \"data_time\",\n                value_str: str = \"value\"\n                ) -> pd.DataFrame:\n    \"\"\"\n        Parameters\n        ----------\n        data : pandas的DataFrame数据结构.\n        index_str : 对象下标字符串，在pandas的DataFrame中的列名.\n        value_str : 对象值字符串，在pandas的DataFrame中的列名\n\n        Returns\n        -------\n        DataFrame or Iterator[DataFrame]\n        例子：{'2007-01-09': 3220.0, '2007-01-10': 3230.0}\n\n        \"\"\"\n    index_list = []  # 空列表\n    value_list = []  # 空列表\n\n    for num in range(0, data.index.size):  # 迭代 所有的指标\n        date=data[index_str][num]\n        if isinstance(date,str) is False:\n            index_list.append(data[index_str][num].strftime(\"%%Y-%%m-%%d\"))\n        else:\n            index_list.append(data[index_str][num])\n        value_list.append(data[value_str][num])\n\n    tmp_data = {\n        \"date\": index_list,\n        \"value\": value_list\n    }\n    pd_data = pd.DataFrame(tmp_data)\n    return pd_data\n\n")
 	return str
 }
 

utils/common.go

@@ -1224,3 +1224,35 @@ func FloatAlmostEqual(a, b float64) bool {
 	epsilon := 1e-9 // 容差值
 	return math.Abs(a-b) <= epsilon
 }
+
+// VerifyFrequency
+// @Description: 校验频度是否合规
+// @author: Roc
+// @datetime 2024-04-26 13:30:22
+// @param frequency string 待校验的频度
+// @return bool
+func VerifyFrequency(frequency string) bool {
+	return InArrayByStr([]string{"年度", "半年度", "季度", "月度", "旬度", "周度", "日度"}, frequency)
+}
+
+// DateConvMysqlConvMongo
+// @Description: 将mysql中的日期比较符转换成mongo中的日期比较符
+// @author: Roc
+// @datetime 2024-05-08 11:03:26
+// @param dateCon string
+func DateConvMysqlConvMongo(dateCon string) string {
+	cond := ""
+	switch dateCon {
+	case "=":
+		cond = "$eq"
+	case "<":
+		cond = "$lt"
+	case "<=":
+		cond = "$lte"
+	case ">":
+		cond = "$gt"
+	case ">=":
+		cond = "$gte"
+	}
+	return cond
+}

utils/config.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	beeLogger "github.com/beego/bee/v2/logger"
 	"github.com/beego/beego/v2/server/web"
+	"github.com/qiniu/qmgo"
 	"strconv"
 )
 
@@ -13,17 +14,22 @@ var (
 	MYSQL_URL     string //数据库连接
 	MYSQL_URL_EDB string
 	MYSQL_URL_GL  string
+	MgoUrlData    string // mongodb数据库连接配置
 
-	PYTHON_MYSQL_HOST   string //python数据库链接主机地址
-	PYTHON_MYSQL_USER   string //python数据库链接账号
-	PYTHON_MYSQL_PASSWD string //python数据库链接密码
-	PYTHON_MYSQL_DB     string //python数据库链接数据库名
-	PYTHON_PATH         string //python可执行文件地址
+	PYTHON_MYSQL_HOST     string // python数据库链接主机地址
+	PYTHON_MYSQL_USER     string // python数据库链接账号
+	PYTHON_MYSQL_PASSWD   string // python数据库链接密码
+	PYTHON_MYSQL_DB       string // python数据库链接数据库名
+	PYTHON_MONGO_HOST     string // python链接mongo主机地址
+	PYTHON_MONGO_DATABASE string // python链接mongo的库名
+	PYTHON_PATH           string // python可执行文件地址
 
 	REDIS_CACHE string //缓存地址
 	//Rc          *cache.Cache //redis缓存
-	Re error       //redis错误
-	Rc RedisClient //redis缓存
+	Re            error        //redis错误
+	Rc            RedisClient  //redis缓存
+	MgoDataCli    *qmgo.Client // mongodb客户端连接
+	MgoDataDbName string       // mongodb指标数据的库名
 )
 
 // DATA_INDEX_NAME 数据指标库ES索引名称
@@ -126,11 +132,18 @@ func init() {
 	MYSQL_URL_EDB = config["mysql_url_edb"]
 	MYSQL_URL_GL = config["mysql_url_gl"]
 
+	// mongodb数据库连接配置
+	MgoUrlData = config["mgo_url_data"]
+
 	//python链接
 	PYTHON_MYSQL_HOST = config["python_mysql_host"]
 	PYTHON_MYSQL_USER = config["python_mysql_user"]
 	PYTHON_MYSQL_PASSWD = config["python_mysql_passwd"]
 	PYTHON_MYSQL_DB = config["python_mysql_db"]
+
+	PYTHON_MONGO_HOST = config["python_mongo_host"]   // python链接mongo主机地址
+	PYTHON_MONGO_DATABASE = config["python_mongo_db"] // python链接mongo的库名
+
 	PYTHON_PATH = config["python_path"]
 
 	REDIS_CACHE = config["beego_cache"]

utils/constants.go

@@ -40,7 +40,7 @@ const (
 	DATA_SOURCE_YS                                              //有色
 	DATA_SOURCE_CALCULATE_HBZ                                   //环比值->12
 	DATA_SOURCE_CALCULATE_HCZ                                   //环差值->13
-	DATA_SOURCE_CALCULATE_BP                                    //变频->14
+	DATA_SOURCE_CALCULATE_BP                                    //变频,2023-2-10 13:56:01调整为"升频"->14
 	DATA_SOURCE_GL                                              //钢联->15
 	DATA_SOURCE_ZZ                                              //郑商所->16
 	DATA_SOURCE_DL                                              //大商所->17
@@ -109,91 +109,93 @@ const (
 	DATA_SOURCE_CALCULATE_SUM = 81
 	DATA_SOURCE_CALCULATE_AVG = 82
 	DATA_SOURCE_BLOOMBERG     = 83 // bloomberg彭博数据
+	DATA_SOURCE_BUSINESS      = 84 // 来源于自有数据
 )
 
 // 指标来源的中文展示
 const (
-	DATA_SOURCE_NAME_THS                                  = `同花顺`               //同花顺
-	DATA_SOURCE_NAME_WIND                                 = `wind`              //wind
-	DATA_SOURCE_NAME_PB                                   = `彭博`                //彭博
+	DATA_SOURCE_NAME_THS                                  = `同花顺`                //同花顺
+	DATA_SOURCE_NAME_WIND                                 = `wind`                  //wind
+	DATA_SOURCE_NAME_PB                                   = `彭博`                  //彭博
 	DATA_SOURCE_NAME_CALCULATE                            = `指标运算`              //指标运算
-	DATA_SOURCE_NAME_CALCULATE_LJZZY                      = `累计值转月值`            //累计值转月
-	DATA_SOURCE_NAME_CALCULATE_TBZ                        = `同比值`               //同比值
-	DATA_SOURCE_NAME_CALCULATE_TCZ                        = `同差值`               //同差值
-	DATA_SOURCE_NAME_CALCULATE_NSZYDPJJS                  = `N数值移动平均计算`         //N数值移动平均计算
+	DATA_SOURCE_NAME_CALCULATE_LJZZY                      = `累计值转月值`          //累计值转月
+	DATA_SOURCE_NAME_CALCULATE_TBZ                        = `同比值`                //同比值
+	DATA_SOURCE_NAME_CALCULATE_TCZ                        = `同差值`                //同差值
+	DATA_SOURCE_NAME_CALCULATE_NSZYDPJJS                  = `N数值移动平均计算`     //N数值移动平均计算
 	DATA_SOURCE_NAME_MANUAL                               = `手工数据`              //手工指标
-	DATA_SOURCE_NAME_LZ                                   = `隆众`                //隆众
-	DATA_SOURCE_NAME_YS                                   = `SMM`               //有色
-	DATA_SOURCE_NAME_CALCULATE_HBZ                        = `环比值`               //环比值->12
-	DATA_SOURCE_NAME_CALCULATE_HCZ                        = `环差值`               //环差值->13
-	DATA_SOURCE_NAME_CALCULATE_BP                         = `升频`                //变频,2023-2-10 13:56:01调整为"升频"->14
-	DATA_SOURCE_NAME_GL                                   = `钢联`                //钢联->15
-	DATA_SOURCE_NAME_ZZ                                   = `郑商所`               //郑商所->16
-	DATA_SOURCE_NAME_DL                                   = `大商所`               //大商所->17
-	DATA_SOURCE_NAME_SH                                   = `上期所`               //上期所->18
-	DATA_SOURCE_NAME_CFFEX                                = `中金所`               //中金所->19
+	DATA_SOURCE_NAME_LZ                                   = `隆众`                  //隆众
+	DATA_SOURCE_NAME_YS                                   = `SMM`                   //有色
+	DATA_SOURCE_NAME_CALCULATE_HBZ                        = `环比值`                //环比值->12
+	DATA_SOURCE_NAME_CALCULATE_HCZ                        = `环差值`                //环差值->13
+	DATA_SOURCE_NAME_CALCULATE_BP                         = `升频`                  //变频,2023-2-10 13:56:01调整为"升频"->14
+	DATA_SOURCE_NAME_GL                                   = `钢联`                  //钢联->15
+	DATA_SOURCE_NAME_ZZ                                   = `郑商所`                //郑商所->16
+	DATA_SOURCE_NAME_DL                                   = `大商所`                //大商所->17
+	DATA_SOURCE_NAME_SH                                   = `上期所`                //上期所->18
+	DATA_SOURCE_NAME_CFFEX                                = `中金所`                //中金所->19
 	DATA_SOURCE_NAME_SHFE                                 = `上期能源`              //上期能源->20
-	DATA_SOURCE_NAME_GIE                                  = `欧洲天然气`             //欧洲天然气->21
+	DATA_SOURCE_NAME_GIE                                  = `欧洲天然气`            //欧洲天然气->21
 	DATA_SOURCE_NAME_CALCULATE_TIME_SHIFT                 = `时间移位`              //时间移位->22
 	DATA_SOURCE_NAME_CALCULATE_ZJPJ                       = `直接拼接`              //直接拼接->23
-	DATA_SOURCE_NAME_CALCULATE_LJZTBPJ                    = `累计值同比拼接`           //累计值同比拼接->24
-	DATA_SOURCE_NAME_LT                                   = `路透`                //路透->25
-	DATA_SOURCE_NAME_COAL                                 = `中国煤炭市场网`           //中国煤炭市场网->26
+	DATA_SOURCE_NAME_CALCULATE_LJZTBPJ                    = `累计值同比拼接`        //累计值同比拼接->24
+	DATA_SOURCE_NAME_LT                                   = `路透`                  //路透->25
+	DATA_SOURCE_NAME_COAL                                 = `中国煤炭市场网`        //中国煤炭市场网->26
 	DATA_SOURCE_NAME_PYTHON                               = `代码运算`              //python代码->27
 	DATA_SOURCE_NAME_PB_FINANCE                           = `彭博财务`              //彭博财务数据->28
-	DATA_SOURCE_NAME_GOOGLE_TRAVEL                        = `our world in data` //谷歌出行数据->29
+	DATA_SOURCE_NAME_GOOGLE_TRAVEL                        = `our world in data`     //谷歌出行数据->29
 	DATA_SOURCE_NAME_PREDICT                              = `预测指标`              //普通预测指标->30
-	DATA_SOURCE_NAME_PREDICT_CALCULATE                    = `预测指标运算`            //预测指标运算->31
+	DATA_SOURCE_NAME_PREDICT_CALCULATE                    = `预测指标运算`          //预测指标运算->31
 	DATA_SOURCE_NAME_PREDICT_CALCULATE_TBZ                = `预测同比`              //预测指标 - 同比值->32
 	DATA_SOURCE_NAME_PREDICT_CALCULATE_TCZ                = `预测同差`              //预测指标 - 同差值->33
 	DATA_SOURCE_NAME_MYSTEEL_CHEMICAL                     = `钢联化工`              //钢联化工->34
 	DATA_SOURCE_NAME_CALCULATE_CJJX                       = `超季节性`              //超季节性->35
-	DATA_SOURCE_NAME_EIA_STEO                             = `EIA STERO报告`       //eia stero报告->36
+	DATA_SOURCE_NAME_EIA_STEO                             = `EIA STERO报告`         //eia stero报告->36
 	DATA_SOURCE_NAME_CALCULATE_NHCC                       = `拟合残差`              //计算指标（拟合残差）->37
-	DATA_SOURCE_NAME_COM_TRADE                            = `UN`                //联合国商品贸易数据->38
-	DATA_SOURCE_NAME_PREDICT_CALCULATE_NSZYDPJJS          = `预测N数值移动平均计算`       //预测指标 - N数值移动平均计算 -> 39
+	DATA_SOURCE_NAME_COM_TRADE                            = `UN`                    //联合国商品贸易数据->38
+	DATA_SOURCE_NAME_PREDICT_CALCULATE_NSZYDPJJS          = `预测N数值移动平均计算` //预测指标 - N数值移动平均计算 -> 39
 	DATA_SOURCE_NAME_CALCULATE_ADJUST                     = `数据调整`              //数据调整->40
-	DATA_SOURCE_NAME_SCI                                  = `SCI`               //卓创数据（红桃三）->41
-	DATA_SOURCE_NAME_PREDICT_CALCULATE_LJZZY              = `预测累计值转月值`          //预测指标 - 累计值转月->42
-	DATA_SOURCE_NAME_PREDICT_CALCULATE_HBZ                = `预测环比值`             //预测指标 - 环比值->43
-	DATA_SOURCE_NAME_PREDICT_CALCULATE_HCZ                = `预测环差值`             //预测指标 - 环差值->44
+	DATA_SOURCE_NAME_SCI                                  = `SCI`                   //卓创数据（红桃三）->41
+	DATA_SOURCE_NAME_PREDICT_CALCULATE_LJZZY              = `预测累计值转月值`      //预测指标 - 累计值转月->42
+	DATA_SOURCE_NAME_PREDICT_CALCULATE_HBZ                = `预测环比值`            //预测指标 - 环比值->43
+	DATA_SOURCE_NAME_PREDICT_CALCULATE_HCZ                = `预测环差值`            //预测指标 - 环差值->44
 	DATA_SOURCE_NAME_PREDICT_CALCULATE_BP                 = `预测升频`              //预测指标 - 升频->45
-	DATA_SOURCE_NAME_PREDICT_CALCULATE_TIME_SHIFT         = `预测时间移位`            //预测指标 - 时间移位->46
-	DATA_SOURCE_NAME_PREDICT_CALCULATE_ZJPJ               = `预测直接拼接`            //预测指标 - 直接拼接->47
-	DATA_SOURCE_NAME_PREDICT_CALCULATE_LJZTBPJ            = `预测累计值同比拼接`         //预测指标 - 累计值同比拼接->48
-	DATA_SOURCE_NAME_PREDICT_CALCULATE_CJJX               = `预测超季节性`            //预测指标 - 超季节性->49
-	DATA_SOURCE_NAME_PREDICT_CALCULATE_NHCC               = `预测拟合残差`            //预测指标 - 计算指标（拟合残差）->50
-	DATA_SOURCE_NAME_CALCULATE_JP                         = `降频`                //降频->51
-	DATA_SOURCE_NAME_CALCULATE_NH                         = `年化`                //年化->52
+	DATA_SOURCE_NAME_PREDICT_CALCULATE_TIME_SHIFT         = `预测时间移位`          //预测指标 - 时间移位->46
+	DATA_SOURCE_NAME_PREDICT_CALCULATE_ZJPJ               = `预测直接拼接`          //预测指标 - 直接拼接->47
+	DATA_SOURCE_NAME_PREDICT_CALCULATE_LJZTBPJ            = `预测累计值同比拼接`    //预测指标 - 累计值同比拼接->48
+	DATA_SOURCE_NAME_PREDICT_CALCULATE_CJJX               = `预测超季节性`          //预测指标 - 超季节性->49
+	DATA_SOURCE_NAME_PREDICT_CALCULATE_NHCC               = `预测拟合残差`          //预测指标 - 计算指标（拟合残差）->50
+	DATA_SOURCE_NAME_CALCULATE_JP                         = `降频`                  //降频->51
+	DATA_SOURCE_NAME_CALCULATE_NH                         = `年化`                  //年化->52
 	DATA_SOURCE_NAME_CALCULATE_KSZS                       = `扩散指数`              //扩散指数->53
 	DATA_SOURCE_NAME_PREDICT_CALCULATE_JP                 = `预测降频`              //预测指标 - 计算指标(降频)->54
 	DATA_SOURCE_NAME_PREDICT_CALCULATE_NH                 = `预测年化`              //预测指标 - 计算指标(年化)->55
-	DATA_SOURCE_NAME_PREDICT_CALCULATE_KSZS               = `预测扩散指数`            //预测指标 - 计算指标(扩散指数)->56
+	DATA_SOURCE_NAME_PREDICT_CALCULATE_KSZS               = `预测扩散指数`          //预测指标 - 计算指标(扩散指数)->56
 	DATA_SOURCE_NAME_BAIINFO                              = `百川盈孚`              //百川盈孚 ->57
 	DATA_SOURCE_NAME_STOCK_PLANT                          = `存量装置`              //存量装置 ->58
-	DATA_SOURCE_NAME_CALCULATE_CORRELATION                = `相关性计算`             //相关性计算->59
-	DATA_SOURCE_NAME_NATIONAL_STATISTICS                  = `国家统计局`             //国家统计局->60
-	DATA_SOURCE_NAME_CALCULATE_LJZZJ                      = `累计值转季值`            //累计值转季 -> 61
-	DATA_SOURCE_NAME_CALCULATE_LJZ                        = `累计值`               //累计值 -> 62
-	DATA_SOURCE_NAME_CALCULATE_LJZNCZJ                    = `年初至今累计值`           //累计值(年初至今） -> 63
-	DATA_SOURCE_NAME_PREDICT_CALCULATE_LJZZJ              = `预测累计值转季值`          //预测指标 - 累计值转季->64
-	DATA_SOURCE_NAME_PREDICT_CALCULATE_LJZ                = `预测累计值`             //预测指标 - 累计值 -> 65
-	DATA_SOURCE_NAME_PREDICT_CALCULATE_LJZNCZJ            = `预测年初至今累计值`         //预测指标 - 累计值(年初至今） -> 66
-	DATA_SOURCE_NAME_CALCULATE_STANDARD_DEVIATION         = `标准差`               //标准差->67
-	DATA_SOURCE_NAME_CALCULATE_PERCENTILE                 = `百分位`               //百分位->68
-	DATA_SOURCE_NAME_PREDICT_CALCULATE_STANDARD_DEVIATION = `预测标准差`             //预测标准差->69
-	DATA_SOURCE_NAME_PREDICT_CALCULATE_PERCENTILE         = `预测百分位`             //预测百分位->70
+	DATA_SOURCE_NAME_CALCULATE_CORRELATION                = `相关性计算`            //相关性计算->59
+	DATA_SOURCE_NAME_NATIONAL_STATISTICS                  = `国家统计局`            //国家统计局->60
+	DATA_SOURCE_NAME_CALCULATE_LJZZJ                      = `累计值转季值`          //累计值转季 -> 61
+	DATA_SOURCE_NAME_CALCULATE_LJZ                        = `累计值`                //累计值 -> 62
+	DATA_SOURCE_NAME_CALCULATE_LJZNCZJ                    = `年初至今累计值`        //累计值(年初至今） -> 63
+	DATA_SOURCE_NAME_PREDICT_CALCULATE_LJZZJ              = `预测累计值转季值`      //预测指标 - 累计值转季->64
+	DATA_SOURCE_NAME_PREDICT_CALCULATE_LJZ                = `预测累计值`            //预测指标 - 累计值 -> 65
+	DATA_SOURCE_NAME_PREDICT_CALCULATE_LJZNCZJ            = `预测年初至今累计值`    //预测指标 - 累计值(年初至今） -> 66
+	DATA_SOURCE_NAME_CALCULATE_STANDARD_DEVIATION         = `标准差`                //标准差->67
+	DATA_SOURCE_NAME_CALCULATE_PERCENTILE                 = `百分位`                //百分位->68
+	DATA_SOURCE_NAME_PREDICT_CALCULATE_STANDARD_DEVIATION = `预测标准差`            //预测标准差->69
+	DATA_SOURCE_NAME_PREDICT_CALCULATE_PERCENTILE         = `预测百分位`            //预测百分位->70
 	DATA_SOURCE_NAME_FUBAO                                = `富宝数据`              //富宝数据->71
 	DATA_SOURCE_NAME_CALCULATE_ZSXY                       = `指数修匀`              //指数修匀->72
-	DATA_SOURCE_NAME_PREDICT_CALCULATE_ZSXY               = `预测指数修匀`            //预测指数修匀->73
-	DATA_SOURCE_NAME_CALCULATE_ZDYFX                      = `自定义分析`             //自定义分析->74
+	DATA_SOURCE_NAME_PREDICT_CALCULATE_ZSXY               = `预测指数修匀`          //预测指数修匀->73
+	DATA_SOURCE_NAME_CALCULATE_ZDYFX                      = `自定义分析`            //自定义分析->74
 	DATA_SOURCE_NAME_YONYI                                = `涌益咨询`              // 涌益咨询
+	DATA_SOURCE_NAME_ICPI                                 = "ICPI消费价格指数"
+	DATA_SOURCE_NAME_FENWEI                               = `汾渭数据` // 汾渭煤炭
+	DATA_SOURCE_NAME_MTJH                                 = `煤炭江湖` // 煤炭江湖->80
 
-	DATA_SOURCE_NAME_FENWEI        = `汾渭数据` // 汾渭煤炭
-	DATA_SOURCE_NAME_MTJH          = `煤炭江湖` // 煤炭江湖->80
-	DATA_SOURCE_NAME_ICPI          = "ICPI消费价格指数"
 	DATA_SOURCE_NAME_CALCULATE_SUM = `多指标求和`
 	DATA_SOURCE_NAME_CALCULATE_AVG = `多指标求平均`
+	DATA_SOURCE_NAME_BUSINESS      = `自有数据`
 )
 
 // 基础数据初始化日期
@@ -217,6 +219,7 @@ const (
 
 const (
 	CACHE_EDB_DATA_ADD               = "CACHE_EDB_DATA_ADD_"
+	CACHE_EDB_DATA_EDIT              = "CACHE_EDB_DATA_EDIT_"
 	CACHE_EDB_DATA_REFRESH           = "CACHE_EDB_DATA_REFRESH_"
 	CACHE_WIND_URL                   = "CACHE_WIND_URL"
 	CACHE_CHART_INFO_DATA            = "chart:info:data:"                 //图表数据

utils/logs.go

@@ -16,6 +16,7 @@ const (
 var FileLog *logs.BeeLogger
 var ApiLog *logs.BeeLogger
 var Binlog *logs.BeeLogger
+var MongoLog *logs.BeeLogger
 
 func init() {
 	if LogMaxDays == 0 {
@@ -43,6 +44,7 @@ func init() {
 
 	initBinlog()
 	initApiLog()
+	initMgoLog()
 }
 
 type logConfig struct {
@@ -103,6 +105,41 @@ func initApiLog() {
 	ApiLog.EnableFuncCallDepth(true)
 }
 
+// initMgoLog
+// @Description: mongo日志
+// @author: Roc
+// @datetime 2024-05-06 16:42:47
+func initMgoLog() {
+	//mgo日志
+
+	var logPath string
+	{
+		binlogPath := BinLogPath
+		if binlogPath == "" {
+			binlogPath = DefaultBinlogPath
+		}
+		logPath = path.Dir(binlogPath)
+	}
+	if logPath == `` {
+		logPath = "./etalogs"
+	}
+
+	mongoLogPath := logPath + "/mongolog"
+	mongoLogFile := "mongolog.log"
+
+	os.MkdirAll(mongoLogPath, os.ModePerm)
+	logFileName := path.Join(mongoLogPath, mongoLogFile)
+	MongoLog = logs.NewLogger(1000000)
+	logConf := getDefaultLogConfig()
+
+	logConf.FileName = logFileName
+	//logConf.MaxLines = 10000000
+	//logConf.Rotate = true
+	b, _ := json.Marshal(logConf)
+	MongoLog.SetLogger(logs.AdapterFile, string(b))
+	MongoLog.EnableFuncCallDepth(true)
+}
+
 func getDefaultLogConfig() logConfig {
 	return logConfig{
 		FileName: "",

utils/mgodb/mgo_config.go

@@ -0,0 +1,127 @@
+package mgodb
+
+import (
+	"context"
+	"eta/eta_index_lib/utils"
+	"github.com/qiniu/qmgo"
+	"github.com/qiniu/qmgo/options"
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/event"
+	mgoptions "go.mongodb.org/mongo-driver/mongo/options"
+	"sync"
+)
+
+var (
+	ConnectTimeoutMS = int64(3000)
+	SocketTimeoutMS  = int64(500000)
+	MaxPoolSize      = uint64(50)
+	MinPoolSize      = uint64(0)
+)
+
+//func init() {
+//	fmt.Println("start MgoNewClient")
+//	MgoNewClient()
+//	fmt.Println("end MgoNewClient")
+//}
+
+type MgoConfig struct {
+	Host       string `json:"host"`
+	Port       string `json:"port"`
+	Username   string `json:"username"`
+	Password   string `json:"password"`
+	AuthSource string `json:"authSource"`
+	Database   string `json:"database"`
+}
+
+// MgoNewClient
+// @Description: 创建一个mongodb客户端链接
+// @author: Roc
+// @datetime 2024-04-25 14:01:14
+// @return *qmgo.Client
+func MgoNewClient(mgoConfig MgoConfig) *qmgo.Client {
+	// 创建cmdMonitor，用于打印SQL
+	//startedCommands := make(map[int64]bson.Raw)
+	startedCommands := sync.Map{} // map[int64]bson.Raw
+
+	cmdMonitor := &event.CommandMonitor{
+		Started: func(_ context.Context, evt *event.CommandStartedEvent) {
+			startedCommands.Store(evt.RequestID, evt.Command)
+			//startedCommands[evt.RequestID] = evt.Command
+		},
+		Succeeded: func(_ context.Context, evt *event.CommandSucceededEvent) {
+			//log.Printf("Command: %v Reply: %v\n",
+			//	startedCommands[evt.RequestID],
+			//	evt.Reply,
+			//)
+			var commands bson.Raw
+			v, ok := startedCommands.Load(evt.RequestID)
+			if ok {
+				commands = v.(bson.Raw)
+			}
+			utils.MongoLog.Info("\n【MongoDB】[%.3fms] [%v] %v \n", float64(evt.Duration)/1e6, commands, evt.Reply)
+		},
+		Failed: func(_ context.Context, evt *event.CommandFailedEvent) {
+			//log.Printf("Command: %v Failure: %v\n",
+			//	startedCommands[evt.RequestID],
+			//	evt.Failure,
+			//)
+			var commands bson.Raw
+			v, ok := startedCommands.Load(evt.RequestID)
+			if ok {
+				commands = v.(bson.Raw)
+			}
+			utils.MongoLog.Info("\n【MongoDB】[%.3fms] [%v] \n %v \n", float64(evt.Duration)/1e6, commands, evt.Failure)
+		},
+	}
+
+	// 创建options
+	ops := options.ClientOptions{ClientOptions: &mgoptions.ClientOptions{}}
+	ops.SetMonitor(cmdMonitor)
+
+	ctx := context.Background()
+
+	//mongodb+srv://myDatabaseUser:D1fficultP%40ssw0rd@mongodb0.example.com/?authSource=admin&replicaSet=myRepl
+
+	var mongoUrl string
+	if mgoConfig.Password != "" {
+		mongoUrl = "mongodb://" + mgoConfig.Username + ":" + mgoConfig.Password + "@" +
+			mgoConfig.Host + ":" + mgoConfig.Port
+
+		if mgoConfig.AuthSource != `` {
+			mongoUrl += mongoUrl + "?&authSource=" + mgoConfig.AuthSource
+		}
+	} else {
+		mongoUrl = "mongodb://" + mgoConfig.Host + ":" + mgoConfig.Port
+	}
+
+	// 创建一个数据库链接
+	client, err := qmgo.NewClient(ctx, &qmgo.Config{
+		Uri:              mongoUrl,
+		Database:         mgoConfig.Database,
+		ConnectTimeoutMS: &ConnectTimeoutMS,
+		SocketTimeoutMS:  &SocketTimeoutMS,
+		MaxPoolSize:      &MaxPoolSize,
+		MinPoolSize:      &MinPoolSize,
+	}, ops)
+
+	if err != nil {
+		panic("MongoDB连接异常：" + err.Error())
+	}
+
+	return client
+}
+
+func MgoGetColl(collName string) (cli *qmgo.QmgoClient, err error) {
+	ctx := context.Background()
+	config := &qmgo.Config{
+		//Uri:              MGO_URL,
+		//Database:         MGO_DB,
+		Coll:             collName,
+		ConnectTimeoutMS: &ConnectTimeoutMS,
+		SocketTimeoutMS:  &SocketTimeoutMS,
+		MaxPoolSize:      &MaxPoolSize,
+		MinPoolSize:      &MinPoolSize,
+	}
+	cli, err = qmgo.Open(ctx, config)
+	return
+}