Merge branch 'debug' of http://8.136.199.33:3000/eta_server/eta_index_lib into bzq1/shanghai_conflict

.gitignore

@@ -12,4 +12,5 @@ eta_index_lib
 /static/imgs/python/*
 /etalogs
 *.gz
-*.exe
+*.exe
+/.vscode/

controllers/base_from_calculate.go

@@ -8,6 +8,7 @@ import (
 	"eta/eta_index_lib/services"
 	"eta/eta_index_lib/utils"
 	"fmt"
+	"sort"
 	"strconv"
 	"strings"
 	"sync"
@@ -248,6 +249,11 @@ func (this *CalculateController) Edit() {
 		br.Msg = "指标信息不存在，EdbInfoId:" + strconv.Itoa(req.EdbInfoId)
 		return
 	}
+	// 记录指标原始的基本信息
+	oldEdbName := edbInfoDetail.EdbName
+	oldFrequency := edbInfoDetail.Frequency
+	oldUnit := edbInfoDetail.Unit
+
 	var needCalculate bool
 
 	if edbInfoDetail.CalculateFormula != req.CalculateFormula || edbInfoDetail.EmptyType != req.EmptyType || edbInfoDetail.MaxEmptyType != req.MaxEmptyType || edbInfoDetail.Extra != req.Extra {
@@ -332,10 +338,31 @@ func (this *CalculateController) Edit() {
 		br.ErrMsg = err.Error()
 		return
 	}
+	// 记录基础信息变更日志
+	oldEdbInfo := new(models.EdbInfo)
+	oldEdbInfo.EdbInfoId = edbInfoDetail.EdbInfoId
+	oldEdbInfo.EdbName = oldEdbName
+	oldEdbInfo.Frequency = oldFrequency
+	oldEdbInfo.Unit = oldUnit
+	newEdbInfoEditRecord := new(models.EdbInfoEditRecord)
+	newEdbInfoEditRecord.EdbName = req.EdbName
+	newEdbInfoEditRecord.Frequency = req.Frequency
+	newEdbInfoEditRecord.Unit = req.Unit
+	newEdbInfoEditRecord.OperateUserId = req.AdminId
+	newEdbInfoEditRecord.OperateUserRealName = req.AdminName
+	err = services.AddEditEdbInfoRecord(oldEdbInfo, newEdbInfoEditRecord)
+	if err != nil {
+		br.Msg = "记录基础信息变更日志失败"
+		br.ErrMsg = err.Error()
+		return
+	}
+
 	resp := models.AddEdbInfoResp{
 		EdbInfoId:  edbInfoDetail.EdbInfoId,
 		UniqueCode: edbInfoDetail.UniqueCode,
 	}
+	// 重置计算指标中的引用关系
+	go services.ResetEdbRelation(edbInfoDetail.EdbInfoId)
 	br.Ret = 200
 	br.Success = true
 	br.Msg = "保存成功"
@@ -778,6 +805,19 @@ func (this *CalculateController) BatchSave() {
 		return
 	}
 
+	newEdbInfo := new(models.EdbInfoEditRecord)
+	newEdbInfo.EdbName = req.EdbName
+	newEdbInfo.Frequency = req.Frequency
+	newEdbInfo.Unit = req.Unit
+	newEdbInfo.OperateUserId = req.AdminId
+	newEdbInfo.OperateUserRealName = req.AdminName
+	err = services.AddEditEdbInfoRecord(edbInfo, newEdbInfo)
+	if err != nil {
+		br.Msg = "保存失败"
+		br.ErrMsg = "保存失败,Err:" + err.Error()
+		return
+	}
+
 	resp := models.AddEdbInfoResp{
 		EdbInfoId:  edbInfo.EdbInfoId,
 		UniqueCode: edbInfo.UniqueCode,
@@ -861,6 +901,10 @@ func (this *CalculateController) BatchEdit() {
 		br.ErrMsg = "获取指标信息失败:Err:" + err.Error()
 		return
 	}
+	// 记录原始指标信息
+	oldEdbName := edbInfo.EdbName
+	oldFrequency := edbInfo.Frequency
+	oldUnit := edbInfo.Unit
 
 	// 基础指标id
 	fromEdbInfoId := req.FromEdbInfoId
@@ -1207,6 +1251,26 @@ func (this *CalculateController) BatchEdit() {
 		br.ErrMsg = err.Error()
 		return
 	}
+	// 记录指标的操作记录
+	oldEdbInfo := new(models.EdbInfo)
+	oldEdbInfo.EdbInfoId = edbInfo.EdbInfoId
+	oldEdbInfo.EdbName = oldEdbName
+	oldEdbInfo.Frequency = oldFrequency
+	oldEdbInfo.Unit = oldUnit
+	newEdbInfoRecord := new(models.EdbInfoEditRecord)
+	newEdbInfoRecord.EdbName = req.EdbName
+	newEdbInfoRecord.Frequency = req.Frequency
+	newEdbInfoRecord.Unit = req.Unit
+	newEdbInfoRecord.OperateUserId = req.AdminId
+	newEdbInfoRecord.OperateUserRealName = req.AdminName
+	err = services.AddEditEdbInfoRecord(oldEdbInfo, newEdbInfoRecord)
+	if err != nil {
+		br.Msg = "保存失败"
+		br.ErrMsg = "记录指标操作记录失败，Err:" + err.Error()
+	}
+
+	// 重置计算指标中的引用关系
+	go services.ResetEdbRelation(edbInfoId)
 
 	resp := models.AddEdbInfoResp{
 		EdbInfoId:  edbInfo.EdbInfoId,
@@ -1767,6 +1831,9 @@ func (this *CalculateController) SaveAdjust() {
 		EdbInfoId:  edbInfo.EdbInfoId,
 		UniqueCode: edbInfo.UniqueCode,
 	}
+
+	// 重置计算指标中的引用关系
+	go services.ResetEdbRelation(edbInfo.EdbInfoId)
 	br.Ret = 200
 	br.Success = true
 	br.Msg = "保存成功"
@@ -2247,6 +2314,8 @@ func (this *CalculateController) BatchEditMulti() {
 			Frequency:        item.Frequency,
 			Unit:             item.Unit,
 			ClassifyId:       item.ClassifyId,
+			AdminId:          req.AdminId,
+			AdminName:        req.AdminName,
 			Formula:          req.Formula, //N数值移动平均计算、环比值、环差值
 			FromEdbInfoId:    item.FromEdbInfoId,
 			CalculateFormula: req.CalculateFormula,
@@ -2302,3 +2371,111 @@ func (this *CalculateController) BatchEditMulti() {
 	br.Data = resp
 	br.IsAddLog = true
 }
+
+// StepCalculate
+// @Title 多步骤计算
+// @Description 多步骤计算
+// @Param request body models.StepCalculateBySearchData true "type json string"
+// @Success Ret=200 返回指标id
+// @router /base/step_calculate [post]
+func (this *CalculateController) StepCalculate() {
+	br := new(models.BaseResponse).Init()
+	defer func() {
+		if br.ErrMsg == "" {
+			br.IsSendEmail = false
+		}
+		this.Data["json"] = br
+		this.ServeJSON()
+	}()
+	var req models.StepCalculateBySearchData
+	if e := json.Unmarshal(this.Ctx.Input.RequestBody, &req); e != nil {
+		br.Msg = "参数解析异常"
+		br.ErrMsg = fmt.Sprintf("参数解析失败, Err: %v", e)
+		return
+	}
+	sort.Slice(req.Calculates, func(i, j int) bool {
+		return req.Calculates[i].Sort < req.Calculates[j].Sort
+	})
+
+	var errMsg string
+	originData, e := models.EdbInfoSearchDataToData(req.DataList)
+	if e != nil {
+		br.Msg = "基础数据异常"
+		br.ErrMsg = fmt.Sprintf("计算失败, 基础数据异常, Err: %v", e)
+		return
+	}
+
+	calculateData := originData
+	dateDataMap := make(map[time.Time]float64)
+	for _, v := range req.Calculates {
+		baseCalculate := models.BaseCalculate{
+			DataList:      calculateData,
+			Frequency:     v.Frequency,
+			Formula:       v.Formula,
+			Calendar:      v.Calendar,
+			MoveType:      v.MoveType,
+			MoveFrequency: v.MoveFrequency,
+			FromFrequency: v.FromFrequency,
+			Source:        v.Source,
+		}
+
+		// 计算方式
+		switch baseCalculate.Source {
+		case utils.EdbBaseCalculateLjzzy:
+			dateDataMap, e, errMsg = baseCalculate.Ljzzy()
+		case utils.EdbBaseCalculateLjzzj:
+			dateDataMap, e, errMsg = baseCalculate.Ljzzj()
+		case utils.EdbBaseCalculateTbz:
+			dateDataMap, e, errMsg = baseCalculate.Tbz()
+		case utils.EdbBaseCalculateTcz:
+			dateDataMap, e, errMsg = baseCalculate.Tcz()
+		case utils.EdbBaseCalculateNszydpjjs:
+			dateDataMap, e, errMsg = baseCalculate.Nszydpjjs()
+		case utils.EdbBaseCalculateHbz:
+			dateDataMap, e, errMsg = baseCalculate.Hbz()
+		case utils.EdbBaseCalculateHcz:
+			dateDataMap, e, errMsg = baseCalculate.Hcz()
+		case utils.EdbBaseCalculateUpFrequency:
+			dateDataMap, e, errMsg = baseCalculate.UpFrequency()
+		case utils.EdbBaseCalculateDownFrequency:
+			dateDataMap, e, errMsg = baseCalculate.DownFrequency()
+		case utils.EdbBaseCalculateTimeShift:
+			dateDataMap, e, errMsg = baseCalculate.TimeShift()
+		case utils.EdbBaseCalculateCjjx:
+			dateDataMap, e, errMsg = baseCalculate.Cjjx()
+		case utils.EdbBaseCalculateAnnualized:
+			dateDataMap, e, errMsg = baseCalculate.Annualized()
+		case utils.EdbBaseCalculateLjz:
+			dateDataMap, e, errMsg = baseCalculate.Ljz()
+		case utils.EdbBaseCalculateLjzNczj:
+			dateDataMap, e, errMsg = baseCalculate.LjzNczj()
+		case utils.EdbBaseCalculateExponentialSmoothing:
+			dateDataMap, e, errMsg = baseCalculate.ExponentialSmoothing()
+		case utils.EdbBaseCalculateRjz:
+			dateDataMap, e, errMsg = baseCalculate.Rjz()
+		default:
+			errMsg = "计算方式无效"
+			e = fmt.Errorf("%s:%d", errMsg, baseCalculate.Source)
+		}
+		if e != nil {
+			br.Msg = "计算失败"
+			if errMsg != "" {
+				br.Msg = errMsg
+			}
+			br.ErrMsg = e.Error()
+			return
+		}
+
+		calculateData = models.TransDateData2EdbData(dateDataMap)
+	}
+	resultData, dates := models.GetDateDataAndDateList(dateDataMap)
+
+	br.Ret = 200
+	br.Success = true
+	br.Msg = "计算成功"
+	br.Data = models.BaseCalculateResp{
+		DataMap:  resultData,
+		DateList: dates,
+	}
+	br.IsAddLog = true
+}

controllers/base_from_ccf.go

@@ -0,0 +1,217 @@
+package controllers
+
+import (
+	"encoding/json"
+	"eta/eta_index_lib/logic"
+	"eta/eta_index_lib/models"
+	"eta/eta_index_lib/services"
+	"eta/eta_index_lib/utils"
+	"strconv"
+	"time"
+)
+
+// CCFController CCF化纤信息
+type CCFController struct {
+	BaseAuthController
+}
+
+// Add
+// @Title 新增CCF指标接口
+// @Description 新增CCF指标接口
+// @Success 200 {object} models.AddEdbInfoReq
+// @router /add [post]
+func (this *CCFController) Add() {
+	br := new(models.BaseResponse).Init()
+	var cacheKey string
+	defer func() {
+		if br.ErrMsg == "" {
+			br.IsSendEmail = false
+		}
+		this.Data["json"] = br
+		this.ServeJSON()
+	}()
+	var req models.AddEdbInfoReq
+	err := json.Unmarshal(this.Ctx.Input.RequestBody, &req)
+	if err != nil {
+		br.Msg = "参数解析异常！"
+		br.ErrMsg = "参数解析失败，Err:" + err.Error()
+		return
+	}
+	if req.EdbCode == "" {
+		br.Msg = "请输入指标编码！"
+		br.ErrMsg = "请输入指标编码，指标编码为空"
+		return
+	}
+
+	ob := new(models.BaseFromCCF)
+	cacheKey = utils.CACHE_EDB_DATA_ADD + strconv.Itoa(ob.GetSource()) + "_" + req.EdbCode
+	if utils.Rc.IsExist(cacheKey) {
+		br.Ret = 501
+		br.Success = true
+		br.Msg = "系统处理中，请稍后重试"
+		return
+	}
+	utils.Rc.SetNX(cacheKey, 1, 1*time.Minute)
+	defer func() {
+		_ = utils.Rc.Delete(cacheKey)
+	}()
+
+	if err = ob.Add(req.EdbCode); err != nil {
+		br.Msg = "获取指标信息失败！"
+		br.ErrMsg = "获取指标信息失败 AddEdbDataFromCCF，Err:" + err.Error()
+		return
+	}
+	br.Ret = 200
+	br.Success = true
+	br.Msg = "获取成功"
+}
+
+// Refresh
+// @Title 刷新CCF指标接口
+// @Description 刷新CCF指标接口
+// @Success 200 {object} models.RefreshEdbInfoReq
+// @router /refresh [post]
+func (this *CCFController) Refresh() {
+	br := new(models.BaseResponse).Init()
+	var cacheKey string
+	defer func() {
+		if br.ErrMsg == "" {
+			br.IsSendEmail = false
+		}
+		this.Data["json"] = br
+		this.ServeJSON()
+	}()
+	var req models.RefreshEdbInfoReq
+	err := json.Unmarshal(this.Ctx.Input.RequestBody, &req)
+	if err != nil {
+		br.Msg = "参数解析异常！"
+		br.ErrMsg = "参数解析失败，Err:" + err.Error()
+		return
+	}
+	if req.EdbCode == "" {
+		br.Msg = "请输入指标编码！"
+		br.ErrMsg = "请输入指标编码，指标编码为空"
+		return
+	}
+	if req.EdbInfoId < 0 {
+		br.Msg = "请输入指标ID！"
+		br.ErrMsg = "请输入指标ID"
+		return
+	}
+
+	ob := new(models.BaseFromCCF)
+	source := ob.GetSource()
+	// 获取指标详情
+	edbInfo, err := models.GetEdbInfoByEdbCode(source, req.EdbCode)
+	if err != nil {
+		br.Msg = "指标不存在！"
+		br.ErrMsg = "指标不存在"
+		return
+	}
+	cacheKey = utils.CACHE_EDB_DATA_REFRESH + strconv.Itoa(source) + "_" + req.EdbCode
+	if utils.Rc.IsExist(cacheKey) {
+		br.Ret = 501
+		br.Success = true
+		br.Msg = "系统处理中，请稍后重试"
+		return
+	}
+	utils.Rc.SetNX(cacheKey, 1, 1*time.Minute)
+	defer func() {
+		_ = utils.Rc.Delete(cacheKey)
+	}()
+	if req.EdbInfoId <= 0 {
+		req.EdbInfoId = edbInfo.EdbInfoId
+	}
+	err = ob.Refresh(req.EdbInfoId, req.EdbCode, req.StartDate)
+	if err != nil && err.Error() != utils.ErrNoRow() {
+		br.Msg = "刷新指标信息失败！"
+		br.ErrMsg = "刷新指标信息失败 RefreshEdbDataFromCCF，Err:" + err.Error()
+		return
+	}
+	// 更新指标最大最小值
+	err, errMsg := models.UnifiedModifyEdbInfoMaxAndMinInfo(edbInfo)
+	if err != nil {
+		br.Msg = errMsg
+		br.ErrMsg = err.Error()
+		return
+	}
+
+	// 更新ES
+	go logic.UpdateEs(edbInfo.EdbInfoId)
+
+	br.Ret = 200
+	br.Success = true
+	br.Msg = "获取成功"
+}
+
+// HandleEdbData
+// @Title 处理CCF指标的接口
+// @Description 处理CCF指标的接口
+// @Success 200 string "操作成功"
+// @router /handle/edb_data [post]
+func (this *CCFController) HandleEdbData() {
+	br := new(models.BaseResponse).Init()
+	defer func() {
+		if br.ErrMsg == "" {
+			br.IsSendEmail = false
+		}
+		this.Data["json"] = br
+		this.ServeJSON()
+	}()
+	var req models.HandleCCFEdbDataReq
+	err := json.Unmarshal(this.Ctx.Input.RequestBody, &req)
+	if err != nil {
+		br.Msg = "参数解析异常！"
+		br.ErrMsg = "参数解析失败，Err:" + err.Error()
+		return
+	}
+
+	// 处理excel表数据
+	if err = services.HandleCCFIndex(&req); err != nil {
+		br.Msg = "处理失败"
+		br.ErrMsg = "处理失败,Err:" + err.Error()
+		return
+	}
+
+	br.Ret = 200
+	br.Success = true
+	br.Msg = "处理成功"
+}
+
+// HandleTableData
+// @Title 处理CCF装置表格的接口
+// @Description 处理CCF装置表格的接口
+// @Success 200 string "操作成功"
+// @router /handle/table_data [post]
+func (this *CCFController) HandleTableData() {
+	br := new(models.BaseResponse).Init()
+	defer func() {
+		if br.ErrMsg == "" {
+			br.IsSendEmail = false
+		}
+		this.Data["json"] = br
+		this.ServeJSON()
+	}()
+	var req models.HandleCCFStockTableReq
+	err := json.Unmarshal(this.Ctx.Input.RequestBody, &req)
+	if err != nil {
+		br.Msg = "参数解析异常！"
+		br.ErrMsg = "参数解析失败，Err:" + err.Error()
+		return
+	}
+	if req.Table == nil {
+		br.Msg = "表格信息为空"
+		br.ErrMsg = "表格信息为空"
+		return
+	}
+
+	if err = services.HandleCCFStockTable(&req); err != nil {
+		br.Msg = "处理失败"
+		br.ErrMsg = "处理失败,Err:" + err.Error()
+		return
+	}
+
+	br.Ret = 200
+	br.Success = true
+	br.Msg = "处理成功"
+}

controllers/base_from_predict_calculate.go

@@ -4,6 +4,7 @@ import (
 	"encoding/json"
 	"eta/eta_index_lib/logic"
 	"eta/eta_index_lib/models"
+	"eta/eta_index_lib/services"
 	"eta/eta_index_lib/utils"
 	"fmt"
 	"strconv"
@@ -383,6 +384,11 @@ func editPredictCalculate(br *models.BaseResponse, req models.EdbInfoCalculateSa
 		return
 	}
 
+	// 记录旧的指标信息
+	oldEdbName := edbInfo.EdbName
+	oldFrequency := edbInfo.Frequency
+	oldUnit := edbInfo.Unit
+
 	// 根据指标名称和指标ID校验库中是否还存在其他同名指标
 	existEdbName, err := logic.CheckExistByEdbNameAndEdbInfoId(edbInfo.EdbInfoType, edbInfo.EdbInfoId, req.EdbName, lang)
 	if err != nil {
@@ -506,6 +512,29 @@ func editPredictCalculate(br *models.BaseResponse, req models.EdbInfoCalculateSa
 		br.ErrMsg = err.Error()
 		return
 	}
+
+	// 记录基础信息变更日志
+	oldEdbInfo := new(models.EdbInfo)
+	oldEdbInfo.EdbInfoId = req.EdbInfoId
+	oldEdbInfo.EdbName = oldEdbName
+	oldEdbInfo.Frequency = oldFrequency
+	oldEdbInfo.Unit = oldUnit
+	newEdbInfoEditRecord := new(models.EdbInfoEditRecord)
+	newEdbInfoEditRecord.EdbName = req.EdbName
+	newEdbInfoEditRecord.Frequency = req.Frequency
+	newEdbInfoEditRecord.Unit = req.Unit
+	newEdbInfoEditRecord.OperateUserId = req.AdminId
+	newEdbInfoEditRecord.OperateUserRealName = req.AdminName
+	err = services.AddEditEdbInfoRecord(oldEdbInfo, newEdbInfoEditRecord)
+	if err != nil {
+		br.Msg = "记录基础信息变更日志失败"
+		br.ErrMsg = err.Error()
+		return
+	}
+
+	// 重置计算指标中的引用关系
+	go services.ResetEdbRelation(edbInfo.EdbInfoId)
+
 	resp := models.AddEdbInfoResp{
 		EdbInfoId:  edbInfo.EdbInfoId,
 		UniqueCode: edbInfo.UniqueCode,
@@ -687,6 +716,17 @@ func (this *PredictCalculateController) CalculateBatchSave() {
 		return
 	}
 
+	// 记录原始指标信息
+	var oldEdbInfo *models.EdbInfo
+	if req.EdbInfoId > 0 {
+		oldEdbInfo, err = models.GetEdbInfoById(req.EdbInfoId)
+		if err != nil {
+			br.Msg = "获取指标信息失败"
+			br.ErrMsg = "获取指标信息失败:Err:" + err.Error()
+			return
+		}
+	}
+
 	// 来源预测指标信息
 	var fromEdbInfo *models.EdbInfo
 	if fromEdbInfoId > 0 {
@@ -875,6 +915,22 @@ func (this *PredictCalculateController) CalculateBatchSave() {
 		return
 	}
 
+	if req.EdbInfoId > 0 {
+		// 记录操作变更记录
+		newEdbInfo := new(models.EdbInfoEditRecord)
+		newEdbInfo.EdbName = req.EdbName
+		newEdbInfo.Frequency = req.Frequency
+		newEdbInfo.Unit = req.Unit
+		newEdbInfo.OperateUserId = req.AdminId
+		newEdbInfo.OperateUserRealName = req.AdminName
+		err = services.AddEditEdbInfoRecord(oldEdbInfo, newEdbInfo)
+		if err != nil {
+			br.Msg = "保存失败"
+			br.ErrMsg = "保存失败,Err:" + err.Error()
+			return
+		}
+	}
+
 	resp := models.AddEdbInfoResp{
 		EdbInfoId:  edbInfo.EdbInfoId,
 		UniqueCode: uniqueCode,

controllers/base_from_python.go

@@ -82,7 +82,8 @@ func (this *PythonController) Add() {
 		br.ErrMsg = "参数解析失败，Err:" + err.Error()
 		return
 	}
-	req.EdbName = strings.Trim(req.EdbName, " ")
+	// 移除首尾空格
+	req.EdbName = strings.TrimSpace(req.EdbName)
 	if req.EdbName == "" {
 		br.Msg = "指标名称不能为空"
 		return
@@ -273,7 +274,9 @@ func (this *PythonController) Edit() {
 		br.ErrMsg = "参数解析失败，Err:" + err.Error()
 		return
 	}
-	req.EdbName = strings.Trim(req.EdbName, " ")
+
+	// 移除首尾空格
+	req.EdbName = strings.TrimSpace(req.EdbName)
 
 	if req.EdbInfoId <= 0 {
 		br.Msg = "指标id不能为空"
@@ -363,8 +366,8 @@ func (this *PythonController) Edit() {
 			return
 		}
 	}
-	edbInfo.EdbName = utils.TrimStr(req.EdbName)
-	edbInfo.EdbNameSource = utils.TrimStr(req.EdbName)
+	edbInfo.EdbName = req.EdbName
+	edbInfo.EdbNameSource = req.EdbName
 	edbInfo.Unit = req.Unit
 	edbInfo.ClassifyId = req.ClassifyId
 	edbInfo.Frequency = req.Frequency

controllers/base_from_wind.go

@@ -57,6 +57,7 @@ func (this *WindController) Add() {
 
 		// 试用平台的话，需要额外从弘则这边获取下地址
 		if utils.BusinessCode == utils.HZ_TRIAL_BUSSINESS_CODE {
+			// todo 从体验版中获取对应的终端信息
 			tmpWindUrl, tmpTerminalCode, err, errMsg := services.GetHzWindUrl(req.EdbCode, source)
 			if err != nil {
 				br.Msg = "添加失败！"

controllers/base_from_wind_wsd.go

@@ -53,7 +53,6 @@ func (this *WindWsdController) Add() {
 
 		windUrl := ``
 		terminalCode := ""
-
 		// 试用平台的话，需要额外从弘则这边获取下地址
 		if utils.BusinessCode == utils.HZ_TRIAL_BUSSINESS_CODE {
 			tmpWindUrl, tmpTerminalCode, err, errMsg := services.GetHzWindUrl(req.EdbCode, source)

logic/base_edb_info.go

@@ -31,6 +31,9 @@ func RefreshBaseEdbInfo(edbInfo *models.EdbInfo, startDate string) (isHandling b
 		err = models.RefreshEdbDataFromSmm(edbInfo.EdbInfoId, edbInfo.EdbCode, startDate)
 	case utils.DATA_SOURCE_BLOOMBERG:
 		err = models.RefreshEdbDataFromBloomberg(edbInfo.EdbInfoId, edbInfo.EdbCode, startDate)
+	case utils.DATA_SOURCE_CCF:
+		ccfOb := new(models.BaseFromCCF)
+		err = ccfOb.Refresh(edbInfo.EdbInfoId, edbInfo.EdbCode, startDate)
 	default:
 		return
 	}

logic/profit_chart_info.go

@@ -261,10 +261,10 @@ func GetProfitChartEdbData(baseEdbInfo *models.EdbInfo, zlFutureGoodEdbInfoList
 	// 特殊的商品期货合约（只有M+N的合约，没有固定日期的合约）
 	specialFutureGoodEdbInfoMap := make(map[int]map[int]*future_good.FutureGoodEdbInfo, 0)
 
-	isAllChina := true // 是否都是国内的期货合约
+	hasChina := false // 是否包含国内的期货合约
 	for _, v := range zlFutureGoodEdbInfoList {
-		if v.RegionType != "国内" {
-			isAllChina = false
+		if v.RegionType == "国内" {
+			hasChina = true
 			break
 		}
 	}
@@ -283,7 +283,7 @@ func GetProfitChartEdbData(baseEdbInfo *models.EdbInfo, zlFutureGoodEdbInfoList
 			return
 		}
 
-		childFutureGoodEdbInfoMap, tmpMaxN, tmpErr := getProfitFutureGoodEdbInfoList(earliestDateTime, v, tmpFutureGoodEdbInfoList, isAllChina, monthNum)
+		childFutureGoodEdbInfoMap, tmpMaxN, tmpErr := getProfitFutureGoodEdbInfoList(earliestDateTime, v, tmpFutureGoodEdbInfoList, monthNum)
 		if tmpErr != nil {
 			err = tmpErr
 			return
@@ -322,6 +322,11 @@ func GetProfitChartEdbData(baseEdbInfo *models.EdbInfo, zlFutureGoodEdbInfoList
 		}
 	}
 
+	// 需求池604，只要包含了国内合约，最大必须是12期
+	if hasChina {
+		maxN = 12
+	}
+
 	// 找出所有的N值，并进行正序排列
 	dateList := make([]string, 0)
 	for _, n := range nMap {
@@ -570,7 +575,7 @@ func ProfitChartChartData(baseDataList []*models.EdbDataList, futureGoodEdbInfoM
 }
 
 // getFutureGoodEdbInfoList 获取适用的指标列表
-func getProfitFutureGoodEdbInfoList(earliestDateTime time.Time, zlFutureGoodEdbInfo *future_good.FutureGoodEdbInfo, tmpFutureGoodEdbInfoList []*future_good.FutureGoodEdbInfo, isAllChina bool, monthNum int) (futureGoodEdbInfoDateMap map[string]*future_good.FutureGoodEdbInfo, newMaxN int, err error) {
+func getProfitFutureGoodEdbInfoList(earliestDateTime time.Time, zlFutureGoodEdbInfo *future_good.FutureGoodEdbInfo, tmpFutureGoodEdbInfoList []*future_good.FutureGoodEdbInfo, monthNum int) (futureGoodEdbInfoDateMap map[string]*future_good.FutureGoodEdbInfo, newMaxN int, err error) {
 	maxN := 36 //最大36期合约
 	futureGoodEdbInfoList := make([]*future_good.FutureGoodEdbInfo, 0)
 	futureGoodEdbInfoDateMap = make(map[string]*future_good.FutureGoodEdbInfo)
@@ -635,7 +640,7 @@ func getProfitFutureGoodEdbInfoList(earliestDateTime time.Time, zlFutureGoodEdbI
 		// 如果（当前年-最新日期的年份） * 12个月 + (当前月-最新日期的月份) 小于总月份
 		tmpN := subYear*12 + subMonth
 		if tmpN < newMaxN {
-			tmpDateTime := time.Date(v.Year, time.Month(v.Month), 0, 0, 0, 0, 0, time.Local)
+			tmpDateTime := time.Date(v.Year, time.Month(v.Month), 1, 0, 0, 0, 0, time.Local)
 			futureGoodEdbInfoDateMap[tmpDateTime.Format(utils.FormatYearMonthDate)] = v
 			if tmpN > newMaxN {
 				newMaxN = tmpN

models/base_calculate.go

@@ -304,6 +304,7 @@ func (obj BaseCalculate) Tbz() (dateDataMap map[time.Time]float64, err error, er
 	var dateArr []time.Time
 	dataMap := make(map[string]*EdbInfoData) // 避免因为时间戳导致的日期不对，还是用string来表示比较合适
 	for _, v := range dataList {
+		v.DataTime = time.Date(v.DataTime.Year(), v.DataTime.Month(), v.DataTime.Day(), 0, 0, 0, 0, v.DataTime.Location())
 		dateArr = append(dateArr, v.DataTime)
 		dataMap[v.DataTime.Format(utils.FormatDate)] = v
 	}
@@ -416,6 +417,7 @@ func (obj BaseCalculate) Tcz() (dateDataMap map[time.Time]float64, err error, er
 	var dateArr []time.Time
 	dataMap := make(map[string]*EdbInfoData)
 	for _, v := range dataList {
+		v.DataTime = time.Date(v.DataTime.Year(), v.DataTime.Month(), v.DataTime.Day(), 0, 0, 0, 0, v.DataTime.Location())
 		dateArr = append(dateArr, v.DataTime)
 		dataMap[v.DataTime.Format(utils.FormatDate)] = v
 	}
@@ -587,6 +589,7 @@ func (obj BaseCalculate) Hbz() (dateDataMap map[time.Time]float64, err error, er
 
 	var dateArr []time.Time
 	for _, v := range dataList {
+		v.DataTime = time.Date(v.DataTime.Year(), v.DataTime.Month(), v.DataTime.Day(), 0, 0, 0, 0, v.DataTime.Location())
 		dateArr = append(dateArr, v.DataTime)
 	}
 
@@ -662,6 +665,7 @@ func (obj BaseCalculate) Hcz() (dateDataMap map[time.Time]float64, err error, er
 
 	var dateArr []time.Time
 	for _, v := range dataList {
+		v.DataTime = time.Date(v.DataTime.Year(), v.DataTime.Month(), v.DataTime.Day(), 0, 0, 0, 0, v.DataTime.Location())
 		dateArr = append(dateArr, v.DataTime)
 	}
 
@@ -722,6 +726,7 @@ func (obj BaseCalculate) UpFrequency() (dateDataMap map[time.Time]float64, err e
 	fromDataMap := make(map[time.Time]float64)
 	//来源指指标数据
 	for _, v := range dataList {
+		v.DataTime = time.Date(v.DataTime.Year(), v.DataTime.Month(), v.DataTime.Day(), 0, 0, 0, 0, v.DataTime.Location())
 		dateArr = append(dateArr, v.DataTime)
 		dataMap[v.DataTime] = v
 		fromDataMap[v.DataTime] = v.Value
@@ -733,8 +738,12 @@ func (obj BaseCalculate) UpFrequency() (dateDataMap map[time.Time]float64, err e
 		return
 	}
 
+	nowDay := time.Now()
+	nowDay = time.Date(nowDay.Year(), nowDay.Month(), nowDay.Day(), 0, 0, 0, 0, nowDay.Location())
+
 	// 数据计算
 	dataLen := len(dataList)
+	lastI := dataLen - 1
 	for i := 0; i < dataLen; i++ {
 		//当期
 		currentItem := dataList[i]
@@ -742,22 +751,39 @@ func (obj BaseCalculate) UpFrequency() (dateDataMap map[time.Time]float64, err e
 		var day int
 		var preItem *EdbInfoData
 		var preDate time.Time
+
+		// 如果是第一期的数据，那么直接赋值
 		if i == 0 {
-			day = int(time.Now().Sub(currentDate).Hours() / float64(24))
-			preDate = time.Now()
-		} else {
-			j := i - 1
-			if j < dataLen {
-				preItem = dataList[j]
-				day = int(preItem.DataTime.Sub(currentDate).Hours() / float64(24))
-				utils.FileLog.Info("preItem.DataTime:" + preItem.DataTime.Format(utils.FormatDate) + ";currentItem.DataTime" + currentItem.DataTime.Format(utils.FormatDate))
-			}
+			dateDataMap[currentItem.DataTime] = currentItem.Value
+			continue
 		}
-		for k := 0; k <= day; k++ {
-			needDay := preDate.AddDate(0, 0, -k)
-			dateDataMap[needDay] = currentItem.Value
+
+		// 上一期
+		j := i - 1
+
+		preItem = dataList[j]
+		preDate = preItem.DataTime
+		day = int(currentDate.Sub(preItem.DataTime).Hours() / float64(24))
+		//utils.FileLog.Info("preItem.DataTime:" + preItem.DataTime.Format(utils.FormatDate) + ";currentItem.DataTime" + currentItem.DataTime.Format(utils.FormatDate))
+		fmt.Println("preItem.DataTime:" + preItem.DataTime.Format(utils.FormatDate) + ";currentItem.DataTime:" + currentItem.DataTime.Format(utils.FormatDate))
+
+		for k := 1; k < day; k++ {
+			needDay := preDate.AddDate(0, 0, k)
+			dateDataMap[needDay] = preItem.Value
 		}
 		dateDataMap[currentItem.DataTime] = currentItem.Value
+
+		// 如果是最后一期的数据
+		if i == lastI {
+			day = int(nowDay.Sub(currentDate).Hours() / float64(24))
+			//utils.FileLog.Info("preItem.DataTime:" + currentDate.Format(utils.FormatDate) + ";currentItem.DataTime" + nowDay.Format(utils.FormatDate))
+
+			for k := 1; k <= day; k++ {
+				needDay := currentDate.AddDate(0, 0, k)
+				dateDataMap[needDay] = currentItem.Value
+			}
+		}
+
 	}
 
 	return
@@ -788,6 +814,7 @@ func (obj BaseCalculate) DownFrequency() (dateDataMap map[time.Time]float64, err
 	fromDataMap := make(map[time.Time]float64)
 	//来源指指标数据
 	for _, v := range dataList {
+		v.DataTime = time.Date(v.DataTime.Year(), v.DataTime.Month(), v.DataTime.Day(), 0, 0, 0, 0, v.DataTime.Location())
 		dateArr = append(dateArr, v.DataTime)
 		dataMap[v.DataTime] = v
 		fromDataMap[v.DataTime] = v.Value
@@ -954,6 +981,7 @@ func (obj BaseCalculate) TimeShift() (dateDataMap map[time.Time]float64, err err
 	var dateArr []time.Time
 	dataMap := make(map[time.Time]*EdbInfoData)
 	for _, v := range dataList {
+		v.DataTime = time.Date(v.DataTime.Year(), v.DataTime.Month(), v.DataTime.Day(), 0, 0, 0, 0, v.DataTime.Location())
 		dateArr = append(dateArr, v.DataTime)
 		dataMap[v.DataTime] = v
 	}
@@ -1017,6 +1045,7 @@ func (obj BaseCalculate) Cjjx() (dateDataMap map[time.Time]float64, err error, e
 	var dateArr []time.Time
 	dataMap := make(map[time.Time]*EdbInfoData)
 	for _, v := range dataList {
+		v.DataTime = time.Date(v.DataTime.Year(), v.DataTime.Month(), v.DataTime.Day(), 0, 0, 0, 0, v.DataTime.Location())
 		dateArr = append(dateArr, v.DataTime)
 		dataMap[v.DataTime] = v
 	}
@@ -1260,6 +1289,7 @@ func (obj BaseCalculate) Ljz() (dateDataMap map[time.Time]float64, err error, er
 		yearMap := make(map[int]float64)
 		yearList := make([]int, 0)
 		for _, item := range dataList {
+			item.DataTime = time.Date(item.DataTime.Year(), item.DataTime.Month(), item.DataTime.Day(), 0, 0, 0, 0, item.DataTime.Location())
 			year := item.DataTime.Year()
 			yearVal, ok := yearMap[year]
 			if ok {
@@ -1278,6 +1308,7 @@ func (obj BaseCalculate) Ljz() (dateDataMap map[time.Time]float64, err error, er
 		yearMonthMap := make(map[string]float64)
 		yearMonthList := make([]string, 0)
 		for _, item := range dataList {
+			item.DataTime = time.Date(item.DataTime.Year(), item.DataTime.Month(), item.DataTime.Day(), 0, 0, 0, 0, item.DataTime.Location())
 			itemDate := item.DataTime
 			year := itemDate.Year()
 			var tmpK string
@@ -1309,6 +1340,7 @@ func (obj BaseCalculate) Ljz() (dateDataMap map[time.Time]float64, err error, er
 		yearMonthMap := make(map[string]float64)
 		yearMonthList := make([]string, 0)
 		for _, item := range dataList {
+			item.DataTime = time.Date(item.DataTime.Year(), item.DataTime.Month(), item.DataTime.Day(), 0, 0, 0, 0, item.DataTime.Location())
 			itemDate := item.DataTime
 			year := itemDate.Year()
 			var tmpK string
@@ -1344,6 +1376,7 @@ func (obj BaseCalculate) Ljz() (dateDataMap map[time.Time]float64, err error, er
 		yearMonthMap := make(map[string]float64)
 		yearMonthList := make([]string, 0)
 		for _, item := range dataList {
+			item.DataTime = time.Date(item.DataTime.Year(), item.DataTime.Month(), item.DataTime.Day(), 0, 0, 0, 0, item.DataTime.Location())
 			itemDate := item.DataTime
 			year := itemDate.Year()
 			var tmpK string
@@ -1371,6 +1404,7 @@ func (obj BaseCalculate) Ljz() (dateDataMap map[time.Time]float64, err error, er
 		tmpDateDataMap := make(map[time.Time]float64)
 		tmpDateList := make([]time.Time, 0)
 		for _, item := range dataList {
+			item.DataTime = time.Date(item.DataTime.Year(), item.DataTime.Month(), item.DataTime.Day(), 0, 0, 0, 0, item.DataTime.Location())
 			itemDate := item.DataTime
 			dayInt := itemDate.Year()*100 + int(itemDate.Month())
 			var currTime time.Time
@@ -1414,6 +1448,7 @@ func (obj BaseCalculate) Ljz() (dateDataMap map[time.Time]float64, err error, er
 		tmpDateDataMap := make(map[time.Time]float64)
 		tmpDateList := make([]time.Time, 0)
 		for _, item := range dataList {
+			item.DataTime = time.Date(item.DataTime.Year(), item.DataTime.Month(), item.DataTime.Day(), 0, 0, 0, 0, item.DataTime.Location())
 			itemDate := item.DataTime
 			var currTime time.Time
 			// 周六周日，这是下一个周五的数据
@@ -1504,6 +1539,7 @@ func (obj BaseCalculate) LjzNczj() (dateDataMap map[time.Time]float64, err error
 		tmpDateDataMap := make(map[time.Time]float64)
 		tmpDateList := make([]time.Time, 0)
 		for _, item := range dataList {
+			item.DataTime = time.Date(item.DataTime.Year(), item.DataTime.Month(), item.DataTime.Day(), 0, 0, 0, 0, item.DataTime.Location())
 			itemDate := item.DataTime
 			var currTime time.Time
 			// 周六周日，这是下一个周五的数据
@@ -1534,6 +1570,7 @@ func (obj BaseCalculate) LjzNczj() (dateDataMap map[time.Time]float64, err error
 		}
 	default:
 		for _, item := range dataList {
+			item.DataTime = time.Date(item.DataTime.Year(), item.DataTime.Month(), item.DataTime.Day(), 0, 0, 0, 0, item.DataTime.Location())
 			itemDate := item.DataTime
 			year := itemDate.Year()
 			yearVal, ok := yearMap[year]
@@ -1624,6 +1661,7 @@ func calculateExponentialSmoothingData(dataList []*EdbInfoData, alpha float64) (
 	alphaDecimal := decimal.NewFromFloat(alpha)
 	subAlpha := decimal.NewFromFloat(1).Sub(alphaDecimal)
 	for k, d := range dataList {
+		d.DataTime = time.Date(d.DataTime.Year(), d.DataTime.Month(), d.DataTime.Day(), 0, 0, 0, 0, d.DataTime.Location())
 		// 首期的值以原始值作为指数修匀的计算值
 		if k == 0 {
 			newDataList = append(newDataList, EdbInfoData{
@@ -1684,6 +1722,7 @@ func (obj BaseCalculate) Rjz() (dateDataMap map[time.Time]float64, err error, er
 	var dateArr []time.Time
 	dataMap := make(map[time.Time]*EdbInfoData)
 	for _, v := range dataList {
+		v.DataTime = time.Date(v.DataTime.Year(), v.DataTime.Month(), v.DataTime.Day(), 0, 0, 0, 0, v.DataTime.Location())
 		dateArr = append(dateArr, v.DataTime)
 		dataMap[v.DataTime] = v
 	}
@@ -1812,3 +1851,33 @@ func GetDateDataAndDateList(dateDataMap map[time.Time]float64) (dateStrDataMap m
 
 	return
 }
+
+// StepCalculateBySearchData
+// @Description: 分步骤计算
+type StepCalculateBySearchData struct {
+	DataList   []*EdbInfoSearchData `description:"基础数据"`
+	Calculates []struct {
+		Formula       interface{}
+		Calendar      string `description:"公历/农历"`
+		Frequency     string `description:"需要转换的频度"`
+		MoveType      int    `description:"移动方式:1:领先(默认),2:滞后"`
+		MoveFrequency string `description:"移动频度"`
+		FromFrequency string `description:"来源的频度"`
+		Source        int    `description:"1：累计值转月；2：累计值转季；3：同比值；4：同差值；5：N数值移动平均数计算；6：环比值；7：环差值；8：升频；9：降频；10：时间移位；11：超季节性；12：年化；13：累计值；14：累计值年初至今；15：指数修匀；16：日均值"`
+		Sort          int    `description:"计算顺序"`
+	}
+}
+
+func TransDateData2EdbData(dateData map[time.Time]float64) (edbData []*EdbInfoData) {
+	edbData = make([]*EdbInfoData, 0)
+	for d, v := range dateData {
+		edbData = append(edbData, &EdbInfoData{
+			DataTime: d,
+			Value:    v,
+		})
+	}
+	sort.Slice(edbData, func(i, j int) bool {
+		return edbData[i].DataTime.Before(edbData[j].DataTime)
+	})
+	return
+}

models/base_from_adjust.go

@@ -306,7 +306,7 @@ func RefreshAllAdjustEdb(edbInfo *EdbInfo, fromEdbInfo *EdbInfo) (err error) {
 		saveVal := utils.SubFloatToString(val, 20)
 		if existVal, ok := existDataMap[currDay]; !ok {
 			//格式化时间
-			currentDate, tmpErr := time.Parse(utils.FormatDate, item.DataTime)
+			currentDate, tmpErr := time.ParseInLocation(utils.FormatDate, item.DataTime, time.Local)
 			if tmpErr != nil {
 				err = tmpErr
 				return

models/base_from_calculate.go

@@ -5,12 +5,13 @@ import (
 	"errors"
 	"eta/eta_index_lib/utils"
 	"fmt"
-	"github.com/beego/beego/v2/client/orm"
-	"github.com/shopspring/decimal"
-	"github.com/yidane/formula"
 	"strconv"
 	"strings"
 	"time"
+
+	"github.com/beego/beego/v2/client/orm"
+	"github.com/shopspring/decimal"
+	"github.com/yidane/formula"
 )
 
 // EdbInfoCalculateSaveReq 计算（运算）指标请求参数
@@ -229,7 +230,6 @@ func EditCalculateInfo(edbInfo *EdbInfo, req EdbInfoCalculateSaveReq, formulaSli
 				}
 			}
 		}
-
 		//计算数据
 		err = refreshAllCalculate(to, edbInfoList, edbInfoTag, edbInfo.EdbInfoId, edbInfo.Source, edbInfo.SubSource, edbInfo.EdbCode, edbInfo.CalculateFormula, "", "", edbInfoIdBytes, edbInfo.EmptyType, edbInfo.MaxEmptyType, edbInfo.Extra)
 	}
@@ -683,6 +683,8 @@ type EdbInfoCalculateBatchEditReq struct {
 	Unit             string                         `description:"单位"`
 	UnitEn           string                         `description:"英文单位"`
 	ClassifyId       int                            `description:"分类id"`
+	AdminId          int                            `description:"操作人id"`
+	AdminName        string                         `description:"操作人姓名"`
 	Formula          string                         `description:"N值"`
 	EdbInfoId        int                            `description:"编辑指标id"`
 	FromEdbInfoId    int                            `description:"计算来源指标id"`
@@ -698,6 +700,14 @@ type EdbInfoCalculateBatchEditReq struct {
 	CalculateFormula string                         `description:"计算公式"`
 }
 
+// DeleteCalculateData 删除计算数据
+func DeleteCalculateData(edbInfoId int) (err error) {
+	o := orm.NewOrm()
+	sql := `DELETE FROM edb_data_calculate WHERE edb_info_id=?`
+	_, err = o.Raw(sql, edbInfoId).Exec()
+	return
+}
+
 // CheckFormula2 校验公式是否正常（比如说除法的分母不能为0之类的，实际上就是用预设的字段数据做一次计算）
 func CheckFormula2(edbInfoArr []*EdbInfo, formulaMap map[string]string, formulaStr string, edbInfoIdBytes []string) (ok bool, err error) {
 	valArr := make(map[int]float64)

models/base_from_ccf.go

@@ -0,0 +1,426 @@
+package models
+
+import (
+	"eta/eta_index_lib/utils"
+	"fmt"
+	"github.com/beego/beego/v2/client/orm"
+	"strconv"
+	"strings"
+	"time"
+)
+
+// BaseFromCCF CCF化纤信息
+type BaseFromCCF struct{}
+
+type BaseFromCCFData struct {
+	BaseFromCcfDataId  int `orm:"column(base_from_ccf_data_id);pk"`
+	BaseFromCcfIndexId int
+	IndexCode          string
+	DataTime           string
+	Value              string
+	CreateTime         time.Time
+	ModifyTime         time.Time
+	DataTimestamp      int64
+}
+
+func (m *BaseFromCCFData) TableName() string {
+	return "base_from_ccf_data"
+}
+
+func GetBaseFromCCFDataByCondition(condition string, pars []interface{}) (list []*BaseFromCCFData, err error) {
+	o := orm.NewOrm()
+	sql := `SELECT * FROM base_from_ccf_data WHERE 1=1 `
+	if condition != "" {
+		sql += condition
+	}
+	_, err = o.Raw(sql, pars).QueryRows(&list)
+	return
+}
+
+// Add 添加
+func (obj BaseFromCCF) Add(edbCode string) (err error) {
+	o := orm.NewOrm()
+
+	var condition string
+	var pars []interface{}
+	if edbCode != "" {
+		condition += " AND index_code = ? "
+		pars = append(pars, edbCode)
+	}
+	ccfBaseDataAll, err := GetBaseFromCCFDataByCondition(condition, pars)
+	if err != nil && err.Error() != utils.ErrNoRow() {
+		return
+	}
+	var isAdd bool
+	addSql := ` INSERT INTO edb_data_ccf(edb_info_id,edb_code,data_time,value,create_time,modify_time,data_timestamp) values `
+	existMap := make(map[string]string)
+	for _, sv := range ccfBaseDataAll {
+		eDate := sv.DataTime
+		dataTime, err := time.Parse(utils.FormatDate, eDate)
+		if err != nil {
+			return err
+		}
+		timestamp := dataTime.UnixNano() / 1e6
+		timeStr := fmt.Sprintf("%d", timestamp)
+		if _, ok := existMap[eDate]; !ok {
+			addSql += GetAddSql("0", edbCode, eDate, timeStr, sv.Value)
+			isAdd = true
+		}
+		existMap[eDate] = sv.Value
+	}
+	if isAdd {
+		addSql = strings.TrimRight(addSql, ",")
+		utils.FileLog.Info("addSql:" + addSql)
+		_, err = o.Raw(addSql).Exec()
+		if err != nil {
+			return err
+		}
+	}
+	return
+}
+
+// Refresh 刷新CCF化纤指标数据
+func (obj BaseFromCCF) Refresh(edbInfoId int, edbCode, startDate string) (err error) {
+	source := obj.GetSource()
+	o := orm.NewOrm()
+	if err != nil {
+		return
+	}
+	edbInfoIdStr := strconv.Itoa(edbInfoId)
+	//计算数据
+	var condition string
+	var pars []interface{}
+
+	if edbCode != "" {
+		condition += " AND index_code=? "
+		pars = append(pars, edbCode)
+	}
+
+	if startDate != "" {
+		condition += " AND data_time>=? "
+		pars = append(pars, startDate)
+	}
+
+	ccfDataList, err := GetBaseFromCCFDataByCondition(condition, pars)
+	if err != nil {
+		return
+	}
+
+	// 真实数据的最大日期  ， 插入规则配置的日期
+	var realDataMaxDate, edbDataInsertConfigDate time.Time
+	var edbDataInsertConfig *EdbDataInsertConfig
+	var isFindConfigDateRealData bool //是否找到配置日期的实际数据的值
+	{
+		edbDataInsertConfig, err = GetEdbDataInsertConfigByEdbId(edbInfoId)
+		if err != nil && err.Error() != utils.ErrNoRow() {
+			return
+		}
+		if edbDataInsertConfig != nil {
+			edbDataInsertConfigDate = edbDataInsertConfig.Date
+		}
+	}
+
+	var existCondition string
+	var existPars []interface{}
+
+	existCondition += " AND edb_info_id=? "
+	existPars = append(existPars, edbInfoId)
+	if startDate != "" {
+		existCondition += " AND data_time>=? "
+		existPars = append(existPars, startDate)
+	}
+
+	existList, err := GetEdbDataByCondition(source, 0, existCondition, existPars)
+	if err != nil {
+		return err
+	}
+	existMap := make(map[string]*EdbInfoSearchData)
+	for _, v := range existList {
+		existMap[v.DataTime] = v
+	}
+	addSql := ` INSERT INTO edb_data_ccf(edb_info_id,edb_code,data_time,value,create_time,modify_time,data_timestamp) values `
+	var isAdd bool
+	for _, v := range ccfDataList {
+		item := v
+		eDate := item.DataTime
+		dataTime, err := time.ParseInLocation(utils.FormatDate, eDate, time.Local)
+		if err != nil {
+			return err
+		}
+		if findItem, ok := existMap[v.DataTime]; !ok {
+			sValue := item.Value
+
+			timestamp := dataTime.UnixNano() / 1e6
+			timeStr := fmt.Sprintf("%d", timestamp)
+
+			addSql += GetAddSql(edbInfoIdStr, edbCode, eDate, timeStr, sValue)
+			isAdd = true
+		} else {
+			if findItem != nil && utils.SubFloatToString(findItem.Value, 30) != item.Value {
+				err = ModifyEdbDataById(source, 0, findItem.EdbDataId, item.Value)
+				if err != nil {
+					return err
+				}
+			}
+		}
+
+		// 下面代码主要目的是处理掉手动插入的数据判断
+		{
+			if realDataMaxDate.IsZero() || dataTime.After(realDataMaxDate) {
+				realDataMaxDate = dataTime
+			}
+			if edbDataInsertConfigDate.IsZero() || dataTime.Equal(edbDataInsertConfigDate) {
+				isFindConfigDateRealData = true
+			}
+		}
+	}
+
+	// 处理手工数据补充的配置
+	HandleConfigInsertEdbData(realDataMaxDate, edbDataInsertConfig, edbInfoId, source, 0, existMap, isFindConfigDateRealData)
+
+	if isAdd {
+		addSql = strings.TrimRight(addSql, ",")
+		_, err = o.Raw(addSql).Exec()
+		if err != nil {
+			fmt.Println("RefreshEdbDataFromBaiinfo add Err", err.Error())
+			return
+		}
+	}
+	return
+}
+
+// GetSource 获取来源编码id
+func (obj BaseFromCCF) GetSource() int {
+	return utils.DATA_SOURCE_CCF
+}
+
+// GetSourceName 获取来源名称
+func (obj BaseFromCCF) GetSourceName() string {
+	return utils.DATA_SOURCE_NAME_CCF
+}
+
+type BaseFromCCFIndex struct {
+	BaseFromCcfIndexId int64 `orm:"column(base_from_ccf_index_id);pk"`
+	IndexCode          string
+	IndexName          string
+	Frequency          string
+	Unit               string
+	ClassifyId         int
+	StartDate          string
+	EndDate            string
+	Sort               int
+	TerminalCode       string
+	CreateTime         time.Time
+	ModifyTime         time.Time
+}
+
+func (m *BaseFromCCFIndex) TableName() string {
+	return "base_from_ccf_index"
+}
+
+func (m *BaseFromCCFIndex) Add() (lastId int64, err error) {
+	o := orm.NewOrm()
+	lastId, err = o.Insert(m)
+	return
+}
+
+func (m *BaseFromCCFIndex) Update(updateCols []string) (err error) {
+	o := orm.NewOrm()
+	_, err = o.Update(m, updateCols...)
+
+	return
+}
+
+func (m *BaseFromCCFIndex) ModifyIndexMaxAndMinDate(indexCode string, item *EdbInfoMaxAndMinInfo) (err error) {
+	o := orm.NewOrm()
+	sql := ` UPDATE base_from_ccf_index SET start_date=?,end_date=?,modify_time=NOW() WHERE index_code=? `
+	_, err = o.Raw(sql, item.MinDate, item.MaxDate, indexCode).Exec()
+	return
+}
+
+func (m *BaseFromCCFIndex) GetByIndexCode(indexCode string) (item *BaseFromCCFIndex, err error) {
+	o := orm.NewOrm()
+	sql := ` SELECT * FROM base_from_ccf_index WHERE index_code=? `
+	err = o.Raw(sql, indexCode).QueryRow(&item)
+	return
+}
+
+func (m *BaseFromCCFIndex) GetItemsByCondition(condition string, pars []interface{}, fieldArr []string, orderRule string) (items []*BaseFromCCFIndex, err error) {
+	o := orm.NewOrm()
+	fields := strings.Join(fieldArr, ",")
+	if len(fieldArr) == 0 {
+		fields = `*`
+	}
+	order := `ORDER BY create_time DESC`
+	if orderRule != "" {
+		order = ` ORDER BY ` + orderRule
+	}
+	sql := fmt.Sprintf(`SELECT %s FROM base_from_ccf_index WHERE 1=1 %s %s`, fields, condition, order)
+	_, err = o.Raw(sql, pars).QueryRows(&items)
+	return
+}
+
+type BaseFromCCFIndexList struct {
+	CcfIndexId   int64 `orm:"column(ccf_index_id);pk"`
+	IndexCode    string
+	IndexName    string
+	Frequency    string
+	Unit         string
+	ClassifyId   int
+	StartDate    string
+	EndDate      string
+	TerminalCode string
+	CreateTime   string
+	ModifyTime   string
+}
+
+func (m *BaseFromCCFData) GetByIndexCode(indexCode string) (list []*BaseFromCCFData, err error) {
+	o := orm.NewOrm()
+	sql := ` SELECT * FROM base_from_ccf_data WHERE index_code=? `
+	_, err = o.Raw(sql, indexCode).QueryRows(&list)
+	return
+}
+
+func (m *BaseFromCCFData) AddMulti(item []*BaseFromCCFData) (err error) {
+	o := orm.NewOrm()
+	_, err = o.InsertMulti(100, item)
+	return
+}
+
+// HandleCCFEdbData CCF化纤的指标数据
+type HandleCCFEdbData struct {
+	IndexName    string            `description:"指标名称"`
+	IndexCode    string            `description:"指标编码"`
+	ClassifyId   int               `description:"分类ID"`
+	Unit         string            `description:"单位"`
+	Sort         int               `description:"排序"`
+	Frequency    string            `description:"频度"`
+	TerminalCode string            `description:"终端编码"`
+	DateData     map[string]string `description:"日期数据"`
+}
+
+type HandleCCFEdbDataReq struct {
+	List         []*HandleCCFEdbData
+	TerminalCode string `description:"编码"`
+}
+
+type HandleCCFTableData struct {
+	ClassifyId   int       `description:"分类ID"`
+	FromPage     string    `description:"表格来源"`
+	TableDate    time.Time `description:"表格日期"`
+	TableContent string    `description:"表格HTML"`
+}
+
+type HandleCCFStockTableReq struct {
+	Table        *HandleCCFTableData
+	TerminalCode string `description:"编码"`
+}
+
+func (m *BaseFromCCFData) GetMaxAndMinDateByIndexCode(indexCode string) (item *EdbInfoMaxAndMinInfo, err error) {
+	o := orm.NewOrm()
+	sql := ` SELECT MIN(data_time) AS min_date,MAX(data_time) AS max_date,MIN(value) AS min_value,MAX(value) AS max_value FROM base_from_ccf_data WHERE index_code=? `
+	err = o.Raw(sql, indexCode).QueryRow(&item)
+	return
+}
+
+// BaseFromCCFClassify CCF化纤数据分类表
+type BaseFromCCFClassify struct {
+	ClassifyId      int       `orm:"column(classify_id);pk"`
+	ClassifyName    string    `description:"分类名称"`
+	ParentId        int       `description:"父级id"`
+	SysUserId       int       `description:"创建人id"`
+	SysUserRealName string    `description:"创建人姓名"`
+	Level           int       `description:"层级"`
+	Sort            int       `description:"排序字段，越小越靠前，默认值：10"`
+	ModifyTime      time.Time `description:"修改时间"`
+	CreateTime      time.Time `description:"创建时间"`
+}
+
+func (m *BaseFromCCFClassify) Add() (lastId int64, err error) {
+	o := orm.NewOrm()
+	lastId, err = o.Insert(m)
+	return
+}
+
+func (m *BaseFromCCFClassify) Update(updateCols []string) (err error) {
+	o := orm.NewOrm()
+	_, err = o.Update(m, updateCols...)
+
+	return
+}
+
+func (m *BaseFromCCFClassify) GetByClassifyName(classifyName string) (item *BaseFromCCFClassify, err error) {
+	o := orm.NewOrm()
+	sql := ` SELECT * FROM base_from_ccf_classify WHERE classify_name=? `
+	err = o.Raw(sql, classifyName).QueryRow(&item)
+	return
+}
+
+func (m *BaseFromCCFClassify) GetItemsByCondition(condition string, pars []interface{}, fieldArr []string, orderRule string) (items []*BaseFromCCFClassify, err error) {
+	o := orm.NewOrm()
+	fields := strings.Join(fieldArr, ",")
+	if len(fieldArr) == 0 {
+		fields = `*`
+	}
+	order := `ORDER BY create_time DESC`
+	if orderRule != "" {
+		order = ` ORDER BY ` + orderRule
+	}
+	sql := fmt.Sprintf(`SELECT %s FROM base_from_ccf_classify WHERE 1=1 %s %s`, fields, condition, order)
+	_, err = o.Raw(sql, pars).QueryRows(&items)
+	return
+}
+
+// BaseFromCCFClassifyItem CCF化纤数据分类信息
+type BaseFromCCFClassifyItem struct {
+	ClassifyId   int                        `description:"分类ID"`
+	ClassifyName string                     `description:"分类名称"`
+	ParentId     int                        `description:"父级id"`
+	Level        int                        `description:"层级"`
+	Sort         int                        `description:"排序字段"`
+	CreateTime   string                     `description:"创建时间"`
+	ModifyTime   string                     `description:"修改时间"`
+	Child        []*BaseFromCCFClassifyItem `description:"子分类"`
+}
+
+func (m *BaseFromCCFClassify) Format2Item(origin *BaseFromCCFClassify) (item *BaseFromCCFClassifyItem) {
+	if origin == nil {
+		return
+	}
+	item = new(BaseFromCCFClassifyItem)
+	item.ClassifyId = origin.ClassifyId
+	item.ClassifyName = origin.ClassifyName
+	item.ParentId = origin.ParentId
+	item.Level = origin.Level
+	item.Sort = origin.Sort
+	item.CreateTime = utils.TimeTransferString(utils.FormatDateTime, origin.CreateTime)
+	item.ModifyTime = utils.TimeTransferString(utils.FormatDateTime, origin.ModifyTime)
+	return
+}
+
+// MultiUpdateBaseFromCCFDataValue 批量更新CCF化纤指标数据
+func MultiUpdateBaseFromCCFDataValue(items []*BaseFromCCFData) (err error) {
+	if len(items) == 0 {
+		return
+	}
+
+	o := orm.NewOrm()
+	sql := `UPDATE base_from_ccf_data SET value = ?, modify_time = NOW() WHERE index_code = ? AND data_time = ? LIMIT 1`
+	p, err := o.Raw(sql).Prepare()
+	if err != nil {
+		return
+	}
+	defer func() {
+		_ = p.Close()
+	}()
+	for _, v := range items {
+		if v.IndexCode == "" || v.DataTime == "" {
+			continue
+		}
+		_, err = p.Exec(v.Value, v.IndexCode, v.DataTime)
+		if err != nil {
+			return
+		}
+	}
+	return
+}

models/base_from_mysteel_chemical.go

@@ -537,3 +537,18 @@ func (m *BaseFromMysteelChemicalIndex) GetIndexByCondition(condition string, par
 	_, err = o.Raw(sql, pars).QueryRows(&items)
 	return
 }
+
+type BaseFromMysteelChemicalRecord struct {
+	BaseFromMysteelChemicalRecordId int64 `orm:"column(base_from_mysteel_chemical_record_id);pk"`
+	BaseFromMysteelChemicalIndexId  int64
+	OldIndexName                    string    `description:"原始名称"`
+	NewIndexName                    string    `description:"新的名称"`
+	CreateTime                      time.Time `description:"记录创建时间"`
+	Timestamp                       int64     `description:"记录创建时间戳"`
+}
+
+func (m *BaseFromMysteelChemicalRecord) AddBaseFromMysteelChemicalRecord() (err error) {
+	o := orm.NewOrm()
+	_, err = o.Insert(m)
+	return
+}

models/base_from_smm.go

@@ -585,3 +585,18 @@ func RefreshEdbDataFromSmmToEdb(edbInfoId int, edbCode, startDate string, smmDat
 	}
 	return
 }
+
+type BaseFromSmmRecord struct {
+	BaseFromSmmRecordId int64 `orm:"column(base_from_smm_record_id);pk"`
+	BaseFromSmmIndexId  int64
+	OldIndexName        string    `description:"原始名称"`
+	NewIndexName        string    `description:"新的名称"`
+	CreateTime          time.Time `description:"记录创建时间"`
+	Timestamp           int64     `description:"记录创建时间戳"`
+}
+
+func AddBaseFromSmmRecord(item *BaseFromSmmRecord) (err error) {
+	o := orm.NewOrm()
+	_, err = o.Insert(item)
+	return
+}

models/base_from_ths.go

@@ -1,6 +1,7 @@
 package models
 
 import (
+	"eta/eta_index_lib/services/alarm_msg"
 	"eta/eta_index_lib/utils"
 	"fmt"
 	"github.com/beego/beego/v2/client/orm"
@@ -26,7 +27,14 @@ type Tables struct {
 
 // 新增同花顺指标数据
 func AddEdbDataFromThs(edbCode string, item EdbDataFromThs) (err error) {
+	var errMsg string
 	o := orm.NewOrm()
+	defer func() {
+		if err != nil {
+			//go utils.SendEmail(utils.APP_NAME_CN+"【"+utils.RunMode+"】"+"失败提醒", " 同花顺数据获取失败:err:"+errMsg, utils.EmailSendToUsers)
+			go alarm_msg.SendAlarmMsg("同花顺数据获取失败:err:"+errMsg, 3)
+		}
+	}()
 
 	if len(item.Tables) > 0 {
 		table := item.Tables[0]
@@ -42,6 +50,7 @@ func AddEdbDataFromThs(edbCode string, item EdbDataFromThs) (err error) {
 			sValue := table.Value[i]
 			dataTime, err := time.ParseInLocation(utils.FormatDate, eDate, time.Local)
 			if err != nil {
+				errMsg = " time.Parse :" + err.Error()
 				return err
 			}
 			timestamp := dataTime.UnixNano() / 1e6
@@ -53,6 +62,7 @@ func AddEdbDataFromThs(edbCode string, item EdbDataFromThs) (err error) {
 			addSql = strings.TrimRight(addSql, ",")
 			_, err = o.Raw(addSql).Exec()
 			if err != nil {
+				errMsg = " tx.Exec Err :" + err.Error()
 				return
 			}
 		}

models/base_predict_from_calculate.go

@@ -303,6 +303,15 @@ func refreshAllPredictCalculate(to orm.TxOrmer, edbInfoIdList []*EdbInfo, edbInf
 		}
 		if maxStartDate.IsZero() || maxStartDate.Before(tmpStartDate) {
 			maxStartDate = tmpStartDate
+		}
+		tmpEndDate, tmpErr := time.ParseInLocation(utils.FormatDate, v.EndDate, time.Local)
+		if tmpErr != nil {
+			err = errors.New(`最近的日期格式化失败;日期：` + v.EndDate + `;err:` + tmpErr.Error())
+			return
+		}
+		if minEndDate.IsZero() || tmpEndDate.Before(minEndDate) {
+			minEndDate = tmpEndDate
+		}
 		}*/
 
 		// 获取关联指标数据

models/ccf_stock_excel.go

@@ -0,0 +1,147 @@
+package models
+
+import (
+	"eta/eta_index_lib/utils"
+	"fmt"
+	"github.com/beego/beego/v2/client/orm"
+	"strings"
+	"time"
+)
+
+// CCFStockExcel CCF化纤装置表格
+type CCFStockExcel struct {
+	CcfStockExcelId int       `orm:"column(ccf_stock_excel_id);pk"`
+	ClassifyId      int       `description:"分类ID"`
+	ExcelDate       time.Time `description:"表格日期"`
+	ExcelContent    string    `description:"表格HTML"`
+	FromPage        string    `description:"表格来源"`
+	CreateTime      time.Time `description:"创建时间"`
+	ModifyTime      time.Time `description:"修改时间"`
+}
+
+func (m *CCFStockExcel) TableName() string {
+	return "ccf_stock_excel"
+}
+
+type CCFStockExcelCols struct {
+	CcfStockExcelId string
+	ClassifyId      string
+	ExcelDate       string
+	ExcelContent    string
+	FromPage        string
+	CreateTime      string
+	ModifyTime      string
+}
+
+func (m *CCFStockExcel) Cols() CCFStockExcelCols {
+	return CCFStockExcelCols{
+		CcfStockExcelId: "ccf_stock_excel_id",
+		ClassifyId:      "classify_id",
+		ExcelDate:       "excel_date",
+		ExcelContent:    "excel_content",
+		FromPage:        "from_page",
+		CreateTime:      "create_time",
+		ModifyTime:      "modify_time",
+	}
+}
+
+func (m *CCFStockExcel) PrimaryId() string {
+	return m.Cols().CcfStockExcelId
+}
+
+func (m *CCFStockExcel) Create() (err error) {
+	o := orm.NewOrm()
+	id, err := o.Insert(m)
+	if err != nil {
+		return
+	}
+	m.CcfStockExcelId = int(id)
+	return
+}
+
+func (m *CCFStockExcel) CreateMulti(items []*CCFStockExcel) (err error) {
+	if len(items) == 0 {
+		return
+	}
+	o := orm.NewOrm()
+	_, err = o.InsertMulti(len(items), items)
+	return
+}
+
+func (m *CCFStockExcel) Update(cols []string) (err error) {
+	o := orm.NewOrm()
+	_, err = o.Update(m, cols...)
+	return
+}
+
+func (m *CCFStockExcel) Del() (err error) {
+	o := orm.NewOrm()
+	sql := fmt.Sprintf(`DELETE FROM %s WHERE %s = ? LIMIT 1`, m.TableName(), m.PrimaryId())
+	_, err = o.Raw(sql, m.CcfStockExcelId).Exec()
+	return
+}
+
+func (m *CCFStockExcel) MultiDel(menuIds []int) (err error) {
+	if len(menuIds) == 0 {
+		return
+	}
+	o := orm.NewOrm()
+	sql := fmt.Sprintf(`DELETE FROM %s WHERE %s IN (%s)`, m.TableName(), m.PrimaryId(), utils.GetOrmInReplace(len(menuIds)))
+	_, err = o.Raw(sql, menuIds).Exec()
+	return
+}
+
+func (m *CCFStockExcel) GetItemById(id int) (item *CCFStockExcel, err error) {
+	o := orm.NewOrm()
+	sql := fmt.Sprintf(`SELECT * FROM %s WHERE %s = ? LIMIT 1`, m.TableName(), m.PrimaryId())
+	err = o.Raw(sql, id).QueryRow(&item)
+	return
+}
+
+func (m *CCFStockExcel) GetItemByCondition(condition string, pars []interface{}, orderRule string) (item *CCFStockExcel, err error) {
+	o := orm.NewOrm()
+	order := ``
+	if orderRule != "" {
+		order = ` ORDER BY ` + orderRule
+	}
+	sql := fmt.Sprintf(`SELECT * FROM %s WHERE 1=1 %s %s LIMIT 1`, m.TableName(), condition, order)
+	err = o.Raw(sql, pars).QueryRow(&item)
+	return
+}
+
+func (m *CCFStockExcel) GetCountByCondition(condition string, pars []interface{}) (count int, err error) {
+	o := orm.NewOrm()
+	sql := fmt.Sprintf(`SELECT COUNT(1) FROM %s WHERE 1=1 %s`, m.TableName(), condition)
+	err = o.Raw(sql, pars).QueryRow(&count)
+	return
+}
+
+func (m *CCFStockExcel) GetItemsByCondition(condition string, pars []interface{}, fieldArr []string, orderRule string) (items []*CCFStockExcel, err error) {
+	o := orm.NewOrm()
+	fields := strings.Join(fieldArr, ",")
+	if len(fieldArr) == 0 {
+		fields = `*`
+	}
+	order := fmt.Sprintf(`ORDER BY %s DESC`, m.Cols().CreateTime)
+	if orderRule != "" {
+		order = ` ORDER BY ` + orderRule
+	}
+	sql := fmt.Sprintf(`SELECT %s FROM %s WHERE 1=1 %s %s`, fields, m.TableName(), condition, order)
+	_, err = o.Raw(sql, pars).QueryRows(&items)
+	return
+}
+
+func (m *CCFStockExcel) GetPageItemsByCondition(condition string, pars []interface{}, fieldArr []string, orderRule string, startSize, pageSize int) (items []*CCFStockExcel, err error) {
+	o := orm.NewOrm()
+	fields := strings.Join(fieldArr, ",")
+	if len(fieldArr) == 0 {
+		fields = `*`
+	}
+	order := fmt.Sprintf(`ORDER BY %s DESC`, m.Cols().CreateTime)
+	if orderRule != "" {
+		order = ` ORDER BY ` + orderRule
+	}
+	sql := fmt.Sprintf(`SELECT %s FROM %s WHERE 1=1 %s %s LIMIT ?,?`, fields, m.TableName(), condition, order)
+	_, err = o.Raw(sql, pars, startSize, pageSize).QueryRows(&items)
+	return
+}

models/db.go

@@ -7,9 +7,10 @@ import (
 	"eta/eta_index_lib/models/future_good"
 	"eta/eta_index_lib/models/supply_analysis"
 	"eta/eta_index_lib/utils"
+	"time"
+
 	"github.com/beego/beego/v2/client/orm"
 	_ "github.com/go-sql-driver/mysql"
-	"time"
 )
 
 func init() {
@@ -42,6 +43,9 @@ func init() {
 		new(EdbDataCalculateZjpj),
 		new(EdbDataCalculateLjztbpj),
 		new(EdbInfo),
+		new(EdbInfoRecord),
+		new(BaseFromSmmRecord),
+		new(BaseFromMysteelChemicalRecord),
 		new(EdbInfoCalculateMapping),
 		new(EdbPythonCode),
 		new(EdbDataPython),
@@ -54,9 +58,10 @@ func init() {
 		new(EdbDataPredictCalculateLjztbpj),
 		new(EdbDataPredictCalculateNhcc),
 		new(EdbDataPredictCalculateZjpj),
-		new(EdbDataInsertConfig),
 		new(EdbAdjustConf), // 数据调整的配置
+		new(EdbDataInsertConfig),
 		new(BaseFromMysteelChemicalClassify),
+		new(EdbInfoRelation), //指标引用记录
 	)
 
 	// 注册期货数据 数据表
@@ -128,15 +133,18 @@ func initBaseIndex() {
 		new(BaseFromCoalmineInlandIndex),
 		new(BaseFromCoalmineCompanyIndex),
 		new(BaseFromCoalmineFirmIndex),
-		new(BaseFromMtjhMapping),
-		new(BaseFromMtjhIndex),
 		new(BaseFromFenweiIndex),
 		new(BaseFromFenweiData),
+		new(BaseFromMtjhMapping),
+		new(BaseFromMtjhIndex),
 		new(BaseFromBloombergIndex),
 		new(BaseFromBloombergData),
 		new(BaseFromSci99Data),
 		new(BaseFromSci99Index),
 		new(BaseFromSci99Classify),
+		new(BaseFromCCFIndex),
+		new(BaseFromCCFData),
+		new(CCFStockExcel),
 	)
 }
 

models/edb_data_business.go

@@ -1,10 +1,12 @@
 package models
 
 import (
+	"errors"
 	"eta/eta_index_lib/models/mgo"
 	"eta/eta_index_lib/utils"
 	"fmt"
 	"github.com/beego/beego/v2/client/orm"
+	"github.com/qiniu/qmgo"
 	"github.com/shopspring/decimal"
 	"go.mongodb.org/mongo-driver/bson"
 	"go.mongodb.org/mongo-driver/bson/primitive"
@@ -406,10 +408,19 @@ func (obj Business) GetEdbInfoMaxAndMinInfo(edbCode string) (item *EdbInfoMaxAnd
 // @return err error
 func (obj Business) UnifiedModifyEdbInfoMaxAndMinInfo(edbInfo *EdbInfo) (err error) {
 	edbInfoMaxAndMinInfo, err := obj.GetEdbInfoMaxAndMinInfo(edbInfo.EdbCode)
-	if err != nil {
+	// 如果有错误，且错误信息是取不到文档，那么就不修改了
+	if err != nil && !errors.Is(err, qmgo.ErrNoSuchDocuments) {
 		return
 	}
-	err = ModifyEdbInfoMaxAndMinInfo(edbInfo.EdbInfoId, edbInfoMaxAndMinInfo)
+
+	// 如果正常获取到了，那就去修改指标的最大最小值
+	if err == nil && edbInfoMaxAndMinInfo != nil {
+		err = ModifyEdbInfoMaxAndMinInfo(edbInfo.EdbInfoId, edbInfoMaxAndMinInfo)
+	} else {
+		// 清空的目的是为了避免异常返回
+		err = nil
+	}
+
 	return
 }
 

models/edb_data_calculate_bp.go

@@ -348,177 +348,107 @@ func refreshAllCalculateBp(to orm.TxOrmer, edbInfoId, source, subSource int, fro
 	if err != nil {
 		return err
 	}
-	var dateArr []string
-	dataMap := make(map[string]*EdbInfoSearchData)
-	fromDataMap := make(map[string]float64)
-	//来源指指标数据
-	for _, v := range dataList {
-		dateArr = append(dateArr, v.DataTime)
-		dataMap[v.DataTime] = v
-		fromDataMap[v.DataTime] = v.Value
+
+	// 来源指标没有数据，那么需要删除所有的计算指标数据
+	if len(dataList) <= 0 {
+		// todo 删除所有的计算指标数据
+		return
+	}
+	// 来源指标的第一个日期
+	fromFirstDate, err := time.ParseInLocation(utils.FormatDate, dataList[0].DataTime, time.Local)
+	if err != nil {
+		return
+	}
+	fromFirstDate = time.Date(fromFirstDate.Year(), fromFirstDate.Month(), fromFirstDate.Day(), 0, 0, 0, 0, time.Local)
+
+	// 变频计算
+	newDataList, err := EdbInfoSearchDataToData(dataList)
+	if err != nil {
+		return
+	}
+
+	baseCalculate := BaseCalculate{
+		DataList:      newDataList,
+		Frequency:     "",
+		Formula:       nil,
+		Calendar:      "",
+		MoveType:      0,
+		MoveFrequency: "",
+		FromFrequency: "",
+		Source:        source,
+	}
+	dateDataMap, err, _ := baseCalculate.UpFrequency()
+	if err != nil {
+		return
 	}
-	fmt.Println("source:", source)
 
-	//获取升频指标所有数据
+	// 获取升频指所有已经存在的计算指标数据
 	existDataList, err := GetAllEdbDataListByTo(to, edbInfoId, source, subSource)
 	if err != nil {
 		return
 	}
 	//计算指标的map
 	existDataMap := make(map[string]*EdbData, 0)
+	for _, v := range existDataList {
+		existDataMap[v.DataTime] = v
+	}
 
 	addSql := ` INSERT INTO edb_data_calculate_bp(edb_info_id,edb_code,data_time,value,create_time,modify_time,data_timestamp) values `
 	var isAdd bool
 
-	var lastValue float64   //最后数据的值(float64)
-	var lastValueStr string //最后数据的值(string)
-	//待删除的日期
-	removeDateList := make([]string, 0)
-	if len(existDataList) > 0 {
-		//第一个已经入库的日期
-		firstExistDataTimeStr := existDataList[0].DataTime //计算指标数据第一条的日期字符串
-		if len(dateArr) > 0 {
-			firstFromDataTimeStr := dateArr[0]                                                                 //来源数据第一条的日期字符串
-			firstExistDataTime, _ := time.ParseInLocation(utils.FormatDate, firstExistDataTimeStr, time.Local) //计算指标数据第一条的日期（time类型）
-			firstFromDataTime, _ := time.ParseInLocation(utils.FormatDate, firstFromDataTimeStr, time.Local)   //来源数据第一条的日期（time类型）
-			nowDateStr := time.Now().Format(utils.FormatDate)                                                  //当天日期字符串
-			nowDate, _ := time.ParseInLocation(utils.FormatDate, nowDateStr, firstFromDataTime.Location())     //当天日期（time类型）
-
-			lastValue = fromDataMap[firstFromDataTimeStr]
-			lastValueStr = decimal.NewFromFloat(lastValue).String()
-			//第一步： 判断来源指标的开始时间与计算指标的开始时间是否相等，相等的话，那么就不需要对两个时间之间的数据做处理
-			if firstExistDataTimeStr != firstFromDataTimeStr {
-				if firstExistDataTime.Before(firstFromDataTime) { //如果计算指标第一条数据的开始时间 早于 来源指标的第一条开始时间，那么需要对两个时间之间的计算指标数据做 删除处理
-					for _, v := range existDataList {
-						if v.DataTime == firstFromDataTimeStr {
-							if tmpLastValue, ok := fromDataMap[firstFromDataTimeStr]; ok { //来源指标当天的数据
-								lastValue = tmpLastValue
-								lastValueStr = decimal.NewFromFloat(lastValue).String()
-							}
-							break
-						}
-						removeDateList = append(removeDateList, v.DataTime)
-					}
-				} else {
-					for _, v := range dateArr { //如果计算指标第一条数据的开始时间 晚于 来源指标的第一条开始时间，那么需要对两个时间之间的计算指标数据做 新增处理
-						vDataTime, _ := time.ParseInLocation(utils.FormatDate, v, time.Local) //当前日期（time类型）
-						if firstExistDataTime.Equal(vDataTime) || firstExistDataTime.Before(vDataTime) {
-							if tmpLastValue, ok := fromDataMap[v]; ok { //来源指标当天的数据
-								lastValue = tmpLastValue
-								lastValueStr = decimal.NewFromFloat(lastValue).String()
-							}
-							break
-						}
+	now := time.Date(time.Now().Year(), time.Now().Month(), time.Now().Day(), 0, 0, 0, 0, time.Local)
 
-						currentDate, _ := time.ParseInLocation(utils.FormatDate, v, time.Local)
-						timestamp := currentDate.UnixNano() / 1e6
-						timestampStr := fmt.Sprintf("%d", timestamp)
-						addSql += GetAddSql(edbInfoIdStr, edbCode, v, timestampStr, lastValueStr)
+	for currDate := fromFirstDate; !currDate.After(now); currDate = currDate.AddDate(0, 0, 1) {
+		currDateStr := currDate.Format(utils.FormatDate)
+		timestamp := currDate.UnixNano() / 1e6
+		timestampStr := fmt.Sprintf("%d", timestamp)
 
-						isAdd = true
-					}
-				}
-			}
-
-			//第二步 剩余数据每天修改
-
-			day := int(nowDate.Sub(firstExistDataTime).Hours() / float64(24))
-
-			//第三步： 已经入库的数据处理
-			for _, v := range existDataList {
-				existDataMap[v.DataTime] = v
-			}
+		// 当前计算的值
+		currValue, ok := dateDataMap[currDate]
+		if !ok {
+			// 没有计算成功就过滤
+			continue
+		}
+		lastValueStr := decimal.NewFromFloat(currValue).Round(4).String()
+
+		// 已经入库的值
+		existData, ok := existDataMap[currDateStr]
+		if !ok {
+			// 没有入库那么就插入添加
+			isAdd = true
+			addSql += GetAddSql(edbInfoIdStr, edbCode, currDateStr, timestampStr, lastValueStr)
+			continue
+		}
 
-			for k := day; k >= 0; k-- {
-				needDay := nowDate.AddDate(0, 0, -k)
-				needDayStr := needDay.Format(utils.FormatDate)
-				tmpExistData, ok := existDataMap[needDayStr]
-				if ok {
-					if tmpLastValue, ok := fromDataMap[tmpExistData.DataTime]; ok { //来源指标当天的数据
-						lastValue = tmpLastValue
-						//lastValueStr = decimal.NewFromFloat(lastValue).String()
-						lastValueStr = fmt.Sprintf("%.4f", lastValue)
-					}
-					//如果对应的值不匹配
-					if tmpExistData.Value != lastValueStr {
-						err = ModifyEdbDataById(source, subSource, tmpExistData.EdbDataId, lastValueStr)
-						if err != nil {
-							return err
-						}
-					}
-				} else {
-					timestamp := needDay.UnixNano() / 1e6
-					timestampStr := fmt.Sprintf("%d", timestamp)
-					addSql += GetAddSql(edbInfoIdStr, edbCode, needDayStr, timestampStr, lastValueStr)
+		// 将已经入库的值转换为decimal类型，然后再保留4位小数，目的是为了做匹配，要不然取出来的数据与计算的数据不一致
+		existDataValueDec, tmpErr := decimal.NewFromString(existData.Value)
+		if tmpErr != nil {
+			err = tmpErr
+			return
+		}
+		existDataValueStr := existDataValueDec.Round(4).String()
 
-					isAdd = true
-				}
-			}
-		} else {
-			//如果没有来源指标数据，那么已经入库的计算指标数据需要全部删除
-			tableName := GetEdbDataTableName(source, subSource)
-			sql := fmt.Sprintf(` DELETE FROM %s WHERE edb_info_id = ?`, tableName)
-			_, err = to.Raw(sql, edbInfoId).Exec()
+		// 如果该日期已经入库了，且两个值不匹配，那么就更新
+		if lastValueStr != existDataValueStr {
+			err = ModifyEdbDataById(source, subSource, existData.EdbDataId, lastValueStr)
 			if err != nil {
-				err = fmt.Errorf("删除所有的升频指标数据失败,Err:" + err.Error())
-				return
+				return err
 			}
-
-			//for _, v := range existDataList {
-			//	removeDateList = append(removeDateList, v.DataTime)
-			//}
 		}
-	} else {
-		existMap := make(map[string]string)
-		dataLen := len(dataList)
 
-		for i := 0; i < dataLen; i++ {
-			//当期
-			currentItem := dataList[i]
-			currentDate, _ := time.ParseInLocation(utils.FormatDate, currentItem.DataTime, time.Local)
-			var day int
-			var preItem *EdbInfoSearchData
-			var preDate time.Time
-			if i == 0 {
-				day = int(time.Now().Sub(currentDate).Hours() / float64(24))
-				preDate = time.Now()
-			} else {
-				j := i - 1
-				if j < dataLen {
-					preItem = dataList[j]
-					preDate, _ = time.ParseInLocation(utils.FormatDate, preItem.DataTime, time.Local)
-					day = int(preDate.Sub(currentDate).Hours() / float64(24))
-					utils.FileLog.Info("preItem.DataTime:" + preItem.DataTime + ";currentItem.DataTime" + currentItem.DataTime)
-				}
-			}
-			for k := 0; k <= day; k++ {
-				needDay := preDate.AddDate(0, 0, -k)
-				needDayStr := needDay.Format(utils.FormatDate)
-				existKey := edbCode + needDayStr
-				if _, ok := existMap[existKey]; !ok {
-					timestamp := needDay.UnixNano() / 1e6
-					timestampStr := fmt.Sprintf("%d", timestamp)
-					valStr := decimal.NewFromFloat(currentItem.Value).String()
-					addSql += GetAddSql(edbInfoIdStr, edbCode, needDayStr, timestampStr, valStr)
-					isAdd = true
-				}
-				existMap[existKey] = needDayStr
-			}
-			existKey := edbCode + currentItem.DataTime
-			if _, ok := existMap[existKey]; !ok {
-				currentDate, _ := time.ParseInLocation(utils.FormatDate, currentItem.DataTime, time.Local)
-				timestamp := currentDate.UnixNano() / 1e6
-				timestampStr := fmt.Sprintf("%d", timestamp)
-				valStr := decimal.NewFromFloat(currentItem.Value).String()
-				addSql += GetAddSql(edbInfoIdStr, edbCode, currentItem.DataTime, timestampStr, valStr)
-				isAdd = true
-			}
-			existMap[existKey] = currentItem.DataTime
-		}
+		// 该日期已经处理过了，所以需要移除，如果后面该map还有数据，那么需要删除该map里面的日期数据
+		delete(existDataMap, currDateStr)
+
 	}
 
 	// 删除不需要的指标数据
-	if len(removeDateList) > 0 {
+	if len(existDataMap) > 0 {
+		//待删除的日期
+		removeDateList := make([]string, 0)
+		for date := range existDataMap {
+			removeDateList = append(removeDateList, date)
+		}
+
 		removeDateStr := strings.Join(removeDateList, `","`)
 		removeDateStr = `"` + removeDateStr + `"`
 		//如果拼接指标变更了，那么需要删除所有的指标数据
@@ -532,9 +462,11 @@ func refreshAllCalculateBp(to orm.TxOrmer, edbInfoId, source, subSource int, fro
 		}
 	}
 
+	// 新增的数据值
 	if isAdd {
 		addSql = strings.TrimRight(addSql, ",")
 		_, err = to.Raw(addSql).Exec()
 	}
+
 	return
 }

models/edb_data_calculate_percentile.go

@@ -244,6 +244,7 @@ func (obj Percentile) GetEdbType() int {
 type PercentileConfig struct {
 	CalculateValue int    `description:"时间长度期数"`
 	CalculateUnit  string `description:"时间长度频度"`
+	PercentType    int    `description:"百分位：0-数据区间(兼容历史数据); 1-数据个数；"`
 }
 
 // refresh 刷新
@@ -259,7 +260,7 @@ func (obj Percentile) refresh(to orm.TxOrmer, edbInfo, fromEdbInfo *EdbInfo, edb
 	}
 
 	// 获取百分位的指标数据
-	fromDataList, err, errMsg := obj.getPercentileData(fromEdbInfo, percentileConfig.CalculateValue, percentileConfig.CalculateUnit)
+	fromDataList, err, errMsg := obj.getPercentileData(fromEdbInfo, percentileConfig.CalculateValue, percentileConfig.CalculateUnit, percentileConfig.PercentType)
 	if err != nil {
 		return
 	}
@@ -374,7 +375,7 @@ func (obj Percentile) calculate(edbInfoId int, date, edbInfoIdStr, edbCode, data
 }
 
 // GetPercentileData 获取百分位图表的指标数据
-func (obj Percentile) getPercentileData(fromEdbInfo *EdbInfo, calculateValue int, calculateUnit string) (newDataList []EdbInfoSearchData, err error, errMsg string) {
+func (obj Percentile) getPercentileData(fromEdbInfo *EdbInfo, calculateValue int, calculateUnit string, percentType int) (newDataList []EdbInfoSearchData, err error, errMsg string) {
 	// 获取时间基准指标在时间区间内的值
 	dataList := make([]*EdbInfoSearchData, 0)
 	switch fromEdbInfo.EdbInfoType {
@@ -408,33 +409,79 @@ func (obj Percentile) getPercentileData(fromEdbInfo *EdbInfo, calculateValue int
 	}
 
 	//百分位：对所选指标滚动地取对应时间长度的数据值，取最大值Max，最小值Min，计算Max-Min，百分位=(现值-Min)/(Max-Min)，Max=Min时不予计算。
-	for i, tmpData := range dataList {
-		currDateTime, _ := time.ParseInLocation(utils.FormatDate, tmpData.DataTime, time.Local)
-		maxVal := tmpData.Value
-		minVal := tmpData.Value
-		for k := 0; k < calculateDay; k++ {
-			preVal, ok2 := dataMap[currDateTime.AddDate(0, 0, -k)]
-			if ok2 {
-				if preVal > maxVal {
-					maxVal = preVal
-				}
-				if preVal < minVal {
-					minVal = preVal
+	if percentType == utils.PercentCalculateTypeRange {
+		for i, tmpData := range dataList {
+			currDateTime, _ := time.ParseInLocation(utils.FormatDate, tmpData.DataTime, time.Local)
+			maxVal := tmpData.Value
+			minVal := tmpData.Value
+			for k := 0; k < calculateDay; k++ {
+				preVal, ok2 := dataMap[currDateTime.AddDate(0, 0, -k)]
+				if ok2 {
+					if preVal > maxVal {
+						maxVal = preVal
+					}
+					if preVal < minVal {
+						minVal = preVal
+					}
 				}
 			}
+
+			if maxVal == minVal {
+				continue
+			}
+			tmpV := (tmpData.Value - minVal) / (maxVal - minVal) * 100
+			tmpV, _ = decimal.NewFromFloat(tmpV).Round(4).Float64()
+			//百分位=(现值-Min)/(Max-Min)
+			newDataList = append(newDataList, EdbInfoSearchData{
+				EdbDataId: i,
+				DataTime:  dataList[i].DataTime,
+				Value:     tmpV,
+			})
 		}
+	}
+	// 百分位数据个数算法
+	// 数据区间第一个和最后一个数据点的时间和数据分别为(T1,S1)(T2,S2); N=T1到T2指标数据个数, n=小于等于S2的数据个数
+	// 个数百分位=(n-1)/(N-1)
+	maxDay := len(dataList) // 往前找数据的边界
+	if percentType == utils.PercentCalculateTypeNum {
+		for i, d := range dataList {
+			// T2为当前日期
+			s2 := decimal.NewFromFloat(d.Value)
+			t2, _ := time.ParseInLocation(utils.FormatDate, d.DataTime, time.Local)
+
+			// 计算N和n
+			var bigN, tinyN int
+			for k := 0; k < maxDay; k++ {
+				// 往前找(时间长度)个有数据的, N理论上只有最前面几个日期<calculateDay, 后面的N=calculateDay
+				if bigN >= calculateDay {
+					break
+				}
+				preVal, preOk := dataMap[t2.AddDate(0, 0, -k)]
+				if !preOk {
+					continue
+				}
+				bigN += 1
+				if decimal.NewFromFloat(preVal).LessThanOrEqual(s2) {
+					tinyN += 1
+				}
+			}
 
-		if maxVal == minVal {
-			continue
+			// N<=1时说明计算无效
+			if bigN <= 1 {
+				continue
+			}
+			numerator := decimal.NewFromInt(int64(tinyN - 1))
+			denominator := decimal.NewFromInt(int64(bigN - 1))
+			// 因为是百分位所以这里是要*100, 跟之前的算法保持同步
+			percentVal, _ := numerator.Div(denominator).Mul(decimal.NewFromFloat(100)).Round(4).Float64()
+
+			// 写进数组并判断指标最大最小值
+			newDataList = append(newDataList, EdbInfoSearchData{
+				EdbDataId: i,
+				DataTime:  dataList[i].DataTime,
+				Value:     percentVal,
+			})
 		}
-		tmpV := (tmpData.Value - minVal) / (maxVal - minVal) * 100
-		tmpV, _ = decimal.NewFromFloat(tmpV).Round(4).Float64()
-		//百分位=(现值-Min)/(Max-Min)
-		newDataList = append(newDataList, EdbInfoSearchData{
-			EdbDataId: i,
-			DataTime:  dataList[i].DataTime,
-			Value:     tmpV,
-		})
 	}
 
 	return

models/edb_data_table.go

@@ -152,7 +152,7 @@ func GetEdbDataTableName(source, subSource int) (tableName string) {
 		tableName = "edb_data_predict_ccalculate_standard_deviation"
 	case utils.DATA_SOURCE_PREDICT_CALCULATE_PERCENTILE: //预测百分位->70
 		tableName = "edb_data_predict_ccalculate_percentile"
-	case utils.DATA_SOURCE_FUBAO:
+	case utils.DATA_SOURCE_FUBAO: //数宝->71
 		tableName = "edb_data_fubao"
 	case utils.DATA_SOURCE_CALCULATE_ZSXY:
 		tableName = "edb_data_calculate_zsxy" // 指数修匀->72

models/edb_info.go

@@ -6,11 +6,13 @@ import (
 	"eta/eta_index_lib/models/mgo"
 	"eta/eta_index_lib/utils"
 	"fmt"
-	"github.com/beego/beego/v2/client/orm"
-	"github.com/shopspring/decimal"
-	"go.mongodb.org/mongo-driver/bson"
 	"strconv"
 	"time"
+
+	"go.mongodb.org/mongo-driver/bson"
+
+	"github.com/beego/beego/v2/client/orm"
+	"github.com/shopspring/decimal"
 )
 
 type EdbInfo struct {
@@ -33,6 +35,7 @@ type EdbInfo struct {
 	UniqueCode       string `description:"指标唯一编码"`
 	CreateTime       time.Time
 	ModifyTime       time.Time
+	BaseModifyTime   time.Time
 	MinValue         float64 `description:"指标最小值"`
 	MaxValue         float64 `description:"指标最大值"`
 	CalculateFormula string  `description:"计算公式"`
@@ -1424,6 +1427,14 @@ func EdbInfoAdd(req *AddEdbInfoParams, serverUrl string, sysUserId int, sysUserR
 	return
 }
 
+func ModifyEdbInfoBaseTimeById(edbInfoId int, cTime time.Time) (err error) {
+	o := orm.NewOrm()
+	// 更新修改时间
+	sql := ` UPDATE edb_info SET base_modify_time = ? WHERE edb_info_id = ? `
+	_, err = o.Raw(sql, cTime, edbInfoId).Exec()
+	return
+}
+
 // EdbInfoRefreshCheckReq 指标数据更新情况查询
 type EdbInfoRefreshCheckReq struct {
 	Source         int    `description:"来源id"`
@@ -1480,6 +1491,17 @@ type CalculateLjzEdbExtra struct {
 	LastValType int `description:"最新值处理：0默认、均值填充"`
 }
 
+type EdbInfoEditRecord struct {
+	EdbInfoId           int    `description:"指标ID"`
+	EdbName             string `description:"指标名称"`
+	Frequency           string `description:"频率"`
+	Unit                string `description:"单位"`
+	ClassifyId          int    `description:"分类id"`
+	CalculateFormula    string `description:"计算公式"`
+	OperateUserId       int    `description:"操作人id"`
+	OperateUserRealName string `description:"操作人姓名"`
+}
+
 // GetEdbInfoByEdbCodeList
 // @Description: 根据来源和指标编码列表获取指标信息列表
 // @author: Roc

models/edb_info_record.go

@@ -0,0 +1,28 @@
+package models
+
+import (
+	"time"
+
+	"github.com/beego/beego/v2/client/orm"
+)
+
+type EdbInfoRecord struct {
+	EdbInfoRecordId     int       `orm:"column(edb_info_record_id);pk"`
+	EdbInfoId           int       `orm:"column(edb_info_id)"`
+	OldEdbName          string    `description:"旧的指标名称"`
+	OldFrequency        string    `description:"旧的频率"`
+	OldUnit             string    `description:"旧的单位"`
+	NewEdbName          string    `description:"新的指标名称"`
+	NewFrequency        string    `description:"新的频率"`
+	NewUnit             string    `description:"新的单位"`
+	OperateUserId       int       `description:"执行人id"`
+	OperateUserRealName string    `description:"执行人名称"`
+	CreateTime          time.Time `description:"记录的生成时间"`
+	Timestamp           int64     `description:"时间戳"`
+}
+
+func AddEditEdbInfoRecord(edbRecord *EdbInfoRecord) (e error) {
+	o := orm.NewOrm()
+	_, e = o.Insert(edbRecord)
+	return
+}

models/edb_info_relation.go

@@ -0,0 +1,123 @@
+package models
+
+import (
+	"eta/eta_index_lib/utils"
+	"github.com/beego/beego/v2/client/orm"
+	"time"
+)
+
+type EdbInfoRelation struct {
+	EdbInfoRelationId  int       `orm:"column(edb_info_relation_id);pk"`
+	EdbInfoId          int       `description:"指标id"`
+	Source             int       `description:"来源:1:同花顺,2:wind,3:彭博,4:指标运算,5:累计值转月,6:同比值,7:同差值,8:N数值移动平均计算,9:手工指标,10:隆众"`
+	EdbName            string    `description:"指标名称"`
+	EdbCode            string    `description:"指标编码"`
+	ReferObjectId      int       `description:"引用对象ID（图表ID，ETA逻辑ID等）"`
+	ReferObjectType    int       `description:"引用对象ID类型（1.图表，2.ETA逻辑）"`
+	ReferObjectSubType int       `description:"引用对象子类"`
+	CreateTime         time.Time `description:"创建时间"`
+	ModifyTime         time.Time `description:"修改时间"`
+	RelationTime       time.Time `description:"引用时间"`
+	RelationType       int       `description:"引用类型，0:直接饮用，1间接引用"`
+	RootEdbInfoId      int       `description:"间接引用时，关联的直接引用的指标ID"`
+	ChildEdbInfoId     int       `description:"间接引用时，计算指标直接关联的指标ID"`
+	RelationCode       string    `description:"引用标识"`
+	ParentRelationId   int       `description:"间接引用关联的直接引用的ID"`
+}
+
+func (e *EdbInfoRelation) TableName() string {
+	return "edb_info_relation"
+}
+
+// GetEdbInfoRelationByChildEdbInfoId 查询引用的指标ID
+func GetEdbInfoRelationByChildEdbInfoId(edbInfoId int) (item *EdbInfoRelation, err error) {
+	o := orm.NewOrm()
+	msql := ` SELECT * FROM edb_info_relation WHERE child_edb_info_id = ?`
+	err = o.Raw(msql, edbInfoId).QueryRow(&item)
+	return
+}
+
+// GetEdbInfoRelationListByChildEdbInfoId 根据间接引用中的的计算指标ID查询引用列表
+func GetEdbInfoRelationListByChildEdbInfoId(edbInfoId int) (items []*EdbInfoRelation, err error) {
+	o := orm.NewOrm()
+	msql := ` SELECT * FROM edb_info_relation WHERE relation_type=1 AND child_edb_info_id=? `
+	_, err = o.Raw(msql, edbInfoId).QueryRows(&items)
+	return
+}
+
+// 新增记录
+func AddOrUpdateEdbInfoRelationByChildEdbInfoId(relationList []*EdbInfoRelation, refreshEdbInfoIds []int, indexCodeList []string, deleteRelationIds []int) (err error) {
+	o, err := orm.NewOrm().Begin()
+	if err != nil {
+		return
+	}
+	defer func() {
+		if err != nil {
+			_ = o.Rollback()
+			return
+		}
+		_ = o.Commit()
+	}()
+
+	relationCodesMap := make(map[string]struct{}, 0)
+	if len(relationList) > 0 {
+		for _, relation := range relationList {
+			if relation.RelationType == 1 {
+				relationCodesMap[relation.RelationCode] = struct{}{}
+			}
+		}
+		_, err = o.InsertMulti(len(relationList), relationList)
+		if err != nil {
+			return
+		}
+	}
+
+	if len(deleteRelationIds) > 0 {
+		// 删除对应的记录
+		sql := ` DELETE FROM edb_info_relation WHERE edb_info_relation_id in (` + utils.GetOrmInReplace(len(deleteRelationIds)) + `)`
+		_, err = o.Raw(sql, deleteRelationIds).Exec()
+		if err != nil {
+			return
+		}
+	}
+
+	if len(refreshEdbInfoIds) > 0 {
+		// todo 更新指标的刷新状态
+		sql := ` UPDATE edb_info SET no_update = 0 WHERE source in (?, ?) AND edb_info_id IN (` + utils.GetOrmInReplace(len(refreshEdbInfoIds)) + `) AND no_update = 1`
+		_, err = o.Raw(sql, utils.DATA_SOURCE_MYSTEEL_CHEMICAL, utils.DATA_SOURCE_WIND, refreshEdbInfoIds).Exec()
+		if err != nil {
+			return
+		}
+	}
+
+	//更新数据源钢联化工指标
+	if len(indexCodeList) > 0 {
+		// 更改数据源的更新状态
+		sql := ` UPDATE base_from_mysteel_chemical_index SET is_stop = 0 WHERE index_code IN (` + utils.GetOrmInReplace(len(indexCodeList)) + `) and is_stop=1`
+		_, err = o.Raw(sql, indexCodeList).Exec()
+		if err != nil {
+			return
+		}
+	}
+
+	if len(relationList) > 0 {
+		// 更新间接引用指标的关联ID
+		relationCodes := make([]string, 0)
+		for relationCode := range relationCodesMap {
+			relationCodes = append(relationCodes, relationCode)
+		}
+		if len(relationCodes) > 0 {
+			sql := ` UPDATE edb_info_relation e1  
+JOIN edb_info_relation e2 ON e1.relation_code = e2.relation_code   
+SET e1.parent_relation_id = e2.edb_info_relation_id  
+WHERE  
+    e1.relation_type = 1   
+    AND e2.relation_type = 0 AND e1.parent_relation_id !=e2.edb_info_relation_id AND e1.relation_code in (` + utils.GetOrmInReplace(len(relationCodes)) + `)`
+			_, err = o.Raw(sql, relationCodes).Exec()
+			if err != nil {
+				return
+			}
+		}
+	}
+	return
+}

models/future_good/future_good_edb_data.go

@@ -1,6 +1,7 @@
 package future_good
 
 import (
+	"eta/eta_index_lib/services/alarm_msg"
 	"eta/eta_index_lib/utils"
 	"fmt"
 	"github.com/beego/beego/v2/client/orm"
@@ -84,9 +85,16 @@ func GetFutureGoodEdbDataList(condition string, pars []interface{}) (list []*Fut
 	return
 }
 
-// AddEdbDataFromThs 添加Ths商品指标数据
+// AddEdbDataFromThs 添加同花顺商品指标数据
 func AddEdbDataFromThs(futureGoodEdbInfoId int, edbCode string, item FutureGoodDataFromThs) (err error) {
+	var errMsg string
 	o := orm.NewOrm()
+	defer func() {
+		if err != nil {
+			//go utils.SendEmail(utils.APP_NAME_CN+"【"+utils.RunMode+"】"+"失败提醒", " 同花顺数据获取失败:err:"+errMsg, utils.EmailSendToUsers)
+			go alarm_msg.SendAlarmMsg("wind商品数据获取失败:err:"+errMsg, 3)
+		}
+	}()
 
 	var isAdd bool
 	addSql := ` INSERT INTO future_good_edb_data(future_good_edb_info_id,future_good_edb_code,data_time,trade_code,open,high,low,close,volume,amt,oi,settle,create_time,modify_time,data_timestamp) values `
@@ -97,6 +105,7 @@ func AddEdbDataFromThs(futureGoodEdbInfoId int, edbCode string, item FutureGoodD
 		eDate := table.Time[k]
 		dataTime, err := time.ParseInLocation(utils.FormatDate, eDate, time.Local)
 		if err != nil {
+			errMsg = " time.Parse :" + err.Error()
 			return err
 		}
 		timestamp := dataTime.UnixNano() / 1e6
@@ -121,6 +130,7 @@ func AddEdbDataFromThs(futureGoodEdbInfoId int, edbCode string, item FutureGoodD
 		addSql = strings.TrimRight(addSql, ",")
 		_, err = o.Raw(addSql).Exec()
 		if err != nil {
+			errMsg = " tx.Exec Err :" + err.Error()
 			return
 		}
 	}

models/predict_edb_data_calculate_bp.go

@@ -217,12 +217,13 @@ func refreshAllPredictCalculateBp(to orm.TxOrmer, edbInfoId, source, subSource i
 	if len(existDataList) > 0 {
 		//第一个已经入库的日期
 		firstExistDataTimeStr := existDataList[0].DataTime //计算指标数据第一条的日期字符串
-		if len(dateArr) > 0 {
-			firstFromDataTimeStr := dateArr[0]                                                                 //来源数据第一条的日期字符串
-			firstExistDataTime, _ := time.ParseInLocation(utils.FormatDate, firstExistDataTimeStr, time.Local) //计算指标数据第一条的日期（time类型）
-			firstFromDataTime, _ := time.ParseInLocation(utils.FormatDate, firstFromDataTimeStr, time.Local)   //来源数据第一条的日期（time类型）
-			nowDateStr := time.Now().Format(utils.FormatDate)                                                  //当天日期字符串
-			nowDate, _ := time.ParseInLocation(utils.FormatDate, nowDateStr, firstFromDataTime.Location())     //当天日期（time类型）
+		lenDateArr := len(dateArr)
+		if lenDateArr > 0 {
+			firstFromDataTimeStr := dateArr[0]                                                                     //来源数据第一条的日期字符串
+			firstExistDataTime, _ := time.ParseInLocation(utils.FormatDate, firstExistDataTimeStr, time.Local)     //计算指标数据第一条的日期（time类型）
+			firstFromDataTime, _ := time.ParseInLocation(utils.FormatDate, firstFromDataTimeStr, time.Local)       //来源数据第一条的日期（time类型）
+			fromEndDateStr := dateArr[lenDateArr-1]                                                                //当天日期字符串
+			fromEndDate, _ := time.ParseInLocation(utils.FormatDate, fromEndDateStr, firstFromDataTime.Location()) //当天日期（time类型）
 
 			lastValue = fromDataMap[firstFromDataTimeStr]
 			lastValueStr = decimal.NewFromFloat(lastValue).String()
@@ -265,7 +266,7 @@ func refreshAllPredictCalculateBp(to orm.TxOrmer, edbInfoId, source, subSource i
 
 			//第二步 剩余数据每天修改
 
-			day := int(nowDate.Sub(firstExistDataTime).Hours() / float64(24))
+			day := int(fromEndDate.Sub(firstExistDataTime).Hours() / float64(24))
 
 			//第三步： 已经入库的数据处理
 			for _, v := range existDataList {
@@ -273,7 +274,7 @@ func refreshAllPredictCalculateBp(to orm.TxOrmer, edbInfoId, source, subSource i
 			}
 
 			for k := day; k >= 0; k-- {
-				needDay := nowDate.AddDate(0, 0, -k)
+				needDay := fromEndDate.AddDate(0, 0, -k)
 				needDayStr := needDay.Format(utils.FormatDate)
 				tmpExistData, ok := existDataMap[needDayStr]
 				if ok {

routers/commentsRouter.go

@@ -187,6 +187,42 @@ func init() {
             Filters: nil,
             Params: nil})
 
+    beego.GlobalControllerRouter["eta/eta_index_lib/controllers:CCFController"] = append(beego.GlobalControllerRouter["eta/eta_index_lib/controllers:CCFController"],
+        beego.ControllerComments{
+            Method: "Add",
+            Router: `/add`,
+            AllowHTTPMethods: []string{"post"},
+            MethodParams: param.Make(),
+            Filters: nil,
+            Params: nil})
+
+    beego.GlobalControllerRouter["eta/eta_index_lib/controllers:CCFController"] = append(beego.GlobalControllerRouter["eta/eta_index_lib/controllers:CCFController"],
+        beego.ControllerComments{
+            Method: "HandleEdbData",
+            Router: `/handle/edb_data`,
+            AllowHTTPMethods: []string{"post"},
+            MethodParams: param.Make(),
+            Filters: nil,
+            Params: nil})
+
+    beego.GlobalControllerRouter["eta/eta_index_lib/controllers:CCFController"] = append(beego.GlobalControllerRouter["eta/eta_index_lib/controllers:CCFController"],
+        beego.ControllerComments{
+            Method: "HandleTableData",
+            Router: `/handle/table_data`,
+            AllowHTTPMethods: []string{"post"},
+            MethodParams: param.Make(),
+            Filters: nil,
+            Params: nil})
+
+    beego.GlobalControllerRouter["eta/eta_index_lib/controllers:CCFController"] = append(beego.GlobalControllerRouter["eta/eta_index_lib/controllers:CCFController"],
+        beego.ControllerComments{
+            Method: "Refresh",
+            Router: `/refresh`,
+            AllowHTTPMethods: []string{"post"},
+            MethodParams: param.Make(),
+            Filters: nil,
+            Params: nil})
+
     beego.GlobalControllerRouter["eta/eta_index_lib/controllers:CalculateController"] = append(beego.GlobalControllerRouter["eta/eta_index_lib/controllers:CalculateController"],
         beego.ControllerComments{
             Method: "Add",
@@ -214,6 +250,15 @@ func init() {
             Filters: nil,
             Params: nil})
 
+    beego.GlobalControllerRouter["eta/eta_index_lib/controllers:CalculateController"] = append(beego.GlobalControllerRouter["eta/eta_index_lib/controllers:CalculateController"],
+        beego.ControllerComments{
+            Method: "StepCalculate",
+            Router: `/base/step_calculate`,
+            AllowHTTPMethods: []string{"post"},
+            MethodParams: param.Make(),
+            Filters: nil,
+            Params: nil})
+
     beego.GlobalControllerRouter["eta/eta_index_lib/controllers:CalculateController"] = append(beego.GlobalControllerRouter["eta/eta_index_lib/controllers:CalculateController"],
         beego.ControllerComments{
             Method: "BatchEdit",

routers/router.go

@@ -100,14 +100,14 @@ func init() {
 				&controllers.GieController{},
 			),
 		),
-		beego.NSNamespace("/coal",
+		beego.NSNamespace("/python",
 			beego.NSInclude(
-				&controllers.CoalController{},
+				&controllers.PythonController{},
 			),
 		),
-		beego.NSNamespace("/python",
+		beego.NSNamespace("/coal",
 			beego.NSInclude(
-				&controllers.PythonController{},
+				&controllers.CoalController{},
 			),
 		),
 		beego.NSNamespace("/google_travel",
@@ -115,6 +115,16 @@ func init() {
 				&controllers.GoogleTravelController{},
 			),
 		),
+		beego.NSNamespace("/predict_calculate",
+			beego.NSInclude(
+				&controllers.PredictCalculateController{},
+			),
+		),
+		beego.NSNamespace("/mysteel_chemical",
+			beego.NSInclude(
+				&controllers.MySteelChemicalController{},
+			),
+		),
 		beego.NSNamespace("/mysteel_chemical",
 			beego.NSInclude(
 				&controllers.MySteelChemicalController{},
@@ -220,24 +230,24 @@ func init() {
 				&controllers.CoalMineDataController{},
 			),
 		),
-		beego.NSNamespace("/fenwei",
+		beego.NSNamespace("/gz",
 			beego.NSInclude(
-				&controllers.FenweiController{},
+				&controllers.GzController{},
 			),
 		),
-		beego.NSNamespace("/gz",
+		beego.NSNamespace("/icpi",
 			beego.NSInclude(
-				&controllers.GzController{},
+				&controllers.IcpiController{},
 			),
 		),
-		beego.NSNamespace("/mtjh",
+		beego.NSNamespace("/fenwei",
 			beego.NSInclude(
-				&controllers.MtjhDataController{},
+				&controllers.FenweiController{},
 			),
 		),
-		beego.NSNamespace("/icpi",
+		beego.NSNamespace("/mtjh",
 			beego.NSInclude(
-				&controllers.IcpiController{},
+				&controllers.MtjhDataController{},
 			),
 		),
 		beego.NSNamespace("/edb_refresh",
@@ -265,6 +275,11 @@ func init() {
 				&controllers.Sci99Crawler{},
 			),
 		),
+		beego.NSNamespace("/ccf",
+			beego.NSInclude(
+				&controllers.CCFController{},
+			),
+		),
 		beego.NSNamespace("/shanghai_smm",
 			beego.NSInclude(
 				&controllers.ShanghaiSmmController{},

services/base_from_business.go

@@ -8,6 +8,7 @@ import (
 	"eta/eta_index_lib/services/alarm_msg"
 	"eta/eta_index_lib/utils"
 	"fmt"
+	"github.com/qiniu/qmgo"
 	"go.mongodb.org/mongo-driver/bson"
 	"strings"
 	"time"
@@ -426,6 +427,22 @@ func DelBusinessIndexData(indexCode string, startDate, endDate string) (err erro
 		return
 	}
 
+	//修改最大最小日期
+	indexMaxAndMinInfo, err := item.GetEdbInfoMaxAndMinInfo(item.IndexCode)
+	// 如果有错误，且错误信息是取不到文档，那么就不修改了
+	if err != nil && !errors.Is(err, qmgo.ErrNoSuchDocuments) {
+		return
+	}
+	if err == nil && indexMaxAndMinInfo != nil {
+		e := item.ModifyIndexMaxAndMinInfo(item.IndexCode, indexMaxAndMinInfo, true)
+		if e != nil {
+			fmt.Println("ModifyIndexMaxAndMinInfo Err:" + e.Error())
+		}
+	} else {
+		// 清空的目的是为了避免异常返回
+		err = nil
+	}
+
 	// 同步刷新指标库的指标
 	go refreshEdbBusiness(item.IndexCode, reqMinDate)
 

services/base_from_calculate.go

@@ -159,6 +159,11 @@ func EdbCalculateBatchSave(req models.EdbInfoCalculateBatchSaveReq, lang string)
 		}
 	}
 
+	// 记录旧的指标信息
+	oldEdbName := fromEdbInfo.EdbName
+	oldFrequency := fromEdbInfo.Frequency
+	oldUnit := fromEdbInfo.Unit
+
 	//生成指标编码
 	edbCode, err := utils.GenerateEdbCode(1, "")
 	if err != nil {
@@ -453,6 +458,24 @@ func EdbCalculateBatchSave(req models.EdbInfoCalculateBatchSaveReq, lang string)
 		return
 	}
 
+	// 记录基础信息操作变更日志
+	oldEdbInfo := new(models.EdbInfo)
+	oldEdbInfo.EdbInfoId = fromEdbInfoId
+	oldEdbInfo.EdbName = oldEdbName
+	oldEdbInfo.Frequency = oldFrequency
+	oldEdbInfo.Unit = oldUnit
+	newEdbInfoEditRecord := new(models.EdbInfoEditRecord)
+	newEdbInfoEditRecord.EdbName = req.EdbName
+	newEdbInfoEditRecord.Frequency = req.Frequency
+	newEdbInfoEditRecord.Unit = req.Unit
+	newEdbInfoEditRecord.OperateUserId = req.AdminId
+	newEdbInfoEditRecord.OperateUserRealName = req.AdminName
+	err = AddEditEdbInfoRecord(oldEdbInfo, newEdbInfoEditRecord)
+	if err != nil {
+		errMsg = "记录基础信息操作变更日志失败，Err:" + err.Error()
+		err = fmt.Errorf("操作记录保存失败")
+		return
+	}
 	// 更新ES
 	go logic.UpdateEs(edbInfo.EdbInfoId)
 	return
@@ -533,6 +556,12 @@ func EdbCalculateBatchEdit(req models.EdbInfoCalculateBatchEditReq) (edbInfo *mo
 
 		return
 	}
+
+	// 记录旧的指标基本信息
+	oldEdbName := edbInfo.EdbName
+	oldFrequency := edbInfo.Frequency
+	oldUnit := edbInfo.Unit
+
 	// 基础指标id
 	fromEdbInfoId := req.FromEdbInfoId
 
@@ -854,8 +883,30 @@ func EdbCalculateBatchEdit(req models.EdbInfoCalculateBatchEditReq) (edbInfo *mo
 		return
 	}
 
+	// 记录基础信息操作变更日志
+	oldEdbInfo := new(models.EdbInfo)
+	oldEdbInfo.EdbInfoId = edbInfoId
+	oldEdbInfo.EdbName = oldEdbName
+	oldEdbInfo.Frequency = oldFrequency
+	oldEdbInfo.Unit = oldUnit
+	newEdbInfoEditRecord := new(models.EdbInfoEditRecord)
+	newEdbInfoEditRecord.EdbName = req.EdbName
+	newEdbInfoEditRecord.Frequency = req.Frequency
+	newEdbInfoEditRecord.Unit = req.Unit
+	newEdbInfoEditRecord.OperateUserId = req.AdminId
+	newEdbInfoEditRecord.OperateUserRealName = req.AdminName
+	err = AddEditEdbInfoRecord(oldEdbInfo, newEdbInfoEditRecord)
+	if err != nil {
+		errMsg = "记录基础信息操作变更日志失败，Err:" + err.Error()
+		err = fmt.Errorf("操作记录保存失败")
+		return
+	}
+
 	// 更新ES
 	go logic.UpdateEs(edbInfo.EdbInfoId)
+
+	// 重置计算指标中的引用关系
+	go ResetEdbRelation(edbInfoId)
 	return
 }
 

services/base_from_ccf.go

@@ -0,0 +1,270 @@
+package services
+
+import (
+	"eta/eta_index_lib/logic"
+	"eta/eta_index_lib/models"
+	"eta/eta_index_lib/services/alarm_msg"
+	"eta/eta_index_lib/utils"
+	"fmt"
+	"html"
+	"strings"
+	"time"
+)
+
+// HandleCCFIndex 处理CCF的excel数据
+func HandleCCFIndex(req *models.HandleCCFEdbDataReq) (err error) {
+	errMsgList := make([]string, 0)
+	defer func() {
+		if len(errMsgList) > 0 {
+			msg := fmt.Sprint("数据源-CCF数据处理失败, err:", strings.Join(errMsgList, "\n"))
+			utils.FileLog.Info(msg)
+			go alarm_msg.SendAlarmMsg(msg, 3)
+		}
+	}()
+	for _, v := range req.List {
+		if v.IndexName == "" || v.IndexCode == "" {
+			errMsgList = append(errMsgList, fmt.Sprintf("新增指标异常，指标编码%s或者指标ID%s为空：", v.IndexCode, v.IndexName))
+			continue
+		}
+		err = handleCCFIndex(v, req.TerminalCode)
+		if err != nil {
+			errMsgList = append(errMsgList, fmt.Sprintf("新增指标异常，指标编码：%s, Err: %s", v.IndexCode, err))
+			return
+		}
+	}
+	return
+}
+
+func handleCCFIndex(req *models.HandleCCFEdbData, terminalCode string) (err error) {
+	indexName := req.IndexName
+	indexCode := req.IndexCode
+	excelDataMap := req.DateData
+	errMsgList := make([]string, 0)
+	defer func() {
+		if len(errMsgList) > 0 {
+			msg := fmt.Sprint("数据源-CCF数据处理失败,err:", strings.Join(errMsgList, "\n"))
+			utils.FileLog.Info(msg)
+			go alarm_msg.SendAlarmMsg(msg, 3)
+		}
+	}()
+	indexObj := new(models.BaseFromCCFIndex)
+	dataObj := new(models.BaseFromCCFData)
+	//classifyObj := new(models.BaseFromCCFClassify)
+
+	var indexId int64
+
+	addDataList := make([]*models.BaseFromCCFData, 0)
+	updateDataList := make([]*models.BaseFromCCFData, 0)
+
+	exitDataMap := make(map[string]*models.BaseFromCCFData)
+
+	// 修改指标信息
+	if indexName == "" {
+		utils.FileLog.Info("未刷新到指标数据:indexName:" + indexName)
+		return
+	}
+
+	//判断指标是否存在
+	var isAdd int
+	item, err := indexObj.GetByIndexCode(indexCode)
+	if err != nil {
+		if err.Error() == utils.ErrNoRow() {
+			isAdd = 1
+			err = nil
+		} else {
+			isAdd = -1
+			err = fmt.Errorf("查询数据源指标库失败 GetByIndexCode Err：%s", err)
+			return
+		}
+	}
+	if item != nil && item.BaseFromCcfIndexId > 0 {
+		fmt.Println("item:", item)
+		isAdd = 2
+	} else {
+		isAdd = 1
+	}
+
+	if isAdd == 1 {
+		indexObj.IndexCode = indexCode
+		indexObj.IndexName = indexName
+		indexObj.Frequency = req.Frequency
+		indexObj.ClassifyId = req.ClassifyId
+		if req.Unit == "" {
+			req.Unit = "无"
+		}
+		indexObj.Unit = req.Unit
+		indexObj.Sort = req.Sort
+		indexObj.ModifyTime = time.Now()
+		indexObj.CreateTime = time.Now()
+		indexObj.TerminalCode = terminalCode
+		indexId, err = indexObj.Add()
+		if err != nil {
+			err = fmt.Errorf("数据源新增CCF指标失败 Err：%s", err)
+			return
+		}
+		indexObj.BaseFromCcfIndexId = indexId
+	} else if isAdd == 2 {
+		if item.TerminalCode == `` && terminalCode != `` {
+			item.TerminalCode = terminalCode
+			err = item.Update([]string{"TerminalCode"})
+			if err != nil {
+				err = fmt.Errorf("数据源更新CCF指标失败 Err：%s", err)
+				return
+			}
+		}
+
+		//获取已存在的所有数据
+		var exitDataList []*models.BaseFromCCFData
+		exitDataList, err = dataObj.GetByIndexCode(indexCode)
+		if err != nil {
+			err = fmt.Errorf("数据源查询CCF指标数据失败 Err：%s", err)
+			return
+		}
+		fmt.Println("exitDataListLen:", len(exitDataList))
+		for _, v := range exitDataList {
+			dateStr := v.DataTime
+			exitDataMap[dateStr] = v
+		}
+		indexId = item.BaseFromCcfIndexId
+	}
+
+	// 遍历excel数据，然后跟现有的数据做校验，不存在则入库
+	for date, value := range excelDataMap {
+		if findData, ok := exitDataMap[date]; !ok {
+			_, err = time.ParseInLocation(utils.FormatDate, date, time.Local)
+			if err != nil {
+				err = fmt.Errorf("%s 转换日期格式失败 Err：%s", date, err)
+				return
+			}
+			if !strings.Contains(value, "#N/A") {
+				var saveDataTime time.Time
+				if strings.Contains(date, "00:00:00") {
+					saveDataTime, err = time.Parse(utils.FormatDateTime, date)
+				} else {
+					saveDataTime, err = time.Parse(utils.FormatDate, date)
+				}
+				if err != nil {
+					err = fmt.Errorf("%s 转换日期格式失败 Err：%s", date, err)
+					continue
+				}
+				timestamp := saveDataTime.UnixNano() / 1e6
+
+				dataItem := new(models.BaseFromCCFData)
+				dataItem.BaseFromCcfIndexId = int(indexId)
+				dataItem.IndexCode = indexCode
+				dataItem.DataTime = date
+				dataItem.Value = value
+				dataItem.CreateTime = time.Now()
+				dataItem.ModifyTime = time.Now()
+				dataItem.DataTimestamp = timestamp
+				addDataList = append(addDataList, dataItem)
+			}
+		} else {
+			if findData != nil && findData.Value != value && !strings.Contains(value, "#N/A") { //修改数据
+				findData.Value = value
+				findData.ModifyTime = time.Now().Local()
+				updateDataList = append(updateDataList, findData)
+			}
+		}
+	}
+
+	if len(addDataList) > 0 {
+		err = dataObj.AddMulti(addDataList)
+		if err != nil {
+			err = fmt.Errorf("批量新增指标失败 Err：%s", err)
+			return
+		}
+
+		var dateItem *models.EdbInfoMaxAndMinInfo
+		dateItem, err = dataObj.GetMaxAndMinDateByIndexCode(indexCode)
+		if err != nil {
+			err = fmt.Errorf("查询指标最新日期失败 Err：%s", err)
+			return
+		}
+
+		go func() {
+			indexObj.ModifyIndexMaxAndMinDate(indexCode, dateItem)
+		}()
+	}
+
+	// 批量更新数据
+	if len(updateDataList) > 0 {
+		e := models.MultiUpdateBaseFromCCFDataValue(updateDataList)
+		if e != nil {
+			err = fmt.Errorf("MultiUpdateBaseFromSmmDataValue err: %s", e.Error())
+			return
+		}
+	}
+
+	// 同步刷新ETA指标库的指标
+	{
+		// 获取指标详情
+		baseObj := new(models.BaseFromCCF)
+		var edbInfo *models.EdbInfo
+		edbInfo, err = models.GetEdbInfoByEdbCode(baseObj.GetSource(), indexCode)
+		if err != nil {
+			if err.Error() != utils.ErrNoRow() {
+				errMsgList = append(errMsgList, fmt.Sprint("刷新ETA指标异常，指标编码：", indexCode, err.Error()))
+				return
+			} else {
+				err = nil
+			}
+		}
+
+		// 已经加入到指标库的话，那么就去更新ETA指标库吧
+		if edbInfo != nil {
+			go logic.RefreshBaseEdbInfo(edbInfo, ``)
+		}
+	}
+	return
+}
+
+// HandleCCFStockTable 处理CCF的表格数据
+func HandleCCFStockTable(req *models.HandleCCFStockTableReq) (err error) {
+	defer func() {
+		if err != nil {
+			tips := fmt.Sprintf("数据源-CCF数据装置处理失败, err: %v", err)
+			utils.FileLog.Info(tips)
+			go alarm_msg.SendAlarmMsg(tips, 3)
+		}
+	}()
+	if req.Table == nil {
+		err = fmt.Errorf("表格信息为空")
+		return
+	}
+
+	// 已存在则更新表格
+	excelOb := new(models.CCFStockExcel)
+	cond := fmt.Sprintf(` AND %s = ? AND %s = ?`, excelOb.Cols().ClassifyId, excelOb.Cols().ExcelDate)
+	pars := make([]interface{}, 0)
+	pars = append(pars, req.Table.ClassifyId, req.Table.TableDate.Format(utils.FormatDate))
+	exists, e := excelOb.GetItemByCondition(cond, pars, "")
+	if e != nil && e.Error() != utils.ErrNoRow() {
+		err = fmt.Errorf("获取已存在表格失败, err: %v", e)
+		return
+	}
+	if exists != nil {
+		exists.ExcelContent = html.EscapeString(req.Table.TableContent)
+		exists.FromPage = req.Table.FromPage
+		exists.ModifyTime = time.Now().Local()
+		cols := []string{excelOb.Cols().ExcelContent, excelOb.Cols().FromPage, excelOb.Cols().ModifyTime}
+		if e = exists.Update(cols); e != nil {
+			err = fmt.Errorf("更新已存在表格失败, err: %v", e)
+			return
+		}
+		return
+	}
+
+	// 新增表格
+	excelOb.ClassifyId = req.Table.ClassifyId
+	excelOb.ExcelDate = req.Table.TableDate
+	excelOb.ExcelContent = html.EscapeString(req.Table.TableContent)
+	excelOb.FromPage = req.Table.FromPage
+	excelOb.CreateTime = time.Now().Local()
+	excelOb.ModifyTime = time.Now().Local()
+	if e = excelOb.Create(); e != nil {
+		err = fmt.Errorf("新增表格失败, err: %v", e)
+		return
+	}
+	return
+}

services/base_from_mysteel_chemical.go

@@ -144,6 +144,20 @@ func handleIndex(indexItem *models.HandleMysteelIndex) (err error) {
 			fmt.Println("Index Update Err:" + e.Error())
 			return
 		}
+		if item.IndexName != indexItem.IndexName {
+			var changeRecord models.BaseFromMysteelChemicalRecord
+			changeRecord.BaseFromMysteelChemicalIndexId = item.BaseFromMysteelChemicalIndexId
+			changeRecord.OldIndexName = item.IndexName
+			changeRecord.NewIndexName = indexItem.IndexName
+			ctime := time.Now()
+			changeRecord.CreateTime = ctime
+			changeRecord.Timestamp = ctime.Unix()
+			e = changeRecord.AddBaseFromMysteelChemicalRecord()
+			if e != nil {
+				fmt.Println("mysteel chemical changeRecord Add Err:" + e.Error())
+				return
+			}
+		}
 
 		dataObj := new(models.BaseFromMysteelChemicalData)
 
@@ -158,6 +172,7 @@ func handleIndex(indexItem *models.HandleMysteelIndex) (err error) {
 			dateStr := v.DataTime.Format(utils.FormatDate)
 			exitDataMap[dateStr] = v
 		}
+
 	}
 
 	dataObj := new(models.BaseFromMysteelChemicalData)

services/base_from_smm.go

@@ -114,6 +114,21 @@ func SmmIndexHandle(baseFilePath, renameFilePath, indexName, indexCode, unit, fr
 			err = fmt.Errorf("smm index update err: %s", e.Error())
 			return
 		}
+		if item.IndexName != indexName {
+			baseFromSmmRecord := new(models.BaseFromSmmRecord)
+			baseFromSmmRecord.BaseFromSmmIndexId = item.BaseFromSmmIndexId
+			baseFromSmmRecord.OldIndexName = item.IndexName
+			baseFromSmmRecord.NewIndexName = indexName
+			ctime := time.Now()
+			baseFromSmmRecord.CreateTime = ctime
+			baseFromSmmRecord.Timestamp = ctime.Unix()
+			e = models.AddBaseFromSmmRecord(baseFromSmmRecord)
+			if e != nil {
+				err = fmt.Errorf("add smm record err: %s", e.Error())
+				return
+			}
+		}
+
 	}
 
 	// 遍历excel数据，然后跟现有的数据做校验，不存在则入库
@@ -243,4 +258,4 @@ func getEdbDataFromSmmData(edbCode string) (smmBaseDataList []models.BaseFromSmm
 	smmBaseDataListItem.IndexCode = edbCode
 	smmBaseDataList = append(smmBaseDataList, smmBaseDataListItem)
 	return
-}
+}

services/edb_info_record.go

@@ -0,0 +1,30 @@
+package services
+
+import (
+	"eta/eta_index_lib/models"
+	"time"
+)
+
+func AddEditEdbInfoRecord(oldEdbInfo *models.EdbInfo, newEdbInfo *models.EdbInfoEditRecord) (err error) {
+	if oldEdbInfo.EdbName != newEdbInfo.EdbName || oldEdbInfo.Frequency != newEdbInfo.Frequency || oldEdbInfo.Unit != newEdbInfo.Unit {
+		edbRecord := new(models.EdbInfoRecord)
+		edbRecord.EdbInfoId = oldEdbInfo.EdbInfoId
+		edbRecord.OldEdbName = oldEdbInfo.EdbName
+		edbRecord.OldFrequency = oldEdbInfo.Frequency
+		edbRecord.OldUnit = oldEdbInfo.Unit
+		edbRecord.NewEdbName = newEdbInfo.EdbName
+		edbRecord.NewFrequency = newEdbInfo.Frequency
+		edbRecord.NewUnit = newEdbInfo.Unit
+		edbRecord.OperateUserId = newEdbInfo.OperateUserId
+		edbRecord.OperateUserRealName = newEdbInfo.OperateUserRealName
+		ctime := time.Now()
+		edbRecord.CreateTime = ctime
+		edbRecord.Timestamp = ctime.Unix()
+		err = models.AddEditEdbInfoRecord(edbRecord)
+		if err != nil {
+			return
+		}
+		err = models.ModifyEdbInfoBaseTimeById(oldEdbInfo.EdbInfoId, ctime)
+	}
+	return
+}

services/edb_info_relation.go

@@ -0,0 +1,122 @@
+package services
+
+import (
+	"eta/eta_index_lib/models"
+	"eta/eta_index_lib/services/alarm_msg"
+	"eta/eta_index_lib/utils"
+	"fmt"
+	"strconv"
+	"strings"
+	"time"
+)
+
+// 重置单个计算指标中的引用关系
+func ResetEdbRelation(edbInfoId int) {
+	var err error
+	defer func() {
+		if err != nil {
+			msg := fmt.Sprintf("重置单个计算指标中的引用关系失败，错误信息：%s", err.Error())
+			go alarm_msg.SendAlarmMsg(msg, 3)
+		}
+	}()
+	//查询与该计算指标相关的间接引用或者间接引用关系，如果记录不存在，则不处理
+	_, err = models.GetEdbInfoRelationByChildEdbInfoId(edbInfoId)
+	if err != nil {
+		if err.Error() == utils.ErrNoRow() {
+			err = nil
+			return
+		}
+		err = fmt.Errorf("查询与该计算指标相关的间接引用或者间接引用关系失败，错误信息：%s", err.Error())
+		return
+	}
+
+	//查询当前计算指标最新的引用指标列表
+	newMappingList, err := models.GetEdbInfoCalculateDetailList(edbInfoId)
+	if err != nil {
+		err = fmt.Errorf("查询当前计算指标最新的指标列表失败，错误信息：%s", err.Error())
+		return
+	}
+	//整理关联的来源指标ID
+	newEdbIdList := make([]int, 0)
+	newMappingListMap := make(map[int]*models.EdbInfoCalculateDetail)
+	for _, v := range newMappingList {
+		newEdbIdList = append(newEdbIdList, v.FromEdbInfoId)
+		newMappingListMap[v.FromEdbInfoId] = v
+	}
+	// 排序
+	//二者匹配一下，如果相同，则不处理，如果不同，先查询所有旧的间接引用记录，整理并分组，则删除旧的间接引用记录，新增新的间接引用记录，
+	relationList, err := models.GetEdbInfoRelationListByChildEdbInfoId(edbInfoId)
+	if err != nil {
+		err = fmt.Errorf("查询当前计算指标的间接引用关系失败，错误信息：%s", err.Error())
+		return
+	}
+	deleteRelationIds := make([]int, 0)
+	// 根据引用对象和直接引用指标 进行分组
+	groupMap := make(map[string]map[int]*models.EdbInfoRelation)
+	// 遍历每组内容，
+	for _, v := range relationList {
+		//如果指标ID不在新的
+		if !utils.InArrayByInt(newEdbIdList, v.EdbInfoId) {
+			deleteRelationIds = append(deleteRelationIds, v.EdbInfoRelationId)
+		}
+		name := fmt.Sprintf("%d_%d_%d_%d", v.ReferObjectId, v.ReferObjectType, v.ReferObjectSubType, v.RootEdbInfoId)
+		childMap, ok := groupMap[name]
+		if !ok {
+			childMap = make(map[int]*models.EdbInfoRelation, 0)
+		}
+		childMap[v.EdbInfoId] = v
+		groupMap[name] = childMap
+	}
+	// 遍历每组内容，如果新ID不在组内，则添加
+	addList := make([]*models.EdbInfoRelation, 0)
+	indexCodeList := make([]string, 0)
+	refreshEdbIds := make([]int, 0)
+	nowTime := time.Now()
+	for name, childMap := range groupMap {
+		for _, edbId := range newEdbIdList {
+			if _, ok := childMap[edbId]; !ok {
+				// 新增记录
+				childEdb, ok1 := newMappingListMap[edbId]
+				if !ok1 {
+					continue
+				}
+				// 获取引用时间
+				relationObjectInfo := strings.Split(name, "_")
+				objectId, _ := strconv.Atoi(relationObjectInfo[0])
+				objectType, _ := strconv.Atoi(relationObjectInfo[1])
+				objectSubType, _ := strconv.Atoi(relationObjectInfo[2])
+				rootEdbId, _ := strconv.Atoi(relationObjectInfo[3])
+				tmp1 := &models.EdbInfoRelation{
+					ReferObjectId:      objectId,
+					ReferObjectType:    objectType,
+					ReferObjectSubType: objectSubType,
+					EdbInfoId:          childEdb.FromEdbInfoId,
+					EdbName:            childEdb.FromEdbName,
+					Source:             childEdb.FromSource,
+					EdbCode:            childEdb.FromEdbCode,
+					CreateTime:         nowTime,
+					ModifyTime:         nowTime,
+					RelationTime:       nowTime,
+					RelationType:       1,
+					RootEdbInfoId:      rootEdbId,
+					ChildEdbInfoId:     edbInfoId,
+				}
+				tmp1.RelationCode = fmt.Sprintf("%d_%d_%d_%d", tmp1.RootEdbInfoId, tmp1.ReferObjectId, tmp1.ReferObjectType, tmp1.ReferObjectSubType)
+				addList = append(addList, tmp1)
+				refreshEdbIds = append(refreshEdbIds, childEdb.FromEdbInfoId)
+				if childEdb.FromSource == utils.DATA_SOURCE_MYSTEEL_CHEMICAL {
+					indexCodeList = append(indexCodeList, childEdb.FromEdbCode)
+				}
+			}
+		}
+	}
+
+	if len(addList) > 0 || len(deleteRelationIds) > 0 {
+		err = models.AddOrUpdateEdbInfoRelationByChildEdbInfoId(addList, refreshEdbIds, indexCodeList, deleteRelationIds)
+		if err != nil {
+			err = fmt.Errorf("新增引用关系失败,err:%v", err)
+			return
+		}
+	}
+	return
+}

utils/constants.go

@@ -20,11 +20,6 @@ const (
 	PageSize30                 = 30
 )
 
-const (
-	DATA_SOURCE_YONYI  = 76 //涌益咨询
-	DATA_SOURCE_FENWEI = 77 //汾渭煤炭
-)
-
 // 数据来源渠道
 const (
 	DATA_SOURCE_THS                                  = iota + 1 //同花顺
@@ -40,7 +35,7 @@ const (
 	DATA_SOURCE_YS                                              //有色
 	DATA_SOURCE_CALCULATE_HBZ                                   //环比值->12
 	DATA_SOURCE_CALCULATE_HCZ                                   //环差值->13
-	DATA_SOURCE_CALCULATE_BP                                    //变频->14
+	DATA_SOURCE_CALCULATE_BP                                    //变频,2023-2-10 13:56:01调整为"升频"->14
 	DATA_SOURCE_GL                                              //钢联->15
 	DATA_SOURCE_ZZ                                              //郑商所->16
 	DATA_SOURCE_DL                                              //大商所->17
@@ -102,15 +97,17 @@ const (
 	DATA_SOURCE_PREDICT_CALCULATE_ZSXY                          // 预测指数修匀->73
 	DATA_SOURCE_CALCULATE_ZDYFX                                 // 自定义分析->74
 	DATA_SOURCE_CALCULATE_RJZ                                   // 日均值计算->75
-
-	DATA_SOURCE_GFEX          = 78 // 广州期货交易所->78
-	DATA_SOURCE_ICPI          = 79 // ICPI消费价格指数->79
-	DATA_SOURCE_MTJH          = 80 // 煤炭江湖->80
-	DATA_SOURCE_CALCULATE_SUM = 81
-	DATA_SOURCE_CALCULATE_AVG = 82
-	DATA_SOURCE_BLOOMBERG     = 83 // bloomberg彭博数据
-	DATA_SOURCE_BUSINESS      = 84 // 来源于自有数据
-	DATA_SOURCE_SCI99         = 85 // 卓创资讯
+	DATA_SOURCE_YONYI                                = 76       //涌益咨询
+	DATA_SOURCE_FENWEI                               = 77       //汾渭煤炭
+	DATA_SOURCE_GFEX                                 = 78       // 广州期货交易所->78
+	DATA_SOURCE_ICPI                                 = 79       // ICPI消费价格指数->79
+	DATA_SOURCE_MTJH                                 = 80       // 煤炭江湖->80
+	DATA_SOURCE_CALCULATE_SUM                        = 81
+	DATA_SOURCE_CALCULATE_AVG                        = 82
+	DATA_SOURCE_BLOOMBERG                            = 83 // bloomberg彭博数据
+	DATA_SOURCE_BUSINESS                             = 84 // 来源于自有数据
+	DATA_SOURCE_SCI99                                = 85 // 卓创资讯
+	DATA_SOURCE_CCF                                  = 86 // CCF化纤信息
 )
 
 // 指标来源的中文展示
@@ -190,13 +187,13 @@ const (
 	DATA_SOURCE_NAME_PREDICT_CALCULATE_ZSXY               = `预测指数修匀`            //预测指数修匀->73
 	DATA_SOURCE_NAME_CALCULATE_ZDYFX                      = `自定义分析`             //自定义分析->74
 	DATA_SOURCE_NAME_YONYI                                = `涌益咨询`              // 涌益咨询
-
-	DATA_SOURCE_NAME_FENWEI        = `汾渭数据` // 汾渭煤炭
-	DATA_SOURCE_NAME_MTJH          = `煤炭江湖` // 煤炭江湖->80
-	DATA_SOURCE_NAME_ICPI          = "ICPI消费价格指数"
-	DATA_SOURCE_NAME_CALCULATE_SUM = `多指标求和`
-	DATA_SOURCE_NAME_CALCULATE_AVG = `多指标求平均`
-	DATA_SOURCE_NAME_BUSINESS      = `自有数据`
+	DATA_SOURCE_NAME_FENWEI                               = `汾渭数据`              // 汾渭煤炭
+	DATA_SOURCE_NAME_MTJH                                 = `煤炭江湖`              // 煤炭江湖->80
+	DATA_SOURCE_NAME_ICPI                                 = "ICPI消费价格指数"
+	DATA_SOURCE_NAME_CALCULATE_SUM                        = `多指标求和`
+	DATA_SOURCE_NAME_CALCULATE_AVG                        = `多指标求平均`
+	DATA_SOURCE_NAME_BUSINESS                             = `自有数据`
+	DATA_SOURCE_NAME_CCF                                  = `CCF` // CCF化纤信息
 )
 
 // 基础数据初始化日期
@@ -290,3 +287,28 @@ const (
 	ZhLangVersion = "zh" // 中文语言版本
 	EnLangVersion = "en" // 英文语言版本
 )
+
+// 指标计算方式
+const (
+	EdbBaseCalculateLjzzy                = 1  // 累计值转月->1
+	EdbBaseCalculateLjzzj                = 2  // 累计值转季->2
+	EdbBaseCalculateTbz                  = 3  // 同比值->3
+	EdbBaseCalculateTcz                  = 4  // 同差值->4
+	EdbBaseCalculateNszydpjjs            = 5  // N数值移动平均数计算->5
+	EdbBaseCalculateHbz                  = 6  // 环比值->6
+	EdbBaseCalculateHcz                  = 7  // 环差值->7
+	EdbBaseCalculateUpFrequency          = 8  // 升频->8
+	EdbBaseCalculateDownFrequency        = 9  // 降频->9
+	EdbBaseCalculateTimeShift            = 10 // 时间移位->10
+	EdbBaseCalculateCjjx                 = 11 // 超季节性->11
+	EdbBaseCalculateAnnualized           = 12 // 年化->12
+	EdbBaseCalculateLjz                  = 13 // 累计值->13
+	EdbBaseCalculateLjzNczj              = 14 // 累计值年初至今->14
+	EdbBaseCalculateExponentialSmoothing = 15 // 指数修匀->15
+	EdbBaseCalculateRjz                  = 16 // 日均值->16
+)
+
+const (
+	PercentCalculateTypeRange = 0 // 百分位算法类型-数据区间
+	PercentCalculateTypeNum   = 1 // 百分位算法类型-数据个数
+)