package services import ( "errors" "eta/eta_forum_hub/models" "eta/eta_forum_hub/models/mgodb" "eta/eta_forum_hub/services/elastic" "eta/eta_forum_hub/utils" "fmt" "github.com/shopspring/decimal" "math" "sort" "strconv" "time" ) // BatchAddOrUpdateEdbInfo 添加批量添加指标到指标库 func BatchAddOrUpdateEdbInfo(edbList []*models.EdbInfo, edbMapping []*models.EdbInfoCalculateMapping, sysUserId int, sysUserRealName string) (newList []*models.EdbInfo, err error, errMsg string, isSendEmail bool) { isSendEmail = true edbCodeMap := make(map[string]*models.EdbInfo) for _, v := range edbList { tmp, e, msg, _ := AddOrUpdateEdbInfo(v, sysUserId, sysUserRealName) if e != nil { err = fmt.Errorf("添加指标失败：%s，%s", e.Error(), msg) errMsg = "添加指标失败" return } newList = append(newList, tmp) edbCodeMap[tmp.EdbCode] = tmp } //批量添加指标的mapping信息 calculateMappingItemList := make([]*models.EdbInfoCalculateMapping, 0) for _, v := range edbMapping { edbInfo, ok := edbCodeMap[v.EdbCode] if !ok { continue } fromEdbInfo, ok := edbCodeMap[v.FromEdbCode] if !ok { continue } calculateMappingItem := new(models.EdbInfoCalculateMapping) calculateMappingItem.CreateTime = time.Now() calculateMappingItem.ModifyTime = time.Now() calculateMappingItem.Sort = v.Sort calculateMappingItem.EdbCode = edbInfo.EdbCode calculateMappingItem.EdbInfoId = edbInfo.EdbInfoId calculateMappingItem.FromEdbInfoId = fromEdbInfo.EdbInfoId calculateMappingItem.FromEdbCode = fromEdbInfo.EdbCode calculateMappingItem.FromEdbName = fromEdbInfo.EdbName calculateMappingItem.FromSource = fromEdbInfo.Source calculateMappingItem.FromSourceName = fromEdbInfo.SourceName calculateMappingItem.FromTag = v.FromTag calculateMappingItem.Source = edbInfo.Source calculateMappingItem.SourceName = edbInfo.SourceName calculateMappingItem.FromSubSource = edbInfo.SubSource calculateMappingItemList = append(calculateMappingItemList, calculateMappingItem) } if len(calculateMappingItemList) > 0 { err = models.AddEdbInfoCalculateMappingMulti(calculateMappingItemList) if err != nil { errMsg = "指标映射关系添加失败" err = fmt.Errorf("指标映射关系添加失败, %s", err.Error()) return } } return } // AddOrUpdateEdbInfo 添加指标到指标库 func AddOrUpdateEdbInfo(edbItem *models.EdbInfo, sysUserId int, sysUserRealName string) (edbInfo *models.EdbInfo, err error, errMsg string, isSendEmail bool) { isSendEmail = true //判断指标是否存在 edbInfo, err = models.GetEdbInfoByEdbCode(edbItem.EdbCode) if err != nil { if err.Error() == utils.ErrNoRow() { err = nil } else { errMsg = "判断指标名称是否存在失败" err = errors.New("判断指标名称是否存在失败,Err:" + err.Error()) return } } var edbInfoId int64 // 指标已存在 if edbInfo != nil && edbInfo.EdbInfoId > 0 { edbInfoId = int64(edbInfo.EdbInfoId) // 更新指标信息 edbInfo.Source = edbItem.Source edbInfo.SourceName = edbItem.SourceName edbInfo.EdbType = edbItem.EdbType edbInfo.EdbCode = edbItem.EdbCode edbInfo.EdbName = edbItem.EdbName edbInfo.EdbNameSource = edbItem.EdbNameSource edbInfo.Frequency = edbItem.Frequency edbInfo.Unit = edbItem.Unit edbInfo.StartDate = edbItem.StartDate edbInfo.EndDate = edbItem.EndDate edbInfo.ClassifyId = edbItem.ClassifyId edbInfo.SysUserId = sysUserId edbInfo.SysUserRealName = sysUserRealName edbInfo.ModifyTime = time.Now() edbInfo.ServerUrl = edbItem.ServerUrl edbInfo.Sort = edbItem.Sort edbInfo.DataDateType = edbItem.DataDateType edbInfo.TerminalCode = edbItem.TerminalCode edbInfo.SourceIndexName = edbItem.SourceIndexName edbInfo.MaxValue = edbItem.MaxValue edbInfo.MinValue = edbItem.MinValue edbInfo.EdbInfoType = edbItem.EdbInfoType edbInfo.LatestDate = edbItem.LatestDate edbInfo.LatestValue = edbItem.LatestValue edbInfo.CalculateFormula = edbItem.CalculateFormula edbInfo.ManualSave = edbItem.ManualSave edbInfo.MoveType = edbItem.MoveType edbInfo.MoveFrequency = edbItem.MoveFrequency edbInfo.NoUpdate = edbItem.NoUpdate edbInfo.EdbNameEn = edbItem.EdbNameEn edbInfo.UnitEn = edbItem.UnitEn edbInfo.ChartImage = edbItem.ChartImage edbInfo.Calendar = edbItem.Calendar edbInfo.EmptyType = edbItem.EmptyType edbInfo.MaxEmptyType = edbItem.MaxEmptyType edbInfo.DataUpdateTime = edbItem.DataUpdateTime edbInfo.ErDataUpdateDate = edbItem.ErDataUpdateDate edbInfo.EndValue = edbItem.EndValue edbInfo.SubSource = edbItem.SubSource edbInfo.SubSourceName = edbItem.SubSourceName edbInfo.IndicatorCode = edbItem.IndicatorCode edbInfo.StockCode = edbItem.StockCode edbInfo.Extra = edbItem.Extra err = edbInfo.Update([]string{}) if err != nil { errMsg = "保存失败" err = errors.New("保存失败，Err:" + err.Error()) return } //同时删除指标的映射信息 err = models.DeleteEdbInfoMapping(edbInfoId) if err != nil { errMsg = "删除指标映射信息失败" err = errors.New("删除指标映射信息失败，Err:" + err.Error()) return } } else { edbInfo = new(models.EdbInfo) edbInfo.Source = edbItem.Source edbInfo.SourceName = edbItem.SourceName edbInfo.EdbType = edbItem.EdbType edbInfo.EdbCode = edbItem.EdbCode edbInfo.EdbName = edbItem.EdbName edbInfo.EdbNameSource = edbItem.EdbNameSource edbInfo.Frequency = edbItem.Frequency edbInfo.Unit = edbItem.Unit edbInfo.StartDate = edbItem.StartDate edbInfo.EndDate = edbItem.EndDate edbInfo.ClassifyId = edbItem.ClassifyId edbInfo.SysUserId = sysUserId edbInfo.SysUserRealName = sysUserRealName edbInfo.CreateTime = time.Now() edbInfo.ModifyTime = time.Now() edbInfo.ServerUrl = edbItem.ServerUrl edbInfo.Sort = edbItem.Sort edbInfo.DataDateType = edbItem.DataDateType edbInfo.TerminalCode = edbItem.TerminalCode edbInfo.SourceIndexName = edbItem.SourceIndexName //timestamp := strconv.FormatInt(time.Now().UnixNano(), 10) //edbInfo.UniqueCode = utils.MD5(utils.DATA_PREFIX + "_" + timestamp) edbInfo.UniqueCode = edbItem.UniqueCode edbInfo.MaxValue = edbItem.MaxValue edbInfo.MinValue = edbItem.MinValue edbInfo.EdbInfoType = edbItem.EdbInfoType edbInfo.LatestDate = edbItem.LatestDate edbInfo.LatestValue = edbItem.LatestValue edbInfo.CalculateFormula = edbItem.CalculateFormula edbInfo.ManualSave = edbItem.ManualSave edbInfo.MoveType = edbItem.MoveType edbInfo.MoveFrequency = edbItem.MoveFrequency edbInfo.NoUpdate = edbItem.NoUpdate edbInfo.EdbNameEn = edbItem.EdbNameEn edbInfo.UnitEn = edbItem.UnitEn edbInfo.ChartImage = edbItem.ChartImage edbInfo.Calendar = edbItem.Calendar edbInfo.EmptyType = edbItem.EmptyType edbInfo.MaxEmptyType = edbItem.MaxEmptyType edbInfo.DataUpdateTime = edbItem.DataUpdateTime edbInfo.ErDataUpdateDate = edbItem.ErDataUpdateDate edbInfo.EndValue = edbItem.EndValue edbInfo.SubSource = edbItem.SubSource edbInfo.SubSourceName = edbItem.SubSourceName edbInfo.IndicatorCode = edbItem.IndicatorCode edbInfo.StockCode = edbItem.StockCode edbInfo.Extra = edbItem.Extra edbInfoId, err = models.AddEdbInfo(edbInfo) if err != nil { errMsg = "保存失败" err = errors.New("保存失败，Err:" + err.Error()) return } edbInfo.EdbInfoId = int(edbInfoId) } if edbInfo.EdbType == 1 { err = mgodb.ModifyEdbDataEdbInfoId(edbInfoId, edbInfo.EdbCode) if err != nil { errMsg = "保存失败" err = errors.New("更新指标数据失败，Err:" + err.Error()) return } } else { err = mgodb.ModifyEdbCalculateDataEdbInfoId(edbInfoId, edbInfo.EdbCode) if err != nil { errMsg = "保存失败" err = errors.New("更新指标数据失败，Err:" + err.Error()) return } } // 更新es go AddOrEditEdbInfoToEs(edbInfo.EdbInfoId) return } // 批量删除指标， func BatchDeleteEdbInfo(edbInfoIds []int) (err error, errMsg string) { for _, v := range edbInfoIds { //查询单个指标是否允许删除 err, errMsg = DeleteEdbInfo(v) if err != nil { errMsg = "删除指标失败" return } } return } // 删除单个指标 func DeleteEdbInfo(edbInfoId int) (err error, errMsg string) { //判断指标是否存在 info, err := models.GetEdbInfoById(edbInfoId) if err != nil { if err.Error() == utils.ErrNoRow() { err = nil return } else { errMsg = "判断指标名称是否存在失败" err = errors.New("判断指标名称是否存在失败,Err:" + err.Error()) return } } //判断指标是否用于作图，如果用于作图，则不可删除 chartCount, tmpErr := models.GetChartEdbMappingCount(edbInfoId) if tmpErr != nil && tmpErr.Error() != utils.ErrNoRow() { errMsg = "删除失败" err = errors.New("判断指标是否被用于作图失败，Err:" + tmpErr.Error()) return } if chartCount > 0 { errMsg = "当前指标已用作画图，不可删除" return } //判断指标是否用作其他指标的计算 calculateCount, tmpErr := models.GetEdbInfoCalculateMappingCount(edbInfoId) if tmpErr != nil { errMsg = "删除失败" err = errors.New("判断指标是否被用于计算失败，GetEdbInfoCalculateCount Err:" + tmpErr.Error()) return } if calculateCount > 0 { errMsg = "当前指标已用作，指标运算，不可删除" return } // 删除指标，删除映射关系，删除指标数据 err = models.DeleteEdbInfo(edbInfoId) if err != nil { errMsg = "删除失败" err = errors.New("删除指标失败，Err:" + err.Error()) return } if info.EdbType == 1 { err = mgodb.DeleteEdbInfoDataByEdbInfoId(edbInfoId) if err != nil { errMsg = "删除失败" err = errors.New("删除指标数据失败，Err:" + err.Error()) return } } else { err = mgodb.DeleteEdbInfoCalculateDataByEdbInfoId(edbInfoId) if err != nil { errMsg = "删除失败" err = errors.New("删除指标数据失败，Err:" + err.Error()) return } } // 删除es中的数据 go DeleteEdbInfoToEs(edbInfoId) return } // AddOrEditEdbInfoToEs 添加/修改ES中的指标 func AddOrEditEdbInfoToEs(edbInfoId int) { //添加es itemInfo, _ := models.GetEdbInfoByCondition("AND edb_info_id=?", []interface{}{edbInfoId}) go elastic.EsAddOrEditEdbInfoData(utils.DATA_INDEX_NAME, strconv.Itoa(itemInfo.EdbInfoId), itemInfo) } // 获取频度的英文版 func GetFrequencyEn(frequency string) (frequencyEn string) { switch frequency { case "日度": frequencyEn = "day" return case "周度": frequencyEn = "week" return case "旬度": frequencyEn = "ten days" return case "月度": frequencyEn = "month" return case "季度": frequencyEn = "quarter" return case "年度": frequencyEn = "year" return } return } func GetLeadUnitEn(unit string) (unitEn string) { switch unit { case "天": unitEn = "day" return case "周": unitEn = "week" return case "月": unitEn = "month" return case "季": unitEn = "quarter" return case "年": unitEn = "year" return } return } // HandleDataByLinearRegressionToList 插值法补充数据(线性方程式) func HandleDataByLinearRegressionToList(edbInfoDataList []*models.EdbDataList, handleDataMap map[string]float64) (dataTimeList []string, valueList []float64, err error) { if len(edbInfoDataList) < 2 { return } var startEdbInfoData *models.EdbDataList for _, v := range edbInfoDataList { handleDataMap[v.DataTime] = v.Value dataTimeList = append(dataTimeList, v.DataTime) // 第一个数据就给过滤了,给后面的试用 if startEdbInfoData == nil { startEdbInfoData = v //startEdbInfoData.DataTime = startEdbInfoData.DataTime[:5]+ "01-01" continue } // 获取两条数据之间相差的天数 startDataTime, _ := time.ParseInLocation(utils.FormatDate, startEdbInfoData.DataTime, time.Local) currDataTime, _ := time.ParseInLocation(utils.FormatDate, v.DataTime, time.Local) betweenHour := int(currDataTime.Sub(startDataTime).Hours()) betweenDay := betweenHour / 24 // 如果相差一天,那么过滤 if betweenDay <= 1 { startEdbInfoData = v continue } // 生成线性方程式 var a, b float64 { coordinateData := make([]utils.Coordinate, 0) tmpCoordinate1 := utils.Coordinate{ X: 1, Y: startEdbInfoData.Value, } coordinateData = append(coordinateData, tmpCoordinate1) tmpCoordinate2 := utils.Coordinate{ X: float64(betweenDay) + 1, Y: v.Value, } coordinateData = append(coordinateData, tmpCoordinate2) a, b = utils.GetLinearResult(coordinateData) if math.IsNaN(a) || math.IsNaN(b) { err = errors.New("线性方程公式生成失败") return } } // 生成对应的值 { for i := 1; i < betweenDay; i++ { tmpDataTime := startDataTime.AddDate(0, 0, i) aDecimal := decimal.NewFromFloat(a) xDecimal := decimal.NewFromInt(int64(i) + 1) bDecimal := decimal.NewFromFloat(b) val, _ := aDecimal.Mul(xDecimal).Add(bDecimal).Round(4).Float64() handleDataMap[tmpDataTime.Format(utils.FormatDate)] = val dataTimeList = append(dataTimeList, tmpDataTime.Format(utils.FormatDate)) valueList = append(valueList, val) } } startEdbInfoData = v } return } // HandleDataByLinearRegressionToList 保证生成365个数据点的线性插值法 func HandleDataByLinearRegressionToListV2(edbInfoDataList []*models.EdbDataList, handleDataMap map[string]float64) (dataTimeList []string, valueList []float64, err error) { if len(edbInfoDataList) < 2 { return } // 确保至少有两天数据来生成线性方程 if len(edbInfoDataList) < 2 { err = errors.New("至少需要两天的数据来执行线性插值") return } // 对数据按日期排序，确保顺序正确 sort.Slice(edbInfoDataList, func(i, j int) bool { t1, _ := time.ParseInLocation(utils.FormatDate, edbInfoDataList[i].DataTime, time.Local) t2, _ := time.ParseInLocation(utils.FormatDate, edbInfoDataList[j].DataTime, time.Local) return t1.Before(t2) }) startEdbInfoData := edbInfoDataList[0] endEdbInfoData := edbInfoDataList[len(edbInfoDataList)-1] // 计算起始和结束日期间实际的天数 startDate, _ := time.ParseInLocation(utils.FormatDate, startEdbInfoData.DataTime, time.Local) endDate, _ := time.ParseInLocation(utils.FormatDate, endEdbInfoData.DataTime, time.Local) actualDays := endDate.Sub(startDate).Hours() / 24 // 生成365个数据点，首先处理已有数据 for _, v := range edbInfoDataList { handleDataMap[v.DataTime] = v.Value dataTimeList = append(dataTimeList, v.DataTime) valueList = append(valueList, v.Value) } // 如果已有数据跨越天数不足365天，则对缺失的日期进行线性插值 if actualDays < 365 { // 使用已有数据点生成线性方程（这里简化处理，实际可能需更细致处理边界情况） var a, b float64 coordinateData := []utils.Coordinate{ {X: 1, Y: startEdbInfoData.Value}, {X: float64(len(edbInfoDataList)), Y: endEdbInfoData.Value}, } a, b = utils.GetLinearResult(coordinateData) if math.IsNaN(a) || math.IsNaN(b) { err = errors.New("线性方程公式生成失败") return } // 对剩余日期进行插值 for i := 1; i < 365; i++ { day := startDate.AddDate(0, 0, i) if _, exists := handleDataMap[day.Format(utils.FormatDate)]; !exists { aDecimal := decimal.NewFromFloat(a) xDecimal := decimal.NewFromInt(int64(i) + 1) bDecimal := decimal.NewFromFloat(b) val, _ := aDecimal.Mul(xDecimal).Add(bDecimal).Round(4).Float64() handleDataMap[day.Format(utils.FormatDate)] = val dataTimeList = append(dataTimeList, day.Format(utils.FormatDate)) valueList = append(valueList, val) } } } return } func GetRefreshEdbInfoFromBase(edbInfoId, source int) (baseEdbInfoArr, calculateInfoArr []*models.EdbInfo, err error) { calculateList, err := models.GetEdbInfoCalculateMap(edbInfoId, source) if err != nil && err.Error() != utils.ErrNoRow() { return } for _, item := range calculateList { if item.EdbInfoId == edbInfoId { // 如果查出来关联的指标就是自己的话，那么就过滤 continue } if item.EdbType == 1 { baseEdbInfoArr = append(baseEdbInfoArr, item) } else { calculateInfoArr = append(calculateInfoArr, item) newBaseEdbInfoArr, newCalculateInfoArr, _ := GetRefreshEdbInfoFromBase(item.EdbInfoId, item.Source) baseEdbInfoArr = append(baseEdbInfoArr, newBaseEdbInfoArr...) calculateInfoArr = append(calculateInfoArr, newCalculateInfoArr...) } } return }