package models import ( "encoding/json" "errors" "eta/eta_index_lib/utils" "fmt" "github.com/beego/beego/v2/client/orm" "github.com/dengsgo/math-engine/engine" "github.com/shopspring/decimal" "strconv" "strings" "time" ) // EdbInfoCalculateSaveReq 计算(运算)指标请求参数 type EdbInfoCalculateSaveReq struct { AdminId int `description:"添加人id"` AdminName string `description:"添加人名称"` EdbInfoId int `description:"指标id"` EdbName string `description:"指标名称"` EdbNameEn string `description:"英文指标名称"` Frequency string `description:"频率"` Unit string `description:"单位"` UnitEn string `description:"英文单位"` ClassifyId int `description:"分类id"` CalculateFormula string `description:"计算公式"` EmptyType int `description:"空值处理类型(0查找前后35天,1不计算,2前值填充,3后值填充,4等于0)"` MaxEmptyType int `description:"MAX、MIN公式空值处理类型(1、等于0;2、跳过空值)"` Extra string `description:"指标额外配置"` EdbInfoIdArr []EdbInfoFromTag } // EdbInfoFromTag 计算指标的关联指标 type EdbInfoFromTag struct { EdbInfoId int `description:"指标id"` FromTag string `description:"指标对应标签"` } // CalculateItems 计算(运算)指标信息 type CalculateItems struct { EdbInfoId int DataMap map[string]float64 } // AddCalculateInfo 新增计算(运算)指标 func AddCalculateInfo(req EdbInfoCalculateSaveReq, calculateMappingList []*EdbInfoCalculateMapping, edbInfoList []*EdbInfo, edbCode, uniqueCode string, edbInfoIdBytes []string) (edbInfo *EdbInfo, err error) { o := orm.NewOrm() to, err := o.Begin() if err != nil { return } defer func() { if err != nil { fmt.Println("AddCalculateHcz,Err:" + err.Error()) _ = to.Rollback() } else { _ = to.Commit() } }() edbInfo = &EdbInfo{ EdbInfoId: 0, SourceName: "指标运算", Source: utils.DATA_SOURCE_CALCULATE, EdbCode: edbCode, EdbName: req.EdbName, EdbNameSource: req.EdbName, Frequency: req.Frequency, Unit: req.Unit, StartDate: "", EndDate: "", ClassifyId: req.ClassifyId, SysUserId: req.AdminId, SysUserRealName: req.AdminName, UniqueCode: uniqueCode, CreateTime: time.Now(), ModifyTime: time.Now(), MinValue: 0, MaxValue: 0, CalculateFormula: req.CalculateFormula, EdbType: 2, Sort: GetAddEdbMaxSortByClassifyId(req.ClassifyId, utils.EDB_INFO_TYPE), MoveType: 0, MoveFrequency: "", NoUpdate: 0, ServerUrl: "", EdbInfoType: 0, EdbNameEn: req.EdbName, UnitEn: req.Unit, LatestDate: "", LatestValue: 0, ChartImage: "", EmptyType: req.EmptyType, MaxEmptyType: req.MaxEmptyType, Extra: req.Extra, } lastId, err := to.Insert(edbInfo) if err != nil { return } edbInfo.EdbInfoId = int(lastId) edbInfoTag := make(map[string]int) //关联关系表 { //处理关联指标 for _, v := range calculateMappingList { edbInfoTag[v.FromTag] = v.FromEdbInfoId v.EdbCode = edbCode v.EdbInfoId = edbInfo.EdbInfoId } if len(calculateMappingList) > 0 { _, err = to.InsertMulti(len(calculateMappingList), calculateMappingList) if err != nil { return } } } //计算数据 err = refreshAllCalculate(to, edbInfoList, edbInfoTag, edbInfo.EdbInfoId, edbInfo.Source, edbInfo.SubSource, edbInfo.EdbCode, edbInfo.CalculateFormula, "", "", edbInfoIdBytes, edbInfo.EmptyType, edbInfo.MaxEmptyType, edbInfo.Extra) return } // EditCalculateInfo 编辑计算(运算)指标 func EditCalculateInfo(edbInfo *EdbInfo, req EdbInfoCalculateSaveReq, formulaSlice []string, edbInfoIdBytes []string, needCalculate bool) (err error, errMsg string) { o := orm.NewOrm() to, err := o.Begin() if err != nil { return } defer func() { if err != nil { fmt.Println("AddCalculateHcz,Err:" + err.Error()) _ = to.Rollback() } else { _ = to.Commit() } }() // 修改指标信息 edbInfo.EdbName = req.EdbName edbInfo.EdbNameEn = req.EdbNameEn edbInfo.EdbNameSource = req.EdbName edbInfo.Frequency = req.Frequency edbInfo.Unit = req.Unit edbInfo.UnitEn = req.UnitEn edbInfo.ClassifyId = req.ClassifyId edbInfo.CalculateFormula = req.CalculateFormula edbInfo.ModifyTime = time.Now() edbInfo.EmptyType = req.EmptyType edbInfo.MaxEmptyType = req.MaxEmptyType edbInfo.Extra = req.Extra _, err = to.Update(edbInfo, "EdbName", "EdbNameSource", "Frequency", "Unit", "EdbNameEn", "UnitEn", "ClassifyId", "CalculateFormula", "ModifyTime", "EmptyType", "MaxEmptyType", "Extra") if err != nil { return } if needCalculate { // 清除计算数据 sql := `DELETE FROM edb_data_calculate WHERE edb_info_id=?` _, err = to.Raw(sql, edbInfo.EdbInfoId).Exec() if err != nil { return } // 清除关联关系表 sql = `DELETE FROM edb_info_calculate_mapping WHERE edb_info_id=?` _, err = to.Raw(sql, edbInfo.EdbInfoId).Exec() if err != nil { return } edbInfoList := make([]*EdbInfo, 0) calculateMappingList := make([]*EdbInfoCalculateMapping, 0) edbInfoTag := make(map[string]int) for k, v := range req.EdbInfoIdArr { edbInfoTag[v.FromTag] = v.EdbInfoId fromEdbInfo, tmpErr := GetEdbInfoById(v.EdbInfoId) if tmpErr != nil { err = tmpErr return } edbInfoList = append(edbInfoList, fromEdbInfo) //关联关系表 { calculateMappingItem := &EdbInfoCalculateMapping{ EdbInfoCalculateMappingId: 0, EdbInfoId: req.EdbInfoId, Source: edbInfo.Source, SourceName: edbInfo.SourceName, EdbCode: edbInfo.EdbCode, FromEdbInfoId: fromEdbInfo.EdbInfoId, FromEdbCode: fromEdbInfo.EdbCode, FromEdbName: fromEdbInfo.EdbName, FromSource: fromEdbInfo.Source, FromSourceName: fromEdbInfo.SourceName, FromTag: v.FromTag, Sort: k + 1, CreateTime: time.Now(), ModifyTime: time.Now(), } calculateMappingList = append(calculateMappingList, calculateMappingItem) } } for _, v := range formulaSlice { formulaMap, e := utils.CheckFormula(v) if e != nil { err = fmt.Errorf("公式错误,请重新填写") return } //预先计算,判断公式是否正常 ok, _ := CheckFormula2(edbInfoList, formulaMap, v, edbInfoIdBytes) if !ok { errMsg = "生成计算指标失败,请使用正确的计算公式" err = errors.New(errMsg) return } } //关联关系表 { if len(calculateMappingList) > 0 { _, err = to.InsertMulti(len(calculateMappingList), calculateMappingList) if err != nil { return } } } //计算数据 err = refreshAllCalculate(to, edbInfoList, edbInfoTag, edbInfo.EdbInfoId, edbInfo.Source, edbInfo.SubSource, edbInfo.EdbCode, edbInfo.CalculateFormula, "", "", edbInfoIdBytes, edbInfo.EmptyType, edbInfo.MaxEmptyType, edbInfo.Extra) } return } // GetCalculateEdbInfo 获取计算指标的关联指标id func GetCalculateEdbInfo(edbInfoId int) (from_edb_info_id string, err error) { o := orm.NewOrm() sql := ` SELECT GROUP_CONCAT(from_edb_info_id ORDER BY sort ASC SEPARATOR ',') AS from_edb_info_id FROM edb_info_calculate_mapping WHERE edb_info_id=? GROUP BY edb_info_id ` err = o.Raw(sql, edbInfoId).QueryRow(&from_edb_info_id) return } // DeleteCalculateEdbInfo 删除指标信息 func DeleteCalculateEdbInfo(edbInfoId int) (err error) { o := orm.NewOrm() to, err := o.Begin() if err != nil { return } defer func() { if err != nil { _ = to.Rollback() } else { _ = to.Commit() } }() sql := `DELETE FROM edb_data_calculate WHERE edb_info_id=?` _, err = to.Raw(sql, edbInfoId).Exec() if err != nil { return } sql = `DELETE FROM edb_info_calculate_mapping WHERE edb_info_id=?` _, err = to.Raw(sql, edbInfoId).Exec() return } // RefreshAllCalculate 刷新全部数据 func RefreshAllCalculate(edbInfoIdArr []*EdbInfo, edbInfoTag map[string]int, edbInfoId, source, subSource int, edbCode, formulaStr, startDate, endDate string, edbInfoIdBytes []string, emptyType, maxEmptyType int, extra string) (err error) { o := orm.NewOrm() to, err := o.Begin() if err != nil { return } defer func() { if err != nil { fmt.Println("RefreshAllCalculate,Err:" + err.Error()) _ = to.Rollback() } else { _ = to.Commit() } }() fmt.Println(startDate, endDate) //计算数据 err = refreshAllCalculate(to, edbInfoIdArr, edbInfoTag, edbInfoId, source, subSource, edbCode, formulaStr, startDate, endDate, edbInfoIdBytes, emptyType, maxEmptyType, extra) return } // refreshAllCalculate 刷新全部数据 func refreshAllCalculate(to orm.TxOrmer, edbInfoIdArr []*EdbInfo, edbInfoTag map[string]int, edbInfoId, source, subSource int, edbCode, formulaStr, startDate, endDate string, edbInfoIdBytes []string, emptyType, maxEmptyType int, extra string) (err error) { realSaveDataMap := make(map[string]map[int]float64) saveDataMap := make(map[string]map[int]float64) // 最小的结束日期 , 最晚的数据开始日期 var minLatestDate, maxStartDate time.Time dateList := make([]string, 0) // 最终的日期数据 dateMap := make(map[string]struct{}) // 最终的日期数据 dateTagConfig := "" if extra != "" { var dateConfig CalculateEdbExtra err = json.Unmarshal([]byte(extra), &dateConfig) if err != nil { err = fmt.Errorf("refreshAllCalculate,extra解析失败,Err:%s", err.Error()) return } dateTagConfig = dateConfig.DateTag } for edbInfoIndex, v := range edbInfoIdArr { dataList, err := GetEdbDataListAllByTo(to, v.Source, v.SubSource, FindEdbDataListAllCond{ EdbInfoId: v.EdbInfoId, StartDataTime: startDate, StartDataTimeCond: ">=", }, 1) if err != nil { return err } for _, dv := range dataList { if val, ok := realSaveDataMap[dv.DataTime]; ok { if _, ok := val[v.EdbInfoId]; !ok { val[v.EdbInfoId] = dv.Value } } else { temp := make(map[int]float64) temp[v.EdbInfoId] = dv.Value realSaveDataMap[dv.DataTime] = temp } // saveDataMap if val, ok := saveDataMap[dv.DataTime]; ok { if _, ok := val[v.EdbInfoId]; !ok { val[v.EdbInfoId] = dv.Value } } else { temp2 := make(map[int]float64) temp2[v.EdbInfoId] = dv.Value saveDataMap[dv.DataTime] = temp2 } if dateTagConfig == "all" { if _, ok := dateMap[dv.DataTime]; !ok { dateList = append(dateList, dv.DataTime) dateMap[dv.DataTime] = struct{}{} } } else if dateTagConfig == "" { // 默认取第一个指标的时间序列 if edbInfoIndex == 0 { if _, ok := dateMap[dv.DataTime]; !ok { dateList = append(dateList, dv.DataTime) dateMap[dv.DataTime] = struct{}{} } } } else { if eId, ok := edbInfoTag[dateTagConfig]; ok { if v.EdbInfoId == eId { if _, ok1 := dateMap[dv.DataTime]; !ok1 { dateList = append(dateList, dv.DataTime) dateMap[dv.DataTime] = struct{}{} } } } } } } // 处理最大日期和最小日期 for _, v := range dateList { tmpDate, _ := time.ParseInLocation(utils.FormatDate, v, time.Local) if minLatestDate.IsZero() || tmpDate.After(minLatestDate) { minLatestDate = tmpDate } if maxStartDate.IsZero() || tmpDate.Before(maxStartDate) { maxStartDate = tmpDate } } //数据处理,将日期内不全的数据做补全 HandleDateSaveDataMap(dateList, maxStartDate, minLatestDate, realSaveDataMap, saveDataMap, edbInfoIdArr, emptyType) formulaDateSlice, formulaDateMap, err := utils.HandleFormulaJson(formulaStr, minLatestDate) if err != nil { return } addSql := ` INSERT INTO edb_data_calculate(edb_info_id,edb_code,data_time,value,create_time,modify_time,data_timestamp) values ` var isAdd bool //获取指标所有数据 dataList := make([]*EdbData, 0) dataTableName := GetEdbDataTableName(source, subSource) sql := `SELECT * FROM %s WHERE edb_info_id=? ` sql = fmt.Sprintf(sql, dataTableName) _, err = to.Raw(sql, edbInfoId).QueryRows(&dataList) if err != nil { return err } dataMap := make(map[string]string) removeDateMap := make(map[string]string) //需要移除的日期 for _, v := range dataList { dataMap[v.DataTime] = v.Value removeDateMap[v.DataTime] = v.DataTime } edbInfoIdStr := strconv.Itoa(edbInfoId) existDataMap := make(map[string]string) // 判断是否特殊处理max和min函数 maxDealFlag := false if emptyType == 4 && maxEmptyType == 2 { maxDealFlag = true } for sk, sv := range saveDataMap { // 当空值处理类型选择了不计算时,只要有一个指标在某个日期没有值(即空值),则计算指标在该日期没有值 if emptyType == 1 { if len(sv) != len(edbInfoIdArr) { continue } } //fmt.Println(sk, sv) // 根据时间范围,选择对应的公式 formulaMap := make(map[string]string) formulaStr = "" for _, fv := range formulaDateSlice { if sk < fv { if f, ok := formulaDateMap[fv]; ok { formulaStr = f formulaMap, err = utils.CheckFormula(formulaStr) if err != nil { err = fmt.Errorf("公式错误,请重新填写") return } } break } } if formulaStr == "" { continue } svMax := make(map[int]float64) if maxDealFlag { // 特殊处理max和min函数,如果原本的值为空,则选择空值参与运算 if svMaxData, ok := realSaveDataMap[sk]; ok { svMax = svMaxData } } formulaStr = strings.ToUpper(formulaStr) formulaFormStr := ReplaceFormula(edbInfoIdArr, sv, svMax, formulaMap, formulaStr, edbInfoIdBytes, maxDealFlag) //计算公式异常,那么就移除该指标 if formulaFormStr == `` { //removeDateList = append(removeDateList, sk) continue } //utils.FileLog.Info(fmt.Sprintf("formulaFormStr:%s", formulaFormStr)) fmt.Println(fmt.Sprintf("%s:formulaFormStr:%s", sk, formulaFormStr)) //expression := formula.NewExpression(formulaFormStr) //calResult, err := expression.Evaluate() //if err != nil { // // 分母为0的报错 // if strings.Contains(err.Error(), "divide by zero") { // //removeDateList = append(removeDateList, sk) // continue // } // err = errors.New("计算失败:Err:" + err.Error() + ";formulaStr:" + formulaFormStr) // fmt.Println(err) // return err //} //// 如果计算结果是NAN,那么就退出当前循环 //if calResult.IsNan() { // continue //} calVal, err := engine.ParseAndExec(formulaFormStr) //calVal, err := calResult.Float64() if err != nil { // 分母为0的报错,忽略该循环 if utils.IsDivideZero(err) { //removeDateList = append(removeDateList, sk) continue } err = errors.New("计算失败:获取计算值失败 Err:" + err.Error() + ";formulaStr:" + formulaFormStr) fmt.Println(err) return err } nanCheck := fmt.Sprintf("%0.f", calVal) // 分母为0.0的报错 if nanCheck == "NaN" || nanCheck == "+Inf" || nanCheck == "-Inf" { continue } // 有计算出来值,那么就从待删除指标中移除 delete(removeDateMap, sk) saveValue := decimal.NewFromFloat(calVal).Round(4).String() //utils.SubFloatToString(calVal, 4) if existVal, ok := dataMap[sk]; !ok { dataTime, _ := time.ParseInLocation(utils.FormatDate, sk, time.Local) timestamp := dataTime.UnixNano() / 1e6 timeStr := fmt.Sprintf("%d", timestamp) if _, existOk := existDataMap[sk]; !existOk { addSql += GetAddSql(edbInfoIdStr, edbCode, sk, timeStr, saveValue) isAdd = true } existDataMap[sk] = sk } else { existValDecimal, err := decimal.NewFromString(existVal) existStr := existValDecimal.String() if existStr != saveValue { sql := ` UPDATE %s SET value=?,modify_time=NOW() WHERE edb_info_id=? AND data_time=? ` sql = fmt.Sprintf(sql, dataTableName) _, err = to.Raw(sql, saveValue, edbInfoId, sk).Exec() if err != nil { return err } } } } if isAdd { addSql = strings.TrimRight(addSql, ",") _, err = to.Raw(addSql).Exec() if err != nil { fmt.Println("RefreshAllCalculate add Err", err.Error()) return } } if len(removeDateMap) > 0 { removeDateList := make([]string, 0) //需要移除的日期 for k := range removeDateMap { removeDateList = append(removeDateList, k) } removeDateStr := strings.Join(removeDateList, `","`) removeDateStr = `"` + removeDateStr + `"` //如果拼接指标变更了,那么需要删除所有的指标数据 tableName := GetEdbDataTableName(source, subSource) sql := fmt.Sprintf(` DELETE FROM %s WHERE edb_info_id = ? and data_time in (%s) `, tableName, removeDateStr) _, err = to.Raw(sql, edbInfoId).Exec() if err != nil { err = fmt.Errorf("删除计算失败的计算指标数据失败,Err:" + err.Error()) return } } return } // ReplaceFormula 替换计算方式 func ReplaceFormula(edbInfoIdArr []*EdbInfo, valArr, valArrMax map[int]float64, formulaMap map[string]string, formulaStr string, edbInfoIdBytes []string, maxDealFlag bool) string { // todo 先处理max和min函数的特殊情况 //if strings.Contains(formulaStr, "MAX") || strings.Contains(formulaStr, "MIN") { funMap := GetFormulaMap() for k, v := range funMap { formulaStr = strings.Replace(formulaStr, k, v, -1) } if maxDealFlag { formulaStr = GetMaxMinEdbInfo(formulaStr) } replaceCount := 0 for dk := len(edbInfoIdArr) - 1; dk >= 0; dk-- { dv := edbInfoIdArr[dk] //for dk, dv := range edbInfoIdArr { var isReplace bool formulaStr, isReplace = GetFormulaReplace(dk, dv.EdbInfoId, formulaStr, edbInfoIdBytes, formulaMap, valArr, valArrMax, maxDealFlag) if isReplace { replaceCount++ } } for k, v := range funMap { formulaStr = strings.Replace(formulaStr, v, k, -1) } if strings.Contains(formulaStr, "MAX()") || strings.Contains(formulaStr, "MIN()") { return "" } formulaStr = strings.Replace(formulaStr, "MAX", "max", -1) formulaStr = strings.Replace(formulaStr, "MIN", "min", -1) formulaStr = strings.Replace(formulaStr, "LN", "ln", -1) formulaStr = strings.Replace(formulaStr, "LOG", "log", -1) formulaStr = strings.Replace(formulaStr, "SIN", "sin", -1) formulaStr = strings.Replace(formulaStr, "COS", "cos", -1) formulaStr = strings.Replace(formulaStr, "TAN", "tan", -1) formulaStr = strings.Replace(formulaStr, "COT", "cot", -1) formulaStr = strings.Replace(formulaStr, "SEC", "sec", -1) formulaStr = strings.Replace(formulaStr, "CSC", "csc", -1) formulaStr = strings.Replace(formulaStr, "ABS", "abs", -1) formulaStr = strings.Replace(formulaStr, "CEIL", "ceil", -1) formulaStr = strings.Replace(formulaStr, "FLOOR", "floor", -1) formulaStr = strings.Replace(formulaStr, "ROUND", "round", -1) formulaStr = strings.Replace(formulaStr, "SQRT", "sqrt", -1) formulaStr = strings.Replace(formulaStr, "CBRT", "cbrt", -1) formulaStr = strings.Replace(formulaStr, "NOERR", "noerr", -1) formulaStr = strings.Replace(formulaStr, "DOUBLE", "double", -1) if replaceCount == len(formulaMap) { return formulaStr } else { return "" } } func GetFormulaReplace(index, edbInfoId int, formulaStr string, edbInfoIdBytes []string, formulaMap map[string]string, valArr, valArrMax map[int]float64, maxDealFlag bool) (formulaResult string, isReplace bool) { formulaResult = formulaStr dKey := edbInfoIdBytes[index] if _, ok := formulaMap[dKey]; ok { //公式中存在 if val, valOk := valArr[edbInfoId]; valOk { //值存在 dvStr := fmt.Sprintf("%v", val) formulaResult = strings.Replace(formulaResult, dKey, dvStr, -1) isReplace = true if maxDealFlag { if valM, valMOk := valArrMax[edbInfoId]; valMOk { //值存在 dvMStr := fmt.Sprintf("%v", valM) formulaResult = strings.Replace(formulaResult, strings.ToLower(dKey), dvMStr, -1) } else { formulaResult = strings.Replace(formulaResult, strings.ToLower(dKey)+",", "", -1) formulaResult = strings.Replace(formulaResult, ","+strings.ToLower(dKey), "", -1) } } } } return } // GetFormulaMap 获取计算公式的map func GetFormulaMap() map[string]string { funMap := make(map[string]string) funMap["MAX"] = "[@@]" funMap["MIN"] = "[@!]" funMap["ABS"] = "[@#]" funMap["CEIL"] = "[@$]" funMap["COS"] = "[@%]" funMap["FLOOR"] = "[@^]" funMap["MOD"] = "[@&]" funMap["POW"] = "[@*]" funMap["ROUND"] = "[@`]" funMap["LN"] = "[@-]" funMap["EXP"] = "[@+]" return funMap } // CalculateBatchSaveReq 添加计算指标的请求参数 type CalculateBatchSaveReq struct { EdbList []*CalculateEdbInfoItem //需要批量计算的指标列表 AdminId int `description:"添加人id"` AdminName string `description:"添加人名称"` Formula string `description:"N值/移动天数"` Source int `description:"来源:1:同花顺,2:wind,3:彭博,4:指标运算,5:累计值转月,6:同比值,7:同差值,8:N数值移动平均计算,12:环比值,13:环差值,14:升频"` CalculateFormula string `description:"计算公式"` EdbInfoIdArr []EdbInfoCalculateEdbInfoIdReq `description:"关联指标列表"` MoveType int `description:"移动方式:1:领先(默认),2:滞后"` MoveFrequency string `description:"移动频度:天/周/月/季/年"` Calendar string `description:"公历/农历"` Data interface{} `description:"数据"` EmptyType int `description:"空值处理类型(0查找前后35天,1不计算,2前值填充,3后值填充,4等于0)"` MaxEmptyType int `description:"MAX、MIN公式空值处理类型(1、等于0;2、跳过空值)"` Extra string `description:"指标额外配置"` } type CalculateEdbInfoItem struct { EdbInfoId int `description:"指标id"` EdbName string `description:"指标名称"` Frequency string `description:"频度"` Unit string `description:"单位"` ClassifyId int `description:"分类id"` CalculateId string `description:"当前请求时,单个计算的唯一标识"` FromEdbInfoId int `description:"计算来源指标id"` } // EdbInfoCalculateBatchSaveReq 添加计算指标的请求参数 type EdbInfoCalculateBatchSaveReq struct { AdminId int `description:"添加人id"` AdminName string `description:"添加人名称"` EdbInfoId int `description:"指标id"` EdbName string `description:"指标名称"` Frequency string `description:"频度"` Unit string `description:"单位"` ClassifyId int `description:"分类id"` Formula string `description:"N值/移动天数"` FromEdbInfoId int `description:"计算来源指标id"` Source int `description:"来源:1:同花顺,2:wind,3:彭博,4:指标运算,5:累计值转月,6:同比值,7:同差值,8:N数值移动平均计算,12:环比值,13:环差值,14:升频"` CalculateFormula string `description:"计算公式"` EdbInfoIdArr []EdbInfoCalculateEdbInfoIdReq `description:"关联指标列表"` MoveType int `description:"移动方式:1:领先(默认),2:滞后"` MoveFrequency string `description:"移动频度:天/周/月/季/年/交易日/自然日"` Calendar string `description:"公历/农历"` Data interface{} `description:"数据"` EmptyType int `description:"空值处理类型(0查找前后35天,1不计算,2前值填充,3后值填充,4等于0)"` MaxEmptyType int `description:"MAX、MIN公式空值处理类型(1、等于0;2、跳过空值)"` Extra string `description:"指标额外配置"` } // EdbInfoCalculateEdbInfoIdReq 新增/编辑请求 关联的指标列表 type EdbInfoCalculateEdbInfoIdReq struct { EdbInfoId int `description:"指标id"` FromTag string `description:"指标对应标签"` MoveValue int `description:"移动的值"` } // EdbInfoCalculateBatchEditReq 编辑计算指标的请求参数 type EdbInfoCalculateBatchEditReq struct { EdbName string `description:"指标名称"` EdbNameEn string `description:"英文指标名称"` Frequency string `description:"频度"` Unit string `description:"单位"` UnitEn string `description:"英文单位"` ClassifyId int `description:"分类id"` AdminId int `description:"操作人id"` AdminName string `description:"操作人姓名"` Formula string `description:"N值"` EdbInfoId int `description:"编辑指标id"` FromEdbInfoId int `description:"计算来源指标id"` Source int `description:"来源:1:同花顺,2:wind,3:彭博,4:指标运算,5:累计值转月,6:同比值,7:同差值,8:N数值移动平均计算,12:环比值,13:环差值,14:升频"` MoveType int `description:"移动方式:1:领先(默认),2:滞后"` MoveFrequency string `description:"移动频度:天/周/月/季/年"` EdbInfoIdArr []EdbInfoCalculateEdbInfoIdReq `description:"关联指标列表"` Calendar string `description:"公历/农历" orm:"default(公历)"` Extra string `description:"指标的额外配置"` Data interface{} `description:"数据"` EmptyType int `description:"空值处理类型(0查找前后35天,1不计算,2前值填充,3后值填充,4等于0)"` MaxEmptyType int `description:"MAX、MIN公式空值处理类型(1、等于0;2、跳过空值)"` CalculateFormula string `description:"计算公式"` } // DeleteCalculateData 删除计算数据 func DeleteCalculateData(edbInfoId int) (err error) { o := orm.NewOrm() sql := `DELETE FROM edb_data_calculate WHERE edb_info_id=?` _, err = o.Raw(sql, edbInfoId).Exec() return } // CheckFormula2 校验公式是否正常(比如说除法的分母不能为0之类的,实际上就是用预设的字段数据做一次计算) func CheckFormula2(edbInfoArr []*EdbInfo, formulaMap map[string]string, formulaStr string, edbInfoIdBytes []string) (ok bool, err error) { valArr := make(map[int]float64) for k, v := range edbInfoArr { valArr[v.EdbInfoId] = 100 + float64(k) } formulaStr = strings.ToUpper(formulaStr) // 预设里的max和min无需特殊处理 valArrMax := make(map[int]float64) formulaFormStr := ReplaceFormula(edbInfoArr, valArr, valArrMax, formulaMap, formulaStr, edbInfoIdBytes, false) if formulaFormStr == "" { return } _, err = engine.ParseAndExec(formulaFormStr) if err != nil { fmt.Println(err) } else { ok = true } //expression := formula.NewExpression(formulaFormStr) //_, err = expression.Evaluate() //if err != nil { //} else { // ok = true //} return } // 处理整个数据 func HandleDateSaveDataMap(dateList []string, maxStartDate, minLatestDate time.Time, realSaveDataMap, saveDataMap map[string]map[int]float64, edbInfoIdArr []*EdbInfo, emptyType int) { for _, date := range dateList { dateTime, _ := time.ParseInLocation(utils.FormatDate, date, time.Local) // 如果当前日期早于数据的最大开始日期,那么不处理,进入下一个循环 if dateTime.Before(maxStartDate) { continue } // todo 规则是否生效如果当前日期晚于数据的最小结束日期,那么不处理,直接返回了 if dateTime.After(minLatestDate) { return } tmpDataMap := realSaveDataMap[date] for _, edbInfo := range edbInfoIdArr { tmpEdbInfoId := edbInfo.EdbInfoId // 当前指标id // 如果该日期不存在该指标数据,那么需要找寻前后日期的数据,进行填补 if _, ok := tmpDataMap[tmpEdbInfoId]; !ok { //day := 0 //switch edbInfo.Frequency { //case "周度": // day = 7 //case "旬度": // day = 15 //case "月度": // day = 30 //case "季度": // day = 90 //case "年度": // day = 365 //} // 需求池 255 指标运算文案修改,补数据遍历区间修改(2023-3-7 09:37:23修改) switch emptyType { case 0: handleDateDataMap(realSaveDataMap, saveDataMap, date, tmpEdbInfoId, 35) case 2: handleDateDataMapBefore(realSaveDataMap, saveDataMap, date, tmpEdbInfoId, edbInfo.StartDate, edbInfo.EndDate) case 3: handleDateDataMapAfter(realSaveDataMap, saveDataMap, date, tmpEdbInfoId, edbInfo.StartDate, edbInfo.EndDate) case 4: handleDateDataMapZero(saveDataMap, date, tmpEdbInfoId) } } } } } // handleDateDataMap 前后35天 处理单个日期的数据 func handleDateDataMap(realSaveDataMap, saveDataMap map[string]map[int]float64, date string, edbInfoId, day int) { currDate, _ := time.ParseInLocation(utils.FormatDate, date, time.Local) // 后一天 nextDateDayStr := currDate.AddDate(0, 0, 1).Format(utils.FormatDate) // 前一天 preDateDayStr := currDate.AddDate(0, 0, -1).Format(utils.FormatDate) for i := 1; i <= day; i++ { // 下个日期的数据 { if i >= 1 { nextDateDayStr = currDate.AddDate(0, 0, i).Format(utils.FormatDate) } if findDataMap, hasFindDataMap := realSaveDataMap[nextDateDayStr]; hasFindDataMap { // 下一个日期有数据 if val, hasFindItem := findDataMap[edbInfoId]; hasFindItem { fmt.Println(fmt.Sprintf("date:%s, 无值,取%s的值%.4f", date, nextDateDayStr, val)) saveDataMap[date][edbInfoId] = val return } } } // 上个日期的数据 { if i >= 1 { preDateDayStr = currDate.AddDate(0, 0, -i).Format(utils.FormatDate) } if findDataMap, hasFindDataMap := realSaveDataMap[preDateDayStr]; hasFindDataMap { // 下一个日期有数据 if val, hasFindItem := findDataMap[edbInfoId]; hasFindItem { fmt.Println(fmt.Sprintf("date:%s, 无值,取%s的值%.4f", date, preDateDayStr, val)) saveDataMap[date][edbInfoId] = val return } } } } } // handleDateDataMapBefore 前值填充:空值优先以最近的前值填充,没有前值时,用后值填充 func handleDateDataMapBefore(realSaveDataMap, saveDataMap map[string]map[int]float64, date string, edbInfoId int, startDate, endDate string) { currDate, _ := time.ParseInLocation(utils.FormatDate, date, time.Local) // 前一天 preDateDay := currDate startDateT, _ := time.ParseInLocation(utils.FormatDate, startDate, time.Local) endDateT, _ := time.ParseInLocation(utils.FormatDate, endDate, time.Local) if currDate.Before(startDateT) { if findDataMap, hasFindDataMap := realSaveDataMap[startDate]; hasFindDataMap { // 下一个日期有数据 if val, hasFindItem := findDataMap[edbInfoId]; hasFindItem { //utils.FileLog.Info(fmt.Sprintf("date:%s, 无值,取%s的值%.4f", date, startDate, val)) saveDataMap[date][edbInfoId] = val return } } return } if currDate.After(endDateT) { if findDataMap, hasFindDataMap := realSaveDataMap[endDate]; hasFindDataMap { // 下一个日期有数据 if val, hasFindItem := findDataMap[edbInfoId]; hasFindItem { //utils.FileLog.Info(fmt.Sprintf("date:%s, 无值,取%s的值%.4f", date, endDate, val)) saveDataMap[date][edbInfoId] = val return } } } for i := 1; preDateDay.After(startDateT) || preDateDay == startDateT; i++ { // 上个日期的数据 { preDateDay = currDate.AddDate(0, 0, -i) preDateDayStr := preDateDay.Format(utils.FormatDate) if findDataMap, hasFindDataMap := realSaveDataMap[preDateDayStr]; hasFindDataMap { // 下一个日期有数据 if val, hasFindItem := findDataMap[edbInfoId]; hasFindItem { //utils.FileLog.Info(fmt.Sprintf("date:%s, 无值,取%s的值%.4f", date, preDateDayStr, val)) saveDataMap[date][edbInfoId] = val return } } } } return } // handleDateDataMapAfter 后值填充:空值优先以最近的后值填充,没有后值时,用前值填充 func handleDateDataMapAfter(realSaveDataMap, saveDataMap map[string]map[int]float64, date string, edbInfoId int, startDate, endDate string) { currDate, _ := time.ParseInLocation(utils.FormatDate, date, time.Local) // 后一天 nextDateDay := currDate startDateT, _ := time.ParseInLocation(utils.FormatDate, startDate, time.Local) endDateT, _ := time.ParseInLocation(utils.FormatDate, endDate, time.Local) if currDate.Before(startDateT) { if findDataMap, hasFindDataMap := realSaveDataMap[startDate]; hasFindDataMap { // 下一个日期有数据 if val, hasFindItem := findDataMap[edbInfoId]; hasFindItem { //utils.FileLog.Info(fmt.Sprintf("date:%s, 无值,取%s的值%.4f", date, startDate, val)) saveDataMap[date][edbInfoId] = val return } } return } if currDate.After(endDateT) { if findDataMap, hasFindDataMap := realSaveDataMap[endDate]; hasFindDataMap { // 下一个日期有数据 if val, hasFindItem := findDataMap[edbInfoId]; hasFindItem { //utils.FileLog.Info(fmt.Sprintf("date:%s, 无值,取%s的值%.4f", date, endDate, val)) saveDataMap[date][edbInfoId] = val return } } } for i := 1; nextDateDay.Before(endDateT) || nextDateDay == endDateT; i++ { // 下个日期的数据 { nextDateDay = currDate.AddDate(0, 0, i) nextDateDayStr := nextDateDay.Format(utils.FormatDate) if findDataMap, hasFindDataMap := realSaveDataMap[nextDateDayStr]; hasFindDataMap { // 下一个日期有数据 if val, hasFindItem := findDataMap[edbInfoId]; hasFindItem { //utils.FileLog.Info(fmt.Sprintf("date:%s, 无值,取%s的值%.4f", date, nextDateDayStr, val)) saveDataMap[date][edbInfoId] = val return } } } } return } // handleDateDataMapZero 等于0 func handleDateDataMapZero(saveDataMap map[string]map[int]float64, date string, edbInfoId int) { saveDataMap[date][edbInfoId] = 0 return } func GetMaxMinEdbInfo(formula string) string { flag := 0 for k, v := range formula { if v >= 65 && v <= 122 { // 从左边找到第一个左括号的位置,往前截取三个字符,判断是不是max或者min,如果是的话,把该入参改成小写 for i := k; i >= 4; i-- { if formula[i:i+1] == "(" && flag >= 1 { t := formula[i-4 : i] if t == "[@@]" || t == "[@!]" { formula = formula[:k] + strings.ToLower(formula[k:k+1]) + formula[k+1:] } break } } } else if formula[k:k+1] == "(" { flag += 1 } else if formula[k:k+1] == ")" { flag -= 1 } } return formula } // PredictRuleCalculateByNineResp 获取预测指标规则9的绘图数据返回 type PredictRuleCalculateByNineResp struct { LatestDate string DataList interface{} } // BatchEdbInfoCalculateBatchSaveResp 批量添加 计算指标 返回数据 type BatchEdbInfoCalculateBatchSaveResp struct { Fail []BatchEdbInfoCalculateBatchSaveFailResp `description:"添加失败的指标"` Success []BatchEdbInfoCalculateBatchSaveSuccessResp `description:"添加成功的指标"` } // BatchEdbInfoCalculateBatchSaveFailResp 添加失败的指标信息 type BatchEdbInfoCalculateBatchSaveFailResp struct { CalculateId string `description:"当前请求时,单个计算的唯一标识"` Msg string `description:"用户提示信息"` ErrMsg string `description:"错误信息,内部查看"` } // BatchEdbInfoCalculateBatchSaveSuccessResp 添加成功的指标信息 type BatchEdbInfoCalculateBatchSaveSuccessResp struct { ClassifyId int `description:"分类id"` CalculateId string `description:"当前请求时,单个计算的唯一标识"` EdbInfoId int `description:"指标ID"` UniqueCode string `description:"指标唯一编码"` }