package data import ( "errors" "eta/eta_api/models" "eta/eta_api/models/data_manage" "eta/eta_api/models/mgo" "eta/eta_api/utils" "fmt" "github.com/shopspring/decimal" "go.mongodb.org/mongo-driver/bson" "time" ) // EdbDataEdit 指标数据修改 type EdbDataEdit struct { OldCreateDate string `description:"旧的录入日期"` CreateDate string `description:"新的录入日期"` TradeCode string `description:"指标唯一编码"` Close string `description:"录入值"` } // BatchEdbData 批量修改指标数据 func BatchEdbData(oldCreateDate, createDate string, list []EdbDataEdit, opUserId int) (failEdbDataList []EdbDataEdit, err error) { if oldCreateDate == createDate { //修改 err = errors.New("开始日期与结束日期一致!") return } for _, newEdbData := range list { edbdata := new(models.Edbdata) edbdata.TradeCode = newEdbData.TradeCode edbdata.Dt = newEdbData.CreateDate edbdata.Close = newEdbData.Close edbdata.ModifyTime = time.Now() item, err := models.GetDataInfo(newEdbData.TradeCode, createDate) if err != nil && err.Error() != utils.ErrNoRow() { failEdbDataList = append(failEdbDataList, newEdbData) continue } if item == nil { _, err = models.AddEdbdata(edbdata) if err != nil { failEdbDataList = append(failEdbDataList, newEdbData) continue } } else { err = models.EditEdbdata(edbdata) if err != nil { failEdbDataList = append(failEdbDataList, newEdbData) continue } } oldItem, err := models.GetDataInfo(newEdbData.TradeCode, oldCreateDate) if err != nil && err.Error() != utils.ErrNoRow() { failEdbDataList = append(failEdbDataList, newEdbData) continue } if oldItem != nil { record := new(models.EdbdataDeleteRecord) record.TradeCode = oldItem.TradeCode record.Dt = oldItem.Dt record.Close = oldItem.Close record.ModifyTime = oldItem.ModifyTime record.CreateTime = time.Now() record.SysUserId = opUserId _, err = models.AddEdbdataDeleteRecord(record) if err != nil { failEdbDataList = append(failEdbDataList, newEdbData) continue } //删除数据 models.DeleteEdbData(newEdbData.TradeCode, newEdbData.OldCreateDate) } } return } // GetEdbDataTbz 获取指标的同比值数据 func GetEdbDataTbz(edbInfo *data_manage.EdbInfo, tmpDataList []*data_manage.EdbDataList, startDateTime time.Time) (dataList []*data_manage.EdbDataList, minValue, maxValue float64, err error) { dataList = make([]*data_manage.EdbDataList, 0) // 数据处理 var dateArr []string dataMap := make(map[string]*data_manage.EdbDataList) for _, v := range tmpDataList { dateArr = append(dateArr, v.DataTime) dataMap[v.DataTime] = v } for _, av := range dateArr { currentItem, ok := dataMap[av] // 如果找不到当前日期的数据,那么终止当前循环,进入下一循环 if !ok { continue } tmpItem := *currentItem var isOk bool //是否计算出来结果 //当前日期 currentDate, tmpErr := time.ParseInLocation(utils.FormatDate, av, time.Local) if tmpErr != nil { err = tmpErr return } // 如果存在开始日期,同时,当前日期早于开始日期,那么终止当前循环,进入下一循环 if !startDateTime.IsZero() && currentDate.Before(startDateTime) { continue } //上一年的日期 preDate := currentDate.AddDate(-1, 0, 0) preDateStr := preDate.Format(utils.FormatDate) if findItem, ok := dataMap[preDateStr]; ok { //上一年同期找到 tmpItem.Value = TbzDiv(currentItem.Value, findItem.Value) isOk = true } else { if edbInfo.Frequency == "月度" { //向上和向下,各找一个月 for i := 0; i <= 35; i++ { nextDateDay := preDate.AddDate(0, 0, i) nextDateDayStr := nextDateDay.Format(utils.FormatDate) if findItem, ok := dataMap[nextDateDayStr]; ok { //上一年同期->下一个月找到 tmpItem.Value = TbzDiv(currentItem.Value, findItem.Value) isOk = true break } else { preDateDay := preDate.AddDate(0, 0, -i) preDateDayStr := preDateDay.Format(utils.FormatDate) if findItem, ok := dataMap[preDateDayStr]; ok { //上一年同期->上一个月找到 tmpItem.Value = TbzDiv(currentItem.Value, findItem.Value) isOk = true break } } } } else if edbInfo.Frequency == "季度" || edbInfo.Frequency == "年度" { if findItem, ok := dataMap[preDateStr]; ok { //上一年同期->下一个月找到 tmpItem.Value = TbzDiv(currentItem.Value, findItem.Value) isOk = true break } } else { nextDateDay := preDate.AddDate(0, 0, 1) nextDateDayStr := nextDateDay.Format(utils.FormatDate) preDateDay := preDate.AddDate(0, 0, -1) preDateDayStr := preDateDay.Format(utils.FormatDate) for i := 0; i < 35; i++ { if i >= 1 { nextDateDay = nextDateDay.AddDate(0, 0, i) nextDateDayStr = nextDateDay.Format(utils.FormatDate) } if findItem, ok := dataMap[nextDateDayStr]; ok { //上一年同期->下一个月找到 tmpItem.Value = TbzDiv(currentItem.Value, findItem.Value) isOk = true break } else { if i >= 1 { preDateDay = preDate.AddDate(0, 0, -i) preDateDayStr = nextDateDay.Format(utils.FormatDate) } if findItem, ok := dataMap[preDateDayStr]; ok { //上一年同期->上一个月找到 tmpItem.Value = TbzDiv(currentItem.Value, findItem.Value) isOk = true break } } } } } if isOk { if tmpItem.Value > maxValue { maxValue = tmpItem.Value } if tmpItem.Value < minValue { minValue = tmpItem.Value } dataList = append(dataList, &tmpItem) } } return } // GetChartEdbSeasonalData 获取指标的季节性数据 func GetChartEdbSeasonalData(calendar, startDate, endDate string, edbInfo *data_manage.EdbInfo) (resultData interface{}, err error) { var startDateReal string startDateReal = startDate calendarPreYear := 0 if calendar == "农历" { newStartDateReal, tmpErr := time.Parse(utils.FormatDate, startDateReal) if tmpErr != nil { err = tmpErr fmt.Println("time.Parse:" + err.Error()) return } calendarPreYear = newStartDateReal.Year() - 1 newStartDateReal = newStartDateReal.AddDate(-1, 0, 0) startDateReal = newStartDateReal.Format(utils.FormatDate) } dataList := make([]*data_manage.EdbDataList, 0) switch edbInfo.EdbInfoType { case 0: dataList, err = data_manage.GetEdbDataList(edbInfo.Source, edbInfo.SubSource, edbInfo.EdbInfoId, startDateReal, endDate) case 1: _, dataList, _, _, err, _ = GetPredictDataListByPredictEdbInfoId(edbInfo.EdbInfoId, startDateReal, endDate, false) default: err = errors.New(fmt.Sprint("获取失败,指标类型异常", edbInfo.EdbInfoType)) } if err != nil { return } latestDateStr := edbInfo.LatestDate //实际数据的截止日期 latestDate, tmpErr := time.Parse(utils.FormatDate, edbInfo.LatestDate) if tmpErr != nil { err = errors.New(fmt.Sprint("获取最后实际数据的日期失败,Err:" + tmpErr.Error() + ";LatestDate:" + edbInfo.LatestDate)) return } latestDateYear := latestDate.Year() //实际数据截止年份 if calendar == "农历" { if len(dataList) <= 0 { resultData = new(data_manage.EdbDataResult) } else { result, tmpErr := data_manage.AddCalculateQuarterV4(dataList) if tmpErr != nil { err = errors.New("获取农历数据失败,Err:" + tmpErr.Error()) return } // 处理季节图的截止日期 for k, edbDataItems := range result.List { var cuttingDataTimestamp int64 // 切割的日期时间字符串 cuttingDataTimeStr := latestDate.AddDate(0, 0, edbDataItems.BetweenDay).Format(utils.FormatDate) //如果等于最后的实际日期,那么遍历找到该日期对应的时间戳,并将其赋值为 切割时间戳 if edbDataItems.Year >= latestDateYear { for _, tmpData := range edbDataItems.Items { if tmpData.DataTime == cuttingDataTimeStr { cuttingDataTimestamp = tmpData.DataTimestamp break } } } edbDataItems.CuttingDataTimestamp = cuttingDataTimestamp result.List[k] = edbDataItems } if result.List[0].Year != calendarPreYear { itemList := make([]*data_manage.EdbDataList, 0) items := new(data_manage.EdbDataItems) //items.Year = calendarPreYear items.Items = itemList newResult := new(data_manage.EdbDataResult) newResult.List = append(newResult.List, items) newResult.List = append(newResult.List, result.List...) resultData = newResult } else { resultData = result } } } else { currentYear := time.Now().Year() quarterDataList := make([]*data_manage.QuarterData, 0) quarterMap := make(map[int][]*data_manage.EdbDataList) var quarterArr []int for _, v := range dataList { itemDate, tmpErr := time.Parse(utils.FormatDate, v.DataTime) if tmpErr != nil { err = errors.New("季度指标日期转换,Err:" + tmpErr.Error() + ";DataTime:" + v.DataTime) return } year := itemDate.Year() newItemDate := itemDate.AddDate(currentYear-year, 0, 0) timestamp := newItemDate.UnixNano() / 1e6 v.DataTimestamp = timestamp if findVal, ok := quarterMap[year]; !ok { quarterArr = append(quarterArr, year) findVal = append(findVal, v) quarterMap[year] = findVal } else { findVal = append(findVal, v) quarterMap[year] = findVal } } for _, v := range quarterArr { itemList := quarterMap[v] quarterItem := new(data_manage.QuarterData) quarterItem.Year = v quarterItem.DataList = itemList //如果等于最后的实际日期,那么将切割时间戳记录 if v == latestDateYear { var cuttingDataTimestamp int64 for _, tmpData := range itemList { if tmpData.DataTime == latestDateStr { cuttingDataTimestamp = tmpData.DataTimestamp break } } quarterItem.CuttingDataTimestamp = cuttingDataTimestamp } else if v > latestDateYear { //如果大于最后的实际日期,那么第一个点就是切割的时间戳 if len(itemList) > 0 { quarterItem.CuttingDataTimestamp = itemList[0].DataTimestamp - 100 } } quarterDataList = append(quarterDataList, quarterItem) } resultData = quarterDataList } return } // TbzDiv 同比值计算 func TbzDiv(a, b float64) float64 { var valFloat float64 if b != 0 { af := decimal.NewFromFloat(a) bf := decimal.NewFromFloat(b) val, _ := af.Div(bf).Float64() val = val - 1 valFloat, _ = decimal.NewFromFloat(val).Round(4).Float64() } else { valFloat = 0 } return valFloat } // GetDateData 获取指标的单个日期的值 func GetDateData(edbInfo *data_manage.EdbInfo, startDate, endDate string) (item *data_manage.EdbDataList, err error) { var dataList []*data_manage.EdbDataList switch edbInfo.EdbInfoType { case 0: dataList, err = data_manage.GetEdbDataList(edbInfo.Source, edbInfo.SubSource, edbInfo.EdbInfoId, startDate, endDate) case 1: _, dataList, _, _, err, _ = GetPredictDataListByPredictEdbInfoId(edbInfo.EdbInfoId, startDate, endDate, false) default: err = errors.New(fmt.Sprint("获取失败,指标类型异常", edbInfo.EdbInfoType)) } if err != nil { return } if len(dataList) <= 0 { return } item = dataList[0] return } // GetEdbBeforeAndAfterDateData 获取指标的单个日期的值 func GetEdbBeforeAndAfterDateData(targetDate string, dataList []*data_manage.EdbDataList) (list []*data_manage.EdbDataList, err error) { lenDataList := len(dataList) if lenDataList <= 0 { return } // 数组按照日期从小到大排序 maxIndex := lenDataList - 1 startIndex := 0 endIndex := 0 for index, v := range dataList { if v.DataTime == targetDate { if index+2 > maxIndex { endIndex = maxIndex } else if index+1 > maxIndex { endIndex = maxIndex } else { endIndex = index + 2 } startIndex = endIndex - 4 if startIndex < 0 { startIndex = 0 } if endIndex == maxIndex { list = dataList[startIndex:] } else { list = dataList[startIndex : endIndex+1] } break } } return } // GetPageData // @Description: 获取指标的分页数据 // @author: Roc // @datetime 2024-05-07 10:33:02 // @param edbInfoId int // @param source int // @param subSource int // @param startSize int // @param pageSize int // @return dataCount int // @return dataList []*data_manage.EdbData // @return err error func GetPageData(edbInfoId, source, subSource int, endDataTime string, startSize, pageSize int) (dataCount int, dataList []*data_manage.EdbData, err error) { dataList = make([]*data_manage.EdbData, 0) // 自有数据需要额外处理(从mongo获取) if source == utils.DATA_SOURCE_BUSINESS && utils.UseMongo { return getPageDataByMongo(edbInfoId, source, subSource, endDataTime, startSize, pageSize) } if source == utils.DATA_SOURCE_THS && subSource == utils.DATA_SUB_SOURCE_HIGH_FREQUENCY && utils.UseMongo { return getThsHfPageDataByMongo(edbInfoId, source, subSource, endDataTime, startSize, pageSize) } // 默认走mysql return getPageDataByMysql(edbInfoId, source, subSource, endDataTime, startSize, pageSize) } // getPageDataByMysql // @Description: 从Mysql获取数据 // @author: Roc // @datetime 2024-05-07 15:08:41 // @param edbInfoId int // @param source int // @param subSource int // @param endDataTime string // @param startSize int // @param pageSize int // @return dataCount int // @return dataList []*data_manage.EdbData // @return err error func getPageDataByMysql(edbInfoId, source, subSource int, endDataTime string, startSize, pageSize int) (dataCount int, dataList []*data_manage.EdbData, err error) { dataList = make([]*data_manage.EdbData, 0) var dataCondition string var dataPars []interface{} dataCondition += ` AND edb_info_id=? ` dataPars = append(dataPars, edbInfoId) // 结束日期 if endDataTime != "" { dataCondition += ` AND data_time <= ? ` dataPars = append(dataPars, endDataTime) } // 获取数据总量 dataCount, err = data_manage.GetEdbDataCountByCondition(dataCondition, dataPars, source, subSource) if err != nil && err.Error() != utils.ErrNoRow() { return } // 获取列表数据 dataList, err = data_manage.GetEdbDataListByCondition(dataCondition, dataPars, source, subSource, pageSize, startSize) if err != nil { return } return } // getPageDataByMongo // @Description: 从MongoDB获取数据 // @author: Roc // @datetime 2024-05-07 15:06:42 // @param edbInfoId int // @param source int // @param subSource int // @param endDataTime string // @param startSize int // @param pageSize int // @return dataCount int // @return dataList []*data_manage.EdbData // @return err error func getPageDataByMongo(edbInfoId, source, subSource int, endDataTime string, startSize, pageSize int) (dataCount int, dataList []*data_manage.EdbData, err error) { dataList = make([]*data_manage.EdbData, 0) mogDataObj := mgo.EdbDataBusiness{} // 构建查询条件 queryConditions := bson.M{ "edb_info_id": edbInfoId, } // 结束日期 dateCondition, err := mgo.BuildDateCondition("", endDataTime) if err != nil { return } if len(dateCondition) > 0 { queryConditions["data_time"] = dateCondition } // 获取数据总量 tmpCount, tmpErr := mogDataObj.GetCountDataList(queryConditions) if tmpErr != nil && tmpErr.Error() != utils.ErrNoRow() { err = tmpErr return } dataCount = int(tmpCount) // 获取列表数据 tmpDataList, tmpErr := mogDataObj.GetPageDataList(queryConditions, int64(startSize), int64(pageSize), []string{"-data_time"}) if tmpErr != nil { err = tmpErr return } for k, v := range tmpDataList { dataList = append(dataList, &data_manage.EdbData{ EdbDataId: k + 1, EdbInfoId: v.EdbInfoId, DataTime: v.DataTime.Format(utils.FormatDate), Value: v.Value, }) } return } // GetEdbDataTbzForSeason 获取指标的同比值数据 func GetEdbDataTbzForSeason(frequency string, tmpDataList []*data_manage.EdbDataList, startDateTime time.Time) (dataList []*data_manage.EdbDataList, minValue, maxValue float64, err error) { dataList = make([]*data_manage.EdbDataList, 0) // 数据处理 var dateArr []string dataMap := make(map[string]*data_manage.EdbDataList) for _, v := range tmpDataList { dateArr = append(dateArr, v.DataTime) dataMap[v.DataTime] = v } for _, av := range dateArr { currentItem, ok := dataMap[av] // 如果找不到当前日期的数据,那么终止当前循环,进入下一循环 if !ok { continue } tmpItem := *currentItem var isOk bool //是否计算出来结果 //当前日期 currentDate, tmpErr := time.ParseInLocation(utils.FormatDate, av, time.Local) if tmpErr != nil { err = tmpErr return } // 如果存在开始日期,同时,当前日期早于开始日期,那么终止当前循环,进入下一循环 if !startDateTime.IsZero() && currentDate.Before(startDateTime) { continue } //上一年的日期 preDate := currentDate.AddDate(-1, 0, 0) preDateStr := preDate.Format(utils.FormatDate) if findItem, ok := dataMap[preDateStr]; ok { //上一年同期找到 tmpItem.Value = TbzDiv(currentItem.Value, findItem.Value) isOk = true } else { if frequency == "月度" { //向上和向下,各找一个月 for i := 0; i <= 35; i++ { nextDateDay := preDate.AddDate(0, 0, i) nextDateDayStr := nextDateDay.Format(utils.FormatDate) if findItem, ok := dataMap[nextDateDayStr]; ok { //上一年同期->下一个月找到 tmpItem.Value = TbzDiv(currentItem.Value, findItem.Value) isOk = true break } else { preDateDay := preDate.AddDate(0, 0, -i) preDateDayStr := preDateDay.Format(utils.FormatDate) if findItem, ok := dataMap[preDateDayStr]; ok { //上一年同期->上一个月找到 tmpItem.Value = TbzDiv(currentItem.Value, findItem.Value) isOk = true break } } } } else if frequency == "季度" || frequency == "年度" { if findItem, ok := dataMap[preDateStr]; ok { //上一年同期->下一个月找到 tmpItem.Value = TbzDiv(currentItem.Value, findItem.Value) isOk = true break } } else { nextDateDay := preDate.AddDate(0, 0, 1) nextDateDayStr := nextDateDay.Format(utils.FormatDate) preDateDay := preDate.AddDate(0, 0, -1) preDateDayStr := preDateDay.Format(utils.FormatDate) for i := 0; i < 35; i++ { if i >= 1 { nextDateDay = nextDateDay.AddDate(0, 0, i) nextDateDayStr = nextDateDay.Format(utils.FormatDate) } if findItem, ok := dataMap[nextDateDayStr]; ok { //上一年同期->下一个月找到 tmpItem.Value = TbzDiv(currentItem.Value, findItem.Value) isOk = true break } else { if i >= 1 { preDateDay = preDate.AddDate(0, 0, -i) preDateDayStr = nextDateDay.Format(utils.FormatDate) } if findItem, ok := dataMap[preDateDayStr]; ok { //上一年同期->上一个月找到 tmpItem.Value = TbzDiv(currentItem.Value, findItem.Value) isOk = true break } } } } } if isOk { if tmpItem.Value > maxValue { maxValue = tmpItem.Value } if tmpItem.Value < minValue { minValue = tmpItem.Value } dataList = append(dataList, &tmpItem) } } return } func getThsHfPageDataByMongo(edbInfoId, source, subSource int, endDataTime string, startSize, pageSize int) (dataCount int, dataList []*data_manage.EdbData, err error) { dataList = make([]*data_manage.EdbData, 0) mogDataObj := mgo.EdbDataThsHf{} // 构建查询条件 queryConditions := bson.M{ "edb_info_id": edbInfoId, } // 结束日期 dateCondition, err := mgo.BuildDateCondition("", endDataTime) if err != nil { return } if len(dateCondition) > 0 { queryConditions["data_time"] = dateCondition } // 获取数据总量 tmpCount, tmpErr := mogDataObj.GetCountDataList(queryConditions) if tmpErr != nil && tmpErr.Error() != utils.ErrNoRow() { err = tmpErr return } dataCount = int(tmpCount) // 获取列表数据 tmpDataList, tmpErr := mogDataObj.GetPageDataList(queryConditions, int64(startSize), int64(pageSize), []string{"-data_time"}) if tmpErr != nil { err = tmpErr return } for k, v := range tmpDataList { dataList = append(dataList, &data_manage.EdbData{ EdbDataId: k + 1, EdbInfoId: v.EdbInfoId, DataTime: v.DataTime.Format(utils.FormatDate), Value: v.Value, }) } return }