eta_forum_hub/services/edb_info.go

package services

import (
	"errors"
	"eta/eta_forum_hub/models"
	"eta/eta_forum_hub/models/mgodb"
	"eta/eta_forum_hub/services/elastic"
	"eta/eta_forum_hub/utils"
	"fmt"
	"github.com/shopspring/decimal"
	"math"
	"sort"
	"strconv"
	"time"
)

// BatchAddOrUpdateEdbInfo 添加批量添加指标到指标库
func BatchAddOrUpdateEdbInfo(edbList []*models.EdbInfo, edbMapping []*models.EdbInfoCalculateMapping, sysUserId int, sysUserRealName string) (newList []*models.EdbInfo, err error, errMsg string, isSendEmail bool) {
	isSendEmail = true
	edbCodeMap := make(map[string]*models.EdbInfo)
	for _, v := range edbList {
		tmp, e, msg, _ := AddOrUpdateEdbInfo(v, sysUserId, sysUserRealName)
		if e != nil {
			err = fmt.Errorf("添加指标失败：%s，%s", e.Error(), msg)
			errMsg = "添加指标失败"
			return
		}
		newList = append(newList, tmp)
		edbCodeMap[tmp.EdbCode] = tmp
	}
	//批量添加指标的mapping信息
	calculateMappingItemList := make([]*models.EdbInfoCalculateMapping, 0)
	for _, v := range edbMapping {
		edbInfo, ok := edbCodeMap[v.EdbCode]
		if !ok {
			continue
		}
		fromEdbInfo, ok := edbCodeMap[v.FromEdbCode]
		if !ok {
			continue
		}

		calculateMappingItem := new(models.EdbInfoCalculateMapping)
		calculateMappingItem.CreateTime = time.Now()
		calculateMappingItem.ModifyTime = time.Now()
		calculateMappingItem.Sort = v.Sort
		calculateMappingItem.EdbCode = edbInfo.EdbCode
		calculateMappingItem.EdbInfoId = edbInfo.EdbInfoId
		calculateMappingItem.FromEdbInfoId = fromEdbInfo.EdbInfoId
		calculateMappingItem.FromEdbCode = fromEdbInfo.EdbCode
		calculateMappingItem.FromEdbName = fromEdbInfo.EdbName
		calculateMappingItem.FromSource = fromEdbInfo.Source
		calculateMappingItem.FromSourceName = fromEdbInfo.SourceName
		calculateMappingItem.FromTag = v.FromTag
		calculateMappingItem.Source = edbInfo.Source
		calculateMappingItem.SourceName = edbInfo.SourceName
		calculateMappingItem.FromSubSource = edbInfo.SubSource
		calculateMappingItemList = append(calculateMappingItemList, calculateMappingItem)
	}
	if len(calculateMappingItemList) > 0 {
		err = models.AddEdbInfoCalculateMappingMulti(calculateMappingItemList)
		if err != nil {
			errMsg = "指标映射关系添加失败"
			err = fmt.Errorf("指标映射关系添加失败, %s", err.Error())
			return
		}
	}

	return
}

// AddOrUpdateEdbInfo 添加指标到指标库
func AddOrUpdateEdbInfo(edbItem *models.EdbInfo, sysUserId int, sysUserRealName string) (edbInfo *models.EdbInfo, err error, errMsg string, isSendEmail bool) {
	isSendEmail = true
	//判断指标是否存在
	//edbInfo, err = models.GetEdbInfoByEdbCode(edbItem.EdbCode)
	edbInfo, err = models.GetEdbInfoById(edbItem.EdbInfoId)
	if err != nil {
		if err.Error() == utils.ErrNoRow() {
			err = nil
		} else {
			errMsg = "判断指标名称是否存在失败"
			err = errors.New("判断指标名称是否存在失败,Err:" + err.Error())
			return
		}
	}
	var edbInfoId int64
	// 指标已存在
	if edbInfo != nil && edbInfo.EdbInfoId > 0 {
		edbInfoId = int64(edbInfo.EdbInfoId)
		// 更新指标信息
		edbInfo.Source = edbItem.Source
		edbInfo.SourceName = edbItem.SourceName
		edbInfo.EdbType = edbItem.EdbType
		edbInfo.EdbCode = edbItem.EdbCode
		edbInfo.EdbName = edbItem.EdbName
		edbInfo.EdbNameSource = edbItem.EdbNameSource
		edbInfo.Frequency = edbItem.Frequency
		edbInfo.Unit = edbItem.Unit
		edbInfo.StartDate = edbItem.StartDate
		edbInfo.EndDate = edbItem.EndDate
		edbInfo.ClassifyId = edbItem.ClassifyId
		edbInfo.SysUserId = sysUserId
		edbInfo.SysUserRealName = sysUserRealName
		edbInfo.ModifyTime = time.Now()
		edbInfo.ServerUrl = edbItem.ServerUrl
		edbInfo.Sort = edbItem.Sort
		edbInfo.DataDateType = edbItem.DataDateType
		edbInfo.TerminalCode = edbItem.TerminalCode
		edbInfo.SourceIndexName = edbItem.SourceIndexName

		edbInfo.MaxValue = edbItem.MaxValue
		edbInfo.MinValue = edbItem.MinValue
		edbInfo.EdbInfoType = edbItem.EdbInfoType
		edbInfo.LatestDate = edbItem.LatestDate
		edbInfo.LatestValue = edbItem.LatestValue
		edbInfo.CalculateFormula = edbItem.CalculateFormula
		edbInfo.ManualSave = edbItem.ManualSave
		edbInfo.MoveType = edbItem.MoveType
		edbInfo.MoveFrequency = edbItem.MoveFrequency
		edbInfo.NoUpdate = edbItem.NoUpdate
		edbInfo.EdbNameEn = edbItem.EdbNameEn
		edbInfo.UnitEn = edbItem.UnitEn
		edbInfo.ChartImage = edbItem.ChartImage
		edbInfo.Calendar = edbItem.Calendar
		edbInfo.EmptyType = edbItem.EmptyType
		edbInfo.MaxEmptyType = edbItem.MaxEmptyType
		edbInfo.DataUpdateTime = edbItem.DataUpdateTime
		edbInfo.ErDataUpdateDate = edbItem.ErDataUpdateDate
		edbInfo.EndValue = edbItem.EndValue
		edbInfo.SubSource = edbItem.SubSource
		edbInfo.SubSourceName = edbItem.SubSourceName
		edbInfo.IndicatorCode = edbItem.IndicatorCode
		edbInfo.StockCode = edbItem.StockCode
		edbInfo.Extra = edbItem.Extra
		err = edbInfo.Update([]string{})
		if err != nil {
			errMsg = "保存失败"
			err = errors.New("保存失败，Err:" + err.Error())
			return
		}
		//同时删除指标的映射信息
		err = models.DeleteEdbInfoMapping(edbInfoId)
		if err != nil {
			errMsg = "删除指标映射信息失败"
			err = errors.New("删除指标映射信息失败，Err:" + err.Error())
			return
		}
	} else {
		edbInfo = new(models.EdbInfo)
		edbInfo.EdbInfoId = edbItem.EdbInfoId
		edbInfo.Source = edbItem.Source
		edbInfo.SourceName = edbItem.SourceName
		edbInfo.EdbType = edbItem.EdbType
		edbInfo.EdbCode = edbItem.EdbCode
		edbInfo.EdbName = edbItem.EdbName
		edbInfo.EdbNameSource = edbItem.EdbNameSource
		edbInfo.Frequency = edbItem.Frequency
		edbInfo.Unit = edbItem.Unit
		edbInfo.StartDate = edbItem.StartDate
		edbInfo.EndDate = edbItem.EndDate
		edbInfo.ClassifyId = edbItem.ClassifyId
		edbInfo.SysUserId = sysUserId
		edbInfo.SysUserRealName = sysUserRealName
		edbInfo.CreateTime = time.Now()
		edbInfo.ModifyTime = time.Now()
		edbInfo.ServerUrl = edbItem.ServerUrl
		edbInfo.Sort = edbItem.Sort
		edbInfo.DataDateType = edbItem.DataDateType
		edbInfo.TerminalCode = edbItem.TerminalCode
		edbInfo.SourceIndexName = edbItem.SourceIndexName
		//timestamp := strconv.FormatInt(time.Now().UnixNano(), 10)
		//edbInfo.UniqueCode = utils.MD5(utils.DATA_PREFIX + "_" + timestamp)
		edbInfo.UniqueCode = edbItem.UniqueCode
		edbInfo.MaxValue = edbItem.MaxValue
		edbInfo.MinValue = edbItem.MinValue
		edbInfo.EdbInfoType = edbItem.EdbInfoType
		edbInfo.LatestDate = edbItem.LatestDate
		edbInfo.LatestValue = edbItem.LatestValue
		edbInfo.CalculateFormula = edbItem.CalculateFormula
		edbInfo.ManualSave = edbItem.ManualSave
		edbInfo.MoveType = edbItem.MoveType
		edbInfo.MoveFrequency = edbItem.MoveFrequency
		edbInfo.NoUpdate = edbItem.NoUpdate
		edbInfo.EdbNameEn = edbItem.EdbNameEn
		edbInfo.UnitEn = edbItem.UnitEn
		edbInfo.ChartImage = edbItem.ChartImage
		edbInfo.Calendar = edbItem.Calendar
		edbInfo.EmptyType = edbItem.EmptyType
		edbInfo.MaxEmptyType = edbItem.MaxEmptyType
		edbInfo.DataUpdateTime = edbItem.DataUpdateTime
		edbInfo.ErDataUpdateDate = edbItem.ErDataUpdateDate
		edbInfo.EndValue = edbItem.EndValue
		edbInfo.SubSource = edbItem.SubSource
		edbInfo.SubSourceName = edbItem.SubSourceName
		edbInfo.IndicatorCode = edbItem.IndicatorCode
		edbInfo.StockCode = edbItem.StockCode
		edbInfo.Extra = edbItem.Extra

		edbInfoId, err = models.AddEdbInfo(edbInfo)
		if err != nil {
			errMsg = "保存失败"
			err = errors.New("保存失败，Err:" + err.Error())
			return
		}
		edbInfo.EdbInfoId = int(edbInfoId)
	}
	// if edbInfo.EdbType == 1 {
	// 	err = mgodb.ModifyEdbDataEdbInfoId(edbInfoId, edbInfo.EdbCode)
	// 	if err != nil {
	// 		errMsg = "保存失败"
	// 		err = errors.New("更新指标数据失败，Err:" + err.Error())
	// 		return
	// 	}
	// } else {
	// 	err = mgodb.ModifyEdbCalculateDataEdbInfoId(edbInfoId, edbInfo.EdbCode)
	// 	if err != nil {
	// 		errMsg = "保存失败"
	// 		err = errors.New("更新指标数据失败，Err:" + err.Error())
	// 		return
	// 	}
	// }

	// 更新es
	go AddOrEditEdbInfoToEs(edbInfo.EdbInfoId)

	return
}

// 批量删除指标，
func BatchDeleteEdbInfo(edbInfoIds []int) (err error, errMsg string) {
	for _, v := range edbInfoIds {
		//查询单个指标是否允许删除
		err, errMsg = DeleteEdbInfo(v)
		if err != nil {
			errMsg = "删除指标失败"
			return
		}
	}
	return
}

// 删除单个指标
func DeleteEdbInfo(edbInfoId int) (err error, errMsg string) {
	defer func() {
		if err != nil {
			utils.FileLog.Error("删除指标失败，Err:" + err.Error(), "edbInfoId", edbInfoId)
		}
		if errMsg != "" {
			utils.FileLog.Error("删除指标失败，ErrMsg:" + errMsg, "edbInfoId", edbInfoId)
		}
	}()
	//判断指标是否存在
	info, err := models.GetEdbInfoById(edbInfoId)
	if err != nil {
		if err.Error() == utils.ErrNoRow() {
			err = nil
			return
		} else {
			errMsg = "判断指标名称是否存在失败"
			err = errors.New("判断指标名称是否存在失败,Err:" + err.Error())
			return
		}
	}
	//判断指标是否用于作图，如果用于作图，则不可删除
	chartCount, tmpErr := models.GetChartEdbMappingCount(edbInfoId)
	if tmpErr != nil && tmpErr.Error() != utils.ErrNoRow() {
		errMsg = "删除失败"
		err = errors.New("判断指标是否被用于作图失败，Err:" + tmpErr.Error())
		return
	}
	if chartCount > 0 {
		errMsg = "当前指标已用作画图，不可删除"
		return
	}

	//判断指标是否用作其他指标的计算
	calculateCount, tmpErr := models.GetEdbInfoCalculateMappingCount(edbInfoId)
	if tmpErr != nil {
		errMsg = "删除失败"
		err = errors.New("判断指标是否被用于计算失败，GetEdbInfoCalculateCount Err:" + tmpErr.Error())
		return
	}
	if calculateCount > 0 {
		errMsg = "当前指标已用作，指标运算，不可删除"
		return
	}

	// 删除指标，删除映射关系，删除指标数据
	err = models.DeleteEdbInfo(edbInfoId)
	if err != nil {
		errMsg = "删除失败"
		err = errors.New("删除指标失败，Err:" + err.Error())
		return
	}
	if info.EdbType == 1 {
		err = mgodb.DeleteEdbInfoDataByEdbInfoId(edbInfoId)
		if err != nil {
			errMsg = "删除失败"
			err = errors.New("删除指标数据失败，Err:" + err.Error())
			return
		}
	} else {
		err = mgodb.DeleteEdbInfoCalculateDataByEdbInfoId(edbInfoId)
		if err != nil {
			errMsg = "删除失败"
			err = errors.New("删除指标数据失败，Err:" + err.Error())
			return
		}
	}

	// 删除es中的数据
	go DeleteEdbInfoToEs(edbInfoId)
	return
}

// AddOrEditEdbInfoToEs 添加/修改ES中的指标
func AddOrEditEdbInfoToEs(edbInfoId int) {
	//添加es
	itemInfo, _ := models.GetEdbInfoByCondition("AND edb_info_id=?", []interface{}{edbInfoId})
	go elastic.EsAddOrEditEdbInfoData(utils.DATA_INDEX_NAME, strconv.Itoa(itemInfo.EdbInfoId), itemInfo)
}

// 获取频度的英文版
func GetFrequencyEn(frequency string) (frequencyEn string) {
	switch frequency {
	case "日度":
		frequencyEn = "day"
		return
	case "周度":
		frequencyEn = "week"
		return
	case "旬度":
		frequencyEn = "ten days"
		return
	case "月度":
		frequencyEn = "month"
		return
	case "季度":
		frequencyEn = "quarter"
		return
	case "年度":
		frequencyEn = "year"
		return
	}
	return
}

func GetLeadUnitEn(unit string) (unitEn string) {
	switch unit {
	case "天":
		unitEn = "day"
		return
	case "周":
		unitEn = "week"
		return
	case "月":
		unitEn = "month"
		return
	case "季":
		unitEn = "quarter"
		return
	case "年":
		unitEn = "year"
		return
	}
	return
}

// HandleDataByLinearRegressionToList 插值法补充数据(线性方程式)
func HandleDataByLinearRegressionToList(edbInfoDataList []*models.EdbDataList, handleDataMap map[string]float64) (dataTimeList []string, valueList []float64, err error) {
	if len(edbInfoDataList) < 2 {
		return
	}

	var startEdbInfoData *models.EdbDataList
	for _, v := range edbInfoDataList {
		handleDataMap[v.DataTime] = v.Value
		dataTimeList = append(dataTimeList, v.DataTime)
		// 第一个数据就给过滤了,给后面的试用
		if startEdbInfoData == nil {
			startEdbInfoData = v
			//startEdbInfoData.DataTime = startEdbInfoData.DataTime[:5]+ "01-01"
			continue
		}

		// 获取两条数据之间相差的天数
		startDataTime, _ := time.ParseInLocation(utils.FormatDate, startEdbInfoData.DataTime, time.Local)
		currDataTime, _ := time.ParseInLocation(utils.FormatDate, v.DataTime, time.Local)
		betweenHour := int(currDataTime.Sub(startDataTime).Hours())
		betweenDay := betweenHour / 24

		// 如果相差一天,那么过滤
		if betweenDay <= 1 {
			startEdbInfoData = v
			continue
		}

		// 生成线性方程式
		var a, b float64
		{
			coordinateData := make([]utils.Coordinate, 0)
			tmpCoordinate1 := utils.Coordinate{
				X: 1,
				Y: startEdbInfoData.Value,
			}
			coordinateData = append(coordinateData, tmpCoordinate1)
			tmpCoordinate2 := utils.Coordinate{
				X: float64(betweenDay) + 1,
				Y: v.Value,
			}
			coordinateData = append(coordinateData, tmpCoordinate2)

			a, b = utils.GetLinearResult(coordinateData)
			if math.IsNaN(a) || math.IsNaN(b) {
				err = errors.New("线性方程公式生成失败")
				return
			}
		}

		// 生成对应的值
		{
			for i := 1; i < betweenDay; i++ {
				tmpDataTime := startDataTime.AddDate(0, 0, i)
				aDecimal := decimal.NewFromFloat(a)
				xDecimal := decimal.NewFromInt(int64(i) + 1)
				bDecimal := decimal.NewFromFloat(b)

				val, _ := aDecimal.Mul(xDecimal).Add(bDecimal).Round(4).Float64()
				handleDataMap[tmpDataTime.Format(utils.FormatDate)] = val
				dataTimeList = append(dataTimeList, tmpDataTime.Format(utils.FormatDate))
				valueList = append(valueList, val)
			}
		}

		startEdbInfoData = v
	}

	return
}

// HandleDataByLinearRegressionToList 保证生成365个数据点的线性插值法
func HandleDataByLinearRegressionToListV2(edbInfoDataList []*models.EdbDataList, handleDataMap map[string]float64) (dataTimeList []string, valueList []float64, err error) {
	if len(edbInfoDataList) < 2 {
		return
	}

	// 确保至少有两天数据来生成线性方程
	if len(edbInfoDataList) < 2 {
		err = errors.New("至少需要两天的数据来执行线性插值")
		return
	}

	// 对数据按日期排序，确保顺序正确
	sort.Slice(edbInfoDataList, func(i, j int) bool {
		t1, _ := time.ParseInLocation(utils.FormatDate, edbInfoDataList[i].DataTime, time.Local)
		t2, _ := time.ParseInLocation(utils.FormatDate, edbInfoDataList[j].DataTime, time.Local)
		return t1.Before(t2)
	})

	startEdbInfoData := edbInfoDataList[0]
	endEdbInfoData := edbInfoDataList[len(edbInfoDataList)-1]

	// 计算起始和结束日期间实际的天数
	startDate, _ := time.ParseInLocation(utils.FormatDate, startEdbInfoData.DataTime, time.Local)
	endDate, _ := time.ParseInLocation(utils.FormatDate, endEdbInfoData.DataTime, time.Local)
	actualDays := endDate.Sub(startDate).Hours() / 24

	// 生成365个数据点，首先处理已有数据
	for _, v := range edbInfoDataList {
		handleDataMap[v.DataTime] = v.Value
		dataTimeList = append(dataTimeList, v.DataTime)
		valueList = append(valueList, v.Value)
	}

	// 如果已有数据跨越天数不足365天，则对缺失的日期进行线性插值
	if actualDays < 365 {
		// 使用已有数据点生成线性方程（这里简化处理，实际可能需更细致处理边界情况）
		var a, b float64
		coordinateData := []utils.Coordinate{
			{X: 1, Y: startEdbInfoData.Value},
			{X: float64(len(edbInfoDataList)), Y: endEdbInfoData.Value},
		}
		a, b = utils.GetLinearResult(coordinateData)
		if math.IsNaN(a) || math.IsNaN(b) {
			err = errors.New("线性方程公式生成失败")
			return
		}

		// 对剩余日期进行插值
		for i := 1; i < 365; i++ {
			day := startDate.AddDate(0, 0, i)
			if _, exists := handleDataMap[day.Format(utils.FormatDate)]; !exists {
				aDecimal := decimal.NewFromFloat(a)
				xDecimal := decimal.NewFromInt(int64(i) + 1)
				bDecimal := decimal.NewFromFloat(b)

				val, _ := aDecimal.Mul(xDecimal).Add(bDecimal).Round(4).Float64()
				handleDataMap[day.Format(utils.FormatDate)] = val
				dataTimeList = append(dataTimeList, day.Format(utils.FormatDate))
				valueList = append(valueList, val)
			}
		}
	}

	return
}

func GetRefreshEdbInfoFromBase(edbInfoId, source int) (baseEdbInfoArr, calculateInfoArr []*models.EdbInfo, err error) {
	calculateList, err := models.GetEdbInfoCalculateMap(edbInfoId, source)
	if err != nil && err.Error() != utils.ErrNoRow() {
		return
	}
	for _, item := range calculateList {
		if item.EdbInfoId == edbInfoId { //	如果查出来关联的指标就是自己的话，那么就过滤
			continue
		}
		if item.EdbType == 1 {
			baseEdbInfoArr = append(baseEdbInfoArr, item)
		} else {
			calculateInfoArr = append(calculateInfoArr, item)
			newBaseEdbInfoArr, newCalculateInfoArr, _ := GetRefreshEdbInfoFromBase(item.EdbInfoId, item.Source)
			baseEdbInfoArr = append(baseEdbInfoArr, newBaseEdbInfoArr...)
			calculateInfoArr = append(calculateInfoArr, newCalculateInfoArr...)
		}
	}
	return
}