fix:指标库兼容外部数据

controllers/data_manage/edb_info.go

@@ -2125,64 +2125,14 @@ func (this *EdbInfoController) EdbInfoList() {
 
 	// 如果有数据权限，那么就去获取指标数据
 	if edbInfoItem.HaveOperaAuth {
-		var dataCondition string
-		var dataPars []interface{}
-
-		dataCondition += ` AND edb_info_id=? `
-		dataPars = append(dataPars, edbInfoItem.EdbInfoId)
-
-		if edbInfoItem.SubSource == utils.DATA_SUB_SOURCE_DATE {
-			if edbInfoItem.Source == utils.DATA_SOURCE_WIND {
-				dataCount, err := data_manage.GetEdbWsdDataCountByCondition(dataCondition, dataPars, edbInfoItem.Source)
-				if err != nil && err.Error() != utils.ErrNoRow() {
-					br.Msg = "获取指标信息失败"
-					br.ErrMsg = "获取指标数据总数失败,Err:" + err.Error()
-					return
-				}
-				page = paging.GetPaging(currentIndex, pageSize, dataCount)
-				dataList, err := data_manage.GetEdbWsdDataListByCondition(dataCondition, dataPars, edbInfoItem.Source, pageSize, startSize)
-				if err != nil {
-					br.Msg = "获取指标信息失败"
-					br.ErrMsg = "获取指标数据失败,Err:" + err.Error()
-					return
-				}
-				edbInfoItem.DataList = dataList
-			} else if edbInfoItem.Source == utils.DATA_SOURCE_THS {
-				dataCount, err := data_manage.GetTHsDsDataCountByCondition(dataCondition, dataPars, edbInfoItem.Source)
-				if err != nil && err.Error() != utils.ErrNoRow() {
-					br.Msg = "获取指标信息失败"
-					br.ErrMsg = "获取指标数据总数失败,Err:" + err.Error()
-					return
-				}
-				page = paging.GetPaging(currentIndex, pageSize, dataCount)
-				dataList, err := data_manage.GetThsDsDataListByCondition(dataCondition, dataPars, edbInfoItem.Source, pageSize, startSize)
-				if err != nil {
-					br.Msg = "获取指标信息失败"
-					br.ErrMsg = "获取指标数据失败,Err:" + err.Error()
-					return
-				}
-				edbInfoItem.DataList = dataList
-			} else {
-				br.Msg = "指标数据源id错误"
-				br.ErrMsg = "指标数据源id错误"
-				return
-			}
-		} else {
-			dataCount, err := data_manage.GetEdbDataCountByCondition(dataCondition, dataPars, edbInfoItem.Source, edbInfoItem.SubSource)
-			if err != nil && err.Error() != utils.ErrNoRow() {
-				br.Msg = "获取指标信息失败"
-				br.ErrMsg = "获取指标数据总数失败,Err:" + err.Error()
-				return
-			}
-			page = paging.GetPaging(currentIndex, pageSize, dataCount)
-			dataList, err := data_manage.GetEdbDataListByCondition(dataCondition, dataPars, edbInfoItem.Source, edbInfoItem.SubSource, pageSize, startSize)
-			if err != nil {
-				br.Msg = "获取指标信息失败"
-				br.ErrMsg = "获取指标数据失败,Err:" + err.Error()
-				return
-			}
-			edbInfoItem.DataList = dataList
+		dataCount, dataList, err := data.GetPageData(edbInfoItem.EdbInfoId, edbInfoItem.Source, edbInfoItem.SubSource, startSize, pageSize)
+		if err != nil && err.Error() != utils.ErrNoRow() {
+			br.Msg = "获取指标信息失败"
+			br.ErrMsg = "获取指标数据总数失败,Err:" + err.Error()
+			return
 		}
+		page = paging.GetPaging(currentIndex, pageSize, dataCount)
+		edbInfoItem.DataList = dataList
 	}
 
 	// 获取指标插入配置

models/data_manage/chart_info.go

@@ -2,10 +2,12 @@ package data_manage
 
 import (
 	"errors"
+	"eta/eta_api/models/mgo"
 	"eta/eta_api/utils"
 	"fmt"
 	"github.com/beego/beego/v2/client/orm"
 	"github.com/rdlucklib/rdluck_tools/paging"
+	"go.mongodb.org/mongo-driver/bson"
 	"strconv"
 	"strings"
 	"time"
@@ -369,8 +371,40 @@ type EdbDataList struct {
 	Value         float64 `description:"数据值"`
 }
 
-// GetEdbDataList 获取指标的数据（日期正序返回）
-func GetEdbDataList(source, subSource, endInfoId int, startDate, endDate string) (list []*EdbDataList, err error) {
+// GetEdbDataList
+// @Description: 获取指标的数据（日期正序返回）
+// @author: Roc
+// @datetime 2024-05-07 10:48:43
+// @param source int
+// @param subSource int
+// @param edbInfoId int
+// @param startDate string
+// @param endDate string
+// @return list []*EdbDataList
+// @return err error
+func GetEdbDataList(source, subSource, edbInfoId int, startDate, endDate string) (list []*EdbDataList, err error) {
+
+	// 外部数据需要额外处理（从mongo获取）
+	if source == utils.DATA_SOURCE_BUSINESS {
+		return getEdbDataListByMongo(source, subSource, edbInfoId, startDate, endDate)
+	}
+
+	return getEdbDataListByMysql(source, subSource, edbInfoId, startDate, endDate)
+
+}
+
+// getEdbDataListByMysql
+// @Description: 通过mysql查询指标数据
+// @author: Roc
+// @datetime 2024-05-07 10:40:43
+// @param source int
+// @param subSource int
+// @param edbInfoId int
+// @param startDate string
+// @param endDate string
+// @return list []*EdbDataList
+// @return err error
+func getEdbDataListByMysql(source, subSource, edbInfoId int, startDate, endDate string) (list []*EdbDataList, err error) {
 	tableName := GetEdbDataTableName(source, subSource)
 	if tableName == "" {
 		err = errors.New("无效的渠道：" + strconv.Itoa(source))
@@ -391,7 +425,64 @@ func GetEdbDataList(source, subSource, endInfoId int, startDate, endDate string)
 	sql += ` ORDER BY data_time ASC `
 	sql = fmt.Sprintf(sql, tableName)
 	o := orm.NewOrmUsingDB("data")
-	_, err = o.Raw(sql, endInfoId, pars).QueryRows(&list)
+	_, err = o.Raw(sql, edbInfoId, pars).QueryRows(&list)
+	return
+}
+
+// getEdbDataListByMongo
+// @Description: 通过mongo查询指标数据
+// @author: Roc
+// @datetime 2024-05-07 10:40:43
+// @param source int
+// @param subSource int
+// @param endInfoId int
+// @param startDate string
+// @param endDate string
+// @return list []*EdbDataList
+// @return err error
+func getEdbDataListByMongo(source, subSource, edbInfoId int, startDate, endDate string) (list []*EdbDataList, err error) {
+	list = make([]*EdbDataList, 0)
+
+	mogDataObj := mgo.EdbDataBusiness{}
+	// 构建查询条件
+	queryConditions := bson.M{
+		"edb_info_id": edbInfoId,
+	}
+
+	// 数据开始日期
+	if startDate != `` {
+		startDateTime, tmpErr := time.ParseInLocation(utils.FormatDate, startDate, time.Local)
+		if tmpErr != nil {
+			err = tmpErr
+			return
+		}
+		queryConditions["data_time"] = bson.M{"$gte": startDateTime}
+	}
+	// 数据结束日期
+	if endDate != "" {
+		endDateTime, tmpErr := time.ParseInLocation(utils.FormatDate, endDate, time.Local)
+		if tmpErr != nil {
+			err = tmpErr
+			return
+		}
+		queryConditions["data_time"] = bson.M{"$lte": endDateTime}
+	}
+	// 获取列表数据
+	tmpDataList, tmpErr := mogDataObj.GetAllDataList(queryConditions, []string{"data_time"})
+	if tmpErr != nil {
+		err = tmpErr
+		return
+	}
+	for k, v := range tmpDataList {
+		list = append(list, &EdbDataList{
+			EdbDataId:     k + 1,
+			EdbInfoId:     v.EdbInfoId,
+			DataTime:      v.DataTime.Format(utils.FormatDate),
+			DataTimestamp: v.DataTimestamp,
+			Value:         v.Value,
+		})
+	}
+
 	return
 }
 

models/mgo/base_from_business_data.go

@@ -60,10 +60,11 @@ func (m *BaseFromBusinessData) GetCollection() *qmgo.Collection {
 // @author: Roc
 // @receiver m
 // @datetime 2024-04-26 13:42:19
+// @param sort []string
 // @param whereParams interface{}
 // @return result []BaseFromBusinessData
 // @return err error
-func (m *BaseFromBusinessData) GetAllDataList(whereParams interface{}) (result []*BaseFromBusinessData, err error) {
+func (m *BaseFromBusinessData) GetAllDataList(whereParams interface{}, sort []string) (result []*BaseFromBusinessData, err error) {
 	if utils.MgoDataCli == nil {
 		err = errors.New("mongodb连接失败")
 		return
@@ -75,7 +76,7 @@ func (m *BaseFromBusinessData) GetAllDataList(whereParams interface{}) (result [
 		fmt.Println("MgoGetColl Err:", err.Error())
 		return
 	}
-	err = coll.Find(ctx, whereParams).All(&result)
+	err = coll.Find(ctx, whereParams).Sort(sort...).All(&result)
 	if err != nil {
 		return
 	}
@@ -124,6 +125,68 @@ func (m *BaseFromBusinessData) GetLimitDataList(whereParams interface{}, size in
 	return
 }
 
+// GetPageDataList
+// @Description: 根据条件获取分页数据列表
+// @author: Roc
+// @receiver m
+// @datetime 2024-05-07 10:21:07
+// @param whereParams interface{}
+// @param startSize int64
+// @param size int64
+// @param sort []string
+// @return result []*BaseFromBusinessData
+// @return err error
+func (m *BaseFromBusinessData) GetPageDataList(whereParams interface{}, startSize, size int64, sort []string) (result []*BaseFromBusinessData, err error) {
+	if utils.MgoDataCli == nil {
+		err = errors.New("mongodb连接失败")
+		return
+	}
+	db := utils.MgoDataCli.Database(m.DataBaseName())
+	coll := db.Collection(m.CollectionName())
+	ctx := context.TODO()
+	if err != nil {
+		fmt.Println("MgoGetColl Err:", err.Error())
+		return
+	}
+	err = coll.Find(ctx, whereParams).Sort(sort...).Skip(startSize).Limit(size).All(&result)
+	if err != nil {
+		return
+	}
+
+	for _, v := range result {
+		v.DataTime = v.DataTime.In(time.Local)
+		v.CreateTime = v.CreateTime.In(time.Local)
+		v.ModifyTime = v.ModifyTime.In(time.Local)
+	}
+
+	return
+}
+
+// GetCountDataList
+// @Description:  根据条件获取数据列表总数
+// @author: Roc
+// @receiver m
+// @datetime 2024-05-07 10:29:00
+// @param whereParams interface{}
+// @return count int64
+// @return err error
+func (m *BaseFromBusinessData) GetCountDataList(whereParams interface{}) (count int64, err error) {
+	if utils.MgoDataCli == nil {
+		err = errors.New("mongodb连接失败")
+		return
+	}
+	db := utils.MgoDataCli.Database(m.DataBaseName())
+	coll := db.Collection(m.CollectionName())
+	ctx := context.TODO()
+	if err != nil {
+		fmt.Println("MgoGetColl Err:", err.Error())
+		return
+	}
+	count, err = coll.Find(ctx, whereParams).Count()
+
+	return
+}
+
 // BatchInsertData
 // @Description: 批量写入数据
 // @author: Roc

models/mgo/edb_data_business.go

@@ -61,9 +61,10 @@ func (m *EdbDataBusiness) GetCollection() *qmgo.Collection {
 // @receiver m
 // @datetime 2024-04-26 13:42:19
 // @param whereParams interface{}
+// @param sort []string
 // @return result []EdbDataBusiness
 // @return err error
-func (m *EdbDataBusiness) GetAllDataList(whereParams interface{}) (result []*EdbDataBusiness, err error) {
+func (m *EdbDataBusiness) GetAllDataList(whereParams interface{}, sort []string) (result []*EdbDataBusiness, err error) {
 	if utils.MgoDataCli == nil {
 		err = errors.New("mongodb连接失败")
 		return
@@ -75,7 +76,7 @@ func (m *EdbDataBusiness) GetAllDataList(whereParams interface{}) (result []*Edb
 		fmt.Println("MgoGetColl Err:", err.Error())
 		return
 	}
-	err = coll.Find(ctx, whereParams).All(&result)
+	err = coll.Find(ctx, whereParams).Sort(sort...).All(&result)
 	if err != nil {
 		return
 	}
@@ -89,6 +90,104 @@ func (m *EdbDataBusiness) GetAllDataList(whereParams interface{}) (result []*Edb
 	return
 }
 
+// GetLimitDataList
+// @Description: 根据条件获取指定数量数据列表
+// @author: Roc
+// @receiver m
+// @datetime 2024-05-06 17:08:32
+// @param whereParams interface{}
+// @param size int64
+// @param sort []string
+// @return result []*BaseFromBusinessData
+// @return err error
+func (m *EdbDataBusiness) GetLimitDataList(whereParams interface{}, size int64, sort []string) (result []*EdbDataBusiness, err error) {
+	if utils.MgoDataCli == nil {
+		err = errors.New("mongodb连接失败")
+		return
+	}
+	db := utils.MgoDataCli.Database(m.DataBaseName())
+	coll := db.Collection(m.CollectionName())
+	ctx := context.TODO()
+	if err != nil {
+		fmt.Println("MgoGetColl Err:", err.Error())
+		return
+	}
+	err = coll.Find(ctx, whereParams).Sort(sort...).Limit(size).All(&result)
+	if err != nil {
+		return
+	}
+
+	for _, v := range result {
+		v.DataTime = v.DataTime.In(time.Local)
+		v.CreateTime = v.CreateTime.In(time.Local)
+		v.ModifyTime = v.ModifyTime.In(time.Local)
+	}
+
+	return
+}
+
+// GetPageDataList
+// @Description: 根据条件获取分页数据列表
+// @author: Roc
+// @receiver m
+// @datetime 2024-05-07 10:21:07
+// @param whereParams interface{}
+// @param startSize int64
+// @param size int64
+// @param sort []string
+// @return result []*EdbDataBusiness
+// @return err error
+func (m *EdbDataBusiness) GetPageDataList(whereParams interface{}, startSize, size int64, sort []string) (result []*EdbDataBusiness, err error) {
+	if utils.MgoDataCli == nil {
+		err = errors.New("mongodb连接失败")
+		return
+	}
+	db := utils.MgoDataCli.Database(m.DataBaseName())
+	coll := db.Collection(m.CollectionName())
+	ctx := context.TODO()
+	if err != nil {
+		fmt.Println("MgoGetColl Err:", err.Error())
+		return
+	}
+	err = coll.Find(ctx, whereParams).Sort(sort...).Skip(startSize).Limit(size).All(&result)
+	if err != nil {
+		return
+	}
+
+	for _, v := range result {
+		v.DataTime = v.DataTime.In(time.Local)
+		v.CreateTime = v.CreateTime.In(time.Local)
+		v.ModifyTime = v.ModifyTime.In(time.Local)
+	}
+
+	return
+}
+
+// GetCountDataList
+// @Description:  根据条件获取数据列表总数
+// @author: Roc
+// @receiver m
+// @datetime 2024-05-07 10:29:00
+// @param whereParams interface{}
+// @return count int64
+// @return err error
+func (m *EdbDataBusiness) GetCountDataList(whereParams interface{}) (count int64, err error) {
+	if utils.MgoDataCli == nil {
+		err = errors.New("mongodb连接失败")
+		return
+	}
+	db := utils.MgoDataCli.Database(m.DataBaseName())
+	coll := db.Collection(m.CollectionName())
+	ctx := context.TODO()
+	if err != nil {
+		fmt.Println("MgoGetColl Err:", err.Error())
+		return
+	}
+	count, err = coll.Find(ctx, whereParams).Count()
+
+	return
+}
+
 // BatchInsertData
 // @Description: 批量写入数据
 // @author: Roc

services/data/edb_data.go

@@ -4,9 +4,11 @@ import (
 	"errors"
 	"eta/eta_api/models"
 	"eta/eta_api/models/data_manage"
+	"eta/eta_api/models/mgo"
 	"eta/eta_api/utils"
 	"fmt"
 	"github.com/shopspring/decimal"
+	"go.mongodb.org/mongo-driver/bson"
 	"time"
 )
 
@@ -391,63 +393,72 @@ func GetEdbBeforeAndAfterDateData(targetDate string, dataList []*data_manage.Edb
 	return
 }
 
-//func GetPageData(edbInfoItem *data_manage.EdbInfoList) {
-//	var dataCondition string
-//	var dataPars []interface{}
-//
-//	dataCondition += ` AND edb_info_id=? `
-//	dataPars = append(dataPars, edbInfoItem.EdbInfoId)
-//
-//	if edbInfoItem.SubSource == utils.DATA_SUB_SOURCE_DATE {
-//		if edbInfoItem.Source == utils.DATA_SOURCE_WIND {
-//			dataCount, err := data_manage.GetEdbWsdDataCountByCondition(dataCondition, dataPars, edbInfoItem.Source)
-//			if err != nil && err.Error() != utils.ErrNoRow() {
-//				br.Msg = "获取指标信息失败"
-//				br.ErrMsg = "获取指标数据总数失败,Err:" + err.Error()
-//				return
-//			}
-//			page = paging.GetPaging(currentIndex, pageSize, dataCount)
-//			dataList, err := data_manage.GetEdbWsdDataListByCondition(dataCondition, dataPars, edbInfoItem.Source, pageSize, startSize)
-//			if err != nil {
-//				br.Msg = "获取指标信息失败"
-//				br.ErrMsg = "获取指标数据失败,Err:" + err.Error()
-//				return
-//			}
-//			edbInfoItem.DataList = dataList
-//		} else if edbInfoItem.Source == utils.DATA_SOURCE_THS {
-//			dataCount, err := data_manage.GetTHsDsDataCountByCondition(dataCondition, dataPars, edbInfoItem.Source)
-//			if err != nil && err.Error() != utils.ErrNoRow() {
-//				br.Msg = "获取指标信息失败"
-//				br.ErrMsg = "获取指标数据总数失败,Err:" + err.Error()
-//				return
-//			}
-//			page = paging.GetPaging(currentIndex, pageSize, dataCount)
-//			dataList, err := data_manage.GetThsDsDataListByCondition(dataCondition, dataPars, edbInfoItem.Source, pageSize, startSize)
-//			if err != nil {
-//				br.Msg = "获取指标信息失败"
-//				br.ErrMsg = "获取指标数据失败,Err:" + err.Error()
-//				return
-//			}
-//			edbInfoItem.DataList = dataList
-//		} else {
-//			br.Msg = "指标数据源id错误"
-//			br.ErrMsg = "指标数据源id错误"
-//			return
-//		}
-//	} else {
-//		dataCount, err := data_manage.GetEdbDataCountByCondition(dataCondition, dataPars, edbInfoItem.Source, edbInfoItem.SubSource)
-//		if err != nil && err.Error() != utils.ErrNoRow() {
-//			br.Msg = "获取指标信息失败"
-//			br.ErrMsg = "获取指标数据总数失败,Err:" + err.Error()
-//			return
-//		}
-//		page = paging.GetPaging(currentIndex, pageSize, dataCount)
-//		dataList, err := data_manage.GetEdbDataListByCondition(dataCondition, dataPars, edbInfoItem.Source, edbInfoItem.SubSource, pageSize, startSize)
-//		if err != nil {
-//			br.Msg = "获取指标信息失败"
-//			br.ErrMsg = "获取指标数据失败,Err:" + err.Error()
-//			return
-//		}
-//		edbInfoItem.DataList = dataList
-//	}
-//}
+// GetPageData
+// @Description: 获取指标的分页数据
+// @author: Roc
+// @datetime 2024-05-07 10:33:02
+// @param edbInfoId int
+// @param source int
+// @param subSource int
+// @param startSize int
+// @param pageSize int
+// @return dataCount int
+// @return dataList []*data_manage.EdbData
+// @return err error
+func GetPageData(edbInfoId, source, subSource, startSize, pageSize int) (dataCount int, dataList []*data_manage.EdbData, err error) {
+	dataList = make([]*data_manage.EdbData, 0)
+
+	// 外部数据需要额外处理（从mongo获取）
+	if source == utils.DATA_SOURCE_BUSINESS {
+		mogDataObj := mgo.EdbDataBusiness{}
+		// 构建查询条件
+		queryConditions := bson.M{
+			"edb_info_id": edbInfoId,
+		}
+
+		// 获取数据总量
+		tmpCount, tmpErr := mogDataObj.GetCountDataList(queryConditions)
+		if tmpErr != nil && tmpErr.Error() != utils.ErrNoRow() {
+			err = tmpErr
+			return
+		}
+		dataCount = int(tmpCount)
+
+		// 获取列表数据
+		tmpDataList, tmpErr := mogDataObj.GetPageDataList(queryConditions, int64(startSize), int64(pageSize), []string{"-data_time"})
+		if tmpErr != nil {
+			err = tmpErr
+			return
+		}
+		for k, v := range tmpDataList {
+			dataList = append(dataList, &data_manage.EdbData{
+				EdbDataId: k + 1,
+				EdbInfoId: v.EdbInfoId,
+				DataTime:  v.DataTime.Format(utils.FormatDate),
+				Value:     v.Value,
+			})
+		}
+
+		return
+	}
+
+	var dataCondition string
+	var dataPars []interface{}
+
+	dataCondition += ` AND edb_info_id=? `
+	dataPars = append(dataPars, edbInfoId)
+
+	// 获取数据总量
+	dataCount, err = data_manage.GetEdbDataCountByCondition(dataCondition, dataPars, source, subSource)
+	if err != nil && err.Error() != utils.ErrNoRow() {
+		return
+	}
+
+	// 获取列表数据
+	dataList, err = data_manage.GetEdbDataListByCondition(dataCondition, dataPars, source, subSource, pageSize, startSize)
+	if err != nil {
+		return
+	}
+
+	return
+}