Merge branch 'feature/eta_1.9.3' into custom

controllers/data_manage/edb_info.go

@@ -10,7 +10,6 @@ import (
 	request2 "eta/eta_mobile/models/data_manage/excel/request"
 	"eta/eta_mobile/models/data_manage/request"
 	"eta/eta_mobile/models/data_manage/response"
-	"eta/eta_mobile/models/mgo"
 	"eta/eta_mobile/models/system"
 	"eta/eta_mobile/services/alarm_msg"
 	"eta/eta_mobile/services/data"
@@ -22,7 +21,6 @@ import (
 	"eta/eta_mobile/utils"
 	"fmt"
 	"github.com/rdlucklib/rdluck_tools/paging"
-	"go.mongodb.org/mongo-driver/bson"
 	"strconv"
 	"strings"
 	"sync"
@@ -1734,8 +1732,9 @@ func (this *EdbInfoController) EdbInfoSearch() {
 				return
 			}
 
-			obj := mgo.BaseFromBusinessData{}
-			tmpDataList, err := obj.GetLimitDataList(bson.M{"index_code": edbCode}, utils.EDB_DATA_LIMIT, []string{"-data_time"})
+			//obj := mgo.BaseFromBusinessData{}
+			//tmpDataList, err := obj.GetLimitDataList(bson.M{"index_code": edbCode}, utils.EDB_DATA_LIMIT, []string{"-data_time"})
+			_, tmpDataList, err := data.GetPageBaseBusinessIndexData(edbCode, 1, utils.EDB_DATA_LIMIT)
 			if err != nil && err.Error() != utils.ErrNoRow() {
 				br.Msg = "获取失败"
 				br.ErrMsg = "获取自有数据已存在信息失败,Err:" + err.Error()
@@ -1744,7 +1743,7 @@ func (this *EdbInfoController) EdbInfoSearch() {
 			dataItems := make([]*data_manage.EdbInfoSearchData, 0)
 			for _, v := range tmpDataList {
 				dataItems = append(dataItems, &data_manage.EdbInfoSearchData{
-					DataTime: v.DataTime.Format(utils.FormatDate),
+					DataTime: v.DataTime,
 					Value:    v.Value,
 				})
 			}

controllers/data_manage/edb_info_calculate.go

@@ -1468,7 +1468,7 @@ func (this *EdbInfoController) QueryEdbDataTable() {
 	tableName := data_manage.GetEdbDataTableName(edbInfo.Source, edbInfo.SubSource)
 
 	var templateStr string
-	if edbInfo.Source == utils.DATA_SOURCE_BUSINESS {
+	if edbInfo.Source == utils.DATA_SOURCE_BUSINESS && utils.UseMongo {
 		templateStr = fmt.Sprintf("# 查询条件\nquery = {\"edb_code\": \"%s\"}\n# 排序\nsort = [(\"data_time\", -1)]  # -1 表示降序排列，对应 SQL 的 DESC\nprojection = {\"data_time\": 1, \"value\": 1, \"_id\": 0}  # 只选择data_time和value字段，忽略_id字段\n# 使用 find() 方法获取数据，然后使用 aggregate() 转换为列表\nraw_data = list(collection.find(query, projection).sort(sort))\n# 将结果转换为 DataFrame\nraw = pd.DataFrame(raw_data)\n# 转换data_time字段为本地时区时间\nraw['data_time'] = raw['data_time'].apply(lambda x: x.replace(tzinfo=utc_tz)).dt.tz_convert(local_tz).dt.strftime('%s')", edbInfo.EdbCode, "%Y-%m-%d")
 	} else {
 		templateStr = fmt.Sprintf("sql1 = f\"\"\"SELECT data_time,`value` FROM %s WHERE edb_code = '%s' ORDER BY data_time DESC;\"\"\"\nraw = pandas_fetch_all(sql1, db)", tableName, edbInfo.EdbCode)

models/business_conf.go

@@ -1,6 +1,7 @@
 package models
 
 import (
+	"eta/eta_mobile/utils"
 	"fmt"
 	"github.com/beego/beego/v2/client/orm"
 	"html"
@@ -220,3 +221,18 @@ func GetBusinessConfByKey(key string) (item *BusinessConf, err error) {
 	err = o.Raw(sql, key).QueryRow(&item)
 	return
 }
+
+// InitUseMongoConf
+// @Description:
+// @author: Roc
+// @datetime 2024-07-01 13:49:09
+func InitUseMongoConf() {
+	useMongo, e := GetBusinessConfByKey("UseMongo")
+	if e != nil {
+		return
+	}
+
+	if useMongo.ConfVal == `true` {
+		utils.UseMongo = true
+	}
+}

models/data_manage/base_from_business_data.go

@@ -0,0 +1,231 @@
+package data_manage
+
+import (
+	"eta/eta_mobile/utils"
+	"fmt"
+	"github.com/beego/beego/v2/client/orm"
+	"strings"
+	"time"
+)
+
+// BaseFromBusinessData
+// @Description: 外部指标（商家系统）原始数据表
+type BaseFromBusinessData struct {
+	BusinessDataId          int       `orm:"column(business_data_id);pk" json:"business_data_id"`
+	BaseFromBusinessIndexId int       `json:"base_from_business_index_id"` // 指标id
+	IndexCode               string    `json:"index_code"`                  // 指标编码
+	DataTime                time.Time `json:"data_time"`                   // 数据日期
+	Value                   float64   `json:"value"`                       // 数据值
+	CreateTime              time.Time `json:"create_time"`                 // 创建时间
+	ModifyTime              time.Time `json:"modify_time"`                 // 修改时间
+}
+
+// TableName
+// @Description:  获取表名
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-26 13:41:36
+// @return string
+func (m *BaseFromBusinessData) TableName() string {
+	return "base_from_business_data"
+}
+
+// CollectionName
+// @Description:  获取集合名称
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-26 13:41:36
+// @return string
+func (m *BaseFromBusinessData) CollectionName() string {
+	return "base_from_business_data"
+}
+
+// DataBaseName
+// @Description: 获取数据库名称
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-26 13:41:33
+// @return string
+func (m *BaseFromBusinessData) DataBaseName() string {
+	return utils.MgoDataDbName
+}
+
+type WhereParams struct {
+	Condition string
+	Pars      []interface{}
+	Order     string `description:"排序字段"`
+}
+
+// GetAllDataList
+// @Description: 根据条件获取所有数据
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-26 13:42:19
+// @param sort []string
+// @param whereParams interface{}
+// @return result []BaseFromBusinessData
+// @return err error
+func (m *BaseFromBusinessData) GetAllDataList(condition string, pars []interface{}, order string) (result []*BaseFromBusinessData, err error) {
+	o := orm.NewOrmUsingDB("data")
+
+	sql := `SELECT * FROM base_from_business_data WHERE 1=1 `
+	if condition != `` {
+		sql += ` ` + condition
+	}
+
+	if order != `` {
+		sql += ` ORDER BY ` + order
+	}
+
+	_, err = o.Raw(sql, pars).QueryRows(&result)
+
+	return
+}
+
+// GetLimitDataList
+// @Description: 根据条件获取指定数量数据列表
+// @author: Roc
+// @receiver m
+// @datetime 2024-05-06 17:08:32
+// @param whereParams interface{}
+// @param size int64
+// @return result []*BaseFromBusinessData
+// @return err error
+func (m *BaseFromBusinessData) GetLimitDataList(condition string, pars []interface{}, order string, size int64) (result []*BaseFromBusinessData, err error) {
+	o := orm.NewOrmUsingDB("data")
+	sql := `SELECT * FROM base_from_business_data WHERE 1=1 `
+	if condition != `` {
+		sql += ` ` + condition
+	}
+
+	if order != `` {
+		sql += ` ORDER BY ` + order
+	}
+
+	sql += fmt.Sprintf(` LIMIT %d`, size)
+
+	_, err = o.Raw(sql, pars).QueryRows(&result)
+
+	return
+}
+
+// GetPageDataList
+// @Description: 根据条件获取分页数据列表
+// @author: Roc
+// @receiver m
+// @datetime 2024-05-07 10:21:07
+// @param whereParams interface{}
+// @param startSize int64
+// @param size int64
+// @param sort []string
+// @return result []*BaseFromBusinessData
+// @return err error
+func (m *BaseFromBusinessData) GetPageDataList(condition []string, pars []interface{}, order string, startSize, size int64) (result []*BaseFromBusinessData, err error) {
+	o := orm.NewOrmUsingDB("data")
+	sql := `SELECT * FROM base_from_business_data `
+	if len(condition) > 0 {
+		sql += ` WHERE ` + strings.Join(condition, " AND ")
+	}
+
+	if order != `` {
+		sql += ` ORDER BY ` + order
+	}
+
+	sql += fmt.Sprintf(` LIMIT %d,%d`, startSize, size)
+
+	_, err = o.Raw(sql, pars).QueryRows(&result)
+
+	return
+}
+
+// GetCountDataList
+// @Description:  根据条件获取数据列表总数
+// @author: Roc
+// @receiver m
+// @datetime 2024-05-07 10:29:00
+// @param whereParams interface{}
+// @return count int64
+// @return err error
+func (m *BaseFromBusinessData) GetCountDataList(condition []string, pars []interface{}) (count int64, err error) {
+	o := orm.NewOrmUsingDB("data")
+	sql := `SELECT COUNT(1) FROM base_from_business_data `
+	if len(condition) > 0 {
+		sql += ` WHERE ` + strings.Join(condition, " AND ")
+	}
+	err = o.Raw(sql, pars).QueryRow(&count)
+
+	return
+}
+
+// InsertDataByColl
+// @Description: 写入单条数据（外部传入集合）
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-26 14:22:18
+// @param addData interface{}
+// @return err error
+func (m *BaseFromBusinessData) InsertDataByColl(addData interface{}) (err error) {
+	o := orm.NewOrmUsingDB("data")
+	_, err = o.Insert(addData)
+
+	return
+}
+
+// BatchInsertData
+// @Description: 批量写入数据
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-26 14:22:18
+// @param bulk int 每次请求保存的数据量
+// @param dataList []interface{}
+// @return err error
+func (m *BaseFromBusinessData) BatchInsertData(bulk int, dataList []interface{}) (err error) {
+	o := orm.NewOrmUsingDB("data")
+	_, err = o.InsertMulti(bulk, dataList)
+
+	return
+}
+
+// UpdateData
+// @Description: 单条数据修改
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-26 15:01:51
+// @param whereParams interface{}
+// @param updateParams interface{}
+// @return err error
+func (m *BaseFromBusinessData) UpdateData(updateCols []string) (err error) {
+	o := orm.NewOrmUsingDB("data")
+	_, err = o.Update(m, updateCols...)
+	if err != nil {
+		fmt.Println("UpdateDataByColl:Err:" + err.Error())
+		return
+	}
+
+	return
+}
+
+// GetEdbInfoMaxAndMinInfo
+// @Description: 获取当前指标的最大最小值
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-30 17:15:39
+// @param whereParams interface{}
+// @return result EdbInfoMaxAndMinInfo
+// @return err error
+func (m *BaseFromBusinessData) GetEdbInfoMaxAndMinInfo(indexCode string) (result EdbInfoMaxAndMinInfo, err error) {
+	o := orm.NewOrmUsingDB("data")
+	sql := ``
+	sql = ` SELECT MIN(data_time) AS min_date,MAX(data_time) AS max_date,MIN(value) AS min_value,MAX(value) AS max_value FROM base_from_business_data WHERE index_code = ? `
+	err = o.Raw(sql, indexCode).QueryRow(&result)
+	if err != nil {
+		return
+	}
+
+	var latestValue float64
+	sql = ` SELECT value AS latest_value FROM base_from_business_data WHERE index_code = ? ORDER BY data_time DESC LIMIT 1 `
+	err = o.Raw(sql, indexCode).QueryRow(&latestValue)
+	result.LatestValue = latestValue
+
+	return
+}

models/data_manage/chart_info.go

@@ -384,9 +384,12 @@ type EdbDataList struct {
 // @return err error
 func GetEdbDataList(source, subSource, edbInfoId int, startDate, endDate string) (list []*EdbDataList, err error) {
 	// 自有数据需要额外处理（从mongo获取）
-	if source == utils.DATA_SOURCE_BUSINESS {
+	if source == utils.DATA_SOURCE_BUSINESS && utils.UseMongo {
 		return getEdbDataListByMongo(source, subSource, edbInfoId, startDate, endDate)
 	}
+	if source == utils.DATA_SOURCE_THS && subSource == utils.DATA_SUB_SOURCE_HIGH_FREQUENCY && utils.UseMongo {
+		return getThsHfEdbDataListByMongo(source, subSource, edbInfoId, startDate, endDate)
+	}
 
 	return getEdbDataListByMysql(source, subSource, edbInfoId, startDate, endDate)
 
@@ -506,9 +509,12 @@ func GetEdbDataLunarList(endInfoId int, startDate, endDate string) (list []*EdbD
 // @return err error
 func GetEdbDataListMinAndMax(source, subSource, edbInfoId int, startDate, endDate string) (min_data, max_data float64, err error) {
 	// 自有数据需要额外处理（从mongo获取）
-	if source == utils.DATA_SOURCE_BUSINESS {
+	if source == utils.DATA_SOURCE_BUSINESS && utils.UseMongo {
 		return getEdbDataListMinAndMaxByMongo(source, subSource, edbInfoId, startDate, endDate)
 	}
+	if source == utils.DATA_SOURCE_THS && subSource == utils.DATA_SUB_SOURCE_HIGH_FREQUENCY && utils.UseMongo {
+		return getThsHfEdbDataListMinAndMaxByMongo(source, subSource, edbInfoId, startDate, endDate)
+	}
 
 	return getEdbDataListMinAndMaxByMysql(source, subSource, edbInfoId, startDate, endDate)
 }
@@ -2343,3 +2349,72 @@ type RadarYData struct {
 	Name  string    `description:"别名"`
 	Value []float64 `description:"每个指标的值"`
 }
+
+func getThsHfEdbDataListByMongo(source, subSource, edbInfoId int, startDate, endDate string) (list []*EdbDataList, err error) {
+	list = make([]*EdbDataList, 0)
+
+	mogDataObj := mgo.EdbDataThsHf{}
+	// 构建查询条件
+	queryConditions := bson.M{
+		"edb_info_id": edbInfoId,
+	}
+
+	// 数据日期
+	dateCondition, err := mgo.BuildDateCondition(startDate, endDate)
+	if err != nil {
+		return
+	}
+	if len(dateCondition) > 0 {
+		queryConditions["data_time"] = dateCondition
+	}
+
+	// 获取列表数据
+	tmpDataList, tmpErr := mogDataObj.GetAllDataList(queryConditions, []string{"data_time"})
+	if tmpErr != nil {
+		err = tmpErr
+		return
+	}
+	for k, v := range tmpDataList {
+		list = append(list, &EdbDataList{
+			EdbDataId:     k + 1,
+			EdbInfoId:     v.EdbInfoId,
+			DataTime:      v.DataTime.Format(utils.FormatDate),
+			DataTimestamp: v.DataTimestamp,
+			Value:         v.Value,
+		})
+	}
+	return
+}
+
+func getThsHfEdbDataListMinAndMaxByMongo(source, subSource, edbInfoId int, startDate, endDate string) (minData, maxData float64, err error) {
+	mogDataObj := mgo.EdbDataThsHf{}
+	// 构建查询条件
+	queryConditions := bson.M{
+		"edb_info_id": edbInfoId,
+	}
+	// 日期
+	dateCondition, err := mgo.BuildDateCondition(startDate, endDate)
+	if err != nil {
+		return
+	}
+	if len(dateCondition) > 0 {
+		queryConditions["data_time"] = dateCondition
+	}
+
+	pipeline := []bson.M{
+		{"$match": queryConditions},
+		{"$group": bson.M{
+			"_id":       nil,
+			"min_value": bson.M{"$min": "$value"},
+			"max_value": bson.M{"$max": "$value"},
+		}},
+		{"$project": bson.M{"_id": 0}}, // 可选，如果不需要_id字段
+	}
+	result, err := mogDataObj.GetEdbInfoMaxAndMinInfo(pipeline)
+	if err != nil {
+		return
+	}
+	minData = result.MinValue
+	maxData = result.MaxValue
+	return
+}

models/data_manage/edb_data_base.go

@@ -13,10 +13,12 @@ import (
 func GetEdbDataTableName(source, subSource int) (tableName string) {
 	switch source {
 	case utils.DATA_SOURCE_THS:
-		tableName = "edb_data_ths"
-		if subSource == utils.DATA_SUB_SOURCE_DATE {
+		switch subSource {
+		case utils.DATA_SUB_SOURCE_DATE:
 			tableName = "edb_data_ths_ds"
-		} else {
+		case utils.DATA_SUB_SOURCE_HIGH_FREQUENCY:
+			tableName = "edb_data_ths_hf"
+		default:
 			tableName = "edb_data_ths"
 		}
 	case utils.DATA_SOURCE_WIND:
@@ -219,9 +221,12 @@ type EdbDataBase struct {
 
 func GetEdbDataAllByEdbCode(edbCode string, source, subSource, limit int) (items []*EdbInfoSearchData, err error) {
 	// 自有数据需要额外处理（从mongo获取）
-	if source == utils.DATA_SOURCE_BUSINESS {
+	if source == utils.DATA_SOURCE_BUSINESS && utils.UseMongo {
 		return GetEdbDataAllByEdbCodeByMongo(edbCode, source, subSource, limit)
 	}
+	if source == utils.DATA_SOURCE_THS && subSource == utils.DATA_SUB_SOURCE_HIGH_FREQUENCY && utils.UseMongo {
+		return GetThsHfEdbDataAllByEdbCodeByMongo(edbCode, source, subSource, limit)
+	}
 
 	return GetEdbDataAllByEdbCodeByMysql(edbCode, source, subSource, limit)
 }
@@ -359,3 +364,31 @@ func GetAddSql(edbInfoId, edbCode, dataTime, timestampStr string, value string)
 	addSql += "),"
 	return
 }
+
+func GetThsHfEdbDataAllByEdbCodeByMongo(edbCode string, source, subSource, limit int) (list []*EdbInfoSearchData, err error) {
+	list = make([]*EdbInfoSearchData, 0)
+
+	mogDataObj := mgo.EdbDataThsHf{}
+	// 构建查询条件
+	queryConditions := bson.M{
+		"edb_code": edbCode,
+	}
+
+	var tmpDataList []*mgo.EdbDataThsHf
+	// 获取列表数据
+	if limit > 0 {
+		tmpDataList, err = mogDataObj.GetLimitDataList(queryConditions, int64(limit), []string{"-data_time"})
+	} else {
+		tmpDataList, err = mogDataObj.GetLimitDataList(queryConditions, int64(limit), []string{"-data_time"})
+	}
+	if err != nil {
+		return
+	}
+	for _, v := range tmpDataList {
+		list = append(list, &EdbInfoSearchData{
+			DataTime: v.DataTime.Format(utils.FormatDate),
+			Value:    v.Value,
+		})
+	}
+	return
+}

models/data_manage/edb_data_insert_config.go

@@ -133,8 +133,10 @@ func CreateEdbDataInsertConfigAndData(edbInfo *EdbInfo, date time.Time, value st
 	}
 
 	// 指标明细数据更新
-	if edbInfo.Source == utils.DATA_SOURCE_BUSINESS {
+	if edbInfo.Source == utils.DATA_SOURCE_BUSINESS && utils.UseMongo {
 		dateStr, err = updateInsertConfigValueByMongo(to, edbInfo, oldConfigDate, date, value)
+	} else if edbInfo.Source == utils.DATA_SOURCE_THS && edbInfo.SubSource == utils.DATA_SUB_SOURCE_HIGH_FREQUENCY && utils.UseMongo {
+		dateStr, err = updateThsHfInsertConfigValueByMongo(to, edbInfo, oldConfigDate, date, value)
 	} else {
 		dateStr, err = updateInsertConfigValueByMysql(to, edbInfo, oldConfigDate, date, value)
 	}
@@ -336,3 +338,102 @@ func updateInsertConfigValueByMongo(to orm.TxOrmer, edbInfo *EdbInfo, oldConfigD
 
 	return
 }
+
+func updateThsHfInsertConfigValueByMongo(to orm.TxOrmer, edbInfo *EdbInfo, oldConfigDate, newDate time.Time, value string) (dateStr string, err error) {
+	tableName := GetEdbDataTableName(edbInfo.Source, edbInfo.SubSource)
+	if tableName == `` {
+		err = errors.New("找不到该指标的数据表")
+		return
+	}
+
+	dateStr = newDate.Format(utils.FormatDate)
+	timestamp := newDate.UnixNano() / 1e6
+	var floatValue float64
+	if value != "" {
+		floatValue, err = strconv.ParseFloat(value, 64)
+		if err != nil {
+			fmt.Println("转换失败:", err.Error())
+			return
+		}
+	}
+
+	mogDataObj := mgo.EdbDataThsHf{}
+	coll := mogDataObj.GetCollection()
+
+	var edbDateData *mgo.EdbDataThsHf
+	if !oldConfigDate.IsZero() {
+		// 构建查询条件
+		queryConditions := bson.M{
+			"edb_info_id": edbInfo.EdbInfoId,
+			"data_time":   oldConfigDate,
+		}
+
+		edbDateData, err = mogDataObj.GetItem(queryConditions)
+		//if tmpErr != nil && tmpErr == mongo.ErrNoDocuments {
+		//	err = tmpErr
+		//	return
+		//}
+		if err != nil && err != mongo.ErrNoDocuments {
+			return
+		}
+		err = nil
+
+	}
+
+	// 如果是没有历史数据，那么就需要增加数据
+	if edbDateData == nil {
+		addDataItem := mgo.EdbDataThsHf{
+			//ID:            primitive.ObjectID{},
+			EdbInfoId:     edbInfo.EdbInfoId,
+			EdbCode:       edbInfo.EdbCode,
+			DataTime:      newDate,
+			Value:         floatValue,
+			CreateTime:    time.Now(),
+			ModifyTime:    time.Now(),
+			DataTimestamp: timestamp,
+		}
+		err = mogDataObj.InsertDataByColl(coll, addDataItem)
+		if err != nil {
+			fmt.Println("mogDataObj.BatchInsertData() Err:" + err.Error())
+			return
+		}
+		return
+	}
+
+	// 数据清空
+	if value == "" {
+		queryConditions := bson.M{
+			"edb_info_id": edbInfo.EdbInfoId,
+		}
+
+		// 获取最新的两条数据
+		tmpDataList, tmpErr := mogDataObj.GetLimitDataList(queryConditions, 2, []string{"-data_time"})
+		if tmpErr != nil {
+			fmt.Println("mogDataObj.GetLimitDataList() Err:" + tmpErr.Error())
+			return
+		}
+		// 如果并没有两条数据，那么就返回
+		if len(tmpDataList) < 2 {
+			return
+		}
+
+		// 实际应该是倒数第二条数据的日期
+		dateStr = tmpDataList[1].DataTime.Format(utils.FormatDate)
+
+		// 删除插入的数据
+		err = mogDataObj.RemoveManyByColl(coll, bson.M{"_id": tmpDataList[0].ID})
+
+		return
+	}
+
+	// 更新配置的数据
+	updateData := bson.M{"$set": bson.M{
+		"value":          floatValue,
+		"modify_time":    time.Now(),
+		"data_time":      newDate,
+		"data_timestamp": timestamp,
+	}}
+	err = mogDataObj.UpdateDataByColl(coll, bson.M{"_id": edbDateData.ID}, updateData)
+
+	return
+}

models/data_manage/edb_info.go

@@ -229,8 +229,11 @@ func DeleteEdbInfoAndData(edbInfoId, source, subSource int) (err error) {
 	}
 
 	// 删除指标的明细数据
-	if source == utils.DATA_SOURCE_BUSINESS {
+	if source == utils.DATA_SOURCE_BUSINESS && utils.UseMongo {
 		err = deleteAllEdbDataByMongo(to, edbInfoId, source, subSource)
+	} else if source == utils.DATA_SOURCE_THS && subSource == utils.DATA_SUB_SOURCE_HIGH_FREQUENCY && utils.UseMongo {
+		mogDataObj := mgo.EdbDataThsHf{}
+		err = mogDataObj.RemoveMany(bson.M{"edb_info_id": edbInfoId})
 	} else {
 		err = deleteAllEdbDataByMysql(to, edbInfoId, source, subSource)
 	}
@@ -578,9 +581,12 @@ type EdbInfoMaxAndMinInfo struct {
 // @return err error
 func GetEdbInfoMaxAndMinInfo(source, subSource int, edbCode string) (item *EdbInfoMaxAndMinInfo, err error) {
 	// 自有数据需要额外处理（从mongo获取）
-	if source == utils.DATA_SOURCE_BUSINESS {
+	if source == utils.DATA_SOURCE_BUSINESS && utils.UseMongo {
 		return GetEdbInfoMaxAndMinInfoByMongo(source, subSource, edbCode)
 	}
+	if source == utils.DATA_SOURCE_THS && subSource == utils.DATA_SUB_SOURCE_HIGH_FREQUENCY && utils.UseMongo {
+		return GetThsHfEdbInfoMaxAndMinInfoByMongo(source, subSource, edbCode)
+	}
 
 	// 默认走mysql
 	return GetEdbInfoMaxAndMinInfoByMysql(source, subSource, edbCode)
@@ -1683,9 +1689,12 @@ func GetEdbCodesBySource(source int) (items []*EdbInfo, err error) {
 // @return err error
 func GetAllEdbDataListData(edbInfoId, source, subSource int, startDataTime string) (dataList []*EdbData, err error) {
 	// 自有数据需要额外处理（从mongo获取）
-	if source == utils.DATA_SOURCE_BUSINESS {
+	if source == utils.DATA_SOURCE_BUSINESS && utils.UseMongo {
 		return getAllDataByMongo(edbInfoId, source, subSource, startDataTime)
 	}
+	if source == utils.DATA_SOURCE_THS && subSource == utils.DATA_SUB_SOURCE_HIGH_FREQUENCY && utils.UseMongo {
+		return getThsHfAllDataByMongo(edbInfoId, source, subSource, startDataTime)
+	}
 
 	// 默认走mysql
 	return getAllDataByMysql(edbInfoId, source, subSource, startDataTime)
@@ -1774,3 +1783,81 @@ type ReplaceEdbInfoItem struct {
 	OldEdbInfo *EdbInfo
 	NewEdbInfo *EdbInfo
 }
+
+func GetThsHfEdbInfoMaxAndMinInfoByMongo(source, subSource int, edbCode string) (item *EdbInfoMaxAndMinInfo, err error) {
+	mogDataObj := new(mgo.EdbDataThsHf)
+	pipeline := []bson.M{
+		{"$match": bson.M{"edb_code": edbCode}},
+		{"$group": bson.M{
+			"_id":       nil,
+			"min_date":  bson.M{"$min": "$data_time"},
+			"max_date":  bson.M{"$max": "$data_time"},
+			"min_value": bson.M{"$min": "$value"},
+			"max_value": bson.M{"$max": "$value"},
+		}},
+		{"$project": bson.M{"_id": 0}}, // 可选，如果不需要_id字段
+	}
+	result, err := mogDataObj.GetEdbInfoMaxAndMinInfo(pipeline)
+	if err != nil {
+		fmt.Println("EdbDataThsHf getEdbDataThsHfList Err:" + err.Error())
+		return
+	}
+
+	if !result.MaxDate.IsZero() {
+		whereQuery := bson.M{"edb_code": edbCode, "data_time": result.MaxDate}
+		selectParam := bson.D{{"value", 1}, {"_id", 0}}
+		latestValue, tmpErr := mogDataObj.GetLatestValue(whereQuery, selectParam)
+		if tmpErr != nil {
+			err = tmpErr
+			return
+		}
+		result.LatestValue = latestValue.Value
+		result.EndValue = latestValue.Value
+	}
+
+	item = &EdbInfoMaxAndMinInfo{
+		MinDate:     result.MinDate.Format(utils.FormatDate),
+		MaxDate:     result.MaxDate.Format(utils.FormatDate),
+		MinValue:    result.MinValue,
+		MaxValue:    result.MaxValue,
+		LatestValue: result.LatestValue,
+	}
+
+	return
+}
+
+func getThsHfAllDataByMongo(edbInfoId, source, subSource int, startDataTime string) (dataList []*EdbData, err error) {
+	dataList = make([]*EdbData, 0)
+
+	mogDataObj := mgo.EdbDataThsHf{}
+	// 构建查询条件
+	queryConditions := bson.M{
+		"edb_info_id": edbInfoId,
+	}
+
+	// 开始日期
+	dateCondition, err := mgo.BuildDateCondition(startDataTime, "")
+	if err != nil {
+		return
+	}
+	if len(dateCondition) > 0 {
+		queryConditions["data_time"] = dateCondition
+	}
+
+	// 获取列表数据
+	tmpDataList, tmpErr := mogDataObj.GetAllDataList(queryConditions, []string{"-data_time"})
+	if tmpErr != nil {
+		err = tmpErr
+		return
+	}
+	for k, v := range tmpDataList {
+		dataList = append(dataList, &EdbData{
+			EdbDataId: k + 1,
+			EdbInfoId: v.EdbInfoId,
+			DataTime:  v.DataTime.Format(utils.FormatDate),
+			Value:     v.Value,
+		})
+	}
+
+	return
+}

models/db.go

@@ -196,7 +196,7 @@ func init() {
 	initFactorEdbSeries()
 
 	// 初始化部分数据表变量(直接init会有顺序问题=_=!)
-	data_manage.InitEdbSourceVar()
+	afterInitTable()
 }
 
 // initSystem 系统表 数据表
@@ -600,3 +600,15 @@ func initFactorEdbSeries() {
 		new(data_manage.FactorEdbSeriesCalculateData), // 因子指标系列-指标关联
 	)
 }
+
+// afterInitTable
+// @Description: 初始化表结构的的后置操作
+// @author: Roc
+// @datetime 2024-07-01 13:31:09
+func afterInitTable() {
+	// 初始化指标来源配置
+	data_manage.InitEdbSourceVar()
+
+	// 初始化是否启用mongo配置
+	InitUseMongoConf()
+}

models/mgo/edb_data_ths_hf.go

@@ -0,0 +1,510 @@
+package mgo
+
+import (
+	"context"
+	"errors"
+	"eta/eta_mobile/utils"
+	"fmt"
+	"github.com/qiniu/qmgo"
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+	"time"
+)
+
+// EdbDataThsHf
+// @Description: 同花顺高频集合（指标库）
+type EdbDataThsHf struct {
+	ID            primitive.ObjectID `json:"_id" bson:"_id,omitempty" `            // 文档id
+	EdbInfoId     int                `json:"edb_info_id" bson:"edb_info_id"`       // 指标ID
+	EdbCode       string             `json:"edb_code" bson:"edb_code"`             // 指标编码
+	DataTime      time.Time          `json:"data_time" bson:"data_time"`           // 数据日期
+	Value         float64            `json:"value" bson:"value"`                   // 数据值
+	CreateTime    time.Time          `json:"create_time" bson:"create_time"`       // 创建时间
+	ModifyTime    time.Time          `json:"modify_time" bson:"modify_time"`       // 修改时间
+	DataTimestamp int64              `json:"data_timestamp" bson:"data_timestamp"` // 数据日期时间戳
+}
+
+// CollectionName
+// @Description:  获取集合名称
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-26 13:41:36
+// @return string
+func (m *EdbDataThsHf) CollectionName() string {
+	return "edb_data_ths_hf"
+}
+
+// DataBaseName
+// @Description: 获取数据库名称
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-26 13:41:33
+// @return string
+func (m *EdbDataThsHf) DataBaseName() string {
+	return utils.MgoDataDbName
+}
+
+// GetCollection
+// @Description: 获取mongodb集合的句柄
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-26 13:41:33
+// @return string
+func (m *EdbDataThsHf) GetCollection() *qmgo.Collection {
+	db := utils.MgoDataCli.Database(m.DataBaseName())
+	return db.Collection(m.CollectionName())
+}
+
+// GetItem
+// @Description: 根据条件获取单条数据
+// @author: Roc
+// @receiver m
+// @datetime 2024-05-09 10:00:49
+// @param whereParams interface{}
+// @return item *EdbDataThsHf
+// @return err error
+func (m *EdbDataThsHf) GetItem(whereParams interface{}) (item *EdbDataThsHf, err error) {
+	if utils.MgoDataCli == nil {
+		err = errors.New("mongodb连接失败")
+		return
+	}
+	db := utils.MgoDataCli.Database(m.DataBaseName())
+	coll := db.Collection(m.CollectionName())
+
+	return m.GetItemByColl(coll, whereParams)
+}
+
+// GetItemByColl
+// @Description: 根据条件获取单条数据
+// @author: Roc
+// @receiver m
+// @datetime 2024-05-09 13:22:06
+// @param coll *qmgo.Collection
+// @param whereParams interface{}
+// @return item *EdbDataThsHf
+// @return err error
+func (m *EdbDataThsHf) GetItemByColl(coll *qmgo.Collection, whereParams interface{}) (item *EdbDataThsHf, err error) {
+	ctx := context.TODO()
+	if err != nil {
+		fmt.Println("MgoGetColl Err:", err.Error())
+		return
+	}
+	err = coll.Find(ctx, whereParams).One(&item)
+	if err != nil {
+		return
+	}
+
+	item.DataTime = item.DataTime.In(time.Local)
+	item.CreateTime = item.CreateTime.In(time.Local)
+	item.ModifyTime = item.ModifyTime.In(time.Local)
+
+	return
+}
+
+// GetAllDataList
+// @Description: 根据条件获取所有数据
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-26 13:42:19
+// @param whereParams interface{}
+// @param sort []string
+// @return result []EdbDataThsHf
+// @return err error
+func (m *EdbDataThsHf) GetAllDataList(whereParams interface{}, sort []string) (result []*EdbDataThsHf, err error) {
+	if utils.MgoDataCli == nil {
+		err = errors.New("mongodb连接失败")
+		return
+	}
+	db := utils.MgoDataCli.Database(m.DataBaseName())
+	coll := db.Collection(m.CollectionName())
+	ctx := context.TODO()
+	if err != nil {
+		fmt.Println("MgoGetColl Err:", err.Error())
+		return
+	}
+	err = coll.Find(ctx, whereParams).Sort(sort...).All(&result)
+	if err != nil {
+		return
+	}
+
+	for _, v := range result {
+		v.DataTime = v.DataTime.In(time.Local)
+		v.CreateTime = v.CreateTime.In(time.Local)
+		v.ModifyTime = v.ModifyTime.In(time.Local)
+	}
+
+	return
+}
+
+// GetLimitDataList
+// @Description: 根据条件获取指定数量数据列表
+// @author: Roc
+// @receiver m
+// @datetime 2024-05-06 17:08:32
+// @param whereParams interface{}
+// @param size int64
+// @param sort []string
+// @return result []*BaseFromBusinessData
+// @return err error
+func (m *EdbDataThsHf) GetLimitDataList(whereParams interface{}, size int64, sort []string) (result []*EdbDataThsHf, err error) {
+	if utils.MgoDataCli == nil {
+		err = errors.New("mongodb连接失败")
+		return
+	}
+	db := utils.MgoDataCli.Database(m.DataBaseName())
+	coll := db.Collection(m.CollectionName())
+	ctx := context.TODO()
+	if err != nil {
+		fmt.Println("MgoGetColl Err:", err.Error())
+		return
+	}
+	err = coll.Find(ctx, whereParams).Sort(sort...).Limit(size).All(&result)
+	if err != nil {
+		return
+	}
+
+	for _, v := range result {
+		v.DataTime = v.DataTime.In(time.Local)
+		v.CreateTime = v.CreateTime.In(time.Local)
+		v.ModifyTime = v.ModifyTime.In(time.Local)
+	}
+
+	return
+}
+
+// GetPageDataList
+// @Description: 根据条件获取分页数据列表
+// @author: Roc
+// @receiver m
+// @datetime 2024-05-07 10:21:07
+// @param whereParams interface{}
+// @param startSize int64
+// @param size int64
+// @param sort []string
+// @return result []*EdbDataThsHf
+// @return err error
+func (m *EdbDataThsHf) GetPageDataList(whereParams interface{}, startSize, size int64, sort []string) (result []*EdbDataThsHf, err error) {
+	if utils.MgoDataCli == nil {
+		err = errors.New("mongodb连接失败")
+		return
+	}
+	db := utils.MgoDataCli.Database(m.DataBaseName())
+	coll := db.Collection(m.CollectionName())
+	ctx := context.TODO()
+	if err != nil {
+		fmt.Println("MgoGetColl Err:", err.Error())
+		return
+	}
+	err = coll.Find(ctx, whereParams).Sort(sort...).Skip(startSize).Limit(size).All(&result)
+	if err != nil {
+		return
+	}
+
+	for _, v := range result {
+		v.DataTime = v.DataTime.In(time.Local)
+		v.CreateTime = v.CreateTime.In(time.Local)
+		v.ModifyTime = v.ModifyTime.In(time.Local)
+	}
+
+	return
+}
+
+// GetCountDataList
+// @Description:  根据条件获取数据列表总数
+// @author: Roc
+// @receiver m
+// @datetime 2024-05-07 10:29:00
+// @param whereParams interface{}
+// @return count int64
+// @return err error
+func (m *EdbDataThsHf) GetCountDataList(whereParams interface{}) (count int64, err error) {
+	if utils.MgoDataCli == nil {
+		err = errors.New("mongodb连接失败")
+		return
+	}
+	db := utils.MgoDataCli.Database(m.DataBaseName())
+	coll := db.Collection(m.CollectionName())
+	ctx := context.TODO()
+	if err != nil {
+		fmt.Println("MgoGetColl Err:", err.Error())
+		return
+	}
+	count, err = coll.Find(ctx, whereParams).Count()
+
+	return
+}
+
+// InsertDataByColl
+// @Description: 写入单条数据（外部传入集合）
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-26 14:22:18
+// @param addData interface{}
+// @return err error
+func (m *EdbDataThsHf) InsertDataByColl(coll *qmgo.Collection, addData interface{}) (err error) {
+	ctx := context.TODO()
+	_, err = coll.InsertOne(ctx, addData)
+	if err != nil {
+		fmt.Println("InsertDataByColl:Err:" + err.Error())
+		return
+	}
+
+	return
+}
+
+// BatchInsertData
+// @Description: 批量写入数据
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-26 14:22:18
+// @param bulk int 每次请求保存的数据量
+// @param dataList []interface{}
+// @return err error
+func (m *EdbDataThsHf) BatchInsertData(bulk int, dataList []interface{}) (err error) {
+	db := utils.MgoDataCli.Database(m.DataBaseName())
+	coll := db.Collection(m.CollectionName())
+
+	return m.BatchInsertDataByColl(coll, bulk, dataList)
+}
+
+// BatchInsertDataByColl
+// @Description: 批量写入数据（外部传入集合）
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-26 14:22:18
+// @param coll *qmgo.Collection
+// @param bulk int 每次请求保存的数据量
+// @param dataList []interface{}
+// @return err error
+func (m *EdbDataThsHf) BatchInsertDataByColl(coll *qmgo.Collection, bulk int, dataList []interface{}) (err error) {
+	ctx := context.TODO()
+	dataNum := len(dataList)
+	if dataNum <= 0 {
+		return
+	}
+
+	// 不设置每次保存切片数量大小，或者实际数据量小于设置的切片数量大小，那么就直接保存吧
+	if bulk <= 0 || dataNum <= bulk {
+		_, err = coll.InsertMany(ctx, dataList)
+		if err != nil {
+			fmt.Println("BatchInsertData:Err:" + err.Error())
+			return
+		}
+		return
+	}
+
+	// 分批保存
+	i := 0
+	tmpAddDataList := make([]interface{}, 0)
+	for _, v := range dataList {
+		tmpAddDataList = append(tmpAddDataList, v)
+		i++
+		if i >= bulk {
+			_, err = coll.InsertMany(ctx, tmpAddDataList)
+			if err != nil {
+				fmt.Println("BatchInsertData:Err:" + err.Error())
+				return
+			}
+			i = 0
+			tmpAddDataList = make([]interface{}, 0)
+		}
+	}
+
+	if len(tmpAddDataList) > 0 {
+		_, err = coll.InsertMany(ctx, tmpAddDataList)
+		if err != nil {
+			fmt.Println("BatchInsertData:Err:" + err.Error())
+			return
+		}
+	}
+
+	return
+}
+
+// UpdateData
+// @Description: 单条数据修改
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-26 15:01:51
+// @param whereParams interface{}
+// @param updateParams interface{}
+// @return err error
+func (m *EdbDataThsHf) UpdateData(whereParams, updateParams interface{}) (err error) {
+	db := utils.MgoDataCli.Database(m.DataBaseName())
+	coll := db.Collection(m.CollectionName())
+
+	return m.UpdateDataByColl(coll, whereParams, updateParams)
+}
+
+// UpdateDataByColl
+// @Description: 单条数据修改（外部传入集合）
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-26 15:01:51
+// @param whereParams interface{}
+// @param updateParams interface{}
+// @return err error
+func (m *EdbDataThsHf) UpdateDataByColl(coll *qmgo.Collection, whereParams, updateParams interface{}) (err error) {
+	ctx := context.TODO()
+	err = coll.UpdateOne(ctx, whereParams, updateParams)
+	if err != nil {
+		fmt.Println("UpdateDataByColl:Err:" + err.Error())
+		return
+	}
+
+	return
+}
+
+// RemoveMany
+// @Description: 根据条件删除多条数据
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-30 13:17:02
+// @param whereParams interface{}
+// @return err error
+func (m *EdbDataThsHf) RemoveMany(whereParams interface{}) (err error) {
+	db := utils.MgoDataCli.Database(m.DataBaseName())
+	coll := db.Collection(m.CollectionName())
+
+	return m.RemoveManyByColl(coll, whereParams)
+}
+
+// RemoveManyByColl
+// @Description: 根据条件删除多条数据（外部传入集合）
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-30 13:18:42
+// @param coll *qmgo.Collection
+// @param whereParams interface{}
+// @return err error
+func (m *EdbDataThsHf) RemoveManyByColl(coll *qmgo.Collection, whereParams interface{}) (err error) {
+	ctx := context.TODO()
+	_, err = coll.RemoveAll(ctx, whereParams)
+	if err != nil {
+		fmt.Println("RemoveManyByColl:Err:" + err.Error())
+		return
+	}
+
+	return
+}
+
+// HandleData
+// @Description: 事务处理数据
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-30 10:39:01
+// @param addDataList []AddEdbDataThsHf
+// @param updateDataList []EdbDataThsHf
+// @return result interface{}
+// @return err error
+func (m *EdbDataThsHf) HandleData(addDataList, updateDataList []EdbDataThsHf) (result interface{}, err error) {
+
+	ctx := context.TODO()
+
+	callback := func(sessCtx context.Context) (interface{}, error) {
+		// 重要：确保事务中的每一个操作，都使用传入的sessCtx参数
+
+		db := utils.MgoDataCli.Database(m.DataBaseName())
+		coll := db.Collection(m.CollectionName())
+
+		// 插入数据
+		if len(addDataList) > 0 {
+			_, err = coll.InsertMany(sessCtx, addDataList)
+			if err != nil {
+				return nil, err
+			}
+		}
+
+		// 修改
+
+		if len(updateDataList) > 0 {
+			for _, v := range updateDataList {
+				err = coll.UpdateOne(ctx, bson.M{"_id": v.ID}, bson.M{"$set": bson.M{"value": v.Value, "modify_time": v.ModifyTime}})
+				if err != nil {
+					fmt.Println("BatchInsertData:Err:" + err.Error())
+					return nil, err
+				}
+			}
+		}
+
+		return nil, nil
+	}
+	result, err = utils.MgoDataCli.DoTransaction(ctx, callback)
+
+	return
+}
+
+// EdbInfoMaxAndMinInfo 指标最新数据记录结构体
+//type EdbInfoMaxAndMinInfo struct {
+//	MinDate     time.Time `description:"最小日期" bson:"min_date"`
+//	MaxDate     time.Time `description:"最大日期" bson:"max_date"`
+//	MinValue    float64   `description:"最小值" bson:"min_value"`
+//	MaxValue    float64   `description:"最大值" bson:"max_value"`
+//	LatestValue float64   `description:"最新值" bson:"latest_value"`
+//	LatestDate  time.Time `description:"实际数据最新日期" bson:"latest_date"`
+//	EndValue    float64   `description:"最新值" bson:"end_value"`
+//}
+
+// GetEdbInfoMaxAndMinInfo
+// @Description: 获取当前指标的最大最小值
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-30 17:15:39
+// @param whereParams interface{}
+// @return result EdbInfoMaxAndMinInfo
+// @return err error
+func (m *EdbDataThsHf) GetEdbInfoMaxAndMinInfo(whereParams interface{}) (result EdbInfoMaxAndMinInfo, err error) {
+	if utils.MgoDataCli == nil {
+		err = errors.New("mongodb连接失败")
+		return
+	}
+	db := utils.MgoDataCli.Database(m.DataBaseName())
+	coll := db.Collection(m.CollectionName())
+	ctx := context.TODO()
+	if err != nil {
+		fmt.Println("MgoGetColl Err:", err.Error())
+		return
+	}
+	err = coll.Aggregate(ctx, whereParams).One(&result)
+	if err != nil {
+		return
+	}
+	result.MinDate = result.MinDate.In(time.Local)
+	result.MaxDate = result.MaxDate.In(time.Local)
+	result.LatestDate = result.LatestDate.In(time.Local)
+
+	return
+}
+
+// LatestValue 指标最新数据记录结构体
+//type LatestValue struct {
+//	Value float64 `description:"值" bson:"value"`
+//}
+
+// GetLatestValue
+// @Description: 获取当前指标的最新数据记录
+// @author: Roc
+// @receiver m
+// @datetime 2024-04-30 17:16:15
+// @param whereParams interface{}
+// @param selectParam interface{}
+// @return latestValue LatestValue
+// @return err error
+func (m *EdbDataThsHf) GetLatestValue(whereParams, selectParam interface{}) (latestValue LatestValue, err error) {
+	if utils.MgoDataCli == nil {
+		err = errors.New("mongodb连接失败")
+		return
+	}
+	db := utils.MgoDataCli.Database(m.DataBaseName())
+	coll := db.Collection(m.CollectionName())
+	ctx := context.TODO()
+	if err != nil {
+		fmt.Println("MgoGetColl Err:", err.Error())
+		return
+	}
+
+	//var result interface{}
+	//err = coll.Find(ctx, whereParams).Select(selectParam).One(&result)
+	err = coll.Find(ctx, whereParams).Select(selectParam).One(&latestValue)
+	return
+}

services/data/base_edb_lib.go

@@ -203,11 +203,13 @@ func RefreshEdbData(edbInfoId, source, subSource int, edbCode, startDate string)
 	urlStr := ``
 	switch source {
 	case utils.DATA_SOURCE_THS:
-		urlStr = "ths/refresh"
-		if subSource == 0 {
-			urlStr = "ths/refresh"
-		} else {
+		switch subSource {
+		case utils.DATA_SUB_SOURCE_DATE:
 			urlStr = "ths/ds/refresh"
+		case utils.DATA_SUB_SOURCE_HIGH_FREQUENCY:
+			urlStr = "ths/hf/edb/refresh"
+		default:
+			urlStr = "ths/refresh"
 		}
 	case utils.DATA_SOURCE_WIND:
 		if subSource == 0 {

services/data/base_from_business.go

@@ -0,0 +1,172 @@
+package data
+
+import (
+	"eta/eta_mobile/models/data_manage"
+	"eta/eta_mobile/models/mgo"
+	"eta/eta_mobile/utils"
+	"fmt"
+	"go.mongodb.org/mongo-driver/bson"
+)
+
+type BusinessIndexSource2EdbReq struct {
+	EdbCode       string
+	EdbName       string
+	Frequency     string
+	Unit          string
+	ClassifyId    int
+	AdminId       int
+	AdminRealName string
+}
+
+// BusinessIndexSource2Edb 新增自有数据源到指标库
+func BusinessIndexSource2Edb(req BusinessIndexSource2EdbReq, lang string) (edb *data_manage.EdbInfo, err error, errMsg string, skip bool) {
+	if req.EdbCode == "" {
+		err = fmt.Errorf("指标ID为空")
+		return
+	}
+	defer func() {
+		if err != nil {
+			tips := fmt.Sprintf("BusinessIndexSource2Edb新增失败, Err: %s", err.Error())
+			fmt.Println(tips)
+			utils.FileLog.Info(tips)
+		}
+	}()
+	source := utils.DATA_SOURCE_BUSINESS
+
+	// 是否新增过指标
+	exist, e := data_manage.GetEdbInfoByEdbCode(source, req.EdbCode)
+	if e != nil && e.Error() != utils.ErrNoRow() {
+		err = fmt.Errorf("获取指标是否存在失败, err: %s", e.Error())
+		return
+	}
+	if exist != nil {
+		skip = true
+		return
+	}
+
+	// 开始结束时间
+	var startDate, endDate string
+
+	// 新增指标库
+	edbInfo, e, msg, _ := EdbInfoAdd(source, utils.DATA_SUB_SOURCE_EDB, req.ClassifyId, req.EdbCode, req.EdbName, req.Frequency, req.Unit, startDate, endDate, req.AdminId, req.AdminRealName, lang)
+	if e != nil {
+		errMsg = msg
+		err = fmt.Errorf("EdbInfo: 新增指标失败, err: %s", e.Error())
+		return
+	}
+
+	edb = edbInfo
+
+	return
+}
+
+// GetPageBaseBusinessIndexData
+// @Description: 获取自有数据的分页数据
+// @author: Roc
+// @datetime 2024-05-07 11:02:51
+// @param indexCode int
+// @param startSize int
+// @param pageSize int
+// @return dataCount int
+// @return dataList []*data_manage.BaseFromBusinessIndexDataItem
+// @return err error
+func GetPageBaseBusinessIndexData(indexCode string, startSize, pageSize int) (dataCount int, dataList []*data_manage.BaseFromBusinessIndexDataItem, err error) {
+	if utils.UseMongo {
+		dataCount, dataList, err = getPageBaseBusinessIndexDataByMongo(indexCode, startSize, pageSize)
+	} else {
+		dataCount, dataList, err = getPageBaseBusinessIndexDataByMysql(indexCode, startSize, pageSize)
+	}
+
+	return
+}
+
+// getPageBaseBusinessIndexDataByMongo
+// @Description: 获取自有数据的分页数据（从mongo中获取）
+// @author: Roc
+// @datetime 2024-07-01 14:01:04
+// @param indexCode string
+// @param startSize int
+// @param pageSize int
+// @return dataCount int
+// @return dataList []*data_manage.BaseFromBusinessIndexDataItem
+// @return err error
+func getPageBaseBusinessIndexDataByMongo(indexCode string, startSize, pageSize int) (dataCount int, dataList []*data_manage.BaseFromBusinessIndexDataItem, err error) {
+	dataList = make([]*data_manage.BaseFromBusinessIndexDataItem, 0)
+
+	mogDataObj := mgo.BaseFromBusinessData{}
+	// 构建查询条件
+	queryConditions := bson.M{
+		"index_code": indexCode,
+	}
+
+	// 获取数据总量
+	tmpCount, tmpErr := mogDataObj.GetCountDataList(queryConditions)
+	if tmpErr != nil && tmpErr.Error() != utils.ErrNoRow() {
+		err = tmpErr
+		return
+	}
+	dataCount = int(tmpCount)
+
+	// 获取列表数据
+	tmpDataList, tmpErr := mogDataObj.GetPageDataList(queryConditions, int64(startSize), int64(pageSize), []string{"-data_time"})
+	if tmpErr != nil {
+		err = tmpErr
+		return
+	}
+	for _, v := range tmpDataList {
+		dataList = append(dataList, &data_manage.BaseFromBusinessIndexDataItem{
+			ID:        v.ID.String(),
+			IndexCode: v.IndexCode,
+			DataTime:  v.DataTime.Format(utils.FormatDate),
+			Value:     v.Value,
+		})
+	}
+
+	return
+}
+
+// getPageBaseBusinessIndexDataByMysql
+// @Description: 获取自有数据的分页数据（从mysql中获取）
+// @author: Roc
+// @datetime 2024-07-01 14:00:41
+// @param indexCode string
+// @param startSize int
+// @param pageSize int
+// @return dataCount int
+// @return dataList []*data_manage.BaseFromBusinessIndexDataItem
+// @return err error
+func getPageBaseBusinessIndexDataByMysql(indexCode string, startSize, pageSize int) (dataCount int, dataList []*data_manage.BaseFromBusinessIndexDataItem, err error) {
+	dataList = make([]*data_manage.BaseFromBusinessIndexDataItem, 0)
+
+	businessDataObj := data_manage.BaseFromBusinessData{}
+	var condition []string
+	var pars []interface{}
+
+	condition = append(condition, "index_code = ?")
+	pars = append(pars, indexCode)
+
+	// 获取数据总量
+	tmpCount, tmpErr := businessDataObj.GetCountDataList(condition, pars)
+	if tmpErr != nil && tmpErr.Error() != utils.ErrNoRow() {
+		err = tmpErr
+		return
+	}
+	dataCount = int(tmpCount)
+
+	// 获取列表数据
+	tmpDataList, tmpErr := businessDataObj.GetPageDataList(condition, pars, " data_time desc", int64(startSize), int64(pageSize))
+	if tmpErr != nil {
+		err = tmpErr
+		return
+	}
+	for _, v := range tmpDataList {
+		dataList = append(dataList, &data_manage.BaseFromBusinessIndexDataItem{
+			ID:        fmt.Sprint(v.BusinessDataId),
+			IndexCode: v.IndexCode,
+			DataTime:  v.DataTime.Format(utils.FormatDate),
+			Value:     v.Value,
+		})
+	}
+
+	return
+}

services/data/edb_data.go

@@ -409,9 +409,12 @@ func GetPageData(edbInfoId, source, subSource int, endDataTime string, startSize
 	dataList = make([]*data_manage.EdbData, 0)
 
 	// 自有数据需要额外处理（从mongo获取）
-	if source == utils.DATA_SOURCE_BUSINESS {
+	if source == utils.DATA_SOURCE_BUSINESS && utils.UseMongo {
 		return getPageDataByMongo(edbInfoId, source, subSource, endDataTime, startSize, pageSize)
 	}
+	if source == utils.DATA_SOURCE_THS && subSource == utils.DATA_SUB_SOURCE_HIGH_FREQUENCY && utils.UseMongo {
+		return getThsHfPageDataByMongo(edbInfoId, source, subSource, endDataTime, startSize, pageSize)
+	}
 
 	// 默认走mysql
 	return getPageDataByMysql(edbInfoId, source, subSource, endDataTime, startSize, pageSize)
@@ -514,3 +517,47 @@ func getPageDataByMongo(edbInfoId, source, subSource int, endDataTime string, st
 
 	return
 }
+
+func getThsHfPageDataByMongo(edbInfoId, source, subSource int, endDataTime string, startSize, pageSize int) (dataCount int, dataList []*data_manage.EdbData, err error) {
+	dataList = make([]*data_manage.EdbData, 0)
+
+	mogDataObj := mgo.EdbDataThsHf{}
+	// 构建查询条件
+	queryConditions := bson.M{
+		"edb_info_id": edbInfoId,
+	}
+
+	// 结束日期
+	dateCondition, err := mgo.BuildDateCondition("", endDataTime)
+	if err != nil {
+		return
+	}
+	if len(dateCondition) > 0 {
+		queryConditions["data_time"] = dateCondition
+	}
+
+	// 获取数据总量
+	tmpCount, tmpErr := mogDataObj.GetCountDataList(queryConditions)
+	if tmpErr != nil && tmpErr.Error() != utils.ErrNoRow() {
+		err = tmpErr
+		return
+	}
+	dataCount = int(tmpCount)
+
+	// 获取列表数据
+	tmpDataList, tmpErr := mogDataObj.GetPageDataList(queryConditions, int64(startSize), int64(pageSize), []string{"-data_time"})
+	if tmpErr != nil {
+		err = tmpErr
+		return
+	}
+	for k, v := range tmpDataList {
+		dataList = append(dataList, &data_manage.EdbData{
+			EdbDataId: k + 1,
+			EdbInfoId: v.EdbInfoId,
+			DataTime:  v.DataTime.Format(utils.FormatDate),
+			Value:     v.Value,
+		})
+	}
+
+	return
+}

services/data/edb_info.go

@@ -103,8 +103,11 @@ func EdbInfoRefreshAllFromBaseV3(edbInfoIdList []int, refreshAll, isSync, isRefr
 	}
 
 	// 需要刷新的指标数量
-	totalEdbInfo := len(newBaseEdbInfoArr) + len(calculateArr) + len(predictCalculateArr)
-
+	// totalEdbInfo := len(newBaseEdbInfoArr) + len(calculateArr) + len(predictCalculateArr)
+	totalEdbInfo := len(newBaseEdbInfoArr) + len(calculateArr) + len(predictCalculateArr) + len(newBasePredictEdbInfoArr)
+	if totalEdbInfo == 0 {
+		return
+	}
 	if totalEdbInfo <= 20 || isSync {
 		err = edbInfoRefreshAll(refreshAll, newBaseEdbInfoArr, newBasePredictEdbInfoArr, newCalculateMap, newPredictCalculateMap, calculateArr, predictCalculateArr)
 	} else {
@@ -173,6 +176,9 @@ func getEdbInfoIdList(edbInfoIdList []int) (newBaseEdbInfoArr, newBasePredictEdb
 		// 普通计算指标
 		for _, edbInfo := range tmpCalculateMap {
 			if _, ok := newCalculateMap[edbInfo.EdbInfoId]; !ok {
+				if edbInfo.NoUpdate == 1 {
+					continue
+				}
 				newCalculateMap[edbInfo.EdbInfoId] = edbInfo
 				calculateArr = append(calculateArr, edbInfo.EdbInfoId)
 			}

utils/config.go

@@ -200,7 +200,11 @@ var (
 
 // chrome配置
 var (
-	ChromePath       string // chrome浏览器地址
+	ChromePath string // chrome浏览器地址
+)
+
+var (
+	UseMongo bool // 是否使用mongo
 )
 
 func init() {

utils/constants.go

@@ -404,8 +404,9 @@ const (
 
 // 子数据来源渠道
 const (
-	DATA_SUB_SOURCE_EDB  = iota //经济数据库
-	DATA_SUB_SOURCE_DATE        //日期序列
+	DATA_SUB_SOURCE_EDB            = iota //经济数据库
+	DATA_SUB_SOURCE_DATE                  //日期序列
+	DATA_SUB_SOURCE_HIGH_FREQUENCY        //高频数据
 )
 
 const (