新增数据管理相关功能并优化数据源处理逻辑

- 在 EdbClassify 和 EdbInfo 模型中添加新增和更新操作- 实现通过来源 ID 和名称获取数据源的功能
- 优化数据源信息的缓存处理，提高查询效率
- 调整数据刷新和桥接服务的逻辑，增加日志记录
- 在任务函数中添加桥接服务同步功能
- 更新常量定义，增加数据前缀和指标类型相关常量
- 在主函数中添加表数据初始化操作

main.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"eta_gn/eta_task/models"
 	_ "eta_gn/eta_task/routers"
 	"eta_gn/eta_task/services"
 	"eta_gn/eta_task/services/alarm_msg"
@@ -19,6 +20,9 @@ func main() {
 		web.BConfig.WebConfig.DirectoryIndex = true
 		web.BConfig.WebConfig.StaticDir["/swagger"] = "swagger"
 	}
+	// 初始化表数据入库
+	models.AfterInitTable()
+
 	go services.Task()
 	//services.TaskInit()
 	web.BConfig.RecoverFunc = Recover

models/data_manage/edb_classify.go

@@ -6,21 +6,23 @@ import (
 )
 
 type EdbClassify struct {
-	ClassifyID         int64 `gorm:"column:classify_id;primaryKey"` //`orm:"column(classify_id);pk"`
-	ClassifyType       uint32
-	ClassifyName       string
-	ParentID           int64
-	HasData            int32
-	CreateTime         time.Time
-	ModifyTime         time.Time
-	SysUserID          int64
-	SysUserRealName    string
-	Level              int64
-	UniqueCode         string
-	Sort               uint32
-	RootID             int64
-	LastModifyUserID   uint32
-	LastModifyUserName string
+	ClassifyID             int64 `gorm:"column:classify_id;primaryKey"` //`orm:"column(classify_id);pk"`
+	ClassifyType           uint32
+	ClassifyName           string
+	ParentID               int64
+	HasData                int32
+	CreateTime             time.Time
+	ModifyTime             time.Time
+	SysUserID              int64
+	SysUserRealName        string
+	Level                  int64
+	UniqueCode             string
+	Sort                   uint32
+	RootID                 int64
+	LastModifyUserID       uint32
+	LastModifyUserRealName string
+	IsJoinPermission       int64
+	ClassifyNameEn         string
 }
 
 // GetAllEdbClassifyListByCondition
@@ -42,3 +44,26 @@ func GetAllEdbClassifyListByCondition(condition string, pars []interface{}) (ite
 	err = global.DmSQL["data"].Raw(sql, pars...).Find(&item).Error
 	return
 }
+
+// AddEdbClassify
+// @Description: 新增分类
+// @param item
+// @return err
+func AddEdbClassify(item *EdbClassify) (err error) {
+	err = global.DmSQL["data"].Create(item).Error
+	if err != nil {
+		return
+	}
+
+	return
+}
+
+// Update
+// @Description: 更新指定参数
+// @receiver m
+// @param cols
+// @return err
+func (m *EdbClassify) Update(cols []string) (err error) {
+	err = global.DmSQL["data"].Select(cols).Updates(m).Error
+	return
+}

models/data_manage/edb_data_base.go

@@ -165,7 +165,7 @@ func GetEdbDataTableName(source, subSource int) (tableName string) {
 	case utils.DATA_SOURCE_CALCULATE_RJZ: //日均值75
 		tableName = "edb_data_calculate_rjz"
 	default:
-		edbSource := EdbSourceIdMap[source]
+		edbSource := GetEdbSourceBySourceId(source)
 		if edbSource != nil {
 			tableName = edbSource.TableName
 		}
@@ -360,7 +360,7 @@ func GetEdbInfoCalculateTableName(source int) (tableName string) {
 	case utils.DATA_SOURCE_CALCULATE_BP:
 		tableName = "edb_info_calculate_bp"
 	default:
-		tableName = EdbDataTableNameMap[source] // 没有对应的从edb_source中取
+		tableName = GetEdbSourceTableNameBySourceId(source) // 没有对应的从edb_source中取
 		return
 	}
 	return

models/data_manage/edb_info.go

@@ -10,28 +10,55 @@ import (
 )
 
 type EdbInfo struct {
-	EdbInfoId        int    `gorm:"column:edb_info_id;primaryKey"` //`orm:"column(edb_info_id);pk"`
-	SourceName       string `description:"来源名称"`
-	Source           int    `description:"来源id"`
-	EdbCode          string `description:"指标编码"`
-	EdbName          string `description:"指标名称"`
-	EdbNameSource    string `description:"指标名称来源"`
-	Frequency        string `description:"频率"`
-	Unit             string `description:"单位"`
-	StartDate        string `description:"起始日期"`
-	EndDate          string `description:"终止日期"`
-	ClassifyId       int    `description:"分类id"`
+	EdbInfoId        int       `gorm:"column:edb_info_id;primaryKey"` //`orm:"column(edb_info_id);pk"`
+	EdbInfoType      int       `description:"指标类型，0：普通指标，1：预测指标"`
+	SourceName       string    `description:"来源名称"`
+	Source           int       `description:"来源id"`
+	EdbCode          string    `description:"指标编码"`
+	EdbName          string    `description:"指标名称"`
+	EdbNameEn        string    `description:"英文指标名称"`
+	EdbNameSource    string    `description:"指标名称来源"`
+	Frequency        string    `description:"频率"`
+	Unit             string    `description:"单位"`
+	UnitEn           string    `description:"英文单位"`
+	StartDate        time.Time `description:"起始日期"`
+	EndDate          time.Time `description:"终止日期"`
+	ClassifyId       int       `description:"分类id"`
 	SysUserId        int
 	SysUserRealName  string
 	UniqueCode       string `description:"指标唯一编码"`
 	CreateTime       time.Time
 	ModifyTime       time.Time
+	BaseModifyTime   time.Time
 	MinValue         float64 `description:"指标最小值"`
 	MaxValue         float64 `description:"指标最大值"`
 	CalculateFormula string  `description:"计算公式"`
-	NoUpdate         int8    `description:"是否停止更新，0：继续更新；1：停止更新"`
-	EdbInfoType      int     `description:"指标类型，0：普通指标，1：预测指标"`
 	EdbType          int     `description:"指标类型:1:基础指标,2:计算指标"`
+	Sort             int     `description:"排序字段"`
+	LatestDate       string  `description:"数据最新日期（实际日期）"`
+	LatestValue      float64 `description:"数据最新值（实际值）"`
+	EndValue         float64 `description:"数据的最新值（预测日期的最新值）"`
+	MoveType         int     `description:"移动方式:1:领先(默认),2:滞后"`
+	MoveFrequency    string  `description:"移动频度"`
+	NoUpdate         int8    `description:"是否停止更新，0：继续更新；1：停止更新"`
+	ServerUrl        string  `description:"服务器地址"`
+	ChartImage       string  `description:"图表图片"`
+	Calendar         string  `description:"公历/农历" orm:"default(公历);"`
+	DataDateType     string  `orm:"column(data_date_type);size(255);null;default(交易日)"`
+	ManualSave       int     `description:"是否有手动保存过上下限: 0-否; 1-是"`
+	EmptyType        int     `description:"空值处理类型（0查找前后35天，1不计算，2前值填充，3后值填充，4等于0）"`
+	MaxEmptyType     int     `description:"MAX、MIN公式空值处理类型（1、等于0；2、跳过空值）"`
+	TerminalCode     string  `description:"终端编码，用于配置在机器上"`
+	DataUpdateTime   string  `description:"最近一次数据发生变化的时间"`
+	ErDataUpdateDate string  `description:"本次更新，数据发生变化的最早日期"`
+	SourceIndexName  string  `description:"数据源中的指标名称"`
+	SubSource        int     `description:"子数据来源:0:经济数据库,1:日期序列"`
+	SubSourceName    string  `description:"子数据来源名称"`
+	IndicatorCode    string  `description:"指标代码"`
+	StockCode        string  `description:"证券代码"`
+	Extra            string  `description:"指标额外配置"`
+	IsJoinPermission int     `description:"是否加入权限管控，0：不加入；1：加入；默认：0"`
+	OriginalEdbCode  string  `description:"指标原始编码"`
 }
 
 type EdbInfoList struct {
@@ -406,3 +433,38 @@ func GetEdbInfoByIdList(edbInfoIdList []int) (items []*EdbInfo, err error) {
 	err = global.DmSQL["data"].Raw(sql, edbInfoIdList).Find(&items).Error
 	return
 }
+
+// GetAllBaseEdbInfo
+// @Description: 获取所有基础指标（只有国能数据节点的数据）
+// @return item
+// @return err
+func GetAllBaseEdbInfo() (item []*EdbInfo, err error) {
+	sql := ` SELECT * FROM edb_info WHERE 1=1 AND "edb_info_type" = ? AND "edb_type" = ?`
+	pars := []interface{}{utils.EDB_INFO_TYPE, utils.EdbTypeBase}
+	err = global.DmSQL["data"].Raw(sql, pars...).Find(&item).Error
+
+	return
+}
+
+// AddEdbInfo
+// @Description: 新增指标
+// @param item
+// @return err
+func AddEdbInfo(item *EdbInfo) (err error) {
+	err = global.DmSQL["data"].Create(item).Error
+	if err != nil {
+		return
+	}
+
+	return
+}
+
+// Update
+// @Description: 更新指定参数
+// @receiver m
+// @param cols
+// @return err
+func (m *EdbInfo) Update(cols []string) (err error) {
+	err = global.DmSQL["data"].Select(cols).Updates(m).Error
+	return
+}

models/data_manage/edb_source.go

@@ -13,6 +13,7 @@ var (
 	EdbTableNameSourceMap   map[string]*EdbSource // 数据表名对应的指标来源
 	EdbSourceIdMap          map[int]*EdbSource    // 指标来源
 	EdbSourceExtendIdMap    map[string]int        // 指标来源字符串对应来源ID
+	EdbNameSourceMap        map[string]*EdbSource // 数据来源名对应的指标来源
 )
 
 // EdbSource 指标来源表
@@ -26,6 +27,9 @@ type EdbSource struct {
 	FromBridge       int    `description:"是否来源于桥接服务: 0-否; 1-是"`
 	BridgeFlag       string `description:"桥接服务对象标识"`
 	SourceExtend     string `description:"扩展字段做查询用"`
+	EdbCodeRequired  int32  `gorm:"column:edb_code_required;type:tinyint(4);comment:指标编码是否必填: 0-否; 1-是;not null;default:0;"` // 指标编码是否必填: 0-否; 1-是
+	IndexTableName   string `gorm:"column:index_table_name;type:varchar(128);comment:源指标表名;not null;"`                       // 源指标表名
+	SourceNameEn     string `gorm:"column:source_name_en;type:varchar(128);comment:指标来源名称-英文;not null;"`                     // 指标来源名称-英文
 }
 
 // GetEdbSourceItemsByCondition 获取指标来源列表
@@ -61,6 +65,7 @@ func InitEdbSourceVar() {
 	EdbDataTableNameMap = make(map[int]string)
 	EdbDataRefreshMethodMap = make(map[int]string)
 	EdbTableNameSourceMap = make(map[string]*EdbSource)
+	EdbNameSourceMap = make(map[string]*EdbSource)
 	EdbSourceIdMap = make(map[int]*EdbSource)
 	EdbSourceExtendIdMap = make(map[string]int)
 	sources, e := GetEdbSourceItemsByCondition(``, make([]interface{}, 0), []string{}, "")
@@ -72,6 +77,7 @@ func InitEdbSourceVar() {
 		EdbDataTableNameMap[v.EdbSourceId] = v.TableName
 		EdbDataRefreshMethodMap[v.EdbSourceId] = v.EdbRefreshMethod
 		EdbTableNameSourceMap[v.TableName] = v
+		EdbNameSourceMap[v.SourceName] = v
 
 		EdbSourceIdMap[v.EdbSourceId] = v
 		if v.SourceExtend != "" {
@@ -85,3 +91,169 @@ func InitEdbSourceVar() {
 		}
 	}
 }
+
+// GetEdbSourceItemsSourceId
+// @Description: 根据来源id获取指标来源
+// @param sourceId
+// @return item
+// @return err
+func GetEdbSourceItemsSourceId(sourceId int) (item *EdbSource, err error) {
+	sql := `SELECT * FROM edb_source WHERE 1=1 AND edb_source_id = ? `
+	err = global.DmSQL["data"].Raw(sql, sourceId).First(&item).Error
+
+	return
+}
+
+// GetEdbSourceBySourceId
+// @Description: 根据来源id获取指标来源
+// @param sourceId
+// @return sourceItem
+func GetEdbSourceBySourceId(sourceId int) (sourceItem *EdbSource) {
+	sourceItem, ok := EdbSourceIdMap[sourceId]
+	if !ok {
+		item, err := GetEdbSourceItemsSourceId(sourceId)
+		if err != nil {
+			return
+		}
+		if item.EdbSourceId > 0 {
+			sourceItem = item
+
+			// 写入到内存中
+			EdbSourceIdMap[sourceId] = sourceItem
+			EdbDataTableNameMap[sourceId] = sourceItem.TableName
+			EdbDataRefreshMethodMap[sourceItem.EdbSourceId] = sourceItem.EdbRefreshMethod
+			EdbTableNameSourceMap[sourceItem.TableName] = sourceItem
+			EdbNameSourceMap[sourceItem.SourceName] = sourceItem
+		}
+	}
+
+	return
+}
+
+// GetEdbSourceItemsSourceName
+// @Description: 根据来源名称获取指标来源
+// @param sourceName
+// @return item
+// @return err
+func GetEdbSourceItemsSourceName(sourceName string) (item *EdbSource, err error) {
+	sql := `SELECT * FROM edb_source WHERE 1=1 AND source_name = ? `
+	err = global.DmSQL["data"].Raw(sql, sourceName).First(&item).Error
+
+	return
+}
+
+// GetEdbSourceBySourceName
+// @Description: 根据来源名称获取指标来源
+// @param sourceId
+// @return sourceItem
+func GetEdbSourceBySourceName(sourceName string) (sourceItem *EdbSource) {
+	sourceItem, ok := EdbNameSourceMap[sourceName]
+	if !ok {
+		item, err := GetEdbSourceItemsSourceName(sourceName)
+		if err != nil {
+			return
+		}
+		if item.EdbSourceId > 0 {
+			sourceItem = item
+
+			// 写入到内存中
+			EdbSourceIdMap[sourceItem.EdbSourceId] = sourceItem
+			EdbDataTableNameMap[sourceItem.EdbSourceId] = sourceItem.TableName
+			EdbDataRefreshMethodMap[sourceItem.EdbSourceId] = sourceItem.EdbRefreshMethod
+			EdbTableNameSourceMap[sourceItem.TableName] = sourceItem
+			EdbNameSourceMap[sourceItem.SourceName] = sourceItem
+		}
+	}
+
+	return
+}
+
+// GetEdbSourceTableNameBySourceId
+// @Description: 根据来源id获取指标来源的归属表
+// @param sourceId
+// @return sourceItem
+func GetEdbSourceTableNameBySourceId(sourceId int) (tableName string) {
+	sourceItem := GetEdbSourceBySourceId(sourceId)
+
+	if sourceItem != nil {
+		tableName = sourceItem.TableName
+	}
+
+	return
+}
+
+// GetEdbSourceRefreshMethodBySourceId
+// @Description: 根据来源id获取指标来源的刷新路径
+// @param sourceId
+// @return sourceItem
+func GetEdbSourceRefreshMethodBySourceId(sourceId int) (refreshMethod string) {
+	sourceItem := GetEdbSourceBySourceId(sourceId)
+
+	if sourceItem != nil {
+		refreshMethod = sourceItem.EdbRefreshMethod
+	}
+
+	return
+}
+
+// AddEdbSource
+// @Description: 添加一个新的数据源
+// @param item
+// @return err
+func AddEdbSource(item *EdbSource, indexNamePrefix string) (err error) {
+	o := global.DmSQL["data"].Begin()
+	if err != nil {
+		return
+	}
+	defer func() {
+		if err != nil {
+			_ = o.Rollback()
+			return
+		}
+		_ = o.Commit()
+	}()
+
+	indexName1 := fmt.Sprintf(`INDEX_%s_EDB_CODE`, indexNamePrefix)
+	indexName2 := fmt.Sprintf(`INDEX_%s_EDB_INFO_ID`, indexNamePrefix)
+	sqlStatements := []string{
+		fmt.Sprintf(`CREATE TABLE "%s"
+(
+"edb_data_id" INT IDENTITY(1, 1) NOT NULL,
+"edb_info_id" INT,
+"edb_code" VARCHAR(50),
+"data_time" DATE,
+"value" DOUBLE,
+"create_time" TIMESTAMP(0),
+"modify_time" TIMESTAMP(0),
+"data_timestamp" BIGINT DEFAULT 0,
+NOT CLUSTER PRIMARY KEY("edb_data_id"),
+UNIQUE("edb_code", "data_time")) STORAGE(ON "MAIN", CLUSTERBTR) ;
+`, item.TableName),
+		fmt.Sprintf(`COMMENT ON COLUMN "%s"."create_time" IS '创建时间';`, item.TableName),
+		fmt.Sprintf(`COMMENT ON COLUMN "%s"."data_time" IS '数据日期';`, item.TableName),
+		fmt.Sprintf(`COMMENT ON COLUMN "%s"."data_timestamp" IS '数据日期时间戳';`, item.TableName),
+		fmt.Sprintf(`COMMENT ON COLUMN "%s"."edb_code" IS '指标编码';`, item.TableName),
+		fmt.Sprintf(`COMMENT ON COLUMN "%s"."edb_info_id" IS '指标id';`, item.TableName),
+		fmt.Sprintf(`COMMENT ON COLUMN "%s"."modify_time" IS '修改时间';`, item.TableName),
+		fmt.Sprintf(`COMMENT ON COLUMN "%s"."value" IS '数据值';`, item.TableName),
+
+		fmt.Sprintf(`CREATE OR REPLACE  INDEX "%s" ON "%s"("edb_code" ASC) STORAGE(ON "MAIN", CLUSTERBTR) ;`, indexName1, item.TableName),
+		fmt.Sprintf(`CREATE OR REPLACE  INDEX "%s" ON "%s"("edb_info_id" ASC) STORAGE(ON "MAIN", CLUSTERBTR) ;`, indexName2, item.TableName),
+	}
+
+	// 创建表和索引
+	for _, sql := range sqlStatements {
+		err = o.Exec(sql).Error
+		if err != nil {
+			return
+		}
+	}
+
+	// 添加来源
+	err = o.Create(item).Error
+	if err != nil {
+		return
+	}
+
+	return
+}

models/db.go

@@ -187,3 +187,10 @@ func initEdbRefresh() {
 		new(edb_refresh.EdbRefreshMapping),       // 指标刷新时间配置关系表
 	)
 }
+
+// AfterInitTable
+// @Description: 系统初始化后的数据操作
+func AfterInitTable() {
+	// 初始化部分数据表变量(直接init会有顺序问题=_=!)
+	data_manage.InitEdbSourceVar()
+}

services/data/base_edb_lib.go

@@ -81,8 +81,9 @@ func RefreshEdbData(edbInfoId, source, subSource int, edbCode, startDate string)
 	case utils.DATA_SOURCE_GFEX:
 		urlStr = "gz/refresh"
 	default:
-		urlStr = data_manage.EdbDataRefreshMethodMap[source] // 没有对应的从edb_source中取
+		urlStr = data_manage.GetEdbSourceRefreshMethodBySourceId(source) // 没有对应的从edb_source中取
 	}
+	fmt.Println("urlStr:", urlStr)
 	if urlStr == "" {
 		err = fmt.Errorf(fmt.Sprint("source:", source, ";未实现该指标的刷新接口，请联系管理员"))
 		return

services/eta_bridge/eta_bridge.go

@@ -72,6 +72,9 @@ func HttpEtaBridgePost(urlStr string, param interface{}) (bResult []byte, err er
 		err = fmt.Errorf("resp body is empty")
 		return
 	}
+
+	utils.FileLog.Info("桥接服务post请求返回数据:\n" + string(bResult))
+
 	// 生产环境解密, 注意有个坑前后的双引号
 	if utils.RunMode == "release" {
 		str := string(bResult)

services/eta_bridge/gn.go

@@ -0,0 +1,690 @@
+package eta_bridge
+
+import (
+	"context"
+	"encoding/json"
+	"eta_gn/eta_task/models/data_manage"
+	"eta_gn/eta_task/services/alarm_msg"
+	"eta_gn/eta_task/services/data"
+	"eta_gn/eta_task/utils"
+	"fmt"
+	"github.com/rdlucklib/rdluck_tools/paging"
+	"net/url"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+)
+
+// 同步指标信息锁
+var lockSyncGnIndex sync.Mutex
+
+const GnEdbListUri = `/index_data/gn/edb/list` // 国能指标列表接口
+
+// CurrLevelParentClassifyMap 当前层级分类map
+var CurrLevelParentClassifyMap map[int64]map[int64]map[string]CurrClassify
+
+// CurrEdbInfoMap 当前库里已有的指标map
+var CurrEdbInfoMap map[string]*data_manage.EdbInfo
+
+type CurrClassify struct {
+	ClassifyId   int64
+	ParentId     int64
+	ClassifyName string
+}
+
+// SyncGnIndex
+// @Description: 定时同步指标信息
+// @author: Roc
+// @datetime 2024-03-07 17:39:34
+// @param cont context.Context
+// @return err error
+func SyncGnIndex(cont context.Context) (err error) {
+	//by, _ := json.Marshal(CurrLevelParentClassifyMap)
+	//utils.FileLog.Info(string(by))
+	//
+	//list := make([]IndexInfo, 0)
+	//list = []IndexInfo{
+	//	{
+	//		ClassifyNameOne:   "宏观经济",
+	//		ClassifyNameTwo:   "美国",
+	//		ClassifyNameThree: "房地产",
+	//	},
+	//	{
+	//		ClassifyNameOne:   "宏观经济",
+	//		ClassifyNameTwo:   "美国",
+	//		ClassifyNameThree: "土地销售面积",
+	//	},
+	//	{
+	//		ClassifyNameOne:   "宏观经济",
+	//		ClassifyNameTwo:   "美国",
+	//		ClassifyNameThree: "GDP",
+	//	},
+	//	{
+	//		ClassifyNameOne:   "宏观经济",
+	//		ClassifyNameTwo:   "中国",
+	//		ClassifyNameThree: "房地产",
+	//	},
+	//	{
+	//		ClassifyNameOne:   "宏观经济",
+	//		ClassifyNameTwo:   "中国",
+	//		ClassifyNameThree: "土地销售面积",
+	//	},
+	//	{
+	//		ClassifyNameOne:   "宏观经济",
+	//		ClassifyNameTwo:   "中国",
+	//		ClassifyNameThree: "GDP",
+	//	},
+	//	{
+	//		ClassifyNameOne:   "宏观经济",
+	//		ClassifyNameTwo:   "韩国",
+	//		ClassifyNameThree: "房地产",
+	//	},
+	//	{
+	//		ClassifyNameOne:   "宏观经济",
+	//		ClassifyNameTwo:   "韩国",
+	//		ClassifyNameThree: "土地销售面积",
+	//	},
+	//	{
+	//		ClassifyNameOne:   "宏观经济",
+	//		ClassifyNameTwo:   "韩国",
+	//		ClassifyNameThree: "GDP",
+	//	},
+	//	{
+	//		ClassifyNameOne:   "宏观经济",
+	//		ClassifyNameTwo:   "中国",
+	//		ClassifyNameThree: "GDP",
+	//	},
+	//}
+	//for _, v := range list {
+	//	fmt.Println(handleClassify(v))
+	//}
+	//return
+
+	lockSyncGnIndex.Lock()
+
+	initCurrEdbInfoMap()
+	initCurrLevelParentClassifyMap()
+
+	errMsgList := make([]string, 0)
+	defer func() {
+		if err != nil {
+			tips := "SyncGnIndex-定时同步国能的指标信息到ETA失败, ErrMsg:\n" + err.Error()
+			utils.FileLog.Info(tips)
+			go alarm_msg.SendAlarmMsg(tips, 3)
+		}
+		if len(errMsgList) > 0 {
+			tips := "SyncGnIndex-定时同步国能的指标信息到ETA失败, ErrMsg:\n" + strings.Join(errMsgList, "\n")
+			utils.FileLog.Info(tips)
+			go alarm_msg.SendAlarmMsg(tips, 3)
+		}
+		lockSyncGnIndex.Unlock()
+	}()
+
+	var lastUpdateTimeStr string // 上一次更新的时间
+
+	err, errMsgList = syncGnIndex(1, utils.SyncCrmIndexNum, lastUpdateTimeStr)
+
+	return
+}
+
+// initCurrLevelParentClassifyMap
+// @Description: 初始化当前层级分类map
+func initCurrLevelParentClassifyMap() {
+	var condition string
+	var pars []interface{}
+
+	// 普通指标分类
+	condition = " AND classify_type = ? "
+	pars = append(pars, 0)
+
+	classifyList, err := data_manage.GetAllEdbClassifyListByCondition(condition, pars)
+	if err != nil {
+		utils.FileLog.Error("获取分类列表数据失败:" + err.Error())
+		return
+	}
+
+	// 清空缓存
+	CurrLevelParentClassifyMap = make(map[int64]map[int64]map[string]CurrClassify)
+
+	for _, v := range classifyList {
+		currParentClassifyMap, ok := CurrLevelParentClassifyMap[v.Level]
+		if !ok {
+			currParentClassifyMap = make(map[int64]map[string]CurrClassify)
+		}
+		currClassifyMap, ok := currParentClassifyMap[v.ParentID]
+		if !ok {
+			currClassifyMap = make(map[string]CurrClassify)
+		}
+
+		classifyName := strings.TrimSpace(v.ClassifyName)
+		currClassifyMap[classifyName] = CurrClassify{
+			ClassifyId:   v.ClassifyID,
+			ParentId:     v.ParentID,
+			ClassifyName: classifyName,
+		}
+		currParentClassifyMap[v.ParentID] = currClassifyMap
+		CurrLevelParentClassifyMap[v.Level] = currParentClassifyMap
+	}
+}
+
+// initCurrEdbInfoMap
+// @Description: 初始化当前指标map
+func initCurrEdbInfoMap() {
+	// 获取指标列表
+	edbInfoList, err := data_manage.GetAllBaseEdbInfo()
+	if err != nil {
+		utils.FileLog.Error("获取指标列表数据失败:" + err.Error())
+		return
+	}
+
+	// 清空缓存
+	CurrEdbInfoMap = make(map[string]*data_manage.EdbInfo)
+
+	for _, v := range edbInfoList {
+		CurrEdbInfoMap[v.OriginalEdbCode] = v
+	}
+}
+
+// EtaBridgeGnIndexListResp
+// @Description: 指标列表返回数据
+type EtaBridgeGnIndexListResp struct {
+	Code int           `json:"code" description:"状态码"`
+	Msg  string        `json:"msg" description:"提示信息"`
+	Data IndexListResp `json:"data" description:"返回数据"`
+}
+
+// IndexListResp
+// @Description: 指标列表数据
+type IndexListResp struct {
+	Paging paging.PagingItem `description:"分页数据"`
+	List   []IndexInfo
+}
+
+// IndexInfo
+// @Description: 指标信息
+type IndexInfo struct {
+	ClassifyNameOne   string `description:"一级目录"`
+	ClassifyNameTwo   string `description:"二级目录"`
+	ClassifyNameThree string `description:"三级目录"`
+	DataIndexCode     string `description:"数据节点指标编码"`
+	SourceEdbCode     string `description:"数据源指标原始编码"`
+	EdbName           string `description:"指标名称"`
+	Frequency         string `description:"频度"`
+	Unit              string `description:"单位"`
+	SourceName        string `description:"来源"`
+}
+
+// BridgeGnIndexParams
+// @Description: 桥接服务-获取国能指标数据入参
+type BridgeGnIndexParams struct {
+	LastModifyTime string `json:"last_modify_time" description:"最近一次更新时间"`
+	PageIndex      int    `json:"page_index" description:"当前页码"`
+	PageSize       int    `json:"page_size" description:"每页数量"`
+}
+
+// syncCrmIndex
+// @Description: 开始同步CRM指标信息
+// @author: Roc
+// @datetime 2024-05-17 15:55:11
+// @param assetPkgCd string
+// @param currIndex int
+// @param pageSize int
+// @param lastUpdateTimeStr string
+// @return err error
+// @return errMsgList []string
+func syncGnIndex(currIndex, pageSize int, baseLastUpdateTimeStr string) (err error, errMsgList []string) {
+	errMsgList = make([]string, 0)
+
+	lastUpdateTimeStr := baseLastUpdateTimeStr
+	if lastUpdateTimeStr != `` {
+		lastUpdateTimeStr = url.QueryEscape(lastUpdateTimeStr)
+	}
+	params := BridgeGnIndexParams{
+		LastModifyTime: lastUpdateTimeStr,
+		PageIndex:      currIndex,
+		PageSize:       pageSize,
+	}
+	bResult, err, _ := HttpEtaBridgePost(GnEdbListUri, params)
+	if err != nil {
+		return
+	}
+
+	var result EtaBridgeGnIndexListResp
+	err = json.Unmarshal(bResult, &result)
+	if err != nil {
+		err = fmt.Errorf("result unmarshal err: %s\nresult: %s", err.Error(), string(bResult))
+		utils.FileLog.Info("桥接服务get请求失败:\n" + string(bResult))
+		return
+	}
+
+	//totalPage := result.Data.Paging.Pages
+
+	// 处理指标信息
+	for _, v := range result.Data.List {
+		tmpErr := handleIndex(v)
+		if tmpErr != nil {
+			errMsgList = append(errMsgList, tmpErr.Error())
+		}
+
+	}
+
+	// 如果还有下一页，那么就继续请求下一页
+	if currIndex < result.Data.Paging.Pages {
+		_, tmpErrMsgList := syncGnIndex(currIndex+1, utils.SyncCrmIndexNum, baseLastUpdateTimeStr)
+		errMsgList = append(errMsgList, tmpErrMsgList...)
+	}
+
+	return
+}
+
+// handleIndex
+// @Description: 指标处理
+// @param index
+// @return err
+func handleIndex(index IndexInfo) (err error) {
+	// 处理分类（如果不存在就创建）
+	_, _, thirdClassifyId, err := handleClassify(index)
+	if err != nil {
+		return
+	}
+
+	// 处理指标（如果不存在就创建）
+	err = handleEdbInfo(index, thirdClassifyId)
+
+	return
+}
+
+// handleClassify
+// @Description: 分类处理
+// @param index
+// @return firstClassifyId
+// @return secondClassifyId
+// @return thirdClassifyId
+// @return err
+func handleClassify(index IndexInfo) (firstClassifyId, secondClassifyId, thirdClassifyId int64, err error) {
+	firstClassifyName := strings.TrimSpace(index.ClassifyNameOne)
+	secondClassifyName := strings.TrimSpace(index.ClassifyNameTwo)
+	thirdClassifyName := strings.TrimSpace(index.ClassifyNameThree)
+
+	var oneLevel, twoLevel, threeLevel int64
+	oneLevel = 1
+	twoLevel = 2
+	threeLevel = 3
+
+	// 一级分类
+	{
+		var parentId int64
+		parentId = 0
+		classifyName := firstClassifyName
+		level := oneLevel
+
+		// 获取层级下的父级分类map
+		currParentClassifyMap, ok := CurrLevelParentClassifyMap[level]
+		if !ok {
+			currParentClassifyMap = make(map[int64]map[string]CurrClassify)
+		}
+
+		// 获取父级id下的分类列表
+		currClassifyListMap, ok := currParentClassifyMap[parentId]
+		if !ok {
+			currClassifyListMap = make(map[string]CurrClassify)
+		}
+
+		// 根据分类名称获取分类
+		currClassify, ok := currClassifyListMap[classifyName]
+		if !ok {
+			timestamp := strconv.FormatInt(time.Now().UnixNano(), 10)
+			classifyInfo := &data_manage.EdbClassify{
+				//ClassifyId:      0,
+				ClassifyType:    0,
+				ClassifyName:    classifyName,
+				ClassifyNameEn:  classifyName,
+				ParentID:        parentId,
+				RootID:          0,
+				HasData:         0,
+				CreateTime:      time.Now(),
+				ModifyTime:      time.Now(),
+				SysUserID:       0,
+				SysUserRealName: "",
+				Level:           level,
+				UniqueCode:      utils.MD5(fmt.Sprint(parentId, "_", utils.DATA_PREFIX+"_"+timestamp)),
+				Sort:            0,
+			}
+			err = data_manage.AddEdbClassify(classifyInfo)
+			if err != nil {
+				return
+			}
+			classifyInfo.RootID = classifyInfo.ClassifyID
+			err = classifyInfo.Update([]string{"root_id"})
+			if err != nil {
+				return
+			}
+
+			currClassify = CurrClassify{
+				ClassifyId:   classifyInfo.ClassifyID,
+				ParentId:     classifyInfo.ParentID,
+				ClassifyName: classifyInfo.ClassifyName,
+			}
+
+			currClassifyListMap[classifyName] = currClassify
+			currParentClassifyMap[parentId] = currClassifyListMap
+			CurrLevelParentClassifyMap[level] = currParentClassifyMap
+		}
+
+		firstClassifyId = currClassify.ClassifyId
+
+	}
+
+	// 二级分类
+	{
+		parentId := firstClassifyId
+		classifyName := secondClassifyName
+		level := twoLevel
+
+		// 获取层级下的父级分类map
+		currParentClassifyMap, ok := CurrLevelParentClassifyMap[level]
+		if !ok {
+			currParentClassifyMap = make(map[int64]map[string]CurrClassify)
+		}
+
+		// 获取父级id下的分类列表
+		currClassifyListMap, ok := currParentClassifyMap[parentId]
+		if !ok {
+			currClassifyListMap = make(map[string]CurrClassify)
+		}
+
+		// 根据分类名称获取分类
+		currClassify, ok := currClassifyListMap[classifyName]
+		if !ok {
+			timestamp := strconv.FormatInt(time.Now().UnixNano(), 10)
+			classifyInfo := &data_manage.EdbClassify{
+				//ClassifyId:      0,
+				ClassifyType:    0,
+				ClassifyName:    classifyName,
+				ClassifyNameEn:  classifyName,
+				ParentID:        parentId,
+				RootID:          firstClassifyId,
+				HasData:         0,
+				CreateTime:      time.Now(),
+				ModifyTime:      time.Now(),
+				SysUserID:       0,
+				SysUserRealName: "",
+				Level:           level,
+				UniqueCode:      utils.MD5(fmt.Sprint(parentId, "_", utils.DATA_PREFIX+"_"+timestamp)),
+				Sort:            0,
+			}
+			err = data_manage.AddEdbClassify(classifyInfo)
+			if err != nil {
+				return
+			}
+			currClassify = CurrClassify{
+				ClassifyId:   classifyInfo.ClassifyID,
+				ParentId:     classifyInfo.ParentID,
+				ClassifyName: classifyInfo.ClassifyName,
+			}
+
+			currClassifyListMap[classifyName] = currClassify
+			currParentClassifyMap[parentId] = currClassifyListMap
+			CurrLevelParentClassifyMap[level] = currParentClassifyMap
+		}
+
+		secondClassifyId = currClassify.ClassifyId
+
+	}
+
+	// 三级分类
+	{
+		parentId := secondClassifyId
+		classifyName := thirdClassifyName
+		level := threeLevel
+
+		// 获取层级下的父级分类map
+		currParentClassifyMap, ok := CurrLevelParentClassifyMap[level]
+		if !ok {
+			currParentClassifyMap = make(map[int64]map[string]CurrClassify)
+		}
+
+		// 获取父级id下的分类列表
+		currClassifyListMap, ok := currParentClassifyMap[parentId]
+		if !ok {
+			currClassifyListMap = make(map[string]CurrClassify)
+		}
+
+		// 根据分类名称获取分类
+		currClassify, ok := currClassifyListMap[classifyName]
+		if !ok {
+			timestamp := strconv.FormatInt(time.Now().UnixNano(), 10)
+			classifyInfo := &data_manage.EdbClassify{
+				//ClassifyId:      0,
+				ClassifyType:    0,
+				ClassifyName:    classifyName,
+				ClassifyNameEn:  classifyName,
+				ParentID:        parentId,
+				RootID:          firstClassifyId,
+				HasData:         1,
+				CreateTime:      time.Now(),
+				ModifyTime:      time.Now(),
+				SysUserID:       0,
+				SysUserRealName: "",
+				Level:           level,
+				UniqueCode:      utils.MD5(fmt.Sprint(parentId, "_", utils.DATA_PREFIX+"_"+timestamp)),
+				Sort:            0,
+			}
+			err = data_manage.AddEdbClassify(classifyInfo)
+			if err != nil {
+				return
+			}
+			currClassify = CurrClassify{
+				ClassifyId:   classifyInfo.ClassifyID,
+				ParentId:     classifyInfo.ParentID,
+				ClassifyName: classifyInfo.ClassifyName,
+			}
+
+			currClassifyListMap[classifyName] = currClassify
+			currParentClassifyMap[parentId] = currClassifyListMap
+			CurrLevelParentClassifyMap[level] = currParentClassifyMap
+		}
+
+		thirdClassifyId = currClassify.ClassifyId
+
+	}
+
+	return
+
+}
+
+// handleEdbInfo
+// @Description: 处理指标
+// @param index
+// @param thirdClassifyId
+// @return err
+func handleEdbInfo(index IndexInfo, thirdClassifyId int64) (err error) {
+	edbInfo, ok := CurrEdbInfoMap[index.DataIndexCode]
+	frequency := Frequency(strings.TrimSpace(index.Frequency))
+	unit := strings.TrimSpace(index.Unit)
+	sourceName, sourceId, err := GetSource(strings.TrimSpace(index.SourceName))
+	if err != nil {
+		return
+	}
+
+	if !ok {
+		endDate := time.Date(1900, 1, 1, 0, 0, 0, 0, time.Local)
+
+		timestamp := strconv.FormatInt(time.Now().UnixNano(), 10)
+
+		edbInfo = &data_manage.EdbInfo{
+			EdbInfoId:        0,
+			EdbInfoType:      utils.EDB_INFO_TYPE,
+			SourceName:       sourceName,
+			Source:           sourceId,
+			EdbCode:          index.SourceEdbCode,
+			EdbName:          index.EdbName,
+			EdbNameEn:        index.EdbName,
+			EdbNameSource:    index.EdbName,
+			Frequency:        frequency,
+			Unit:             unit,
+			UnitEn:           unit,
+			StartDate:        endDate,
+			EndDate:          endDate,
+			ClassifyId:       int(thirdClassifyId),
+			SysUserId:        0,
+			SysUserRealName:  "",
+			UniqueCode:       utils.MD5(fmt.Sprint(index.SourceEdbCode, "_", utils.DATA_PREFIX+"_"+timestamp)),
+			CreateTime:       time.Now(),
+			ModifyTime:       time.Now(),
+			BaseModifyTime:   time.Now(),
+			MinValue:         0,
+			MaxValue:         0,
+			CalculateFormula: "",
+			EdbType:          utils.EdbTypeBase,
+			Sort:             0,
+			LatestDate:       "",
+			LatestValue:      0,
+			EndValue:         0,
+			MoveType:         0,
+			MoveFrequency:    "",
+			NoUpdate:         0,
+			ServerUrl:        "",
+			ChartImage:       "", // 缩略图
+			Calendar:         "",
+			DataDateType:     "",
+			ManualSave:       0,
+			EmptyType:        0,
+			MaxEmptyType:     0,
+			TerminalCode:     "",
+			DataUpdateTime:   "",
+			ErDataUpdateDate: "",
+			SourceIndexName:  index.EdbName,
+			SubSource:        0,
+			SubSourceName:    "",
+			IndicatorCode:    "",
+			StockCode:        "",
+			Extra:            "",
+			IsJoinPermission: 0,
+			OriginalEdbCode:  index.DataIndexCode,
+		}
+		err = data_manage.AddEdbInfo(edbInfo)
+		if err != nil {
+			return
+		}
+
+		CurrEdbInfoMap[index.DataIndexCode] = edbInfo
+
+		// TODO 刷新指标明细数据
+		fmt.Println(data.RefreshEdbData(edbInfo.EdbInfoId, edbInfo.Source, edbInfo.SubSource, edbInfo.EdbCode, edbInfo.EndDate.Format(utils.FormatDate)))
+		return
+	}
+
+	updateCols := make([]string, 0)
+	if edbInfo.EdbNameEn == edbInfo.EdbName && edbInfo.EdbName != index.EdbName {
+		edbInfo.EdbNameEn = index.EdbName
+		updateCols = append(updateCols, "edb_name_en")
+	}
+	if edbInfo.EdbName != index.EdbName {
+		edbInfo.EdbName = index.EdbName
+		updateCols = append(updateCols, "edb_name")
+	}
+	if edbInfo.Frequency != index.Frequency {
+		edbInfo.Frequency = index.Frequency
+		updateCols = append(updateCols, "frequency")
+	}
+	if edbInfo.UnitEn == edbInfo.Unit && edbInfo.Unit != unit {
+		edbInfo.UnitEn = unit
+		updateCols = append(updateCols, "unit_en")
+	}
+	if edbInfo.Unit != unit {
+		edbInfo.Unit = unit
+		updateCols = append(updateCols, "unit")
+	}
+	if edbInfo.ClassifyId != int(thirdClassifyId) {
+		edbInfo.ClassifyId = int(thirdClassifyId)
+		updateCols = append(updateCols, "classify_id")
+	}
+
+	if len(updateCols) > 0 {
+		err = edbInfo.Update(updateCols)
+	}
+
+	return
+}
+
+// Frequency
+// @Description: 获取频度
+// @param unit
+// @return string
+func Frequency(unit string) string {
+	switch unit {
+	case "半月度":
+		unit = `周度`
+	case "不定期":
+		unit = `日度`
+	case `日度`, `周度`, `旬度`, `月度`, `季度`, `半年度`, `年度`:
+	default:
+		unit = ``
+	}
+
+	return unit
+
+}
+
+// GetSource
+// @Description: 获取来源
+// @param sourceName
+// @return gnSourceName
+// @return source
+// @return err
+func GetSource(sourceName string) (gnSourceName string, source int, err error) {
+	gnSourceName = sourceName
+	var tableNameSuffix, indexNamePrefix string
+	tableNamePrefix := "edb_data_gn_"
+	switch sourceName {
+	case "CCTD":
+		tableNameSuffix = "cctd"
+	case "mysteel":
+		tableNameSuffix = "mysteel"
+	case "wind":
+		tableNameSuffix = "wind"
+	case "卓创":
+		tableNameSuffix = "sci"
+	case "CCI":
+		tableNameSuffix = "cci"
+		//return
+	default:
+		if strings.Contains(sourceName, "国能购销辅助决策系统") {
+			gnSourceName = `国能购销辅助决策系统`
+		} else if strings.Contains(sourceName, "国能市场分析平台") {
+			gnSourceName = `国能市场分析平台`
+		}
+	}
+
+	sourceItem := data_manage.GetEdbSourceBySourceName(gnSourceName)
+	// 如果找不到，说明是
+	if sourceItem == nil {
+		indexNamePrefix = strings.ToUpper(tableNameSuffix)
+
+		sourceItem = &data_manage.EdbSource{
+			EdbSourceId:      0,
+			SourceName:       gnSourceName,
+			TableName:        tableNamePrefix + tableNameSuffix,
+			EdbAddMethod:     "gn_index/add",
+			EdbRefreshMethod: "gn_index/refresh",
+			IsBase:           1,
+			FromBridge:       1,
+			BridgeFlag:       "bridge_gn",
+			SourceExtend:     gnSourceName,
+			EdbCodeRequired:  1,
+			IndexTableName:   "",
+			SourceNameEn:     gnSourceName,
+		}
+
+		err = data_manage.AddEdbSource(sourceItem, indexNamePrefix)
+		if err != nil {
+			return
+		}
+	}
+
+	source = sourceItem.EdbSourceId
+
+	return
+}

services/task.go

@@ -5,6 +5,7 @@ import (
 	"eta_gn/eta_task/services/data"
 	"eta_gn/eta_task/services/data/future_good"
 	"eta_gn/eta_task/services/data_stat"
+	"eta_gn/eta_task/services/eta_bridge"
 	"eta_gn/eta_task/utils"
 	"fmt"
 	"sync"
@@ -15,6 +16,8 @@ import (
 
 func Task() {
 	fmt.Println("task start")
+	fmt.Println(eta_bridge.SyncGnIndex(nil))
+	return
 	//如果是生产环境，才需要走这些任务
 	if utils.RunMode == "release" {
 		releaseTask()

utils/constants.go

@@ -195,3 +195,22 @@ const (
 
 // MultiAddNum 批量插入的数据量
 const MultiAddNum = 500
+
+const (
+	DATA_PREFIX       = "hz_data"
+	CHART_PREFIX      = "hz_chart"
+	EXCEL_DATA_PREFIX = "hz_excel_data"
+)
+
+// 指标类型
+const (
+	EDB_INFO_TYPE         = 0 //指标类型 0：普通指标
+	PREDICT_EDB_INFO_TYPE = 1 //指标类型 1：预测指标
+)
+
+const (
+	EdbTypeBase      = 1 // 指标类型-基础指标
+	EdbTypeCalculate = 2 // 指标类型-计算指标
+)
+
+//const END_DATE = time.Date(1900, 1, 1, 0, 0, 0, 0, time.Local)