Merge remote-tracking branch 'origin/feature/eta_2.3.5'

# Conflicts:
#	services/task.go
#	utils/constants.go

models/ai_predict_model/ai_predict_model_index.go

@@ -0,0 +1,91 @@
+package data_manage
+
+import (
+	"fmt"
+	"github.com/beego/beego/v2/client/orm"
+	"strings"
+	"time"
+)
+
+// AiPredictModelIndex AI预测模型标的
+type AiPredictModelIndex struct {
+	AiPredictModelIndexId int       `orm:"column(ai_predict_model_index_id);pk"`
+	IndexName             string    `description:"标的名称"`
+	IndexCode             string    `description:"自生成的指标编码"`
+	ClassifyId            int       `description:"分类ID"`
+	ModelFramework        string    `description:"模型框架"`
+	PredictDate           time.Time `description:"预测日期"`
+	PredictValue          float64   `description:"预测值"`
+	PredictFrequency      string    `description:"预测频度"`
+	DirectionAccuracy     string    `description:"方向准确度"`
+	AbsoluteDeviation     string    `description:"绝对偏差"`
+	ExtraConfig           string    `description:"模型参数"`
+	Sort                  int       `description:"排序"`
+	SysUserId             int       `description:"创建人ID"`
+	SysUserRealName       string    `description:"创建人姓名"`
+	LeftMin               string    `description:"图表左侧最小值"`
+	LeftMax               string    `description:"图表左侧最大值"`
+	CreateTime            time.Time `description:"创建时间"`
+	ModifyTime            time.Time `description:"修改时间"`
+}
+
+func (m *AiPredictModelIndex) TableName() string {
+	return "ai_predict_model_index"
+}
+
+type AiPredictModelIndexCols struct {
+	PrimaryId         string
+	IndexName         string
+	IndexCode         string
+	ClassifyId        string
+	ModelFramework    string
+	PredictDate       string
+	PredictValue      string
+	DirectionAccuracy string
+	AbsoluteDeviation string
+	ExtraConfig       string
+	Sort              string
+	SysUserId         string
+	SysUserRealName   string
+	LeftMin           string
+	LeftMax           string
+	CreateTime        string
+	ModifyTime        string
+}
+
+func (m *AiPredictModelIndex) Cols() AiPredictModelIndexCols {
+	return AiPredictModelIndexCols{
+		PrimaryId:         "ai_predict_model_index_id",
+		IndexName:         "index_name",
+		IndexCode:         "index_code",
+		ClassifyId:        "classify_id",
+		ModelFramework:    "model_framework",
+		PredictDate:       "predict_date",
+		PredictValue:      "predict_value",
+		DirectionAccuracy: "direction_accuracy",
+		AbsoluteDeviation: "absolute_deviation",
+		ExtraConfig:       "extra_config",
+		Sort:              "sort",
+		SysUserId:         "sys_user_id",
+		SysUserRealName:   "sys_user_real_name",
+		LeftMin:           "left_min",
+		LeftMax:           "left_max",
+		CreateTime:        "create_time",
+		ModifyTime:        "modify_time",
+	}
+}
+
+func (m *AiPredictModelIndex) GetItemsByCondition(condition string, pars []interface{}, fieldArr []string, orderRule string) (items []*AiPredictModelIndex, err error) {
+	o := orm.NewOrmUsingDB("data")
+	fields := strings.Join(fieldArr, ",")
+	if len(fieldArr) == 0 {
+		fields = `*`
+	}
+	order := fmt.Sprintf(`ORDER BY %s DESC`, m.Cols().CreateTime)
+	if orderRule != "" {
+		order = ` ORDER BY ` + orderRule
+	}
+	sql := fmt.Sprintf(`SELECT %s FROM %s WHERE 1=1 %s %s`, fields, m.TableName(), condition, order)
+	_, err = o.Raw(sql, pars).QueryRows(&items)
+	return
+}

models/business_conf.go

@@ -1,6 +1,7 @@
 package models
 
 import (
+	"eta/eta_task/utils"
 	"fmt"
 	"github.com/beego/beego/v2/client/orm"
 	"html"
@@ -8,13 +9,20 @@ import (
 	"time"
 )
 
+var (
+	BusinessConfMap map[string]string
+)
+
 const (
-	BusinessConfUseXf          = "UseXf"
-	BusinessConfXfAppid        = "XfAppid"
-	BusinessConfXfApiKey       = "XfApiKey"
-	BusinessConfXfApiSecret    = "XfApiSecret"
-	BusinessConfXfVcn          = "XfVcn"
-	BusinessConfEnPptCoverImgs = "EnPptCoverImgs"
+	BusinessConfUseXf                 = "UseXf"
+	BusinessConfXfAppid               = "XfAppid"
+	BusinessConfXfApiKey              = "XfApiKey"
+	BusinessConfXfApiSecret           = "XfApiSecret"
+	BusinessConfXfVcn                 = "XfVcn"
+	BusinessConfEnPptCoverImgs        = "EnPptCoverImgs"
+	BusinessConfEsIndexNameExcel      = "EsIndexNameExcel"      // ES索引名称-表格
+	BusinessConfEsIndexNameDataSource = "EsIndexNameDataSource" // ES索引名称-数据源
+	BusinessConfSyncDataEsDaily       = "SyncDataEsDaily"       // 每日同步数据源ES开关
 )
 
 // BusinessConf 商户配置表
@@ -178,3 +186,18 @@ type EdbStopRefreshRule struct {
 	BaseIndexStopDays int `description:"数据源间隔天数未加入指标库则停用"`
 	EdbStopDays       int `description:"指标库间隔天数未引用则停用"`
 }
+
+func InitBusinessConf() {
+	var e error
+	BusinessConfMap, e = GetBusinessConf()
+	if e != nil {
+		return
+	}
+	// ES索引名称
+	if BusinessConfMap[BusinessConfEsIndexNameExcel] != "" {
+		utils.EsExcelIndexName = BusinessConfMap[BusinessConfEsIndexNameExcel]
+	}
+	if BusinessConfMap[BusinessConfEsIndexNameDataSource] != "" {
+		utils.EsDataSourceIndexName = BusinessConfMap[BusinessConfEsIndexNameDataSource]
+	}
+}

models/data_source/data_source.go

@@ -0,0 +1,2017 @@
+package data_source
+
+import (
+	"eta/eta_task/utils"
+	"fmt"
+	"github.com/beego/beego/v2/client/orm"
+	"github.com/rdlucklib/rdluck_tools/paging"
+	"strings"
+	"time"
+)
+
+// SearchDataSource 数据源ES搜索
+type SearchDataSource struct {
+	PrimaryId   int     `description:"主键ID"`
+	IndexCode   string  `description:"指标编码"`
+	IndexName   string  `description:"指标名称"`
+	ClassifyId  int     `description:"分类ID"`
+	Unit        string  `description:"单位"`
+	Frequency   string  `description:"频度"`
+	StartDate   string  `description:"开始日期"`
+	EndDate     string  `description:"结束日期"`
+	LatestValue float64 `description:"最新值"`
+	Source      int     `description:"来源"`
+	SourceName  string  `description:"数据源名称"`
+	SubSource   int     `description:"子来源"`
+	IsDeleted   int     `description:"是否已删除：0-正常；1-已删除"`
+	CreateTime  string  `description:"创建时间"`
+	ModifyTime  string  `description:"修改时间"`
+}
+
+// SearchDataSourceItem 数据源ES搜索
+type SearchDataSourceItem struct {
+	SearchDataSource
+	SearchText string `description:"搜索结果（含高亮）"`
+}
+
+// ToMap 为了方便前端那边的修改，这里兼容一下部分Key的变动，不然改动量很大
+func (s *SearchDataSourceItem) ToMap(primaryIdKey, indexNameKey, classifyIdKey string) map[string]interface{} {
+	data := make(map[string]interface{})
+	if primaryIdKey != "" {
+		data[primaryIdKey] = s.PrimaryId
+	} else {
+		data["PrimaryId"] = s.PrimaryId
+	}
+	if indexNameKey != "" {
+		data[indexNameKey] = s.IndexName
+	} else {
+		data["IndexName"] = s.IndexName
+	}
+	if classifyIdKey != "" {
+		data[classifyIdKey] = s.ClassifyId
+	} else {
+		data["ClassifyId"] = s.ClassifyId
+	}
+	data["IndexCode"] = s.IndexCode
+	data["Unit"] = s.Unit
+	data["Frequency"] = s.Frequency
+	data["StartDate"] = s.StartDate
+	data["EndDate"] = s.EndDate
+	data["LatestValue"] = s.LatestValue
+	data["Source"] = s.Source
+	data["SourceName"] = s.SourceName
+	data["IsDeleted"] = s.IsDeleted
+	data["CreateTime"] = s.CreateTime
+	data["ModifyTime"] = s.ModifyTime
+	data["SearchText"] = s.SearchText
+	return data
+}
+
+// SearchDataSourceResp 数据源ES-分页搜索响应
+type SearchDataSourceResp struct {
+	Paging *paging.PagingItem
+	List   []map[string]interface{}
+}
+
+// SearchEsCols 数据源ES搜索字段
+type SearchEsCols struct {
+	PrimaryId   string
+	IndexCode   string
+	IndexName   string
+	ClassifyId  string
+	Unit        string
+	Frequency   string
+	StartDate   string
+	EndDate     string
+	LatestValue string
+	CreateTime  string
+	ModifyTime  string
+}
+
+// EsBaseFromIndex 数据源ES统一实现的接口
+type EsBaseFromIndex interface {
+	EsCols() SearchEsCols
+	SourceInfo() (int, int, string)
+}
+
+// GetEsBaseFromIndexByTableName 根据表名获取对应数据源
+func GetEsBaseFromIndexByTableName(tableName string) EsBaseFromIndex {
+	switch tableName {
+	case "base_from_rzd_index":
+		return &BaseFromRzdIndex{}
+	case "base_from_hisugar_index":
+		return &BaseFromHisugarIndex{}
+	case "base_from_ly_index":
+		return &BaseFromLyIndex{}
+	case "base_from_sci_hq_index":
+		return &BaseFromSciHqIndex{}
+	case "base_from_oilchem_index":
+		return &BaseFromOilchemIndex{}
+	case "base_from_ths_hf_index":
+		return &BaseFromThsHfIndex{}
+	case "base_from_ccf_index":
+		return &BaseFromCcfIndex{}
+	case "base_from_usda_fas_index":
+		return &BaseFromUsdaFasIndex{}
+	case "base_from_mysteel_chemical_index":
+		return &BaseFromMysteelChemicalIndex{}
+	case "base_from_smm_index":
+		return &BaseFromSmmIndex{}
+	case "base_from_baiinfo_index":
+		return &BaseFromBaiinfoIndex{}
+	case "base_from_sci_index":
+		return &BaseFromSciIndex{}
+	case "base_from_coalmine_mapping":
+		return &BaseFromCoalmineMapping{}
+	case "base_from_eia_steo_index":
+		return &BaseFromEiaSteoIndex{}
+	case "base_from_icpi_index":
+		return &BaseFromIcpiIndex{}
+	case "base_from_yongyi_index":
+		return &BaseFromYongyiIndex{}
+	case "base_from_fenwei_index":
+		return &BaseFromFenweiIndex{}
+	case "base_from_sci99_index":
+		return &BaseFromSci99Index{}
+	case "mb_index_main_info":
+		return &BaseFromGlIndex{}
+	case "edbinfo":
+		return &BaseFromManualEdb{}
+	case "base_from_business_index":
+		return &BaseFromBusinessIndex{}
+	case "base_from_bloomberg_index":
+		return &BaseFromBloombergIndex{}
+	case "base_from_mtjh_mapping":
+		return &BaseFromMtjhMapping{}
+	}
+
+	return nil
+}
+
+// BaseFromRzdIndex 睿咨得
+type BaseFromRzdIndex struct {
+	BaseFromRzdIndexId    int       `orm:"column(base_from_rzd_index_id);pk"`
+	BaseFromRzdClassifyId int       `description:"分类ID"`
+	IndexCode             string    `description:"指标编码"`
+	IndexName             string    `description:"指标名称"`
+	Unit                  string    `description:"单位"`
+	Frequency             string    `description:"频度"`
+	StartDate             time.Time `description:"开始日期"`
+	EndDate               time.Time `description:"结束日期"`
+	LatestValue           float64   `description:"最新值"`
+	CreateTime            time.Time `description:"创建时间"`
+	ModifyTime            time.Time `description:"更新时间"`
+}
+
+func (m *BaseFromRzdIndex) SourceInfo() (int, int, string) {
+	return utils.DATA_SOURCE_RZD, 0, "睿咨得"
+}
+
+func (m *BaseFromRzdIndex) EsCols() SearchEsCols {
+	return SearchEsCols{
+		PrimaryId:   "base_from_rzd_index_id",
+		IndexCode:   "index_code",
+		IndexName:   "index_name",
+		ClassifyId:  "base_from_rzd_classify_id",
+		Unit:        "unit",
+		Frequency:   "frequency",
+		StartDate:   "start_date",
+		EndDate:     "end_date",
+		LatestValue: "latest_value",
+		CreateTime:  "create_time",
+		ModifyTime:  "modify_time",
+	}
+}
+
+func (m *BaseFromRzdIndex) Format2SearchDataSource(origin *BaseFromRzdIndex) (item *SearchDataSource) {
+	if origin == nil {
+		return
+	}
+	source, subSource, sourceName := m.SourceInfo()
+	item = new(SearchDataSource)
+	item.PrimaryId = origin.BaseFromRzdIndexId
+	item.IndexCode = origin.IndexCode
+	item.IndexName = origin.IndexName
+	item.ClassifyId = origin.BaseFromRzdClassifyId
+	item.Unit = origin.Unit
+	item.Frequency = origin.Frequency
+	item.StartDate = utils.TimeTransferString(utils.FormatDate, origin.StartDate)
+	item.EndDate = utils.TimeTransferString(utils.FormatDate, origin.EndDate)
+	item.LatestValue = origin.LatestValue
+	item.Source = source
+	item.SubSource = subSource
+	item.SourceName = sourceName
+	item.CreateTime = utils.TimeTransferString(utils.FormatDateTime, origin.CreateTime)
+	item.ModifyTime = utils.TimeTransferString(utils.FormatDateTime, origin.ModifyTime)
+	return
+}
+
+func (m *BaseFromRzdIndex) GetItemsByCondition(condition string, pars []interface{}, fieldArr []string, orderRule string) (items []*BaseFromRzdIndex, err error) {
+	o := orm.NewOrmUsingDB("data")
+	fields := strings.Join(fieldArr, ",")
+	if len(fieldArr) == 0 {
+		fields = `*`
+	}
+	order := `ORDER BY create_time DESC`
+	if orderRule != "" {
+		order = ` ORDER BY ` + orderRule
+	}
+	sql := fmt.Sprintf(`SELECT %s FROM base_from_rzd_index WHERE 1=1 %s %s`, fields, condition, order)
+	_, err = o.Raw(sql, pars).QueryRows(&items)
+	return
+}
+
+// BaseFromHisugarIndex 泛糖科技
+type BaseFromHisugarIndex struct {
+	BaseFromHisugarIndexId int       `orm:"column(base_from_hisugar_index_id);pk"`
+	ClassifyId             int       `description:"分类ID"`
+	IndexCode              string    `description:"指标编码"`
+	IndexName              string    `description:"指标名称"`
+	Unit                   string    `description:"单位"`
+	Frequency              string    `description:"频度"`
+	Source                 string    `description:"数据来源"`
+	Describe               string    `description:"指标描述"`
+	Sort                   int       `description:"排序"`
+	StartDate              time.Time `description:"开始日期"`
+	EndDate                time.Time `description:"结束日期"`
+	LatestValue            float64   `description:"最新值"`
+	CreateTime             time.Time `description:"创建时间"`
+	ModifyTime             time.Time `description:"更新时间"`
+}
+
+func (m *BaseFromHisugarIndex) SourceInfo() (int, int, string) {
+	return utils.DATA_SOURCE_HISUGAR, 0, "泛糖科技"
+}
+
+func (m *BaseFromHisugarIndex) EsCols() SearchEsCols {
+	return SearchEsCols{
+		PrimaryId:   "base_from_hisugar_index_id",
+		IndexCode:   "index_code",
+		IndexName:   "index_name",
+		ClassifyId:  "classify_id",
+		Unit:        "unit",
+		Frequency:   "frequency",
+		StartDate:   "start_date",
+		EndDate:     "end_date",
+		LatestValue: "latest_value",
+		CreateTime:  "create_time",
+		ModifyTime:  "modify_time",
+	}
+}
+
+func (m *BaseFromHisugarIndex) Format2SearchDataSource(origin *BaseFromHisugarIndex) (item *SearchDataSource) {
+	if origin == nil {
+		return
+	}
+	source, subSource, sourceName := m.SourceInfo()
+	item = new(SearchDataSource)
+	item.PrimaryId = origin.BaseFromHisugarIndexId
+	item.IndexCode = origin.IndexCode
+	item.IndexName = origin.IndexName
+	item.ClassifyId = origin.ClassifyId
+	item.Unit = origin.Unit
+	item.Frequency = origin.Frequency
+	item.StartDate = utils.TimeTransferString(utils.FormatDate, origin.StartDate)
+	item.EndDate = utils.TimeTransferString(utils.FormatDate, origin.EndDate)
+	item.LatestValue = origin.LatestValue
+	item.Source = source
+	item.SubSource = subSource
+	item.SourceName = sourceName
+	item.CreateTime = utils.TimeTransferString(utils.FormatDateTime, origin.CreateTime)
+	item.ModifyTime = utils.TimeTransferString(utils.FormatDateTime, origin.ModifyTime)
+	return
+}
+
+func (m *BaseFromHisugarIndex) GetItemsByCondition(condition string, pars []interface{}, fieldArr []string, orderRule string) (items []*BaseFromHisugarIndex, err error) {
+	o := orm.NewOrmUsingDB("data")
+	fields := strings.Join(fieldArr, ",")
+	if len(fieldArr) == 0 {
+		fields = `*`
+	}
+	order := `ORDER BY create_time DESC`
+	if orderRule != "" {
+		order = ` ORDER BY ` + orderRule
+	}
+	sql := fmt.Sprintf(`SELECT %s FROM base_from_hisugar_index WHERE 1=1 %s %s`, fields, condition, order)
+	_, err = o.Raw(sql, pars).QueryRows(&items)
+	return
+}
+
+// BaseFromLyIndex 粮油商务网
+type BaseFromLyIndex struct {
+	BaseFromLyIndexId    int       `orm:"column(base_from_ly_index_id);pk"`
+	BaseFromLyClassifyId int       `description:"分类ID"`
+	IndexCode            string    `description:"指标编码"`
+	IndexName            string    `description:"指标名称"`
+	Unit                 string    `description:"单位"`
+	Frequency            string    `description:"频度"`
+	StartDate            time.Time `description:"开始日期"`
+	EndDate              time.Time `description:"结束日期"`
+	LatestValue          float64   `description:"最新值"`
+	EdbExist             int       `description:"指标库是否已添加：0-否；1-是"`
+	CreateTime           time.Time `description:"创建时间"`
+	ModifyTime           time.Time `description:"更新时间"`
+}
+
+func (m *BaseFromLyIndex) SourceInfo() (int, int, string) {
+	return utils.DATA_SOURCE_LY, 0, "粮油商务网"
+}
+
+func (m *BaseFromLyIndex) EsCols() SearchEsCols {
+	return SearchEsCols{
+		PrimaryId:   "base_from_ly_index_id",
+		IndexCode:   "index_code",
+		IndexName:   "index_name",
+		ClassifyId:  "base_from_ly_classify_id",
+		Unit:        "unit",
+		Frequency:   "frequency",
+		StartDate:   "start_date",
+		EndDate:     "end_date",
+		LatestValue: "latest_value",
+		CreateTime:  "create_time",
+		ModifyTime:  "modify_time",
+	}
+}
+
+func (m *BaseFromLyIndex) Format2SearchDataSource(origin *BaseFromLyIndex) (item *SearchDataSource) {
+	if origin == nil {
+		return
+	}
+	source, subSource, sourceName := m.SourceInfo()
+	item = new(SearchDataSource)
+	item.PrimaryId = origin.BaseFromLyIndexId
+	item.IndexCode = origin.IndexCode
+	item.IndexName = origin.IndexName
+	item.ClassifyId = origin.BaseFromLyClassifyId
+	item.Unit = origin.Unit
+	item.Frequency = origin.Frequency
+	item.StartDate = utils.TimeTransferString(utils.FormatDate, origin.StartDate)
+	item.EndDate = utils.TimeTransferString(utils.FormatDate, origin.EndDate)
+	item.LatestValue = origin.LatestValue
+	item.Source = source
+	item.SubSource = subSource
+	item.SourceName = sourceName
+	item.CreateTime = utils.TimeTransferString(utils.FormatDateTime, origin.CreateTime)
+	item.ModifyTime = utils.TimeTransferString(utils.FormatDateTime, origin.ModifyTime)
+	return
+}
+
+func (m *BaseFromLyIndex) GetItemsByCondition(condition string, pars []interface{}, fieldArr []string, orderRule string) (items []*BaseFromLyIndex, err error) {
+	o := orm.NewOrmUsingDB("data")
+	fields := strings.Join(fieldArr, ",")
+	if len(fieldArr) == 0 {
+		fields = `*`
+	}
+	order := `ORDER BY create_time DESC`
+	if orderRule != "" {
+		order = ` ORDER BY ` + orderRule
+	}
+	sql := fmt.Sprintf(`SELECT %s FROM base_from_ly_index WHERE 1=1 %s %s`, fields, condition, order)
+	_, err = o.Raw(sql, pars).QueryRows(&items)
+	return
+}
+
+// BaseFromSciHqIndex 卓创红期
+type BaseFromSciHqIndex struct {
+	BaseFromSciHqIndexId int       `orm:"column(base_from_sci_hq_index_id);pk"`
+	ClassifyId           int       `description:"分类ID"`
+	IndexCode            string    `description:"指标编码"`
+	IndexName            string    `description:"指标名称"`
+	Unit                 string    `description:"单位"`
+	Frequency            string    `description:"频度"`
+	Sort                 int       `description:"排序"`
+	StartDate            time.Time `description:"开始日期"`
+	EndDate              time.Time `description:"结束日期"`
+	LatestValue          float64   `description:"最新值"`
+	LatestDate           time.Time `description:"最新更新时间"`
+	TerminalCode         string    `description:"指标描述"`
+	FilePath             string    `description:"文件路径"`
+	CreateTime           time.Time `description:"创建时间"`
+	ModifyTime           time.Time `description:"更新时间"`
+}
+
+func (m *BaseFromSciHqIndex) SourceInfo() (int, int, string) {
+	return utils.DATA_SOURCE_SCI_HQ, 0, "卓创红期"
+}
+
+func (m *BaseFromSciHqIndex) EsCols() SearchEsCols {
+	return SearchEsCols{
+		PrimaryId:   "base_from_sci_hq_index_id",
+		IndexCode:   "index_code",
+		IndexName:   "index_name",
+		ClassifyId:  "classify_id",
+		Unit:        "unit",
+		Frequency:   "frequency",
+		StartDate:   "start_date",
+		EndDate:     "end_date",
+		LatestValue: "latest_value",
+		CreateTime:  "create_time",
+		ModifyTime:  "modify_time",
+	}
+}
+
+func (m *BaseFromSciHqIndex) Format2SearchDataSource(origin *BaseFromSciHqIndex) (item *SearchDataSource) {
+	if origin == nil {
+		return
+	}
+	source, subSource, sourceName := m.SourceInfo()
+	item = new(SearchDataSource)
+	item.PrimaryId = origin.BaseFromSciHqIndexId
+	item.IndexCode = origin.IndexCode
+	item.IndexName = origin.IndexName
+	item.ClassifyId = origin.ClassifyId
+	item.Unit = origin.Unit
+	item.Frequency = origin.Frequency
+	item.StartDate = utils.TimeTransferString(utils.FormatDate, origin.StartDate)
+	item.EndDate = utils.TimeTransferString(utils.FormatDate, origin.EndDate)
+	item.LatestValue = origin.LatestValue
+	item.Source = source
+	item.SubSource = subSource
+	item.SourceName = sourceName
+	item.CreateTime = utils.TimeTransferString(utils.FormatDateTime, origin.CreateTime)
+	item.ModifyTime = utils.TimeTransferString(utils.FormatDateTime, origin.ModifyTime)
+	return
+}
+
+func (m *BaseFromSciHqIndex) GetItemsByCondition(condition string, pars []interface{}, fieldArr []string, orderRule string) (items []*BaseFromSciHqIndex, err error) {
+	o := orm.NewOrmUsingDB("data")
+	fields := strings.Join(fieldArr, ",")
+	if len(fieldArr) == 0 {
+		fields = `*`
+	}
+	order := `ORDER BY create_time DESC`
+	if orderRule != "" {
+		order = ` ORDER BY ` + orderRule
+	}
+	sql := fmt.Sprintf(`SELECT %s FROM base_from_sci_hq_index WHERE 1=1 %s %s`, fields, condition, order)
+	_, err = o.Raw(sql, pars).QueryRows(&items)
+	return
+}
+
+// BaseFromOilchemIndex 隆众资讯
+type BaseFromOilchemIndex struct {
+	BaseFromOilchemIndexId int       `orm:"column(base_from_oilchem_index_id);pk"`
+	ClassifyId             int       `description:"分类ID"`
+	IndexCode              string    `description:"指标编码"`
+	IndexName              string    `description:"指标名称"`
+	Unit                   string    `description:"单位"`
+	Frequency              string    `description:"频度"`
+	StartDate              time.Time `description:"开始日期"`
+	EndDate                time.Time `description:"结束日期"`
+	LatestValue            float64   `description:"最新值"`
+	CreateTime             time.Time `description:"创建时间"`
+	ModifyTime             time.Time `description:"更新时间"`
+}
+
+func (m *BaseFromOilchemIndex) SourceInfo() (int, int, string) {
+	return utils.DATA_SOURCE_OILCHEM, 0, "隆众资讯"
+}
+
+func (m *BaseFromOilchemIndex) EsCols() SearchEsCols {
+	return SearchEsCols{
+		PrimaryId:   "base_from_oilchem_index_id",
+		IndexCode:   "index_code",
+		IndexName:   "index_name",
+		ClassifyId:  "classify_id",
+		Unit:        "unit",
+		Frequency:   "frequency",
+		StartDate:   "start_date",
+		EndDate:     "end_date",
+		LatestValue: "latest_value",
+		CreateTime:  "create_time",
+		ModifyTime:  "modify_time",
+	}
+}
+
+func (m *BaseFromOilchemIndex) Format2SearchDataSource(origin *BaseFromOilchemIndex) (item *SearchDataSource) {
+	if origin == nil {
+		return
+	}
+	source, subSource, sourceName := m.SourceInfo()
+	item = new(SearchDataSource)
+	item.PrimaryId = origin.BaseFromOilchemIndexId
+	item.IndexCode = origin.IndexCode
+	item.IndexName = origin.IndexName
+	item.ClassifyId = origin.ClassifyId
+	item.Unit = origin.Unit
+	item.Frequency = origin.Frequency
+	item.StartDate = utils.TimeTransferString(utils.FormatDate, origin.StartDate)
+	item.EndDate = utils.TimeTransferString(utils.FormatDate, origin.EndDate)
+	item.LatestValue = origin.LatestValue
+	item.Source = source
+	item.SubSource = subSource
+	item.SourceName = sourceName
+	item.CreateTime = utils.TimeTransferString(utils.FormatDateTime, origin.CreateTime)
+	item.ModifyTime = utils.TimeTransferString(utils.FormatDateTime, origin.ModifyTime)
+	return
+}
+
+func (m *BaseFromOilchemIndex) GetItemsByCondition(condition string, pars []interface{}, fieldArr []string, orderRule string) (items []*BaseFromOilchemIndex, err error) {
+	o := orm.NewOrmUsingDB("data")
+	fields := strings.Join(fieldArr, ",")
+	if len(fieldArr) == 0 {
+		fields = `*`
+	}
+	order := `ORDER BY create_time DESC`
+	if orderRule != "" {
+		order = ` ORDER BY ` + orderRule
+	}
+	sql := fmt.Sprintf(`SELECT %s FROM base_from_oilchem_index WHERE 1=1 %s %s`, fields, condition, order)
+	_, err = o.Raw(sql, pars).QueryRows(&items)
+	return
+}
+
+// BaseFromThsHfIndex 同花顺高频数据
+type BaseFromThsHfIndex struct {
+	BaseFromThsHfIndexId    int       `orm:"column(base_from_ths_hf_index_id);pk"`
+	BaseFromThsHfClassifyId int       `description:"分类ID"`
+	IndexCode               string    `description:"指标编码"`
+	IndexName               string    `description:"指标名称"`
+	Unit                    string    `description:"单位"`
+	Source                  string    `description:"数据来源"`
+	Frequency               string    `description:"频度"`
+	StartDate               time.Time `description:"开始日期（至时分秒）"`
+	EndDate                 time.Time `description:"结束日期（至时分秒）"`
+	Describe                string    `description:"指标描述"`
+	Sort                    int       `description:"排序"`
+	IsStop                  int       `description:"是否停更：0-否；1-停更"`
+	TerminalCode            string    `description:"所属终端编码"`
+	StockCode               string    `description:"证券代码"`
+	Indicator               string    `description:"同花顺指标代码"`
+	ApiPars                 string    `description:"API请求参数"`
+	LatestValue             float64   `description:"最新值"`
+	SysUserId               int       `description:"创建人ID"`
+	SysUserRealName         string    `description:"创建人姓名"`
+	CreateTime              time.Time `description:"创建时间"`
+	ModifyTime              time.Time `description:"修改时间"`
+}
+
+func (m *BaseFromThsHfIndex) SourceInfo() (int, int, string) {
+	return utils.DATA_SOURCE_THS, utils.DATA_SUB_SOURCE_HIGH_FREQUENCY, "同花顺高频"
+}
+
+func (m *BaseFromThsHfIndex) EsCols() SearchEsCols {
+	return SearchEsCols{
+		PrimaryId:   "base_from_ths_hf_index_id",
+		IndexCode:   "index_code",
+		IndexName:   "index_name",
+		ClassifyId:  "base_from_ths_hf_classify_id",
+		Unit:        "unit",
+		Frequency:   "frequency",
+		StartDate:   "start_date",
+		EndDate:     "end_date",
+		LatestValue: "latest_value",
+		CreateTime:  "create_time",
+		ModifyTime:  "modify_time",
+	}
+}
+
+func (m *BaseFromThsHfIndex) Format2SearchDataSource(origin *BaseFromThsHfIndex) (item *SearchDataSource) {
+	if origin == nil {
+		return
+	}
+	source, subSource, sourceName := m.SourceInfo()
+	item = new(SearchDataSource)
+	item.PrimaryId = origin.BaseFromThsHfIndexId
+	item.IndexCode = origin.IndexCode
+	item.IndexName = origin.IndexName
+	item.ClassifyId = origin.BaseFromThsHfClassifyId
+	item.Unit = origin.Unit
+	item.Frequency = origin.Frequency
+	item.StartDate = utils.TimeTransferString(utils.FormatDateTime, origin.StartDate) // 高频这里是到时分秒
+	item.EndDate = utils.TimeTransferString(utils.FormatDateTime, origin.EndDate)
+	item.LatestValue = origin.LatestValue
+	item.Source = source
+	item.SubSource = subSource
+	item.SourceName = sourceName
+	item.CreateTime = utils.TimeTransferString(utils.FormatDateTime, origin.CreateTime)
+	item.ModifyTime = utils.TimeTransferString(utils.FormatDateTime, origin.ModifyTime)
+	return
+}
+
+func (m *BaseFromThsHfIndex) GetItemsByCondition(condition string, pars []interface{}, fieldArr []string, orderRule string) (items []*BaseFromThsHfIndex, err error) {
+	o := orm.NewOrmUsingDB("data")
+	fields := strings.Join(fieldArr, ",")
+	if len(fieldArr) == 0 {
+		fields = `*`
+	}
+	order := `ORDER BY create_time DESC`
+	if orderRule != "" {
+		order = ` ORDER BY ` + orderRule
+	}
+	sql := fmt.Sprintf(`SELECT %s FROM base_from_ths_hf_index WHERE 1=1 %s %s`, fields, condition, order)
+	_, err = o.Raw(sql, pars).QueryRows(&items)
+	return
+}
+
+// BaseFromCcfIndex CCF化纤信息
+type BaseFromCcfIndex struct {
+	BaseFromCcfIndexId int       `orm:"column(base_from_ccf_index_id);pk"`
+	ClassifyId         int       `description:"分类ID"`
+	IndexCode          string    `description:"指标编码"`
+	IndexName          string    `description:"指标名称"`
+	Unit               string    `description:"单位"`
+	Frequency          string    `description:"频度"`
+	StartDate          time.Time `description:"开始日期"`
+	EndDate            time.Time `description:"结束日期"`
+	LatestValue        float64   `description:"最新值"`
+	CreateTime         time.Time `description:"创建时间"`
+	ModifyTime         time.Time `description:"更新时间"`
+}
+
+func (m *BaseFromCcfIndex) SourceInfo() (int, int, string) {
+	return utils.DATA_SOURCE_CCF, 0, "CCF化纤信息"
+}
+
+func (m *BaseFromCcfIndex) EsCols() SearchEsCols {
+	return SearchEsCols{
+		PrimaryId:   "base_from_ccf_index_id",
+		IndexCode:   "index_code",
+		IndexName:   "index_name",
+		ClassifyId:  "classify_id",
+		Unit:        "unit",
+		Frequency:   "frequency",
+		StartDate:   "start_date",
+		EndDate:     "end_date",
+		LatestValue: "latest_value",
+		CreateTime:  "create_time",
+		ModifyTime:  "modify_time",
+	}
+}
+
+func (m *BaseFromCcfIndex) Format2SearchDataSource(origin *BaseFromCcfIndex) (item *SearchDataSource) {
+	if origin == nil {
+		return
+	}
+	source, subSource, sourceName := m.SourceInfo()
+	item = new(SearchDataSource)
+	item.PrimaryId = origin.BaseFromCcfIndexId
+	item.IndexCode = origin.IndexCode
+	item.IndexName = origin.IndexName
+	item.ClassifyId = origin.ClassifyId
+	item.Unit = origin.Unit
+	item.Frequency = origin.Frequency
+	item.StartDate = utils.TimeTransferString(utils.FormatDate, origin.StartDate)
+	item.EndDate = utils.TimeTransferString(utils.FormatDate, origin.EndDate)
+	item.LatestValue = origin.LatestValue
+	item.Source = source
+	item.SubSource = subSource
+	item.SourceName = sourceName
+	item.CreateTime = utils.TimeTransferString(utils.FormatDateTime, origin.CreateTime)
+	item.ModifyTime = utils.TimeTransferString(utils.FormatDateTime, origin.ModifyTime)
+	return
+}
+
+func (m *BaseFromCcfIndex) GetItemsByCondition(condition string, pars []interface{}, fieldArr []string, orderRule string) (items []*BaseFromCcfIndex, err error) {
+	o := orm.NewOrmUsingDB("data")
+	fields := strings.Join(fieldArr, ",")
+	if len(fieldArr) == 0 {
+		fields = `*`
+	}
+	order := `ORDER BY create_time DESC`
+	if orderRule != "" {
+		order = ` ORDER BY ` + orderRule
+	}
+	sql := fmt.Sprintf(`SELECT %s FROM base_from_ccf_index WHERE 1=1 %s %s`, fields, condition, order)
+	_, err = o.Raw(sql, pars).QueryRows(&items)
+	return
+}
+
+// BaseFromUsdaFasIndex 美国农业部
+type BaseFromUsdaFasIndex struct {
+	BaseFromUsdaFasIndexId int       `orm:"column(base_from_usda_fas_index_id);pk"`
+	ClassifyId             int       `description:"分类ID"`
+	IndexCode              string    `description:"指标编码"`
+	IndexName              string    `description:"指标名称"`
+	Unit                   string    `description:"单位"`
+	Frequency              string    `description:"频度"`
+	StartDate              time.Time `description:"开始日期"`
+	EndDate                time.Time `description:"结束日期"`
+	LatestValue            float64   `description:"最新值"`
+	CreateTime             time.Time `description:"创建时间"`
+	ModifyTime             time.Time `description:"更新时间"`
+}
+
+func (m *BaseFromUsdaFasIndex) SourceInfo() (int, int, string) {
+	return utils.DATA_SOURCE_USDA_FAS, 0, "美国农业部"
+}
+
+func (m *BaseFromUsdaFasIndex) EsCols() SearchEsCols {
+	return SearchEsCols{
+		PrimaryId:   "base_from_usda_fas_index_id",
+		IndexCode:   "index_code",
+		IndexName:   "index_name",
+		ClassifyId:  "classify_id",
+		Unit:        "unit",
+		Frequency:   "frequency",
+		StartDate:   "start_date",
+		EndDate:     "end_date",
+		LatestValue: "end_value",
+		CreateTime:  "create_time",
+		ModifyTime:  "modify_time",
+	}
+}
+
+func (m *BaseFromUsdaFasIndex) Format2SearchDataSource(origin *BaseFromUsdaFasIndex) (item *SearchDataSource) {
+	if origin == nil {
+		return
+	}
+	source, subSource, sourceName := m.SourceInfo()
+	item = new(SearchDataSource)
+	item.PrimaryId = origin.BaseFromUsdaFasIndexId
+	item.IndexCode = origin.IndexCode
+	item.IndexName = origin.IndexName
+	item.ClassifyId = origin.ClassifyId
+	item.Unit = origin.Unit
+	item.Frequency = origin.Frequency
+	item.StartDate = utils.TimeTransferString(utils.FormatDate, origin.StartDate)
+	item.EndDate = utils.TimeTransferString(utils.FormatDate, origin.EndDate)
+	item.LatestValue = origin.LatestValue
+	item.Source = source
+	item.SubSource = subSource
+	item.SourceName = sourceName
+	item.CreateTime = utils.TimeTransferString(utils.FormatDateTime, origin.CreateTime)
+	item.ModifyTime = utils.TimeTransferString(utils.FormatDateTime, origin.ModifyTime)
+	return
+}
+
+func (m *BaseFromUsdaFasIndex) GetItemsByCondition(condition string, pars []interface{}, fieldArr []string, orderRule string) (items []*BaseFromUsdaFasIndex, err error) {
+	o := orm.NewOrmUsingDB("data")
+	fields := strings.Join(fieldArr, ",")
+	if len(fieldArr) == 0 {
+		fields = `*`
+	}
+	order := `ORDER BY create_time DESC`
+	if orderRule != "" {
+		order = ` ORDER BY ` + orderRule
+	}
+	sql := fmt.Sprintf(`SELECT %s, end_value AS latest_value FROM base_from_usda_fas_index WHERE 1=1 %s %s`, fields, condition, order)
+	_, err = o.Raw(sql, pars).QueryRows(&items)
+	return
+}
+
+// BaseFromMysteelChemicalIndex 上海钢联
+type BaseFromMysteelChemicalIndex struct {
+	BaseFromMysteelChemicalIndexId    int       `orm:"column(base_from_mysteel_chemical_index_id);pk"`
+	BaseFromMysteelChemicalClassifyId int       `description:"分类ID"`
+	IndexCode                         string    `description:"指标编码"`
+	IndexName                         string    `description:"指标名称"`
+	Unit                              string    `description:"单位"`
+	Frequency                         string    `description:"频度"`
+	StartDate                         time.Time `description:"开始日期"`
+	EndDate                           time.Time `description:"结束日期"`
+	LatestValue                       float64   `description:"最新值"`
+	CreateTime                        time.Time `description:"创建时间"`
+	ModifyTime                        time.Time `description:"更新时间"`
+}
+
+func (m *BaseFromMysteelChemicalIndex) SourceInfo() (int, int, string) {
+	return utils.DATA_SOURCE_MYSTEEL_CHEMICAL, 0, "上海钢联"
+}
+
+func (m *BaseFromMysteelChemicalIndex) EsCols() SearchEsCols {
+	return SearchEsCols{
+		PrimaryId:   "base_from_mysteel_chemical_index_id",
+		IndexCode:   "index_code",
+		IndexName:   "index_name",
+		ClassifyId:  "base_from_mysteel_chemical_classify_id",
+		Unit:        "unit",
+		Frequency:   "frequency",
+		StartDate:   "start_date",
+		EndDate:     "end_date",
+		LatestValue: "end_value",
+		CreateTime:  "create_time",
+		ModifyTime:  "modify_time",
+	}
+}
+
+func (m *BaseFromMysteelChemicalIndex) Format2SearchDataSource(origin *BaseFromMysteelChemicalIndex) (item *SearchDataSource) {
+	if origin == nil {
+		return
+	}
+	source, subSource, sourceName := m.SourceInfo()
+	item = new(SearchDataSource)
+	item.PrimaryId = origin.BaseFromMysteelChemicalIndexId
+	item.IndexCode = origin.IndexCode
+	item.IndexName = origin.IndexName
+	item.ClassifyId = origin.BaseFromMysteelChemicalClassifyId
+	item.Unit = origin.Unit
+	item.Frequency = origin.Frequency
+	item.StartDate = utils.TimeTransferString(utils.FormatDate, origin.StartDate)
+	item.EndDate = utils.TimeTransferString(utils.FormatDate, origin.EndDate)
+	item.LatestValue = origin.LatestValue
+	item.Source = source
+	item.SubSource = subSource
+	item.SourceName = sourceName
+	item.CreateTime = utils.TimeTransferString(utils.FormatDateTime, origin.CreateTime)
+	item.ModifyTime = utils.TimeTransferString(utils.FormatDateTime, origin.ModifyTime)
+	return
+}
+
+func (m *BaseFromMysteelChemicalIndex) GetItemsByCondition(condition string, pars []interface{}, fieldArr []string, orderRule string) (items []*BaseFromMysteelChemicalIndex, err error) {
+	o := orm.NewOrmUsingDB("data")
+	fields := strings.Join(fieldArr, ",")
+	if len(fieldArr) == 0 {
+		fields = `*`
+	}
+	order := `ORDER BY create_time DESC`
+	if orderRule != "" {
+		order = ` ORDER BY ` + orderRule
+	}
+	sql := fmt.Sprintf(`SELECT %s, end_value AS latest_value FROM base_from_mysteel_chemical_index WHERE 1=1 %s %s`, fields, condition, order)
+	_, err = o.Raw(sql, pars).QueryRows(&items)
+	return
+}
+
+// BaseFromSmmIndex 有色原始数据库
+type BaseFromSmmIndex struct {
+	BaseFromSmmIndexId int       `orm:"column(base_from_smm_index_id);pk"`
+	ClassifyId         int       `description:"分类ID"`
+	IndexCode          string    `description:"指标编码"`
+	IndexName          string    `description:"指标名称"`
+	Unit               string    `description:"单位"`
+	Frequency          string    `description:"频度"`
+	StartDate          time.Time `description:"开始日期"`
+	EndDate            time.Time `description:"结束日期"`
+	LatestValue        float64   `description:"最新值"`
+	CreateTime         time.Time `description:"创建时间"`
+	ModifyTime         time.Time `description:"更新时间"`
+}
+
+func (m *BaseFromSmmIndex) SourceInfo() (int, int, string) {
+	return utils.DATA_SOURCE_YS, 0, "SMM原始数据库"
+}
+
+func (m *BaseFromSmmIndex) EsCols() SearchEsCols {
+	return SearchEsCols{
+		PrimaryId:   "base_from_smm_index_id",
+		IndexCode:   "index_code",
+		IndexName:   "index_name",
+		ClassifyId:  "classify_id",
+		Unit:        "unit",
+		Frequency:   "frequency",
+		StartDate:   "start_date",
+		EndDate:     "end_date",
+		LatestValue: "end_value",
+		CreateTime:  "create_time",
+		ModifyTime:  "modify_time",
+	}
+}
+
+func (m *BaseFromSmmIndex) Format2SearchDataSource(origin *BaseFromSmmIndex) (item *SearchDataSource) {
+	if origin == nil {
+		return
+	}
+	source, subSource, sourceName := m.SourceInfo()
+	item = new(SearchDataSource)
+	item.PrimaryId = origin.BaseFromSmmIndexId
+	item.IndexCode = origin.IndexCode
+	item.IndexName = origin.IndexName
+	item.ClassifyId = origin.ClassifyId
+	item.Unit = origin.Unit
+	item.Frequency = origin.Frequency
+	item.StartDate = utils.TimeTransferString(utils.FormatDate, origin.StartDate)
+	item.EndDate = utils.TimeTransferString(utils.FormatDate, origin.EndDate)
+	item.LatestValue = origin.LatestValue
+	item.Source = source
+	item.SubSource = subSource
+	item.SourceName = sourceName
+	item.CreateTime = utils.TimeTransferString(utils.FormatDateTime, origin.CreateTime)
+	item.ModifyTime = utils.TimeTransferString(utils.FormatDateTime, origin.ModifyTime)
+	return
+}
+
+func (m *BaseFromSmmIndex) GetItemsByCondition(condition string, pars []interface{}, fieldArr []string, orderRule string) (items []*BaseFromSmmIndex, err error) {
+	o := orm.NewOrmUsingDB("data")
+	fields := strings.Join(fieldArr, ",")
+	if len(fieldArr) == 0 {
+		fields = `*`
+	}
+	order := `ORDER BY create_time DESC`
+	if orderRule != "" {
+		order = ` ORDER BY ` + orderRule
+	}
+	sql := fmt.Sprintf(`SELECT %s, end_value AS latest_value FROM base_from_smm_index WHERE 1=1 %s %s`, fields, condition, order)
+	_, err = o.Raw(sql, pars).QueryRows(&items)
+	return
+}
+
+// BaseFromBaiinfoIndex 百川盈孚
+type BaseFromBaiinfoIndex struct {
+	BaseFromBaiinfoIndexId int       `orm:"column(base_from_baiinfo_index_id);pk"`
+	ClassifyId             int       `description:"分类ID"`
+	IndexCode              string    `description:"指标编码"`
+	IndexName              string    `description:"指标名称"`
+	Unit                   string    `description:"单位"`
+	Frequency              string    `description:"频度"`
+	StartDate              time.Time `description:"开始日期"`
+	EndDate                time.Time `description:"结束日期"`
+	LatestValue            float64   `description:"最新值"`
+	CreateTime             time.Time `description:"创建时间"`
+	ModifyTime             time.Time `description:"更新时间"`
+}
+
+func (m *BaseFromBaiinfoIndex) SourceInfo() (int, int, string) {
+	return utils.DATA_SOURCE_BAIINFO, 0, "百川盈孚"
+}
+
+func (m *BaseFromBaiinfoIndex) EsCols() SearchEsCols {
+	return SearchEsCols{
+		PrimaryId:   "base_from_baiinfo_index_id",
+		IndexCode:   "index_code",
+		IndexName:   "index_name",
+		ClassifyId:  "classify_id",
+		Unit:        "unit",
+		Frequency:   "frequency",
+		StartDate:   "start_date",
+		EndDate:     "end_date",
+		LatestValue: "latest_value",
+		CreateTime:  "create_time",
+		ModifyTime:  "modify_time",
+	}
+}
+
+func (m *BaseFromBaiinfoIndex) Format2SearchDataSource(origin *BaseFromBaiinfoIndex) (item *SearchDataSource) {
+	if origin == nil {
+		return
+	}
+	source, subSource, sourceName := m.SourceInfo()
+	item = new(SearchDataSource)
+	item.PrimaryId = origin.BaseFromBaiinfoIndexId
+	item.IndexCode = origin.IndexCode
+	item.IndexName = origin.IndexName
+	item.ClassifyId = origin.ClassifyId
+	item.Unit = origin.Unit
+	item.Frequency = origin.Frequency
+	item.StartDate = utils.TimeTransferString(utils.FormatDate, origin.StartDate)
+	item.EndDate = utils.TimeTransferString(utils.FormatDate, origin.EndDate)
+	item.LatestValue = origin.LatestValue
+	item.Source = source
+	item.SubSource = subSource
+	item.SourceName = sourceName
+	item.CreateTime = utils.TimeTransferString(utils.FormatDateTime, origin.CreateTime)
+	item.ModifyTime = utils.TimeTransferString(utils.FormatDateTime, origin.ModifyTime)
+	return
+}
+
+func (m *BaseFromBaiinfoIndex) GetItemsByCondition(condition string, pars []interface{}, fieldArr []string, orderRule string) (items []*BaseFromBaiinfoIndex, err error) {
+	o := orm.NewOrmUsingDB("data")
+	fields := strings.Join(fieldArr, ",")
+	if len(fieldArr) == 0 {
+		fields = `*`
+	}
+	order := `ORDER BY create_time DESC`
+	if orderRule != "" {
+		order = ` ORDER BY ` + orderRule
+	}
+	sql := fmt.Sprintf(`SELECT %s FROM base_from_baiinfo_index WHERE 1=1 %s %s`, fields, condition, order)
+	_, err = o.Raw(sql, pars).QueryRows(&items)
+	return
+}
+
+// BaseFromSciIndex 卓创数据（红桃3）
+type BaseFromSciIndex struct {
+	BaseFromSciIndexId int       `orm:"column(base_from_sci_index_id);pk"`
+	ClassifyId         int       `description:"分类ID"`
+	IndexCode          string    `description:"指标编码"`
+	IndexName          string    `description:"指标名称"`
+	Unit               string    `description:"单位"`
+	Frequency          string    `description:"频度"`
+	StartDate          time.Time `description:"开始日期"`
+	EndDate            time.Time `description:"结束日期"`
+	LatestValue        float64   `description:"最新值"`
+	CreateTime         time.Time `description:"创建时间"`
+	ModifyTime         time.Time `description:"更新时间"`
+}
+
+func (m *BaseFromSciIndex) SourceInfo() (int, int, string) {
+	return utils.DATA_SOURCE_SCI, 0, "卓创数据（红桃3）"
+}
+
+func (m *BaseFromSciIndex) EsCols() SearchEsCols {
+	return SearchEsCols{
+		PrimaryId:   "base_from_sci_index_id",
+		IndexCode:   "index_code",
+		IndexName:   "index_name",
+		ClassifyId:  "classify_id",
+		Unit:        "unit",
+		Frequency:   "frequency",
+		StartDate:   "start_date",
+		EndDate:     "end_date",
+		LatestValue: "latest_value",
+		CreateTime:  "create_time",
+		ModifyTime:  "modify_time",
+	}
+}
+
+func (m *BaseFromSciIndex) Format2SearchDataSource(origin *BaseFromSciIndex) (item *SearchDataSource) {
+	if origin == nil {
+		return
+	}
+	source, subSource, sourceName := m.SourceInfo()
+	item = new(SearchDataSource)
+	item.PrimaryId = origin.BaseFromSciIndexId
+	item.IndexCode = origin.IndexCode
+	item.IndexName = origin.IndexName
+	item.ClassifyId = origin.ClassifyId
+	item.Unit = origin.Unit
+	item.Frequency = origin.Frequency
+	item.StartDate = utils.TimeTransferString(utils.FormatDate, origin.StartDate)
+	item.EndDate = utils.TimeTransferString(utils.FormatDate, origin.EndDate)
+	item.LatestValue = origin.LatestValue
+	item.Source = source
+	item.SubSource = subSource
+	item.SourceName = sourceName
+	item.CreateTime = utils.TimeTransferString(utils.FormatDateTime, origin.CreateTime)
+	item.ModifyTime = utils.TimeTransferString(utils.FormatDateTime, origin.ModifyTime)
+	return
+}
+
+func (m *BaseFromSciIndex) GetItemsByCondition(condition string, pars []interface{}, fieldArr []string, orderRule string) (items []*BaseFromSciIndex, err error) {
+	o := orm.NewOrmUsingDB("data")
+	fields := strings.Join(fieldArr, ",")
+	if len(fieldArr) == 0 {
+		fields = `*`
+	}
+	order := `ORDER BY create_time DESC`
+	if orderRule != "" {
+		order = ` ORDER BY ` + orderRule
+	}
+	sql := fmt.Sprintf(`SELECT %s FROM base_from_sci_index WHERE 1=1 %s %s`, fields, condition, order)
+	_, err = o.Raw(sql, pars).QueryRows(&items)
+	return
+}
+
+// BaseFromCoalmineMapping 中国煤炭市场网
+type BaseFromCoalmineMapping struct {
+	BaseFromCoalmineMappingId int       `orm:"column(base_from_coalmine_mapping_id);pk"`
+	ClassifyId                int       `description:"分类ID"`
+	IndexCode                 string    `description:"指标编码"`
+	IndexName                 string    `description:"指标名称"`
+	Unit                      string    `description:"单位"`
+	Frequency                 string    `description:"频度"`
+	StartDate                 time.Time `description:"开始日期"`
+	EndDate                   time.Time `description:"结束日期"`
+	LatestValue               float64   `description:"最新值"`
+	CreateTime                time.Time `description:"创建时间"`
+	ModifyTime                time.Time `description:"更新时间"`
+}
+
+func (m *BaseFromCoalmineMapping) SourceInfo() (int, int, string) {
+	return utils.DATA_SOURCE_COAL, 0, "中国煤炭市场网"
+}
+
+func (m *BaseFromCoalmineMapping) EsCols() SearchEsCols {
+	return SearchEsCols{
+		PrimaryId:   "base_from_coalmine_mapping_id",
+		IndexCode:   "index_code",
+		IndexName:   "index_name",
+		ClassifyId:  "",
+		Unit:        "",
+		Frequency:   "",
+		StartDate:   "",
+		EndDate:     "",
+		LatestValue: "",
+		CreateTime:  "create_time",
+		ModifyTime:  "",
+	}
+}
+
+func (m *BaseFromCoalmineMapping) Format2SearchDataSource(origin *BaseFromCoalmineMapping) (item *SearchDataSource) {
+	if origin == nil {
+		return
+	}
+	source, subSource, sourceName := m.SourceInfo()
+	item = new(SearchDataSource)
+	item.PrimaryId = origin.BaseFromCoalmineMappingId
+	item.IndexCode = origin.IndexCode
+	item.IndexName = origin.IndexName
+	item.ClassifyId = origin.ClassifyId
+	item.Unit = origin.Unit
+	item.Frequency = origin.Frequency
+	item.StartDate = utils.TimeTransferString(utils.FormatDate, origin.StartDate)
+	item.EndDate = utils.TimeTransferString(utils.FormatDate, origin.EndDate)
+	item.LatestValue = origin.LatestValue
+	item.Source = source
+	item.SubSource = subSource
+	item.SourceName = sourceName
+	item.CreateTime = utils.TimeTransferString(utils.FormatDateTime, origin.CreateTime)
+	item.ModifyTime = utils.TimeTransferString(utils.FormatDateTime, origin.ModifyTime)
+	return
+}
+
+func (m *BaseFromCoalmineMapping) GetItemsByCondition(condition string, pars []interface{}, fieldArr []string, orderRule string) (items []*BaseFromCoalmineMapping, err error) {
+	o := orm.NewOrmUsingDB("data")
+	fields := strings.Join(fieldArr, ",")
+	if len(fieldArr) == 0 {
+		fields = `*`
+	}
+	order := `ORDER BY create_time DESC`
+	if orderRule != "" {
+		order = ` ORDER BY ` + orderRule
+	}
+	sql := fmt.Sprintf(`SELECT %s FROM base_from_coalmine_mapping WHERE 1=1 %s %s`, fields, condition, order)
+	_, err = o.Raw(sql, pars).QueryRows(&items)
+	return
+}
+
+// BaseFromEiaSteoIndex EIA STEO报告
+type BaseFromEiaSteoIndex struct {
+	BaseFromEiaSteoIndexId    int       `orm:"column(base_from_eia_steo_index_id);pk"`
+	BaseFromEiaSteoClassifyId int       `description:"分类ID"`
+	IndexCode                 string    `description:"指标编码"`
+	IndexName                 string    `description:"指标名称"`
+	Unit                      string    `description:"单位"`
+	Frequency                 string    `description:"频度"`
+	StartDate                 time.Time `description:"开始日期"`
+	EndDate                   time.Time `description:"结束日期"`
+	LatestValue               float64   `description:"最新值"`
+	CreateTime                time.Time `description:"创建时间"`
+	ModifyTime                time.Time `description:"更新时间"`
+}
+
+func (m *BaseFromEiaSteoIndex) SourceInfo() (int, int, string) {
+	return utils.DATA_SOURCE_EIA_STEO, 0, "EIA STEO报告"
+}
+
+func (m *BaseFromEiaSteoIndex) EsCols() SearchEsCols {
+	return SearchEsCols{
+		PrimaryId:   "base_from_eia_steo_index_id",
+		IndexCode:   "index_code",
+		IndexName:   "index_name",
+		ClassifyId:  "base_from_eia_steo_classify_id",
+		Unit:        "unit",
+		Frequency:   "frequency",
+		StartDate:   "start_date",
+		EndDate:     "end_date",
+		LatestValue: "latest_value",
+		CreateTime:  "create_time",
+		ModifyTime:  "modify_time",
+	}
+}
+
+func (m *BaseFromEiaSteoIndex) Format2SearchDataSource(origin *BaseFromEiaSteoIndex) (item *SearchDataSource) {
+	if origin == nil {
+		return
+	}
+	source, subSource, sourceName := m.SourceInfo()
+	item = new(SearchDataSource)
+	item.PrimaryId = origin.BaseFromEiaSteoIndexId
+	item.IndexCode = origin.IndexCode
+	item.IndexName = origin.IndexName
+	item.ClassifyId = origin.BaseFromEiaSteoClassifyId
+	item.Unit = origin.Unit
+	item.Frequency = origin.Frequency
+	item.StartDate = utils.TimeTransferString(utils.FormatDate, origin.StartDate)
+	item.EndDate = utils.TimeTransferString(utils.FormatDate, origin.EndDate)
+	item.LatestValue = origin.LatestValue
+	item.Source = source
+	item.SubSource = subSource
+	item.SourceName = sourceName
+	item.CreateTime = utils.TimeTransferString(utils.FormatDateTime, origin.CreateTime)
+	item.ModifyTime = utils.TimeTransferString(utils.FormatDateTime, origin.ModifyTime)
+	return
+}
+
+func (m *BaseFromEiaSteoIndex) GetItemsByCondition(condition string, pars []interface{}, fieldArr []string, orderRule string) (items []*BaseFromEiaSteoIndex, err error) {
+	o := orm.NewOrmUsingDB("data")
+	fields := strings.Join(fieldArr, ",")
+	if len(fieldArr) == 0 {
+		fields = `*`
+	}
+	order := `ORDER BY create_time DESC`
+	if orderRule != "" {
+		order = ` ORDER BY ` + orderRule
+	}
+	sql := fmt.Sprintf(`SELECT %s FROM base_from_eia_steo_index WHERE 1=1 %s %s`, fields, condition, order)
+	_, err = o.Raw(sql, pars).QueryRows(&items)
+	return
+}
+
+// BaseFromIcpiIndex ICPI消费价格指数
+type BaseFromIcpiIndex struct {
+	BaseFromIcpiIndexId    int       `orm:"column(base_from_icpi_index_id);pk"`
+	BaseFromIcpiClassifyId int       `description:"分类ID"`
+	IndexCode              string    `description:"指标编码"`
+	IndexName              string    `description:"指标名称"`
+	Unit                   string    `description:"单位"`
+	Frequency              string    `description:"频度"`
+	StartDate              time.Time `description:"开始日期"`
+	EndDate                time.Time `description:"结束日期"`
+	LatestValue            float64   `description:"最新值"`
+	CreateTime             time.Time `description:"创建时间"`
+	ModifyTime             time.Time `description:"更新时间"`
+}
+
+func (m *BaseFromIcpiIndex) SourceInfo() (int, int, string) {
+	return utils.DATA_SOURCE_ICPI, 0, "ICPI消费价格指数"
+}
+
+func (m *BaseFromIcpiIndex) EsCols() SearchEsCols {
+	return SearchEsCols{
+		PrimaryId:   "base_from_icpi_index_id",
+		IndexCode:   "index_code",
+		IndexName:   "index_name",
+		ClassifyId:  "base_from_icpi_classify_id",
+		Unit:        "unit",
+		Frequency:   "frequency",
+		StartDate:   "start_date",
+		EndDate:     "end_date",
+		LatestValue: "latest_value",
+		CreateTime:  "create_time",
+		ModifyTime:  "modify_time",
+	}
+}
+
+func (m *BaseFromIcpiIndex) Format2SearchDataSource(origin *BaseFromIcpiIndex) (item *SearchDataSource) {
+	if origin == nil {
+		return
+	}
+	source, subSource, sourceName := m.SourceInfo()
+	item = new(SearchDataSource)
+	item.PrimaryId = origin.BaseFromIcpiIndexId
+	item.IndexCode = origin.IndexCode
+	item.IndexName = origin.IndexName
+	item.ClassifyId = origin.BaseFromIcpiClassifyId
+	item.Unit = origin.Unit
+	item.Frequency = origin.Frequency
+	item.StartDate = utils.TimeTransferString(utils.FormatDate, origin.StartDate)
+	item.EndDate = utils.TimeTransferString(utils.FormatDate, origin.EndDate)
+	item.LatestValue = origin.LatestValue
+	item.Source = source
+	item.SubSource = subSource
+	item.SourceName = sourceName
+	item.CreateTime = utils.TimeTransferString(utils.FormatDateTime, origin.CreateTime)
+	item.ModifyTime = utils.TimeTransferString(utils.FormatDateTime, origin.ModifyTime)
+	return
+}
+
+func (m *BaseFromIcpiIndex) GetItemsByCondition(condition string, pars []interface{}, fieldArr []string, orderRule string) (items []*BaseFromIcpiIndex, err error) {
+	o := orm.NewOrmUsingDB("data")
+	fields := strings.Join(fieldArr, ",")
+	if len(fieldArr) == 0 {
+		fields = `*`
+	}
+	order := `ORDER BY create_time DESC`
+	if orderRule != "" {
+		order = ` ORDER BY ` + orderRule
+	}
+	sql := fmt.Sprintf(`SELECT %s FROM base_from_icpi_index WHERE 1=1 %s %s`, fields, condition, order)
+	_, err = o.Raw(sql, pars).QueryRows(&items)
+	return
+}
+
+// BaseFromYongyiIndex 涌益咨询
+type BaseFromYongyiIndex struct {
+	YongyiIndexId int       `orm:"column(yongyi_index_id);pk"`
+	ClassifyId    int       `description:"分类ID"`
+	IndexCode     string    `description:"指标编码"`
+	IndexName     string    `description:"指标名称"`
+	Unit          string    `description:"单位"`
+	Frequency     string    `description:"频度"`
+	StartDate     time.Time `description:"开始日期"`
+	EndDate       time.Time `description:"结束日期"`
+	LatestValue   float64   `description:"最新值"`
+	CreateTime    time.Time `description:"创建时间"`
+	ModifyTime    time.Time `description:"更新时间"`
+}
+
+func (m *BaseFromYongyiIndex) SourceInfo() (int, int, string) {
+	return utils.DATA_SOURCE_YONYI, 0, "涌益咨询"
+}
+
+func (m *BaseFromYongyiIndex) EsCols() SearchEsCols {
+	return SearchEsCols{
+		PrimaryId:   "yongyi_index_id",
+		IndexCode:   "index_code",
+		IndexName:   "index_name",
+		ClassifyId:  "classify_id",
+		Unit:        "unit",
+		Frequency:   "frequency",
+		StartDate:   "start_date",
+		EndDate:     "end_date",
+		LatestValue: "latest_value",
+		CreateTime:  "create_time",
+		ModifyTime:  "modify_time",
+	}
+}
+
+func (m *BaseFromYongyiIndex) Format2SearchDataSource(origin *BaseFromYongyiIndex) (item *SearchDataSource) {
+	if origin == nil {
+		return
+	}
+	source, subSource, sourceName := m.SourceInfo()
+	item = new(SearchDataSource)
+	item.PrimaryId = origin.YongyiIndexId
+	item.IndexCode = origin.IndexCode
+	item.IndexName = origin.IndexName
+	item.ClassifyId = origin.ClassifyId
+	item.Unit = origin.Unit
+	item.Frequency = origin.Frequency
+	item.StartDate = utils.TimeTransferString(utils.FormatDate, origin.StartDate)
+	item.EndDate = utils.TimeTransferString(utils.FormatDate, origin.EndDate)
+	item.LatestValue = origin.LatestValue
+	item.Source = source
+	item.SubSource = subSource
+	item.SourceName = sourceName
+	item.CreateTime = utils.TimeTransferString(utils.FormatDateTime, origin.CreateTime)
+	item.ModifyTime = utils.TimeTransferString(utils.FormatDateTime, origin.ModifyTime)
+	return
+}
+
+func (m *BaseFromYongyiIndex) GetItemsByCondition(condition string, pars []interface{}, fieldArr []string, orderRule string) (items []*BaseFromYongyiIndex, err error) {
+	o := orm.NewOrmUsingDB("data")
+	fields := strings.Join(fieldArr, ",")
+	if len(fieldArr) == 0 {
+		fields = `*`
+	}
+	order := `ORDER BY create_time DESC`
+	if orderRule != "" {
+		order = ` ORDER BY ` + orderRule
+	}
+	sql := fmt.Sprintf(`SELECT %s FROM base_from_yongyi_index WHERE 1=1 %s %s`, fields, condition, order)
+	_, err = o.Raw(sql, pars).QueryRows(&items)
+	return
+}
+
+// BaseFromFenweiIndex 汾渭数据
+type BaseFromFenweiIndex struct {
+	FenweiIndexId int       `orm:"column(fenwei_index_id);pk"`
+	ClassifyId    int       `description:"分类ID"`
+	IndexCode     string    `description:"指标编码"`
+	IndexName     string    `description:"指标名称"`
+	Unit          string    `description:"单位"`
+	Frequency     string    `description:"频度"`
+	StartDate     time.Time `description:"开始日期"`
+	EndDate       time.Time `description:"结束日期"`
+	LatestValue   float64   `description:"最新值"`
+	CreateTime    time.Time `description:"创建时间"`
+	ModifyTime    time.Time `description:"更新时间"`
+}
+
+func (m *BaseFromFenweiIndex) SourceInfo() (int, int, string) {
+	return utils.DATA_SOURCE_FENWEI, 0, "汾渭数据"
+}
+
+func (m *BaseFromFenweiIndex) EsCols() SearchEsCols {
+	return SearchEsCols{
+		PrimaryId:   "fenwei_index_id",
+		IndexCode:   "index_code",
+		IndexName:   "index_name",
+		ClassifyId:  "classify_id",
+		Unit:        "unit",
+		Frequency:   "frequency",
+		StartDate:   "start_date",
+		EndDate:     "end_date",
+		LatestValue: "latest_value",
+		CreateTime:  "create_time",
+		ModifyTime:  "modify_time",
+	}
+}
+
+func (m *BaseFromFenweiIndex) Format2SearchDataSource(origin *BaseFromFenweiIndex) (item *SearchDataSource) {
+	if origin == nil {
+		return
+	}
+	source, subSource, sourceName := m.SourceInfo()
+	item = new(SearchDataSource)
+	item.PrimaryId = origin.FenweiIndexId
+	item.IndexCode = origin.IndexCode
+	item.IndexName = origin.IndexName
+	item.ClassifyId = origin.ClassifyId
+	item.Unit = origin.Unit
+	item.Frequency = origin.Frequency
+	item.StartDate = utils.TimeTransferString(utils.FormatDate, origin.StartDate)
+	item.EndDate = utils.TimeTransferString(utils.FormatDate, origin.EndDate)
+	item.LatestValue = origin.LatestValue
+	item.Source = source
+	item.SubSource = subSource
+	item.SourceName = sourceName
+	item.CreateTime = utils.TimeTransferString(utils.FormatDateTime, origin.CreateTime)
+	item.ModifyTime = utils.TimeTransferString(utils.FormatDateTime, origin.ModifyTime)
+	return
+}
+
+func (m *BaseFromFenweiIndex) GetItemsByCondition(condition string, pars []interface{}, fieldArr []string, orderRule string) (items []*BaseFromFenweiIndex, err error) {
+	o := orm.NewOrmUsingDB("data")
+	fields := strings.Join(fieldArr, ",")
+	if len(fieldArr) == 0 {
+		fields = `*`
+	}
+	order := `ORDER BY create_time DESC`
+	if orderRule != "" {
+		order = ` ORDER BY ` + orderRule
+	}
+	sql := fmt.Sprintf(`SELECT %s FROM base_from_fenwei_index WHERE 1=1 %s %s`, fields, condition, order)
+	_, err = o.Raw(sql, pars).QueryRows(&items)
+	return
+}
+
+// BaseFromSci99Index 卓创资讯
+type BaseFromSci99Index struct {
+	BaseFromSciIndexId int       `orm:"column(base_from_sci_index_id);pk"`
+	ClassifyId         int       `description:"分类ID"`
+	IndexCode          string    `description:"指标编码"`
+	IndexName          string    `description:"指标名称"`
+	Unit               string    `description:"单位"`
+	Frequency          string    `description:"频度"`
+	StartDate          time.Time `description:"开始日期"`
+	EndDate            time.Time `description:"结束日期"`
+	LatestValue        float64   `description:"最新值"`
+	CreateTime         time.Time `description:"创建时间"`
+	ModifyTime         time.Time `description:"更新时间"`
+}
+
+func (m *BaseFromSci99Index) SourceInfo() (int, int, string) {
+	return utils.DATA_SOURCE_SCI99, 0, "卓创资讯"
+}
+
+func (m *BaseFromSci99Index) EsCols() SearchEsCols {
+	return SearchEsCols{
+		PrimaryId:   "base_from_sci_index_id",
+		IndexCode:   "index_code",
+		IndexName:   "index_name",
+		ClassifyId:  "classify_id",
+		Unit:        "unit",
+		Frequency:   "frequency",
+		StartDate:   "start_date",
+		EndDate:     "end_date",
+		LatestValue: "latest_value",
+		CreateTime:  "create_time",
+		ModifyTime:  "modify_time",
+	}
+}
+
+func (m *BaseFromSci99Index) Format2SearchDataSource(origin *BaseFromSci99Index) (item *SearchDataSource) {
+	if origin == nil {
+		return
+	}
+	source, subSource, sourceName := m.SourceInfo()
+	item = new(SearchDataSource)
+	item.PrimaryId = origin.BaseFromSciIndexId
+	item.IndexCode = origin.IndexCode
+	item.IndexName = origin.IndexName
+	item.ClassifyId = origin.ClassifyId
+	item.Unit = origin.Unit
+	item.Frequency = origin.Frequency
+	item.StartDate = utils.TimeTransferString(utils.FormatDate, origin.StartDate)
+	item.EndDate = utils.TimeTransferString(utils.FormatDate, origin.EndDate)
+	item.LatestValue = origin.LatestValue
+	item.Source = source
+	item.SubSource = subSource
+	item.SourceName = sourceName
+	item.CreateTime = utils.TimeTransferString(utils.FormatDateTime, origin.CreateTime)
+	item.ModifyTime = utils.TimeTransferString(utils.FormatDateTime, origin.ModifyTime)
+	return
+}
+
+func (m *BaseFromSci99Index) GetItemsByCondition(condition string, pars []interface{}, fieldArr []string, orderRule string) (items []*BaseFromSci99Index, err error) {
+	o := orm.NewOrmUsingDB("data")
+	fields := strings.Join(fieldArr, ",")
+	if len(fieldArr) == 0 {
+		fields = `*`
+	}
+	order := `ORDER BY create_time DESC`
+	if orderRule != "" {
+		order = ` ORDER BY ` + orderRule
+	}
+	sql := fmt.Sprintf(`SELECT %s FROM base_from_sci99_index WHERE 1=1 %s %s`, fields, condition, order)
+	_, err = o.Raw(sql, pars).QueryRows(&items)
+	return
+}
+
+// BaseFromGlIndex 钢联原始指标库
+type BaseFromGlIndex struct {
+	PrimaryId   int       `orm:"column(ID);pk"`
+	IndexCode   string    `orm:"column(INDEX_CODE)" description:"指标编码"`
+	IndexName   string    `orm:"column(INDEX_NAME)" description:"指标名称"`
+	Unit        string    `orm:"column(UNIT_NAME)" description:"单位"`
+	Frequency   string    `orm:"column(FREQUENCY_NAME)" description:"频度"`
+	StartDate   time.Time `orm:"column(BEGIN_DATE)" description:"开始日期"`
+	EndDate     time.Time `orm:"column(END_DATE)" description:"结束日期"`
+	LatestValue float64   `orm:"column(DATA_VALUE)" description:"最新值"`
+	CreateTime  time.Time `orm:"column(CREATE_TIME)" description:"创建时间"`
+	ModifyTime  time.Time `orm:"column(UPDATE_TIME)" description:"更新时间"`
+}
+
+func (m *BaseFromGlIndex) SourceInfo() (int, int, string) {
+	return utils.DATA_SOURCE_GL, 0, "钢联原始数据库"
+}
+
+func (m *BaseFromGlIndex) EsCols() SearchEsCols {
+	return SearchEsCols{
+		PrimaryId:   "ID",
+		IndexCode:   "INDEX_CODE",
+		IndexName:   "INDEX_NAME",
+		ClassifyId:  "",
+		Unit:        "UNIT_NAME",
+		Frequency:   "FREQUENCY_NAME",
+		StartDate:   "BEGIN_DATE",
+		EndDate:     "END_DATE",
+		LatestValue: "DATA_VALUE",
+		CreateTime:  "CREATE_TIME",
+		ModifyTime:  "UPDATE_TIME",
+	}
+}
+
+func (m *BaseFromGlIndex) Format2SearchDataSource(origin *BaseFromGlIndex) (item *SearchDataSource) {
+	if origin == nil {
+		return
+	}
+	source, subSource, sourceName := m.SourceInfo()
+	item = new(SearchDataSource)
+	item.PrimaryId = origin.PrimaryId
+	item.IndexCode = origin.IndexCode
+	item.IndexName = origin.IndexName
+	//item.ClassifyId = origin.ClassifyId
+	item.Unit = origin.Unit
+	item.Frequency = origin.Frequency
+	item.StartDate = utils.TimeTransferString(utils.FormatDate, origin.StartDate)
+	item.EndDate = utils.TimeTransferString(utils.FormatDate, origin.EndDate)
+	item.LatestValue = origin.LatestValue
+	item.Source = source
+	item.SubSource = subSource
+	item.SourceName = sourceName
+	item.CreateTime = utils.TimeTransferString(utils.FormatDateTime, origin.CreateTime)
+	item.ModifyTime = utils.TimeTransferString(utils.FormatDateTime, origin.ModifyTime)
+	return
+}
+
+func (m *BaseFromGlIndex) GetItemsByCondition(condition string, pars []interface{}, fieldArr []string, orderRule string) (items []*BaseFromGlIndex, err error) {
+	o := orm.NewOrmUsingDB("gl")
+	fields := strings.Join(fieldArr, ",")
+	if len(fieldArr) == 0 {
+		fields = `*`
+	}
+	order := `ORDER BY CREATE_TIME DESC`
+	if orderRule != "" {
+		order = ` ORDER BY ` + orderRule
+	}
+	sql := fmt.Sprintf(`SELECT %s FROM mb_index_main_info WHERE 1=1 %s %s`, fields, condition, order)
+	_, err = o.Raw(sql, pars).QueryRows(&items)
+	return
+}
+
+// BaseFromManualEdb 手工数据录入
+type BaseFromManualEdb struct {
+	//PrimaryId   int       `orm:"column(TRADE_CODE)"` // 注手工指标没自增ID...
+	IndexCode   string    `orm:"column(TRADE_CODE)" description:"指标编码"`
+	IndexName   string    `orm:"column(SEC_NAME)" description:"指标名称"`
+	ClassifyId  int       `orm:"column(classify_id)" description:"分类ID"`
+	Unit        string    `orm:"column(UNIT)" description:"单位"`
+	Frequency   string    `orm:"column(frequency)" description:"频度"`
+	StartDate   time.Time `orm:"column(start_date)" description:"开始日期"`
+	EndDate     time.Time `orm:"column(end_date)" description:"结束日期"`
+	LatestValue float64   `orm:"column(latest_value)" description:"最新值"`
+	CreateTime  time.Time `orm:"column(create_date)" description:"创建时间"`
+	ModifyTime  time.Time `orm:"column(modify_time)" description:"更新时间"`
+}
+
+func (m *BaseFromManualEdb) SourceInfo() (int, int, string) {
+	return utils.DATA_SOURCE_MANUAL, 0, "手工指标录入"
+}
+
+func (m *BaseFromManualEdb) EsCols() SearchEsCols {
+	return SearchEsCols{
+		PrimaryId:   "",
+		IndexCode:   "TRADE_CODE",
+		IndexName:   "SEC_NAME",
+		ClassifyId:  "classify_id",
+		Unit:        "UNIT",
+		Frequency:   "frequency",
+		StartDate:   "start_date",
+		EndDate:     "end_date",
+		LatestValue: "latest_value",
+		CreateTime:  "create_date",
+		ModifyTime:  "modify_time",
+	}
+}
+
+func (m *BaseFromManualEdb) Format2SearchDataSource(origin *BaseFromManualEdb) (item *SearchDataSource) {
+	if origin == nil {
+		return
+	}
+	source, subSource, sourceName := m.SourceInfo()
+	item = new(SearchDataSource)
+	//item.PrimaryId = origin.PrimaryId
+	item.IndexCode = origin.IndexCode
+	item.IndexName = origin.IndexName
+	item.ClassifyId = origin.ClassifyId
+	item.Unit = origin.Unit
+	item.Frequency = origin.Frequency
+	item.StartDate = utils.TimeTransferString(utils.FormatDate, origin.StartDate)
+	item.EndDate = utils.TimeTransferString(utils.FormatDate, origin.EndDate)
+	item.LatestValue = origin.LatestValue
+	item.Source = source
+	item.SubSource = subSource
+	item.SourceName = sourceName
+	item.CreateTime = utils.TimeTransferString(utils.FormatDateTime, origin.CreateTime)
+	item.ModifyTime = utils.TimeTransferString(utils.FormatDateTime, origin.ModifyTime)
+	return
+}
+
+func (m *BaseFromManualEdb) GetItemsByCondition(condition string, pars []interface{}, fieldArr []string, orderRule string) (items []*BaseFromManualEdb, err error) {
+	o := orm.NewOrmUsingDB("edb")
+	fields := strings.Join(fieldArr, ",")
+	if len(fieldArr) == 0 {
+		fields = `*`
+	}
+	order := `ORDER BY create_date DESC`
+	if orderRule != "" {
+		order = ` ORDER BY ` + orderRule
+	}
+	sql := fmt.Sprintf(`SELECT %s FROM edbinfo WHERE 1=1 %s %s`, fields, condition, order)
+	_, err = o.Raw(sql, pars).QueryRows(&items)
+	return
+}
+
+// BaseFromBusinessIndex 自有数据
+type BaseFromBusinessIndex struct {
+	BaseFromBusinessIndexId int `orm:"column(base_from_business_index_id);pk"`
+	//ClassifyId              int       `description:"分类ID"`
+	IndexCode   string    `description:"指标编码"`
+	IndexName   string    `description:"指标名称"`
+	Unit        string    `description:"单位"`
+	Frequency   string    `description:"频度"`
+	StartDate   time.Time `description:"开始日期"`
+	EndDate     time.Time `description:"结束日期"`
+	LatestValue float64   `description:"最新值"`
+	CreateTime  time.Time `description:"创建时间"`
+	ModifyTime  time.Time `description:"更新时间"`
+}
+
+func (m *BaseFromBusinessIndex) SourceInfo() (int, int, string) {
+	return utils.DATA_SOURCE_BUSINESS, 0, "自有数据"
+}
+
+func (m *BaseFromBusinessIndex) EsCols() SearchEsCols {
+	return SearchEsCols{
+		PrimaryId:   "base_from_business_index_id",
+		IndexCode:   "index_code",
+		IndexName:   "index_name",
+		ClassifyId:  "",
+		Unit:        "unit",
+		Frequency:   "frequency",
+		StartDate:   "start_date",
+		EndDate:     "end_date",
+		LatestValue: "latest_value",
+		CreateTime:  "create_time",
+		ModifyTime:  "modify_time",
+	}
+}
+
+func (m *BaseFromBusinessIndex) Format2SearchDataSource(origin *BaseFromBusinessIndex) (item *SearchDataSource) {
+	if origin == nil {
+		return
+	}
+	source, subSource, sourceName := m.SourceInfo()
+	item = new(SearchDataSource)
+	item.PrimaryId = origin.BaseFromBusinessIndexId
+	item.IndexCode = origin.IndexCode
+	item.IndexName = origin.IndexName
+	//item.ClassifyId = origin.ClassifyId
+	item.Unit = origin.Unit
+	item.Frequency = origin.Frequency
+	item.StartDate = utils.TimeTransferString(utils.FormatDate, origin.StartDate)
+	item.EndDate = utils.TimeTransferString(utils.FormatDate, origin.EndDate)
+	item.LatestValue = origin.LatestValue
+	item.Source = source
+	item.SubSource = subSource
+	item.SourceName = sourceName
+	item.CreateTime = utils.TimeTransferString(utils.FormatDateTime, origin.CreateTime)
+	item.ModifyTime = utils.TimeTransferString(utils.FormatDateTime, origin.ModifyTime)
+	return
+}
+
+func (m *BaseFromBusinessIndex) GetItemsByCondition(condition string, pars []interface{}, fieldArr []string, orderRule string) (items []*BaseFromBusinessIndex, err error) {
+	o := orm.NewOrmUsingDB("data")
+	fields := strings.Join(fieldArr, ",")
+	if len(fieldArr) == 0 {
+		fields = `*`
+	}
+	order := `ORDER BY create_time DESC`
+	if orderRule != "" {
+		order = ` ORDER BY ` + orderRule
+	}
+	sql := fmt.Sprintf(`SELECT %s FROM base_from_business_index WHERE 1=1 %s %s`, fields, condition, order)
+	_, err = o.Raw(sql, pars).QueryRows(&items)
+	return
+}
+
+// BaseFromBloombergIndex Bloomberg
+type BaseFromBloombergIndex struct {
+	BaseFromBloombergIndexId int `orm:"column(base_from_bloomberg_index_id);pk"`
+	//ClassifyId              int       `description:"分类ID"`
+	IndexCode   string    `description:"指标编码"`
+	IndexName   string    `description:"指标名称"`
+	Unit        string    `description:"单位"`
+	Frequency   string    `description:"频度"`
+	StartDate   time.Time `description:"开始日期"`
+	EndDate     time.Time `description:"结束日期"`
+	LatestValue float64   `description:"最新值"`
+	CreateTime  time.Time `description:"创建时间"`
+	ModifyTime  time.Time `description:"更新时间"`
+}
+
+func (m *BaseFromBloombergIndex) SourceInfo() (int, int, string) {
+	return utils.DATA_SOURCE_BLOOMBERG, 0, "Bloomberg"
+}
+
+func (m *BaseFromBloombergIndex) EsCols() SearchEsCols {
+	return SearchEsCols{
+		PrimaryId:   "base_from_bloomberg_index_id",
+		IndexCode:   "index_code",
+		IndexName:   "index_name",
+		ClassifyId:  "",
+		Unit:        "unit",
+		Frequency:   "frequency",
+		StartDate:   "start_date",
+		EndDate:     "end_date",
+		LatestValue: "latest_value",
+		CreateTime:  "create_time",
+		ModifyTime:  "modify_time",
+	}
+}
+
+func (m *BaseFromBloombergIndex) Format2SearchDataSource(origin *BaseFromBloombergIndex) (item *SearchDataSource) {
+	if origin == nil {
+		return
+	}
+	source, subSource, sourceName := m.SourceInfo()
+	item = new(SearchDataSource)
+	item.PrimaryId = origin.BaseFromBloombergIndexId
+	item.IndexCode = origin.IndexCode
+	item.IndexName = origin.IndexName
+	//item.ClassifyId = origin.ClassifyId
+	item.Unit = origin.Unit
+	item.Frequency = origin.Frequency
+	item.StartDate = utils.TimeTransferString(utils.FormatDate, origin.StartDate)
+	item.EndDate = utils.TimeTransferString(utils.FormatDate, origin.EndDate)
+	item.LatestValue = origin.LatestValue
+	item.Source = source
+	item.SubSource = subSource
+	item.SourceName = sourceName
+	item.CreateTime = utils.TimeTransferString(utils.FormatDateTime, origin.CreateTime)
+	item.ModifyTime = utils.TimeTransferString(utils.FormatDateTime, origin.ModifyTime)
+	return
+}
+
+func (m *BaseFromBloombergIndex) GetItemsByCondition(condition string, pars []interface{}, fieldArr []string, orderRule string) (items []*BaseFromBloombergIndex, err error) {
+	o := orm.NewOrmUsingDB("data")
+	fields := strings.Join(fieldArr, ",")
+	if len(fieldArr) == 0 {
+		fields = `*`
+	}
+	order := `ORDER BY create_time DESC`
+	if orderRule != "" {
+		order = ` ORDER BY ` + orderRule
+	}
+	sql := fmt.Sprintf(`SELECT %s FROM base_from_bloomberg_index WHERE 1=1 %s %s`, fields, condition, order)
+	_, err = o.Raw(sql, pars).QueryRows(&items)
+	return
+}
+
+// BaseFromMtjhMapping 煤炭江湖
+type BaseFromMtjhMapping struct {
+	BaseFromMtjhMappingId int `orm:"column(base_from_mtjh_mapping_id);pk"`
+	//ClassifyId            int       `description:"分类ID"`
+	IndexCode   string    `description:"指标编码"`
+	IndexName   string    `description:"指标名称"`
+	Unit        string    `description:"单位"`
+	Frequency   string    `description:"频度"`
+	StartDate   time.Time `description:"开始日期"`
+	EndDate     time.Time `description:"结束日期"`
+	LatestValue float64   `description:"最新值"`
+	CreateTime  time.Time `description:"创建时间"`
+	ModifyTime  time.Time `description:"更新时间"`
+}
+
+func (m *BaseFromMtjhMapping) SourceInfo() (int, int, string) {
+	return utils.DATA_SOURCE_MTJH, 0, "煤炭江湖"
+}
+
+func (m *BaseFromMtjhMapping) EsCols() SearchEsCols {
+	return SearchEsCols{
+		PrimaryId:   "base_from_mtjh_mapping_id",
+		IndexCode:   "index_code",
+		IndexName:   "index_name",
+		ClassifyId:  "",
+		Unit:        "unit",
+		Frequency:   "frequency",
+		StartDate:   "start_date",
+		EndDate:     "end_date",
+		LatestValue: "latest_value",
+		CreateTime:  "create_time",
+		ModifyTime:  "",
+	}
+}
+
+func (m *BaseFromMtjhMapping) Format2SearchDataSource(origin *BaseFromMtjhMapping) (item *SearchDataSource) {
+	if origin == nil {
+		return
+	}
+	source, subSource, sourceName := m.SourceInfo()
+	item = new(SearchDataSource)
+	item.PrimaryId = origin.BaseFromMtjhMappingId
+	item.IndexCode = origin.IndexCode
+	item.IndexName = origin.IndexName
+	//item.ClassifyId = origin.ClassifyId
+	item.Unit = origin.Unit
+	item.Frequency = origin.Frequency
+	item.StartDate = utils.TimeTransferString(utils.FormatDate, origin.StartDate)
+	item.EndDate = utils.TimeTransferString(utils.FormatDate, origin.EndDate)
+	item.LatestValue = origin.LatestValue
+	item.Source = source
+	item.SubSource = subSource
+	item.SourceName = sourceName
+	item.CreateTime = utils.TimeTransferString(utils.FormatDateTime, origin.CreateTime)
+	item.ModifyTime = utils.TimeTransferString(utils.FormatDateTime, origin.ModifyTime)
+	return
+}
+
+func (m *BaseFromMtjhMapping) GetItemsByCondition(condition string, pars []interface{}, fieldArr []string, orderRule string) (items []*BaseFromMtjhMapping, err error) {
+	o := orm.NewOrmUsingDB("data")
+	fields := strings.Join(fieldArr, ",")
+	if len(fieldArr) == 0 {
+		fields = `*`
+	}
+	order := `ORDER BY create_time DESC`
+	if orderRule != "" {
+		order = ` ORDER BY ` + orderRule
+	}
+	sql := fmt.Sprintf(`SELECT %s FROM base_from_mtjh_mapping WHERE 1=1 %s %s`, fields, condition, order)
+	_, err = o.Raw(sql, pars).QueryRows(&items)
+	return
+}
+
+// BaseIndexDataMinMax 数据源极值信息
+type BaseIndexDataMinMax struct {
+	MinDate     string  `description:"最小日期"`
+	MaxDate     string  `description:"最大日期"`
+	LatestValue float64 `description:"最新值"`
+	//MinValue    float64 `description:"最小值"`
+	//MaxValue    float64 `description:"最大值"`
+}
+
+// GetBaseIndexDataTableName 根据来源获取原始指标表和数据表名（很大一部分来源不存在原始指标）
+func GetBaseIndexDataTableName(source, subSource int) (indexTable, dataTable string) {
+	switch source {
+	case utils.DATA_SOURCE_THS:
+		if subSource == utils.DATA_SUB_SOURCE_HIGH_FREQUENCY {
+			indexTable = "base_from_ths_hf_index"
+			dataTable = "base_from_ths_hf_data"
+		}
+	case utils.DATA_SOURCE_RZD:
+		indexTable = "base_from_rzd_index"
+		dataTable = "base_from_rzd_data"
+	case utils.DATA_SOURCE_HISUGAR:
+		indexTable = "base_from_hisugar_index"
+		dataTable = "base_from_hisugar_data"
+	case utils.DATA_SOURCE_LY:
+		indexTable = "base_from_ly_index"
+		dataTable = "base_from_ly_data"
+	case utils.DATA_SOURCE_SCI_HQ:
+		indexTable = "base_from_sci_hq_index"
+		dataTable = "base_from_sci_hq_data"
+	case utils.DATA_SOURCE_OILCHEM:
+		indexTable = "base_from_oilchem_index"
+		dataTable = "base_from_oilchem_data"
+	case utils.DATA_SOURCE_CCF:
+		indexTable = "base_from_ccf_index"
+		dataTable = "base_from_ccf_data"
+	case utils.DATA_SOURCE_USDA_FAS:
+		indexTable = "base_from_usda_fas_index"
+		dataTable = "base_from_usda_fas_data"
+	case utils.DATA_SOURCE_MYSTEEL_CHEMICAL:
+		indexTable = "base_from_mysteel_chemical_index"
+		dataTable = "base_from_mysteel_chemical_data"
+	case utils.DATA_SOURCE_YS:
+		indexTable = "base_from_smm_index"
+		dataTable = "base_from_smm_data"
+	case utils.DATA_SOURCE_BAIINFO:
+		indexTable = "base_from_baiinfo_index"
+		dataTable = "base_from_baiinfo_data"
+	case utils.DATA_SOURCE_SCI:
+		indexTable = "base_from_sci_index"
+		dataTable = "base_from_sci_data"
+	case utils.DATA_SOURCE_EIA_STEO:
+		indexTable = "base_from_eia_steo_index"
+		dataTable = "base_from_eia_steo_data"
+	case utils.DATA_SOURCE_ICPI:
+		indexTable = "base_from_icpi_index"
+		dataTable = "base_from_icpi_data"
+	case utils.DATA_SOURCE_YONYI:
+		indexTable = "base_from_yongyi_index"
+		dataTable = "base_from_yongyi_data"
+	case utils.DATA_SOURCE_FENWEI:
+		indexTable = "base_from_fenwei_index"
+		dataTable = "base_from_fenwei_data"
+	case utils.DATA_SOURCE_SCI99:
+		indexTable = "base_from_icpi_index"
+		dataTable = "base_from_icpi_data"
+	case utils.DATA_SOURCE_BUSINESS:
+		indexTable = "base_from_business_index"
+		dataTable = "base_from_business_data"
+	case utils.DATA_SOURCE_BLOOMBERG:
+		indexTable = "base_from_bloomberg_index"
+		dataTable = "base_from_bloomberg_data"
+	default:
+		utils.FileLog.Info(fmt.Sprintf("数据源无对应表名, source: %d, sub: %d", source, subSource))
+	}
+	return
+}
+
+// getCoalmineDataTableName 获取中国煤炭市场网数据表名
+func getCoalmineDataTableName(indexCode string) string {
+	if strings.Contains(indexCode, "jsm") {
+		return "base_from_coalmine_jsm_index"
+	}
+	if strings.Contains(indexCode, "company") {
+		return "base_from_coalmine_company_index"
+	}
+	if strings.Contains(indexCode, "firm") {
+		return "base_from_coalmine_firm_index"
+	}
+	if strings.Contains(indexCode, "coastal") {
+		return "base_from_coalmine_coastal_index"
+	}
+	if strings.Contains(indexCode, "inland") {
+		return "base_from_coalmine_inland_index"
+	}
+	return ""
+}
+
+// GetBaseIndexDataMinMax 获取数据源极值
+func GetBaseIndexDataMinMax(source, subSource int, indexCode string) (item *BaseIndexDataMinMax, err error) {
+	o := orm.NewOrmUsingDB("data")
+	var sql string
+	var latestVal float64
+
+	// 煤炭江湖
+	if source == utils.DATA_SOURCE_MTJH {
+		sql = `SELECT MIN(data_time) AS min_date, MAX(data_time) AS max_date FROM base_from_mtjh_index WHERE index_code = ?`
+		if e := o.Raw(sql, indexCode).QueryRow(&item); e != nil {
+			err = fmt.Errorf("获取数据源开始结束时间失败, %v", e)
+			return
+		}
+
+		sql = `SELECT deal_value AS latest_value FROM base_from_mtjh_index WHERE index_code = ? ORDER BY data_time DESC LIMIT 1`
+		if e := o.Raw(sql, indexCode).QueryRow(&latestVal); e != nil {
+			err = fmt.Errorf("获取数据源最新值失败, %v", e)
+			return
+		}
+		item.LatestValue = latestVal
+		return
+	}
+
+	// 中国煤炭市场网
+	if source == utils.DATA_SOURCE_COAL {
+		dataTable := getCoalmineDataTableName(indexCode)
+		if dataTable == "" {
+			err = fmt.Errorf("中国煤炭市场网-指标无对应表名: %s", indexCode)
+			return
+		}
+		fieldDataTime := "data_time"
+		if dataTable == "base_from_coalmine_firm_index" {
+			fieldDataTime = "data_time_date"
+		}
+		sql = `SELECT MIN(%s) AS min_date, MAX(%s) AS max_date FROM %s WHERE index_code = ?`
+		sql = fmt.Sprintf(sql, fieldDataTime, fieldDataTime, dataTable)
+		if e := o.Raw(sql, indexCode).QueryRow(&item); e != nil {
+			err = fmt.Errorf("获取数据源开始结束时间失败, %v", e)
+			return
+		}
+
+		sql = `SELECT deal_value AS latest_value FROM %s WHERE index_code = ? ORDER BY %s DESC LIMIT 1`
+		sql = fmt.Sprintf(sql, dataTable, fieldDataTime)
+		if e := o.Raw(sql, indexCode).QueryRow(&latestVal); e != nil {
+			err = fmt.Errorf("获取数据源最新值失败, %v", e)
+			return
+		}
+		item.LatestValue = latestVal
+		return
+	}
+
+	// 其他数据源
+	_, dataTable := GetBaseIndexDataTableName(source, subSource)
+	if dataTable == "" {
+		err = fmt.Errorf("数据源无对应数据表, source: %d, sub: %d, code: %s", source, subSource, indexCode)
+		return
+	}
+
+	sql = `SELECT MIN(data_time) AS min_date, MAX(data_time) AS max_date FROM %s WHERE index_code = ?`
+	sql = fmt.Sprintf(sql, dataTable)
+	if e := o.Raw(sql, indexCode).QueryRow(&item); e != nil {
+		err = fmt.Errorf("获取数据源开始结束时间失败, %v", e)
+		return
+	}
+
+	sql = `SELECT value AS latest_value FROM %s WHERE index_code = ? ORDER BY data_time DESC LIMIT 1`
+	sql = fmt.Sprintf(sql, dataTable)
+	if e := o.Raw(sql, indexCode).QueryRow(&latestVal); e != nil {
+		err = fmt.Errorf("获取数据源最新值失败, %v", e)
+		return
+	}
+	item.LatestValue = latestVal
+	return
+}

models/db.go

@@ -99,7 +99,7 @@ func init() {
 	initEdbRefresh()
 
 	// 初始化部分数据表变量(直接init会有顺序问题=_=!)
-	data_manage.InitEdbSourceVar()
+	afterInitTable()
 }
 
 // initEdbDataTable 注册Edb指标 数据表
@@ -204,3 +204,15 @@ func initEdbRefresh() {
 		new(edb_refresh.EdbRefreshMapping),       // 指标刷新时间配置关系表
 	)
 }
+
+// afterInitTable
+// @Description: 初始化表结构的的后置操作
+// @author: Roc
+// @datetime 2024-07-01 13:31:09
+func afterInitTable() {
+	// 初始化指标来源配置
+	data_manage.InitEdbSourceVar()
+
+	// 初始化商家基本配置
+	InitBusinessConf()
+}

services/data/data_source.go

@@ -0,0 +1,506 @@
+package data
+
+import (
+	"context"
+	"eta/eta_task/models"
+	aiPredictModel "eta/eta_task/models/ai_predict_model"
+	dataSourceModel "eta/eta_task/models/data_source"
+	"eta/eta_task/services/alarm_msg"
+	"eta/eta_task/services/elastic"
+	"eta/eta_task/utils"
+	"fmt"
+)
+
+// SyncDataSourceEs 同步数据源ES
+func SyncDataSourceEs(cont context.Context) (err error) {
+	utils.FileLog.Info("SyncDataSourceEs-start")
+	defer func() {
+		if err != nil {
+			tips := fmt.Sprintf("SyncDataSourceEs-同步数据源ES失败, %v", err)
+			utils.FileLog.Info(tips)
+			go alarm_msg.SendAlarmMsg(tips, 3)
+		}
+		utils.FileLog.Info("SyncDataSourceEs-end")
+	}()
+	confOb := new(models.BusinessConf)
+	conf, e := confOb.GetItemByConfKey(models.BusinessConfSyncDataEsDaily)
+	if e != nil {
+		if e.Error() == utils.ErrNoRow() {
+			utils.FileLog.Info("SyncDataSourceEs-无刷新配置，不同步")
+			return
+		}
+		err = fmt.Errorf("获取刷新配置失败, %v", e)
+		return
+	}
+	if conf.ConfVal != "true" {
+		return
+	}
+
+	var cond string
+	var pars []interface{}
+	// 睿咨得
+	rzdOb := new(dataSourceModel.BaseFromRzdIndex)
+	{
+		list, e := rzdOb.GetItemsByCondition(cond, pars, []string{}, "")
+		if e != nil {
+			err = fmt.Errorf("获取睿咨得失败, %v", e)
+			return
+		}
+		for _, v := range list {
+			indexItem := v.Format2SearchDataSource(v)
+			fmt.Printf("写入中：睿咨得-%d\n", indexItem.PrimaryId)
+			docId := fmt.Sprintf("%d-%d", indexItem.Source, indexItem.PrimaryId)
+			if e := elastic.EsAddOrEditDataSourceIndex(utils.EsDataSourceIndexName, docId, indexItem); e != nil {
+				err = fmt.Errorf("睿咨得-写入es失败, %v", e)
+				return
+			}
+		}
+	}
+
+	// 泛糖科技
+	hisugarOb := new(dataSourceModel.BaseFromHisugarIndex)
+	{
+		list, e := hisugarOb.GetItemsByCondition(cond, pars, []string{}, "")
+		if e != nil {
+			err = fmt.Errorf("获取泛糖科技失败, %v", e)
+			return
+		}
+		for _, v := range list {
+			indexItem := v.Format2SearchDataSource(v)
+			fmt.Printf("写入中：泛糖科技-%d\n", indexItem.PrimaryId)
+			docId := fmt.Sprintf("%d-%d", indexItem.Source, indexItem.PrimaryId)
+			if e := elastic.EsAddOrEditDataSourceIndex(utils.EsDataSourceIndexName, docId, indexItem); e != nil {
+				err = fmt.Errorf("泛糖科技-写入es失败, %v", e)
+				return
+			}
+		}
+	}
+
+	// 粮油商务网
+	lyOb := new(dataSourceModel.BaseFromLyIndex)
+	{
+		list, e := lyOb.GetItemsByCondition(cond, pars, []string{}, "")
+		if e != nil {
+			err = fmt.Errorf("获取粮油商务网失败, %v", e)
+			return
+		}
+		for _, v := range list {
+			indexItem := v.Format2SearchDataSource(v)
+			fmt.Printf("写入中：粮油商务网-%d\n", indexItem.PrimaryId)
+			docId := fmt.Sprintf("%d-%d", indexItem.Source, indexItem.PrimaryId)
+			if e := elastic.EsAddOrEditDataSourceIndex(utils.EsDataSourceIndexName, docId, indexItem); e != nil {
+				err = fmt.Errorf("粮油商务网-写入es失败, %v", e)
+				return
+			}
+		}
+	}
+
+	// 卓创红期
+	sciHqOb := new(dataSourceModel.BaseFromSciHqIndex)
+	{
+		list, e := sciHqOb.GetItemsByCondition(cond, pars, []string{}, "")
+		if e != nil {
+			err = fmt.Errorf("获取卓创红期失败, %v", e)
+			return
+		}
+		for _, v := range list {
+			indexItem := v.Format2SearchDataSource(v)
+			fmt.Printf("写入中：卓创红期-%d\n", indexItem.PrimaryId)
+			docId := fmt.Sprintf("%d-%d", indexItem.Source, indexItem.PrimaryId)
+			if e := elastic.EsAddOrEditDataSourceIndex(utils.EsDataSourceIndexName, docId, indexItem); e != nil {
+				err = fmt.Errorf("卓创红期-写入es失败, %v", e)
+				return
+			}
+		}
+	}
+
+	// 同花顺高频
+	thsHfOb := new(dataSourceModel.BaseFromThsHfIndex)
+	{
+		list, e := thsHfOb.GetItemsByCondition(cond, pars, []string{}, "")
+		if e != nil {
+			err = fmt.Errorf("获取同花顺高频失败, %v", e)
+			return
+		}
+		for _, v := range list {
+			indexItem := v.Format2SearchDataSource(v)
+			fmt.Printf("写入中：同花顺高频-%d\n", indexItem.PrimaryId)
+			docId := fmt.Sprintf("%d-%d", indexItem.Source, indexItem.PrimaryId)
+			if e := elastic.EsAddOrEditDataSourceIndex(utils.EsDataSourceIndexName, docId, indexItem); e != nil {
+				err = fmt.Errorf("同花顺高频-写入es失败, %v", e)
+				return
+			}
+		}
+	}
+
+	// 隆众资讯
+	oilchemOb := new(dataSourceModel.BaseFromOilchemIndex)
+	{
+		list, e := oilchemOb.GetItemsByCondition(cond, pars, []string{}, "")
+		if e != nil {
+			err = fmt.Errorf("获取隆众资讯失败, %v", e)
+			return
+		}
+		for _, v := range list {
+			indexItem := v.Format2SearchDataSource(v)
+			fmt.Printf("写入中：隆众资讯-%d\n", indexItem.PrimaryId)
+			docId := fmt.Sprintf("%d-%d", indexItem.Source, indexItem.PrimaryId)
+			if e := elastic.EsAddOrEditDataSourceIndex(utils.EsDataSourceIndexName, docId, indexItem); e != nil {
+				err = fmt.Errorf("隆众资讯-写入es失败, %v", e)
+				return
+			}
+		}
+	}
+
+	// CCF化纤信息
+	ccfOb := new(dataSourceModel.BaseFromCcfIndex)
+	{
+		list, e := ccfOb.GetItemsByCondition(cond, pars, []string{}, "")
+		if e != nil {
+			err = fmt.Errorf("获取CCF化纤信息失败, %v", e)
+			return
+		}
+		for _, v := range list {
+			indexItem := v.Format2SearchDataSource(v)
+			fmt.Printf("写入中：CCF化纤信息-%d\n", indexItem.PrimaryId)
+			docId := fmt.Sprintf("%d-%d", indexItem.Source, indexItem.PrimaryId)
+			if e := elastic.EsAddOrEditDataSourceIndex(utils.EsDataSourceIndexName, docId, indexItem); e != nil {
+				err = fmt.Errorf("CCF化纤信息-写入es失败, %v", e)
+				return
+			}
+		}
+	}
+
+	// 上海钢联
+	mysteelOb := new(dataSourceModel.BaseFromMysteelChemicalIndex)
+	{
+		list, e := mysteelOb.GetItemsByCondition(cond, pars, []string{}, "")
+		if e != nil {
+			err = fmt.Errorf("获取上海钢联失败, %v", e)
+			return
+		}
+		for _, v := range list {
+			indexItem := v.Format2SearchDataSource(v)
+			fmt.Printf("写入中：上海钢联-%d\n", indexItem.PrimaryId)
+			docId := fmt.Sprintf("%d-%d", indexItem.Source, indexItem.PrimaryId)
+			if e := elastic.EsAddOrEditDataSourceIndex(utils.EsDataSourceIndexName, docId, indexItem); e != nil {
+				err = fmt.Errorf("上海钢联-写入es失败, %v", e)
+				return
+			}
+		}
+	}
+
+	// SMM、有色原始数据库
+	smmOb := new(dataSourceModel.BaseFromSmmIndex)
+	{
+		list, e := smmOb.GetItemsByCondition(cond, pars, []string{}, "")
+		if e != nil {
+			err = fmt.Errorf("获取有色原始数据库失败, %v", e)
+			return
+		}
+		for _, v := range list {
+			indexItem := v.Format2SearchDataSource(v)
+			fmt.Printf("写入中：有色原始数据库-%d\n", indexItem.PrimaryId)
+			docId := fmt.Sprintf("%d-%d", indexItem.Source, indexItem.PrimaryId)
+			if e := elastic.EsAddOrEditDataSourceIndex(utils.EsDataSourceIndexName, docId, indexItem); e != nil {
+				err = fmt.Errorf("有色原始数据库-写入es失败, %v", e)
+				return
+			}
+		}
+	}
+
+	// 百川盈孚
+	baiinfoOb := new(dataSourceModel.BaseFromBaiinfoIndex)
+	{
+		list, e := baiinfoOb.GetItemsByCondition(cond, pars, []string{}, "")
+		if e != nil {
+			err = fmt.Errorf("获取百川盈孚失败, %v", e)
+			return
+		}
+		for _, v := range list {
+			indexItem := v.Format2SearchDataSource(v)
+			fmt.Printf("写入中：百川盈孚-%d\n", indexItem.PrimaryId)
+			docId := fmt.Sprintf("%d-%d", indexItem.Source, indexItem.PrimaryId)
+			if e := elastic.EsAddOrEditDataSourceIndex(utils.EsDataSourceIndexName, docId, indexItem); e != nil {
+				err = fmt.Errorf("百川盈孚-写入es失败, %v", e)
+				return
+			}
+		}
+	}
+
+	// 红桃3
+	sciOb := new(dataSourceModel.BaseFromSciIndex)
+	{
+		list, e := sciOb.GetItemsByCondition(cond, pars, []string{}, "")
+		if e != nil {
+			err = fmt.Errorf("获取红桃3失败, %v", e)
+			return
+		}
+		for _, v := range list {
+			indexItem := v.Format2SearchDataSource(v)
+			fmt.Printf("写入中：红桃3-%d\n", indexItem.PrimaryId)
+			docId := fmt.Sprintf("%d-%d", indexItem.Source, indexItem.PrimaryId)
+			if e := elastic.EsAddOrEditDataSourceIndex(utils.EsDataSourceIndexName, docId, indexItem); e != nil {
+				err = fmt.Errorf("红桃3-写入es失败, %v", e)
+				return
+			}
+		}
+	}
+
+	// 中国煤炭市场网
+	coalmineOb := new(dataSourceModel.BaseFromCoalmineMapping)
+	{
+		list, e := coalmineOb.GetItemsByCondition(cond, pars, []string{}, "")
+		if e != nil {
+			err = fmt.Errorf("获取中国煤炭市场网失败, %v", e)
+			return
+		}
+		for _, v := range list {
+			indexItem := v.Format2SearchDataSource(v)
+			fmt.Printf("写入中：中国煤炭市场网-%d\n", indexItem.PrimaryId)
+			docId := fmt.Sprintf("%d-%d", indexItem.Source, indexItem.PrimaryId)
+			if e := elastic.EsAddOrEditDataSourceIndex(utils.EsDataSourceIndexName, docId, indexItem); e != nil {
+				err = fmt.Errorf("中国煤炭市场网-写入es失败, %v", e)
+				return
+			}
+		}
+	}
+
+	// EIA STEO报告
+	eiaOb := new(dataSourceModel.BaseFromEiaSteoIndex)
+	{
+		list, e := eiaOb.GetItemsByCondition(cond, pars, []string{}, "")
+		if e != nil {
+			err = fmt.Errorf("获取EIA STEO报告失败, %v", e)
+			return
+		}
+		for _, v := range list {
+			indexItem := v.Format2SearchDataSource(v)
+			fmt.Printf("写入中：EIA STEO报告-%d\n", indexItem.PrimaryId)
+			docId := fmt.Sprintf("%d-%d", indexItem.Source, indexItem.PrimaryId)
+			if e := elastic.EsAddOrEditDataSourceIndex(utils.EsDataSourceIndexName, docId, indexItem); e != nil {
+				err = fmt.Errorf("EIA STEO报告-写入es失败, %v", e)
+				return
+			}
+		}
+	}
+
+	// ICPI消费价格指数
+	icpiOb := new(dataSourceModel.BaseFromIcpiIndex)
+	{
+		list, e := icpiOb.GetItemsByCondition(cond, pars, []string{}, "")
+		if e != nil {
+			err = fmt.Errorf("获取ICPI消费价格指数失败, %v", e)
+			return
+		}
+		for _, v := range list {
+			indexItem := v.Format2SearchDataSource(v)
+			fmt.Printf("写入中：ICPI消费价格指数-%d\n", indexItem.PrimaryId)
+			docId := fmt.Sprintf("%d-%d", indexItem.Source, indexItem.PrimaryId)
+			if e := elastic.EsAddOrEditDataSourceIndex(utils.EsDataSourceIndexName, docId, indexItem); e != nil {
+				err = fmt.Errorf("ICPI消费价格指数-写入es失败, %v", e)
+				return
+			}
+		}
+	}
+
+	// 涌益咨询
+	yongyiOb := new(dataSourceModel.BaseFromYongyiIndex)
+	{
+		list, e := yongyiOb.GetItemsByCondition(cond, pars, []string{}, "")
+		if e != nil {
+			err = fmt.Errorf("获取涌益咨询失败, %v", e)
+			return
+		}
+		for _, v := range list {
+			indexItem := v.Format2SearchDataSource(v)
+			fmt.Printf("写入中：涌益咨询-%d\n", indexItem.PrimaryId)
+			docId := fmt.Sprintf("%d-%d", indexItem.Source, indexItem.PrimaryId)
+			if e := elastic.EsAddOrEditDataSourceIndex(utils.EsDataSourceIndexName, docId, indexItem); e != nil {
+				err = fmt.Errorf("涌益咨询-写入es失败, %v", e)
+				return
+			}
+		}
+	}
+
+	// 汾渭数据
+	fenweiOb := new(dataSourceModel.BaseFromFenweiIndex)
+	{
+		list, e := fenweiOb.GetItemsByCondition(cond, pars, []string{}, "")
+		if e != nil {
+			err = fmt.Errorf("获取汾渭数据失败, %v", e)
+			return
+		}
+		for _, v := range list {
+			indexItem := v.Format2SearchDataSource(v)
+			fmt.Printf("写入中：汾渭数据-%d\n", indexItem.PrimaryId)
+			docId := fmt.Sprintf("%d-%d", indexItem.Source, indexItem.PrimaryId)
+			if e := elastic.EsAddOrEditDataSourceIndex(utils.EsDataSourceIndexName, docId, indexItem); e != nil {
+				err = fmt.Errorf("汾渭数据-写入es失败, %v", e)
+				return
+			}
+		}
+	}
+
+	// 卓创数据
+	sci99Ob := new(dataSourceModel.BaseFromSci99Index)
+	{
+		list, e := sci99Ob.GetItemsByCondition(cond, pars, []string{}, "")
+		if e != nil {
+			err = fmt.Errorf("获取卓创数据失败, %v", e)
+			return
+		}
+		for _, v := range list {
+			indexItem := v.Format2SearchDataSource(v)
+			fmt.Printf("写入中：卓创数据-%d\n", indexItem.PrimaryId)
+			docId := fmt.Sprintf("%d-%d", indexItem.Source, indexItem.PrimaryId)
+			if e := elastic.EsAddOrEditDataSourceIndex(utils.EsDataSourceIndexName, docId, indexItem); e != nil {
+				err = fmt.Errorf("卓创数据-写入es失败, %v", e)
+				return
+			}
+		}
+	}
+
+	// 钢联原始指标库
+	glOb := new(dataSourceModel.BaseFromGlIndex)
+	{
+		list, e := glOb.GetItemsByCondition(cond, pars, []string{}, "")
+		if e != nil {
+			err = fmt.Errorf("获取钢联原始指标库失败, %v", e)
+			return
+		}
+		for _, v := range list {
+			indexItem := v.Format2SearchDataSource(v)
+			fmt.Printf("写入中：钢联原始指标库-%d\n", indexItem.PrimaryId)
+			docId := fmt.Sprintf("%d-%d", indexItem.Source, indexItem.PrimaryId)
+			if e := elastic.EsAddOrEditDataSourceIndex(utils.EsDataSourceIndexName, docId, indexItem); e != nil {
+				err = fmt.Errorf("钢联原始指标库-写入es失败, %v", e)
+				return
+			}
+		}
+	}
+
+	// 手工指标录入
+	manualOb := new(dataSourceModel.BaseFromManualEdb)
+	{
+		list, e := manualOb.GetItemsByCondition(cond, pars, []string{}, "")
+		if e != nil {
+			err = fmt.Errorf("获取手工指标失败, %v", e)
+			return
+		}
+		for _, v := range list {
+			indexItem := v.Format2SearchDataSource(v)
+			fmt.Printf("写入中：手工指标-%s\n", indexItem.IndexCode)
+			docId := fmt.Sprintf("%d-%s", indexItem.Source, indexItem.IndexCode)
+			if e := elastic.EsAddOrEditDataSourceIndex(utils.EsDataSourceIndexName, docId, indexItem); e != nil {
+				err = fmt.Errorf("手工指标-写入es失败, %v", e)
+				return
+			}
+		}
+	}
+
+	// Bloomberg
+	bloombergOb := new(dataSourceModel.BaseFromBloombergIndex)
+	{
+		list, e := bloombergOb.GetItemsByCondition(cond, pars, []string{}, "")
+		if e != nil {
+			err = fmt.Errorf("获取Bloomberg失败, %v", e)
+			return
+		}
+		for _, v := range list {
+			indexItem := v.Format2SearchDataSource(v)
+			fmt.Printf("写入中：Bloomberg-%d\n", indexItem.PrimaryId)
+			docId := fmt.Sprintf("%d-%d", indexItem.Source, indexItem.PrimaryId)
+			if e := elastic.EsAddOrEditDataSourceIndex(utils.EsDataSourceIndexName, docId, indexItem); e != nil {
+				err = fmt.Errorf("Bloomberg-写入es失败, %v", e)
+				return
+			}
+		}
+	}
+
+	// 煤炭江湖
+	mtjhOb := new(dataSourceModel.BaseFromMtjhMapping)
+	{
+		list, e := mtjhOb.GetItemsByCondition(cond, pars, []string{}, "")
+		if e != nil {
+			err = fmt.Errorf("获取煤炭江湖失败, %v", e)
+			return
+		}
+		for _, v := range list {
+			indexItem := v.Format2SearchDataSource(v)
+			fmt.Printf("写入中：煤炭江湖-%d\n", indexItem.PrimaryId)
+			docId := fmt.Sprintf("%d-%d", indexItem.Source, indexItem.PrimaryId)
+			if e := elastic.EsAddOrEditDataSourceIndex(utils.EsDataSourceIndexName, docId, indexItem); e != nil {
+				err = fmt.Errorf("煤炭江湖-写入es失败, %v", e)
+				return
+			}
+		}
+	}
+
+	// AI预测模型
+	aiPredictOb := new(aiPredictModel.AiPredictModelIndex)
+	{
+		list, e := aiPredictOb.GetItemsByCondition(cond, pars, []string{}, "")
+		if e != nil {
+			err = fmt.Errorf("获取AI预测模型失败, %v", e)
+			return
+		}
+		for _, v := range list {
+			indexItem := new(dataSourceModel.SearchDataSource)
+			indexItem.PrimaryId = v.AiPredictModelIndexId
+			indexItem.IndexName = v.IndexName
+			indexItem.IndexCode = v.IndexCode
+			indexItem.ClassifyId = v.ClassifyId
+			indexItem.Source = utils.DATA_SOURCE_AI_PREDICT_MODEL
+			indexItem.SourceName = "AI预测模型"
+			indexItem.CreateTime = utils.TimeTransferString(utils.FormatDateTime, v.CreateTime)
+			indexItem.ModifyTime = utils.TimeTransferString(utils.FormatDateTime, v.ModifyTime)
+			fmt.Printf("写入中：AI预测模型-%d\n", indexItem.PrimaryId)
+			docId := fmt.Sprintf("%d-%d", indexItem.Source, indexItem.PrimaryId)
+			if e := elastic.EsAddOrEditDataSourceIndex(utils.EsDataSourceIndexName, docId, indexItem); e != nil {
+				err = fmt.Errorf("AI预测模型-写入es失败, %v", e)
+				return
+			}
+		}
+	}
+
+	// 美国农业部
+	usdaOb := new(dataSourceModel.BaseFromUsdaFasIndex)
+	{
+		list, e := usdaOb.GetItemsByCondition(cond, pars, []string{}, "")
+		if e != nil {
+			err = fmt.Errorf("获取美国农业部失败, %v", e)
+			return
+		}
+		for _, v := range list {
+			indexItem := v.Format2SearchDataSource(v)
+			fmt.Printf("写入中：美国农业部-%d\n", indexItem.PrimaryId)
+			docId := fmt.Sprintf("%d-%d", indexItem.Source, indexItem.PrimaryId)
+			if e := elastic.EsAddOrEditDataSourceIndex(utils.EsDataSourceIndexName, docId, indexItem); e != nil {
+				err = fmt.Errorf("美国农业部-写入es失败, %v", e)
+				return
+			}
+		}
+	}
+
+	// 自有数据
+	businessOb := new(dataSourceModel.BaseFromBusinessIndex)
+	{
+		list, e := businessOb.GetItemsByCondition(cond, pars, []string{}, "")
+		if e != nil {
+			err = fmt.Errorf("获取自有数据失败, %v", e)
+			return
+		}
+		for _, v := range list {
+			indexItem := v.Format2SearchDataSource(v)
+			fmt.Printf("写入中：自有数据-%d\n", indexItem.PrimaryId)
+			docId := fmt.Sprintf("%d-%d", indexItem.Source, indexItem.PrimaryId)
+			if e := elastic.EsAddOrEditDataSourceIndex(utils.EsDataSourceIndexName, docId, indexItem); e != nil {
+				err = fmt.Errorf("自有数据-写入es失败, %v", e)
+				return
+			}
+		}
+	}
+
+	return
+}

services/elastic/elastic.go

@@ -0,0 +1,32 @@
+package elastic
+
+import (
+	"context"
+	dataSourceModel "eta/eta_task/models/data_source"
+	"eta/eta_task/utils"
+	"fmt"
+)
+
+// EsAddOrEditDataSourceIndex 新增/修改es中的数据源指标
+func EsAddOrEditDataSourceIndex(indexName, docId string, item *dataSourceModel.SearchDataSource) (err error) {
+	defer func() {
+		if err != nil {
+			tips := fmt.Sprintf("EsAddOrEditDataSourceIndex err: %v", err)
+			utils.FileLog.Info(tips)
+		}
+	}()
+	client := utils.EsClient
+
+	resp, e := client.Index().Index(indexName).Id(docId).BodyJson(item).Do(context.Background())
+	if e != nil {
+		err = fmt.Errorf("resp err, %v", e)
+		return
+	}
+	if resp.Status != 0 {
+		err = fmt.Errorf("result err, status: %d, result: %s", resp.Status, resp.Result)
+		return
+	}
+	err = nil
+	fmt.Println("data source write to es success", resp.Result)
+	return
+}

services/task.go

@@ -40,6 +40,12 @@ func Task() {
 		task.AddTask("refreshTradeAnalysisTables", refreshTradeAnalysisTables)
 	}
 
+	// 测试-同步数据源ES
+	if utils.RunMode == "debug" {
+		syncDataSourceEs := task.NewTask("syncDataSourceEs", "0 30 4 * * *", data.SyncDataSourceEs)
+		task.AddTask("syncDataSourceEs", syncDataSourceEs)
+	}
+
 	task.StartTask()
 	fmt.Println("task end")
 }
@@ -151,6 +157,10 @@ func releaseTask() {
 	// (基于交易所数据)多空分析/相关性表格刷新
 	refreshTradeAnalysisTables := task.NewTask("refreshTradeAnalysisTables", "0 0 20 * * *", data.RefreshTradeAnalysisTables)
 	task.AddTask("refreshTradeAnalysisTables", refreshTradeAnalysisTables)
+
+	// 每日同步数据源ES（business_conf可开关）
+	syncDataSourceEs := task.NewTask("syncDataSourceEs", "0 30 4 * * *", data.SyncDataSourceEs)
+	task.AddTask("syncDataSourceEs", syncDataSourceEs)
 }
 
 func RefreshData(cont context.Context) (err error) {

utils/common.go

@@ -951,3 +951,11 @@ func HmacSha256(key string, data string) []byte {
 func HmacSha256ToBase64(key string, data string) string {
 	return base64.URLEncoding.EncodeToString(HmacSha256(key, data))
 }
+
+func TimeTransferString(format string, t time.Time) string {
+	str := t.Format(format)
+	if t.IsZero() {
+		return ""
+	}
+	return str
+}

utils/config.go

@@ -100,6 +100,8 @@ var (
 	EsReportIndexName        string //研报ES索引
 	EsEnglishReportIndexName string //英文研报ES索引
 	SmartReportIndexName     string //智能研报ES索引
+	EsExcelIndexName         string // 表格ES索引名称
+	EsDataSourceIndexName    string // 数据源ES索引名称
 )
 
 // 科大讯飞--语音合成
@@ -345,4 +347,7 @@ func init() {
 
 	//任务刷新时间
 	RefreshCalculateEdbTaskTime = config["refresh_calculate_edb_task_time"]
+
+	// 初始化ES
+	initEs()
 }

utils/constants.go

@@ -101,14 +101,27 @@ const (
 	DATA_SOURCE_PREDICT_CALCULATE_ZSXY                          // 预测指数修匀->73
 	DATA_SOURCE_CALCULATE_ZDYFX                                 // 自定义分析->74
 	DATA_SOURCE_CALCULATE_RJZ                                   // 日均值计算->75
+	DATA_SOURCE_YONYI                                = 76       //涌益咨询
+	DATA_SOURCE_FENWEI                               = 77       // 汾渭数据->77
 	DATA_SOURCE_GFEX                                 = 78       // 广州期货交易所->78
 	DATA_SOURCE_ICPI                                 = 79       // ICPI消费价格指数->79
+	DATA_SOURCE_MTJH                                 = 80       // 煤炭江湖
 	DATA_SOURCE_BLOOMBERG                            = 83       // bloomberg彭博数据
 	DATA_SOURCE_BUSINESS                             = 84       // 来源于自有数据
+	DATA_SOURCE_SCI99                                = 85       // 卓创资讯 -> 85
 	DATA_SOURCE_CCF                                  = 86       // CCF化纤信息
+	DATA_SOURCE_CALCULATE_RANGEANLYSIS               = 87       //区间计算->87
+	DATA_SOURCE_SCI_HQ                               = 88       // 卓创红期->88
+	DATA_SOURCE_OILCHEM                              = 89       // 隆众资讯 -> 89
+	DATA_SOURCE_PREDICT_CALCULATE_RANGEANLYSIS       = 90       // 预测指标区间计算->90
+	DATA_SOURCE_LY                                   = 91       // 粮油商务网
 	DATA_SOURCE_TRADE_ANALYSIS                       = 92       // 持仓分析
+	DATA_SOURCE_HISUGAR                              = 93       // 泛糖科技 -> 93
 	DATA_SOURCE_USDA_FAS                             = 96       //美国农业部
+	DATA_SOURCE_RZD                                  = 97       // 睿姿得数据
+	DATA_SOURCE_CALCULATE_STL                        = 98       // STL趋势分解 -> 98
 	DATA_SOURCE_GPR_RISK                             = 102      //GPR地缘风险指数
+	DATA_SOURCE_AI_PREDICT_MODEL                     = 103      // AI预测模型
 )
 
 // 数据刷新频率
@@ -156,8 +169,9 @@ const (
 
 // 子数据来源渠道
 const (
-	DATA_SUB_SOURCE_EDB  = iota //经济数据库
-	DATA_SUB_SOURCE_DATE        //日期序列
+	DATA_SUB_SOURCE_EDB            = iota //经济数据库
+	DATA_SUB_SOURCE_DATE                  //日期序列
+	DATA_SUB_SOURCE_HIGH_FREQUENCY        //高频数据
 )
 
 // 已经处理了的变更id

utils/elastic.go

@@ -0,0 +1,21 @@
+package utils
+
+import (
+	"github.com/olivere/elastic/v7"
+)
+
+// EsClient es客户端
+var EsClient *elastic.Client
+
+func initEs() {
+	client, err := elastic.NewClient(
+		elastic.SetURL(ES_URL),
+		elastic.SetBasicAuth(ES_USERNAME, ES_PASSWORD),
+		elastic.SetSniff(false))
+	if err != nil {
+		panic("ElasticSearch连接失败,err:" + err.Error())
+		//go alarm_msg.SendAlarmMsg("ElasticSearch连接失败", 2)
+	}
+	EsClient = client
+	return
+}