Merge branch 'feature/eta_2.4.8' into debug

controllers/base_from_radish_research.go

@@ -0,0 +1,187 @@
+package controllers
+
+import (
+	"encoding/json"
+	"eta/eta_index_lib/logic"
+	"eta/eta_index_lib/models"
+	"eta/eta_index_lib/services"
+	"eta/eta_index_lib/utils"
+	"fmt"
+	"strconv"
+	"time"
+)
+
+// RadishResearchController 萝卜投研
+type RadishResearchController struct {
+	BaseAuthController
+}
+
+// Add
+// @Title 新增萝卜投研接口
+// @Description 新增萝卜投研接口
+// @Success 200 {object} models.AddEdbInfoReq
+// @router /add [post]
+func (this *RadishResearchController) Add() {
+	br := new(models.BaseResponse).Init()
+	var cacheKey string
+	defer func() {
+		if br.ErrMsg == "" {
+			br.IsSendEmail = false
+		}
+		this.Data["json"] = br
+		this.ServeJSON()
+	}()
+	var req models.AddEdbInfoReq
+	err := json.Unmarshal(this.Ctx.Input.RequestBody, &req)
+	if err != nil {
+		br.Msg = "参数解析异常！"
+		br.ErrMsg = "参数解析失败，Err:" + err.Error()
+		return
+	}
+	if req.EdbCode == "" {
+		br.Msg = "请输入指标编码！"
+		br.ErrMsg = "请输入指标编码，指标编码为空"
+		return
+	}
+
+	ob := new(models.BaseFromRadishResearch)
+	cacheKey = fmt.Sprintf("%s_%d_%s", utils.CACHE_EDB_DATA_ADD, ob.GetSource(), req.EdbCode)
+	if utils.Rc.IsExist(cacheKey) {
+		br.Ret = 501
+		br.Success = true
+		br.Msg = "系统处理中，请稍后重试"
+		return
+	}
+	utils.Rc.SetNX(cacheKey, 1, 1*time.Minute)
+	defer func() {
+		_ = utils.Rc.Delete(cacheKey)
+	}()
+
+	if err = ob.Add(req.EdbCode); err != nil {
+		br.Msg = "操作失败"
+		br.ErrMsg = fmt.Sprintf("新增萝卜投研指标数据失败, %v", err)
+		return
+	}
+
+	br.Ret = 200
+	br.Success = true
+	br.Msg = "操作失败"
+}
+
+// Refresh
+// @Title 刷新萝卜投研接口
+// @Description 刷新萝卜投研接口
+// @Success 200 {object} models.RefreshEdbInfoReq
+// @router /refresh [post]
+func (this *RadishResearchController) Refresh() {
+	br := new(models.BaseResponse).Init()
+	var cacheKey string
+	defer func() {
+		if br.ErrMsg == "" {
+			br.IsSendEmail = false
+		}
+		this.Data["json"] = br
+		this.ServeJSON()
+	}()
+	var req models.RefreshEdbInfoReq
+	err := json.Unmarshal(this.Ctx.Input.RequestBody, &req)
+	if err != nil {
+		br.Msg = "参数解析异常！"
+		br.ErrMsg = "参数解析失败，Err:" + err.Error()
+		return
+	}
+	if req.EdbCode == "" {
+		br.Msg = "请输入指标编码！"
+		br.ErrMsg = "请输入指标编码，指标编码为空"
+		return
+	}
+	if req.EdbInfoId < 0 {
+		br.Msg = "请输入指标ID！"
+		br.ErrMsg = "请输入指标ID"
+		return
+	}
+
+	ob := new(models.BaseFromRadishResearch)
+	source := ob.GetSource()
+	// 获取指标详情
+	edbInfo, err := models.GetEdbInfoByEdbCode(source, req.EdbCode)
+	if err != nil {
+		br.Msg = "指标不存在！"
+		br.ErrMsg = "指标不存在"
+		return
+	}
+	if edbInfo != nil && edbInfo.EdbInfoId <= 0 {
+		br.Msg = "指标不存在！"
+		br.ErrMsg = "指标不存在"
+		return
+	}
+	cacheKey = utils.CACHE_EDB_DATA_REFRESH + strconv.Itoa(source) + "_" + req.EdbCode
+	if utils.Rc.IsExist(cacheKey) {
+		br.Ret = 501
+		br.Success = true
+		br.Msg = "系统处理中，请稍后重试"
+		return
+	}
+	utils.Rc.SetNX(cacheKey, 1, 1*time.Minute)
+	defer func() {
+		_ = utils.Rc.Delete(cacheKey)
+	}()
+	if req.EdbInfoId <= 0 {
+		req.EdbInfoId = edbInfo.EdbInfoId
+	}
+
+	if err = ob.Refresh(req.EdbInfoId, req.EdbCode, req.StartDate); err != nil {
+		br.Msg = "刷新指标信息失败！"
+		br.ErrMsg = fmt.Sprintf("刷新萝卜投研指标失败, %v", err)
+		return
+	}
+
+	// 更新指标最大最小值
+	err, errMsg := models.UnifiedModifyEdbInfoMaxAndMinInfo(edbInfo)
+	if err != nil {
+		br.Msg = errMsg
+		br.ErrMsg = err.Error()
+		return
+	}
+
+	// 更新ES
+	go logic.UpdateEs(edbInfo.EdbInfoId)
+
+	br.Ret = 200
+	br.Success = true
+	br.Msg = "获取成功"
+}
+
+// HandleIndex
+// @Title 处理萝卜投研的指标数据
+// @Description 处理萝卜投研的指标数据接口
+// @Success 200 string "操作成功"
+// @router /handle/index [post]
+func (this *RadishResearchController) HandleIndex() {
+	br := new(models.BaseResponse).Init()
+	defer func() {
+		if br.ErrMsg == "" {
+			br.IsSendEmail = false
+		}
+		this.Data["json"] = br
+		this.ServeJSON()
+	}()
+	var req models.HandleRadishResearchEdbDataReq
+	err := json.Unmarshal(this.Ctx.Input.RequestBody, &req)
+	if err != nil {
+		br.Msg = "参数解析异常！"
+		br.ErrMsg = "参数解析失败，Err:" + err.Error()
+		return
+	}
+
+	// 处理excel表数据
+	if err = services.HandleRadishResearchIndex(&req); err != nil {
+		br.Msg = "处理失败"
+		br.ErrMsg = "处理失败,Err:" + err.Error()
+		return
+	}
+
+	br.Ret = 200
+	br.Success = true
+	br.Msg = "处理成功"
+}

logic/base_edb_info.go

@@ -36,6 +36,9 @@ func RefreshBaseEdbInfo(edbInfo *models.EdbInfo, startDate string) (isHandling b
 		err = ccfOb.Refresh(edbInfo.EdbInfoId, edbInfo.EdbCode, startDate)
 	case utils.DATA_SOURCE_SCI_HQ:
 		err = models.RefreshEdbDataFromSciHq(edbInfo.EdbInfoId, edbInfo.EdbCode, startDate)
+	case utils.DATA_SOURCE_RADISH_RESEARCH:
+		ob := new(models.BaseFromRadishResearch)
+		err = ob.Refresh(edbInfo.EdbInfoId, edbInfo.EdbCode, startDate)
 	default:
 		return
 	}

models/base_from_radish_research.go

@@ -0,0 +1,435 @@
+package models
+
+import (
+	sql2 "database/sql"
+	"eta/eta_index_lib/global"
+	"eta/eta_index_lib/utils"
+	"fmt"
+	"gorm.io/gorm"
+	"strconv"
+	"strings"
+	"time"
+)
+
+// BaseFromRadishResearch 萝卜投研
+type BaseFromRadishResearch struct{}
+
+type BaseFromRadishResearchData struct {
+	BaseFromRadishResearchDataId  int `gorm:"column:base_from_radish_research_data_id;primaryKey"`
+	BaseFromRadishResearchIndexId int
+	IndexCode                     string
+	DataTime                      string
+	Value                         string
+	CreateTime                    time.Time
+	ModifyTime                    time.Time
+	DataTimestamp                 int64
+}
+
+func (m *BaseFromRadishResearchData) TableName() string {
+	return "base_from_radish_research_data"
+}
+
+func (e *BaseFromRadishResearchData) AfterFind(db *gorm.DB) (err error) {
+	e.DataTime = utils.GormDateStrToDateStr(e.DataTime)
+	return
+}
+
+func GetBaseFromRadishResearchDataByCondition(condition string, pars []interface{}) (list []*BaseFromRadishResearchData, err error) {
+	sql := `SELECT * FROM base_from_radish_research_data WHERE 1=1 `
+	if condition != "" {
+		sql += condition
+	}
+	err = global.DEFAULT_DB.Raw(sql, pars...).Find(&list).Error
+	return
+}
+
+// Add 添加
+func (obj BaseFromRadishResearch) Add(edbCode string) (err error) {
+	var condition string
+	var pars []interface{}
+	if edbCode != "" {
+		condition += " AND index_code = ? "
+		pars = append(pars, edbCode)
+	}
+	dataList, err := GetBaseFromRadishResearchDataByCondition(condition, pars)
+	if err != nil {
+		return
+	}
+	var isAdd bool
+	addSql := ` INSERT INTO edb_data_radish_research(edb_info_id,edb_code,data_time,value,create_time,modify_time,data_timestamp) values `
+	addSql = utils.ReplaceDriverKeywords("", addSql)
+	existMap := make(map[string]string)
+	for _, sv := range dataList {
+		eDate := sv.DataTime
+		dataTime, err := time.Parse(utils.FormatDate, eDate)
+		if err != nil {
+			return err
+		}
+		timestamp := dataTime.UnixNano() / 1e6
+		timeStr := fmt.Sprintf("%d", timestamp)
+		if _, ok := existMap[eDate]; !ok {
+			addSql += GetAddSql("0", edbCode, eDate, timeStr, sv.Value)
+			isAdd = true
+		}
+		existMap[eDate] = sv.Value
+	}
+	if isAdd {
+		addSql = strings.TrimRight(addSql, ",")
+		utils.FileLog.Info("addSql:" + addSql)
+		err = global.DEFAULT_DB.Exec(addSql).Error
+		if err != nil {
+			return err
+		}
+	}
+	return
+}
+
+// Refresh 刷新萝卜投研指标数据
+func (obj BaseFromRadishResearch) Refresh(edbInfoId int, edbCode, startDate string) (err error) {
+	source := obj.GetSource()
+	edbInfoIdStr := strconv.Itoa(edbInfoId)
+
+	var condition string
+	var pars []interface{}
+	if edbCode != "" {
+		condition += " AND index_code=? "
+		pars = append(pars, edbCode)
+	}
+	if startDate != "" {
+		condition += " AND data_time>=? "
+		pars = append(pars, startDate)
+	}
+	dataList, e := GetBaseFromRadishResearchDataByCondition(condition, pars)
+	if e != nil {
+		err = fmt.Errorf("GetBaseFromRadishResearchDataByCondition err: %v", e)
+		return
+	}
+
+	// 真实数据的最大日期  ， 插入规则配置的日期
+	var realDataMaxDate, edbDataInsertConfigDate time.Time
+	var edbDataInsertConfig *EdbDataInsertConfig
+	var isFindConfigDateRealData bool //是否找到配置日期的实际数据的值
+	{
+		edbDataInsertConfig, err = GetEdbDataInsertConfigByEdbId(edbInfoId)
+		if err != nil && !utils.IsErrNoRow(err) {
+			return
+		}
+		if edbDataInsertConfig != nil {
+			edbDataInsertConfigDate = edbDataInsertConfig.Date
+		}
+	}
+
+	var existCondition string
+	var existPars []interface{}
+
+	existCondition += " AND edb_info_id=? "
+	existPars = append(existPars, edbInfoId)
+	if startDate != "" {
+		existCondition += " AND data_time>=? "
+		existPars = append(existPars, startDate)
+	}
+
+	existList, err := GetEdbDataByCondition(source, 0, existCondition, existPars)
+	if err != nil {
+		return err
+	}
+	existMap := make(map[string]*EdbInfoSearchData)
+	for _, v := range existList {
+		existMap[v.DataTime] = v
+	}
+	addSql := ` INSERT INTO edb_data_radish_research(edb_info_id,edb_code,data_time,value,create_time,modify_time,data_timestamp) values `
+	addSql = utils.ReplaceDriverKeywords("", addSql)
+	var isAdd bool
+	for _, v := range dataList {
+		item := v
+		eDate := item.DataTime
+		dataTime, err := time.ParseInLocation(utils.FormatDate, eDate, time.Local)
+		if err != nil {
+			return err
+		}
+		if findItem, ok := existMap[v.DataTime]; !ok {
+			sValue := item.Value
+
+			timestamp := dataTime.UnixNano() / 1e6
+			timeStr := fmt.Sprintf("%d", timestamp)
+
+			addSql += GetAddSql(edbInfoIdStr, edbCode, eDate, timeStr, sValue)
+			isAdd = true
+		} else {
+			if findItem != nil && utils.SubFloatToString(findItem.Value, 30) != item.Value {
+				err = ModifyEdbDataById(source, 0, findItem.EdbDataId, item.Value)
+				if err != nil {
+					return err
+				}
+			}
+		}
+
+		// 下面代码主要目的是处理掉手动插入的数据判断
+		{
+			if realDataMaxDate.IsZero() || dataTime.After(realDataMaxDate) {
+				realDataMaxDate = dataTime
+			}
+			if edbDataInsertConfigDate.IsZero() || dataTime.Equal(edbDataInsertConfigDate) {
+				isFindConfigDateRealData = true
+			}
+		}
+	}
+
+	// 处理手工数据补充的配置
+	HandleConfigInsertEdbData(realDataMaxDate, edbDataInsertConfig, edbInfoId, source, 0, existMap, isFindConfigDateRealData)
+
+	if isAdd {
+		addSql = strings.TrimRight(addSql, ",")
+		err = global.DEFAULT_DB.Exec(addSql).Error
+		if err != nil {
+			fmt.Println("BaseFromRadishResearch add Err", err.Error())
+			return
+		}
+	}
+	return
+}
+
+// GetSource 获取来源编码id
+func (obj BaseFromRadishResearch) GetSource() int {
+	return utils.DATA_SOURCE_RADISH_RESEARCH
+}
+
+// GetSourceName 获取来源名称
+func (obj BaseFromRadishResearch) GetSourceName() string {
+	return utils.DATA_SOURCE_NAME_RADISH_RESEARCH
+}
+
+type BaseFromRadishResearchIndex struct {
+	BaseFromRadishResearchIndexId int `gorm:"column:base_from_radish_research_index_id;primaryKey"`
+	IndexCode                     string
+	IndexName                     string
+	Frequency                     string
+	Unit                          string
+	Source                        string `description:"数据来源"`
+	ClassifyId                    int
+	StartDate                     string
+	EndDate                       string
+	Sort                          int
+	TerminalCode                  string
+	FilePath                      string
+	LatestValue                   float64
+	CreateTime                    time.Time
+	ModifyTime                    time.Time
+}
+
+func (e *BaseFromRadishResearchIndex) AfterFind(db *gorm.DB) (err error) {
+	e.StartDate = utils.GormDateStrToDateStr(e.StartDate)
+	e.EndDate = utils.GormDateStrToDateStr(e.EndDate)
+	return
+}
+
+func (m *BaseFromRadishResearchIndex) TableName() string {
+	return "base_from_radish_research_index"
+}
+
+func (m *BaseFromRadishResearchIndex) Add() (lastId int64, err error) {
+	err = global.DEFAULT_DB.Create(&m).Error
+	if err != nil {
+		return
+	}
+	lastId = int64(m.BaseFromRadishResearchIndexId)
+	return
+}
+
+func (m *BaseFromRadishResearchIndex) Update(updateCols []string) (err error) {
+	err = global.DEFAULT_DB.Model(&m).Select(updateCols).Updates(&m).Error
+	return
+}
+
+func (m *BaseFromRadishResearchIndex) ModifyIndexMaxAndMinDate(indexCode string, item *EdbInfoMaxAndMinInfo) (err error) {
+	sql := ` UPDATE base_from_radish_research_index SET start_date=?,end_date=?,latest_value=?,modify_time=NOW() WHERE index_code=? `
+	err = global.DEFAULT_DB.Exec(sql, item.MinDate, item.MaxDate, item.LatestValue, indexCode).Error
+	return
+}
+
+func (m *BaseFromRadishResearchIndex) GetByIndexCode(indexCode string) (item *BaseFromRadishResearchIndex, err error) {
+	sql := ` SELECT * FROM base_from_radish_research_index WHERE index_code = ? `
+	err = global.DEFAULT_DB.Raw(sql, indexCode).First(&item).Error
+	return
+}
+
+func (m *BaseFromRadishResearchIndex) GetItemsByCondition(condition string, pars []interface{}, fieldArr []string, orderRule string) (items []*BaseFromRadishResearchIndex, err error) {
+	fields := strings.Join(fieldArr, ",")
+	if len(fieldArr) == 0 {
+		fields = `*`
+	}
+	order := `ORDER BY create_time DESC`
+	if orderRule != "" {
+		order = ` ORDER BY ` + orderRule
+	}
+	sql := fmt.Sprintf(`SELECT %s FROM base_from_radish_research_index WHERE 1=1 %s %s`, fields, condition, order)
+	err = global.DEFAULT_DB.Raw(sql, pars...).Find(&items).Error
+	return
+}
+
+type BaseFromRadishResearchIndexList struct {
+	RadishResearchIndexId int64 `gorm:"column:ccf_index_id;primaryKey"`
+	IndexCode             string
+	IndexName             string
+	Frequency             string
+	Unit                  string
+	ClassifyId            int
+	StartDate             string
+	EndDate               string
+	TerminalCode          string
+	CreateTime            string
+	ModifyTime            string
+}
+
+func (m *BaseFromRadishResearchData) GetByIndexCode(indexCode string) (list []*BaseFromRadishResearchData, err error) {
+	sql := ` SELECT * FROM base_from_radish_research_data WHERE index_code=? `
+	err = global.DEFAULT_DB.Raw(sql, indexCode).Find(&list).Error
+	return
+}
+
+func (m *BaseFromRadishResearchData) AddMulti(item []*BaseFromRadishResearchData) (err error) {
+	err = global.DEFAULT_DB.CreateInBatches(&item, utils.MultiAddNum).Error
+	return
+}
+
+// HandleRadishResearchEdbData 萝卜投研的指标数据
+type HandleRadishResearchEdbData struct {
+	IndexName    string            `description:"指标名称"`
+	IndexCode    string            `description:"指标编码"`
+	ClassifyId   int               `description:"分类ID"`
+	Unit         string            `description:"单位"`
+	Source       string            `description:"数据来源"`
+	Sort         int               `description:"排序"`
+	Frequency    string            `description:"频度"`
+	ExcelDataMap map[string]string `description:"指标数据"`
+}
+
+type HandleRadishResearchEdbDataReq struct {
+	List         []*HandleRadishResearchEdbData
+	TerminalCode string `description:"终端编码"`
+	FilePath     string `description:"文件存储路径"`
+}
+
+type HandleRadishResearchTableData struct {
+	ClassifyId   int       `description:"分类ID"`
+	FromPage     string    `description:"表格来源"`
+	TableDate    time.Time `description:"表格日期"`
+	TableContent string    `description:"表格HTML"`
+}
+
+type HandleRadishResearchStockTableReq struct {
+	Table        *HandleRadishResearchTableData
+	TerminalCode string `description:"编码"`
+}
+
+func (m *BaseFromRadishResearchData) GetMaxAndMinDateByIndexCode(indexCode string) (item *EdbInfoMaxAndMinInfo, err error) {
+	sql := ` SELECT MIN(data_time) AS min_date,MAX(data_time) AS max_date,MIN(value) AS min_value,MAX(value) AS max_value FROM base_from_radish_research_data WHERE index_code=? `
+	err = global.DEFAULT_DB.Raw(sql, indexCode).First(&item).Error
+	sql = ` SELECT value AS latest_value FROM base_from_radish_research_data WHERE index_code=? ORDER BY data_time DESC LIMIT 1 `
+	var latestValueNull sql2.NullFloat64
+	err = global.DEFAULT_DB.Raw(sql, indexCode).Scan(&latestValueNull).Error
+	if err == nil && latestValueNull.Valid {
+		item.LatestValue = latestValueNull.Float64
+	}
+	return
+}
+
+// BaseFromRadishResearchClassify 萝卜投研数据分类表
+type BaseFromRadishResearchClassify struct {
+	ClassifyId      int       `gorm:"column:classify_id;primaryKey"`
+	ClassifyName    string    `description:"分类名称"`
+	ParentId        int       `description:"父级id"`
+	SysUserId       int       `description:"创建人id"`
+	SysUserRealName string    `description:"创建人姓名"`
+	Level           int       `description:"层级"`
+	Sort            int       `description:"排序字段，越小越靠前，默认值：10"`
+	ModifyTime      time.Time `description:"修改时间"`
+	CreateTime      time.Time `description:"创建时间"`
+}
+
+func (m *BaseFromRadishResearchClassify) Add() (lastId int64, err error) {
+	err = global.DEFAULT_DB.Create(&m).Error
+	if err != nil {
+		return
+	}
+	lastId = int64(m.ClassifyId)
+	return
+}
+
+func (m *BaseFromRadishResearchClassify) Update(updateCols []string) (err error) {
+	err = global.DEFAULT_DB.Model(&m).Select(updateCols).Updates(&m).Error
+	return
+}
+
+func (m *BaseFromRadishResearchClassify) GetByClassifyName(classifyName string) (item *BaseFromRadishResearchClassify, err error) {
+	sql := ` SELECT * FROM base_from_ccf_classify WHERE classify_name=? `
+	err = global.DEFAULT_DB.Raw(sql, classifyName).First(&item).Error
+	return
+}
+
+func (m *BaseFromRadishResearchClassify) GetItemsByCondition(condition string, pars []interface{}, fieldArr []string, orderRule string) (items []*BaseFromRadishResearchClassify, err error) {
+	fields := strings.Join(fieldArr, ",")
+	if len(fieldArr) == 0 {
+		fields = `*`
+	}
+	order := `ORDER BY create_time DESC`
+	if orderRule != "" {
+		order = ` ORDER BY ` + orderRule
+	}
+	sql := fmt.Sprintf(`SELECT %s FROM base_from_ccf_classify WHERE 1=1 %s %s`, fields, condition, order)
+	err = global.DEFAULT_DB.Raw(sql, pars...).Find(&items).Error
+	return
+}
+
+// BaseFromRadishResearchClassifyItem 萝卜投研数据分类信息
+type BaseFromRadishResearchClassifyItem struct {
+	ClassifyId   int                        `description:"分类ID"`
+	ClassifyName string                     `description:"分类名称"`
+	ParentId     int                        `description:"父级id"`
+	Level        int                        `description:"层级"`
+	Sort         int                        `description:"排序字段"`
+	CreateTime   string                     `description:"创建时间"`
+	ModifyTime   string                     `description:"修改时间"`
+	Child        []*BaseFromCCFClassifyItem `description:"子分类"`
+}
+
+func (m *BaseFromRadishResearchClassify) Format2Item(origin *BaseFromRadishResearchClassify) (item *BaseFromCCFClassifyItem) {
+	if origin == nil {
+		return
+	}
+	item = new(BaseFromCCFClassifyItem)
+	item.ClassifyId = origin.ClassifyId
+	item.ClassifyName = origin.ClassifyName
+	item.ParentId = origin.ParentId
+	item.Level = origin.Level
+	item.Sort = origin.Sort
+	item.CreateTime = utils.TimeTransferString(utils.FormatDateTime, origin.CreateTime)
+	item.ModifyTime = utils.TimeTransferString(utils.FormatDateTime, origin.ModifyTime)
+	return
+}
+
+// MultiUpdateBaseFromRadishResearchDataValue 批量更新萝卜投研指标数据
+func MultiUpdateBaseFromRadishResearchDataValue(items []*BaseFromRadishResearchData) (err error) {
+	if len(items) == 0 {
+		return
+	}
+	to := global.DEFAULT_DB.Begin()
+	defer func() {
+		if err != nil {
+			_ = to.Rollback()
+		} else {
+			_ = to.Commit()
+		}
+	}()
+	sql := `UPDATE base_from_radish_research_data SET value = ?, modify_time = NOW() WHERE index_code = ? AND data_time = ? LIMIT 1`
+	for _, v := range items {
+		if v.IndexCode == "" || v.DataTime == "" {
+			continue
+		}
+		err = to.Exec(sql, v.Value, v.IndexCode, v.DataTime).Error
+		if err != nil {
+			return
+		}
+	}
+	return
+}

models/business_conf.go

@@ -48,6 +48,7 @@ const (
 	BusinessConfTencentApiSecretKey          = "TencentApiSecretKey"          // 腾讯云API-密钥对
 	BusinessConfTencentApiRecTaskCallbackUrl = "TencentApiRecTaskCallbackUrl" // 腾讯云API-语音识别回调地址
 	BusinessConfSmsJhgjVariable              = "SmsJhgjVariable"              // 聚合国际短信变量
+	BusinessConfEsIndexNameDataSource        = "EsIndexNameDataSource"        // ES索引名称-数据源
 )
 
 const (
@@ -240,3 +241,19 @@ func InitSmmDataMethodConf() {
 		utils.SmmDataMethod = "api"
 	}
 }
+
+var (
+	BusinessConfMap map[string]string
+)
+
+func InitBusinessConf() {
+	var e error
+	BusinessConfMap, e = GetBusinessConf()
+	if e != nil {
+		return
+	}
+	// ES索引名称
+	if BusinessConfMap[BusinessConfEsIndexNameDataSource] != "" {
+		utils.EsDataSourceIndexName = BusinessConfMap[BusinessConfEsIndexNameDataSource]
+	}
+}

models/data_source.go

@@ -0,0 +1,20 @@
+package models
+
+// SearchDataSource 数据源ES搜索
+type SearchDataSource struct {
+	PrimaryId   int    `description:"主键ID"`
+	IndexCode   string `description:"指标编码"`
+	IndexName   string `description:"指标名称"`
+	ClassifyId  int    `description:"分类ID"`
+	Unit        string `description:"单位"`
+	Frequency   string `description:"频度"`
+	StartDate   string `description:"开始日期"`
+	EndDate     string `description:"结束日期"`
+	LatestValue string `description:"最新值(部分数据源如中国煤炭网存在97%这种数据...)"`
+	Source      int    `description:"来源"`
+	SourceName  string `description:"数据源名称"`
+	SubSource   int    `description:"子来源"`
+	IsDeleted   int    `description:"是否已删除：0-正常；1-已删除"`
+	CreateTime  string `description:"创建时间"`
+	ModifyTime  string `description:"修改时间"`
+}

models/db.go

@@ -248,4 +248,7 @@ func AfterInitTable() {
 
 	// 初始化Smm有色指标数据获取方式
 	InitSmmDataMethodConf()
+
+	// 初始化商家基本配置
+	InitBusinessConf()
 }

routers/commentsRouter.go

@@ -1654,6 +1654,33 @@ func init() {
             Filters: nil,
             Params: nil})
 
+    beego.GlobalControllerRouter["eta/eta_index_lib/controllers:RadishResearchController"] = append(beego.GlobalControllerRouter["eta/eta_index_lib/controllers:RadishResearchController"],
+        beego.ControllerComments{
+            Method: "Add",
+            Router: `/add`,
+            AllowHTTPMethods: []string{"post"},
+            MethodParams: param.Make(),
+            Filters: nil,
+            Params: nil})
+
+    beego.GlobalControllerRouter["eta/eta_index_lib/controllers:RadishResearchController"] = append(beego.GlobalControllerRouter["eta/eta_index_lib/controllers:RadishResearchController"],
+        beego.ControllerComments{
+            Method: "HandleIndex",
+            Router: `/handle/index`,
+            AllowHTTPMethods: []string{"post"},
+            MethodParams: param.Make(),
+            Filters: nil,
+            Params: nil})
+
+    beego.GlobalControllerRouter["eta/eta_index_lib/controllers:RadishResearchController"] = append(beego.GlobalControllerRouter["eta/eta_index_lib/controllers:RadishResearchController"],
+        beego.ControllerComments{
+            Method: "Refresh",
+            Router: `/refresh`,
+            AllowHTTPMethods: []string{"post"},
+            MethodParams: param.Make(),
+            Filters: nil,
+            Params: nil})
+
     beego.GlobalControllerRouter["eta/eta_index_lib/controllers:Sci99Controller"] = append(beego.GlobalControllerRouter["eta/eta_index_lib/controllers:Sci99Controller"],
         beego.ControllerComments{
             Method: "Add",

routers/router.go

@@ -352,6 +352,11 @@ func init() {
 				&controllers.BaseFromPurangController{},
 			),
 		),
+		beego.NSNamespace("/radish_research",
+			beego.NSInclude(
+				&controllers.RadishResearchController{},
+			),
+		),
 	)
 	beego.AddNamespace(ns)
 }

services/base_from_radish_research.go

@@ -0,0 +1,277 @@
+package services
+
+import (
+	"eta/eta_index_lib/logic"
+	"eta/eta_index_lib/models"
+	"eta/eta_index_lib/services/alarm_msg"
+	"eta/eta_index_lib/services/elastic"
+	"eta/eta_index_lib/utils"
+	"fmt"
+	"strings"
+	"time"
+)
+
+// HandleRadishResearchIndex 处理萝卜投研的excel数据
+func HandleRadishResearchIndex(req *models.HandleRadishResearchEdbDataReq) (err error) {
+	errMsgList := make([]string, 0)
+	defer func() {
+		if len(errMsgList) > 0 {
+			msg := fmt.Sprint("HandleRadishResearchIndex, err:", strings.Join(errMsgList, "\n"))
+			utils.FileLog.Info(msg)
+			go alarm_msg.SendAlarmMsg(msg, 3)
+		}
+	}()
+	for _, v := range req.List {
+		if v.IndexName == "" || v.IndexCode == "" {
+			errMsgList = append(errMsgList, fmt.Sprintf("新增指标异常，指标编码%s或者指标ID%s为空：", v.IndexCode, v.IndexName))
+			continue
+		}
+		err = handleRadishResearchIndex(v, req.TerminalCode, req.FilePath)
+		if err != nil {
+			errMsgList = append(errMsgList, fmt.Sprintf("新增指标异常，指标编码：%s, Err: %s", v.IndexCode, err))
+			return
+		}
+	}
+	return
+}
+
+func handleRadishResearchIndex(req *models.HandleRadishResearchEdbData, terminalCode, filePath string) (err error) {
+	indexName := req.IndexName
+	indexCode := req.IndexCode
+	excelDataMap := req.ExcelDataMap
+	errMsgList := make([]string, 0)
+	defer func() {
+		if len(errMsgList) > 0 {
+			msg := fmt.Sprint("handleRadishResearchIndex, err:", strings.Join(errMsgList, "\n"))
+			utils.FileLog.Info(msg)
+			go alarm_msg.SendAlarmMsg(msg, 3)
+		}
+	}()
+	indexObj := new(models.BaseFromRadishResearchIndex)
+	dataObj := new(models.BaseFromRadishResearchData)
+	//classifyObj := new(models.BaseFromCCFClassify)
+
+	var indexId int64
+
+	addDataList := make([]*models.BaseFromRadishResearchData, 0)
+	updateDataList := make([]*models.BaseFromRadishResearchData, 0)
+
+	exitDataMap := make(map[string]*models.BaseFromRadishResearchData)
+
+	// 修改指标信息
+	if indexName == "" {
+		utils.FileLog.Info("未刷新到指标数据:indexName:" + indexName)
+		return
+	}
+
+	//判断指标是否存在
+	var isAdd int
+	item, err := indexObj.GetByIndexCode(indexCode)
+	if err != nil {
+		if err.Error() == utils.ErrNoRow() {
+			isAdd = 1
+			err = nil
+		} else {
+			isAdd = -1
+			err = fmt.Errorf("查询数据源指标库失败 GetByIndexCode Err：%s", err)
+			return
+		}
+	}
+	if item != nil && item.BaseFromRadishResearchIndexId > 0 {
+		fmt.Println("item:", item)
+		isAdd = 2
+	} else {
+		isAdd = 1
+	}
+
+	if isAdd == 1 {
+		indexObj.IndexCode = indexCode
+		indexObj.IndexName = indexName
+		indexObj.Frequency = req.Frequency
+		indexObj.ClassifyId = req.ClassifyId
+		if req.Unit == "" {
+			req.Unit = "无"
+		}
+		indexObj.Unit = req.Unit
+		indexObj.Source = req.Source
+		indexObj.Sort = req.Sort
+		indexObj.ModifyTime = time.Now()
+		indexObj.CreateTime = time.Now()
+		indexObj.TerminalCode = terminalCode
+		indexObj.FilePath = filePath
+		indexId, err = indexObj.Add()
+		if err != nil {
+			err = fmt.Errorf("数据源新增萝卜投研指标失败 Err：%s", err)
+			return
+		}
+		indexObj.BaseFromRadishResearchIndexId = int(indexId)
+	} else if isAdd == 2 {
+		item.TerminalCode = terminalCode
+		item.FilePath = filePath
+		item.Source = req.Source
+		item.ModifyTime = time.Now().Local()
+		updateCols := []string{"TerminalCode", "FilePath", "Source", "ModifyTime"}
+		err = item.Update(updateCols)
+		if err != nil {
+			err = fmt.Errorf("数据源更新萝卜投研指标失败 Err：%s", err)
+			return
+		}
+
+		//获取已存在的所有数据
+		var exitDataList []*models.BaseFromRadishResearchData
+		exitDataList, err = dataObj.GetByIndexCode(indexCode)
+		if err != nil {
+			err = fmt.Errorf("数据源查询萝卜投研指标数据失败 Err：%s", err)
+			return
+		}
+		fmt.Println("exitDataListLen:", len(exitDataList))
+		for _, v := range exitDataList {
+			dateStr := v.DataTime
+			exitDataMap[dateStr] = v
+		}
+		indexId = int64(item.BaseFromRadishResearchIndexId)
+	}
+
+	// 遍历excel数据，然后跟现有的数据做校验，不存在则入库
+	for date, value := range excelDataMap {
+		if findData, ok := exitDataMap[date]; !ok {
+			_, err = time.ParseInLocation(utils.FormatDate, date, time.Local)
+			if err != nil {
+				err = fmt.Errorf("%s 转换日期格式失败 Err：%s", date, err)
+				return
+			}
+			if !strings.Contains(value, "#N/A") {
+				var saveDataTime time.Time
+				if strings.Contains(date, "00:00:00") {
+					saveDataTime, err = time.Parse(utils.FormatDateTime, date)
+				} else {
+					saveDataTime, err = time.Parse(utils.FormatDate, date)
+				}
+				if err != nil {
+					err = fmt.Errorf("%s 转换日期格式失败 Err：%s", date, err)
+					continue
+				}
+				timestamp := saveDataTime.UnixNano() / 1e6
+
+				dataItem := new(models.BaseFromRadishResearchData)
+				dataItem.BaseFromRadishResearchIndexId = int(indexId)
+				dataItem.IndexCode = indexCode
+				dataItem.DataTime = date
+				dataItem.Value = value
+				dataItem.CreateTime = time.Now()
+				dataItem.ModifyTime = time.Now()
+				dataItem.DataTimestamp = timestamp
+				addDataList = append(addDataList, dataItem)
+			}
+		} else {
+			if findData != nil && findData.Value != value && !strings.Contains(value, "#N/A") { //修改数据
+				findData.Value = value
+				findData.ModifyTime = time.Now().Local()
+				updateDataList = append(updateDataList, findData)
+			}
+		}
+	}
+
+	var esUpdated bool
+	if len(addDataList) > 0 {
+		err = dataObj.AddMulti(addDataList)
+		if err != nil {
+			err = fmt.Errorf("批量新增萝卜投研指标数据失败 Err：%s", err)
+			return
+		}
+
+		var dateItem *models.EdbInfoMaxAndMinInfo
+		dateItem, err = dataObj.GetMaxAndMinDateByIndexCode(indexCode)
+		if err != nil {
+			err = fmt.Errorf("查询指标最新日期失败 Err：%s", err)
+			return
+		}
+
+		go func() {
+			indexObj.ModifyIndexMaxAndMinDate(indexCode, dateItem)
+			_ = UpdateRadishResearchIndexEs(indexCode)
+		}()
+		esUpdated = true
+	}
+
+	// 批量更新数据
+	if len(updateDataList) > 0 {
+		e := models.MultiUpdateBaseFromRadishResearchDataValue(updateDataList)
+		if e != nil {
+			err = fmt.Errorf("MultiUpdateBaseFromRadishResearchDataValue err: %s", e.Error())
+			return
+		}
+	}
+
+	// 同步刷新ETA指标库的指标
+	{
+		// 获取指标详情
+		baseObj := new(models.BaseFromRadishResearch)
+		var edbInfo *models.EdbInfo
+		edbInfo, err = models.GetEdbInfoByEdbCode(baseObj.GetSource(), indexCode)
+		if err != nil {
+			if err.Error() != utils.ErrNoRow() {
+				errMsgList = append(errMsgList, fmt.Sprint("刷新ETA指标异常，指标编码：", indexCode, err.Error()))
+				return
+			} else {
+				err = nil
+			}
+		}
+
+		// 已经加入到指标库的话，那么就去更新ETA指标库吧
+		if edbInfo != nil {
+			go logic.RefreshBaseEdbInfo(edbInfo, ``)
+		}
+	}
+
+	// 写入es
+	if !esUpdated {
+		_ = UpdateRadishResearchIndexEs(indexCode)
+	}
+	return
+}
+
+// UpdateRadishResearchIndexEs 更新萝卜投研指标ES
+func UpdateRadishResearchIndexEs(indexCode string) (err error) {
+	defer func() {
+		if err != nil {
+			tips := fmt.Sprintf("UpdateRadishResearchIndexEs, IndexCode: %s, errMsg: %v", indexCode, err)
+			fmt.Println(tips)
+			utils.FileLog.Info(tips)
+		}
+	}()
+	indexObj := new(models.BaseFromRadishResearchIndex)
+	item, e := indexObj.GetByIndexCode(indexCode)
+	if e != nil {
+		if utils.IsErrNoRow(e) {
+			err = fmt.Errorf("指标不存在, IndexCode: %s", indexCode)
+			return
+		}
+	}
+	if item.BaseFromRadishResearchIndexId <= 0 {
+		err = fmt.Errorf("指标不存在, IndexCode: %s", indexCode)
+		return
+	}
+	indexItem := new(models.SearchDataSource)
+	indexItem.PrimaryId = item.BaseFromRadishResearchIndexId
+	indexItem.IndexCode = item.IndexCode
+	indexItem.IndexName = item.IndexName
+	indexItem.ClassifyId = item.ClassifyId
+	indexItem.Unit = item.Unit
+	indexItem.Frequency = item.Frequency
+	indexItem.StartDate = item.StartDate
+	indexItem.EndDate = item.EndDate
+	indexItem.LatestValue = fmt.Sprint(item.LatestValue)
+	indexItem.Source = utils.DATA_SOURCE_RADISH_RESEARCH
+	indexItem.SourceName = utils.DATA_SOURCE_NAME_RADISH_RESEARCH
+	indexItem.IsDeleted = 0
+	indexItem.CreateTime = item.CreateTime.Format(utils.FormatDateTime)
+	indexItem.ModifyTime = item.ModifyTime.Format(utils.FormatDateTime)
+
+	docId := fmt.Sprintf("%d-%d", utils.DATA_SOURCE_RADISH_RESEARCH, item.BaseFromRadishResearchIndexId)
+	if e := elastic.EsAddOrEditDataSourceIndex(utils.EsDataSourceIndexName, docId, indexItem); e != nil {
+		err = fmt.Errorf("指标写入ES失败, %v", e)
+		return
+	}
+	return
+}

services/elastic/elastic.go

@@ -33,3 +33,28 @@ func EsAddOrEditEdbInfoData(indexName, docId string, item *models.EdbInfoList) (
 	}
 	return
 }
+
+// EsAddOrEditDataSourceIndex 新增/修改es中的数据源指标
+func EsAddOrEditDataSourceIndex(indexName, docId string, item *models.SearchDataSource) (err error) {
+	fmt.Println("索引名：", indexName)
+	defer func() {
+		if err != nil {
+			tips := fmt.Sprintf("EsAddOrEditDataSourceIndex err: %v", err)
+			utils.FileLog.Info(tips)
+		}
+	}()
+	client := utils.EsClient
+
+	resp, e := client.Index().Index(indexName).Id(docId).BodyJson(item).Refresh("true").Do(context.Background())
+	if e != nil {
+		err = fmt.Errorf("resp err, %v", e)
+		return
+	}
+	if resp.Status != 0 {
+		err = fmt.Errorf("result err, status: %d, result: %s", resp.Status, resp.Result)
+		return
+	}
+	err = nil
+	fmt.Println("data source write to es success", resp.Result)
+	return
+}

utils/config.go

@@ -87,9 +87,10 @@ type WindUrlMap struct {
 
 // ES配置
 var (
-	ES_URL      string // ES服务器地址
-	ES_USERNAME string // ES账号
-	ES_PASSWORD string // ES密码
+	ES_URL                string // ES服务器地址
+	ES_USERNAME           string // ES账号
+	ES_PASSWORD           string // ES密码
+	EsDataSourceIndexName string // 数据源ES索引名称
 )
 
 // 初始化数据用户信息

utils/constants.go

@@ -126,6 +126,7 @@ const (
 	DATA_SOURCE_CLARKSONS                            = 101 // 克拉克森 -> 101
 	DATA_SOURCE_GPR_RISK                             = 102
 	DATA_SOURCE_PuRang                               = 104 // 普兰金融-> 104
+	DATA_SOURCE_RADISH_RESEARCH                      = 105 // 萝卜投研
 )
 
 // 指标来源的中文展示
@@ -221,6 +222,7 @@ const (
 	DATA_SOURCE_NAME_HISUGAR                              = `泛糖科技`      // 泛糖科技 -> 93
 	DATA_SOURCE_NAME_GPR_RISK                             = `GPR地缘风险指数` // GPR地缘风险指数 -> 102
 	DATA_SOURCE_NAME_PuRang                               = `普兰金融`      // 普兰金融 -> 104
+	DATA_SOURCE_NAME_RADISH_RESEARCH                      = `萝卜投研`      // 萝卜投研 -> 105
 )
 
 // 基础数据初始化日期