Merge branch 'master' of http://8.136.199.33:3000/eta_server/eta_index_lib into bzq1/shanghai

controllers/base_from_bloomberg.go

@@ -149,12 +149,12 @@ func (this *BloombergController) Refresh() {
 	br.Msg = "获取成功"
 }
 
-// PCSGRefreshDaily
-// @Title 中石油新加坡-刷新日度指标
-// @Description  中石油新加坡-刷新日度指标
-// @Success 200 {object} models.AddEdbInfoReq
-// @router /pcsg/refresh_daily [post]
-func (this *BloombergController) PCSGRefreshDaily() {
+// PCSGImportHistoryData
+// @Title 中石油新加坡-导入历史数据
+// @Description  中石油新加坡-导入历史数据
+// @Success 200 {object} models.PCSGImportHistoryDataReq
+// @router /pcsg/import_history_data [post]
+func (this *BloombergController) PCSGImportHistoryData() {
 	br := new(models.BaseResponse).Init()
 	defer func() {
 		if br.ErrMsg == "" {
@@ -163,64 +163,28 @@ func (this *BloombergController) PCSGRefreshDaily() {
 		this.Data["json"] = br
 		this.ServeJSON()
 	}()
-
-	// 获取数据
-	indexes, e := services.GetPCSGBloombergDailyFromBridge()
-	if e != nil {
-		br.Msg = "刷新失败"
-		br.ErrMsg = "Bridge-获取PCSG彭博日度指标失败, Err: " + e.Error()
-		return
-	}
-	if len(indexes) == 0 {
-		br.Ret = 200
-		br.Success = true
-		br.Msg = "操作成功"
-		return
-	}
-
-	// 写入数据
-	if e = services.PCSGWrite2BaseBloomberg(indexes); e != nil {
-		br.Msg = "刷新失败"
-		br.ErrMsg = "PCSG-写入Bloomberg数据源失败, Err: " + e.Error()
+	var req models.PCSGImportHistoryDataReq
+	if e := json.Unmarshal(this.Ctx.Input.RequestBody, &req); e != nil {
+		br.Msg = "参数解析异常！"
+		br.ErrMsg = "参数解析失败，Err:" + e.Error()
 		return
 	}
 
-	br.Ret = 200
-	br.Success = true
-	br.Msg = "操作成功"
-}
-
-// PCSGRefreshWeekly
-// @Title 中石油新加坡-刷新周度指标
-// @Description  中石油新加坡-刷新周度指标
-// @Success 200 {object} models.AddEdbInfoReq
-// @router /pcsg/refresh_weekly [post]
-func (this *BloombergController) PCSGRefreshWeekly() {
-	br := new(models.BaseResponse).Init()
-	defer func() {
-		if br.ErrMsg == "" {
-			br.IsSendEmail = false
-		}
-		this.Data["json"] = br
-		this.ServeJSON()
-	}()
-
-	// 获取数据
-	indexes, e := services.GetPCSGBloombergWeeklyFromBridge()
-	if e != nil {
-		br.Msg = "刷新失败"
-		br.ErrMsg = "Bridge-获取PCSG彭博周度指标失败, Err: " + e.Error()
-		return
-	}
-	if len(indexes) == 0 {
-		br.Ret = 200
-		br.Success = true
-		br.Msg = "操作成功"
-		return
+	var indexes []models.BaseFromBloombergApiIndexAndData
+	var index models.BaseFromBloombergApiIndexAndData
+	var indexData []models.BaseFromBloombergApiIndexData
+	index.IndexCode = req.IndexCode
+	for k, v := range req.DataMap {
+		indexData = append(indexData, models.BaseFromBloombergApiIndexData{
+			DataTime: k,
+			Value:    v,
+		})
 	}
+	index.Data = indexData
+	indexes = append(indexes, index)
 
 	// 写入数据
-	if e = services.PCSGWrite2BaseBloomberg(indexes); e != nil {
+	if e := services.PCSGWrite2BaseBloomberg(indexes, req.IsVCode); e != nil {
 		br.Msg = "刷新失败"
 		br.ErrMsg = "PCSG-写入Bloomberg数据源失败, Err: " + e.Error()
 		return
@@ -231,12 +195,12 @@ func (this *BloombergController) PCSGRefreshWeekly() {
 	br.Msg = "操作成功"
 }
 
-// PCSGRefreshMonthly
-// @Title 中石油新加坡-刷新月度指标
-// @Description  中石油新加坡-刷新周度指标
+// PCSGRefreshTask
+// @Title 中石油新加坡-刷新任务
+// @Description  中石油新加坡-刷新任务
 // @Success 200 {object} models.AddEdbInfoReq
-// @router /pcsg/refresh_monthly [post]
-func (this *BloombergController) PCSGRefreshMonthly() {
+// @router /pcsg/refresh_task [post]
+func (this *BloombergController) PCSGRefreshTask() {
 	br := new(models.BaseResponse).Init()
 	defer func() {
 		if br.ErrMsg == "" {
@@ -245,72 +209,49 @@ func (this *BloombergController) PCSGRefreshMonthly() {
 		this.Data["json"] = br
 		this.ServeJSON()
 	}()
-
-	// 获取数据
-	indexes, e := services.GetPCSGBloombergMonthlyFromBridge()
-	if e != nil {
-		br.Msg = "刷新失败"
-		br.ErrMsg = "Bridge-获取PCSG彭博月度指标失败, Err: " + e.Error()
-		return
-	}
-	if len(indexes) == 0 {
-		br.Ret = 200
-		br.Success = true
-		br.Msg = "操作成功"
-		return
+	// 非必传, 只是手动请求的时候用
+	var req struct {
+		TaskKey string
 	}
+	_ = json.Unmarshal(this.Ctx.Input.RequestBody, &req)
 
-	// 写入数据
-	if e = services.PCSGWrite2BaseBloomberg(indexes); e != nil {
-		br.Msg = "刷新失败"
-		br.ErrMsg = "PCSG-写入Bloomberg数据源失败, Err: " + e.Error()
+	tasks, e := services.LoadPCSGBloombergTask()
+	if e != nil {
+		br.Msg = "加载配置失败"
+		br.ErrMsg = fmt.Sprintf("加载配置失败, Err: %v", e)
 		return
 	}
 
-	br.Ret = 200
-	br.Success = true
-	br.Msg = "操作成功"
-}
-
-// PCSGImportHistoryData
-// @Title 中石油新加坡-导入历史数据
-// @Description  中石油新加坡-导入历史数据
-// @Success 200 {object} models.PCSGImportHistoryDataReq
-// @router /pcsg/import_history_data [post]
-func (this *BloombergController) PCSGImportHistoryData() {
-	br := new(models.BaseResponse).Init()
-	defer func() {
-		if br.ErrMsg == "" {
-			br.IsSendEmail = false
+	for _, v := range tasks {
+		if req.TaskKey != "" && v.TaskKey != req.TaskKey {
+			continue
 		}
-		this.Data["json"] = br
-		this.ServeJSON()
-	}()
-	var req models.PCSGImportHistoryDataReq
-	if e := json.Unmarshal(this.Ctx.Input.RequestBody, &req); e != nil {
-		br.Msg = "参数解析异常！"
-		br.ErrMsg = "参数解析失败，Err:" + e.Error()
-		return
-	}
+		fmt.Println(v)
+		time.Sleep(5 * time.Second)
 
-	var indexes []models.BaseFromBloombergApiIndexAndData
-	var index models.BaseFromBloombergApiIndexAndData
-	var indexData []models.BaseFromBloombergApiIndexData
-	index.IndexCode = req.IndexCode
-	for k, v := range req.DataMap {
-		indexData = append(indexData, models.BaseFromBloombergApiIndexData{
-			DataTime: k,
-			Value:    v,
-		})
-	}
-	index.Data = indexData
-	indexes = append(indexes, index)
+		// 获取数据
+		var r services.PCSGBloombergApiReq
+		r.TaskKey = v.TaskKey
+		r.Frequency = v.Frequency
+		indexes, e := services.GetPCSGBloombergGeneralIndexFromBridge(r)
+		if e != nil {
+			br.Msg = "刷新失败"
+			br.ErrMsg = "Bridge-GetPCSGBloombergGeneralIndexFromBridge, Err: " + e.Error()
+			return
+		}
+		if len(indexes) == 0 {
+			br.Ret = 200
+			br.Success = true
+			br.Msg = "操作成功"
+			return
+		}
 
-	// 写入数据
-	if e := services.PCSGWrite2BaseBloomberg(indexes); e != nil {
-		br.Msg = "刷新失败"
-		br.ErrMsg = "PCSG-写入Bloomberg数据源失败, Err: " + e.Error()
-		return
+		// 写入数据
+		if e = services.PCSGWrite2BaseBloomberg(indexes, v.VCode); e != nil {
+			br.Msg = "刷新失败"
+			br.ErrMsg = "PCSG-写入Bloomberg数据源失败, Err: " + e.Error()
+			return
+		}
 	}
 
 	br.Ret = 200

controllers/base_from_ccf.go

@@ -0,0 +1,217 @@
+package controllers
+
+import (
+	"encoding/json"
+	"eta/eta_index_lib/logic"
+	"eta/eta_index_lib/models"
+	"eta/eta_index_lib/services"
+	"eta/eta_index_lib/utils"
+	"strconv"
+	"time"
+)
+
+// CCFController CCF化纤信息
+type CCFController struct {
+	BaseAuthController
+}
+
+// Add
+// @Title 新增CCF指标接口
+// @Description 新增CCF指标接口
+// @Success 200 {object} models.AddEdbInfoReq
+// @router /add [post]
+func (this *CCFController) Add() {
+	br := new(models.BaseResponse).Init()
+	var cacheKey string
+	defer func() {
+		if br.ErrMsg == "" {
+			br.IsSendEmail = false
+		}
+		this.Data["json"] = br
+		this.ServeJSON()
+	}()
+	var req models.AddEdbInfoReq
+	err := json.Unmarshal(this.Ctx.Input.RequestBody, &req)
+	if err != nil {
+		br.Msg = "参数解析异常！"
+		br.ErrMsg = "参数解析失败，Err:" + err.Error()
+		return
+	}
+	if req.EdbCode == "" {
+		br.Msg = "请输入指标编码！"
+		br.ErrMsg = "请输入指标编码，指标编码为空"
+		return
+	}
+
+	ob := new(models.BaseFromCCF)
+	cacheKey = utils.CACHE_EDB_DATA_ADD + strconv.Itoa(ob.GetSource()) + "_" + req.EdbCode
+	if utils.Rc.IsExist(cacheKey) {
+		br.Ret = 501
+		br.Success = true
+		br.Msg = "系统处理中，请稍后重试"
+		return
+	}
+	utils.Rc.SetNX(cacheKey, 1, 1*time.Minute)
+	defer func() {
+		_ = utils.Rc.Delete(cacheKey)
+	}()
+
+	if err = ob.Add(req.EdbCode); err != nil {
+		br.Msg = "获取指标信息失败！"
+		br.ErrMsg = "获取指标信息失败 AddEdbDataFromCCF，Err:" + err.Error()
+		return
+	}
+	br.Ret = 200
+	br.Success = true
+	br.Msg = "获取成功"
+}
+
+// Refresh
+// @Title 刷新CCF指标接口
+// @Description 刷新CCF指标接口
+// @Success 200 {object} models.RefreshEdbInfoReq
+// @router /refresh [post]
+func (this *CCFController) Refresh() {
+	br := new(models.BaseResponse).Init()
+	var cacheKey string
+	defer func() {
+		if br.ErrMsg == "" {
+			br.IsSendEmail = false
+		}
+		this.Data["json"] = br
+		this.ServeJSON()
+	}()
+	var req models.RefreshEdbInfoReq
+	err := json.Unmarshal(this.Ctx.Input.RequestBody, &req)
+	if err != nil {
+		br.Msg = "参数解析异常！"
+		br.ErrMsg = "参数解析失败，Err:" + err.Error()
+		return
+	}
+	if req.EdbCode == "" {
+		br.Msg = "请输入指标编码！"
+		br.ErrMsg = "请输入指标编码，指标编码为空"
+		return
+	}
+	if req.EdbInfoId < 0 {
+		br.Msg = "请输入指标ID！"
+		br.ErrMsg = "请输入指标ID"
+		return
+	}
+
+	ob := new(models.BaseFromCCF)
+	source := ob.GetSource()
+	// 获取指标详情
+	edbInfo, err := models.GetEdbInfoByEdbCode(source, req.EdbCode)
+	if err != nil {
+		br.Msg = "指标不存在！"
+		br.ErrMsg = "指标不存在"
+		return
+	}
+	cacheKey = utils.CACHE_EDB_DATA_REFRESH + strconv.Itoa(source) + "_" + req.EdbCode
+	if utils.Rc.IsExist(cacheKey) {
+		br.Ret = 501
+		br.Success = true
+		br.Msg = "系统处理中，请稍后重试"
+		return
+	}
+	utils.Rc.SetNX(cacheKey, 1, 1*time.Minute)
+	defer func() {
+		_ = utils.Rc.Delete(cacheKey)
+	}()
+	if req.EdbInfoId <= 0 {
+		req.EdbInfoId = edbInfo.EdbInfoId
+	}
+	err = ob.Refresh(req.EdbInfoId, req.EdbCode, req.StartDate)
+	if err != nil && err.Error() != utils.ErrNoRow() {
+		br.Msg = "刷新指标信息失败！"
+		br.ErrMsg = "刷新指标信息失败 RefreshEdbDataFromCCF，Err:" + err.Error()
+		return
+	}
+	// 更新指标最大最小值
+	err, errMsg := models.UnifiedModifyEdbInfoMaxAndMinInfo(edbInfo)
+	if err != nil {
+		br.Msg = errMsg
+		br.ErrMsg = err.Error()
+		return
+	}
+
+	// 更新ES
+	go logic.UpdateEs(edbInfo.EdbInfoId)
+
+	br.Ret = 200
+	br.Success = true
+	br.Msg = "获取成功"
+}
+
+// HandleEdbData
+// @Title 处理CCF指标的接口
+// @Description 处理CCF指标的接口
+// @Success 200 string "操作成功"
+// @router /handle/edb_data [post]
+func (this *CCFController) HandleEdbData() {
+	br := new(models.BaseResponse).Init()
+	defer func() {
+		if br.ErrMsg == "" {
+			br.IsSendEmail = false
+		}
+		this.Data["json"] = br
+		this.ServeJSON()
+	}()
+	var req models.HandleCCFEdbDataReq
+	err := json.Unmarshal(this.Ctx.Input.RequestBody, &req)
+	if err != nil {
+		br.Msg = "参数解析异常！"
+		br.ErrMsg = "参数解析失败，Err:" + err.Error()
+		return
+	}
+
+	// 处理excel表数据
+	if err = services.HandleCCFIndex(&req); err != nil {
+		br.Msg = "处理失败"
+		br.ErrMsg = "处理失败,Err:" + err.Error()
+		return
+	}
+
+	br.Ret = 200
+	br.Success = true
+	br.Msg = "处理成功"
+}
+
+// HandleTableData
+// @Title 处理CCF装置表格的接口
+// @Description 处理CCF装置表格的接口
+// @Success 200 string "操作成功"
+// @router /handle/table_data [post]
+func (this *CCFController) HandleTableData() {
+	br := new(models.BaseResponse).Init()
+	defer func() {
+		if br.ErrMsg == "" {
+			br.IsSendEmail = false
+		}
+		this.Data["json"] = br
+		this.ServeJSON()
+	}()
+	var req models.HandleCCFStockTableReq
+	err := json.Unmarshal(this.Ctx.Input.RequestBody, &req)
+	if err != nil {
+		br.Msg = "参数解析异常！"
+		br.ErrMsg = "参数解析失败，Err:" + err.Error()
+		return
+	}
+	if req.Table == nil {
+		br.Msg = "表格信息为空"
+		br.ErrMsg = "表格信息为空"
+		return
+	}
+
+	if err = services.HandleCCFStockTable(&req); err != nil {
+		br.Msg = "处理失败"
+		br.ErrMsg = "处理失败,Err:" + err.Error()
+		return
+	}
+
+	br.Ret = 200
+	br.Success = true
+	br.Msg = "处理成功"
+}

controllers/base_from_python.go

@@ -82,7 +82,8 @@ func (this *PythonController) Add() {
 		br.ErrMsg = "参数解析失败，Err:" + err.Error()
 		return
 	}
-	req.EdbName = strings.Trim(req.EdbName, " ")
+	// 移除首尾空格
+	req.EdbName = strings.TrimSpace(req.EdbName)
 	if req.EdbName == "" {
 		br.Msg = "指标名称不能为空"
 		return
@@ -273,7 +274,9 @@ func (this *PythonController) Edit() {
 		br.ErrMsg = "参数解析失败，Err:" + err.Error()
 		return
 	}
-	req.EdbName = strings.Trim(req.EdbName, " ")
+
+	// 移除首尾空格
+	req.EdbName = strings.TrimSpace(req.EdbName)
 
 	if req.EdbInfoId <= 0 {
 		br.Msg = "指标id不能为空"
@@ -363,8 +366,8 @@ func (this *PythonController) Edit() {
 			return
 		}
 	}
-	edbInfo.EdbName = utils.TrimStr(req.EdbName)
-	edbInfo.EdbNameSource = utils.TrimStr(req.EdbName)
+	edbInfo.EdbName = req.EdbName
+	edbInfo.EdbNameSource = req.EdbName
 	edbInfo.Unit = req.Unit
 	edbInfo.ClassifyId = req.ClassifyId
 	edbInfo.Frequency = req.Frequency

logic/base_edb_info.go

@@ -31,6 +31,9 @@ func RefreshBaseEdbInfo(edbInfo *models.EdbInfo, startDate string) (isHandling b
 		err = models.RefreshEdbDataFromSmm(edbInfo.EdbInfoId, edbInfo.EdbCode, startDate)
 	case utils.DATA_SOURCE_BLOOMBERG:
 		err = models.RefreshEdbDataFromBloomberg(edbInfo.EdbInfoId, edbInfo.EdbCode, startDate)
+	case utils.DATA_SOURCE_CCF:
+		ccfOb := new(models.BaseFromCCF)
+		err = ccfOb.Refresh(edbInfo.EdbInfoId, edbInfo.EdbCode, startDate)
 	default:
 		return
 	}

models/base_from_bloomberg.go

@@ -356,4 +356,5 @@ type BaseFromBloombergApiIndexData struct {
 type PCSGImportHistoryDataReq struct {
 	IndexCode string                `description:"指标编码"`
 	DataMap   map[time.Time]float64 `description:"数据日期/值"`
+	IsVCode   bool                  `description:"是否指标编码中间加V"`
 }

models/base_from_ccf.go

@@ -0,0 +1,426 @@
+package models
+
+import (
+	"eta/eta_index_lib/utils"
+	"fmt"
+	"github.com/beego/beego/v2/client/orm"
+	"strconv"
+	"strings"
+	"time"
+)
+
+// BaseFromCCF CCF化纤信息
+type BaseFromCCF struct{}
+
+type BaseFromCCFData struct {
+	BaseFromCcfDataId  int `orm:"column(base_from_ccf_data_id);pk"`
+	BaseFromCcfIndexId int
+	IndexCode          string
+	DataTime           string
+	Value              string
+	CreateTime         time.Time
+	ModifyTime         time.Time
+	DataTimestamp      int64
+}
+
+func (m *BaseFromCCFData) TableName() string {
+	return "base_from_ccf_data"
+}
+
+func GetBaseFromCCFDataByCondition(condition string, pars []interface{}) (list []*BaseFromCCFData, err error) {
+	o := orm.NewOrm()
+	sql := `SELECT * FROM base_from_ccf_data WHERE 1=1 `
+	if condition != "" {
+		sql += condition
+	}
+	_, err = o.Raw(sql, pars).QueryRows(&list)
+	return
+}
+
+// Add 添加
+func (obj BaseFromCCF) Add(edbCode string) (err error) {
+	o := orm.NewOrm()
+
+	var condition string
+	var pars []interface{}
+	if edbCode != "" {
+		condition += " AND index_code = ? "
+		pars = append(pars, edbCode)
+	}
+	ccfBaseDataAll, err := GetBaseFromCCFDataByCondition(condition, pars)
+	if err != nil && err.Error() != utils.ErrNoRow() {
+		return
+	}
+	var isAdd bool
+	addSql := ` INSERT INTO edb_data_ccf(edb_info_id,edb_code,data_time,value,create_time,modify_time,data_timestamp) values `
+	existMap := make(map[string]string)
+	for _, sv := range ccfBaseDataAll {
+		eDate := sv.DataTime
+		dataTime, err := time.Parse(utils.FormatDate, eDate)
+		if err != nil {
+			return err
+		}
+		timestamp := dataTime.UnixNano() / 1e6
+		timeStr := fmt.Sprintf("%d", timestamp)
+		if _, ok := existMap[eDate]; !ok {
+			addSql += GetAddSql("0", edbCode, eDate, timeStr, sv.Value)
+			isAdd = true
+		}
+		existMap[eDate] = sv.Value
+	}
+	if isAdd {
+		addSql = strings.TrimRight(addSql, ",")
+		utils.FileLog.Info("addSql:" + addSql)
+		_, err = o.Raw(addSql).Exec()
+		if err != nil {
+			return err
+		}
+	}
+	return
+}
+
+// Refresh 刷新CCF化纤指标数据
+func (obj BaseFromCCF) Refresh(edbInfoId int, edbCode, startDate string) (err error) {
+	source := obj.GetSource()
+	o := orm.NewOrm()
+	if err != nil {
+		return
+	}
+	edbInfoIdStr := strconv.Itoa(edbInfoId)
+	//计算数据
+	var condition string
+	var pars []interface{}
+
+	if edbCode != "" {
+		condition += " AND index_code=? "
+		pars = append(pars, edbCode)
+	}
+
+	if startDate != "" {
+		condition += " AND data_time>=? "
+		pars = append(pars, startDate)
+	}
+
+	ccfDataList, err := GetBaseFromCCFDataByCondition(condition, pars)
+	if err != nil {
+		return
+	}
+
+	// 真实数据的最大日期  ， 插入规则配置的日期
+	var realDataMaxDate, edbDataInsertConfigDate time.Time
+	var edbDataInsertConfig *EdbDataInsertConfig
+	var isFindConfigDateRealData bool //是否找到配置日期的实际数据的值
+	{
+		edbDataInsertConfig, err = GetEdbDataInsertConfigByEdbId(edbInfoId)
+		if err != nil && err.Error() != utils.ErrNoRow() {
+			return
+		}
+		if edbDataInsertConfig != nil {
+			edbDataInsertConfigDate = edbDataInsertConfig.Date
+		}
+	}
+
+	var existCondition string
+	var existPars []interface{}
+
+	existCondition += " AND edb_info_id=? "
+	existPars = append(existPars, edbInfoId)
+	if startDate != "" {
+		existCondition += " AND data_time>=? "
+		existPars = append(existPars, startDate)
+	}
+
+	existList, err := GetEdbDataByCondition(source, 0, existCondition, existPars)
+	if err != nil {
+		return err
+	}
+	existMap := make(map[string]*EdbInfoSearchData)
+	for _, v := range existList {
+		existMap[v.DataTime] = v
+	}
+	addSql := ` INSERT INTO edb_data_ccf(edb_info_id,edb_code,data_time,value,create_time,modify_time,data_timestamp) values `
+	var isAdd bool
+	for _, v := range ccfDataList {
+		item := v
+		eDate := item.DataTime
+		dataTime, err := time.ParseInLocation(utils.FormatDate, eDate, time.Local)
+		if err != nil {
+			return err
+		}
+		if findItem, ok := existMap[v.DataTime]; !ok {
+			sValue := item.Value
+
+			timestamp := dataTime.UnixNano() / 1e6
+			timeStr := fmt.Sprintf("%d", timestamp)
+
+			addSql += GetAddSql(edbInfoIdStr, edbCode, eDate, timeStr, sValue)
+			isAdd = true
+		} else {
+			if findItem != nil && utils.SubFloatToString(findItem.Value, 30) != item.Value {
+				err = ModifyEdbDataById(source, 0, findItem.EdbDataId, item.Value)
+				if err != nil {
+					return err
+				}
+			}
+		}
+
+		// 下面代码主要目的是处理掉手动插入的数据判断
+		{
+			if realDataMaxDate.IsZero() || dataTime.After(realDataMaxDate) {
+				realDataMaxDate = dataTime
+			}
+			if edbDataInsertConfigDate.IsZero() || dataTime.Equal(edbDataInsertConfigDate) {
+				isFindConfigDateRealData = true
+			}
+		}
+	}
+
+	// 处理手工数据补充的配置
+	HandleConfigInsertEdbData(realDataMaxDate, edbDataInsertConfig, edbInfoId, source, 0, existMap, isFindConfigDateRealData)
+
+	if isAdd {
+		addSql = strings.TrimRight(addSql, ",")
+		_, err = o.Raw(addSql).Exec()
+		if err != nil {
+			fmt.Println("RefreshEdbDataFromBaiinfo add Err", err.Error())
+			return
+		}
+	}
+	return
+}
+
+// GetSource 获取来源编码id
+func (obj BaseFromCCF) GetSource() int {
+	return utils.DATA_SOURCE_CCF
+}
+
+// GetSourceName 获取来源名称
+func (obj BaseFromCCF) GetSourceName() string {
+	return utils.DATA_SOURCE_NAME_CCF
+}
+
+type BaseFromCCFIndex struct {
+	BaseFromCcfIndexId int64 `orm:"column(base_from_ccf_index_id);pk"`
+	IndexCode          string
+	IndexName          string
+	Frequency          string
+	Unit               string
+	ClassifyId         int
+	StartDate          string
+	EndDate            string
+	Sort               int
+	TerminalCode       string
+	CreateTime         time.Time
+	ModifyTime         time.Time
+}
+
+func (m *BaseFromCCFIndex) TableName() string {
+	return "base_from_ccf_index"
+}
+
+func (m *BaseFromCCFIndex) Add() (lastId int64, err error) {
+	o := orm.NewOrm()
+	lastId, err = o.Insert(m)
+	return
+}
+
+func (m *BaseFromCCFIndex) Update(updateCols []string) (err error) {
+	o := orm.NewOrm()
+	_, err = o.Update(m, updateCols...)
+
+	return
+}
+
+func (m *BaseFromCCFIndex) ModifyIndexMaxAndMinDate(indexCode string, item *EdbInfoMaxAndMinInfo) (err error) {
+	o := orm.NewOrm()
+	sql := ` UPDATE base_from_ccf_index SET start_date=?,end_date=?,modify_time=NOW() WHERE index_code=? `
+	_, err = o.Raw(sql, item.MinDate, item.MaxDate, indexCode).Exec()
+	return
+}
+
+func (m *BaseFromCCFIndex) GetByIndexCode(indexCode string) (item *BaseFromCCFIndex, err error) {
+	o := orm.NewOrm()
+	sql := ` SELECT * FROM base_from_ccf_index WHERE index_code=? `
+	err = o.Raw(sql, indexCode).QueryRow(&item)
+	return
+}
+
+func (m *BaseFromCCFIndex) GetItemsByCondition(condition string, pars []interface{}, fieldArr []string, orderRule string) (items []*BaseFromCCFIndex, err error) {
+	o := orm.NewOrm()
+	fields := strings.Join(fieldArr, ",")
+	if len(fieldArr) == 0 {
+		fields = `*`
+	}
+	order := `ORDER BY create_time DESC`
+	if orderRule != "" {
+		order = ` ORDER BY ` + orderRule
+	}
+	sql := fmt.Sprintf(`SELECT %s FROM base_from_ccf_index WHERE 1=1 %s %s`, fields, condition, order)
+	_, err = o.Raw(sql, pars).QueryRows(&items)
+	return
+}
+
+type BaseFromCCFIndexList struct {
+	CcfIndexId   int64 `orm:"column(ccf_index_id);pk"`
+	IndexCode    string
+	IndexName    string
+	Frequency    string
+	Unit         string
+	ClassifyId   int
+	StartDate    string
+	EndDate      string
+	TerminalCode string
+	CreateTime   string
+	ModifyTime   string
+}
+
+func (m *BaseFromCCFData) GetByIndexCode(indexCode string) (list []*BaseFromCCFData, err error) {
+	o := orm.NewOrm()
+	sql := ` SELECT * FROM base_from_ccf_data WHERE index_code=? `
+	_, err = o.Raw(sql, indexCode).QueryRows(&list)
+	return
+}
+
+func (m *BaseFromCCFData) AddMulti(item []*BaseFromCCFData) (err error) {
+	o := orm.NewOrm()
+	_, err = o.InsertMulti(100, item)
+	return
+}
+
+// HandleCCFEdbData CCF化纤的指标数据
+type HandleCCFEdbData struct {
+	IndexName    string            `description:"指标名称"`
+	IndexCode    string            `description:"指标编码"`
+	ClassifyId   int               `description:"分类ID"`
+	Unit         string            `description:"单位"`
+	Sort         int               `description:"排序"`
+	Frequency    string            `description:"频度"`
+	TerminalCode string            `description:"终端编码"`
+	DateData     map[string]string `description:"日期数据"`
+}
+
+type HandleCCFEdbDataReq struct {
+	List         []*HandleCCFEdbData
+	TerminalCode string `description:"编码"`
+}
+
+type HandleCCFTableData struct {
+	ClassifyId   int       `description:"分类ID"`
+	FromPage     string    `description:"表格来源"`
+	TableDate    time.Time `description:"表格日期"`
+	TableContent string    `description:"表格HTML"`
+}
+
+type HandleCCFStockTableReq struct {
+	Table        *HandleCCFTableData
+	TerminalCode string `description:"编码"`
+}
+
+func (m *BaseFromCCFData) GetMaxAndMinDateByIndexCode(indexCode string) (item *EdbInfoMaxAndMinInfo, err error) {
+	o := orm.NewOrm()
+	sql := ` SELECT MIN(data_time) AS min_date,MAX(data_time) AS max_date,MIN(value) AS min_value,MAX(value) AS max_value FROM base_from_ccf_data WHERE index_code=? `
+	err = o.Raw(sql, indexCode).QueryRow(&item)
+	return
+}
+
+// BaseFromCCFClassify CCF化纤数据分类表
+type BaseFromCCFClassify struct {
+	ClassifyId      int       `orm:"column(classify_id);pk"`
+	ClassifyName    string    `description:"分类名称"`
+	ParentId        int       `description:"父级id"`
+	SysUserId       int       `description:"创建人id"`
+	SysUserRealName string    `description:"创建人姓名"`
+	Level           int       `description:"层级"`
+	Sort            int       `description:"排序字段，越小越靠前，默认值：10"`
+	ModifyTime      time.Time `description:"修改时间"`
+	CreateTime      time.Time `description:"创建时间"`
+}
+
+func (m *BaseFromCCFClassify) Add() (lastId int64, err error) {
+	o := orm.NewOrm()
+	lastId, err = o.Insert(m)
+	return
+}
+
+func (m *BaseFromCCFClassify) Update(updateCols []string) (err error) {
+	o := orm.NewOrm()
+	_, err = o.Update(m, updateCols...)
+
+	return
+}
+
+func (m *BaseFromCCFClassify) GetByClassifyName(classifyName string) (item *BaseFromCCFClassify, err error) {
+	o := orm.NewOrm()
+	sql := ` SELECT * FROM base_from_ccf_classify WHERE classify_name=? `
+	err = o.Raw(sql, classifyName).QueryRow(&item)
+	return
+}
+
+func (m *BaseFromCCFClassify) GetItemsByCondition(condition string, pars []interface{}, fieldArr []string, orderRule string) (items []*BaseFromCCFClassify, err error) {
+	o := orm.NewOrm()
+	fields := strings.Join(fieldArr, ",")
+	if len(fieldArr) == 0 {
+		fields = `*`
+	}
+	order := `ORDER BY create_time DESC`
+	if orderRule != "" {
+		order = ` ORDER BY ` + orderRule
+	}
+	sql := fmt.Sprintf(`SELECT %s FROM base_from_ccf_classify WHERE 1=1 %s %s`, fields, condition, order)
+	_, err = o.Raw(sql, pars).QueryRows(&items)
+	return
+}
+
+// BaseFromCCFClassifyItem CCF化纤数据分类信息
+type BaseFromCCFClassifyItem struct {
+	ClassifyId   int                        `description:"分类ID"`
+	ClassifyName string                     `description:"分类名称"`
+	ParentId     int                        `description:"父级id"`
+	Level        int                        `description:"层级"`
+	Sort         int                        `description:"排序字段"`
+	CreateTime   string                     `description:"创建时间"`
+	ModifyTime   string                     `description:"修改时间"`
+	Child        []*BaseFromCCFClassifyItem `description:"子分类"`
+}
+
+func (m *BaseFromCCFClassify) Format2Item(origin *BaseFromCCFClassify) (item *BaseFromCCFClassifyItem) {
+	if origin == nil {
+		return
+	}
+	item = new(BaseFromCCFClassifyItem)
+	item.ClassifyId = origin.ClassifyId
+	item.ClassifyName = origin.ClassifyName
+	item.ParentId = origin.ParentId
+	item.Level = origin.Level
+	item.Sort = origin.Sort
+	item.CreateTime = utils.TimeTransferString(utils.FormatDateTime, origin.CreateTime)
+	item.ModifyTime = utils.TimeTransferString(utils.FormatDateTime, origin.ModifyTime)
+	return
+}
+
+// MultiUpdateBaseFromCCFDataValue 批量更新CCF化纤指标数据
+func MultiUpdateBaseFromCCFDataValue(items []*BaseFromCCFData) (err error) {
+	if len(items) == 0 {
+		return
+	}
+
+	o := orm.NewOrm()
+	sql := `UPDATE base_from_ccf_data SET value = ?, modify_time = NOW() WHERE index_code = ? AND data_time = ? LIMIT 1`
+	p, err := o.Raw(sql).Prepare()
+	if err != nil {
+		return
+	}
+	defer func() {
+		_ = p.Close()
+	}()
+	for _, v := range items {
+		if v.IndexCode == "" || v.DataTime == "" {
+			continue
+		}
+		_, err = p.Exec(v.Value, v.IndexCode, v.DataTime)
+		if err != nil {
+			return
+		}
+	}
+	return
+}

models/base_from_mysteel_chemical.go

@@ -245,9 +245,8 @@ func (m *BaseFromMysteelChemicalIndex) GetIndexItem(indexCode string) (item *Bas
 
 func (m *BaseFromMysteelChemicalIndex) GetIndexCreate(terminalCode string) (items []*BaseFromMysteelChemicalIndex, err error) {
 	o := orm.NewOrm()
-	endTime := time.Now().Add(-2 * time.Minute).Format(utils.FormatDateTime)
-	sql := `SELECT * FROM base_from_mysteel_chemical_index WHERE index_name = '' AND create_time <= ? AND terminal_code = ? `
-	_, err = o.Raw(sql, endTime, terminalCode).QueryRows(&items)
+	sql := `SELECT * FROM base_from_mysteel_chemical_index WHERE index_name = '' AND terminal_code = ? `
+	_, err = o.Raw(sql, terminalCode).QueryRows(&items)
 	return
 }
 

models/base_from_wind_wsd.go

@@ -1,6 +1,7 @@
 package models
 
 import (
+	"errors"
 	"eta/eta_index_lib/utils"
 	"fmt"
 	"github.com/beego/beego/v2/client/orm"
@@ -31,6 +32,12 @@ func AddEdbDataFromWindWsd(stockCode string, item map[string]map[string]interfac
 				return err
 			}
 
+			if vk == "OUTMESSAGE" {
+				utils.FileLog.Info("OUTMESSAGE:" + vv.(string))
+				err = errors.New("OUTMESSAGE:" + vv.(string))
+				return err
+			}
+			
 			vk = strings.ToLower(vk)
 			indexCode = windWsd + stockCode + vk
 

models/ccf_stock_excel.go

@@ -0,0 +1,147 @@
+package models
+
+import (
+	"eta/eta_index_lib/utils"
+	"fmt"
+	"github.com/beego/beego/v2/client/orm"
+	"strings"
+	"time"
+)
+
+// CCFStockExcel CCF化纤装置表格
+type CCFStockExcel struct {
+	CcfStockExcelId int       `orm:"column(ccf_stock_excel_id);pk"`
+	ClassifyId      int       `description:"分类ID"`
+	ExcelDate       time.Time `description:"表格日期"`
+	ExcelContent    string    `description:"表格HTML"`
+	FromPage        string    `description:"表格来源"`
+	CreateTime      time.Time `description:"创建时间"`
+	ModifyTime      time.Time `description:"修改时间"`
+}
+
+func (m *CCFStockExcel) TableName() string {
+	return "ccf_stock_excel"
+}
+
+type CCFStockExcelCols struct {
+	CcfStockExcelId string
+	ClassifyId      string
+	ExcelDate       string
+	ExcelContent    string
+	FromPage        string
+	CreateTime      string
+	ModifyTime      string
+}
+
+func (m *CCFStockExcel) Cols() CCFStockExcelCols {
+	return CCFStockExcelCols{
+		CcfStockExcelId: "ccf_stock_excel_id",
+		ClassifyId:      "classify_id",
+		ExcelDate:       "excel_date",
+		ExcelContent:    "excel_content",
+		FromPage:        "from_page",
+		CreateTime:      "create_time",
+		ModifyTime:      "modify_time",
+	}
+}
+
+func (m *CCFStockExcel) PrimaryId() string {
+	return m.Cols().CcfStockExcelId
+}
+
+func (m *CCFStockExcel) Create() (err error) {
+	o := orm.NewOrm()
+	id, err := o.Insert(m)
+	if err != nil {
+		return
+	}
+	m.CcfStockExcelId = int(id)
+	return
+}
+
+func (m *CCFStockExcel) CreateMulti(items []*CCFStockExcel) (err error) {
+	if len(items) == 0 {
+		return
+	}
+	o := orm.NewOrm()
+	_, err = o.InsertMulti(len(items), items)
+	return
+}
+
+func (m *CCFStockExcel) Update(cols []string) (err error) {
+	o := orm.NewOrm()
+	_, err = o.Update(m, cols...)
+	return
+}
+
+func (m *CCFStockExcel) Del() (err error) {
+	o := orm.NewOrm()
+	sql := fmt.Sprintf(`DELETE FROM %s WHERE %s = ? LIMIT 1`, m.TableName(), m.PrimaryId())
+	_, err = o.Raw(sql, m.CcfStockExcelId).Exec()
+	return
+}
+
+func (m *CCFStockExcel) MultiDel(menuIds []int) (err error) {
+	if len(menuIds) == 0 {
+		return
+	}
+	o := orm.NewOrm()
+	sql := fmt.Sprintf(`DELETE FROM %s WHERE %s IN (%s)`, m.TableName(), m.PrimaryId(), utils.GetOrmInReplace(len(menuIds)))
+	_, err = o.Raw(sql, menuIds).Exec()
+	return
+}
+
+func (m *CCFStockExcel) GetItemById(id int) (item *CCFStockExcel, err error) {
+	o := orm.NewOrm()
+	sql := fmt.Sprintf(`SELECT * FROM %s WHERE %s = ? LIMIT 1`, m.TableName(), m.PrimaryId())
+	err = o.Raw(sql, id).QueryRow(&item)
+	return
+}
+
+func (m *CCFStockExcel) GetItemByCondition(condition string, pars []interface{}, orderRule string) (item *CCFStockExcel, err error) {
+	o := orm.NewOrm()
+	order := ``
+	if orderRule != "" {
+		order = ` ORDER BY ` + orderRule
+	}
+	sql := fmt.Sprintf(`SELECT * FROM %s WHERE 1=1 %s %s LIMIT 1`, m.TableName(), condition, order)
+	err = o.Raw(sql, pars).QueryRow(&item)
+	return
+}
+
+func (m *CCFStockExcel) GetCountByCondition(condition string, pars []interface{}) (count int, err error) {
+	o := orm.NewOrm()
+	sql := fmt.Sprintf(`SELECT COUNT(1) FROM %s WHERE 1=1 %s`, m.TableName(), condition)
+	err = o.Raw(sql, pars).QueryRow(&count)
+	return
+}
+
+func (m *CCFStockExcel) GetItemsByCondition(condition string, pars []interface{}, fieldArr []string, orderRule string) (items []*CCFStockExcel, err error) {
+	o := orm.NewOrm()
+	fields := strings.Join(fieldArr, ",")
+	if len(fieldArr) == 0 {
+		fields = `*`
+	}
+	order := fmt.Sprintf(`ORDER BY %s DESC`, m.Cols().CreateTime)
+	if orderRule != "" {
+		order = ` ORDER BY ` + orderRule
+	}
+	sql := fmt.Sprintf(`SELECT %s FROM %s WHERE 1=1 %s %s`, fields, m.TableName(), condition, order)
+	_, err = o.Raw(sql, pars).QueryRows(&items)
+	return
+}
+
+func (m *CCFStockExcel) GetPageItemsByCondition(condition string, pars []interface{}, fieldArr []string, orderRule string, startSize, pageSize int) (items []*CCFStockExcel, err error) {
+	o := orm.NewOrm()
+	fields := strings.Join(fieldArr, ",")
+	if len(fieldArr) == 0 {
+		fields = `*`
+	}
+	order := fmt.Sprintf(`ORDER BY %s DESC`, m.Cols().CreateTime)
+	if orderRule != "" {
+		order = ` ORDER BY ` + orderRule
+	}
+	sql := fmt.Sprintf(`SELECT %s FROM %s WHERE 1=1 %s %s LIMIT ?,?`, fields, m.TableName(), condition, order)
+	_, err = o.Raw(sql, pars, startSize, pageSize).QueryRows(&items)
+	return
+}

models/db.go

@@ -137,6 +137,9 @@ func initBaseIndex() {
 		new(BaseFromSci99Data),
 		new(BaseFromSci99Index),
 		new(BaseFromSci99Classify),
+		new(BaseFromCCFIndex),
+		new(BaseFromCCFData),
+		new(CCFStockExcel),
 	)
 }
 

models/edb_data_calculate_percentile.go

@@ -244,6 +244,7 @@ func (obj Percentile) GetEdbType() int {
 type PercentileConfig struct {
 	CalculateValue int    `description:"时间长度期数"`
 	CalculateUnit  string `description:"时间长度频度"`
+	PercentType    int    `description:"百分位：0-数据区间(兼容历史数据); 1-数据个数；"`
 }
 
 // refresh 刷新
@@ -259,7 +260,7 @@ func (obj Percentile) refresh(to orm.TxOrmer, edbInfo, fromEdbInfo *EdbInfo, edb
 	}
 
 	// 获取百分位的指标数据
-	fromDataList, err, errMsg := obj.getPercentileData(fromEdbInfo, percentileConfig.CalculateValue, percentileConfig.CalculateUnit)
+	fromDataList, err, errMsg := obj.getPercentileData(fromEdbInfo, percentileConfig.CalculateValue, percentileConfig.CalculateUnit, percentileConfig.PercentType)
 	if err != nil {
 		return
 	}
@@ -374,7 +375,7 @@ func (obj Percentile) calculate(edbInfoId int, date, edbInfoIdStr, edbCode, data
 }
 
 // GetPercentileData 获取百分位图表的指标数据
-func (obj Percentile) getPercentileData(fromEdbInfo *EdbInfo, calculateValue int, calculateUnit string) (newDataList []EdbInfoSearchData, err error, errMsg string) {
+func (obj Percentile) getPercentileData(fromEdbInfo *EdbInfo, calculateValue int, calculateUnit string, percentType int) (newDataList []EdbInfoSearchData, err error, errMsg string) {
 	// 获取时间基准指标在时间区间内的值
 	dataList := make([]*EdbInfoSearchData, 0)
 	switch fromEdbInfo.EdbInfoType {
@@ -408,33 +409,79 @@ func (obj Percentile) getPercentileData(fromEdbInfo *EdbInfo, calculateValue int
 	}
 
 	//百分位：对所选指标滚动地取对应时间长度的数据值，取最大值Max，最小值Min，计算Max-Min，百分位=(现值-Min)/(Max-Min)，Max=Min时不予计算。
-	for i, tmpData := range dataList {
-		currDateTime, _ := time.ParseInLocation(utils.FormatDate, tmpData.DataTime, time.Local)
-		maxVal := tmpData.Value
-		minVal := tmpData.Value
-		for k := 0; k < calculateDay; k++ {
-			preVal, ok2 := dataMap[currDateTime.AddDate(0, 0, -k)]
-			if ok2 {
-				if preVal > maxVal {
-					maxVal = preVal
-				}
-				if preVal < minVal {
-					minVal = preVal
+	if percentType == utils.PercentCalculateTypeRange {
+		for i, tmpData := range dataList {
+			currDateTime, _ := time.ParseInLocation(utils.FormatDate, tmpData.DataTime, time.Local)
+			maxVal := tmpData.Value
+			minVal := tmpData.Value
+			for k := 0; k < calculateDay; k++ {
+				preVal, ok2 := dataMap[currDateTime.AddDate(0, 0, -k)]
+				if ok2 {
+					if preVal > maxVal {
+						maxVal = preVal
+					}
+					if preVal < minVal {
+						minVal = preVal
+					}
 				}
 			}
+
+			if maxVal == minVal {
+				continue
+			}
+			tmpV := (tmpData.Value - minVal) / (maxVal - minVal) * 100
+			tmpV, _ = decimal.NewFromFloat(tmpV).Round(4).Float64()
+			//百分位=(现值-Min)/(Max-Min)
+			newDataList = append(newDataList, EdbInfoSearchData{
+				EdbDataId: i,
+				DataTime:  dataList[i].DataTime,
+				Value:     tmpV,
+			})
 		}
+	}
+	// 百分位数据个数算法
+	// 数据区间第一个和最后一个数据点的时间和数据分别为(T1,S1)(T2,S2); N=T1到T2指标数据个数, n=小于等于S2的数据个数
+	// 个数百分位=(n-1)/(N-1)
+	maxDay := len(dataList) // 往前找数据的边界
+	if percentType == utils.PercentCalculateTypeNum {
+		for i, d := range dataList {
+			// T2为当前日期
+			s2 := decimal.NewFromFloat(d.Value)
+			t2, _ := time.ParseInLocation(utils.FormatDate, d.DataTime, time.Local)
+
+			// 计算N和n
+			var bigN, tinyN int
+			for k := 0; k < maxDay; k++ {
+				// 往前找(时间长度)个有数据的, N理论上只有最前面几个日期<calculateDay, 后面的N=calculateDay
+				if bigN >= calculateDay {
+					break
+				}
+				preVal, preOk := dataMap[t2.AddDate(0, 0, -k)]
+				if !preOk {
+					continue
+				}
+				bigN += 1
+				if decimal.NewFromFloat(preVal).LessThanOrEqual(s2) {
+					tinyN += 1
+				}
+			}
 
-		if maxVal == minVal {
-			continue
+			// N<=1时说明计算无效
+			if bigN <= 1 {
+				continue
+			}
+			numerator := decimal.NewFromInt(int64(tinyN - 1))
+			denominator := decimal.NewFromInt(int64(bigN - 1))
+			// 因为是百分位所以这里是要*100, 跟之前的算法保持同步
+			percentVal, _ := numerator.Div(denominator).Mul(decimal.NewFromFloat(100)).Round(4).Float64()
+
+			// 写进数组并判断指标最大最小值
+			newDataList = append(newDataList, EdbInfoSearchData{
+				EdbDataId: i,
+				DataTime:  dataList[i].DataTime,
+				Value:     percentVal,
+			})
 		}
-		tmpV := (tmpData.Value - minVal) / (maxVal - minVal) * 100
-		tmpV, _ = decimal.NewFromFloat(tmpV).Round(4).Float64()
-		//百分位=(现值-Min)/(Max-Min)
-		newDataList = append(newDataList, EdbInfoSearchData{
-			EdbDataId: i,
-			DataTime:  dataList[i].DataTime,
-			Value:     tmpV,
-		})
 	}
 
 	return

routers/commentsRouter.go

@@ -99,26 +99,8 @@ func init() {
 
     beego.GlobalControllerRouter["eta/eta_index_lib/controllers:BloombergController"] = append(beego.GlobalControllerRouter["eta/eta_index_lib/controllers:BloombergController"],
         beego.ControllerComments{
-            Method: "PCSGRefreshDaily",
-            Router: `/pcsg/refresh_daily`,
-            AllowHTTPMethods: []string{"post"},
-            MethodParams: param.Make(),
-            Filters: nil,
-            Params: nil})
-
-    beego.GlobalControllerRouter["eta/eta_index_lib/controllers:BloombergController"] = append(beego.GlobalControllerRouter["eta/eta_index_lib/controllers:BloombergController"],
-        beego.ControllerComments{
-            Method: "PCSGRefreshMonthly",
-            Router: `/pcsg/refresh_monthly`,
-            AllowHTTPMethods: []string{"post"},
-            MethodParams: param.Make(),
-            Filters: nil,
-            Params: nil})
-
-    beego.GlobalControllerRouter["eta/eta_index_lib/controllers:BloombergController"] = append(beego.GlobalControllerRouter["eta/eta_index_lib/controllers:BloombergController"],
-        beego.ControllerComments{
-            Method: "PCSGRefreshWeekly",
-            Router: `/pcsg/refresh_weekly`,
+            Method: "PCSGRefreshTask",
+            Router: `/pcsg/refresh_task`,
             AllowHTTPMethods: []string{"post"},
             MethodParams: param.Make(),
             Filters: nil,
@@ -187,6 +169,42 @@ func init() {
             Filters: nil,
             Params: nil})
 
+    beego.GlobalControllerRouter["eta/eta_index_lib/controllers:CCFController"] = append(beego.GlobalControllerRouter["eta/eta_index_lib/controllers:CCFController"],
+        beego.ControllerComments{
+            Method: "Add",
+            Router: `/add`,
+            AllowHTTPMethods: []string{"post"},
+            MethodParams: param.Make(),
+            Filters: nil,
+            Params: nil})
+
+    beego.GlobalControllerRouter["eta/eta_index_lib/controllers:CCFController"] = append(beego.GlobalControllerRouter["eta/eta_index_lib/controllers:CCFController"],
+        beego.ControllerComments{
+            Method: "HandleEdbData",
+            Router: `/handle/edb_data`,
+            AllowHTTPMethods: []string{"post"},
+            MethodParams: param.Make(),
+            Filters: nil,
+            Params: nil})
+
+    beego.GlobalControllerRouter["eta/eta_index_lib/controllers:CCFController"] = append(beego.GlobalControllerRouter["eta/eta_index_lib/controllers:CCFController"],
+        beego.ControllerComments{
+            Method: "HandleTableData",
+            Router: `/handle/table_data`,
+            AllowHTTPMethods: []string{"post"},
+            MethodParams: param.Make(),
+            Filters: nil,
+            Params: nil})
+
+    beego.GlobalControllerRouter["eta/eta_index_lib/controllers:CCFController"] = append(beego.GlobalControllerRouter["eta/eta_index_lib/controllers:CCFController"],
+        beego.ControllerComments{
+            Method: "Refresh",
+            Router: `/refresh`,
+            AllowHTTPMethods: []string{"post"},
+            MethodParams: param.Make(),
+            Filters: nil,
+            Params: nil})
+
     beego.GlobalControllerRouter["eta/eta_index_lib/controllers:CalculateController"] = append(beego.GlobalControllerRouter["eta/eta_index_lib/controllers:CalculateController"],
         beego.ControllerComments{
             Method: "Add",

routers/router.go

@@ -265,6 +265,11 @@ func init() {
 				&controllers.Sci99Crawler{},
 			),
 		),
+		beego.NSNamespace("/ccf",
+			beego.NSInclude(
+				&controllers.CCFController{},
+			),
+		),
 		beego.NSNamespace("/shanghai_smm",
 			beego.NSInclude(
 				&controllers.ShanghaiSmmController{},

services/base_from_ccf.go

@@ -0,0 +1,270 @@
+package services
+
+import (
+	"eta/eta_index_lib/logic"
+	"eta/eta_index_lib/models"
+	"eta/eta_index_lib/services/alarm_msg"
+	"eta/eta_index_lib/utils"
+	"fmt"
+	"html"
+	"strings"
+	"time"
+)
+
+// HandleCCFIndex 处理CCF的excel数据
+func HandleCCFIndex(req *models.HandleCCFEdbDataReq) (err error) {
+	errMsgList := make([]string, 0)
+	defer func() {
+		if len(errMsgList) > 0 {
+			msg := fmt.Sprint("数据源-CCF数据处理失败, err:", strings.Join(errMsgList, "\n"))
+			utils.FileLog.Info(msg)
+			go alarm_msg.SendAlarmMsg(msg, 3)
+		}
+	}()
+	for _, v := range req.List {
+		if v.IndexName == "" || v.IndexCode == "" {
+			errMsgList = append(errMsgList, fmt.Sprintf("新增指标异常，指标编码%s或者指标ID%s为空：", v.IndexCode, v.IndexName))
+			continue
+		}
+		err = handleCCFIndex(v, req.TerminalCode)
+		if err != nil {
+			errMsgList = append(errMsgList, fmt.Sprintf("新增指标异常，指标编码：%s, Err: %s", v.IndexCode, err))
+			return
+		}
+	}
+	return
+}
+
+func handleCCFIndex(req *models.HandleCCFEdbData, terminalCode string) (err error) {
+	indexName := req.IndexName
+	indexCode := req.IndexCode
+	excelDataMap := req.DateData
+	errMsgList := make([]string, 0)
+	defer func() {
+		if len(errMsgList) > 0 {
+			msg := fmt.Sprint("数据源-CCF数据处理失败,err:", strings.Join(errMsgList, "\n"))
+			utils.FileLog.Info(msg)
+			go alarm_msg.SendAlarmMsg(msg, 3)
+		}
+	}()
+	indexObj := new(models.BaseFromCCFIndex)
+	dataObj := new(models.BaseFromCCFData)
+	//classifyObj := new(models.BaseFromCCFClassify)
+
+	var indexId int64
+
+	addDataList := make([]*models.BaseFromCCFData, 0)
+	updateDataList := make([]*models.BaseFromCCFData, 0)
+
+	exitDataMap := make(map[string]*models.BaseFromCCFData)
+
+	// 修改指标信息
+	if indexName == "" {
+		utils.FileLog.Info("未刷新到指标数据:indexName:" + indexName)
+		return
+	}
+
+	//判断指标是否存在
+	var isAdd int
+	item, err := indexObj.GetByIndexCode(indexCode)
+	if err != nil {
+		if err.Error() == utils.ErrNoRow() {
+			isAdd = 1
+			err = nil
+		} else {
+			isAdd = -1
+			err = fmt.Errorf("查询数据源指标库失败 GetByIndexCode Err：%s", err)
+			return
+		}
+	}
+	if item != nil && item.BaseFromCcfIndexId > 0 {
+		fmt.Println("item:", item)
+		isAdd = 2
+	} else {
+		isAdd = 1
+	}
+
+	if isAdd == 1 {
+		indexObj.IndexCode = indexCode
+		indexObj.IndexName = indexName
+		indexObj.Frequency = req.Frequency
+		indexObj.ClassifyId = req.ClassifyId
+		if req.Unit == "" {
+			req.Unit = "无"
+		}
+		indexObj.Unit = req.Unit
+		indexObj.Sort = req.Sort
+		indexObj.ModifyTime = time.Now()
+		indexObj.CreateTime = time.Now()
+		indexObj.TerminalCode = terminalCode
+		indexId, err = indexObj.Add()
+		if err != nil {
+			err = fmt.Errorf("数据源新增CCF指标失败 Err：%s", err)
+			return
+		}
+		indexObj.BaseFromCcfIndexId = indexId
+	} else if isAdd == 2 {
+		if item.TerminalCode == `` && terminalCode != `` {
+			item.TerminalCode = terminalCode
+			err = item.Update([]string{"TerminalCode"})
+			if err != nil {
+				err = fmt.Errorf("数据源更新CCF指标失败 Err：%s", err)
+				return
+			}
+		}
+
+		//获取已存在的所有数据
+		var exitDataList []*models.BaseFromCCFData
+		exitDataList, err = dataObj.GetByIndexCode(indexCode)
+		if err != nil {
+			err = fmt.Errorf("数据源查询CCF指标数据失败 Err：%s", err)
+			return
+		}
+		fmt.Println("exitDataListLen:", len(exitDataList))
+		for _, v := range exitDataList {
+			dateStr := v.DataTime
+			exitDataMap[dateStr] = v
+		}
+		indexId = item.BaseFromCcfIndexId
+	}
+
+	// 遍历excel数据，然后跟现有的数据做校验，不存在则入库
+	for date, value := range excelDataMap {
+		if findData, ok := exitDataMap[date]; !ok {
+			_, err = time.ParseInLocation(utils.FormatDate, date, time.Local)
+			if err != nil {
+				err = fmt.Errorf("%s 转换日期格式失败 Err：%s", date, err)
+				return
+			}
+			if !strings.Contains(value, "#N/A") {
+				var saveDataTime time.Time
+				if strings.Contains(date, "00:00:00") {
+					saveDataTime, err = time.Parse(utils.FormatDateTime, date)
+				} else {
+					saveDataTime, err = time.Parse(utils.FormatDate, date)
+				}
+				if err != nil {
+					err = fmt.Errorf("%s 转换日期格式失败 Err：%s", date, err)
+					continue
+				}
+				timestamp := saveDataTime.UnixNano() / 1e6
+
+				dataItem := new(models.BaseFromCCFData)
+				dataItem.BaseFromCcfIndexId = int(indexId)
+				dataItem.IndexCode = indexCode
+				dataItem.DataTime = date
+				dataItem.Value = value
+				dataItem.CreateTime = time.Now()
+				dataItem.ModifyTime = time.Now()
+				dataItem.DataTimestamp = timestamp
+				addDataList = append(addDataList, dataItem)
+			}
+		} else {
+			if findData != nil && findData.Value != value && !strings.Contains(value, "#N/A") { //修改数据
+				findData.Value = value
+				findData.ModifyTime = time.Now().Local()
+				updateDataList = append(updateDataList, findData)
+			}
+		}
+	}
+
+	if len(addDataList) > 0 {
+		err = dataObj.AddMulti(addDataList)
+		if err != nil {
+			err = fmt.Errorf("批量新增指标失败 Err：%s", err)
+			return
+		}
+
+		var dateItem *models.EdbInfoMaxAndMinInfo
+		dateItem, err = dataObj.GetMaxAndMinDateByIndexCode(indexCode)
+		if err != nil {
+			err = fmt.Errorf("查询指标最新日期失败 Err：%s", err)
+			return
+		}
+
+		go func() {
+			indexObj.ModifyIndexMaxAndMinDate(indexCode, dateItem)
+		}()
+	}
+
+	// 批量更新数据
+	if len(updateDataList) > 0 {
+		e := models.MultiUpdateBaseFromCCFDataValue(updateDataList)
+		if e != nil {
+			err = fmt.Errorf("MultiUpdateBaseFromSmmDataValue err: %s", e.Error())
+			return
+		}
+	}
+
+	// 同步刷新ETA指标库的指标
+	{
+		// 获取指标详情
+		baseObj := new(models.BaseFromCCF)
+		var edbInfo *models.EdbInfo
+		edbInfo, err = models.GetEdbInfoByEdbCode(baseObj.GetSource(), indexCode)
+		if err != nil {
+			if err.Error() != utils.ErrNoRow() {
+				errMsgList = append(errMsgList, fmt.Sprint("刷新ETA指标异常，指标编码：", indexCode, err.Error()))
+				return
+			} else {
+				err = nil
+			}
+		}
+
+		// 已经加入到指标库的话，那么就去更新ETA指标库吧
+		if edbInfo != nil {
+			go logic.RefreshBaseEdbInfo(edbInfo, ``)
+		}
+	}
+	return
+}
+
+// HandleCCFStockTable 处理CCF的表格数据
+func HandleCCFStockTable(req *models.HandleCCFStockTableReq) (err error) {
+	defer func() {
+		if err != nil {
+			tips := fmt.Sprintf("数据源-CCF数据装置处理失败, err: %v", err)
+			utils.FileLog.Info(tips)
+			go alarm_msg.SendAlarmMsg(tips, 3)
+		}
+	}()
+	if req.Table == nil {
+		err = fmt.Errorf("表格信息为空")
+		return
+	}
+
+	// 已存在则更新表格
+	excelOb := new(models.CCFStockExcel)
+	cond := fmt.Sprintf(` AND %s = ? AND %s = ?`, excelOb.Cols().ClassifyId, excelOb.Cols().ExcelDate)
+	pars := make([]interface{}, 0)
+	pars = append(pars, req.Table.ClassifyId, req.Table.TableDate.Format(utils.FormatDate))
+	exists, e := excelOb.GetItemByCondition(cond, pars, "")
+	if e != nil && e.Error() != utils.ErrNoRow() {
+		err = fmt.Errorf("获取已存在表格失败, err: %v", e)
+		return
+	}
+	if exists != nil {
+		exists.ExcelContent = html.EscapeString(req.Table.TableContent)
+		exists.FromPage = req.Table.FromPage
+		exists.ModifyTime = time.Now().Local()
+		cols := []string{excelOb.Cols().ExcelContent, excelOb.Cols().FromPage, excelOb.Cols().ModifyTime}
+		if e = exists.Update(cols); e != nil {
+			err = fmt.Errorf("更新已存在表格失败, err: %v", e)
+			return
+		}
+		return
+	}
+
+	// 新增表格
+	excelOb.ClassifyId = req.Table.ClassifyId
+	excelOb.ExcelDate = req.Table.TableDate
+	excelOb.ExcelContent = html.EscapeString(req.Table.TableContent)
+	excelOb.FromPage = req.Table.FromPage
+	excelOb.CreateTime = time.Now().Local()
+	excelOb.ModifyTime = time.Now().Local()
+	if e = excelOb.Create(); e != nil {
+		err = fmt.Errorf("新增表格失败, err: %v", e)
+		return
+	}
+	return
+}

services/base_from_pcsg.go

@@ -14,143 +14,53 @@ import (
 )
 
 var (
-	BridgeApiPCSGBloombergDailyUrl   = "/api/pcsg/bloomberg/daily_index"   // 日度指标API
-	BridgeApiPCSGBloombergWeeklyUrl  = "/api/pcsg/bloomberg/weekly_index"  // 周度指标API
-	BridgeApiPCSGBloombergMonthlyUrl = "/api/pcsg/bloomberg/monthly_index" // 月度指标API
+	PCSGBloombergGeneralIndexDataUrl = "/api/pcsg/bloomberg/index_data/general" // 通用指标API
 )
 
-// GetPCSGBloombergDailyFromBridge 获取彭博日度指标
-func GetPCSGBloombergDailyFromBridge() (indexes []models.BaseFromBloombergApiIndexAndData, err error) {
-	defer func() {
-		if err != nil {
-			tips := fmt.Sprintf("GetPCSGBloombergDailyFromBridge-获取彭博日度指标失败, err: %s", err.Error())
-			utils.FileLog.Info(tips)
-			go alarm_msg.SendAlarmMsg(tips, 3)
-		}
-	}()
+type PCSGBloombergApiReq struct {
+	TaskKey   string `description:"任务key"`
+	Frequency string `description:"指标频度"`
+}
 
-	url := fmt.Sprint(utils.EtaBridgeUrl, BridgeApiPCSGBloombergDailyUrl)
-	body := ioutil.NopCloser(strings.NewReader(""))
-	client := &http.Client{}
-	req, e := http.NewRequest("POST", url, body)
-	if e != nil {
-		err = fmt.Errorf("http create request err: %s", e.Error())
-		return
-	}
+type PCSGBloombergTask struct {
+	TaskKey   string `json:"TaskKey"`
+	Frequency string `json:"Frequency"`
+	VCode     bool   `json:"VCode"`
+}
 
-	checkToken := utils.MD5(utils.EtaBridgeAppNameEn + utils.EtaBridgeMd5Key)
-	contentType := "application/json;charset=utf-8"
-	req.Header.Set("Content-Type", contentType)
-	req.Header.Set("Authorization", checkToken)
-	resp, e := client.Do(req)
-	if e != nil {
-		err = fmt.Errorf("http client do err: %s", e.Error())
-		return
-	}
-	defer func() {
-		_ = resp.Body.Close()
-	}()
-	b, e := ioutil.ReadAll(resp.Body)
+// LoadPCSGBloombergTask 加载配置
+func LoadPCSGBloombergTask() (tasks []*PCSGBloombergTask, err error) {
+	filePath := "./static/pcsg_task.json"
+	b, e := ioutil.ReadFile(filePath)
 	if e != nil {
-		err = fmt.Errorf("resp body read err: %s", e.Error())
+		err = fmt.Errorf("读取配置失败, err: %v", e)
 		return
 	}
-	if len(b) == 0 {
-		err = fmt.Errorf("resp body is empty")
+	if e = json.Unmarshal(b, &tasks); e != nil {
+		err = fmt.Errorf("解析配置失败, err: %v", e)
 		return
 	}
-	// 生产环境解密
-	if utils.RunMode == "release" {
-		str := string(b)
-		str = strings.Trim(str, `"`)
-		b = utils.DesBase64Decrypt([]byte(str), utils.EtaBridgeDesKey)
-	}
-
-	result := new(models.BridgePCSGBloombergResultData)
-	if e = json.Unmarshal(b, &result); e != nil {
-		err = fmt.Errorf("result unmarshal err: %s\nresult: %s", e.Error(), string(b))
-		return
-	}
-	if result.Code != 200 {
-		err = fmt.Errorf("result: %s", string(b))
-		return
-	}
-	indexes = result.Data
 	return
 }
 
-// GetPCSGBloombergWeeklyFromBridge 获取彭博周度指标
-func GetPCSGBloombergWeeklyFromBridge() (indexes []models.BaseFromBloombergApiIndexAndData, err error) {
+// GetPCSGBloombergGeneralIndexFromBridge 获取通用数据类型指标
+func GetPCSGBloombergGeneralIndexFromBridge(params PCSGBloombergApiReq) (indexes []models.BaseFromBloombergApiIndexAndData, err error) {
 	defer func() {
 		if err != nil {
-			tips := fmt.Sprintf("GetPCSGBloombergWeeklyFromBridge-获取彭博周度指标失败, err: %s", err.Error())
+			tips := fmt.Sprintf("GetPCSGBloombergGeneralIndexFromBridge-获取指标数据失败, err: %s", err.Error())
 			utils.FileLog.Info(tips)
 			go alarm_msg.SendAlarmMsg(tips, 3)
 		}
 	}()
 
-	url := fmt.Sprint(utils.EtaBridgeUrl, BridgeApiPCSGBloombergWeeklyUrl)
-	body := ioutil.NopCloser(strings.NewReader(""))
-	client := &http.Client{}
-	req, e := http.NewRequest("POST", url, body)
+	p, e := json.Marshal(params)
 	if e != nil {
-		err = fmt.Errorf("http create request err: %s", e.Error())
+		err = fmt.Errorf("params json marshal err: %v", e)
 		return
 	}
 
-	checkToken := utils.MD5(utils.EtaBridgeAppNameEn + utils.EtaBridgeMd5Key)
-	contentType := "application/json;charset=utf-8"
-	req.Header.Set("Content-Type", contentType)
-	req.Header.Set("Authorization", checkToken)
-	resp, e := client.Do(req)
-	if e != nil {
-		err = fmt.Errorf("http client do err: %s", e.Error())
-		return
-	}
-	defer func() {
-		_ = resp.Body.Close()
-	}()
-	b, e := ioutil.ReadAll(resp.Body)
-	if e != nil {
-		err = fmt.Errorf("resp body read err: %s", e.Error())
-		return
-	}
-	if len(b) == 0 {
-		err = fmt.Errorf("resp body is empty")
-		return
-	}
-	// 生产环境解密
-	if utils.RunMode == "release" {
-		str := string(b)
-		str = strings.Trim(str, `"`)
-		b = utils.DesBase64Decrypt([]byte(str), utils.EtaBridgeDesKey)
-	}
-
-	result := new(models.BridgePCSGBloombergResultData)
-	if e = json.Unmarshal(b, &result); e != nil {
-		err = fmt.Errorf("result unmarshal err: %s\nresult: %s", e.Error(), string(b))
-		return
-	}
-	if result.Code != 200 {
-		err = fmt.Errorf("result: %s", string(b))
-		return
-	}
-	indexes = result.Data
-	return
-}
-
-// GetPCSGBloombergMonthlyFromBridge 获取彭博月度指标
-func GetPCSGBloombergMonthlyFromBridge() (indexes []models.BaseFromBloombergApiIndexAndData, err error) {
-	defer func() {
-		if err != nil {
-			tips := fmt.Sprintf("GetPCSGBloombergMonthlyFromBridge-获取彭博月度指标失败, err: %s", err.Error())
-			utils.FileLog.Info(tips)
-			go alarm_msg.SendAlarmMsg(tips, 3)
-		}
-	}()
-
-	url := fmt.Sprint(utils.EtaBridgeUrl, BridgeApiPCSGBloombergMonthlyUrl)
-	body := ioutil.NopCloser(strings.NewReader(""))
+	url := fmt.Sprint(utils.EtaBridgeUrl, PCSGBloombergGeneralIndexDataUrl)
+	body := ioutil.NopCloser(strings.NewReader(string(p)))
 	client := &http.Client{}
 	req, e := http.NewRequest("POST", url, body)
 	if e != nil {
@@ -200,7 +110,7 @@ func GetPCSGBloombergMonthlyFromBridge() (indexes []models.BaseFromBloombergApiI
 }
 
 // PCSGWrite2BaseBloomberg 写入彭博数据源
-func PCSGWrite2BaseBloomberg(indexes []models.BaseFromBloombergApiIndexAndData) (err error) {
+func PCSGWrite2BaseBloomberg(indexes []models.BaseFromBloombergApiIndexAndData, isVCode bool) (err error) {
 	defer func() {
 		if err != nil {
 			tips := fmt.Sprintf("PCSGWrite2BaseBloomberg-写入彭博数据源失败, err: %s", err.Error())
@@ -217,6 +127,9 @@ func PCSGWrite2BaseBloomberg(indexes []models.BaseFromBloombergApiIndexAndData)
 		if len(v.Data) == 0 {
 			continue
 		}
+		if isVCode {
+			v.IndexCode = utils.InsertStr2StrIdx(v.IndexCode, " ", 1, "V")
+		}
 
 		// 指标是否存在
 		index, e := models.GetBaseFromBloombergIndexByCode(v.IndexCode)

static/pcsg_task.json

@@ -0,0 +1,22 @@
+[
+  {
+    "TaskKey": "IDpcsgDailyRun4",
+    "Frequency": "日度",
+    "VCode": false
+  },
+  {
+    "TaskKey": "IDpcsgDailyRun5",
+    "Frequency": "日度",
+    "VCode": true
+  },
+  {
+    "TaskKey": "IDpcsgMonthRun2",
+    "Frequency": "月度",
+    "VCode": false
+  },
+  {
+    "TaskKey": "IDpcsgDailyRunHist1",
+    "Frequency": "日度",
+    "VCode": false
+  }
+]

utils/common.go

@@ -1289,3 +1289,24 @@ func GenerateEdbCode(num int, pre string) (edbCode string, err error) {
 
 	return
 }
+
+// InsertStr2StrIdx 可分隔的字符串中插入指定字符串, 例如: CO1 Comdty插入V => CO1 V Comdty
+func InsertStr2StrIdx(str, sep string, idx int, value string) string {
+	str = strings.TrimSpace(str)
+	// 默认以空格作为分隔符
+	if sep == "" {
+		sep = " "
+	}
+	slice := strings.Split(str, sep)
+	if len(slice) < 2 {
+		return str
+	}
+
+	// 如果idx不在切片的有效范围内，直接返回原字符串
+	if idx < 0 || idx > len(slice) {
+		return str
+	}
+
+	slice = append(slice[:idx], append([]string{value}, slice[idx:]...)...)
+	return strings.Join(slice, sep)
+}

utils/constants.go

@@ -20,11 +20,6 @@ const (
 	PageSize30                 = 30
 )
 
-const (
-	DATA_SOURCE_YONYI  = 76 //涌益咨询
-	DATA_SOURCE_FENWEI = 77 //汾渭煤炭
-)
-
 // 数据来源渠道
 const (
 	DATA_SOURCE_THS                                  = iota + 1 //同花顺
@@ -103,6 +98,8 @@ const (
 	DATA_SOURCE_CALCULATE_ZDYFX                                 // 自定义分析->74
 	DATA_SOURCE_CALCULATE_RJZ                                   // 日均值计算->75
 
+	DATA_SOURCE_YONYI         = 76 //涌益咨询
+	DATA_SOURCE_FENWEI        = 77 //汾渭煤炭
 	DATA_SOURCE_GFEX          = 78 // 广州期货交易所->78
 	DATA_SOURCE_ICPI          = 79 // ICPI消费价格指数->79
 	DATA_SOURCE_MTJH          = 80 // 煤炭江湖->80
@@ -111,6 +108,7 @@ const (
 	DATA_SOURCE_BLOOMBERG     = 83 // bloomberg彭博数据
 	DATA_SOURCE_BUSINESS      = 84 // 来源于自有数据
 	DATA_SOURCE_SCI99         = 85 // 卓创资讯
+	DATA_SOURCE_CCF           = 86 // CCF化纤信息
 )
 
 // 指标来源的中文展示
@@ -190,13 +188,13 @@ const (
 	DATA_SOURCE_NAME_PREDICT_CALCULATE_ZSXY               = `预测指数修匀`            //预测指数修匀->73
 	DATA_SOURCE_NAME_CALCULATE_ZDYFX                      = `自定义分析`             //自定义分析->74
 	DATA_SOURCE_NAME_YONYI                                = `涌益咨询`              // 涌益咨询
-
-	DATA_SOURCE_NAME_FENWEI        = `汾渭数据` // 汾渭煤炭
-	DATA_SOURCE_NAME_MTJH          = `煤炭江湖` // 煤炭江湖->80
-	DATA_SOURCE_NAME_ICPI          = "ICPI消费价格指数"
-	DATA_SOURCE_NAME_CALCULATE_SUM = `多指标求和`
-	DATA_SOURCE_NAME_CALCULATE_AVG = `多指标求平均`
-	DATA_SOURCE_NAME_BUSINESS      = `自有数据`
+	DATA_SOURCE_NAME_FENWEI                               = `汾渭数据`              // 汾渭煤炭
+	DATA_SOURCE_NAME_MTJH                                 = `煤炭江湖`              // 煤炭江湖->80
+	DATA_SOURCE_NAME_ICPI                                 = "ICPI消费价格指数"
+	DATA_SOURCE_NAME_CALCULATE_SUM                        = `多指标求和`
+	DATA_SOURCE_NAME_CALCULATE_AVG                        = `多指标求平均`
+	DATA_SOURCE_NAME_BUSINESS                             = `自有数据`
+	DATA_SOURCE_NAME_CCF                                  = `CCF` // CCF化纤信息
 )
 
 // 基础数据初始化日期
@@ -290,3 +288,8 @@ const (
 	ZhLangVersion = "zh" // 中文语言版本
 	EnLangVersion = "en" // 英文语言版本
 )
+
+const (
+	PercentCalculateTypeRange = 0 // 百分位算法类型-数据区间
+	PercentCalculateTypeNum   = 1 // 百分位算法类型-数据个数
+)