Merge branch 'master' into feature/eta1.6.5_edb_calculate

# Conflicts:
#	utils/constants.go

controllers/base_from_fenwei.go

@@ -0,0 +1,255 @@
+package controllers
+
+import (
+	"encoding/json"
+	"eta/eta_index_lib/logic"
+	"eta/eta_index_lib/models"
+	"eta/eta_index_lib/services"
+	"eta/eta_index_lib/utils"
+	"strconv"
+	"time"
+)
+
+// FenweiController 汾渭煤炭
+type FenweiController struct {
+	BaseAuthController
+}
+
+// Add
+// @Title 新增汾渭煤炭指标接口
+// @Description 新增汾渭煤炭指标接口
+// @Success 200 {object} models.AddEdbInfoReq
+// @router /add [post]
+func (this *FenweiController) Add() {
+	br := new(models.BaseResponse).Init()
+	var cacheKey string
+	defer func() {
+		if br.ErrMsg == "" {
+			br.IsSendEmail = false
+		}
+		this.Data["json"] = br
+		this.ServeJSON()
+	}()
+	var req models.AddEdbInfoReq
+	err := json.Unmarshal(this.Ctx.Input.RequestBody, &req)
+	if err != nil {
+		br.Msg = "参数解析异常！"
+		br.ErrMsg = "参数解析失败，Err:" + err.Error()
+		return
+	}
+	if req.EdbCode == "" {
+		br.Msg = "请输入指标编码！"
+		br.ErrMsg = "请输入指标编码，指标编码为空"
+		return
+	}
+
+	ob := new(models.BaseFromFenwei)
+	cacheKey = utils.CACHE_EDB_DATA_ADD + strconv.Itoa(ob.GetSource()) + "_" + req.EdbCode
+	if utils.Rc.IsExist(cacheKey) {
+		br.Ret = 501
+		br.Success = true
+		br.Msg = "系统处理中，请稍后重试"
+		return
+	}
+	utils.Rc.SetNX(cacheKey, 1, 1*time.Minute)
+	defer func() {
+		_ = utils.Rc.Delete(cacheKey)
+	}()
+
+	if err = ob.Add(req.EdbCode); err != nil {
+		br.Msg = "获取指标信息失败！"
+		br.ErrMsg = "获取指标信息失败 AddEdbDataFromFenwei，Err:" + err.Error()
+		return
+	}
+	br.Ret = 200
+	br.Success = true
+	br.Msg = "获取成功"
+}
+
+// Refresh
+// @Title 刷新汾渭煤炭指标接口
+// @Description 刷新汾渭煤炭指标接口
+// @Success 200 {object} models.RefreshEdbInfoReq
+// @router /refresh [post]
+func (this *FenweiController) Refresh() {
+	br := new(models.BaseResponse).Init()
+	var cacheKey string
+	defer func() {
+		if br.ErrMsg == "" {
+			br.IsSendEmail = false
+		}
+		this.Data["json"] = br
+		this.ServeJSON()
+	}()
+	var req models.RefreshEdbInfoReq
+	err := json.Unmarshal(this.Ctx.Input.RequestBody, &req)
+	if err != nil {
+		br.Msg = "参数解析异常！"
+		br.ErrMsg = "参数解析失败，Err:" + err.Error()
+		return
+	}
+	if req.EdbCode == "" {
+		br.Msg = "请输入指标编码！"
+		br.ErrMsg = "请输入指标编码，指标编码为空"
+		return
+	}
+	if req.EdbInfoId < 0 {
+		br.Msg = "请输入指标ID！"
+		br.ErrMsg = "请输入指标ID"
+		return
+	}
+
+	ob := new(models.BaseFromFenwei)
+	source := ob.GetSource()
+	// 获取指标详情
+	edbInfo, err := models.GetEdbInfoByEdbCode(source, req.EdbCode)
+	if err != nil {
+		br.Msg = "指标不存在！"
+		br.ErrMsg = "指标不存在"
+		return
+	}
+	cacheKey = utils.CACHE_EDB_DATA_REFRESH + strconv.Itoa(source) + "_" + req.EdbCode
+	if utils.Rc.IsExist(cacheKey) {
+		br.Ret = 501
+		br.Success = true
+		br.Msg = "系统处理中，请稍后重试"
+		return
+	}
+	utils.Rc.SetNX(cacheKey, 1, 1*time.Minute)
+	defer func() {
+		_ = utils.Rc.Delete(cacheKey)
+	}()
+	if req.EdbInfoId <= 0 {
+		req.EdbInfoId = edbInfo.EdbInfoId
+	}
+	err = ob.Refresh(req.EdbInfoId, req.EdbCode, req.StartDate)
+	if err != nil && err.Error() != utils.ErrNoRow() {
+		br.Msg = "刷新指标信息失败！"
+		br.ErrMsg = "刷新指标信息失败 RefreshEdbDataFromFenwei，Err:" + err.Error()
+		return
+	}
+	// 更新指标最大最小值
+	err, errMsg := models.UnifiedModifyEdbInfoMaxAndMinInfo(edbInfo)
+	if err != nil {
+		br.Msg = errMsg
+		br.ErrMsg = err.Error()
+		return
+	}
+
+	// 更新ES
+	go logic.UpdateEs(edbInfo.EdbInfoId)
+
+	br.Ret = 200
+	br.Success = true
+	br.Msg = "获取成功"
+}
+
+// HandleExcelData
+// @Title 处理汾渭煤炭指标的接口
+// @Description 处理汾渭煤炭指标的接口
+// @Success 200 string "操作成功"
+// @router /handle/excel_data [post]
+func (this *FenweiController) HandleExcelData() {
+	br := new(models.BaseResponse).Init()
+	defer func() {
+		if br.ErrMsg == "" {
+			br.IsSendEmail = false
+		}
+		this.Data["json"] = br
+		this.ServeJSON()
+	}()
+	var req models.HandleFenweiExcelDataReq
+	err := json.Unmarshal(this.Ctx.Input.RequestBody, &req)
+	if err != nil {
+		br.Msg = "参数解析异常！"
+		br.ErrMsg = "参数解析失败，Err:" + err.Error()
+		return
+	}
+
+	// 处理excel表数据
+	if err = services.HandleFenweiIndex(&req); err != nil {
+		br.Msg = "处理失败"
+		br.ErrMsg = "处理失败,Err:" + err.Error()
+		return
+	}
+
+	br.Ret = 200
+	br.Success = true
+	br.Msg = "处理成功"
+}
+
+// ClassifyTree
+// @Title 汾渭数据分类
+// @Description 汾渭数据分类接口
+// @Success 200 {object} models.BaseFromFenweiClassifyItem
+// @router /classify_tree [post]
+func (this *FenweiController) ClassifyTree() {
+	br := new(models.BaseResponse).Init()
+	defer func() {
+		if br.ErrMsg == "" {
+			br.IsSendEmail = false
+		}
+		this.Data["json"] = br
+		this.ServeJSON()
+	}()
+
+	classifyOb := new(models.BaseFromFenweiClassify)
+	classifies, e := classifyOb.GetItemsByCondition(``, make([]interface{}, 0), []string{}, "parent_id ASC, sort ASC")
+	if e != nil {
+		br.Msg = "获取失败"
+		br.ErrMsg = "获取汾渭数据分类失败, Err: " + e.Error()
+		return
+	}
+
+	resp := make([]*models.BaseFromFenweiClassifyItem, 0)
+	parentMap := make(map[int][]*models.BaseFromFenweiClassifyItem)
+	for _, v := range classifies {
+		t := classifyOb.Format2Item(v)
+		if v.ParentId == 0 {
+			resp = append(resp, t)
+		}
+		if v.ParentId > 0 {
+			if parentMap[v.ParentId] == nil {
+				parentMap[v.ParentId] = make([]*models.BaseFromFenweiClassifyItem, 0)
+			}
+			parentMap[v.ParentId] = append(parentMap[v.ParentId], t)
+		}
+	}
+	for _, v := range resp {
+		v.Child = parentMap[v.ClassifyId]
+	}
+
+	br.Data = resp
+	br.Ret = 200
+	br.Success = true
+	br.Msg = "获取成功"
+}
+
+// BaseIndexList
+// @Title 汾渭指标基础信息列表
+// @Description 汾渭指标基础信息列表接口
+// @Success 200 {object} models.BaseFromFenweiIndex
+// @router /base_index_list [post]
+func (this *FenweiController) BaseIndexList() {
+	br := new(models.BaseResponse).Init()
+	defer func() {
+		if br.ErrMsg == "" {
+			br.IsSendEmail = false
+		}
+		this.Data["json"] = br
+		this.ServeJSON()
+	}()
+
+	ob := new(models.BaseFromFenweiIndex)
+	list, e := ob.GetItemsByCondition(``, make([]interface{}, 0), []string{}, "")
+	if e != nil {
+		br.Msg = "获取失败"
+		br.ErrMsg = "获取汾渭指标列表失败, Err: " + e.Error()
+		return
+	}
+
+	br.Data = list
+	br.Ret = 200
+	br.Success = true
+	br.Msg = "获取成功"
+}

controllers/base_from_gz.go

@@ -0,0 +1,128 @@
+package controllers
+
+import (
+	"encoding/json"
+	"eta/eta_index_lib/logic"
+	"eta/eta_index_lib/models"
+	"eta/eta_index_lib/utils"
+	"strconv"
+	"time"
+)
+
+// 广期所
+type GzController struct {
+	BaseAuthController
+}
+
+// @Title 新增广期所指标接口
+// @Description 新增广期所指标接口
+// @Success 200 {object} models.AddEdbInfoReq
+// @router /add [post]
+func (this *GzController) Add() {
+	br := new(models.BaseResponse).Init()
+	var cacheKey string
+	defer func() {
+		utils.Rc.Delete(cacheKey)
+		this.Data["json"] = br
+		this.ServeJSON()
+	}()
+	source := utils.DATA_SOURCE_GFEX
+	var req models.AddEdbInfoReq
+	err := json.Unmarshal(this.Ctx.Input.RequestBody, &req)
+	if err != nil {
+		br.Msg = "参数解析异常！"
+		br.ErrMsg = "参数解析失败，Err:" + err.Error()
+		return
+	}
+	if req.EdbCode == "" {
+		br.Msg = "请输入指标编码！"
+		br.ErrMsg = "请输入指标编码，指标编码为空"
+		return
+	}
+	cacheKey = utils.CACHE_EDB_DATA_ADD + strconv.Itoa(source) + "_" + req.EdbCode
+	if !utils.Rc.IsExist(cacheKey) {
+		utils.Rc.SetNX(cacheKey, 1, 1*time.Minute)
+		err = models.AddEdbDataFromGz(req.EdbCode)
+		if err != nil {
+			br.Msg = "获取指标信息失败！"
+			br.ErrMsg = "获取指标信息失败 AddEdbDataFromGz，Err:" + err.Error()
+			return
+		}
+		br.Ret = 200
+		br.Success = true
+		br.Msg = "获取成功"
+	} else {
+		br.Ret = 501
+		br.Success = true
+		br.Msg = "系统处理中，请稍后重试"
+	}
+}
+
+// @Title 刷新广期所指标接口
+// @Description 刷新广期所指标接口
+// @Success 200 {object} models.RefreshEdbInfoReq
+// @router /refresh [post]
+func (this *GzController) Refresh() {
+	br := new(models.BaseResponse).Init()
+	var cacheKey string
+	defer func() {
+		utils.Rc.Delete(cacheKey)
+		this.Data["json"] = br
+		this.ServeJSON()
+	}()
+	source := utils.DATA_SOURCE_GFEX
+	var req models.RefreshEdbInfoReq
+	err := json.Unmarshal(this.Ctx.Input.RequestBody, &req)
+	if err != nil {
+		br.Msg = "参数解析异常！"
+		br.ErrMsg = "参数解析失败，Err:" + err.Error()
+		return
+	}
+	if req.EdbCode == "" {
+		br.Msg = "请输入指标编码！"
+		br.ErrMsg = "请输入指标编码，指标编码为空"
+		return
+	}
+	if req.EdbInfoId <= 0 {
+		br.Msg = "请输入指标ID！"
+		br.ErrMsg = "请输入指标ID"
+		return
+	}
+
+	// 获取指标详情
+	edbInfo, err := models.GetEdbInfoByEdbCode(source, req.EdbCode)
+	if err != nil {
+		br.Msg = "指标不存在！"
+		br.ErrMsg = "指标不存在"
+		return
+	}
+	cacheKey = utils.CACHE_EDB_DATA_REFRESH + strconv.Itoa(source) + "_" + req.EdbCode
+	if utils.Rc.IsExist(cacheKey) {
+		br.Ret = 501
+		br.Success = true
+		br.Msg = "系统处理中，请稍后重试"
+		return
+	}
+
+	utils.Rc.SetNX(cacheKey, 1, 1*time.Minute)
+	err = models.RefreshEdbDataFromGz(req.EdbInfoId, req.EdbCode, req.StartDate)
+	if err != nil && err.Error() != utils.ErrNoRow() {
+		br.Msg = "刷新指标信息失败！"
+		br.ErrMsg = "刷新指标信息失败 RefreshEdbDataFromGz，Err:" + err.Error()
+		return
+	}
+	// 更新指标最大最小值
+	err, errMsg := models.UnifiedModifyEdbInfoMaxAndMinInfo(edbInfo)
+	if err != nil {
+		br.Msg = errMsg
+		br.ErrMsg = err.Error()
+		return
+	}
+
+	// 更新ES
+	go logic.UpdateEs(edbInfo.EdbInfoId)
+
+	br.Ret = 200
+	br.Success = true
+	br.Msg = "获取成功"
+}

controllers/base_from_icpi.go

@@ -0,0 +1,128 @@
+package controllers
+
+import (
+	"encoding/json"
+	"eta/eta_index_lib/logic"
+	"eta/eta_index_lib/models"
+	"eta/eta_index_lib/utils"
+	"strconv"
+	"time"
+)
+
+// ICPI消费价格指数
+type IcpiController struct {
+	BaseAuthController
+}
+
+// @Title 新增上期所指标接口
+// @Description 新增上期所指标接口
+// @Success 200 {object} models.AddEdbInfoReq
+// @router /add [post]
+func (this *IcpiController) Add() {
+	br := new(models.BaseResponse).Init()
+	var cacheKey string
+	defer func() {
+		utils.Rc.Delete(cacheKey)
+		this.Data["json"] = br
+		this.ServeJSON()
+	}()
+	source := utils.DATA_SOURCE_ICPI
+	var req models.AddEdbInfoReq
+	err := json.Unmarshal(this.Ctx.Input.RequestBody, &req)
+	if err != nil {
+		br.Msg = "参数解析异常！"
+		br.ErrMsg = "参数解析失败，Err:" + err.Error()
+		return
+	}
+	if req.EdbCode == "" {
+		br.Msg = "请输入指标编码！"
+		br.ErrMsg = "请输入指标编码，指标编码为空"
+		return
+	}
+	cacheKey = utils.CACHE_EDB_DATA_ADD + strconv.Itoa(source) + "_" + req.EdbCode
+	if !utils.Rc.IsExist(cacheKey) {
+		utils.Rc.SetNX(cacheKey, 1, 1*time.Minute)
+		err = models.AddEdbDataFromIcpi(req.EdbCode)
+		if err != nil {
+			br.Msg = "获取指标信息失败！"
+			br.ErrMsg = "获取指标信息失败 AddEdbDataFromSh，Err:" + err.Error()
+			return
+		}
+		br.Ret = 200
+		br.Success = true
+		br.Msg = "获取成功"
+	} else {
+		br.Ret = 501
+		br.Success = true
+		br.Msg = "系统处理中，请稍后重试"
+	}
+}
+
+// @Title 刷新上期所指标接口
+// @Description 刷新上期所指标接口
+// @Success 200 {object} models.RefreshEdbInfoReq
+// @router /refresh [post]
+func (this *IcpiController) Refresh() {
+	br := new(models.BaseResponse).Init()
+	var cacheKey string
+	defer func() {
+		utils.Rc.Delete(cacheKey)
+		this.Data["json"] = br
+		this.ServeJSON()
+	}()
+	source := utils.DATA_SOURCE_ICPI
+	var req models.RefreshEdbInfoReq
+	err := json.Unmarshal(this.Ctx.Input.RequestBody, &req)
+	if err != nil {
+		br.Msg = "参数解析异常！"
+		br.ErrMsg = "参数解析失败，Err:" + err.Error()
+		return
+	}
+	if req.EdbCode == "" {
+		br.Msg = "请输入指标编码！"
+		br.ErrMsg = "请输入指标编码，指标编码为空"
+		return
+	}
+	if req.EdbInfoId <= 0 {
+		br.Msg = "请输入指标ID！"
+		br.ErrMsg = "请输入指标ID"
+		return
+	}
+
+	// 获取指标详情
+	edbInfo, err := models.GetEdbInfoByEdbCode(source, req.EdbCode)
+	if err != nil {
+		br.Msg = "指标不存在！"
+		br.ErrMsg = "指标不存在"
+		return
+	}
+	cacheKey = utils.CACHE_EDB_DATA_REFRESH + strconv.Itoa(source) + "_" + req.EdbCode
+	if utils.Rc.IsExist(cacheKey) {
+		br.Ret = 501
+		br.Success = true
+		br.Msg = "系统处理中，请稍后重试"
+		return
+	}
+
+	utils.Rc.SetNX(cacheKey, 1, 1*time.Minute)
+	err = models.RefreshEdbDataFromIcpi(req.EdbInfoId, req.EdbCode, req.StartDate)
+	if err != nil && err.Error() != utils.ErrNoRow() {
+		br.Msg = "刷新指标信息失败！"
+		br.ErrMsg = "刷新指标信息失败 RefreshEdbDataFromIcpi，Err:" + err.Error()
+		return
+	}
+	// 更新指标最大最小值
+	err, errMsg := models.UnifiedModifyEdbInfoMaxAndMinInfo(edbInfo)
+	if err != nil {
+		br.Msg = errMsg
+		br.ErrMsg = err.Error()
+		return
+	}
+
+	// 更新ES
+	go logic.UpdateEs(edbInfo.EdbInfoId)
+
+	br.Ret = 200
+	br.Success = true
+	br.Msg = "获取成功"
+}

controllers/base_from_lt.go

@@ -112,13 +112,13 @@ func (this *LtController) Refresh() {
 		return
 	}
 
-	dataItem, err := services.GetEdbDataFromLt(req.EdbCode, req.StartDate, utils.BASE_END_DATE, edbInfo.TerminalCode)
+	dataItem, err := services.GetEdbDataFromLt(req.EdbCode, utils.GetEdbRefreshStartDate(req.StartDate), utils.BASE_END_DATE, edbInfo.TerminalCode)
 	if err != nil {
 		br.Msg = "获取指标信息失败！"
 		br.ErrMsg = "获取指标信息失败 GetEdbDataFromLt，Err:" + err.Error()
 		return
 	}
-	err = models.RefreshEdbDataFromLt(req.EdbInfoId, req.EdbCode, req.StartDate, dataItem)
+	err = models.RefreshEdbDataFromLt(req.EdbInfoId, req.EdbCode, utils.GetEdbRefreshStartDate(req.StartDate), dataItem)
 	if err != nil && err.Error() != utils.ErrNoRow() {
 		br.Msg = "刷新指标信息失败！"
 		br.ErrMsg = "刷新指标信息失败 RefreshEdbDataFromLt，Err:" + err.Error()

controllers/base_from_mtjh.go

@@ -0,0 +1,379 @@
+package controllers
+
+import (
+	"encoding/json"
+	"eta/eta_index_lib/logic"
+	"eta/eta_index_lib/models"
+	"eta/eta_index_lib/utils"
+	"fmt"
+	"github.com/mozillazg/go-pinyin"
+	"strconv"
+	"time"
+)
+
+type MtjhDataController struct {
+	BaseAuthController
+}
+
+
+// @Title 煤炭江湖数据
+// @Description 刷新煤炭江湖数据接口
+// @Param	request	body models.CoalMineDataReq true "type json string"
+// @Success 200 {object} models.EdbClassify
+// @router /data [post]
+func (this *MtjhDataController) Mtjh() {
+	br := new(models.BaseResponse).Init()
+	defer func() {
+		this.Data["json"] = br
+		this.ServeJSON()
+	}()
+	var req models.CoalMineDataReq
+
+	err := json.Unmarshal(this.Ctx.Input.RequestBody, &req)
+	if err != nil {
+		br.Msg = "参数解析异常！"
+		br.ErrMsg = "参数解析失败，Err:" + err.Error()
+		return
+	}
+
+	var area string
+	var port string
+	var description string
+
+	var dateMap = make(map[int]string)
+	var varietyMap = make(map[int]string)
+	var codeMap = make(map[string]string)
+	var indexMap = make(map[string]string)
+	var codeCompanyMap = make(map[string]string)
+	var indexCompanyMap = make(map[string]string)
+	var items []*models.BaseFromMtjhMapping
+	var itemsIndex []*models.BaseFromMtjhIndex
+	codeList, err := models.GetBaseFromMtjhMapping()
+	if err != nil && err.Error() != utils.ErrNoRow() {
+		utils.FileLog.Info("获取煤炭指标失败：", err)
+		return
+	}
+
+	if len(codeList) > 0 {
+		for _, v := range codeList {
+			codeMap[v.IndexName] = v.IndexCode
+		}
+	}
+
+	codeCompanyList, err := models.GetBaseFromMtjhMapping()
+	if err != nil && err.Error() != utils.ErrNoRow() {
+		utils.FileLog.Info("获取煤炭公司指标失败：", err)
+		return
+	}
+	if len(codeCompanyList) > 0 {
+		for _, v := range codeCompanyList {
+			codeCompanyMap[v.IndexName] = v.IndexCode
+		}
+	}
+
+	indexList, err := models.GetBaseFromMtjhIndex()
+	if err != nil && err.Error() != utils.ErrNoRow() {
+		utils.FileLog.Info("获取煤炭公司指标失败：", err)
+		return
+	}
+	if len(indexList) > 0 {
+		for _, v := range indexList {
+			indexMap[v.IndexName+v.DataTime] = v.DealValue
+		}
+	}
+
+	indexCompanyList, err := models.GetBaseFromCoalmineCompanyIndex()
+	if err != nil && err.Error() != utils.ErrNoRow() {
+		utils.FileLog.Info("获取煤炭公司指标失败：", err)
+		return
+	}
+	if len(indexCompanyList) > 0 {
+		for _, v := range indexCompanyList {
+			indexCompanyMap[v.IndexName+v.DataTime] = v.DealValue
+		}
+	}
+
+	for _, sheet := range req.SheetData {
+		//遍历行读取
+		maxRow := sheet.MaxRow
+		for i := 1; i < maxRow; i++ {
+			//获取样本情况
+			row := sheet.Rows[i]
+			cells := row.Cells
+
+			// 获取日期
+			if i == 1 {
+				var date string
+				for k, cell := range cells {
+					if k > 2 {
+						text := cell.Value
+						if text != "" {
+							stamp, _ := time.ParseInLocation("01-02-06", text, time.Now().Location())
+							date = stamp.Format(utils.FormatDate)
+						}
+						dateMap[k] = date
+					}
+				}
+			}
+
+			// 获取品种
+			if i == 2 {
+				for k, cell := range cells {
+					if k > 2 {
+						text := cell.Value
+						if text != "" {
+							varietyMap[k] = text
+						}
+					}
+				}
+			}
+
+			//for i := 0 ;i < len(cells);i ++ {
+			//	fmt.Println("k:",i)
+			//	fmt.Println("dateMap:",dateMap[i])
+			//	fmt.Println("varietyMap:",varietyMap[i])
+			//}
+			if i > 2 {
+				for k, cell := range cells {
+					text := cell.Value
+					if text != "" {
+						if k == 1 {
+							area = text
+						}
+						if k == 2 {
+							port = text
+
+							for j := 0; j < 3; j++ {
+								item := new(models.BaseFromMtjhMapping)
+								switch j {
+								case 0:
+									description = "动力煤"
+								case 1:
+									description = "焦煤"
+								case 2:
+									description = "合计"
+								}
+								itemName := area + port + description
+
+								//取处理后公司名首字母缩写
+								var strResult string
+
+								a := pinyin.NewArgs()
+								pys := [][]string{}
+								for _, r := range itemName {
+									py := pinyin.SinglePinyin(r, a)
+									if len(py) > 0 {
+										pys = append(pys, py)
+									} else {
+										if r != '-' && r != '（'  && r != '）'{
+											pys = append(pys, []string{string(r)})
+										}
+									}
+								}
+								for i := 0; i < len(pys); i++ {
+									if len(pys[i]) != 0 {
+										str := pys[i][0]
+										pi := str[0:1]
+										strResult += pi
+									}
+								}
+
+								strResult = "mtjh" + strResult
+
+								item.IndexCode = strResult
+								item.Area = area
+								item.Port = port
+								item.Variety = description
+								item.Unit = "万吨"
+								item.Frequency = "周度"
+								item.IndexName = itemName
+
+								item.CreateTime = time.Now()
+								items = append(items, item)
+							}
+						}
+						if k > 2 {
+							item := new(models.BaseFromMtjhIndex)
+							item.IndexName = area + port + varietyMap[k]
+							item.IndexCode = codeMap[item.IndexName]
+							item.DealValue = text
+							item.DataTime = dateMap[k]
+							item.Area = area
+							item.Port = port
+							item.Variety = varietyMap[k]
+							item.Unit = "万吨"
+							item.Frequency = "周度"
+							item.ModifyTime = time.Now()
+							item.CreateTime = time.Now()
+							itemsIndex = append(itemsIndex, item)
+						}
+					}
+				}
+			}
+		}
+	}
+
+	//添加数据到数据库
+	mappingAddList := make([]*models.BaseFromMtjhMapping, 0)
+	for _, v := range items {
+		if codeMap[v.IndexName] == "" {
+			codeMap[v.IndexName] = v.IndexCode
+			mappingAddList = append(mappingAddList, v)
+		}
+	}
+	if len(mappingAddList) > 0 {
+		newId, err := models.AddBaseFromMtjhMappingMuti(mappingAddList)
+		if err != nil {
+			fmt.Println("添加指标名称错误",err.Error())
+		} else {
+			fmt.Println("添加指标名称成功", newId)
+		}
+	}
+
+	fmt.Println("指标操作完成")
+
+	indexAddList := make([]*models.BaseFromMtjhIndex, 0)
+	for _, v := range itemsIndex {
+		v.IndexCode = codeMap[v.IndexName]
+		if indexMap[v.IndexName+v.DataTime] == "" && v.DealValue != "" {
+			indexAddList = append(indexAddList, v)
+		} else {
+			if indexMap[v.IndexName+v.DataTime] != v.DealValue && v.DealValue != "" {
+				err = models.UpdateBaseFromMtjhIndex(v)
+				if err != nil {
+					fmt.Println("修改数据错误错误", err)
+					return
+				}
+			}
+		}
+	}
+	if len(indexAddList) > 0 {
+		newId, err := models.AddBaseFromMtjhIndexMuti(indexAddList)
+		if err != nil {
+			fmt.Println("添加指标名称错误",err.Error())
+		} else {
+			fmt.Println("添加指标名称成功", newId)
+		}
+	}
+	fmt.Println("数据操作完成")
+
+	br.Ret = 200
+	br.Success = true
+	br.Msg = "获取成功"
+	return
+}
+
+
+// @Title 新增煤炭网指标接口
+// @Description 新增煤炭网指标接口
+// @Success 200 {object} models.AddEdbInfoReq
+// @router /add [post]
+func (this *MtjhDataController) Add() {
+	br := new(models.BaseResponse).Init()
+	var cacheKey string
+	defer func() {
+		utils.Rc.Delete(cacheKey)
+		this.Data["json"] = br
+		this.ServeJSON()
+	}()
+	source := utils.DATA_SOURCE_MTJH
+	var req models.AddEdbInfoReq
+	err := json.Unmarshal(this.Ctx.Input.RequestBody, &req)
+	if err != nil {
+		br.Msg = "参数解析异常！"
+		br.ErrMsg = "参数解析失败，Err:" + err.Error()
+		return
+	}
+	if req.EdbCode == "" {
+		br.Msg = "请输入指标编码！"
+		br.ErrMsg = "请输入指标编码，指标编码为空"
+		return
+	}
+	cacheKey = utils.CACHE_EDB_DATA_ADD + strconv.Itoa(source) + "_" + req.EdbCode
+	if !utils.Rc.IsExist(cacheKey) {
+		utils.Rc.SetNX(cacheKey, 1, 1*time.Minute)
+		err = models.AddEdbDataFromMtjh(req.EdbCode)
+		if err != nil {
+			br.Msg = "获取指标信息失败！"
+			br.ErrMsg = "获取指标信息失败 AddEdbDataFromMtjh，Err:" + err.Error()
+			return
+		}
+		br.Ret = 200
+		br.Success = true
+		br.Msg = "获取成功"
+	} else {
+		br.Ret = 501
+		br.Success = true
+		br.Msg = "系统处理中，请稍后重试"
+	}
+}
+
+// @Title 刷新中国煤炭网指标接口
+// @Description 刷新中国煤炭网指标接口
+// @Success 200 {object} models.RefreshEdbInfoReq
+// @router /refresh [post]
+func (this *MtjhDataController) Refresh() {
+	br := new(models.BaseResponse).Init()
+	var cacheKey string
+	defer func() {
+		utils.Rc.Delete(cacheKey)
+		this.Data["json"] = br
+		this.ServeJSON()
+	}()
+	source := utils.DATA_SOURCE_MTJH
+	var req models.RefreshEdbInfoReq
+	err := json.Unmarshal(this.Ctx.Input.RequestBody, &req)
+	if err != nil {
+		br.Msg = "参数解析异常！"
+		br.ErrMsg = "参数解析失败，Err:" + err.Error()
+		return
+	}
+	if req.EdbCode == "" {
+		br.Msg = "请输入指标编码！"
+		br.ErrMsg = "请输入指标编码，指标编码为空"
+		return
+	}
+	if req.EdbInfoId <= 0 {
+		br.Msg = "请输入指标ID！"
+		br.ErrMsg = "请输入指标ID"
+		return
+	}
+
+	cacheKey = utils.CACHE_EDB_DATA_REFRESH + strconv.Itoa(source) + "_" + req.EdbCode
+	if utils.Rc.IsExist(cacheKey) {
+		br.Ret = 501
+		br.Success = true
+		br.Msg = "系统处理中，请稍后重试"
+		return
+	}
+
+	utils.Rc.SetNX(cacheKey, 1, 1*time.Minute)
+	// 获取指标详情
+	edbInfo, err := models.GetEdbInfoById(req.EdbInfoId)
+	if err != nil {
+		br.Msg = "指标不存在！"
+		br.ErrMsg = "指标不存在"
+		return
+	}
+
+	err = models.RefreshEdbDataFromMtjh(req.EdbInfoId, req.EdbCode, req.StartDate)
+	if err != nil && err.Error() != utils.ErrNoRow() {
+		br.Msg = "刷新指标信息失败！"
+		br.ErrMsg = "刷新指标信息失败 RefreshEdbDataFromCoal，Err:" + err.Error()
+		return
+	}
+	// 更新指标最大最小值
+	err, errMsg := models.UnifiedModifyEdbInfoMaxAndMinInfo(edbInfo)
+	if err != nil {
+		br.Msg = errMsg
+		br.ErrMsg = err.Error()
+		return
+	}
+
+	// 更新ES
+	go logic.UpdateEs(edbInfo.EdbInfoId)
+
+	br.Ret = 200
+	br.Success = true
+	br.Msg = "获取成功"
+}

controllers/base_from_ths.go

@@ -114,13 +114,13 @@ func (this *ThsController) Refresh() {
 	defer func() {
 		utils.Rc.Delete(cacheKey)
 	}()
-	dataItem, err := services.GetEdbDataFromThs(req.EdbCode, req.StartDate, utils.BASE_END_DATE, edbInfo.TerminalCode)
+	dataItem, err := services.GetEdbDataFromThs(req.EdbCode, utils.GetEdbRefreshStartDate(req.StartDate), utils.BASE_END_DATE, edbInfo.TerminalCode)
 	if err != nil {
 		br.Msg = "获取指标信息失败！"
 		br.ErrMsg = "获取指标信息失败 GetEdbDataFromThs，Err:" + err.Error()
 		return
 	}
-	err = models.RefreshEdbDataFromThs(req.EdbInfoId, req.EdbCode, req.StartDate, dataItem)
+	err = models.RefreshEdbDataFromThs(req.EdbInfoId, req.EdbCode, utils.GetEdbRefreshStartDate(req.StartDate), dataItem)
 	if err != nil && err.Error() != utils.ErrNoRow() {
 		br.Msg = "刷新指标信息失败！"
 		br.ErrMsg = "刷新指标信息失败 RefreshEdbDataFromThs，Err:" + err.Error()

controllers/base_from_ths_ds.go

@@ -52,7 +52,7 @@ func (this *ThsDsController) Add() {
 			br.ErrMsg = "获取指标信息失败 GetEdbDataFromThsDs，Err:" + err.Error()
 			return
 		}
-		err = models.AddEdbDataFromThsDs(req.StockCode,req.EdbCode, dataItem)
+		err = models.AddEdbDataFromThsDs(req.StockCode, req.EdbCode, dataItem)
 		if err != nil {
 			br.Msg = "获取指标信息失败！"
 			br.ErrMsg = "获取指标信息失败 AddEdbDataFromThs，Err:" + err.Error()
@@ -118,13 +118,13 @@ func (this *ThsDsController) Refresh() {
 	defer func() {
 		utils.Rc.Delete(cacheKey)
 	}()
-	dataItem, err := services.GetEdbDataFromThsDs(edbInfo.StockCode, edbInfo.IndicatorCode, req.StartDate, endDate, edbInfo.TerminalCode)
+	dataItem, err := services.GetEdbDataFromThsDs(edbInfo.StockCode, edbInfo.IndicatorCode, utils.GetEdbRefreshStartDate(req.StartDate), endDate, edbInfo.TerminalCode)
 	if err != nil {
 		br.Msg = "获取指标信息失败！"
 		br.ErrMsg = "获取指标信息失败 GetEdbDataFromThsDs，Err:" + err.Error()
 		return
 	}
-	err = models.RefreshEdbDataFromThsDs(req.EdbInfoId, req.EdbCode, req.StartDate, dataItem)
+	err = models.RefreshEdbDataFromThsDs(req.EdbInfoId, req.EdbCode, utils.GetEdbRefreshStartDate(req.StartDate), dataItem)
 	if err != nil && err.Error() != utils.ErrNoRow() {
 		br.Msg = "刷新指标信息失败！"
 		br.ErrMsg = "刷新指标信息失败 RefreshEdbDataFromThsDs，Err:" + err.Error()

controllers/base_from_wind.go

@@ -209,7 +209,7 @@ func (this *WindController) Refresh() {
 		br.Msg = "终端未配置"
 		return
 	}
-	dataItem, errCode, err := services.GetEdbDataFromWindUrl(terminal.ServerUrl, req.EdbCode, req.StartDate, endDate)
+	dataItem, errCode, err := services.GetEdbDataFromWindUrl(terminal.ServerUrl, req.EdbCode, utils.GetEdbRefreshStartDate(req.StartDate), utils.GetEdbRefreshEndDate(endDate))
 	if errCode == 421 { //指标超限
 		br.Ret = 421
 		br.Msg = "指标超限！"
@@ -233,7 +233,7 @@ func (this *WindController) Refresh() {
 		br.ErrMsg = "获取指标信息失败 GetEdbDataFromWind，Err:" + err.Error()
 		return
 	}
-	err = models.RefreshEdbDataFromWind(req.EdbInfoId, req.EdbCode, req.StartDate, dataItem)
+	err = models.RefreshEdbDataFromWind(req.EdbInfoId, req.EdbCode, utils.GetEdbRefreshStartDate(req.StartDate), dataItem)
 	if err != nil && err.Error() != utils.ErrNoRow() {
 		br.Msg = "刷新指标信息失败！"
 		br.ErrMsg = "刷新指标信息失败 RefreshEdbDataFromWind，Err:" + err.Error()

controllers/base_from_wind_wsd.go

@@ -197,7 +197,7 @@ func (this *WindWsdController) Refresh() {
 		endDate = time.Now().Format(utils.FormatDate)
 	}
 
-	dataItem, errCode, err := services.GetEdbDataFromWindUrlWsd(edbInfo.ServerUrl, edbInfo.StockCode, edbInfo.IndicatorCode, req.StartDate, endDate)
+	dataItem, errCode, err := services.GetEdbDataFromWindUrlWsd(edbInfo.ServerUrl, edbInfo.StockCode, edbInfo.IndicatorCode, utils.GetEdbRefreshStartDate(req.StartDate), utils.GetEdbRefreshEndDate(endDate))
 	if errCode == 421 { //指标超限
 		br.Ret = 421
 		br.Msg = "指标超限！"
@@ -221,7 +221,7 @@ func (this *WindWsdController) Refresh() {
 		br.ErrMsg = "获取指标信息失败 GetEdbDataFromWind，Err:" + err.Error()
 		return
 	}
-	err = models.RefreshEdbDataFromWindWsd(req.EdbInfoId, edbInfo.EdbCode, req.StartDate, dataItem)
+	err = models.RefreshEdbDataFromWindWsd(req.EdbInfoId, edbInfo.EdbCode, utils.GetEdbRefreshStartDate(req.StartDate), dataItem)
 	if err != nil && err.Error() != utils.ErrNoRow() {
 		br.Msg = "刷新指标信息失败！"
 		br.ErrMsg = "刷新指标信息失败 RefreshEdbDataFromWind，Err:" + err.Error()

controllers/commodity_coal.go

@@ -659,8 +659,8 @@ func (this *CoalMineDataController) InlandHistory() {
 	return
 }
 
-// @Title 刷新数据
-// @Description 刷新数据接口
+// @Title 晋陕蒙数据
+// @Description 刷新晋陕蒙数据接口
 // @Param	request	body models.CoalMineDataReq true "type json string"
 // @Success 200 {object} models.EdbClassify
 // @router /jsm [post]
@@ -1007,8 +1007,8 @@ func (this *CoalMineDataController) Jsm() {
 	return
 }
 
-// @Title 刷新数据
-// @Description 刷新数据接口
+// @Title 沿海数据
+// @Description 刷新沿海数据接口
 // @Param	request	body models.CoalMineDataReq true "type json string"
 // @Success 200 {object} models.EdbClassify
 // @router /coastal [post]
@@ -1209,8 +1209,8 @@ func (this *CoalMineDataController) Coastal() {
 	return
 }
 
-// @Title 刷新数据
-// @Description 刷新数据接口
+// @Title 内陆数据
+// @Description 刷新内陆数据接口
 // @Param	request	body models.CoalMineDataReq true "type json string"
 // @Success 200 {object} models.EdbClassify
 // @router /inland [post]
@@ -1414,6 +1414,238 @@ func (this *CoalMineDataController) Inland() {
 	br.Msg = "获取成功"
 	return
 }
+
+// @Title 内陆数据
+// @Description 刷新内陆数据接口
+// @Param	request	body models.CoalMineDataReq true "type json string"
+// @Success 200 {object} models.EdbClassify
+// @router /firm [post]
+func (this *CoalMineDataController) Firm() {
+	br := new(models.BaseResponse).Init()
+	defer func() {
+		this.Data["json"] = br
+		this.ServeJSON()
+	}()
+	var req models.CoalMineDataReq
+
+	err := json.Unmarshal(this.Ctx.Input.RequestBody, &req)
+	if err != nil {
+		br.Msg = "参数解析异常！"
+		br.ErrMsg = "参数解析失败，Err:" + err.Error()
+		return
+	}
+
+
+	var year, month, day string
+	var yearMap = make(map[int]string)
+	var monthMap = make(map[int]string)
+	var dateMap = make(map[int]string)
+	var mappingItems []*models.BaseFromCoalmineMapping
+	var indexItems []*models.BaseFromCoalmineFirmIndex
+	var codeMap = make(map[string]string)
+	var indexMap = make(map[string]string)
+	var groupName string
+	codeList, err := models.GetBaseFromCoalmineMapping()
+	if err != nil && err.Error() != utils.ErrNoRow() {
+		utils.FileLog.Info("获取煤炭指标失败：", err)
+		return
+	}
+	if len(codeList) > 0 {
+		for _, v := range codeList {
+			codeMap[v.IndexName] = v.IndexCode
+		}
+	}
+	indexCompanyList, err := models.GetBaseFromCoalmineFirmIndex()
+	if err != nil && err.Error() != utils.ErrNoRow() {
+		utils.FileLog.Info("获取煤炭公司指标失败：", err)
+		return
+	}
+	if len(indexCompanyList) > 0 {
+		for _, v := range indexCompanyList {
+			indexMap[v.IndexName+v.DataTime] = v.DealValue
+		}
+	}
+
+	for _, sheet := range req.SheetData {
+		//遍历行读取
+		maxRow := sheet.MaxRow
+		for i := 0; i < maxRow; i++ {
+			//获取年份
+			//if i == 2 {
+			//	row := sheet.Row(i)
+			//	cells := row.Cells
+			//	for k, cell := range cells {
+			//		text := cell.String()
+			//		if k > 0 && text != "" {
+			//			year = text
+			//			fmt.Println("year:",year)
+			//			fmt.Println("k:",k)
+			//			yearMap[k] = year
+			//		}
+			//	}
+			//}
+			//获取月份和年度
+			if i == 3 {
+				row := sheet.Rows[i]
+				cells := row.Cells
+				for k, cell := range cells {
+					text := cell.Value
+					if k > 0 && text != "" {
+						month = text
+						month = utils.ConvertToFormatDay(month)
+						time, _ := time.Parse(utils.FormatDate, month)
+						yearInt := time.Year()
+						monthInt := int(time.Month())
+						year = strconv.Itoa(yearInt)
+						yearMap[k] = year
+						month = strconv.Itoa(monthInt)
+						monthMap[k] = month
+					}
+				}
+			}
+			//获取旬度
+			if i == 4 {
+				row := sheet.Rows[i]
+				cells := row.Cells
+				for k, cell := range cells {
+					text := cell.Value
+					if k > 0 && text != "" {
+						if yearMap[k] != "" {
+							year = yearMap[k]
+						}
+						if monthMap[k] != "" {
+							month = monthMap[k]
+						}
+						day = text
+						dateMap[k] = year + "年" + month + "月" + day
+					}
+				}
+			}
+			//获取企业或地区名及信息
+			if i > 4 {
+				row := sheet.Rows[i]
+				cells := row.Cells
+				var companyName string
+				for k, cell := range cells {
+					if k == 0 {
+						companyName = cell.Value
+
+						//省名
+						if !strings.Contains(companyName, " ") {
+							groupName = companyName
+						}
+						if groupName == "全国合计" {
+							groupName = "全国"
+						}
+						//若有下面四个名称要拼上省名不然会重复
+						if companyName == "  地方合计" || companyName == "    地方国有" ||
+							companyName == "    地方乡镇" || companyName == "  国有重点" {
+							companyName = groupName + companyName
+						}
+						companyName = strings.Replace(companyName, " ", "", -1)
+						var item models.BaseFromCoalmineMapping
+						//生成code
+						exists := ContainsSpecialName(companyName)
+						var code string
+						if exists {
+							abbr := trimProvinceName(companyName)
+							//取处理后公司名首字母缩写
+							a := pinyin.NewArgs()
+							rows := pinyin.Pinyin(companyName[9:], a)
+							for i := 0; i < len(rows); i++ {
+								if len(rows[i]) != 0 {
+									str := rows[i][0]
+									pi := str[0:1]
+									code += pi
+								}
+							}
+							item.IndexCode = abbr[:2] + code + "firm"
+
+						} else {
+							a := pinyin.NewArgs()
+							rows := pinyin.Pinyin(companyName, a)
+							for i := 0; i < len(rows); i++ {
+								if len(rows[i]) != 0 {
+									str := rows[i][0]
+									pi := str[0:1]
+									code += pi
+								}
+							}
+							item.IndexCode = code + "firm"
+						}
+						item.IndexName = companyName
+						item.CreateTime = time.Now()
+						mappingItems = append(mappingItems, &item)
+					} else {
+						dealValue := cell.Value
+
+						item := models.BaseFromCoalmineFirmIndex{
+							IndexName:  companyName,
+							IndexCode:  codeMap[companyName],
+							DataTime:   dateMap[k],
+							DealValue:  dealValue,
+							GroupName:  groupName,
+							Source:     "全国分企业",
+							Unit:       "万吨",
+							Frequency:  "旬度",
+							CreateTime: time.Now(),
+							ModifyTime: time.Now(),
+						}
+
+						indexItems = append(indexItems, &item)
+					}
+				}
+			}
+		}
+	}
+	//添加数据到数据库
+	for _, v := range mappingItems {
+		if codeMap[v.IndexName] == "" {
+			codeMap[v.IndexName] = v.IndexCode
+			newId, err := models.AddBaseFromCoalmineMapping(v)
+			if err != nil {
+				for i := 0; i < 10; i++ {
+					v.IndexCode = v.IndexCode + strconv.Itoa(i)
+					codeMap[v.IndexName] = v.IndexCode
+					newId, err := models.AddBaseFromCoalmineMapping(v)
+					if err != nil {
+						fmt.Println("再次添加公司指标名称错误", err)
+						continue
+					} else {
+						fmt.Println("新增公司成功", newId)
+						break
+					}
+				}
+			} else {
+				fmt.Println("新增公司成功", newId)
+			}
+		}
+	}
+	fmt.Println("指标操作完成")
+
+	//给indexItem中的code赋值并插入index表
+	for _, v := range indexItems {
+		v.IndexCode = codeMap[v.IndexName]
+		if indexMap[v.IndexName+v.DataTime] == "" && v.DealValue != "" {
+			newId, err := models.AddBaseFromCoalFirmIndex(v)
+			if err != nil {
+				fmt.Println("添加数据错误", err)
+			} else {
+				fmt.Println("新增成功", newId)
+			}
+		} else {
+			if indexMap[v.IndexName+v.DataTime] != v.DealValue && v.DealValue != "" {
+				err = models.UpdateBaseFromCoalFirmIndex(v)
+				if err != nil {
+					fmt.Println("修改数据错误错误", err)
+					return
+				}
+			}
+		}
+	}
+	return
+}
+
 func ttrimProvinceName(name string) string {
 	name = strings.Replace(name, "陕西省", "sn", -1)
 	name = strings.Replace(name, "陕西", "sn", -1)

controllers/edb_refresh.go

@@ -0,0 +1,61 @@
+package controllers
+
+import (
+	"eta/eta_index_lib/models"
+	"eta/eta_index_lib/services/edb_refresh"
+	"eta/eta_index_lib/utils"
+)
+
+type EdbRefreshController struct {
+	BaseAuthController
+}
+
+// GetMysteelIndexConfigList
+// @Title 获取钢联的刷新配置
+// @Description 获取钢联的刷新配置
+// @Success 200 {object}
+// @router /config/mysteel_index/list [post]
+func (this *EdbRefreshController) GetMysteelIndexConfigList() {
+	br := new(models.BaseResponse).Init()
+	defer func() {
+		this.Data["json"] = br
+		this.ServeJSON()
+	}()
+
+	list, err := edb_refresh.GetDefaultConfigList(utils.DATA_SOURCE_MYSTEEL_CHEMICAL, 0)
+
+	if err != nil {
+		br.Msg = "获取失败"
+		br.ErrMsg = "获取失败，ERR：" + err.Error()
+	}
+
+	br.Ret = 200
+	br.Success = true
+	br.Msg = "获取成功"
+	br.Data = list
+}
+
+// GetSmmConfigList
+// @Title 获取有色的刷新配置
+// @Description 获取有色的刷新配置
+// @Success 200 {object}
+// @router /config/smm/list [post]
+func (this *EdbRefreshController) GetSmmConfigList() {
+	br := new(models.BaseResponse).Init()
+	defer func() {
+		this.Data["json"] = br
+		this.ServeJSON()
+	}()
+
+	list, err := edb_refresh.GetDefaultConfigList(utils.DATA_SOURCE_YS, 0)
+
+	if err != nil {
+		br.Msg = "获取失败"
+		br.ErrMsg = "获取失败，ERR：" + err.Error()
+	}
+
+	br.Ret = 200
+	br.Success = true
+	br.Msg = "获取成功"
+	br.Data = list
+}

logic/profit_chart_info.go

@@ -552,7 +552,7 @@ func ProfitChartChartData(baseDataList []*models.EdbDataList, futureGoodEdbInfoM
 
 		yDataList = append(yDataList, YData{
 			Date:           yDate,
-			ConfigDate:     findDateTime,
+			ConfigDate:     realDateTime,
 			Value:          findDataList,
 			NoDataEdbList:  noDataIdList,
 			XEdbInfoIdList: xEdbInfoIdList,

models/base_from_coal_firm.go

@@ -20,23 +20,23 @@ type BaseFromCoalmineFirmIndex struct {
 	ModifyTime                  time.Time `description:"修改时间"`
 }
 
-//查询指标
+// 查询指标
 func GetBaseFromCoalmineFirmIndex() (items []*BaseFromCoalmineFirmIndex, err error) {
-	o := orm.NewOrmUsingDB("data")
+	o := orm.NewOrm()
 	sql := `SELECT * FROM base_from_coalmine_firm_index`
 	_, err = o.Raw(sql).QueryRows(&items)
 	return
 }
 
-//添加数据
+// 添加数据
 func AddBaseFromCoalFirmIndex(item *BaseFromCoalmineFirmIndex) (lastId int64, err error) {
-	o := orm.NewOrmUsingDB("data")
+	o := orm.NewOrm()
 	lastId, err = o.Insert(item)
 	return
 }
 
 func UpdateBaseFromCoalFirmIndex(item *BaseFromCoalmineFirmIndex) (err error) {
-	o := orm.NewOrmUsingDB("data")
+	o := orm.NewOrm()
 	sql := `UPDATE base_from_coalmine_firm_index SET deal_value=?  WHERE index_name=?  AND  data_time = ?`
 	_, err = o.Raw(sql, item.DealValue, item.IndexName, item.DataTime).Exec()
 	return

models/base_from_fenwei.go

@@ -0,0 +1,415 @@
+package models
+
+import (
+	"eta/eta_index_lib/utils"
+	"fmt"
+	"github.com/beego/beego/v2/client/orm"
+	"strconv"
+	"strings"
+	"time"
+)
+
+// BaseFromFenwei 涌溢咨询
+type BaseFromFenwei struct{}
+
+type BaseFromFenweiData struct {
+	FenweiDataId  int `orm:"column(fenwei_data_id);pk"`
+	FenweiIndexId int
+	IndexCode     string
+	DataTime      string
+	Value         string
+	CreateTime    time.Time
+	ModifyTime    time.Time
+	DataTimestamp int64
+}
+
+func (m *BaseFromFenweiData) TableName() string {
+	return "base_from_fenwei_data"
+}
+
+func GetBaseFromFenweiDataByCondition(condition string, pars []interface{}) (list []*BaseFromFenweiData, err error) {
+	o := orm.NewOrm()
+	sql := `SELECT * FROM base_from_fenwei_data WHERE 1=1 `
+	if condition != "" {
+		sql += condition
+	}
+	_, err = o.Raw(sql, pars).QueryRows(&list)
+	return
+}
+
+// Add 添加
+func (obj BaseFromFenwei) Add(edbCode string) (err error) {
+	o := orm.NewOrm()
+
+	var condition string
+	var pars []interface{}
+	if edbCode != "" {
+		condition += " AND index_code=? "
+		pars = append(pars, edbCode)
+	}
+	fenweiBaseDataAll, err := GetBaseFromFenweiDataByCondition(condition, pars)
+	if err != nil && err.Error() != utils.ErrNoRow() {
+		return
+	}
+	var isAdd bool
+	addSql := ` INSERT INTO edb_data_fenwei(edb_info_id,edb_code,data_time,value,create_time,modify_time,data_timestamp) values `
+	existMap := make(map[string]string)
+	for _, sv := range fenweiBaseDataAll {
+		eDate := sv.DataTime
+		dataTime, err := time.Parse(utils.FormatDate, eDate)
+		if err != nil {
+			return err
+		}
+		timestamp := dataTime.UnixNano() / 1e6
+		timeStr := fmt.Sprintf("%d", timestamp)
+		if _, ok := existMap[eDate]; !ok {
+			addSql += GetAddSql("0", edbCode, eDate, timeStr, sv.Value)
+			isAdd = true
+		}
+		existMap[eDate] = sv.Value
+	}
+	if isAdd {
+		addSql = strings.TrimRight(addSql, ",")
+		utils.FileLog.Info("addSql:" + addSql)
+		_, err = o.Raw(addSql).Exec()
+		if err != nil {
+			return err
+		}
+	}
+	return
+}
+
+// Refresh 刷新汾渭数据指标数据
+func (obj BaseFromFenwei) Refresh(edbInfoId int, edbCode, startDate string) (err error) {
+	source := obj.GetSource()
+	o := orm.NewOrm()
+	if err != nil {
+		return
+	}
+	edbInfoIdStr := strconv.Itoa(edbInfoId)
+	//计算数据
+	var condition string
+	var pars []interface{}
+
+	if edbCode != "" {
+		condition += " AND index_code=? "
+		pars = append(pars, edbCode)
+	}
+
+	if startDate != "" {
+		condition += " AND data_time>=? "
+		pars = append(pars, startDate)
+	}
+
+	fenweiDataList, err := GetBaseFromFenweiDataByCondition(condition, pars)
+	if err != nil {
+		return
+	}
+
+	// 真实数据的最大日期  ， 插入规则配置的日期
+	var realDataMaxDate, edbDataInsertConfigDate time.Time
+	var edbDataInsertConfig *EdbDataInsertConfig
+	var isFindConfigDateRealData bool //是否找到配置日期的实际数据的值
+	{
+		edbDataInsertConfig, err = GetEdbDataInsertConfigByEdbId(edbInfoId)
+		if err != nil && err.Error() != utils.ErrNoRow() {
+			return
+		}
+		if edbDataInsertConfig != nil {
+			edbDataInsertConfigDate = edbDataInsertConfig.Date
+		}
+	}
+
+	var existCondition string
+	var existPars []interface{}
+
+	existCondition += " AND edb_info_id=? "
+	existPars = append(existPars, edbInfoId)
+	if startDate != "" {
+		existCondition += " AND data_time>=? "
+		existPars = append(existPars, startDate)
+	}
+
+	existList, err := GetEdbDataByCondition(source, 0, existCondition, existPars)
+	if err != nil {
+		return err
+	}
+	existMap := make(map[string]*EdbInfoSearchData)
+	for _, v := range existList {
+		existMap[v.DataTime] = v
+	}
+	addSql := ` INSERT INTO edb_data_fenwei(edb_info_id,edb_code,data_time,value,create_time,modify_time,data_timestamp) values `
+	var isAdd bool
+	for _, v := range fenweiDataList {
+		item := v
+		eDate := item.DataTime
+		dataTime, err := time.ParseInLocation(utils.FormatDate, eDate, time.Local)
+		if err != nil {
+			return err
+		}
+		if findItem, ok := existMap[v.DataTime]; !ok {
+			sValue := item.Value
+
+			timestamp := dataTime.UnixNano() / 1e6
+			timeStr := fmt.Sprintf("%d", timestamp)
+
+			addSql += GetAddSql(edbInfoIdStr, edbCode, eDate, timeStr, sValue)
+			isAdd = true
+		} else {
+			if findItem != nil && utils.SubFloatToString(findItem.Value, 30) != item.Value {
+				err = ModifyEdbDataById(source, 0, findItem.EdbDataId, item.Value)
+				if err != nil {
+					return err
+				}
+			}
+		}
+
+		// 下面代码主要目的是处理掉手动插入的数据判断
+		{
+			if realDataMaxDate.IsZero() || dataTime.After(realDataMaxDate) {
+				realDataMaxDate = dataTime
+			}
+			if edbDataInsertConfigDate.IsZero() || dataTime.Equal(edbDataInsertConfigDate) {
+				isFindConfigDateRealData = true
+			}
+		}
+	}
+
+	// 处理手工数据补充的配置
+	HandleConfigInsertEdbData(realDataMaxDate, edbDataInsertConfig, edbInfoId, source, 0, existMap, isFindConfigDateRealData)
+
+	if isAdd {
+		addSql = strings.TrimRight(addSql, ",")
+		_, err = o.Raw(addSql).Exec()
+		if err != nil {
+			fmt.Println("RefreshEdbDataFromBaiinfo add Err", err.Error())
+			return
+		}
+	}
+	return
+}
+
+// GetSource 获取来源编码id
+func (obj BaseFromFenwei) GetSource() int {
+	return utils.DATA_SOURCE_FENWEI
+}
+
+// GetSourceName 获取来源名称
+func (obj BaseFromFenwei) GetSourceName() string {
+	return utils.DATA_SOURCE_NAME_FENWEI
+}
+
+type BaseFromFenweiIndex struct {
+	FenweiIndexId int64 `orm:"column(fenwei_index_id);pk"`
+	IndexCode     string
+	IndexName     string
+	Frequency     string
+	Unit          string
+	ClassifyId    int
+	StartDate     string
+	EndDate       string
+	Sort          int
+	TerminalCode  string
+	CreateTime    time.Time
+	ModifyTime    time.Time
+}
+
+func (m *BaseFromFenweiIndex) TableName() string {
+	return "base_from_fenwei_index"
+}
+
+func (m *BaseFromFenweiIndex) Add() (lastId int64, err error) {
+	o := orm.NewOrm()
+	lastId, err = o.Insert(m)
+	return
+}
+
+func (m *BaseFromFenweiIndex) Update(updateCols []string) (err error) {
+	o := orm.NewOrm()
+	_, err = o.Update(m, updateCols...)
+
+	return
+}
+
+func (m *BaseFromFenweiIndex) ModifyIndexMaxAndMinDate(indexCode string, item *EdbInfoMaxAndMinInfo) (err error) {
+	o := orm.NewOrm()
+	sql := ` UPDATE base_from_fenwei_index SET start_date=?,end_date=?,modify_time=NOW() WHERE index_code=? `
+	_, err = o.Raw(sql, item.MinDate, item.MaxDate, indexCode).Exec()
+	return
+}
+
+func (m *BaseFromFenweiIndex) GetByIndexCode(indexCode string) (item *BaseFromFenweiIndex, err error) {
+	o := orm.NewOrm()
+	sql := ` SELECT * FROM base_from_fenwei_index WHERE index_code=? `
+	err = o.Raw(sql, indexCode).QueryRow(&item)
+	return
+}
+
+func (m *BaseFromFenweiIndex) GetItemsByCondition(condition string, pars []interface{}, fieldArr []string, orderRule string) (items []*BaseFromFenweiIndex, err error) {
+	o := orm.NewOrm()
+	fields := strings.Join(fieldArr, ",")
+	if len(fieldArr) == 0 {
+		fields = `*`
+	}
+	order := `ORDER BY create_time DESC`
+	if orderRule != "" {
+		order = ` ORDER BY ` + orderRule
+	}
+	sql := fmt.Sprintf(`SELECT %s FROM base_from_fenwei_index WHERE 1=1 %s %s`, fields, condition, order)
+	_, err = o.Raw(sql, pars).QueryRows(&items)
+	return
+}
+
+type BaseFromFenweiIndexList struct {
+	FenweiIndexId int64 `orm:"column(fenwei_index_id);pk"`
+	IndexCode     string
+	IndexName     string
+	Frequency     string
+	Unit          string
+	ClassifyId    int
+	StartDate     string
+	EndDate       string
+	TerminalCode  string
+	CreateTime    string
+	ModifyTime    string
+}
+
+func (y *BaseFromFenweiData) GetByIndexCode(indexCode string) (list []*BaseFromFenweiData, err error) {
+	o := orm.NewOrm()
+	sql := ` SELECT * FROM base_from_fenwei_data WHERE index_code=? `
+	_, err = o.Raw(sql, indexCode).QueryRows(&list)
+	return
+}
+
+func (y *BaseFromFenweiData) AddMulti(item []*BaseFromFenweiData) (err error) {
+	o := orm.NewOrm()
+	_, err = o.InsertMulti(1, item)
+	return
+}
+
+// HandleFenweiExcelData 汾渭数据的excel数据
+type HandleFenweiExcelData struct {
+	//ClassifyName string `description:"指标目录"`
+	//ClassifySort int    `description:"指标目录排序号"`
+	IndexName    string `description:"指标名称"`
+	IndexCode    string `description:"指标编码"`
+	Unit         string `description:"单位"`
+	Sort         int    `description:"排序号"`
+	Frequency    string `description:"频度"`
+	ClassifyId   int    `description:"分类ID"`
+	ExcelDataMap map[string]string
+}
+
+type HandleFenweiExcelDataReq struct {
+	List         []*HandleFenweiExcelData
+	TerminalCode string `description:"编码"`
+}
+
+func (y *BaseFromFenweiData) GetMaxAndMinDateByIndexCode(indexCode string) (item *EdbInfoMaxAndMinInfo, err error) {
+	o := orm.NewOrm()
+	sql := ` SELECT MIN(data_time) AS min_date,MAX(data_time) AS max_date,MIN(value) AS min_value,MAX(value) AS max_value FROM base_from_fenwei_data WHERE index_code=? `
+	err = o.Raw(sql, indexCode).QueryRow(&item)
+	return
+}
+
+// BaseFromFenweiClassify 汾渭数据分类表
+type BaseFromFenweiClassify struct {
+	ClassifyId      int       `orm:"column(classify_id);pk"`
+	ClassifyName    string    `description:"分类名称"`
+	ParentId        int       `description:"父级id"`
+	SysUserId       int       `description:"创建人id"`
+	SysUserRealName string    `description:"创建人姓名"`
+	Level           int       `description:"层级"`
+	Sort            int       `description:"排序字段，越小越靠前，默认值：10"`
+	ModifyTime      time.Time `description:"修改时间"`
+	CreateTime      time.Time `description:"创建时间"`
+}
+
+func (y *BaseFromFenweiClassify) Add() (lastId int64, err error) {
+	o := orm.NewOrm()
+	lastId, err = o.Insert(y)
+	return
+}
+
+func (y *BaseFromFenweiClassify) Update(updateCols []string) (err error) {
+	o := orm.NewOrm()
+	_, err = o.Update(y, updateCols...)
+
+	return
+}
+
+func (y *BaseFromFenweiClassify) GetByClassifyName(classifyName string) (item *BaseFromFenweiClassify, err error) {
+	o := orm.NewOrm()
+	sql := ` SELECT * FROM base_from_fenwei_classify WHERE classify_name=? `
+	err = o.Raw(sql, classifyName).QueryRow(&item)
+	return
+}
+
+func (y *BaseFromFenweiClassify) GetItemsByCondition(condition string, pars []interface{}, fieldArr []string, orderRule string) (items []*BaseFromFenweiClassify, err error) {
+	o := orm.NewOrm()
+	fields := strings.Join(fieldArr, ",")
+	if len(fieldArr) == 0 {
+		fields = `*`
+	}
+	order := `ORDER BY create_time DESC`
+	if orderRule != "" {
+		order = ` ORDER BY ` + orderRule
+	}
+	sql := fmt.Sprintf(`SELECT %s FROM base_from_fenwei_classify WHERE 1=1 %s %s`, fields, condition, order)
+	_, err = o.Raw(sql, pars).QueryRows(&items)
+	return
+}
+
+// BaseFromFenweiClassifyItem 汾渭数据分类信息
+type BaseFromFenweiClassifyItem struct {
+	ClassifyId   int                           `description:"分类ID"`
+	ClassifyName string                        `description:"分类名称"`
+	ParentId     int                           `description:"父级id"`
+	Level        int                           `description:"层级"`
+	Sort         int                           `description:"排序字段"`
+	CreateTime   string                        `description:"创建时间"`
+	ModifyTime   string                        `description:"修改时间"`
+	Child        []*BaseFromFenweiClassifyItem `description:"子分类"`
+}
+
+func (y *BaseFromFenweiClassify) Format2Item(origin *BaseFromFenweiClassify) (item *BaseFromFenweiClassifyItem) {
+	if origin == nil {
+		return
+	}
+	item = new(BaseFromFenweiClassifyItem)
+	item.ClassifyId = origin.ClassifyId
+	item.ClassifyName = origin.ClassifyName
+	item.ParentId = origin.ParentId
+	item.Level = origin.Level
+	item.Sort = origin.Sort
+	item.CreateTime = utils.TimeTransferString(utils.FormatDateTime, origin.CreateTime)
+	item.ModifyTime = utils.TimeTransferString(utils.FormatDateTime, origin.ModifyTime)
+	return
+}
+
+// MultiUpdateBaseFromFenweiDataValue 批量更新汾渭指标数据
+func MultiUpdateBaseFromFenweiDataValue(items []*BaseFromFenweiData) (err error) {
+	if len(items) == 0 {
+		return
+	}
+
+	o := orm.NewOrm()
+	sql := `UPDATE base_from_fenwei_data SET value = ?, modify_time = NOW() WHERE index_code = ? AND data_time = ? LIMIT 1`
+	p, err := o.Raw(sql).Prepare()
+	if err != nil {
+		return
+	}
+	defer func() {
+		_ = p.Close()
+	}()
+	for _, v := range items {
+		if v.IndexCode == "" || v.DataTime == "" {
+			continue
+		}
+		_, err = p.Exec(v.Value, v.IndexCode, v.DataTime)
+		if err != nil {
+			return
+		}
+	}
+	return
+}

models/base_from_gz.go

@@ -0,0 +1,291 @@
+package models
+
+import (
+	"eta/eta_index_lib/utils"
+	"fmt"
+	"github.com/beego/beego/v2/client/orm"
+	"strconv"
+	"strings"
+	"time"
+)
+
+type BaseFromTradeGuangzhouIndex struct {
+	BaseFromTradeGuangzhouIndexId    int       `orm:"column(base_from_trade_guangzhou_index_id);pk"`
+	BaseFromTradeGuangzhouClassifyId int       `description:"分类id"`
+	IndexCode                        string    `description:"指标编码"`
+	IndexName                        string    `description:"指标名称"`
+	Frequency                        string    `description:"频率"`
+	Unit                             string    `description:"单位"`
+	StartDate                        string    `description:"开始日期"`
+	EndDate                          string    `description:"结束日期"`
+	CreateTime                       time.Time `description:"创建日期"`
+	ModifyTime                       time.Time `description:"修改日期"`
+}
+
+type BaseFromTradeGuangzhouData struct {
+	BaseFromTradeGuangzhouDataId  int       `orm:"column(base_from_trade_guangzhou_data_id);pk"`
+	BaseFromTradeGuangzhouIndexId int       `description:"指标id"`
+	IndexCode                     string    `description:"指标编码"`
+	DataTime                      string    `description:"数据日期"`
+	Value                         string    `description:"数据值"`
+	QtySub                        float64   `description:"增减"`
+	CreateTime                    time.Time `description:"创建日期"`
+	ModifyTime                    time.Time `description:"修改日期"`
+}
+
+func GetBaseFromGuangzhouDataByIndexCode(indexCode string) (items []*BaseFromTradeGuangzhouData, err error) {
+	o := orm.NewOrm()
+	sql := `SELECT * FROM base_from_trade_guangzhou_data WHERE index_code=? `
+	_, err = o.Raw(sql, indexCode).QueryRows(&items)
+	return
+}
+
+func GetBaseFromGuangzhouDataDataByCondition(condition string, pars []interface{}) (item []*BaseFromTradeGuangzhouData, err error) {
+	sql := ` SELECT * FROM base_from_trade_guangzhou_data WHERE 1=1 `
+	o := orm.NewOrm()
+	if condition != "" {
+		sql += condition
+	}
+	sql += ` ORDER BY data_time DESC `
+	_, err = o.Raw(sql, pars).QueryRows(&item)
+	return
+}
+
+// 新增广期所指标数据
+func AddEdbDataFromGz(edbCode string) (err error) {
+	o := orm.NewOrm()
+
+	dataAll, err := GetBaseFromGuangzhouDataByIndexCode(edbCode)
+	if err != nil && err.Error() != utils.ErrNoRow() {
+		return
+	}
+
+	var isAdd bool
+	addSql := ` INSERT INTO edb_data_gz(edb_info_id,edb_code,data_time,value,create_time,modify_time,data_timestamp) values `
+	existMap := make(map[string]string)
+
+	for _, sv := range dataAll {
+		eDate := sv.DataTime
+		dataTime, err := time.ParseInLocation(utils.FormatDate, eDate, time.Local)
+		if err != nil {
+			fmt.Println("time.Parse Err:" + eDate)
+			return err
+		}
+		timestamp := dataTime.UnixNano() / 1e6
+		timeStr := fmt.Sprintf("%d", timestamp)
+		if _, ok := existMap[eDate]; !ok {
+			addSql += GetAddSql("0", edbCode, eDate, timeStr, sv.Value)
+			isAdd = true
+		}
+	}
+	if isAdd {
+		addSql = strings.TrimRight(addSql, ",")
+		utils.FileLog.Info("addSql:" + addSql)
+		_, err = o.Raw(addSql).Exec()
+		if err != nil {
+			return err
+		}
+	}
+	return
+}
+
+// 刷新广期所指标数据
+func RefreshEdbDataFromGz(edbInfoId int, edbCode, startDate string) (err error) {
+	source := utils.DATA_SOURCE_GFEX
+	subSource := utils.DATA_SUB_SOURCE_EDB
+
+	o := orm.NewOrm()
+	if err != nil {
+		return
+	}
+	edbInfoIdStr := strconv.Itoa(edbInfoId)
+	//计算数据
+	var condition string
+	var pars []interface{}
+
+	if edbCode != "" {
+		condition += " AND index_code=? "
+		pars = append(pars, edbCode)
+	}
+
+	if startDate != "" {
+		condition += " AND data_time>=? "
+		pars = append(pars, startDate)
+	}
+
+	dataList, err := GetBaseFromGuangzhouDataDataByCondition(condition, pars)
+	if err != nil {
+		return
+	}
+
+	// 真实数据的最大日期  ， 插入规则配置的日期
+	var realDataMaxDate, edbDataInsertConfigDate time.Time
+	var edbDataInsertConfig *EdbDataInsertConfig
+	var isFindConfigDateRealData bool //是否找到配置日期的实际数据的值
+	{
+		edbDataInsertConfig, err = GetEdbDataInsertConfigByEdbId(edbInfoId)
+		if err != nil && err.Error() != utils.ErrNoRow() {
+			return
+		}
+		if edbDataInsertConfig != nil {
+			edbDataInsertConfigDate = edbDataInsertConfig.Date
+		}
+	}
+
+	//获取指标所有数据
+	var existCondition string
+	var existPars []interface{}
+
+	existCondition += " AND edb_info_id=? "
+	existPars = append(existPars, edbInfoId)
+	if startDate != "" {
+		existCondition += " AND data_time>=? "
+		existPars = append(existPars, startDate)
+	}
+
+	existList, err := GetEdbDataByCondition(source, subSource, existCondition, existPars)
+	if err != nil {
+		return err
+	}
+	existMap := make(map[string]*EdbInfoSearchData)
+	for _, v := range existList {
+		existMap[v.DataTime] = v
+	}
+
+	addSql := ` INSERT INTO edb_data_gz(edb_info_id,edb_code,data_time,value,create_time,modify_time,data_timestamp) values `
+	var isAdd bool
+	for _, v := range dataList {
+		item := v
+		itemValue := v.Value
+		eDate := item.DataTime
+		dataTime, err := time.ParseInLocation(utils.FormatDate, eDate, time.Local)
+		if err != nil {
+			return err
+		}
+		if _, ok := existMap[v.DataTime]; !ok {
+			sValue := itemValue
+			if sValue != "" {
+				timestamp := dataTime.UnixNano() / 1e6
+				timeStr := fmt.Sprintf("%d", timestamp)
+				saveValue := sValue
+
+				if findItem, ok := existMap[eDate]; !ok {
+					addSql += GetAddSql(edbInfoIdStr, edbCode, eDate, timeStr, saveValue)
+					isAdd = true
+				} else {
+					if findItem != nil && utils.SubFloatToString(findItem.Value, 30) != sValue {
+						err = ModifyEdbDataById(source, subSource, findItem.EdbDataId, sValue)
+						if err != nil {
+							return err
+						}
+					}
+				}
+			}
+		}
+
+		// 下面代码主要目的是处理掉手动插入的数据判断
+		{
+			if realDataMaxDate.IsZero() || dataTime.After(realDataMaxDate) {
+				realDataMaxDate = dataTime
+			}
+			if edbDataInsertConfigDate.IsZero() || dataTime.Equal(edbDataInsertConfigDate) {
+				isFindConfigDateRealData = true
+			}
+		}
+	}
+
+	// 处理手工数据补充的配置
+	HandleConfigInsertEdbData(realDataMaxDate, edbDataInsertConfig, edbInfoId, source, subSource, existMap, isFindConfigDateRealData)
+
+	if isAdd {
+		addSql = strings.TrimRight(addSql, ",")
+		_, err = o.Raw(addSql).Exec()
+		if err != nil {
+			return err
+		}
+	}
+	return
+}
+
+//
+//type RefreshSHExchangeReq struct {
+//	Url      string `description:"交易所链接"`
+//	Exchange string `description:"交易所"`
+//	Date     string `description:"日期"`
+//	Data     SHMessage
+//}
+//
+//type Position []struct {
+//	ContractCode     string      `json:"INSTRUMENTID"`
+//	ProductSortNo    int         `json:"PRODUCTSORTNO"`
+//	Rank             int         `json:"RANK"`
+//	ParticipantID1   string      `json:"PARTICIPANTID1"`
+//	ParticipantName1 string      `json:"PARTICIPANTABBR1"`
+//	Deal             interface{} `json:"CJ1"`
+//	Change1          interface{} `json:"CJ1_CHG"`
+//	ParticipantID2   string      `json:"PARTICIPANTID2"`
+//	ParticipantName2 string      `json:"PARTICIPANTABBR2"`
+//	BuyIn            interface{} `json:"CJ2"`
+//	Change2          interface{} `json:"CJ2_CHG"`
+//	ParticipantID3   string      `json:"PARTICIPANTID3"`
+//	ParticipantName3 string      `json:"PARTICIPANTABBR3"`
+//	SoldOut          interface{} `json:"CJ3"`
+//	Change3          interface{} `json:"CJ3_CHG"`
+//	ProductName      string      `json:"PRODUCTNAME"`
+//}
+//
+//type SHMessage struct {
+//	Position   Position `json:"o_cursor"`
+//	Length     string   `json:"showlength"`
+//	Code       int      `json:"o_code"`
+//	Msg        string   `json:"o_msg"`
+//	ReportDate string   `json:"report_date"`
+//	UpdateDate string   `json:"update_date"`
+//	PrintDate  string   `json:"print_date"`
+//}
+//
+//type BaseFromTradeShanghaiIndex struct {
+//	BaseFromTradeShangHaiIndexId int `orm:"column(base_from_trade_shanghai_index_id);pk"`
+//	Rank                         int
+//	DealShortName                string
+//	DealName                     string
+//	DealCode                     string
+//	DealValue                    int
+//	DealChange                   int
+//	BuyShortName                 string
+//	BuyName                      string
+//	BuyCode                      string
+//	BuyValue                     int
+//	BuyChange                    int
+//	SoldShortName                string
+//	SoldName                     string
+//	SoldCode                     string
+//	SoldValue                    int
+//	SoldChange                   int
+//	Frequency                    string
+//	ClassifyName                 string
+//	ClassifyType                 string
+//	CreateTime                   time.Time
+//	ModifyTime                   time.Time
+//	DataTime                     string
+//}
+//
+//func AddBaseFromTradeShangHaiIndex(item *BaseFromTradeShanghaiIndex) (lastId int64, err error) {
+//	o := orm.NewOrm()
+//	lastId, err = o.Insert(item)
+//	return
+//}
+//
+//func GetBaseFromTradeShangHaiIndexAll(dateStr string) (list []*BaseFromTradeShanghaiIndex, err error) {
+//	o := orm.NewOrm()
+//	sql := `SELECT * FROM base_from_trade_shanghai_index WHERE data_time=?`
+//	_, err = o.Raw(sql, dateStr).QueryRows(&list)
+//	return
+//}
+//
+//func ModifyBaseFromTradeShangHaiIndex(dealValue, buyValue, soldValue int, dataId int) (err error) {
+//	o := orm.NewOrm()
+//	sql := `UPDATE base_from_trade_shanghai_index SET deal_value=?,buy_value=?,sold_value=?,modify_time=NOW() WHERE base_from_trade_shanghai_index_id=? `
+//	_, err = o.Raw(sql, dealValue, buyValue, soldValue, dataId).Exec()
+//	return
+//}

models/base_from_icpi.go

@@ -0,0 +1,214 @@
+package models
+
+import (
+	"eta/eta_index_lib/utils"
+	"fmt"
+	"github.com/beego/beego/v2/client/orm"
+	"strconv"
+	"strings"
+	"time"
+)
+
+type BaseFromIcpiIndex struct {
+	BaseFromIcpiIndexId    int       `orm:"column(base_from_icpi_index_id);pk"`
+	BaseFromIcpiClassifyId int       `description:"分类id"`
+	IndexCode              string    `description:"指标编码"`
+	IndexName              string    `description:"指标名称"`
+	Frequency              string    `description:"频度"`
+	StartDate              time.Time `description:"开始日期"`
+	EndDate                time.Time `description:"结束日期"`
+	CreateTime             time.Time `description:"创建时间"`
+	ModifyTime             time.Time `description:"修改时间"`
+}
+
+type BaseFromIcpiData struct {
+	BaseFromIcpiDataId  int       `orm:"column(base_from_icpi_data_id);pk"`
+	BaseFromIcpiIndexId int       `description:"指标id"`
+	IndexCode           string    `description:"指标编码"`
+	DataTime            string    `description:"日期"`
+	Value               string    `description:"值"`
+	CreateTime          time.Time `description:"创建时间"`
+	ModifyTime          time.Time `description:"修改时间"`
+}
+
+type BaseFromIcpiClassify struct {
+	BaseFromIcpiClassifyId int       `orm:"column(base_from_icpi_classify_id);pk"`
+	ClassifyName           string    `description:"分类名称"`
+	ClassifyNameEn         string    `description:"英文名称"`
+	ParentId               int       `description:"上级id"`
+	CreateTime             time.Time `description:"创建时间"`
+	ModifyTime             time.Time `description:"修改时间"`
+}
+
+func GetBaseFromIcpiDataByIndexCode(indexCode string) (items []*BaseFromIcpiData, err error) {
+	o := orm.NewOrm()
+	sql := `SELECT * FROM base_from_icpi_data WHERE index_code=? `
+	_, err = o.Raw(sql, indexCode).QueryRows(&items)
+	return
+}
+
+func GetBaseFromIcpiDataDataByCondition(condition string, pars []interface{}) (item []*BaseFromIcpiData, err error) {
+	sql := ` SELECT * FROM base_from_icpi_data WHERE 1=1 `
+	o := orm.NewOrm()
+	if condition != "" {
+		sql += condition
+	}
+	sql += ` ORDER BY data_time DESC `
+	_, err = o.Raw(sql, pars).QueryRows(&item)
+	return
+}
+
+// 新增广期所指标数据
+func AddEdbDataFromIcpi(edbCode string) (err error) {
+	o := orm.NewOrm()
+	dataAll, err := GetBaseFromIcpiDataByIndexCode(edbCode)
+	if err != nil && err.Error() != utils.ErrNoRow() {
+		return
+	}
+
+	var isAdd bool
+	addSql := ` INSERT INTO edb_data_icpi(edb_info_id,edb_code,data_time,value,create_time,modify_time,data_timestamp) values `
+	existMap := make(map[string]string)
+
+	for _, sv := range dataAll {
+		eDate := sv.DataTime
+		dataTime, err := time.ParseInLocation(utils.FormatDate, eDate, time.Local)
+		if err != nil {
+			fmt.Println("time.Parse Err:" + eDate)
+			return err
+		}
+		timestamp := dataTime.UnixNano() / 1e6
+		timeStr := fmt.Sprintf("%d", timestamp)
+		if _, ok := existMap[eDate]; !ok {
+			addSql += GetAddSql("0", edbCode, eDate, timeStr, sv.Value)
+			isAdd = true
+		}
+	}
+	if isAdd {
+		addSql = strings.TrimRight(addSql, ",")
+		utils.FileLog.Info("addSql:" + addSql)
+		_, err = o.Raw(addSql).Exec()
+		if err != nil {
+			return err
+		}
+	}
+	return
+}
+
+// 刷新广期所指标数据
+func RefreshEdbDataFromIcpi(edbInfoId int, edbCode, startDate string) (err error) {
+	source := utils.DATA_SOURCE_ICPI
+	subSource := utils.DATA_SUB_SOURCE_EDB
+
+	o := orm.NewOrm()
+	if err != nil {
+		return
+	}
+	edbInfoIdStr := strconv.Itoa(edbInfoId)
+	//计算数据
+	var condition string
+	var pars []interface{}
+
+	if edbCode != "" {
+		condition += " AND index_code=? "
+		pars = append(pars, edbCode)
+	}
+
+	if startDate != "" {
+		condition += " AND data_time>=? "
+		pars = append(pars, startDate)
+	}
+
+	dataList, err := GetBaseFromIcpiDataDataByCondition(condition, pars)
+	if err != nil {
+		return
+	}
+
+	// 真实数据的最大日期  ， 插入规则配置的日期
+	var realDataMaxDate, edbDataInsertConfigDate time.Time
+	var edbDataInsertConfig *EdbDataInsertConfig
+	var isFindConfigDateRealData bool //是否找到配置日期的实际数据的值
+	{
+		edbDataInsertConfig, err = GetEdbDataInsertConfigByEdbId(edbInfoId)
+		if err != nil && err.Error() != utils.ErrNoRow() {
+			return
+		}
+		if edbDataInsertConfig != nil {
+			edbDataInsertConfigDate = edbDataInsertConfig.Date
+		}
+	}
+
+	//获取指标所有数据
+	var existCondition string
+	var existPars []interface{}
+
+	existCondition += " AND edb_info_id=? "
+	existPars = append(existPars, edbInfoId)
+	if startDate != "" {
+		existCondition += " AND data_time>=? "
+		existPars = append(existPars, startDate)
+	}
+
+	existList, err := GetEdbDataByCondition(source, subSource, existCondition, existPars)
+	if err != nil {
+		return err
+	}
+	existMap := make(map[string]*EdbInfoSearchData)
+	for _, v := range existList {
+		existMap[v.DataTime] = v
+	}
+
+	addSql := ` INSERT INTO edb_data_icpi(edb_info_id,edb_code,data_time,value,create_time,modify_time,data_timestamp) values `
+	var isAdd bool
+	for _, v := range dataList {
+		item := v
+		itemValue := v.Value
+		eDate := item.DataTime
+		dataTime, err := time.ParseInLocation(utils.FormatDate, eDate, time.Local)
+		if err != nil {
+			return err
+		}
+		if _, ok := existMap[v.DataTime]; !ok {
+			sValue := itemValue
+			if sValue != "" {
+				timestamp := dataTime.UnixNano() / 1e6
+				timeStr := fmt.Sprintf("%d", timestamp)
+				saveValue := sValue
+
+				if findItem, ok := existMap[eDate]; !ok {
+					addSql += GetAddSql(edbInfoIdStr, edbCode, eDate, timeStr, saveValue)
+					isAdd = true
+				} else {
+					if findItem != nil && utils.SubFloatToString(findItem.Value, 30) != sValue {
+						err = ModifyEdbDataById(source, subSource, findItem.EdbDataId, sValue)
+						if err != nil {
+							return err
+						}
+					}
+				}
+			}
+		}
+
+		// 下面代码主要目的是处理掉手动插入的数据判断
+		{
+			if realDataMaxDate.IsZero() || dataTime.After(realDataMaxDate) {
+				realDataMaxDate = dataTime
+			}
+			if edbDataInsertConfigDate.IsZero() || dataTime.Equal(edbDataInsertConfigDate) {
+				isFindConfigDateRealData = true
+			}
+		}
+	}
+
+	// 处理手工数据补充的配置
+	HandleConfigInsertEdbData(realDataMaxDate, edbDataInsertConfig, edbInfoId, source, subSource, existMap, isFindConfigDateRealData)
+
+	if isAdd {
+		addSql = strings.TrimRight(addSql, ",")
+		_, err = o.Raw(addSql).Exec()
+		if err != nil {
+			return err
+		}
+	}
+	return
+}

models/base_from_mtjh.go

@@ -0,0 +1,283 @@
+package models
+
+import (
+	"eta/eta_index_lib/utils"
+	"fmt"
+	"github.com/beego/beego/v2/client/orm"
+	"strconv"
+	"strings"
+	"time"
+)
+
+type BaseFromMtjhMapping struct {
+	BaseFromMtjhMappingId int       `orm:"column(base_from_Mtjh_mapping_id);pk"`
+	IndexName             string    `description:"持买单量指标名称"`
+	IndexCode             string    `description:"持买单量指标编码"`
+	CreateTime            time.Time `description:"时间"`
+	Area                  string    `description:"区域"`
+	Port                  string    `description:"港口或码头"`
+	Variety               string    `description:"品种"`
+	Unit                  string    `description:"单位"`
+	Frequency             string    `description:"频率"`
+}
+
+type BaseFromMtjhIndex struct {
+	BaseFromMtjhIndexId int       `orm:"column(base_from_mtjh_index_id);pk"`
+	IndexName           string    `description:"持买单量指标名称"`
+	IndexCode           string    `description:"持买单量指标编码"`
+	DealValue           string    `description:"成交量"`
+	DataTime            string    `description:"数据日期"`
+	Area                string    `description:"区域"`
+	Port                string    `description:"港口或码头"`
+	Variety             string    `description:"品种"`
+	Unit                string    `description:"单位"`
+	Frequency           string    `description:"频率"`
+	CreateTime          time.Time `description:"插入时间"`
+	ModifyTime          time.Time `description:"修改时间"`
+}
+
+// 查询指标
+func GetBaseFromMtjhMapping() (items []*BaseFromMtjhMapping, err error) {
+	o := orm.NewOrm()
+	sql := `SELECT * FROM base_from_mtjh_mapping`
+	_, err = o.Raw(sql).QueryRows(&items)
+	return
+}
+
+// 查询指标
+func GetBaseFromMtjhIndex() (items []*BaseFromMtjhIndex, err error) {
+	o := orm.NewOrm()
+	sql := `SELECT * FROM base_from_mtjh_index`
+	_, err = o.Raw(sql).QueryRows(&items)
+	return
+}
+
+// 添加数据
+func AddBaseFromMtjhIndex(item *BaseFromMtjhIndex) (lastId int64, err error) {
+	o := orm.NewOrm()
+	lastId, err = o.Insert(item)
+	return
+}
+
+func AddBaseFromMtjhIndexMuti(items []*BaseFromMtjhIndex) (lastId int64, err error) {
+	o := orm.NewOrm()
+	lastId, err = o.InsertMulti(500, items)
+	return
+}
+
+// 添加指标
+func AddBaseFromMtjhMapping(item *BaseFromMtjhMapping) (lastId int64, err error) {
+	o := orm.NewOrm()
+	lastId, err = o.Insert(item)
+	return
+}
+
+func AddBaseFromMtjhMappingMuti(items []*BaseFromMtjhMapping) (lastId int64, err error) {
+	o := orm.NewOrm()
+	lastId, err = o.InsertMulti(500, items)
+	return
+}
+
+func UpdateBaseFromMtjhIndex(item *BaseFromMtjhIndex) (err error) {
+	o := orm.NewOrm()
+	sql := `UPDATE base_from_mtjh_index SET deal_value=?  WHERE index_name=?  AND  data_time = ?`
+	_, err = o.Raw(sql, item.DealValue, item.IndexName, item.DataTime).Exec()
+	return
+}
+
+func AddEdbDataFromMtjh(edbCode string) (err error) {
+	o := orm.NewOrm()
+
+	coalBaseDataAll, err := GetMtjhindexByCode(edbCode)
+	if err != nil && err.Error() != utils.ErrNoRow() {
+		return
+	}
+
+	var isAdd bool
+	addSql := ` INSERT INTO edb_data_mtjh(edb_info_id,edb_code,data_time,value,create_time,modify_time,data_timestamp) values `
+	existMap := make(map[string]string)
+	for _, sv := range coalBaseDataAll {
+		eDate := sv.DataTime
+		var timeStr string
+		var dataTime time.Time
+		var sDataTime string
+		var timestamp int64
+
+		sDataTime = eDate
+		dataTime, err = time.ParseInLocation(utils.FormatDate, eDate, time.Local)
+		if err != nil {
+			fmt.Println("time.Parse Err:" + eDate)
+			return err
+		}
+		timestamp = dataTime.UnixNano() / 1e6
+		timeStr = fmt.Sprintf("%d", timestamp)
+
+		value := strings.Replace(sv.DealValue, "%", "", -1)
+		if _, ok := existMap[sDataTime]; !ok {
+			addSql += GetAddSql("0", edbCode, sDataTime, timeStr, value)
+			fmt.Println("edbCode：", edbCode)
+			fmt.Println("sDataTime：", sDataTime)
+			fmt.Println("timeStr：", timeStr)
+			fmt.Println("value：", value)
+			isAdd = true
+		}
+		existMap[eDate] = value
+	}
+	if isAdd {
+		addSql = strings.TrimRight(addSql, ",")
+		utils.FileLog.Info("addSql:" + addSql)
+		_, err = o.Raw(addSql).Exec()
+		if err != nil {
+			return err
+		}
+	}
+	return
+}
+
+func RefreshEdbDataFromMtjh(edbInfoId int, edbCode, startDate string) (err error) {
+	source := utils.DATA_SOURCE_MTJH
+	subSource := utils.DATA_SUB_SOURCE_EDB
+	o := orm.NewOrm()
+	if err != nil {
+		return
+	}
+
+	edbInfoIdStr := strconv.Itoa(edbInfoId)
+	//计算数据
+	var condition string
+	var pars []interface{}
+
+	if edbCode != "" {
+		condition += " AND index_code=? "
+		pars = append(pars, edbCode)
+	}
+
+	if startDate != "" {
+		condition += " AND data_time>=? "
+		pars = append(pars, startDate)
+	}
+
+	glDataList, err := GetMtjhDataByTradeCode(condition, pars)
+	if err != nil {
+		return
+	}
+
+	// 真实数据的最大日期  ， 插入规则配置的日期
+	var realDataMaxDate, edbDataInsertConfigDate time.Time
+	var edbDataInsertConfig *EdbDataInsertConfig
+	var isFindConfigDateRealData bool //是否找到配置日期的实际数据的值
+	{
+		edbDataInsertConfig, err = GetEdbDataInsertConfigByEdbId(edbInfoId)
+		if err != nil && err.Error() != utils.ErrNoRow() {
+			return
+		}
+		if edbDataInsertConfig != nil {
+			edbDataInsertConfigDate = edbDataInsertConfig.Date
+		}
+	}
+
+	//获取指标所有数据
+	var existCondition string
+	var existPars []interface{}
+
+	existCondition += " AND edb_info_id=? "
+	existPars = append(existPars, edbInfoId)
+	if startDate != "" {
+		existCondition += " AND data_time>=? "
+		existPars = append(existPars, startDate)
+	}
+
+	existList, err := GetEdbDataByCondition(source, subSource, existCondition, existPars)
+	if err != nil {
+		return err
+	}
+	existMap := make(map[string]*EdbInfoSearchData)
+	for _, v := range existList {
+		existMap[v.DataTime] = v
+	}
+
+	addSql := ` INSERT INTO edb_data_mtjh(edb_info_id,edb_code,data_time,value,create_time,modify_time,data_timestamp) values `
+	var isAdd bool
+	for _, v := range glDataList {
+		var value string
+		value = strings.Replace(v.DealValue, "%", "", -1)
+		item := v
+		itemValue := value
+		if _, ok := existMap[v.DataTime]; !ok {
+			eDate := item.DataTime
+			var timeStr string
+			var dataTime time.Time
+			var sDataTime string
+			var timestamp int64
+
+			sDataTime = eDate
+			dataTime, err = time.ParseInLocation(utils.FormatDate, eDate, time.Local)
+			if err != nil {
+				fmt.Println("time.Parse Err:" + eDate)
+				return err
+			}
+			timestamp = dataTime.UnixNano() / 1e6
+			timeStr = fmt.Sprintf("%d", timestamp)
+
+			sValue := itemValue
+			if sValue != "" {
+				saveValue := sValue
+
+				if findItem, ok := existMap[eDate]; !ok {
+					addSql += GetAddSql(edbInfoIdStr, edbCode, sDataTime, timeStr, saveValue)
+					isAdd = true
+				} else {
+					if findItem != nil && utils.SubFloatToString(findItem.Value, 30) != sValue {
+						err = ModifyEdbDataById(source, subSource, findItem.EdbDataId, sValue)
+						if err != nil {
+							return err
+						}
+					}
+				}
+			}
+
+			// 下面代码主要目的是处理掉手动插入的数据判断
+			{
+				if realDataMaxDate.IsZero() || dataTime.After(realDataMaxDate) {
+					realDataMaxDate = dataTime
+				}
+				if edbDataInsertConfigDate.IsZero() || dataTime.Equal(edbDataInsertConfigDate) {
+					isFindConfigDateRealData = true
+				}
+			}
+
+		}
+	}
+
+	// 处理手工数据补充的配置
+	HandleConfigInsertEdbData(realDataMaxDate, edbDataInsertConfig, edbInfoId, source, subSource, existMap, isFindConfigDateRealData)
+
+	if isAdd {
+		addSql = strings.TrimRight(addSql, ",")
+		_, err = o.Raw(addSql).Exec()
+		if err != nil {
+			return err
+		}
+	}
+
+	return
+}
+
+// GetMtjhindexByCode
+func GetMtjhindexByCode(indexCode string) (items []*BaseFromMtjhIndex, err error) {
+	o := orm.NewOrm()
+	sql := "SELECT * FROM base_from_mtjh_index WHERE index_code=? "
+	_, err = o.Raw(sql, indexCode).QueryRows(&items)
+	return
+}
+
+func GetMtjhDataByTradeCode(condition string, pars []interface{}) (item []*BaseFromMtjhIndex, err error) {
+	sql := ` SELECT * FROM base_from_mtjh_index WHERE 1=1 `
+	o := orm.NewOrm()
+	if condition != "" {
+		sql += condition
+	}
+	sql += ` ORDER BY data_time DESC `
+	_, err = o.Raw(sql, pars).QueryRows(&item)
+	return
+}

models/base_from_mysteel_chemical.go

@@ -503,13 +503,27 @@ func (d *BaseFromMysteelChemicalData) GetMysteelIndexInfoMaxAndMinInfo(indexCode
 	o := orm.NewOrm()
 	sql := ` SELECT MIN(data_time) AS min_date,MAX(data_time) AS max_date,MIN(value) AS min_value,MAX(value) AS max_value FROM base_from_mysteel_chemical_data WHERE index_code=? `
 	err = o.Raw(sql, indexCode).QueryRow(&item)
+
+	if err != nil {
+		return
+	}
+
+	// 获取最新值
+	var latest_value float64
+	sql = ` SELECT value AS latest_value FROM base_from_mysteel_chemical_data WHERE index_code=? ORDER BY data_time DESC LIMIT 1 `
+	err = o.Raw(sql, indexCode).QueryRow(&latest_value)
+	if err != nil {
+		return
+	}
+	item.LatestValue = latest_value
+
 	return
 }
 
 func (d *BaseFromMysteelChemicalData) ModifyMysteelIndexMaxAndMinInfo(indexCode string, item *EdbInfoMaxAndMinInfo) (err error) {
 	o := orm.NewOrm()
-	sql := ` UPDATE base_from_mysteel_chemical_index SET start_date=?,end_date=?,modify_time=NOW() WHERE index_code=? `
-	_, err = o.Raw(sql, item.MinDate, item.MaxDate, indexCode).Exec()
+	sql := ` UPDATE base_from_mysteel_chemical_index SET start_date=?,end_date=?,end_value=?,modify_time=NOW() WHERE index_code=? `
+	_, err = o.Raw(sql, item.MinDate, item.MaxDate, item.LatestValue, indexCode).Exec()
 	return
 }
 

models/base_from_smm.go

@@ -247,13 +247,26 @@ func GetSmmIndexInfoMaxAndMinInfo(indexCode string) (item *EdbInfoMaxAndMinInfo,
 	o := orm.NewOrm()
 	sql := ` SELECT MIN(data_time) AS min_date,MAX(data_time) AS max_date,MIN(value) AS min_value,MAX(value) AS max_value FROM base_from_smm_data WHERE index_code=? `
 	err = o.Raw(sql, indexCode).QueryRow(&item)
+	if err != nil {
+		return
+	}
+
+	// 获取最新值
+	var latest_value float64
+	sql = ` SELECT value AS latest_value FROM base_from_smm_data WHERE index_code=? ORDER BY data_time DESC LIMIT 1 `
+	err = o.Raw(sql, indexCode).QueryRow(&latest_value)
+	if err != nil {
+		return
+	}
+	item.LatestValue = latest_value
+
 	return
 }
 
 func ModifySmmIndexMaxAndMinInfo(indexCode string, item *EdbInfoMaxAndMinInfo) (err error) {
 	o := orm.NewOrm()
-	sql := ` UPDATE base_from_smm_index SET start_date=?,end_date=?,modify_time=NOW() WHERE index_code=? `
-	_, err = o.Raw(sql, item.MinDate, item.MaxDate, indexCode).Exec()
+	sql := ` UPDATE base_from_smm_index SET start_date=?,end_date=?,end_value=?,modify_time=NOW() WHERE index_code=? `
+	_, err = o.Raw(sql, item.MinDate, item.MaxDate, item.LatestValue, indexCode).Exec()
 	return
 }
 
@@ -410,7 +423,7 @@ func AddBaseFromSmmIndex(list []*BaseFromSmmIndex) (lastId int64, err error) {
 // BridgeZhongJiIndexLatestDataParams 桥接服务-获取中基更新指标数据入参
 type BridgeZhongJiIndexLatestDataParams struct {
 	EdbCode  string `json:"edb_code" form:"edb_code" description:"指标编码"`
-	LastTime int64 `json:"last_time" form:"last_time" description:"上次更新时间戳（秒）"`
+	LastTime int64  `json:"last_time" form:"last_time" description:"上次更新时间戳（秒）"`
 }
 
 func GetBaseFromSmmIndex() (list []*BaseFromSmmIndex, err error) {

models/db.go

@@ -2,6 +2,7 @@ package models
 
 import (
 	"eta/eta_index_lib/models/data_stat"
+	"eta/eta_index_lib/models/edb_refresh"
 	"eta/eta_index_lib/models/excel"
 	"eta/eta_index_lib/models/future_good"
 	"eta/eta_index_lib/models/supply_analysis"
@@ -69,9 +70,13 @@ func init() {
 
 	// initDataStat 数据源统计管理相关表
 	initDataStat()
+
 	// Eta表格相关
 	initExcel()
 
+	// 初始化指标刷新
+	initEdbRefresh()
+
 	// 初始化部分数据表变量(直接init会有顺序问题=_=!)
 	InitEdbSource()
 }
@@ -119,6 +124,11 @@ func initBaseIndex() {
 		new(BaseFromCoalmineCoastalIndex),
 		new(BaseFromCoalmineInlandIndex),
 		new(BaseFromCoalmineCompanyIndex),
+		new(BaseFromCoalmineFirmIndex),
+		new(BaseFromMtjhMapping),
+		new(BaseFromMtjhIndex),
+		new(BaseFromFenweiIndex),
+		new(BaseFromFenweiData),
 	)
 }
 
@@ -139,3 +149,13 @@ func initExcel() {
 		new(excel.ExcelEdbMapping), //excel与指标的关系表
 	)
 }
+
+// initEdbRefresh 初始化指标刷新
+func initEdbRefresh() {
+	orm.RegisterModel(
+		new(edb_refresh.EdbRefreshSource),        // 刷新的数据源表
+		new(edb_refresh.EdbRefreshDefaultConfig), // 指标的默认刷新时间配置表
+		new(edb_refresh.EdbRefreshConfig),        // 指标的刷新时间配置表
+		new(edb_refresh.EdbRefreshMapping),       // 指标刷新时间配置关系表
+	)
+}

models/edb_data_calculate_zjpj.go

@@ -508,11 +508,28 @@ func refreshAllCalculateZjpj(to orm.TxOrmer, edbInfo *EdbInfo, existItemA, exist
 
 	//数据入库
 	if len(addDataList) > 0 {
-		_, tmpErr := to.InsertMulti(len(addDataList), addDataList)
-		if tmpErr != nil {
-			err = tmpErr
-			return
+		tmpAddDataList := make([]*EdbDataCalculateZjpj, 0)
+		i := 0
+		for _, v := range addDataList {
+			tmpAddDataList = append(tmpAddDataList, v)
+			i++
+			if i >= 500 {
+				_, err = to.InsertMulti(len(tmpAddDataList), tmpAddDataList)
+				if err != nil {
+					return
+				}
+				i = 0
+				tmpAddDataList = make([]*EdbDataCalculateZjpj, 0)
+			}
+		}
+
+		if len(tmpAddDataList) > 0 {
+			_, err = to.InsertMulti(len(tmpAddDataList), tmpAddDataList)
+			if err != nil {
+				return
+			}
 		}
 	}
+
 	return
 }

models/edb_info.go

@@ -107,6 +107,14 @@ type EdbInfoList struct {
 	PredictDataList []*EdbData             `description:"预测指标数据"`
 	Button          EdbClassifyItemsButton `description:"操作权限"`
 	IsEnEdb         bool                   `description:"是否展示英文标识"`
+	DataDateType    string                 `description:"数据日期类型，枚举值：交易日、自然日"`
+	EmptyType       int                    `description:"空值处理类型（0查找前后35天，1不计算，2前值填充，3后值填充，4等于0）"`
+	MaxEmptyType    int                    `description:"MAX、MIN公式空值处理类型（1、等于0；2、跳过空值）"`
+	SubSource       int                    `description:"子数据来源:0:经济数据库,1:日期序列"`
+	SubSourceName   string                 `description:"子数据来源名称"`
+	IndicatorCode   string                 `description:"指标代码"`
+	StockCode       string                 `description:"证券代码"`
+	NoUpdate        int8                   `description:"是否停止更新，0：继续更新；1：停止更新"`
 }
 
 // EdbClassifyItemsButton 操作按钮

models/edb_refresh/edb_refresh_config.go

@@ -0,0 +1,89 @@
+package edb_refresh
+
+import (
+	"errors"
+	"github.com/beego/beego/v2/client/orm"
+	"time"
+)
+
+// EdbRefreshConfig
+// @Description: 指标的刷新时间配置表
+type EdbRefreshConfig struct {
+	EdbRefreshConfigId  int       `orm:"column(edb_refresh_config_id);pk"`
+	RefreshFrequency    string    `description:"刷新频率"`
+	RefreshFrequencyDay int       `description:"具体刷新的日期"`
+	RefreshTime         string    `description:"刷新时间"`
+	RefreshAllData      int       `description:"是否刷新所有数据，0：否，1：刷新所有数据"`
+	RefreshDataNum      int       `description:"刷新单元格数"`
+	ModifyTime          time.Time `description:"最晚一次的更新时间"`
+	CreateTime          time.Time `description:"添加时间"`
+}
+
+// Add
+// @Description: 添加
+// @author: Roc
+// @receiver m
+// @datetime 2023-12-14 16:11:10
+// @param cols []string
+// @return err error
+func (m *EdbRefreshConfig) Add() (err error) {
+	if m.EdbRefreshConfigId > 0 {
+		err = errors.New("该配置已存在")
+		return
+	}
+	o := orm.NewOrm()
+	lastId, err := o.Insert(m)
+	if err != nil {
+		return
+	}
+	m.EdbRefreshConfigId = int(lastId)
+
+	return
+}
+
+// Update
+// @Description: 更新
+// @author: Roc
+// @receiver m
+// @datetime 2023-12-14 16:11:10
+// @param cols []string
+// @return err error
+func (m *EdbRefreshConfig) Update(cols []string) (err error) {
+	o := orm.NewOrm()
+	_, err = o.Update(m, cols...)
+	return
+}
+
+// Delete
+// @Description: 删除
+// @author: Roc
+// @receiver m
+// @datetime 2023-12-14 16:11:10
+// @return err error
+func (m *EdbRefreshConfig) Delete() (err error) {
+	o := orm.NewOrm()
+	_, err = o.Delete(m)
+	return
+}
+
+// GetEdbRefreshConfigListByCondition
+// @Description: 根据条条件获取刷新配置列表
+// @author: Roc
+// @datetime 2024-01-09 13:28:49
+// @param condition string
+// @param pars []interface{}
+// @return list []*EdbRefreshDefaultConfig
+// @return err error
+func GetEdbRefreshConfigListByCondition(condition string, pars []interface{}) (list []*EdbRefreshConfig, err error) {
+	o := orm.NewOrm()
+	sql := `SELECT * FROM edb_refresh_config
+         WHERE 1 = 1 `
+
+	if condition != "" {
+		sql += condition
+	}
+	sql += ` ORDER BY edb_refresh_config_id ASC `
+	_, err = o.Raw(sql, pars).QueryRows(&list)
+
+	return
+}

models/edb_refresh/edb_refresh_default_config.go

@@ -0,0 +1,134 @@
+package edb_refresh
+
+import (
+	"errors"
+	"eta/eta_index_lib/utils"
+	"github.com/beego/beego/v2/client/orm"
+	"time"
+)
+
+// EdbRefreshDefaultConfig
+// @Description: 指标的默认刷新时间配置表
+type EdbRefreshDefaultConfig struct {
+	Id                  int       `orm:"column(id);pk"`
+	Source              int       `description:"来源"`
+	SubSource           int       `description:"来源名称"`
+	Frequency           string    `description:"频度"`
+	RefreshFrequency    string    `description:"刷新频率"`
+	RefreshFrequencyDay int       `description:"具体刷新的日期"`
+	RefreshTime         string    `description:"刷新时间"`
+	RefreshAllData      int       `description:"是否刷新所有数据，0：否，1：刷新所有数据"`
+	RefreshDataNum      int       `description:"刷新单元格数"`
+	ModifyTime          time.Time `description:"最晚一次的更新时间"`
+	CreateTime          time.Time `description:"添加时间"`
+}
+
+// Add
+// @Description: 添加
+// @author: Roc
+// @receiver m
+// @datetime 2023-12-14 16:11:10
+// @param cols []string
+// @return err error
+func (m *EdbRefreshDefaultConfig) Add() (err error) {
+	if m.Id > 0 {
+		err = errors.New("该配置已存在")
+		return
+	}
+	o := orm.NewOrm()
+	lastId, err := o.Insert(m)
+	if err != nil {
+		return
+	}
+	m.Id = int(lastId)
+
+	return
+}
+
+// Update
+// @Description: 更新
+// @author: Roc
+// @receiver m
+// @datetime 2023-12-14 16:11:10
+// @param cols []string
+// @return err error
+func (m *EdbRefreshDefaultConfig) Update(cols []string) (err error) {
+	o := orm.NewOrm()
+	_, err = o.Update(m, cols...)
+	return
+}
+
+// Delete
+// @Description: 删除
+// @author: Roc
+// @receiver m
+// @datetime 2023-12-14 16:11:10
+// @return err error
+func (m *EdbRefreshDefaultConfig) Delete() (err error) {
+	o := orm.NewOrm()
+	_, err = o.Delete(m)
+	return
+}
+
+// GetListBySourceAndFrequency
+// @Description: 根据来源和频度获取列表
+// @author: Roc
+// @datetime 2024-01-04 17:39:47
+// @param source int
+// @param subSource int
+// @param frequency string
+// @return list []*EdbRefreshDefaultConfig
+// @return err error
+func GetListBySourceAndFrequency(source, subSource int, frequency string) (list []*EdbRefreshDefaultConfig, err error) {
+	o := orm.NewOrm()
+	sql := `SELECT * FROM edb_refresh_default_config
+         WHERE source = ? AND sub_source = ? AND frequency = ? ORDER BY id ASC `
+	_, err = o.Raw(sql, source, subSource, frequency).QueryRows(&list)
+
+	return
+}
+
+// GetListByCondition
+// @Description: 根据条条件获取默认配置列表
+// @author: Roc
+// @datetime 2024-01-09 13:28:49
+// @param condition string
+// @param pars []interface{}
+// @return list []*EdbRefreshDefaultConfig
+// @return err error
+func GetListByCondition(condition string, pars []interface{}) (list []*EdbRefreshDefaultConfig, err error) {
+	o := orm.NewOrm()
+	sql := `SELECT * FROM edb_refresh_default_config
+         WHERE 1 = 1 `
+
+	if condition != "" {
+		sql += condition
+	}
+	sql += ` ORDER BY id ASC `
+	_, err = o.Raw(sql, pars).QueryRows(&list)
+
+	return
+}
+
+// GetDefaultRefreshEdbInfoListBySourceAndSubSource
+// @Description: 根据来源获取默认刷新的指标列表
+// @author: Roc
+// @datetime 2024-01-09 16:22:49
+// @param source int
+// @param subSource int
+// @return list []*data_manage.EdbInfo
+// @return err error
+func GetDefaultRefreshEdbInfoListBySourceAndSubSource(source, subSource int, frequencyList []string) (list []*EdbInfoListAndRefreshConfig, err error) {
+	num := len(frequencyList)
+	if num <= 0 {
+		return
+	}
+
+	o := orm.NewOrm()
+	sql := `SELECT a.* FROM edb_info a 
+LEFT JOIN edb_refresh_mapping b ON a.edb_info_id = b.edb_info_id
+WHERE a.source = ? AND a.sub_source = ? AND a.frequency IN (` + utils.GetOrmInReplace(num) + `)  AND b.edb_info_id is null`
+	_, err = o.Raw(sql, source, subSource, frequencyList).QueryRows(&list)
+
+	return
+}

models/edb_refresh/edb_refresh_mapping.go

@@ -0,0 +1,122 @@
+package edb_refresh
+
+import (
+	"errors"
+	"eta/eta_index_lib/utils"
+	"github.com/beego/beego/v2/client/orm"
+	"time"
+)
+
+// EdbRefreshMapping
+// @Description: 指标刷新时间配置关系表
+type EdbRefreshMapping struct {
+	EdbRefreshMappingId int       `orm:"column(edb_refresh_mapping_id);pk"`
+	Source              int       `description:"来源"`
+	SubSource           int       `description:"来源名称"`
+	EdbInfoId           int       `description:"指标id，如果是数据源（钢联、有色）的，那么就是数据源里面的id"`
+	EdbRefreshConfigId  int       `description:"刷新配置id"`
+	SysUserId           int       `description:"操作人id"`
+	SysUserRealName     string    `description:"操作人真实姓名"`
+	ModifyTime          time.Time `description:"最晚一次的更新时间"`
+	CreateTime          time.Time `description:"添加时间"`
+}
+
+// Add
+// @Description: 添加
+// @author: Roc
+// @receiver m
+// @datetime 2023-12-14 16:11:10
+// @param cols []string
+// @return err error
+func (m *EdbRefreshMapping) Add() (err error) {
+	if m.EdbRefreshMappingId > 0 {
+		err = errors.New("该配置已存在")
+		return
+	}
+	o := orm.NewOrm()
+	lastId, err := o.Insert(m)
+	if err != nil {
+		return
+	}
+	m.EdbRefreshMappingId = int(lastId)
+
+	return
+}
+
+// Update
+// @Description: 更新
+// @author: Roc
+// @receiver m
+// @datetime 2023-12-14 16:11:10
+// @param cols []string
+// @return err error
+func (m *EdbRefreshMapping) Update(cols []string) (err error) {
+	o := orm.NewOrm()
+	_, err = o.Update(m, cols...)
+	return
+}
+
+// Delete
+// @Description: 删除
+// @author: Roc
+// @receiver m
+// @datetime 2023-12-14 16:11:10
+// @return err error
+func (m *EdbRefreshMapping) Delete() (err error) {
+	o := orm.NewOrm()
+	_, err = o.Delete(m)
+	return
+}
+
+type EdbInfoListAndRefreshConfig struct {
+	EdbInfoId          int       `orm:"column(edb_info_id);pk"`
+	SourceName         string    `description:"来源名称"`
+	Source             int       `description:"来源id"`
+	SubSource          int       `description:"子数据来源:0:经济数据库,1:日期序列"`
+	SubSourceName      string    `description:"子数据来源名称"`
+	EdbCode            string    `description:"指标编码"`
+	EdbName            string    `description:"指标名称"`
+	Frequency          string    `description:"频率"`
+	Unit               string    `description:"单位"`
+	StartDate          time.Time `description:"起始日期"`
+	EndDate            time.Time `description:"终止日期"`
+	ClassifyId         int       `description:"分类id"`
+	UniqueCode         string    `description:"指标唯一编码"`
+	CalculateFormula   string    `description:"计算公式"`
+	ModifyTime         string    `description:"更新时间"`
+	NoUpdate           int8      `description:"是否停止更新，0：继续更新；1：停止更新"`
+	EdbRefreshConfigId int       `description:"刷新配置id"`
+	DataRefreshNum     int       `description:"刷新的期数"`
+}
+
+// GetConfigRefreshEdbInfoListBySourceAndSubSource
+// @Description:  根据来源和配置id列表获取指标列表
+// @author: Roc
+// @datetime 2024-01-09 17:28:06
+// @param sourceList []int
+// @param configIdList []int
+// @return list []*data_manage.EdbInfoList
+// @return err error
+func GetConfigRefreshEdbInfoListBySourceAndSubSource(sourceList, configIdList []int) (list []*EdbInfoListAndRefreshConfig, err error) {
+	num := len(configIdList)
+	if num <= 0 {
+		return
+	}
+
+	var pars []interface{}
+
+	o := orm.NewOrm()
+	sql := `SELECT a.*,b.edb_refresh_config_id FROM edb_info a 
+ JOIN edb_refresh_mapping b ON a.edb_info_id = b.edb_info_id
+WHERE b.edb_refresh_config_id IN (` + utils.GetOrmInReplace(num) + `) `
+	pars = append(pars, configIdList)
+
+	sourceNum := len(sourceList)
+	if sourceNum > 0 {
+		sql += ` AND b.source not in (` + utils.GetOrmInReplace(sourceNum) + `) `
+		pars = append(pars, sourceList)
+	}
+	_, err = o.Raw(sql, pars).QueryRows(&list)
+
+	return
+}

models/edb_refresh/edb_refresh_source.go

@@ -0,0 +1,51 @@
+package edb_refresh
+
+import (
+	"github.com/beego/beego/v2/client/orm"
+	"time"
+)
+
+// EdbRefreshSource
+// @Description: 刷新的数据源表
+type EdbRefreshSource struct {
+	Id            int       `orm:"column(id);pk"`
+	Source        int       `orm:"column(source)" description:"来源"`
+	SourceName    string    `description:"来源名称"`
+	SubSource     int       `description:"子数据来源:0:经济数据库,1:日期序列"`
+	SubSourceName string    `description:"子来源名称"`
+	HasChild      int       `description:"是否有子来源，0：否，1：是"`
+	ModifyTime    time.Time `description:"修改时间"`
+	CreateTime    time.Time `description:"创建时间"`
+}
+
+// EdbRefreshSourceList
+// @Description: 获取刷新的数据源列表
+type EdbRefreshSourceList struct {
+	Source        int                    `orm:"column(source)" description:"来源"`
+	SourceName    string                 `description:"来源名称"`
+	SubSource     int                    `description:"子数据来源:0:经济数据库,1:日期序列"`
+	SubSourceName string                 `description:"子来源名称"`
+	Child         []EdbRefreshSourceList `description:"子来源"`
+	HasChild      int                    `description:"是否有子来源，0：否，1：是"`
+}
+
+// GetAllList
+// @Description: 获取刷新数据源列表
+// @author: Roc
+// @datetime 2024-01-03 15:03:24
+// @return items []*EdbRefreshSource
+// @return err error
+func (m EdbRefreshSource) GetAllList() (items []*EdbRefreshSource, err error) {
+	o := orm.NewOrm()
+	sql := ` SELECT *  FROM edb_refresh_source ORDER BY id ASC `
+	_, err = o.Raw(sql).QueryRows(&items)
+	return
+}
+
+type BaseClassifyItems struct {
+	ClassifyId   int                  `description:"分类id"`
+	ClassifyName string               `description:"分类名称"`
+	ParentId     int                  `description:"父级id"`
+	UniqueCode   string               `description:"唯一编码"`
+	Children     []*BaseClassifyItems `description:"下级"`
+}

models/edb_refresh/request/edb_info_refresh.go

@@ -0,0 +1,52 @@
+package request
+
+// SaveEdbRefreshDefaultConfigReq
+// @Description: 设置默认刷新时间配置
+type SaveEdbRefreshDefaultConfigReq struct {
+	Source    int                `description:"来源"`
+	SubSource int                `description:"子来源"`
+	Frequency string             `description:"频度"`
+	List      []RefreshConfigReq `description:"刷新配置项"`
+}
+
+// RefreshConfigReq
+// @Description: 刷新时间配置项
+type RefreshConfigReq struct {
+	RefreshFrequency    string `description:"刷新频率"`
+	RefreshFrequencyDay int    `description:"具体刷新的日期"`
+	RefreshTime         string `description:"刷新时间"`
+	RefreshAllData      int    `description:"是否刷新所有数据，0：否，1：刷新所有数据"`
+	RefreshDataNum      int    `description:"刷新单元格数"`
+}
+
+// SaveEdbRefreshConfigReq
+// @Description: 设置指标的刷新时间配置
+type SaveEdbRefreshConfigReq struct {
+	Source          int                `description:"来源"`
+	SubSource       int                `description:"子来源"`
+	ClassifyId      int                `description:"分类id"`
+	TerminalCode    string             `description:"终端编码"`
+	SysUserId       string             `description:"操作人id"`
+	Frequency       string             `description:"频度"`
+	Keyword         string             `description:"关键字"`
+	Status          string             `description:"状态，枚举值：启用、暂停"`
+	IsSelectAll     bool               `description:"是否选择所有指标"`
+	EdbSelectIdList []int              `description:"选择的指标id列表"`
+	List            []RefreshConfigReq `description:"刷新配置项"`
+}
+
+// SaveEdbRefreshStatusReq
+// @Description: 设置指标的刷新状态
+type SaveEdbRefreshStatusReq struct {
+	Source          int    `description:"来源"`
+	SubSource       int    `description:"子来源"`
+	ClassifyId      int    `description:"分类id"`
+	TerminalCode    string `description:"终端编码"`
+	SysUserId       string `description:"操作人id"`
+	Frequency       string `description:"频度"`
+	Keyword         string `description:"关键字"`
+	Status          string `description:"状态，枚举值：启用、暂停"`
+	IsSelectAll     bool   `description:"是否选择所有指标"`
+	EdbSelectIdList []int  `description:"选择的指标id列表"`
+	ModifyStatus    string `description:"需要更改的状态，枚举值：启用、暂停"`
+}

routers/commentsRouter.go

@@ -232,6 +232,15 @@ func init() {
             Filters: nil,
             Params: nil})
 
+    beego.GlobalControllerRouter["eta/eta_index_lib/controllers:CoalMineDataController"] = append(beego.GlobalControllerRouter["eta/eta_index_lib/controllers:CoalMineDataController"],
+        beego.ControllerComments{
+            Method: "Firm",
+            Router: `/firm`,
+            AllowHTTPMethods: []string{"post"},
+            MethodParams: param.Make(),
+            Filters: nil,
+            Params: nil})
+
     beego.GlobalControllerRouter["eta/eta_index_lib/controllers:CoalMineDataController"] = append(beego.GlobalControllerRouter["eta/eta_index_lib/controllers:CoalMineDataController"],
         beego.ControllerComments{
             Method: "Inland",
@@ -376,6 +385,24 @@ func init() {
             Filters: nil,
             Params: nil})
 
+    beego.GlobalControllerRouter["eta/eta_index_lib/controllers:EdbRefreshController"] = append(beego.GlobalControllerRouter["eta/eta_index_lib/controllers:EdbRefreshController"],
+        beego.ControllerComments{
+            Method: "GetMysteelIndexConfigList",
+            Router: `/config/mysteel_index/list`,
+            AllowHTTPMethods: []string{"post"},
+            MethodParams: param.Make(),
+            Filters: nil,
+            Params: nil})
+
+    beego.GlobalControllerRouter["eta/eta_index_lib/controllers:EdbRefreshController"] = append(beego.GlobalControllerRouter["eta/eta_index_lib/controllers:EdbRefreshController"],
+        beego.ControllerComments{
+            Method: "GetSmmConfigList",
+            Router: `/config/smm/list`,
+            AllowHTTPMethods: []string{"post"},
+            MethodParams: param.Make(),
+            Filters: nil,
+            Params: nil})
+
     beego.GlobalControllerRouter["eta/eta_index_lib/controllers:EiaSteoController"] = append(beego.GlobalControllerRouter["eta/eta_index_lib/controllers:EiaSteoController"],
         beego.ControllerComments{
             Method: "Add",
@@ -412,6 +439,51 @@ func init() {
             Filters: nil,
             Params: nil})
 
+    beego.GlobalControllerRouter["eta/eta_index_lib/controllers:FenweiController"] = append(beego.GlobalControllerRouter["eta/eta_index_lib/controllers:FenweiController"],
+        beego.ControllerComments{
+            Method: "Add",
+            Router: `/add`,
+            AllowHTTPMethods: []string{"post"},
+            MethodParams: param.Make(),
+            Filters: nil,
+            Params: nil})
+
+    beego.GlobalControllerRouter["eta/eta_index_lib/controllers:FenweiController"] = append(beego.GlobalControllerRouter["eta/eta_index_lib/controllers:FenweiController"],
+        beego.ControllerComments{
+            Method: "BaseIndexList",
+            Router: `/base_index_list`,
+            AllowHTTPMethods: []string{"post"},
+            MethodParams: param.Make(),
+            Filters: nil,
+            Params: nil})
+
+    beego.GlobalControllerRouter["eta/eta_index_lib/controllers:FenweiController"] = append(beego.GlobalControllerRouter["eta/eta_index_lib/controllers:FenweiController"],
+        beego.ControllerComments{
+            Method: "ClassifyTree",
+            Router: `/classify_tree`,
+            AllowHTTPMethods: []string{"post"},
+            MethodParams: param.Make(),
+            Filters: nil,
+            Params: nil})
+
+    beego.GlobalControllerRouter["eta/eta_index_lib/controllers:FenweiController"] = append(beego.GlobalControllerRouter["eta/eta_index_lib/controllers:FenweiController"],
+        beego.ControllerComments{
+            Method: "HandleExcelData",
+            Router: `/handle/excel_data`,
+            AllowHTTPMethods: []string{"post"},
+            MethodParams: param.Make(),
+            Filters: nil,
+            Params: nil})
+
+    beego.GlobalControllerRouter["eta/eta_index_lib/controllers:FenweiController"] = append(beego.GlobalControllerRouter["eta/eta_index_lib/controllers:FenweiController"],
+        beego.ControllerComments{
+            Method: "Refresh",
+            Router: `/refresh`,
+            AllowHTTPMethods: []string{"post"},
+            MethodParams: param.Make(),
+            Filters: nil,
+            Params: nil})
+
     beego.GlobalControllerRouter["eta/eta_index_lib/controllers:FuBaoController"] = append(beego.GlobalControllerRouter["eta/eta_index_lib/controllers:FuBaoController"],
         beego.ControllerComments{
             Method: "Add",
@@ -466,6 +538,42 @@ func init() {
             Filters: nil,
             Params: nil})
 
+    beego.GlobalControllerRouter["eta/eta_index_lib/controllers:GzController"] = append(beego.GlobalControllerRouter["eta/eta_index_lib/controllers:GzController"],
+        beego.ControllerComments{
+            Method: "Add",
+            Router: `/add`,
+            AllowHTTPMethods: []string{"post"},
+            MethodParams: param.Make(),
+            Filters: nil,
+            Params: nil})
+
+    beego.GlobalControllerRouter["eta/eta_index_lib/controllers:GzController"] = append(beego.GlobalControllerRouter["eta/eta_index_lib/controllers:GzController"],
+        beego.ControllerComments{
+            Method: "Refresh",
+            Router: `/refresh`,
+            AllowHTTPMethods: []string{"post"},
+            MethodParams: param.Make(),
+            Filters: nil,
+            Params: nil})
+
+    beego.GlobalControllerRouter["eta/eta_index_lib/controllers:IcpiController"] = append(beego.GlobalControllerRouter["eta/eta_index_lib/controllers:IcpiController"],
+        beego.ControllerComments{
+            Method: "Add",
+            Router: `/add`,
+            AllowHTTPMethods: []string{"post"},
+            MethodParams: param.Make(),
+            Filters: nil,
+            Params: nil})
+
+    beego.GlobalControllerRouter["eta/eta_index_lib/controllers:IcpiController"] = append(beego.GlobalControllerRouter["eta/eta_index_lib/controllers:IcpiController"],
+        beego.ControllerComments{
+            Method: "Refresh",
+            Router: `/refresh`,
+            AllowHTTPMethods: []string{"post"},
+            MethodParams: param.Make(),
+            Filters: nil,
+            Params: nil})
+
     beego.GlobalControllerRouter["eta/eta_index_lib/controllers:JiaYueController"] = append(beego.GlobalControllerRouter["eta/eta_index_lib/controllers:JiaYueController"],
         beego.ControllerComments{
             Method: "Add",
@@ -547,6 +655,33 @@ func init() {
             Filters: nil,
             Params: nil})
 
+    beego.GlobalControllerRouter["eta/eta_index_lib/controllers:MtjhDataController"] = append(beego.GlobalControllerRouter["eta/eta_index_lib/controllers:MtjhDataController"],
+        beego.ControllerComments{
+            Method: "Add",
+            Router: `/add`,
+            AllowHTTPMethods: []string{"post"},
+            MethodParams: param.Make(),
+            Filters: nil,
+            Params: nil})
+
+    beego.GlobalControllerRouter["eta/eta_index_lib/controllers:MtjhDataController"] = append(beego.GlobalControllerRouter["eta/eta_index_lib/controllers:MtjhDataController"],
+        beego.ControllerComments{
+            Method: "Mtjh",
+            Router: `/data`,
+            AllowHTTPMethods: []string{"post"},
+            MethodParams: param.Make(),
+            Filters: nil,
+            Params: nil})
+
+    beego.GlobalControllerRouter["eta/eta_index_lib/controllers:MtjhDataController"] = append(beego.GlobalControllerRouter["eta/eta_index_lib/controllers:MtjhDataController"],
+        beego.ControllerComments{
+            Method: "Refresh",
+            Router: `/refresh`,
+            AllowHTTPMethods: []string{"post"},
+            MethodParams: param.Make(),
+            Filters: nil,
+            Params: nil})
+
     beego.GlobalControllerRouter["eta/eta_index_lib/controllers:MySteelChemicalController"] = append(beego.GlobalControllerRouter["eta/eta_index_lib/controllers:MySteelChemicalController"],
         beego.ControllerComments{
             Method: "Add",

routers/router.go

@@ -219,6 +219,31 @@ func init() {
 				&controllers.CoalMineDataController{},
 			),
 		),
+		beego.NSNamespace("/fenwei",
+			beego.NSInclude(
+				&controllers.FenweiController{},
+			),
+		),
+		beego.NSNamespace("/gz",
+			beego.NSInclude(
+				&controllers.GzController{},
+			),
+		),
+		beego.NSNamespace("/mtjh",
+			beego.NSInclude(
+				&controllers.MtjhDataController{},
+			),
+		),
+		beego.NSNamespace("/icpi",
+			beego.NSInclude(
+				&controllers.IcpiController{},
+			),
+		),
+		beego.NSNamespace("/edb_refresh",
+			beego.NSInclude(
+				&controllers.EdbRefreshController{},
+			),
+		),
 	)
 	beego.AddNamespace(ns)
 }

services/base_from_fenwei.go

@@ -0,0 +1,249 @@
+package services
+
+import (
+	"eta/eta_index_lib/logic"
+	"eta/eta_index_lib/models"
+	"eta/eta_index_lib/services/alarm_msg"
+	"eta/eta_index_lib/utils"
+	"fmt"
+	"strings"
+	"time"
+)
+
+// HandleFenweiIndex 处理汾渭数据的excel数据
+func HandleFenweiIndex(req *models.HandleFenweiExcelDataReq) (err error) {
+	errMsgList := make([]string, 0)
+	defer func() {
+		if len(errMsgList) > 0 {
+			msg := fmt.Sprint("数据源-汾渭数据数据处理失败, err:", strings.Join(errMsgList, "\n"))
+			utils.FileLog.Info(msg)
+			go alarm_msg.SendAlarmMsg(msg, 3)
+		}
+	}()
+	for _, v := range req.List {
+		if v.IndexName == "" || v.IndexCode == "" {
+			errMsgList = append(errMsgList, fmt.Sprintf("新增指标异常，指标编码%s或者指标ID%s为空：", v.IndexCode, v.IndexName))
+			continue
+		}
+		err = handleFenweiIndex(v, req.TerminalCode)
+		if err != nil {
+			errMsgList = append(errMsgList, fmt.Sprintf("新增指标异常，指标编码：%s, Err: %s", v.IndexCode, err))
+			return
+		}
+	}
+	return
+}
+
+func handleFenweiIndex(req *models.HandleFenweiExcelData, terminalCode string) (err error) {
+	indexName := req.IndexName
+	indexCode := req.IndexCode
+	excelDataMap := req.ExcelDataMap
+	errMsgList := make([]string, 0)
+	defer func() {
+		if len(errMsgList) > 0 {
+			msg := fmt.Sprint("数据源-汾渭数据数据处理失败,err:", strings.Join(errMsgList, "\n"))
+			utils.FileLog.Info(msg)
+			go alarm_msg.SendAlarmMsg(msg, 3)
+		}
+	}()
+	indexObj := new(models.BaseFromFenweiIndex)
+	dataObj := new(models.BaseFromFenweiData)
+	//classifyObj := new(models.BaseFromFenweiClassify)
+
+	var indexId int64
+
+	addDataList := make([]*models.BaseFromFenweiData, 0)
+	updateDataList := make([]*models.BaseFromFenweiData, 0)
+
+	exitDataMap := make(map[string]*models.BaseFromFenweiData)
+
+	// 修改指标信息
+	if indexName == "" {
+		utils.FileLog.Info("未刷新到指标数据:indexName:" + indexName)
+		return
+	}
+
+	// 判断目录是否存在
+	//var classifyId int64
+	//now := time.Now()
+	//if req.ClassifyName != "" {
+	//	classifyObj, err = classifyObj.GetByClassifyName(req.ClassifyName)
+	//	if err != nil {
+	//		if err.Error() == utils.ErrNoRow() {
+	//			//新增分类
+	//			classifyObj = &models.BaseFromFenweiClassify{
+	//				ClassifyName:    req.ClassifyName,
+	//				ParentId:        0,
+	//				SysUserId:       0,
+	//				SysUserRealName: "",
+	//				Level:           1,
+	//				Sort:            req.ClassifySort,
+	//				ModifyTime:      now,
+	//				CreateTime:      now,
+	//			}
+	//
+	//			classifyId, err = classifyObj.Add()
+	//			if err != nil {
+	//				err = fmt.Errorf("新增分类失败 Err：%s", err)
+	//				return
+	//			}
+	//			classifyObj.ClassifyId = classifyId
+	//		} else {
+	//			return
+	//		}
+	//	} else {
+	//		classifyId = classifyObj.ClassifyId
+	//	}
+	//}
+
+	//判断指标是否存在
+	var isAdd int
+	item, err := indexObj.GetByIndexCode(indexCode)
+	if err != nil {
+		if err.Error() == utils.ErrNoRow() {
+			isAdd = 1
+			err = nil
+		} else {
+			isAdd = -1
+			err = fmt.Errorf("查询数据源指标库失败 GetByIndexCode Err：%s", err)
+			return
+		}
+	}
+	if item != nil && item.FenweiIndexId > 0 {
+		fmt.Println("item:", item)
+		isAdd = 2
+	} else {
+		isAdd = 1
+	}
+
+	if isAdd == 1 {
+		indexObj.IndexCode = indexCode
+		indexObj.IndexName = indexName
+		indexObj.Frequency = req.Frequency
+		indexObj.ClassifyId = req.ClassifyId
+		indexObj.Unit = req.Unit
+		indexObj.Sort = req.Sort
+		indexObj.ModifyTime = time.Now()
+		indexObj.CreateTime = time.Now()
+		indexObj.TerminalCode = terminalCode
+		indexId, err = indexObj.Add()
+		if err != nil {
+			err = fmt.Errorf("数据源新增涌溢指标失败 Err：%s", err)
+			return
+		}
+		indexObj.FenweiIndexId = indexId
+	} else if isAdd == 2 {
+		if item.TerminalCode == `` && terminalCode != `` {
+			item.TerminalCode = terminalCode
+			err = item.Update([]string{"TerminalCode"})
+			if err != nil {
+				err = fmt.Errorf("数据源更新涌溢指标失败 Err：%s", err)
+				return
+			}
+		}
+
+		//获取已存在的所有数据
+		var exitDataList []*models.BaseFromFenweiData
+		exitDataList, err = dataObj.GetByIndexCode(indexCode)
+		if err != nil {
+			err = fmt.Errorf("数据源查询涌溢指标数据失败 Err：%s", err)
+			return
+		}
+		fmt.Println("exitDataListLen:", len(exitDataList))
+		for _, v := range exitDataList {
+			dateStr := v.DataTime
+			exitDataMap[dateStr] = v
+		}
+		indexId = item.FenweiIndexId
+	}
+
+	// 遍历excel数据，然后跟现有的数据做校验，不存在则入库
+	for date, value := range excelDataMap {
+		if findData, ok := exitDataMap[date]; !ok {
+			_, err = time.ParseInLocation(utils.FormatDate, date, time.Local)
+			if err != nil {
+				err = fmt.Errorf("%s 转换日期格式失败 Err：%s", date, err)
+				return
+			}
+			if !strings.Contains(value, "#N/A") {
+				var saveDataTime time.Time
+				if strings.Contains(date, "00:00:00") {
+					saveDataTime, err = time.Parse(utils.FormatDateTime, date)
+				} else {
+					saveDataTime, err = time.Parse(utils.FormatDate, date)
+				}
+				if err != nil {
+					err = fmt.Errorf("%s 转换日期格式失败 Err：%s", date, err)
+					continue
+				}
+				timestamp := saveDataTime.UnixNano() / 1e6
+
+				dataItem := new(models.BaseFromFenweiData)
+				dataItem.FenweiIndexId = int(indexId)
+				dataItem.IndexCode = indexCode
+				dataItem.DataTime = date
+				dataItem.Value = value
+				dataItem.CreateTime = time.Now()
+				dataItem.ModifyTime = time.Now()
+				dataItem.DataTimestamp = timestamp
+				addDataList = append(addDataList, dataItem)
+			}
+		} else {
+			if findData != nil && findData.Value != value && !strings.Contains(value, "#N/A") { //修改数据
+				findData.Value = value
+				findData.ModifyTime = time.Now().Local()
+				updateDataList = append(updateDataList, findData)
+			}
+		}
+	}
+
+	if len(addDataList) > 0 {
+		err = dataObj.AddMulti(addDataList)
+		if err != nil {
+			err = fmt.Errorf("批量新增指标失败 Err：%s", err)
+			return
+		}
+
+		var dateItem *models.EdbInfoMaxAndMinInfo
+		dateItem, err = dataObj.GetMaxAndMinDateByIndexCode(indexCode)
+		if err != nil {
+			err = fmt.Errorf("查询指标最新日期失败 Err：%s", err)
+			return
+		}
+
+		go func() {
+			indexObj.ModifyIndexMaxAndMinDate(indexCode, dateItem)
+		}()
+	}
+
+	// 批量更新数据
+	if len(updateDataList) > 0 {
+		e := models.MultiUpdateBaseFromFenweiDataValue(updateDataList)
+		if e != nil {
+			err = fmt.Errorf("MultiUpdateBaseFromSmmDataValue err: %s", e.Error())
+			return
+		}
+	}
+
+	// 同步刷新ETA指标库的指标
+	{
+		// 获取指标详情
+		baseObj := new(models.BaseFromFenwei)
+		var edbInfo *models.EdbInfo
+		edbInfo, err = models.GetEdbInfoByEdbCode(baseObj.GetSource(), indexCode)
+		if err != nil {
+			if err.Error() != utils.ErrNoRow() {
+				errMsgList = append(errMsgList, fmt.Sprint("刷新ETA指标异常，指标编码：", indexCode, err.Error()))
+				return
+			} else {
+				err = nil
+			}
+		}
+
+		// 已经加入到指标库的话，那么就去更新ETA指标库吧
+		if edbInfo != nil {
+			go logic.RefreshBaseEdbInfo(edbInfo, ``)
+		}
+	}
+	return
+}

services/edb_refresh/edb_refresh.go

@@ -0,0 +1,565 @@
+package edb_refresh
+
+import (
+	"context"
+	"eta/eta_index_lib/models/edb_refresh"
+	"eta/eta_index_lib/services/alarm_msg"
+	"eta/eta_index_lib/utils"
+	"fmt"
+	"strings"
+	"sync"
+	"time"
+)
+
+// GetDefaultConfigList
+// @Description: 获取默认配置列表
+// @author: Roc
+// @datetime 2024-01-11 14:46:54
+// @param source int
+// @param subSource int
+// @return list []*edb_refresh.EdbRefreshDefaultConfig
+// @return err error
+func GetDefaultConfigList(source, subSource int) (list []*edb_refresh.EdbRefreshDefaultConfig, err error) {
+	defer func() {
+		if err != nil {
+			fmt.Println(err)
+		}
+	}()
+	//刷新频率，枚举值：每自然日、每交易日、每周、每旬、每月、每季、每半年、每年
+	refreshFrequencyList := []string{"每自然日", "每交易日", "每周", "每旬", "每月", "每季", "每半年", "每年"}
+	now := time.Now()
+	//now = time.Date(2023, 12, 31, 19, 10, 59, 0, time.Local)
+	//now = time.Date(2023, 12, 31, 04, 10, 59, 0, time.Local)
+
+	currTimeStr := getPreviousHalfHour(now)
+	fmt.Println(currTimeStr)
+
+	// 所有默认配置刷新项
+	list = make([]*edb_refresh.EdbRefreshDefaultConfig, 0)
+	// 获取各个刷新频率的配置
+	for _, refreshFrequency := range refreshFrequencyList {
+		// 获取刷新频率条件
+		condition, pars, isHandler := getRefreshFrequencyCondition(now, refreshFrequency)
+		if !isHandler {
+			// 可能是非交易日，所以过滤不处理
+			continue
+		}
+
+		condition += ` AND refresh_frequency = ? AND refresh_time = ? AND source = ? AND sub_source = ? `
+		pars = append(pars, refreshFrequency, currTimeStr, source, subSource)
+
+		tmpList, tmpErr := edb_refresh.GetListByCondition(condition, pars)
+		if tmpErr != nil {
+			err = tmpErr
+			return
+		}
+		list = append(list, tmpList...)
+	}
+
+	return
+}
+
+// ConfigRefreshData
+// @Description:  配置刷新数据
+// @author: Roc
+// @datetime 2024-01-10 13:55:05
+// @param cont context.Context
+// @return err error
+func ConfigRefreshData(cont context.Context) (err error) {
+	errMsgList := make([]string, 0)
+	defer func() {
+		if err != nil {
+			fmt.Println(err)
+		}
+	}()
+	// 一期是只做wind、同花顺、钢联、有色
+
+	now := time.Now()
+	//now = time.Date(2023, 12, 31, 19, 10, 59, 0, time.Local)
+	//now = time.Date(2023, 12, 31, 16, 50, 59, 0, time.Local)
+	defaultSourceEdbInfoListMap, err := getDefaultRefreshData(now)
+	if err != nil {
+		errMsgList = append(errMsgList, "获取默认刷新数据失败，Err:"+err.Error())
+	}
+	sourceEdbInfoListMap, err := getConfigRefreshData(now)
+	if err != nil {
+		errMsgList = append(errMsgList, "获取指标配置刷新数据失败，Err:"+err.Error())
+	}
+
+	// 将两个合并
+	allSourceEdbInfoListMap := mergeMaps(defaultSourceEdbInfoListMap, sourceEdbInfoListMap)
+	wgNum := len(allSourceEdbInfoListMap)
+	if wgNum <= 0 {
+		return
+	}
+	wg := sync.WaitGroup{}
+	wg.Add(wgNum)
+
+	for _, edbList := range allSourceEdbInfoListMap {
+		go BaseRefreshData(&wg, edbList[0].Source, edbList[0].SubSource, edbList)
+	}
+
+	wg.Wait()
+
+	fmt.Println("Refresh End")
+
+	return
+}
+
+// Function to merge two maps
+func mergeMaps(dst map[string][]*edb_refresh.EdbInfoListAndRefreshConfig, src map[string][]*edb_refresh.EdbInfoListAndRefreshConfig) map[string][]*edb_refresh.EdbInfoListAndRefreshConfig {
+	if dst == nil {
+		return src
+	}
+	if src == nil {
+		return dst
+	}
+	for k, v := range src {
+		if dstk, ok := dst[k]; ok {
+			dstk = append(dstk, v...)
+			dst[k] = dstk
+		} else {
+			dst[k] = v
+		}
+	}
+	return dst
+}
+
+// getDefaultRefreshData
+// @Description: 根据默认配置获取需要刷新的指标列表
+// @author: Roc
+// @datetime 2024-01-10 13:55:38
+// @param now time.Time
+// @return sourceEdbInfoListMap map[string][]*edb_refresh.EdbInfoListAndRefreshConfig
+// @return err error
+func getDefaultRefreshData(now time.Time) (sourceEdbInfoListMap map[string][]*edb_refresh.EdbInfoListAndRefreshConfig, err error) {
+	errMsgList := make([]string, 0)
+	defer func() {
+		if err != nil {
+			fmt.Println(err)
+		}
+	}()
+	// 一期是只做wind、同花顺、钢联、有色
+
+	sourceEdbInfoListMap = make(map[string][]*edb_refresh.EdbInfoListAndRefreshConfig)
+
+	currTimeStr := getPreviousHalfHour(now)
+	fmt.Println(currTimeStr)
+
+	// 所有默认配置刷新项
+	list := make([]*edb_refresh.EdbRefreshDefaultConfig, 0)
+
+	//刷新频率，枚举值：每自然日、每交易日、每周、每旬、每月、每季、每半年、每年
+	refreshFrequencyList := []string{"每自然日", "每交易日", "每周", "每旬", "每月", "每季", "每半年", "每年"}
+
+	// 获取各个刷新频率的配置
+	for _, refreshFrequency := range refreshFrequencyList {
+		// 获取刷新频率条件
+		condition, pars, isHandler := getRefreshFrequencyCondition(now, refreshFrequency)
+		if !isHandler {
+			// 可能是非交易日，所以过滤不处理
+			continue
+		}
+
+		condition += ` AND refresh_frequency = ? AND refresh_time = ?`
+		pars = append(pars, refreshFrequency, currTimeStr)
+
+		// 这两个是excel的数据源，他是从公共机更新的，需要过滤掉
+		condition += ` AND source not in (?,?)`
+		pars = append(pars, utils.DATA_SOURCE_MYSTEEL_CHEMICAL, utils.DATA_SOURCE_YS)
+
+		tmpList, tmpErr := edb_refresh.GetListByCondition(condition, pars)
+		if tmpErr != nil {
+			err = tmpErr
+			return
+		}
+		list = append(list, tmpList...)
+	}
+
+	// 更新的单元格数
+	refreshDataNumMap := make(map[string]*edb_refresh.EdbRefreshDefaultConfig)
+	// 数据源刷新频度的列表数组
+	refreshDataFrequencyListMap := make(map[int]map[int][]string)
+
+	wgNum := 0
+	// 处理待刷新的数据源，整理成数组，方便获取对应的指标
+	for _, item := range list {
+		// 更新的单元格数
+		key := fmt.Sprintf("%d_%d_%s", item.Source, item.SubSource, item.Frequency)
+		refreshDataNumMap[key] = item
+
+		// 数据源刷新频度的列表数组
+		subSourceFrequencyList, ok := refreshDataFrequencyListMap[item.Source]
+		if !ok {
+			subSourceFrequencyList = make(map[int][]string)
+		}
+		frequencyList, ok := subSourceFrequencyList[item.SubSource]
+		if !ok {
+			wgNum++
+			frequencyList = make([]string, 0)
+		}
+		subSourceFrequencyList[item.SubSource] = append(frequencyList, item.Frequency)
+		refreshDataFrequencyListMap[item.Source] = subSourceFrequencyList
+	}
+
+	for source, subSourceFrequencyListMap := range refreshDataFrequencyListMap {
+		switch source {
+		case utils.DATA_SOURCE_MYSTEEL_CHEMICAL, utils.DATA_SOURCE_YS:
+			// 这种不处理
+		default:
+			for subSource, frequencyList := range subSourceFrequencyListMap {
+				edbList, tmpErr := edb_refresh.GetDefaultRefreshEdbInfoListBySourceAndSubSource(source, subSource, frequencyList)
+				if tmpErr != nil {
+					errMsgList = append(errMsgList, fmt.Sprint("source:", source, "subSource:", subSource, "frequencyList:", strings.Join(frequencyList, ","), "err:", tmpErr.Error()))
+				}
+
+				for _, v := range edbList {
+					// 数据刷新的期数
+					dataRefreshNum := utils.DATA_REFRESH
+					key := fmt.Sprintf("%d_%d_%s", v.Source, v.SubSource, v.Frequency)
+					if edbRefreshDefaultConfig, ok := refreshDataNumMap[key]; ok {
+						if edbRefreshDefaultConfig.RefreshAllData == 1 { // 刷新所有数据期数
+							dataRefreshNum = 0
+						} else if edbRefreshDefaultConfig.RefreshDataNum > 0 { //
+							dataRefreshNum = edbRefreshDefaultConfig.RefreshDataNum
+						}
+					}
+					v.DataRefreshNum = dataRefreshNum
+				}
+
+				key := fmt.Sprint(source, "_", subSource)
+				sourceEdbInfoListMap[key] = edbList
+			}
+		}
+	}
+
+	fmt.Println("Get Refresh End")
+	return
+}
+
+// getConfigRefreshData
+// @Description: 根据指标配置获取需要刷新的指标列表
+// @author: Roc
+// @datetime 2024-01-10 13:55:59
+// @param now time.Time
+// @return sourceEdbInfoListMap map[string][]*edb_refresh.EdbInfoListAndRefreshConfig
+// @return err error
+func getConfigRefreshData(now time.Time) (sourceEdbInfoListMap map[string][]*edb_refresh.EdbInfoListAndRefreshConfig, err error) {
+	defer func() {
+		if err != nil {
+			fmt.Println(err)
+		}
+	}()
+	// 一期是只做wind、同花顺、钢联、有色
+
+	sourceEdbInfoListMap = make(map[string][]*edb_refresh.EdbInfoListAndRefreshConfig)
+
+	currTimeStr := getPreviousHalfHour(now)
+
+	// 所有默认配置刷新项
+	list := make([]*edb_refresh.EdbRefreshConfig, 0)
+
+	//刷新频率，枚举值：每自然日、每交易日、每周、每旬、每月、每季、每半年、每年
+	refreshFrequencyList := []string{"每自然日", "每交易日", "每周", "每旬", "每月", "每季", "每半年", "每年"}
+
+	// 获取各个刷新频率的配置
+	for _, refreshFrequency := range refreshFrequencyList {
+		// 获取刷新频率条件
+		condition, pars, isHandler := getRefreshFrequencyCondition(now, refreshFrequency)
+		if !isHandler {
+			// 可能是非交易日，所以过滤不处理
+			continue
+		}
+
+		condition += ` AND refresh_frequency = ? AND refresh_time = ?`
+		pars = append(pars, refreshFrequency, currTimeStr)
+
+		tmpList, tmpErr := edb_refresh.GetEdbRefreshConfigListByCondition(condition, pars)
+		if tmpErr != nil {
+			err = tmpErr
+			return
+		}
+		list = append(list, tmpList...)
+	}
+
+	// 配置列表
+	configIdEdbRefreshConfigMap := make(map[int]*edb_refresh.EdbRefreshConfig)
+	configIdList := make([]int, 0)
+	for _, v := range list {
+		configIdList = append(configIdList, v.EdbRefreshConfigId)
+		configIdEdbRefreshConfigMap[v.EdbRefreshConfigId] = v
+	}
+
+	edbInfoList, err := edb_refresh.GetConfigRefreshEdbInfoListBySourceAndSubSource([]int{utils.DATA_SOURCE_MYSTEEL_CHEMICAL, utils.DATA_SOURCE_YS}, configIdList)
+	if err != nil {
+		return
+	}
+
+	for _, v := range edbInfoList {
+		key := fmt.Sprint(v.Source, "_", v.SubSource)
+		tmpList, ok := sourceEdbInfoListMap[key]
+		if !ok {
+			tmpList = make([]*edb_refresh.EdbInfoListAndRefreshConfig, 0)
+		}
+		// 数据刷新的期数
+		dataRefreshNum := utils.DATA_REFRESH
+		if edbRefreshConfig, ok2 := configIdEdbRefreshConfigMap[v.EdbRefreshConfigId]; ok2 {
+			if edbRefreshConfig.RefreshAllData == 1 { // 刷新所有数据期数
+				dataRefreshNum = 0
+			} else if edbRefreshConfig.RefreshDataNum > 0 { //
+				dataRefreshNum = edbRefreshConfig.RefreshDataNum
+			}
+		}
+		v.DataRefreshNum = dataRefreshNum
+		sourceEdbInfoListMap[key] = append(tmpList, v)
+	}
+
+	fmt.Println("Get ConfigRefreshData End")
+	return
+}
+
+// BaseRefreshData
+// @Description: 基础数据刷新
+// @author: Roc
+// @datetime 2024-01-09 16:27:45
+// @param wg *sync.WaitGroup
+// @return err error
+func BaseRefreshData(wg *sync.WaitGroup, source, subSource int, items []*edb_refresh.EdbInfoListAndRefreshConfig) (err error) {
+	errMsgList := make([]string, 0)
+	defer func() {
+		if err != nil {
+			fmt.Println("来源：", source, "；子来源：", subSource, "；BaseRefreshData Err:"+err.Error())
+			go alarm_msg.SendAlarmMsg(fmt.Sprint("来源：", source, "；子来源：", subSource, "；BaseRefreshData ErrMsg:"+err.Error()), 3)
+		}
+		if len(errMsgList) > 0 {
+			errMsg := fmt.Sprint("来源：", source, "；子来源：", subSource, "；BaseRefreshData Err:"+strings.Join(errMsgList, "\n"))
+			fmt.Println(errMsg)
+			go alarm_msg.SendAlarmMsg(errMsg, 3)
+		}
+		wg.Done()
+	}()
+
+	// 数据刷新的期数
+	dataRefreshNum := utils.DATA_REFRESH
+	// 是否从最开始的日期更新
+	var isRefreshByStartDate bool
+
+	for _, v := range items {
+		if v.DataRefreshNum > 0 {
+			dataRefreshNum = v.DataRefreshNum
+		}
+
+		startDate := ""
+		if isRefreshByStartDate {
+			startDate = v.StartDate.Format(utils.FormatDate)
+		} else {
+			if v.Frequency == "日度" {
+				startDate = v.EndDate.AddDate(0, 0, -dataRefreshNum).Format(utils.FormatDate)
+			} else if v.Frequency == "周度" {
+				startDate = v.EndDate.AddDate(0, 0, -(dataRefreshNum * 7)).Format(utils.FormatDate)
+			} else if v.Frequency == "旬度" {
+				startDate = v.EndDate.AddDate(0, 0, -(dataRefreshNum * 10)).Format(utils.FormatDate)
+			} else if v.Frequency == "月度" {
+				startDate = v.EndDate.AddDate(0, -dataRefreshNum, 0).Format(utils.FormatDate)
+			} else if v.Frequency == "季度" {
+				startDate = v.EndDate.AddDate(0, -dataRefreshNum*3, 0).Format(utils.FormatDate)
+			} else if v.Frequency == "半年度" {
+				startDate = v.EndDate.AddDate(0, -dataRefreshNum*6, 0).Format(utils.FormatDate)
+			} else if v.Frequency == "年度" {
+				startDate = v.EndDate.AddDate(-dataRefreshNum, 0, 0).Format(utils.FormatDate)
+			} else {
+				startDate = v.EndDate.AddDate(0, 0, -utils.DATA_REFRESH).Format(utils.FormatDate)
+			}
+		}
+		fmt.Println(startDate)
+
+		// 数据更新
+		//resp, tmpErr := data.RefreshEdbData(v.EdbInfoId, v.Source, v.SubSource, v.EdbCode, startDate)
+		//if tmpErr != nil {
+		//	errMsgList = append(errMsgList, v.EdbCode+"RefreshEdbData Err:"+tmpErr.Error())
+		//	continue
+		//}
+		//if resp.Ret != 200 {
+		//	errMsgList = append(errMsgList, v.EdbCode+";RefreshEdbData Err:"+resp.Msg+";ErrMsg:"+resp.ErrMsg)
+		//	continue
+		//}
+	}
+
+	fmt.Println("来源：", source, ";子来源：", subSource, "刷新结束")
+
+	return err
+}
+
+// getRefreshFrequencyCondition
+// @Description: 根据时间和刷新频率获取条件
+// @author: Roc
+// @datetime 2024-01-09 16:27:11
+// @param now time.Time
+// @param refreshFrequency string
+// @return condition string
+// @return pars []interface{}
+// @return isHandler bool
+func getRefreshFrequencyCondition(now time.Time, refreshFrequency string) (condition string, pars []interface{}, isHandler bool) {
+	isHandler = true
+
+	var dayNum int
+	var isLastDay bool
+
+	//刷新频率，枚举值：每自然日、每交易日、每周、每旬、每月、每季、每半年、每年
+	switch refreshFrequency {
+	case "每自然日":
+		// 自然日不需要额外条件
+		return
+	case "每交易日":
+		// 周六日不处理
+		if now.Weekday() == time.Saturday || now.Weekday() == time.Sunday {
+			isHandler = false
+		}
+		return
+	case "每周":
+		currWeekDay := now.Weekday()
+		if currWeekDay == time.Sunday {
+			currWeekDay = 7
+			isLastDay = true
+		}
+		dayNum = int(currWeekDay)
+	case "每旬":
+		currDay := now.Day()
+		if currDay <= 10 {
+			dayNum = currDay
+			// 如果是这旬的最后一天
+			if currDay == 10 {
+				isLastDay = true
+			}
+		} else if currDay <= 20 {
+			dayNum = currDay - 10
+			// 如果是这旬的最后一天
+			if currDay == 20 {
+				isLastDay = true
+			}
+		} else {
+			dayNum = currDay - 20
+
+			// 当月的最后一天
+			monthLastDay := time.Date(now.Year(), now.Month()+1, 1, 0, 0, 0, 0, time.Local).AddDate(0, 0, -1)
+			// 如果是这旬的最后一天
+			if currDay == monthLastDay.Day() {
+				isLastDay = true
+			}
+		}
+	case "每月":
+		// 当前日期
+		currDay := now.Day()
+		dayNum = currDay
+
+		// 当期的最后一天
+		monthLastDay := time.Date(now.Year(), now.Month()+1, 1, 0, 0, 0, 0, time.Local).AddDate(0, 0, -1)
+
+		// 如果是这期的最后一天
+		if currDay == monthLastDay.Day() {
+			isLastDay = true
+		}
+	case "每季":
+		// 当期的第一天  ;  当期的最后一天
+		var startDay, endDay time.Time
+		currMonth := now.Month()
+		currDay := now.Day()
+		if currMonth <= 3 {
+			// 当季的第一天
+			startDay = time.Date(now.Year(), 1, 1, 0, 0, 0, 0, time.Local)
+			// 当季的最后一天
+			endDay = time.Date(now.Year(), 4, 1, 0, 0, 0, 0, time.Local).AddDate(0, 0, -1)
+		} else if currMonth <= 6 {
+			// 当期的第一天
+			startDay = time.Date(now.Year(), 4, 1, 0, 0, 0, 0, time.Local)
+			// 当期的最后一天
+			endDay = time.Date(now.Year(), 7, 1, 0, 0, 0, 0, time.Local).AddDate(0, 0, -1)
+		} else if currMonth <= 9 {
+			// 当期的第一天
+			startDay = time.Date(now.Year(), 7, 1, 0, 0, 0, 0, time.Local)
+			// 当期的最后一天
+			endDay = time.Date(now.Year(), 10, 1, 0, 0, 0, 0, time.Local).AddDate(0, 0, -1)
+		} else {
+			// 当期的第一天
+			startDay = time.Date(now.Year(), 10, 1, 0, 0, 0, 0, time.Local)
+			// 当期的最后一天
+			endDay = time.Date(now.Year()+1, 1, 1, 0, 0, 0, 0, time.Local).AddDate(0, 0, -1)
+		}
+
+		// 计算这期的第一天和当日的天数
+		dayNum = utils.GetTimeSubDay(startDay, now) + 1
+
+		// 如果是这期的最后一天
+		if currMonth == endDay.Month() && currDay == endDay.Day() {
+			isLastDay = true
+		}
+	case "每半年":
+		// 当期的第一天  ;  当期的最后一天
+		var startDay, endDay time.Time
+		currMonth := now.Month()
+		currDay := now.Day()
+		if currMonth <= 6 {
+			// 当期的第一天
+			startDay = time.Date(now.Year(), 1, 1, 0, 0, 0, 0, time.Local)
+			// 当期的最后一天
+			endDay = time.Date(now.Year(), 7, 1, 0, 0, 0, 0, time.Local).AddDate(0, 0, -1)
+		} else {
+			// 当期的第一天
+			startDay = time.Date(now.Year(), 7, 1, 0, 0, 0, 0, time.Local)
+			// 当期的最后一天
+			endDay = time.Date(now.Year()+1, 1, 1, 0, 0, 0, 0, time.Local).AddDate(0, 0, -1)
+		}
+
+		// 计算这期的第一天和当日的天数
+		dayNum = utils.GetTimeSubDay(startDay, now) + 1
+
+		// 如果是这期的最后一天
+		if currMonth == endDay.Month() && currDay == endDay.Day() {
+			isLastDay = true
+		}
+	case "每年":
+		currMonth := now.Month()
+		currDay := now.Day()
+
+		// 当期的第一天
+		startDay := time.Date(now.Year(), 1, 1, 0, 0, 0, 0, time.Local)
+		// 当期的最后一天
+		endDay := time.Date(now.Year()+1, 1, 1, 0, 0, 0, 0, time.Local).AddDate(0, 0, -1)
+
+		// 计算这期的第一天和当日的天数
+		dayNum = utils.GetTimeSubDay(startDay, now) + 1
+
+		// 如果是这期的最后一天
+		if currMonth == endDay.Month() && currDay == endDay.Day() {
+			isLastDay = true
+		}
+
+	}
+
+	// 如果是这期的最后一天，那么就是判断refresh_frequency_day是否配置为0，或者配置的天数大于这期的最大天数
+	if isLastDay {
+		condition += ` AND ( refresh_frequency_day = ? OR refresh_frequency_day >= ? )`
+		pars = append(pars, 0, dayNum)
+	} else {
+		// 如果不是这期的最后一天，那么就是判断refresh_frequency_day是否等于配置的天数
+		condition += ` AND refresh_frequency_day = ?  `
+		pars = append(pars, dayNum)
+	}
+
+	return
+}
+
+// getPreviousHalfHour
+// @Description: 根据传入的时间获取该时间的前整半小时的时间字符串
+// @author: Roc
+// @datetime 2024-01-09 14:27:34
+// @param now time.Time
+// @return string
+func getPreviousHalfHour(now time.Time) string {
+	minute := now.Minute()
+
+	if minute >= 30 {
+		return fmt.Sprintf("%02d:%02d", now.Hour(), 30)
+	}
+	return fmt.Sprintf("%02d:%02d", now.Hour(), 0)
+}

utils/common.go

@@ -1162,4 +1162,42 @@ func CamelToSnake(s string) string {
 		}
 	}
 	return result
-}
+}
+
+func TimeTransferString(format string, t time.Time) string {
+	str := t.Format(format)
+	if t.IsZero() {
+		return ""
+	}
+	return str
+}
+
+// GetEdbRefreshStartDate
+// @Description: 获取开始刷新时间
+// @author: Roc
+// @datetime 2024-02-05 11:23:55
+// @param startDate string
+// @return string
+func GetEdbRefreshStartDate(startDate string) string {
+	// 没有传入日期，或者日期异常的话，那么就是从1990-01-01开始
+	if startDate == `` || strings.Contains(startDate, "0000-") {
+		return "1990-01-01"
+	}
+
+	return startDate
+}
+
+// GetEdbRefreshEndDate
+// @Description: 获取结束刷新时间
+// @author: Roc
+// @datetime 2024-02-05 11:23:55
+// @param startDate string
+// @return string
+func GetEdbRefreshEndDate(endDate string) string {
+	// 没有传入日期，或者日期异常的话，那么就是从1990-01-01开始
+	if endDate == `` || strings.Contains(endDate, "0000-") {
+		return time.Now().Format(FormatDate)
+	}
+
+	return endDate
+}

utils/constants.go

@@ -20,7 +20,10 @@ const (
 	PageSize30                 = 30
 )
 
-const DATA_SOURCE_YONYI = 76 //涌益咨询 =
+const (
+	DATA_SOURCE_YONYI  = 76 //涌益咨询
+	DATA_SOURCE_FENWEI = 77 //汾渭煤炭
+)
 
 // 数据来源渠道
 const (
@@ -100,6 +103,9 @@ const (
 	DATA_SOURCE_CALCULATE_ZDYFX                                 // 自定义分析->74
 	DATA_SOURCE_CALCULATE_RJZ                                   // 日均值计算->75
 
+	DATA_SOURCE_GFEX          = 78 // 广州期货交易所->78
+	DATA_SOURCE_ICPI          = 79 // ICPI消费价格指数->79
+	DATA_SOURCE_MTJH          = 80 // 煤炭江湖->80
 	DATA_SOURCE_CALCULATE_SUM = 81
 	DATA_SOURCE_CALCULATE_AVG = 82
 )
@@ -181,8 +187,12 @@ const (
 	DATA_SOURCE_NAME_PREDICT_CALCULATE_ZSXY               = `预测指数修匀`            //预测指数修匀->73
 	DATA_SOURCE_NAME_CALCULATE_ZDYFX                      = `自定义分析`             //自定义分析->74
 	DATA_SOURCE_NAME_YONYI                                = `涌益咨询`              // 涌益咨询
-	DATA_SOURCE_NAME_CALCULATE_SUM                        = `多指标求和`
-	DATA_SOURCE_NAME_CALCULATE_AVG                        = `多指标求平均`
+
+	DATA_SOURCE_NAME_FENWEI        = `汾渭数据` // 汾渭煤炭
+	DATA_SOURCE_NAME_MTJH          = `煤炭江湖` // 煤炭江湖->80
+	DATA_SOURCE_NAME_ICPI          = "ICPI消费价格指数"
+	DATA_SOURCE_NAME_CALCULATE_SUM = `多指标求和`
+	DATA_SOURCE_NAME_CALCULATE_AVG = `多指标求平均`
 )
 
 // 基础数据初始化日期