fix:红桃3指标刷新

controllers/base_from_sci.go

@@ -4,16 +4,18 @@ import (
 	"encoding/json"
 	"hongze/hongze_edb_lib/logic"
 	"hongze/hongze_edb_lib/models"
+	"hongze/hongze_edb_lib/services"
 	"hongze/hongze_edb_lib/utils"
 	"strconv"
 	"time"
 )
 
-//有色
+// SciController 卓创（红桃3）
 type SciController struct {
 	BaseAuthController
 }
 
+// Add
 // @Title 新增Sci指标接口
 // @Description 新增Sci指标接口
 // @Success 200 {object} models.AddEdbInfoReq
@@ -58,13 +60,13 @@ func (this *SciController) Add() {
 	}
 }
 
+// Refresh
 // @Title 刷新sci指标接口
 // @Description 刷新sci指标接口
 // @Success 200 {object} models.RefreshEdbInfoReq
 // @router /refresh [post]
 func (this *SciController) Refresh() {
 	br := new(models.BaseResponse).Init()
-	var cacheKey string
 	defer func() {
 		this.Data["json"] = br
 		this.ServeJSON()
@@ -95,38 +97,47 @@ func (this *SciController) Refresh() {
 		br.ErrMsg = "指标不存在"
 		return
 	}
-	cacheKey = utils.CACHE_EDB_DATA_REFRESH + strconv.Itoa(source) + "_" + req.EdbCode
-	if utils.Rc.IsExist(cacheKey) {
+	isHandling, errMsg, err := logic.RefreshSci(edbInfo, req.StartDate)
+	if isHandling {
 		br.Ret = 501
 		br.Success = true
 		br.Msg = "系统处理中，请稍后重试"
 		return
 	}
-	utils.Rc.SetNX(cacheKey, 1, 1*time.Minute)
-	defer func() {
-		utils.Rc.Delete(cacheKey)
-	}()
-	if req.EdbInfoId <= 0 {
-		req.EdbInfoId = edbInfo.EdbInfoId
-	}
-	err = models.RefreshEdbDataFromSci(req.EdbInfoId, req.EdbCode, req.StartDate)
-	if err != nil && err.Error() != utils.ErrNoRow() {
-		br.Msg = "刷新指标信息失败！"
-		br.ErrMsg = "刷新指标信息失败 RefreshEdbDataFromSci，Err:" + err.Error()
-		return
-	}
-	// 更新指标最大最小值
-	err, errMsg := models.UnifiedModifyEdbInfoMaxAndMinInfo(edbInfo)
 	if err != nil {
 		br.Msg = errMsg
 		br.ErrMsg = err.Error()
 		return
 	}
 
-	// 更新ES
-	go logic.UpdateEs(edbInfo.EdbInfoId)
-
 	br.Ret = 200
 	br.Success = true
 	br.Msg = "获取成功"
 }
+
+// HandleExcelData
+// @Title 处理卓创（红桃3）指标的接口
+// @Description 处理卓创（红桃3）指标的接口
+// @Success 200 {object} models.HandleMysteelIndexResp
+// @router /handle/excel_data [post]
+func (this *SciController) HandleExcelData() {
+	br := new(models.BaseResponse).Init()
+	defer func() {
+		this.Data["json"] = br
+		this.ServeJSON()
+	}()
+	var req models.HandleSciExcelDataReq
+	err := json.Unmarshal(this.Ctx.Input.RequestBody, &req)
+	if err != nil {
+		br.Msg = "参数解析异常！"
+		br.ErrMsg = "参数解析失败，Err:" + err.Error()
+		return
+	}
+
+	// 处理excel表数据
+	services.HandleSciIndex(req.IndexNameList, req.ThirdIndexIdList, req.FrequencyList, req.UnitList, req.DataMap)
+
+	br.Ret = 200
+	br.Success = true
+	br.Msg = "处理成功"
+}

logic/sic.go

@@ -0,0 +1,39 @@
+package logic
+
+import (
+	"errors"
+	"hongze/hongze_edb_lib/models"
+	"hongze/hongze_edb_lib/utils"
+	"strconv"
+	"time"
+)
+
+// RefreshSci 卓创（红桃3）刷新
+func RefreshSci(edbInfo *models.EdbInfo, startDate string) (isHandling bool, errMsg string, err error) {
+	cacheKey := utils.CACHE_EDB_DATA_REFRESH + strconv.Itoa(edbInfo.Source) + "_" + edbInfo.EdbCode
+	if utils.Rc.IsExist(cacheKey) {
+		isHandle = true
+		return
+	}
+	utils.Rc.SetNX(cacheKey, 1, 1*time.Minute)
+	defer func() {
+		utils.Rc.Delete(cacheKey)
+	}()
+
+	err = models.RefreshEdbDataFromSci(edbInfo.EdbInfoId, edbInfo.EdbCode, startDate)
+	if err != nil && err.Error() != utils.ErrNoRow() {
+		errMsg = "刷新指标信息失败！"
+		err = errors.New("刷新指标信息失败 RefreshEdbDataFromSci，Err:" + err.Error())
+		return
+	}
+	// 更新指标最大最小值
+	err, errMsg = models.UnifiedModifyEdbInfoMaxAndMinInfo(edbInfo)
+	if err != nil {
+		return
+	}
+
+	// 更新ES
+	go UpdateEs(edbInfo.EdbInfoId)
+
+	return
+}

models/base_from_mysteel_chemical.go

@@ -280,6 +280,7 @@ func (m *BaseFromMysteelChemicalIndex) Update(cols []string) (err error) {
 	return
 }
 
+// GetIndexDataList 根据指标编码获取数据
 func (d *BaseFromMysteelChemicalData) GetIndexDataList(indexCode string) (item []*BaseFromMysteelChemicalData, err error) {
 	o := orm.NewOrm()
 	sql := `SELECT * FROM base_from_mysteel_chemical_data WHERE index_code = ? `
@@ -287,14 +288,14 @@ func (d *BaseFromMysteelChemicalData) GetIndexDataList(indexCode string) (item [
 	return
 }
 
-// 修改
+// Update 修改
 func (r *BaseFromMysteelChemicalData) Update(updateCols []string) (err error) {
 	o := orm.NewOrm()
 	_, err = o.Update(r, updateCols...)
 	return
 }
 
-// 新增
+// Add 新增
 func (r *BaseFromMysteelChemicalData) Add(list []BaseFromMysteelChemicalData) (err error) {
 	o := orm.NewOrm()
 	_, err = o.InsertMulti(len(list), list)

models/base_from_sci.go

@@ -21,6 +21,31 @@ type BaseFromSciData struct {
 	DataTimestamp      int64
 }
 
+// Update 修改
+func (r *BaseFromSciData) Update(updateCols []string) (err error) {
+	o := orm.NewOrm()
+	_, err = o.Update(r, updateCols...)
+
+	return
+}
+
+// GetIndexDataList 获取所有的指标数据
+func (r *BaseFromSciData) GetIndexDataList(indexCode string) (items []*BaseFromSciData, err error) {
+	o := orm.NewOrm()
+	sql := `SELECT * FROM base_from_sci_index WHERE 1=1 AND index_code = ? `
+	_, err = o.Raw(sql, indexCode).QueryRows(&items)
+
+	return
+}
+
+// BatchAdd 批量添加指标
+func (r *BaseFromSciData) BatchAdd(list []*BaseFromSciData) (err error) {
+	o := orm.NewOrm()
+	_, err = o.InsertMulti(len(list), list)
+
+	return
+}
+
 func GetBaseFromSciDataByCondition(condition string, pars []interface{}) (list []*BaseFromSciData, err error) {
 	o := orm.NewOrm()
 	sql := `SELECT * FROM base_from_sci_data WHERE 1=1 `
@@ -31,7 +56,7 @@ func GetBaseFromSciDataByCondition(condition string, pars []interface{}) (list [
 	return
 }
 
-// 新增有色指标数据
+// AddEdbDataFromSci 新增卓创（红桃3）指标数据
 func AddEdbDataFromSci(edbCode string) (err error) {
 	o := orm.NewOrm()
 
@@ -73,7 +98,7 @@ func AddEdbDataFromSci(edbCode string) (err error) {
 	return
 }
 
-// 刷新有色指标数据
+// RefreshEdbDataFromSci 刷新卓创（红桃3）指标数据
 func RefreshEdbDataFromSci(edbInfoId int, edbCode, startDate string) (err error) {
 	source := utils.DATA_SOURCE_SCI
 	o := orm.NewOrm()
@@ -182,3 +207,55 @@ func RefreshEdbDataFromSci(edbInfoId int, edbCode, startDate string) (err error)
 	}
 	return
 }
+
+// HandleSciExcelDataReq 卓创（红桃3）的excel数据
+type HandleSciExcelDataReq struct {
+	DataMap          map[string]map[string]string
+	IndexNameList    []string
+	ThirdIndexIdList []string
+	FrequencyList    []string
+	UnitList         []string
+}
+
+// BaseFromSciIndex 红桃3指标表格
+type BaseFromSciIndex struct {
+	BaseFromSciIndexId int       `orm:"column(base_from_sci_index_id);pk"  json:"base_from_sci_index_id"` //序号
+	ClassifyId         int       `gorm:"column:classify_id" json:"classify_id"`
+	IndexCode          string    `gorm:"column:index_code" json:"index_code" description:"指标编码"`
+	IndexName          string    `gorm:"column:index_name" json:"index_name" description:"指标名称"`
+	Frequency          string    `gorm:"column:frequency" json:"frequency"`
+	Unit               string    `gorm:"column:unit" json:"unit"`
+	StartDate          time.Time `gorm:"column:start_date" json:"start_date"`
+	EndDate            time.Time `gorm:"column:end_date" json:"end_date"`
+	CreateTime         time.Time `gorm:"autoCreateTime;column:create_time" json:"create_time"`       //创建时间
+	ModifyTime         time.Time `gorm:"autoUpdateTime:milli;column:modify_time" json:"modify_time"` //最后更新时间
+}
+
+// TableName get sql table name.获取数据库表名
+func (r *BaseFromSciIndex) TableName() string {
+	return "base_from_sci_index"
+}
+
+// Update 修改
+func (r *BaseFromSciIndex) Update(updateCols []string) (err error) {
+	o := orm.NewOrm()
+	_, err = o.Update(r, updateCols...)
+
+	return
+}
+
+// GetAllIndex 获取所有的指标
+func (r *BaseFromSciIndex) GetAllIndex() (items []*BaseFromSciIndex, err error) {
+	o := orm.NewOrm()
+	sql := `SELECT * FROM base_from_sci_index WHERE 1=1 `
+	_, err = o.Raw(sql).QueryRows(&items)
+	return
+}
+
+// BatchAdd 批量添加指标
+func (r *BaseFromSciIndex) BatchAdd(list []*BaseFromSciIndex) (err error) {
+	o := orm.NewOrm()
+	_, err = o.InsertMulti(len(list), list)
+
+	return
+}

services/base_from_sci.go

@@ -0,0 +1,175 @@
+package services
+
+import (
+	"fmt"
+	"hongze/hongze_edb_lib/logic"
+	"hongze/hongze_edb_lib/models"
+	"hongze/hongze_edb_lib/services/alarm_msg"
+	"hongze/hongze_edb_lib/utils"
+	"strings"
+	"time"
+)
+
+// HandleSciIndex 处理卓创（红桃3）的Excel数据
+func HandleSciIndex(indexNameList, thirdIndexIdList, frequencyList, unitList []string, dataMap map[string]map[string]string) {
+	// 卓创（红桃3）指标id列表
+
+	sciIndexModel := new(models.BaseFromSciIndex)
+	list, err := sciIndexModel.GetAllIndex()
+	if err != nil {
+		fmt.Println("获取指标失败:", err)
+		return
+	}
+	allIndexMap := make(map[string]string)
+	for _, v := range list {
+		allIndexMap[v.IndexCode] = ``
+	}
+
+	// 需要入库的指标下标
+	needAddIndexKeyList := make([]int, 0)
+	needAddIndexMap := make(map[string]int, 0)
+	for key, v := range thirdIndexIdList {
+		if _, ok := allIndexMap[v]; !ok {
+			if _, ok2 := needAddIndexMap[v]; !ok2 {
+				needAddIndexKeyList = append(needAddIndexKeyList, key)
+				needAddIndexMap[v] = key
+			}
+		}
+	}
+	//fmt.Println(needAddIndexKeyList)
+	//return
+	addSciIndexList := make([]*models.BaseFromSciIndex, 0)
+	// 新的指标入库
+	for _, key := range needAddIndexKeyList {
+		tmpSciIndex := &models.BaseFromSciIndex{
+			//BaseFromSciIndexId: 0,
+			//ClassifyId:         0,
+			IndexCode: thirdIndexIdList[key],
+			IndexName: indexNameList[key],
+			Frequency: frequencyList[key],
+			Unit:      unitList[key],
+			//StartDate: time.Time{},
+			//EndDate:      time.Time{},
+			CreateTime: time.Now(),
+			ModifyTime: time.Now(),
+		}
+		addSciIndexList = append(addSciIndexList, tmpSciIndex)
+	}
+	//fmt.Println(addSciIndexList)
+	if len(addSciIndexList) > 0 {
+		err = sciIndexModel.BatchAdd(addSciIndexList)
+		if err != nil {
+			fmt.Println("批量添加指标失败:", err)
+			return
+		}
+		fmt.Println("添加成功")
+	}
+
+	// 红桃3实际数据处理
+	HandleData(dataMap)
+
+}
+
+// HandleData 红桃3实际数据处理
+func HandleData(dataMap map[string]map[string]string) {
+	errMsgList := make([]string, 0)
+	defer func() {
+		if len(errMsgList) > 0 {
+			go alarm_msg.SendAlarmMsg(fmt.Sprint("红桃3实际数据处理失败,err:", strings.Join(errMsgList, "\n")), 3)
+		}
+	}()
+	// 获取所有的指标
+	sciIndexModel := new(models.BaseFromSciIndex)
+	list, err := sciIndexModel.GetAllIndex()
+	if err != nil {
+		fmt.Println("获取指标失败:", err)
+		return
+	}
+	allIndexMap := make(map[string]*models.BaseFromSciIndex)
+	for _, v := range list {
+		allIndexMap[v.IndexCode] = v
+	}
+
+	sciIndexDataModel := new(models.BaseFromSciData)
+	for indexCode, data := range dataMap {
+		indexInfo, ok := allIndexMap[indexCode]
+		if !ok {
+			fmt.Println("找不到该指标:", indexCode)
+			continue
+		}
+
+		indexDataList, err := sciIndexDataModel.GetIndexDataList(indexCode)
+		if err != nil {
+			errMsgList = append(errMsgList, fmt.Sprint("查找卓创基础指标失败,指标编码：", indexCode, ";错误原因：", err.Error()))
+			fmt.Println(indexCode, "寻找数据失败:", err)
+			continue
+		}
+		indexDataExistMap := make(map[string]*models.BaseFromSciData)
+		for _, indexData := range indexDataList {
+			indexDataExistMap[indexData.DataTime] = indexData
+		}
+		addSciDataList := make([]*models.BaseFromSciData, 0)
+
+		for currDate, currVal := range data {
+			currDataTime, tmpErr := time.ParseInLocation(utils.FormatDate, currDate, time.Local)
+			if tmpErr != nil {
+				fmt.Println("时间格式化失败:", currDate)
+				continue
+			}
+			timestamp := currDataTime.UnixNano() / 1e6
+
+			sciData, ok := indexDataExistMap[currDate]
+			//判断是否存在数据,如果不存在,那么插入数据,存在的话那么修改数据
+			if !ok {
+				tmpBaseFromSciData := &models.BaseFromSciData{
+					//SciDataId:          0,
+					BaseFromSciIndexId: indexInfo.BaseFromSciIndexId,
+					IndexCode:          indexInfo.IndexCode,
+					DataTime:           currDate,
+					Value:              currVal,
+					DataTimestamp:      timestamp,
+					CreateTime:         time.Now(),
+					ModifyTime:         time.Now(),
+				}
+				indexDataExistMap[currDate] = tmpBaseFromSciData
+				addSciDataList = append(addSciDataList, tmpBaseFromSciData)
+			} else {
+				// 更新数据
+				existValue := sciData.Value
+				if existValue != currVal {
+					//fmt.Println("existValue:", existValue, ";====;currVal:", currVal)
+					sciData.Value = currVal
+					sciData.ModifyTime = time.Now()
+					// 如果是已经入库了数据,那么就更新,否则只是单纯更改内存数据,而不去更新数据库
+					if sciData.SciDataId > 0 {
+						err = sciData.Update([]string{"Value", "ModifyTime"})
+						if err != nil {
+							fmt.Println(indexCode, "更新,", currDate, "数据失败,err:", err)
+						}
+					}
+				}
+			}
+		}
+
+		if len(addSciDataList) > 0 {
+			err = sciIndexDataModel.BatchAdd(addSciDataList)
+			if err != nil {
+				fmt.Println(indexCode, "批量添加数据失败:", err)
+			}
+		}
+		indexInfo.ModifyTime = time.Now()
+		indexInfo.Update([]string{"ModifyTime"})
+
+		// 同步刷新ETA图库红桃3的指标
+		{
+			// 获取指标详情
+			edbInfo, tmpErr := models.GetEdbInfoByEdbCode(utils.DATA_SOURCE_SCI, indexInfo.IndexCode)
+			if tmpErr != nil && tmpErr.Error() != utils.ErrNoRow() {
+				br.Msg = "指标不存在！"
+				br.ErrMsg = "指标不存在"
+				return
+			}
+			go logic.RefreshSci(edbInfo, ``)
+		}
+	}
+}