fix:新增百川盈弗EXCEL处理

controllers/base_from_baiinfo.go

@@ -4,12 +4,13 @@ import (
 	"encoding/json"
 	"hongze/hongze_edb_lib/logic"
 	"hongze/hongze_edb_lib/models"
+	"hongze/hongze_edb_lib/services"
 	"hongze/hongze_edb_lib/utils"
 	"strconv"
 	"time"
 )
 
-//百川盈孚
+// 百川盈孚
 type BaiinfoController struct {
 	BaseAuthController
 }
@@ -130,3 +131,30 @@ func (this *BaiinfoController) Refresh() {
 	br.Success = true
 	br.Msg = "获取成功"
 }
+
+// HandleExcelData
+// @Title 处理卓创（红桃3）指标的接口
+// @Description 处理卓创（红桃3）指标的接口
+// @Success 200 {object} models.HandleMysteelIndexResp
+// @router /handle/excel_data [post]
+func (this *BaiinfoController) HandleExcelData() {
+	br := new(models.BaseResponse).Init()
+	defer func() {
+		this.Data["json"] = br
+		this.ServeJSON()
+	}()
+	var req models.HandleBaiinfoExcelDataReq
+	err := json.Unmarshal(this.Ctx.Input.RequestBody, &req)
+	if err != nil {
+		br.Msg = "参数解析异常！"
+		br.ErrMsg = "参数解析失败，Err:" + err.Error()
+		return
+	}
+
+	// 处理excel表数据
+	services.HandleBaiinfoIndex(req.BaseFilePath, req.RenameFilePath, req.IndexName, req.IndexCode, req.ExcelDataMap)
+
+	br.Ret = 200
+	br.Success = true
+	br.Msg = "处理成功"
+}

controllers/base_from_sci.go

@@ -97,7 +97,7 @@ func (this *SciController) Refresh() {
 		br.ErrMsg = "指标不存在"
 		return
 	}
-	isHandling, errMsg, err := logic.RefreshSci(edbInfo, req.StartDate)
+	isHandling, errMsg, err := logic.RefreshBaseEdbInfo(edbInfo, req.StartDate)
 	if isHandling {
 		br.Ret = 501
 		br.Success = true

logic/sic.go → logic/base_edb_info.go

@@ -8,8 +8,8 @@ import (
 	"time"
 )
 
-// RefreshSci 卓创（红桃3）刷新
-func RefreshSci(edbInfo *models.EdbInfo, startDate string) (isHandling bool, errMsg string, err error) {
+// RefreshBaseEdbInfo 刷新基础指标
+func RefreshBaseEdbInfo(edbInfo *models.EdbInfo, startDate string) (isHandling bool, errMsg string, err error) {
 	cacheKey := utils.CACHE_EDB_DATA_REFRESH + strconv.Itoa(edbInfo.Source) + "_" + edbInfo.EdbCode
 	if utils.Rc.IsExist(cacheKey) {
 		isHandling = true
@@ -20,7 +20,14 @@ func RefreshSci(edbInfo *models.EdbInfo, startDate string) (isHandling bool, err
 		utils.Rc.Delete(cacheKey)
 	}()
 
-	err = models.RefreshEdbDataFromSci(edbInfo.EdbInfoId, edbInfo.EdbCode, startDate)
+	switch edbInfo.Source {
+	case utils.DATA_SOURCE_SCI:
+		err = models.RefreshEdbDataFromSci(edbInfo.EdbInfoId, edbInfo.EdbCode, startDate)
+	case utils.DATA_SOURCE_BAIINFO:
+		err = models.RefreshEdbDataFromBaiinfo(edbInfo.EdbInfoId, edbInfo.EdbCode, startDate)
+	default:
+		return
+	}
 	if err != nil && err.Error() != utils.ErrNoRow() {
 		errMsg = "刷新指标信息失败！"
 		err = errors.New("刷新指标信息失败 RefreshEdbDataFromSci，Err:" + err.Error())

models/base_from_baiinfo.go

@@ -31,7 +31,7 @@ func GetBaseFromBaiinfoDataByCondition(condition string, pars []interface{}) (li
 	return
 }
 
-// 新增有色指标数据
+// AddEdbDataFromBaiinfo 新增百川盈弗指标数据
 func AddEdbDataFromBaiinfo(edbCode string) (err error) {
 	o := orm.NewOrm()
 
@@ -182,3 +182,74 @@ func RefreshEdbDataFromBaiinfo(edbInfoId int, edbCode, startDate string) (err er
 	}
 	return
 }
+
+// HandleBaiinfoExcelDataReq 百川盈弗的excel数据
+type HandleBaiinfoExcelDataReq struct {
+	BaseFilePath   string
+	RenameFilePath string
+	IndexName      string
+	IndexCode      string
+	ExcelDataMap   map[string]string
+}
+
+type BaseFromBaiinfoIndex struct {
+	BaseFromBaiinfoIndexId int64 `orm:"column(base_from_baiinfo_index_id);pk"`
+	IndexCode              string
+	IndexName              string
+	Frequency              string
+	Unit                   string
+	StartDate              string
+	EndDate                string
+	Sort                   int
+	BaseFileName           string
+	RenameFileName         string
+	CreateTime             time.Time
+	ModifyTime             time.Time
+}
+
+func (d *BaseFromBaiinfoIndex) AddBaseFromBaiinfoIndex() (lastId int64, err error) {
+	o := orm.NewOrm()
+	lastId, err = o.Insert(d)
+	return
+}
+
+func AddBaseFromBaiinfoData(item []*BaseFromBaiinfoData) (err error) {
+	o := orm.NewOrm()
+	_, err = o.InsertMulti(1, item)
+	return
+}
+
+func (d *BaseFromBaiinfoIndex) GetSmmIndexItem(indexCode string) (item *BaseFromBaiinfoIndex, err error) {
+	o := orm.NewOrm()
+	sql := ` SELECT * FROM base_from_baiinfo_index WHERE index_code=? `
+	err = o.Raw(sql, indexCode).QueryRow(&item)
+	return
+}
+
+func GetBaseFromBaiinfoDataByCode(indexCode string) (list []*BaseFromBaiinfoData, err error) {
+	o := orm.NewOrm()
+	sql := ` SELECT * FROM base_from_baiinfo_data WHERE index_code=? `
+	_, err = o.Raw(sql, indexCode).QueryRows(&list)
+	return
+}
+
+func ModifyBaiinfoIndexSort(baseFromBaiinfoIndexId int64) (err error) {
+	o := orm.NewOrm()
+	sql := ` UPDATE base_from_baiinfo_index SET sort=? WHERE base_from_baiinfo_index_id=? `
+	_, err = o.Raw(sql, baseFromBaiinfoIndexId, baseFromBaiinfoIndexId).Exec()
+	return
+}
+
+func GetBaiinfoIndexInfoMaxAndMinInfo(indexCode string) (item *EdbInfoMaxAndMinInfo, err error) {
+	o := orm.NewOrm()
+	sql := ` SELECT MIN(data_time) AS min_date,MAX(data_time) AS max_date,MIN(value) AS min_value,MAX(value) AS max_value FROM base_from_baiinfo_data WHERE index_code=? `
+	err = o.Raw(sql, indexCode).QueryRow(&item)
+	return
+}
+
+func ModifyBaiinfoIndexMaxAndMinInfo(indexCode string, item *EdbInfoMaxAndMinInfo) (err error) {
+	o := orm.NewOrm()
+	sql := ` UPDATE base_from_baiinfo_index SET start_date=?,end_date=?,modify_time=NOW() WHERE index_code=? `
+	_, err = o.Raw(sql, item.MinDate, item.MaxDate, indexCode).Exec()
+	return
+}

models/db.go

@@ -91,5 +91,7 @@ func initBaseIndex() {
 	orm.RegisterModel(
 		new(BaseFromSciIndex),
 		new(BaseFromSciData),
+		new(BaseFromBaiinfoIndex),
+		new(BaseFromBaiinfoData),
 	)
 }

routers/commentsRouter.go

@@ -43,6 +43,15 @@ func init() {
             Filters: nil,
             Params: nil})
 
+    beego.GlobalControllerRouter["hongze/hongze_edb_lib/controllers:BaiinfoController"] = append(beego.GlobalControllerRouter["hongze/hongze_edb_lib/controllers:BaiinfoController"],
+        beego.ControllerComments{
+            Method: "HandleExcelData",
+            Router: `/handle/excel_data`,
+            AllowHTTPMethods: []string{"post"},
+            MethodParams: param.Make(),
+            Filters: nil,
+            Params: nil})
+
     beego.GlobalControllerRouter["hongze/hongze_edb_lib/controllers:BaiinfoController"] = append(beego.GlobalControllerRouter["hongze/hongze_edb_lib/controllers:BaiinfoController"],
         beego.ControllerComments{
             Method: "Refresh",

services/base_from_baiinfo.go

@@ -0,0 +1,150 @@
+package services
+
+import (
+	"fmt"
+	"hongze/hongze_edb_lib/logic"
+	"hongze/hongze_edb_lib/models"
+	"hongze/hongze_edb_lib/services/alarm_msg"
+	"hongze/hongze_edb_lib/utils"
+	"strings"
+	"time"
+)
+
+// HandleBaiinfoIndex 处理百川盈弗的excel数据
+func HandleBaiinfoIndex(baseFilePath, renameFilePath, indexName, indexCode string, excelDataMap map[string]string) {
+	var err error
+
+	errMsgList := make([]string, 0)
+	defer func() {
+		if len(errMsgList) > 0 {
+			go alarm_msg.SendAlarmMsg(fmt.Sprint("红桃3实际数据处理失败,err:", strings.Join(errMsgList, "\n")), 3)
+		}
+	}()
+
+	indexObj := new(models.BaseFromBaiinfoIndex)
+	var indexId int64
+
+	addDataList := make([]*models.BaseFromBaiinfoData, 0)
+
+	exitDataMap := make(map[string]*models.BaseFromBaiinfoData)
+
+	// 修改指标信息
+	if indexName == "" {
+		utils.FileLog.Info("未刷新到指标数据:indexName:" + indexName)
+		return
+	}
+	//判断指标是否存在
+	var isAdd int
+	item, err := indexObj.GetSmmIndexItem(indexCode)
+	if err != nil {
+		if err.Error() == utils.ErrNoRow() {
+			isAdd = 1
+		} else {
+			isAdd = -1
+			fmt.Println("GetSmmIndexItem Err:" + err.Error())
+			return
+		}
+	}
+	if item != nil && item.BaseFromBaiinfoIndexId > 0 {
+		fmt.Println("item:", item)
+		isAdd = 2
+	} else {
+		isAdd = 1
+	}
+
+	if isAdd == 1 {
+		indexObj.IndexCode = indexCode
+		indexObj.IndexName = indexName
+		indexObj.ModifyTime = time.Now()
+		indexObj.CreateTime = time.Now()
+		indexObj.BaseFileName = baseFilePath
+		indexObj.RenameFileName = renameFilePath
+
+		lastId, err := indexObj.AddBaseFromBaiinfoIndex()
+		if err != nil {
+			fmt.Println("add err:" + err.Error())
+			return
+		}
+		indexId = lastId
+		//修改排序
+		go models.ModifyBaiinfoIndexSort(indexId)
+	} else if isAdd == 2 {
+		//获取已存在的所有数据
+		exitDataList, err := models.GetBaseFromBaiinfoDataByCode(indexCode)
+		if err != nil {
+			fmt.Println("GetIndexDataList Err:" + err.Error())
+			return
+		}
+		fmt.Println("exitDataListLen:", len(exitDataList))
+		for _, v := range exitDataList {
+			dateStr := v.DataTime
+			exitDataMap[dateStr] = v
+		}
+		indexId = item.BaseFromBaiinfoIndexId
+	}
+
+	// 遍历excel数据，然后跟现有的数据做校验，不存在则入库
+	for date, value := range excelDataMap {
+		if findData, ok := exitDataMap[date]; !ok {
+			_, err := time.ParseInLocation(utils.FormatDate, date, time.Local)
+			if err != nil {
+				fmt.Println("time.ParseInLocation Err:" + err.Error())
+				return
+			}
+			if !strings.Contains(value, "#N/A") {
+				var saveDataTime time.Time
+				if strings.Contains(date, "00:00:00") {
+					saveDataTime, err = time.Parse(utils.FormatDateTime, date)
+				} else {
+					saveDataTime, err = time.Parse(utils.FormatDate, date)
+				}
+				if err != nil {
+					errMsg := "saveDataTime parse err:" + err.Error() + " dataTime:" + date
+					fmt.Println(errMsg)
+					continue
+				}
+				timestamp := saveDataTime.UnixNano() / 1e6
+
+				dataItem := new(models.BaseFromBaiinfoData)
+				dataItem.BaseFromBaiinfoIndexId = int(indexId)
+				dataItem.IndexCode = indexCode
+				dataItem.DataTime = date
+				dataItem.Value = value
+				dataItem.CreateTime = time.Now()
+				dataItem.ModifyTime = time.Now()
+				dataItem.DataTimestamp = timestamp
+				addDataList = append(addDataList, dataItem)
+			}
+		} else {
+			if findData != nil && findData.Value != value && !strings.Contains(value, "#N/A") { //修改数据
+
+			}
+		}
+	}
+
+	if len(addDataList) > 0 {
+		err = models.AddBaseFromBaiinfoData(addDataList)
+		if err != nil {
+			fmt.Println("dataObj.Add() Err:" + err.Error())
+		}
+
+		item, err := models.GetBaiinfoIndexInfoMaxAndMinInfo(indexCode)
+		if err == nil && item != nil {
+			go models.ModifyBaiinfoIndexMaxAndMinInfo(indexCode, item)
+		}
+	}
+
+	// 同步刷新ETA图库红桃3的指标
+	{
+		// 获取指标详情
+		edbInfo, err := models.GetEdbInfoByEdbCode(utils.DATA_SOURCE_BAIINFO, indexCode)
+		if err != nil && err.Error() != utils.ErrNoRow() {
+			errMsgList = append(errMsgList, fmt.Sprint("刷新ETA指标异常，指标编码：", indexCode, err.Error()))
+		}
+
+		// 已经加入到指标库的话，那么就去更新ETA指标库吧
+		if edbInfo != nil {
+			go logic.RefreshBaseEdbInfo(edbInfo, ``)
+		}
+	}
+}

services/base_from_sci.go

@@ -66,12 +66,12 @@ func HandleSciIndex(indexNameList, thirdIndexIdList, frequencyList, unitList []s
 	}
 
 	// 红桃3实际数据处理
-	HandleData(dataMap)
+	HandleSciData(dataMap)
 
 }
 
 // HandleData 红桃3实际数据处理
-func HandleData(dataMap map[string]map[string]string) {
+func HandleSciData(dataMap map[string]map[string]string) {
 	errMsgList := make([]string, 0)
 	defer func() {
 		if len(errMsgList) > 0 {
@@ -172,7 +172,7 @@ func HandleData(dataMap map[string]map[string]string) {
 
 			// 已经加入到指标库的话，那么就去更新ETA指标库吧
 			if edbInfo != nil {
-				go logic.RefreshSci(edbInfo, ``)
+				go logic.RefreshBaseEdbInfo(edbInfo, ``)
 			}
 		}
 	}