Merge branch 'fix/future_good_ths' into debug

# Conflicts:
#	routers/router.go

controllers/base_from_smm.go

@@ -6,6 +6,7 @@ import (
 	"eta/eta_index_lib/models"
 	"eta/eta_index_lib/services"
 	"eta/eta_index_lib/utils"
+	"fmt"
 	"strconv"
 	"time"
 )
@@ -153,7 +154,12 @@ func (this *SmmController) HandleExcelData() {
 	}
 
 	// 处理excel表数据
-	services.SmmIndexHandle(req.BaseFilePath, req.RenameFilePath, req.IndexName, req.IndexCode, req.Unit, req.Frequency, req.Source, req.ExcelDataMap)
+	e, errMsg := services.SmmIndexHandle(req.BaseFilePath, req.RenameFilePath, req.IndexName, req.IndexCode, req.Unit, req.Frequency, req.Source, req.ExcelDataMap)
+	if e != nil {
+		br.Msg = "处理失败"
+		br.ErrMsg = fmt.Sprintf("处理有色excel表数据失败, Err: %s, ErrMsg: %s", e.Error(), errMsg)
+		return
+	}
 
 	br.Ret = 200
 	br.Success = true

controllers/fix/custom_analysis.go

@@ -17,7 +17,7 @@ var fixTable = false
 // @Title 指标详情接口
 // @Description  指标详情接口
 // @Success 200 {object} models.EdbInfoDetailReq
-// @router /fix/v1 [post]
+// @router /v1 [post]
 func (this *CustomAnalysisController) FixTableV1() {
 	br := new(models.BaseResponse).Init()
 	defer func() {

models/db.go

@@ -55,7 +55,6 @@ func init() {
 		new(EdbAdjustConf), // 数据调整的配置
 		new(EdbDataInsertConfig),
 		new(BaseFromMysteelChemicalClassify),
-		new(ExcelEdbMapping), //excel与指标的关系表
 	)
 
 	// 注册期货数据 数据表
@@ -117,6 +116,6 @@ func initExcel() {
 	orm.RegisterModel(
 		new(excel.ExcelInfo),
 		new(excel.ExcelClassify),
-		new(excel.ExcelEdbMapping),
+		new(excel.ExcelEdbMapping), //excel与指标的关系表
 	)
 }

models/edb_data_calculate_zdyfx.go

@@ -3,6 +3,7 @@ package models
 import (
 	"encoding/json"
 	"errors"
+	"eta/eta_index_lib/models/excel"
 	"eta/eta_index_lib/utils"
 	"fmt"
 	"github.com/beego/beego/v2/client/orm"
@@ -78,7 +79,7 @@ func (obj CustomAnalysis) Add(params AddCalculateBatchParams) (edbInfo *EdbInfo,
 
 	//关联关系
 	{
-		excelEdbMappingItem := new(ExcelEdbMapping)
+		excelEdbMappingItem := new(excel.ExcelEdbMapping)
 		excelEdbMappingItem.CreateTime = time.Now()
 		excelEdbMappingItem.ModifyTime = time.Now()
 		excelEdbMappingItem.ExcelInfoId = customAnalysisData.ExcelInfoId
@@ -142,7 +143,7 @@ func (obj CustomAnalysis) Edit(params EditCalculateBatchParams) (err error, errM
 	// 只有excel的id有传递，同时有数据的情况下，才去做处理
 	if customAnalysisData.ExcelInfoId > 0 && len(customAnalysisData.DateList) > 0 && len(customAnalysisData.DataList) > 0 {
 		//判断计算指标是否被更换
-		excelEdbMapping, tmpErr := GetExcelEdbMappingByEdbInfoId(edbInfo.EdbInfoId)
+		excelEdbMapping, tmpErr := excel.GetExcelEdbMappingByEdbInfoId(edbInfo.EdbInfoId)
 		if tmpErr != nil {
 			err = errors.New("获取excel与指标的关系失败,Err:" + tmpErr.Error())
 			return

models/excel/excel_edb_mapping.go

@@ -46,3 +46,12 @@ type CalculateFormula struct {
 	DateSequenceStr string `json:"DateSequenceStr"`
 	DataSequenceStr string `json:"DataSequenceStr"`
 }
+
+// GetExcelEdbMappingByEdbInfoId 根据指标id获取配置关系
+func GetExcelEdbMappingByEdbInfoId(edbInfoId int) (item *ExcelEdbMapping, err error) {
+	o := orm.NewOrm()
+	sql := ` SELECT *  FROM excel_edb_mapping WHERE 1=1 AND edb_info_id = ? `
+
+	err = o.Raw(sql, edbInfoId).QueryRow(&item)
+	return
+}

models/excel_edb_mapping.go

@@ -1,39 +0,0 @@
-package models
-
-import (
-	"github.com/beego/beego/v2/client/orm"
-	"time"
-)
-
-// ExcelEdbMapping excel与指标的关系表
-type ExcelEdbMapping struct {
-	ExcelEdbMappingId int       `orm:"column(excel_edb_mapping_id);pk"`
-	ExcelInfoId       int       `description:"excel的id"`
-	Source            int       `description:"表格来源，1：excel插件的表格，2：自定义表格，3：混合表格，4：自定义分析，默认：1"`
-	EdbInfoId         int       `description:"计算指标id"`
-	CreateTime        time.Time `description:"创建时间"`
-	ModifyTime        time.Time `description:"修改时间"`
-}
-
-// AddExcelEdbMappingMulti 批量添加excel与指标的关系
-func AddExcelEdbMappingMulti(items []*ExcelEdbMapping) (err error) {
-	o := orm.NewOrm()
-	_, err = o.InsertMulti(len(items), items)
-	return
-}
-
-// Add 添加excel与指标的关系
-func (e *ExcelEdbMapping) Add() (err error) {
-	o := orm.NewOrm()
-	_, err = o.Insert(e)
-	return
-}
-
-// GetExcelEdbMappingByEdbInfoId 根据指标id获取配置关系
-func GetExcelEdbMappingByEdbInfoId(edbInfoId int) (item *ExcelEdbMapping, err error) {
-	o := orm.NewOrm()
-	sql := ` SELECT *  FROM excel_edb_mapping WHERE 1=1 AND edb_info_id = ? `
-
-	err = o.Raw(sql, edbInfoId).QueryRow(&item)
-	return
-}

routers/commentsRouter.go

@@ -10,7 +10,7 @@ func init() {
     beego.GlobalControllerRouter["eta/eta_index_lib/controllers/fix:CustomAnalysisController"] = append(beego.GlobalControllerRouter["eta/eta_index_lib/controllers/fix:CustomAnalysisController"],
         beego.ControllerComments{
             Method: "FixTableV1",
-            Router: `/fix/v1`,
+            Router: `/v1`,
             AllowHTTPMethods: []string{"post"},
             MethodParams: param.Make(),
             Filters: nil,

services/base_from_smm.go

@@ -1,7 +1,6 @@
 package services
 
 import (
-	"errors"
 	"eta/eta_index_lib/logic"
 	"eta/eta_index_lib/models"
 	"eta/eta_index_lib/services/alarm_msg"
@@ -11,14 +10,17 @@ import (
 	"time"
 )
 
-func SmmIndexHandle(baseFilePath, renameFilePath, indexName, indexCode, unit, frequency, source string, excelDataMap map[string]string) {
-	var err error
-
-	errMsgList := make([]string, 0)
+func SmmIndexHandle(baseFilePath, renameFilePath, indexName, indexCode, unit, frequency, source string, excelDataMap map[string]string) (err error, errMsg string) {
+	messages := make([]string, 0)
 	defer func() {
-		if len(errMsgList) > 0 {
-			fmt.Println(fmt.Sprint("SMM有色实际数据处理失败,err:", strings.Join(errMsgList, "\n")))
-			go alarm_msg.SendAlarmMsg(fmt.Sprint("SMM有色实际数据处理失败,err:", strings.Join(errMsgList, "\n")), 3)
+		if len(messages) > 0 {
+			errMsg = strings.Join(messages, "\n")
+			utils.FileLog.Info("SmmIndexHandle ErrMsg: %s", errMsg)
+		}
+		if err != nil {
+			tips := fmt.Sprintf("SMM有色excel数据处理失败, Err: %s, ErrMsg: %s", err.Error(), errMsg)
+			utils.FileLog.Info(tips)
+			go alarm_msg.SendAlarmMsg(tips, 3)
 		}
 	}()
 
@@ -35,26 +37,24 @@ func SmmIndexHandle(baseFilePath, renameFilePath, indexName, indexCode, unit, fr
 		utils.FileLog.Info("未刷新到指标数据:indexName:" + indexName)
 		return
 	}
-	//判断指标是否存在
+	// 判断指标是否存在
 	var isAdd int
-	item, err := indexObj.GetSmmIndexItem(indexCode)
-	if err != nil {
-		if err.Error() == utils.ErrNoRow() {
+	item, e := indexObj.GetSmmIndexItem(indexCode)
+	if e != nil {
+		if e.Error() == utils.ErrNoRow() {
 			isAdd = 1
 		} else {
 			isAdd = -1
-			fmt.Println("GetSmmIndexItem Err:" + err.Error())
+			err = fmt.Errorf("GetSmmIndexItem Err: %s", e.Error())
 			return
 		}
 	}
 	if item != nil && item.BaseFromSmmIndexId > 0 {
-		fmt.Println("item:", item)
 		isAdd = 2
 	} else {
 		isAdd = 1
 	}
 
-	fmt.Println("isAdd:", isAdd)
 	if !strings.Contains(frequency, "度") {
 		frequency = frequency + "度"
 	}
@@ -69,18 +69,18 @@ func SmmIndexHandle(baseFilePath, renameFilePath, indexName, indexCode, unit, fr
 		indexObj.CreateTime = time.Now()
 		indexObj.BaseFileName = baseFilePath
 		indexObj.RenameFileName = renameFilePath
-		lastId, err := indexObj.AddBaseFromSmmIndex()
-		if err != nil {
-			fmt.Println("add err:" + err.Error())
+		lastId, e := indexObj.AddBaseFromSmmIndex()
+		if e != nil {
+			err = fmt.Errorf("AddBaseFromSmmIndex err: %s", e.Error())
 			return
 		}
 		indexId = lastId
 		go models.ModifySmmIndexSort(indexId)
 	} else if isAdd == 2 {
 		//获取已存在的所有数据
-		exitDataList, err := models.GetBaseFromSmmDataBySmmCode(indexCode)
-		if err != nil {
-			fmt.Println("GetIndexDataList Err:" + err.Error())
+		exitDataList, e := models.GetBaseFromSmmDataBySmmCode(indexCode)
+		if e != nil {
+			err = fmt.Errorf("GetBaseFromSmmDataBySmmCode err: %s", e.Error())
 			return
 		}
 		fmt.Println("exitDataListLen:", len(exitDataList))
@@ -101,9 +101,9 @@ func SmmIndexHandle(baseFilePath, renameFilePath, indexName, indexCode, unit, fr
 		whereParam["index_code"] = indexCode
 
 		smmIndexObj := new(models.BaseFromSmmIndex)
-		err = smmIndexObj.Update(updateParams, whereParam)
-		if err != nil {
-			err = errors.New("smm index update err:" + err.Error())
+		e = smmIndexObj.Update(updateParams, whereParam)
+		if e != nil {
+			err = fmt.Errorf("smm index update err: %s", e.Error())
 			return
 		}
 	}
@@ -112,20 +112,27 @@ func SmmIndexHandle(baseFilePath, renameFilePath, indexName, indexCode, unit, fr
 	for date, value := range excelDataMap {
 		dateTime, e := time.ParseInLocation(utils.FormatDate, date, time.Local)
 		if e != nil {
-			fmt.Println("time.ParseInLocation Err:" + e.Error())
-			return
+			messages = append(messages, fmt.Sprintf("data time parse err: %s, date: %s", e.Error(), date))
+			continue
 		}
 		if _, ok := exitDataMap[date]; !ok {
 			if !strings.Contains(value, "#N/A") {
 				var saveDataTime time.Time
 				if strings.Contains(date, "00:00:00") {
-					saveDataTime, err = time.Parse(utils.FormatDateTime, date)
+					saveDataTime, e = time.Parse(utils.FormatDateTime, date)
+					if e != nil {
+						messages = append(messages, fmt.Sprintf("save data time parse err: %s, date: %s", e.Error(), date))
+						continue
+					}
 				} else {
-					saveDataTime, err = time.Parse(utils.FormatDate, date)
+					saveDataTime, e = time.Parse(utils.FormatDate, date)
+					if e != nil {
+						messages = append(messages, fmt.Sprintf("save data time parse err: %s, date: %s", e.Error(), date))
+						continue
+					}
 				}
-				if err != nil {
-					errMsg := "saveDataTime parse err:" + err.Error() + " dataTime:" + date
-					fmt.Println(errMsg)
+				if saveDataTime.IsZero() {
+					messages = append(messages, fmt.Sprintf("save data time empty"))
 					continue
 				}
 				timestamp := saveDataTime.UnixNano() / 1e6
@@ -142,7 +149,12 @@ func SmmIndexHandle(baseFilePath, renameFilePath, indexName, indexCode, unit, fr
 			}
 		} else {
 			// 更新对应日期数据
-			if strings.Contains(value, "#N/A") {
+			if value == "" || strings.Contains(value, "#N/A") {
+				continue
+			}
+			// 若数值相同则忽略更新
+			existData := exitDataMap[date]
+			if existData != nil && existData.Value == value {
 				continue
 			}
 
@@ -155,38 +167,40 @@ func SmmIndexHandle(baseFilePath, renameFilePath, indexName, indexCode, unit, fr
 		}
 	}
 
+	//fmt.Println("addDataList len: ", len(addDataList))
 	if len(addDataList) > 0 {
-		err = models.AddBaseFromSmmData(addDataList)
-		if err != nil {
-			fmt.Println("AddBaseFromSmmData Err:" + err.Error())
-			errMsgList = append(errMsgList, "AddBaseFromSmmData err: "+err.Error())
+		e = models.AddBaseFromSmmData(addDataList)
+		if e != nil {
+			err = fmt.Errorf("AddBaseFromSmmData err: %s", e.Error())
+			return
 		}
 	}
 
 	// 已存在的日期数据更新
+	//fmt.Println("updateDataList len: ", len(updateDataList))
 	if len(updateDataList) > 0 {
-		err = models.MultiUpdateBaseFromSmmDataValue(updateDataList)
-		if err != nil {
-			fmt.Println("MultiUpdateBaseFromSmmDataValue Err:" + err.Error())
-			errMsgList = append(errMsgList, "MultiUpdateBaseFromSmmDataValue err: "+err.Error())
+		e = models.MultiUpdateBaseFromSmmDataValue(updateDataList)
+		if e != nil {
+			err = fmt.Errorf("MultiUpdateBaseFromSmmDataValue err: %s", e.Error())
+			return
 		}
 	}
 
 	itemInfo, err := models.GetSmmIndexInfoMaxAndMinInfo(indexCode)
 	if err == nil && item != nil {
-		e := models.ModifySmmIndexMaxAndMinInfo(indexCode, itemInfo)
+		e = models.ModifySmmIndexMaxAndMinInfo(indexCode, itemInfo)
 		if e != nil {
-			fmt.Println("ModifySmmIndexMaxAndMinInfo Err:" + e.Error())
-			errMsgList = append(errMsgList, "ModifySmmIndexMaxAndMinInfo err: "+e.Error())
+			err = fmt.Errorf("ModifySmmIndexMaxAndMinInfo err: %s", e.Error())
+			return
 		}
 	}
 
 	// 同步刷新ETA图库有色的指标
 	{
 		// 获取指标详情
-		edbInfo, err := models.GetEdbInfoByEdbCode(utils.DATA_SOURCE_YS, indexCode)
-		if err != nil && err.Error() != utils.ErrNoRow() {
-			errMsgList = append(errMsgList, fmt.Sprint("刷新ETA指标异常，指标编码：", indexCode, err.Error()))
+		edbInfo, e := models.GetEdbInfoByEdbCode(utils.DATA_SOURCE_YS, indexCode)
+		if e != nil && e.Error() != utils.ErrNoRow() {
+			messages = append(messages, fmt.Sprintf("刷新ETA指标异常, indexCode: %s, err: %s", indexCode, e.Error()))
 		}
 
 		// 已经加入到指标库的话，那么就去更新ETA指标库吧
@@ -194,4 +208,5 @@ func SmmIndexHandle(baseFilePath, renameFilePath, indexName, indexCode, unit, fr
 			go logic.RefreshBaseEdbInfo(edbInfo, ``)
 		}
 	}
+	return
 }

services/base_from_ths.go

@@ -159,7 +159,7 @@ type FutureGoodDataFromThsInterface struct {
 }
 
 func GetFutureGoodDataFromThs(edbCode, startDate, endDate string) (item future_good.FutureGoodDataFromThs, err error) {
-	if utils.RunMode == `release` { // 生产环境走官方http请求，测试环境走终端
+	if utils.ThsDataMethod == "" || utils.ThsDataMethod == "api" { // 生产环境走官方http请求，测试环境走终端
 		return getFutureGoodDataFromThsHttp(edbCode, startDate, endDate)
 	} else {
 		return getFutureGoodDataFromThsApp(edbCode, startDate, endDate, 0)