Merge remote-tracking branch 'origin/aj_mysteel'

controllers/base_auth.go

@@ -50,6 +50,7 @@ func (this *BaseAuthController) Prepare() {
 				return
 			}
 			checkAuthorization := utils.MD5(utils.APP_NAME_EN + utils.Md5Key)
+			fmt.Println(checkAuthorization)
 			if authorization != checkAuthorization {
 				this.JSON(models.BaseResponse{Ret: 408, Msg: "签名错误！", ErrMsg: "签名错误:authorization is err "}, false, false)
 				this.StopRun()

controllers/base_from_calculate.go

@@ -138,7 +138,7 @@ func (this *CalculateController) Add() {
 		formulaStr += v.FromTag + ","
 		edbInfoIdBytes = append(edbInfoIdBytes, v.FromTag)
 	}
-	formulaMap := services.CheckFormula(req.CalculateFormula)
+	formulaMap := utils.CheckFormula(req.CalculateFormula)
 	for _, v := range formulaMap {
 		if !strings.Contains(formulaStr, v) {
 			br.Msg = "公式错误，请重新填写"
@@ -402,7 +402,7 @@ func (this *CalculateController) Edit() {
 		}
 	}
 
-	formulaMap := services.CheckFormula(req.CalculateFormula)
+	formulaMap := utils.CheckFormula(req.CalculateFormula)
 	for _, v := range formulaMap {
 		if !strings.Contains(formulaStr, v) {
 			br.Msg = "公式错误，请重新填写"

controllers/base_from_mysteel.go

@@ -9,7 +9,7 @@ import (
 	"time"
 )
 
-//钢联
+// 钢联
 type MySteelController struct {
 	BaseAuthController
 }

controllers/base_from_mysteel_chemical.go

@@ -4,6 +4,7 @@ import (
 	"encoding/json"
 	"hongze/hongze_edb_lib/logic"
 	"hongze/hongze_edb_lib/models"
+	"hongze/hongze_edb_lib/services"
 	"hongze/hongze_edb_lib/utils"
 	"strconv"
 	"time"
@@ -128,3 +129,51 @@ func (this *MySteelChemicalController) Refresh() {
 	br.Success = true
 	br.Msg = "获取成功"
 }
+
+// @Title 查询刷新钢联指标的接口
+// @Description 查询刷新钢联指标的接口
+// @Success 200 {object} models.CheckDeleteClassifyResp
+// @router /query/refresh [post]
+func (this *MySteelChemicalController) QueryRefresh() {
+	br := new(models.BaseResponse).Init()
+	defer func() {
+		this.Data["json"] = br
+		this.ServeJSON()
+	}()
+	mysteelIndex := new(models.BaseFromMysteelChemicalIndex)
+	list, err := mysteelIndex.GetIndexRefreshAllByMergeFile()
+	if err != nil && err.Error() != utils.ErrNoRow() {
+		br.Msg = "获取失败！"
+		br.ErrMsg = "获取失败，Err:" + err.Error()
+		return
+	}
+	br.Ret = 200
+	br.Data = list
+	br.Success = true
+	br.Msg = "获取成功"
+}
+
+// @Title 处理钢联指标的接口
+// @Description 处理钢联指标的接口
+// @Success 200 {object} models.HandleMysteelIndexResp
+// @router /handle/mysteel/index [post]
+func (this *MySteelChemicalController) HandleMysteelIndex() {
+	br := new(models.BaseResponse).Init()
+	defer func() {
+		this.Data["json"] = br
+		this.ServeJSON()
+	}()
+	var req models.HandleMysteelIndexResp
+	err := json.Unmarshal(this.Ctx.Input.RequestBody, &req)
+	if err != nil {
+		br.Msg = "参数解析异常！"
+		br.ErrMsg = "参数解析失败，Err:" + err.Error()
+		return
+	}
+
+	services.HandleMysteelIndex(&req)
+
+	br.Ret = 200
+	br.Success = true
+	br.Msg = "处理成功"
+}

controllers/base_from_predict_calculate.go

@@ -5,7 +5,6 @@ import (
 	"fmt"
 	"hongze/hongze_edb_lib/logic"
 	"hongze/hongze_edb_lib/models"
-	"hongze/hongze_edb_lib/services"
 	"hongze/hongze_edb_lib/utils"
 	"strconv"
 	"strings"
@@ -107,7 +106,7 @@ func addPredictCalculate(br *models.BaseResponse, req models.EdbInfoCalculateSav
 		formulaStr += v.FromTag + ","
 		edbInfoIdBytes = append(edbInfoIdBytes, v.FromTag)
 	}
-	formulaMap := services.CheckFormula(req.CalculateFormula)
+	formulaMap := utils.CheckFormula(req.CalculateFormula)
 	for _, v := range formulaMap {
 		if !strings.Contains(formulaStr, v) {
 			br.Msg = "公式错误，请重新填写"
@@ -335,7 +334,7 @@ func editPredictCalculate(br *models.BaseResponse, req models.EdbInfoCalculateSa
 		formulaStr += v.FromTag + ","
 		edbInfoIdBytes = append(edbInfoIdBytes, v.FromTag)
 	}
-	formulaMap := services.CheckFormula(req.CalculateFormula)
+	formulaMap := utils.CheckFormula(req.CalculateFormula)
 	for _, v := range formulaMap {
 		if !strings.Contains(formulaStr, v) {
 			br.Msg = "公式错误，请重新填写"

controllers/base_from_sci.go

@@ -4,16 +4,18 @@ import (
 	"encoding/json"
 	"hongze/hongze_edb_lib/logic"
 	"hongze/hongze_edb_lib/models"
+	"hongze/hongze_edb_lib/services"
 	"hongze/hongze_edb_lib/utils"
 	"strconv"
 	"time"
 )
 
-//有色
+// SciController 卓创（红桃3）
 type SciController struct {
 	BaseAuthController
 }
 
+// Add
 // @Title 新增Sci指标接口
 // @Description 新增Sci指标接口
 // @Success 200 {object} models.AddEdbInfoReq
@@ -58,13 +60,13 @@ func (this *SciController) Add() {
 	}
 }
 
+// Refresh
 // @Title 刷新sci指标接口
 // @Description 刷新sci指标接口
 // @Success 200 {object} models.RefreshEdbInfoReq
 // @router /refresh [post]
 func (this *SciController) Refresh() {
 	br := new(models.BaseResponse).Init()
-	var cacheKey string
 	defer func() {
 		this.Data["json"] = br
 		this.ServeJSON()
@@ -95,38 +97,47 @@ func (this *SciController) Refresh() {
 		br.ErrMsg = "指标不存在"
 		return
 	}
-	cacheKey = utils.CACHE_EDB_DATA_REFRESH + strconv.Itoa(source) + "_" + req.EdbCode
-	if utils.Rc.IsExist(cacheKey) {
+	isHandling, errMsg, err := logic.RefreshSci(edbInfo, req.StartDate)
+	if isHandling {
 		br.Ret = 501
 		br.Success = true
 		br.Msg = "系统处理中，请稍后重试"
 		return
 	}
-	utils.Rc.SetNX(cacheKey, 1, 1*time.Minute)
-	defer func() {
-		utils.Rc.Delete(cacheKey)
-	}()
-	if req.EdbInfoId <= 0 {
-		req.EdbInfoId = edbInfo.EdbInfoId
-	}
-	err = models.RefreshEdbDataFromSci(req.EdbInfoId, req.EdbCode, req.StartDate)
-	if err != nil && err.Error() != utils.ErrNoRow() {
-		br.Msg = "刷新指标信息失败！"
-		br.ErrMsg = "刷新指标信息失败 RefreshEdbDataFromSci，Err:" + err.Error()
-		return
-	}
-	// 更新指标最大最小值
-	err, errMsg := models.UnifiedModifyEdbInfoMaxAndMinInfo(edbInfo)
 	if err != nil {
 		br.Msg = errMsg
 		br.ErrMsg = err.Error()
 		return
 	}
 
-	// 更新ES
-	go logic.UpdateEs(edbInfo.EdbInfoId)
-
 	br.Ret = 200
 	br.Success = true
 	br.Msg = "获取成功"
 }
+
+// HandleExcelData
+// @Title 处理卓创（红桃3）指标的接口
+// @Description 处理卓创（红桃3）指标的接口
+// @Success 200 {object} models.HandleMysteelIndexResp
+// @router /handle/excel_data [post]
+func (this *SciController) HandleExcelData() {
+	br := new(models.BaseResponse).Init()
+	defer func() {
+		this.Data["json"] = br
+		this.ServeJSON()
+	}()
+	var req models.HandleSciExcelDataReq
+	err := json.Unmarshal(this.Ctx.Input.RequestBody, &req)
+	if err != nil {
+		br.Msg = "参数解析异常！"
+		br.ErrMsg = "参数解析失败，Err:" + err.Error()
+		return
+	}
+
+	// 处理excel表数据
+	services.HandleSciIndex(req.IndexNameList, req.ThirdIndexIdList, req.FrequencyList, req.UnitList, req.DataMap)
+
+	br.Ret = 200
+	br.Success = true
+	br.Msg = "处理成功"
+}

controllers/base_from_ths.go

@@ -10,7 +10,7 @@ import (
 	"time"
 )
 
-//同花顺
+// 同花顺
 type ThsController struct {
 	BaseAuthController
 }
@@ -141,3 +141,27 @@ func (this *ThsController) Refresh() {
 	br.Success = true
 	br.Msg = "获取成功"
 }
+
+//func init() {
+//	//?EdbCode=s005696248&StartDate=2023-02-03&EndDate=2027-03-23
+//	//edbCode := `s005696248`
+//	//startDate := `2023-02-03`
+//	//endDate := `2027-03-23`
+//	//EdbCode=S011292460&StartDate=1993-03-23&EndDate=2027-03-23
+//	edbCode := `S011292460`
+//	startDate := `1993-03-23`
+//	endDate := `2027-03-23`
+//	//edbCode := `@CL0W.NMX`
+//	//startDate := `20221218`
+//	//endDate := `20230118`
+//	list, err := services.GetEdbDataFromThsHttp(edbCode, startDate, endDate, 0)
+//	//list, err := services.GetFutureGoodDataFromThsHttp(edbCode, startDate, endDate)
+//
+//	//token, err := services.GetAccessToken()
+//	if err != nil {
+//		fmt.Println("err:", err)
+//		return
+//	}
+//	fmt.Println(list)
+//	//fmt.Println(token)
+//}

controllers/future_good/future_good_edb_info.go

@@ -70,14 +70,14 @@ func (this *FutureGoodEdbInfoController) Add() {
 		}
 		endDate := time.Now().Format(utils.FormatDate)
 
-		dataItem, err := services.GetFutureGoodDataFromThs(req.EdbCode, utils.BASE_START_DATE, endDate, 0)
+		dataItem, err := services.GetFutureGoodDataFromThs(req.EdbCode, utils.BASE_START_DATE, endDate)
 		if err != nil {
 			br.Msg = "获取指标信息失败！"
-			br.ErrMsg = "获取指标信息失败 GetEdbDataFromWind，Err:" + err.Error()
+			br.ErrMsg = "获取指标信息失败 GetFutureGoodDataFromThs，Err:" + err.Error()
 			return
 		}
 
-		err = future_good.AddEdbDataFromWind(futureGoodEdbInfo.FutureGoodEdbInfoId, req.EdbCode, dataItem)
+		err = future_good.AddEdbDataFromThs(futureGoodEdbInfo.FutureGoodEdbInfoId, req.EdbCode, dataItem)
 		if err != nil {
 			br.Msg = "获取指标信息失败！"
 			br.ErrMsg = "获取指标信息失败 AddEdbDataFromWind，Err:" + err.Error()
@@ -169,10 +169,10 @@ func (this *FutureGoodEdbInfoController) Refresh() {
 	if startDate == `` { // 如果传入的日期为空的话,那么就默认兼容一周的数据吧
 		startDate = endDateTime.AddDate(0, 0, -7).Format(utils.FormatDate)
 	}
-	dataItem, err := services.GetFutureGoodDataFromThs(req.FutureGoodEdbCode, req.StartDate, endDate, 0)
+	dataItem, err := services.GetFutureGoodDataFromThs(req.FutureGoodEdbCode, req.StartDate, endDate)
 	if err != nil {
 		br.Msg = "获取指标信息失败！"
-		br.ErrMsg = "获取指标信息失败 GetEdbDataFromWind，Err:" + err.Error()
+		br.ErrMsg = "获取指标信息失败 GetFutureGoodDataFromThs，Err:" + err.Error()
 		return
 	}
 	err = future_good.RefreshFutureGoodEdbDataFromThs(req.FutureGoodEdbInfoId, req.FutureGoodEdbCode, req.StartDate, dataItem)

logic/predict_edb.go

@@ -3,7 +3,6 @@ package logic
 import (
 	"errors"
 	"hongze/hongze_edb_lib/models"
-	"hongze/hongze_edb_lib/services"
 	"hongze/hongze_edb_lib/utils"
 	"strconv"
 	"strings"
@@ -196,7 +195,7 @@ func AddPredictEdbInfo(sourceEdbInfoId, classifyId int, edbName string, ruleList
 				formulaStr += tmpEdbInfoId.FromTag + ","
 				edbInfoIdBytes = append(edbInfoIdBytes, tmpEdbInfoId.FromTag)
 			}
-			formulaMap := services.CheckFormula(formula)
+			formulaMap := utils.CheckFormula(formula)
 			for _, formula := range formulaMap {
 				if !strings.Contains(formulaStr, formula) {
 					errMsg = "公式错误，请重新填写"
@@ -562,7 +561,7 @@ func EditPredictEdbInfo(edbInfoId, classifyId int, edbName string, ruleList []mo
 				formulaStr += tmpEdbInfoId.FromTag + ","
 				edbInfoIdBytes = append(edbInfoIdBytes, tmpEdbInfoId.FromTag)
 			}
-			formulaMap := services.CheckFormula(formula)
+			formulaMap := utils.CheckFormula(formula)
 			for _, formula := range formulaMap {
 				if !strings.Contains(formulaStr, formula) {
 					errMsg = "公式错误，请重新填写"
@@ -865,7 +864,7 @@ func RefreshPredictEdbInfo(edbInfoId int) (edbInfo *models.EdbInfo, err error, e
 				formulaStr += tmpEdbInfoId.FromTag + ","
 				edbInfoIdBytes = append(edbInfoIdBytes, tmpEdbInfoId.FromTag)
 			}
-			formulaMap := services.CheckFormula(formula)
+			formulaMap := utils.CheckFormula(formula)
 			for _, formula := range formulaMap {
 				if !strings.Contains(formulaStr, formula) {
 					errMsg = "公式错误，请重新填写"

logic/sic.go

@@ -0,0 +1,39 @@
+package logic
+
+import (
+	"errors"
+	"hongze/hongze_edb_lib/models"
+	"hongze/hongze_edb_lib/utils"
+	"strconv"
+	"time"
+)
+
+// RefreshSci 卓创（红桃3）刷新
+func RefreshSci(edbInfo *models.EdbInfo, startDate string) (isHandling bool, errMsg string, err error) {
+	cacheKey := utils.CACHE_EDB_DATA_REFRESH + strconv.Itoa(edbInfo.Source) + "_" + edbInfo.EdbCode
+	if utils.Rc.IsExist(cacheKey) {
+		isHandling = true
+		return
+	}
+	utils.Rc.SetNX(cacheKey, 1, 1*time.Minute)
+	defer func() {
+		utils.Rc.Delete(cacheKey)
+	}()
+
+	err = models.RefreshEdbDataFromSci(edbInfo.EdbInfoId, edbInfo.EdbCode, startDate)
+	if err != nil && err.Error() != utils.ErrNoRow() {
+		errMsg = "刷新指标信息失败！"
+		err = errors.New("刷新指标信息失败 RefreshEdbDataFromSci，Err:" + err.Error())
+		return
+	}
+	// 更新指标最大最小值
+	err, errMsg = models.UnifiedModifyEdbInfoMaxAndMinInfo(edbInfo)
+	if err != nil {
+		return
+	}
+
+	// 更新ES
+	go UpdateEs(edbInfo.EdbInfoId)
+
+	return
+}

models/base_from_adjust.go

@@ -4,7 +4,6 @@ import (
 	"errors"
 	"fmt"
 	"github.com/beego/beego/v2/client/orm"
-	"hongze/hongze_edb_lib/services/alarm_msg"
 	"hongze/hongze_edb_lib/utils"
 	"strconv"
 	"strings"
@@ -41,8 +40,6 @@ func SaveAdjustEdb(req SaveAdjustEdbReq) (edbInfo *EdbInfo, err error, errMsg st
 	defer func() {
 		if err != nil {
 			_ = to.Rollback()
-			fmt.Println("SaveAdjustEdb,Err:" + err.Error())
-			go alarm_msg.SendAlarmMsg(" 保存数据调整指标失败:err:"+err.Error(), 3)
 		} else {
 			_ = to.Commit()
 		}

models/base_from_calculate.go

@@ -6,7 +6,6 @@ import (
 	"github.com/beego/beego/v2/client/orm"
 	"github.com/shopspring/decimal"
 	"github.com/yidane/formula"
-	"hongze/hongze_edb_lib/services"
 	"hongze/hongze_edb_lib/utils"
 	"strconv"
 	"strings"
@@ -72,7 +71,7 @@ func AddCalculate(edbInfoIdArr []*EdbInfo, edbInfoId int, edbCode, formulaStr st
 		item.EdbInfoId = v.EdbInfoId
 		item.DataMap = dataMap
 	}
-	formulaMap := services.CheckFormula(formulaStr)
+	formulaMap := utils.CheckFormula(formulaStr)
 	addSql := ` INSERT INTO edb_data_calculate(edb_info_id,edb_code,data_time,value,create_time,modify_time,data_timestamp) values `
 	nowStr := time.Now().Format(utils.FormatDateTime)
 	var isAdd bool
@@ -415,7 +414,7 @@ func refreshAllCalculate(to orm.TxOrmer, edbInfoIdArr []*EdbInfo, edbInfoId, sou
 	//数据处理，将日期内不全的数据做补全
 	handleDateSaveDataMap(dateList, realSaveDataMap, saveDataMap, edbInfoIdArr)
 
-	formulaMap := services.CheckFormula(formulaStr)
+	formulaMap := utils.CheckFormula(formulaStr)
 	addSql := ` INSERT INTO edb_data_calculate(edb_info_id,edb_code,data_time,value,create_time,modify_time,data_timestamp) values `
 	var isAdd bool
 

models/base_from_fubao.go

@@ -3,15 +3,31 @@ package models
 import (
 	"fmt"
 	"github.com/beego/beego/v2/client/orm"
-	"hongze/hongze_edb_lib/services"
 	"hongze/hongze_edb_lib/utils"
 	"strconv"
 	"strings"
 	"time"
 )
 
+type FuBaoData struct {
+	ReturnValue struct {
+		Product struct {
+			AlikeName    string `json:"alikeName"`
+			DiffName     string `json:"diffName"`
+			PropValueStr string `json:"propValueStr"`
+			Pattern      string `json:"pattern"`
+		} `json:"product"`
+		Data []struct {
+			PriceStr string `json:"priceStr"`
+			Uad      string `json:"uad"`
+			Datadate string `json:"datadate"`
+		} `json:"data"`
+	} `json:"returnValue"`
+	Code int `json:"code"`
+}
+
 // 新增富宝指标数据
-func AddEdbDataFromFubao(edbCode string, fubaoData *services.FuBaoData) (err error) {
+func AddEdbDataFromFubao(edbCode string, fubaoData *FuBaoData) (err error) {
 	o := orm.NewOrm()
 	var isAdd bool
 	addSql := ` INSERT INTO edb_data_fubao(edb_info_id,edb_code,data_time,value,create_time,modify_time,data_timestamp) values `
@@ -41,7 +57,7 @@ func AddEdbDataFromFubao(edbCode string, fubaoData *services.FuBaoData) (err err
 }
 
 // 刷新路透指标数据
-func RefreshEdbDataFromFubao(edbInfoId int, edbCode, startDate string, fubaoData *services.FuBaoData) (err error) {
+func RefreshEdbDataFromFubao(edbInfoId int, edbCode, startDate string, fubaoData *FuBaoData) (err error) {
 	source := utils.DATA_SOURCE_FUBAO
 	o := orm.NewOrm()
 

models/base_from_mysteel_chemical.go

@@ -197,3 +197,107 @@ func RefreshEdbDataFromMysteelChemical(edbInfoId int, edbCode, startDate string)
 	}
 	return
 }
+
+// 钢联化工指标数据
+type BaseFromMysteelChemicalIndex struct {
+	BaseFromMysteelChemicalIndexId int64     `orm:"column(base_from_mysteel_chemical_index_id);pk"`
+	IndexCode                      string    `description:"指标编码"`
+	IndexName                      string    `description:"指标名称"`
+	Unit                           string    `description:"单位"`
+	Source                         string    `description:"数据来源"`
+	Frequency                      string    `description:"频度"`
+	StartDate                      time.Time `description:"开始日期"`
+	EndDate                        time.Time `description:"结束日期"`
+	Describe                       string    `description:"指标描述"`
+	UpdateWeek                     string    `description:"更新周期"`
+	UpdateTime                     string    `description:"更新时间，多个时间点用英文,隔开"`
+	UpdateTime2                    string    `description:"更新时间2"`
+	SysUserId                      int       `description:"创建人id"`
+	SysUserRealName                string    `description:"创建人姓名"`
+	FilePath                       string    `description:"文件存储路径"`
+	MergeFilePath                  string    `description:"更新文件"`
+	FileIndex                      int       `description:"文件索引"`
+	MergeUpdateWeek                string    `description:"合并文件的更新周"`
+	UpdateDate                     string    `description:"更新日期"`
+	CreateTime                     time.Time `description:"创建时间"`
+	ModifyTime                     time.Time `description:"修改时间"`
+}
+
+// GetIndexRefreshAllByMergeFile 根据合并文件去分组查询需要刷新的文件
+func (d *BaseFromMysteelChemicalIndex) GetIndexRefreshAllByMergeFile() (items []*BaseFromMysteelChemicalIndex, err error) {
+	o := orm.NewOrm()
+	sql := `SELECT * FROM base_from_mysteel_chemical_index WHERE merge_file_path <>'' AND base_from_mysteel_chemical_classify_id NOT IN(54,55,56,57)`
+	_, err = o.Raw(sql).QueryRows(&items)
+	return
+}
+
+type HandleMysteelIndex struct {
+	IndexName    string `description:"指标名称"`
+	IndexCode    string `description:"指标编码"`
+	Unit         string `description:"单位"`
+	Source       string `description:"数据来源"`
+	Frequency    string `description:"频度"`
+	StartDate    string `description:"开始日期"`
+	EndDate      string `description:"结束日期"`
+	Describe     string `description:"指标描述"`
+	UpdateDate   string `description:"更新日期"`
+	ExcelDataMap map[string]string
+}
+
+type HandleMysteelIndexResp struct {
+	List []*HandleMysteelIndex
+}
+
+// 钢联化工指标数据
+type BaseFromMysteelChemicalData struct {
+	BaseFromMysteelChemicalDataId  int64 `orm:"column(base_from_mysteel_chemical_data_id);pk"`
+	BaseFromMysteelChemicalIndexId int64
+	IndexCode                      string
+	DataTime                       time.Time
+	Value                          string
+	UpdateDate                     string
+	CreateTime                     time.Time `description:"创建时间"`
+	ModifyTime                     time.Time `description:"修改时间"`
+}
+
+func (d *BaseFromMysteelChemicalIndex) GetIndexItem(indexCode string) (item *BaseFromMysteelChemicalIndex, err error) {
+	o := orm.NewOrm()
+	sql := `SELECT * FROM base_from_mysteel_chemical_index WHERE index_code = ? `
+	err = o.Raw(sql, indexCode).QueryRow(&item)
+	return
+}
+
+// 新增
+func (r *BaseFromMysteelChemicalIndex) Add() (err error) {
+	o := orm.NewOrm()
+	_, err = o.Insert(r)
+	return
+}
+
+func (m *BaseFromMysteelChemicalIndex) Update(cols []string) (err error) {
+	o := orm.NewOrm()
+	_, err = o.Update(m, cols...)
+	return
+}
+
+// GetIndexDataList 根据指标编码获取数据
+func (d *BaseFromMysteelChemicalData) GetIndexDataList(indexCode string) (item []*BaseFromMysteelChemicalData, err error) {
+	o := orm.NewOrm()
+	sql := `SELECT * FROM base_from_mysteel_chemical_data WHERE index_code = ? `
+	_, err = o.Raw(sql, indexCode).QueryRows(&item)
+	return
+}
+
+// Update 修改
+func (r *BaseFromMysteelChemicalData) Update(updateCols []string) (err error) {
+	o := orm.NewOrm()
+	_, err = o.Update(r, updateCols...)
+	return
+}
+
+// Add 新增
+func (r *BaseFromMysteelChemicalData) Add(list []BaseFromMysteelChemicalData) (err error) {
+	o := orm.NewOrm()
+	_, err = o.InsertMulti(len(list), list)
+	return
+}

models/base_from_pb.go

@@ -3,25 +3,21 @@ package models
 import (
 	"fmt"
 	"github.com/beego/beego/v2/client/orm"
-	"hongze/hongze_edb_lib/services"
-	"hongze/hongze_edb_lib/services/alarm_msg"
 	"hongze/hongze_edb_lib/utils"
 	"strconv"
 	"strings"
 	"time"
 )
 
-func AddEdbDataFromPb(edbCode string, item *services.EdbDataFromPb) (err error) {
-	var errMsg string
-	o := orm.NewOrm()
-	defer func() {
-		if err != nil {
-			//go utils.SendEmail(utils.APP_NAME_CN+"【"+utils.RunMode+"】"+"失败提醒", " 彭博数据获取失败:err:"+errMsg, utils.EmailSendToUsers)
-			go alarm_msg.SendAlarmMsg("彭博数据获取失败:err:"+errMsg, 3)
-
-		}
-	}()
+type EdbDataFromPb struct {
+	Date   map[string]int64   `json:"date"`
+	Ticker map[string]string  `json:"ticker"`
+	Field  map[string]string  `json:"field"`
+	Value  map[string]float64 `json:"value"`
+}
 
+func AddEdbDataFromPb(edbCode string, item *EdbDataFromPb) (err error) {
+	o := orm.NewOrm()
 	if len(item.Date) > 0 {
 		var isAdd bool
 		dateMap := item.Date
@@ -49,7 +45,6 @@ func AddEdbDataFromPb(edbCode string, item *services.EdbDataFromPb) (err error)
 			addSql = strings.TrimRight(addSql, ",")
 			_, err = o.Raw(addSql).Exec()
 			if err != nil {
-				errMsg = " tx.Exec Err :" + err.Error()
 				return
 			}
 		}
@@ -58,7 +53,7 @@ func AddEdbDataFromPb(edbCode string, item *services.EdbDataFromPb) (err error)
 }
 
 // 刷新彭博指标数据
-func RefreshEdbDataFromPb(edbInfoId int, edbCode, startDate string, item *services.EdbDataFromPb) (err error) {
+func RefreshEdbDataFromPb(edbInfoId int, edbCode, startDate string, item *EdbDataFromPb) (err error) {
 	o := orm.NewOrm()
 	source := utils.DATA_SOURCE_PB
 

models/base_from_pb_finance.go

@@ -3,25 +3,14 @@ package models
 import (
 	"fmt"
 	"github.com/beego/beego/v2/client/orm"
-	"hongze/hongze_edb_lib/services"
-	"hongze/hongze_edb_lib/services/alarm_msg"
 	"hongze/hongze_edb_lib/utils"
 	"strconv"
 	"strings"
 	"time"
 )
 
-func AddEdbDataFromPbFinance(edbCode string, item *services.EdbDataFromPb) (err error) {
-	var errMsg string
+func AddEdbDataFromPbFinance(edbCode string, item *EdbDataFromPb) (err error) {
 	o := orm.NewOrm()
-	defer func() {
-		if err != nil {
-			//go utils.SendEmail(utils.APP_NAME_CN+"【"+utils.RunMode+"】"+"失败提醒", " 彭博数据获取失败:err:"+errMsg, utils.EmailSendToUsers)
-			go alarm_msg.SendAlarmMsg("彭博数据获取失败:err:"+errMsg, 3)
-
-		}
-	}()
-
 	if len(item.Date) > 0 {
 		var isAdd bool
 		dateMap := item.Date
@@ -47,7 +36,6 @@ func AddEdbDataFromPbFinance(edbCode string, item *services.EdbDataFromPb) (err
 			addSql = strings.TrimRight(addSql, ",")
 			_, err = o.Raw(addSql).Exec()
 			if err != nil {
-				errMsg = " tx.Exec Err :" + err.Error()
 				return
 			}
 		}
@@ -56,7 +44,7 @@ func AddEdbDataFromPbFinance(edbCode string, item *services.EdbDataFromPb) (err
 }
 
 // 刷新彭博指标数据
-func RefreshEdbDataFromPbFinance(edbInfoId int, edbCode, startDate string, item *services.EdbDataFromPb) (err error) {
+func RefreshEdbDataFromPbFinance(edbInfoId int, edbCode, startDate string, item *EdbDataFromPb) (err error) {
 	o := orm.NewOrm()
 	source := utils.DATA_SOURCE_PB_FINANCE
 

models/base_from_python.go

@@ -5,7 +5,6 @@ import (
 	"fmt"
 	"github.com/beego/beego/v2/client/orm"
 	"github.com/shopspring/decimal"
-	"hongze/hongze_edb_lib/services"
 	"hongze/hongze_edb_lib/services/alarm_msg"
 	"hongze/hongze_edb_lib/utils"
 	"strings"
@@ -24,15 +23,15 @@ type EdbDataPython struct {
 	DataTimestamp int64
 }
 
+// EdbDataFromPython 通过python代码获取到的指标数据
+type EdbDataFromPython struct {
+	Date  map[int]string  `json:"date"`
+	Value map[int]float64 `json:"value"`
+}
+
 // AddPythonEdb 新增python运算指标
-func AddPythonEdb(edbInfoId int, edbCode string, item services.EdbDataFromPython, edbInfoList []*EdbInfo) (err error) {
-	var errMsg string
+func AddPythonEdb(edbInfoId int, edbCode string, item EdbDataFromPython, edbInfoList []*EdbInfo) (err error) {
 	o := orm.NewOrm()
-	defer func() {
-		if err != nil {
-			go alarm_msg.SendAlarmMsg(" python代码运算数据添加失败:err:"+errMsg, 3)
-		}
-	}()
 
 	//添加指标关系
 	for _, tmpEdbInfo := range edbInfoList {
@@ -79,7 +78,6 @@ func AddPythonEdb(edbInfoId int, edbCode string, item services.EdbDataFromPython
 		addSql = strings.TrimRight(addSql, ",")
 		_, err = o.Raw(addSql).Exec()
 		if err != nil {
-			errMsg = " tx.Exec Err :" + err.Error()
 			return
 		}
 	}
@@ -87,15 +85,8 @@ func AddPythonEdb(edbInfoId int, edbCode string, item services.EdbDataFromPython
 }
 
 // EditPythonEdb 编辑python运算指标
-func EditPythonEdb(edbInfoId int, edbCode string, item services.EdbDataFromPython) (err error) {
-	var errMsg string
+func EditPythonEdb(edbInfoId int, edbCode string, item EdbDataFromPython) (err error) {
 	o := orm.NewOrm()
-	defer func() {
-		if err != nil {
-			go alarm_msg.SendAlarmMsg(" 编辑python运算指标:err:"+errMsg, 3)
-		}
-	}()
-
 	var isAdd bool
 	addSql := ` INSERT INTO edb_data_python (edb_info_id,edb_code,data_time,value,create_time,modify_time,data_timestamp) values `
 	for k, dateTimeStr := range item.Date {
@@ -120,7 +111,6 @@ func EditPythonEdb(edbInfoId int, edbCode string, item services.EdbDataFromPytho
 		addSql = strings.TrimRight(addSql, ",")
 		_, err = o.Raw(addSql).Exec()
 		if err != nil {
-			errMsg = " tx.Exec Err :" + err.Error()
 			return
 		}
 	}
@@ -128,7 +118,7 @@ func EditPythonEdb(edbInfoId int, edbCode string, item services.EdbDataFromPytho
 }
 
 // RefreshAllPythonEdb 刷新所有 python运算指标
-func RefreshAllPythonEdb(edbInfo *EdbInfo, item services.EdbDataFromPython) (err error) {
+func RefreshAllPythonEdb(edbInfo *EdbInfo, item EdbDataFromPython) (err error) {
 	o := orm.NewOrm()
 	to, err := o.Begin()
 	if err != nil {

models/base_from_sci.go

@@ -21,6 +21,31 @@ type BaseFromSciData struct {
 	DataTimestamp      int64
 }
 
+// Update 修改
+func (r *BaseFromSciData) Update(updateCols []string) (err error) {
+	o := orm.NewOrm()
+	_, err = o.Update(r, updateCols...)
+
+	return
+}
+
+// GetIndexDataList 获取所有的指标数据
+func (r *BaseFromSciData) GetIndexDataList(indexCode string) (items []*BaseFromSciData, err error) {
+	o := orm.NewOrm()
+	sql := `SELECT * FROM base_from_sci_index WHERE 1=1 AND index_code = ? `
+	_, err = o.Raw(sql, indexCode).QueryRows(&items)
+
+	return
+}
+
+// BatchAdd 批量添加指标
+func (r *BaseFromSciData) BatchAdd(list []*BaseFromSciData) (err error) {
+	o := orm.NewOrm()
+	_, err = o.InsertMulti(len(list), list)
+
+	return
+}
+
 func GetBaseFromSciDataByCondition(condition string, pars []interface{}) (list []*BaseFromSciData, err error) {
 	o := orm.NewOrm()
 	sql := `SELECT * FROM base_from_sci_data WHERE 1=1 `
@@ -31,7 +56,7 @@ func GetBaseFromSciDataByCondition(condition string, pars []interface{}) (list [
 	return
 }
 
-// 新增有色指标数据
+// AddEdbDataFromSci 新增卓创（红桃3）指标数据
 func AddEdbDataFromSci(edbCode string) (err error) {
 	o := orm.NewOrm()
 
@@ -73,7 +98,7 @@ func AddEdbDataFromSci(edbCode string) (err error) {
 	return
 }
 
-// 刷新有色指标数据
+// RefreshEdbDataFromSci 刷新卓创（红桃3）指标数据
 func RefreshEdbDataFromSci(edbInfoId int, edbCode, startDate string) (err error) {
 	source := utils.DATA_SOURCE_SCI
 	o := orm.NewOrm()
@@ -182,3 +207,55 @@ func RefreshEdbDataFromSci(edbInfoId int, edbCode, startDate string) (err error)
 	}
 	return
 }
+
+// HandleSciExcelDataReq 卓创（红桃3）的excel数据
+type HandleSciExcelDataReq struct {
+	DataMap          map[string]map[string]string
+	IndexNameList    []string
+	ThirdIndexIdList []string
+	FrequencyList    []string
+	UnitList         []string
+}
+
+// BaseFromSciIndex 红桃3指标表格
+type BaseFromSciIndex struct {
+	BaseFromSciIndexId int       `orm:"column(base_from_sci_index_id);pk"  json:"base_from_sci_index_id"` //序号
+	ClassifyId         int       `gorm:"column:classify_id" json:"classify_id"`
+	IndexCode          string    `gorm:"column:index_code" json:"index_code" description:"指标编码"`
+	IndexName          string    `gorm:"column:index_name" json:"index_name" description:"指标名称"`
+	Frequency          string    `gorm:"column:frequency" json:"frequency"`
+	Unit               string    `gorm:"column:unit" json:"unit"`
+	StartDate          time.Time `gorm:"column:start_date" json:"start_date"`
+	EndDate            time.Time `gorm:"column:end_date" json:"end_date"`
+	CreateTime         time.Time `gorm:"autoCreateTime;column:create_time" json:"create_time"`       //创建时间
+	ModifyTime         time.Time `gorm:"autoUpdateTime:milli;column:modify_time" json:"modify_time"` //最后更新时间
+}
+
+// TableName get sql table name.获取数据库表名
+func (r *BaseFromSciIndex) TableName() string {
+	return "base_from_sci_index"
+}
+
+// Update 修改
+func (r *BaseFromSciIndex) Update(updateCols []string) (err error) {
+	o := orm.NewOrm()
+	_, err = o.Update(r, updateCols...)
+
+	return
+}
+
+// GetAllIndex 获取所有的指标
+func (r *BaseFromSciIndex) GetAllIndex() (items []*BaseFromSciIndex, err error) {
+	o := orm.NewOrm()
+	sql := `SELECT * FROM base_from_sci_index WHERE 1=1 `
+	_, err = o.Raw(sql).QueryRows(&items)
+	return
+}
+
+// BatchAdd 批量添加指标
+func (r *BaseFromSciIndex) BatchAdd(list []*BaseFromSciIndex) (err error) {
+	o := orm.NewOrm()
+	_, err = o.InsertMulti(len(list), list)
+
+	return
+}

models/base_from_ths.go

@@ -3,24 +3,30 @@ package models
 import (
 	"fmt"
 	"github.com/beego/beego/v2/client/orm"
-	"hongze/hongze_edb_lib/services"
-	"hongze/hongze_edb_lib/services/alarm_msg"
 	"hongze/hongze_edb_lib/utils"
 	"strconv"
 	"strings"
 	"time"
 )
 
+type EdbDataFromThs struct {
+	DataVol   int64       `json:"dataVol"`
+	Errmsg    string      `json:"errmsg"`
+	Errorcode int64       `json:"errorcode"`
+	Perf      interface{} `json:"perf"`
+	Tables    []Tables    `json:"tables"`
+}
+
+// Tables 表格数据
+type Tables struct {
+	ID    []string  `json:"id"`
+	Time  []string  `json:"time"`
+	Value []float64 `json:"value"`
+}
+
 // 新增同花顺指标数据
-func AddEdbDataFromThs(edbCode string, item *services.EdbDataFromThs) (err error) {
-	var errMsg string
+func AddEdbDataFromThs(edbCode string, item EdbDataFromThs) (err error) {
 	o := orm.NewOrm()
-	defer func() {
-		if err != nil {
-			//go utils.SendEmail(utils.APP_NAME_CN+"【"+utils.RunMode+"】"+"失败提醒", " 同花顺数据获取失败:err:"+errMsg, utils.EmailSendToUsers)
-			go alarm_msg.SendAlarmMsg("同花顺数据获取失败:err:"+errMsg, 3)
-		}
-	}()
 
 	if len(item.Tables) > 0 {
 		table := item.Tables[0]
@@ -36,7 +42,6 @@ func AddEdbDataFromThs(edbCode string, item *services.EdbDataFromThs) (err error
 			sValue := table.Value[i]
 			dataTime, err := time.ParseInLocation(utils.FormatDate, eDate, time.Local)
 			if err != nil {
-				errMsg = " time.Parse :" + err.Error()
 				return err
 			}
 			timestamp := dataTime.UnixNano() / 1e6
@@ -48,7 +53,6 @@ func AddEdbDataFromThs(edbCode string, item *services.EdbDataFromThs) (err error
 			addSql = strings.TrimRight(addSql, ",")
 			_, err = o.Raw(addSql).Exec()
 			if err != nil {
-				errMsg = " tx.Exec Err :" + err.Error()
 				return
 			}
 		}
@@ -57,7 +61,7 @@ func AddEdbDataFromThs(edbCode string, item *services.EdbDataFromThs) (err error
 }
 
 // 刷新同花顺指标数据
-func RefreshEdbDataFromThs(edbInfoId int, edbCode, startDate string, item *services.EdbDataFromThs) (err error) {
+func RefreshEdbDataFromThs(edbInfoId int, edbCode, startDate string, item EdbDataFromThs) (err error) {
 	o := orm.NewOrm()
 	source := utils.DATA_SOURCE_THS
 

models/base_from_wind.go

@@ -3,23 +3,21 @@ package models
 import (
 	"fmt"
 	"github.com/beego/beego/v2/client/orm"
-	"hongze/hongze_edb_lib/services"
-	"hongze/hongze_edb_lib/services/alarm_msg"
 	"hongze/hongze_edb_lib/utils"
 	"strconv"
 	"strings"
 	"time"
 )
 
-func AddEdbDataFromWind(edbCode string, item *services.EdbDataFromWind) (err error) {
-	var errMsg string
+type EdbDataFromWind struct {
+	Close     map[string]float64 `json:"CLOSE"`
+	Dt        map[string]int64   `json:"DT"`
+	ErrorCode map[string]int64   `json:"ErrorCode"`
+	ErrMsg    string
+}
+
+func AddEdbDataFromWind(edbCode string, item *EdbDataFromWind) (err error) {
 	o := orm.NewOrm()
-	defer func() {
-		if err != nil {
-			//go utils.SendEmail(utils.APP_NAME_CN+"【"+utils.RunMode+"】"+"失败提醒", " 同花顺数据获取失败:err:"+errMsg, utils.EmailSendToUsers)
-			go alarm_msg.SendAlarmMsg("wind数据获取失败:err:"+errMsg, 3)
-		}
-	}()
 
 	var isAdd bool
 	addSql := ` INSERT INTO edb_data_wind(edb_info_id,edb_code,data_time,value,create_time,modify_time,data_timestamp) values `
@@ -39,7 +37,6 @@ func AddEdbDataFromWind(edbCode string, item *services.EdbDataFromWind) (err err
 		addSql = strings.TrimRight(addSql, ",")
 		_, err = o.Raw(addSql).Exec()
 		if err != nil {
-			errMsg = " tx.Exec Err :" + err.Error()
 			return
 		}
 	}
@@ -47,7 +44,7 @@ func AddEdbDataFromWind(edbCode string, item *services.EdbDataFromWind) (err err
 }
 
 // RefreshEdbDataFromWind 刷新wind指标数据
-func RefreshEdbDataFromWind(edbInfoId int, edbCode, startDate string, item *services.EdbDataFromWind) (err error) {
+func RefreshEdbDataFromWind(edbInfoId int, edbCode, startDate string, item *EdbDataFromWind) (err error) {
 	o := orm.NewOrm()
 	source := utils.DATA_SOURCE_WIND
 

models/base_predict_from_calculate.go

@@ -6,7 +6,6 @@ import (
 	"github.com/beego/beego/v2/client/orm"
 	"github.com/shopspring/decimal"
 	"github.com/yidane/formula"
-	"hongze/hongze_edb_lib/services"
 	"hongze/hongze_edb_lib/utils"
 	"strconv"
 	"strings"
@@ -158,7 +157,7 @@ func AddPredictCalculateDataBak(edbInfoIdList []*EdbInfo, edbInfoId int, edbCode
 	//数据处理，将日期内不全的数据做补全
 	handleDateSaveDataMap(dateList, oldSaveDataMap, saveDataMap, edbInfoIdList)
 
-	formulaMap := services.CheckFormula(formulaStr)
+	formulaMap := utils.CheckFormula(formulaStr)
 	addSql := ` INSERT INTO edb_data_predict_calculate (edb_info_id,edb_code,data_time,value,create_time,modify_time,data_timestamp) values `
 	nowStr := time.Now().Format(utils.FormatDateTime)
 	var isAdd bool
@@ -318,7 +317,7 @@ func refreshAllPredictCalculate(to orm.TxOrmer, edbInfoIdList []*EdbInfo, edbInf
 	//数据处理，将日期内不全的数据做补全
 	handleDateSaveDataMap(dateList, realSaveDataMap, saveDataMap, edbInfoIdList)
 
-	formulaMap := services.CheckFormula(formulaStr)
+	formulaMap := utils.CheckFormula(formulaStr)
 	addSql := ` INSERT INTO edb_data_predict_calculate (edb_info_id,edb_code,data_time,value,create_time,modify_time,data_timestamp) values `
 	var isAdd bool
 	if !minLatestDate.IsZero() {

models/db.go

@@ -60,6 +60,9 @@ func init() {
 
 	// 供应分析 数据表
 	initSupplyAnalysis()
+
+	// 基础指标表
+	initBaseIndex()
 }
 
 // initFutureGood 注册期货数据 数据表
@@ -82,3 +85,11 @@ func initSupplyAnalysis() {
 		new(supply_analysis.BaseFromStockPlantData),
 	)
 }
+
+// initBaseIndex 基础指标表
+func initBaseIndex() {
+	orm.RegisterModel(
+		new(BaseFromSciIndex),
+		new(BaseFromSciData),
+	)
+}

models/future_good/future_good_edb_data.go

@@ -3,8 +3,6 @@ package future_good
 import (
 	"fmt"
 	"github.com/beego/beego/v2/client/orm"
-	"hongze/hongze_edb_lib/services"
-	"hongze/hongze_edb_lib/services/alarm_msg"
 	"hongze/hongze_edb_lib/utils"
 	"strconv"
 	"strings"
@@ -51,6 +49,29 @@ type FutureGoodEdbDataItem struct {
 	CreateTime          time.Time
 }
 
+// FutureGoodDataFromThs 同花顺期货数据
+type FutureGoodDataFromThs struct {
+	DataVol   int64                `json:"dataVol"`
+	Errmsg    string               `json:"errmsg"`
+	Errorcode int64                `json:"errorcode"`
+	Perf      interface{}          `json:"perf"`
+	Tables    FutureGoodDataTables `json:"tables"`
+}
+
+// FutureGoodDataTables 同花顺表格数据
+type FutureGoodDataTables struct {
+	//TradeCode    []string  `json:"id"`
+	Time       []string  `json:"time"`
+	Open       []float64 `json:"open"`
+	High       []float64 `json:"high"`
+	Low        []float64 `json:"low"`
+	Close      []float64 `json:"close"`
+	Volume     []float64 `json:"volume"`
+	Amount     []float64 `json:"amount"`
+	Ccl        []float64 `json:"ccl"`
+	Settlement []float64 `json:"settlement"`
+}
+
 // GetFutureGoodEdbDataList 获取期货指标数据列表
 func GetFutureGoodEdbDataList(condition string, pars []interface{}) (list []*FutureGoodEdbDataItem, err error) {
 	o := orm.NewOrm()
@@ -63,16 +84,9 @@ func GetFutureGoodEdbDataList(condition string, pars []interface{}) (list []*Fut
 	return
 }
 
-// AddEdbDataFromWind 添加wind商品指标数据
-func AddEdbDataFromWind(futureGoodEdbInfoId int, edbCode string, item *services.FutureGoodDataFromThs) (err error) {
-	var errMsg string
+// AddEdbDataFromThs 添加Ths商品指标数据
+func AddEdbDataFromThs(futureGoodEdbInfoId int, edbCode string, item FutureGoodDataFromThs) (err error) {
 	o := orm.NewOrm()
-	defer func() {
-		if err != nil {
-			//go utils.SendEmail(utils.APP_NAME_CN+"【"+utils.RunMode+"】"+"失败提醒", " 同花顺数据获取失败:err:"+errMsg, utils.EmailSendToUsers)
-			go alarm_msg.SendAlarmMsg("wind商品数据获取失败:err:"+errMsg, 3)
-		}
-	}()
 
 	var isAdd bool
 	addSql := ` INSERT INTO future_good_edb_data(future_good_edb_info_id,future_good_edb_code,data_time,trade_code,open,high,low,close,volume,amt,oi,settle,create_time,modify_time,data_timestamp) values `
@@ -83,7 +97,6 @@ func AddEdbDataFromWind(futureGoodEdbInfoId int, edbCode string, item *services.
 		eDate := table.Time[k]
 		dataTime, err := time.ParseInLocation(utils.FormatDate, eDate, time.Local)
 		if err != nil {
-			errMsg = " time.Parse :" + err.Error()
 			return err
 		}
 		timestamp := dataTime.UnixNano() / 1e6
@@ -108,7 +121,6 @@ func AddEdbDataFromWind(futureGoodEdbInfoId int, edbCode string, item *services.
 		addSql = strings.TrimRight(addSql, ",")
 		_, err = o.Raw(addSql).Exec()
 		if err != nil {
-			errMsg = " tx.Exec Err :" + err.Error()
 			return
 		}
 	}
@@ -123,7 +135,7 @@ type RefreshFutureEdbEdbInfoReq struct {
 }
 
 // RefreshFutureGoodEdbDataFromThs 刷新wind期货指标数据
-func RefreshFutureGoodEdbDataFromThs(futureGoodEdbInfoId int, edbCode, startDate string, item *services.FutureGoodDataFromThs) (err error) {
+func RefreshFutureGoodEdbDataFromThs(futureGoodEdbInfoId int, edbCode, startDate string, item FutureGoodDataFromThs) (err error) {
 	o := orm.NewOrm()
 	to, err := o.Begin()
 	if err != nil {

models/predict_edb.go

@@ -7,8 +7,6 @@ import (
 	"github.com/beego/beego/v2/client/orm"
 	"github.com/shopspring/decimal"
 	"github.com/yidane/formula"
-	"hongze/hongze_edb_lib/services"
-	"hongze/hongze_edb_lib/services/alarm_msg"
 	"hongze/hongze_edb_lib/utils"
 	"strings"
 	"time"
@@ -36,10 +34,7 @@ func RefreshCalculateByRuleBy9(rule CalculateRule) (resultDataList []*EdbInfoSea
 	}
 	defer func() {
 		if err != nil {
-			tmpErr := to.Rollback()
-			if tmpErr != nil {
-				go alarm_msg.SendAlarmMsg("RefreshCalculateByRuleBy9 事务回滚失败，Err："+tmpErr.Error(), 3)
-			}
+			to.Rollback()
 		} else {
 			err = to.Commit()
 		}
@@ -105,7 +100,7 @@ func CalculateByRuleBy9(to orm.TxOrmer, rule CalculateRule) (resultDataList []*E
 	addDataList := make([]*PredictEdbRuleData, 0)
 
 	// 计算规则
-	formulaMap := services.CheckFormula(formulaStr)
+	formulaMap := utils.CheckFormula(formulaStr)
 
 	//获取指标所有数据
 	dataList := make([]*PredictEdbRuleData, 0)
@@ -229,10 +224,7 @@ func RefreshCalculateByRuleByLineNh(predictEdbInfo EdbInfo, predictEdbConfAndDat
 	}
 	defer func() {
 		if err != nil {
-			tmpErr := to.Rollback()
-			if tmpErr != nil {
-				go alarm_msg.SendAlarmMsg("RefreshCalculateByRuleBy9 事务回滚失败，Err："+tmpErr.Error(), 3)
-			}
+			to.Rollback()
 		} else {
 			err = to.Commit()
 		}

models/predict_edb_conf.go

@@ -2,7 +2,6 @@ package models
 
 import (
 	"github.com/beego/beego/v2/client/orm"
-	"hongze/hongze_edb_lib/services/alarm_msg"
 	"hongze/hongze_edb_lib/utils"
 	"time"
 )
@@ -156,10 +155,7 @@ func AddPredictEdb(item *EdbInfo, calculateMappingList []*EdbInfoCalculateMappin
 	}
 	defer func() {
 		if err != nil {
-			tmpErr := tx.Rollback()
-			if tmpErr != nil {
-				go alarm_msg.SendAlarmMsg("AddPredictEdb 事务回滚失败，Err："+tmpErr.Error(), 3)
-			}
+			tx.Rollback()
 		} else {
 			err = tx.Commit()
 		}
@@ -275,10 +271,7 @@ func EditPredictEdb(edbInfo *EdbInfo, updateEdbInfoCol []string, calculateMappin
 	}
 	defer func() {
 		if err != nil {
-			tmpErr := tx.Rollback()
-			if tmpErr != nil {
-				go alarm_msg.SendAlarmMsg("AddPredictEdb 事务回滚失败，Err："+tmpErr.Error(), 3)
-			}
+			tx.Rollback()
 		} else {
 			err = tx.Commit()
 		}

routers/commentsRouter.go

@@ -313,6 +313,24 @@ func init() {
             Filters: nil,
             Params: nil})
 
+    beego.GlobalControllerRouter["hongze/hongze_edb_lib/controllers:MySteelChemicalController"] = append(beego.GlobalControllerRouter["hongze/hongze_edb_lib/controllers:MySteelChemicalController"],
+        beego.ControllerComments{
+            Method: "HandleMysteelIndex",
+            Router: `/handle/mysteel/index`,
+            AllowHTTPMethods: []string{"post"},
+            MethodParams: param.Make(),
+            Filters: nil,
+            Params: nil})
+
+    beego.GlobalControllerRouter["hongze/hongze_edb_lib/controllers:MySteelChemicalController"] = append(beego.GlobalControllerRouter["hongze/hongze_edb_lib/controllers:MySteelChemicalController"],
+        beego.ControllerComments{
+            Method: "QueryRefresh",
+            Router: `/query/refresh`,
+            AllowHTTPMethods: []string{"post"},
+            MethodParams: param.Make(),
+            Filters: nil,
+            Params: nil})
+
     beego.GlobalControllerRouter["hongze/hongze_edb_lib/controllers:MySteelChemicalController"] = append(beego.GlobalControllerRouter["hongze/hongze_edb_lib/controllers:MySteelChemicalController"],
         beego.ControllerComments{
             Method: "Refresh",
@@ -475,6 +493,15 @@ func init() {
             Filters: nil,
             Params: nil})
 
+    beego.GlobalControllerRouter["hongze/hongze_edb_lib/controllers:SciController"] = append(beego.GlobalControllerRouter["hongze/hongze_edb_lib/controllers:SciController"],
+        beego.ControllerComments{
+            Method: "HandleExcelData",
+            Router: `/handle/excel_data`,
+            AllowHTTPMethods: []string{"post"},
+            MethodParams: param.Make(),
+            Filters: nil,
+            Params: nil})
+
     beego.GlobalControllerRouter["hongze/hongze_edb_lib/controllers:SciController"] = append(beego.GlobalControllerRouter["hongze/hongze_edb_lib/controllers:SciController"],
         beego.ControllerComments{
             Method: "Refresh",

services/base_from_fubao.go

@@ -4,6 +4,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"github.com/rdlucklib/rdluck_tools/http"
+	"hongze/hongze_edb_lib/models"
 	"hongze/hongze_edb_lib/utils"
 )
 
@@ -12,25 +13,8 @@ const (
 	FuBaoQueryLastDataUrl = "http://data.f139.com/cooperation/wande/queryLastData.do"
 )
 
-type FuBaoData struct {
-	ReturnValue struct {
-		Product struct {
-			AlikeName    string `json:"alikeName"`
-			DiffName     string `json:"diffName"`
-			PropValueStr string `json:"propValueStr"`
-			Pattern      string `json:"pattern"`
-		} `json:"product"`
-		Data []struct {
-			PriceStr string `json:"priceStr"`
-			Uad      string `json:"uad"`
-			Datadate string `json:"datadate"`
-		} `json:"data"`
-	} `json:"returnValue"`
-	Code int `json:"code"`
-}
-
 // GetEdbDataFromLt 获取路透数据
-func GetEdbDataFromFuBao(edbCode, startDate, endDate string) (item *FuBaoData, err error) {
+func GetEdbDataFromFuBao(edbCode, startDate, endDate string) (item *models.FuBaoData, err error) {
 	sbUrl := fmt.Sprintf(FuBaoQueryDataUrl, edbCode)
 	utils.FileLog.Info("sbUrl:%s", sbUrl)
 	body, err := http.Get(sbUrl)
@@ -39,7 +23,7 @@ func GetEdbDataFromFuBao(edbCode, startDate, endDate string) (item *FuBaoData, e
 	if err != nil {
 		return
 	}
-	item = new(FuBaoData)
+	item = new(models.FuBaoData)
 	err = json.Unmarshal(body, &item)
 	return
 }

services/base_from_mysteel_chemical.go

@@ -0,0 +1,148 @@
+package services
+
+import (
+	"fmt"
+	"hongze/hongze_edb_lib/models"
+	"hongze/hongze_edb_lib/utils"
+	"strings"
+	"time"
+)
+
+func HandleMysteelIndex(req *models.HandleMysteelIndexResp) {
+	for _, v := range req.List {
+		if v.IndexName == "" || v.IndexCode == "" {
+			continue
+		}
+		handleIndex(v)
+	}
+}
+
+func handleIndex(indexItem *models.HandleMysteelIndex) {
+	var err error
+
+	//return
+	indexObj := new(models.BaseFromMysteelChemicalIndex)
+	var indexId int64
+
+	addDataList := make([]models.BaseFromMysteelChemicalData, 0)
+
+	exitDataMap := make(map[string]*models.BaseFromMysteelChemicalData)
+
+	//判断指标是否存在
+	var isAdd int
+	item, err := indexObj.GetIndexItem(indexItem.IndexCode)
+	if err != nil {
+		if err.Error() == err.Error() {
+			isAdd = 1
+		} else {
+			isAdd = -1
+			return
+		}
+	}
+	if item != nil && item.BaseFromMysteelChemicalIndexId > 0 {
+		isAdd = 2
+	} else {
+		isAdd = 1
+	}
+
+	fmt.Println("isAdd:", isAdd)
+	if !strings.Contains(indexItem.Frequency, "度") {
+		indexItem.Frequency = indexItem.Frequency + "度"
+	}
+
+	if isAdd == 1 {
+		indexObj.IndexCode = indexItem.IndexCode
+		indexObj.IndexName = indexItem.IndexName
+		indexObj.Unit = indexItem.Unit
+		indexObj.Source = indexItem.Source
+		indexObj.Describe = indexItem.Describe
+		indexObj.StartDate, _ = time.ParseInLocation(utils.FormatDate, indexItem.StartDate, time.Local)
+		indexObj.EndDate, _ = time.ParseInLocation(utils.FormatDate, indexItem.EndDate, time.Local)
+		indexObj.Frequency = indexItem.Frequency
+		err = indexObj.Add()
+		if err != nil {
+			fmt.Println("add err:" + err.Error())
+			return
+		}
+		indexId = indexObj.BaseFromMysteelChemicalIndexId
+	} else if isAdd == 2 {
+		indexObj.IndexCode = indexItem.IndexCode
+		indexObj.IndexName = indexItem.IndexName
+		indexObj.Unit = indexItem.Unit
+		indexObj.Source = indexItem.Source
+		indexObj.Describe = indexItem.Describe
+		indexObj.StartDate, _ = time.ParseInLocation(utils.FormatDate, indexItem.StartDate, time.Local)
+		indexObj.EndDate, _ = time.ParseInLocation(utils.FormatDate, indexItem.EndDate, time.Local)
+		indexObj.Frequency = indexItem.Frequency
+		indexObj.ModifyTime = time.Now()
+		indexId = item.BaseFromMysteelChemicalIndexId
+		//修改数据
+		updateColsArr := make([]string, 0)
+		updateColsArr = append(updateColsArr, "index_name")
+		updateColsArr = append(updateColsArr, "unit")
+		updateColsArr = append(updateColsArr, "source")
+		updateColsArr = append(updateColsArr, "frequency")
+		updateColsArr = append(updateColsArr, "start_date")
+		updateColsArr = append(updateColsArr, "end_date")
+		updateColsArr = append(updateColsArr, "describe")
+		updateColsArr = append(updateColsArr, "end_date")
+		updateColsArr = append(updateColsArr, "modify_time")
+
+		indexObj.Update(updateColsArr)
+
+		dataObj := new(models.BaseFromMysteelChemicalData)
+
+		//获取已存在的所有数据
+		exitDataList, err := dataObj.GetIndexDataList(indexItem.IndexCode)
+		if err != nil {
+			fmt.Println("GetIndexDataList Err:" + err.Error())
+			return
+		}
+		fmt.Println("exitDataListLen:", len(exitDataList))
+		for _, v := range exitDataList {
+			dateStr := v.DataTime.Format(utils.FormatDate)
+			exitDataMap[dateStr] = v
+		}
+	}
+
+	dataObj := new(models.BaseFromMysteelChemicalData)
+	// 遍历excel数据，然后跟现有的数据做校验，不存在则入库
+	for date, value := range indexItem.ExcelDataMap {
+		if findData, ok := exitDataMap[date]; !ok {
+			dateTime, err := time.ParseInLocation(utils.FormatDate, date, time.Local)
+			if err != nil {
+				fmt.Println("time.ParseInLocation Err:" + err.Error())
+				return
+			}
+			if !strings.Contains(value, "#N/A") {
+				dataItem := new(models.BaseFromMysteelChemicalData)
+				dataItem.BaseFromMysteelChemicalIndexId = indexId
+				dataItem.IndexCode = indexItem.IndexCode
+				dataItem.DataTime = dateTime
+				dataItem.Value = value
+				dataItem.UpdateDate = indexItem.UpdateDate
+				dataItem.CreateTime = time.Now()
+				dataItem.ModifyTime = time.Now()
+				addDataList = append(addDataList, *dataItem)
+			}
+		} else {
+			if findData != nil && findData.Value != value && !strings.Contains(value, "#N/A") { //修改数据
+				dataObj.BaseFromMysteelChemicalDataId = findData.BaseFromMysteelChemicalDataId
+				dataObj.Value = value
+				dataObj.ModifyTime = time.Now()
+
+				updateDataColsArr := make([]string, 0)
+				updateDataColsArr = append(updateDataColsArr, "value")
+				updateDataColsArr = append(updateDataColsArr, "modify_time")
+				dataObj.Update(updateDataColsArr)
+			}
+		}
+	}
+
+	if len(addDataList) > 0 {
+		err = dataObj.Add(addDataList)
+		if err != nil {
+			fmt.Println("dataObj.Add() Err:" + err.Error())
+		}
+	}
+}

services/base_from_pb.go

@@ -4,19 +4,13 @@ import (
 	"encoding/json"
 	"fmt"
 	"github.com/rdlucklib/rdluck_tools/http"
+	"hongze/hongze_edb_lib/models"
 	"hongze/hongze_edb_lib/utils"
 	"net/url"
 )
 
-type EdbDataFromPb struct {
-	Date   map[string]int64   `json:"date"`
-	Ticker map[string]string  `json:"ticker"`
-	Field  map[string]string  `json:"field"`
-	Value  map[string]float64 `json:"value"`
-}
-
 // GetEdbDataFromPb 获取Pb数据
-func GetEdbDataFromPb(edbCode, startDate, endDate string) (item *EdbDataFromPb, err error) {
+func GetEdbDataFromPb(edbCode, startDate, endDate string) (item *models.EdbDataFromPb, err error) {
 	edbCode = url.QueryEscape(edbCode)
 	bpUrl := utils.Hz_Pb_Data_Url + `edbInfo/pb?EdbCode=%s&StartDate=%s&EndDate=%s`
 	bpUrl = fmt.Sprintf(bpUrl, edbCode, startDate, endDate)
@@ -27,7 +21,7 @@ func GetEdbDataFromPb(edbCode, startDate, endDate string) (item *EdbDataFromPb,
 		return
 	}
 	utils.FileLog.Info("GetEdbDataByPb result:" + string(body))
-	item = new(EdbDataFromPb)
+	item = new(models.EdbDataFromPb)
 	err = json.Unmarshal(body, &item)
 	if err != nil {
 		utils.FileLog.Info("GetEdbDataByPb Unmarshal Err:" + err.Error())
@@ -38,7 +32,7 @@ func GetEdbDataFromPb(edbCode, startDate, endDate string) (item *EdbDataFromPb,
 }
 
 // GetEdbDataFromPbFinance 获取Pb财务数据
-func GetEdbDataFromPbFinance(companyCode, edbCode, startDate, endDate string) (item *EdbDataFromPb, err error) {
+func GetEdbDataFromPbFinance(companyCode, edbCode, startDate, endDate string) (item *models.EdbDataFromPb, err error) {
 	companyCode = url.QueryEscape(companyCode)
 	edbCode = url.QueryEscape(edbCode)
 	bpUrl := utils.Hz_Pb_Data_Url + `edbInfo/pb/finance?CompanyCode=%s&EdbCode=%s&StartDate=%s&EndDate=%s`
@@ -50,7 +44,7 @@ func GetEdbDataFromPbFinance(companyCode, edbCode, startDate, endDate string) (i
 		return
 	}
 	utils.FileLog.Info("GetEdbDataByPb result:" + string(body))
-	item = new(EdbDataFromPb)
+	item = new(models.EdbDataFromPb)
 	err = json.Unmarshal(body, &item)
 	if err != nil {
 		utils.FileLog.Info("GetEdbDataByPb Unmarshal Err:" + err.Error())

services/base_from_python.go

@@ -5,6 +5,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"github.com/shopspring/decimal"
+	"hongze/hongze_edb_lib/models"
 	"hongze/hongze_edb_lib/utils"
 	"os"
 	"os/exec"
@@ -54,12 +55,6 @@ func Test() (err error) {
 	return
 }
 
-// EdbDataFromPython 通过python代码获取到的指标数据
-type EdbDataFromPython struct {
-	Date  map[int]string  `json:"date"`
-	Value map[int]float64 `json:"value"`
-}
-
 // EdbDataStrFromPython 通过python代码获取到的指标数据（interface数据）
 type EdbDataStrFromPython struct {
 	Date  map[int]string      `json:"date"`
@@ -67,7 +62,7 @@ type EdbDataStrFromPython struct {
 }
 
 // ExecPythonCode 执行Python代码
-func ExecPythonCode(edbCode, reqCode string) (dataMap EdbDataFromPython, err error, errMsg string) {
+func ExecPythonCode(edbCode, reqCode string) (dataMap models.EdbDataFromPython, err error, errMsg string) {
 	defer func() {
 		if err != nil {
 			fmt.Println("err:", err)

services/base_from_sci.go

@@ -0,0 +1,179 @@
+package services
+
+import (
+	"fmt"
+	"hongze/hongze_edb_lib/logic"
+	"hongze/hongze_edb_lib/models"
+	"hongze/hongze_edb_lib/services/alarm_msg"
+	"hongze/hongze_edb_lib/utils"
+	"strings"
+	"time"
+)
+
+// HandleSciIndex 处理卓创（红桃3）的Excel数据
+func HandleSciIndex(indexNameList, thirdIndexIdList, frequencyList, unitList []string, dataMap map[string]map[string]string) {
+	// 卓创（红桃3）指标id列表
+
+	sciIndexModel := new(models.BaseFromSciIndex)
+	list, err := sciIndexModel.GetAllIndex()
+	if err != nil {
+		fmt.Println("获取指标失败:", err)
+		return
+	}
+	allIndexMap := make(map[string]string)
+	for _, v := range list {
+		allIndexMap[v.IndexCode] = ``
+	}
+
+	// 需要入库的指标下标
+	needAddIndexKeyList := make([]int, 0)
+	needAddIndexMap := make(map[string]int, 0)
+	for key, v := range thirdIndexIdList {
+		if _, ok := allIndexMap[v]; !ok {
+			if _, ok2 := needAddIndexMap[v]; !ok2 {
+				needAddIndexKeyList = append(needAddIndexKeyList, key)
+				needAddIndexMap[v] = key
+			}
+		}
+	}
+	//fmt.Println(needAddIndexKeyList)
+	//return
+	addSciIndexList := make([]*models.BaseFromSciIndex, 0)
+	// 新的指标入库
+	for _, key := range needAddIndexKeyList {
+		tmpSciIndex := &models.BaseFromSciIndex{
+			//BaseFromSciIndexId: 0,
+			//ClassifyId:         0,
+			IndexCode: thirdIndexIdList[key],
+			IndexName: indexNameList[key],
+			Frequency: frequencyList[key],
+			Unit:      unitList[key],
+			//StartDate: time.Time{},
+			//EndDate:      time.Time{},
+			CreateTime: time.Now(),
+			ModifyTime: time.Now(),
+		}
+		addSciIndexList = append(addSciIndexList, tmpSciIndex)
+	}
+	//fmt.Println(addSciIndexList)
+	if len(addSciIndexList) > 0 {
+		err = sciIndexModel.BatchAdd(addSciIndexList)
+		if err != nil {
+			fmt.Println("批量添加指标失败:", err)
+			return
+		}
+		fmt.Println("添加成功")
+	}
+
+	// 红桃3实际数据处理
+	HandleData(dataMap)
+
+}
+
+// HandleData 红桃3实际数据处理
+func HandleData(dataMap map[string]map[string]string) {
+	errMsgList := make([]string, 0)
+	defer func() {
+		if len(errMsgList) > 0 {
+			go alarm_msg.SendAlarmMsg(fmt.Sprint("红桃3实际数据处理失败,err:", strings.Join(errMsgList, "\n")), 3)
+		}
+	}()
+	// 获取所有的指标
+	sciIndexModel := new(models.BaseFromSciIndex)
+	list, err := sciIndexModel.GetAllIndex()
+	if err != nil {
+		fmt.Println("获取指标失败:", err)
+		return
+	}
+	allIndexMap := make(map[string]*models.BaseFromSciIndex)
+	for _, v := range list {
+		allIndexMap[v.IndexCode] = v
+	}
+
+	sciIndexDataModel := new(models.BaseFromSciData)
+	for indexCode, data := range dataMap {
+		indexInfo, ok := allIndexMap[indexCode]
+		if !ok {
+			fmt.Println("找不到该指标:", indexCode)
+			continue
+		}
+
+		indexDataList, err := sciIndexDataModel.GetIndexDataList(indexCode)
+		if err != nil {
+			errMsgList = append(errMsgList, fmt.Sprint("查找卓创基础指标失败,指标编码：", indexCode, ";错误原因：", err.Error()))
+			continue
+		}
+		indexDataExistMap := make(map[string]*models.BaseFromSciData)
+		for _, indexData := range indexDataList {
+			indexDataExistMap[indexData.DataTime] = indexData
+		}
+		addSciDataList := make([]*models.BaseFromSciData, 0)
+
+		for currDate, currVal := range data {
+			currDataTime, tmpErr := time.ParseInLocation(utils.FormatDate, currDate, time.Local)
+			if tmpErr != nil {
+				errMsgList = append(errMsgList, fmt.Sprint("时间格式化失败,指标编码：", currDate, ";错误原因：", tmpErr.Error()))
+				continue
+			}
+			timestamp := currDataTime.UnixNano() / 1e6
+
+			sciData, ok := indexDataExistMap[currDate]
+			//判断是否存在数据,如果不存在,那么插入数据,存在的话那么修改数据
+			if !ok {
+				tmpBaseFromSciData := &models.BaseFromSciData{
+					//SciDataId:          0,
+					BaseFromSciIndexId: indexInfo.BaseFromSciIndexId,
+					IndexCode:          indexInfo.IndexCode,
+					DataTime:           currDate,
+					Value:              currVal,
+					DataTimestamp:      timestamp,
+					CreateTime:         time.Now(),
+					ModifyTime:         time.Now(),
+				}
+				indexDataExistMap[currDate] = tmpBaseFromSciData
+				addSciDataList = append(addSciDataList, tmpBaseFromSciData)
+			} else {
+				// 更新数据
+				existValue := sciData.Value
+				if existValue != currVal {
+					//fmt.Println("existValue:", existValue, ";====;currVal:", currVal)
+					sciData.Value = currVal
+					sciData.ModifyTime = time.Now()
+					// 如果是已经入库了数据,那么就更新,否则只是单纯更改内存数据,而不去更新数据库
+					if sciData.SciDataId > 0 {
+						tmpErr = sciData.Update([]string{"Value", "ModifyTime"})
+						if tmpErr != nil {
+							errMsgList = append(errMsgList, fmt.Sprint("指标数据更新失败,指标编码：", indexCode, "，时间：", currDate, ";错误原因：", tmpErr.Error()))
+							continue
+						}
+					}
+				}
+			}
+		}
+
+		if len(addSciDataList) > 0 {
+			err = sciIndexDataModel.BatchAdd(addSciDataList)
+			if err != nil {
+				errMsgList = append(errMsgList, fmt.Sprint("批量添加数据失败,指标编码：", indexCode, err.Error()))
+				continue
+			}
+		}
+		indexInfo.ModifyTime = time.Now()
+		indexInfo.Update([]string{"ModifyTime"})
+
+		// 同步刷新ETA图库红桃3的指标
+		{
+			// 获取指标详情
+			edbInfo, err := models.GetEdbInfoByEdbCode(utils.DATA_SOURCE_SCI, indexInfo.IndexCode)
+			if err != nil && err.Error() != utils.ErrNoRow() {
+				errMsgList = append(errMsgList, fmt.Sprint("刷新ETA指标异常，指标编码：", indexCode, err.Error()))
+				continue
+			}
+
+			// 已经加入到指标库的话，那么就去更新ETA指标库吧
+			if edbInfo != nil {
+				go logic.RefreshSci(edbInfo, ``)
+			}
+		}
+	}
+}

services/base_from_ths.go

@@ -6,25 +6,12 @@ import (
 	"fmt"
 	"github.com/rdlucklib/rdluck_tools/http"
 	"github.com/shopspring/decimal"
+	"hongze/hongze_edb_lib/models"
+	"hongze/hongze_edb_lib/models/future_good"
 	"hongze/hongze_edb_lib/utils"
 	"reflect"
 )
 
-type EdbDataFromThs struct {
-	DataVol   int64       `json:"dataVol"`
-	Errmsg    string      `json:"errmsg"`
-	Errorcode int64       `json:"errorcode"`
-	Perf      interface{} `json:"perf"`
-	Tables    []Tables    `json:"tables"`
-}
-
-// Tables 表格数据
-type Tables struct {
-	ID    []string  `json:"id"`
-	Time  []string  `json:"time"`
-	Value []float64 `json:"value"`
-}
-
 // EdbDataFromThsInterface 数据类型转为interface
 type EdbDataFromThsInterface struct {
 	DataVol   int64       `json:"dataVol"`
@@ -38,12 +25,16 @@ type EdbDataFromThsInterface struct {
 	} `json:"tables"`
 }
 
-func GetEdbDataFromThs(edbCode, startDate, endDate string) (item *EdbDataFromThs, err error) {
-	return getEdbDataFromThs(edbCode, startDate, endDate, 0)
+func GetEdbDataFromThs(edbCode, startDate, endDate string) (item models.EdbDataFromThs, err error) {
+	if utils.RunMode == `release` { // 生产环境走官方http请求，测试环境走终端
+		return getEdbDataFromThsHttp(edbCode, startDate, endDate)
+	} else {
+		return getEdbDataFromThsApp(edbCode, startDate, endDate, 0)
+	}
 }
 
 // getEdbDataFromThs 获取同花顺接口数据
-func getEdbDataFromThs(edbCode, startDate, endDate string, num int) (item *EdbDataFromThs, err error) {
+func getEdbDataFromThsApp(edbCode, startDate, endDate string, num int) (item models.EdbDataFromThs, err error) {
 	thsUrl := utils.Hz_Wind_Data_Url + `edbInfo/ths?EdbCode=%s&StartDate=%s&EndDate=%s`
 	thsUrl = fmt.Sprintf(thsUrl, edbCode, startDate, endDate)
 	utils.FileLog.Info("thsUrl:" + thsUrl)
@@ -69,7 +60,7 @@ func getEdbDataFromThs(edbCode, startDate, endDate string, num int) (item *EdbDa
 		//session has expired,please re-login after using the system
 		//如果是同花顺登录session失效了，那么就重新请求获取数据
 		if tmpItems.Errorcode == -1020 && num == 0 {
-			return getEdbDataFromThs(edbCode, startDate, endDate, 1)
+			return getEdbDataFromThsApp(edbCode, startDate, endDate, 1)
 		}
 
 		err = errors.New(string(body))
@@ -77,7 +68,7 @@ func getEdbDataFromThs(edbCode, startDate, endDate string, num int) (item *EdbDa
 	}
 
 	// 因为table里面的value有的时候返回的是string,有的是float64，所以需要用interface来反射取值
-	tablesList := make([]Tables, 0)
+	tablesList := make([]models.Tables, 0)
 	for _, table := range tmpItems.Tables {
 		tableIdList := make([]string, 0)
 		tableTimeList := make([]string, 0)
@@ -108,14 +99,14 @@ func getEdbDataFromThs(edbCode, startDate, endDate string, num int) (item *EdbDa
 			}
 			tableValueList = append(tableValueList, tableValue)
 		}
-		tmpTable := Tables{
+		tmpTable := models.Tables{
 			ID:    tableIdList,
 			Time:  tableTimeList,
 			Value: tableValueList,
 		}
 		tablesList = append(tablesList, tmpTable)
 	}
-	item = &EdbDataFromThs{
+	item = models.EdbDataFromThs{
 		DataVol:   tmpItems.DataVol,
 		Errmsg:    tmpItems.Errmsg,
 		Errorcode: tmpItems.Errorcode,
@@ -159,31 +150,16 @@ type FutureGoodDataFromThsInterface struct {
 	} `json:"tables"`
 }
 
-// FutureGoodDataFromThs 同花顺期货数据
-type FutureGoodDataFromThs struct {
-	DataVol   int64                `json:"dataVol"`
-	Errmsg    string               `json:"errmsg"`
-	Errorcode int64                `json:"errorcode"`
-	Perf      interface{}          `json:"perf"`
-	Tables    FutureGoodDataTables `json:"tables"`
-}
-
-// FutureGoodDataTables 同花顺表格数据
-type FutureGoodDataTables struct {
-	//TradeCode    []string  `json:"id"`
-	Time       []string  `json:"time"`
-	Open       []float64 `json:"open"`
-	High       []float64 `json:"high"`
-	Low        []float64 `json:"low"`
-	Close      []float64 `json:"close"`
-	Volume     []float64 `json:"volume"`
-	Amount     []float64 `json:"amount"`
-	Ccl        []float64 `json:"ccl"`
-	Settlement []float64 `json:"settlement"`
+func GetFutureGoodDataFromThs(edbCode, startDate, endDate string) (item future_good.FutureGoodDataFromThs, err error) {
+	if utils.RunMode == `release` { // 生产环境走官方http请求，测试环境走终端
+		return getFutureGoodDataFromThsHttp(edbCode, startDate, endDate)
+	} else {
+		return getFutureGoodDataFromThsApp(edbCode, startDate, endDate, 0)
+	}
 }
 
-// GetFutureGoodDataFromThs 通过url获取wind的商品数据
-func GetFutureGoodDataFromThs(edbCode, startDate, endDate string, num int) (item *FutureGoodDataFromThs, err error) {
+// getFutureGoodDataFromThsApp 通过终端获取wind的商品数据
+func getFutureGoodDataFromThsApp(edbCode, startDate, endDate string, num int) (item future_good.FutureGoodDataFromThs, err error) {
 	thsUrl := utils.Hz_Wind_Data_Url + `edbInfo/ths/future_good?EdbCode=%s&StartDate=%s&EndDate=%s`
 	thsUrl = fmt.Sprintf(thsUrl, edbCode, startDate, endDate)
 	utils.FileLog.Info("thsUrl:" + thsUrl)
@@ -209,7 +185,7 @@ func GetFutureGoodDataFromThs(edbCode, startDate, endDate string, num int) (item
 		//session has expired,please re-login after using the system
 		//如果是同花顺登录session失效了，那么就重新请求获取数据
 		if tmpItems.Errorcode == -1020 && num == 0 {
-			return GetFutureGoodDataFromThs(edbCode, startDate, endDate, 1)
+			return getFutureGoodDataFromThsApp(edbCode, startDate, endDate, 1)
 		}
 
 		err = errors.New(string(body))
@@ -220,12 +196,12 @@ func GetFutureGoodDataFromThs(edbCode, startDate, endDate string, num int) (item
 		return
 	}
 	table := tmpItems.Tables[0]
-	item = &FutureGoodDataFromThs{
+	item = future_good.FutureGoodDataFromThs{
 		DataVol:   tmpItems.DataVol,
 		Errmsg:    tmpItems.Errmsg,
 		Errorcode: tmpItems.Errorcode,
 		Perf:      tmpItems.Perf,
-		Tables: FutureGoodDataTables{
+		Tables: future_good.FutureGoodDataTables{
 			Time:       table.Time,
 			Open:       table.Table.Open,
 			High:       table.Table.High,

services/base_from_ths_http.go

@@ -0,0 +1,323 @@
+package services
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"github.com/shopspring/decimal"
+	"hongze/hongze_edb_lib/models"
+	"hongze/hongze_edb_lib/models/future_good"
+	"hongze/hongze_edb_lib/services/alarm_msg"
+	"hongze/hongze_edb_lib/utils"
+	"io"
+	netHttp "net/http"
+	"reflect"
+	"strings"
+	"time"
+)
+
+// refreshToken 同花顺刷新token
+var refreshToken = `eyJzaWduX3RpbWUiOiIyMDIzLTAzLTI0IDEzOjQ3OjExIn0=.eyJ1aWQiOiI1NzY2NDgxMDkifQ==.339B8D21168AC21A0F80840544E38378AB2D04A02D325F0CD1C44251915233F6`
+var tokenRedisKey = `THS_SERVER_ACCESS_TOKEN`
+
+// getEdbDataFromThsHttp 通过url获取同花顺的普通数据
+func getEdbDataFromThsHttp(edbCode, startDate, endDate string) (item models.EdbDataFromThs, err error) {
+	thsUrl := `https://quantapi.51ifind.com/api/v1/edb_service`
+	//indicators 是 半角逗号分隔的所有指标，宏观指标过多，推荐使用Windows超级命令生成。 "indicators":"M001620326,M002822183"
+	//functionpara 否 key-value格式,省略时不进行更新时间筛选。两个时间控件更新起始时间（startrtime）和更新结束时间（endrtime），不勾选时省略见下方代码块
+	//startdate 是 开始日期，支持”YYYYMMDD"”YYYY-MM-DD"”YYYY/MM/DD"三种时间格式 "startdate":"2018-01-01"
+	//enddate 是 结束日期，支持”YYYYMMDD"”YYYY-MM-DD"”YYYY/MM/DD"三种日期格式 "enddate":"2018-01-01
+	//发送创建请求
+	dataMap := map[string]interface{}{
+		"indicators": edbCode,
+		"startdate":  startDate,
+		"enddate":    endDate,
+	}
+
+	body, err, _ := postCurl(thsUrl, dataMap, 0)
+	if err != nil {
+		return
+	}
+
+	tmpItems := new(EdbDataFromThsInterface)
+	err = json.Unmarshal(body, &tmpItems)
+	if err != nil {
+		err = errors.New("GetEdbDataFromThs json.Unmarshal Err:" + err.Error())
+		return
+	}
+	if tmpItems.Errorcode != 0 {
+		err = errors.New(tmpItems.Errmsg)
+		return
+	}
+	// 因为table里面的value有的时候返回的是string,有的是float64，所以需要用interface来反射取值
+	tablesList := make([]models.Tables, 0)
+	for _, table := range tmpItems.Tables {
+		tableIdList := make([]string, 0)
+		tableTimeList := make([]string, 0)
+		tableValueList := make([]float64, 0)
+
+		for _, tableId := range table.ID {
+			tableIdList = append(tableIdList, tableId)
+		}
+		for _, tableTime := range table.Time {
+			tableTimeList = append(tableTimeList, tableTime)
+		}
+
+		//指标数据
+		for _, tmpValue := range table.Value {
+			var tableValue float64
+			if reflect.TypeOf(tmpValue).Kind() == reflect.Float64 {
+				tableValue = reflect.ValueOf(tmpValue).Float()
+			} else if reflect.TypeOf(tmpValue).Kind() == reflect.String {
+				tmpTableValue, tmpErr := decimal.NewFromString(reflect.ValueOf(tmpValue).String())
+				if tmpErr != nil {
+					err = tmpErr
+					return
+				}
+				tableValue, _ = tmpTableValue.Truncate(4).Float64()
+			} else {
+				err = errors.New("错误的数据类型" + reflect.TypeOf(tmpValue).String())
+				return
+			}
+			tableValueList = append(tableValueList, tableValue)
+		}
+		tmpTable := models.Tables{
+			ID:    tableIdList,
+			Time:  tableTimeList,
+			Value: tableValueList,
+		}
+		tablesList = append(tablesList, tmpTable)
+	}
+	item = models.EdbDataFromThs{
+		DataVol:   tmpItems.DataVol,
+		Errmsg:    tmpItems.Errmsg,
+		Errorcode: tmpItems.Errorcode,
+		Perf:      tmpItems.Perf,
+		Tables:    tablesList,
+	}
+	return
+}
+
+// getFutureGoodDataFromThsHttp 通过url获取同花顺的商品数据
+func getFutureGoodDataFromThsHttp(edbCode, startDate, endDate string) (item future_good.FutureGoodDataFromThs, err error) {
+	thsUrl := `https://quantapi.51ifind.com/api/v1/cmd_history_quotation`
+
+	//codes 是 半角逗号分隔的所有代码 "codes":"300033.SZ,600030.SH"
+	//indicators 是 半角逗号分隔的所有指标 "indicators":"preClose,open"
+	//functionpara 否 /key-value格式。所有key均取默认时，functionpara省略。 见下方说明
+	//startdate 是 开始日期，支持"YYYYMMDD""YYYY-MMDD""YYYY/MM/DD"三种日期格式
+	//"startdate":"2018-01-01"
+	//enddate 是 结束日期，支持"YYYYMMDD""YYYY-MMDD""YYYY/MM/DD"三种日期格式
+	//发送创建请求
+	dataMap := map[string]interface{}{
+		"codes":      edbCode,
+		"indicators": `lastclose,open,high,low,close,avgprice,change,changeper,volume,amount,hsl,lastsettlement,settlement,zdsettlement,zdfsettlement,ccl,ccbd,zf,zjlx,zjcd`,
+		"startdate":  startDate,
+		"enddate":    endDate,
+	}
+
+	body, err, _ := postCurl(thsUrl, dataMap, 0)
+	if err != nil {
+		return
+	}
+
+	tmpItems := new(FutureGoodDataFromThsInterface)
+	err = json.Unmarshal(body, &tmpItems)
+	if err != nil {
+		err = errors.New("GetEdbDataFromThs json.Unmarshal Err:" + err.Error())
+		return
+	}
+	if tmpItems.Errorcode != 0 {
+		err = errors.New(tmpItems.Errmsg)
+		return
+	}
+
+	if len(tmpItems.Tables) <= 0 {
+		return
+	}
+	table := tmpItems.Tables[0]
+	item = future_good.FutureGoodDataFromThs{
+		DataVol:   tmpItems.DataVol,
+		Errmsg:    tmpItems.Errmsg,
+		Errorcode: tmpItems.Errorcode,
+		Perf:      tmpItems.Perf,
+		Tables: future_good.FutureGoodDataTables{
+			Time:       table.Time,
+			Open:       table.Table.Open,
+			High:       table.Table.High,
+			Low:        table.Table.Low,
+			Close:      table.Table.Close,
+			Volume:     table.Table.Volume,
+			Amount:     table.Table.Amount,
+			Ccl:        table.Table.Ccl,
+			Settlement: table.Table.Settlement,
+		},
+	}
+	return
+}
+
+// BaseThsInterface 同花顺基础返回
+type BaseThsInterface struct {
+	ErrMsg    string      `json:"errmsg"`
+	ErrorCode int64       `json:"errorcode"`
+	Tables    interface{} `json:"tables"`
+}
+
+// postCurl post请求同花顺接口
+func postCurl(urlStr string, dataMap map[string]interface{}, num int) (body []byte, err error, errMsg string) {
+	logMsg := ``
+	defer func() {
+		if err != nil {
+			if logMsg != `` {
+				errMsg = logMsg
+				go alarm_msg.SendAlarmMsg("post请求同花顺接口失败，ERR："+err.Error()+";errMsg:"+errMsg, 3)
+			}
+		}
+	}()
+	token, err := GetAccessToken(false)
+	if err != nil {
+		return
+	}
+
+	jsonStrByte, err := json.Marshal(dataMap)
+	if err != nil {
+		return
+	}
+	reqStr := string(jsonStrByte)
+	req, _ := netHttp.NewRequest("POST", urlStr, strings.NewReader(reqStr))
+	req.Header.Add("Content-Type", "application/json")
+	req.Header.Add("access_token", token)
+
+	res, _ := netHttp.DefaultClient.Do(req)
+	defer res.Body.Close()
+	//解析resp并且存入关联表
+	body, err = io.ReadAll(res.Body)
+	if err != nil {
+		logMsg = fmt.Sprint("post err; request:", reqStr, "; errMsg:", err.Error())
+		utils.FileLog.Info(logMsg)
+		return
+	}
+	//logMsg = fmt.Sprint("post request:", reqStr, "; response:", string(body))
+	//utils.FileLog.Info(logMsg)
+	logMsg = fmt.Sprint("post request url:", urlStr, ";token:", token, ";params:", reqStr, ";response:", string(body))
+
+	var response BaseThsInterface
+	err = json.Unmarshal(body, &response)
+	if err != nil {
+		utils.FileLog.Info("post Err:", err.Error(), ";url:", urlStr, ";params:", reqStr, ";response:", string(body))
+		err = errors.New("Unmarshal Err:" + err.Error())
+		return
+	}
+	utils.FileLog.Info(fmt.Sprint("post request url:", urlStr, ";token:", token, ";params:", reqStr, ";response:", string(body)))
+
+	//如果是token失效，同时只是第一次请求（没有尝试强制刷新token，那么重新请求）
+	if utils.InArrayByInt([]int{-1010, -1302}, int(response.ErrorCode)) && num <= 0 {
+		//token失效
+		_, tmpErr := refreshAccessToken()
+		if tmpErr != nil {
+			err = tmpErr
+		}
+		num++
+		return postCurl(urlStr, dataMap, num)
+	} else if response.ErrorCode != 0 {
+		utils.FileLog.Info(fmt.Sprint("post data err:", response.ErrMsg, ";url:", urlStr, ";params:", reqStr, ";response:", string(body)))
+		err = errors.New(response.ErrMsg)
+		return
+	}
+
+	return
+}
+
+// GetAccessToken 获取accessToken
+func GetAccessToken(isRefresh bool) (token string, err error) {
+	defer func() {
+		if err != nil {
+			go alarm_msg.SendAlarmMsg("获取同花顺的token失败，ERR："+err.Error(), 3)
+		}
+	}()
+	token, redisErr := utils.Rc.RedisString(tokenRedisKey)
+	//如果从redis中accessToken 获取失败或者token为空了，再或者需要强制刷新了，那么重新获取accessToken
+	if redisErr != nil || token == `` || isRefresh {
+		return refreshAccessToken()
+	}
+	return
+}
+
+// refreshAccessToken 强制刷新获取accessToken
+func refreshAccessToken() (token string, err error) {
+	defer func() {
+		if err != nil {
+			go alarm_msg.SendAlarmMsg("刷新同花顺的token失败；ERR："+err.Error(), 3)
+		}
+	}()
+	tokenInfo, tmpErr := getAccessToken()
+	if tmpErr != nil {
+		err = tmpErr
+		return
+	}
+	token = tokenInfo.AccessToken
+
+	expireTime, err := time.ParseInLocation(utils.FormatDateTime, tokenInfo.ExpiredTime, time.Local)
+	if err != nil {
+		go alarm_msg.SendAlarmMsg("获取同花顺的token失败；同花顺token截止日期转换失败，ERR："+err.Error(), 3)
+		return
+	}
+
+	//token存入redis
+	//err = utils.Rc.Put(tokenRedisKey, token, time.Duration(expireTime.Unix()-600)*time.Second)
+	// 本来是要设置下600s的过期时间，但因为不是强制刷新token，就不获取了
+	err = utils.Rc.Put(tokenRedisKey, token, time.Duration(expireTime.Unix())*time.Second)
+	if err != nil {
+		go alarm_msg.SendAlarmMsg("获取同花顺的token成功；同花顺token存入redis失败，ERR："+err.Error(), 3)
+	}
+	return
+}
+
+type GetTokenResp struct {
+	ErrorCode int       `json:"errorcode"`
+	ErrMsg    string    `json:"errmsg"`
+	Data      TokenData `json:"data"`
+}
+
+type TokenData struct {
+	AccessToken string `json:"access_token"`
+	//ExpireIn    int    `json:"expire_in"`
+	ExpiredTime string `json:"expired_time"`
+}
+
+// getAccessToken token内部请求接口
+func getAccessToken() (tokenData TokenData, err error) {
+	defer func() {
+		if err != nil {
+			go alarm_msg.SendAlarmMsg("更新同花顺的token失败；ERR："+err.Error(), 3)
+		}
+	}()
+	//getUrl := `https://quantapi.51ifind.com/api/v1/update_access_token`	// 强制刷新token；目前因为生产和测试都是使用的一个账号，所以token不能因为环境的不同而改变
+	getUrl := `https://quantapi.51ifind.com/api/v1/get_access_token` // 获取当前token
+
+	req, _ := netHttp.NewRequest("GET", getUrl, nil)
+	req.Header.Add("Content-Type", "application/json")
+	req.Header.Add("refresh_token", refreshToken)
+
+	res, _ := netHttp.DefaultClient.Do(req)
+	defer res.Body.Close()
+	body, err := io.ReadAll(res.Body)
+	if err != nil {
+		err = errors.New("NewRequest Err:" + err.Error())
+		return
+	}
+	utils.FileLog.Info("同花顺刷新token:" + string(body))
+
+	var tokenResp GetTokenResp
+	err = json.Unmarshal(body, &tokenResp)
+	if err != nil {
+		err = errors.New("Unmarshal Err:" + err.Error())
+		return
+	}
+	if tokenResp.ErrorCode != 0 {
+		err = errors.New("getAccessToken err:" + tokenResp.ErrMsg)
+		return
+	}
+	tokenData = tokenResp.Data
+	return
+}

services/base_from_wind.go

@@ -5,20 +5,14 @@ import (
 	"errors"
 	"fmt"
 	"github.com/rdlucklib/rdluck_tools/http"
+	"hongze/hongze_edb_lib/models"
 	"hongze/hongze_edb_lib/services/alarm_msg"
 	"hongze/hongze_edb_lib/utils"
 	"time"
 )
 
-type EdbDataFromWind struct {
-	Close     map[string]float64 `json:"CLOSE"`
-	Dt        map[string]int64   `json:"DT"`
-	ErrorCode map[string]int64   `json:"ErrorCode"`
-	ErrMsg    string
-}
-
 // GetEdbDataFromWind 获取wind数据
-func GetEdbDataFromWind(edbCode, startDate, endDate string) (item *EdbDataFromWind, errorCode int, err error) {
+func GetEdbDataFromWind(edbCode, startDate, endDate string) (item *models.EdbDataFromWind, errorCode int, err error) {
 	windUrl, err := GetWindUrl(edbCode)
 	if err != nil {
 		errorCode = 421
@@ -36,7 +30,7 @@ func GetEdbDataFromWind(edbCode, startDate, endDate string) (item *EdbDataFromWi
 	if err != nil {
 		return
 	}
-	item = new(EdbDataFromWind)
+	item = new(models.EdbDataFromWind)
 	err = json.Unmarshal(body, &item)
 
 	//异常的话，需要邮件通知
@@ -80,7 +74,7 @@ func GetEdbDataFromWind(edbCode, startDate, endDate string) (item *EdbDataFromWi
 }
 
 // GetEdbDataFromWindUrl 通过url获取wind数据
-func GetEdbDataFromWindUrl(windUrl, edbCode, startDate, endDate string) (item *EdbDataFromWind, errorCode int, err error) {
+func GetEdbDataFromWindUrl(windUrl, edbCode, startDate, endDate string) (item *models.EdbDataFromWind, errorCode int, err error) {
 	if windUrl == `` {
 		return GetEdbDataFromWind(edbCode, startDate, endDate)
 	}
@@ -96,7 +90,7 @@ func GetEdbDataFromWindUrl(windUrl, edbCode, startDate, endDate string) (item *E
 	if err != nil {
 		return
 	}
-	item = new(EdbDataFromWind)
+	item = new(models.EdbDataFromWind)
 	err = json.Unmarshal(body, &item)
 
 	//异常的话，需要邮件通知

services/base_from_calculate.go → utils/base_from_calculate.go

@@ -1,4 +1,4 @@
-package services
+package utils
 
 import (
 	"strings"