feat：调整同花顺的数据获取逻辑

controllers/base_from_ths.go

@@ -10,7 +10,7 @@ import (
 	"time"
 )
 
-//同花顺
+// 同花顺
 type ThsController struct {
 	BaseAuthController
 }
@@ -43,7 +43,7 @@ func (this *ThsController) Add() {
 	cacheKey = utils.CACHE_EDB_DATA_ADD + strconv.Itoa(source) + "_" + req.EdbCode
 	if !utils.Rc.IsExist(cacheKey) {
 		utils.Rc.SetNX(cacheKey, 1, 1*time.Minute)
-		dataItem, err := services.GetEdbDataFromThs(req.EdbCode, utils.BASE_START_DATE, utils.BASE_END_DATE)
+		dataItem, err := services.GetEdbDataFromThsHttp(req.EdbCode, utils.BASE_START_DATE, utils.BASE_END_DATE)
 		if err != nil {
 			br.Msg = "获取指标信息失败！"
 			br.ErrMsg = "获取指标信息失败 GetEdbDataFromThs，Err:" + err.Error()
@@ -114,7 +114,7 @@ func (this *ThsController) Refresh() {
 	defer func() {
 		utils.Rc.Delete(cacheKey)
 	}()
-	dataItem, err := services.GetEdbDataFromThs(req.EdbCode, req.StartDate, utils.BASE_END_DATE)
+	dataItem, err := services.GetEdbDataFromThsHttp(req.EdbCode, req.StartDate, utils.BASE_END_DATE)
 	if err != nil {
 		br.Msg = "获取指标信息失败！"
 		br.ErrMsg = "获取指标信息失败 GetEdbDataFromThs，Err:" + err.Error()
@@ -141,3 +141,27 @@ func (this *ThsController) Refresh() {
 	br.Success = true
 	br.Msg = "获取成功"
 }
+
+//func init() {
+//	//?EdbCode=s005696248&StartDate=2023-02-03&EndDate=2027-03-23
+//	//edbCode := `s005696248`
+//	//startDate := `2023-02-03`
+//	//endDate := `2027-03-23`
+//	//EdbCode=S011292460&StartDate=1993-03-23&EndDate=2027-03-23
+//	edbCode := `S011292460`
+//	startDate := `1993-03-23`
+//	endDate := `2027-03-23`
+//	//edbCode := `@CL0W.NMX`
+//	//startDate := `20221218`
+//	//endDate := `20230118`
+//	list, err := services.GetEdbDataFromThsHttp(edbCode, startDate, endDate, 0)
+//	//list, err := services.GetFutureGoodDataFromThsHttp(edbCode, startDate, endDate)
+//
+//	//token, err := services.GetAccessToken()
+//	if err != nil {
+//		fmt.Println("err:", err)
+//		return
+//	}
+//	fmt.Println(list)
+//	//fmt.Println(token)
+//}

controllers/future_good/future_good_edb_info.go

@@ -68,7 +68,7 @@ func (this *FutureGoodEdbInfoController) Add() {
 		}
 		endDate := time.Now().Format(utils.FormatDate)
 
-		dataItem, err := services.GetFutureGoodDataFromThs(req.EdbCode, utils.BASE_START_DATE, endDate, 0)
+		dataItem, err := services.GetFutureGoodDataFromThsHttp(req.EdbCode, utils.BASE_START_DATE, endDate)
 		if err != nil {
 			br.Msg = "获取指标信息失败！"
 			br.ErrMsg = "获取指标信息失败 GetEdbDataFromWind，Err:" + err.Error()
@@ -167,7 +167,7 @@ func (this *FutureGoodEdbInfoController) Refresh() {
 	if startDate == `` { // 如果传入的日期为空的话,那么就默认兼容一周的数据吧
 		startDate = endDateTime.AddDate(0, 0, -7).Format(utils.FormatDate)
 	}
-	dataItem, err := services.GetFutureGoodDataFromThs(req.FutureGoodEdbCode, req.StartDate, endDate, 0)
+	dataItem, err := services.GetFutureGoodDataFromThsHttp(req.FutureGoodEdbCode, req.StartDate, endDate)
 	if err != nil {
 		br.Msg = "获取指标信息失败！"
 		br.ErrMsg = "获取指标信息失败 GetEdbDataFromWind，Err:" + err.Error()

models/base_from_ths.go

@@ -12,7 +12,7 @@ import (
 )
 
 // 新增同花顺指标数据
-func AddEdbDataFromThs(edbCode string, item *services.EdbDataFromThs) (err error) {
+func AddEdbDataFromThs(edbCode string, item services.EdbDataFromThs) (err error) {
 	var errMsg string
 	o := orm.NewOrm()
 	defer func() {
@@ -57,7 +57,7 @@ func AddEdbDataFromThs(edbCode string, item *services.EdbDataFromThs) (err error
 }
 
 // 刷新同花顺指标数据
-func RefreshEdbDataFromThs(edbInfoId int, edbCode, startDate string, item *services.EdbDataFromThs) (err error) {
+func RefreshEdbDataFromThs(edbInfoId int, edbCode, startDate string, item services.EdbDataFromThs) (err error) {
 	o := orm.NewOrm()
 	source := utils.DATA_SOURCE_THS
 

models/future_good/future_good_edb_data.go

@@ -64,7 +64,7 @@ func GetFutureGoodEdbDataList(condition string, pars []interface{}) (list []*Fut
 }
 
 // AddEdbDataFromWind 添加wind商品指标数据
-func AddEdbDataFromWind(futureGoodEdbInfoId int, edbCode string, item *services.FutureGoodDataFromThs) (err error) {
+func AddEdbDataFromWind(futureGoodEdbInfoId int, edbCode string, item services.FutureGoodDataFromThs) (err error) {
 	var errMsg string
 	o := orm.NewOrm()
 	defer func() {
@@ -123,7 +123,7 @@ type RefreshFutureEdbEdbInfoReq struct {
 }
 
 // RefreshFutureGoodEdbDataFromThs 刷新wind期货指标数据
-func RefreshFutureGoodEdbDataFromThs(futureGoodEdbInfoId int, edbCode, startDate string, item *services.FutureGoodDataFromThs) (err error) {
+func RefreshFutureGoodEdbDataFromThs(futureGoodEdbInfoId int, edbCode, startDate string, item services.FutureGoodDataFromThs) (err error) {
 	o := orm.NewOrm()
 	to, err := o.Begin()
 	if err != nil {

services/base_from_ths_http.go

@@ -0,0 +1,316 @@
+package services
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"github.com/shopspring/decimal"
+	"hongze/hongze_edb_lib/services/alarm_msg"
+	"hongze/hongze_edb_lib/utils"
+	"io"
+	netHttp "net/http"
+	"reflect"
+	"strings"
+	"time"
+)
+
+// refreshToken 同花顺刷新token
+var refreshToken = `eyJzaWduX3RpbWUiOiIyMDIzLTAzLTI0IDEzOjQ3OjExIn0=.eyJ1aWQiOiI1NzY2NDgxMDkifQ==.339B8D21168AC21A0F80840544E38378AB2D04A02D325F0CD1C44251915233F6`
+
+func GetEdbDataFromThsHttp(edbCode, startDate, endDate string) (item EdbDataFromThs, err error) {
+	thsUrl := `https://quantapi.51ifind.com/api/v1/edb_service`
+	//indicators 是 半角逗号分隔的所有指标，宏观指标过多，推荐使用Windows超级命令生成。 "indicators":"M001620326,M002822183"
+	//functionpara 否 key-value格式,省略时不进行更新时间筛选。两个时间控件更新起始时间（startrtime）和更新结束时间（endrtime），不勾选时省略见下方代码块
+	//startdate 是 开始日期，支持”YYYYMMDD"”YYYY-MM-DD"”YYYY/MM/DD"三种时间格式 "startdate":"2018-01-01"
+	//enddate 是 结束日期，支持”YYYYMMDD"”YYYY-MM-DD"”YYYY/MM/DD"三种日期格式 "enddate":"2018-01-01
+	//发送创建请求
+	dataMap := map[string]interface{}{
+		"indicators": edbCode,
+		"startdate":  startDate,
+		"enddate":    endDate,
+	}
+
+	body, err, _ := postCurl(thsUrl, dataMap, 0)
+
+	tmpItems := new(EdbDataFromThsInterface)
+	err = json.Unmarshal(body, &tmpItems)
+	if err != nil {
+		err = errors.New("GetEdbDataFromThs json.Unmarshal Err:" + err.Error())
+		return
+	}
+	if tmpItems.Errorcode != 0 {
+		err = errors.New(tmpItems.Errmsg)
+		return
+	}
+	// 因为table里面的value有的时候返回的是string,有的是float64，所以需要用interface来反射取值
+	tablesList := make([]Tables, 0)
+	for _, table := range tmpItems.Tables {
+		tableIdList := make([]string, 0)
+		tableTimeList := make([]string, 0)
+		tableValueList := make([]float64, 0)
+
+		for _, tableId := range table.ID {
+			tableIdList = append(tableIdList, tableId)
+		}
+		for _, tableTime := range table.Time {
+			tableTimeList = append(tableTimeList, tableTime)
+		}
+
+		//指标数据
+		for _, tmpValue := range table.Value {
+			var tableValue float64
+			if reflect.TypeOf(tmpValue).Kind() == reflect.Float64 {
+				tableValue = reflect.ValueOf(tmpValue).Float()
+			} else if reflect.TypeOf(tmpValue).Kind() == reflect.String {
+				tmpTableValue, tmpErr := decimal.NewFromString(reflect.ValueOf(tmpValue).String())
+				if tmpErr != nil {
+					err = tmpErr
+					return
+				}
+				tableValue, _ = tmpTableValue.Truncate(4).Float64()
+			} else {
+				err = errors.New("错误的数据类型" + reflect.TypeOf(tmpValue).String())
+				return
+			}
+			tableValueList = append(tableValueList, tableValue)
+		}
+		tmpTable := Tables{
+			ID:    tableIdList,
+			Time:  tableTimeList,
+			Value: tableValueList,
+		}
+		tablesList = append(tablesList, tmpTable)
+	}
+	item = EdbDataFromThs{
+		DataVol:   tmpItems.DataVol,
+		Errmsg:    tmpItems.Errmsg,
+		Errorcode: tmpItems.Errorcode,
+		Perf:      tmpItems.Perf,
+		Tables:    tablesList,
+	}
+	return
+}
+
+// GetFutureGoodDataFromThsHttp 通过url获取wind的商品数据
+func GetFutureGoodDataFromThsHttp(edbCode, startDate, endDate string) (item FutureGoodDataFromThs, err error) {
+	//thsUrl := utils.Hz_Wind_Data_Url + `edbInfo/ths/future_good?EdbCode=%s&StartDate=%s&EndDate=%s`
+	thsUrl := `https://quantapi.51ifind.com/api/v1/cmd_history_quotation`
+
+	//codes 是 半角逗号分隔的所有代码 "codes":"300033.SZ,600030.SH"
+	//indicators 是 半角逗号分隔的所有指标 "indicators":"preClose,open"
+	//functionpara 否 /key-value格式。所有key均取默认时，functionpara省略。 见下方说明
+	//startdate 是 开始日期，支持"YYYYMMDD""YYYY-MMDD""YYYY/MM/DD"三种日期格式
+	//"startdate":"2018-01-01"
+	//enddate 是 结束日期，支持"YYYYMMDD""YYYY-MMDD""YYYY/MM/DD"三种日期格式
+	//发送创建请求
+	dataMap := map[string]interface{}{
+		"codes":      edbCode,
+		"indicators": `lastclose,open,high,low,close,avgprice,change,changeper,volume,amount,hsl,lastsettlement,settlement,zdsettlement,zdfsettlement,ccl,ccbd,zf,zjlx,zjcd`,
+		"startdate":  startDate,
+		"enddate":    endDate,
+	}
+
+	body, err, _ := postCurl(thsUrl, dataMap, 0)
+
+	tmpItems := new(FutureGoodDataFromThsInterface)
+	err = json.Unmarshal(body, &tmpItems)
+	if err != nil {
+		err = errors.New("GetEdbDataFromThs json.Unmarshal Err:" + err.Error())
+		return
+	}
+	if tmpItems.Errorcode != 0 {
+		err = errors.New(tmpItems.Errmsg)
+		return
+	}
+
+	if len(tmpItems.Tables) <= 0 {
+		return
+	}
+	table := tmpItems.Tables[0]
+	item = FutureGoodDataFromThs{
+		DataVol:   tmpItems.DataVol,
+		Errmsg:    tmpItems.Errmsg,
+		Errorcode: tmpItems.Errorcode,
+		Perf:      tmpItems.Perf,
+		Tables: FutureGoodDataTables{
+			Time:       table.Time,
+			Open:       table.Table.Open,
+			High:       table.Table.High,
+			Low:        table.Table.Low,
+			Close:      table.Table.Close,
+			Volume:     table.Table.Volume,
+			Amount:     table.Table.Amount,
+			Ccl:        table.Table.Ccl,
+			Settlement: table.Table.Settlement,
+		},
+	}
+	return
+}
+
+// BaseThsInterface 同花顺基础返回
+type BaseThsInterface struct {
+	ErrMsg    string      `json:"errmsg"`
+	ErrorCode int64       `json:"errorcode"`
+	Tables    interface{} `json:"tables"`
+}
+
+// postCurl post请求上海接口
+func postCurl(urlStr string, dataMap map[string]interface{}, num int) (body []byte, err error, errMsg string) {
+	logMsg := ``
+	defer func() {
+		if err != nil {
+			if logMsg != `` {
+				errMsg = logMsg
+				go alarm_msg.SendAlarmMsg("post请求上海接口失败，ERR："+err.Error()+";errMsg:"+errMsg, 3)
+				//go utils.SendEmail(utils.APPNAME+"post请求上海接口失败："+time.Now().Format("2006-01-02 15:04:05"), "post请求上海接口失败："+errMsg, utils.EmailSendToUsers)
+			}
+		}
+	}()
+	token, err := GetAccessToken(false)
+	if err != nil {
+		return
+	}
+
+	jsonStrByte, err := json.Marshal(dataMap)
+	if err != nil {
+		return
+	}
+	reqStr := string(jsonStrByte)
+	req, _ := netHttp.NewRequest("POST", urlStr, strings.NewReader(reqStr))
+	req.Header.Add("Content-Type", "application/json")
+	req.Header.Add("access_token", token)
+
+	res, _ := netHttp.DefaultClient.Do(req)
+	defer res.Body.Close()
+	//解析resp并且存入关联表
+	body, err = io.ReadAll(res.Body)
+	if err != nil {
+		logMsg = fmt.Sprint("post err; request:", reqStr, "; errMsg:", err.Error())
+		utils.FileLog.Info(logMsg)
+		return
+	}
+	//logMsg = fmt.Sprint("post request:", reqStr, "; response:", string(body))
+	//utils.FileLog.Info(logMsg)
+	logMsg = fmt.Sprint("post request url:", urlStr, ";params:", reqStr, ";response:", string(body))
+
+	var response BaseThsInterface
+	err = json.Unmarshal(body, &response)
+	if err != nil {
+		utils.FileLog.Info("post Err:", err.Error(), ";url:", urlStr, ";params:", reqStr, ";response:", string(body))
+		err = errors.New("Unmarshal Err:" + err.Error())
+		return
+	}
+	utils.FileLog.Info(fmt.Sprint("post request url:", urlStr, ";params:", reqStr, ";response:", string(body)))
+
+	//如果是token失效，同时只是第一次请求（没有尝试强制刷新token，那么重新请求）
+	if response.ErrorCode == -1010 && num <= 0 {
+		//token失效
+		_, tmpErr := refreshAccessToken()
+		if tmpErr != nil {
+			err = tmpErr
+		}
+		num++
+		return postCurl(urlStr, dataMap, num)
+	} else if response.ErrorCode != 1 {
+		utils.FileLog.Info(fmt.Sprint("post data err", ";url:", urlStr, ";params:", reqStr, ";response:", string(body)))
+		err = errors.New(response.ErrMsg)
+		return
+	}
+
+	return
+}
+
+// GetAccessToken 获取accessToken
+
+func GetAccessToken(isRefresh bool) (token string, err error) {
+	defer func() {
+		if err != nil {
+			go alarm_msg.SendAlarmMsg("获取上海的token失败，ERR："+err.Error(), 3)
+			//go utils.SendEmail(utils.APPNAME+"获取上海的token失败："+time.Now().Format("2006-01-02 15:04:05"), err.Error(), utils.EmailSendToUsers)
+		}
+	}()
+	token, redisErr := utils.Rc.RedisString("SH_ACCESS_TOKEN")
+	//如果从redis中accessToken 获取失败或者token为空了，再或者需要强制刷新了，那么重新获取accessToken
+	if redisErr != nil || token == `` || isRefresh {
+		return refreshAccessToken()
+	}
+	return
+}
+
+// refreshAccessToken 强制刷新获取accessToken
+func refreshAccessToken() (token string, err error) {
+	defer func() {
+		if err != nil {
+			go alarm_msg.SendAlarmMsg("刷新上海的token失败；ERR："+err.Error(), 3)
+			//go utils.SendEmail(utils.APPNAME+"刷新上海的token失败："+time.Now().Format("2006-01-02 15:04:05"), err.Error(), utils.EmailSendToUsers)
+		}
+	}()
+	tokenInfo, tmpErr := getAccessToken()
+	if tmpErr != nil {
+		err = tmpErr
+		return
+	}
+	token = tokenInfo.AccessToken
+
+	expireTime, err := time.ParseInLocation(utils.FormatDateTime, tokenInfo.ExpiredTime, time.Local)
+	if err != nil {
+		go alarm_msg.SendAlarmMsg("获取同花顺的token失败；同花顺token截止日期转换失败，ERR："+err.Error(), 3)
+
+	}
+
+	//token存入redis
+	err = utils.Rc.Put("THS_SERVER_ACCESS_TOKEN", token, time.Duration(expireTime.Unix()-600)*time.Second)
+	if err != nil {
+		go alarm_msg.SendAlarmMsg("获取同花顺的token失败；同花顺token存入redis失败，ERR："+err.Error(), 3)
+		//go utils.SendEmail(utils.APPNAME+"获取上海的token失败："+time.Now().Format("2006-01-02 15:04:05"), "上海token存入redis失败：", utils.EmailSendToUsers)
+	}
+	return
+}
+
+type GetTokenResp struct {
+	ErrorCode int       `json:"errorcode"`
+	ErrMsg    string    `json:"errmsg"`
+	Data      TokenData `json:"data"`
+}
+
+type TokenData struct {
+	AccessToken string `json:"access_token"`
+	//ExpireIn    int    `json:"expire_in"`
+	ExpiredTime string `json:"expired_time"`
+}
+
+// getAccessToken token内部请求接口
+func getAccessToken() (tokenData TokenData, err error) {
+	defer func() {
+		if err != nil {
+			go alarm_msg.SendAlarmMsg("更新上海的token失败；ERR："+err.Error(), 3)
+			//go utils.SendEmail(utils.APPNAME+"更新上海的token失败："+time.Now().Format("2006-01-02 15:04:05"), err.Error(), utils.EmailSendToUsers)
+		}
+	}()
+	getUrl := `https://quantapi.51ifind.com/api/v1/update_access_token`
+
+	req, _ := netHttp.NewRequest("GET", getUrl, nil)
+	req.Header.Add("Content-Type", "application/json")
+	req.Header.Add("refresh_token", refreshToken)
+
+	res, _ := netHttp.DefaultClient.Do(req)
+	defer res.Body.Close()
+	body, err := io.ReadAll(res.Body)
+	if err != nil {
+		err = errors.New("NewRequest Err:" + err.Error())
+		return
+	}
+
+	var tokenResp GetTokenResp
+	err = json.Unmarshal(body, &tokenResp)
+	if err != nil {
+		err = errors.New("Unmarshal Err:" + err.Error())
+		return
+	}
+	if tokenResp.ErrorCode != 0 {
+		err = errors.New("getAccessToken err:" + tokenResp.ErrMsg)
+		return
+	}
+	tokenData = tokenResp.Data
+	return
+}