Merge branch 'feature/pcsg_0827' into debug

models/pcsg/bloomberg.go

@@ -1,31 +1,34 @@
 package pcsg
 
 import (
-	"eta/eta_bridge/utils"
+	"eta/eta_bridge/global"
+	"fmt"
 	"strings"
 	"time"
 )
 
-// PythonBloombergGeneralData 通用数据格式
+// PythonBloombergGeneralData 通用数据格式(有空改成Map吧...然后做成配置方便加新字段...)
 type PythonBloombergGeneralData struct {
-	NAME                 string   `json:"NAME" description:"指标名称, 可能为null"`
-	IDENTIFIER           string   `json:"IDENTIFIER" description:"指标编码"`
-	PX_LAST_EOD          *float64 `json:"PX_LAST_EOD" description:"数据值, 可能为null"`
-	PX_VOLUME_EOD        *float64 `json:"PX_VOLUME_EOD" description:"数据值, 可能为null"`
-	PX_LAST              *float64 `json:"PX_LAST" description:"数据值, 可能为null"`
-	PX_VOLUME            *float64 `json:"PX_VOLUME" description:"数据值, 可能为null"`
-	LAST_UPDATE_DATE_EOD string   `json:"LAST_UPDATE_DATE_EOD" description:"数据日期, 可能为null"`
-	DATE                 string   `json:"DATE" description:"数据日期, 可能为null"`
-
-	PX_SETTLE         *float64 `json:"PX_SETTLE" description:"数据值, 可能为null"`
-	PX_SETTLE_LAST_DT string   `json:"PX_SETTLE_LAST_DT" description:"数据日期, 可能为null"`
+	NAME                   string   `json:"NAME" description:"指标名称, 可能为null"`
+	IDENTIFIER             string   `json:"IDENTIFIER" description:"指标编码"`
+	PX_LAST_EOD            *float64 `json:"PX_LAST_EOD" description:"数据值, 可能为null"`
+	PX_VOLUME_EOD          *float64 `json:"PX_VOLUME_EOD" description:"数据值, 可能为null"`
+	PX_LAST                *float64 `json:"PX_LAST" description:"数据值, 可能为null"`
+	PX_VOLUME              *float64 `json:"PX_VOLUME" description:"数据值, 可能为null"`
+	LAST_UPDATE_DATE_EOD   string   `json:"LAST_UPDATE_DATE_EOD" description:"数据日期"`
+	DATE                   string   `json:"DATE" description:"数据日期"`
+	PX_SETTLE              *float64 `json:"PX_SETTLE" description:"数据值, 可能为null"`
+	PX_SETTLE_LAST_DT      string   `json:"PX_SETTLE_LAST_DT" description:"数据日期"`
+	OPEN_INT               *float64 `json:"OPEN_INT"`
+	OPEN_INT_DATE          string   `json:"OPEN_INT_DATE"`
+	DL_SNAPSHOT_START_TIME string   `json:"DL_SNAPSHOT_START_TIME" description:"数据日期"`
 }
 
 // PythonBloombergGeneralResult API响应体
 type PythonBloombergGeneralResult struct {
-	Code int                          `json:"code"`
-	Msg  string                       `json:"msg"`
-	Data []PythonBloombergGeneralData `json:"data"`
+	Code int                      `json:"code"`
+	Msg  string                   `json:"msg"`
+	Data []map[string]interface{} `json:"data"`
 }
 
 // BaseFromBloombergApiIndexAndData Bloomberg原始指标及数据
@@ -47,44 +50,122 @@ type BaseFromBloombergApiIndexData struct {
 	Value    float64   `description:"数据值"`
 }
 
-func FormatPythonBloombergGeneralData2Base(origin PythonBloombergGeneralData, frequency string) (item BaseFromBloombergApiIndexAndData) {
-	if origin.IDENTIFIER == "" {
-		return
-	}
-	item.IndexCode = strings.TrimSpace(origin.IDENTIFIER)
-	item.IndexName = strings.TrimSpace(origin.NAME)
-	item.Frequency = frequency
-	item.Unit = "无"
-	item.Data = make([]BaseFromBloombergApiIndexData, 0)
+// TaskConfPCSGBloomberg 配置
+type TaskConfPCSGBloomberg struct {
+	TaskKey             string `json:"TaskKey" description:"任务名称"`
+	IndexCodeField      string `json:"IndexCodeField" description:"指标编码字段名"`
+	IndexNameField      string `json:"IndexNameField" description:"指标名称字段名"`
+	FrequencyVal        string `json:"FrequencyVal" description:"频度"`
+	UnitVal             string `json:"UnitVal" description:"单位"`
+	DataTimeField       string `json:"DataTimeField" description:"数据日期字段名"`
+	DataTimeFormat      string `json:"DataTimeFormat" description:"数据日期格式"`
+	DataTimeMoveDays    int    `json:"DataTimeMoveDays" description:"数据日期位移天数"`
+	DataValField        string `json:"DataValField" description:"数据值字段名"`
+	CheckDataTimeField  string `json:"CheckDataTimeField" description:"需要校验的数据日期字段名"`
+	CheckDataTimeFormat string `json:"CheckDataTimeFormat" description:"需要校验的数据日期格式"`
+}
 
-	// 数据来源的值和日期对应字段不同
-	var dataTime time.Time
-	var val *float64
-	if origin.LAST_UPDATE_DATE_EOD != "" && origin.PX_LAST_EOD != nil {
-		dataTime, _ = time.ParseInLocation(utils.FormatDate, origin.LAST_UPDATE_DATE_EOD, time.Local)
-		val = origin.PX_LAST_EOD
+func FormatPythonBloombergGeneralData2Base(origin map[string]interface{}, conf *TaskConfPCSGBloomberg) (item BaseFromBloombergApiIndexAndData) {
+	// 指标编码
+	if v, ok := origin[conf.IndexCodeField]; ok {
+		indexCode, yes := v.(string)
+		if !yes {
+			return
+		}
+		item.IndexCode = strings.TrimSpace(indexCode)
 	}
-	if origin.LAST_UPDATE_DATE_EOD != "" && origin.PX_VOLUME_EOD != nil {
-		dataTime, _ = time.ParseInLocation(utils.FormatDate, origin.LAST_UPDATE_DATE_EOD, time.Local)
-		val = origin.PX_VOLUME_EOD
+
+	// 指标名称/频率/单位
+	if v, ok := origin[conf.IndexNameField]; ok {
+		indexName, yes := v.(string)
+		if yes {
+			item.IndexName = strings.TrimSpace(indexName)
+		}
 	}
-	if origin.DATE != "" && origin.PX_LAST != nil {
-		dataTime, _ = time.ParseInLocation(utils.FormatDate, origin.DATE, time.Local)
-		val = origin.PX_LAST
+	item.Frequency = conf.FrequencyVal
+	item.Unit = conf.UnitVal
+
+	// 校验一下指定字段的日期格式, 如果格式不对那么不处理
+	checkTimeFunc := func(dataMap map[string]interface{}) (pass bool) {
+		pass = true // 默认处理
+		f, ok := dataMap[conf.CheckDataTimeField]
+		if !ok {
+			return
+		}
+		checkField, yes := f.(string)
+		if !yes {
+			return
+		}
+		f, ok = dataMap[conf.CheckDataTimeFormat]
+		if !ok {
+			return
+		}
+		checkFormat, yes := f.(string)
+		if !yes {
+			return
+		}
+		f, ok = dataMap[checkField]
+		if !ok {
+			return
+		}
+		fieldVal, yes := f.(string)
+		if !yes {
+			return
+		}
+		// 格式不一致, 那么不处理
+		_, e := time.ParseInLocation(checkFormat, fieldVal, time.Local)
+		if e != nil {
+			pass = false
+			return
+		}
+		return
 	}
-	if origin.DATE != "" && origin.PX_VOLUME != nil {
-		dataTime, _ = time.ParseInLocation(utils.FormatDate, origin.DATE, time.Local)
-		val = origin.PX_VOLUME
+	if !checkTimeFunc(origin) {
+		return
 	}
-	if origin.PX_SETTLE_LAST_DT != "" && origin.PX_SETTLE != nil {
-		dataTime, _ = time.ParseInLocation(utils.FormatDate, origin.PX_SETTLE_LAST_DT, time.Local)
-		val = origin.PX_SETTLE
+
+	// 数据日期
+	item.Data = make([]BaseFromBloombergApiIndexData, 0)
+	var dataTime time.Time
+	if v, ok := origin[conf.DataTimeField]; ok {
+		strTime, yes := v.(string)
+		if !yes {
+			return
+		}
+		if strTime == "" {
+			return
+		}
+		d, e := time.ParseInLocation(conf.DataTimeFormat, strTime, time.Local)
+		if e != nil {
+			global.LOG.Info(fmt.Sprintf("数据日期解析失败, %s, err: %v", strTime, e))
+			return
+		}
+		if d.IsZero() {
+			global.LOG.Info(fmt.Sprintf("数据日期解析为空, %s", strTime))
+			return
+		}
+		if conf.DataTimeMoveDays != 0 {
+			d = d.AddDate(0, 0, conf.DataTimeMoveDays)
+		}
+		dataTime = d
 	}
-	if !dataTime.IsZero() && val != nil {
-		item.Data = append(item.Data, BaseFromBloombergApiIndexData{
-			DataTime: dataTime,
-			Value:    *val,
-		})
+
+	// 数据值
+	var dataVal *float64
+	if v, ok := origin[conf.DataValField]; ok {
+		val, yes := v.(*float64)
+		if !yes {
+			global.LOG.Info(fmt.Sprintf("数据float断言失败, %v", v))
+			return
+		}
+		if val == nil {
+			return
+		}
+		dataVal = val
 	}
+	item.Data = append(item.Data, BaseFromBloombergApiIndexData{
+		DataTime: dataTime,
+		Value:    *dataVal,
+	})
 	return
 }

services/index_data/pcsg_bloomberg.go

@@ -19,8 +19,18 @@ func GetPCSGBloombergGeneralIndex(taskKey, frequency string) (indexes []pcsg.Bas
 	if len(apiData) == 0 {
 		return
 	}
+
+	conf, e := LoadPCSGBloombergTask(taskKey)
+	if e != nil {
+		err = fmt.Errorf("加载配置失败, Err: %v", e)
+		return
+	}
+	if conf == nil {
+		err = fmt.Errorf("加载配置异常, %s", taskKey)
+		return
+	}
 	for _, v := range apiData {
-		t := pcsg.FormatPythonBloombergGeneralData2Base(v, frequency)
+		t := pcsg.FormatPythonBloombergGeneralData2Base(v, conf)
 		if t.IndexCode != "" {
 			indexes = append(indexes, t)
 		}
@@ -29,7 +39,7 @@ func GetPCSGBloombergGeneralIndex(taskKey, frequency string) (indexes []pcsg.Bas
 }
 
 // CurlPCSGBloombergGeneralDataApi 请求通用格式数据接口
-func CurlPCSGBloombergGeneralDataApi(taskKey string) (resultData []pcsg.PythonBloombergGeneralData, err error) {
+func CurlPCSGBloombergGeneralDataApi(taskKey string) (resultData []map[string]interface{}, err error) {
 	if taskKey == "" {
 		err = fmt.Errorf("任务Key为空")
 		return
@@ -40,7 +50,6 @@ func CurlPCSGBloombergGeneralDataApi(taskKey string) (resultData []pcsg.PythonBl
 	}
 	url := fmt.Sprintf("%s/api/bloomberg/general_data?task_key=%s", global.CONFIG.PCSG.BloombergApiUrl, taskKey)
 
-	//resp, e := http.Post(url, "application/json", bytes.NewBuffer([]byte("")))
 	resp, e := http.Get(url)
 	if e != nil {
 		err = fmt.Errorf("http post err: %s", e.Error())
@@ -58,6 +67,9 @@ func CurlPCSGBloombergGeneralDataApi(taskKey string) (resultData []pcsg.PythonBl
 		return
 	}
 
+	//str := `{"Code":200,"Msg":"","Data":[{"DL_REQUEST_ID":"IDpcsgDailySnap0330","DL_REQUEST_NAME":"contractsLastHistory","DL_SNAPSHOT_START_TIME":"2024-07-09T08:30:03","DL_SNAPSHOT_TZ":"JST","IDENTIFIER":"Comdty B001","RC":0,"DATE":"2024-05-09","PX_LAST":1183.88},{"DL_REQUEST_ID":"IDpcsgDailySnap0330","DL_REQUEST_NAME":"contractsLastHistory","DL_SNAPSHOT_START_TIME":"2024-07-10T08:30:03","DL_SNAPSHOT_TZ":"JST","IDENTIFIER":"Comdty B001","RC":0,"DATE":"2024-05-10","PX_LAST":1182.79},{"DL_REQUEST_ID":"IDpcsgDailySnap0330","DL_REQUEST_NAME":"contractsLastHistory","DL_SNAPSHOT_START_TIME":"2024-07-09T08:30:03","DL_SNAPSHOT_TZ":"JST","IDENTIFIER":"Comdty B002","RC":0,"DATE":"2024-05-10","PX_LAST":1162.63}]}`
+	//b := []byte(str)
+
 	result := new(pcsg.PythonBloombergGeneralResult)
 	if e = json.Unmarshal(b, &result); e != nil {
 		err = fmt.Errorf("result unmarshal err: %s\nresult: %s", e.Error(), string(b))
@@ -70,3 +82,25 @@ func CurlPCSGBloombergGeneralDataApi(taskKey string) (resultData []pcsg.PythonBl
 	resultData = result.Data
 	return
 }
+
+// LoadPCSGBloombergTask 加载配置
+func LoadPCSGBloombergTask(taskName string) (conf *pcsg.TaskConfPCSGBloomberg, err error) {
+	filePath := "./static/pcsg_task.json"
+	b, e := ioutil.ReadFile(filePath)
+	if e != nil {
+		err = fmt.Errorf("读取配置失败, err: %v", e)
+		return
+	}
+	tasks := make([]*pcsg.TaskConfPCSGBloomberg, 0)
+	if e = json.Unmarshal(b, &tasks); e != nil {
+		err = fmt.Errorf("解析配置失败, err: %v", e)
+		return
+	}
+	for _, v := range tasks {
+		if v.TaskKey == taskName {
+			conf = v
+			break
+		}
+	}
+	return
+}

static/pcsg_task.json

@@ -0,0 +1,171 @@
+[
+  {
+    "TaskKey": "IDpcsgDailyRunHistU2",
+    "IndexCodeField": "IDENTIFIER",
+    "IndexNameField": "NAME",
+    "FrequencyVal": "日度",
+    "UnitVal": "无",
+    "DataTimeField": "DATE",
+    "DataTimeFormat": "2006-01-02",
+    "DataTimeMoveDays": 0,
+    "DataValField": "PX_LAST",
+    "CheckDataTimeField": "",
+    "CheckDataTimeFormat": ""
+  },
+  {
+    "TaskKey": "IDpcsgDailyRunHist4",
+    "IndexCodeField": "IDENTIFIER",
+    "IndexNameField": "NAME",
+    "FrequencyVal": "日度",
+    "UnitVal": "无",
+    "DataTimeField": "DATE",
+    "DataTimeFormat": "2006-01-02",
+    "DataTimeMoveDays": 0,
+    "DataValField": "PX_LAST",
+    "CheckDataTimeField": "",
+    "CheckDataTimeFormat": ""
+  },
+  {
+    "TaskKey": "IDpcsgDailyRunHist1",
+    "IndexCodeField": "IDENTIFIER",
+    "IndexNameField": "NAME",
+    "FrequencyVal": "日度",
+    "UnitVal": "无",
+    "DataTimeField": "DATE",
+    "DataTimeFormat": "2006-01-02",
+    "DataTimeMoveDays": 0,
+    "DataValField": "PX_LAST",
+    "CheckDataTimeField": "",
+    "CheckDataTimeFormat": ""
+  },
+  {
+    "TaskKey": "IDpcsgDailyRunHist2",
+    "IndexCodeField": "IDENTIFIER",
+    "IndexNameField": "NAME",
+    "FrequencyVal": "日度",
+    "UnitVal": "无",
+    "DataTimeField": "DATE",
+    "DataTimeFormat": "2006-01-02",
+    "DataTimeMoveDays": 0,
+    "DataValField": "PX_LAST",
+    "CheckDataTimeField": "",
+    "CheckDataTimeFormat": ""
+  },
+  {
+    "TaskKey": "IDpcsgDailyRunHistV1",
+    "IndexCodeField": "IDENTIFIER",
+    "IndexNameField": "NAME",
+    "FrequencyVal": "日度",
+    "UnitVal": "无",
+    "DataTimeField": "DATE",
+    "DataTimeFormat": "2006-01-02",
+    "DataTimeMoveDays": 0,
+    "DataValField": "PX_VOLUME",
+    "CheckDataTimeField": "",
+    "CheckDataTimeFormat": ""
+  },
+  {
+    "TaskKey": "IDpcsgDailyRun4",
+    "IndexCodeField": "IDENTIFIER",
+    "IndexNameField": "NAME",
+    "FrequencyVal": "日度",
+    "UnitVal": "无",
+    "DataTimeField": "LAST_UPDATE_DATE_EOD",
+    "DataTimeFormat": "2006-01-02",
+    "DataTimeMoveDays": 0,
+    "DataValField": "PX_LAST_EOD",
+    "CheckDataTimeField": "",
+    "CheckDataTimeFormat": ""
+  },
+  {
+    "TaskKey": "IDpcsgDailyRun6",
+    "IndexCodeField": "IDENTIFIER",
+    "IndexNameField": "NAME",
+    "FrequencyVal": "日度",
+    "UnitVal": "无",
+    "DataTimeField": "PX_SETTLE_LAST_DT",
+    "DataTimeFormat": "2006-01-02",
+    "DataTimeMoveDays": 0,
+    "DataValField": "PX_SETTLE",
+    "CheckDataTimeField": "",
+    "CheckDataTimeFormat": ""
+  },
+  {
+    "TaskKey": "IDpcsgDailyRun7",
+    "IndexCodeField": "IDENTIFIER",
+    "IndexNameField": "NAME",
+    "FrequencyVal": "日度",
+    "UnitVal": "无",
+    "DataTimeField": "OPEN_INT_DATE",
+    "DataTimeFormat": "2006-01-02",
+    "DataTimeMoveDays": 0,
+    "DataValField": "OPEN_INT",
+    "CheckDataTimeField": "",
+    "CheckDataTimeFormat": ""
+  },
+  {
+    "TaskKey": "IDpcsgDailyRun8",
+    "IndexCodeField": "IDENTIFIER",
+    "IndexNameField": "NAME",
+    "FrequencyVal": "日度",
+    "UnitVal": "无",
+    "DataTimeField": "LAST_UPDATE_DATE_EOD",
+    "DataTimeFormat": "2006-01-02",
+    "DataTimeMoveDays": 0,
+    "DataValField": "PX_LAST_EOD",
+    "CheckDataTimeField": "",
+    "CheckDataTimeFormat": ""
+  },
+  {
+    "TaskKey": "IDpcsgDailySnap0000",
+    "IndexCodeField": "IDENTIFIER",
+    "IndexNameField": "NAME",
+    "FrequencyVal": "日度",
+    "UnitVal": "无",
+    "DataTimeField": "DL_SNAPSHOT_START_TIME",
+    "DataTimeFormat": "2006-01-02T15:04:05",
+    "DataTimeMoveDays": 0,
+    "DataValField": "PX_LAST",
+    "CheckDataTimeField": "LAST_UPDATE",
+    "CheckDataTimeFormat": "15:04:05"
+  },
+  {
+    "TaskKey": "IDpcsgDailySnap0330",
+    "IndexCodeField": "IDENTIFIER",
+    "IndexNameField": "NAME",
+    "FrequencyVal": "日度",
+    "UnitVal": "无",
+    "DataTimeField": "DL_SNAPSHOT_START_TIME",
+    "DataTimeFormat": "2006-01-02T15:04:05",
+    "DataTimeMoveDays": -1,
+    "DataValField": "PX_LAST",
+    "CheckDataTimeField": "LAST_UPDATE",
+    "CheckDataTimeFormat": "15:04:05"
+  },
+  {
+    "TaskKey": "IDpcsgDailySnap0345",
+    "IndexCodeField": "IDENTIFIER",
+    "IndexNameField": "NAME",
+    "FrequencyVal": "日度",
+    "UnitVal": "无",
+    "DataTimeField": "PX_CLOSE_DT",
+    "DataTimeFormat": "2006-01-02",
+    "DataTimeMoveDays": 0,
+    "DataValField": "PX_LAST",
+    "CheckDataTimeField": "",
+    "CheckDataTimeFormat": ""
+  },
+  {
+    "TaskKey": "IDpcsgMonthRun2",
+    "IndexCodeField": "IDENTIFIER",
+    "IndexNameField": "NAME",
+    "FrequencyVal": "月度",
+    "UnitVal": "无",
+    "DataTimeField": "LAST_UPDATE_DATE_EOD",
+    "DataTimeFormat": "2006-01-02",
+    "DataTimeMoveDays": 0,
+    "DataValField": "PX_LAST_EOD",
+    "CheckDataTimeField": "",
+    "CheckDataTimeFormat": ""
+  }
+]