pcsg优化,待测试

models/pcsg/bloomberg.go

@@ -2,18 +2,11 @@ package pcsg
 
 import (
 	"eta/eta_bridge/global"
-	"eta/eta_bridge/utils"
 	"fmt"
 	"strings"
 	"time"
 )
 
-const (
-	TaskKey0000      = "IDpcsgDailySnap0000"
-	SubOneDayTaskKey = "IDpcsgDailySnap0330" // 日期-1的key
-	TaskKey0345      = "IDpcsgDailySnap0345"
-)
-
 // PythonBloombergGeneralData 通用数据格式(有空改成Map吧...然后做成配置方便加新字段...)
 type PythonBloombergGeneralData struct {
 	NAME                   string   `json:"NAME" description:"指标名称, 可能为null"`
@@ -29,15 +22,13 @@ type PythonBloombergGeneralData struct {
 	OPEN_INT               *float64 `json:"OPEN_INT"`
 	OPEN_INT_DATE          string   `json:"OPEN_INT_DATE"`
 	DL_SNAPSHOT_START_TIME string   `json:"DL_SNAPSHOT_START_TIME" description:"数据日期"`
-	LAST_UPDATE            string   `json:"LAST_UPDATE" description:"IDpcsgDailySnap0000和IDpcsgDailySnap0330判断用的"`
-	PX_CLOSE_DT            string   `json:"PX_CLOSE_DT" description:"数据日期"`
 }
 
 // PythonBloombergGeneralResult API响应体
 type PythonBloombergGeneralResult struct {
-	Code int                          `json:"code"`
-	Msg  string                       `json:"msg"`
-	Data []PythonBloombergGeneralData `json:"data"`
+	Code int                      `json:"code"`
+	Msg  string                   `json:"msg"`
+	Data []map[string]interface{} `json:"data"`
 }
 
 // BaseFromBloombergApiIndexAndData Bloomberg原始指标及数据
@@ -59,70 +50,122 @@ type BaseFromBloombergApiIndexData struct {
 	Value    float64   `description:"数据值"`
 }
 
-func FormatPythonBloombergGeneralData2Base(origin PythonBloombergGeneralData, frequency, taskKey string) (item BaseFromBloombergApiIndexAndData) {
-	if origin.IDENTIFIER == "" {
-		return
-	}
-	item.IndexCode = strings.TrimSpace(origin.IDENTIFIER)
-	item.IndexName = strings.TrimSpace(origin.NAME)
-	item.Frequency = frequency
-	item.Unit = "无"
-	item.Data = make([]BaseFromBloombergApiIndexData, 0)
+// TaskConfPCSGBloomberg 配置
+type TaskConfPCSGBloomberg struct {
+	TaskKey             string `json:"TaskKey" description:"任务名称"`
+	IndexCodeField      string `json:"IndexCodeField" description:"指标编码字段名"`
+	IndexNameField      string `json:"IndexNameField" description:"指标名称字段名"`
+	FrequencyVal        string `json:"FrequencyVal" description:"频度"`
+	UnitVal             string `json:"UnitVal" description:"单位"`
+	DataTimeField       string `json:"DataTimeField" description:"数据日期字段名"`
+	DataTimeFormat      string `json:"DataTimeFormat" description:"数据日期格式"`
+	DataTimeMoveDays    int    `json:"DataTimeMoveDays" description:"数据日期位移天数"`
+	DataValField        string `json:"DataValField" description:"数据值字段名"`
+	CheckDataTimeField  string `json:"CheckDataTimeField" description:"需要校验的数据日期字段名"`
+	CheckDataTimeFormat string `json:"CheckDataTimeFormat" description:"需要校验的数据日期格式"`
+}
 
-	// 数据来源的值和日期对应字段不同
-	var dataTime time.Time
-	var val *float64
-	if origin.LAST_UPDATE_DATE_EOD != "" && origin.PX_LAST_EOD != nil {
-		dataTime, _ = time.ParseInLocation(utils.FormatDate, origin.LAST_UPDATE_DATE_EOD, time.Local)
-		val = origin.PX_LAST_EOD
-	}
-	if origin.LAST_UPDATE_DATE_EOD != "" && origin.PX_VOLUME_EOD != nil {
-		dataTime, _ = time.ParseInLocation(utils.FormatDate, origin.LAST_UPDATE_DATE_EOD, time.Local)
-		val = origin.PX_VOLUME_EOD
-	}
-	if origin.DATE != "" && origin.PX_LAST != nil {
-		dataTime, _ = time.ParseInLocation(utils.FormatDate, origin.DATE, time.Local)
-		val = origin.PX_LAST
-	}
-	if origin.DATE != "" && origin.PX_VOLUME != nil {
-		dataTime, _ = time.ParseInLocation(utils.FormatDate, origin.DATE, time.Local)
-		val = origin.PX_VOLUME
-	}
-	if origin.PX_SETTLE_LAST_DT != "" && origin.PX_SETTLE != nil {
-		dataTime, _ = time.ParseInLocation(utils.FormatDate, origin.PX_SETTLE_LAST_DT, time.Local)
-		val = origin.PX_SETTLE
-	}
-	if origin.OPEN_INT_DATE != "" && origin.OPEN_INT != nil {
-		dataTime, _ = time.ParseInLocation(utils.FormatDate, origin.OPEN_INT_DATE, time.Local)
-		val = origin.OPEN_INT
+func FormatPythonBloombergGeneralData2Base(origin map[string]interface{}, conf *TaskConfPCSGBloomberg) (item BaseFromBloombergApiIndexAndData) {
+	// 指标编码
+	if v, ok := origin[conf.IndexCodeField]; ok {
+		indexCode, yes := v.(string)
+		if !yes {
+			return
+		}
+		item.IndexCode = strings.TrimSpace(indexCode)
 	}
-	if taskKey == TaskKey0345 && origin.PX_CLOSE_DT != "" && origin.PX_LAST != nil {
-		dataTime, _ = time.ParseInLocation(utils.FormatDate, origin.PX_CLOSE_DT, time.Local)
-		val = origin.PX_LAST
+
+	// 指标名称/频率/单位
+	if v, ok := origin[conf.IndexNameField]; ok {
+		indexName, yes := v.(string)
+		if yes {
+			item.IndexName = strings.TrimSpace(indexName)
+		}
 	}
-	if (taskKey == TaskKey0000 || taskKey == SubOneDayTaskKey) && origin.DL_SNAPSHOT_START_TIME != "" && origin.PX_LAST != nil {
-		// 根据LAST_UPDATE的格式进行判断, 仅加入格式为hh:mm:ss的
-		_, e := time.ParseInLocation("15:04:05", origin.LAST_UPDATE, time.Local)
+	item.Frequency = conf.FrequencyVal
+	item.Unit = conf.UnitVal
+
+	// 校验一下指定字段的日期格式, 如果格式不对那么不处理
+	checkTimeFunc := func(dataMap map[string]interface{}) (pass bool) {
+		pass = true // 默认处理
+		f, ok := dataMap[conf.CheckDataTimeField]
+		if !ok {
+			return
+		}
+		checkField, yes := f.(string)
+		if !yes {
+			return
+		}
+		f, ok = dataMap[conf.CheckDataTimeFormat]
+		if !ok {
+			return
+		}
+		checkFormat, yes := f.(string)
+		if !yes {
+			return
+		}
+		f, ok = dataMap[checkField]
+		if !ok {
+			return
+		}
+		fieldVal, yes := f.(string)
+		if !yes {
+			return
+		}
+		// 格式不一致, 那么不处理
+		_, e := time.ParseInLocation(checkFormat, fieldVal, time.Local)
 		if e != nil {
+			pass = false
 			return
 		}
+		return
+	}
+	if !checkTimeFunc(origin) {
+		return
+	}
 
-		d, e := time.ParseInLocation("2006-01-02T15:04:05", origin.DL_SNAPSHOT_START_TIME, time.Local)
+	// 数据日期
+	item.Data = make([]BaseFromBloombergApiIndexData, 0)
+	var dataTime time.Time
+	if v, ok := origin[conf.DataTimeField]; ok {
+		strTime, yes := v.(string)
+		if !yes {
+			return
+		}
+		if strTime == "" {
+			return
+		}
+		d, e := time.ParseInLocation(conf.DataTimeFormat, strTime, time.Local)
 		if e != nil {
-			global.LOG.Info(fmt.Sprintf("DL_SNAPSHOT_START_TIME日期解析失败, %s", origin.DL_SNAPSHOT_START_TIME))
+			global.LOG.Info(fmt.Sprintf("数据日期解析失败, %s, err: %v", strTime, e))
+			return
 		}
-		dataTime = d
-		// 这个任务日期需要-1
-		if taskKey == SubOneDayTaskKey {
-			dataTime = dataTime.AddDate(0, 0, -1)
+		if d.IsZero() {
+			global.LOG.Info(fmt.Sprintf("数据日期解析为空, %s", strTime))
+			return
+		}
+		if conf.DataTimeMoveDays != 0 {
+			d = d.AddDate(0, 0, conf.DataTimeMoveDays)
 		}
-		val = origin.PX_LAST
+		dataTime = d
 	}
-	if !dataTime.IsZero() && val != nil {
-		item.Data = append(item.Data, BaseFromBloombergApiIndexData{
-			DataTime: dataTime,
-			Value:    *val,
-		})
+
+	// 数据值
+	var dataVal *float64
+	if v, ok := origin[conf.DataValField]; ok {
+		val, yes := v.(*float64)
+		if !yes {
+			global.LOG.Info(fmt.Sprintf("数据float断言失败, %v", v))
+			return
+		}
+		if val == nil {
+			return
+		}
+		dataVal = val
 	}
+	item.Data = append(item.Data, BaseFromBloombergApiIndexData{
+		DataTime: dataTime,
+		Value:    *dataVal,
+	})
 	return
 }

services/index_data/pcsg_bloomberg.go

@@ -19,8 +19,18 @@ func GetPCSGBloombergGeneralIndex(taskKey, frequency string) (indexes []pcsg.Bas
 	if len(apiData) == 0 {
 		return
 	}
+
+	conf, e := LoadPCSGBloombergTask(taskKey)
+	if e != nil {
+		err = fmt.Errorf("加载配置失败, Err: %v", e)
+		return
+	}
+	if conf == nil {
+		err = fmt.Errorf("加载配置异常, %s", taskKey)
+		return
+	}
 	for _, v := range apiData {
-		t := pcsg.FormatPythonBloombergGeneralData2Base(v, frequency, taskKey)
+		t := pcsg.FormatPythonBloombergGeneralData2Base(v, conf)
 		if t.IndexCode != "" {
 			indexes = append(indexes, t)
 		}
@@ -29,7 +39,7 @@ func GetPCSGBloombergGeneralIndex(taskKey, frequency string) (indexes []pcsg.Bas
 }
 
 // CurlPCSGBloombergGeneralDataApi 请求通用格式数据接口
-func CurlPCSGBloombergGeneralDataApi(taskKey string) (resultData []pcsg.PythonBloombergGeneralData, err error) {
+func CurlPCSGBloombergGeneralDataApi(taskKey string) (resultData []map[string]interface{}, err error) {
 	if taskKey == "" {
 		err = fmt.Errorf("任务Key为空")
 		return
@@ -72,3 +82,25 @@ func CurlPCSGBloombergGeneralDataApi(taskKey string) (resultData []pcsg.PythonBl
 	resultData = result.Data
 	return
 }
+
+// LoadPCSGBloombergTask 加载配置
+func LoadPCSGBloombergTask(taskName string) (conf *pcsg.TaskConfPCSGBloomberg, err error) {
+	filePath := "./static/pcsg_task.json"
+	b, e := ioutil.ReadFile(filePath)
+	if e != nil {
+		err = fmt.Errorf("读取配置失败, err: %v", e)
+		return
+	}
+	tasks := make([]*pcsg.TaskConfPCSGBloomberg, 0)
+	if e = json.Unmarshal(b, &tasks); e != nil {
+		err = fmt.Errorf("解析配置失败, err: %v", e)
+		return
+	}
+	for _, v := range tasks {
+		if v.TaskKey == taskName {
+			conf = v
+			break
+		}
+	}
+	return
+}

static/pcsg_task.json

@@ -0,0 +1,171 @@
+[
+  {
+    "TaskKey": "IDpcsgDailyRunHistU2",
+    "IndexCodeField": "IDENTIFIER",
+    "IndexNameField": "NAME",
+    "FrequencyVal": "日度",
+    "UnitVal": "无",
+    "DataTimeField": "DATE",
+    "DataTimeFormat": "2006-01-02",
+    "DataTimeMoveDays": 0,
+    "DataValField": "PX_LAST",
+    "CheckDataTimeField": "",
+    "CheckDataTimeFormat": ""
+  },
+  {
+    "TaskKey": "IDpcsgDailyRunHist4",
+    "IndexCodeField": "IDENTIFIER",
+    "IndexNameField": "NAME",
+    "FrequencyVal": "日度",
+    "UnitVal": "无",
+    "DataTimeField": "DATE",
+    "DataTimeFormat": "2006-01-02",
+    "DataTimeMoveDays": 0,
+    "DataValField": "PX_LAST",
+    "CheckDataTimeField": "",
+    "CheckDataTimeFormat": ""
+  },
+  {
+    "TaskKey": "IDpcsgDailyRunHist1",
+    "IndexCodeField": "IDENTIFIER",
+    "IndexNameField": "NAME",
+    "FrequencyVal": "日度",
+    "UnitVal": "无",
+    "DataTimeField": "DATE",
+    "DataTimeFormat": "2006-01-02",
+    "DataTimeMoveDays": 0,
+    "DataValField": "PX_LAST",
+    "CheckDataTimeField": "",
+    "CheckDataTimeFormat": ""
+  },
+  {
+    "TaskKey": "IDpcsgDailyRunHist2",
+    "IndexCodeField": "IDENTIFIER",
+    "IndexNameField": "NAME",
+    "FrequencyVal": "日度",
+    "UnitVal": "无",
+    "DataTimeField": "DATE",
+    "DataTimeFormat": "2006-01-02",
+    "DataTimeMoveDays": 0,
+    "DataValField": "PX_LAST",
+    "CheckDataTimeField": "",
+    "CheckDataTimeFormat": ""
+  },
+  {
+    "TaskKey": "IDpcsgDailyRunHistV1",
+    "IndexCodeField": "IDENTIFIER",
+    "IndexNameField": "NAME",
+    "FrequencyVal": "日度",
+    "UnitVal": "无",
+    "DataTimeField": "DATE",
+    "DataTimeFormat": "2006-01-02",
+    "DataTimeMoveDays": 0,
+    "DataValField": "PX_VOLUME",
+    "CheckDataTimeField": "",
+    "CheckDataTimeFormat": ""
+  },
+  {
+    "TaskKey": "IDpcsgDailyRun4",
+    "IndexCodeField": "IDENTIFIER",
+    "IndexNameField": "NAME",
+    "FrequencyVal": "日度",
+    "UnitVal": "无",
+    "DataTimeField": "LAST_UPDATE_DATE_EOD",
+    "DataTimeFormat": "2006-01-02",
+    "DataTimeMoveDays": 0,
+    "DataValField": "PX_LAST_EOD",
+    "CheckDataTimeField": "",
+    "CheckDataTimeFormat": ""
+  },
+  {
+    "TaskKey": "IDpcsgDailyRun6",
+    "IndexCodeField": "IDENTIFIER",
+    "IndexNameField": "NAME",
+    "FrequencyVal": "日度",
+    "UnitVal": "无",
+    "DataTimeField": "PX_SETTLE_LAST_DT",
+    "DataTimeFormat": "2006-01-02",
+    "DataTimeMoveDays": 0,
+    "DataValField": "PX_SETTLE",
+    "CheckDataTimeField": "",
+    "CheckDataTimeFormat": ""
+  },
+  {
+    "TaskKey": "IDpcsgDailyRun7",
+    "IndexCodeField": "IDENTIFIER",
+    "IndexNameField": "NAME",
+    "FrequencyVal": "日度",
+    "UnitVal": "无",
+    "DataTimeField": "OPEN_INT_DATE",
+    "DataTimeFormat": "2006-01-02",
+    "DataTimeMoveDays": 0,
+    "DataValField": "OPEN_INT",
+    "CheckDataTimeField": "",
+    "CheckDataTimeFormat": ""
+  },
+  {
+    "TaskKey": "IDpcsgDailyRun8",
+    "IndexCodeField": "IDENTIFIER",
+    "IndexNameField": "NAME",
+    "FrequencyVal": "日度",
+    "UnitVal": "无",
+    "DataTimeField": "LAST_UPDATE_DATE_EOD",
+    "DataTimeFormat": "2006-01-02",
+    "DataTimeMoveDays": 0,
+    "DataValField": "PX_LAST_EOD",
+    "CheckDataTimeField": "",
+    "CheckDataTimeFormat": ""
+  },
+  {
+    "TaskKey": "IDpcsgDailySnap0000",
+    "IndexCodeField": "IDENTIFIER",
+    "IndexNameField": "NAME",
+    "FrequencyVal": "日度",
+    "UnitVal": "无",
+    "DataTimeField": "DL_SNAPSHOT_START_TIME",
+    "DataTimeFormat": "2006-01-02T15:04:05",
+    "DataTimeMoveDays": 0,
+    "DataValField": "PX_LAST",
+    "CheckDataTimeField": "LAST_UPDATE",
+    "CheckDataTimeFormat": "15:04:05"
+  },
+  {
+    "TaskKey": "IDpcsgDailySnap0330",
+    "IndexCodeField": "IDENTIFIER",
+    "IndexNameField": "NAME",
+    "FrequencyVal": "日度",
+    "UnitVal": "无",
+    "DataTimeField": "DL_SNAPSHOT_START_TIME",
+    "DataTimeFormat": "2006-01-02T15:04:05",
+    "DataTimeMoveDays": -1,
+    "DataValField": "PX_LAST",
+    "CheckDataTimeField": "LAST_UPDATE",
+    "CheckDataTimeFormat": "15:04:05"
+  },
+  {
+    "TaskKey": "IDpcsgDailySnap0345",
+    "IndexCodeField": "IDENTIFIER",
+    "IndexNameField": "NAME",
+    "FrequencyVal": "日度",
+    "UnitVal": "无",
+    "DataTimeField": "PX_CLOSE_DT",
+    "DataTimeFormat": "2006-01-02",
+    "DataTimeMoveDays": 0,
+    "DataValField": "PX_LAST",
+    "CheckDataTimeField": "",
+    "CheckDataTimeFormat": ""
+  },
+  {
+    "TaskKey": "IDpcsgMonthRun2",
+    "IndexCodeField": "IDENTIFIER",
+    "IndexNameField": "NAME",
+    "FrequencyVal": "月度",
+    "UnitVal": "无",
+    "DataTimeField": "LAST_UPDATE_DATE_EOD",
+    "DataTimeFormat": "2006-01-02",
+    "DataTimeMoveDays": 0,
+    "DataValField": "PX_LAST_EOD",
+    "CheckDataTimeField": "",
+    "CheckDataTimeFormat": ""
+  }
+]