Merge remote-tracking branch 'origin/master' into xy_data

# Conflicts:
#	utils/config.go

models/data_manage/base_from_eia_steo.go

@@ -32,6 +32,12 @@ func AddBaseFromEiaSteoIndex(item *BaseFromEiaSteoIndex) (lastId int64, err erro
 	return
 }
 
+func InsertOrUpdateBaseFromEiaSteoIndex(item *BaseFromEiaSteoIndex) (lastId int64, err error) {
+	o := orm.NewOrm()
+	lastId, err = o.InsertOrUpdate(item)
+	return
+}
+
 // Add 新增指标
 func (item *BaseFromEiaSteoIndex) Add() (err error) {
 	o := orm.NewOrm()

models/data_manage/base_from_icpi.go

@@ -46,6 +46,12 @@ func (obj *BaseFromIcpiIndex) AddBaseFromIcpiIndex(item *BaseFromIcpiIndex) (las
 	return
 }
 
+func (obj *BaseFromIcpiIndex) InsertOrUpdateBaseFromIcpiIndex(item *BaseFromIcpiIndex) (lastId int64, err error) {
+	o := orm.NewOrm()
+	lastId, err = o.InsertOrUpdate(item)
+	return
+}
+
 type BaseFromIcpiClassify struct {
 	BaseFromIcpiClassifyId int    `orm:"column(base_from_icpi_classify_id);pk"`
 	ClassifyName           string `description:"分类名称"`
@@ -76,6 +82,12 @@ func AddBaseFromIcpiClassify(item *BaseFromIcpiClassify) (lastId int64, err erro
 	return
 }
 
+func InsertOrUpdateBaseFromIcpiClassify(item *BaseFromIcpiClassify) (lastId int64, err error) {
+	o := orm.NewOrm()
+	lastId, err = o.InsertOrUpdate(item)
+	return
+}
+
 // GetBaseFromComTradeMaxDate 获取ICPI消费者指数最大数据
 func GetBaseFromIcpiMaxDate() (max_date time.Time, err error) {
 	o := orm.NewOrm()

services/data/base_edb_lib.go

@@ -21,7 +21,11 @@ func RefreshEdbData(edbInfoId, source, subSource int, edbCode, startDate string)
 	urlStr := ``
 	switch source {
 	case utils.DATA_SOURCE_THS:
-		urlStr = "ths/refresh"
+		if subSource == 0 {
+			urlStr = "ths/refresh"
+		} else {
+			urlStr = "ths/ds/refresh"
+		}
 	case utils.DATA_SOURCE_WIND:
 		if subSource == 0 {
 			urlStr = "wind/refresh"

services/data/edb_info.go

@@ -135,9 +135,18 @@ func RefreshDataFromPb(wg *sync.WaitGroup) (err error) {
 		}
 		wg.Done()
 	}()
+	if utils.PbRefreshOpen == "0" {
+		return
+	}
 	var condition string
 	var pars []interface{}
-	condition += " AND source=? AND frequency in ('日度','周度') "
+	var frequencyStr string
+	if utils.PbRefreshFrequency == "" {
+		frequencyStr = "'日度','周度'"
+	} else {
+		frequencyStr = utils.PbRefreshFrequency
+	}
+	condition += " AND source=? AND frequency in (" + frequencyStr + ") "
 	pars = append(pars, utils.DATA_SOURCE_PB)
 
 	items, err := data_manage.GetEdbInfoByCondition(condition, pars, 0)

services/eia_steo.go

@@ -34,7 +34,7 @@ func SyncEiaSteoIndex() (err error) {
 		return err
 	}
 	//获取所有指标信息  某一天的
-	allIndex, err := data_manage.GetBaseFromEiaSteoIndexAll(startDate)
+	/*allIndex, err := data_manage.GetBaseFromEiaSteoIndexAll(startDate)
 	if err != nil {
 		fmt.Println("get GetBaseFromEiaSteoIndexAll err:" + err.Error())
 		return
@@ -43,16 +43,16 @@ func SyncEiaSteoIndex() (err error) {
 	existIndexMap := make(map[string]*data_manage.BaseFromEiaSteoIndex)
 	for _, v := range allIndex {
 		existIndexMap[v.IndexCode] = v
-	}
+	}*/
 
 	for _, zv := range respObj.Data {
-		if _, ok := existIndexMap[zv.IndexCode]; !ok {
-			newID, err := data_manage.AddBaseFromEiaSteoIndex(zv)
-			if err != nil {
-				fmt.Println("insert error:", err)
-			}
-			fmt.Println("insert new indexID:", newID)
+		//if _, ok := existIndexMap[zv.IndexCode]; !ok {
+		newID, err := data_manage.InsertOrUpdateBaseFromEiaSteoIndex(zv)
+		if err != nil {
+			fmt.Println("InsertOrUpdateBaseFromEiaSteoIndex error:", err)
 		}
+		fmt.Println("InsertOrUpdateBaseFromEiaSteoIndex new indexID:", newID)
+		//}
 	}
 	return err
 }

services/icpi.go

@@ -31,12 +31,12 @@ func SyncBaseFromIcpi() (err error) {
 	respObj := new(data_manage.IcpiIndexResp)
 	err = json.Unmarshal([]byte(result), &respObj)
 	if err != nil {
-		utils.FileLog.Info("err:",err.Error())
-		fmt.Println("err:",err.Error())
+		utils.FileLog.Info("err:", err.Error())
+		fmt.Println("err:", err.Error())
 		return err
 	}
 	//获取所有指标信息  某一天的
-	allIndex, err := data_manage.GetBaseFromIcpiIndexAll(startDate)
+	/*allIndex, err := data_manage.GetBaseFromIcpiIndexAll(startDate)
 	if err != nil {
 		return
 	}
@@ -44,18 +44,16 @@ func SyncBaseFromIcpi() (err error) {
 	existIndexMap := make(map[int]*data_manage.BaseFromIcpiIndex)
 	for _, v := range allIndex {
 		existIndexMap[v.BaseFromIcpiIndexId] = v
-	}
+	}*/
 
 	icpiObj := new(data_manage.BaseFromIcpiIndex)
 
 	for _, zv := range respObj.Data {
-		if _, ok := existIndexMap[zv.BaseFromIcpiIndexId]; !ok {
-			newID, err := icpiObj.AddBaseFromIcpiIndex(zv)
-			if err != nil {
-				fmt.Println("insert error:", err)
-			}
-			fmt.Println("insert new indexID:", newID)
+		newID, err := icpiObj.InsertOrUpdateBaseFromIcpiIndex(zv)
+		if err != nil {
+			fmt.Println("InsertOrUpdateBaseFromIcpiIndex error:", err)
 		}
+		fmt.Println("InsertOrUpdateBaseFromIcpiIndex new indexID:", newID)
 	}
 	return err
 }
@@ -78,24 +76,24 @@ func SyncBaseFromIcpiClassify() (err error) {
 		return err
 	}
 	//获取所有分类
-	allClassify, err := data_manage.GetBaseFromIcpiClassifyAll()
-	if err != nil {
-		return
-	}
+	/*	allClassify, err := data_manage.GetBaseFromIcpiClassifyAll()
+		if err != nil {
+			return
+		}
 
-	existIndexMap := make(map[int]*data_manage.BaseFromIcpiClassify)
-	for _, v := range allClassify {
-		existIndexMap[v.BaseFromIcpiClassifyId] = v
-	}
+		existIndexMap := make(map[int]*data_manage.BaseFromIcpiClassify)
+		for _, v := range allClassify {
+			existIndexMap[v.BaseFromIcpiClassifyId] = v
+		}*/
 
 	for _, item := range respObj.Data {
-		if _, ok := existIndexMap[item.BaseFromIcpiClassifyId]; !ok {
-			newID, err := data_manage.AddBaseFromIcpiClassify(item)
-			if err != nil {
-				fmt.Println("AddBaseFromIcpiClassify error:", err)
-			}
-			fmt.Println("AddBaseFromIcpiClassify new indexID:", newID)
+		//if _, ok := existIndexMap[item.BaseFromIcpiClassifyId]; !ok {
+		newID, err := data_manage.InsertOrUpdateBaseFromIcpiClassify(item)
+		if err != nil {
+			fmt.Println("InsertOrUpdateBaseFromIcpiClassify error:", err)
 		}
+		fmt.Println("InsertOrUpdateBaseFromIcpiClassify new indexID:", newID)
+		//}
 	}
 	return err
 }

services/meeting_probabilities.go

@@ -21,7 +21,7 @@ func SyncMeetingProbabilities() (err error) {
 
 	method := `index/list`
 	data := make(map[string]interface{})
-	data["Source"] = utils.DATA_SOURCE_MEETING_PROBABILITIES
+	data["Source"] = 61
 	data["StartDate"] = startDate
 	//data["EndDate"] = endDate
 	result, err := HttpPost("SyncMeetingProbabilities", method, data)

services/sync_hz_data.go

@@ -11,39 +11,39 @@ import (
 func SyncHzDataIndex(cont context.Context) (err error) {
 	fmt.Println("SyncHzDataIndex start:", time.Now().Format(utils.FormatDateTime))
 	//郑州交易所
-	err = SyncRankingFromZhengzhou()
-	if err != nil {
-		fmt.Println("SyncRankingFromZhengzhou Err:" + err.Error())
-		return
-	}
+	go SyncRankingFromZhengzhou()
+	//if err != nil {
+	//	fmt.Println("SyncRankingFromZhengzhou Err:" + err.Error())
+	//	return
+	//}
 
 	//大商所
-	err = SyncRankingFromDalian()
-	if err != nil {
-		fmt.Println("SyncRankingFromDalian Err:" + err.Error())
-		return
-	}
+	go SyncRankingFromDalian()
+	//if err != nil {
+	//	fmt.Println("SyncRankingFromDalian Err:" + err.Error())
+	//	return
+	//}
 
 	//上期所
-	err = SyncRankingFromShanghai()
-	if err != nil {
-		fmt.Println("SyncRankingFromDalian Err:" + err.Error())
-		return
-	}
+	go SyncRankingFromShanghai()
+	//if err != nil {
+	//	fmt.Println("SyncRankingFromDalian Err:" + err.Error())
+	//	return
+	//}
 
 	//中金所
-	err = SyncRankingFromCffex()
-	if err != nil {
-		fmt.Println("SyncRankingFromCffex Err:" + err.Error())
-		return
-	}
+	go SyncRankingFromCffex()
+	//if err != nil {
+	//	fmt.Println("SyncRankingFromCffex Err:" + err.Error())
+	//	return
+	//}
 
 	//上期能源
-	err = SyncRankingFromIne()
-	if err != nil {
-		fmt.Println("SyncRankingFromIne Err:" + err.Error())
-		return
-	}
+	go SyncRankingFromIne()
+	//if err != nil {
+	//	fmt.Println("SyncRankingFromIne Err:" + err.Error())
+	//	return
+	//}
 
 	//欧洲天然气
 	err = SyncRankingFromEic()

utils/config.go

@@ -132,6 +132,10 @@ var (
 	SyncIndexClassifyTime string // 桥接服务-指标分类同步的时间配置
 )
 
+var (
+	PbRefreshFrequency string // 彭博更新频度
+	PbRefreshOpen      string // 彭博刷新开关 0关闭，默认都是开启的
+)
 var HzDataApi string
 
 func init() {
@@ -284,4 +288,9 @@ func init() {
 	if HzDataApi == "" {
 		HzDataApi = "https://hzdataapi.hzinsights.com/hzdataapi/"
 	}
+
+	// 彭博刷新频度
+	PbRefreshFrequency = config["pb_refresh_frequency"]
+	// 彭博刷新开关
+	PbRefreshOpen = config["pb_refresh_open"]
 }

utils/constants.go

@@ -92,10 +92,10 @@ const (
 	DATA_SOURCE_PREDICT_CALCULATE_LJZ                           //预测指标 - 累计值 -> 65
 	DATA_SOURCE_PREDICT_CALCULATE_LJZNCZJ                       //预测指标 - 累计值(年初至今） -> 66
 	DATA_SOURCE_CALCULATE_STANDARD_DEVIATION                    //标准差->67
-	DATA_SOURCE_CALCULATE_PERCENTILE                            //百分位图表->68
+	DATA_SOURCE_CALCULATE_PERCENTILE                            //百分位->68
 	DATA_SOURCE_PREDICT_CALCULATE_STANDARD_DEVIATION            //预测标准差->69
 	DATA_SOURCE_PREDICT_CALCULATE_PERCENTILE                    //预测百分位->70
-	DATA_SOURCE_FUBAO                                           //富宝->71
+	DATA_SOURCE_FUBAO                                           //富宝数据->71
 	DATA_SOURCE_CALCULATE_ZSXY                                  // 指数修匀->72
 	DATA_SOURCE_PREDICT_CALCULATE_ZSXY                          // 预测指数修匀->73
 	DATA_SOURCE_CALCULATE_ZDYFX                                 // 自定义分析->74