新增富宝数据

controllers/base_from_fubao.go

@@ -0,0 +1,146 @@
+package controllers
+
+import (
+	"encoding/json"
+	"hongze/hongze_edb_lib/logic"
+	"hongze/hongze_edb_lib/models"
+	"hongze/hongze_edb_lib/services"
+	"hongze/hongze_edb_lib/utils"
+	"strconv"
+	"time"
+)
+
+// 富宝-数据对接
+type FuBaoController struct {
+	BaseAuthController
+}
+
+// @Title 新增富宝指标接口
+// @Description  新增富宝指标接口
+// @Success 200 {object} models.AddEdbInfoReq
+// @router /add [post]
+func (this *FuBaoController) Add() {
+	br := new(models.BaseResponse).Init()
+	var cacheKey string
+	defer func() {
+		utils.Rc.Delete(cacheKey)
+		this.Data["json"] = br
+		this.ServeJSON()
+	}()
+	source := utils.DATA_SOURCE_FUBAO
+	var req models.AddEdbInfoReq
+	err := json.Unmarshal(this.Ctx.Input.RequestBody, &req)
+	if err != nil {
+		br.Msg = "参数解析异常！"
+		br.ErrMsg = "参数解析失败，Err:" + err.Error()
+		return
+	}
+	if req.EdbCode == "" {
+		br.Msg = "请输入指标编码！"
+		br.ErrMsg = "请输入指标编码，指标编码为空"
+		return
+	}
+	cacheKey = utils.CACHE_EDB_DATA_ADD + strconv.Itoa(source) + "_" + req.EdbCode
+	if !utils.Rc.IsExist(cacheKey) {
+		utils.Rc.SetNX(cacheKey, 1, 1*time.Minute)
+		dataItem, err := services.GetEdbDataFromFuBao(req.EdbCode, utils.BASE_START_DATE, utils.BASE_END_DATE)
+		if err != nil {
+			br.Msg = "获取指标信息失败！"
+			br.ErrMsg = "获取指标信息失败 GetEdbDataFromLt，Err:" + err.Error()
+			return
+		}
+		if dataItem.Code != 200 {
+			br.Msg = "获取富宝指标数据失败！"
+			br.ErrMsg = "获取富宝指标数据失败 GetEdbDataFromLt，Code:" + strconv.Itoa(dataItem.Code)
+			return
+		}
+		err = models.AddEdbDataFromFubao(req.EdbCode, dataItem)
+		if err != nil {
+			br.Msg = "获取指标信息失败！"
+			br.ErrMsg = "获取指标信息失败 AddEdbDataFromFubao，Err:" + err.Error()
+			return
+		}
+		br.Ret = 200
+		br.Success = true
+		br.Msg = "获取成功"
+	} else {
+		br.Ret = 501
+		br.Success = true
+		br.Msg = "系统处理中，请稍后重试"
+	}
+}
+
+// @Title 刷新富宝指标接口
+// @Description 刷新路透社指标接口
+// @Success 200 {object} models.RefreshEdbInfoReq
+// @router /refresh [post]
+func (this *FuBaoController) Refresh() {
+	br := new(models.BaseResponse).Init()
+	var cacheKey string
+	defer func() {
+		utils.Rc.Delete(cacheKey)
+		this.Data["json"] = br
+		this.ServeJSON()
+	}()
+	source := utils.DATA_SOURCE_FUBAO
+	var req models.RefreshEdbInfoReq
+	err := json.Unmarshal(this.Ctx.Input.RequestBody, &req)
+	if err != nil {
+		br.Msg = "参数解析异常！"
+		br.ErrMsg = "参数解析失败，Err:" + err.Error()
+		return
+	}
+	if req.EdbCode == "" {
+		br.Msg = "请输入指标编码！"
+		br.ErrMsg = "请输入指标编码，指标编码为空"
+		return
+	}
+	if req.EdbInfoId <= 0 {
+		br.Msg = "请输入指标ID！"
+		br.ErrMsg = "请输入指标ID"
+		return
+	}
+	cacheKey = utils.CACHE_EDB_DATA_REFRESH + strconv.Itoa(source) + "_" + req.EdbCode
+	if utils.Rc.IsExist(cacheKey) {
+		br.Ret = 501
+		br.Success = true
+		br.Msg = "系统处理中，请稍后重试"
+		return
+	}
+	utils.Rc.SetNX(cacheKey, 1, 1*time.Minute)
+
+	// 获取指标详情
+	edbInfo, err := models.GetEdbInfoByEdbCode(source, req.EdbCode)
+	if err != nil {
+		br.Msg = "指标不存在！"
+		br.ErrMsg = "指标不存在"
+		return
+	}
+
+	dataItem, err := services.GetEdbDataFromFuBao(req.EdbCode, req.StartDate, utils.BASE_END_DATE)
+	if err != nil {
+		br.Msg = "获取指标信息失败！"
+		br.ErrMsg = "获取指标信息失败 GetEdbDataFromLt，Err:" + err.Error()
+		return
+	}
+	err = models.RefreshEdbDataFromFubao(req.EdbInfoId, req.EdbCode, req.StartDate, dataItem)
+	if err != nil && err.Error() != utils.ErrNoRow() {
+		br.Msg = "刷新指标信息失败！"
+		br.ErrMsg = "刷新指标信息失败 RefreshEdbDataFromFubao，Err:" + err.Error()
+		return
+	}
+	// 更新指标最大最小值
+	err, errMsg := models.UnifiedModifyEdbInfoMaxAndMinInfo(edbInfo)
+	if err != nil {
+		br.Msg = errMsg
+		br.ErrMsg = err.Error()
+		return
+	}
+
+	// 更新ES
+	go logic.UpdateEs(edbInfo.EdbInfoId)
+
+	br.Ret = 200
+	br.Success = true
+	br.Msg = "获取成功"
+}

docs/富宝数据API.md

@@ -0,0 +1,118 @@
+### 查询单个产品及数据(json)
+
+> get HTTP://{ip}:{port}/cooperation/wande/queryData.do
+
+#### 参数
+
+- `pid` 产品id
+
+#### 响应 
+
+```json5
+{
+	"code": 200,
+	"returnValue": {
+		"product": {
+			"alikeName": "147家钢厂废钢日耗",
+			"diffName": "华北",
+			"propValueStr": "区域:华北地区,报价类型:废钢日耗",
+			"pattern":"华北:147家钢厂废钢日耗"
+		},
+		"data": [{
+				"priceStr": "344460",
+				"uad": "涨2170",
+				"datadate": "2019-07-04 16:31:33"
+			},
+			{
+				"priceStr": "342290",
+				"uad": "跌4010",
+				"datadate": "2019-07-01 09:14:14"
+			}
+		]
+	}
+}
+```
+
+### 查询所有产品的数据(json)
+
+> get HTTP://{ip}:{port}/cooperation/wande/queryLastData.do
+
+#### 响应
+
+```json5
+{
+	"code": 200,
+	"returnValue": {
+		"data": [{
+				"priceStr": "2459.82",
+				"uad": "跌0.07%",
+				"datadate": "2019-07-08 16:14:08",
+				"pid": 13170
+			},
+			{
+				"priceStr": "2470.32",
+				"uad": "跌0.05%",
+				"datadate": "2019-07-08 16:14:08", 
+				"pid": 16865
+			}
+		]
+	}
+}
+
+```
+
+### 查询修正产品的数据(json)
+
+> get HTTP://{ip}:{port}/cooperation/wande/queryReviseData.do
+
+- 如果当天没有修正的价格,不返回returnValue
+- 如果当天有修正的价格,返回的returnValue中的key是产品的id,value[product产品类型，data价格数组]
+
+#### 响应
+
+```json5
+
+	{
+	"code": 200,
+	"returnValue": {
+		"23617": {
+			"product": {
+				"alikeName": "废钢日耗（按区）",
+				"diffName": "华北",
+				"propValueStr": "区域:华北地区,报价类型:废钢日耗"
+			},
+			"data": [{
+					"priceStr": "79000",
+					"uad": "跌500",
+					"datadate": "2019-07-11 15:59:23"
+				},
+				{
+					"priceStr": "79500",
+					"uad": "涨1800",
+					"datadate": "2019-07-04 00:00:00"
+				}
+			]
+		},
+		"23622": {
+			"product": {
+				"alikeName": "废钢日耗（按省）",
+				"diffName": "江苏",
+				"propValueStr": "区域:华东地区,省份:江苏,报价类型:废钢日耗"
+			},
+			"data": [{
+					"priceStr": "82300",
+					"uad": "稳",
+					"datadate": "2019-07-11 15:59:23"
+				},
+				{
+					"priceStr": "82300",
+					"uad": "跌2000",
+					"datadate": "2019-07-04 00:00:00"
+				}
+			]
+		}
+	}
+}
+
+
+``` 

models/base_from_fubao.go

@@ -0,0 +1,131 @@
+package models
+
+import (
+	"fmt"
+	"github.com/beego/beego/v2/client/orm"
+	"hongze/hongze_edb_lib/services"
+	"hongze/hongze_edb_lib/utils"
+	"strconv"
+	"strings"
+	"time"
+)
+
+// 新增富宝指标数据
+func AddEdbDataFromFubao(edbCode string, fubaoData *services.FuBaoData) (err error) {
+	o := orm.NewOrm()
+	var isAdd bool
+	addSql := ` INSERT INTO edb_data_fubao(edb_info_id,edb_code,data_time,value,create_time,modify_time,data_timestamp) values `
+	for _, dataItem := range fubaoData.ReturnValue.Data {
+		dataTime, err := time.ParseInLocation(utils.FormatDate, dataItem.Datadate, time.Local)
+		if err != nil {
+			return err
+		}
+		eDate := dataTime.Format(utils.FormatDate)
+		if err != nil {
+			return err
+		}
+		timestamp := dataTime.UnixNano() / 1e6
+		timeStr := fmt.Sprintf("%d", timestamp)
+		addSql += GetAddSql("0", edbCode, eDate, timeStr, dataItem.PriceStr)
+		isAdd = true
+	}
+	if isAdd {
+		addSql = strings.TrimRight(addSql, ",")
+		_, err = o.Raw(addSql).Exec()
+		if err != nil {
+			fmt.Println("RefreshAllEdbDataByFubao add Err", err.Error())
+			return
+		}
+	}
+	return
+}
+
+// 刷新路透指标数据
+func RefreshEdbDataFromFubao(edbInfoId int, edbCode, startDate string, fubaoData *services.FuBaoData) (err error) {
+	source := utils.DATA_SOURCE_FUBAO
+	o := orm.NewOrm()
+
+	// 真实数据的最大日期  ， 插入规则配置的日期
+	var realDataMaxDate, edbDataInsertConfigDate time.Time
+	var edbDataInsertConfig *EdbDataInsertConfig
+	var isFindConfigDateRealData bool //是否找到配置日期的实际数据的值
+	{
+		edbDataInsertConfig, err = GetEdbDataInsertConfigByEdbId(edbInfoId)
+		if err != nil && err.Error() != utils.ErrNoRow() {
+			return
+		}
+		if edbDataInsertConfig != nil {
+			edbDataInsertConfigDate = edbDataInsertConfig.Date
+		}
+	}
+
+	var existCondition string
+	var existPars []interface{}
+
+	existCondition += " AND edb_info_id=? "
+	existPars = append(existPars, edbInfoId)
+	if startDate != "" {
+		existCondition += " AND data_time>=? "
+		existPars = append(existPars, startDate)
+	}
+
+	existList, err := GetEdbDataByCondition(source, existCondition, existPars)
+	if err != nil {
+		return err
+	}
+	existMap := make(map[string]*EdbInfoSearchData)
+	for _, v := range existList {
+		existMap[v.DataTime] = v
+	}
+
+	edbInfoIdStr := strconv.Itoa(edbInfoId)
+
+	addSql := ` INSERT INTO edb_data_fubao(edb_info_id,edb_code,data_time,value,create_time,modify_time,data_timestamp) values `
+	var isAdd bool
+
+	for _, dataItem := range fubaoData.ReturnValue.Data {
+
+		dataTime, err := time.ParseInLocation(utils.FormatDate, dataItem.Datadate, time.Local)
+		if err != nil {
+			return err
+		}
+
+		if findItem, ok := existMap[dataItem.Datadate]; !ok {
+			timestamp := dataTime.UnixNano() / 1e6
+			timeStr := fmt.Sprintf("%d", timestamp)
+
+			addSql += GetAddSql(edbInfoIdStr, edbCode, dataItem.Datadate, timeStr, dataItem.PriceStr)
+			isAdd = true
+		} else {
+			if findItem != nil && utils.SubFloatToString(findItem.Value, 30) != dataItem.PriceStr {
+				err = ModifyEdbDataById(source, findItem.EdbDataId, dataItem.PriceStr)
+				if err != nil {
+					return err
+				}
+			}
+		}
+
+		// 下面代码主要目的是处理掉手动插入的数据判断
+		{
+			if realDataMaxDate.IsZero() || dataTime.After(realDataMaxDate) {
+				realDataMaxDate = dataTime
+			}
+			if edbDataInsertConfigDate.IsZero() || dataTime.Equal(edbDataInsertConfigDate) {
+				isFindConfigDateRealData = true
+			}
+		}
+	}
+
+	// 处理手工数据补充的配置
+	HandleConfigInsertEdbData(realDataMaxDate, edbDataInsertConfig, edbInfoId, source, existMap, isFindConfigDateRealData)
+
+	if isAdd {
+		addSql = strings.TrimRight(addSql, ",")
+		_, err = o.Raw(addSql).Exec()
+		if err != nil {
+			fmt.Println("RefreshAllEdbDataByLt add Err", err.Error())
+			return
+		}
+	}
+	return
+}

routers/router.go

@@ -160,6 +160,11 @@ func init() {
 				&controllers.NationalStatisticsController{},
 			),
 		),
+		beego.NSNamespace("/fubao",
+			beego.NSInclude(
+				&controllers.FuBaoController{},
+			),
+		),
 	)
 	beego.AddNamespace(ns)
 }

services/base_from_fubao.go

@@ -0,0 +1,45 @@
+package services
+
+import (
+	"encoding/json"
+	"fmt"
+	"github.com/rdlucklib/rdluck_tools/http"
+	"hongze/hongze_edb_lib/utils"
+)
+
+const (
+	FuBaoQueryDataUrl     = "http://data.f139.com/cooperation/wande/queryData.do?pid=%s"
+	FuBaoQueryLastDataUrl = "http://data.f139.com/cooperation/wande/queryLastData.do"
+)
+
+type FuBaoData struct {
+	ReturnValue struct {
+		Product struct {
+			AlikeName    string `json:"alikeName"`
+			DiffName     string `json:"diffName"`
+			PropValueStr string `json:"propValueStr"`
+			Pattern      string `json:"pattern"`
+		} `json:"product"`
+		Data []struct {
+			PriceStr string `json:"priceStr"`
+			Uad      string `json:"uad"`
+			Datadate string `json:"datadate"`
+		} `json:"data"`
+	} `json:"returnValue"`
+	Code int `json:"code"`
+}
+
+// GetEdbDataFromLt 获取路透数据
+func GetEdbDataFromFuBao(edbCode, startDate, endDate string) (item *FuBaoData, err error) {
+	sbUrl := fmt.Sprintf(FuBaoQueryDataUrl, edbCode)
+	utils.FileLog.Info("sbUrl:%s", sbUrl)
+	body, err := http.Get(sbUrl)
+	utils.FileLog.Info("fubao result:%s", string(body))
+
+	if err != nil {
+		return
+	}
+	item = new(FuBaoData)
+	err = json.Unmarshal(body, &item)
+	return
+}

utils/constants.go

@@ -90,6 +90,7 @@ const (
 	DATA_SOURCE_STOCK_PLANT                             //存量装置 ->58
 	DATA_SOURCE_CALCULATE_CORRELATION                   //相关性计算->59
 	DATA_SOURCE_NATIONAL_STATISTICS                     //国家统计局->60
+	DATA_SOURCE_FUBAO                                   //富宝->61
 )
 
 // 指标来源的中文展示