Merge branch 'feature/jiayue_index' into debug

.gitignore

@@ -8,4 +8,7 @@ binlog/
 /eta_index_lib.exe
 /eta_index_lib.exe~
 eta_index_lib
-/static/imgs/python/*
+/static/imgs/python/*
+/etalogs
+*.gz
+*.exe

controllers/base_from_jiayue.go

@@ -216,3 +216,50 @@ func (this *JiaYueController) Refresh() {
 	br.Success = true
 	br.Msg = "操作成功"
 }
+
+// SyncNewIndex
+// @Title 嘉悦物产-同步增量指标
+// @Description  同步增量指标
+// @Success 200 {object} models.AddEdbInfoReq
+// @router /sync_new_index [post]
+func (this *JiaYueController) SyncNewIndex() {
+	br := new(models.BaseResponse).Init()
+	defer func() {
+		if br.ErrMsg == "" {
+			br.IsSendEmail = false
+		}
+		this.Data["json"] = br
+		this.ServeJSON()
+	}()
+
+	// 从桥接服务获取增量指标
+	newIndexes, e := services.GetJiaYueNewIndexFromBridge()
+	if e != nil {
+		br.Msg = "同步增量指标失败"
+		br.ErrMsg = "同步增量指标失败, Err: " + e.Error()
+		return
+	}
+	if len(newIndexes) == 0 {
+		utils.FileLog.Info("无增量指标同步")
+		return
+	}
+
+	// 获取指标目录
+	indexMenus, e := services.GetJiaYueMenuListFromBridge()
+	if e != nil {
+		br.Msg = "获取指标目录失败"
+		br.ErrMsg = "获取指标目录失败, Err: " + e.Error()
+		return
+	}
+
+	go func() {
+		for _, v := range newIndexes {
+			// 错误信息在FileLog中这边不做返回
+			_ = services.SyncJiaYueNewIndex(v, indexMenus)
+		}
+	}()
+
+	br.Ret = 200
+	br.Success = true
+	br.Msg = "操作成功"
+}

models/base_from_jiayue.go

@@ -26,15 +26,17 @@ type BridgeJiaYueResultIndexData struct {
 
 // BridgeJiaYueIndexAndData 桥接服务-嘉悦指标和数据
 type BridgeJiaYueIndexAndData struct {
-	Id             int                     `description:"指标自增ID" json:"id"`
-	IndexCode      string                  `description:"指标编码" json:"index_code"`
-	IndexName      string                  `description:"指标名称" json:"index_name"`
-	Unit           string                  `description:"单位" json:"unit"`
-	Frequency      string                  `description:"频度" json:"frequency"`
-	LastDate       time.Time               `description:"指标最新时间" json:"last_date"`
-	LastUpdateTime time.Time               `description:"最新更新时间" json:"last_update_time"`
-	Status         int                     `description:"指标状态" json:"status"`
-	IndexData      []BridgeJiaYueIndexData `description:"指标数据" json:"index_data"`
+	Id             int                       `description:"指标自增ID" json:"id"`
+	IndexCode      string                    `description:"指标编码" json:"index_code"`
+	IndexName      string                    `description:"指标名称" json:"index_name"`
+	SourceType     string                    `description:"指标来源" json:"source_type"`
+	Unit           string                    `description:"单位" json:"unit"`
+	Frequency      string                    `description:"频度" json:"frequency"`
+	LastDate       time.Time                 `description:"指标最新时间" json:"last_date"`
+	LastUpdateTime time.Time                 `description:"最新更新时间" json:"last_update_time"`
+	Status         int                       `description:"指标状态" json:"status"`
+	IndexData      []BridgeJiaYueIndexData   `description:"指标数据" json:"index_data"`
+	MenuData       BridgeJiaYueIndexMenuData `description:"指标目录信息" json:"menu_data"`
 }
 
 // BridgeJiaYueIndexData 桥接服务-嘉悦指标数据
@@ -44,6 +46,39 @@ type BridgeJiaYueIndexData struct {
 	UpdateTime time.Time `json:"update_time"`
 }
 
+// BridgeJiaYueIndexMenuData 桥接服务-嘉悦指标目录信息
+type BridgeJiaYueIndexMenuData struct {
+	Id         int    `description:"目录ID" json:"id"`
+	Type       string `description:"目录类型" json:"type"`
+	Code       string `description:"目录编码" json:"code"`
+	Name       string `description:"目录名称" json:"name"`
+	Icon       string `description:"目录图标" json:"icon"`
+	Sort       int    `description:"排序" json:"sort"`
+	ParentId   int    `description:"父级目录ID" json:"parent_id"`
+	ParentName string `description:"父级目录名称" json:"parent_name"`
+	Path       string `description:"目录全路径" json:"path"`
+}
+
+// BridgeJiaYueIndexMenuWithLevel 桥接服务-嘉悦指标目录带层级
+type BridgeJiaYueIndexMenuWithLevel struct {
+	Level int `description:"层级"`
+	Menu  BridgeJiaYueIndexMenuData
+}
+
+// BridgeJiaYueResultNewIndexData 桥接服务-获取嘉悦增量指标数据响应体
+type BridgeJiaYueResultNewIndexData struct {
+	Code int                        `json:"code" description:"状态码"`
+	Msg  string                     `json:"msg" description:"提示信息"`
+	Data []BridgeJiaYueIndexAndData `json:"data" description:"返回数据"`
+}
+
+// BridgeJiaYueResultMenuListData 桥接服务-获取嘉悦指标目录数据响应体
+type BridgeJiaYueResultMenuListData struct {
+	Code int                         `json:"code" description:"状态码"`
+	Msg  string                      `json:"msg" description:"提示信息"`
+	Data []BridgeJiaYueIndexMenuData `json:"data" description:"返回数据"`
+}
+
 // AddEdbDataFromJiaYue 新增嘉悦指标数据
 func AddEdbDataFromJiaYue(tableName, edbCode string, dataList []BridgeJiaYueIndexData) (err error) {
 	if tableName == "" {

models/edb_data_table.go

@@ -2,15 +2,20 @@ package models
 
 import (
 	"eta/eta_index_lib/utils"
+	"strings"
 )
 
 var (
-	EdbDataTableNameMap map[int]string // 指标来源对应数据表名
+	EdbDataTableNameMap  map[int]string // 指标来源对应数据表名
+	EdbSourceExtendIdMap map[string]int // 指标来源字符串对应来源ID
+	EdbSourceNameMap     map[int]string // 指标来源名称
 )
 
 // InitEdbSource 初始化时加载指标来源对应表名, 避免循环中查库, 注意edb_source表修改table_name的话需要重启服务
 func InitEdbSource() {
 	EdbDataTableNameMap = make(map[int]string)
+	EdbSourceExtendIdMap = make(map[string]int)
+	EdbSourceNameMap = make(map[int]string)
 	sources, e := GetEdbSourceItemsByCondition(``, make([]interface{}, 0), []string{}, "")
 	if e != nil {
 		utils.FileLog.Info("init source table err: %s", e.Error())
@@ -18,6 +23,16 @@ func InitEdbSource() {
 	}
 	for _, v := range sources {
 		EdbDataTableNameMap[v.EdbSourceId] = v.TableName
+		EdbSourceNameMap[v.EdbSourceId] = v.SourceName
+		if v.SourceExtend != "" {
+			arr := strings.Split(v.SourceExtend, ",")
+			if len(arr) == 0 {
+				continue
+			}
+			for _, s := range arr {
+				EdbSourceExtendIdMap[s] = v.EdbSourceId
+			}
+		}
 	}
 }
 

models/edb_info.go

@@ -1071,8 +1071,11 @@ func EdbInfoAdd(req *AddEdbInfoParams, serverUrl string, sysUserId int, sysUserR
 
 	sourceName, ok := sourceNameMap[source]
 	if !ok {
-		err = errors.New("指标来源异常")
-		return
+		sourceName = EdbSourceNameMap[source]
+		if sourceName == "" {
+			err = errors.New("指标来源异常")
+			return
+		}
 	}
 	edbInfo.SourceName = sourceName
 

routers/commentsRouter.go

@@ -394,6 +394,15 @@ func init() {
             Filters: nil,
             Params: nil})
 
+    beego.GlobalControllerRouter["eta/eta_index_lib/controllers:JiaYueController"] = append(beego.GlobalControllerRouter["eta/eta_index_lib/controllers:JiaYueController"],
+        beego.ControllerComments{
+            Method: "SyncNewIndex",
+            Router: `/sync_new_index`,
+            AllowHTTPMethods: []string{"post"},
+            MethodParams: param.Make(),
+            Filters: nil,
+            Params: nil})
+
     beego.GlobalControllerRouter["eta/eta_index_lib/controllers:LtController"] = append(beego.GlobalControllerRouter["eta/eta_index_lib/controllers:LtController"],
         beego.ControllerComments{
             Method: "Add",

services/base_from_jiayue.go

@@ -2,17 +2,22 @@ package services
 
 import (
 	"encoding/json"
+	"eta/eta_index_lib/logic"
 	"eta/eta_index_lib/models"
 	"eta/eta_index_lib/services/alarm_msg"
 	"eta/eta_index_lib/utils"
 	"fmt"
 	"io/ioutil"
 	"net/http"
+	"sort"
+	"strconv"
 	"strings"
 )
 
 var (
 	BridgeApiJiaYueIndexDataUrl = "/api/index_data/jiayue/index_data" // 获取指标数据API
+	BridgeApiJiaYueNewIndexUrl  = "/api/index_data/jiayue/new_index"  // 获取增量指标API
+	BridgeApiJiaYueMenuListUrl  = "/api/index_data/jiayue/menu_list"  // 获取指标目录API
 )
 
 // GetJiaYueIndexDataFromBridge 从桥接服务获取指标数据
@@ -20,7 +25,8 @@ func GetJiaYueIndexDataFromBridge(param models.BridgeJiaYueIndexDataParams) (ind
 	defer func() {
 		if err != nil {
 			b, _ := json.Marshal(param)
-			tips := fmt.Sprintf("桥接服务-获取嘉悦指标数据, err: %s, params: %s", err.Error(), string(b))
+			tips := fmt.Sprintf("桥接服务-获取嘉悦指标数据失败, err: %s, params: %s", err.Error(), string(b))
+			utils.FileLog.Info(tips)
 			go alarm_msg.SendAlarmMsg(tips, 3)
 		}
 	}()
@@ -79,3 +85,263 @@ func GetJiaYueIndexDataFromBridge(param models.BridgeJiaYueIndexDataParams) (ind
 	indexData = result.Data
 	return
 }
+
+// GetJiaYueNewIndexFromBridge 从桥接服务获取增量指标
+func GetJiaYueNewIndexFromBridge() (indexData []models.BridgeJiaYueIndexAndData, err error) {
+	defer func() {
+		if err != nil {
+			tips := fmt.Sprintf("桥接服务-获取嘉悦增量指标失败, err: %s", err.Error())
+			utils.FileLog.Info(tips)
+			go alarm_msg.SendAlarmMsg(tips, 3)
+		}
+	}()
+
+	url := fmt.Sprint(utils.EtaBridgeUrl, BridgeApiJiaYueNewIndexUrl)
+	body := ioutil.NopCloser(strings.NewReader(""))
+	client := &http.Client{}
+	req, e := http.NewRequest("POST", url, body)
+	if e != nil {
+		err = fmt.Errorf("http create request err: %s", e.Error())
+		return
+	}
+
+	checkToken := utils.MD5(utils.EtaBridgeAppNameEn + utils.EtaBridgeMd5Key)
+	contentType := "application/json;charset=utf-8"
+	req.Header.Set("Content-Type", contentType)
+	req.Header.Set("Authorization", checkToken)
+	resp, e := client.Do(req)
+	if e != nil {
+		err = fmt.Errorf("http client do err: %s", e.Error())
+		return
+	}
+	defer func() {
+		_ = resp.Body.Close()
+	}()
+	b, e := ioutil.ReadAll(resp.Body)
+	if e != nil {
+		err = fmt.Errorf("resp body read err: %s", e.Error())
+		return
+	}
+	if len(b) == 0 {
+		err = fmt.Errorf("resp body is empty")
+		return
+	}
+	// 生产环境解密
+	if utils.RunMode == "release" {
+		str := string(b)
+		str = strings.Trim(str, `"`)
+		b = utils.DesBase64Decrypt([]byte(str), utils.EtaBridgeDesKey)
+	}
+
+	result := new(models.BridgeJiaYueResultNewIndexData)
+	if e = json.Unmarshal(b, &result); e != nil {
+		err = fmt.Errorf("result unmarshal err: %s\nresult: %s", e.Error(), string(b))
+		return
+	}
+	if result.Code != 200 {
+		err = fmt.Errorf("result: %s", string(b))
+		return
+	}
+	indexData = result.Data
+	return
+}
+
+// GetJiaYueMenuListFromBridge 从桥接服务获取指标目录列表
+func GetJiaYueMenuListFromBridge() (indexData []models.BridgeJiaYueIndexMenuData, err error) {
+	defer func() {
+		if err != nil {
+			tips := fmt.Sprintf("桥接服务-获取嘉悦增量指标失败, err: %s", err.Error())
+			utils.FileLog.Info(tips)
+			go alarm_msg.SendAlarmMsg(tips, 3)
+		}
+	}()
+
+	url := fmt.Sprint(utils.EtaBridgeUrl, BridgeApiJiaYueMenuListUrl)
+	body := ioutil.NopCloser(strings.NewReader(""))
+	client := &http.Client{}
+	req, e := http.NewRequest("POST", url, body)
+	if e != nil {
+		err = fmt.Errorf("http create request err: %s", e.Error())
+		return
+	}
+
+	checkToken := utils.MD5(utils.EtaBridgeAppNameEn + utils.EtaBridgeMd5Key)
+	contentType := "application/json;charset=utf-8"
+	req.Header.Set("Content-Type", contentType)
+	req.Header.Set("Authorization", checkToken)
+	resp, e := client.Do(req)
+	if e != nil {
+		err = fmt.Errorf("http client do err: %s", e.Error())
+		return
+	}
+	defer func() {
+		_ = resp.Body.Close()
+	}()
+	b, e := ioutil.ReadAll(resp.Body)
+	if e != nil {
+		err = fmt.Errorf("resp body read err: %s", e.Error())
+		return
+	}
+	if len(b) == 0 {
+		err = fmt.Errorf("resp body is empty")
+		return
+	}
+	// 生产环境解密
+	if utils.RunMode == "release" {
+		str := string(b)
+		str = strings.Trim(str, `"`)
+		b = utils.DesBase64Decrypt([]byte(str), utils.EtaBridgeDesKey)
+	}
+
+	result := new(models.BridgeJiaYueResultMenuListData)
+	if e = json.Unmarshal(b, &result); e != nil {
+		err = fmt.Errorf("result unmarshal err: %s\nresult: %s", e.Error(), string(b))
+		return
+	}
+	if result.Code != 200 {
+		err = fmt.Errorf("result: %s", string(b))
+		return
+	}
+	indexData = result.Data
+	return
+}
+
+// SyncJiaYueNewIndex 同步嘉悦增量指标
+func SyncJiaYueNewIndex(item models.BridgeJiaYueIndexAndData, menus []models.BridgeJiaYueIndexMenuData) (err error) {
+	defer func() {
+		if err != nil {
+			b, _ := json.Marshal(item)
+			tips := fmt.Sprintf("SyncJiaYueNewIndex同步增量指标失败, Err: %s\nIndex: %s", err.Error(), string(b))
+			utils.FileLog.Info(tips)
+			go alarm_msg.SendAlarmMsg(tips, 3)
+		}
+	}()
+
+	// 校验指标来源
+	sourceId := models.EdbSourceExtendIdMap[item.SourceType]
+	if sourceId <= 0 {
+		utils.FileLog.Info(fmt.Sprintf("指标来源ID有误, 忽略同步, sourceId: %d; sourceType: %s", sourceId, item.SourceType))
+		return
+	}
+	sourceTableName := models.EdbDataTableNameMap[sourceId]
+	if sourceTableName == "" {
+		utils.FileLog.Info(fmt.Sprintf("指标数据表有误, 忽略同步, sourceId: %d", sourceId))
+		return
+	}
+	// 指标编码默认使用自增ID
+	indexCode := strconv.Itoa(item.Id)
+
+	// 从桥接服务获取指标和数据
+	var params models.BridgeJiaYueIndexDataParams
+	params.IndexCode = indexCode
+	params.SourceExtend = item.SourceType
+	params.StartDate = utils.BASE_START_DATE
+	params.EndDate = utils.BASE_END_DATE
+	indexData, e := GetJiaYueIndexDataFromBridge(params)
+	if e != nil {
+		err = fmt.Errorf("GetJiaYueIndexDataFromBridge err: %s", e.Error())
+		return
+	}
+
+	// 没找到数据, 频度有误的忽略掉
+	if indexData.Id <= 0 {
+		utils.FileLog.Info(fmt.Sprintf("SyncJiaYueNewIndex indexCode empty, IndexCode: %s", indexCode))
+		return
+	}
+	frequency := TransJiaYueFrequency(item.Frequency)
+	if frequency == "" {
+		utils.FileLog.Info(fmt.Sprintf("SyncJiaYueNewIndex frequency empty, Frequency: %s", item.Frequency))
+		return
+	}
+
+	// 获取指标目录对应的所有父级目录
+	parentId := 0
+	parentMenus := GetJiaYueParentMenusByMenu(indexData.MenuData, menus, 0)
+	level := len(parentMenus)
+	if level > 0 {
+		// 将父级目录按照Level倒序排序(因为是从子目录往上找的, 所以Level最大的对应为顶级目录), 再进行遍历
+		sort.Slice(parentMenus, func(i, j int) bool {
+			return parentMenus[i].Level > parentMenus[j].Level
+		})
+		for _, p := range parentMenus {
+			l := level - p.Level - 1
+			c, _, msg := models.SaveEdbClassify(p.Menu.Name, parentId, l, 0, utils.InitAdminId, utils.InitAdminName)
+			if msg != "" {
+				err = fmt.Errorf("SaveEdbClassify err: %s", msg)
+				return
+			}
+			parentId = c.ClassifyId
+		}
+	}
+	classify, _, msg := models.SaveEdbClassify(indexData.MenuData.Name, parentId, level, 0, utils.InitAdminId, utils.InitAdminName)
+	if msg != "" {
+		err = fmt.Errorf("SaveEdbClassify err: %s", msg)
+		return
+	}
+
+	// 新增指标
+	addParams := new(models.AddEdbInfoParams)
+	addParams.EdbCode = indexCode
+	addParams.EdbName = item.IndexName
+	addParams.Frequency = frequency
+	addParams.Unit = item.Unit
+	addParams.ClassifyId = classify.ClassifyId
+	addParams.Source = sourceId
+	edbInfo, e := models.EdbInfoAdd(addParams, "", 0, "")
+	if e != nil {
+		err = fmt.Errorf("EdbInfoAdd err: %s", e.Error())
+		return
+	}
+
+	// 刷新指标数据
+	e = models.RefreshEdbDataFromJiaYue(sourceId, edbInfo.EdbInfoId, sourceTableName, edbInfo.EdbCode, params.StartDate, indexData.IndexData)
+	if e != nil {
+		err = fmt.Errorf("RefreshEdbDataFromJiaYue err: %s", e.Error())
+		return
+	}
+
+	// 更新指标最大最小值
+	e, errMsg := models.UnifiedModifyEdbInfoMaxAndMinInfo(edbInfo)
+	if e != nil {
+		err = fmt.Errorf("UnifiedModifyEdbInfoMaxAndMinInfo err: %s, errMsg: %s", e.Error(), errMsg)
+		return
+	}
+
+	// 更新ES
+	go logic.UpdateEs(edbInfo.EdbInfoId)
+
+	return
+}
+
+// TransJiaYueFrequency 频度转换
+func TransJiaYueFrequency(origin string) string {
+	mapping := map[string]string{
+		"日":  "日度",
+		"周":  "周度",
+		"旬":  "旬度",
+		"半月": "旬度",
+		"月":  "月度",
+		"季":  "季度",
+		"半年": "半年度",
+		"年":  "年度",
+	}
+	return mapping[origin]
+}
+
+// GetJiaYueParentMenusByMenu 获取指定目录的父级目录
+func GetJiaYueParentMenusByMenu(menu models.BridgeJiaYueIndexMenuData, menus []models.BridgeJiaYueIndexMenuData, level int) (results []models.BridgeJiaYueIndexMenuWithLevel) {
+	results = make([]models.BridgeJiaYueIndexMenuWithLevel, 0)
+	for _, m := range menus {
+		if menu.ParentId == m.Id {
+			results = append(results, models.BridgeJiaYueIndexMenuWithLevel{
+				Level: level,
+				Menu:  m,
+			})
+			ps := GetJiaYueParentMenusByMenu(m, menus, level+1)
+			if len(ps) > 0 {
+				results = append(results, ps...)
+			}
+		}
+	}
+	return
+}