Merge branch 'feature/eta2.6.5_kpler' into debug

controllers/base_from_kpler.go

@@ -58,7 +58,7 @@ func (this *KplerController) GetData() {
 	}
 	
 	// 根据配置获取指标数据
-	indexes, _, _, e := services.GetKplerDataByApi(params, "")
+	indexes, _, _, e := services.GetKplerDataByApi(params, "", false)
 	if e != nil {
 		br.Msg = "获取失败"
 		br.ErrMsg = fmt.Sprintf("获取开普勒高频指标失败, %v", e)
@@ -119,7 +119,7 @@ func (this *KplerController) IndexAdd() {
 	}
 	
 	// 根据配置获取指标数据
-	indexes, apiQueryUrl, terminalInfo, e := services.GetKplerDataByApi(params, "")
+	indexes, apiQueryUrl, terminalInfo, e := services.GetKplerDataByApi(params, "", false)
 	if e != nil {
 		br.Msg = "获取失败"
 		br.ErrMsg = fmt.Sprintf("获取开普勒高频指标失败, %v", e)
@@ -307,4 +307,78 @@ func (this *KplerController) HandleExcelData() {
 	br.Ret = 200
 	br.Success = true
 	br.Msg = "处理成功"
+}
+
+// RefreshByApi
+// @Title 刷新Kpler指标接口
+// @Description 刷新Kpler指标接口
+// @Success 200 {object} models.RefreshEdbInfoReq
+// @router /index/refresh_by_api [post]
+func (this *KplerController) RefreshByApi() {
+	br := new(models.BaseResponse).Init()
+	defer func() {
+		this.Data["json"] = br
+		this.ServeJSON()
+	}()
+	var req models.KplerRefreshByApiReq
+	err := json.Unmarshal(this.Ctx.Input.RequestBody, &req)
+	if err != nil {
+		br.Msg = "参数解析异常！"
+		br.ErrMsg = "参数解析失败，Err:" + err.Error()
+		return
+	}
+	if req.StartDate == "" {
+		req.StartDate = time.Now().Format("2006-01-02")
+	}
+	// 查询所有通过API获取的指标
+	refreshIndexes, err := models.GetApiKplerIndexesByFrequencyEndDateTime(req.Frequency, req.StartDate)
+	if err != nil {
+		br.Msg = "获取指标失败"
+		br.ErrMsg = "获取指标失败,Err:" + err.Error()
+		return
+	}
+    
+	if len(refreshIndexes) == 0 {
+		br.Ret = 200
+		br.Success = true
+		br.Msg = "没有需要刷新的指标"
+		return
+	}
+
+	hasRefreshIndexMap := make(map[string]struct{})
+	for _, index := range refreshIndexes {
+		if _, ok := hasRefreshIndexMap[index.IndexCode]; ok {
+			continue
+		}
+		hasRefreshIndexMap[index.IndexCode] = struct{}{}
+
+		reqLib := models.KplerSearchEdbReq{
+			ProductNames: index.ProductNames,
+			FromZoneNames: index.FromZoneName,
+			ToZoneNames: index.ToZoneName,
+			Granularity: index.Granularity,
+			Split: index.Split,
+			Unit: index.Unit,
+			FlowDirection: index.FlowDirection,
+		}
+		// 获取指标数据
+		indexes, apiQueryUrl, _, err := services.GetKplerDataByApi(reqLib, index.TerminalCode, true)
+		if err != nil {
+			utils.FileLog.Error("RefreshByApi Err:" + err.Error())
+			continue
+		}
+        
+        for _, hasIndex := range indexes {
+			hasRefreshIndexMap[hasIndex.IndexCode] = struct{}{}
+		}
+		err = services.AddKplerIndexByApi(indexes, &reqLib, apiQueryUrl, index.ClassifyId, index.TerminalCode)
+		if err != nil {
+			utils.FileLog.Error("RefreshByApi Err:" + err.Error())
+			continue
+		}
+	}
+	br.Ret = 200
+	br.Success = true
+	br.Msg = "获取成功"
+
 }

models/base_from_kpler.go

@@ -521,6 +521,12 @@ func GetAllBaseFromKplerClassify() (items []*BaseFromKplerClassify, err error) {
 	return
 }
 
+func GetAllBaseFromKplerClassifyByClassifyType(classifyType string) (items []*BaseFromKplerClassify, err error) {
+	sql := ` SELECT * FROM base_from_kpler_classify WHERE classify_type=? and modify_time < "2025-06-18 15:37:00" limit 100`
+	err = global.DEFAULT_DB.Raw(sql, classifyType).Find(&items).Error
+	return
+}
+
 // 获取分类
 func (y *BaseFromKplerClassify) GetByProductIds(productIds []int) (items []*BaseFromKplerClassify, err error) {
 	sql := ` SELECT * FROM base_from_kpler_classify WHERE product_id in (?) `
@@ -528,6 +534,13 @@ func (y *BaseFromKplerClassify) GetByProductIds(productIds []int) (items []*Base
 	return
 }
 
+// 获取单个分类
+func (y *BaseFromKplerClassify) GetByProductId(productId int) (item *BaseFromKplerClassify, err error) {
+	sql := ` SELECT * FROM base_from_kpler_classify WHERE product_id=? `
+	err = global.DEFAULT_DB.Raw(sql, productId).First(&item).Error
+	return
+}
+
 // 获取分类
 func (y *BaseFromKplerClassify) GetByProductName(productName string) (item *BaseFromKplerClassify, err error) {
 	sql := ` SELECT * FROM base_from_kpler_classify WHERE product_name=? `
@@ -615,4 +628,28 @@ type KplerZone struct {
 type KplerZoneDataLibReq struct {
 	AncestorName string `description:"祖先名称"`
 	DescendantName string `description:"子名称"`
+}
+
+type KplerRefreshByApiReq struct {
+	Frequency string `description:"频度"`
+	StartDate string `description:"开始日期"`
+}
+
+func GetApiKplerIndexesByFrequencyEndDateTime(frequency string, endDateTime string) (items []*BaseFromKplerIndex, err error) {
+	sql := ` SELECT * FROM base_from_kpler_index WHERE frequency=? AND end_date < ? and api_query_url != '' order by base_from_kpler_index_id asc, api_query_url asc `
+	err = global.DEFAULT_DB.Raw(sql, frequency, endDateTime).Find(&items).Error
+	return
+}
+
+type KplerProductLibReq struct {
+	AncestorFamilyIds string `description:"祖先家族ID"`
+	AncestorFamilyNames string `description:"祖先家族名称"`
+	AncestorGroupIds string `description:"祖先组ID"`
+	AncestorGroupNames string `description:"祖先组名称"`
+	AncestorProductIds string `description:"祖先产品ID"`
+	AncestorProductNames string `description:"祖先产品名称"`
+	AncestorGradeIds string `description:"祖先等级ID"`
+	AncestorGradeNames string `description:"祖先等级名称"`
+	Products string `description:"产品"`
+	ProductIds string `description:"产品ID"`
 }

routers/commentsRouter.go

@@ -1249,6 +1249,15 @@ func init() {
             Filters: nil,
             Params: nil})
 
+    beego.GlobalControllerRouter["eta/eta_index_lib/controllers:KplerController"] = append(beego.GlobalControllerRouter["eta/eta_index_lib/controllers:KplerController"],
+        beego.ControllerComments{
+            Method: "RefreshByApi",
+            Router: `/index/refresh_by_api`,
+            AllowHTTPMethods: []string{"post"},
+            MethodParams: param.Make(),
+            Filters: nil,
+            Params: nil})
+
     beego.GlobalControllerRouter["eta/eta_index_lib/controllers:KplerController"] = append(beego.GlobalControllerRouter["eta/eta_index_lib/controllers:KplerController"],
         beego.ControllerComments{
             Method: "IndexAdd",

services/base_from_kpler.go

@@ -15,7 +15,7 @@ import (
 )
 
 // GetKplerDataByApi 获取开普勒数据
-func GetKplerDataByApi(params models.KplerSearchEdbReq, terminalCode string) (indexes []*models.KplerIndexItem, apiQueryUrl string, terminalInfo *models.EdbTerminal, err error) {
+func GetKplerDataByApi(params models.KplerSearchEdbReq, terminalCode string, isRefresh bool) (indexes []*models.KplerIndexItem, apiQueryUrl string, terminalInfo *models.EdbTerminal, err error) {
 	terminal, e := GetApiTerminal(utils.DATA_SOURCE_KPLER, terminalCode)
 	if e != nil {
 		err = fmt.Errorf("获取开普勒终端配置失败, %v", e)
@@ -29,7 +29,7 @@ func GetKplerDataByApi(params models.KplerSearchEdbReq, terminalCode string) (in
     
 	// 走API
 	if terminal.IsApi == 1 {
-		indexes, apiQueryUrl, err = getKplerDataByApi(params, terminal.ServerUrl)
+		indexes, apiQueryUrl, err = getKplerDataByApi(params, terminal.ServerUrl, isRefresh)
 		if err != nil {
 			err = fmt.Errorf("获取开普勒指标数据失败, %v", err)
 			return
@@ -41,7 +41,7 @@ func GetKplerDataByApi(params models.KplerSearchEdbReq, terminalCode string) (in
 }
 
 // getEdbDataFromThsHfHttp API-获取高频指标数据
-func getKplerDataByApi(params models.KplerSearchEdbReq, serverUrl string) (list []*models.KplerIndexItem, apiQueryUrl string, err error) {
+func getKplerDataByApi(params models.KplerSearchEdbReq, serverUrl string, isRefresh bool) (list []*models.KplerIndexItem, apiQueryUrl string, err error) {
 	defer func() {
 		if err != nil {
 			tips := fmt.Sprintf("开普勒指标API-getKplerDataByApi err: %v", err)
@@ -66,7 +66,7 @@ func getKplerDataByApi(params models.KplerSearchEdbReq, serverUrl string) (list
 	libreq.WithIntraRegion = "true"
 	libreq.WithForecast = "true"
 	libreq.OnlyRealized = "false"
-	libreq.StartDate = "2025-01-01"
+	libreq.StartDate = "2013-01-01"
 	libreq.EndDate = time.Now().Format(utils.FormatDate)
 	// 请求接口
 	apiResp, err := getKplerFlowDataLib(serverUrl, libreq)
@@ -121,10 +121,10 @@ func getKplerDataByApi(params models.KplerSearchEdbReq, serverUrl string) (list
 	for _, v := range existList {
 		existIndexCodeMap[v.IndexCode] = true
 	}
-	// 过滤已经存在的指标
+	// 过滤已经存在的指标, 如果是刷新指标，则无需过滤
 	list = make([]*models.KplerIndexItem, 0)
 	for _, v := range indexes {
-		if _, ok := existIndexCodeMap[v.IndexCode]; !ok {
+		if _, ok := existIndexCodeMap[v.IndexCode]; !ok || isRefresh {
 			list = append(list, v)
 		}
 	}
@@ -182,6 +182,7 @@ func AddKplerIndexByApi(indexList []*models.KplerIndexItem, req *models.KplerSea
 			indexObj.IndexName = indexInfo.IndexName
 			indexObj.Frequency = indexInfo.Frequency
 			indexObj.ClassifyId = int(classifyId)
+			indexObj.ProductNames = req.ProductNames
 			indexObj.FromZoneId = fromZoneId
 			indexObj.ToZoneId = toZoneId
 			indexObj.FromZoneName = req.FromZoneNames
@@ -221,7 +222,7 @@ func AddKplerIndexByApi(indexList []*models.KplerIndexItem, req *models.KplerSea
 			indexObj.FromZoneName = req.FromZoneNames
 			indexObj.ToZoneName = req.ToZoneNames
 			indexObj.FlowDirection = req.FlowDirection
-			indexObj.Granularity = req.Granularity
+			indexObj.Granularity = GetKplerGranularity(req.Granularity)
 			indexObj.Split = req.Split
 			indexObj.Unit = req.Unit
 			indexObj.ApiQueryUrl = apiQueryUrl
@@ -415,9 +416,13 @@ func getKplerFlowDataLib(libUrl string, dataMap *models.KplerFlowDataLibReq) (re
 	return resp, nil
 }
 
-func getKplerProductLib(libUrl string) (resp *models.KplerProductLibResp, err error) {
+func getKplerProductLib(libUrl string, req *models.KplerProductLibReq) (resp *models.KplerProductLibResp, err error) {
 	postUrl := fmt.Sprintf("%s%s", libUrl, "/v1/kpler/getProductData")
-	result, err := HttpPost(postUrl, "", "application/json")
+	postData, err := json.Marshal(req)
+	if err != nil {
+		return
+	}
+	result, err := HttpPost(postUrl, string(postData), "application/json")
 	if err != nil {
 		return
 	}
@@ -781,15 +786,15 @@ func getKplerFrequency(granularity string) (frequency string) {
 
 func GetKplerGranularity(frequency string) (granularity string) {
 	switch frequency {
-	case "daily", "Daily":
+	case "daily", "Daily", "days":
 		return "days"
-	case "weekly", "Weekly":
+	case "weekly", "Weekly", "weeks":
 		return "weeks"
-	case "monthly", "Monthly":
+	case "monthly", "Monthly", "months":
 		return "months"
-	case "yearly", "Yearly":
+	case "yearly", "Yearly", "years":
 		return "years"
-	case "quarterly", "Quarters":
+	case "quarterly", "Quarters", "quarters":
 		return "quarters"
 	}
 	return ""
@@ -797,7 +802,18 @@ func GetKplerGranularity(frequency string) (granularity string) {
 
 func InitKplerProduct ()(err error) {
 	libUrl := "http://127.0.0.1:8915"
-	libResp, err := getKplerProductLib(libUrl)
+	libResp, err := getKplerProductLib(libUrl, &models.KplerProductLibReq{
+		AncestorFamilyIds: "",
+		AncestorFamilyNames: "",
+		AncestorGroupIds: "",
+		AncestorGroupNames: "",
+		AncestorProductIds: "",
+		AncestorProductNames: "",
+		AncestorGradeIds: "",
+		AncestorGradeNames: "",
+		Products: "",
+		ProductIds: "",
+	})
 	if err != nil {
 		return
 	}
@@ -871,6 +887,77 @@ func InitKplerProduct ()(err error) {
 	return
 }
 
+func InitKplerProductGrade ()(err error) {
+	libUrl := "http://127.0.0.1:8915"
+	
+	classifyList, err := models.GetAllBaseFromKplerClassifyByClassifyType("subgrade4")
+	if err != nil {
+		return
+	}
+	for _, c := range classifyList {
+		c.ModifyTime = time.Now()
+	err = c.Update([]string{"ModifyTime"})
+	if err != nil {
+		fmt.Println("更新开普勒产品库失败", err)
+		utils.FileLog.Info("更新开普勒产品库失败", err)
+	}
+		// 每个分类都发起分组请求
+	libResp, er := getKplerProductLib(libUrl, &models.KplerProductLibReq{
+		// AncestorFamilyIds: familyIds,
+		// AncestorFamilyNames: familyNames,
+		// AncestorGroupIds: groupIds,
+		// AncestorGroupNames: groupNames,
+		// AncestorProductIds: productIds,
+		// AncestorProductNames: productNames,
+		AncestorGradeIds: strconv.Itoa(c.ProductId),
+		AncestorGradeNames: c.ProductName,
+	})
+	if er != nil {
+		fmt.Println("获取开普勒产品库失败", er)
+		utils.FileLog.Info("获取开普勒产品库失败", er)
+		continue
+	}
+	if libResp.Ret != 200 {
+		fmt.Println("获取开普勒产品库失败", libResp.ErrMsg)
+		utils.FileLog.Info("获取开普勒产品库失败", libResp.ErrMsg)
+		continue
+	}
+	classifyObj := new(models.BaseFromKplerClassify)
+	for _, v := range libResp.Data {
+		id, _ := strconv.Atoi(v.Id)
+		if id == 0 {
+			continue
+		}
+		if id == c.ProductId {
+			continue
+		}
+		// 更新对应的分类等级和父级
+		// 查找所有子分类
+
+		classifyItem, er := classifyObj.GetByProductId(id)
+		if er != nil {
+			err = fmt.Errorf("子分类不存在 Err：%s", er)
+			continue
+		}
+		
+		classifyItem.Level = c.Level + 1
+		classifyItem.ParentId = c.ClassifyId
+		classifyItem.ClassifyType = "subgrade5"
+		classifyItem.ModifyTime = time.Now()
+		err = classifyItem.Update([]string{"Level", "ParentId", "ClassifyType", "ModifyTime"})
+		if err != nil {
+			fmt.Println("更新开普勒产品库失败", err)
+			utils.FileLog.Info("更新开普勒产品库失败", err)
+			}
+		}
+		
+	}
+	
+	fmt.Println("classifyList:", classifyList)
+	
+	return
+}
+
 
 func InitKplerZone() (err error) {
 	libUrl := "http://127.0.0.1:8915"