kpler数据解析

routers/commentsRouter.go

@@ -7,6 +7,33 @@ import (
 
 func init() {
 
+    beego.GlobalControllerRouter["eta/eta_data_analysis/controllers:KplerController"] = append(beego.GlobalControllerRouter["eta/eta_data_analysis/controllers:KplerController"],
+        beego.ControllerComments{
+            Method: "GetFlowData",
+            Router: `/getFlowData`,
+            AllowHTTPMethods: []string{"post"},
+            MethodParams: param.Make(),
+            Filters: nil,
+            Params: nil})
+
+    beego.GlobalControllerRouter["eta/eta_data_analysis/controllers:KplerController"] = append(beego.GlobalControllerRouter["eta/eta_data_analysis/controllers:KplerController"],
+        beego.ControllerComments{
+            Method: "GetProductData",
+            Router: `/getProductData`,
+            AllowHTTPMethods: []string{"post"},
+            MethodParams: param.Make(),
+            Filters: nil,
+            Params: nil})
+
+    beego.GlobalControllerRouter["eta/eta_data_analysis/controllers:KplerController"] = append(beego.GlobalControllerRouter["eta/eta_data_analysis/controllers:KplerController"],
+        beego.ControllerComments{
+            Method: "GetZoneData",
+            Router: `/getZoneData`,
+            AllowHTTPMethods: []string{"post"},
+            MethodParams: param.Make(),
+            Filters: nil,
+            Params: nil})
+
     beego.GlobalControllerRouter["eta/eta_data_analysis/controllers:RzdController"] = append(beego.GlobalControllerRouter["eta/eta_data_analysis/controllers:RzdController"],
         beego.ControllerComments{
             Method: "DealData",

routers/router.go

@@ -24,6 +24,11 @@ func init() {
 				&controllers.RzdController{},
 			),
 		),
+		web.NSNamespace("/kpler",
+			web.NSInclude(
+				&controllers.KplerController{},
+			),
+		),
 	)
 	web.AddNamespace(ns)
 }

services/base_from_kpler.go

@@ -0,0 +1,155 @@
+package services
+
+import (
+	"encoding/json"
+	"eta/eta_data_analysis/models"
+	"eta/eta_data_analysis/services/kpler"
+	"eta/eta_data_analysis/utils"
+	"fmt"
+	"time"
+
+	"github.com/patrickmn/go-cache"
+)
+
+func ExcelDataWatch() {
+	fmt.Println("kplerExcelWatch start")
+	utils.FileLog.Info("kplerExcelWatch start")
+	var err error
+	defer func() {
+		if err != nil {
+			fmt.Println("kplerExcelDataWatch Err:" + err.Error())
+			utils.FileLog.Info(fmt.Sprintf("kplerExcelDataWatch, Err: %s", err))
+		}
+	}()
+	var cacheClient *cache.Cache
+	if cacheClient == nil {
+		cacheClient = cache.New(365*24*time.Hour, 365*24*time.Hour)
+	}
+	/*err = filepath.Walk(utils.KplerExcelFilePath, func(path string, info fs.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		if !info.IsDir() {
+			fileInfo, e := os.Stat(path)
+			if e != nil {
+				err = e
+				fmt.Println("os.Stat:", err.Error())
+				utils.FileLog.Info(fmt.Sprintf("os.Stat, Err: %s", err))
+				return err
+			}
+			winFileAttr := fileInfo.Sys().(*syscall.Win32FileAttributeData)
+			modifyTimeStr := utils.SecondToTime(winFileAttr.LastWriteTime.Nanoseconds() / 1e9).Format(utils.FormatDateTime)
+
+			existModifyTime, ok := cacheClient.Get(path)
+			if ok {
+				existModifyTimeStr := existModifyTime.(string)
+				if existModifyTimeStr != modifyTimeStr {
+					_, err = GetKplerDataByExcel(path)
+				}
+			} else {
+				_, err = GetKplerDataByExcel(path)
+			}
+			cacheClient.Delete(path)
+			cacheClient.Set(path, modifyTimeStr, 24*time.Hour)
+		}
+		return nil
+	})*/
+}
+
+// Main function for standalone testing
+func GetKplerDataByExcel(filePath string) {
+	filePath = "services/kpler/crude.xlsx"
+	fmt.Println("Starting Kpler data processing...")
+
+	// Process the Excel data
+	indexData, err :=kpler.ProcessKplerData(filePath)
+	if err != nil {
+		fmt.Printf("Error processing Excel data: %v\n", err)
+		return
+	}
+    indexList := make([]*models.HandleKplerExcelData, 0)
+	// Print the processed data
+	for k, index := range indexData {
+		// 解析请求参数
+		if index.Request != "" {
+			flowsRequestItem, err := kpler.ParseSpecificKplerFormulaV2(index.Request)
+			if err != nil {
+				fmt.Printf("Error parsing formula: %v\n", err)
+				continue
+			}
+			
+			indexName := index.Name
+			unit := flowsRequestItem.Unit
+			sort := k
+			productNameSlice := flowsRequestItem.Products
+			productNames := ""
+			if len(productNameSlice) > 0 {
+				for _, productName := range productNameSlice {
+					productNames += productName.Name + ","
+				}
+			}
+			fromZoneNameSlice := flowsRequestItem.Origins
+			fromZoneNames := ""
+			if len(fromZoneNameSlice) > 0 {
+				for _, fromZoneName := range fromZoneNameSlice {
+					fromZoneNames += fromZoneName.Name + ","
+				}
+			}
+			toZoneNames := ""
+			toZoneNameSlice := flowsRequestItem.Destinations
+			if len(toZoneNameSlice) > 0 {
+				for _, toZoneName := range toZoneNameSlice {
+					toZoneNames += toZoneName.Name + ","
+				}
+			}
+			flowDirection := flowsRequestItem.FlowDirection
+			granularity := flowsRequestItem.Granularity
+			split := flowsRequestItem.Split
+			excelDataMap := make(map[string]string)
+            if len(index.DataPoints) > 0 {
+                for _, dataPoint := range index.DataPoints {
+                    excelDataMap[dataPoint.EndDate] = dataPoint.Value
+                }
+            }
+			tmp := models.HandleKplerExcelData{
+				IndexName: indexName,
+				Unit: unit,
+				Sort: sort,
+				ProductNames: productNames,
+				FromZoneNames: fromZoneNames,
+				ToZoneNames: toZoneNames,
+				FlowDirection: flowDirection,
+				Granularity: granularity,
+				Split: split,
+				ExcelDataMap: excelDataMap,
+			}
+			indexList = append(indexList, &tmp)
+		}
+	}
+
+	if len(indexList) > 0 {
+		params := make(map[string]interface{})
+		params["List"] = indexList
+		params["TerminalCode"] = ""
+		result, e := PostEdbLib(params, utils.LIB_ROUTE_KPLER_DATA)
+		if e != nil {
+			b, _ := json.Marshal(params)
+			utils.FileLog.Info(fmt.Sprintf("sheet ：GetKplerDataByExcel PostEdbLib err: %s, params: %s", e.Error(), string(b)))
+			return
+		}
+		resp := new(models.BaseEdbLibResponse)
+		if e := json.Unmarshal(result, &resp); e != nil {
+			utils.FileLog.Info(fmt.Sprintf("sheet ：GetKplerDataByExcel json.Unmarshal err: %s", e))
+			return
+		}
+		if resp.Ret != 200 {
+			utils.FileLog.Info(fmt.Sprintf("sheet ：GetKplerDataByExcel Msg: %s, ErrMsg: %s", resp.Msg, resp.ErrMsg))
+			return
+		}
+	}
+	// 传递list给指标服务
+
+	fmt.Println("GetKplerDataByExcel completed successfully!")
+}
+
+

services/kpler/excel.go

@@ -2,9 +2,12 @@ package kpler
 
 import (
 	"encoding/json"
+	"eta/eta_data_analysis/models"
+	"eta/eta_data_analysis/utils"
 	"fmt"
 	"net/url"
 	"regexp"
+	"strconv"
 	"strings"
 
 	"github.com/xuri/excelize/v2"
@@ -17,15 +20,8 @@ type ExcelData struct {
 	SheetName   string
 }
 
-// KplerFormulaData represents parsed data from a Kpler formula
-type KplerFormulaData struct {
-	Function    string
-	Parameters  []string
-	FullFormula string
-}
-
 // ParseExcel reads and parses data from an Excel file
-func ParseExcel(filePath string) (*ExcelData, error) {
+func parseExcel(filePath string) (*ExcelData, error) {
 	// Open the Excel file
 	f, err := excelize.OpenFile(filePath)
 	if err != nil {
@@ -57,67 +53,8 @@ func ParseExcel(filePath string) (*ExcelData, error) {
 	return excelData, nil
 }
 
-// ParseKplerFormula extracts the function name and parameters from a Kpler Excel formula
-func ParseKplerFormula(formula string) (*KplerFormulaData, error) {
-	result := &KplerFormulaData{
-		FullFormula: formula,
-	}
-
-	// Regular expression to match the function name and parameters
-	// This pattern looks for: =@'path\to\file.xlam'!FunctionName(param1,param2,...)
-	re := regexp.MustCompile(`=@'[^']*'!([A-Za-z0-9_]+)\((.*)\)`)
-	matches := re.FindStringSubmatch(formula)
-	
-	if len(matches) < 3 {
-		// Try another pattern without the @' prefix
-		re = regexp.MustCompile(`=([A-Za-z0-9_]+)\((.*)\)`)
-		matches = re.FindStringSubmatch(formula)
-		
-		if len(matches) < 3 {
-			return nil, fmt.Errorf("could not parse Kpler formula: %s", formula)
-		}
-	}
-	
-	// Extract function name
-	result.Function = matches[1]
-	
-	// Extract parameters
-	paramsStr := matches[2]
-	
-	// Split parameters, handling commas inside quotes
-	var params []string
-	inQuote := false
-	currentParam := ""
-	
-	for _, char := range paramsStr {
-		switch char {
-		case '"':
-			inQuote = !inQuote
-			currentParam += string(char)
-		case ',':
-			if inQuote {
-				currentParam += string(char)
-			} else {
-				params = append(params, strings.TrimSpace(currentParam))
-				currentParam = ""
-			}
-		default:
-			currentParam += string(char)
-		}
-	}
-	
-	// Add the last parameter
-	if currentParam != "" {
-		params = append(params, strings.TrimSpace(currentParam))
-	}
-	
-	result.Parameters = params
-	return result, nil
-}
-
-
 // ScanSheetForFormulas scans an entire sheet for formulas
-func ScanSheetForFormulas(filePath, sheetName string) (map[int]map[int]string, error) {
+func scanSheetForFormulas(filePath, sheetName string) (map[int]string, error) {
 	// Open the Excel file
 	f, err := excelize.OpenFile(filePath)
 	if err != nil {
@@ -125,7 +62,7 @@ func ScanSheetForFormulas(filePath, sheetName string) (map[int]map[int]string, e
 	}
 	defer f.Close()
 
-	formulas := make(map[int]map[int]string)
+	formulas := make(map[int]string)
 	
 	// Get sheet dimensions
 	dimension, err := f.GetSheetDimension(sheetName)
@@ -169,10 +106,9 @@ func ScanSheetForFormulas(filePath, sheetName string) (map[int]map[int]string, e
 					// fmt.Println("row: ", row)
 					// fmt.Println("col: ", col)
 					// fmt.Println("GetCellFormula: ", formula)
-					if _, ok := formulas[row-1]; !ok {
-						formulas[row-1] = make(map[int]string)
+					if _, ok := formulas[col-1]; !ok {
+						formulas[col-1] = formula
 					}
-					formulas[row-1][col-1] = formula
 				}
 			}
 		}
@@ -181,201 +117,208 @@ func ScanSheetForFormulas(filePath, sheetName string) (map[int]map[int]string, e
 	return formulas, nil
 }
 
+// ProcessKplerData 解析excel获取指标对应的公式和数据
+func ProcessKplerData(filePath string) (indexData []models.KplerExcelIndexData, err error) {
+	defer func() {
+		if err != nil {
+			utils.FileLog.Info(fmt.Sprintf("ProcessKplerData error: %v\n", err))
+		}
+	}()
 
-
-
-// ProcessKplerData demonstrates how to parse and process Kpler crude flow data
-func ProcessKplerData() error {
-	// Path to the Kpler crude flow Excel file
-	//filePath := "services/kpler/Kpler crude flow (自动保存的).xlsx"
-	filePath := "services/kpler/最新版kpler插件.xlsx"
-	// First, list all sheets in the Excel file
 	// Open the Excel file
 	f, err := excelize.OpenFile(filePath)
 	if err != nil {
-		return fmt.Errorf("error opening Excel file: %w", err)
-	}
-	// Return the list of sheets
-	sheets := f.GetSheetList()
-	if err != nil {
-		return fmt.Errorf("error listing sheets: %w", err)
+		return nil, fmt.Errorf("error opening Excel file: %w", err)
 	}
 	defer f.Close()
-	
-	fmt.Println("Available sheets:")
-	for i, sheet := range sheets {
-		fmt.Printf("%d. %s\n", i+1, sheet)
-	}
-	
-	// Parse the Excel file using the first sheet (default)
-	data, err := ParseExcel(filePath)
+
+	// Get the first sheet by default
+	data, err := parseExcel(filePath)
 	if err != nil {
-		return fmt.Errorf("error parsing Excel file: %w", err)
+		return nil, fmt.Errorf("error parsing Excel file: %w", err)
 	}
-	
-	// Print the headers and a sample of data rows
-	// fmt.Println("\nHeaders found in the sheet:")
-	// for i, header := range data.Headers {
-	// 	fmt.Printf("%d. %s\n", i+1, header)
-	// }
-	
+
 	// Look for Kpler formulas
-	fmt.Println("\nLooking for Kpler formulas across the sheet...")
-	formulas, err := ScanSheetForFormulas(filePath, data.SheetName)
+	formulas, err := scanSheetForFormulas(filePath, data.SheetName)
 	if err != nil {
-		fmt.Printf("Error scanning for formulas: %v\n", err)
-		return err
-	} else {
-		fmt.Printf("Found %d formulas in the sheet.\n", len(formulas))
+		return nil, fmt.Errorf("error scanning for formulas: %v", err)
 	}
-	
-	fmt.Println("\nSample data (first 5 rows):")
-	rowCount := 5
-	if len(data.Rows) < rowCount {
-		rowCount = len(data.Rows)
+
+	// Initialize maps to store column information
+	indexMap := make(map[int]*models.KplerExcelIndexData)    // Maps column to index data
+	dateMap := make(map[int]int)                             // Maps data column to its end date column
+
+	// First pass: identify data columns and their corresponding date columns
+	// Headers are in the third row (index 2)
+	if len(data.Rows) < 3 {
+		return nil, fmt.Errorf("Excel file does not have enough rows")
+	}
+
+	headers := data.Rows[1] // Get headers from the second row
+	for j, header := range headers {
+		// Skip empty headers
+		if header == "" {
+			continue
+		}
+
+		// Check if this is a Period End Date column
+		if header == "Period End Date" {
+			// The data column is typically one column before the date
+			if j > 0 && headers[j-1] != "Date" {
+				dateMap[j-1] = j // Map the previous column (data) to this date column
+			}
+		} else if header != "Date" {
+			// This is a data column
+			indexMap[j] = &models.KplerExcelIndexData{
+				Name:       header,
+				DataPoints: make([]models.KplerDataPoint, 0),
+			}
+
+			// Process formula for this column if it exists
+			if formula, ok := formulas[j]; ok {
+				indexMap[j].Request = formula
+			} else {
+				// Look backwards for the formula
+				for k := j; k >= 0; k-- {
+					if formula, ok := formulas[k]; ok {
+						indexMap[j].Request = formula
+						break
+					}
+				}
+			}
+		}
 	}
+
 	
-	for i := 0; i < rowCount; i++ {
-		fmt.Printf("Row %d:\n", i+1)
+	pendingData := make(map[int][]models.DataPoint) // Maps column index to pending data points
+
+	for i := 2; i < len(data.Rows); i++ { // Start from row 3 (index 2) for data
 		row := data.Rows[i]
+		if len(row) == 0 {
+			continue
+		}
+
 		for j, cell := range row {
-			// fmt.Println("i: ", i)
-			// fmt.Println("j: ", j)
-			// //fmt.Println("data.Headers[j]: ", data.Headers[j])
-			// fmt.Println("cell: ", cell)
-			if i == 1 {
-				formula, ok:= formulas[i][j]
-				if ok {
-					fmt.Printf("  %s formula: %s\n", data.Headers[j], formula)
-					// 解析公式
-					ParseSpecificKplerFormulaV2(formula)
+			if cell == "" {
+				continue
+			}
+
+			// Check if this is a date column
+			if _, exists := indexMap[j]; !exists {
+				// This might be a date column, check if it's used as a date column
+				isDateCol := false
+				for dataCol, dateCol := range dateMap {
+					if dateCol == j {
+						isDateCol = true
+						// This is a date column, update all pending data points for the corresponding data column
+						if pending, hasPending := pendingData[dataCol]; hasPending {
+							for _, dp := range pending {
+								if idx, exists := indexMap[dataCol]; exists {
+									idx.DataPoints = append(idx.DataPoints, models.KplerDataPoint{
+										EndDate: cell,
+										Value:   dp.Value,
+									})
+								}
+							}
+							// Clear pending data for this column
+							delete(pendingData, dataCol)
+						}
+						break
+					}
+				}
+				if isDateCol {
+					continue
 				}
 			}
-			if j < len(data.Headers) && cell != "" {
-				fmt.Printf("  %s: %s\n", data.Headers[j], cell)
+
+			// Try to convert cell value to float64
+			_, err := strconv.ParseFloat(cell, 64)
+			if err != nil {
+				fmt.Printf("Warning: Error parsing value at row %d, col %d: %v\n", i+1, j+1, err)
+				continue
+			}
+
+			// Store the data point for later processing when we find the date
+			if _, exists := indexMap[j]; exists {
+				if _, hasPending := pendingData[j]; !hasPending {
+					pendingData[j] = make([]models.DataPoint, 0)
+				}
+				pendingData[j] = append(pendingData[j], models.DataPoint{
+					Value: cell,
+					Row:   i,
+				})
 			}
 		}
-		fmt.Println()
 	}
-	
-	// Print total number of data rows
-	fmt.Printf("Total data rows: %d\n", len(data.Rows))
-	
-	return nil
+
+	// Convert map to slice
+	for _, index := range indexMap {
+		if len(index.DataPoints) > 0 {
+			indexData = append(indexData, *index)
+		}
+	}
+
+	return indexData, nil
 }
 
 
+func ParseSpecificKplerFormulaV2(specificFormula string) (reqObj models.KplerFlowsRequest, err error) {
+	// Remove Excel string concatenation
+	specificFormula = strings.ReplaceAll(specificFormula, `" & "`, "")
+	specificFormula = strings.ReplaceAll(specificFormula, `"&"`, "")
+	specificFormula = strings.ReplaceAll(specificFormula, `&amp;`, "")
+	specificFormula = strings.ReplaceAll(specificFormula, `\"`, `"`)
 
-// 旧版本的kpler插件生成的excel文件解析
-func ParseSpecificKplerFormulaV1() {
-	// The specific formula provided in the user's question
-	specificFormula := `=@'F:\Desktop\kpler\kpler-excel-addin.xlam'!GetFlows("China,","Russian Federation,Venezuela,Iran,EOPL,",,,,,"import","weekly","Total","kbd",FALSE,,FALSE,FALSE,FALSE,FALSE,TRUE,"CPP",)`
-	
-	fmt.Println("Analyzing the specific Kpler formula from the user's question:")
-	fmt.Println(specificFormula)
-	
-	parsedFormula, err := ParseKplerFormula(specificFormula)
-	if err != nil {
-		fmt.Printf("Error parsing formula: %v\n", err)
-		return
+	// Get content inside parentheses
+	re := regexp.MustCompile(`_xldudf_KPLER_GETFLOWS\((.*)\)`)
+	matches := re.FindStringSubmatch(specificFormula)
+	if len(matches) < 2 {
+		// Try the old format
+		re = regexp.MustCompile(`\((.*)\)`)
+		matches = re.FindStringSubmatch(specificFormula)
+		if len(matches) < 2 {
+			err = fmt.Errorf("没有找到括号里的内容")
+			return
+		}
 	}
+
+	// Get the parameter string
+	encodedParam := matches[1]
+	// Remove surrounding quotes if present
+	encodedParam = strings.Trim(encodedParam, `"`)
 	
-	fmt.Println("\nFormula breakdown:")
-	fmt.Printf("Function name: %s\n", parsedFormula.Function)
-	fmt.Println("Parameters list:")
-	
-	// Define parameter names for GetFlows function based on Kpler documentation
-	paramNames := []string{
-		"fromZones",            // 1. "Saudi Arabia,"
-		"toZones",              // 2. empty
-		"fromInstallations",        // 3. empty
-		"toInstallations",          // 4. empty
-		"fromRegions",          // 5. empty
-		"toRegions",            // 6. empty
-		"flowDirection",        // 7. "import"
-		"granularity",          // 8. "weekly"
-		"split",                // 9. "Total"
-		"unit",                 // 10. "kbd"
-		"withForecast",            // 14. FALSE
-		"products",
-		"withProductEstimation",             // 15. FALSE
-		"withIntraCountry",   // 16. FALSE
-		"withIntraRegion",   // 17. TRUE
-		"withFreightView",  // false
-		"withPeriodEndTime", // false
-		"productFilter",        // 18. "CPP"
-		"lastDataPoints",       // 19. empty
+	// Try direct JSON parsing first
+	var jsonObj models.KplerFlowsRequest
+	if err = json.Unmarshal([]byte(encodedParam), &jsonObj); err == nil {
+		return jsonObj, nil
 	}
-	// Print each parameter with its meaning
-	for i, param := range parsedFormula.Parameters {
-		if i < len(paramNames) {
-			paramName := paramNames[i]
-			if param == "" {
-				fmt.Printf("  %s: [empty]\n", paramName)
-			} else {
-				fmt.Printf("  %s: %s\n", paramName, param)
+
+	// If direct parsing fails, try URL decoding
+	decodedStr, err := url.QueryUnescape(encodedParam)
+	if err != nil {
+		// If URL decoding fails, try removing escapes and parse again
+		cleanStr := strings.ReplaceAll(encodedParam, `\`, "")
+		if err = json.Unmarshal([]byte(cleanStr), &jsonObj); err != nil {
+			// Try one more time with manual concatenation cleanup
+			cleanStr = strings.ReplaceAll(cleanStr, `" "`, "")
+			if err = json.Unmarshal([]byte(cleanStr), &jsonObj); err != nil {
+				return reqObj, fmt.Errorf("error parsing formula: %v", err)
 			}
-		} else {
-			fmt.Printf("  Parameter %d: %s\n", i+1, param)
 		}
+		return jsonObj, nil
 	}
 
-}
-
-// Main function for standalone testing
-func GetKplerDataByExcel() {
-	//fmt.Println("Testing Kpler formula parsing...")
-//	FormulaExample()
-fmt.Println("Starting Kpler data processing...")
-//ParseSpecificKplerFormulaV2()
-// First demonstrate the specific formula parsing
-// ParseSpecificKplerFormula()
-
-// Then process the Excel data
-err := ProcessKplerData()
-if err != nil {
-	fmt.Println("error processing Excel data: %w", err)
-}
-
-fmt.Println("Kpler data processing completed successfully!")
-}
-
-func ParseSpecificKplerFormulaV2(specificFormula string) {
-	specificFormula = strings.ReplaceAll(specificFormula, `"&"`, "")
-	//specificFormula := `=Kpler.getFlows("%7B%22platform%22%3A%22liquids%22%2C%22origins%22%3A%5B%7B%22id%22%3A%22Angola%22%2C%22name%22%3A%22Angola%22%7D%5D%2C%22destinations%22%3A%5B%5D%2C%22fromInstallations%22%3A%5B%5D%2C%22toInstallations%22%3A%5B%5D%2C%22flowDirection%22%3A%22export%22%2C%2" &amp; "2products%22%3A%5B%7B%22id%22%3A%22Crude%22%2C%22name%22%3A%22Crude%22%7D%5D%2C%22unit%22%3A%22kbd%22%2C%22isProductEstimation%22%3Afalse%2C%22isIntracountry%22%3Afalse%2C%22isIntraRegion%22%3Afalse%2C%22isWithForecast%22%3Afalse%2C%22granularity%22%3A%22" &amp; "weeks%22%2C%22vesselClassification%22%3A%22CPP%22%2C%22vesselsTypes%22%3A%5B%5D%2C%22split%22%3A%22Total%22%2C%22isFreightView%22%3Afalse%2C%22isWithPeriodEndTime%22%3Atrue%2C%22projection%22%3A%22actual%22%2C%22selectedPreset%22%3A%223m%22%2C%22startDate" &amp; "%22%3Anull%2C%22endDate%22%3Anull%7D")`
-	// 吧函数的入参解析成json串，并转成结构体
-	 // 手动解码URL编码字符串以获取JSON
-  // 这是完整的公式参数 - Excel中使用"&"连接字符串
-  //encodedParam := `%7B%22platform%22%3A%22liquids%22%2C%22origins%22%3A%5B%7B%22id%22%3A%22Angola%22%2C%22name%22%3A%22Angola%22%7D%5D%2C%22destinations%22%3A%5B%5D%2C%22fromInstallations%22%3A%5B%5D%2C%22toInstallations%22%3A%5B%5D%2C%22flowDirection%22%3A%22export%22%2C%22products%22%3A%5B%7B%22id%22%3A%22Crude%22%2C%22name%22%3A%22Crude%22%7D%5D%2C%22unit%22%3A%22kbd%22%2C%22isProductEstimation%22%3Afalse%2C%22isIntracountry%22%3Afalse%2C%22isIntraRegion%22%3Afalse%2C%22isWithForecast%22%3Afalse%2C%22granularity%22%3A%22weeks%22%2C%22vesselClassification%22%3A%22CPP%22%2C%22vesselsTypes%22%3A%5B%5D%2C%22split%22%3A%22Total%22%2C%22isFreightView%22%3Afalse%2C%22isWithPeriodEndTime%22%3Atrue%2C%22projection%22%3A%22actual%22%2C%22selectedPreset%22%3A%223m%22%2C%22startDate%22%3Anull%2C%22endDate%22%3Anull%7D`
-  // 获取括号里的内容
-  re := regexp.MustCompile(`\((.*)\)`)
-  matches := re.FindStringSubmatch(specificFormula)
-  if len(matches) < 2 {
-    fmt.Println("没有找到括号里的内容")
-    return
-  }
-  encodedParam := matches[1]
-  fmt.Println("encodedParam: ", encodedParam)
-  // 解码URL编码的字符串
-  decodedStr, err := url.QueryUnescape(encodedParam)
-  if err != nil {
-    fmt.Printf("Error decoding URL: %v\n", err)
-    return
-  } else {
-    // 打印解码后的JSON字符串
-    fmt.Println("Decoded parameter JSON:")
-    fmt.Println(decodedStr)
-    
-    // 使解码后的JSON更易读
-    var jsonObj interface{}
-    if err := json.Unmarshal([]byte(decodedStr), &jsonObj); err == nil {
-      prettyJSON, _ := json.MarshalIndent(jsonObj, "", "  ")
-      fmt.Println("\nPretty JSON format:")
-      fmt.Println(string(prettyJSON))
-    } else {
-      fmt.Printf("Error parsing JSON: %v\n", err)
-      return
-    }
-  }
-}
+	// Remove surrounding quotes if present in decoded string
+	decodedStr = strings.Trim(decodedStr, `"`)
+	
+	// Try parsing the decoded string
+	if err = json.Unmarshal([]byte(decodedStr), &jsonObj); err != nil {
+		// Try one more time with manual cleanup
+		decodedStr = strings.ReplaceAll(decodedStr, `" "`, "")
+		if err = json.Unmarshal([]byte(decodedStr), &jsonObj); err != nil {
+			return reqObj, fmt.Errorf("error parsing decoded JSON: %v", err)
+		}
+	}
 
+	return jsonObj, nil
+}

services/kpler/kpler.go

@@ -1,6 +1,139 @@
 package kpler
 
-func GetKplerData() {
-	GetKplerDataByApi()
-	//GetKplerDataByExcel()
+import (
+	"eta/eta_data_analysis/models"
+	"eta/eta_data_analysis/services/alarm_msg"
+	"fmt"
+	"time"
+
+	"github.com/patrickmn/go-cache"
+)
+
+func GetProducts() (products []models.KplerProduct, err error) {
+	token, err := GetKplerAccessToken()
+	if err != nil {
+		return nil, err
+	}
+	products, err = GetProductsByApi(token)
+	if err != nil {
+		if err.Error() == "Unauthorized" {
+			token, err = GetKplerAccessToken()
+			if err != nil {
+				return 
+			}
+			products, err = GetProductsByApi(token)
+			if err != nil {
+				return 
+			}
+			return
+		}
+		return 
+	}
+	return 
+}
+//   token := ""
+func GetKplerData(req models.KplerFlowDataLibReq) (data []models.KplerFlowData, err error) {
+//   token := ""
+//   flowDirection := "import"
+//   granularity := "monthly"
+//   split := "Destination%20Countries"
+//   withIntraRegion := "true"
+//   startDate := "2024-01-01"
+//   endDate := "2025-06-30"
+//   unit := "kbd"
+//   products := "CPC%20Russia,Eastern%20Russia%20Crude,Western%20Russia%20Crude"
+//   fromZones := ""
+//   toZones := ""
+//   onlyRealized := "true"
+//   req = models.KplerFlowDataLibReq{
+// 		Granularity: granularity,
+// 		Split: split,
+// 		Unit: unit,
+// 		FlowDirection: flowDirection,
+// 		FromZones: fromZones,
+// 		ToZones: toZones,
+// 		OnlyRealized: onlyRealized,
+// 		WithIntraRegion: withIntraRegion,
+// 		StartDate: startDate,
+// 		EndDate: endDate,
+// 		Products: products,
+// 	}
+    token, err := GetKplerAccessToken()
+	if err != nil {
+		return nil, err
+	}
+	data, err = GetKplerDataByApi(req, token)
+
+	if err != nil {
+		fmt.Println("GetKplerDataByApi error", err)
+		if err.Error() == "Unauthorized" {
+			token, err = GetKplerAccessToken()
+			if err != nil {
+				err = fmt.Errorf("获取开普勒API-AccessToken失败, %v", err)
+				return
+			}
+			data, err = GetKplerDataByApi(req, token)
+			if err != nil {
+				fmt.Println("GetKplerDataByApi error", err)
+				return nil, err
+			}
+			fmt.Println("GetKplerDataByApi success", data)
+			return
+		}
+		return nil, err
+	}
+	fmt.Println("GetKplerDataByApi success", data)
+	return
+}
+
+// GetKplerAccessToken 获取登录凭证
+func GetKplerAccessToken() (token string, err error) {
+	defer func() {
+		if err != nil {
+			go alarm_msg.SendAlarmMsg("获取开普勒的登录凭证失败，ERR："+err.Error(), 3)
+		}
+	}()
+	redisKey := "kpler_access_token"
+	var cacheClient *cache.Cache
+	if cacheClient == nil {
+		cacheClient = cache.New(365*24*time.Hour, 365*24*time.Hour)
+	}
+	tokenTmp, ok := cacheClient.Get(redisKey)
+	//如果从redis中accessToken 获取失败或者token为空了，再或者需要强制刷新了，那么重新获取accessToken
+	if !ok {
+		token, err = refreshKplerAccessToken(cacheClient,redisKey)
+		return
+	}
+	fmt.Println("tokenTmp", tokenTmp)
+	if tokenTmp == nil {
+		token, err = refreshKplerAccessToken(cacheClient,redisKey)
+		return
+	}
+	return
+}
+
+// refreshKplerAccessToken 强制刷新获取登录凭证
+func refreshKplerAccessToken(cacheClient *cache.Cache, redisKey string) (token string, err error) {
+	defer func() {
+		if err != nil {
+			go alarm_msg.SendAlarmMsg("获取开普勒的登录凭证失败；ERR："+err.Error(), 3)
+		}
+	}()
+	token, tmpErr := login()
+	if tmpErr != nil {
+		err = tmpErr
+		return
+	}
+
+	expireTime := time.Now().Add(time.Hour * 24 * 30)
+
+	//token存入redis
+	//err = utils.Rc.Put(tokenRedisKey, token, time.Duration(expireTime.Unix()-600)*time.Second)
+	// 本来是要设置下600s的过期时间，但因为不是强制刷新token，就不获取了
+	cacheClient.Set(redisKey, token, time.Duration(expireTime.Unix())*time.Second)
+	if err != nil {
+		err = fmt.Errorf("获取开普勒的登录凭证成功；开普勒登录凭证存入redis失败，ERR：%s", err.Error())
+		return
+	}
+	return
 }

services/kpler/liquid.go

@@ -2,27 +2,13 @@ package kpler
 
 import (
 	"encoding/json"
+	"eta/eta_data_analysis/models"
 	"fmt"
 	"io/ioutil"
 	"net/http"
 	"strings"
 )
 
-// RunKplerAPI is the main entry point for Kpler API functionality
-func GetKplerDataByApi() {
-  // token, err := login()
-	// if err != nil {
-	// 	fmt.Println(err)
-	// 	return
-	// }
-	//  fmt.Println(token)
-  token := "cU5INqLhj5FuKoC_sNVl4jWlH2jU2Jl2qVndCMuCKL2yT"
-	initFlowIndex(token)
-  //getFlow(token)
-  //getProduct(token)
-  //getZone(token)
-}
-
 // 获取token登录凭证
 func login()(token string, err error){
 
@@ -70,19 +56,19 @@ func login()(token string, err error){
 }
 
 // 分别获取group为：Clean Products；Crude/Co; DPP)的产品
-func getProduct(token string) {
+func GetProductsByApi(token string) (data []models.KplerProduct, err error) {
   url := "https://api.kpler.com/v1/products"
   ancestorFamilyIds := ""
   ancestorFamilyNames := ""
-  ancestorGroupIds := "1370"
-  ancestorGroupNames := "Crude/Co"
+  ancestorGroupIds := ""
+  ancestorGroupNames := ""
   ancestorProductIds := ""
   ancestorProductNames := ""
   ancestorGradeIds := ""
   ancestorGradeNames := ""
   products := ""
   productIds := ""
-  url = fmt.Sprintf("%s?size=1000&ancestorFamilyIds=%s&ancestorFamilyNames=%s&ancestorGroupIds=%s&ancestorGroupNames=%s&ancestorProductIds=%s&ancestorProductNames=%s&ancestorGradeIds=%s&ancestorGradeNames=%s&products=%s&productIds=%s", url, ancestorFamilyIds, ancestorFamilyNames, ancestorGroupIds, ancestorGroupNames, ancestorProductIds, ancestorProductNames, ancestorGradeIds, ancestorGradeNames, products, productIds)
+  url = fmt.Sprintf("%s?ancestorFamilyIds=%s&ancestorFamilyNames=%s&ancestorGroupIds=%s&ancestorGroupNames=%s&ancestorProductIds=%s&ancestorProductNames=%s&ancestorGradeIds=%s&ancestorGradeNames=%s&products=%s&productIds=%s", url, ancestorFamilyIds, ancestorFamilyNames, ancestorGroupIds, ancestorGroupNames, ancestorProductIds, ancestorProductNames, ancestorGradeIds, ancestorGradeNames, products, productIds)
   method := "GET"
 
   client := &http.Client {
@@ -112,40 +98,75 @@ func getProduct(token string) {
 // 2952;CPC Russia;grade;Dirty;1398;Crude/Co;1370;Crude;1368;CPC Russia;2952;805.0;kg/cm;26948.236;MJ/cm;1.0
 // 2953;CPC Kazakhstan;grade;Dirty;1398;Crude/Co;1370;Crude;1368;CPC Kazakhstan;2953;805.0;kg/cm;26948.236;MJ/cm;1.0
 // 1360;CPC;grade;Dirty;1398;Crude/Co;1370;Crude;1368;CPC;1360;805.0;kg/cm;26948.236;MJ/cm;1.0"
+// 解析body
+  var result map[string]interface{}
+  resErr := json.Unmarshal(body, &result)
+  if resErr == nil {
+    if result["message"] == "Unauthorized" {
+      fmt.Println("Unauthorized")
+      return
+    }
+    err = fmt.Errorf(result["message"].(string))
+    return
+  }
+  // 解析result
+  bodyStr := string(body)
+  lines := strings.Split(bodyStr, "\n")
+  for _, line := range lines {
+    fields := strings.Split(line, ";")
+    if len(fields) < 10 {
+      continue
+    }
+    data = append(data, models.KplerProduct{
+      Id: fields[0],
+      Name: fields[1],
+      Type: fields[2],
+      Family: fields[3],
+      FamilyId: fields[4],
+      Group: fields[5],
+      GroupId: fields[6],
+      Product: fields[7],
+      ProductId: fields[8],
+      Grade: fields[9],
+      GradeId: fields[10],
+      Density: fields[11],
+      DensityUnit: fields[12],
+      EnergyDensity: fields[13],
+      EnergyDensityUnit: fields[14],
+      ExpansionRatio: fields[15],
+    })
+  }
+  return
 }
 
 
 
 // 根据flowDirection 和 products 循环调用
-func getFlow(token, flowDirection, split, granularity, products, startDate, endDate string) {
-  //url := "https://api.kpler.com/v1/flows?unit=kb&flowDirection=export&granularity=monthly&toZones=China&products=CPC%20Russia,Eastern%20Russia%20Crude,Western%20Russia%20Crude&split=Destination%20Countries&withIntraRegion=true"
-  //flowDirection := "import"
-  //granularity := "monthly"
-  //split := "Destination%20Countries"
-  withIntraRegion := "true"
-  //startDate := "2024-01-01"
-  //endDate := "2025-06-30"
-  unit := "kbd"
- //products := "CPC%20Russia,Eastern%20Russia%20Crude,Western%20Russia%20Crude"
-
-
-  // fromInstallations := ""
-  // toInstallations := ""
-  // fromZones := ""
-  // toZones := ""
-  // fromCountries := ""
-  // toCountries := ""
-  //onlyRealized := false
-  // vesselTypes := ""
-  // vesselTypesAlt := ""
-  // withIntraCountry := false
-  // withIntraRegion := true
-  // withForecast := true
-  // withFreightView := false
-  // withProductEstimation := false
-
-
-  url := fmt.Sprintf("https://api.kpler.com/v1/flows?unit=%s&flowDirection=%s&granularity=%s&products=%s&split=%s&withIntraRegion=%s&startDate=%s&endDate=%s", unit, flowDirection, granularity, products, split, withIntraRegion, startDate, endDate)
+func GetKplerDataByApi(params models.KplerFlowDataLibReq, token string) (data []models.KplerFlowData, err error) {
+  flowDirection := params.FlowDirection
+  granularity := params.Granularity
+  products := params.Products
+  split := params.Split
+  startDate := params.StartDate
+  endDate := params.EndDate
+  unit := params.Unit
+  withIntraRegion := params.WithIntraRegion
+  fromZones := params.FromZones
+  toZones := params.ToZones
+  onlyRealized := params.OnlyRealized
+  withForecast := params.WithForecast
+  withProductEstimation := params.WithProductEstimation
+  // fromInstallations := req.FromInstallations
+  // toInstallations := req.ToInstallations
+  // fromCountries := req.FromCountries
+  // toCountries := req.ToCountries
+  // vesselTypes := req.VesselTypes
+  // vesselTypesAlt := req.VesselTypesAlt
+  // withIntraCountry := req.WithIntraCountry
+  // 
+  // withFreightView := req.WithFreightView
+
+  url := fmt.Sprintf("https://api.kpler.com/v1/flows?unit=%s&flowDirection=%s&granularity=%s&products=%s&split=%s&withIntraRegion=%s&startDate=%s&endDate=%s&fromZones=%s&toZones=%s&onlyRealized=%s&withForecast=%s&withProductEstimation=%s", unit, flowDirection, granularity, products, split, withIntraRegion, startDate, endDate, fromZones, toZones, onlyRealized, withForecast, withProductEstimation)
   method := "GET"
 
   client := &http.Client {
@@ -171,6 +192,19 @@ func getFlow(token, flowDirection, split, granularity, products, startDate, endD
     return
   }
   fmt.Println(string(body))
+  //{"message":"Unauthorized"}
+  // 解析body
+  var result map[string]interface{}
+  resErr := json.Unmarshal(body, &result)
+  if resErr == nil {
+    if result["message"] == "Unauthorized" {
+      fmt.Println("Unauthorized")
+      return
+    }
+    err = fmt.Errorf(result["message"].(string))
+    return
+  }
+
 //   bodystr :=`Date;China;Period End Date
 // 2024-07;35763.15;2024-07-31
 // 2024-08;35386.42;2024-08-31
@@ -185,26 +219,61 @@ func getFlow(token, flowDirection, split, granularity, products, startDate, endD
 // 2025-05;3795.25;2025-05-31
 // 2025-06;0;2025-06-30`
 
+  // 解析result
+  bodyStr := string(body)
+  lines := strings.Split(bodyStr, "\n")
+	
+	// 解析lines
+  splitNameMap := make(map[int]string)
+  splitDataMap := make(map[int][]models.KplerBaseExcelData)
+  endDateCol := 0
+	for row, line := range lines {
+		fields := strings.Split(line, ";")
+		if len(fields) < 3 {
+			continue
+		}
+    for col, field := range fields {
+      if col == 0 {
+        continue
+      }
+      // 处理表头
+      if row == 0 {
+         if field == "Period End Date" {
+          endDateCol = col
+         }else if field == "Date" {
+          continue
+         }else{
+          splitNameMap[col] = field
+         }
+      }else{
+        if col == endDateCol {
+          continue
+        }
+        date := fields[endDateCol]
+        value := fields[col]
+        splitDataMap[col] = append(splitDataMap[col], models.KplerBaseExcelData{
+          DataTime: date,
+          Value: value,
+        })
+      }
+    }
+	}
+
+  for col, name := range splitNameMap {
+    data = append(data, models.KplerFlowData{
+      SplitItem: name,
+      ApiQueryUrl: url,
+      IndexData: splitDataMap[col],
+    })
+  }
+  return
 }
 
-// 获取所有产品信息，初始化产品分类
-// 初始化指标，
-func initFlowIndex(token string) {
-  //flowDirection := "import"
-  flowDirection := "export"
-  granularity := "monthly"
-  //split := "Destination%20Countries"
-  split := "Origin%20Countries"
-  startDate := "2025-03-30"
-  endDate := "2025-06-25"
-  products := "Crude/Co"
-  getFlow(token, flowDirection, split, granularity, products, startDate, endDate)
-}
 
 
-func getZone(token string) {
+func GetZonesByApi(token string, ancestorName string, descendantType string) (data []models.KplerZone, err error) {
   //url := "https://api.kpler.com/v1/zones"
-  url := "https://api.kpler.com/v1/zones?ancestorName=Baltic%20Sea&descendantType=port"
+  url := fmt.Sprintf("https://api.kpler.com/v1/zones?ancestorName=%s&descendantType=%s", ancestorName, descendantType)
 
   method := "GET"
 
@@ -341,4 +410,26 @@ func getZone(token string) {
 // 87;Baltic Sea;gulf;3382;Kopli;port
 // 87;Baltic Sea;gulf;1156;Muuga Harbour;port
 // 87;Baltic Sea;gulf;2601;Ust Luga;port`
+
+  // 解析result
+  bodyStr := string(body)
+  lines := strings.Split(bodyStr, "\n")
+  for i, line := range lines {
+    if i == 0 {
+      continue
+    }
+    fields := strings.Split(line, ";")
+    if len(fields) < 6 {
+      continue
+    }
+    data = append(data, models.KplerZone{
+      AncestorId: fields[0],
+      AncestorName: fields[1],
+      AncestorType: fields[2],
+      DescendantId: fields[3],
+      DescendantName: fields[4],
+      DescendantType: fields[5],
+    })
+  }
+  return
 }

utils/config.go

@@ -132,6 +132,12 @@ var (
 	ClarkSonsOpen     string //是否配置克拉克森数据源，1已配置
 )
 
+// Kpler
+var (
+	KplerExcelFilePath string //excel文件地址
+	KplerExcelOpen     string //是否配置Kpler数据源，1已配置
+)
+
 func init() {
 	tmpRunMode, err := web.AppConfig.String("run_mode")
 	if err != nil {

utils/constants.go

@@ -274,6 +274,7 @@ const (
 	GET_RZD_EDB_INFO_BY_INDEX_CODE                    = "/rzd/get/rzd/edb/info/by/code"                         // 根据指标code获取指标信息
 	UPDATE_RZD_EDB_DATA                               = "/rzd/update/rzd/edb/data"                              // 修改指标库指标数据
 	LIB_ROUTE_CLARKSONS                               = "/clarksons/data"                                       // 克拉克森
+	LIB_ROUTE_KPLER_DATA                               = "/kpler/handle/excel_data"                                       // 克拉克森
 )
 
 const (