eta_data_analysis/services/base_from_kpler.go

package services

import (
	"context"
	"encoding/json"
	"eta/eta_data_analysis/models"
	"eta/eta_data_analysis/services/kpler"
	"eta/eta_data_analysis/utils"
	"fmt"
	"io/fs"
	"os"
	"path/filepath"
	"strings"
	"syscall"
	"time"

	"github.com/patrickmn/go-cache"
)

func KplerExcelDataWatch(cont context.Context) (err error) {
	fmt.Println("kplerExcelWatch start")
	utils.FileLog.Info("kplerExcelWatch start")
	defer func() {
		if err != nil {
			fmt.Println("kplerExcelDataWatch Err:" + err.Error())
			utils.FileLog.Info(fmt.Sprintf("kplerExcelDataWatch, Err: %s", err))
		}
	}()
	cacheClient := utils.CacheClient
	if cacheClient == nil {
		 utils.CacheClient = cache.New(365*24*time.Hour, 365*24*time.Hour)
	}
	err = filepath.Walk(utils.KplerExcelFilePath, func(path string, info fs.FileInfo, err error) error {
		if err != nil {
			return err
		}
		if !info.IsDir() {
			fileInfo, e := os.Stat(path)
			if e != nil {
				err = e
				fmt.Println("os.Stat:", err.Error())
				utils.FileLog.Info(fmt.Sprintf("os.Stat, Err: %s", err))
				return err
			}
			winFileAttr := fileInfo.Sys().(*syscall.Win32FileAttributeData)
			modifyTimeStr := utils.SecondToTime(winFileAttr.LastWriteTime.Nanoseconds() / 1e9).Format(utils.FormatDateTime)
			fmt.Println("文件的修改时间modifyTimeStr:", modifyTimeStr)
			existModifyTime, ok := cacheClient.Get(path)
			fmt.Println("缓存里的时间existModifyTime:", existModifyTime)
			if ok {
				existModifyTimeStr := existModifyTime.(string)
				if existModifyTimeStr != modifyTimeStr {
					 err = GetKplerDataByExcel(path)
				}
			} else {
				 err = GetKplerDataByExcel(path)
			}
			cacheClient.Delete(path)
			cacheClient.Set(path, modifyTimeStr, 24*time.Hour)
		}
		return nil
	})
	return
}

// Main function for standalone testing
func GetKplerDataByExcel(filePath string) (err error) {
	//filePath = "services/kpler/crude.xlsx"
	fmt.Println("Starting Kpler data processing...")

	// Process the Excel data
	indexData, err :=kpler.ProcessKplerData(filePath)
	if err != nil {
		fmt.Printf("Error processing Excel data: %v\n", err)
		return
	}
    indexList := make([]*models.HandleKplerExcelData, 0)
	// Print the processed data
	for k, index := range indexData {
		// 解析请求参数
		if index.Request != "" {
			flowsRequestItem, err := kpler.ParseSpecificKplerFormulaV2(index.Request)
			if err != nil {
				fmt.Printf("Error parsing formula: %v\n", err)
				continue
			}
			
			indexName := fmt.Sprintf("%s_%s", index.Title, index.Name)
			unit := flowsRequestItem.Unit
			sort := k
			classifyName := ""
			productNameSlice := flowsRequestItem.Products
			productNames := ""
			if len(productNameSlice) > 0 {
				for _, productName := range productNameSlice {
					if classifyName == "" {
						classifyName = productName.Name
					}
					productNames += productName.Name + ","
				}
			}
			
			productNames = strings.TrimSuffix(productNames, ",")
			fromZoneNameSlice := flowsRequestItem.Origins
			fromZoneNames := ""
			if len(fromZoneNameSlice) > 0 {
				for _, fromZoneName := range fromZoneNameSlice {
					fromZoneNames += fromZoneName.Name + ","
				}
			}
			fromZoneNames = strings.TrimSuffix(fromZoneNames, ",")
			toZoneNames := ""
			toZoneNameSlice := flowsRequestItem.Destinations
			if len(toZoneNameSlice) > 0 {
				for _, toZoneName := range toZoneNameSlice {
					toZoneNames += toZoneName.Name + ","
				}
			}
			toZoneNames = strings.TrimSuffix(toZoneNames, ",")
			flowDirection := flowsRequestItem.FlowDirection
			granularity := flowsRequestItem.Granularity
			split := flowsRequestItem.Split
			excelDataMap := make(map[string]string)
            if len(index.DataPoints) > 0 {
                for _, dataPoint := range index.DataPoints {
                    excelDataMap[dataPoint.EndDate] = dataPoint.Value
                }
            }
			tmp := models.HandleKplerExcelData{
				IndexName: indexName,
				Unit: unit,
				Sort: sort,
				ClassifyName: classifyName,
				ProductNames: productNames,
				FromZoneNames: fromZoneNames,
				ToZoneNames: toZoneNames,
				FlowDirection: flowDirection,
				Granularity: granularity,
				Split: split,
				SplitName: index.Name,
				ExcelQueryUrl: index.Request,
				ExcelDataMap: excelDataMap,
			}
			indexList = append(indexList, &tmp)
		}
	}

	if len(indexList) > 0 {
		params := make(map[string]interface{})
		params["List"] = indexList
		params["TerminalCode"] = ""
		result, e := PostEdbLib(params, utils.LIB_ROUTE_KPLER_DATA)
		if e != nil {
			b, _ := json.Marshal(params)
			utils.FileLog.Info(fmt.Sprintf("sheet ：GetKplerDataByExcel PostEdbLib err: %s, params: %s", e.Error(), string(b)))
			return
		}
		resp := new(models.BaseEdbLibResponse)
		if e := json.Unmarshal(result, &resp); e != nil {
			utils.FileLog.Info(fmt.Sprintf("sheet ：GetKplerDataByExcel json.Unmarshal err: %s", e))
			return
		}
		if resp.Ret != 200 {
			utils.FileLog.Info(fmt.Sprintf("sheet ：GetKplerDataByExcel Msg: %s, ErrMsg: %s", resp.Msg, resp.ErrMsg))
			return
		}
	}
	// 传递list给指标服务

	fmt.Println("GetKplerDataByExcel completed successfully!")
	return
}

// 定时调用python脚本刷新kpler
func RefreshKplerByExcel(cont context.Context) (err error) {
	if utils.KplerRefreshUrl == "" {
		return
	}
    //查询utils.KplerExcelFilePath目录下所有excel文件
	files, err := filepath.Glob(utils.KplerExcelFilePath + "/*.xlsx")
	if err != nil {
		utils.FileLog.Info("RefreshKplerByExcel Err:" + err.Error())
		return
	}
	for _, file := range files {
		fmt.Println("RefreshKplerByExcel file:" + file)
		kplerRefreshUrl := fmt.Sprintf("%s/kpler/refresh?FilePath=%s", utils.KplerRefreshUrl, file)
		body, er := HttpGet(kplerRefreshUrl)
		if er != nil {
			utils.FileLog.Info("RefreshKplerByExcel Err:" + er.Error())
			return
		}
		utils.FileLog.Info("RefreshKplerByExcel Result:" + string(body))
	}
	
	return
}