自动检测文件

controllers/yongyi.go

@@ -32,7 +32,7 @@ func (this *YongyiController) Analysis() {
 	weekFlag, _ := this.GetInt("WeekFlag")
 	var err error
 	if dayFlag == 1 {
-		err = services.HandleYongyiExcelDaily(context.TODO())
+		err = services.YongyiDownloadDaily(context.TODO())
 		if err != nil {
 			fmt.Println(err)
 			br.Msg = "操作异常"
@@ -42,7 +42,7 @@ func (this *YongyiController) Analysis() {
 	}
 
 	if weekFlag == 1 {
-		err = services.HandleYongyiExcelWeekly(context.TODO())
+		err = services.YongyiDownloadWeekyly(context.TODO())
 		if err != nil {
 			fmt.Println(err)
 			br.Msg = "操作异常"

services/base_from_yongyi.go

@@ -7,8 +7,13 @@ import (
 	"eta/eta_data_analysis/services/base_from_yongyi"
 	"eta/eta_data_analysis/utils"
 	"fmt"
+	"github.com/patrickmn/go-cache"
 	"github.com/rdlucklib/rdluck_tools/http"
 	"github.com/tealeg/xlsx"
+	"io/fs"
+	"os"
+	"path/filepath"
+	"strings"
 	"time"
 )
 
@@ -35,8 +40,8 @@ import (
 月度-大猪存栏（2020年5月新增）
 月度-商品猪出栏量
 */
-func HandleYongyiExcelDaily(cont context.Context) (err error) {
-	filePath := fmt.Sprintf("%s/%s_day.xlsx", utils.YongyiFilePath, time.Now().Format(utils.FormatDate))
+func HandleYongyiExcelDaily(filePath string) (err error) {
+	//filePath := fmt.Sprintf("%s/%s_day.xlsx", utils.YongyiFilePath, time.Now().Format(utils.FormatDate))
 	xlFile, err := xlsx.OpenFile(filePath)
 	if err != nil {
 		err = fmt.Errorf("打开文件失败, Err: %s", err)
@@ -87,8 +92,8 @@ func HandleYongyiExcelDaily(cont context.Context) (err error) {
 	return
 }
 
-func HandleYongyiExcelWeekly(cont context.Context) (err error) {
-	filePath := fmt.Sprintf("%s/%s_week.xlsx", utils.YongyiFilePath, time.Now().Format(utils.FormatDate))
+func HandleYongyiExcelWeekly(filePath string) (err error) {
+	// filePath := fmt.Sprintf("%s/%s_week.xlsx", utils.YongyiFilePath, time.Now().Format(utils.FormatDate))
 	xlFile, err := xlsx.OpenFile(filePath)
 	if err != nil {
 		err = fmt.Errorf("打开文件失败, Err: %s", err)
@@ -158,6 +163,19 @@ const (
 // @Description: 调用python服务去涌益咨询官网下载日度excel文件
 // @datetime 2023-12-19 09:39:05
 func YongyiDownloadDaily(cont context.Context) (err error) {
+	// todo 判断文件是否已经下载，如果已经下载到则无需重复下载
+	var cacheClient *cache.Cache
+	if cacheClient == nil {
+		cacheClient = cache.New(365*24*time.Hour, 365*24*time.Hour)
+	}
+	// 2023年12月19日涌益咨询日度数据
+	path := fmt.Sprintf("%s/%s%s", utils.YongyiFilePath, time.Now().Format(utils.FormatDateYearMonthDay), "涌益咨询日度数据.xlsx")
+	_, ok := cacheClient.Get(path)
+	fmt.Println("YongyiDownloadDaily: " + path)
+	if ok {
+		utils.FileLog.Info("YongyiDownloadDaily: 文件已存在无需再下载")
+		return
+	}
 	url := fmt.Sprintf("%s?dayFlag=1&weekFlag=0", YongyiDownloadUrl)
 	fmt.Println("YongyiDownload URL:" + url)
 	body, err := http.Get(url)
@@ -169,15 +187,31 @@ func YongyiDownloadDaily(cont context.Context) (err error) {
 	return
 }
 
-// YongyiDownloadWeeyly
+// YongyiDownloadWeekyly
 // @Description: 调用python服务去涌益咨询官网下载周度excel文件
 // @datetime 2023-12-19 09:39:05
-func YongyiDownloadWeeyly(cont context.Context) (err error) {
+func YongyiDownloadWeekyly(cont context.Context) (err error) {
 	weekFlag := 1
 	week := time.Now().Weekday()
 	if week != time.Thursday { //每周四，处理周度文件
 		return
 	}
+	// 判断文件是否已经下载，如果已经下载到则无需重复下载
+	var cacheClient *cache.Cache
+	if cacheClient == nil {
+		cacheClient = cache.New(365*24*time.Hour, 365*24*time.Hour)
+	}
+	// 2023.11.10-2023.11.16涌益咨询 周度数据.xlsx
+	// 获取本周的时间范围, 即当前时间
+	endDate := time.Now().Format(utils.FormatDatePoint)
+	startDate := time.Now().AddDate(0, 0, -6).Format(utils.FormatDatePoint)
+	path := fmt.Sprintf("%s/%s-%s%s", utils.YongyiFilePath, startDate, endDate, "涌益咨询 周度数据.xlsx")
+	fmt.Println("YongyiDownloadWeekyly: " + path)
+	_, ok := cacheClient.Get(path)
+	if ok {
+		utils.FileLog.Info("YongyiDownloadWeekyly: 文件已存在无需再下载")
+		return
+	}
 	url := fmt.Sprintf("%s?dayFlag=0&weekFlag=%d", YongyiDownloadUrl, weekFlag)
 	fmt.Println("YongyiDownload URL:" + url)
 	body, err := http.Get(url)
@@ -185,6 +219,62 @@ func YongyiDownloadWeeyly(cont context.Context) (err error) {
 		utils.FileLog.Info("YongyiDownload Err:" + err.Error())
 		return
 	}
+
 	utils.FileLog.Debug("YongyiDownload Result:" + string(body))
 	return
 }
+
+// 监听涌溢咨询文件夹是否有新增的excel文件
+func ReadWatchYongyiFile(cont context.Context) (err error) {
+	fmt.Println("ReadWatchYongyiFile start")
+	defer func() {
+		if err != nil {
+			fmt.Println("ReadWatchYongyiFile Err:" + err.Error())
+		}
+	}()
+	var cacheClient *cache.Cache
+	if cacheClient == nil {
+		cacheClient = cache.New(365*24*time.Hour, 365*24*time.Hour)
+	}
+	err = filepath.Walk(utils.YongyiFilePath, func(path string, info fs.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		if !info.IsDir() {
+			fmt.Println("ReadWatchYongyiFile path" + path)
+			fileInfo, err := os.Stat(path)
+			if err != nil {
+				fmt.Println("os.Stat:", err.Error())
+			}
+			winFileAttr := fileInfo.Sys().(*syscall.Win32FileAttributeData)
+			modifyTimeStr := utils.SecondToTime(winFileAttr.LastWriteTime.Nanoseconds() / 1e9).Format(utils.FormatDateTime)
+			fmt.Println("ReadWatchYongyiFile modifyTimeStr" + modifyTimeStr)
+			existModifyTime, ok := cacheClient.Get(path)
+			fmt.Println("ReadWatchYongyiFile existModifyTime" + existModifyTime.(string))
+			if ok {
+				existModifyTimeStr := existModifyTime.(string)
+				if existModifyTimeStr != modifyTimeStr {
+					if strings.Contains(path, "涌益咨询") && strings.Contains(path, "日度") {
+						time.Sleep(time.Second * 10)
+						err = HandleYongyiExcelDaily(path)
+					} else if strings.Contains(path, "涌益咨询") && strings.Contains(path, "周度") {
+						time.Sleep(time.Second * 10)
+						err = HandleYongyiExcelWeekly(path)
+					}
+				}
+			} else {
+				if strings.Contains(path, "涌益咨询") && strings.Contains(path, "日度") {
+					time.Sleep(time.Second * 10)
+					err = HandleYongyiExcelDaily(path)
+				} else if strings.Contains(path, "涌益咨询") && strings.Contains(path, "周度") {
+					time.Sleep(time.Second * 10)
+					err = HandleYongyiExcelWeekly(path)
+				}
+			}
+			cacheClient.Delete(path)
+			cacheClient.Set(path, modifyTimeStr, 24*time.Hour)
+		}
+		return nil
+	})
+	return
+}

services/task.go

@@ -17,18 +17,22 @@ import (
 func Task() {
 	fmt.Println("task start")
 	if utils.YongyiOpen == "1" {
-		handleYongyiExcelDaily := task.NewTask("handleYongyiExcelDaily", "0 35 13,23 * * *", HandleYongyiExcelDaily)
+		// 每隔两分钟检测文件是否变动，如果发生变动，则自动解析数据到库里
+		readWatchYongyiFile := task.NewTask("ReadWatchYongyiFile", "0 */2 * * * *", ReadWatchYongyiFile)
+		task.AddTask("监听涌溢咨询文件夹并解析Excel", readWatchYongyiFile)
+
+		/*handleYongyiExcelDaily := task.NewTask("handleYongyiExcelDaily", "0 35 13,23 * * *", HandleYongyiExcelDaily)
 		task.AddTask("涌益咨询日度指标处理", handleYongyiExcelDaily)
 
 		handleYongyiExcelWeekly := task.NewTask("handleYongyiExcelWeekly", "0 5 17,23 * * *", HandleYongyiExcelWeekly)
-		task.AddTask("涌益咨询周度指标处理", handleYongyiExcelWeekly)
+		task.AddTask("涌益咨询周度指标处理", handleYongyiExcelWeekly)*/
 		//HandleYongyiExcelDaily("/Users/xiexiaoyuan/Downloads/2023年11月21日涌益咨询日度数据 (1).xlsx")
 		//HandleYongyiExcelWeekly()
 
-		yongyiDownloadDaily := task.NewTask("YongyiDownloadDaily", "0 30 13,23 * * *", YongyiDownloadDaily)
+		yongyiDownloadDaily := task.NewTask("YongyiDownloadDaily", "0 */30 13-23 * * *", YongyiDownloadDaily)
 		task.AddTask("涌益咨询日度指标下载", yongyiDownloadDaily)
 
-		yongyiDownloadWeekly := task.NewTask("YongyiDownloadWeekly", "0 0 17,23 * * *", YongyiDownloadWeeyly)
+		yongyiDownloadWeekly := task.NewTask("YongyiDownloadWeekly", "0 30 17-23 * * *", YongyiDownloadWeekyly)
 		task.AddTask("涌益咨询周度指标下载", yongyiDownloadWeekly)
 		task.StartTask()
 	}
@@ -47,7 +51,6 @@ func Task() {
 	fmt.Println("task end")
 }
 
-
 func ReadWatchIndexFile() {
 	fmt.Println("ReadWatchIndexFile start")
 	var err error
@@ -86,7 +89,7 @@ func ReadWatchIndexFile() {
 						err = JsmHistory(path)
 					} else if strings.Contains(path, "CⅢ-8-16 25省市库存和日耗情况") {
 						err = CoastalHistory(path)
-						time.Sleep(time.Second*10)
+						time.Sleep(time.Second * 10)
 						err = InlandHistory(path)
 					}
 				}
@@ -101,7 +104,7 @@ func ReadWatchIndexFile() {
 					err = JsmHistory(path)
 				} else if strings.Contains(path, "CⅢ-8-16 25省市库存和日耗情况") {
 					err = CoastalHistory(path)
-					time.Sleep(time.Second*10)
+					time.Sleep(time.Second * 10)
 					err = InlandHistory(path)
 				}
 			}

utils/constants.go

@@ -15,6 +15,8 @@ const (
 	FormatYearMonthUnSpace     = "200601"                  //年月的日期格式
 	PageSize15                 = 15                        //列表页每页数据量
 	FormatDate1                = "2006/1/02"               //日期格式
+	FormatDateYearMonthDay     = "2006年01月02日"             //日期格式
+	FormatDatePoint            = "2006.01.02"              //日期格式
 	PageSize5                  = 5
 	PageSize10                 = 10
 	PageSize20                 = 20
@@ -230,11 +232,11 @@ var FrequencyDaysMap = map[string]int{
 
 // edb_index_lib 的接口名称
 const (
-	LIB_ROUTE_YONGYI_HANDLE = "yongyi/handle/excel_data" //涌益咨询处理excel数据并入库 数据地址
-	LIB_ROUTE_COAL_MINE_JSM_HISTORY = "/coal_mine/jsm/history" //jsm三省煤炭网历史数据处理excel数据并入库 数据地址
+	LIB_ROUTE_YONGYI_HANDLE             = "yongyi/handle/excel_data"   //涌益咨询处理excel数据并入库 数据地址
+	LIB_ROUTE_COAL_MINE_JSM_HISTORY     = "/coal_mine/jsm/history"     //jsm三省煤炭网历史数据处理excel数据并入库 数据地址
 	LIB_ROUTE_COAL_MINE_COASTAL_HISTORY = "/coal_mine/coastal/history" //沿海煤炭网历史数据处理excel数据并入库 数据地址
-	LIB_ROUTE_COAL_MINE_INLAND_HISTORY = "/coal_mine/inland/history" //内陆三省煤炭网历史数据处理excel数据并入库 数据地址
-	LIB_ROUTE_COAL_MINE_JSM = "/coal_mine/jsm" //jsm三省煤炭网历史数据处理excel数据并入库 数据地址
-	LIB_ROUTE_COAL_MINE_COASTAL = "/coal_mine/coastal" //沿海煤炭网历史数据处理excel数据并入库 数据地址
-	LIB_ROUTE_COAL_MINE_INLAND = "/coal_mine/inland" //内陆三省煤炭网历史数据处理excel数据并入库 数据地址
+	LIB_ROUTE_COAL_MINE_INLAND_HISTORY  = "/coal_mine/inland/history"  //内陆三省煤炭网历史数据处理excel数据并入库 数据地址
+	LIB_ROUTE_COAL_MINE_JSM             = "/coal_mine/jsm"             //jsm三省煤炭网历史数据处理excel数据并入库 数据地址
+	LIB_ROUTE_COAL_MINE_COASTAL         = "/coal_mine/coastal"         //沿海煤炭网历史数据处理excel数据并入库 数据地址
+	LIB_ROUTE_COAL_MINE_INLAND          = "/coal_mine/inland"          //内陆三省煤炭网历史数据处理excel数据并入库 数据地址
 )