eta_data_analysis/services/task.go

package services

import (
	ccfService "eta/eta_data_analysis/services/base_from_ccf"
	hisugarService "eta/eta_data_analysis/services/base_from_hisugar"
	oilchemService "eta/eta_data_analysis/services/base_from_oilchem"
	"eta/eta_data_analysis/services/fenwei"
	"eta/eta_data_analysis/services/liangyou"
	"eta/eta_data_analysis/services/ruizide"
	"eta/eta_data_analysis/utils"
	"fmt"
	"io/fs"
	"os"
	"path/filepath"
	"strings"
	"syscall"
	"time"

	"github.com/beego/beego/v2/task"
	"github.com/patrickmn/go-cache"
	"github.com/robfig/cron/v3"
)

func Task() {
	fmt.Println("task start")
	if utils.YongyiOpen == "1" {
		// 每隔两分钟检测文件是否变动，如果发生变动，则自动解析数据到库里
		readWatchYongyiFile := task.NewTask("ReadWatchYongyiFile", "0 */2 * * * *", ReadWatchYongyiFile)
		task.AddTask("监听涌溢咨询文件夹并解析Excel", readWatchYongyiFile)

		yongyiDownloadDaily := task.NewTask("YongyiDownloadDaily", "0 */25 13-23 * * *", YongyiDownloadDaily)
		task.AddTask("涌益咨询日度指标下载", yongyiDownloadDaily)

		yongyiDownloadWeekly := task.NewTask("YongyiDownloadWeekly", "0 */30 16-23 * * 4,5,6,0", YongyiDownloadWeekyly)
		task.AddTask("涌益咨询周度指标下载", yongyiDownloadWeekly)
	}

	if utils.CoalMineOpen == "1" {
		c := cron.New(cron.WithSeconds())
		//每2分钟检测一次指标文件是否更新
		_, err := c.AddFunc("0 */2 * * * *", ReadWatchIndexFile)
		if err != nil {
			fmt.Println("watch.ReadWatchIndexFile err" + err.Error())
			utils.FileLog.Info("watch.ReadWatchIndexFile err" + err.Error())
		}
		c.Start()
	}

	// 汾渭数据
	if utils.FenweiOpen == "1" {
		// 每5分钟检测一次目录是否有新文件
		fenWeiReadWatchIndexFile := task.NewTask("fenWeiReadWatchIndexFile", "0 */5 * * * *", FenweiReadWatchIndexFile)
		task.AddTask("汾渭数据指标文件检测", fenWeiReadWatchIndexFile)
	}

	// 汾渭网络数据
	if utils.FenweiNetOpen == "1" {
		// 汾渭网络数据处理 每天16点和19点 执行爬取
		fenWeiNetDataDeal := task.NewTask("fenWeiNetDataDeal", "0 0 16,19 * * *", fenwei.FenWeiNetDataDeal)
		task.AddTask("汾渭网络数据处理", fenWeiNetDataDeal)
	}

	if utils.LY_OPEN == "1" {
		lyData := task.NewTask("refreshData", "0 0 12 * * *", liangyou.LyDataDeal) // 粮油商务网

		task.AddTask("粮油商务网", lyData)
	}

	if utils.RZD_OPEN == "1" {
		rzdData := task.NewTask("refreshRzdData", "0 0 1 * * 4", ruizide.ResolverNet) // 睿姿得数据

		task.AddTask("睿咨得数据", rzdData)
	}

	if utils.MtjhOpen == "1" {
		c := cron.New(cron.WithSeconds())
		//每2分钟检测一次指标文件是否更新
		_, err := c.AddFunc("0 */2 * * * *", mtjhWatch)
		if err != nil {
			fmt.Println("watch.mtjhWatch err" + err.Error())
			utils.FileLog.Info("watch.mtjhWatch err" + err.Error())
		}
		c.Start()
	}

	if utils.CoalMailAttachmentOpen == "1" {
		mailAttachment := task.NewTask("MailAttachment", utils.CoalMailAttachmentTime, MailAttachment)
		task.AddTask("启动获取邮件附件脚本", mailAttachment)
	}

	// CCF化纤信息
	if utils.CCFOpen == "1" {
		// 原油石化早报-默认每日9:00
		ccfDailyTime := utils.CCFDailyTaskTime
		if ccfDailyTime == "" {
			ccfDailyTime = "0 0 9 * * *"
		}
		taskCCFOilDailyEdb := task.NewTask("taskCCFOilDailyEdb", ccfDailyTime, ccfService.TaskOilDailyEdb)
		task.AddTask("CCF原油石化早报", taskCCFOilDailyEdb)

		// 各品种周报-默认每周五17-19点每隔半小时
		ccfWeeklyTime := utils.CCFWeeklyTaskTime
		if ccfWeeklyTime == "" {
			ccfWeeklyTime = "0 */30 17-19 * * 5"
		}
		taskCCFWeeklyEdb := task.NewTask("taskCCFWeeklyEdb", ccfWeeklyTime, ccfService.TaskWeeklyEdb)
		task.AddTask("CCF周度指标", taskCCFWeeklyEdb)

		// 各品种装置-默认每周四15-18点每隔半小时
		ccfStockTime := utils.CCFStockTaskTime
		if ccfStockTime == "" {
			ccfStockTime = "0 */30 15-18 * * 4"
		}
		taskCCFStockTable := task.NewTask("taskCCFStockTable", ccfStockTime, ccfService.TaskStockTable)
		task.AddTask("CCF装置检修", taskCCFStockTable)

		// ccf图表数据-默认每天下午4点
		ccfChartTime := utils.CCFChartTaskTime
		if ccfChartTime == "" {
			ccfChartTime = "0 16 * * * *"
		}
		taskGetCCFChartEdb := task.NewTask("taskGetCCFChartEdb", ccfChartTime, ccfService.TaskGetCCFChartEdb)
		task.AddTask("CCF图表数据", taskGetCCFChartEdb)
	}

	// 隆众资讯
	if utils.OilchemDataInit == "1" && utils.OilchemOpen == "1" {
		err := oilchemService.OilchemList(nil)
		if err != nil {
			utils.FileLog.Info("InitOilchemList Err:" + err.Error())
		}
	} else if utils.OilchemOpen == "1" {
		oilchemData := task.NewTask("oilchemData", "0 0 16 * * *", oilchemService.OilchemList)
		task.AddTask("卓创资讯", oilchemData)
	}

	if utils.HisugarOpen == "1" {
		hisugarData := task.NewTask("hisugarData", "0 30 16 * * *", hisugarService.HiugarCrawler)
		task.AddTask("泛糖科技", hisugarData)
	}

	if utils.ClarkSonsOpen == "1" {
		c := cron.New(cron.WithSeconds())
		//每2分钟检测一次指标文件是否更新
		_, err := c.AddFunc("0 */2 * * * *", clarksonsWatch)
		if err != nil {
			fmt.Println("watch.clarksonsWatch err" + err.Error())
			utils.FileLog.Info("watch.clarksonsWatch err" + err.Error())
		}
		c.Start()
	}

	task.StartTask()

	fmt.Println("task end")
}

func ReadWatchIndexFile() {
	fmt.Println("ReadWatchIndexFile start")
	var err error
	defer func() {
		if err != nil {
			fmt.Println("ReadWatchIndexFile Err:" + err.Error())
		}
	}()
	var cacheClient *cache.Cache
	if cacheClient == nil {
		cacheClient = cache.New(365*24*time.Hour, 365*24*time.Hour)
	}
	err = filepath.Walk(utils.CoalMineFilePath, func(path string, info fs.FileInfo, err error) error {
		if err != nil {
			return err
		}
		if !info.IsDir() {
			fileInfo, err := os.Stat(path)
			if err != nil {
				fmt.Println("os.Stat:", err.Error())
			}
			winFileAttr := fileInfo.Sys().(*syscall.Win32FileAttributeData)
			modifyTimeStr := utils.SecondToTime(winFileAttr.LastWriteTime.Nanoseconds() / 1e9).Format(utils.FormatDateTime)

			existModifyTime, ok := cacheClient.Get(path)
			if ok {
				existModifyTimeStr := existModifyTime.(string)
				if existModifyTimeStr != modifyTimeStr {
					if strings.Contains(path, "442家晋陕蒙煤矿周度产量数据") {
						err = Jsm(path)
					} else if strings.Contains(path, "内陆17省动力煤终端用户供耗存") {
						err = Inland(path)
					} else if strings.Contains(path, "沿海八省动力煤终端用户供耗存数据更新") {
						err = Coastal(path)
					} else if strings.Contains(path, "442家晋陕蒙历史数据") {
						err = JsmHistory(path)
					} else if strings.Contains(path, "CⅢ-8-16 25省市库存和日耗情况") {
						err = CoastalHistory(path)
						time.Sleep(time.Second * 10)
						err = InlandHistory(path)
					} else if strings.Contains(path, "分企业煤炭产量旬度数据") {
						err = Firm(path)
					}
				}
			} else {
				if strings.Contains(path, "442家晋陕蒙煤矿周度产量数据") {
					err = Jsm(path)
				} else if strings.Contains(path, "内陆17省动力煤终端用户供耗存") {
					err = Inland(path)
				} else if strings.Contains(path, "沿海八省动力煤终端用户供耗存数据更新") {
					err = Coastal(path)
				} else if strings.Contains(path, "442家晋陕蒙历史数据") {
					err = JsmHistory(path)
				} else if strings.Contains(path, "CⅢ-8-16 25省市库存和日耗情况") {
					err = CoastalHistory(path)
					time.Sleep(time.Second * 10)
					err = InlandHistory(path)
				} else if strings.Contains(path, "分企业煤炭产量旬度数据") {
					err = Firm(path)
				}
			}
			cacheClient.Delete(path)
			cacheClient.Set(path, modifyTimeStr, 24*time.Hour)
		}
		return nil
	})
}