feat：调整日志相关，调整定时刷新的逻辑（从配置中获取）

config/config.go

@@ -8,20 +8,20 @@ type Config struct {
 
 // Serve gin服务配置
 type Serve struct {
-	Port               int    `mapstructure:"port" json:"port" yaml:"port" description:"gin开启监听http服务的端口"`
-	RunMode            string `mapstructure:"run-mode" json:"run-mode" yaml:"run-mode" description:"gin运行模式的默认，枚举值：debug,release"`
-	UseRedis           bool   `mapstructure:"use-redis" json:"use-redis" yaml:"use-redis" description:"是否使用redis"`
-	AppName            string `mapstructure:"app-name" json:"app-name" yaml:"app-name" description:"项目名称"`
-	StaticDir          string `mapstructure:"static-dir" json:"static-dir" yaml:"static-dir" description:"上传的文件存储目录地址"`
-	IndexSaveDir       string `mapstructure:"index-save-dir" json:"index-save-dir" yaml:"index-save-dir" description:"监听文件夹的路径"`
-	IndexMergeSaveDir  string `mapstructure:"index-merge-save-dir" json:"index-merge-save-dir" yaml:"index-merge-save-dir" description:"监听合并文件夹的路径"`
-	Frequency          string `mapstructure:"frequency" json:"frequency" yaml:"frequency" description:"频度"`
-	SystemType         string `mapstructure:"frequency" json:"frequency" yaml:"frequency" description:"类型定义"`
-	EdbLibUrl          string `mapstructure:"edb-lib-url" json:"edb-lib-url" yaml:"edb-lib-url" description:"公共指标库的地址"`
-	AppEdbLibNameEn    string `mapstructure:"app_edb_lib_name_en" json:"app_edb_lib_name_en" yaml:"app_edb_lib_name_en" description:"指标库的英文名称"`
-	EdbLibMd5Key       string `mapstructure:"edb_lib_md5_key" json:"edb_lib_md5_key" yaml:"edb_lib_md5_key" description:"指标库服务秘钥"`
-	TerminalCode       string `mapstructure:"terminal_code" json:"terminal_code" yaml:"terminal_code" description:"终端编码"`
-	IsCheckIndexUpdate bool   `mapstructure:"is_check_index_update" json:"is_check_index_update" yaml:"is_check_index_update" description:"是否检测指标更新状态"`
+	Port               int      `mapstructure:"port" json:"port" yaml:"port" description:"gin开启监听http服务的端口"`
+	RunMode            string   `mapstructure:"run-mode" json:"run-mode" yaml:"run-mode" description:"gin运行模式的默认，枚举值：debug,release"`
+	UseRedis           bool     `mapstructure:"use-redis" json:"use-redis" yaml:"use-redis" description:"是否使用redis"`
+	AppName            string   `mapstructure:"app-name" json:"app-name" yaml:"app-name" description:"项目名称"`
+	StaticDir          string   `mapstructure:"static-dir" json:"static-dir" yaml:"static-dir" description:"上传的文件存储目录地址"`
+	IndexSaveDir       string   `mapstructure:"index-save-dir" json:"index-save-dir" yaml:"index-save-dir" description:"监听文件夹的路径"`
+	IndexMergeSaveDir  string   `mapstructure:"index-merge-save-dir" json:"index-merge-save-dir" yaml:"index-merge-save-dir" description:"监听合并文件夹的路径"`
+	Frequency          string   `mapstructure:"frequency" json:"frequency" yaml:"frequency" description:"频度"`
+	EdbLibUrl          string   `mapstructure:"edb-lib-url" json:"edb-lib-url" yaml:"edb-lib-url" description:"公共指标库的地址"`
+	AppEdbLibNameEn    string   `mapstructure:"app_edb_lib_name_en" json:"app_edb_lib_name_en" yaml:"app_edb_lib_name_en" description:"指标库的英文名称"`
+	EdbLibMd5Key       string   `mapstructure:"edb_lib_md5_key" json:"edb_lib_md5_key" yaml:"edb_lib_md5_key" description:"指标库服务秘钥"`
+	TerminalCode       string   `mapstructure:"terminal_code" json:"terminal_code" yaml:"terminal_code" description:"终端编码"`
+	IsCheckIndexUpdate bool     `mapstructure:"is_check_index_update" json:"is_check_index_update" yaml:"is_check_index_update" description:"是否检测指标更新状态"`
+	RefreshTimeList    []string `mapstructure:"refresh_time_list" json:"refresh_time_list" yaml:"refresh_time_list" description:"刷新服务任务更新时间列表"`
 }
 
 // Log 日志配置

init_serve/task.go

@@ -13,31 +13,58 @@ import (
 
 func InitTask() {
 	c := cron.New(cron.WithSeconds())
+	//awk '{ if ($9 <= 200) print "<= 200ms"; else if ($9 <= 500) print "201-500ms"; else if ($9 <= 1000) print "501-1000ms"; else if ($9 <= 2000) print "1001-2000ms"; else print "> 2000ms"; }' eta_api.log | sort | uniq -c
 
-	//每1分钟检测一次，指标生成
+	//每5分钟检测一次，指标生成
 	_, err := c.AddFunc("0 */5 * * * *", CheckIndexCreateMerge)
 	if err != nil {
-		global.LOG.Info("DownloadCvTask err" + err.Error())
+		global.LOG.Error("DownloadCvTask err" + err.Error())
 	}
 
-	//每天17点刷新指标
-	_, err = c.AddFunc("0 0 17 * * *", services.MergeV2)
-	if err != nil {
-		global.LOG.Info("DownloadCvTask err" + err.Error())
+	// 定时统一更新
+	if len(global.CONFIG.Serve.RefreshTimeList) <= 0 {
+		panic("未配置数据统一刷新时间")
+	}
+	for _, timeStr := range global.CONFIG.Serve.RefreshTimeList {
+		_, tmpErr := c.AddFunc(timeStr, services.MergeV2)
+		if tmpErr != nil {
+			global.LOG.Error("services.MergeV2 服务启动失败" + "时间：" + timeStr + "；err：" + tmpErr.Error())
+		}
 	}
+	//
+	////每天17点刷新指标
+	//_, err = c.AddFunc("0 0 14 * * *", services.MergeV2)
+	////_, err = c.AddFunc("0 */2 * * * *", services.MergeV2)
+	//if err != nil {
+	//	global.LOG.Info("services.MergeV2 14:00 err" + err.Error())
+	//}
+	//
+	////每天17点刷新指标
+	//_, err = c.AddFunc("0 0 16 * * *", services.MergeV2)
+	////_, err = c.AddFunc("0 */2 * * * *", services.MergeV2)
+	//if err != nil {
+	//	global.LOG.Info("services.MergeV2 16:00 err" + err.Error())
+	//}
+	//
+	////每天17点刷新指标
+	//_, err = c.AddFunc("0 0 20 * * *", services.MergeV2)
+	////_, err = c.AddFunc("0 */2 * * * *", services.MergeV2)
+	//if err != nil {
+	//	global.LOG.Info("services.MergeV2 20:00 err" + err.Error())
+	//}
 
 	//每2分钟检测一次指标文件是否更新
 	_, err = c.AddFunc("0 */2 * * * *", watch.ReadWatchIndexFile)
 
 	_, err = c.AddFunc("0 0 17 1 * *", services.MergeMonthSeasonYearV2)
 	if err != nil {
-		global.LOG.Info("DownloadCvTask err" + err.Error())
+		global.LOG.Error("DownloadCvTask err" + err.Error())
 	}
 
 	// 服务检测
 	_, err = c.AddFunc("0 */1 * * * *", services.ServerCheck)
 	if err != nil {
-		global.LOG.Info("ServerCheck err" + err.Error())
+		global.LOG.Error("ServerCheck err" + err.Error())
 	}
 
 	// 指标更新检测
@@ -45,7 +72,7 @@ func InitTask() {
 		services.IndexUpdateCheck()
 		_, err = c.AddFunc("0 30 17 * * *", services.IndexUpdateCheck)
 		if err != nil {
-			global.LOG.Info("IndexUpdateCheck err" + err.Error())
+			global.LOG.Error("IndexUpdateCheck err" + err.Error())
 		}
 
 	}

services/api_tool.go

@@ -1,6 +1,7 @@
 package services
 
 import (
+	"eta/mysteel_watch/global"
 	"fmt"
 	"github.com/rdlucklib/rdluck_tools/http"
 	"net/url"
@@ -14,11 +15,11 @@ func MysteelChemicalRefresh(filePath string) {
 	var refreshUrl string
 	//filePathStr:=url.PathEscape(filePath)
 	refreshUrl = RefreshUrl + "?FilePath=" + url.QueryEscape(filePath)
-	fmt.Println("MysteelChemicalRefresh URL:"+refreshUrl)
+	fmt.Println("MysteelChemicalRefresh URL:" + refreshUrl)
 	body, err := http.Get(refreshUrl)
 	if err != nil {
-		fmt.Println("MysteelChemicalRefresh Err:" + err.Error())
+		global.LOG.Error("MysteelChemicalRefresh Err:" + err.Error())
 		return
 	}
-	fmt.Println("MysteelChemicalRefresh Result:" + string(body))
+	global.LOG.Debug("MysteelChemicalRefresh Result:" + string(body))
 }

services/index.go

@@ -346,7 +346,7 @@ func IndexUpdateCheck() {
 	req.FrequencyBatch = strings.Join(frequencyBatch, ",")
 	ret, err := watch.RefreshCheckByDay(req)
 	if err != nil {
-		global.LOG.Info(utils.APPNAME + " 钢联指标更新检查 出错" + time.Now().Format("2006-01-02 15:04:05") + ";Err:" + err.Error())
+		global.LOG.Error(utils.APPNAME + " 钢联指标更新检查 出错" + time.Now().Format("2006-01-02 15:04:05") + ";Err:" + err.Error())
 		go alarm_msg.SendAlarmMsg(utils.APPNAME+" 钢联指标更新检查 出错"+time.Now().Format("2006-01-02 15:04:05")+";Err:"+err.Error(), 3)
 		return
 	}

services/index_create.go

@@ -9,6 +9,7 @@ import (
 	"github.com/xuri/excelize/v2"
 	"os"
 	"strconv"
+	"strings"
 	"sync"
 	"time"
 )
@@ -19,19 +20,46 @@ const (
 
 var indexCreateCheckLock sync.RWMutex
 
+// 有更新的指标
+var hasRefreshIndexMap = make(map[string]int)
+
 func IndexCreateCheckMerge() (err error) {
 	indexCreateCheckLock.Lock()
 	defer func() {
 		indexCreateCheckLock.Unlock()
 	}()
 	fmt.Println("IndexCreateCheck")
-	list, err := watch.GetIndexCreate()
+	findList, err := watch.GetIndexCreate()
 	if err != nil {
 		return
 	}
-	lenList := len(list)
-	fmt.Println("listLen:", len(list))
+	lenFindList := len(findList)
+	fmt.Println("查出来的指标数量:", lenFindList)
 	time.Sleep(3 * time.Second)
+	if lenFindList <= 0 {
+		return nil
+	}
+
+	// 超限的指标编码
+	codeList := make([]string, 0)
+	// 需要去查询的指标编码
+	list := make([]index.BaseFromMysteelChemicalIndex, 0)
+	for _, v := range findList {
+		if tmpV, ok := hasRefreshIndexMap[v.IndexCode]; ok {
+			// 超过5次没刷到数据，就不处理了
+			if tmpV >= 5 {
+				codeList = append(codeList, v.IndexCode)
+				continue
+			}
+		}
+		list = append(list, v)
+	}
+	if len(codeList) > 0 {
+		fmt.Println(strings.Join(codeList, " , "), "；刷新超限了")
+	}
+
+	lenList := len(list)
+	fmt.Println("需要生成的指标数量:", lenList)
 	if lenList <= 0 {
 		return nil
 	}
@@ -51,6 +79,15 @@ func IndexCreateCheckMerge() (err error) {
 		}
 	}
 
+	// 将指标标记下，不重复刷新数据
+	for _, v := range list {
+		tmpV, ok := hasRefreshIndexMap[v.IndexCode]
+		if !ok {
+			tmpV = 0
+		}
+		hasRefreshIndexMap[v.IndexCode] = tmpV + 1
+	}
+
 	templateFile := excelize.NewFile()
 	defer func() {
 		if err := templateFile.Close(); err != nil {
@@ -71,7 +108,7 @@ func IndexCreateCheckMerge() (err error) {
 		} else {
 			pageList = list[IndexCreatePageSize*i:]
 		}
-		fmt.Println(pageList)
+		//fmt.Println(pageList)
 
 		// 生成批注
 		commentResult, tmpErr := GetIndexCommentV3(mergeIndexCreateFilePath, startDate, pageList)

services/server_check.go

@@ -1,7 +1,7 @@
 package services
 
 import (
-	"fmt"
+	"eta/mysteel_watch/global"
 	"github.com/rdlucklib/rdluck_tools/http"
 )
 
@@ -12,8 +12,8 @@ const (
 func ServerCheck() {
 	body, err := http.Get(ServerCheckUrl)
 	if err != nil {
-		fmt.Println("ServerCheck Err:" + err.Error())
+		global.LOG.Error("ServerCheck Err:" + err.Error())
 		return
 	}
-	fmt.Println("ServerCheck Result:" + string(body))
+	global.LOG.Debug("ServerCheck Result:" + string(body))
 }

watch/edb_lib.go

@@ -87,13 +87,13 @@ func postHandleMysteelIndex(req *HandleMysteelIndexReq) (resp *BaseResponse, err
 		return
 	}
 	//fmt.Println("postData:" + string(postData))
-	global.LOG.Info(" postData: " + string(postData))
+	global.LOG.Debug(" postData: " + string(postData))
 	result, err := HttpPost(postUrl, string(postData), "application/json")
 	if err != nil {
 		return
 	}
 	if result != nil {
-		global.LOG.Info(" postHandleMysteelIndex Result: " + string(result))
+		global.LOG.Debug(" postHandleMysteelIndex Result: " + string(result))
 		err = json.Unmarshal(result, &resp)
 		if err != nil {
 			return
@@ -129,7 +129,7 @@ func GetNoMergeIndexByFrequencyCount(req GetMaxFileIndexReq) (total float64, err
 	}
 
 	var resp BaseResponse
-	global.LOG.Info(" GetNoMergeIndexByFrequencyCount Result: " + string(result))
+	global.LOG.Debug(" GetNoMergeIndexByFrequencyCount Result: " + string(result))
 	err = json.Unmarshal(result, &resp)
 	if err != nil {
 		return
@@ -161,7 +161,7 @@ func GetMaxFileIndex(req GetMaxFileIndexReq) (baseFromMysteelChemicalIndex index
 	}
 
 	var resp IndexResponse
-	global.LOG.Info(" Refresh Result: " + string(result))
+	global.LOG.Debug(" Refresh Result: " + string(result))
 	err = json.Unmarshal(result, &resp)
 	if err != nil {
 		return
@@ -207,7 +207,7 @@ func GetNoMergeIndexByFrequencyPage(frequency string, limit int) (item []index.B
 	}
 
 	var resp IndexesResponse
-	global.LOG.Info(" GetNoMergeIndexByFrequencyPage Result: " + string(result))
+	global.LOG.Debug(" GetNoMergeIndexByFrequencyPage Result: " + string(result))
 	err = json.Unmarshal(result, &resp)
 	if err != nil {
 		return
@@ -244,7 +244,7 @@ func GetIndexCountByMergeFilePath(mergeFilePath string) (total float64, err erro
 	}
 
 	var resp BaseResponse
-	global.LOG.Info(" GetIndexCountByMergeFilePath Result: " + string(result))
+	global.LOG.Debug(" GetIndexCountByMergeFilePath Result: " + string(result))
 	err = json.Unmarshal(result, &resp)
 	if err != nil {
 		return
@@ -271,7 +271,7 @@ func GetIndexCreate() (items []index.BaseFromMysteelChemicalIndex, err error) {
 	}
 
 	var resp IndexesResponse
-	global.LOG.Info(" GetIndexCreate Result: " + string(result))
+	global.LOG.Debug(" GetIndexCreate Result: " + string(result))
 	err = json.Unmarshal(result, &resp)
 	if err != nil {
 		return
@@ -297,7 +297,7 @@ func GetIndexRefreshAllByMergeFile() (items []index.BaseFromMysteelChemicalIndex
 	}
 
 	var resp IndexesResponse
-	global.LOG.Info(" GetIndexRefreshAllByMergeFile Result: " + string(result))
+	global.LOG.Debug(" GetIndexRefreshAllByMergeFile Result: " + string(result))
 	err = json.Unmarshal(result, &resp)
 	if err != nil {
 		return
@@ -322,7 +322,7 @@ func GetIndexRefreshMethanolByMergeFile() (items []index.BaseFromMysteelChemical
 	}
 
 	var resp IndexesResponse
-	global.LOG.Info(" GetIndexRefreshMethanolByMergeFile Result: " + string(result))
+	global.LOG.Debug(" GetIndexRefreshMethanolByMergeFile Result: " + string(result))
 	err = json.Unmarshal(result, &resp)
 	if err != nil {
 		return
@@ -347,7 +347,7 @@ func GetIndexRefreshMethanolByTimely() (items []index.BaseFromMysteelChemicalInd
 	}
 
 	var resp IndexesResponse
-	global.LOG.Info(" GetIndexRefreshMethanolByTimely Result: " + string(result))
+	global.LOG.Debug(" GetIndexRefreshMethanolByTimely Result: " + string(result))
 	err = json.Unmarshal(result, &resp)
 	if err != nil {
 		return
@@ -383,7 +383,7 @@ func GetIndexByFrequency(frequency string) (items []index.BaseFromMysteelChemica
 	}
 
 	var resp IndexesResponse
-	global.LOG.Info(" GetIndexByFrequency Result: " + string(result))
+	global.LOG.Debug(" GetIndexByFrequency Result: " + string(result))
 	err = json.Unmarshal(result, &resp)
 	if err != nil {
 		return
@@ -423,7 +423,7 @@ func GetIndexRefreshWeek(nowWeekZn, startTime, endTime string) (items []index.Ba
 	}
 
 	var resp IndexesResponse
-	global.LOG.Info(" GetIndexRefreshWeek Result: " + string(result))
+	global.LOG.Debug(" GetIndexRefreshWeek Result: " + string(result))
 	err = json.Unmarshal(result, &resp)
 	if err != nil {
 		return
@@ -462,7 +462,7 @@ func MultiUpdateIndex(req []UpdateIndexReq) (err error) {
 	}
 
 	var resp BaseResponse
-	global.LOG.Info(" MultiUpdateIndex Result: " + string(result))
+	global.LOG.Debug(" MultiUpdateIndex Result: " + string(result))
 	err = json.Unmarshal(result, &resp)
 	if err != nil {
 		return
@@ -512,7 +512,7 @@ func RefreshCheckByDay(req RefreshCheckByDayReq) (data RefreshCheckByDayData, er
 	}
 
 	var resp RefreshCheckByDayResponse
-	global.LOG.Info(" RefreshCheckByDay Result: " + string(result))
+	global.LOG.Debug(" RefreshCheckByDay Result: " + string(result))
 	err = json.Unmarshal(result, &resp)
 	if err != nil {
 		return