Merge branch 'feature/eta2.5.9_api_stat' into debug

controllers/edb_info_stat.go

@@ -3,6 +3,7 @@ package controllers
 import (
 	"eta/eta_index_lib/models"
 	"eta/eta_index_lib/services"
+	"eta/eta_index_lib/utils"
 )
 
 type EdbInfoStatController struct {
@@ -20,8 +21,21 @@ func (this *EdbInfoStatController) SetEdbSourceStat() {
 		this.Data["json"] = br
 		this.ServeJSON()
 	}()
-	// 钢联终端统计汇总
-	_ = services.SetMysteelChemicalEdbInfoUpdateStat(true)
+
+	conf, err := models.GetBusinessConf()
+	if err != nil {
+		br.Msg = "添加指标失败"
+		br.ErrMsg = "获取业务配置失败,Err:" + err.Error()
+		return
+	}
+	// 判断钢联化工的数据刷新方式
+	if conf["MySteelDataMethod"] == "api" {
+		_ = services.SetEdbInfoUpdateStat(false, utils.DATA_SOURCE_MYSTEEL_CHEMICAL)
+	}else{
+		_ = services.SetMysteelChemicalEdbInfoUpdateStat(true)
+	}
+	_ = services.SetEdbInfoUpdateStat(false, utils.DATA_SOURCE_THS)
+	_ = services.SetEdbInfoUpdateStat(false, utils.DATA_SOURCE_WIND)
 	_ = services.SetEdbSourceStat(true)
 
 	br.Ret = 200

models/data_stat/edb_info_update_log.go

@@ -71,9 +71,9 @@ func GetEdbUpdateLogByCondition(condition string, pars []interface{}) (item []*E
 }
 
 // GetEdbUpdateSourceLogByCreateDate 获取指标终端更新记录
-func GetEdbUpdateSourceLogByCreateDate(source int, startDate, endDate string) (item []*EdbInfoUpdateLog, err error) {
-	sql := ` SELECT * FROM edb_info_update_log WHERE source = ? and create_time >= ? and create_time < ? and is_source_refresh =1 order by data_update_result asc, id desc`
-	err = global.DEFAULT_DB.Raw(sql, source, startDate, endDate).Find(&item).Error
+func GetEdbUpdateSourceLogByCreateDate(source int, startDate, endDate string, isSourceRefresh int) (item []*EdbInfoUpdateLog, err error) {
+	sql := ` SELECT * FROM edb_info_update_log WHERE source = ? and create_time >= ? and create_time < ? and is_source_refresh = ? order by data_update_result asc, id desc`
+	err = global.DEFAULT_DB.Raw(sql, source, startDate, endDate, isSourceRefresh).Find(&item).Error
 
 	return
 }

models/edb_refresh/edb_refresh_mapping.go

@@ -102,7 +102,7 @@ func GetConfigRefreshEdbInfoListBySourceAndSubSource(sourceList, configIdList []
 
 	sql := `SELECT a.*,b.edb_refresh_config_id FROM edb_info a 
  JOIN edb_refresh_mapping b ON a.edb_info_id = b.edb_info_id
-WHERE b.edb_refresh_config_id IN (` + utils.GetOrmInReplace(num) + `) `
+WHERE b.edb_refresh_config_id IN (` + utils.GetOrmInReplace(num) + `) and b.source = ?`
 	pars = append(pars, configIdList)
 
 	sourceNum := len(sourceList)
@@ -114,3 +114,19 @@ WHERE b.edb_refresh_config_id IN (` + utils.GetOrmInReplace(num) + `) `
 
 	return
 }
+
+func GetConfigRefreshEdbInfoListBySource(source int, configIdList []int) (list []*EdbRefreshMapping, err error) {
+	num := len(configIdList)
+	if num <= 0 {
+		return
+	}
+
+	var pars []interface{}
+
+	
+	sql := `select * from edb_refresh_mapping where edb_refresh_config_id in (?) and source = ?`
+	pars = append(pars, configIdList, source)
+	err = global.DEFAULT_DB.Raw(sql, pars...).Find(&list).Error
+
+	return
+}

services/edb_info_stat.go

@@ -3,6 +3,7 @@ package services
 import (
 	"eta/eta_index_lib/models"
 	"eta/eta_index_lib/models/data_stat"
+	"eta/eta_index_lib/models/edb_refresh"
 	"eta/eta_index_lib/services/alarm_msg"
 	"eta/eta_index_lib/utils"
 	"fmt"
@@ -79,7 +80,7 @@ func SetMysteelChemicalEdbInfoUpdateStat(needStat bool) (err error) {
 	todayT, _ := time.ParseInLocation(utils.FormatDate, today, time.Local)
 	nextDay := time.Now().AddDate(0, 0, 1).Format(utils.FormatDate)
 	//查询当日所有钢联指标的终端更新记录
-	updateLogList, err := data_stat.GetEdbUpdateSourceLogByCreateDate(utils.DATA_SOURCE_MYSTEEL_CHEMICAL, today, nextDay)
+	updateLogList, err := data_stat.GetEdbUpdateSourceLogByCreateDate(utils.DATA_SOURCE_MYSTEEL_CHEMICAL, today, nextDay, 1)
 	if err != nil {
 		err = fmt.Errorf("查询上海钢联指标终端更新日志报错，err: %s", err)
 		return
@@ -289,9 +290,9 @@ func SetEdbSourceStat(needStat bool) (err error) {
 		}
 	}()
 	//查询钢联的所有指标信息
-	nowTime := time.Now()
-	today := time.Now().Format(utils.FormatDate)
-	nextDay := time.Now().AddDate(0, 0, 1).Format(utils.FormatDate)
+	nowTime := time.Now().AddDate(0, 0, -1)
+	today := time.Now().AddDate(0, 0, -1).Format(utils.FormatDate)
+	nextDay := time.Now().Format(utils.FormatDate)
 
 	statCond := "  and  create_time >= ? and create_time < ?"
 	var statPars []interface{}
@@ -458,3 +459,360 @@ func EdbDataUpdateRecord(edbInfoId int, hasUpdate bool, updateResult bool, updat
 	}
 	return
 }
+
+// SetEdbInfoUpdateStat 定时统计同花顺的数据源明细表
+func SetEdbInfoUpdateStat(needStat bool, source int) (err error) {
+	defer func() {
+		if err != nil {
+			tips := fmt.Sprintf("同花顺数据刷新-ETA指标刷新统计异常 Err: %s", err.Error())
+			utils.FileLog.Info(tips)
+			alarm_msg.SendAlarmMsg(tips, 3)
+		}
+	}()
+	//查询同花顺的所有在更新的指标信息
+	condition := " and source = ? and no_update=0"
+	var pars []interface{}
+	pars = append(pars, source)
+	edbList, err := models.GetEdbInfoByCondition(condition, pars, 0)
+	if err != nil {
+		err = fmt.Errorf("查询同花顺指标信息出错，err: %s", err)
+		return
+	}
+	nowTime := time.Now()
+	today := time.Now().Format(utils.FormatDate)
+	todayT, _ := time.ParseInLocation(utils.FormatDate, today, time.Local)
+	nextDay := time.Now().AddDate(0, 0, 1).Format(utils.FormatDate)
+	//查询当日所有同花顺指标的终端更新记录
+	updateLogList, err := data_stat.GetEdbUpdateSourceLogByCreateDate(source, today, nextDay, 0)
+	if err != nil {
+		err = fmt.Errorf("查询同花顺指标终端更新日志报错，err: %s", err)
+		return
+	}
+	fmt.Println(len(updateLogList))
+	if !needStat && len(updateLogList) == 0 { //如果不存在变更记录 则不进行汇总
+		return
+	}
+	updateLogMap := make(map[int]*data_stat.EdbInfoUpdateLog)
+	if len(updateLogList) > 0 {
+		for _, v := range updateLogList {
+			if _, ok := updateLogMap[v.EdbInfoId]; !ok {
+				updateLogMap[v.EdbInfoId] = v
+			}
+		}
+	}
+	statCond := "  and source = ? and create_time >= ? and create_time < ?"
+	var statPars []interface{}
+	statPars = append(statPars, source, today, nextDay)
+	//查询当日同花顺所有的刷新记录
+	updateStatList, err := data_stat.GetEdbUpdateStatByCondition(statCond, statPars)
+	if err != nil {
+		err = fmt.Errorf("查询同花顺数据源明细记录统计报错，err: %s", err)
+		return
+	}
+	updateStatMap := make(map[int]*data_stat.EdbInfoUpdateStat)
+	if len(updateStatList) > 0 {
+		for _, v := range updateStatList {
+			updateStatMap[v.EdbInfoId] = v
+		}
+	}
+	//查询自动刷新设置成今天的指标信息
+	// 找出当天的时间
+	needRefreshEdbInfoIdsMap, err := GetTodayNeedRefreshEdbInfoIds(source)
+	if err != nil {
+		err = fmt.Errorf("查询同花顺自动刷新设置成今天的指标信息报错，err: %s", err)
+		return
+	}
+
+	logStat := new(data_stat.EdbInfoUpdateStat)
+	//组装新增数据
+	addList := make([]*data_stat.EdbInfoUpdateStat, 0)
+	modifyList := make([]*data_stat.EdbInfoUpdateStat, 0)
+	if len(edbList) > 0 {
+		for _, v := range edbList {
+			tmp := &data_stat.EdbInfoUpdateStat{
+				EdbInfoId:        v.EdbInfoId,
+				SourceName:       v.SourceName,
+				Source:           v.Source,
+				EdbCode:          v.EdbCode,
+				EdbName:          v.EdbName,
+				EdbNameSource:    v.EdbNameSource,
+				Frequency:        v.Frequency,
+				Unit:             v.Unit,
+				StartDate:        v.StartDate,
+				EndDate:          v.EndDate,
+				SysUserId:        v.SysUserId,
+				SysUserRealName:  v.SysUserRealName,
+				UniqueCode:       v.UniqueCode,
+				EdbCreateTime:    v.CreateTime,
+				EdbModifyTime:    v.ModifyTime,
+				LatestDate:       v.LatestDate,
+				LatestValue:      v.LatestValue,
+				TerminalCode:     v.TerminalCode,
+				DataUpdateTime:   v.DataUpdateTime,
+				ErDataUpdateDate: v.ErDataUpdateDate,
+				ModifyTime:       nowTime,
+			}
+			_, ok := needRefreshEdbInfoIdsMap[v.EdbInfoId]
+			if ok {
+				tmp.NeedRefresh = 1
+			} else {
+				tmp.NeedRefresh = 0
+			}
+
+			// 判断是否当日新增
+			if v.CreateTime.After(todayT) || v.CreateTime == todayT {
+				tmp.IsAdd = 1
+			} else {
+				tmp.IsAdd = 2
+			}
+			if up, ok := updateLogMap[v.EdbInfoId]; ok {
+				tmp.DataUpdateTime = up.DataUpdateTime
+				tmp.ErDataUpdateDate = up.ErDataUpdateDate
+				tmp.DataUpdateResult = up.DataUpdateResult
+				tmp.DataUpdateFailedReason = up.DataUpdateFailedReason
+				tmp.HasRefresh = 1
+				tmp.UpdateResult = up.UpdateResult
+				tmp.UpdateFailedReason = up.UpdateFailedReason
+				tmp.UpdateTime = up.CreateTime.Format(utils.FormatDateTime)
+			} else if tmp.NeedRefresh == 1 {
+				tmp.HasRefresh = 0
+				tmp.DataUpdateResult = 2
+				tmp.DataUpdateFailedReason = "服务异常"
+			}
+
+			// 判断是否需要新增还是更新
+			exist, existOk := updateStatMap[v.EdbInfoId]
+			if existOk {
+				tmp.Id = exist.Id
+				modifyList = append(modifyList, tmp)
+			} else {
+				tmp.CreateTime = nowTime
+				addList = append(addList, tmp)
+			}
+			if len(addList) >= 500 {
+				err = logStat.Add(addList)
+				if err != nil {
+					err = fmt.Errorf("新增同花顺明细记录报错，err: %s", err)
+					return
+				}
+				addList = addList[:0]
+			}
+
+			if len(modifyList) >= 500 {
+				err = data_stat.UpdateEdbUpdateStatMulti(modifyList)
+				if err != nil {
+					err = fmt.Errorf("更新同花顺明细记录报错，err: %s", err)
+					return
+				}
+				modifyList = modifyList[:0]
+			}
+		}
+	}
+
+	//判断当日指标统计数据是否存在，如果存在则更新，不存在则新增
+	if len(addList) > 0 {
+		err = logStat.Add(addList)
+	}
+	if len(modifyList) > 0 {
+		err = data_stat.UpdateEdbUpdateStatMulti(modifyList)
+	}
+	return
+}
+
+func GetTodayNeedRefreshEdbInfoIds(source int) (needRefreshMap map[int]struct{}, err error) {
+	//刷新频率，枚举值：每自然日、每交易日、每周、每旬、每月、每季、每半年、每年
+	refreshFrequencyList := []string{"每自然日", "每交易日", "每周", "每旬", "每月", "每季", "每半年", "每年"}
+	// 获取刷新频率条件
+	now := time.Now()
+	configIds := make([]int, 0)
+	needRefreshMap = make(map[int]struct{})
+	for _, refreshFrequency := range refreshFrequencyList {
+		condition, pars, isHandler := getRefreshFrequencyCondition(now, refreshFrequency)
+		if !isHandler {
+			// 可能是非交易日，所以过滤不处理
+			continue
+		}
+
+		condition += ` AND refresh_frequency = ? AND source = ?`
+		pars = append(pars, refreshFrequency, source)	
+
+		tmpList, tmpErr := edb_refresh.GetListByCondition(condition, pars)
+		if tmpErr != nil {
+			err = tmpErr
+			return
+		}
+		for _, v := range tmpList {
+			configIds = append(configIds, v.Id)
+		}
+	}
+
+	tmpList, tmpErr := edb_refresh.GetConfigRefreshEdbInfoListBySource(source, configIds)
+	if tmpErr != nil {
+		err = tmpErr
+		return
+	}
+	for _, v := range tmpList {	
+		needRefreshMap[v.EdbInfoId] = struct{}{}
+	}
+	
+	return
+}
+
+// getRefreshFrequencyCondition
+// @Description: 根据时间和刷新频率获取条件
+// @author: Roc
+// @datetime 2024-01-09 16:27:11
+// @param now time.Time
+// @param refreshFrequency string
+// @return condition string
+// @return pars []interface{}
+// @return isHandler bool
+func getRefreshFrequencyCondition(now time.Time, refreshFrequency string) (condition string, pars []interface{}, isHandler bool) {
+	isHandler = true
+
+	var dayNum int
+	var isLastDay bool
+
+	//刷新频率，枚举值：每自然日、每交易日、每周、每旬、每月、每季、每半年、每年
+	switch refreshFrequency {
+	case "每自然日":
+		// 自然日不需要额外条件
+		return
+	case "每交易日":
+		// 周六日不处理
+		if now.Weekday() == time.Saturday || now.Weekday() == time.Sunday {
+			isHandler = false
+		}
+		return
+	case "每周":
+		currWeekDay := now.Weekday()
+		if currWeekDay == time.Sunday {
+			currWeekDay = 7
+			isLastDay = true
+		}
+		dayNum = int(currWeekDay)
+	case "每旬":
+		currDay := now.Day()
+		if currDay <= 10 {
+			dayNum = currDay
+			// 如果是这旬的最后一天
+			if currDay == 10 {
+				isLastDay = true
+			}
+		} else if currDay <= 20 {
+			dayNum = currDay - 10
+			// 如果是这旬的最后一天
+			if currDay == 20 {
+				isLastDay = true
+			}
+		} else {
+			dayNum = currDay - 20
+
+			// 当月的最后一天
+			monthLastDay := time.Date(now.Year(), now.Month()+1, 1, 0, 0, 0, 0, time.Local).AddDate(0, 0, -1)
+			// 如果是这旬的最后一天
+			if currDay == monthLastDay.Day() {
+				isLastDay = true
+			}
+		}
+	case "每月":
+		// 当前日期
+		currDay := now.Day()
+		dayNum = currDay
+
+		// 当期的最后一天
+		monthLastDay := time.Date(now.Year(), now.Month()+1, 1, 0, 0, 0, 0, time.Local).AddDate(0, 0, -1)
+
+		// 如果是这期的最后一天
+		if currDay == monthLastDay.Day() {
+			isLastDay = true
+		}
+	case "每季":
+		// 当期的第一天  ;  当期的最后一天
+		var startDay, endDay time.Time
+		currMonth := now.Month()
+		currDay := now.Day()
+		if currMonth <= 3 {
+			// 当季的第一天
+			startDay = time.Date(now.Year(), 1, 1, 0, 0, 0, 0, time.Local)
+			// 当季的最后一天
+			endDay = time.Date(now.Year(), 4, 1, 0, 0, 0, 0, time.Local).AddDate(0, 0, -1)
+		} else if currMonth <= 6 {
+			// 当期的第一天
+			startDay = time.Date(now.Year(), 4, 1, 0, 0, 0, 0, time.Local)
+			// 当期的最后一天
+			endDay = time.Date(now.Year(), 7, 1, 0, 0, 0, 0, time.Local).AddDate(0, 0, -1)
+		} else if currMonth <= 9 {
+			// 当期的第一天
+			startDay = time.Date(now.Year(), 7, 1, 0, 0, 0, 0, time.Local)
+			// 当期的最后一天
+			endDay = time.Date(now.Year(), 10, 1, 0, 0, 0, 0, time.Local).AddDate(0, 0, -1)
+		} else {
+			// 当期的第一天
+			startDay = time.Date(now.Year(), 10, 1, 0, 0, 0, 0, time.Local)
+			// 当期的最后一天
+			endDay = time.Date(now.Year()+1, 1, 1, 0, 0, 0, 0, time.Local).AddDate(0, 0, -1)
+		}
+
+		// 计算这期的第一天和当日的天数
+		dayNum = utils.GetTimeSubDay(startDay, now) + 1
+
+		// 如果是这期的最后一天
+		if currMonth == endDay.Month() && currDay == endDay.Day() {
+			isLastDay = true
+		}
+	case "每半年":
+		// 当期的第一天  ;  当期的最后一天
+		var startDay, endDay time.Time
+		currMonth := now.Month()
+		currDay := now.Day()
+		if currMonth <= 6 {
+			// 当期的第一天
+			startDay = time.Date(now.Year(), 1, 1, 0, 0, 0, 0, time.Local)
+			// 当期的最后一天
+			endDay = time.Date(now.Year(), 7, 1, 0, 0, 0, 0, time.Local).AddDate(0, 0, -1)
+		} else {
+			// 当期的第一天
+			startDay = time.Date(now.Year(), 7, 1, 0, 0, 0, 0, time.Local)
+			// 当期的最后一天
+			endDay = time.Date(now.Year()+1, 1, 1, 0, 0, 0, 0, time.Local).AddDate(0, 0, -1)
+		}
+
+		// 计算这期的第一天和当日的天数
+		dayNum = utils.GetTimeSubDay(startDay, now) + 1
+
+		// 如果是这期的最后一天
+		if currMonth == endDay.Month() && currDay == endDay.Day() {
+			isLastDay = true
+		}
+	case "每年":
+		currMonth := now.Month()
+		currDay := now.Day()
+
+		// 当期的第一天
+		startDay := time.Date(now.Year(), 1, 1, 0, 0, 0, 0, time.Local)
+		// 当期的最后一天
+		endDay := time.Date(now.Year()+1, 1, 1, 0, 0, 0, 0, time.Local).AddDate(0, 0, -1)
+
+		// 计算这期的第一天和当日的天数
+		dayNum = utils.GetTimeSubDay(startDay, now) + 1
+
+		// 如果是这期的最后一天
+		if currMonth == endDay.Month() && currDay == endDay.Day() {
+			isLastDay = true
+		}
+
+	}
+
+	// 如果是这期的最后一天，那么就是判断refresh_frequency_day是否配置为0，或者配置的天数大于这期的最大天数
+	if isLastDay {
+		condition += ` AND ( refresh_frequency_day = ? OR refresh_frequency_day >= ? )`
+		pars = append(pars, 0, dayNum)
+	} else {
+		// 如果不是这期的最后一天，那么就是判断refresh_frequency_day是否等于配置的天数
+		condition += ` AND refresh_frequency_day = ?  `
+		pars = append(pars, dayNum)
+	}
+
+	return
+}