Browse Source

Merge branch 'feature/eta2.5.9_api_stat' of eta_server/eta_index_lib into master

xyxie 1 week ago
parent
commit
c882080416

+ 14 - 3
controllers/base_from_ths.go

@@ -43,7 +43,7 @@ func (this *ThsController) Add() {
 	cacheKey = utils.CACHE_EDB_DATA_ADD + strconv.Itoa(source) + "_" + req.EdbCode
 	if !utils.Rc.IsExist(cacheKey) {
 		utils.Rc.SetNX(cacheKey, 1, 1*time.Minute)
-		dataItem, err := services.GetEdbDataFromThs(req.EdbCode, utils.BASE_START_DATE, utils.BASE_END_DATE, "")
+		dataItem, _, err := services.GetEdbDataFromThs(req.EdbCode, utils.BASE_START_DATE, utils.BASE_END_DATE, "")
 		if err != nil {
 			br.Msg = "获取指标信息失败!"
 			br.ErrMsg = "获取指标信息失败 GetEdbDataFromThs,Err:" + err.Error()
@@ -111,16 +111,27 @@ func (this *ThsController) Refresh() {
 	}
 
 	utils.Rc.SetNX(cacheKey, 1, 1*time.Minute)
+	var hasUpdate bool
+	var updateResult bool
+	var realApiErrMsg string
+	var dataItem models.EdbDataFromThs
 	defer func() {
 		utils.Rc.Delete(cacheKey)
+		var updateFailedReason string
+		if err == nil {
+			updateResult = true
+		} else {
+			updateFailedReason = err.Error()
+		}
+		_ = services.EdbDataUpdateRecord(edbInfo.EdbInfoId, hasUpdate, updateResult, updateFailedReason, realApiErrMsg)
 	}()
-	dataItem, err := services.GetEdbDataFromThs(req.EdbCode, utils.GetEdbRefreshStartDate(req.StartDate), utils.BASE_END_DATE, edbInfo.TerminalCode)
+	dataItem, realApiErrMsg, err = services.GetEdbDataFromThs(req.EdbCode, utils.GetEdbRefreshStartDate(req.StartDate), utils.BASE_END_DATE, edbInfo.TerminalCode)
 	if err != nil {
 		br.Msg = "获取指标信息失败!"
 		br.ErrMsg = "获取指标信息失败 GetEdbDataFromThs,Err:" + err.Error()
 		return
 	}
-	err = models.RefreshEdbDataFromThs(req.EdbInfoId, req.EdbCode, utils.GetEdbRefreshStartDate(req.StartDate), dataItem)
+	hasUpdate, err = models.RefreshEdbDataFromThs(req.EdbInfoId, req.EdbCode, utils.GetEdbRefreshStartDate(req.StartDate), dataItem)
 	if err != nil && !utils.IsErrNoRow(err) {
 		br.Msg = "刷新指标信息失败!"
 		br.ErrMsg = "刷新指标信息失败 RefreshEdbDataFromThs,Err:" + err.Error()

+ 14 - 3
controllers/base_from_ths_ds.go

@@ -47,7 +47,7 @@ func (this *ThsDsController) Add() {
 	cacheKey = utils.CACHE_EDB_DATA_ADD + strconv.Itoa(source) + "_" + req.StockCode + req.EdbCode
 	if !utils.Rc.IsExist(cacheKey) {
 		utils.Rc.SetNX(cacheKey, 1, 1*time.Minute)
-		dataItem, err := services.GetEdbDataFromThsDs(req.StockCode, req.EdbCode, utils.BASE_START_DATE, endDate, "", req.ExtraPars)
+		dataItem, _, err := services.GetEdbDataFromThsDs(req.StockCode, req.EdbCode, utils.BASE_START_DATE, endDate, "", req.ExtraPars)
 		if err != nil {
 			br.Msg = "获取指标信息失败!"
 			br.ErrMsg = "获取指标信息失败 GetEdbDataFromThsDs,Err:" + err.Error()
@@ -116,8 +116,19 @@ func (this *ThsDsController) Refresh() {
 	}
 
 	utils.Rc.SetNX(cacheKey, 1, 1*time.Minute)
+	var hasUpdate bool
+	var updateResult bool
+	var realApiErrMsg string
+	var dataItem models.EdbDataFromThs
 	defer func() {
 		utils.Rc.Delete(cacheKey)
+		var updateFailedReason string
+		if err == nil {
+			updateResult = true
+		} else {
+			updateFailedReason = err.Error()
+		}
+		_ = services.EdbDataUpdateRecord(edbInfo.EdbInfoId, hasUpdate, updateResult, updateFailedReason, realApiErrMsg)
 	}()
 
 	var extra models.EdbInfoExtra
@@ -129,13 +140,13 @@ func (this *ThsDsController) Refresh() {
 		}
 	}
 
-	dataItem, err := services.GetEdbDataFromThsDs(edbInfo.StockCode, edbInfo.IndicatorCode, utils.GetEdbRefreshStartDate(req.StartDate), endDate, edbInfo.TerminalCode, extra.ApiExtraPars)
+	dataItem, realApiErrMsg, err = services.GetEdbDataFromThsDs(edbInfo.StockCode, edbInfo.IndicatorCode, utils.GetEdbRefreshStartDate(req.StartDate), endDate, edbInfo.TerminalCode, extra.ApiExtraPars)
 	if err != nil {
 		br.Msg = "获取指标信息失败!"
 		br.ErrMsg = "获取指标信息失败 GetEdbDataFromThsDs,Err:" + err.Error()
 		return
 	}
-	err = models.RefreshEdbDataFromThsDs(req.EdbInfoId, req.EdbCode, utils.GetEdbRefreshStartDate(req.StartDate), dataItem)
+	hasUpdate, err = models.RefreshEdbDataFromThsDs(req.EdbInfoId, req.EdbCode, utils.GetEdbRefreshStartDate(req.StartDate), dataItem)
 	if err != nil && err.Error() != utils.ErrNoRow() {
 		br.Msg = "刷新指标信息失败!"
 		br.ErrMsg = "刷新指标信息失败 RefreshEdbDataFromThsDs,Err:" + err.Error()

+ 23 - 8
controllers/base_from_wind.go

@@ -113,7 +113,7 @@ func (this *WindController) Add() {
 		_ = utils.Rc.Put(terminalCodeCacheKey, terminalCode, utils.GetTodayLastSecond())
 
 		//windUrl = utils.Hz_Wind_Data_Url_LIST[1].Url
-		dataItem, errCode, err := services.GetEdbDataFromWindUrl(windUrl, req.EdbCode, utils.BASE_START_DATE, endDate)
+		dataItem, errCode, _, err := services.GetEdbDataFromWindUrl(windUrl, req.EdbCode, utils.BASE_START_DATE, endDate)
 		//dataItem, errCode, err := services.GetEdbDataFromWind(req.EdbCode, utils.BASE_START_DATE, endDate)
 		if errCode == 421 { //指标超限
 			br.Ret = 421
@@ -185,11 +185,6 @@ func (this *WindController) Refresh() {
 		return
 	}
 
-	utils.Rc.SetNX(cacheKey, 1, 1*time.Minute)
-	defer func() {
-		utils.Rc.Delete(cacheKey)
-	}()
-
 	//获取指标信息
 	edbInfo, err := models.GetEdbInfoById(req.EdbInfoId)
 	if err != nil {
@@ -198,6 +193,23 @@ func (this *WindController) Refresh() {
 		return
 	}
 
+	utils.Rc.SetNX(cacheKey, 1, 1*time.Minute)
+	var hasUpdate bool
+	var updateResult bool
+	var realApiErrMsg string
+	var dataItem *models.EdbDataFromWind
+	defer func() {
+		utils.Rc.Delete(cacheKey)
+		var updateFailedReason string
+		if err == nil {
+			updateResult = true
+		} else {
+			updateFailedReason = err.Error()
+		}
+		_ = services.EdbDataUpdateRecord(edbInfo.EdbInfoId, hasUpdate, updateResult, updateFailedReason, realApiErrMsg)
+	}()
+
+
 	//期货数据,就默认到今天,特殊处理下
 	endDate := utils.BASE_END_DATE
 	if strings.Contains(req.EdbCode, ".") {
@@ -209,10 +221,12 @@ func (this *WindController) Refresh() {
 		br.Msg = "终端未配置"
 		return
 	}
-	dataItem, errCode, err := services.GetEdbDataFromWindUrl(terminal.ServerUrl, req.EdbCode, utils.GetEdbRefreshStartDate(req.StartDate), utils.GetEdbRefreshEndDate(endDate))
+	var errCode int
+	dataItem, errCode, realApiErrMsg, err = services.GetEdbDataFromWindUrl(terminal.ServerUrl, req.EdbCode, utils.GetEdbRefreshStartDate(req.StartDate), utils.GetEdbRefreshEndDate(endDate))
 	if errCode == 421 { //指标超限
 		br.Ret = 421
 		br.Msg = "指标超限!"
+		err = errors.New(realApiErrMsg)
 		return
 	}
 	// 忽略掉指标下架的错误, 并更新指标为停止更新
@@ -224,6 +238,7 @@ func (this *WindController) Refresh() {
 			br.ErrMsg = "更新wind指标停更失败, Err: " + e.Error()
 			return
 		}
+		err = errors.New(realApiErrMsg)
 		br.Ret = 200
 		br.Msg = "操作成功"
 		return
@@ -233,7 +248,7 @@ func (this *WindController) Refresh() {
 		br.ErrMsg = "获取指标信息失败 GetEdbDataFromWind,Err:" + err.Error()
 		return
 	}
-	err = models.RefreshEdbDataFromWind(req.EdbInfoId, req.EdbCode, utils.GetEdbRefreshStartDate(req.StartDate), dataItem)
+	hasUpdate, err = models.RefreshEdbDataFromWind(req.EdbInfoId, req.EdbCode, utils.GetEdbRefreshStartDate(req.StartDate), dataItem)
 	if err != nil && err.Error() != utils.ErrNoRow() {
 		br.Msg = "刷新指标信息失败!"
 		br.ErrMsg = "刷新指标信息失败 RefreshEdbDataFromWind,Err:" + err.Error()

+ 23 - 9
controllers/base_from_wind_wsd.go

@@ -2,6 +2,7 @@ package controllers
 
 import (
 	"encoding/json"
+	"errors"
 	"eta/eta_index_lib/logic"
 	"eta/eta_index_lib/models"
 	"eta/eta_index_lib/services"
@@ -95,7 +96,7 @@ func (this *WindWsdController) Add() {
 		_ = utils.Rc.Put(terminalCodeCacheKey, terminalCode, utils.GetTodayLastSecond())
 
 		//windUrl = utils.Hz_Wind_Data_Url_LIST[1].Url
-		dataItem, errCode, err := services.GetEdbDataFromWindUrlWsd(windUrl, req.StockCode, req.EdbCode, utils.BASE_START_DATE, endDate)
+		dataItem, errCode, _, err := services.GetEdbDataFromWindUrlWsd(windUrl, req.StockCode, req.EdbCode, utils.BASE_START_DATE, endDate)
 		if errCode == 421 { //指标超限
 			br.Ret = 421
 			br.Msg = "指标超限!"
@@ -166,12 +167,6 @@ func (this *WindWsdController) Refresh() {
 		br.Msg = "系统处理中,请稍后重试"
 		return
 	}
-
-	utils.Rc.SetNX(cacheKey, 1, 1*time.Minute)
-	defer func() {
-		utils.Rc.Delete(cacheKey)
-	}()
-
 	//获取指标信息
 	edbInfo, err := models.GetEdbInfoById(req.EdbInfoId)
 	if err != nil {
@@ -180,6 +175,22 @@ func (this *WindWsdController) Refresh() {
 		return
 	}
 
+	utils.Rc.SetNX(cacheKey, 1, 1*time.Minute)
+	var hasUpdate bool
+	var updateResult bool
+	var realApiErrMsg string
+	var dataItem map[string]map[string]interface{}
+	defer func() {
+		utils.Rc.Delete(cacheKey)
+		var updateFailedReason string
+		if err == nil {
+			updateResult = true
+		} else {
+			updateFailedReason = err.Error()
+		}
+		_ = services.EdbDataUpdateRecord(edbInfo.EdbInfoId, hasUpdate, updateResult, updateFailedReason, realApiErrMsg)
+	}()
+
 	//期货数据,就默认到今天,特殊处理下
 	endDate := utils.BASE_END_DATE
 	if strings.Contains(req.EdbCode, ".") {
@@ -191,10 +202,12 @@ func (this *WindWsdController) Refresh() {
 		br.Msg = "终端未配置"
 		return
 	}
-	dataItem, errCode, err := services.GetEdbDataFromWindUrlWsd(terminal.ServerUrl, edbInfo.StockCode, edbInfo.IndicatorCode, utils.GetEdbRefreshStartDate(req.StartDate), utils.GetEdbRefreshEndDate(endDate))
+	var errCode int
+	dataItem, errCode, realApiErrMsg, err = services.GetEdbDataFromWindUrlWsd(terminal.ServerUrl, edbInfo.StockCode, edbInfo.IndicatorCode, utils.GetEdbRefreshStartDate(req.StartDate), utils.GetEdbRefreshEndDate(endDate))
 	if errCode == 421 { //指标超限
 		br.Ret = 421
 		br.Msg = "指标超限!"
+		err = errors.New(realApiErrMsg)
 		return
 	}
 	// 忽略掉指标下架的错误, 并更新指标为停止更新
@@ -206,6 +219,7 @@ func (this *WindWsdController) Refresh() {
 			br.ErrMsg = "更新wind指标停更失败, Err: " + e.Error()
 			return
 		}
+		err = errors.New(realApiErrMsg)
 		br.Ret = 200
 		br.Msg = "操作成功"
 		return
@@ -215,7 +229,7 @@ func (this *WindWsdController) Refresh() {
 		br.ErrMsg = "获取指标信息失败 GetEdbDataFromWind,Err:" + err.Error()
 		return
 	}
-	err = models.RefreshEdbDataFromWindWsd(req.EdbInfoId, edbInfo.EdbCode, utils.GetEdbRefreshStartDate(req.StartDate), dataItem)
+	hasUpdate, err = models.RefreshEdbDataFromWindWsd(req.EdbInfoId, edbInfo.EdbCode, utils.GetEdbRefreshStartDate(req.StartDate), dataItem)
 	if err != nil && err.Error() != utils.ErrNoRow() {
 		br.Msg = "刷新指标信息失败!"
 		br.ErrMsg = "刷新指标信息失败 RefreshEdbDataFromWind,Err:" + err.Error()

+ 16 - 2
controllers/edb_info_stat.go

@@ -3,6 +3,7 @@ package controllers
 import (
 	"eta/eta_index_lib/models"
 	"eta/eta_index_lib/services"
+	"eta/eta_index_lib/utils"
 )
 
 type EdbInfoStatController struct {
@@ -20,8 +21,21 @@ func (this *EdbInfoStatController) SetEdbSourceStat() {
 		this.Data["json"] = br
 		this.ServeJSON()
 	}()
-	// 钢联终端统计汇总
-	_ = services.SetMysteelChemicalEdbInfoUpdateStat(true)
+
+	conf, err := models.GetBusinessConf()
+	if err != nil {
+		br.Msg = "添加指标失败"
+		br.ErrMsg = "获取业务配置失败,Err:" + err.Error()
+		return
+	}
+	// 判断钢联化工的数据刷新方式
+	if conf["MySteelDataMethod"] == "api" {
+		_ = services.SetEdbInfoUpdateStat(false, utils.DATA_SOURCE_MYSTEEL_CHEMICAL)
+	}else{
+		_ = services.SetMysteelChemicalEdbInfoUpdateStat(true)
+	}
+	_ = services.SetEdbInfoUpdateStat(false, utils.DATA_SOURCE_THS)
+	_ = services.SetEdbInfoUpdateStat(false, utils.DATA_SOURCE_WIND)
 	_ = services.SetEdbSourceStat(true)
 
 	br.Ret = 200

+ 7 - 5
models/base_from_ths.go

@@ -63,7 +63,7 @@ func AddEdbDataFromThs(edbCode string, item EdbDataFromThs) (err error) {
 }
 
 // 刷新同花顺指标数据
-func RefreshEdbDataFromThs(edbInfoId int, edbCode, startDate string, item EdbDataFromThs) (err error) {
+func RefreshEdbDataFromThs(edbInfoId int, edbCode, startDate string, item EdbDataFromThs) (hasUpdate bool, err error) {
 	//o := orm.NewOrm()
 	source := utils.DATA_SOURCE_THS
 	subSource := utils.DATA_SUB_SOURCE_EDB
@@ -95,7 +95,7 @@ func RefreshEdbDataFromThs(edbInfoId int, edbCode, startDate string, item EdbDat
 
 		existList, err := GetEdbDataByCondition(source, subSource, condition, pars)
 		if err != nil {
-			return err
+			return false, err
 		}
 
 		existMap := make(map[string]*EdbInfoSearchData)
@@ -117,7 +117,7 @@ func RefreshEdbDataFromThs(edbInfoId int, edbCode, startDate string, item EdbDat
 
 			dataTime, err := time.ParseInLocation(utils.FormatDate, eDate, time.Local)
 			if err != nil {
-				return err
+				return false, err
 			}
 			if findItem, ok := existMap[eDate]; !ok {
 				if _, addOk := addMap[eDate]; !addOk {
@@ -131,8 +131,9 @@ func RefreshEdbDataFromThs(edbInfoId int, edbCode, startDate string, item EdbDat
 				if findItem != nil && utils.SubFloatToString(findItem.Value, 30) != utils.SubFloatToString(sValue, 30) {
 					err = ModifyEdbDataById(source, subSource, findItem.EdbDataId, sValueStr)
 					if err != nil {
-						return err
+						return false, err
 					}
+					hasUpdate = true
 				}
 			}
 
@@ -155,8 +156,9 @@ func RefreshEdbDataFromThs(edbInfoId int, edbCode, startDate string, item EdbDat
 			//_, err = o.Raw(addSql).Exec()
 			err = global.DEFAULT_DB.Exec(addSql).Error
 			if err != nil {
-				return err
+				return false, err
 			}
+			hasUpdate = true
 		}
 	}
 	return

+ 7 - 5
models/base_from_ths_ds.go

@@ -73,7 +73,7 @@ func AddEdbDataFromThsDs(stockCode, edbCode string, item EdbDataFromThs) (err er
 }
 
 // 刷新同花顺指标数据
-func RefreshEdbDataFromThsDs(edbInfoId int, edbCode, startDate string, item EdbDataFromThs) (err error) {
+func RefreshEdbDataFromThsDs(edbInfoId int, edbCode, startDate string, item EdbDataFromThs) (hasUpdate bool, err error) {
 	//o := orm.NewOrm()
 	source := utils.DATA_SOURCE_THS
 	subSource := utils.DATA_SUB_SOURCE_DATE
@@ -105,7 +105,7 @@ func RefreshEdbDataFromThsDs(edbInfoId int, edbCode, startDate string, item EdbD
 
 		existList, err := GetEdbDataByCondition(source, subSource, condition, pars)
 		if err != nil {
-			return err
+			return false, err
 		}
 
 		existMap := make(map[string]*EdbInfoSearchData)
@@ -127,7 +127,7 @@ func RefreshEdbDataFromThsDs(edbInfoId int, edbCode, startDate string, item EdbD
 
 			dataTime, err := time.ParseInLocation(utils.FormatDate, eDate, time.Local)
 			if err != nil {
-				return err
+				return false, err
 			}
 			if findItem, ok := existMap[eDate]; !ok {
 				if _, addOk := addMap[eDate]; !addOk {
@@ -141,8 +141,9 @@ func RefreshEdbDataFromThsDs(edbInfoId int, edbCode, startDate string, item EdbD
 				if findItem != nil && utils.SubFloatToString(findItem.Value, 30) != utils.SubFloatToString(sValue, 30) {
 					err = ModifyEdbDataById(source, subSource, findItem.EdbDataId, sValueStr)
 					if err != nil {
-						return err
+						return false, err
 					}
+					hasUpdate = true
 				}
 			}
 
@@ -165,8 +166,9 @@ func RefreshEdbDataFromThsDs(edbInfoId int, edbCode, startDate string, item EdbD
 			//_, err = o.Raw(addSql).Exec()
 			err = global.DEFAULT_DB.Exec(addSql).Error
 			if err != nil {
-				return err
+				return false, err
 			}
+			hasUpdate = true
 		}
 	}
 	return

+ 5 - 3
models/base_from_wind.go

@@ -50,7 +50,7 @@ func AddEdbDataFromWind(edbCode string, item *EdbDataFromWind) (err error) {
 }
 
 // RefreshEdbDataFromWind 刷新wind指标数据
-func RefreshEdbDataFromWind(edbInfoId int, edbCode, startDate string, item *EdbDataFromWind) (err error) {
+func RefreshEdbDataFromWind(edbInfoId int, edbCode, startDate string, item *EdbDataFromWind) (hasUpdate bool, err error) {
 	//o := orm.NewOrm()
 	source := utils.DATA_SOURCE_WIND
 	subSource := utils.DATA_SUB_SOURCE_EDB
@@ -122,8 +122,9 @@ func RefreshEdbDataFromWind(edbInfoId int, edbCode, startDate string, item *EdbD
 			if findItem != nil && utils.SubFloatToString(findItem.Value, 30) != saveVal {
 				err = ModifyEdbDataById(source, subSource, findItem.EdbDataId, saveVal)
 				if err != nil {
-					return err
+					return false, err
 				}
+				hasUpdate = true
 			}
 		}
 
@@ -147,8 +148,9 @@ func RefreshEdbDataFromWind(edbInfoId int, edbCode, startDate string, item *EdbD
 		err = global.DEFAULT_DB.Exec(addSql).Error
 		if err != nil {
 			fmt.Println("RefreshEdbDataFromWind add Err", err.Error())
-			return
+			return false, err
 		}
+		hasUpdate = true
 	}
 
 	return

+ 7 - 5
models/base_from_wind_wsd.go

@@ -105,7 +105,7 @@ func AddEdbDataFromWindWsd(stockCode string, item map[string]map[string]interfac
 }
 
 // RefreshEdbDataFromWind 刷新wind指标数据
-func RefreshEdbDataFromWindWsd(edbInfoId int, edbCode, startDate string, item map[string]map[string]interface{}) (err error) {
+func RefreshEdbDataFromWindWsd(edbInfoId int, edbCode, startDate string, item map[string]map[string]interface{}) (hasUpdate bool, err error) {
 	//o := orm.NewOrm()
 	source := utils.DATA_SOURCE_WIND
 	subSource := utils.DATA_SUB_SOURCE_DATE
@@ -157,7 +157,7 @@ func RefreshEdbDataFromWindWsd(edbInfoId int, edbCode, startDate string, item ma
 			wkInt, err := strconv.ParseInt(wk, 10, 64)
 			if err != nil {
 				fmt.Println("ParseInt Err:" + err.Error() + ";wk:" + wk)
-				return err
+				return false, err
 			}
 
 			vk = strings.ToLower(vk)
@@ -171,7 +171,7 @@ func RefreshEdbDataFromWindWsd(edbInfoId int, edbCode, startDate string, item ma
 
 			dataTime, err := time.ParseInLocation(utils.FormatDate, dateTime, time.Local)
 			if err != nil {
-				return err
+				return false, err
 			}
 			timestamp := dataTime.UnixNano() / 1e6
 			timeStr := fmt.Sprintf("%d", timestamp)
@@ -204,8 +204,9 @@ func RefreshEdbDataFromWindWsd(edbInfoId int, edbCode, startDate string, item ma
 				if findItem != nil && utils.SubFloatToString(findItem.Value, 30) != saveVal {
 					err = ModifyEdbDataById(source, subSource, findItem.EdbDataId, saveVal)
 					if err != nil {
-						return err
+						return false, err
 					}
+					hasUpdate = true
 				}
 			}
 
@@ -230,8 +231,9 @@ func RefreshEdbDataFromWindWsd(edbInfoId int, edbCode, startDate string, item ma
 		err = global.DEFAULT_DB.Exec(addSql).Error
 		if err != nil {
 			fmt.Println("RefreshEdbDataFromWind add Err", err.Error())
-			return
+			return false, err
 		}
+		hasUpdate = true
 	}
 
 	return

+ 3 - 3
models/data_stat/edb_info_update_log.go

@@ -71,9 +71,9 @@ func GetEdbUpdateLogByCondition(condition string, pars []interface{}) (item []*E
 }
 
 // GetEdbUpdateSourceLogByCreateDate 获取指标终端更新记录
-func GetEdbUpdateSourceLogByCreateDate(source int, startDate, endDate string) (item []*EdbInfoUpdateLog, err error) {
-	sql := ` SELECT * FROM edb_info_update_log WHERE source = ? and create_time >= ? and create_time < ? and is_source_refresh =1 order by data_update_result asc, id desc`
-	err = global.DEFAULT_DB.Raw(sql, source, startDate, endDate).Find(&item).Error
+func GetEdbUpdateSourceLogByCreateDate(source int, startDate, endDate string, isSourceRefresh int) (item []*EdbInfoUpdateLog, err error) {
+	sql := ` SELECT * FROM edb_info_update_log WHERE source = ? and create_time >= ? and create_time < ? and is_source_refresh = ? order by data_update_result asc, id desc`
+	err = global.DEFAULT_DB.Raw(sql, source, startDate, endDate, isSourceRefresh).Find(&item).Error
 
 	return
 }

+ 17 - 1
models/edb_refresh/edb_refresh_mapping.go

@@ -102,7 +102,7 @@ func GetConfigRefreshEdbInfoListBySourceAndSubSource(sourceList, configIdList []
 
 	sql := `SELECT a.*,b.edb_refresh_config_id FROM edb_info a 
  JOIN edb_refresh_mapping b ON a.edb_info_id = b.edb_info_id
-WHERE b.edb_refresh_config_id IN (` + utils.GetOrmInReplace(num) + `) `
+WHERE b.edb_refresh_config_id IN (` + utils.GetOrmInReplace(num) + `) and b.source = ?`
 	pars = append(pars, configIdList)
 
 	sourceNum := len(sourceList)
@@ -114,3 +114,19 @@ WHERE b.edb_refresh_config_id IN (` + utils.GetOrmInReplace(num) + `) `
 
 	return
 }
+
+func GetConfigRefreshEdbInfoListBySource(source int, configIdList []int) (list []*EdbRefreshMapping, err error) {
+	num := len(configIdList)
+	if num <= 0 {
+		return
+	}
+
+	var pars []interface{}
+
+	
+	sql := `select * from edb_refresh_mapping where edb_refresh_config_id in (?) and source = ?`
+	pars = append(pars, configIdList, source)
+	err = global.DEFAULT_DB.Raw(sql, pars...).Find(&list).Error
+
+	return
+}

+ 24 - 1
services/base_from_mysteel_chemical.go

@@ -682,6 +682,25 @@ func RefreshDataFromMysteelChemical(edbCode, startDate, endDate string) (err err
 
 	// 如果配置了api的token, 那么就走api接口
 	if utils.MysteelChemicalApiToken != "" {
+		var apiErrMsg string
+		var updateResult bool
+		hasUpdateMap := make(map[string]bool)
+		defer func() {
+			// 查询指标编码
+			edbList, tErr := models.GetEdbInfoByEdbCodeList(utils.DATA_SOURCE_MYSTEEL_CHEMICAL, []string{edbCode})
+			if tErr != nil {
+				return
+			}
+			for _, v := range edbList {
+				hasUpdate, ok := hasUpdateMap[edbCode]
+				if !ok {
+					hasUpdate = false
+				}
+				EdbDataUpdateRecord(v.EdbInfoId, hasUpdate, updateResult, err.Error(), apiErrMsg)
+			}
+			
+		}()
+	
 		resp, er := GetEdbDataFromMySteelChemical([]string{edbCode}, startDate, endDate, "desc")
 		if er != nil {
 			err = er
@@ -689,8 +708,10 @@ func RefreshDataFromMysteelChemical(edbCode, startDate, endDate string) (err err
 		}
 		if !resp.Success {
 			err = errors.New(resp.Message)
+			apiErrMsg = fmt.Sprintf("请求失败, ErrCode: %s, ErrMsg: %s", resp.Code, resp.Message)
 			return
 		}
+		updateResult = true
 
 		dataObj := new(models.BaseFromMysteelChemicalData)
 		exitDataList, er := dataObj.GetIndexDataList(edbCode)
@@ -708,7 +729,7 @@ func RefreshDataFromMysteelChemical(edbCode, startDate, endDate string) (err err
 			err = er
 			return
 		}
-
+		
 		addItems := make([]*models.BaseFromMysteelChemicalData, 0)
 		indexObj := &models.BaseFromMysteelChemicalIndex{}
 		existIndex, er := indexObj.GetIndexItem(edbCode)
@@ -737,6 +758,7 @@ func RefreshDataFromMysteelChemical(edbCode, startDate, endDate string) (err err
 					if err != nil {
 						return
 					}
+					hasUpdateMap[edbCode] = true
 				}
 			}
 		}
@@ -744,6 +766,7 @@ func RefreshDataFromMysteelChemical(edbCode, startDate, endDate string) (err err
 		if err != nil {
 			return
 		}
+		hasUpdateMap[edbCode] = true
 		return
 	}
 	return

+ 9 - 7
services/base_from_ths.go

@@ -25,7 +25,7 @@ type EdbDataFromThsInterface struct {
 	} `json:"tables"`
 }
 
-func GetEdbDataFromThs(edbCode, startDate, endDate, edbTerminalCode string) (item models.EdbDataFromThs, err error) {
+func GetEdbDataFromThs(edbCode, startDate, endDate, edbTerminalCode string) (item models.EdbDataFromThs, apiErrMsg string, err error) {
 	terminal, err := GetTerminal(utils.DATA_SOURCE_THS, edbTerminalCode)
 	if err != nil {
 		err = fmt.Errorf("获取同花顺接口配置出错 Err: %s", err)
@@ -47,16 +47,18 @@ func GetEdbDataFromThs(edbCode, startDate, endDate, edbTerminalCode string) (ite
 		var token string
 		token, err = GetAccessToken(false, terminal.Value)
 		if err != nil {
+			apiErrMsg = err.Error()
 			return
 		}
-		return getEdbDataFromThsHttp(edbCode, startDate, endDate, terminal.Value, token)
+		item, apiErrMsg, err = getEdbDataFromThsHttp(edbCode, startDate, endDate, terminal.Value, token)
+		return
 	}
-
-	return getEdbDataFromThsApp(edbCode, startDate, endDate, 0, terminal.ServerUrl)
+	item, apiErrMsg, err = getEdbDataFromThsApp(edbCode, startDate, endDate, 0, terminal.ServerUrl)
+	return
 }
 
 // getEdbDataFromThs 获取同花顺接口数据
-func getEdbDataFromThsApp(edbCode, startDate, endDate string, num int, serverUrl string) (item models.EdbDataFromThs, err error) {
+func getEdbDataFromThsApp(edbCode, startDate, endDate string, num int, serverUrl string) (item models.EdbDataFromThs, apiErrMsg string, err error) {
 	if serverUrl == `` {
 		err = errors.New("同花顺接口未配置")
 		return
@@ -82,7 +84,7 @@ func getEdbDataFromThsApp(edbCode, startDate, endDate string, num int, serverUrl
 		return
 	}
 	if tmpItems.Errorcode != 0 {
-
+		apiErrMsg = fmt.Sprintf("%d:%s", tmpItems.Errorcode, tmpItems.Errmsg)
 		//session has expired,please re-login after using the system
 		//如果是同花顺登录session失效了,那么就重新请求获取数据
 		if tmpItems.Errorcode == -1020 && num == 0 {
@@ -139,7 +141,7 @@ func getEdbDataFromThsApp(edbCode, startDate, endDate string, num int, serverUrl
 		Perf:      tmpItems.Perf,
 		Tables:    tablesList,
 	}
-	return item, nil
+	return item, apiErrMsg, err
 }
 
 // FutureGoodDataFromThsInterface 同花顺商品数据类型转为interface

+ 10 - 5
services/base_from_ths_ds.go

@@ -10,7 +10,7 @@ import (
 	"strings"
 )
 
-func GetEdbDataFromThsDs(stockCode, edbCode, startDate, endDate, edbTerminalCode, extraPars string) (item models.EdbDataFromThs, err error) {
+func GetEdbDataFromThsDs(stockCode, edbCode, startDate, endDate, edbTerminalCode, extraPars string) (item models.EdbDataFromThs, apiErrMsg string, err error) {
 	terminal, err := GetTerminal(utils.DATA_SOURCE_THS, edbTerminalCode)
 	if err != nil {
 		err = fmt.Errorf("获取同花顺接口配置出错 Err: %s", err)
@@ -32,15 +32,18 @@ func GetEdbDataFromThsDs(stockCode, edbCode, startDate, endDate, edbTerminalCode
 		var token string
 		token, err = GetAccessToken(false, terminal.Value)
 		if err != nil {
+			apiErrMsg = err.Error()
 			return
 		}
 
 		// TEST
 		//token = "23f339e97fac48d8b99024228fafb6f0128cfbb7.signs_NTc2NjQ4MTA5"
-		return getEdbDataFromThsDsHttp(stockCode, edbCode, startDate, endDate, terminal.Value, token, extraPars)
+		item, apiErrMsg, err = getEdbDataFromThsDsHttp(stockCode, edbCode, startDate, endDate, terminal.Value, token, extraPars)
+		return
 	}
 
-	return getEdbDataFromThsDsApp(stockCode, edbCode, startDate, endDate, 0, terminal.ServerUrl, extraPars)
+	item, apiErrMsg, err = getEdbDataFromThsDsApp(stockCode, edbCode, startDate, endDate, 0, terminal.ServerUrl, extraPars)
+	return
 }
 
 type EdbDataFromThsSdInterface struct {
@@ -97,7 +100,7 @@ type Type struct {
 //}
 
 // getEdbDataFromThsDs 获取同花顺接口数据
-func getEdbDataFromThsDsApp(stockCode, edbCode, startDate, endDate string, num int, serverUrl, extraPars string) (item models.EdbDataFromThs, err error) {
+func getEdbDataFromThsDsApp(stockCode, edbCode, startDate, endDate string, num int, serverUrl, extraPars string) (item models.EdbDataFromThs, apiErrMsg string, err error) {
 	if serverUrl == `` {
 		err = errors.New("同花顺接口未配置")
 		return
@@ -139,6 +142,7 @@ func getEdbDataFromThsDsApp(stockCode, edbCode, startDate, endDate string, num i
 
 			errCode = int64(response.ErrorCode)
 			if response.ErrorCode != 0 {
+				apiErrMsg = fmt.Sprintf("%d:%s", response.ErrorCode, response.ErrMsg)
 				//session has expired,please re-login after using the system
 				//如果是同花顺登录session失效了,那么就重新请求获取数据
 				if response.ErrorCode == -1020 && num == 0 {
@@ -188,6 +192,7 @@ func getEdbDataFromThsDsApp(stockCode, edbCode, startDate, endDate string, num i
 
 		errCode = int64(response.ErrorCode)
 		if response.ErrorCode != 0 {
+			apiErrMsg = fmt.Sprintf("%d:%s", response.ErrorCode, response.ErrMsg)
 			//session has expired,please re-login after using the system
 			//如果是同花顺登录session失效了,那么就重新请求获取数据
 			if response.ErrorCode == -1020 && num == 0 {
@@ -279,5 +284,5 @@ func getEdbDataFromThsDsApp(stockCode, edbCode, startDate, endDate string, num i
 	//	Perf:      tmpItems.Perf,
 	//	Tables:    tablesList,
 	//}
-	return item, nil
+	return item, apiErrMsg, err
 }

+ 4 - 1
services/base_from_ths_ds_http.go

@@ -10,7 +10,7 @@ import (
 )
 
 // getEdbDataFromThsDsHttp 通过url获取同花顺的日期序列数据
-func getEdbDataFromThsDsHttp(stockCode, edbCode, startDate, endDate, thsRefreshToken, token, extraPars string) (item models.EdbDataFromThs, err error) {
+func getEdbDataFromThsDsHttp(stockCode, edbCode, startDate, endDate, thsRefreshToken, token, extraPars string) (item models.EdbDataFromThs, apiErrMsg string, err error) {
 	thsUrl := "https://quantapi.51ifind.com/api/v1/date_sequence"
 	//indicators 是 半角逗号分隔的所有指标,宏观指标过多,推荐使用Windows超级命令生成。 "indicators":"M001620326,M002822183"
 	//functionpara 否 key-value格式,省略时不进行更新时间筛选。两个时间控件更新起始时间(startrtime)和更新结束时间(endrtime),不勾选时省略见下方代码块
@@ -51,6 +51,8 @@ func getEdbDataFromThsDsHttp(stockCode, edbCode, startDate, endDate, thsRefreshT
 
 	body, err, _ := postCurl(thsUrl, dataMap, 0, thsRefreshToken, token)
 	if err != nil {
+		// 返回接口报错的实际原因
+		apiErrMsg = err.Error()
 		return
 	}
 
@@ -62,6 +64,7 @@ func getEdbDataFromThsDsHttp(stockCode, edbCode, startDate, endDate, thsRefreshT
 	}
 	if tmpItems.Errorcode != 0 {
 		err = errors.New(tmpItems.Errmsg)
+		apiErrMsg = fmt.Sprintf("%d:%s", tmpItems.Errorcode, tmpItems.Errmsg)
 		return
 	}
 

+ 4 - 1
services/base_from_ths_http.go

@@ -19,7 +19,7 @@ import (
 var tokenRedisKey = `THS_SERVER_ACCESS_TOKEN`
 
 // getEdbDataFromThsHttp 通过url获取同花顺的普通数据
-func getEdbDataFromThsHttp(edbCode, startDate, endDate, thsRefreshToken, token string) (item models.EdbDataFromThs, err error) {
+func getEdbDataFromThsHttp(edbCode, startDate, endDate, thsRefreshToken, token string) (item models.EdbDataFromThs, apiErrMsg string, err error) {
 	thsUrl := `https://quantapi.51ifind.com/api/v1/edb_service`
 	//indicators 是 半角逗号分隔的所有指标,宏观指标过多,推荐使用Windows超级命令生成。 "indicators":"M001620326,M002822183"
 	//functionpara 否 key-value格式,省略时不进行更新时间筛选。两个时间控件更新起始时间(startrtime)和更新结束时间(endrtime),不勾选时省略见下方代码块
@@ -34,6 +34,8 @@ func getEdbDataFromThsHttp(edbCode, startDate, endDate, thsRefreshToken, token s
 
 	body, err, _ := postCurl(thsUrl, dataMap, 0, thsRefreshToken, token)
 	if err != nil {
+		// 返回接口报错的实际原因
+		apiErrMsg = err.Error()
 		return
 	}
 
@@ -45,6 +47,7 @@ func getEdbDataFromThsHttp(edbCode, startDate, endDate, thsRefreshToken, token s
 	}
 	if tmpItems.Errorcode != 0 {
 		err = errors.New(tmpItems.Errmsg)
+		apiErrMsg = fmt.Sprintf("%d:%s", tmpItems.Errorcode, tmpItems.Errmsg)
 		return
 	}
 	// 因为table里面的value有的时候返回的是string,有的是float64,所以需要用interface来反射取值

+ 13 - 4
services/base_from_wind.go

@@ -16,7 +16,7 @@ const (
 )
 
 // GetEdbDataFromWind 获取wind数据
-func GetEdbDataFromWind(edbCode, startDate, endDate string) (item *models.EdbDataFromWind, errorCode int, err error) {
+func GetEdbDataFromWind(edbCode, startDate, endDate string) (item *models.EdbDataFromWind, errorCode int, apiErrMsg string, err error) {
 	windUrl, err := GetWindUrl(edbCode)
 	if err != nil {
 		errorCode = 421
@@ -32,6 +32,7 @@ func GetEdbDataFromWind(edbCode, startDate, endDate string) (item *models.EdbDat
 	utils.FileLog.Info(fmt.Sprint("指标编码:", edbCode, ";wind result:", string(body)))
 
 	if err != nil {
+		apiErrMsg = err.Error()
 		return
 	}
 	item = new(models.EdbDataFromWind)
@@ -58,6 +59,7 @@ func GetEdbDataFromWind(edbCode, startDate, endDate string) (item *models.EdbDat
 				if err != nil {
 					return
 				}
+				apiErrMsg = fmt.Sprintf("ErrorCode:%d, wind数据接口超限", item.ErrorCode["0"])
 				return GetEdbDataFromWind(edbCode, startDate, endDate)
 			} else if item.ErrorCode["0"] == -40520005 {
 				//.ErrorCode=-40520005
@@ -67,12 +69,15 @@ func GetEdbDataFromWind(edbCode, startDate, endDate string) (item *models.EdbDat
 				if err != nil {
 					return
 				}
+				apiErrMsg = fmt.Sprintf("ErrorCode:%d, 无API接口权限", item.ErrorCode["0"])
 				go alarm_msg.SendAlarmMsg(fmt.Sprintf("wind数据服务异常,edbCode:%s,ErrorCode:%d,result:%s", edbCode, item.ErrorCode["0"], string(body)), 3)
 				return GetEdbDataFromWind(edbCode, startDate, endDate)
 			} else if item.ErrorCode["0"] == WindNoAuthCode {
 				// 指标下架, 无权限
-				return nil, WindNoAuthCode, nil
+				apiErrMsg = fmt.Sprintf("ErrorCode:%d, 指标已下架", item.ErrorCode["0"])
+				return nil, WindNoAuthCode, apiErrMsg, nil
 			} else {
+				apiErrMsg = fmt.Sprintf("ErrorCode:%d, wind数据服务异常", item.ErrorCode["0"])
 				go alarm_msg.SendAlarmMsg(fmt.Sprintf("wind数据服务异常,edbCode:%s,ErrorCode:%d,result:%s", edbCode, item.ErrorCode["0"], string(body)), 3)
 			}
 		}
@@ -81,7 +86,7 @@ func GetEdbDataFromWind(edbCode, startDate, endDate string) (item *models.EdbDat
 }
 
 // GetEdbDataFromWindUrl 通过url获取wind数据
-func GetEdbDataFromWindUrl(windUrl, edbCode, startDate, endDate string) (item *models.EdbDataFromWind, errorCode int, err error) {
+func GetEdbDataFromWindUrl(windUrl, edbCode, startDate, endDate string) (item *models.EdbDataFromWind, errorCode int, apiErrMsg string, err error) {
 	// todo 刷新后的处理
 	if windUrl == `` {
 		return GetEdbDataFromWind(edbCode, startDate, endDate)
@@ -118,6 +123,7 @@ func GetEdbDataFromWindUrl(windUrl, edbCode, startDate, endDate string) (item *m
 				//}
 				// 设置服务器已超限
 				errorCode = 421
+				apiErrMsg = fmt.Sprintf("ErrorCode:%d, wind数据接口超限", item.ErrorCode["0"])
 				go alarm_msg.SendAlarmMsg(fmt.Sprintf("wind数据接口超限,地址:%s", requestWindUrl), 2)
 				//go utils.SendEmail("wind数据接口超限", "地址:"+requestWindUrl, utils.EmailSendToUsers)
 				return
@@ -125,12 +131,15 @@ func GetEdbDataFromWindUrl(windUrl, edbCode, startDate, endDate string) (item *m
 				//.ErrorCode=-40520005
 				//.Data=[No Python API Authority
 				err = errors.New("No Python API Authority")
+				apiErrMsg = fmt.Sprintf("ErrorCode:%d, 无API接口权限", item.ErrorCode["0"])
 				go alarm_msg.SendAlarmMsg(fmt.Sprintf("wind数据服务异常,edbCode:%s,ErrorCode:%d,result:%s", edbCode, item.ErrorCode["0"], string(body)), 3)
 				return
 			} else if item.ErrorCode["0"] == WindNoAuthCode {
 				// 指标下架, 无权限
-				return nil, WindNoAuthCode, nil
+				apiErrMsg = fmt.Sprintf("ErrorCode:%d, 指标已下架", item.ErrorCode["0"])
+				return nil, WindNoAuthCode, apiErrMsg, nil
 			} else {
+				apiErrMsg = fmt.Sprintf("ErrorCode:%d, wind数据服务异常", item.ErrorCode["0"])
 				go alarm_msg.SendAlarmMsg(fmt.Sprintf("wind数据服务异常,edbCode:%s,ErrorCode:%d,result:%s", edbCode, item.ErrorCode["0"], string(body)), 3)
 			}
 		}

+ 11 - 3
services/base_from_wind_wsd.go

@@ -10,7 +10,7 @@ import (
 )
 
 // GetEdbDataFromWind 获取wind数据
-func GetEdbDataFromWindWsd(securitiesCode, edbCode, startDate, endDate string) (item *models.EdbDataFromWind, errorCode int, err error) {
+func GetEdbDataFromWindWsd(securitiesCode, edbCode, startDate, endDate string) (item *models.EdbDataFromWind, errorCode int, apiErrMsg string, err error) {
 	windUrl, err := GetWindUrl(edbCode)
 	if err != nil {
 		errorCode = 421
@@ -52,6 +52,7 @@ func GetEdbDataFromWindWsd(securitiesCode, edbCode, startDate, endDate string) (
 				if err != nil {
 					return
 				}
+				apiErrMsg = fmt.Sprintf("ErrorCode:%d, wind数据接口超限", item.ErrorCode["0"])
 				return GetEdbDataFromWind(edbCode, startDate, endDate)
 			} else if item.ErrorCode["0"] == -40520005 {
 				//.ErrorCode=-40520005
@@ -61,12 +62,15 @@ func GetEdbDataFromWindWsd(securitiesCode, edbCode, startDate, endDate string) (
 				if err != nil {
 					return
 				}
+				apiErrMsg = fmt.Sprintf("ErrorCode:%d, 无API接口权限", item.ErrorCode["0"])
 				go alarm_msg.SendAlarmMsg(fmt.Sprintf("wind数据服务异常,edbCode:%s,ErrorCode:%d,result:%s", edbCode, item.ErrorCode["0"], string(body)), 3)
 				return GetEdbDataFromWind(edbCode, startDate, endDate)
 			} else if item.ErrorCode["0"] == WindNoAuthCode {
 				// 指标下架, 无权限
-				return nil, WindNoAuthCode, nil
+				apiErrMsg = fmt.Sprintf("ErrorCode:%d, 指标已下架", item.ErrorCode["0"])
+				return nil, WindNoAuthCode, apiErrMsg, nil
 			} else {
+				apiErrMsg = fmt.Sprintf("ErrorCode:%d, wind数据服务异常", item.ErrorCode["0"])
 				go alarm_msg.SendAlarmMsg(fmt.Sprintf("wind数据服务异常,edbCode:%s,ErrorCode:%d,result:%s", edbCode, item.ErrorCode["0"], string(body)), 3)
 			}
 		}
@@ -75,13 +79,17 @@ func GetEdbDataFromWindWsd(securitiesCode, edbCode, startDate, endDate string) (
 }
 
 // GetEdbDataFromWindUrlWsd 通过url获取wind wsd数据
-func GetEdbDataFromWindUrlWsd(windUrl, stockCode, edbCode, startDate, endDate string) (item map[string]map[string]interface{}, errorCode int, err error) {
+func GetEdbDataFromWindUrlWsd(windUrl, stockCode, edbCode, startDate, endDate string) (item map[string]map[string]interface{}, errorCode int, apiErrMsg string, err error) {
 	requestWindUrl := windUrl + `edbInfo/wind/wsd?StockCode=%s&EdbCode=%s&StartDate=%s&EndDate=%s`
 	requestWindUrl = fmt.Sprintf(requestWindUrl, stockCode, edbCode, startDate, endDate)
 	utils.FileLog.Info(fmt.Sprintf("windUrl:%s", requestWindUrl))
 	body, err := http.Get(requestWindUrl)
 	utils.FileLog.Info(fmt.Sprint("wind result:", string(body)))
 	err = json.Unmarshal(body, &item)
+	if err != nil {
+		apiErrMsg = fmt.Sprintf("服务异常")
+		return
+	}	
 	return
 }
 

+ 422 - 13
services/edb_info_stat.go

@@ -3,6 +3,7 @@ package services
 import (
 	"eta/eta_index_lib/models"
 	"eta/eta_index_lib/models/data_stat"
+	"eta/eta_index_lib/models/edb_refresh"
 	"eta/eta_index_lib/services/alarm_msg"
 	"eta/eta_index_lib/utils"
 	"fmt"
@@ -56,7 +57,7 @@ func AddEdbInfoUpdateLog(edbInfoId int, updateResult int, updateFailedReason str
 	return
 }
 
-// SetMysteelChemicalEdbInfoUpdateStat 定时统计钢联化工的数据源明细表
+// SetMysteelChemicalEdbInfoUpdateStat 定时统计上海钢联的数据源明细表
 func SetMysteelChemicalEdbInfoUpdateStat(needStat bool) (err error) {
 	defer func() {
 		if err != nil {
@@ -71,7 +72,7 @@ func SetMysteelChemicalEdbInfoUpdateStat(needStat bool) (err error) {
 	pars = append(pars, utils.DATA_SOURCE_MYSTEEL_CHEMICAL)
 	edbList, err := models.GetEdbInfoByCondition(condition, pars, 0)
 	if err != nil {
-		err = fmt.Errorf("查询钢联化工指标信息出错,err: %s", err)
+		err = fmt.Errorf("查询上海钢联指标信息出错,err: %s", err)
 		return
 	}
 	nowTime := time.Now()
@@ -79,9 +80,9 @@ func SetMysteelChemicalEdbInfoUpdateStat(needStat bool) (err error) {
 	todayT, _ := time.ParseInLocation(utils.FormatDate, today, time.Local)
 	nextDay := time.Now().AddDate(0, 0, 1).Format(utils.FormatDate)
 	//查询当日所有钢联指标的终端更新记录
-	updateLogList, err := data_stat.GetEdbUpdateSourceLogByCreateDate(utils.DATA_SOURCE_MYSTEEL_CHEMICAL, today, nextDay)
+	updateLogList, err := data_stat.GetEdbUpdateSourceLogByCreateDate(utils.DATA_SOURCE_MYSTEEL_CHEMICAL, today, nextDay, 1)
 	if err != nil {
-		err = fmt.Errorf("查询钢联化工指标终端更新日志报错,err: %s", err)
+		err = fmt.Errorf("查询上海钢联指标终端更新日志报错,err: %s", err)
 		return
 	}
 	fmt.Println(len(updateLogList))
@@ -102,7 +103,7 @@ func SetMysteelChemicalEdbInfoUpdateStat(needStat bool) (err error) {
 	//查询当日钢联所有的刷新记录
 	updateStatList, err := data_stat.GetEdbUpdateStatByCondition(statCond, statPars)
 	if err != nil {
-		err = fmt.Errorf("查询钢联化工数据源明细记录统计报错,err: %s", err)
+		err = fmt.Errorf("查询上海钢联数据源明细记录统计报错,err: %s", err)
 		return
 	}
 	updateStatMap := make(map[int]*data_stat.EdbInfoUpdateStat)
@@ -123,7 +124,7 @@ func SetMysteelChemicalEdbInfoUpdateStat(needStat bool) (err error) {
 		//查询所有需要当日刷新的周度指标
 		indexTotal, tErr := indexObj.GetIndexByCondition(cond, tmpPars)
 		if tErr != nil {
-			err = fmt.Errorf("查询钢联化工原始指标报错,err: %s", tErr)
+			err = fmt.Errorf("查询上海钢联原始指标报错,err: %s", tErr)
 			return
 		}
 		for _, v := range indexTotal {
@@ -136,7 +137,7 @@ func SetMysteelChemicalEdbInfoUpdateStat(needStat bool) (err error) {
 	//查询所有需要当日刷新的周度指标
 	indexStop, tErr := indexObj.GetIndexByCondition(tmpCond, []interface{}{})
 	if tErr != nil {
-		err = fmt.Errorf("查询钢联化工原始指标报错,err: %s", tErr)
+		err = fmt.Errorf("查询上海钢联原始指标报错,err: %s", tErr)
 		return
 	}
 	for _, v := range indexStop {
@@ -224,7 +225,7 @@ func SetMysteelChemicalEdbInfoUpdateStat(needStat bool) (err error) {
 			if len(addList) >= 500 {
 				err = logStat.Add(addList)
 				if err != nil {
-					err = fmt.Errorf("新增钢联化工明细记录报错,err: %s", err)
+					err = fmt.Errorf("新增上海钢联明细记录报错,err: %s", err)
 					return
 				}
 				addList = addList[:0]
@@ -233,7 +234,7 @@ func SetMysteelChemicalEdbInfoUpdateStat(needStat bool) (err error) {
 			if len(modifyList) >= 500 {
 				err = data_stat.UpdateEdbUpdateStatMulti(modifyList)
 				if err != nil {
-					err = fmt.Errorf("更新钢联化工明细记录报错,err: %s", err)
+					err = fmt.Errorf("更新上海钢联明细记录报错,err: %s", err)
 					return
 				}
 				modifyList = modifyList[:0]
@@ -299,7 +300,7 @@ func SetEdbSourceStat(needStat bool) (err error) {
 	//查询当日钢联所有的统计数据
 	updateStatList, err := data_stat.GetEdbUpdateStatByCondition(statCond, statPars)
 	if err != nil {
-		err = fmt.Errorf("查询钢联化工数据源明细记录统计报错,err: %s", err)
+		err = fmt.Errorf("查询上海钢联数据源明细记录统计报错,err: %s", err)
 		return
 	}
 	if !needStat && len(updateStatList) == 0 {
@@ -318,7 +319,7 @@ func SetEdbSourceStat(needStat bool) (err error) {
 	//查询当日钢联所有的统计数据
 	statList, err := data_stat.GetEdbSourceStatByCondition(cond, pars)
 	if err != nil {
-		err = fmt.Errorf("查询钢联化工数据源统计报错,err: %s", err)
+		err = fmt.Errorf("查询上海钢联数据源统计报错,err: %s", err)
 		return
 	}
 	statMap := make(map[string]*data_stat.EdbSourceStat)
@@ -395,7 +396,7 @@ func SetEdbSourceStat(needStat bool) (err error) {
 		if len(addList) >= 500 {
 			err = logStat.Add(addList)
 			if err != nil {
-				err = fmt.Errorf("新增钢联化工统计表报错,err: %s", err)
+				err = fmt.Errorf("新增上海钢联统计表报错,err: %s", err)
 				return
 			}
 			addList = addList[:0]
@@ -404,7 +405,7 @@ func SetEdbSourceStat(needStat bool) (err error) {
 		if len(modifyList) >= 500 {
 			err = data_stat.UpdateEdbSourceStatMulti(modifyList)
 			if err != nil {
-				err = fmt.Errorf("更新钢联化工统计表报错,err: %s", err)
+				err = fmt.Errorf("更新上海钢联统计表报错,err: %s", err)
 				return
 			}
 			modifyList = modifyList[:0]
@@ -420,3 +421,411 @@ func SetEdbSourceStat(needStat bool) (err error) {
 	}
 	return
 }
+
+// 根据标识判断指标数据是否更新成功
+func EdbDataUpdateRecord(edbInfoId int, hasUpdate bool, updateResult bool, updateFailedReason string, updateRealReason string) (err error) {
+	defer func() {
+		if err != nil {
+			tips := fmt.Sprintf("ETA指标刷新统计异常, 指标ID: %d, err: %s", edbInfoId, err.Error())
+			alarm_msg.SendAlarmMsg(tips, 3)
+		}
+	}()
+	dataUpdateResult := 2
+	dataUpdateFailedReason := "服务异常"
+	if !updateResult {
+		if updateRealReason != "" {
+			dataUpdateFailedReason = updateRealReason
+		}
+		err = AddEdbInfoUpdateLog(edbInfoId, 2, updateFailedReason, dataUpdateResult, dataUpdateFailedReason, 0, 0)
+		if err != nil {
+			err = fmt.Errorf("新增指标更新日志失败,Err: %s", err)
+			return
+		}
+		return
+	}
+
+	if hasUpdate {
+		dataUpdateResult = 1
+		dataUpdateFailedReason = ""
+	} else {
+		dataUpdateFailedReason = "未刷新到数据"
+	}
+
+	// 添加刷新成功日志
+	err = AddEdbInfoUpdateLog(edbInfoId, 1, "", dataUpdateResult, dataUpdateFailedReason, 0, 0)
+	if err != nil {
+		err = fmt.Errorf("新增指标更新日志失败,Err: %s", err)
+		return
+	}
+	return
+}
+
+// SetEdbInfoUpdateStat 定时统计同花顺的数据源明细表
+func SetEdbInfoUpdateStat(needStat bool, source int) (err error) {
+	defer func() {
+		if err != nil {
+			tips := fmt.Sprintf("同花顺数据刷新-ETA指标刷新统计异常 Err: %s", err.Error())
+			utils.FileLog.Info(tips)
+			alarm_msg.SendAlarmMsg(tips, 3)
+		}
+	}()
+	//查询同花顺的所有在更新的指标信息
+	condition := " and source = ? and no_update=0"
+	var pars []interface{}
+	pars = append(pars, source)
+	edbList, err := models.GetEdbInfoByCondition(condition, pars, 0)
+	if err != nil {
+		err = fmt.Errorf("查询同花顺指标信息出错,err: %s", err)
+		return
+	}
+	nowTime := time.Now()
+	today := time.Now().Format(utils.FormatDate)
+	todayT, _ := time.ParseInLocation(utils.FormatDate, today, time.Local)
+	nextDay := time.Now().AddDate(0, 0, 1).Format(utils.FormatDate)
+	//查询当日所有同花顺指标的终端更新记录
+	updateLogList, err := data_stat.GetEdbUpdateSourceLogByCreateDate(source, today, nextDay, 0)
+	if err != nil {
+		err = fmt.Errorf("查询同花顺指标终端更新日志报错,err: %s", err)
+		return
+	}
+	fmt.Println(len(updateLogList))
+	if !needStat && len(updateLogList) == 0 { //如果不存在变更记录 则不进行汇总
+		return
+	}
+	updateLogMap := make(map[int]*data_stat.EdbInfoUpdateLog)
+	if len(updateLogList) > 0 {
+		for _, v := range updateLogList {
+			if _, ok := updateLogMap[v.EdbInfoId]; !ok {
+				updateLogMap[v.EdbInfoId] = v
+			}
+		}
+	}
+	statCond := "  and source = ? and create_time >= ? and create_time < ?"
+	var statPars []interface{}
+	statPars = append(statPars, source, today, nextDay)
+	//查询当日同花顺所有的刷新记录
+	updateStatList, err := data_stat.GetEdbUpdateStatByCondition(statCond, statPars)
+	if err != nil {
+		err = fmt.Errorf("查询同花顺数据源明细记录统计报错,err: %s", err)
+		return
+	}
+	updateStatMap := make(map[int]*data_stat.EdbInfoUpdateStat)
+	if len(updateStatList) > 0 {
+		for _, v := range updateStatList {
+			updateStatMap[v.EdbInfoId] = v
+		}
+	}
+	//查询自动刷新设置成今天的指标信息
+	// 找出当天的时间
+	needRefreshEdbInfoIdsMap, mysteelNeedRefreshMap, err := GetTodayNeedRefreshEdbInfoIds(source)
+	if err != nil {
+		err = fmt.Errorf("查询同花顺自动刷新设置成今天的指标信息报错,err: %s", err)
+		return
+	}
+
+	logStat := new(data_stat.EdbInfoUpdateStat)
+	//组装新增数据
+	addList := make([]*data_stat.EdbInfoUpdateStat, 0)
+	modifyList := make([]*data_stat.EdbInfoUpdateStat, 0)
+	if len(edbList) > 0 {
+		for _, v := range edbList {
+			tmp := &data_stat.EdbInfoUpdateStat{
+				EdbInfoId:        v.EdbInfoId,
+				SourceName:       v.SourceName,
+				Source:           v.Source,
+				EdbCode:          v.EdbCode,
+				EdbName:          v.EdbName,
+				EdbNameSource:    v.EdbNameSource,
+				Frequency:        v.Frequency,
+				Unit:             v.Unit,
+				StartDate:        v.StartDate,
+				EndDate:          v.EndDate,
+				SysUserId:        v.SysUserId,
+				SysUserRealName:  v.SysUserRealName,
+				UniqueCode:       v.UniqueCode,
+				EdbCreateTime:    v.CreateTime,
+				EdbModifyTime:    v.ModifyTime,
+				LatestDate:       v.LatestDate,
+				LatestValue:      v.LatestValue,
+				TerminalCode:     v.TerminalCode,
+				DataUpdateTime:   v.DataUpdateTime,
+				ErDataUpdateDate: v.ErDataUpdateDate,
+				ModifyTime:       nowTime,
+			}
+			if source == utils.DATA_SOURCE_MYSTEEL_CHEMICAL {
+				_, ok := mysteelNeedRefreshMap[v.Frequency]
+				if ok {
+					tmp.NeedRefresh = 1
+				} else {
+					tmp.NeedRefresh = 0
+				}
+			}else {
+				_, ok := needRefreshEdbInfoIdsMap[v.EdbInfoId]
+				if ok {
+					tmp.NeedRefresh = 1
+				} else {
+					tmp.NeedRefresh = 0
+				}
+			}
+			
+
+			// 判断是否当日新增
+			if v.CreateTime.After(todayT) || v.CreateTime == todayT {
+				tmp.IsAdd = 1
+			} else {
+				tmp.IsAdd = 2
+			}
+			if up, ok := updateLogMap[v.EdbInfoId]; ok {
+				tmp.DataUpdateTime = up.DataUpdateTime
+				tmp.ErDataUpdateDate = up.ErDataUpdateDate
+				tmp.DataUpdateResult = up.DataUpdateResult
+				tmp.DataUpdateFailedReason = up.DataUpdateFailedReason
+				tmp.HasRefresh = 1
+				tmp.UpdateResult = up.UpdateResult
+				tmp.UpdateFailedReason = up.UpdateFailedReason
+				tmp.UpdateTime = up.CreateTime.Format(utils.FormatDateTime)
+			} else if tmp.NeedRefresh == 1 {
+				tmp.HasRefresh = 0
+				tmp.DataUpdateResult = 2
+				tmp.DataUpdateFailedReason = "服务异常"
+			}
+
+			// 判断是否需要新增还是更新
+			exist, existOk := updateStatMap[v.EdbInfoId]
+			if existOk {
+				tmp.Id = exist.Id
+				modifyList = append(modifyList, tmp)
+			} else {
+				tmp.CreateTime = nowTime
+				addList = append(addList, tmp)
+			}
+			if len(addList) >= 500 {
+				err = logStat.Add(addList)
+				if err != nil {
+					err = fmt.Errorf("新增同花顺明细记录报错,err: %s", err)
+					return
+				}
+				addList = addList[:0]
+			}
+
+			if len(modifyList) >= 500 {
+				err = data_stat.UpdateEdbUpdateStatMulti(modifyList)
+				if err != nil {
+					err = fmt.Errorf("更新同花顺明细记录报错,err: %s", err)
+					return
+				}
+				modifyList = modifyList[:0]
+			}
+		}
+	}
+
+	//判断当日指标统计数据是否存在,如果存在则更新,不存在则新增
+	if len(addList) > 0 {
+		err = logStat.Add(addList)
+	}
+	if len(modifyList) > 0 {
+		err = data_stat.UpdateEdbUpdateStatMulti(modifyList)
+	}
+	return
+}
+
+func GetTodayNeedRefreshEdbInfoIds(source int) (needRefreshMap map[int]struct{}, mysteelNeedRefreshMap map[string]struct{}, err error) {
+	//刷新频率,枚举值:每自然日、每交易日、每周、每旬、每月、每季、每半年、每年
+	refreshFrequencyList := []string{"每自然日", "每交易日", "每周", "每旬", "每月", "每季", "每半年", "每年"}
+	// 获取刷新频率条件
+	now := time.Now()
+	configIds := make([]int, 0)
+	needRefreshMap = make(map[int]struct{})
+	mysteelNeedRefreshMap = make(map[string]struct{})
+	for _, refreshFrequency := range refreshFrequencyList {
+		condition, pars, isHandler := getRefreshFrequencyCondition(now, refreshFrequency)
+		if !isHandler {
+			// 可能是非交易日,所以过滤不处理
+			continue
+		}
+		condition += ` AND refresh_frequency = ? AND source = ?`
+		pars = append(pars, refreshFrequency, source)	
+
+		tmpList, tmpErr := edb_refresh.GetListByCondition(condition, pars)
+		if tmpErr != nil {
+			err = tmpErr
+			return
+		}
+		for _, v := range tmpList {
+			configIds = append(configIds, v.Id)
+			if source == utils.DATA_SOURCE_MYSTEEL_CHEMICAL {
+				mysteelNeedRefreshMap[v.Frequency] = struct{}{}
+			}
+		}
+	}
+
+	tmpList, tmpErr := edb_refresh.GetConfigRefreshEdbInfoListBySource(source, configIds)
+	if tmpErr != nil {
+		err = tmpErr
+		return
+	}
+	for _, v := range tmpList {	
+		needRefreshMap[v.EdbInfoId] = struct{}{}
+	}
+	
+	return
+}
+
+// getRefreshFrequencyCondition
+// @Description: 根据时间和刷新频率获取条件
+// @author: Roc
+// @datetime 2024-01-09 16:27:11
+// @param now time.Time
+// @param refreshFrequency string
+// @return condition string
+// @return pars []interface{}
+// @return isHandler bool
+func getRefreshFrequencyCondition(now time.Time, refreshFrequency string) (condition string, pars []interface{}, isHandler bool) {
+	isHandler = true
+
+	var dayNum int
+	var isLastDay bool
+
+	//刷新频率,枚举值:每自然日、每交易日、每周、每旬、每月、每季、每半年、每年
+	switch refreshFrequency {
+	case "每自然日":
+		// 自然日不需要额外条件
+		return
+	case "每交易日":
+		// 周六日不处理
+		if now.Weekday() == time.Saturday || now.Weekday() == time.Sunday {
+			isHandler = false
+		}
+		return
+	case "每周":
+		currWeekDay := now.Weekday()
+		if currWeekDay == time.Sunday {
+			currWeekDay = 7
+			isLastDay = true
+		}
+		dayNum = int(currWeekDay)
+	case "每旬":
+		currDay := now.Day()
+		if currDay <= 10 {
+			dayNum = currDay
+			// 如果是这旬的最后一天
+			if currDay == 10 {
+				isLastDay = true
+			}
+		} else if currDay <= 20 {
+			dayNum = currDay - 10
+			// 如果是这旬的最后一天
+			if currDay == 20 {
+				isLastDay = true
+			}
+		} else {
+			dayNum = currDay - 20
+
+			// 当月的最后一天
+			monthLastDay := time.Date(now.Year(), now.Month()+1, 1, 0, 0, 0, 0, time.Local).AddDate(0, 0, -1)
+			// 如果是这旬的最后一天
+			if currDay == monthLastDay.Day() {
+				isLastDay = true
+			}
+		}
+	case "每月":
+		// 当前日期
+		currDay := now.Day()
+		dayNum = currDay
+
+		// 当期的最后一天
+		monthLastDay := time.Date(now.Year(), now.Month()+1, 1, 0, 0, 0, 0, time.Local).AddDate(0, 0, -1)
+
+		// 如果是这期的最后一天
+		if currDay == monthLastDay.Day() {
+			isLastDay = true
+		}
+	case "每季":
+		// 当期的第一天  ;  当期的最后一天
+		var startDay, endDay time.Time
+		currMonth := now.Month()
+		currDay := now.Day()
+		if currMonth <= 3 {
+			// 当季的第一天
+			startDay = time.Date(now.Year(), 1, 1, 0, 0, 0, 0, time.Local)
+			// 当季的最后一天
+			endDay = time.Date(now.Year(), 4, 1, 0, 0, 0, 0, time.Local).AddDate(0, 0, -1)
+		} else if currMonth <= 6 {
+			// 当期的第一天
+			startDay = time.Date(now.Year(), 4, 1, 0, 0, 0, 0, time.Local)
+			// 当期的最后一天
+			endDay = time.Date(now.Year(), 7, 1, 0, 0, 0, 0, time.Local).AddDate(0, 0, -1)
+		} else if currMonth <= 9 {
+			// 当期的第一天
+			startDay = time.Date(now.Year(), 7, 1, 0, 0, 0, 0, time.Local)
+			// 当期的最后一天
+			endDay = time.Date(now.Year(), 10, 1, 0, 0, 0, 0, time.Local).AddDate(0, 0, -1)
+		} else {
+			// 当期的第一天
+			startDay = time.Date(now.Year(), 10, 1, 0, 0, 0, 0, time.Local)
+			// 当期的最后一天
+			endDay = time.Date(now.Year()+1, 1, 1, 0, 0, 0, 0, time.Local).AddDate(0, 0, -1)
+		}
+
+		// 计算这期的第一天和当日的天数
+		dayNum = utils.GetTimeSubDay(startDay, now) + 1
+
+		// 如果是这期的最后一天
+		if currMonth == endDay.Month() && currDay == endDay.Day() {
+			isLastDay = true
+		}
+	case "每半年":
+		// 当期的第一天  ;  当期的最后一天
+		var startDay, endDay time.Time
+		currMonth := now.Month()
+		currDay := now.Day()
+		if currMonth <= 6 {
+			// 当期的第一天
+			startDay = time.Date(now.Year(), 1, 1, 0, 0, 0, 0, time.Local)
+			// 当期的最后一天
+			endDay = time.Date(now.Year(), 7, 1, 0, 0, 0, 0, time.Local).AddDate(0, 0, -1)
+		} else {
+			// 当期的第一天
+			startDay = time.Date(now.Year(), 7, 1, 0, 0, 0, 0, time.Local)
+			// 当期的最后一天
+			endDay = time.Date(now.Year()+1, 1, 1, 0, 0, 0, 0, time.Local).AddDate(0, 0, -1)
+		}
+
+		// 计算这期的第一天和当日的天数
+		dayNum = utils.GetTimeSubDay(startDay, now) + 1
+
+		// 如果是这期的最后一天
+		if currMonth == endDay.Month() && currDay == endDay.Day() {
+			isLastDay = true
+		}
+	case "每年":
+		currMonth := now.Month()
+		currDay := now.Day()
+
+		// 当期的第一天
+		startDay := time.Date(now.Year(), 1, 1, 0, 0, 0, 0, time.Local)
+		// 当期的最后一天
+		endDay := time.Date(now.Year()+1, 1, 1, 0, 0, 0, 0, time.Local).AddDate(0, 0, -1)
+
+		// 计算这期的第一天和当日的天数
+		dayNum = utils.GetTimeSubDay(startDay, now) + 1
+
+		// 如果是这期的最后一天
+		if currMonth == endDay.Month() && currDay == endDay.Day() {
+			isLastDay = true
+		}
+
+	}
+
+	// 如果是这期的最后一天,那么就是判断refresh_frequency_day是否配置为0,或者配置的天数大于这期的最大天数
+	if isLastDay {
+		condition += ` AND ( refresh_frequency_day = ? OR refresh_frequency_day >= ? )`
+		pars = append(pars, 0, dayNum)
+	} else {
+		// 如果不是这期的最后一天,那么就是判断refresh_frequency_day是否等于配置的天数
+		condition += ` AND refresh_frequency_day = ?  `
+		pars = append(pars, dayNum)
+	}
+
+	return
+}