钢联化工更新统计优化

controllers/base_from_mysteel_chemical.go

@@ -125,9 +125,9 @@ func (this *MySteelChemicalController) Refresh() {
 	}
 	// 添加指标刷新成功日志
 	if erDataUpdateDate != "" {
-		_ = services.AddEdbInfoUpdateLog(edbInfo.EdbInfoId, 1, "", 1, "", 0)
+		_ = services.AddEdbInfoUpdateLog(edbInfo.EdbInfoId, 1, "", 1, "", 0, 0)
 	} else {
-		_ = services.AddEdbInfoUpdateLog(edbInfo.EdbInfoId, 1, "", 2, "未刷新到数据", 0)
+		_ = services.AddEdbInfoUpdateLog(edbInfo.EdbInfoId, 1, "", 2, "未刷新到数据", 0, 0)
 	}
 
 	// 更新ES

models/base_from_mysteel_chemical.go

@@ -512,3 +512,14 @@ func (d *BaseFromMysteelChemicalData) ModifyMysteelIndexMaxAndMinInfo(indexCode
 	_, err = o.Raw(sql, item.MinDate, item.MaxDate, indexCode).Exec()
 	return
 }
+
+// GetIndexByCondition 获取指标
+func (m *BaseFromMysteelChemicalIndex) GetIndexByCondition(condition string, pars []interface{}) (items []*BaseFromMysteelChemicalIndex, err error) {
+	o := orm.NewOrm()
+	sql := `SELECT * FROM base_from_mysteel_chemical_index WHERE 1=1 `
+	if condition != "" {
+		sql += condition
+	}
+	_, err = o.Raw(sql, pars).QueryRows(&items)
+	return
+}

models/data_stat/edb_info_update_log.go

@@ -36,6 +36,7 @@ type EdbInfoUpdateLog struct {
 	DataUpdateFailedReason string    `description:"数据未正常更新原因"`
 	DataUpdateTime         string    `description:"数据更新时间"`
 	IsSourceRefresh        int       `description:"是否为终端刷新到数据源的刷新操作：0否，1是"`
+	UpdateType             int       `description:"变更类型，0:数据明细变更，1:基础信息变更, 2:新增指标"`
 }
 
 func AddEdbUpdateLog(item *EdbInfoUpdateLog) (lastId int64, err error) {

services/base_from_mysteel_chemical.go

@@ -38,7 +38,7 @@ func handleIndex(indexItem *models.HandleMysteelIndex) (err error) {
 			edbInfo, e := models.GetEdbInfoByEdbCode(utils.DATA_SOURCE_MYSTEEL_CHEMICAL, indexItem.IndexCode)
 			if e == nil {
 				//查询指标存在，才添加刷新日志
-				_ = AddEdbInfoUpdateLog(edbInfo.EdbInfoId, 2, err.Error(), dataUpdateResult, dataUpdateFailedReason, 1)
+				_ = AddEdbInfoUpdateLog(edbInfo.EdbInfoId, 2, err.Error(), dataUpdateResult, dataUpdateFailedReason, 1, 0)
 			}
 		}
 	}()
@@ -61,8 +61,12 @@ func handleIndex(indexItem *models.HandleMysteelIndex) (err error) {
 			return
 		}
 	}
+	nameChange := false
 	if item != nil && item.BaseFromMysteelChemicalIndexId > 0 {
 		isAdd = 2
+		if item.IndexName != indexItem.IndexName {
+			nameChange = true
+		}
 	} else {
 		isAdd = 1
 	}
@@ -222,7 +226,7 @@ func handleIndex(indexItem *models.HandleMysteelIndex) (err error) {
 		dataUpdateFailedReason := "服务异常"
 		_, logErrMsg, logErr := logic.RefreshBaseEdbInfo(edbInfo, ``)
 		if logErr != nil {
-			lErr = AddEdbInfoUpdateLog(edbInfo.EdbInfoId, 2, logErrMsg+logErr.Error(), dataUpdateResult, dataUpdateFailedReason, 1)
+			lErr = AddEdbInfoUpdateLog(edbInfo.EdbInfoId, 2, logErrMsg+logErr.Error(), dataUpdateResult, dataUpdateFailedReason, 1, 0)
 			return
 		}
 
@@ -234,7 +238,19 @@ func handleIndex(indexItem *models.HandleMysteelIndex) (err error) {
 		}
 
 		// 添加刷新成功日志
-		lErr = AddEdbInfoUpdateLog(edbInfo.EdbInfoId, 1, "", dataUpdateResult, dataUpdateFailedReason, 1)
+		lErr = AddEdbInfoUpdateLog(edbInfo.EdbInfoId, 1, "", dataUpdateResult, dataUpdateFailedReason, 1, 0)
+		if lErr != nil {
+			return
+		}
+		//如果变更了指标名称，则添加指标信息变更日志
+		if nameChange {
+			edbInfo.SourceIndexName = indexItem.IndexName
+			lErr = edbInfo.Update([]string{"SourceIndexName"})
+			if lErr != nil {
+				return
+			}
+			lErr = AddEdbInfoUpdateLog(edbInfo.EdbInfoId, 1, "", 0, "", 0, 1)
+		}
 	}
 	//}()
 

services/edb_info_stat.go

@@ -10,7 +10,7 @@ import (
 )
 
 // AddEdbInfoUpdateLog 添加指标编辑/刷新日志
-func AddEdbInfoUpdateLog(edbInfoId int, updateResult int, updateFailedReason string, dataUpdateResult int, dataUpdateFailedReason string, isSourceRefresh int) (err error) {
+func AddEdbInfoUpdateLog(edbInfoId int, updateResult int, updateFailedReason string, dataUpdateResult int, dataUpdateFailedReason string, isSourceRefresh int, updateType int) (err error) {
 	var edbInfo *models.EdbInfo
 	if edbInfoId > 0 {
 		// 获取指标详情
@@ -46,6 +46,7 @@ func AddEdbInfoUpdateLog(edbInfoId int, updateResult int, updateFailedReason str
 		log.DataUpdateResult = dataUpdateResult
 		log.DataUpdateFailedReason = dataUpdateFailedReason
 		log.IsSourceRefresh = isSourceRefresh
+		log.UpdateType = updateType
 		_, err = data_stat.AddEdbUpdateLog(log)
 		if err != nil {
 			err = fmt.Errorf("新增指标更新日志失败，Err: %s", err)
@@ -108,6 +109,24 @@ func SetMysteelChemicalEdbInfoUpdateStat(needStat bool) (err error) {
 			updateStatMap[v.EdbInfoId] = v
 		}
 	}
+	indexObj := new(models.BaseFromMysteelChemicalIndex)
+	week := int(nowTime.Weekday())
+	weekNeedRefreshMap := make(map[string]struct{})
+	if week >= 3 && week <= 6 {
+		yesterday := nowTime.AddDate(0, 0, -1).Format(utils.FormatDate)
+		cond := ` AND frequency = ? AND end_date < ? AND is_stop = 0`
+		var tmpPars []interface{}
+		tmpPars = append(tmpPars, "周度", yesterday)
+		//查询所有需要当日刷新的周度指标
+		indexTotal, tErr := indexObj.GetIndexByCondition(cond, tmpPars)
+		if tErr != nil {
+			err = fmt.Errorf("查询钢联化工原始指标报错，err: %s", tErr)
+			return
+		}
+		for _, v := range indexTotal {
+			weekNeedRefreshMap[v.IndexCode] = struct{}{}
+		}
+	}
 	logStat := new(data_stat.EdbInfoUpdateStat)
 	//组装新增数据
 	addList := make([]*data_stat.EdbInfoUpdateStat, 0)
@@ -137,7 +156,15 @@ func SetMysteelChemicalEdbInfoUpdateStat(needStat bool) (err error) {
 				ErDataUpdateDate: v.ErDataUpdateDate,
 				ModifyTime:       nowTime,
 			}
-			needRefresh, _ := checkMySteelEdbInfoNeedRefresh(v.Frequency)
+			frequency := v.Frequency
+			if v.Frequency == "旬度" { //特殊处理指标库里和数据源里频度不一致的情况
+				//查询源指标库的频度
+				indexTmp, e := indexObj.GetIndexItem(v.EdbCode)
+				if e == nil {
+					frequency = indexTmp.Frequency
+				}
+			}
+			needRefresh, _ := checkMySteelEdbInfoNeedRefresh(v.EdbCode, frequency, weekNeedRefreshMap)
 			tmp.NeedRefresh = needRefresh
 
 			// 判断是否当日新增
@@ -182,7 +209,7 @@ func SetMysteelChemicalEdbInfoUpdateStat(needStat bool) (err error) {
 	return
 }
 
-func checkMySteelEdbInfoNeedRefresh(frequency string) (needRefresh int, err error) {
+func checkMySteelEdbInfoNeedRefresh(edbCode, frequency string, weekNeedRefreshMap map[string]struct{}) (needRefresh int, err error) {
 	now := time.Now()
 	week := int(now.Weekday())
 	//日度
@@ -194,14 +221,15 @@ func checkMySteelEdbInfoNeedRefresh(frequency string) (needRefresh int, err erro
 	}
 	//周度
 	if week >= 3 && week <= 6 {
-		if frequency == "周度" {
+		_, ok := weekNeedRefreshMap[edbCode]
+		if frequency == "周度" && ok {
 			needRefresh = 1
 			return
 		}
 	}
 
-	day := now.Day() //季度，月度，年度都是每个月1号刷新
-	if day == 1 {
+	//季度，月度，年度都是每个周末刷新
+	if week == 0 {
 		needRefresh = 1
 	}
 	return
@@ -297,7 +325,7 @@ func SetEdbSourceStat(needStat bool) (err error) {
 			// 区分刷新成功和更新成功
 			if v.DataUpdateResult == 1 { //处理更新结果
 				tmp.UpdateSuccessNum = tmp.UpdateSuccessNum + 1
-			} else if v.NeedRefresh == 1 || v.HasRefresh == 1 {
+			} else if v.NeedRefresh == 1 {
 				tmp.UpdateFailedNum = tmp.UpdateFailedNum + 1
 			}