Merge branch 'feature/eta1.5.2_mysteel_stat' into debug

controllers/base_auth.go

@@ -39,19 +39,19 @@ func (this *BaseAuthController) Prepare() {
 	fmt.Println("Url:", uri)
 	if method != "HEAD" {
 		if method == "POST" {
-			//authorization := this.Ctx.Input.Header("authorization")
-			//if authorization == "" {
-			//	this.JSON(models.BaseResponse{Ret: 408, Msg: "请重新授权！", ErrMsg: "请重新授权:authorization is empty "}, false, false)
-			//	this.StopRun()
-			//	return
-			//}
-			//checkAuthorization := utils.MD5(utils.APP_NAME_EN + utils.Md5Key)
-			//fmt.Println(checkAuthorization)
-			//if authorization != checkAuthorization {
-			//	this.JSON(models.BaseResponse{Ret: 408, Msg: "签名错误！", ErrMsg: "签名错误:authorization is err "}, false, false)
-			//	this.StopRun()
-			//	return
-			//}
+			authorization := this.Ctx.Input.Header("authorization")
+			if authorization == "" {
+				this.JSON(models.BaseResponse{Ret: 408, Msg: "请重新授权！", ErrMsg: "请重新授权:authorization is empty "}, false, false)
+				this.StopRun()
+				return
+			}
+			checkAuthorization := utils.MD5(utils.APP_NAME_EN + utils.Md5Key)
+			fmt.Println(checkAuthorization)
+			if authorization != checkAuthorization {
+				this.JSON(models.BaseResponse{Ret: 408, Msg: "签名错误！", ErrMsg: "签名错误:authorization is err "}, false, false)
+				this.StopRun()
+				return
+			}
 		} else {
 			this.JSON(models.BaseResponse{Ret: 408, Msg: "请求异常，请联系客服!", ErrMsg: "POST之外的请求，暂不支持"}, false, false)
 			this.StopRun()

controllers/base_from_mysteel_chemical.go

@@ -125,9 +125,9 @@ func (this *MySteelChemicalController) Refresh() {
 	}
 	// 添加指标刷新成功日志
 	if erDataUpdateDate != "" {
-		_ = services.AddEdbInfoUpdateLog(edbInfo.EdbInfoId, 1, "", 1, "", 0)
+		_ = services.AddEdbInfoUpdateLog(edbInfo.EdbInfoId, 1, "", 1, "", 0, 0)
 	} else {
-		_ = services.AddEdbInfoUpdateLog(edbInfo.EdbInfoId, 1, "", 2, "未刷新到数据", 0)
+		_ = services.AddEdbInfoUpdateLog(edbInfo.EdbInfoId, 1, "", 2, "未刷新到数据", 0, 0)
 	}
 
 	// 更新ES

logic/base_edb_info.go

@@ -27,6 +27,8 @@ func RefreshBaseEdbInfo(edbInfo *models.EdbInfo, startDate string) (isHandling b
 		err = models.RefreshEdbDataFromBaiinfo(edbInfo.EdbInfoId, edbInfo.EdbCode, startDate)
 	case utils.DATA_SOURCE_MYSTEEL_CHEMICAL:
 		err = models.RefreshEdbDataFromMysteelChemical(edbInfo.EdbInfoId, edbInfo.EdbCode, startDate)
+	case utils.DATA_SOURCE_YS:
+		err = models.RefreshEdbDataFromSmm(edbInfo.EdbInfoId, edbInfo.EdbCode, startDate)
 	default:
 		return
 	}

models/base_from_mysteel_chemical.go

@@ -512,3 +512,14 @@ func (d *BaseFromMysteelChemicalData) ModifyMysteelIndexMaxAndMinInfo(indexCode
 	_, err = o.Raw(sql, item.MinDate, item.MaxDate, indexCode).Exec()
 	return
 }
+
+// GetIndexByCondition 获取指标
+func (m *BaseFromMysteelChemicalIndex) GetIndexByCondition(condition string, pars []interface{}) (items []*BaseFromMysteelChemicalIndex, err error) {
+	o := orm.NewOrm()
+	sql := `SELECT * FROM base_from_mysteel_chemical_index WHERE 1=1 `
+	if condition != "" {
+		sql += condition
+	}
+	_, err = o.Raw(sql, pars).QueryRows(&items)
+	return
+}

models/data_stat/edb_info_update_log.go

@@ -36,6 +36,7 @@ type EdbInfoUpdateLog struct {
 	DataUpdateFailedReason string    `description:"数据未正常更新原因"`
 	DataUpdateTime         string    `description:"数据更新时间"`
 	IsSourceRefresh        int       `description:"是否为终端刷新到数据源的刷新操作：0否，1是"`
+	UpdateType             int       `description:"变更类型，0:数据明细变更，1:基础信息变更, 2:新增指标"`
 }
 
 func AddEdbUpdateLog(item *EdbInfoUpdateLog) (lastId int64, err error) {

services/base_from_mysteel_chemical.go

@@ -38,7 +38,7 @@ func handleIndex(indexItem *models.HandleMysteelIndex) (err error) {
 			edbInfo, e := models.GetEdbInfoByEdbCode(utils.DATA_SOURCE_MYSTEEL_CHEMICAL, indexItem.IndexCode)
 			if e == nil {
 				//查询指标存在，才添加刷新日志
-				_ = AddEdbInfoUpdateLog(edbInfo.EdbInfoId, 2, err.Error(), dataUpdateResult, dataUpdateFailedReason, 1)
+				_ = AddEdbInfoUpdateLog(edbInfo.EdbInfoId, 2, err.Error(), dataUpdateResult, dataUpdateFailedReason, 1, 0)
 			}
 		}
 	}()
@@ -61,8 +61,12 @@ func handleIndex(indexItem *models.HandleMysteelIndex) (err error) {
 			return
 		}
 	}
+	nameChange := false
 	if item != nil && item.BaseFromMysteelChemicalIndexId > 0 {
 		isAdd = 2
+		if item.IndexName != indexItem.IndexName {
+			nameChange = true
+		}
 	} else {
 		isAdd = 1
 	}
@@ -222,7 +226,7 @@ func handleIndex(indexItem *models.HandleMysteelIndex) (err error) {
 		dataUpdateFailedReason := "服务异常"
 		_, logErrMsg, logErr := logic.RefreshBaseEdbInfo(edbInfo, ``)
 		if logErr != nil {
-			lErr = AddEdbInfoUpdateLog(edbInfo.EdbInfoId, 2, logErrMsg+logErr.Error(), dataUpdateResult, dataUpdateFailedReason, 1)
+			lErr = AddEdbInfoUpdateLog(edbInfo.EdbInfoId, 2, logErrMsg+logErr.Error(), dataUpdateResult, dataUpdateFailedReason, 1, 0)
 			return
 		}
 
@@ -234,7 +238,19 @@ func handleIndex(indexItem *models.HandleMysteelIndex) (err error) {
 		}
 
 		// 添加刷新成功日志
-		lErr = AddEdbInfoUpdateLog(edbInfo.EdbInfoId, 1, "", dataUpdateResult, dataUpdateFailedReason, 1)
+		lErr = AddEdbInfoUpdateLog(edbInfo.EdbInfoId, 1, "", dataUpdateResult, dataUpdateFailedReason, 1, 0)
+		if lErr != nil {
+			return
+		}
+		//如果变更了指标名称，则添加指标信息变更日志
+		if nameChange {
+			edbInfo.SourceIndexName = indexItem.IndexName
+			lErr = edbInfo.Update([]string{"SourceIndexName"})
+			if lErr != nil {
+				return
+			}
+			lErr = AddEdbInfoUpdateLog(edbInfo.EdbInfoId, 1, "", 0, "", 0, 1)
+		}
 	}
 	//}()
 

services/base_from_ths_ds.go

@@ -148,11 +148,12 @@ func getEdbDataFromThsDsApp(stockCode, edbCode, startDate, endDate string, num i
 			for _, stockData := range response.Data {
 				time := stockData["time"].(string)
 				//thsCode := stockData["thscode"].(string)
-				tableTimeList = append(tableTimeList, time)
+				//tableTimeList = append(tableTimeList, time)
 
 				for k, v := range stockData {
 					if k != "time" && k != "thscode" {
 						if v != nil {
+							tableTimeList = append(tableTimeList, time)
 							tableValueList = append(tableValueList, v.(float64))
 						}
 					}

services/edb_info_stat.go

@@ -10,7 +10,7 @@ import (
 )
 
 // AddEdbInfoUpdateLog 添加指标编辑/刷新日志
-func AddEdbInfoUpdateLog(edbInfoId int, updateResult int, updateFailedReason string, dataUpdateResult int, dataUpdateFailedReason string, isSourceRefresh int) (err error) {
+func AddEdbInfoUpdateLog(edbInfoId int, updateResult int, updateFailedReason string, dataUpdateResult int, dataUpdateFailedReason string, isSourceRefresh int, updateType int) (err error) {
 	var edbInfo *models.EdbInfo
 	if edbInfoId > 0 {
 		// 获取指标详情
@@ -46,6 +46,7 @@ func AddEdbInfoUpdateLog(edbInfoId int, updateResult int, updateFailedReason str
 		log.DataUpdateResult = dataUpdateResult
 		log.DataUpdateFailedReason = dataUpdateFailedReason
 		log.IsSourceRefresh = isSourceRefresh
+		log.UpdateType = updateType
 		_, err = data_stat.AddEdbUpdateLog(log)
 		if err != nil {
 			err = fmt.Errorf("新增指标更新日志失败，Err: %s", err)
@@ -63,8 +64,8 @@ func SetMysteelChemicalEdbInfoUpdateStat(needStat bool) (err error) {
 			alarm_msg.SendAlarmMsg(tips, 3)
 		}
 	}()
-	//查询钢联的所有指标信息
-	condition := " and source = ? "
+	//查询钢联的所有在更新的指标信息
+	condition := " and source = ? and no_update=0"
 	var pars []interface{}
 	pars = append(pars, utils.DATA_SOURCE_MYSTEEL_CHEMICAL)
 	edbList, err := models.GetEdbInfoByCondition(condition, pars, 0)
@@ -108,6 +109,24 @@ func SetMysteelChemicalEdbInfoUpdateStat(needStat bool) (err error) {
 			updateStatMap[v.EdbInfoId] = v
 		}
 	}
+	indexObj := new(models.BaseFromMysteelChemicalIndex)
+	week := int(nowTime.Weekday())
+	weekNeedRefreshMap := make(map[string]struct{})
+	if week >= 3 && week <= 6 {
+		yesterday := nowTime.AddDate(0, 0, -1).Format(utils.FormatDate)
+		cond := ` AND frequency = ? AND end_date < ? AND is_stop = 0`
+		var tmpPars []interface{}
+		tmpPars = append(tmpPars, "周度", yesterday)
+		//查询所有需要当日刷新的周度指标
+		indexTotal, tErr := indexObj.GetIndexByCondition(cond, tmpPars)
+		if tErr != nil {
+			err = fmt.Errorf("查询钢联化工原始指标报错，err: %s", tErr)
+			return
+		}
+		for _, v := range indexTotal {
+			weekNeedRefreshMap[v.IndexCode] = struct{}{}
+		}
+	}
 	logStat := new(data_stat.EdbInfoUpdateStat)
 	//组装新增数据
 	addList := make([]*data_stat.EdbInfoUpdateStat, 0)
@@ -137,7 +156,15 @@ func SetMysteelChemicalEdbInfoUpdateStat(needStat bool) (err error) {
 				ErDataUpdateDate: v.ErDataUpdateDate,
 				ModifyTime:       nowTime,
 			}
-			needRefresh, _ := checkMySteelEdbInfoNeedRefresh(v.Frequency)
+			frequency := v.Frequency
+			if v.Frequency == "旬度" { //特殊处理指标库里和数据源里频度不一致的情况
+				//查询源指标库的频度
+				indexTmp, e := indexObj.GetIndexItem(v.EdbCode)
+				if e == nil {
+					frequency = indexTmp.Frequency
+				}
+			}
+			needRefresh, _ := checkMySteelEdbInfoNeedRefresh(v.EdbCode, frequency, weekNeedRefreshMap)
 			tmp.NeedRefresh = needRefresh
 
 			// 判断是否当日新增
@@ -182,7 +209,7 @@ func SetMysteelChemicalEdbInfoUpdateStat(needStat bool) (err error) {
 	return
 }
 
-func checkMySteelEdbInfoNeedRefresh(frequency string) (needRefresh int, err error) {
+func checkMySteelEdbInfoNeedRefresh(edbCode, frequency string, weekNeedRefreshMap map[string]struct{}) (needRefresh int, err error) {
 	now := time.Now()
 	week := int(now.Weekday())
 	//日度
@@ -194,14 +221,15 @@ func checkMySteelEdbInfoNeedRefresh(frequency string) (needRefresh int, err erro
 	}
 	//周度
 	if week >= 3 && week <= 6 {
-		if frequency == "周度" {
+		_, ok := weekNeedRefreshMap[edbCode]
+		if frequency == "周度" && ok {
 			needRefresh = 1
 			return
 		}
 	}
 
-	day := now.Day() //季度，月度，年度都是每个月1号刷新
-	if day == 1 {
+	//季度，月度，年度都是每个周末刷新
+	if week == 0 {
 		needRefresh = 1
 	}
 	return
@@ -303,7 +331,7 @@ func SetEdbSourceStat(needStat bool) (err error) {
 
 			if v.UpdateResult == 1 { //刷新结果
 				tmp.RefreshSuccessNum = tmp.RefreshSuccessNum + 1
-			} else if v.NeedRefresh == 1 {
+			} else if v.HasRefresh == 1 {
 				tmp.RefreshFailedNum = tmp.RefreshFailedNum + 1
 			}
 		}