Merge branch 'pool/374' into debug

controllers/base_from_pb.go

@@ -43,7 +43,13 @@ func (this *PbController) Add() {
 	cacheKey = utils.CACHE_EDB_DATA_ADD + strconv.Itoa(source) + "_" + req.EdbCode
 	if !utils.Rc.IsExist(cacheKey) {
 		utils.Rc.SetNX(cacheKey, 1, 1*time.Minute)
-		dataItem, err := services.GetEdbDataFromPb(req.EdbCode, utils.BASE_START_DATE_UnSpace, utils.BASE_END_DATE_UnSpace, "")
+		dataItem, err := services.GetEdbDataFromPb(req.EdbCode, utils.BASE_START_DATE_UnSpace, utils.PB_BASE_END_DATE_UnSpace, "")
+		// 日志记录
+		num := 0
+		if dataItem != nil {
+			num = len(dataItem.Date)
+		}
+		models.AddEdbRefreshApiRecordByRefresh(&models.EdbInfo{EdbCode: req.EdbCode, Source: source}, utils.BASE_START_DATE_UnSpace, num, err)
 		if err != nil {
 			br.Msg = "获取指标信息失败！"
 			br.ErrMsg = "获取指标信息失败 GetEdbDataFromPb，Err:" + err.Error()
@@ -115,7 +121,13 @@ func (this *PbController) Refresh() {
 
 	//开始日期格式化
 	startDateTime, _ := time.ParseInLocation(utils.FormatDate, req.StartDate, time.Local)
-	dataItem, err := services.GetEdbDataFromPb(req.EdbCode, startDateTime.Format(utils.FormatDateUnSpace), utils.BASE_END_DATE_UnSpace, edbInfo.TerminalCode)
+	dataItem, err := services.GetEdbDataFromPb(req.EdbCode, startDateTime.Format(utils.FormatDateUnSpace), utils.PB_BASE_END_DATE_UnSpace, edbInfo.TerminalCode)
+	// 日志记录
+	num := 0
+	if dataItem != nil {
+		num = len(dataItem.Date)
+	}
+	models.AddEdbRefreshApiRecordByRefresh(edbInfo, startDateTime.Format(utils.FormatDateUnSpace), num, err)
 	if err != nil {
 		br.Msg = "获取指标信息失败！"
 		br.ErrMsg = "获取指标信息失败 GetEdbDataFromPb，Err:" + err.Error()

controllers/base_from_pb_finance.go

@@ -54,7 +54,13 @@ func (this *PbFinanceController) Add() {
 			br.ErrMsg = "指标编码异常，请传入客户编码"
 			return
 		}
-		dataItem, err := services.GetEdbDataFromPbFinance(edbCodeArr[0], edbCodeArr[1], utils.BASE_START_DATE_UnSpace, utils.BASE_END_DATE_UnSpace, "")
+		dataItem, err := services.GetEdbDataFromPbFinance(edbCodeArr[0], edbCodeArr[1], utils.BASE_START_DATE_UnSpace, utils.PB_BASE_END_DATE_UnSpace, "")
+		// 日志记录
+		num := 0
+		if dataItem != nil {
+			num = len(dataItem.Date)
+		}
+		models.AddEdbRefreshApiRecordByRefresh(&models.EdbInfo{EdbCode: edbCode, Source: source}, utils.BASE_START_DATE_UnSpace, num, err)
 		if err != nil {
 			br.Msg = "获取指标信息失败！"
 			br.ErrMsg = "获取指标信息失败 GetEdbDataFromPb，Err:" + err.Error()
@@ -136,7 +142,13 @@ func (this *PbFinanceController) Refresh() {
 		br.ErrMsg = "指标编码异常，请传入客户编码"
 		return
 	}
-	dataItem, err := services.GetEdbDataFromPbFinance(edbCodeArr[0], edbCodeArr[1], startDateTime.Format(utils.FormatDateUnSpace), utils.BASE_END_DATE_UnSpace, edbInfo.TerminalCode)
+	dataItem, err := services.GetEdbDataFromPbFinance(edbCodeArr[0], edbCodeArr[1], startDateTime.Format(utils.FormatDateUnSpace), utils.PB_BASE_END_DATE_UnSpace, edbInfo.TerminalCode)
+	// 日志记录
+	num := 0
+	if dataItem != nil {
+		num = len(dataItem.Date)
+	}
+	models.AddEdbRefreshApiRecordByRefresh(edbInfo, startDateTime.Format(utils.FormatDateUnSpace), num, err)
 	if err != nil {
 		br.Msg = "获取指标信息失败！"
 		br.ErrMsg = "获取指标信息失败 GetEdbDataFromPb，Err:" + err.Error()

controllers/commodity_coal.go

@@ -1806,6 +1806,9 @@ func (this *CoalMineDataController) Firm() {
 					err = fmt.Errorf("查询指标最新日期失败 Err：%s", err)
 					return
 				}
+				dateItem.LatestDate, _ = utils.FormatZhDate(dateItem.LatestDate)
+				dateItem.MinDate, _ = utils.FormatZhDate(dateItem.MinDate)
+				dateItem.MaxDate, _ = utils.FormatZhDate(dateItem.MaxDate)
 				obj.ModifyIndexMaxAndMinDate(indexCode, dateItem)
 			}
 		}()

models/base_from_coalmine.go

@@ -160,7 +160,7 @@ func (m *BaseFromCoalmineMapping) GetMaxAndMinDateByIndexCode(indexCode, tableNa
 	}
 
 	// 获取最新值
-	sql = ` SELECT value AS latest_value FROM %s WHERE index_code=? ORDER BY data_time DESC LIMIT 1 `
+	sql = ` SELECT deal_value AS latest_value FROM %s WHERE index_code=? ORDER BY data_time DESC LIMIT 1 `
 	sql = fmt.Sprintf(sql, tableName)
 	//err = o.Raw(sql, indexCode).QueryRow(&latest_value)
 	var latestValueNull sql2.NullFloat64

models/base_from_ths_hf_data.go

@@ -5,6 +5,7 @@ import (
 	"eta/eta_index_lib/global"
 	"eta/eta_index_lib/utils"
 	"fmt"
+	"gorm.io/gorm"
 	"strings"
 	"time"
 )
@@ -207,7 +208,28 @@ func (m *BaseFromThsHfData) MultiInsertOrUpdate(inserts, updates []*BaseFromThsH
 	return
 }
 
-func (m *BaseFromThsHfData) GetIndexMinMax(indexCode string) (item *EdbInfoMaxAndMinInfo, err error) {
+// ThsHfEdbInfoMaxAndMinInfo 指标最新数据记录结构体
+type ThsHfEdbInfoMaxAndMinInfo struct {
+	MinDate     string  `description:"最小日期" bson:"min_date"`
+	MaxDate     string  `description:"最大日期" bson:"max_date"`
+	MinValue    float64 `description:"最小值" bson:"min_value"`
+	MaxValue    float64 `description:"最大值" bson:"max_value"`
+	LatestValue float64 `gorm:"-" description:"最新值" bson:"latest_value"`
+	LatestDate  string  `gorm:"-" description:"实际数据最新日期" bson:"latest_date"`
+	EndValue    float64 `description:"最新值" bson:"end_value"`
+	IndexCode   string  `description:"指标编码" bson:"index_code"`
+}
+
+// AfterFind 在该模型上设置钩子函数，把日期转成正确的string，所以查询函数只能用Find函数，First或者Scan是不会触发该函数的来获取数据
+func (m *ThsHfEdbInfoMaxAndMinInfo) AfterFind(db *gorm.DB) (err error) {
+	m.MinDate = utils.GormDateStrToDateTimeStr(m.MinDate)
+	m.MaxDate = utils.GormDateStrToDateTimeStr(m.MaxDate)
+	m.LatestDate = utils.GormDateStrToDateStr(m.LatestDate)
+
+	return
+}
+
+func (m *BaseFromThsHfData) GetIndexMinMax(indexCode string) (item *ThsHfEdbInfoMaxAndMinInfo, err error) {
 	sql := fmt.Sprintf(`SELECT MIN(%s) AS min_date, MAX(%s) AS max_date, MIN(%s) AS min_value,MAX(%s) AS max_value FROM %s WHERE %s = ?`, m.Cols().DataTime, m.Cols().DataTime, m.Cols().Value, m.Cols().Value, m.TableName(), m.Cols().IndexCode)
 	err = global.DEFAULT_DB.Raw(sql, indexCode).First(&item).Error
 	if err != nil {

models/edb_info.go

@@ -543,8 +543,8 @@ type EdbInfoMaxAndMinInfo struct {
 
 // AfterFind 在该模型上设置钩子函数，把日期转成正确的string，所以查询函数只能用Find函数，First或者Scan是不会触发该函数的来获取数据
 func (m *EdbInfoMaxAndMinInfo) AfterFind(db *gorm.DB) (err error) {
-	m.MinDate = utils.GormDateStrToDateTimeStr(m.MinDate)
-	m.MaxDate = utils.GormDateStrToDateTimeStr(m.MaxDate)
+	m.MinDate = utils.GormDateStrToDateStr(m.MinDate)
+	m.MaxDate = utils.GormDateStrToDateStr(m.MaxDate)
 	m.LatestDate = utils.GormDateStrToDateStr(m.LatestDate)
 
 	return

models/edb_refresh_api_record.go

@@ -0,0 +1,57 @@
+package models
+
+import (
+	"eta/eta_index_lib/global"
+	"eta/eta_index_lib/utils"
+	"fmt"
+	"time"
+)
+
+type EdbRefreshApiRecord struct {
+	EdbRefreshApiRecordId int       `gorm:"column:edb_refresh_api_record_id;type:int(10) UNSIGNED;primaryKey;not null;" json:"edb_refresh_api_record_id"`
+	EdbCode               string    `gorm:"column:edb_code;type:varchar(255);comment:指标编码;" json:"edb_code"`                            // 指标编码
+	EdbName               string    `gorm:"column:edb_name;type:varchar(255);comment:指标名称;" json:"edb_name"`                            // 指标名称
+	Source                int       `gorm:"column:source;type:int(9) UNSIGNED;comment:指标来源;default:0;" json:"source"`                   // 指标来源
+	Frequency             string    `gorm:"column:frequency;type:varchar(32);comment:指标频度;" json:"frequency"`                           // 指标频度
+	StartDate             string    `gorm:"column:start_date;type:varchar(64);comment:刷新开始时间;" json:"start_date"`                       // 刷新开始时间
+	Result                int       `gorm:"column:result;type:tinyint(4) UNSIGNED;comment:刷新成功与否，0：未成功，1：已成功;default:0;" json:"result"` // 刷新成功与否，0：未成功，1：已成功
+	RefreshDesc           string    `gorm:"column:refresh_desc;type:text;comment:刷新的备注;" json:"refresh_desc"`                           // 刷新的备注
+	RefreshNum            int       `gorm:"column:refresh_num;type:int(9) UNSIGNED;comment:刷新到了的数据数量;default:0;" json:"refresh_num"`    // 刷新到了的数据数量
+	CreateTime            time.Time `gorm:"column:create_time;type:datetime;comment:刷新时间;default:NULL;" json:"create_time"`             // 刷新时间
+}
+
+func (m *EdbRefreshApiRecord) Add() (err error) {
+	err = global.DEFAULT_DB.Create(m).Error
+
+	return
+}
+
+func AddEdbRefreshApiRecordByRefresh(edbInfo *EdbInfo, startDate string, num int, resultErr error) {
+	var err error
+	defer func() {
+		if err != nil {
+			utils.FileLog.Info(fmt.Sprintf("AddEdbRefreshApiRecordByRefresh, err: %v", err))
+		}
+	}()
+	result := 1
+	resultErrMsg := ``
+	if resultErr != nil {
+		result = 0
+		resultErrMsg = resultErr.Error()
+	}
+	record := &EdbRefreshApiRecord{
+		EdbRefreshApiRecordId: 0,
+		EdbCode:               edbInfo.EdbCode,
+		EdbName:               edbInfo.EdbName,
+		Source:                edbInfo.Source,
+		Frequency:             edbInfo.Frequency,
+		StartDate:             startDate,
+		Result:                result,
+		RefreshDesc:           resultErrMsg,
+		RefreshNum:            num,
+		CreateTime:            time.Now(),
+	}
+	err = global.DEFAULT_DB.Create(record).Error
+
+	return
+}

utils/common.go

@@ -1810,3 +1810,92 @@ func GormDateStrToDateStr(originalString string) (formatStr string) {
 
 	return
 }
+
+func FormatZhDateBak(dateTimeStr string) (newDateTimeStr string, err error) {
+	if dateTimeStr == `` {
+		return
+	}
+	if strings.Contains(dateTimeStr, "年") {
+		dateTimeStr = strings.ReplaceAll(dateTimeStr, "年", "-")
+	}
+	if strings.Contains(dateTimeStr, "月份") {
+		dateTimeStr = strings.ReplaceAll(dateTimeStr, "月份", "-")
+		dateTimeStrList := strings.Split(dateTimeStr, "-")
+		if len(dateTimeStrList) == 3 {
+			monthInt, tmpErr := strconv.Atoi(dateTimeStrList[1])
+			if tmpErr != nil {
+				return ``, tmpErr
+			}
+			if monthInt < 10 {
+				dateTimeStrList[1] = fmt.Sprint("0", dateTimeStrList[1])
+				dateTimeStr = strings.Join(dateTimeStrList, `-`)
+			}
+		}
+	} else if strings.Contains(dateTimeStr, "月") {
+		dateTimeStr = strings.ReplaceAll(dateTimeStr, "月", "-")
+		dateTimeStrList := strings.Split(dateTimeStr, "-")
+		if len(dateTimeStrList) == 3 {
+			monthInt, tmpErr := strconv.Atoi(dateTimeStrList[1])
+			if tmpErr != nil {
+				return ``, tmpErr
+			}
+			if monthInt < 10 {
+				dateTimeStrList[1] = fmt.Sprint("0", dateTimeStrList[1])
+				dateTimeStr = strings.Join(dateTimeStrList, `-`)
+			}
+		}
+	}
+
+	if strings.Contains(dateTimeStr, "上旬") {
+		dateTimeStr = strings.ReplaceAll(dateTimeStr, "上旬", "10")
+	} else if strings.Contains(dateTimeStr, "中旬") {
+		dateTimeStr = strings.ReplaceAll(dateTimeStr, "中旬", "20")
+	} else if strings.Contains(dateTimeStr, "本月") {
+		dateTimeStr = strings.ReplaceAll(dateTimeStr, "本月", "")
+		dateTime, tmpErr := time.ParseInLocation(FormatYearMonthDate, dateTimeStr, time.Local)
+		if tmpErr != nil {
+			err = tmpErr
+			return
+		}
+		dateTimeStr = time.Date(dateTime.Year(), dateTime.Month(), 1, 0, 0, 0, 0, time.Local).AddDate(0, 1, -1).Format(FormatDate)
+	}
+
+	return dateTimeStr, nil
+}
+
+func FormatZhDate(dateStr string) (newDateTimeStr string, err error) {
+	// Regular expression to extract year, month, and period
+	re := regexp.MustCompile(`(\d{4})年?(\d{1,2})月(?:份)?(上旬|中旬|本月)`)
+	matches := re.FindStringSubmatch(dateStr)
+
+	if len(matches) != 4 {
+		return "", fmt.Errorf("unsupported date string format: %s", dateStr)
+	}
+
+	year, err := strconv.Atoi(matches[1])
+	if err != nil {
+		return
+	}
+	month, err := strconv.Atoi(matches[2])
+	if err != nil {
+		return
+	}
+	period := matches[3]
+
+	var day int
+	switch period {
+	case "上旬":
+		day = 10
+	case "中旬":
+		day = 20
+	case "本月":
+		// Get the last day of the month
+		lastDayOfMonth := time.Date(year, time.Month(month+1), 0, 0, 0, 0, 0, time.UTC).Day()
+		day = lastDayOfMonth
+	default:
+		return "", fmt.Errorf("unsupported period in date string: %s", dateStr)
+	}
+
+	targetDate := time.Date(year, time.Month(month), day, 0, 0, 0, 0, time.UTC)
+	return targetDate.Format(FormatDate), nil
+}

utils/constants.go

@@ -223,10 +223,11 @@ const (
 
 // 基础数据初始化日期
 var (
-	BASE_START_DATE         = `1900-01-01`                                          //基础数据开始日期
-	BASE_END_DATE           = time.Now().AddDate(4, 0, 0).Format(FormatDate)        //基础数据结束日期
-	BASE_START_DATE_UnSpace = "19000101"                                            //基础数据开始日期
-	BASE_END_DATE_UnSpace   = time.Now().AddDate(4, 0, 0).Format(FormatDateUnSpace) //基础数据结束日期
+	BASE_START_DATE          = `1900-01-01`                                          //基础数据开始日期
+	BASE_END_DATE            = time.Now().AddDate(4, 0, 0).Format(FormatDate)        //基础数据结束日期
+	BASE_START_DATE_UnSpace  = "19000101"                                            //基础数据开始日期
+	BASE_END_DATE_UnSpace    = time.Now().AddDate(4, 0, 0).Format(FormatDateUnSpace) //基础数据结束日期
+	PB_BASE_END_DATE_UnSpace = time.Now().Format(FormatDateUnSpace)                  //彭博的基础数据结束日期
 )
 
 const (