Merge branch 'refs/heads/eta_2.0.7_fenwei_0820@guomengyuan'

# Conflicts:
#	models/db.go

controllers/base_from_fenwei.go

@@ -6,6 +6,7 @@ import (
 	"eta/eta_index_lib/models"
 	"eta/eta_index_lib/services"
 	"eta/eta_index_lib/utils"
+	"github.com/beego/beego/v2/core/logs"
 	"strconv"
 	"time"
 )
@@ -253,3 +254,37 @@ func (this *FenweiController) BaseIndexList() {
 	br.Success = true
 	br.Msg = "获取成功"
 }
+
+// NetDataHandle
+// @Title 汾渭网页数据落库
+// @Description 汾渭网页数据落库
+// @Success 200 string "操作成功"
+// @router /net/data/handle [post]
+func (this *FenweiController) NetDataHandle() {
+	br := new(models.BaseResponse).Init()
+	defer func() {
+		if br.ErrMsg == "" {
+			br.IsSendEmail = false
+		}
+		this.Data["json"] = br
+		this.ServeJSON()
+	}()
+
+	var req models.HandleFenWeiNetDataReq
+	err := json.Unmarshal(this.Ctx.Input.RequestBody, &req)
+	if err != nil {
+		br.Msg = "参数解析异常！"
+		br.ErrMsg = "参数解析失败，Err:" + err.Error()
+		return
+	}
+
+	logs.Info("汾渭网页数据落库请求参数 List size：", len(req.List))
+	err = services.NetDataHandle(req)
+	if err != nil {
+		return
+	}
+
+	br.Ret = 200
+	br.Success = true
+	br.Msg = "操作成功"
+}

models/base_from_fenwei.go

@@ -1,6 +1,7 @@
 package models
 
 import (
+	"errors"
 	"eta/eta_index_lib/utils"
 	"fmt"
 	"github.com/beego/beego/v2/client/orm"
@@ -305,6 +306,23 @@ type HandleFenweiExcelDataReq struct {
 	TerminalCode string `description:"编码"`
 }
 
+type FenWeiNetIndexInfo struct {
+	FenweiIndexId int64  `orm:"column(fenwei_index_id);pk"`
+	IndexName     string `description:"指标名称"`
+	//IndexCode    string  `description:"指标编码"`
+	Unit         string      `description:"单位"`
+	Frequency    string      `description:"频度"`
+	TerminalCode string      `description:"编码"`
+	ClassifyName string      `description:"分类名称"`
+	DataTime     string      `description:"数据时间"`
+	Value        interface{} `description:"数据值"`
+}
+
+type HandleFenWeiNetDataReq struct {
+	List         []*FenWeiNetIndexInfo
+	TerminalCode string `description:"编码"`
+}
+
 func (y *BaseFromFenweiData) GetMaxAndMinDateByIndexCode(indexCode string) (item *EdbInfoMaxAndMinInfo, err error) {
 	o := orm.NewOrm()
 	sql := ` SELECT MIN(data_time) AS min_date,MAX(data_time) AS max_date,MIN(value) AS min_value,MAX(value) AS max_value FROM base_from_fenwei_data WHERE index_code=? `
@@ -338,10 +356,15 @@ func (y *BaseFromFenweiClassify) Update(updateCols []string) (err error) {
 	return
 }
 
-func (y *BaseFromFenweiClassify) GetByClassifyName(classifyName string) (item *BaseFromFenweiClassify, err error) {
+func (y *BaseFromFenweiClassify) GetByClassifyName(classifyName string) (item []*BaseFromFenweiClassify, err error) {
 	o := orm.NewOrm()
 	sql := ` SELECT * FROM base_from_fenwei_classify WHERE classify_name=? `
-	err = o.Raw(sql, classifyName).QueryRow(&item)
+	_, err = o.Raw(sql, classifyName).QueryRows(&item)
+
+	if errors.Is(err, orm.ErrNoRows) {
+		return nil, nil
+	}
+
 	return
 }
 
@@ -413,3 +436,59 @@ func MultiUpdateBaseFromFenweiDataValue(items []*BaseFromFenweiData) (err error)
 	}
 	return
 }
+
+// GetBaseFromFenWeiIndexByIndexName 根据指标名称查询指标
+func GetBaseFromFenWeiIndexByIndexName(indexName string) (item *BaseFromFenweiIndex, err error) {
+	o := orm.NewOrm()
+	sql := ` SELECT * FROM base_from_fenwei_index WHERE index_name=? `
+	err = o.Raw(sql, indexName).QueryRow(&item)
+
+	if errors.Is(err, orm.ErrNoRows) {
+		return nil, nil
+	}
+	return
+}
+
+// GetBaseFromFenweiDataByIndexCodeAndDataTime 根据指标编码和dataTime查询指标数据
+func GetBaseFromFenweiDataByIndexCodeAndDataTime(indexCode, dataTime string) (item *BaseFromFenweiData, err error) {
+	o := orm.NewOrm()
+	sql := ` SELECT * FROM base_from_fenwei_data WHERE index_code=? AND data_time=? `
+	err = o.Raw(sql, indexCode, dataTime).QueryRow(&item)
+
+	if errors.Is(err, orm.ErrNoRows) {
+		return nil, nil
+	}
+
+	return
+}
+
+// BatchAddBaseFromFenWeiData 批量插入指标数据
+func BatchAddBaseFromFenWeiData(items []*BaseFromFenweiData) (err error) {
+	if len(items) == 0 {
+		return
+	}
+
+	o := orm.NewOrm()
+	_, err = o.InsertMulti(1, items)
+	return
+}
+
+// AddBaseFromFenWeiClassify 新增汾渭分类
+func AddBaseFromFenWeiClassify(item *BaseFromFenweiClassify) (classifyId int64, err error) {
+	o := orm.NewOrm()
+	classifyId, err = o.Insert(item)
+	return
+}
+
+// GetBaseFromFenWeiIndexByIndexCode 跟据指标编码查询指标信息
+func GetBaseFromFenWeiIndexByIndexCode(indexCode string) (item *BaseFromFenweiIndex, err error) {
+	o := orm.NewOrm()
+	sql := ` SELECT * FROM base_from_fenwei_index WHERE index_code=? `
+	err = o.Raw(sql, indexCode).QueryRow(&item)
+
+	if errors.Is(err, orm.ErrNoRows) {
+		return nil, nil
+	}
+
+	return
+}

models/db.go

@@ -168,6 +168,7 @@ func initBaseIndex() {
 		new(BaseFromBusinessData), // 数据源中自有数据的明细数据表
 		new(BaseFromOilchemIndex),
 		new(BaseFromOilchemData),
+		new(BaseFromFenweiClassify),
 	)
 }
 

routers/commentsRouter.go

@@ -637,6 +637,15 @@ func init() {
             Filters: nil,
             Params: nil})
 
+    beego.GlobalControllerRouter["eta/eta_index_lib/controllers:FenweiController"] = append(beego.GlobalControllerRouter["eta/eta_index_lib/controllers:FenweiController"],
+        beego.ControllerComments{
+            Method: "NetDataHandle",
+            Router: `/net/data/handle`,
+            AllowHTTPMethods: []string{"post"},
+            MethodParams: param.Make(),
+            Filters: nil,
+            Params: nil})
+
     beego.GlobalControllerRouter["eta/eta_index_lib/controllers:FenweiController"] = append(beego.GlobalControllerRouter["eta/eta_index_lib/controllers:FenweiController"],
         beego.ControllerComments{
             Method: "Refresh",

services/base_from_fenwei.go

@@ -6,8 +6,11 @@ import (
 	"eta/eta_index_lib/services/alarm_msg"
 	"eta/eta_index_lib/utils"
 	"fmt"
+	"github.com/beego/beego/v2/core/logs"
+	"github.com/mozillazg/go-pinyin"
 	"strings"
 	"time"
+	"unicode"
 )
 
 // HandleFenweiIndex 处理汾渭数据的excel数据
@@ -247,3 +250,287 @@ func handleFenweiIndex(req *models.HandleFenweiExcelData, terminalCode string) (
 	}
 	return
 }
+
+func NetDataHandle(req models.HandleFenWeiNetDataReq) error {
+	indexInfoList := req.List
+	classifyObj := new(models.BaseFromFenweiClassify)
+
+	// 因为传递过来得List是 指标数据 维度得，获取全量得指标数据比对，会导致分类和指标 频繁查询，会频繁访问数据库，做个简单优化吧
+	classifyMap := make(map[string]models.FenWeiNetIndexInfo)
+	indexMap := make(map[string]models.FenWeiNetIndexInfo)
+	for _, indexInfo := range indexInfoList {
+		classifyMap[indexInfo.ClassifyName] = *indexInfo
+		indexMap[indexInfo.IndexName] = *indexInfo
+	}
+
+	// 处理分类信息
+	classifyIdMap := make(map[string]int)
+	for classifyName, reqData := range classifyMap {
+		var classifyId int
+		classify, err := classifyObj.GetByClassifyName(classifyName)
+		if err != nil {
+			return err
+		}
+		if classify == nil {
+			logs.Info("NetDataHandle addClassify classifyName: ", classifyName)
+			classifyId, err = addFenWeiClassify(&reqData)
+			if err != nil {
+				return err
+			}
+		} else {
+			classifyId = classify[0].ClassifyId
+		}
+
+		// 保存分类ID
+		classifyIdMap[classifyName] = classifyId
+	}
+
+	// 处理指标信息
+	indexIdMap := make(map[string]int64)
+	indexCodeMap := make(map[string]string)
+	for indexName, reqData := range indexMap {
+		indexCode := GenerateIndexCode(indexName)
+		indexCodeMap[indexName] = indexCode
+		index, err := models.GetBaseFromFenWeiIndexByIndexName(indexName)
+		if err != nil {
+			return err
+		}
+		if index == nil {
+			logs.Info("NetDataHandle indexName: ", indexName)
+			classifyId := classifyIdMap[reqData.ClassifyName]
+			logs.Info("NetDataHandle addIndex indexName: ", indexName)
+			indexId, err := addIndex(&reqData, classifyId, indexCode)
+			if err != nil {
+				return err
+			}
+			indexIdMap[reqData.IndexName] = indexId
+		} else {
+			logs.Info("NetDataHandle exist indexName: ", indexName)
+			indexIdMap[reqData.IndexName] = index.FenweiIndexId
+		}
+	}
+
+	indexDataList := make([]*models.BaseFromFenweiData, 0)
+	for _, indexInfo := range indexInfoList {
+		// 处理指标数据
+		indexData, err := handleIndexData(indexInfo, indexIdMap, indexCodeMap)
+		if err != nil {
+			return err
+		}
+		if indexData != nil {
+			indexDataList = append(indexDataList, indexData)
+		}
+	}
+	// 批量插入指标数据
+	logs.Info("NetDataHandle addIndexData indexDataList.size: ", len(indexDataList))
+	if len(indexDataList) == 0 {
+		return nil
+	}
+	err := models.BatchAddBaseFromFenWeiData(indexDataList)
+	if err != nil {
+		return err
+	}
+	logs.Info("NetDataHandle addIndexData success")
+	return nil
+}
+
+func handleIndexData(indexInfo *models.FenWeiNetIndexInfo, indexIdMap map[string]int64, indexCodeMap map[string]string) (*models.BaseFromFenweiData, error) {
+	var format string
+	if isYearMonth(indexInfo.DataTime) {
+		format = convertYearMonthToLastDay(indexInfo.DataTime)
+	} else if indexInfo.Frequency == "月度" && isFirstDayOfMonth(indexInfo.DataTime) {
+		format = convertYearMonthDayToLastDay(indexInfo.DataTime)
+	} else {
+		format = indexInfo.DataTime
+	}
+
+	// 获取指标数据
+	indexCode := indexCodeMap[indexInfo.IndexName]
+	indexData, err := models.GetBaseFromFenweiDataByIndexCodeAndDataTime(indexCode, format)
+	if err != nil {
+		return nil, err
+	}
+	if indexData != nil {
+		// 汾渭不存在数据更新和预测值情况，所以此处未做更新逻辑
+		return nil, nil
+	}
+
+	timestamp := time.Now().UnixNano() / 1e6
+
+	indexId := indexIdMap[indexInfo.IndexName]
+
+	data := new(models.BaseFromFenweiData)
+	data.FenweiIndexId = int(indexId)
+	data.IndexCode = indexCode
+	data.DataTime = format
+	data.Value = fmt.Sprintf("%v", indexInfo.Value)
+	data.CreateTime = time.Now()
+	data.ModifyTime = time.Now()
+	data.DataTimestamp = timestamp
+
+	// 打印data对象日志
+
+	logs.Info("handleIndexData data : ", data)
+
+	return data, nil
+}
+
+func addIndex(info *models.FenWeiNetIndexInfo, classifyId int, indexCode string) (indexId int64, err error) {
+	byIndexCode, err := models.GetBaseFromFenWeiIndexByIndexCode(indexCode)
+	if err != nil {
+		return 0, err
+	}
+	if byIndexCode != nil {
+		return byIndexCode.FenweiIndexId, nil
+	}
+
+	index := new(models.BaseFromFenweiIndex)
+	index.IndexName = info.IndexName
+	index.IndexCode = indexCode
+	index.IndexCode = indexCode
+	index.Frequency = info.Frequency
+	index.ClassifyId = classifyId
+	index.Unit = info.Unit
+	index.ModifyTime = time.Now()
+	index.CreateTime = time.Now()
+	index.TerminalCode = info.TerminalCode
+	indexId, err = index.Add()
+	if err != nil {
+		return 0, err
+	}
+	return indexId, nil
+}
+
+// addFenWeiClassify 添加分类
+func addFenWeiClassify(info *models.FenWeiNetIndexInfo) (int, error) {
+	// 初始化分类结构
+	classify := new(models.BaseFromFenweiClassify)
+	classify.ClassifyName = info.ClassifyName
+
+	// 递归处理分类层级
+	return handleClassify(classify, 0, 1)
+}
+
+func handleClassify(classify *models.BaseFromFenweiClassify, parentId int, level int) (int, error) {
+	// 将分类名按 "-" 分割
+	classifyArr := strings.Split(classify.ClassifyName, "-")
+
+	// 当前分类名称
+	currentClassifyName := classifyArr[0]
+	classify.ClassifyName = currentClassifyName
+	classify.ParentId = parentId
+	classify.Level = level
+
+	// 检查当前分类是否已存在
+	existingClassifyList, err := classify.GetByClassifyName(currentClassifyName)
+	if err != nil {
+		return 0, err
+	}
+	var existingClassify *models.BaseFromFenweiClassify
+	for _, item := range existingClassifyList {
+		if item.ParentId == parentId {
+			existingClassify = item
+		}
+	}
+
+	// 如果分类存在，且父级ID相同，则递归处理下一级
+	if existingClassify != nil && existingClassify != nil {
+		// 如果存在且有下一级，则递归处理下一级
+		if len(classifyArr) > 1 {
+			nextClassifyName := strings.Join(classifyArr[1:], "-")
+			nextClassify := new(models.BaseFromFenweiClassify)
+			nextClassify.ClassifyName = nextClassifyName
+			return handleClassify(nextClassify, existingClassify.ClassifyId, existingClassify.Level+1)
+		}
+		return existingClassify.ClassifyId, nil
+	}
+
+	// 如果不存在，新增当前分类
+	classify.SysUserId = 0
+	classify.SysUserRealName = ""
+	classify.ModifyTime = time.Now()
+	classify.CreateTime = time.Now()
+	classifyId, err := models.AddBaseFromFenWeiClassify(classify)
+	if err != nil {
+		return 0, err
+	}
+
+	// 如果有下一级，递归处理下一级
+	if len(classifyArr) > 1 {
+		nextClassifyName := strings.Join(classifyArr[1:], "-")
+		nextClassify := new(models.BaseFromFenweiClassify)
+		nextClassify.ClassifyName = nextClassifyName
+		return handleClassify(nextClassify, int(classifyId), level+1)
+	}
+
+	return int(classifyId), nil
+}
+
+// 判断传入参数 dataText 是否是yyyy-MM格式，如果是则返回true，否则返回false
+func isYearMonth(dataText string) bool {
+	_, err := time.Parse("2006-01", dataText)
+	if err != nil {
+		return false
+	}
+	return true
+}
+
+// 判断传入参数 dataText yyyy-MM-dd格式，是否是该月第一天，如果是则返回true，否则返回false
+func isFirstDayOfMonth(dataText string) bool {
+	t, _ := time.Parse("2006-01-02", dataText)
+	if t.Day() == 1 {
+		return true
+	}
+	return false
+}
+
+// 转换时间 dataText yyyy-MM-dd格式，获取该月最后一天的日期
+func convertYearMonthDayToLastDay(dataText string) string {
+	t, _ := time.Parse("2006-01-02", dataText)
+	lastDay := t.AddDate(0, 1, -1)
+	return lastDay.Format("2006-01-02")
+}
+
+// 转换时间 dataText 2024-08 yyyy-MM 格式的时间字符串转换为该月最后一天的日期Date --> 2024-08-31
+func convertYearMonthToLastDay(dataText string) string {
+	t, _ := time.Parse("2006-01", dataText)
+	lastDay := t.AddDate(0, 1, -1)
+	return lastDay.Format("2006-01-02")
+}
+
+// GenerateIndexCode 指标编码规则：指标名称拼音首字母，数字、字母保留，特殊字符拿掉
+// 例：美湾：9月U：国际大豆进口成本价：期货收盘：张家港 -----> lyswwmw9yUgjddjkcbjqhspzjg
+func GenerateIndexCode(indexName string) string {
+
+	// 获取汉字的拼音首字母，保留数字和大写字母
+	indexCode := getFirstLetters(indexName)
+
+	return indexCode
+}
+
+// getFirstLetters 获取汉字的拼音首字母，并保留数字和大写字母
+func getFirstLetters(input string) string {
+	// 设置拼音转换选项，只获取首字母
+	args := pinyin.NewArgs()
+	args.Style = pinyin.FirstLetter
+
+	// 定义用于存储结果的字符串
+	var result strings.Builder
+
+	// 遍历输入字符串中的每个字符
+	for _, r := range input {
+		if unicode.IsDigit(r) || unicode.IsUpper(r) {
+			// 保留数字和大写字母
+			result.WriteRune(r)
+		} else if unicode.Is(unicode.Han, r) {
+			// 如果是汉字，则获取其拼音首字母
+			py := pinyin.Pinyin(string(r), args)
+			if len(py) > 0 && len(py[0]) > 0 {
+				result.WriteString(py[0][0])
+			}
+		}
+		// 对于其他字符，忽略处理
+	}
+
+	return result.String()
+}