汾渭数据源-数据落库

controllers/base_from_fenwei.go

@@ -253,3 +253,35 @@ func (this *FenweiController) BaseIndexList() {
 	br.Success = true
 	br.Msg = "获取成功"
 }
+
+// NetDataHandle
+// @Title 汾渭网页数据落库
+// @Description 汾渭网页数据落库
+// @Success 200 string "操作成功"
+// @router /net/data/handle [post]
+func (this *FenweiController) NetDataHandle() {
+	br := new(models.BaseResponse).Init()
+	defer func() {
+		if br.ErrMsg == "" {
+			br.IsSendEmail = false
+		}
+		this.Data["json"] = br
+		this.ServeJSON()
+	}()
+	var req models.HandleFenWeiNetDataReq
+	err := json.Unmarshal(this.Ctx.Input.RequestBody, &req)
+	if err != nil {
+		br.Msg = "参数解析异常！"
+		br.ErrMsg = "参数解析失败，Err:" + err.Error()
+		return
+	}
+
+	err = services.NetDataHandle(req)
+	if err != nil {
+		return
+	}
+
+	br.Ret = 200
+	br.Success = true
+	br.Msg = "操作成功"
+}

models/base_from_fenwei.go

@@ -305,6 +305,22 @@ type HandleFenweiExcelDataReq struct {
 	TerminalCode string `description:"编码"`
 }
 
+type FenWeiNetIndexInfo struct {
+	IndexName string `description:"指标名称"`
+	//IndexCode    string  `description:"指标编码"`
+	Unit         string  `description:"单位"`
+	Frequency    string  `description:"频度"`
+	TerminalCode string  `description:"编码"`
+	ClassifyName string  `description:"分类名称"`
+	DataTime     string  `description:"数据时间"`
+	Value        float64 `description:"数据值"`
+}
+
+type HandleFenWeiNetDataReq struct {
+	List         []*FenWeiNetIndexInfo
+	TerminalCode string `description:"编码"`
+}
+
 func (y *BaseFromFenweiData) GetMaxAndMinDateByIndexCode(indexCode string) (item *EdbInfoMaxAndMinInfo, err error) {
 	o := orm.NewOrm()
 	sql := ` SELECT MIN(data_time) AS min_date,MAX(data_time) AS max_date,MIN(value) AS min_value,MAX(value) AS max_value FROM base_from_fenwei_data WHERE index_code=? `
@@ -413,3 +429,26 @@ func MultiUpdateBaseFromFenweiDataValue(items []*BaseFromFenweiData) (err error)
 	}
 	return
 }
+
+// GetBaseFromFenweiIndexByIndexName 根据指标名称查询指标
+func GetBaseFromFenweiIndexByIndexName(indexName string) (item *BaseFromFenweiIndex, err error) {
+	o := orm.NewOrm()
+	sql := ` SELECT * FROM base_from_fenwei_index WHERE index_name=? `
+	err = o.Raw(sql, indexName).QueryRow(&item)
+	return
+}
+
+// GetBaseFromFenweiDataByIndexCodeAndDataTime 根据指标编码和dataTime查询指标数据
+func GetBaseFromFenweiDataByIndexCodeAndDataTime(indexCode, dataTime string) (item *BaseFromFenweiData, err error) {
+	o := orm.NewOrm()
+	sql := ` SELECT * FROM base_from_fenwei_data WHERE index_code=? AND data_time=? `
+	err = o.Raw(sql, indexCode, dataTime).QueryRow(&item)
+	return
+}
+
+// 添加指标数据
+func AddBaseFromFenweiData(item *BaseFromFenweiData) (err error) {
+	o := orm.NewOrm()
+	_, err = o.Insert(item)
+	return
+}

services/base_from_fenwei.go

@@ -6,8 +6,10 @@ import (
 	"eta/eta_index_lib/services/alarm_msg"
 	"eta/eta_index_lib/utils"
 	"fmt"
+	"github.com/mozillazg/go-pinyin"
 	"strings"
 	"time"
+	"unicode"
 )
 
 // HandleFenweiIndex 处理汾渭数据的excel数据
@@ -247,3 +249,203 @@ func handleFenweiIndex(req *models.HandleFenweiExcelData, terminalCode string) (
 	}
 	return
 }
+
+func NetDataHandle(req models.HandleFenWeiNetDataReq) error {
+	indexInfoList := req.List
+	classifyObj := new(models.BaseFromFenweiClassify)
+
+	for _, indexInfo := range indexInfoList {
+
+		// 处理分类信息
+		var classifyId int
+		classify, err := classifyObj.GetByClassifyName(indexInfo.ClassifyName)
+		if err != nil {
+			return err
+		}
+		if classify == nil {
+			classifyId, err = addFenWeiClassify(indexInfo)
+			if err != nil {
+				return err
+			}
+		}
+		classifyId = classify.ClassifyId
+
+		// 处理指标
+		indexCode := GenerateIndexCode(indexInfo.IndexName)
+		index, err := models.GetBaseFromFenweiIndexByIndexName(indexInfo.IndexName)
+		if err != nil {
+			return err
+		}
+		if index == nil {
+			addIndex(indexInfo, classifyId, indexCode)
+		}
+
+		// 处理指标数据
+		err = handleIndexData(indexInfo, indexCode)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func handleIndexData(indexInfo *models.FenWeiNetIndexInfo, indexCode string) error {
+	var format string
+	if isYearMonth(indexInfo.DataTime) {
+		format = convertYearMonthToLastDay(indexInfo.DataTime)
+	}
+	if indexInfo.Frequency == "月度" && isFirstDayOfMonth(indexInfo.DataTime) {
+		format = convertYearMonthDayToLastDay(indexInfo.DataTime)
+	}
+	// 获取指标数据
+	indexData, err := models.GetBaseFromFenweiDataByIndexCodeAndDataTime(indexCode, format)
+	if err != nil {
+		return err
+	}
+	if indexData != nil {
+		// 汾渭不存在数据更新和预测值情况，所以此处未做更新逻辑
+		return nil
+	}
+
+	timestamp := time.Now().UnixNano() / 1e6
+
+	data := new(models.BaseFromFenweiData)
+	data.IndexCode = indexCode
+	data.DataTime = format
+	data.Value = fmt.Sprintf("%v", indexInfo.Value)
+	data.CreateTime = time.Now()
+	data.ModifyTime = time.Now()
+	data.DataTimestamp = timestamp
+
+	err = models.AddBaseFromFenweiData(data)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func addIndex(info *models.FenWeiNetIndexInfo, classifyId int, indexCode string) {
+	index := new(models.BaseFromFenweiIndex)
+	index.IndexName = info.IndexName
+	index.IndexCode = indexCode
+	index.IndexCode = indexCode
+	index.Frequency = info.Frequency
+	index.ClassifyId = classifyId
+	index.Unit = info.Unit
+	index.ModifyTime = time.Now()
+	index.CreateTime = time.Now()
+	index.TerminalCode = info.TerminalCode
+	index.Add()
+}
+
+// addFenWeiClassify 添加分类
+func addFenWeiClassify(info *models.FenWeiNetIndexInfo) (int, error) {
+	classify := new(models.BaseFromFenweiClassify)
+	classify.ClassifyName = info.ClassifyName
+	classify.ParentId = 0
+	classify.Level = 1
+
+	// ClassifyName 拿到”-“分割的字符串 拿到前面的字符串
+	if strings.Contains(classify.ClassifyName, "-") {
+		classifyArr := strings.Split(classify.ClassifyName, "-")
+		if len(classifyArr) > 1 {
+			classify.ClassifyName = classifyArr[0]
+			parentClassify, err := classify.GetByClassifyName(classify.ClassifyName)
+			if err != nil {
+				return 0, err
+			}
+			if parentClassify != nil {
+				classify.ParentId = parentClassify.ClassifyId
+				classify.Level = parentClassify.Level + 1
+			} else {
+				//新增分类
+				classify.ParentId = 0
+				classify.Level = 1
+			}
+		}
+		return 0, nil
+	}
+
+	classify.SysUserId = 0
+	classify.SysUserRealName = ""
+	classify.ModifyTime = time.Now()
+	classify.CreateTime = time.Now()
+	classifyId, err := classify.Add()
+	if err != nil {
+		return 0, err
+	}
+
+	return int(classifyId), nil
+}
+
+// 判断传入参数 dataText 是否是yyyy-MM格式，如果是则返回true，否则返回false
+func isYearMonth(dataText string) bool {
+	_, err := time.Parse("2006-01", dataText)
+	if err != nil {
+		return false
+	}
+	return true
+}
+
+// 判断传入参数 dataText yyyy-MM-dd格式，是否是该月第一天，如果是则返回true，否则返回false
+func isFirstDayOfMonth(dataText string) bool {
+	t, _ := time.Parse("2006-01-02", dataText)
+	if t.Day() == 1 {
+		return true
+	}
+	return false
+}
+
+// 转换时间 dataText yyyy-MM-dd格式，获取该月最后一天的日期
+func convertYearMonthDayToLastDay(dataText string) string {
+	t, _ := time.Parse("2006-01-02", dataText)
+	lastDay := t.AddDate(0, 1, -1)
+	return lastDay.Format("2006-01-02")
+}
+
+// 转换时间 dataText yyyy-MM 格式的时间字符串转换为该月最后一天的日期Date
+func convertYearMonthToLastDay(dataText string) string {
+	t, _ := time.Parse("2006-01", dataText)
+	lastDay := t.AddDate(0, 1, -1)
+	return lastDay.Format("2006-01-02")
+}
+
+// GenerateIndexCode 指标编码规则：粮油商务网拼音首字母+指标名称拼音首字母，数字、字母保留，特殊字符拿掉
+// 例：美湾：9月U：国际大豆进口成本价：期货收盘：张家港 -----> lyswwmw9yUgjddjkcbjqhspzjg
+func GenerateIndexCode(indexName string) string {
+
+	// 获取汉字的拼音首字母，保留数字和大写字母
+	firstLetters := getFirstLetters(indexName)
+
+	// 组合 sourceName 和处理后的拼音首字母
+	indexCode := fmt.Sprintf("%s%s", firstLetters)
+
+	return indexCode
+}
+
+// getFirstLetters 获取汉字的拼音首字母，并保留数字和大写字母
+func getFirstLetters(input string) string {
+	// 设置拼音转换选项，只获取首字母
+	args := pinyin.NewArgs()
+	args.Style = pinyin.FirstLetter
+
+	// 定义用于存储结果的字符串
+	var result strings.Builder
+
+	// 遍历输入字符串中的每个字符
+	for _, r := range input {
+		if unicode.IsDigit(r) || unicode.IsUpper(r) {
+			// 保留数字和大写字母
+			result.WriteRune(r)
+		} else if unicode.Is(unicode.Han, r) {
+			// 如果是汉字，则获取其拼音首字母
+			py := pinyin.Pinyin(string(r), args)
+			if len(py) > 0 && len(py[0]) > 0 {
+				result.WriteString(py[0][0])
+			}
+		}
+		// 对于其他字符，忽略处理
+	}
+
+	return result.String()
+}