..

models/base_from_pig.go

@@ -1,26 +0,0 @@
-package models
-
-import (
-	"github.com/beego/beego/v2/client/orm"
-)
-
-type BaseFromPigTest struct {
-	Id              int `orm:"column(id);pk"`
-	Date            string
-	ScaleField      string
-	Retail          string
-	AveragePrice    string
-	YesterdayChange string
-	LastYear        string
-	Yoy             string
-	Tomorrow        string
-	Province        string
-	Area            string
-}
-
-// Add 新增
-func (r *BaseFromPigTest) Add(list []BaseFromPigTest) (err error) {
-	o := orm.NewOrmUsingDB("data")
-	_, err = o.InsertMulti(len(list), list)
-	return
-}

models/base_from_yongyi.go

@@ -0,0 +1,70 @@
+package models
+
+import (
+	"github.com/beego/beego/v2/client/orm"
+	"time"
+)
+
+type BaseFromYongyiIndex struct {
+	YongyiIndexId int `orm:"column(yongyi_index_id);pk"`
+	ClassifyId    int
+	IndexCode     string
+	IndexName     string
+	Frequency     string
+	Unit          string
+	Sort          int
+	CreateTime    time.Time
+	ModifyTime    time.Time
+}
+
+// Add 新增
+func (y *BaseFromYongyiIndex) Add(list []*BaseFromYongyiIndex) (err error) {
+	o := orm.NewOrmUsingDB("data")
+	_, err = o.InsertMulti(len(list), list)
+	return
+}
+
+func GetBaseFromYongyiIndexByClassifyId(classifyId int) (list []*BaseFromYongyiIndex, err error) {
+	o := orm.NewOrmUsingDB("data")
+	sql := `SELECT * FROM base_from_yongyi_index where classify_id=?`
+	_, err = o.Raw(sql, classifyId).QueryRows(&list)
+	return
+}
+
+type BaseFromYongyiData struct {
+	YongyiDataId  int `orm:"column(yongyi_data_id);pk"`
+	YongyiIndexId int
+	IndexCode     string
+	DataTime      string
+	Value         string
+	CreateTime    time.Time
+	ModifyTime    time.Time
+	DataTimestamp int64
+}
+
+// Add 新增
+func (y *BaseFromYongyiData) Add(list []*BaseFromYongyiData) (err error) {
+	o := orm.NewOrmUsingDB("data")
+	_, err = o.InsertMulti(len(list), list)
+	return
+}
+
+// BaseFromYongyiClassify 涌益咨询原始数据分类表
+type BaseFromYongyiClassify struct {
+	ClassifyId      int       `orm:"column(classify_id);pk"`
+	ClassifyName    string    `description:"分类名称"`
+	ParentId        int       `description:"父级id"`
+	SysUserId       int       `description:"创建人id"`
+	SysUserRealName string    `description:"创建人姓名"`
+	Level           int       `description:"层级"`
+	Sort            int       `description:"排序字段，越小越靠前，默认值：10"`
+	ModifyTime      time.Time `description:"修改时间"`
+	CreateTime      time.Time `description:"创建时间"`
+}
+
+// Add 新增
+func (y *BaseFromYongyiClassify) Add(list []*BaseFromYongyiClassify) (err error) {
+	o := orm.NewOrmUsingDB("data")
+	_, err = o.InsertMulti(len(list), list)
+	return
+}

models/db.go

@@ -56,6 +56,8 @@ func init() {
 		new(BaseFromNationalStatisticsIndex),
 		new(BaseFromNationalStatisticsData),
 		new(MeetingProbabilities),
-		new(BaseFromPigTest),
+		new(BaseFromYongyiClassify),
+		new(BaseFromYongyiIndex),
+		new(BaseFromYongyiData),
 	)
 }

services/yongyi_pig.go

@@ -2,6 +2,7 @@ package services
 
 import (
 	"eta/eta_crawler/models"
+	"eta/eta_crawler/utils"
 	"fmt"
 	"github.com/tealeg/xlsx"
 	"strconv"
@@ -10,7 +11,6 @@ import (
 
 func HandleYongyiExcelDaily(uploadPath string) {
 	var err error
-
 	xlFile, err := xlsx.OpenFile(uploadPath)
 	if err != nil {
 		err = fmt.Errorf("打开文件失败, Err: %s", err)
@@ -20,21 +20,37 @@ func HandleYongyiExcelDaily(uploadPath string) {
 	dateMap := make(map[int]string)
 	nameMap := make(map[int]string)
 
-	dataProvinceMap := make([]models.BaseFromPigTest, 0)
+	indexDataList := make([]*models.BaseFromYongyiData, 0)
+	indexList := make([]*models.BaseFromYongyiIndex, 0)
 	dailyPriceSheet, ok := xlFile.Sheet["出栏价"]
 	if ok {
+		// todo 处理分类信息，一个sheet表示一个分类
+		//classifyName := "日度-商品猪出栏价"
+		classifyId := 1
+		now := time.Now()
+		frequency := "日度"
+		unit := "元/公斤"
+		namePrefix := "商品猪出栏价"
+
 		sheet := dailyPriceSheet
 		// 遍历行读取
 		maxRow := sheet.MaxRow
 		fmt.Println("最大行")
 		fmt.Println(maxRow)
+		// 指标名称
+		indexMap := make(map[string]*models.BaseFromYongyiIndex)
 		for i := 0; i < maxRow; i++ {
+			fmt.Printf("当前第%d行 \n", i)
 			if i == 0 { // 首行，表示时间
 				row := sheet.Row(i)
 				cells := row.Cells
 				for k, cell := range cells {
 					text := cell.String()
 					if k > 1 && text != "" {
+						if cell.IsTime() {
+							dateText, _ := cell.GetTime(false)
+							text = dateText.Format(utils.FormatDate)
+						}
 						// 检查单元格是否为合并单元格
 						if cell.HMerge > 0 {
 							for j := 1; j <= cell.HMerge; j++ {
@@ -46,6 +62,7 @@ func HandleYongyiExcelDaily(uploadPath string) {
 					}
 				}
 			} else if i == 1 { //表示表头
+				// 处理 index指标表
 				row := sheet.Row(i)
 				cells := row.Cells
 				for k, cell := range cells {
@@ -53,18 +70,28 @@ func HandleYongyiExcelDaily(uploadPath string) {
 					nameMap[k] = text
 				}
 			} else { //数据列
+				// 新增指标记录，并返回对应的ID
+				existIndexMap := make(map[string]*models.BaseFromYongyiIndex)
+
+				//获取所有指标信息
+				allIndex, err := models.GetBaseFromYongyiIndexByClassifyId(classifyId)
+				if err != nil {
+					return
+				}
+				for _, v := range allIndex {
+					indexKey := v.IndexName
+					existIndexMap[indexKey] = v
+				}
 				row := sheet.Row(i)
 				cells := row.Cells
 				province := ""
-				area := ""
-				dataMap := make(map[string]models.BaseFromPigTest)
 				for k, cell := range cells {
+					fmt.Printf("当前第%d列 \n", k)
 					text := cell.String()
 					if k == 0 {
 						province = text
 						continue
 					} else if k == 1 {
-						area = text
 						continue
 					}
 
@@ -78,41 +105,69 @@ func HandleYongyiExcelDaily(uploadPath string) {
 						err = fmt.Errorf("找不到对应的列名，第%d行，第%d列", i, k)
 						return
 					}
-					var item models.BaseFromPigTest
-					item, _ = dataMap[date]
-					switch name {
-					case "规模场":
-						item.ScaleField = text
-					case "小散户":
-						item.Retail = text
-					case "均价":
-						item.AveragePrice = text
-					case "较昨日涨跌":
-						item.YesterdayChange = text
-					case "去年同期":
-						item.LastYear = text
-					case "同比":
-						item.Yoy = text
-					case "明日预计":
-						item.Tomorrow = text
+					// todo 放到对应的指标名称的下方
+					if name != "规模场" && name != "小散户" && name != "均价" {
+						// 只处理以上三个类型，其余过滤
+						continue
 					}
-					dataMap[date] = item
-				}
+					fmt.Printf("当前第%d行第%d列, 当前省份%s \n", i, k, province)
+					// 处理指标名称
+					fullIndexName := fmt.Sprintf("%s/%s/%s", namePrefix, province, name)
+					indexItem, okIndex := indexMap[fullIndexName]
+
+					if !okIndex {
+						// 新增指标
+						indexItem = new(models.BaseFromYongyiIndex)
+						indexItem.IndexName = fullIndexName
+						indexItem.ClassifyId = classifyId
+						// todo 处理indexCode
+						indexItem.IndexCode = fullIndexName
+						indexItem.Frequency = frequency
+						indexItem.Unit = unit
+						indexItem.ModifyTime = now
+						indexItem.CreateTime = now
+						// todo 查询是否已存在，如果已存在，则获取已存在的指标ID
+						if exist, ok3 := existIndexMap[fullIndexName]; ok3 {
+							indexItem.YongyiIndexId = exist.YongyiIndexId
+						} else {
+							indexList = append(indexList, indexItem)
+						}
+						indexMap[fullIndexName] = indexItem
+					} else {
+						// todo 查询是否已存在，如果已存在，则获取已存在的指标ID
+						if exist, ok3 := existIndexMap[fullIndexName]; ok3 {
+							indexItem.YongyiIndexId = exist.YongyiIndexId
+						}
+					}
+					fmt.Printf("indexItem%s", indexItem.IndexCode)
+
+					dataItem := new(models.BaseFromYongyiData)
+					dataItem.IndexCode = indexItem.IndexCode
+					dataItem.YongyiIndexId = indexItem.YongyiIndexId
+					dataItem.DataTime = date
+					dataItem.Value = text
+					dataItem.CreateTime = now
+					dataItem.ModifyTime = now
+					// 判断指标是否已存在
+					indexDataList = append(indexDataList, dataItem)
+					continue
 
-				// 重新整理并汇总
-				for k, item := range dataMap {
-					item.Province = province
-					item.Area = area
-					item.Date = k
-					dataProvinceMap = append(dataProvinceMap, item)
 				}
 			}
 		}
 
+		if len(indexList) > 0 {
+			index := new(models.BaseFromYongyiIndex)
+			err = index.Add(indexList)
+			if err != nil {
+				fmt.Println(err)
+				return
+			}
+		}
 		// 处理最终的数据并入库
-		if len(dataProvinceMap) > 0 {
-			pig := new(models.BaseFromPigTest)
-			err = pig.Add(dataProvinceMap)
+		if len(indexDataList) > 0 {
+			pigData := new(models.BaseFromYongyiData)
+			err = pigData.Add(indexDataList)
 			if err != nil {
 				fmt.Println(err)
 				return