涌益日度指标解析

go.mod

@@ -6,6 +6,7 @@ require (
 	github.com/beego/bee/v2 v2.1.0
 	github.com/beego/beego/v2 v2.1.4
 	github.com/gonum/stat v0.0.0-20181125101827-41a0da705a5b
+	github.com/mozillazg/go-pinyin v0.20.0
 	github.com/shopspring/decimal v1.3.1
 	github.com/tealeg/xlsx v1.0.5
 )

go.sum

@@ -42,6 +42,8 @@ github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zk
 github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
 github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
 github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
+github.com/mozillazg/go-pinyin v0.20.0 h1:BtR3DsxpApHfKReaPO1fCqF4pThRwH9uwvXzm+GnMFQ=
+github.com/mozillazg/go-pinyin v0.20.0/go.mod h1:iR4EnMMRXkfpFVV5FMi4FNB6wGq9NV6uDWbUuPhP4Yc=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=

models/base_from_yongyi.go

@@ -50,3 +50,10 @@ type BaseFromYongyiData struct {
 	ModifyTime    time.Time
 	DataTimestamp int64
 }
+
+type YongyiSheet struct {
+	NamePrefix   string
+	ClassifyName string
+	Frequency    string
+	Unit         string
+}

services/base_from_yongyi.go

@@ -3,6 +3,7 @@ package services
 import (
 	"encoding/json"
 	"eta/eta_data_analysis/models"
+	"eta/eta_data_analysis/services/base_from_yongyi"
 	"eta/eta_data_analysis/utils"
 	"fmt"
 	"github.com/tealeg/xlsx"
@@ -22,7 +23,20 @@ func HandleYongyiExcelDaily(uploadPath string) (err error) {
 
 	for _, sheet := range xlFile.Sheet {
 		var indexList []*models.YongyiExcelIndex
-		indexList, err = handleYongyiExcelDaily(sheet)
+
+		switch sheet.Name {
+		/*case "出栏价":
+			indexList, err = base_from_yongyi.HandleYongyiExcelDaily1(sheet)
+		case "标肥价差":
+			indexList, err = base_from_yongyi.HandleYongyiExcelDaily2(sheet)
+		case "价格+宰量":
+			indexList, err = base_from_yongyi.HandleYongyiExcelDaily3(sheet)*/
+		case "屠宰企业日度屠宰量":
+			indexList, err = base_from_yongyi.HandleYongyiExcelDaily4(sheet)
+		default:
+			continue
+		}
+
 		if len(indexList) > 0 {
 			params := make(map[string]interface{})
 			params["List"] = indexList
@@ -46,110 +60,3 @@ func HandleYongyiExcelDaily(uploadPath string) (err error) {
 
 	return
 }
-
-func handleYongyiExcelDaily(sheet *xlsx.Sheet) (indexList []*models.YongyiExcelIndex, err error) {
-	// todo 处理分类信息，一个sheet表示一个分类
-	classifyName := "日度-商品猪出栏价"
-	frequency := "日度"
-	unit := "元/公斤"
-	namePrefix := "商品猪出栏价"
-
-	//sheet := dailyPriceSheet
-	// 遍历行读取
-	indexList = make([]*models.YongyiExcelIndex, 0)
-	dateMap := make(map[int]string)
-	nameMap := make(map[int]string)
-	maxRow := sheet.MaxRow
-	fmt.Println("最大行")
-	fmt.Println(maxRow)
-	// 指标名称
-	indexMap := make(map[string]*models.YongyiExcelIndex)
-	for i := 0; i < maxRow; i++ {
-		fmt.Printf("当前第%d行 \n", i)
-		if i == 0 { // 首行，表示时间
-			row := sheet.Row(i)
-			cells := row.Cells
-			for k, cell := range cells {
-				text := cell.String()
-				if k > 1 && text != "" {
-					if cell.IsTime() {
-						dateText, _ := cell.GetTime(false)
-						text = dateText.Format(utils.FormatDate)
-					}
-					// 检查单元格是否为合并单元格
-					if cell.HMerge > 0 {
-						for j := 1; j <= cell.HMerge; j++ {
-							dateMap[k+j] = text
-						}
-					}
-					fmt.Printf("合并单元格开始列：%d \n", k)
-					dateMap[k] = text
-				}
-			}
-		} else if i == 1 { //表示表头
-			// 处理 index指标表
-			row := sheet.Row(i)
-			cells := row.Cells
-			for k, cell := range cells {
-				text := cell.String()
-				nameMap[k] = text
-			}
-		} else { //数据列
-			row := sheet.Row(i)
-			cells := row.Cells
-			province := ""
-			for k, cell := range cells {
-				fmt.Printf("当前第%d列 \n", k)
-				text := cell.String()
-				if k == 0 {
-					province = text
-					continue
-				} else if k == 1 {
-					continue
-				}
-
-				date, ok1 := dateMap[k]
-				if !ok1 {
-					err = fmt.Errorf("找不到对应的日期，第%d行，第%d列", i, k)
-					return
-				}
-				name, ok2 := nameMap[k]
-				if !ok2 {
-					err = fmt.Errorf("找不到对应的列名，第%d行，第%d列", i, k)
-					return
-				}
-				// todo 放到对应的指标名称的下方
-				if name != "规模场" && name != "小散户" && name != "均价" {
-					// 只处理以上三个类型，其余过滤
-					continue
-				}
-				fmt.Printf("当前第%d行第%d列, 当前省份%s \n", i, k, province)
-				// 处理指标名称
-				fullIndexName := fmt.Sprintf("%s/%s/%s", namePrefix, province, name)
-				indexItem, okIndex := indexMap[fullIndexName]
-
-				if !okIndex {
-					// 新增指标
-					indexItem = new(models.YongyiExcelIndex)
-					indexItem.IndexName = fullIndexName
-					indexItem.ClassifyName = classifyName
-					// todo 处理indexCode
-					indexItem.IndexCode = fullIndexName
-					indexItem.Frequency = frequency
-					indexItem.Unit = unit
-					indexItem.ExcelDataMap = make(map[string]string)
-				}
-				fmt.Printf("indexItem%s", indexItem.IndexCode)
-
-				indexItem.ExcelDataMap[date] = text
-				indexMap[fullIndexName] = indexItem
-				continue
-			}
-		}
-	}
-	for _, v := range indexMap {
-		indexList = append(indexList, v)
-	}
-	// todo 整理请求入参，并加入，改成一个指标请求一次还是多个指标请求一次
-	return
-}

services/base_from_yongyi/daily.go

@@ -0,0 +1,426 @@
+package base_from_yongyi
+
+import (
+	"eta/eta_data_analysis/models"
+	"eta/eta_data_analysis/utils"
+	"fmt"
+	"github.com/tealeg/xlsx"
+	"strings"
+	"time"
+)
+
+var yongyiSheetMap map[string]*models.YongyiSheet
+var yongyiNotAddMap map[string][]string
+var yongyiSpecialNameMap map[string]map[string]string
+var yongyiSpecialUnitMap map[string]string
+
+func init() {
+	yongyiSheetMap = map[string]*models.YongyiSheet{
+		"出栏价":       {"商品猪出栏价", "日度-商品猪出栏价", "日度", "元/公斤"},
+		"标肥价差":      {"", "日度-商品猪标肥价差", "日度", "元/公斤"},
+		"价格+宰量":     {"", "", "日度", "元/公斤"},
+		"屠宰企业日度屠宰量": {"商品猪日屠宰量", "日度-屠宰企业屠宰量", "日度", "头/日"},
+	}
+	yongyiNotAddMap = map[string][]string{"出栏价": {"商品猪出栏价/全国/均价"}, "屠宰企业日度屠宰量": {"商品猪日屠宰量/合计（单位：头）"}}
+	yongyiSpecialNameMap = map[string]map[string]string{"价格+宰量": {"全国均价": "商品猪出栏价/全国/均价", "日屠宰量": "商品猪日屠宰量/全国"}}
+	yongyiSpecialUnitMap = map[string]string{"商品猪日屠宰量/全国": "头/日"}
+}
+
+// HandleYongyiExcelDaily1 日度-商品猪出栏价
+func HandleYongyiExcelDaily1(sheet *xlsx.Sheet) (indexList []*models.YongyiExcelIndex, err error) {
+	classifyName := "日度-商品猪出栏价"
+	frequency := "日度"
+	unit := "元/公斤"
+	namePrefix := "商品猪出栏价"
+
+	//sheet := dailyPriceSheet
+	// 遍历行读取
+	indexList = make([]*models.YongyiExcelIndex, 0)
+	dateMap := make(map[int]string)
+	nameMap := make(map[int]string)
+	maxRow := sheet.MaxRow
+	fmt.Println("最大行")
+	fmt.Println(maxRow)
+	// 指标名称
+	indexMap := make(map[string]*models.YongyiExcelIndex)
+	for i := 0; i < maxRow; i++ {
+		fmt.Printf("当前第%d行 \n", i)
+		if i == 0 { // 首行，表示时间
+			row := sheet.Row(i)
+			cells := row.Cells
+			for k, cell := range cells {
+				text := cell.String()
+				if k > 1 && text != "" {
+					if cell.IsTime() {
+						dateText, _ := cell.GetTime(false)
+						text = dateText.Format(utils.FormatDate)
+					}
+					// 检查单元格是否为合并单元格
+					if cell.HMerge > 0 {
+						for j := 1; j <= cell.HMerge; j++ {
+							dateMap[k+j] = text
+						}
+					}
+					fmt.Printf("合并单元格开始列：%d \n", k)
+					dateMap[k] = text
+				}
+			}
+		} else if i == 1 { //表示表头
+			// 处理 index指标表
+			row := sheet.Row(i)
+			cells := row.Cells
+			for k, cell := range cells {
+				text := cell.String()
+				nameMap[k] = text
+			}
+		} else { //数据列
+			row := sheet.Row(i)
+			cells := row.Cells
+			province := ""
+			for k, cell := range cells {
+				fmt.Printf("当前第%d列 \n", k)
+				text := cell.String()
+				if k == 0 {
+					province = text
+					continue
+				} else if k == 1 {
+					continue
+				}
+
+				date, ok1 := dateMap[k]
+				if !ok1 {
+					err = fmt.Errorf("找不到对应的日期，第%d行，第%d列", i, k)
+					return
+				}
+				name, ok2 := nameMap[k]
+				if !ok2 {
+					err = fmt.Errorf("找不到对应的列名，第%d行，第%d列", i, k)
+					return
+				}
+				fmt.Printf("当前第%d行第%d列, 当前省份%s \n", i, k, province)
+				// 处理指标名称
+				fullIndexName := fmt.Sprintf("%s/%s/%s", namePrefix, province, name)
+
+				if name != "规模场" && name != "小散户" && name != "均价" {
+					// 只处理以上三个类型，其余过滤
+					continue
+				}
+
+				if fullIndexName == "商品猪出栏价/全国/均价" {
+					continue
+				}
+
+				indexItem, okIndex := indexMap[fullIndexName]
+
+				if !okIndex {
+					// 新增指标
+					indexItem = new(models.YongyiExcelIndex)
+					indexItem.IndexName = fullIndexName
+					indexItem.ClassifyName = classifyName
+					// todo 处理indexCode
+					indexItem.IndexCode = fullIndexName
+					indexItem.Frequency = frequency
+					indexItem.Unit = unit
+					indexItem.ExcelDataMap = make(map[string]string)
+				}
+				fmt.Printf("indexItem%s", indexItem.IndexCode)
+
+				indexItem.ExcelDataMap[date] = text
+				indexMap[fullIndexName] = indexItem
+				continue
+			}
+		}
+	}
+	for _, v := range indexMap {
+		indexList = append(indexList, v)
+	}
+	// todo 整理请求入参，并加入，改成一个指标请求一次还是多个指标请求一次
+	return
+}
+
+// HandleYongyiExcelDaily2 日度-商品猪标肥价差
+func HandleYongyiExcelDaily2(sheet *xlsx.Sheet) (indexList []*models.YongyiExcelIndex, err error) {
+	classifyName := "日度-商品猪标肥价差"
+	frequency := "日度"
+	unit := "元/公斤"
+
+	// 遍历行读取
+	indexList = make([]*models.YongyiExcelIndex, 0)
+	dateMap := make(map[int]string)
+	nameMap := make(map[int]string)
+	maxRow := sheet.MaxRow
+	fmt.Println("最大行")
+	fmt.Println(maxRow)
+	// 指标名称
+	indexMap := make(map[string]*models.YongyiExcelIndex)
+	for i := 0; i < maxRow; i++ {
+		fmt.Printf("当前第%d行 \n", i)
+		if i == 0 { // 首行，表示时间
+			row := sheet.Row(i)
+			cells := row.Cells
+			for k, cell := range cells {
+				text := cell.String()
+				if text != "" {
+					if cell.IsTime() {
+						dateText, _ := cell.GetTime(false)
+						text = dateText.Format(utils.FormatDate)
+					}
+					// 检查单元格是否为合并单元格
+					if cell.HMerge > 0 {
+						for j := 1; j <= cell.HMerge; j++ {
+							dateMap[k+j] = text
+						}
+					}
+					fmt.Printf("合并单元格开始列：%d \n", k)
+					dateMap[k] = text
+				}
+			}
+		} else if i == 1 { //表示表头
+			// 处理 index指标表
+			row := sheet.Row(i)
+			cells := row.Cells
+			for k, cell := range cells {
+				text := cell.String()
+				nameMap[k] = text
+			}
+		} else { //数据列
+			row := sheet.Row(i)
+			cells := row.Cells
+			province := ""
+			for k, cell := range cells {
+				fmt.Printf("当前第%d列 \n", k)
+				text := cell.String()
+				if k == 0 {
+					province = text
+					continue
+				}
+
+				date, ok1 := dateMap[k]
+				if !ok1 {
+					err = fmt.Errorf("找不到对应的日期，第%d行，第%d列", i, k)
+					return
+				}
+				name, ok2 := nameMap[k]
+				if !ok2 {
+					err = fmt.Errorf("找不到对应的列名，第%d行，第%d列", i, k)
+					return
+				}
+				fmt.Printf("当前第%d行第%d列, 当前省份%s \n", i, k, province)
+				// 处理指标名称
+
+				// 过滤特殊的指标 放到对应的指标名称的下方
+
+				if name != "市场标重猪价" && name != "150公斤左右较标猪" && name != "175公斤左右较标猪" {
+					// 只处理以上三个类型，其余过滤
+					continue
+				}
+				if name == "150公斤左右较标猪" {
+					name = "150公斤较标猪价差"
+				} else if name == "175公斤左右较标猪" {
+					name = "175公斤较标猪价差"
+				}
+				fullIndexName := fmt.Sprintf("%s/%s", name, province)
+
+				indexItem, okIndex := indexMap[fullIndexName]
+
+				if !okIndex {
+					// 新增指标
+					indexItem = new(models.YongyiExcelIndex)
+					indexItem.IndexName = fullIndexName
+					indexItem.ClassifyName = classifyName
+					// todo 处理indexCode
+					indexItem.IndexCode = fullIndexName
+					indexItem.Frequency = frequency
+					indexItem.Unit = unit
+					indexItem.ExcelDataMap = make(map[string]string)
+				}
+				fmt.Printf("indexItem%s", indexItem.IndexCode)
+
+				indexItem.ExcelDataMap[date] = text
+				indexMap[fullIndexName] = indexItem
+				continue
+			}
+		}
+	}
+	for _, v := range indexMap {
+		indexList = append(indexList, v)
+	}
+	// todo 整理请求入参，并加入，改成一个指标请求一次还是多个指标请求一次
+	return
+}
+
+// HandleYongyiExcelDaily3 日度-商品猪全国均价和宰量
+func HandleYongyiExcelDaily3(sheet *xlsx.Sheet) (indexList []*models.YongyiExcelIndex, err error) {
+	classifyName := "日度-商品猪全国均价和宰量"
+	frequency := "日度"
+	unit := ""
+
+	// 遍历行读取
+	indexList = make([]*models.YongyiExcelIndex, 0)
+	nameMap := make(map[int]string)
+	maxRow := sheet.MaxRow
+	fmt.Println("最大行")
+	fmt.Println(maxRow)
+	// 指标名称
+	indexMap := make(map[string]*models.YongyiExcelIndex)
+	for i := 0; i < maxRow; i++ {
+		fmt.Printf("当前第%d行 \n", i)
+		if i == 0 { // 首行，名称
+			row := sheet.Row(i)
+			cells := row.Cells
+			for k, cell := range cells {
+				text := cell.String()
+				nameMap[k] = text
+			}
+		} else { //数据列
+			row := sheet.Row(i)
+			cells := row.Cells
+			date := ""
+			for k, cell := range cells {
+				fmt.Printf("当前第%d列 \n", k)
+				text := cell.String()
+				if k == 0 {
+					if cell.IsTime() {
+						dateText, _ := cell.GetTime(false)
+						text = dateText.Format(utils.FormatDate)
+					}
+					date = text
+					continue
+				}
+
+				name, ok2 := nameMap[k]
+				if !ok2 {
+					err = fmt.Errorf("找不到对应的列名，第%d行，第%d列", i, k)
+					return
+				}
+				if name != "全国均价" && name != "日屠宰量" {
+					// 只处理以上三个类型，其余过滤
+					continue
+				}
+				if name == "全国均价" {
+					name = "商品猪出拦价/全国/均价"
+					unit = "元/公斤"
+				} else if name == "日屠宰量" {
+					name = "商品猪日屠宰量/全国"
+					unit = "头/日"
+				}
+				fullIndexName := name
+
+				indexItem, okIndex := indexMap[fullIndexName]
+
+				if !okIndex {
+					// 新增指标
+					indexItem = new(models.YongyiExcelIndex)
+					indexItem.IndexName = fullIndexName
+					indexItem.ClassifyName = classifyName
+					// todo 处理indexCode
+					indexItem.IndexCode = fullIndexName
+					indexItem.Frequency = frequency
+					indexItem.Unit = unit
+					indexItem.ExcelDataMap = make(map[string]string)
+				}
+				fmt.Printf("indexItem%s", indexItem.IndexCode)
+
+				indexItem.ExcelDataMap[date] = text
+				indexMap[fullIndexName] = indexItem
+			}
+		}
+	}
+	for _, v := range indexMap {
+		indexList = append(indexList, v)
+	}
+	// todo 整理请求入参，并加入，改成一个指标请求一次还是多个指标请求一次
+	return
+}
+
+// HandleYongyiExcelDaily4 日度-屠宰企业屠宰量
+func HandleYongyiExcelDaily4(sheet *xlsx.Sheet) (indexList []*models.YongyiExcelIndex, err error) {
+	classifyName := "日度-屠宰企业屠宰量"
+	frequency := "日度"
+	unit := "头/日"
+	namePrefix := "商品猪日屠宰量"
+	namePrefixPingin := utils.GetFirstPingYin(namePrefix)
+	// 遍历行读取
+	indexList = make([]*models.YongyiExcelIndex, 0)
+	dateMap := make(map[int]string)
+	maxRow := sheet.MaxRow
+	fmt.Println("最大行")
+	fmt.Println(maxRow)
+	// 指标名称
+	indexMap := make(map[string]*models.YongyiExcelIndex)
+	for i := 0; i < maxRow; i++ {
+		fmt.Printf("当前第%d行 \n", i)
+		if i == 0 { // 首行，表示时间
+			row := sheet.Row(i)
+			cells := row.Cells
+			for k, cell := range cells {
+				text := cell.String()
+				if text != "" && text != "省份" {
+					if cell.IsTime() {
+						dateText, _ := cell.GetTime(false)
+						text = dateText.Format(utils.FormatDate)
+					} else {
+						if strings.Contains(text, "（") {
+							tmp := strings.Split(text, "（")
+							text = tmp[0]
+						}
+						timeTmp, tErr := time.ParseInLocation(utils.FormatDate1, text, time.Local)
+						if tErr != nil {
+							err = fmt.Errorf("时间格式转换失败，Err:%s", tErr)
+							return
+						}
+						text = timeTmp.Format(utils.FormatDate)
+					}
+
+					dateMap[k] = text
+				}
+			}
+		} else { //数据列
+			row := sheet.Row(i)
+			cells := row.Cells
+			province := ""
+			for k, cell := range cells {
+				fmt.Printf("当前第%d列 \n", k)
+				text := cell.String()
+				if k == 0 {
+					province = text
+					continue
+				}
+
+				date, ok1 := dateMap[k]
+				if !ok1 {
+					err = fmt.Errorf("找不到对应的日期，第%d行，第%d列", i, k)
+					return
+				}
+				fmt.Printf("当前第%d行第%d列, 当前省份%s \n", i, k, province)
+				// 处理指标名称
+
+				// 过滤特殊的指标 放到对应的指标名称的下方
+				fullIndexName := fmt.Sprintf("%s/%s", namePrefix, province)
+
+				provincePingyin := utils.GetFullPingYin(province)
+				fullIndexNamePingyin := fmt.Sprintf("%s/%s", namePrefixPingin, provincePingyin)
+				indexItem, okIndex := indexMap[fullIndexName]
+
+				if !okIndex {
+					// 新增指标
+					indexItem = new(models.YongyiExcelIndex)
+					indexItem.IndexName = fullIndexName
+					indexItem.ClassifyName = classifyName
+					indexItem.IndexCode = fullIndexNamePingyin
+					indexItem.Frequency = frequency
+					indexItem.Unit = unit
+					indexItem.ExcelDataMap = make(map[string]string)
+				}
+				fmt.Printf("indexItem%s", indexItem.IndexCode)
+
+				indexItem.ExcelDataMap[date] = text
+				indexMap[fullIndexName] = indexItem
+			}
+		}
+	}
+	for _, v := range indexMap {
+		indexList = append(indexList, v)
+	}
+	return
+}

services/base_from_yongyi/week.go

@@ -0,0 +1 @@
+package base_from_yongyi

services/task.go

@@ -7,6 +7,6 @@ import (
 func Task() {
 	fmt.Println("task start")
 
-	HandleYongyiExcelDaily("/Users/xiexiaoyuan/Downloads/test.xlsx")
+	//HandleYongyiExcelDaily("/Users/xiexiaoyuan/Downloads/test.xlsx")
 	fmt.Println("task end")
 }

utils/common.go

@@ -9,6 +9,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"github.com/mozillazg/go-pinyin"
 	"github.com/shopspring/decimal"
 	"image"
 	"image/png"
@@ -27,6 +28,35 @@ import (
 	"time"
 )
 
+// GetFirstPingYin 拼音首字母
+func GetFirstPingYin(name string) string {
+	a := pinyin.NewArgs()
+	code := ""
+	rows := pinyin.Pinyin(name, a)
+	for i := 0; i < len(rows); i++ {
+		if len(rows[i]) != 0 {
+			str := rows[i][0]
+			pi := str[0:1]
+			code += pi
+		}
+	}
+	return code
+}
+
+// GetFullPingYin 获取全拼音
+func GetFullPingYin(name string) string {
+	a := pinyin.NewArgs()
+	code := ""
+	rows := pinyin.Pinyin(name, a)
+	for i := 0; i < len(rows); i++ {
+		if len(rows[i]) != 0 {
+			str := rows[i][0]
+			code += str
+		}
+	}
+	return code
+}
+
 // 随机数种子
 var rnd = rand.New(rand.NewSource(time.Now().UnixNano()))
 

utils/constants.go

@@ -14,6 +14,7 @@ const (
 	FormatYearMonthDate        = "2006-01"                 //日期格式
 	FormatYearMonthUnSpace     = "200601"                  //年月的日期格式
 	PageSize15                 = 15                        //列表页每页数据量
+	FormatDate1                = "2006/1/02"               //日期格式
 	PageSize5                  = 5
 	PageSize10                 = 10
 	PageSize20                 = 20