add:涌益三期指标爬取

services/base_from_yongyi.go

@@ -7,7 +7,6 @@ import (
 	"eta/eta_data_analysis/services/base_from_yongyi_v2"
 	"eta/eta_data_analysis/utils"
 	"fmt"
-	"github.com/xuri/excelize/v2"
 	"io/fs"
 	"os"
 	"path"
@@ -15,6 +14,8 @@ import (
 	"strings"
 	"sync"
 	"time"
+
+	"github.com/xuri/excelize/v2"
 )
 
 /*
@@ -343,55 +344,69 @@ func ReadWatchYongyiFile(cont context.Context) (err error) {
 }
 
 var YongyiDailyHandlers = map[string]func(f *excelize.File, sheetName string) ([]*models.YongyiExcelIndex, error){
-	"出栏价":             base_from_yongyi_v2.HandleYongyiExcelDaily1,
-	"标肥价差":           base_from_yongyi_v2.HandleYongyiExcelDaily2,
-	"价格+宰量":          base_from_yongyi_v2.HandleYongyiExcelDaily3,
+	"出栏价":       base_from_yongyi_v2.HandleYongyiExcelDaily1,
+	"标肥价差":      base_from_yongyi_v2.HandleYongyiExcelDaily2,
+	"价格+宰量":     base_from_yongyi_v2.HandleYongyiExcelDaily3,
 	"屠宰企业日度屠宰量": base_from_yongyi_v2.HandleYongyiExcelDaily4,
 }
 
 var YongyiWeeklyHandlers = map[string]func(f *excelize.File, sheetName string) ([]*models.YongyiExcelIndex, error){
-	"周度-商品猪出栏价":                     base_from_yongyi_v2.HandleYongyiExcelWeekly1,
-	"周度-体重":                             base_from_yongyi_v2.HandleYongyiExcelWeekly2,
-	"周度-屠宰厂宰前活猪重":                 base_from_yongyi_v2.HandleYongyiExcelWeekly3,
-	"周度-各体重段价差":                     base_from_yongyi_v2.HandleYongyiExcelWeekly4,
-	"周度-50公斤二元母猪价格":               base_from_yongyi_v2.HandleYongyiExcelWeekly5,
-	"周度-规模场15公斤仔猪出栏价":           base_from_yongyi_v2.HandleYongyiExcelWeekly5,
-	"周度-宰后结算价":                       base_from_yongyi_v2.HandleYongyiExcelWeekly5,
-	"周度-冻品库存":                         base_from_yongyi_v2.HandleYongyiExcelWeekly5,
-	"周度-猪肉价（前三等级白条均价）":         base_from_yongyi_v2.HandleYongyiExcelWeekly5,
-	"周度-冻品库存多样本":                   base_from_yongyi_v2.HandleYongyiExcelWeekly6,
-	"月度出栏完成率":                        base_from_yongyi_v2.HandleYongyiExcelWeekly6,
-	"月度计划出栏量":                        base_from_yongyi_v2.HandleYongyiExcelWeekly7,
-	"月度-能繁母猪存栏（2020年2月新增）":      base_from_yongyi_v2.HandleYongyiExcelWeekly8,
-	"月度-小猪存栏（2020年5月新增）":          base_from_yongyi_v2.HandleYongyiExcelWeekly9,
-	"月度-中猪存栏（2020年5月新增）":          base_from_yongyi_v2.HandleYongyiExcelWeekly9,
-	"月度-大猪存栏（2020年5月新增）":          base_from_yongyi_v2.HandleYongyiExcelWeekly9,
-	"月度-商品猪出栏量":                     base_from_yongyi_v2.HandleYongyiExcelWeekly10,
-	"历史猪价":                              base_from_yongyi_v2.HandleYongyiExcelWeekly12,
-	"二育成本":                              base_from_yongyi_v2.HandleYongyiExcelWeekly13,
-	"二育销量":                              base_from_yongyi_v2.HandleYongyiExcelWeekly14,
-	"育肥栏舍利用率":                        base_from_yongyi_v2.HandleYongyiExcelWeekly15,
-	"周度-养殖利润最新":                     base_from_yongyi_v2.HandleYongyiExcelWeekly16,
-	"周度-当期、预期成本":                    base_from_yongyi_v2.HandleYongyiExcelWeekly17,
-	"育肥全价料出厂价":                      base_from_yongyi_v2.HandleYongyiExcelWeekly18,
-	"周度-成本计算附件":                     base_from_yongyi_v2.HandleYongyiExcelWeekly19,
-	"周度-毛白价差":                         base_from_yongyi_v2.HandleYongyiExcelWeekly20,
-	"仔猪、母猪":                             base_from_yongyi_v2.HandleYongyiExcelWeekly21,
-	"周度-河南屠宰白条成本":                 base_from_yongyi_v2.HandleYongyiExcelWeekly22,
-	"周度-淘汰母猪价格":                     base_from_yongyi_v2.HandleYongyiExcelWeekly23, //todo 少两个指标，云南和贵州，数据都不符合规范
-	"鲜销率":                                base_from_yongyi_v2.HandleYongyiExcelWeekly24, //todo 等待解析（正月初六-二月初四）2022.2.6-3.6
-	"周度-猪肉产品价格":                     base_from_yongyi_v2.HandleYongyiExcelWeekly25,
-	"周度-屠宰企业日度屠宰量":               base_from_yongyi_v2.HandleYongyiExcelWeekly26,
-	"周度-屠宰新2022.10.28":                 base_from_yongyi_v2.HandleYongyiExcelWeekly27,
-	"月度-淘汰母猪屠宰厂宰杀量":             base_from_yongyi_v2.HandleYongyiExcelWeekly28,
-	"月度-猪料销量":                         base_from_yongyi_v2.HandleYongyiExcelWeekly29,
-	"月度-屠宰企业开工率":                   base_from_yongyi_v2.HandleYongyiExcelWeekly30,
-	"月度-屠宰厂公母比例":                   base_from_yongyi_v2.HandleYongyiExcelWeekly31,
-	"月度-生产指标（2021.5.7新增）":           base_from_yongyi_v2.HandleYongyiExcelWeekly32,
-	"月度-生产指标2":                        base_from_yongyi_v2.HandleYongyiExcelWeekly33,
-	"月度-二元三元能繁比例":                 base_from_yongyi_v2.HandleYongyiExcelWeekly34, //todo 2019年1-6月
-	"月度-能繁母猪存栏量":                   base_from_yongyi_v2.HandleYongyiExcelWeekly35,
-	"月度-原种场二元后备母猪销量及出栏日龄": base_from_yongyi_v2.HandleYongyiExcelWeekly11,
+	"周度-商品猪出栏价":            base_from_yongyi_v2.HandleYongyiExcelWeekly1,
+	"周度-体重":                base_from_yongyi_v2.HandleYongyiExcelWeekly2,
+	"周度-屠宰厂宰前活猪重":          base_from_yongyi_v2.HandleYongyiExcelWeekly3,
+	"周度-各体重段价差":            base_from_yongyi_v2.HandleYongyiExcelWeekly4,
+	"周度-50公斤二元母猪价格":        base_from_yongyi_v2.HandleYongyiExcelWeekly5,
+	"周度-规模场15公斤仔猪出栏价":      base_from_yongyi_v2.HandleYongyiExcelWeekly5,
+	"周度-宰后结算价":             base_from_yongyi_v2.HandleYongyiExcelWeekly5,
+	"周度-冻品库存":              base_from_yongyi_v2.HandleYongyiExcelWeekly5,
+	"周度-猪肉价（前三等级白条均价）":     base_from_yongyi_v2.HandleYongyiExcelWeekly5,
+	"周度-冻品库存多样本":           base_from_yongyi_v2.HandleYongyiExcelWeekly6,
+	"月度出栏完成率":              base_from_yongyi_v2.HandleYongyiExcelWeekly6,
+	"月度计划出栏量":              base_from_yongyi_v2.HandleYongyiExcelWeekly7,
+	"月度-能繁母猪存栏（2020年2月新增）": base_from_yongyi_v2.HandleYongyiExcelWeekly8,
+	"月度-小猪存栏（2020年5月新增）":   base_from_yongyi_v2.HandleYongyiExcelWeekly9,
+	"月度-中猪存栏（2020年5月新增）":   base_from_yongyi_v2.HandleYongyiExcelWeekly9,
+	"月度-大猪存栏（2020年5月新增）":   base_from_yongyi_v2.HandleYongyiExcelWeekly9,
+	"月度-商品猪出栏量":            base_from_yongyi_v2.HandleYongyiExcelWeekly10,
+	"历史猪价":                 base_from_yongyi_v2.HandleYongyiExcelWeekly12,
+	"二育成本":                 base_from_yongyi_v2.HandleYongyiExcelWeekly13,
+	"二育销量":                 base_from_yongyi_v2.HandleYongyiExcelWeekly14,
+	"育肥栏舍利用率":              base_from_yongyi_v2.HandleYongyiExcelWeekly15,
+	"周度-养殖利润最新":            base_from_yongyi_v2.HandleYongyiExcelWeekly16,
+	"周度-当期、预期成本":           base_from_yongyi_v2.HandleYongyiExcelWeekly17,
+	"育肥全价料出厂价":             base_from_yongyi_v2.HandleYongyiExcelWeekly18,
+	"周度-成本计算附件":            base_from_yongyi_v2.HandleYongyiExcelWeekly19,
+	"周度-毛白价差":              base_from_yongyi_v2.HandleYongyiExcelWeekly20,
+	"仔猪、母猪":                base_from_yongyi_v2.HandleYongyiExcelWeekly21,
+	"周度-河南屠宰白条成本":          base_from_yongyi_v2.HandleYongyiExcelWeekly22,
+	"周度-淘汰母猪价格":            base_from_yongyi_v2.HandleYongyiExcelWeekly23, //todo 少两个指标，云南和贵州，数据都不符合规范
+	"鲜销率":                  base_from_yongyi_v2.HandleYongyiExcelWeekly24, //todo 等待解析（正月初六-二月初四）2022.2.6-3.6
+	"周度-猪肉产品价格":            base_from_yongyi_v2.HandleYongyiExcelWeekly25,
+	"周度-屠宰企业日度屠宰量":         base_from_yongyi_v2.HandleYongyiExcelWeekly26,
+	"周度-屠宰新2022.10.28":     base_from_yongyi_v2.HandleYongyiExcelWeekly27,
+	"月度-淘汰母猪屠宰厂宰杀量":        base_from_yongyi_v2.HandleYongyiExcelWeekly28,
+	"月度-猪料销量":              base_from_yongyi_v2.HandleYongyiExcelWeekly29,
+	"月度-屠宰企业开工率":           base_from_yongyi_v2.HandleYongyiExcelWeekly30,
+	"月度-屠宰厂公母比例":           base_from_yongyi_v2.HandleYongyiExcelWeekly31,
+	"月度-生产指标（2021.5.7新增）":  base_from_yongyi_v2.HandleYongyiExcelWeekly32,
+	"月度-生产指标2":             base_from_yongyi_v2.HandleYongyiExcelWeekly33,
+	"月度-二元三元能繁比例":          base_from_yongyi_v2.HandleYongyiExcelWeekly34, //todo 2019年1-6月
+	"月度-能繁母猪存栏量":           base_from_yongyi_v2.HandleYongyiExcelWeekly35,
+	"月度-原种场二元后备母猪销量及出栏日龄":  base_from_yongyi_v2.HandleYongyiExcelWeekly11,
+	"周度-养殖利润":              base_from_yongyi_v2.HandleYongyiExcelWeekly37,
+	"华东冻品价格":               base_from_yongyi_v2.HandleYongyiExcelWeekly38,
+	"运费":                   base_from_yongyi_v2.HandleYongyiExcelWeekly39,
+	"国产冻品2-4号肉价格":          base_from_yongyi_v2.HandleYongyiExcelWeekly40,
+	"月度-小猪（50公斤以下）存栏":      base_from_yongyi_v2.HandleYongyiExcelWeekly41,
+	"月度-中猪（50-80公斤）存栏":     base_from_yongyi_v2.HandleYongyiExcelWeekly41,
+	"月度-大猪（80公斤以上）月度存栏":    base_from_yongyi_v2.HandleYongyiExcelWeekly41,
+	"月度猪肉供应占比":             base_from_yongyi_v2.HandleYongyiExcelWeekly42,
+	"历史出栏体重":               base_from_yongyi_v2.HandleYongyiExcelWeekly43,
+	"重要部位冻品进口":             base_from_yongyi_v2.HandleYongyiExcelWeekly44,
+	"各存栏规模":                base_from_yongyi_v2.HandleYongyiExcelWeekly45,
+	"进口肉":                  base_from_yongyi_v2.HandleYongyiExcelWeekly46,
+	"猪料原料占比":               base_from_yongyi_v2.HandleYongyiExcelWeekly47,
+	"MSY":                  base_from_yongyi_v2.HandleYongyiExcelWeekly48,
 }
 var YongyiWeeklyChartHandlers = map[string]func(f *excelize.File, sheetName string) ([]*models.YongyiExcelIndex, error){
 	"涌益样本测算": base_from_yongyi_v2.HandleYongyiExcelWeekly36,

services/base_from_yongyi_v2/base.go

@@ -73,6 +73,8 @@ func getClassifySortMap() map[string]int {
 		"重要部位冻品进口":             57,
 		"各存栏规模":                58,
 		"进口肉":                  59,
+		"猪料原料占比":               60,
+		"MSY":                  61,
 	}
 	return classifyMap
 }
@@ -160,7 +162,9 @@ func init() {
 		"历史出栏体重":               {"商品猪历史出栏体重", "历史出栏体重", "周度", "公斤"},
 		"重要部位冻品进口":             {"重要部位冻品进口", "重要部位冻品进口", "月度", "-"},
 		"各存栏规模":                {"半年度各规模存栏", "各存栏规模", "半年度", "%"},
-		"进口肉":                  {"-", "进口肉", "月度", "-"},
+		"进口肉":                  {"", "进口肉", "月度", ""},
+		"猪料原料占比":               {"猪料育肥全价料中各原料占比", "猪料原料占比", "月度", "%"},
+		"MSY":                  {"MSY", "MSY", "年度", "%"},
 	}
 }
 

services/base_from_yongyi_v2/week50.go

@@ -396,13 +396,14 @@ func HandleYongyiExcelWeekly46(f *excelize.File, sheetName string) (indexList []
 	sort := 0
 	var indexName string
 	re := regexp.MustCompile(`\d+`)
+	regZr := regexp.MustCompile(`\([^()]*\)`)
+	zrzuDateMap := make(map[int]string) // 猪肉及杂碎进口量
 	indexInfoMap := make(map[string]*models.YongyiExcelIndex)
 	for _, col := range cols {
 		if indexName == "" {
 			indexName = col[1]
-			continue
 		}
-		if len(col) == 0 {
+		if col[2] == "" && col[3] == "" {
 			indexName = ""
 			continue
 		}
@@ -421,7 +422,7 @@ func HandleYongyiExcelWeekly46(f *excelize.File, sheetName string) (indexList []
 				}
 				monthArr := strings.Split(cols[0][j], "-")
 				var month int
-				switch len(monthArr[1]) {
+				switch len(monthArr) {
 				case 2:
 					month, err = strconv.Atoi(re.FindString(monthArr[1]))
 					if err != nil {
@@ -456,9 +457,221 @@ func HandleYongyiExcelWeekly46(f *excelize.File, sheetName string) (indexList []
 				}
 			}
 		} else if indexName != "" && strings.Contains(indexName, "猪肉及杂碎进口量") {
-			// preName := "猪肉及杂碎进口量"
+			preName := "猪肉及杂碎进口量"
+			if len(col) <= 3 {
+				continue
+			}
+			colTitleName := col[2]
+			if colTitleName == `` {
+				continue
+			}
+			if colTitleName == `日期` {
+				for j, cell := range col {
+					if cell == "" || j <= 2 {
+						continue
+					}
+					excelDate, err := strconv.ParseFloat(cell, 64)
+					if err != nil {
+						continue
+					}
+					tmpDate, err := excelize.ExcelDateToTime(excelDate, false)
+					if err != nil {
+						continue
+					}
+					LastDayDate := LastDayOfMonth(tmpDate.Year(), tmpDate.Month())
+					zrzuDateMap[j] = LastDayDate
+				}
+			} else {
+				unit := regZr.FindString(colTitleName)
+				if unit == "" {
+					unit = "%"
+				} else {
+					unit = strings.ReplaceAll(unit, "(", "")
+					unit = strings.ReplaceAll(unit, ")", "")
+				}
+				namePart := regZr.ReplaceAllString(colTitleName, "")
+				tmpIndexName := fmt.Sprintf("%s/%s", preName, namePart)
+				if indexInfo, ok := indexInfoMap[tmpIndexName]; !ok {
+					yongyiIndex := new(models.YongyiExcelIndex)
+					yongyiIndex.ClassifyName = classifyName
+					yongyiIndex.ClassifySort = classifySort
+					yongyiIndex.Frequency = frequency
+					yongyiIndex.Unit = unit
+					yongyiIndex.IndexName = tmpIndexName
+					tmpIndexName = strings.ReplaceAll(tmpIndexName, "/", "")
+					yongyiIndex.IndexCode = namePrefixPingin + utils.GetFirstPingYin(tmpIndexName)
+					yongyiIndex.Sort = sort
+					yongyiIndex.ExcelDataMap = make(map[string]string)
+					for j, cell := range col {
+						if j == 0 {
+							continue
+						}
+						cell = strings.ReplaceAll(cell, ",", "")
+						if unit == "%" {
+							cell = strings.ReplaceAll(cell, "%", "")
+						}
+						if !isValueValid(cell) {
+							continue
+						}
+						yongyiIndex.ExcelDataMap[zrzuDateMap[j]] = cell
+					}
+					indexInfoMap[yongyiIndex.IndexName] = yongyiIndex
+					indexList = append(indexList, yongyiIndex)
+					sort++
+				} else {
+					for j, cell := range col {
+						if j == 0 {
+							continue
+						}
+						cell = strings.ReplaceAll(cell, ",", "")
+						if !isValueValid(cell) {
+							continue
+						}
+						indexInfo.ExcelDataMap[zrzuDateMap[j]] = cell
+					}
+				}
+			}
+		}
+	}
+	utils.FileLog.Info(fmt.Sprintf("HandleYongyiExcelWeekly46 %s 更新指标数量：%d", sheetName, len(indexList)))
+	return
+}
+
+// HandleYongyiExcelWeekly47 猪料原料占比
+func HandleYongyiExcelWeekly47(f *excelize.File, sheetName string) (indexList []*models.YongyiExcelIndex, err error) {
+	defer func() {
+		if err != nil {
+			utils.FileLog.Info(fmt.Sprintf("HandleYongyiExcelWeekly47 %s ErrMsg: %s", sheetName, err.Error()))
+		}
+	}()
+	rows, err := f.GetRows(sheetName)
+	if err != nil {
+		err = fmt.Errorf("f GetRows err: %s", err.Error())
+		return
+	}
+	classifyName, classifySort, frequency, unit, namePrefix, namePrefixPingin := GetBaseInfo(sheetName)
 
+	namePart2Map := map[string]int{
+		"玉米": 0, "小麦": 0, "稻谷、糙米": 0, "豆粕": 0, "棉粕": 0, "菜粕": 0,
+	}
+	regionMap := map[string]struct{}{
+		"东北": {}, "华北": {}, "华中": {}, "华南": {}, "西北": {}, "西南": {}, "全国": {},
+	}
+	regionIndexMap := make(map[int]string)
+	indexInfoMap := make(map[string]*models.YongyiExcelIndex)
+
+	sort := 0
+	for i, row := range rows {
+		if i == 0 {
+			continue
+		}
+		if len(row) <= 1 {
+			continue
+		}
+		var preName string
+		for j, cell := range row {
+			if _, ok := namePart2Map[cell]; ok {
+				namePart2Map[cell] = j
+				preName = cell
+				continue
+			}
+			if _, ok := regionMap[cell]; ok {
+				regionIndexMap[j] = preName + "/" + cell
+				continue
+			}
+			if region, ok := regionIndexMap[j]; ok {
+				if !isValueValid(cell) {
+					continue
+				}
+				kind := strings.Split(region, "/")[0]
+				dateIndex := namePart2Map[kind]
+				excelDate, err := strconv.ParseFloat(row[dateIndex], 64)
+				if err != nil {
+					continue
+				}
+				curTime, err := excelize.ExcelDateToTime(excelDate, false)
+				if err != nil {
+					continue
+				}
+				dateTime := LastDayOfMonth(curTime.Year(), curTime.Month())
+
+				indexName := fmt.Sprintf("%s/%s", namePrefix, region)
+				if indexInfo, ok := indexInfoMap[indexName]; !ok {
+					yongyiIndex := new(models.YongyiExcelIndex)
+					yongyiIndex.ClassifyName = classifyName
+					yongyiIndex.ClassifySort = classifySort
+					yongyiIndex.Frequency = frequency
+					yongyiIndex.Unit = unit
+					yongyiIndex.IndexName = indexName
+					tmpSuffix := strings.ReplaceAll(region, "/", "")
+					tmpSuffix = strings.ReplaceAll(tmpSuffix, "、", "")
+					yongyiIndex.IndexCode = namePrefixPingin + utils.GetFirstPingYin(tmpSuffix)
+					yongyiIndex.Sort = sort
+					yongyiIndex.ExcelDataMap = make(map[string]string)
+					cell = strings.ReplaceAll(cell, "%", "")
+					yongyiIndex.ExcelDataMap[dateTime] = cell
+					indexInfoMap[indexName] = yongyiIndex
+					indexList = append(indexList, yongyiIndex)
+					sort++
+				} else {
+					cell = strings.ReplaceAll(cell, "%", "")
+					indexInfo.ExcelDataMap[dateTime] = cell
+				}
+			}
+
+		}
+	}
+	utils.FileLog.Info(fmt.Sprintf("HandleYongyiExcelWeekly47 %s 更新指标数量：%d", sheetName, len(indexList)))
+	return
+}
+
+// HandleYongyiExcelWeekly48 MSY
+func HandleYongyiExcelWeekly48(f *excelize.File, sheetName string) (indexList []*models.YongyiExcelIndex, err error) {
+	defer func() {
+		if err != nil {
+			utils.FileLog.Info(fmt.Sprintf("HandleYongyiExcelWeekly48 %s ErrMsg: %s", sheetName, err.Error()))
+		}
+	}()
+	cols, err := f.GetCols(sheetName)
+	if err != nil {
+		err = fmt.Errorf("f GetRows err: %s", err.Error())
+		return
+	}
+	classifyName, classifySort, frequency, unit, namePrefix, namePrefixPingin := GetBaseInfo(sheetName)
+
+	re := regexp.MustCompile(`\d+`)
+	dateMap := make(map[int]string)
+	for _, col := range cols {
+		if len(col) <= 3 {
+			continue
+		}
+		if col[2] == "时间" {
+			for j, cell := range col {
+				year := re.FindString(cell)
+				if year == "" {
+					continue
+				}
+				dateMap[j] = year + "-12-31"
+			}
+		} else if col[2] == "MSY" {
+			yongyiIndex := new(models.YongyiExcelIndex)
+			yongyiIndex.ClassifyName = classifyName
+			yongyiIndex.ClassifySort = classifySort
+			yongyiIndex.Frequency = frequency
+			yongyiIndex.Unit = unit
+			yongyiIndex.IndexName = namePrefix
+			yongyiIndex.IndexCode = namePrefixPingin
+			yongyiIndex.Sort = 0
+			yongyiIndex.ExcelDataMap = make(map[string]string)
+			for j, cell := range col {
+				if !isValueValid(cell) {
+					continue
+				}
+				yongyiIndex.ExcelDataMap[dateMap[j]] = cell
+			}
+			indexList = append(indexList, yongyiIndex)
 		}
 	}
+	utils.FileLog.Info(fmt.Sprintf("HandleYongyiExcelWeekly48 %s 更新指标数量：%d", sheetName, len(indexList)))
 	return
 }