汾渭数据

models/base_from_fenwei.go

@@ -1,7 +1,9 @@
 package models
 
-// FenWeiExcelIndex 汾渭Excel指标数据
-type FenWeiExcelIndex struct {
+import "time"
+
+// FenweiExcelIndex 汾渭Excel指标数据
+type FenweiExcelIndex struct {
 	//ClassifyName string `description:"指标目录"`
 	//ClassifySort int    `description:"指标目录排序号"`
 	IndexName    string `description:"指标名称"`
@@ -14,8 +16,8 @@ type FenWeiExcelIndex struct {
 	ExcelDataMap map[string]string
 }
 
-// BaseFromFenWeiClassifyItem 汾渭数据分类信息
-type BaseFromFenWeiClassifyItem struct {
+// BaseFromFenweiClassifyItem 汾渭数据分类信息
+type BaseFromFenweiClassifyItem struct {
 	ClassifyId   int                           `description:"分类ID"`
 	ClassifyName string                        `description:"分类名称"`
 	ParentId     int                           `description:"父级id"`
@@ -23,13 +25,36 @@ type BaseFromFenWeiClassifyItem struct {
 	Sort         int                           `description:"排序字段"`
 	CreateTime   string                        `description:"创建时间"`
 	ModifyTime   string                        `description:"修改时间"`
-	Child        []*BaseFromFenWeiClassifyItem `description:"子分类"`
+	Child        []*BaseFromFenweiClassifyItem `description:"子分类"`
+}
+
+type EdbLibFenweiClassifyResponse struct {
+	Ret     int
+	Msg     string
+	ErrMsg  string
+	ErrCode string
+	Data    []BaseFromFenweiClassifyItem
+}
+
+type BaseFromFenweiIndex struct {
+	FenweiIndexId int64 `orm:"column(fenwei_index_id);pk"`
+	IndexCode     string
+	IndexName     string
+	Frequency     string
+	Unit          string
+	ClassifyId    int
+	StartDate     string
+	EndDate       string
+	Sort          int
+	TerminalCode  string
+	CreateTime    time.Time
+	ModifyTime    time.Time
 }
 
-type EdbLibFenWeiClassifyResponse struct {
+type EdbLibFenweiIndexListResponse struct {
 	Ret     int
 	Msg     string
 	ErrMsg  string
 	ErrCode string
-	Data    []BaseFromFenWeiClassifyItem
+	Data    []BaseFromFenweiIndex
 }

services/base_from_fenwei.go

@@ -6,6 +6,7 @@ import (
 	"eta/eta_data_analysis/utils"
 	"fmt"
 	"github.com/shakinm/xlsReader/xls"
+	"github.com/tealeg/xlsx"
 	"github.com/xuri/excelize/v2"
 	"path"
 	"strings"
@@ -13,24 +14,26 @@ import (
 )
 
 const (
-	FenWeiIndexNamePrefix = "fwsj" // 指标名称前缀
-
-	FenWeiExcelHistoryNameA = "动力煤产销存-66家"
-	FenWeiExcelHistoryNameB = "动力煤产销存-100家"
-	FenWeiExcelHistoryNameC = "焦煤产销存-53家"
-	FenWeiExcelHistoryNameD = "焦煤产销存-88家"
-	FenWeiExcelHistoryNameE = "样本焦化厂焦煤"
-
-	FenWeiYangBenIndexNamePre    = "样本焦企炼焦煤库存可用天数"
-	FenWeiYangBenIndexClassifyId = 5
+	FenweiIndexNamePrefix = "fwsj" // 指标名称前缀
+
+	FenweiExcelNameA    = "动力煤产销存-66家"
+	FenweiExcelNameB    = "动力煤产销存-100家"
+	FenweiExcelNameC    = "焦煤产销存-53家"
+	FenweiNewExcelNameC = "炼焦煤产销存分省-53家"
+	FenweiExcelNameD    = "焦煤产销存-88家"
+	FenweiNewExcelNameD = "炼焦煤产销存分省-88家"
+	FenweiExcelNameE    = "样本焦化厂焦煤"
+
+	FenweiYangBenIndexNamePre    = "样本焦企炼焦煤库存可用天数"
+	FenweiYangBenIndexClassifyId = 5 // 样本焦企炼焦分类ID
 )
 
-var FenWeiProvinces = []string{
-	"内蒙古", "山西", "陕西", "三省合计", "临汾市", "临汾", "吕梁市", "吕梁", "长治市", "长治", "山西其他", "河北", "合计",
+var FenweiProvinces = []string{
+	"内蒙古", "山西", "陕西", "三省合计", "临汾市", "临汾", "吕梁市", "吕梁", "长治市", "长治", "山西其他", "河北", "合计", "山东", "其他",
 }
 
-// FenWeiProvinceClassifyMapping 指标名称省份->分类省份
-var FenWeiProvinceClassifyMapping = map[string]string{
+// FenweiProvinceClassifyMapping 指标名称省份->分类省份
+var FenweiProvinceClassifyMapping = map[string]string{
 	"内蒙古":  "内蒙古",
 	"山西":   "山西",
 	"陕西":   "陕西",
@@ -43,40 +46,189 @@ var FenWeiProvinceClassifyMapping = map[string]string{
 	"长治":   "长治市",
 	"山西其他": "山西其他",
 	"河北":   "河北",
+	"山东":   "山东",
 	"合计":   "合计",
+	"其他":   "其他",
+}
+
+// FenweiFirstClassifyMap excel表名->一级分类ID
+var FenweiFirstClassifyMap = map[string]int{
+	FenweiExcelNameA: 1,
+	FenweiExcelNameB: 2,
+	FenweiExcelNameC: 3,
+	FenweiExcelNameD: 4,
+	FenweiExcelNameE: 5,
+}
+
+// FenweiNewDlm66DiffIndexNameReplaceMapping 动力煤66家-指标名称不同的进行替换
+var FenweiNewDlm66DiffIndexNameReplaceMapping = map[string]string{
+	"价格平均上涨幅度（元/吨）": "煤矿价格平均上涨幅度",
+	"库存库存量（万吨）":     "煤矿库存",
+	"盈亏情况盈利煤矿数量":    "盈利煤矿数量",
+	"盈亏情况平均利润（元/吨）": "煤矿平均利润",
+	"盈亏情况亏损煤矿数量":    "亏损煤矿数量",
+	"盈亏情况平均亏损（元/吨）": "煤矿平均亏损",
+	"周度产量产量（万吨）":    "煤矿周度产量",
+	"销售好":           "销售好煤矿数量",
+	"销售一般":          "销售一般煤矿数量",
+	"销售不好":          "销售不好煤矿数量",
+	"销售关闭":          "关闭煤矿数量",
+	"产能利用率产能利用率":    "煤矿产能利用率",
+	"开工率开工率":        "煤矿算术平均开工率（含停产）",
+	"价格平均下跌幅度":      "煤矿价格平均下跌幅度",
+}
+
+// FenweiNewDlm66IgnoreIndexNameMapping 动力煤66家-需要忽略的指标名称
+var FenweiNewDlm66IgnoreIndexNameMapping = []string{
+	"库存环比", "发运至港口平均利润", "周度产量环比", "开工率算数平均（不含停产矿）", "变化", "产能利用率变化", "开工率变化",
+}
+
+// FenweiNewDlm100DiffIndexNameReplaceMapping 动力煤100家-指标名称不同的进行替换
+var FenweiNewDlm100DiffIndexNameReplaceMapping = map[string]string{
+	"价格平均上涨幅度（元/吨）": "煤矿价格平均上涨幅度",
+	"价格平均下跌幅度":      "煤矿价格平均下跌幅度",
+	"库存库存量（万吨）":     "煤矿库存",
+	"库存环比":          "煤矿库存环比",
+	"盈亏情况盈利煤矿数量":    "盈利煤矿数量",
+	"盈亏情况平均利润（元/吨）": "煤矿平均利润",
+	"盈亏情况亏损煤矿数量":    "亏损煤矿数量",
+	"盈亏情况平均亏损（元/吨）": "煤矿平均亏损",
+	"发运至港口平均利润":     "煤炭发运至港口平均利润",
+	"周度产量产量（万吨）":    "煤矿周度产量",
+	"周度产量环比":        "煤矿周度产量环比",
+	"产能利用率产能利用率":    "煤矿产能利用率",
+	"产能利用率变化":       "煤矿产能利用率环比",
+	"开工率开工率":        "煤矿平均开工率",
+	"开工率变化":         "煤矿平均开工率环比",
+	"销售好":           "销售好煤矿数量",
+	"销售一般":          "销售一般煤矿数量",
+	"销售不好":          "销售不好煤矿数量",
+	"销售关闭":          "关闭煤矿数量",
+}
+
+// FenweiNewDlm100IgnoreIndexNameMapping 动力煤100家-需要忽略的指标名称
+var FenweiNewDlm100IgnoreIndexNameMapping = make([]string, 0)
+
+// FenweiNewJm53DiffIndexNameReplaceMapping 焦煤53家-指标名称不同的进行替换
+var FenweiNewJm53DiffIndexNameReplaceMapping = map[string]string{
+	"价格情况煤矿数量":       "煤矿数量",
+	"价格情况上涨煤矿数量":     "上涨煤矿数量",
+	"价格情况平均上涨（元/吨）":  "煤矿平均上涨幅度",
+	"价格情况下跌煤矿数量":     "下跌煤矿数量",
+	"价格情况平均下跌（元/吨）":  "煤矿平均下跌幅度",
+	"价格情况持平煤矿数量":     "持平煤矿数量",
+	"原煤库存情况库存总量（万吨）": "原煤库存",
+	"精煤库存情况库存总量（万吨）": "精煤库存",
+	"开工情况权重平均开工率":    "煤矿加权开工率",
+	"开工情况算数平均开工率":    "煤矿算术平均开工率",
+	"开工情况停产煤矿数量":     "停产煤矿数量",
+	"开工情况本周原煤产量（万吨）": "本周原煤产量",
+	"销售良好":     "销售好煤矿数量",
+	"销售正常":     "销售一般煤矿数量",
+	"销售不好":     "销售不好煤矿数量",
+	"成本盈利煤矿数量": "盈利煤矿数量",
+	"成本平均利润":   "煤矿平均利润",
+	"成本亏损煤矿数量": "亏损煤矿数量",
+	"成本平均亏损":   "煤矿平均亏损",
+}
+
+// FenweiNewJm53IgnoreIndexNameMapping 焦煤53家-需要忽略的指标名称
+var FenweiNewJm53IgnoreIndexNameMapping = []string{
+	"原煤库存情况环比（万吨）", "精煤库存情况环比（万吨）", "开工情况开工率变化", "开工情况停产煤矿数量变化", "开工情况环比产量变化",
+}
+
+// FenweiNewJm88DiffIndexNameReplaceMapping 焦煤88家-指标名称不同的进行替换
+var FenweiNewJm88DiffIndexNameReplaceMapping = map[string]string{
+	"价格情况煤企数量":       "煤企数量",
+	"价格情况上涨煤企数量":     "上涨煤企数量",
+	"价格情况平均上涨（元/吨）":  "煤企平均上涨幅度",
+	"价格情况下跌煤矿数量":     "下跌煤企数量",
+	"价格情况平均下跌（元/吨）":  "煤企平均下跌幅度",
+	"价格情况持平煤企数量":     "持平煤企数量",
+	"原煤库存情况库存总量（万吨）": "原煤库存",
+	"原煤库存情况环比（万吨）":   "原煤库存环比",
+	"精煤库存情况库存总量（万吨）": "精煤库存",
+	"精煤库存情况环比（万吨）":   "精煤库存环比",
+	"开工情况产能利用率":      "煤企产能利用率",
+	"开工情况产能利用率变化":    "煤企产能利用率环比",
+	"开工情况开工率":        "煤企开工率",
+	"开工情况开工率变化":      "煤企开工率环比",
+	"开工情况停产煤矿数量":     "停产煤矿数量",
+	"开工情况停产煤矿数量变化":   "停产煤矿数量变化",
+	"开工情况本周原煤产量（万吨）": "本周原煤产量",
+	"开工情况环比产量变化":     "本周原煤产量变化",
+	"本周精煤产量（万吨）":     "本周精煤产量",
+	"环比产量变化":         "本周精煤产量变化",
+	"良好":             "销售好煤企数量",
+	"正常":             "销售一般煤企数量",
+	"不好":             "销售不好煤企数量",
+	"盈利煤企数量":         "盈利煤企数量",
+	"商品煤平均利润":        "煤企平均利润",
+	"亏损煤企数量":         "亏损煤企数量",
+	"商品煤平均亏损":        "煤企平均亏损",
+}
+
+// FenweiNewJm88IgnoreIndexNameMapping 焦煤88家-需要忽略的指标名称
+var FenweiNewJm88IgnoreIndexNameMapping = make([]string, 0)
+
+var FenweiNewDiffIndexNameReplaceMapping = map[string]map[string]string{
+	FenweiExcelNameA: FenweiNewDlm66DiffIndexNameReplaceMapping,
+	FenweiExcelNameB: FenweiNewDlm100DiffIndexNameReplaceMapping,
+	FenweiExcelNameC: FenweiNewJm53DiffIndexNameReplaceMapping,
+	FenweiExcelNameD: FenweiNewJm88DiffIndexNameReplaceMapping,
+}
+
+var FenweiNewIgnoreIndexNameMapping = map[string][]string{
+	FenweiExcelNameA: FenweiNewDlm66IgnoreIndexNameMapping,
+	FenweiExcelNameB: FenweiNewDlm100IgnoreIndexNameMapping,
+	FenweiExcelNameC: FenweiNewJm53IgnoreIndexNameMapping,
+	FenweiExcelNameD: FenweiNewJm88IgnoreIndexNameMapping,
+}
+
+var FenweiNewIndexNameSuffixMapping = map[string]string{
+	FenweiExcelNameA: "/动力煤/66家样本",
+	FenweiExcelNameB: "/动力煤/100家样本",
+	FenweiExcelNameC: "/焦煤/53家样本",
+	FenweiExcelNameD: "/焦煤/88家样本",
 }
 
-// FenWeiFirstClassifyMap excel表名->一级分类ID
-var FenWeiFirstClassifyMap = map[string]int{
-	FenWeiExcelHistoryNameA: 1,
-	FenWeiExcelHistoryNameB: 2,
-	FenWeiExcelHistoryNameC: 3,
-	FenWeiExcelHistoryNameD: 4,
-	FenWeiExcelHistoryNameE: 5,
+var FenweiNewExcelProvinceMapping = map[string]string{
+	"内蒙古":    "内蒙古",
+	"山西":     "山西",
+	"陕西":     "陕西",
+	"三省合计":   "三省合计",
+	"山西-临汾市": "临汾",
+	"山西-吕梁市": "吕梁",
+	"山西-长治市": "长治",
+	"山西-其他":  "山西其他",
+	"山东":     "山东",
+	"其他":     "其他",
+	"合计":     "合计",
+	"河北":     "河北",
 }
 
-// AnalysisFenWeiHistoryExcel 解析汾渭历史Excel数据
-func AnalysisFenWeiHistoryExcel(filePath string) (err error) {
+// AnalysisFenweiHistoryExcel 解析汾渭历史Excel数据
+func AnalysisFenweiHistoryExcel(filePath string) (err error) {
 	defer func() {
 		if err != nil {
-			fmt.Printf("AnalysisFenWeiExcel ErrMsg: %s\n", err.Error())
-			utils.FileLog.Info(fmt.Sprintf("AnalysisFenWeiExcel ErrMsg: %s", err.Error()))
+			fmt.Printf("AnalysisFenweiExcel ErrMsg: %s\n", err.Error())
+			utils.FileLog.Info(fmt.Sprintf("AnalysisFenweiExcel ErrMsg: %s", err.Error()))
 		}
 	}()
 
 	fileName := path.Base(filePath)
-	var indexes []*models.FenWeiExcelIndex
+	var indexes []*models.FenweiExcelIndex
 	var e error
-	if strings.Contains(fileName, FenWeiExcelHistoryNameA) {
-		indexes, e = AnalysisFenWeiHistoryExcelNormal(filePath, FenWeiExcelHistoryNameA)
-	} else if strings.Contains(fileName, FenWeiExcelHistoryNameB) {
-		indexes, e = AnalysisFenWeiHistoryExcelNormal(filePath, FenWeiExcelHistoryNameB)
-	} else if strings.Contains(fileName, FenWeiExcelHistoryNameC) {
-		indexes, e = AnalysisFenWeiHistoryExcelNormal(filePath, FenWeiExcelHistoryNameC)
-	} else if strings.Contains(fileName, FenWeiExcelHistoryNameD) {
-		indexes, e = AnalysisFenWeiHistoryExcelNormal(filePath, FenWeiExcelHistoryNameD)
-	} else if strings.Contains(fileName, FenWeiExcelHistoryNameE) {
-		indexes, e = AnalysisFenWeiExcelYangBen(filePath)
+	if strings.Contains(fileName, FenweiExcelNameA) {
+		indexes, e = AnalysisFenweiHistoryExcelNormal(filePath, FenweiExcelNameA)
+	} else if strings.Contains(fileName, FenweiExcelNameB) {
+		indexes, e = AnalysisFenweiHistoryExcelNormal(filePath, FenweiExcelNameB)
+	} else if strings.Contains(fileName, FenweiExcelNameC) {
+		indexes, e = AnalysisFenweiHistoryExcelNormal(filePath, FenweiExcelNameC)
+	} else if strings.Contains(fileName, FenweiExcelNameD) {
+		indexes, e = AnalysisFenweiHistoryExcelNormal(filePath, FenweiExcelNameD)
+	} else if strings.Contains(fileName, FenweiExcelNameE) {
+		indexes, e = AnalysisFenweiExcelYangBen(filePath)
 	} else {
 		utils.FileLog.Info(fmt.Sprintf("文件名有误, FileName: %s", fileName))
 		return
@@ -112,12 +264,12 @@ func AnalysisFenWeiHistoryExcel(filePath string) (err error) {
 	return
 }
 
-// AnalysisFenWeiHistoryExcelNormal 历史数据-动力煤产销存/焦煤产销存
-func AnalysisFenWeiHistoryExcelNormal(filePath, excelName string) (indexes []*models.FenWeiExcelIndex, err error) {
+// AnalysisFenweiHistoryExcelNormal 历史数据-动力煤产销存/焦煤产销存
+func AnalysisFenweiHistoryExcelNormal(filePath, excelName string) (indexes []*models.FenweiExcelIndex, err error) {
 	defer func() {
 		if err != nil {
-			fmt.Printf("AnalysisFenWeiHistoryExcelNormal ErrMsg: %s\n", err.Error())
-			utils.FileLog.Info(fmt.Sprintf("AnalysisFenWeiHistoryExcelNormal ErrMsg: %s", err.Error()))
+			fmt.Printf("AnalysisFenweiHistoryExcelNormal ErrMsg: %s\n", err.Error())
+			utils.FileLog.Info(fmt.Sprintf("AnalysisFenweiHistoryExcelNormal ErrMsg: %s", err.Error()))
 		}
 	}()
 
@@ -139,12 +291,12 @@ func AnalysisFenWeiHistoryExcelNormal(filePath, excelName string) (indexes []*mo
 	}
 
 	// 获取指标分类
-	firstId := FenWeiFirstClassifyMap[excelName]
+	firstId := FenweiFirstClassifyMap[excelName]
 	if firstId <= 0 {
 		err = fmt.Errorf("excel名称有误, 分类不匹配, excelName: %s", excelName)
 		return
 	}
-	classifies, e := GetFenWeiClassify()
+	classifies, e := GetFenweiClassify()
 	if e != nil {
 		err = fmt.Errorf("获取指标分类失败, Err: %s", e.Error())
 		return
@@ -159,8 +311,10 @@ func AnalysisFenWeiHistoryExcelNormal(filePath, excelName string) (indexes []*mo
 		}
 	}
 
-	indexes = make([]*models.FenWeiExcelIndex, 0)
-	indexColIndex := make(map[int]*models.FenWeiExcelIndex) // 列对应的指标(B列-?列)
+	indexes = make([]*models.FenweiExcelIndex, 0)
+	indexColIndex := make(map[int]*models.FenweiExcelIndex) // 列对应的指标(B列-?列)
+	maxCol := 0
+	sortArr := make([]string, 0) // 排序用的
 	for i, row := range rows {
 		// 首行为标题, 忽略
 		if i == 0 {
@@ -171,25 +325,28 @@ func AnalysisFenWeiHistoryExcelNormal(filePath, excelName string) (indexes []*mo
 		if i == 1 {
 			for k, cell := range row {
 				if k == 0 {
+					maxCol += 1
 					continue
 				}
 				name := strings.TrimSpace(cell)
+				name = strings.ReplaceAll(name, " ", "")
 				if name == "" {
 					break
 				}
+				maxCol += 1
 				if indexColIndex[k] == nil {
-					indexColIndex[k] = new(models.FenWeiExcelIndex)
+					indexColIndex[k] = new(models.FenweiExcelIndex)
 					indexColIndex[k].ExcelDataMap = make(map[string]string)
 					indexColIndex[k].TerminalCode = utils.TerminalCode
+					indexColIndex[k].IndexName = name
 				}
-				indexColIndex[k].IndexName = name
 			}
 			continue
 		}
 
 		// 第三行-指标ID、指标分类
 		if i == 2 {
-			for k := range row {
+			for k := 0; k < maxCol; k++ {
 				if k == 0 {
 					continue
 				}
@@ -200,7 +357,14 @@ func AnalysisFenWeiHistoryExcelNormal(filePath, excelName string) (indexes []*mo
 				// 例:内蒙古煤矿数量/动力煤/66家样本->fwsjneimenggumksldlm66jyb
 				name := strings.ReplaceAll(indexColIndex[k].IndexName, "/", "")
 				var middle, suffix, province string
-				for _, p := range FenWeiProvinces {
+				for _, p := range FenweiProvinces {
+					// 此处山西其他可能会被错误的给鉴别为山西, 所以优先鉴别一下山西其他
+					if strings.Contains(name, "山西其他") {
+						province = "山西其他"
+						middle = utils.GetFullPingYin("山西其他")
+						name = strings.ReplaceAll(name, p, "")
+						break
+					}
 					if strings.Contains(name, p) {
 						province = p
 						middle = utils.GetFullPingYin(p)
@@ -212,7 +376,7 @@ func AnalysisFenWeiHistoryExcelNormal(filePath, excelName string) (indexes []*mo
 					utils.FileLog.Info(fmt.Sprintf("指标名称转换编码有误, IndexName: %s", indexColIndex[k].IndexName))
 					continue
 				}
-				classifyProvince := FenWeiProvinceClassifyMapping[province]
+				classifyProvince := FenweiProvinceClassifyMapping[province]
 				classifyId := secondClassifyMap[classifyProvince]
 				if classifyId <= 0 {
 					utils.FileLog.Info(fmt.Sprintf("指标名称分类不匹配, IndexName: %s", indexColIndex[k].IndexName))
@@ -220,8 +384,9 @@ func AnalysisFenWeiHistoryExcelNormal(filePath, excelName string) (indexes []*mo
 				}
 
 				suffix = utils.GetFirstPingYin(name)
-				indexColIndex[k].IndexCode = fmt.Sprint(FenWeiIndexNamePrefix, middle, suffix)
+				indexColIndex[k].IndexCode = fmt.Sprint(FenweiIndexNamePrefix, middle, suffix)
 				indexColIndex[k].ClassifyId = classifyId
+				sortArr = append(sortArr, indexColIndex[k].IndexCode)
 			}
 			continue
 		}
@@ -292,38 +457,44 @@ func AnalysisFenWeiHistoryExcelNormal(filePath, excelName string) (indexes []*mo
 		}
 	}
 
+	codeIndexes := make(map[string]*models.FenweiExcelIndex)
 	for _, v := range indexColIndex {
 		if v.IndexCode == "" {
 			continue
 		}
-		indexes = append(indexes, v)
+		codeIndexes[v.IndexCode] = v
+	}
+	for _, v := range sortArr {
+		if codeIndexes[v] != nil {
+			indexes = append(indexes, codeIndexes[v])
+		}
 	}
 	return
 }
 
-// AnalysisFenWeiExcelYangBen 样本焦化厂
-func AnalysisFenWeiExcelYangBen(filePath string) (indexes []*models.FenWeiExcelIndex, err error) {
+// AnalysisFenweiExcelYangBen 样本焦化厂
+func AnalysisFenweiExcelYangBen(filePath string) (indexes []*models.FenweiExcelIndex, err error) {
 	defer func() {
 		if err != nil {
-			fmt.Printf("AnalysisFenWeiExcelYangBen ErrMsg: %s\n", err.Error())
-			utils.FileLog.Info(fmt.Sprintf("AnalysisFenWeiExcelYangBen ErrMsg: %s", err.Error()))
+			fmt.Printf("AnalysisFenweiExcelYangBen ErrMsg: %s\n", err.Error())
+			utils.FileLog.Info(fmt.Sprintf("AnalysisFenweiExcelYangBen ErrMsg: %s", err.Error()))
 		}
 	}()
 
-	indexColIndex := make(map[int]*models.FenWeiExcelIndex) // 列对应的指标(B列-E列)
+	indexColIndex := make(map[int]*models.FenweiExcelIndex) // 列对应的指标(B列-E列)
 	provinces := []string{"地区", "全国", "山西", "河北", "山东"}
 	for k, v := range provinces {
 		if k == 0 {
 			continue
 		}
 		if indexColIndex[k] == nil {
-			indexColIndex[k] = new(models.FenWeiExcelIndex)
+			indexColIndex[k] = new(models.FenweiExcelIndex)
 			indexColIndex[k].ExcelDataMap = make(map[string]string)
 			indexColIndex[k].TerminalCode = utils.TerminalCode
-			indexColIndex[k].ClassifyId = FenWeiYangBenIndexClassifyId
+			indexColIndex[k].ClassifyId = FenweiYangBenIndexClassifyId
 			indexColIndex[k].Frequency = "周度"
 			indexColIndex[k].Unit = "天"
-			indexColIndex[k].IndexName = fmt.Sprintf("%s/%s", FenWeiYangBenIndexNamePre, v)
+			indexColIndex[k].IndexName = fmt.Sprintf("%s/%s", FenweiYangBenIndexNamePre, v)
 
 			// 指标编码, 例:样本焦企炼焦煤库存可用天数/全国->fwsjybjqljmkckytsquanguo
 			name := strings.ReplaceAll(indexColIndex[k].IndexName, "/", "")
@@ -340,7 +511,7 @@ func AnalysisFenWeiExcelYangBen(filePath string) (indexes []*models.FenWeiExcelI
 				continue
 			}
 			middle = utils.GetFirstPingYin(name)
-			indexColIndex[k].IndexCode = fmt.Sprint(FenWeiIndexNamePrefix, middle, suffix)
+			indexColIndex[k].IndexCode = fmt.Sprint(FenweiIndexNamePrefix, middle, suffix)
 		}
 	}
 
@@ -422,7 +593,7 @@ func AnalysisFenWeiExcelYangBen(filePath string) (indexes []*models.FenWeiExcelI
 		}
 	}
 
-	indexes = make([]*models.FenWeiExcelIndex, 0)
+	indexes = make([]*models.FenweiExcelIndex, 0)
 	for _, v := range indexColIndex {
 		if v.IndexCode == "" {
 			continue
@@ -432,50 +603,72 @@ func AnalysisFenWeiExcelYangBen(filePath string) (indexes []*models.FenWeiExcelI
 	return
 }
 
-// GetFenWeiClassify 获取汾渭数据分类
-func GetFenWeiClassify() (classifies []models.BaseFromFenWeiClassifyItem, err error) {
+// GetFenweiClassify 获取汾渭数据分类
+func GetFenweiClassify() (classifies []models.BaseFromFenweiClassifyItem, err error) {
 	params := make(map[string]interface{})
 	result, e := PostEdbLib(params, utils.LIB_ROUTE_FENWEI_CLASSIFY)
 	if e != nil {
 		b, _ := json.Marshal(params)
-		err = fmt.Errorf("GetFenWeiClassify, PostEdbLib err: %s, params: %s", e.Error(), string(b))
+		err = fmt.Errorf("GetFenweiClassify, PostEdbLib err: %s, params: %s", e.Error(), string(b))
 		return
 	}
-	resp := new(models.EdbLibFenWeiClassifyResponse)
+	resp := new(models.EdbLibFenweiClassifyResponse)
 	if e = json.Unmarshal(result, &resp); e != nil {
-		err = fmt.Errorf(fmt.Sprintf("GetFenWeiClassify, json.Unmarshal err: %s", e.Error()))
+		err = fmt.Errorf(fmt.Sprintf("GetFenweiClassify, json.Unmarshal err: %s", e.Error()))
 		return
 	}
 	if resp.Ret != 200 {
-		err = fmt.Errorf(fmt.Sprintf("GetFenWeiClassify, Msg: %s, ErrMsg: %s", resp.Msg, resp.ErrMsg))
+		err = fmt.Errorf(fmt.Sprintf("GetFenweiClassify, Msg: %s, ErrMsg: %s", resp.Msg, resp.ErrMsg))
 		return
 	}
 	classifies = resp.Data
 	return
 }
 
-// AnalysisFenWeiNewExcel 解析汾渭增量Excel数据
-func AnalysisFenWeiNewExcel(filePath string) (err error) {
+// GetFenweiIndexList 获取汾渭指标列表
+func GetFenweiIndexList() (list []models.BaseFromFenweiIndex, err error) {
+	params := make(map[string]interface{})
+	result, e := PostEdbLib(params, utils.LIB_ROUTE_FENWEI_INDEX_LIST)
+	if e != nil {
+		b, _ := json.Marshal(params)
+		err = fmt.Errorf("GetFenweiIndexList, PostEdbLib err: %s, params: %s", e.Error(), string(b))
+		return
+	}
+	resp := new(models.EdbLibFenweiIndexListResponse)
+	if e = json.Unmarshal(result, &resp); e != nil {
+		err = fmt.Errorf(fmt.Sprintf("GetFenweiIndexList, json.Unmarshal err: %s", e.Error()))
+		return
+	}
+	if resp.Ret != 200 {
+		err = fmt.Errorf(fmt.Sprintf("GetFenweiIndexList, Msg: %s, ErrMsg: %s", resp.Msg, resp.ErrMsg))
+		return
+	}
+	list = resp.Data
+	return
+}
+
+// AnalysisFenweiNewExcel 解析汾渭增量Excel数据
+func AnalysisFenweiNewExcel(filePath string) (err error) {
 	defer func() {
 		if err != nil {
-			fmt.Printf("AnalysisFenWeiNewExcel ErrMsg: %s\n", err.Error())
-			utils.FileLog.Info(fmt.Sprintf("AnalysisFenWeiNewExcel ErrMsg: %s", err.Error()))
+			fmt.Printf("AnalysisFenweiNewExcel ErrMsg: %s\n", err.Error())
+			utils.FileLog.Info(fmt.Sprintf("AnalysisFenweiNewExcel ErrMsg: %s", err.Error()))
 		}
 	}()
 
 	fileName := path.Base(filePath)
-	var indexes []*models.FenWeiExcelIndex
+	var indexes []*models.FenweiExcelIndex
 	var e error
-	if strings.Contains(fileName, FenWeiExcelHistoryNameA) {
-		indexes, e = AnalysisFenWeiNewExcelNormal(filePath, FenWeiExcelHistoryNameA)
-	} else if strings.Contains(fileName, FenWeiExcelHistoryNameB) {
-		indexes, e = AnalysisFenWeiNewExcelNormal(filePath, FenWeiExcelHistoryNameB)
-	} else if strings.Contains(fileName, FenWeiExcelHistoryNameC) {
-		indexes, e = AnalysisFenWeiNewExcelNormal(filePath, FenWeiExcelHistoryNameC)
-	} else if strings.Contains(fileName, FenWeiExcelHistoryNameD) {
-		indexes, e = AnalysisFenWeiNewExcelNormal(filePath, FenWeiExcelHistoryNameD)
-	} else if strings.Contains(fileName, FenWeiExcelHistoryNameE) {
-		indexes, e = AnalysisFenWeiExcelYangBen(filePath) // 样本焦化厂-增量数据和历史数据excel格式相同
+	if strings.Contains(fileName, FenweiExcelNameA) {
+		indexes, e = AnalysisFenweiNewExcelNormal(filePath, FenweiExcelNameA)
+	} else if strings.Contains(fileName, FenweiExcelNameB) {
+		indexes, e = AnalysisFenweiNewExcelNormal(filePath, FenweiExcelNameB)
+	} else if strings.Contains(fileName, FenweiNewExcelNameC) {
+		indexes, e = AnalysisFenweiNewExcelNormal(filePath, FenweiExcelNameC)
+	} else if strings.Contains(fileName, FenweiNewExcelNameD) {
+		indexes, e = AnalysisFenweiNewExcelNormal(filePath, FenweiExcelNameD)
+	} else if strings.Contains(fileName, FenweiExcelNameE) {
+		indexes, e = AnalysisFenweiExcelYangBen(filePath) // 样本焦化厂-增量数据和历史数据excel格式相同
 	} else {
 		utils.FileLog.Info(fmt.Sprintf("文件名有误, FileName: %s", fileName))
 		return
@@ -511,14 +704,199 @@ func AnalysisFenWeiNewExcel(filePath string) (err error) {
 	return
 }
 
-// TODO:AnalysisFenWeiNewExcelNormal 增量数据-动力煤/焦煤产
-func AnalysisFenWeiNewExcelNormal(filePath, excelName string) (indexes []*models.FenWeiExcelIndex, err error) {
+// AnalysisFenweiNewExcelNormal 增量数据-动力煤/焦煤
+func AnalysisFenweiNewExcelNormal(filePath, excelName string) (indexes []*models.FenweiExcelIndex, err error) {
 	defer func() {
 		if err != nil {
-			fmt.Printf("AnalysisFenWeiNewExcelNormal ErrMsg: %s\n", err.Error())
-			utils.FileLog.Info(fmt.Sprintf("AnalysisFenWeiNewExcelNormal ErrMsg: %s", err.Error()))
+			fmt.Printf("AnalysisFenweiNewExcelNormal ErrMsg: %s\n", err.Error())
+			utils.FileLog.Info(fmt.Sprintf("AnalysisFenweiNewExcelNormal ErrMsg: %s", err.Error()))
 		}
 	}()
 
+	xlsFile, e := xlsx.OpenFile(filePath)
+	if e != nil {
+		err = fmt.Errorf("OpenFile Err: %s", e.Error())
+		return
+	}
+
+	// 获取存量指标, 增量数据只新增存量指标的数据
+	originIndexes, e := GetFenweiIndexList()
+	if e != nil {
+		err = fmt.Errorf("GetFenweiIndexList err: %s", e.Error())
+		return
+	}
+	indexMapping := make(map[string]models.BaseFromFenweiIndex)
+	for _, v := range originIndexes {
+		indexMapping[v.IndexCode] = v
+	}
+
+	diffNameReplace := FenweiNewDiffIndexNameReplaceMapping[excelName]
+	ignoreIndexName := FenweiNewIgnoreIndexNameMapping[excelName]
+	indexNameSuffix := FenweiNewIndexNameSuffixMapping[excelName]
+	var dataRowMin, dataRowMax int // 数据起始行数
+	if excelName == FenweiExcelNameA || excelName == FenweiExcelNameB {
+		dataRowMin = 3
+		dataRowMax = 6
+	}
+	if excelName == FenweiExcelNameC {
+		dataRowMin = 3
+		dataRowMax = 10
+	}
+	if excelName == FenweiExcelNameD {
+		dataRowMin = 3
+		dataRowMax = 13
+	}
+
+	var dataDate string
+	secondRowNames := make(map[int]string) // 第二行-指标名称的上半部分
+	thirdRowNames := make(map[int]string)  // 第三行-指标名称的另一部分
+	indexes = make([]*models.FenweiExcelIndex, 0)
+	indexColIndex := make(map[string]*models.FenweiExcelIndex) // 行-列对应的指标
+	rowProvince := make(map[int]string)                        // 行对应的省份名
+	for _, sheet := range xlsFile.Sheet {
+		maxRow := sheet.MaxRow
+		for i := 0; i < maxRow; i++ {
+			cells := sheet.Row(i).Cells
+
+			// 首行为数据日期
+			if i == 0 {
+				for _, cell := range cells {
+					str := strings.TrimSpace(cell.String())
+					if str == "" {
+						continue
+					}
+					dateArr := strings.Split(str, "-")
+					if len(dateArr) != 2 {
+						err = fmt.Errorf("数据日期有误, len(dateArr): %d", len(dateArr))
+						return
+					}
+					t, e := time.ParseInLocation("2006.1.2", dateArr[1], time.Local)
+					if e != nil {
+						t, e = time.ParseInLocation("2006.01.02", dateArr[1], time.Local)
+						if e != nil {
+							err = fmt.Errorf("数据日期有误, date: %s", dateArr[1])
+							return
+						}
+					}
+					dataDate = t.Format(utils.FormatDate)
+					break
+				}
+			}
+
+			// 第二行-指标名称上半部分
+			if i == 1 {
+				for k, cell := range cells {
+					text := strings.TrimSpace(cell.String())
+					if k > 0 && text != "" {
+						// 纵向合并
+						if cell.VMerge > 0 {
+							thirdRowNames[k] = text
+							continue
+						}
+						secondRowNames[k] = text
+						// 横向合并
+						if cell.HMerge > 0 {
+							for j := 1; j <= cell.HMerge; j++ {
+								if secondRowNames[k+j] != "" {
+									continue
+								}
+								secondRowNames[k+j] = text
+							}
+						}
+					}
+				}
+			}
+
+			// 第三行-指标名称下半部分, 指标名称, 指标编码在这一步生成
+			if i == 2 {
+				for k, cell := range cells {
+					text := strings.TrimSpace(cell.String())
+					if k > 0 && text != "" {
+						thirdRowNames[k] = text
+					}
+				}
+			}
+
+			// 数据行
+			if i >= dataRowMin && i <= dataRowMax {
+				for k, cell := range cells {
+					text := strings.TrimSpace(cell.String())
+					if text == "" {
+						continue
+					}
+					// 第一个单元格为地区
+					if k == 0 {
+						if text == "合计" && (excelName == FenweiExcelNameA || excelName == FenweiExcelNameB) {
+							text = "三省合计"
+						}
+						rowProvince[i] = FenweiNewExcelProvinceMapping[text]
+						continue
+					}
+					rowsName := strings.TrimSpace(fmt.Sprint(secondRowNames[k], thirdRowNames[k]))
+					rowsName = strings.ReplaceAll(rowsName, "\n", "")
+					namePrefix := diffNameReplace[rowsName]
+					if namePrefix == "" {
+						namePrefix = rowsName
+					}
+					// 忽略部分名称
+					if utils.InArrayByStr(ignoreIndexName, namePrefix) {
+						continue
+					}
+					indexName := fmt.Sprintf("%s%s%s", rowProvince[i], namePrefix, indexNameSuffix)
+					//fmt.Printf("k: %d, %s\n", k, indexName)
+					name := strings.ReplaceAll(indexName, "/", "")
+					var middle, suffix string
+					for _, p := range FenweiProvinces {
+						// 此处山西其他可能会被错误的给鉴别为山西, 所以优先鉴别一下山西其他
+						if strings.Contains(name, "山西其他") {
+							middle = utils.GetFullPingYin("山西其他")
+							name = strings.ReplaceAll(name, p, "")
+							break
+						}
+						if strings.Contains(name, p) {
+							middle = utils.GetFullPingYin(p)
+							name = strings.ReplaceAll(name, p, "")
+							break
+						}
+					}
+					if middle == "" {
+						utils.FileLog.Info(fmt.Sprintf("指标名称转换编码有误, IndexName: %s", indexName))
+						continue
+					}
+					suffix = utils.GetFirstPingYin(name)
+					indexCode := fmt.Sprint(FenweiIndexNamePrefix, middle, suffix)
+
+					// 根据转换的编码, 匹配存量指标, 无则忽略
+					originIndex := indexMapping[indexCode]
+					if originIndex.FenweiIndexId <= 0 {
+						utils.FileLog.Info(fmt.Sprintf("无对应存量指标, IndexName: %s, IndexCode: %s", indexName, indexCode))
+						continue
+					}
+					indexKey := fmt.Sprintf("%d-%d", i, k)
+					if indexColIndex[indexKey] == nil {
+						indexColIndex[indexKey] = new(models.FenweiExcelIndex)
+						indexColIndex[indexKey].TerminalCode = utils.TerminalCode
+						indexColIndex[indexKey].ClassifyId = originIndex.ClassifyId
+						indexColIndex[indexKey].Frequency = originIndex.Frequency
+						indexColIndex[indexKey].Unit = originIndex.Unit
+						indexColIndex[indexKey].IndexName = indexName
+						indexColIndex[indexKey].IndexCode = indexCode
+						indexColIndex[indexKey].ExcelDataMap = make(map[string]string)
+						indexColIndex[indexKey].ExcelDataMap[dataDate] = text
+					}
+				}
+			}
+		}
+
+		// 只读取第一个sheet
+		break
+	}
+
+	for _, v := range indexColIndex {
+		if v.IndexCode == "" {
+			continue
+		}
+		indexes = append(indexes, v)
+	}
 	return
 }

services/fenwei_watch.go

@@ -0,0 +1,67 @@
+package services
+
+import (
+	"context"
+	"eta/eta_data_analysis/utils"
+	"fmt"
+	"io/fs"
+	"os"
+	"path/filepath"
+	"sync"
+)
+
+// HandleExcelFilePathMutex 创建一个互斥锁
+var HandleExcelFilePathMutex sync.Mutex
+
+func FenweiReadWatchIndexFile(context.Context) (err error) {
+	defer func() {
+		if err != nil {
+			fmt.Println("FenweiReadWatchIndexFile Err:" + err.Error())
+			utils.FileLog.Info(fmt.Sprintf("FenweiReadWatchIndexFile Err: %s", err.Error()))
+		}
+	}()
+	if utils.FenweiFileDir == "" {
+		utils.FileLog.Info("文件目录未配置")
+		return
+	}
+
+	// 上一个监听未结束时不执行
+	HandleExcelFilePathMutex.Lock()
+	defer HandleExcelFilePathMutex.Unlock()
+	//fmt.Println("FenweiFileDir: ", utils.FenweiFileDir)
+
+	err = filepath.Walk(utils.FenweiFileDir, func(path string, info fs.FileInfo, e error) error {
+		if e != nil {
+			err = fmt.Errorf("walk err: %s", e.Error())
+			return err
+		}
+		if info.IsDir() {
+			return nil
+		}
+		if path == "" {
+			return nil
+		}
+
+		//fmt.Println("path", path)
+		e = AnalysisFenweiNewExcel(path)
+		if e != nil {
+			err = fmt.Errorf("AnalysisFenweiNewExcel err: %s", e.Error())
+			return err
+		}
+
+		// 移动文件至已处理目录
+		e = utils.CreateDirIfNotExists(utils.FenweiOldFileDir)
+		if e != nil {
+			err = fmt.Errorf("CreateDirIfNotExists err: %s", e.Error())
+			return err
+		}
+		destPath := filepath.Join(utils.FenweiOldFileDir, info.Name())
+		e = os.Rename(path, destPath)
+		if e != nil {
+			err = fmt.Errorf("file move err: %s", e.Error())
+			return err
+		}
+		return nil
+	})
+	return
+}

services/task.go

@@ -2,7 +2,6 @@ package services
 
 import (
 	"eta/eta_data_analysis/utils"
-	"eta/eta_data_analysis/watch"
 	"fmt"
 	"github.com/beego/beego/v2/task"
 )
@@ -23,17 +22,16 @@ func Task() {
 
 		yongyiDownloadWeekly := task.NewTask("YongyiDownloadWeekly", "0 0 17,23 * * *", YongyiDownloadWeeyly)
 		task.AddTask("涌益咨询周度指标下载", yongyiDownloadWeekly)
-		task.StartTask()
 	}
 
 	// 汾渭数据
 	if utils.FenweiOpen == "1" {
-		// 每2分钟检测一次指标文件是否更新
-		fenWeiReadWatchIndexFile := task.NewTask("fenWeiReadWatchIndexFile", "0 */2 * * * *", watch.FenWeiReadWatchIndexFile)
+		// 每5分钟检测一次目录是否有新文件
+		fenWeiReadWatchIndexFile := task.NewTask("fenWeiReadWatchIndexFile", "0 */5 * * * *", FenweiReadWatchIndexFile)
 		task.AddTask("汾渭数据指标文件检测", fenWeiReadWatchIndexFile)
 	}
 
-	//go AnalysisFenWeiHistoryExcel("样本焦化厂焦煤可用天数指标转换2.xls")
+	task.StartTask()
 
 	fmt.Println("task end")
 }

utils/common.go

@@ -1231,3 +1231,25 @@ func CheckFrequency(leftFrequency, rightFrequency string) int {
 
 	return frequencyMap[leftFrequency] - frequencyMap[rightFrequency]
 }
+
+// CreateDirIfNotExists 目录是否存在
+func CreateDirIfNotExists(directoryPath string) error {
+	// 检查目录是否存在
+	_, err := os.Stat(directoryPath)
+	if os.IsNotExist(err) {
+		// 目录不存在，创建目录
+		err := os.MkdirAll(directoryPath, 0755)
+		if err != nil {
+			return fmt.Errorf("failed to create directory: %v", err)
+		}
+		fmt.Printf("Directory '%s' created successfully.\n", directoryPath)
+	} else if err != nil {
+		// 发生其他错误
+		return fmt.Errorf("failed to check directory: %v", err)
+	} else {
+		// 目录已存在
+		//fmt.Printf("Directory '%s' already exists.\n", directoryPath)
+	}
+
+	return nil
+}

utils/config.go

@@ -45,6 +45,7 @@ var (
 // 汾渭煤炭
 var (
 	FenweiFileDir      string // excel文件目录
+	FenweiOldFileDir   string // 已读取过的excel文件目录
 	FenweiOpen         string // 是否配置汾渭数据源
 	FenweiTerminalCode string // 汾渭终端号
 )
@@ -98,6 +99,7 @@ func init() {
 	{
 		FenweiOpen = config["fenwei_open"]
 		FenweiFileDir = config["fenwei_file_dir"]
+		FenweiOldFileDir = config["fenwei_old_file_dir"]
 		FenweiTerminalCode = config["fenwei_terminal_code"]
 	}
 

utils/constants.go

@@ -230,7 +230,8 @@ var FrequencyDaysMap = map[string]int{
 
 // edb_index_lib 的接口名称
 const (
-	LIB_ROUTE_YONGYI_HANDLE   = "yongyi/handle/excel_data" //涌益咨询处理excel数据并入库 数据地址
-	LIB_ROUTE_FENWEI_HANDLE   = "fenwei/handle/excel_data" // 汾渭煤炭excel数据入库接口地址
-	LIB_ROUTE_FENWEI_CLASSIFY = "fenwei/classify_tree"     // 汾渭煤炭分类接口地址
+	LIB_ROUTE_YONGYI_HANDLE     = "yongyi/handle/excel_data" //涌益咨询处理excel数据并入库 数据地址
+	LIB_ROUTE_FENWEI_HANDLE     = "fenwei/handle/excel_data" // 汾渭煤炭excel数据入库接口地址
+	LIB_ROUTE_FENWEI_CLASSIFY   = "fenwei/classify_tree"     // 汾渭煤炭分类接口地址
+	LIB_ROUTE_FENWEI_INDEX_LIST = "fenwei/base_index_list"   // 汾渭煤炭指标列表接口地址
 )

watch/fenwei_watch.go

@@ -1,47 +0,0 @@
-package watch
-
-import (
-	"context"
-	"fmt"
-)
-
-func FenWeiReadWatchIndexFile(context.Context) (err error) {
-	defer func() {
-		if err != nil {
-			fmt.Println("FenWeiReadWatchIndexFile Err:" + err.Error())
-		}
-	}()
-	//cacheClient := global.CacheClient
-	//if cacheClient == nil {
-	//	cacheClient = cache.New(365*24*time.Hour, 365*24*time.Hour)
-	//}
-	//err = filepath.Walk(global.CONFIG.Serve.IndexSaveDir, func(path string, info fs.FileInfo, err error) error {
-	//	if err != nil {
-	//		return err
-	//	}
-	//	if !info.IsDir() {
-	//		fileInfo, err := os.Stat(path)
-	//		if err != nil {
-	//			fmt.Println("os.Stat:", err.Error())
-	//		}
-	//		winFileAttr := fileInfo.Sys().(*syscall.Win32FileAttributeData)
-	//		modifyTimeStr := utils.SecondToTime(winFileAttr.LastWriteTime.Nanoseconds() / 1e9).Format(utils.FormatDateTime)
-	//
-	//		existModifyTime, ok := cacheClient.Get(path)
-	//		if ok {
-	//			existModifyTimeStr := existModifyTime.(string)
-	//			if existModifyTimeStr != modifyTimeStr {
-	//				localCache.AddIndexHandleExcel(path)
-	//				//WatchIndexFile(path)
-	//			}
-	//		} else {
-	//			localCache.AddIndexHandleExcel(path)
-	//			//WatchIndexFile(path)
-	//		}
-	//		cacheClient.Delete(path)
-	//		cacheClient.Set(path, modifyTimeStr, 24*time.Hour)
-	//	}
-	//	return nil
-	//})
-	return
-}