煤炭网

models/base_from_coal_coastal.go

@@ -0,0 +1,69 @@
+package models
+
+import (
+	"github.com/rdlucklib/rdluck_tools/orm"
+	"time"
+)
+
+// BaseFromCoalmineCoastalMapping 沿海八省动力煤用户供耗存数据映射表
+type BaseFromCoalmineCoastalMapping struct {
+	BaseFromCoalmineCoastalMappingID int    `orm:"column(base_from_coalmine_coastal_mapping_id);pk"`
+	IndexName                     string // 省份/企业名称
+	IndexCode                     string // 持买单量指标编码
+	CreateTime time.Time
+}
+
+// BaseFromCoalmineCoastalIndex 沿海八省动力煤用户供耗存数据指标表
+type BaseFromCoalmineCoastalIndex struct {
+	BaseFromCoalmineCoastalIndexID int       `orm:"column(base_from_coalmine_coastal_index_id);pk"`
+	IndexName                   string    // 省份/企业名称
+	IndexCode                   string    // 持买单量指标编码
+	DataTime                    string    // 指标时间
+	DealValue                   string    // 数据量
+	GroupName                   string    // 地区
+	Unit                        string    //单位
+	CreateTime                  time.Time `description:"插入时间"`
+	ModifyTime                  time.Time `description:"修改时间"`
+}
+
+//添加指标
+func AddBaseFromCoalmineCoastalMapping(item *BaseFromCoalmineCoastalMapping) (lastId int64, err error) {
+	o := orm.NewOrm()
+	o.Using("data")
+	lastId, err = o.Insert(item)
+	return
+}
+
+//查询指标
+func GetBaseFromCoalmineCoastalMapping() (items []*BaseFromCoalmineCoastalMapping, err error) {
+	o := orm.NewOrm()
+	o.Using("data")
+	sql := `SELECT * FROM base_from_coalmine_coastal_mapping`
+	_, err = o.Raw(sql).QueryRows(&items)
+	return
+}
+
+//查询指标
+func GetBaseFromCoalmineCoastalIndex() (items []*BaseFromCoalmineCoastalIndex, err error) {
+	o := orm.NewOrm()
+	o.Using("data")
+	sql := `SELECT * FROM base_from_coalmine_coastal_index`
+	_, err = o.Raw(sql).QueryRows(&items)
+	return
+}
+
+//添加数据
+func AddBaseFromCoalCoastalIndex(item *BaseFromCoalmineCoastalIndex) (lastId int64, err error) {
+	o := orm.NewOrm()
+	o.Using("data")
+	lastId, err = o.Insert(item)
+	return
+}
+
+func UpdateBaseFromCoalCoastalIndex(item *BaseFromCoalmineCoastalIndex) (err error) {
+	o := orm.NewOrm()
+	o.Using("data")
+	sql := `UPDATE base_from_coalmine_coastal_index SET deal_value=?  WHERE index_name=?  AND  data_time = ?`
+	_, err = o.Raw(sql, item.DealValue, item.IndexName, item.DataTime).Exec()
+	return
+}

models/base_from_coal_firm.go

@@ -0,0 +1,69 @@
+package models
+
+import (
+	"github.com/rdlucklib/rdluck_tools/orm"
+	"time"
+)
+
+// BaseFromCoalmineFirmMapping 煤矿产量周度公司数据指标表
+type BaseFromCoalmineFirmMapping struct {
+	BaseFromCoalmineFirmMappingID int    `orm:"column(base_from_coalmine_firm_mapping_id);pk"`
+	IndexName                     string // 省份/企业名称
+	IndexCode                     string // 持买单量指标编码
+	Source                        string // 来源
+	CreateTime                    time.Time
+}
+
+// BaseFromCoalmineFirmIndex 煤矿产量周度公司数据表
+type BaseFromCoalmineFirmIndex struct {
+	BaseFromCoalmineFirmIndexID int       `orm:"column(base_from_coalmine_firm_index_id);pk"`
+	IndexName                   string    // 省份/企业名称
+	IndexCode                   string    // 持买单量指标编码
+	DataTime                    string    // 指标时间
+	DealValue                   string    // 数据量
+	GroupName                   string    // 集团名
+	CreateTime                  time.Time `description:"插入时间"`
+	ModifyTime                  time.Time `description:"修改时间"`
+}
+
+//添加指标
+func AddBaseFromCoalmineFirmMapping(item *BaseFromCoalmineFirmMapping) (lastId int64, err error) {
+	o := orm.NewOrm()
+	o.Using("data")
+	lastId, err = o.Insert(item)
+	return
+}
+
+//查询指标
+func GetBaseFromCoalmineFirmMapping() (items []*BaseFromCoalmineFirmMapping, err error) {
+	o := orm.NewOrm()
+	o.Using("data")
+	sql := `SELECT * FROM base_from_coalmine_firm_mapping`
+	_, err = o.Raw(sql).QueryRows(&items)
+	return
+}
+
+//查询指标
+func GetBaseFromCoalmineFirmIndex() (items []*BaseFromCoalmineFirmIndex, err error) {
+	o := orm.NewOrm()
+	o.Using("data")
+	sql := `SELECT * FROM base_from_coalmine_firm_index`
+	_, err = o.Raw(sql).QueryRows(&items)
+	return
+}
+
+//添加数据
+func AddBaseFromCoalFirmIndex(item *BaseFromCoalmineFirmIndex) (lastId int64, err error) {
+	o := orm.NewOrm()
+	o.Using("data")
+	lastId, err = o.Insert(item)
+	return
+}
+
+func UpdateBaseFromCoalFirmIndex(item *BaseFromCoalmineFirmIndex) (err error) {
+	o := orm.NewOrm()
+	o.Using("data")
+	sql := `UPDATE base_from_coalmine_firm_index SET deal_value=?  WHERE index_name=?  AND  data_time = ?`
+	_, err = o.Raw(sql, item.DealValue, item.IndexName, item.DataTime).Exec()
+	return
+}

models/base_from_coal_inland.go

@@ -0,0 +1,68 @@
+package models
+
+import (
+	"github.com/rdlucklib/rdluck_tools/orm"
+	"time"
+)
+
+// BaseFromCoalmineInlandMapping 沿海八省动力煤用户供耗存数据映射表
+type BaseFromCoalmineInlandMapping struct {
+	BaseFromCoalmineInlandMappingID int    `orm:"column(base_from_coalmine_inland_mapping_id);pk"`
+	IndexName                     string // 省份/企业名称
+	IndexCode                     string // 持买单量指标编码
+	CreateTime time.Time
+}
+
+// BaseFromCoalmineInlandIndex 沿海八省动力煤用户供耗存数据指标表
+type BaseFromCoalmineInlandIndex struct {
+	BaseFromCoalmineInlandIndexID int       `orm:"column(base_from_coalmine_inland_index_id);pk"`
+	IndexName                   string    // 省份/企业名称
+	IndexCode                   string    // 持买单量指标编码
+	DataTime                    string    // 指标时间
+	DealValue                   string    // 数据量
+	GroupName                   string    // 地区
+	CreateTime                  time.Time `description:"插入时间"`
+	ModifyTime                  time.Time `description:"修改时间"`
+}
+
+//添加指标
+func AddBaseFromCoalmineInlandMapping(item *BaseFromCoalmineInlandMapping) (lastId int64, err error) {
+	o := orm.NewOrm()
+	o.Using("data")
+	lastId, err = o.Insert(item)
+	return
+}
+
+//查询指标
+func GetBaseFromCoalmineInlandMapping() (items []*BaseFromCoalmineInlandMapping, err error) {
+	o := orm.NewOrm()
+	o.Using("data")
+	sql := `SELECT * FROM base_from_coalmine_inland_mapping`
+	_, err = o.Raw(sql).QueryRows(&items)
+	return
+}
+
+//查询指标
+func GetBaseFromCoalmineInlandIndex() (items []*BaseFromCoalmineInlandIndex, err error) {
+	o := orm.NewOrm()
+	o.Using("data")
+	sql := `SELECT * FROM base_from_coalmine_inland_index`
+	_, err = o.Raw(sql).QueryRows(&items)
+	return
+}
+
+//添加数据
+func AddBaseFromCoalInlandIndex(item *BaseFromCoalmineInlandIndex) (lastId int64, err error) {
+	o := orm.NewOrm()
+	o.Using("data")
+	lastId, err = o.Insert(item)
+	return
+}
+
+func UpdateBaseFromCoalInlandIndex(item *BaseFromCoalmineInlandIndex) (err error) {
+	o := orm.NewOrm()
+	o.Using("data")
+	sql := `UPDATE base_from_coalmine_inland_index SET deal_value=?  WHERE index_name=?  AND  data_time = ?`
+	_, err = o.Raw(sql, item.DealValue, item.IndexName, item.DataTime).Exec()
+	return
+}

models/base_from_coalmine.go

@@ -116,3 +116,20 @@ func AddBaseFromCoalmineCompanyIndex(item *BaseFromCoalmineCompanyIndex) (lastId
 	lastId, err = o.Insert(item)
 	return
 }
+
+//查询公司指标
+func GetBaseFromCoalmineCompanyIndex() (items []*BaseFromCoalmineCompanyIndex,err error){
+	o := orm.NewOrm()
+	o.Using("data")
+	sql := `SELECT * FROM base_from_coalmine_company_index`
+	_, err = o.Raw(sql).QueryRows(&items)
+	return
+}
+
+func UpdateBaseFromCoalmineCompanyIndex(item *BaseFromCoalmineCompanyIndex) (err error) {
+	o := orm.NewOrm()
+	o.Using("data")
+	sql := `UPDATE base_from_coalmine_company_index SET deal_value=?  WHERE index_name=?  AND  data_time = ?`
+	_, err = o.Raw(sql, item.DealValue, item.IndexName, item.DataTime).Exec()
+	return
+}

models/db.go

@@ -39,7 +39,12 @@ func init() {
 		new(BaseFromCoalmineIndex),
 		new(BaseFromCoalmineCompanyMapping),
 		new(BaseFromCoalmineCompanyIndex),
-
+		new(BaseFromCoalmineFirmIndex),
+		new(BaseFromCoalmineFirmMapping),
+		new(BaseFromCoalmineCoastalIndex),
+		new(BaseFromCoalmineCoastalMapping),
+		new(BaseFromCoalmineInlandIndex),
+		new(BaseFromCoalmineInlandMapping),
 		//new(BaseFromTradeDalianData),
 	)
 }

services/commodity_coal_coastal.go

@@ -0,0 +1,199 @@
+package services
+
+import (
+	"fmt"
+	"github.com/mozillazg/go-pinyin"
+	"github.com/tealeg/xlsx"
+	"hongze/hongze_data_crawler/models"
+	"hongze/hongze_data_crawler/utils"
+	"strconv"
+	"strings"
+	"time"
+)
+
+func FileCoalCoastal() (err error) {
+	defer func() {
+		if err != nil {
+			fmt.Println("RefreshDataFromDalian Err:" + err.Error())
+			go utils.SendEmail(utils.APPNAME+"【"+utils.RunMode+"】"+"失败提醒", "RefreshDataFromDalian ErrMsg:"+err.Error(), utils.EmailSendToUsers)
+		}
+	}()
+	path := "static/file/coastal.xlsx"
+	xlFile, err := xlsx.OpenFile(path)
+	if err != nil {
+		fmt.Println(err)
+		return
+	}
+
+	var mappingItems []*models.BaseFromCoalmineCoastalMapping
+	var indexItems []*models.BaseFromCoalmineCoastalIndex
+	var codeMap = make(map[string]string)
+	var indexMap = make(map[string]string)
+	var nameMap = make(map[int]string)
+	var groupMap = make(map[int]string)
+	var dataTime string
+	var sheetName, unit string
+	codeList, err := models.GetBaseFromCoalmineCoastalMapping()
+	if err != nil && err.Error() != utils.ErrNoRow() {
+		utils.FileLog.Info("获取煤炭指标失败：", err)
+		return err
+	}
+	if len(codeList) > 0 {
+		for _, v := range codeList {
+			codeMap[v.IndexName] = v.IndexCode
+		}
+	}
+	indexCompanyList, err := models.GetBaseFromCoalmineCoastalIndex()
+	if err != nil && err.Error() != utils.ErrNoRow() {
+		utils.FileLog.Info("获取煤炭公司指标失败：", err)
+		return err
+	}
+	if len(indexCompanyList) > 0 {
+		for _, v := range indexCompanyList {
+			indexMap[v.IndexName+v.DataTime] = v.DealValue
+		}
+	}
+
+	for _, sheet := range xlFile.Sheets {
+		sheetName = sheet.Name
+		//遍历行读取
+		maxCol := sheet.MaxCol
+		for i := 0; i < maxCol; i++ {
+			if i == 0 {
+				row := sheet.Row(i)
+				cells := row.Cells
+				for k, cell := range cells {
+					if k > 0 && k < 9 {
+						text := cell.String()
+						groupMap[k] = text
+						var item models.BaseFromCoalmineCoastalMapping
+						//合计命名
+						if k == 9 {
+							item.IndexName = sheetName
+						}
+						item.IndexName = text + sheetName
+						nameMap[k] = item.IndexName
+						//code
+						exists := ContainsSpecialName(text)
+						var code string
+						if exists {
+							abbr := trimProvinceName(text)
+							//取处理后公司名首字母缩写
+							a := pinyin.NewArgs()
+							rows := pinyin.Pinyin(sheetName, a)
+							for i := 0; i < len(rows); i++ {
+								if len(rows[i]) != 0 {
+									str := rows[i][0]
+									pi := str[0:1]
+									code += pi
+								}
+							}
+							item.IndexCode = abbr + code
+						} else {
+							a := pinyin.NewArgs()
+							rows := pinyin.Pinyin(text+sheetName, a)
+							for i := 0; i < len(rows); i++ {
+								if len(rows[i]) != 0 {
+									str := rows[i][0]
+									pi := str[0:1]
+									code += pi
+								}
+							}
+							item.IndexCode = code
+						}
+						item.CreateTime = time.Now()
+						mappingItems = append(mappingItems, &item)
+					}
+					if k == 10 {
+						unit = strings.Replace(cell.String(),"单位：","",-1)
+					}
+				}
+			}
+			if i > 0 {
+				row := sheet.Row(i)
+				cells := row.Cells
+				for k, cell := range cells {
+					if k < 9 {
+						var item models.BaseFromCoalmineCoastalIndex
+						if k == 0 {
+							text := cell.String()
+							dataTime = ExcelDateToDate(text).Format(utils.FormatDate)
+						}
+						if k > 0 {
+							text := cell.String()
+							item.IndexName = nameMap[k]
+							item.IndexCode = codeMap[nameMap[k]]
+							item.DealValue = text
+							item.GroupName = groupMap[k]
+							item.DataTime = dataTime
+							item.ModifyTime = time.Now()
+							item.CreateTime = time.Now()
+						}
+						item.Unit = unit
+						indexItems = append(indexItems, &item)
+					}
+				}
+			}
+		}
+	}
+	//添加数据到数据库
+	for _, v := range mappingItems {
+		if codeMap[v.IndexName] == "" {
+			codeMap[v.IndexName] = v.IndexCode
+			newId, err := models.AddBaseFromCoalmineCoastalMapping(v)
+			if err != nil {
+				fmt.Println("添加指标名称错误")
+			} else {
+				fmt.Println("添加指标名称成功", newId)
+			}
+		}
+	}
+	fmt.Println("指标操作完成")
+
+	//给indexItem中的code赋值并插入index表
+	for _, v := range indexItems {
+		v.IndexCode = codeMap[v.IndexName]
+		if indexMap[v.IndexName+v.DataTime] == "" && v.DealValue != "" {
+			newId, err := models.AddBaseFromCoalCoastalIndex(v)
+			if err != nil {
+				fmt.Println("添加数据错误", err)
+			} else {
+				fmt.Println("新增成功", newId)
+			}
+		} else {
+			if indexMap[v.IndexName+v.DataTime] != v.DealValue && v.DealValue != "" {
+				err = models.UpdateBaseFromCoalCoastalIndex(v)
+				if err != nil {
+					fmt.Println("修改数据错误错误", err)
+					return
+				}
+			}
+		}
+	}
+	return
+}
+
+func ttrimProvinceName(name string) string {
+	name = strings.Replace(name, "陕西省", "sn", -1)
+	name = strings.Replace(name, "陕西", "sn", -1)
+	name = strings.Replace(name, "海南市", "hi", -1)
+	name = strings.Replace(name, "海南", "hi", -1)
+	name = strings.Replace(name, "河南省", "ha", -1)
+	name = strings.Replace(name, "河南", "ha", -1)
+	name = strings.Replace(name, "河北省", "he", -1)
+	name = strings.Replace(name, "河北", "he", -1)
+	name = strings.Replace(name, "澳门", "mo", -1)
+	name = strings.Replace(name, "内蒙古自治区", "nm", -1)
+	name = strings.Replace(name, "内蒙古", "nm", -1)
+	name = strings.Replace(name, "黑龙江", "hl", -1)
+	name = strings.Replace(name, "（", "", -1)
+	name = strings.Replace(name, "）", "", -1)
+	name = strings.Replace(name, "+", "", -1)
+	return name
+}
+
+func ExcelDateToDate(excelDate string) time.Time {
+	excelTime := time.Date(1899, time.December, 30, 0, 0, 0, 0, time.UTC)
+	var days, _ = strconv.Atoi(excelDate)
+	return excelTime.Add(time.Second * time.Duration(days*86400))
+}

services/commodity_coal_firm.go

@@ -0,0 +1,251 @@
+package services
+
+import (
+	"fmt"
+	"github.com/mozillazg/go-pinyin"
+	"github.com/tealeg/xlsx"
+	"hongze/hongze_data_crawler/models"
+	"hongze/hongze_data_crawler/utils"
+	"strings"
+	"time"
+)
+
+func FileCoalFirm() (err error) {
+	defer func() {
+		if err != nil {
+			fmt.Println("RefreshDataFromDalian Err:" + err.Error())
+			go utils.SendEmail(utils.APPNAME+"【"+utils.RunMode+"】"+"失败提醒", "RefreshDataFromDalian ErrMsg:"+err.Error(), utils.EmailSendToUsers)
+		}
+	}()
+	path := "static/file/firm.xlsx"
+	xlFile, err := xlsx.OpenFile(path)
+	if err != nil {
+		fmt.Println(err)
+		return
+	}
+
+	var source string
+	var year, month, day string
+	var yearMap = make(map[int]string)
+	var monthMap = make(map[int]string)
+	var dateMap = make(map[int]string)
+	var mappingItems []*models.BaseFromCoalmineFirmMapping
+	var indexItems []*models.BaseFromCoalmineFirmIndex
+	var codeMap = make(map[string]string)
+	var indexMap = make(map[string]string)
+	var groupName string
+	codeList, err := models.GetBaseFromCoalmineFirmMapping()
+	if err != nil && err.Error() != utils.ErrNoRow() {
+		utils.FileLog.Info("获取煤炭指标失败：", err)
+		return err
+	}
+	if len(codeList) > 0 {
+		for _, v := range codeList {
+			codeMap[v.IndexName] = v.IndexCode
+		}
+	}
+	indexCompanyList, err := models.GetBaseFromCoalmineFirmIndex()
+	if err != nil && err.Error() != utils.ErrNoRow() {
+		utils.FileLog.Info("获取煤炭公司指标失败：", err)
+		return err
+	}
+	if len(indexCompanyList) > 0 {
+		for _, v := range indexCompanyList {
+			indexMap[v.IndexName+v.DataTime] = v.DealValue
+		}
+	}
+
+	for _, sheet := range xlFile.Sheets {
+		//遍历行读取
+		maxRow := sheet.MaxRow
+		for i := 0; i < maxRow; i++ {
+			//获取制表来源
+			if i == 1 {
+				row := sheet.Row(i)
+				cells := row.Cells
+			Loop:
+				for k, cell := range cells {
+					text := cell.String()
+					if k == 0 {
+						source = text
+						break Loop
+					}
+				}
+				if source != "" {
+					source = strings.Replace(source, "制表单位:", "", -1)
+				}
+			}
+			//获取年份
+			if i == 2 {
+				row := sheet.Row(i)
+				cells := row.Cells
+				for k, cell := range cells {
+					text := cell.String()
+					if k > 0 && text != "" {
+						year = text
+						yearMap[k] = year
+					}
+				}
+			}
+			//获取月份
+			if i == 3 {
+				row := sheet.Row(i)
+				cells := row.Cells
+				for k, cell := range cells {
+					text := cell.String()
+					if k > 0 && text != "" {
+						month = text
+						monthMap[k] = month
+					}
+				}
+			}
+			//获取旬度
+			if i == 4 {
+				row := sheet.Row(i)
+				cells := row.Cells
+				for k, cell := range cells {
+					text := cell.String()
+					if k > 0 && text != "" {
+						if yearMap[k] != "" {
+							year = yearMap[k]
+						}
+						if monthMap[k] != "" {
+							month = monthMap[k]
+						}
+						day = text
+						dateMap[k] = year + month + day
+					}
+				}
+			}
+			//获取企业或地区名及信息
+			if i > 4 {
+				row := sheet.Row(i)
+				cells := row.Cells
+				var companyName string
+				for k, cell := range cells {
+					if k == 0 {
+						companyName = cell.String()
+
+						//省名
+						if !strings.Contains(companyName, " ") {
+							groupName = companyName
+						}
+						if groupName == "全国合计" {
+							groupName = "全国"
+						}
+						//若有下面四个名称要拼上省名不然会重复
+						if companyName == "  地方合计" || companyName == "    地方国有" ||
+							companyName == "    地方乡镇" || companyName == "  国有重点" {
+							companyName = groupName + companyName
+						}
+						companyName = strings.Replace(companyName, " ", "", -1)
+						var item models.BaseFromCoalmineFirmMapping
+						//生成code
+						exists := ContainsSpecialName(companyName)
+						var code string
+						if exists {
+							abbr := trimProvinceName(companyName)
+							//取处理后公司名首字母缩写
+							a := pinyin.NewArgs()
+							rows := pinyin.Pinyin(companyName[2:], a)
+							for i := 0; i < len(rows); i++ {
+								if len(rows[i]) != 0 {
+									str := rows[i][0]
+									pi := str[0:1]
+									code += pi
+								}
+							}
+							item.IndexCode = abbr[:2] + code
+
+						} else {
+							a := pinyin.NewArgs()
+							rows := pinyin.Pinyin(companyName, a)
+							for i := 0; i < len(rows); i++ {
+								if len(rows[i]) != 0 {
+									str := rows[i][0]
+									pi := str[0:1]
+									code += pi
+								}
+							}
+							item.IndexCode = code
+						}
+
+						item.IndexName = companyName
+						item.IndexCode = code
+						item.Source = source
+						item.CreateTime = time.Now()
+						mappingItems = append(mappingItems, &item)
+					} else {
+						dealValue := cell.String()
+
+						item := models.BaseFromCoalmineFirmIndex{
+							IndexName:  companyName,
+							IndexCode:  codeMap[companyName],
+							DataTime:   dateMap[k],
+							DealValue:  dealValue,
+							GroupName:  groupName,
+							CreateTime: time.Now(),
+							ModifyTime: time.Now(),
+						}
+
+						indexItems = append(indexItems, &item)
+					}
+				}
+			}
+		}
+	}
+	//添加数据到数据库
+	for _, v := range mappingItems {
+		if codeMap[v.IndexName] == "" {
+			codeMap[v.IndexName] = v.IndexCode
+			newId, err := models.AddBaseFromCoalmineFirmMapping(v)
+			if err != nil {
+				fmt.Println("添加指标名称错误")
+			} else {
+				fmt.Println("添加指标名称成功", newId)
+			}
+		}
+	}
+	fmt.Println("指标操作完成")
+
+	//给indexItem中的code赋值并插入index表
+	for _, v := range indexItems {
+		v.IndexCode = codeMap[v.IndexName]
+		if indexMap[v.IndexName+v.DataTime] == "" && v.DealValue != "" {
+			newId, err := models.AddBaseFromCoalFirmIndex(v)
+			if err != nil {
+				fmt.Println("添加数据错误", err)
+			} else {
+				fmt.Println("新增成功", newId)
+			}
+		} else {
+			if indexMap[v.IndexName+v.DataTime] != v.DealValue && v.DealValue != "" {
+				err = models.UpdateBaseFromCoalFirmIndex(v)
+				if err != nil {
+					fmt.Println("修改数据错误错误", err)
+					return
+				}
+			}
+		}
+	}
+	return
+}
+
+func trimProvinceName(name string) string {
+	name = strings.Replace(name, "陕西省", "sn", -1)
+	name = strings.Replace(name, "陕西", "sn", -1)
+	name = strings.Replace(name, "海南市", "hi", -1)
+	name = strings.Replace(name, "海南", "hi", -1)
+	name = strings.Replace(name, "河南省", "ha", -1)
+	name = strings.Replace(name, "河南", "ha", -1)
+	name = strings.Replace(name, "河北省", "he", -1)
+	name = strings.Replace(name, "河北", "he", -1)
+	name = strings.Replace(name, "澳门", "mo", -1)
+	name = strings.Replace(name, "内蒙古自治区", "nm", -1)
+	name = strings.Replace(name, "内蒙古", "nm", -1)
+	name = strings.Replace(name, "黑龙江", "hl", -1)
+	name = strings.Replace(name, "（", "", -1)
+	name = strings.Replace(name, "）", "", -1)
+	name = strings.Replace(name, "+", "", -1)
+	return name
+}

services/commodity_coal_inland.go

@@ -0,0 +1,175 @@
+package services
+
+import (
+	"fmt"
+	"github.com/mozillazg/go-pinyin"
+	"github.com/tealeg/xlsx"
+	"hongze/hongze_data_crawler/models"
+	"hongze/hongze_data_crawler/utils"
+	"time"
+)
+
+func FileCoalInland() (err error) {
+	defer func() {
+		if err != nil {
+			fmt.Println("RefreshDataFromDalian Err:" + err.Error())
+			go utils.SendEmail(utils.APPNAME+"【"+utils.RunMode+"】"+"失败提醒", "RefreshDataFromDalian ErrMsg:"+err.Error(), utils.EmailSendToUsers)
+		}
+	}()
+	path := "static/file/inland.xlsx"
+	xlFile, err := xlsx.OpenFile(path)
+	if err != nil {
+		fmt.Println(err)
+		return
+	}
+
+	var mappingItems []*models.BaseFromCoalmineInlandMapping
+	var indexItems []*models.BaseFromCoalmineInlandIndex
+	var codeMap = make(map[string]string)
+	var indexMap = make(map[string]string)
+	var nameMap = make(map[int]string)
+	var groupMap = make(map[int]string)
+	var dataTime string
+	var sheetName string
+	codeList, err := models.GetBaseFromCoalmineInlandMapping()
+	if err != nil && err.Error() != utils.ErrNoRow() {
+		utils.FileLog.Info("获取煤炭指标失败：", err)
+		return err
+	}
+	if len(codeList) > 0 {
+		for _, v := range codeList {
+			codeMap[v.IndexName] = v.IndexCode
+		}
+	}
+	indexCompanyList, err := models.GetBaseFromCoalmineInlandIndex()
+	if err != nil && err.Error() != utils.ErrNoRow() {
+		utils.FileLog.Info("获取煤炭公司指标失败：", err)
+		return err
+	}
+	if len(indexCompanyList) > 0 {
+		for _, v := range indexCompanyList {
+			indexMap[v.IndexName+v.DataTime] = v.DealValue
+		}
+	}
+
+	for s, sheet := range xlFile.Sheets {
+		sheetName = sheet.Name
+		//遍历行读取
+		maxCol := sheet.MaxCol
+		for i := 0; i < maxCol; i++ {
+			if i == 0 {
+				row := sheet.Row(i)
+				cells := row.Cells
+				for k, cell := range cells {
+					if k > 0 && k < 18 {
+						text := cell.String()
+						groupMap[k] = text
+						var item models.BaseFromCoalmineInlandMapping
+						//合计命名
+						if k == 9 {
+							item.IndexName = sheetName
+						}
+						item.IndexName = text + sheetName
+						nameMap[k] = item.IndexName
+						//code
+						exists := ContainsSpecialName(text)
+						var code string
+						if exists {
+							abbr := trimProvinceName(text)
+							//取处理后公司名首字母缩写
+							a := pinyin.NewArgs()
+							rows := pinyin.Pinyin(sheetName, a)
+							for i := 0; i < len(rows); i++ {
+								if len(rows[i]) != 0 {
+									str := rows[i][0]
+									pi := str[0:1]
+									code += pi
+								}
+							}
+							item.IndexCode = abbr + code
+						} else {
+							a := pinyin.NewArgs()
+							rows := pinyin.Pinyin(text+sheetName, a)
+							for i := 0; i < len(rows); i++ {
+								if len(rows[i]) != 0 {
+									str := rows[i][0]
+									pi := str[0:1]
+									code += pi
+								}
+							}
+							item.IndexCode = code
+						}
+						item.CreateTime = time.Now()
+						mappingItems = append(mappingItems, &item)
+					}
+				}
+			}
+			if i > 0 {
+				row := sheet.Row(i)
+				cells := row.Cells
+				for k, cell := range cells {
+					if k < 18 {
+						var item models.BaseFromCoalmineInlandIndex
+						if k == 0 {
+							if s == 0 {
+								text := cell.String()
+								t, _ := time.ParseInLocation("01-02-06", text, time.Local)
+								dataTime = t.Format(utils.FormatDate)
+							} else {
+								text := cell.String()
+								dataTime = ExcelDateToDate(text).Format(utils.FormatDate)
+							}
+						}
+						if k > 0 {
+							text := cell.String()
+							item.IndexName = nameMap[k]
+							item.IndexCode = codeMap[nameMap[k]]
+							item.DealValue = text
+							item.GroupName = groupMap[k]
+							item.DataTime = dataTime
+							item.ModifyTime = time.Now()
+							item.CreateTime = time.Now()
+						}
+						indexItems = append(indexItems, &item)
+					}
+				}
+			}
+		}
+	}
+	//添加数据到数据库
+	for _, v := range mappingItems {
+		if codeMap[v.IndexName] == "" {
+			codeMap[v.IndexName] = v.IndexCode
+			newId, err := models.AddBaseFromCoalmineInlandMapping(v)
+			if err != nil {
+				fmt.Println("添加指标名称错误")
+			} else {
+				fmt.Println("添加指标名称成功", newId)
+			}
+		}
+	}
+	fmt.Println("指标操作完成")
+
+	//给indexItem中的code赋值并插入index表
+	for _, v := range indexItems {
+		v.IndexCode = codeMap[v.IndexName]
+		if indexMap[v.IndexName+v.DataTime] == "" && v.DealValue != "" {
+			newId, err := models.AddBaseFromCoalInlandIndex(v)
+			if err != nil {
+				fmt.Println("添加数据错误", err)
+			} else {
+				fmt.Println("新增成功", newId)
+			}
+		} else {
+			if indexMap[v.IndexName+v.DataTime] != v.DealValue && v.DealValue != "" {
+				err = models.UpdateBaseFromCoalInlandIndex(v)
+				if err != nil {
+					fmt.Println("修改数据错误错误", err)
+					return
+				}
+			}
+		}
+	}
+	return
+}
+

services/commodity_coal_mine.go

@@ -2,6 +2,7 @@ package services
 
 import (
 	"fmt"
+	"github.com/mozillazg/go-pinyin"
 	"github.com/tealeg/xlsx"
 	"hongze/hongze_data_crawler/models"
 	"hongze/hongze_data_crawler/utils"
@@ -36,6 +37,7 @@ func FileCoalJsm() (err error) {
 	var codeMap = make(map[string]string)
 	var indexMap = make(map[string]string)
 	var codeCompanyMap = make(map[string]string)
+	var indexCompanyMap = make(map[string]string)
 	var items []*models.BaseFromCoalmineMapping
 	var itemsCompany []*models.BaseFromCoalmineCompanyMapping
 	var itemsIndex []*models.BaseFromCoalmineIndex
@@ -63,14 +65,25 @@ func FileCoalJsm() (err error) {
 		}
 	}
 
-	indexCompanyList, err := models.GetBaseFromCoalmineIndex()
+	indexList, err := models.GetBaseFromCoalmineIndex()
+	if err != nil && err.Error() != utils.ErrNoRow() {
+		utils.FileLog.Info("获取煤炭公司指标失败：", err)
+		return err
+	}
+	if len(indexList) > 0 {
+		for _, v := range indexList {
+			indexMap[v.IndexName+v.DataTime] = v.DealValue
+		}
+	}
+
+	indexCompanyList, err := models.GetBaseFromCoalmineCompanyIndex()
 	if err != nil && err.Error() != utils.ErrNoRow() {
 		utils.FileLog.Info("获取煤炭公司指标失败：", err)
 		return err
 	}
 	if len(indexCompanyList) > 0 {
 		for _, v := range indexCompanyList {
-			indexMap[v.IndexName+v.DataTime] = v.DealValue
+			indexCompanyMap[v.IndexName+v.DataTime] = v.DealValue
 		}
 	}
 
@@ -111,8 +124,38 @@ func FileCoalJsm() (err error) {
 						if k == 4 {
 							item := new(models.BaseFromCoalmineMapping)
 							exchange = text
-							item.IndexName = source + "_" + province + "_" + exchange
-							item.IndexCode = utils.MD5(item.IndexName)
+
+							exists := ContainsSpecialName(province)
+							var strResult string
+							if exists {
+								abbr := TrimProvinceName(province)
+								//取处理后公司名首字母缩写
+								a := pinyin.NewArgs()
+								rows := pinyin.Pinyin(exchange, a)
+								for i := 0; i < len(rows); i++ {
+									if len(rows[i]) != 0 {
+										str := rows[i][0]
+										pi := str[0:1]
+										strResult += pi
+									}
+								}
+								item.IndexCode = abbr + strResult
+							} else {
+								//取处理后公司名首字母缩写
+								a := pinyin.NewArgs()
+								rows := pinyin.Pinyin(province+exchange, a)
+								for i := 0; i < len(rows); i++ {
+									if len(rows[i]) != 0 {
+										str := rows[i][0]
+										pi := str[0:1]
+										strResult += pi
+									}
+								}
+								item.IndexCode = strResult
+							}
+
+							item.IndexName = province + exchange
+
 							item.Exchange = exchange
 							item.Description = description
 							item.Province = province
@@ -130,6 +173,7 @@ func FileCoalJsm() (err error) {
 				cells := row.Cells
 				for k, cell := range cells {
 					text := cell.String()
+					fmt.Println("text：",text)
 					stamp, _ := time.ParseInLocation("01-02-06", text, time.Now().Location())
 					if k > 4 {
 						dateMap[k] = stamp.Format(utils.FormatDate)
@@ -144,7 +188,7 @@ func FileCoalJsm() (err error) {
 					text := cell.String()
 					if k > 4 {
 						item := new(models.BaseFromCoalmineIndex)
-						item.IndexName = source + "_" + province + "_" + exchange
+						item.IndexName = province + exchange
 						item.IndexCode = codeMap[item.IndexName]
 						item.Exchange = exchange
 						item.DealValue = text
@@ -179,13 +223,33 @@ func FileCoalJsm() (err error) {
 						if k == 4 {
 							item := new(models.BaseFromCoalmineCompanyMapping)
 							group = text
-							item.IndexName = source + "_" + province + "_" + city + "_" + companyName
-							item.IndexCode = utils.MD5(item.IndexName)
+							item.CompanyName = companyName
+							trimName := TrimCompanyName(companyName)
+							item.IndexName = trimName
+							//取处理后公司名首字母缩写
+							strResult := ""
+							a := pinyin.NewArgs()
+							rows := pinyin.Pinyin(trimName, a)
+							for i := 0; i < len(rows); i++ {
+								if len(rows[i]) != 0 {
+									str := rows[i][0]
+									pi := str[0:1]
+									strResult += pi
+								}
+							}
+							item.IndexCode = strResult
+
 							item.Rank = rank
 							item.Province = province
 							item.City = city
-							item.CompanyName = companyName
-							item.GroupName = group
+
+							//处理无类别名时的情况
+							if group == "无" {
+								item.GroupName = trimName
+							} else {
+								item.GroupName = group
+							}
+
 							item.Source = source
 							item.CreateTime = time.Now()
 							itemsCompany = append(itemsCompany, item)
@@ -203,7 +267,8 @@ func FileCoalJsm() (err error) {
 					text := cell.String()
 					if k > 4 {
 						item := new(models.BaseFromCoalmineCompanyIndex)
-						item.IndexName = source + "_" + province + "_" + city + "_" + companyName
+						companyName = TrimCompanyName(companyName)
+						item.IndexName = companyName
 						item.IndexCode = codeCompanyMap[item.IndexName]
 						item.DealValue = text
 						item.DataTime = dateMap[k]
@@ -241,7 +306,7 @@ func FileCoalJsm() (err error) {
 				fmt.Println("新增成功", newId)
 			}
 		} else {
-			if indexMap[v.IndexName+v.DataTime] == v.IndexName+v.DataTime && indexMap[v.IndexName+v.DataTime] != v.DealValue {
+			if indexMap[v.IndexName+v.DataTime] != v.DealValue && v.DealValue != "" {
 				err = models.UpdateBaseFromCoalmineIndex(v)
 				if err != nil {
 					fmt.Println("修改数据错误错误", err)
@@ -265,14 +330,63 @@ func FileCoalJsm() (err error) {
 	}
 	fmt.Println("公司指标操作完成")
 
-	//for _, v := range itemsCompanyIndex {
-	//	newId, err := models.AddBaseFromCoalmineCompanyIndex(v)
-	//	if err != nil {
-	//		fmt.Println("添加公司数据错误", err)
-	//return
-	//	} else {
-	//		fmt.Println("新增公司数据成功", newId)
-	//	}
-	//}
+	for _, v := range itemsCompanyIndex {
+		v.IndexCode = codeCompanyMap[v.IndexName]
+		if indexCompanyMap[v.IndexName+v.DataTime] == "" && v.DealValue != "" {
+			newId, err := models.AddBaseFromCoalmineCompanyIndex(v)
+			if err != nil {
+				fmt.Println("添加公司数据错误", err)
+			} else {
+				fmt.Println("新增公司数据成功", newId)
+			}
+		} else {
+			if indexCompanyMap[v.IndexName+v.DataTime] != v.DealValue && v.DealValue != "" {
+				err = models.UpdateBaseFromCoalmineCompanyIndex(v)
+				if err != nil {
+					fmt.Println("修改数据错误错误", err)
+				}
+			}
+		}
+
+	}
 	return
 }
+
+func TrimCompanyName(name string) string {
+	name = strings.Replace(name, "有限", "", -1)
+	name = strings.Replace(name, "股份", "", -1)
+	name = strings.Replace(name, "责任", "", -1)
+	name = strings.Replace(name, "公司", "", -1)
+	name = strings.Replace(name, "（", "", -1)
+	name = strings.Replace(name, "）", "", -1)
+	return name
+}
+
+func ContainsSpecialName(name string) bool {
+	return strings.Contains(name, "陕西") ||
+		strings.Contains(name, "海南") ||
+		strings.Contains(name, "河南") ||
+		strings.Contains(name, "河北") ||
+		strings.Contains(name, "澳门") ||
+		strings.Contains(name, "内蒙古") ||
+		strings.Contains(name, "黑龙江")
+}
+
+func TrimProvinceName(name string) string {
+	name = strings.Replace(name, "陕西省", "sns", -1)
+	name = strings.Replace(name, "陕西", "sn", -1)
+	name = strings.Replace(name, "海南市", "hi", -1)
+	name = strings.Replace(name, "海南", "hi", -1)
+	name = strings.Replace(name, "河南省", "has", -1)
+	name = strings.Replace(name, "河南", "ha", -1)
+	name = strings.Replace(name, "河北省", "hes", -1)
+	name = strings.Replace(name, "河北", "he", -1)
+	name = strings.Replace(name, "澳门", "mo", -1)
+	name = strings.Replace(name, "内蒙古自治区", "nm", -1)
+	name = strings.Replace(name, "内蒙古", "nm", -1)
+	name = strings.Replace(name, "黑龙江", "hl", -1)
+	name = strings.Replace(name, "（", "", -1)
+	name = strings.Replace(name, "）", "", -1)
+	name = strings.Replace(name, "+", "", -1)
+	return name
+}

services/task.go

@@ -3,17 +3,19 @@ package services
 import (
 	"context"
 	"fmt"
-	"github.com/beego/beego/v2/task"
 )
 
 func Task() {
 	fmt.Println("start crawler")
-	refreshData := task.NewTask("refreshData", "0 0,30 16-20 * * *", RefreshData)
-	refreshEic := task.NewTask("refreshData", "0 0 1,6 * * *", RefreshEic)
-	task.AddTask("数据爬取", refreshData)
-	task.AddTask("欧洲天然气爬取", refreshEic)
+	//refreshData := task.NewTask("refreshData", "0 0,30 16-20 * * *", RefreshData)
+	//refreshEic := task.NewTask("refreshData", "0 0 1,6 * * *", RefreshEic)
+	//task.AddTask("数据爬取", refreshData)
+	//task.AddTask("欧洲天然气爬取", refreshEic)
 	//task.StartTask()
 	//FileCoalJsm()
+	//FileCoalFirm()
+	//FileCoalCoastal()
+	FileCoalInland()
 	fmt.Println("end crawler")
 }