煤炭

models/base_from_coal_coastal.go

@@ -8,15 +8,16 @@ import (
 // BaseFromCoalmineCoastalIndex 沿海八省动力煤用户供耗存数据指标表
 type BaseFromCoalmineCoastalIndex struct {
 	BaseFromCoalmineCoastalIndexID int       `orm:"column(base_from_coalmine_coastal_index_id);pk"`
-	IndexName                   string    // 省份/企业名称
-	IndexCode                   string    // 持买单量指标编码
-	DataTime                    string    // 指标时间
-	DealValue                   string    // 数据量
-	GroupName                   string    // 地区
-	Source                      string    // 来源
-	Unit                        string    //单位
-	CreateTime                  time.Time `description:"插入时间"`
-	ModifyTime                  time.Time `description:"修改时间"`
+	IndexName                      string    // 省份/企业名称
+	IndexCode                      string    // 持买单量指标编码
+	DataTime                       string    // 指标时间
+	DealValue                      string    // 数据量
+	GroupName                      string    // 地区
+	Source                         string    // 来源
+	Unit                           string    //单位
+	Frequency                      string    `description:"频率"`
+	CreateTime                     time.Time `description:"插入时间"`
+	ModifyTime                     time.Time `description:"修改时间"`
 }
 
 //查询指标

models/base_from_coal_firm.go

@@ -14,6 +14,8 @@ type BaseFromCoalmineFirmIndex struct {
 	DealValue                   string    // 数据量
 	GroupName                   string    // 集团名
 	Source                      string    // 来源
+	Unit                        string    // 来源
+	Frequency                   string    `description:"频率"`
 	CreateTime                  time.Time `description:"插入时间"`
 	ModifyTime                  time.Time `description:"修改时间"`
 }

models/base_from_coal_inland.go

@@ -5,7 +5,6 @@ import (
 	"time"
 )
 
-
 // BaseFromCoalmineInlandIndex 沿海八省动力煤用户供耗存数据指标表
 type BaseFromCoalmineInlandIndex struct {
 	BaseFromCoalmineInlandIndexID int       `orm:"column(base_from_coalmine_inland_index_id);pk"`
@@ -15,11 +14,12 @@ type BaseFromCoalmineInlandIndex struct {
 	DealValue                     string    // 数据量
 	GroupName                     string    // 地区
 	Source                        string    // 来源
+	Unit                          string    // 来源
+	Frequency                     string    `description:"频率"`
 	CreateTime                    time.Time `description:"插入时间"`
 	ModifyTime                    time.Time `description:"修改时间"`
 }
 
-
 //查询指标
 func GetBaseFromCoalmineInlandIndex() (items []*BaseFromCoalmineInlandIndex, err error) {
 	o := orm.NewOrm()

models/base_from_coalmine.go

@@ -12,31 +12,34 @@ type BaseFromCoalmineMapping struct {
 	CreateTime                time.Time `description:"时间"`
 }
 
-type BaseFromCoalmineIndex struct {
-	BaseFromCoalmineIndexId int       `orm:"column(base_from_coalmine_index_id);pk"`
-	IndexName               string    `description:"持买单量指标名称"`
-	IndexCode               string    `description:"持买单量指标编码"`
-	Exchange                string    `description:"样本统计类别"`
-	DealValue               string    `description:"成交量"`
-	DataTime                string    `description:"数据日期"`
-	Source                    string    `description:"来源"`
-	Province                  string    `description:"省份"`
-	Description               string    `description:"描述"`
-	CreateTime              time.Time `description:"插入时间"`
-	ModifyTime              time.Time `description:"修改时间"`
+type BaseFromCoalmineJsmIndex struct {
+	BaseFromCoalmineJsmIndexId int       `orm:"column(base_from_coalmine_jsm_index_id);pk"`
+	IndexName                  string    `description:"持买单量指标名称"`
+	IndexCode                  string    `description:"持买单量指标编码"`
+	Exchange                   string    `description:"样本统计类别"`
+	DealValue                  string    `description:"成交量"`
+	DataTime                   string    `description:"数据日期"`
+	Source                     string    `description:"来源"`
+	Province                   string    `description:"省份"`
+	Description                string    `description:"描述"`
+	Unit                       string    `description:"单位"`
+	Frequency                  string    `description:"频率"`
+	CreateTime                 time.Time `description:"插入时间"`
+	ModifyTime                 time.Time `description:"修改时间"`
 }
 
-
 type BaseFromCoalmineCompanyIndex struct {
 	BaseFromCoalmineCompanyIndexId int       `orm:"column(base_from_coalmine_company_index_id);pk"`
 	IndexName                      string    `description:"持买单量指标名称"`
 	IndexCode                      string    `description:"持买单量指标编码"`
 	DealValue                      string    `description:"成交量"`
 	DataTime                       string    `description:"数据日期"`
-	Source                           string    `description:"来源"`
-	Province                         string    `description:"省份"`
-	City                             string    `description:"城市"`
-	GroupName                        string    `description:"集团名称"`
+	Source                         string    `description:"来源"`
+	Province                       string    `description:"省份"`
+	City                           string    `description:"城市"`
+	GroupName                      string    `description:"集团名称"`
+	Unit                           string    `description:"单位"`
+	Frequency                      string    `description:"频率"`
 	CreateTime                     time.Time `description:"插入时间"`
 	ModifyTime                     time.Time `description:"修改时间"`
 }
@@ -59,47 +62,30 @@ func GetBaseFromCoalmineMapping() (items []*BaseFromCoalmineMapping, err error)
 }
 
 //查询数据
-func GetBaseFromCoalmineIndex() (items []*BaseFromCoalmineIndex, err error) {
+func GetBaseFromCoalmineIndex() (items []*BaseFromCoalmineJsmIndex, err error) {
 	o := orm.NewOrm()
 	o.Using("data")
-	sql := `SELECT * FROM base_from_coalmine_index`
+	sql := `SELECT * FROM base_from_coalmine_jsm_index`
 	_, err = o.Raw(sql).QueryRows(&items)
 	return
 }
 
-func UpdateBaseFromCoalmineIndex(item *BaseFromCoalmineIndex) (err error) {
+func UpdateBaseFromCoalmineIndex(item *BaseFromCoalmineJsmIndex) (err error) {
 	o := orm.NewOrm()
 	o.Using("data")
-	sql := `UPDATE base_from_coalmine_index SET deal_value=?  WHERE index_name=?  AND  data_time = ?`
+	sql := `UPDATE base_from_coalmine_jsm_index SET deal_value=?  WHERE index_name=?  AND  data_time = ?`
 	_, err = o.Raw(sql, item.DealValue, item.IndexName, item.DataTime).Exec()
 	return
 }
 
 //添加数据
-func AddBaseFromCoalmineIndex(item *BaseFromCoalmineIndex) (lastId int64, err error) {
+func AddBaseFromCoalmineIndex(item *BaseFromCoalmineJsmIndex) (lastId int64, err error) {
 	o := orm.NewOrm()
 	o.Using("data")
 	lastId, err = o.Insert(item)
 	return
 }
 
-////添加公司指标
-//func AddBaseFromCoalmineCompanyMapping(item *BaseFromCoalmineCompanyMapping) (lastId int64, err error) {
-//	o := orm.NewOrm()
-//	o.Using("data")
-//	lastId, err = o.Insert(item)
-//	return
-//}
-//
-////查询指标
-//func GetBaseFromCoalmineCompanyMapping() (items []*BaseFromCoalmineCompanyMapping, err error) {
-//	o := orm.NewOrm()
-//	o.Using("data")
-//	sql := `SELECT * FROM base_from_coalmine_company_mapping`
-//	_, err = o.Raw(sql).QueryRows(&items)
-//	return
-//}
-
 //添加公司指标
 func AddBaseFromCoalmineCompanyIndex(item *BaseFromCoalmineCompanyIndex) (lastId int64, err error) {
 	o := orm.NewOrm()
@@ -109,7 +95,7 @@ func AddBaseFromCoalmineCompanyIndex(item *BaseFromCoalmineCompanyIndex) (lastId
 }
 
 //查询公司指标
-func GetBaseFromCoalmineCompanyIndex() (items []*BaseFromCoalmineCompanyIndex,err error){
+func GetBaseFromCoalmineCompanyIndex() (items []*BaseFromCoalmineCompanyIndex, err error) {
 	o := orm.NewOrm()
 	o.Using("data")
 	sql := `SELECT * FROM base_from_coalmine_company_index`
@@ -123,4 +109,4 @@ func UpdateBaseFromCoalmineCompanyIndex(item *BaseFromCoalmineCompanyIndex) (err
 	sql := `UPDATE base_from_coalmine_company_index SET deal_value=?  WHERE index_name=?  AND  data_time = ?`
 	_, err = o.Raw(sql, item.DealValue, item.IndexName, item.DataTime).Exec()
 	return
-}
+}

models/db.go

@@ -36,7 +36,7 @@ func init() {
 		//
 		new(BaseFromTradeDalianIndex),
 		new(BaseFromCoalmineMapping),
-		new(BaseFromCoalmineIndex),
+		new(BaseFromCoalmineJsmIndex),
 		new(BaseFromCoalmineCompanyIndex),
 		new(BaseFromCoalmineFirmIndex),
 		new(BaseFromCoalmineCoastalIndex),

services/commodity_coal_coastal.go

@@ -88,7 +88,7 @@ func FileCoalCoastal() (err error) {
 									code += pi
 								}
 							}
-							item.IndexCode = abbr + code
+							item.IndexCode = abbr + code + "coastal"
 						} else {
 							a := pinyin.NewArgs()
 							rows := pinyin.Pinyin(text+sheetName, a)
@@ -99,7 +99,7 @@ func FileCoalCoastal() (err error) {
 									code += pi
 								}
 							}
-							item.IndexCode = code
+							item.IndexCode = code + "coastal"
 						}
 						item.CreateTime = time.Now()
 						mappingItems = append(mappingItems, &item)
@@ -127,6 +127,7 @@ func FileCoalCoastal() (err error) {
 							item.GroupName = groupMap[k]
 							item.Source = "沿海八省"
 							item.DataTime = dataTime
+							item.Frequency = "日度"
 							item.ModifyTime = time.Now()
 							item.CreateTime = time.Now()
 						}

services/commodity_coal_firm.go

@@ -131,7 +131,7 @@ func FileCoalFirm() (err error) {
 							abbr := trimProvinceName(companyName)
 							//取处理后公司名首字母缩写
 							a := pinyin.NewArgs()
-							rows := pinyin.Pinyin(companyName[2:], a)
+							rows := pinyin.Pinyin(companyName[9:], a)
 							for i := 0; i < len(rows); i++ {
 								if len(rows[i]) != 0 {
 									str := rows[i][0]
@@ -139,7 +139,7 @@ func FileCoalFirm() (err error) {
 									code += pi
 								}
 							}
-							item.IndexCode = abbr[:2] + code
+							item.IndexCode = abbr[:2] + code + "firm"
 
 						} else {
 							a := pinyin.NewArgs()
@@ -151,11 +151,9 @@ func FileCoalFirm() (err error) {
 									code += pi
 								}
 							}
-							item.IndexCode = code
+							item.IndexCode = code + "firm"
 						}
-
 						item.IndexName = companyName
-						item.IndexCode = code
 						item.CreateTime = time.Now()
 						mappingItems = append(mappingItems, &item)
 					} else {
@@ -167,7 +165,9 @@ func FileCoalFirm() (err error) {
 							DataTime:                    dateMap[k],
 							DealValue:                   dealValue,
 							GroupName:                   groupName,
-							Source:                      "全国分企业旬度",
+							Source:                      "全国分企业",
+							Unit:                        "万吨",
+							Frequency:                   "旬度",
 							CreateTime:                  time.Now(),
 							ModifyTime:                  time.Now(),
 						}

services/commodity_coal_inland.go

@@ -87,7 +87,7 @@ func FileCoalInland() (err error) {
 									code += pi
 								}
 							}
-							item.IndexCode = abbr + code
+							item.IndexCode = abbr + code + "inland"
 						} else {
 							a := pinyin.NewArgs()
 							rows := pinyin.Pinyin(text+sheetName, a)
@@ -98,7 +98,7 @@ func FileCoalInland() (err error) {
 									code += pi
 								}
 							}
-							item.IndexCode = code
+							item.IndexCode = code + "inland"
 						}
 						item.CreateTime = time.Now()
 						mappingItems = append(mappingItems, &item)
@@ -129,6 +129,8 @@ func FileCoalInland() (err error) {
 							item.GroupName = groupMap[k]
 							item.Source = "内陆十七省"
 							item.DataTime = dataTime
+							item.Unit = "万吨"
+							item.Frequency = "日度"
 							item.ModifyTime = time.Now()
 							item.CreateTime = time.Now()
 						}

services/commodity_coal_mine.go

@@ -39,7 +39,7 @@ func FileCoalJsm() (err error) {
 	var indexCompanyMap = make(map[string]string)
 	var items []*models.BaseFromCoalmineMapping
 	var itemsCompany []*models.BaseFromCoalmineMapping
-	var itemsIndex []*models.BaseFromCoalmineIndex
+	var itemsIndex []*models.BaseFromCoalmineJsmIndex
 	var itemsCompanyIndex []*models.BaseFromCoalmineCompanyIndex
 	codeList, err := models.GetBaseFromCoalmineMapping()
 	if err != nil && err.Error() != utils.ErrNoRow() {
@@ -122,7 +122,7 @@ func FileCoalJsm() (err error) {
 										strResult += pi
 									}
 								}
-								item.IndexCode = abbr + strResult
+								item.IndexCode = abbr + strResult + "jsm"
 							} else {
 								//取处理后公司名首字母缩写
 								a := pinyin.NewArgs()
@@ -134,7 +134,7 @@ func FileCoalJsm() (err error) {
 										strResult += pi
 									}
 								}
-								item.IndexCode = strResult
+								item.IndexCode = strResult + "jsm"
 							}
 
 							item.IndexName = province + exchange
@@ -166,7 +166,7 @@ func FileCoalJsm() (err error) {
 				for k, cell := range cells {
 					text := cell.String()
 					if k > 4 {
-						item := new(models.BaseFromCoalmineIndex)
+						item := new(models.BaseFromCoalmineJsmIndex)
 						item.IndexName = province + exchange
 						item.IndexCode = codeMap[item.IndexName]
 						item.Exchange = exchange
@@ -176,6 +176,8 @@ func FileCoalJsm() (err error) {
 						item.Description = description
 						item.Province = province
 						item.Source = "三省周度"
+						item.Unit = "万吨"
+						item.Frequency = "周度"
 						item.ModifyTime = time.Now()
 						item.CreateTime = time.Now()
 						itemsIndex = append(itemsIndex, item)
@@ -216,7 +218,7 @@ func FileCoalJsm() (err error) {
 									strResult += pi
 								}
 							}
-							item.IndexCode = strResult
+							item.IndexCode = strResult + "company"
 							item.CreateTime = time.Now()
 							itemsCompany = append(itemsCompany, item)
 							break Loop3
@@ -249,6 +251,8 @@ func FileCoalJsm() (err error) {
 						}
 
 						item.Source = "三省企业"
+						item.Unit = "万吨"
+						item.Frequency = "周度"
 						item.ModifyTime = time.Now()
 						item.CreateTime = time.Now()
 						//fmt.Println(item)

services/task.go

@@ -12,9 +12,9 @@ func Task() {
 	//task.AddTask("数据爬取", refreshData)
 	//task.AddTask("欧洲天然气爬取", refreshEic)
 	//task.StartTask()
-	//FileCoalJsm()
-	//FileCoalFirm()
-	//FileCoalCoastal()
+	FileCoalJsm()
+	FileCoalFirm()
+	FileCoalCoastal()
 	FileCoalInland()
 	fmt.Println("end crawler")
 }