Merge branch 'coal_fix'

services/commodity_coal_firm.go

@@ -13,64 +13,61 @@ import (
 	"time"
 )
 
-func FileCoalFirm() () {
+func FileCoalFirm() {
 	var err error
 	defer func() {
 		if err != nil {
 			fmt.Println("RefreshDataFromCoalFirm Err:" + err.Error())
-			msg := "失败提醒"+"RefreshDataFromCoalFirm ErrMsg:"+err.Error()
+			msg := "失败提醒" + "RefreshDataFromCoalFirm ErrMsg:" + err.Error()
 			go alarm_msg.SendAlarmMsg(msg, 3)
 			//go utils.SendEmail(utils.APPNAME+"【"+utils.RunMode+"】"+"失败提醒", "RefreshDataFromCoalFirm ErrMsg:"+err.Error(), utils.EmailSendToUsers)
 		}
 	}()
 	var xlFile *xlsx.File
-	//syear := time.Now().Year()
-	//nyear := strconv.Itoa(syear)
-	//var smonth string
-	//dateMonth := int(time.Now().Month())
-	//smonth = strconv.Itoa(dateMonth)
-	//path := "static/file/firm.xlsx"
-	//path := "/home/code/python/coal_mail/emailFile/全国分企业煤炭产量旬度数据（%s年%s月）.xlsx"
-	path := "/home/code/python/coal_mail/emailFile/全国分企业煤炭产量旬度数据（2022年3月）.xlsx"
-	//path = fmt.Sprintf(path,nyear,smonth)
-	//exist,err := PathExists(path)
-	//if err != nil {
-	//	fmt.Println(err)
-	//	return
-	//}
-	//if !exist {
-	//	path = "/home/code/python/coal_mail/emailFile/全国分企业煤炭产量旬度数据（%s年%s月中旬）.xlsx"
-	//	path = fmt.Sprintf(path,nyear,smonth)
-	//	exist,err = PathExists(path)
-	//	if err != nil {
-	//		fmt.Println(err)
-	//		return
-	//	}
-	//}
-	//if !exist {
-	//	path = "/home/code/python/coal_mail/emailFile/全国分企业煤炭产量旬度数据（%s年%s月上旬）.xlsx"
-	//	path = fmt.Sprintf(path,nyear,smonth)
-	//	exist,err = PathExists(path)
-	//	if err != nil {
-	//		fmt.Println(err)
-	//		return
-	//	}
-	//}
-	//if !exist {
-	//	//本月三个文件都没有，去取上一个月的
-	//	dateMonth = dateMonth - 1
-	//	smonth = strconv.Itoa(dateMonth)
-	//	path = "/home/code/python/coal_mail/emailFile/全国分企业煤炭产量旬度数据（%s年%s月）.xlsx"
-	//	path = fmt.Sprintf(path,nyear,smonth)
-	//	exist,err = PathExists(path)
-	//	if err != nil {
-	//		fmt.Println(err)
-	//		return
-	//	}
-	//}
-
+	syear := time.Now().Year()
+	nyear := strconv.Itoa(syear)
+	var smonth string
+	dateMonth := int(time.Now().Month())
+	smonth = strconv.Itoa(dateMonth)
+	path := "/home/code/python/coal_mail/emailFile/全国分企业煤炭产量旬度数据（%s年%s月）.xlsx"
+	path = fmt.Sprintf(path,nyear,smonth)
+	exist,err := PathExists(path)
+	if err != nil {
+		fmt.Println(err)
+		return
+	}
+	if !exist {
+		path = "/home/code/python/coal_mail/emailFile/全国分企业煤炭产量旬度数据（%s年%s月中旬）.xlsx"
+		path = fmt.Sprintf(path,nyear,smonth)
+		exist,err = PathExists(path)
+		if err != nil {
+			fmt.Println(err)
+			return
+		}
+	}
+	if !exist {
+		path = "/home/code/python/coal_mail/emailFile/全国分企业煤炭产量旬度数据（%s年%s月上旬）.xlsx"
+		path = fmt.Sprintf(path,nyear,smonth)
+		exist,err = PathExists(path)
+		if err != nil {
+			fmt.Println(err)
+			return
+		}
+	}
+	if !exist {
+		//本月三个文件都没有，去取上一个月的
+		dateMonth = dateMonth - 1
+		smonth = strconv.Itoa(dateMonth)
+		path = "/home/code/python/coal_mail/emailFile/全国分企业煤炭产量旬度数据（%s年%s月）.xlsx"
+		path = fmt.Sprintf(path,nyear,smonth)
+		exist,err = PathExists(path)
+		if err != nil {
+			fmt.Println(err)
+			return
+		}
+	}
 
-	//path = fmt.Sprintf(path,nyear,smonth)
+	path = fmt.Sprintf(path,nyear,smonth)
 	fmt.Println(path)
 	//path = "static/file/全国分企业煤炭产量旬度数据（2022年3月上旬）.xlsx"
 	xlFile, _ = xlsx.OpenFile(path)
@@ -114,18 +111,20 @@ func FileCoalFirm() () {
 		maxRow := sheet.MaxRow
 		for i := 0; i < maxRow; i++ {
 			//获取年份
-			if i == 2 {
-				row := sheet.Row(i)
-				cells := row.Cells
-				for k, cell := range cells {
-					text := cell.String()
-					if k > 0 && text != "" {
-						year = text
-						yearMap[k] = year
-					}
-				}
-			}
-			//获取月份
+			//if i == 2 {
+			//	row := sheet.Row(i)
+			//	cells := row.Cells
+			//	for k, cell := range cells {
+			//		text := cell.String()
+			//		if k > 0 && text != "" {
+			//			year = text
+			//			fmt.Println("year:",year)
+			//			fmt.Println("k:",k)
+			//			yearMap[k] = year
+			//		}
+			//	}
+			//}
+			//获取月份和年度
 			if i == 3 {
 				row := sheet.Row(i)
 				cells := row.Cells
@@ -133,6 +132,13 @@ func FileCoalFirm() () {
 					text := cell.String()
 					if k > 0 && text != "" {
 						month = text
+						month = utils.ConvertToFormatDay(month)
+						time, _ := time.Parse(utils.FormatDate, month)
+						yearInt := time.Year()
+						monthInt := int(time.Month())
+						year = strconv.Itoa(yearInt)
+						yearMap[k] = year
+						month = strconv.Itoa(monthInt)
 						monthMap[k] = month
 					}
 				}
@@ -151,7 +157,7 @@ func FileCoalFirm() () {
 							month = monthMap[k]
 						}
 						day = text
-						dateMap[k] = year + month + day
+						dateMap[k] = year + "年" + month + "月" + day
 					}
 				}
 			}
@@ -214,16 +220,16 @@ func FileCoalFirm() () {
 						dealValue := cell.String()
 
 						item := models.BaseFromCoalmineFirmIndex{
-							IndexName:                   companyName,
-							IndexCode:                   codeMap[companyName],
-							DataTime:                    dateMap[k],
-							DealValue:                   dealValue,
-							GroupName:                   groupName,
-							Source:                      "全国分企业",
-							Unit:                        "万吨",
-							Frequency:                   "旬度",
-							CreateTime:                  time.Now(),
-							ModifyTime:                  time.Now(),
+							IndexName:  companyName,
+							IndexCode:  codeMap[companyName],
+							DataTime:   dateMap[k],
+							DealValue:  dealValue,
+							GroupName:  groupName,
+							Source:     "全国分企业",
+							Unit:       "万吨",
+							Frequency:  "旬度",
+							CreateTime: time.Now(),
+							ModifyTime: time.Now(),
 						}
 
 						indexItems = append(indexItems, &item)
@@ -233,50 +239,50 @@ func FileCoalFirm() () {
 		}
 	}
 	//添加数据到数据库
-	for _, v := range mappingItems {
-		if codeMap[v.IndexName] == "" {
-			codeMap[v.IndexName] = v.IndexCode
-			newId, err := models.AddBaseFromCoalmineMapping(v)
-			if err != nil {
-				for i := 0; i<10; i++ {
-					v.IndexCode = v.IndexCode + strconv.Itoa(i)
-					codeMap[v.IndexName] = v.IndexCode
-					newId, err := models.AddBaseFromCoalmineMapping(v)
-					if err != nil {
-						fmt.Println("再次添加公司指标名称错误", err)
-						continue
-					} else {
-						fmt.Println("新增公司成功", newId)
-						break
-					}
-				}
-			} else {
-				fmt.Println("新增公司成功", newId)
-			}
-		}
-	}
-	fmt.Println("指标操作完成")
-
-	//给indexItem中的code赋值并插入index表
-	for _, v := range indexItems {
-		v.IndexCode = codeMap[v.IndexName]
-		if indexMap[v.IndexName+v.DataTime] == "" && v.DealValue != "" {
-			newId, err := models.AddBaseFromCoalFirmIndex(v)
-			if err != nil {
-				fmt.Println("添加数据错误", err)
-			} else {
-				fmt.Println("新增成功", newId)
-			}
-		} else {
-			if indexMap[v.IndexName+v.DataTime] != v.DealValue && v.DealValue != "" {
-				err = models.UpdateBaseFromCoalFirmIndex(v)
-				if err != nil {
-					fmt.Println("修改数据错误错误", err)
-					return
-				}
-			}
-		}
-	}
+	//for _, v := range mappingItems {
+	//	if codeMap[v.IndexName] == "" {
+	//		codeMap[v.IndexName] = v.IndexCode
+	//		newId, err := models.AddBaseFromCoalmineMapping(v)
+	//		if err != nil {
+	//			for i := 0; i < 10; i++ {
+	//				v.IndexCode = v.IndexCode + strconv.Itoa(i)
+	//				codeMap[v.IndexName] = v.IndexCode
+	//				newId, err := models.AddBaseFromCoalmineMapping(v)
+	//				if err != nil {
+	//					fmt.Println("再次添加公司指标名称错误", err)
+	//					continue
+	//				} else {
+	//					fmt.Println("新增公司成功", newId)
+	//					break
+	//				}
+	//			}
+	//		} else {
+	//			fmt.Println("新增公司成功", newId)
+	//		}
+	//	}
+	//}
+	//fmt.Println("指标操作完成")
+	//
+	////给indexItem中的code赋值并插入index表
+	//for _, v := range indexItems {
+	//	v.IndexCode = codeMap[v.IndexName]
+	//	if indexMap[v.IndexName+v.DataTime] == "" && v.DealValue != "" {
+	//		newId, err := models.AddBaseFromCoalFirmIndex(v)
+	//		if err != nil {
+	//			fmt.Println("添加数据错误", err)
+	//		} else {
+	//			fmt.Println("新增成功", newId)
+	//		}
+	//	} else {
+	//		if indexMap[v.IndexName+v.DataTime] != v.DealValue && v.DealValue != "" {
+	//			err = models.UpdateBaseFromCoalFirmIndex(v)
+	//			if err != nil {
+	//				fmt.Println("修改数据错误错误", err)
+	//				return
+	//			}
+	//		}
+	//	}
+	//}
 	return
 }
 
@@ -299,8 +305,7 @@ func trimProvinceName(name string) string {
 	return name
 }
 
-
-func PathExists(path string)(bool, error) {
+func PathExists(path string) (bool, error) {
 	_, err := os.Stat(path)
 	if err == nil { //文件或者目录存在
 		return true, nil
@@ -309,4 +314,4 @@ func PathExists(path string)(bool, error) {
 		return false, nil
 	}
 	return false, err
-}
+}

services/commodity_coal_inland.go

@@ -22,7 +22,8 @@ func FileCoalInland() () {
 		}
 	}()
 	//path := "static/file/内陆17省动力煤终端用户供耗存20220408.xlsx"
-	path := "/home/code/python/coal_mail/emailFile/内陆17省动力煤终端用户供耗存%s.xlsx"
+	//path := "/home/code/python/coal_mail/emailFile/内陆17省动力煤终端用户供耗存%s.xlsx"
+	path := "/home/code/python/coal_mail/emailFile/内陆17省动力煤终端用户供耗存20220525.xlsx"
 	for i := 0;i < 7;i++{
 		npath := fmt.Sprintf(path,time.Now().AddDate(0,0,-i).Format(utils.FormatDateUnSpace))
 		fmt.Println(npath)

services/task.go

@@ -10,7 +10,7 @@ func Task() {
 	fmt.Println("start crawler")
 	refreshData := task.NewTask("refreshData", "0 0,30 16-20 * * *", RefreshData)
 	refreshEic := task.NewTask("refreshData", "0 0 1,6 * * *", RefreshEic)
-	refreshCoal := task.NewTask("refreshData", "0 0 18-23 * * *", RefreshCoal)
+	refreshCoal := task.NewTask("refreshData", "0 27 16 * * *", RefreshCoal)
 	task.AddTask("数据爬取", refreshData)
 	task.AddTask("欧洲天然气爬取", refreshEic)
 	task.AddTask("中国煤炭网爬取", refreshCoal)