Merge remote-tracking branch 'origin/master' into chart/11.2

services/eia_stero.go

@@ -1,154 +1,22 @@
 package services
 
 import (
+	"context"
 	"encoding/json"
 	"fmt"
 	"github.com/rdlucklib/rdluck_tools/http"
 	"hongze/hongze_data_crawler/models"
-	"io/ioutil"
-	"os"
 	"reflect"
 	"strconv"
 	"time"
 )
 
-// SyncEiaStero 同步Eia Stero 报告数据
-func SyncEiaStero() {
-	var err error
-	defer func() {
-		if err != nil {
-			fmt.Println("err:", err)
-		}
-	}()
-	fileName := `data.json`
-	f, err := os.Open(fileName)
-	if err != nil {
-		fmt.Println("打开失败", err)
-		return
-	}
-	defer f.Close()
-
-	b, err := ioutil.ReadAll(f)
-	if err != nil {
-		fmt.Println("读取失败", err)
-		return
-	}
-	//content := string(b)
-	//fmt.Println(string(b))
-	fmt.Println("长度：", len(b))
-
-	var eiaSteroData EiaSteroData
-	err = json.Unmarshal(b, &eiaSteroData)
-	if err != nil {
-		fmt.Println("数据格式化失败", err)
-		return
-	}
-
-	// 获取分类列表
-	classifyList, err := models.GetBaseFromEiaSteroClassifyAll()
-	if err != nil {
-		fmt.Println("获取分类失败：", err)
-		return
-	}
-	classifyMap := make(map[string]*models.BaseFromEiaSteroClassify)
-	for _, v := range classifyList {
-		classifyMap[v.ClassifyNameOriginal] = v
-	}
-
-	// 获取指标列表
-	indexList, err := models.GetBaseFromEiaSteroIndexAll()
-	if err != nil {
-		fmt.Println("获取分类失败：", err)
-		return
-	}
-	indexMap := make(map[string]*models.BaseFromEiaSteroIndex)
-	for _, v := range indexList {
-		indexMap[v.IndexNameOriginal] = v
-	}
-
-	var nowClassify *models.BaseFromEiaSteroClassify
-	for _, v := range eiaSteroData.VIEWSDATA.ROWS {
-		if v.LEVEL == 1 {
-			fmt.Println(v.CHARTNAME)
-			tmpNowClassify, ok := classifyMap[v.CHARTNAME]
-			if !ok {
-				nowClassify = &models.BaseFromEiaSteroClassify{
-					BaseFromEiaSteroClassifyId: 0,
-					ClassifyName:               "",
-					ClassifyNameOriginal:       v.CHARTNAME,
-					ModifyTime:                 time.Now(),
-					CreateTime:                 time.Now(),
-				}
-				// 新增指标
-				err = nowClassify.AddBaseFromEiaSteroClassify()
-				if err != nil {
-					return
-				}
-				classifyMap[v.CHARTNAME] = nowClassify
-			} else {
-				nowClassify = tmpNowClassify
-			}
-			continue
-		}
-		if v.LEVEL > 1 && v.SERIESID == `` {
-			continue
-		}
-
-		var eiaSteroIndex *models.BaseFromEiaSteroIndex
-
-		eiaSteroIndex, ok := indexMap[v.CHARTNAME]
-		if !ok {
-			eiaSteroIndex = &models.BaseFromEiaSteroIndex{
-				//BaseFromEiaSteroIndexId:    0,
-				BaseFromEiaSteroClassifyId: nowClassify.BaseFromEiaSteroClassifyId,
-				IndexCode:                  v.SERIESID,
-				IndexName:                  EiaSteroNameMap[v.SERIESID],
-				IndexNameOriginal:          v.CHARTNAME,
-				Frequency:                  "月度",
-				Level:                      v.LEVEL,
-				Unit:                       v.UNITS,
-				Super:                      v.SUPER,
-				Precision:                  v.PRECISION,
-				LastHistorical:             strconv.Itoa(v.LASTHISTORICAL),
-				Description:                v.DESCRIPTION,
-				IsMappable:                 v.ISMAPPABLE,
-				StartDate:                  time.Now(),
-				EndDate:                    time.Now(),
-				ModifyTime:                 time.Now(),
-				CreateTime:                 time.Now(),
-			}
-			// 新增指标
-			err = eiaSteroIndex.Add()
-			if err != nil {
-				return
-			}
-			indexMap[v.CHARTNAME] = eiaSteroIndex
-		}
-
-		//if v.SERIESID != "PAPR_OECD" {
-		//	continue
-		//}
-
-		//校验数据类型对不对
-		valType := reflect.TypeOf(v.DATA)
-		//if valType.String() == "map[string]interface{}"{
-		//	fmt.Println("匹配上了")
-		//}
-		switch valType.String() {
-		case "[]interface {}": // 没有数据
-		case "map[string]interface {}": // 有数据
-			data := v.DATA.(map[string]interface{})
-			err = models.HandleEiaSteroData(data, eiaSteroIndex)
-			if err != nil {
-				return
-			}
-		}
-
-		fmt.Println(v.CHARTNAME, v.SERIESID, "==", v.PRECISION, "===========")
-	}
+func SyncEiaSteroData(cont context.Context) (err error) {
+	err = syncEiaSteroData()
+	return
 }
 
-func SyncEiaSteroData() {
+func syncEiaSteroData() (err error) {
 	// 获取数据
 
 	// 这是获取数据的链接（月度的）

services/task.go

@@ -7,17 +7,18 @@ import (
 )
 
 func Task() {
-	SyncEiaSteroData()
-	return
 	fmt.Println("start crawler")
 	refreshData := task.NewTask("refreshData", "0 0,30 16-20 * * *", RefreshData)
 	refreshEic := task.NewTask("refreshData", "0 0 2,6 * * *", RefreshEic)
 	refreshCoal := task.NewTask("refreshData", "0 0,30 17-23 * * *", RefreshCoal)
 	refreshVisitors := task.NewTask("RefreshChangesVisitorsCovid", "0 30 2-22/10 * * *", RefreshChangesVisitorsCovid)
+	syncEiaSteroData := task.NewTask("SyncEiaSteroData", "0 0 22 * * *", SyncEiaSteroData)
+
 	task.AddTask("数据爬取", refreshData)
 	task.AddTask("欧洲天然气爬取", refreshEic)
 	task.AddTask("中国煤炭网爬取", refreshCoal)
 	task.AddTask("谷歌出行指数爬取", refreshVisitors)
+	task.AddTask("eia stero报告", syncEiaSteroData) //每天22点爬一次
 	task.StartTask()
 	//FileCoalJsm()
 	//FileCoalFirm()