统计局-优化数据更新规则、月度日期规则

models/base_from_national_statistics_index.go

@@ -3,6 +3,7 @@ package models
 import (
 	"fmt"
 	"github.com/beego/beego/v2/client/orm"
+	"hongze/hongze_data_crawler/utils"
 	"strings"
 	"time"
 )
@@ -99,20 +100,40 @@ func (m *BaseFromNationalStatisticsIndex) GetItemsByCondition(condition string,
 
 // SaveNationalStatisticsIndexAndDataReq 保存指标和数据请求体
 type SaveNationalStatisticsIndexAndDataReq struct {
-	Index      *BaseFromNationalStatisticsIndex  `description:"指标信息"`
-	IndexExist bool                              `description:"指标是否存在"`
-	DataList   []*BaseFromNationalStatisticsData `description:"新增的指标数据"`
+	//IndexExist bool                              `description:"指标是否存在"`
+	Index    *BaseFromNationalStatisticsIndex  `description:"指标信息"`
+	DataList []*BaseFromNationalStatisticsData `description:"新增的指标数据"`
 }
 
 // SaveNationalStatisticsIndexAndData 保存指标和值
-func SaveNationalStatisticsIndexAndData(req *SaveNationalStatisticsIndexAndDataReq) (err error) {
-	if req.Index == nil {
+func SaveNationalStatisticsIndexAndData(index *BaseFromNationalStatisticsIndex, data []*BaseFromNationalStatisticsData) (err error) {
+	if index == nil {
 		return
 	}
+	indexExist := new(BaseFromNationalStatisticsIndex)
+	indexCond := ` AND index_code = ?`
+	indexPars := make([]interface{}, 0)
+	indexPars = append(indexPars, index.IndexCode)
+	if e := indexExist.GetItemByCondition(indexCond, indexPars); e != nil && e.Error() != utils.ErrNoRow() {
+		return e
+	}
+	dataOB := new(BaseFromNationalStatisticsData)
+	dataCond := ` AND index_code = ?`
+	dataPars := make([]interface{}, 0)
+	dataPars = append(dataPars, index.IndexCode)
+	dataList, e := dataOB.GetItemsByCondition(dataCond, dataPars, []string{}, "data_time ASC")
+	if e != nil {
+		return e
+	}
+	dataMap := make(map[string]*BaseFromNationalStatisticsData)
+	for _, d := range dataList {
+		dataMap[d.DataTime.Format(utils.FormatDate)] = d
+	}
+
 	o := orm.NewOrmUsingDB("data")
-	tx, err := o.Begin()
-	if err != nil {
-		return
+	tx, e := o.Begin()
+	if e != nil {
+		return e
 	}
 	defer func() {
 		if err != nil {
@@ -122,19 +143,41 @@ func SaveNationalStatisticsIndexAndData(req *SaveNationalStatisticsIndexAndDataR
 		_ = tx.Commit()
 	}()
 
-	if !req.IndexExist {
-		id, e := tx.Insert(req.Index)
+	indexId := indexExist.BaseFromNationalStatisticsIndexId
+	if indexExist.BaseFromNationalStatisticsIndexId == 0 {
+		id, e := tx.Insert(index)
 		if e != nil {
 			return e
 		}
-		req.Index.BaseFromNationalStatisticsIndexId = int(id)
+		indexId = int(id)
+	}
+	if indexExist.BaseFromNationalStatisticsIndexId > 0 {
+		indexExist.ModifyTime = time.Now().Local()
+		if e = indexExist.Update([]string{"ModifyTime"}); e != nil {
+			return e
+		}
 	}
-	indexId := req.Index.BaseFromNationalStatisticsIndexId
-	if req.DataList != nil && len(req.DataList) > 0 {
-		for _, d := range req.DataList {
-			d.BaseFromNationalStatisticsIndexId = indexId
+	inserts := make([]*BaseFromNationalStatisticsData, 0)
+	updateCols := []string{"Value", "ModifyTime"}
+	for _, d := range data {
+		k := d.DataTime.Format(utils.FormatDate)
+		// 更新数据
+		v := dataMap[k]
+		if v != nil {
+			v.Value = d.Value
+			v.ModifyTime = time.Now().Local()
+			if _, e = tx.Update(v, updateCols...); e != nil {
+				err = e
+				return
+			}
+			continue
 		}
-		_, e := tx.InsertMulti(len(req.DataList), req.DataList)
+		d.BaseFromNationalStatisticsIndexId = indexId
+		inserts = append(inserts, d)
+	}
+	// 批量插入新数据
+	if len(inserts) > 0 {
+		_, e = tx.InsertMulti(len(inserts), inserts)
 		if e != nil {
 			return e
 		}

services/national_data/common.go

@@ -261,7 +261,24 @@ func formatMonth2YearDateCode(dateCode string) (date time.Time, err error) {
 	// 根据日期code长度进行区分, 格式为三种: 月度-200601; 季度-2006A; 年度-2006
 	switch len([]rune(dateCode)) {
 	case 6:
-		t, e := time.ParseInLocation("200601", dateCode, time.Local)
+		// 月度日期取每月最后一天
+		m := dateCode[4:]
+		replaceMonth := map[string]string{
+			"01": "0131",
+			"02": "0228",
+			"03": "0331",
+			"04": "0430",
+			"05": "0531",
+			"06": "0630",
+			"07": "0731",
+			"08": "0831",
+			"09": "0930",
+			"10": "1031",
+			"11": "1130",
+			"12": "1231",
+		}
+		md := fmt.Sprintf("%s%s", dateCode[:4], replaceMonth[m])
+		t, e := time.ParseInLocation("20060102", md, time.Local)
 		if e != nil {
 			err = fmt.Errorf("月度指标日期转换失败, Err: %s", e.Error())
 			return

services/national_data/national_data.go

@@ -239,42 +239,12 @@ func SyncXDateYQuotaData(classifyId int, dbCode, classifyCode string) (err error
 		}
 	}
 
-	// 指标编码去重, 指标编码+日期数据去重
-	indexOB := new(models.BaseFromNationalStatisticsIndex)
-	indexCond := ` AND dbcode = ?`
-	indexPars := make([]interface{}, 0)
-	indexPars = append(indexPars, dbCode)
-	indexList, e := indexOB.GetItemsByCondition(indexCond, indexPars, []string{"index_code"}, "")
-	if e != nil {
-		err = fmt.Errorf("获取指标列表失败, Err: %s", e.Error())
-		return
-	}
-	indexExistMap := make(map[string]bool)
-	for _, v := range indexList {
-		indexExistMap[v.IndexCode] = true
-	}
-
 	// 遍历XY轴
 	indexDataList := make([]*models.SaveNationalStatisticsIndexAndDataReq, 0)
 	indexDataMap := make(map[string][]*models.BaseFromNationalStatisticsData)
 	for _, q := range quotaNodes {
 		indexCode := fmt.Sprintf("%s%s", dbCode, q.Code)
 
-		// 数据去重
-		dataExistMap := make(map[string]bool)
-		dataOB := new(models.BaseFromNationalStatisticsData)
-		dataCond := ` AND index_code = ?`
-		dataPars := make([]interface{}, 0)
-		dataPars = append(dataPars, indexCode)
-		dataList, e := dataOB.GetItemsByCondition(dataCond, dataPars, []string{"index_code", "data_time"}, "")
-		if e != nil {
-			err = fmt.Errorf("获取指标数据列表失败, Err: %s", e.Error())
-			return
-		}
-		for _, v := range dataList {
-			dataExistMap[v.DataTime.Format(utils.FormatDate)] = true
-		}
-
 		// 指标
 		r := new(models.SaveNationalStatisticsIndexAndDataReq)
 		r.Index = &models.BaseFromNationalStatisticsIndex{
@@ -287,9 +257,6 @@ func SyncXDateYQuotaData(classifyId int, dbCode, classifyCode string) (err error
 			CreateTime:                           time.Now().Local(),
 			ModifyTime:                           time.Now().Local(),
 		}
-		if indexExistMap[indexCode] {
-			r.IndexExist = true
-		}
 
 		// 数据
 		for _, d := range dateNodes {
@@ -299,14 +266,12 @@ func SyncXDateYQuotaData(classifyId int, dbCode, classifyCode string) (err error
 				continue
 			}
 
+			// 日期处理
 			t, e := formatMonth2YearDateCode(d.Code)
 			if e != nil {
 				err = fmt.Errorf("格式化日期code失败, Err: %s", e.Error())
 				return
 			}
-			if dataExistMap[t.Format(utils.FormatDate)] {
-				continue
-			}
 
 			// 数据map
 			if indexDataMap[indexCode] == nil {
@@ -330,7 +295,7 @@ func SyncXDateYQuotaData(classifyId int, dbCode, classifyCode string) (err error
 			continue
 		}
 		v.DataList = ds
-		if e := models.SaveNationalStatisticsIndexAndData(v); e != nil {
+		if e := models.SaveNationalStatisticsIndexAndData(v.Index, v.DataList); e != nil {
 			err = fmt.Errorf("保存指标和数据失败, Err: %s", e.Error())
 			return
 		}
@@ -478,21 +443,6 @@ func SyncXDateYQuotaZRegData(classifyId int, dbCode, classifyCode string, regLis
 			}
 		}
 
-		// 指标编码去重, 指标编码+日期数据去重
-		indexOB := new(models.BaseFromNationalStatisticsIndex)
-		indexCond := ` AND dbcode = ?`
-		indexPars := make([]interface{}, 0)
-		indexPars = append(indexPars, dbCode)
-		indexList, e := indexOB.GetItemsByCondition(indexCond, indexPars, []string{"index_code"}, "")
-		if e != nil {
-			err = fmt.Errorf("获取指标列表失败, Err: %s", e.Error())
-			return
-		}
-		indexExistMap := make(map[string]bool)
-		for _, v := range indexList {
-			indexExistMap[v.IndexCode] = true
-		}
-
 		// 遍历XY轴
 		indexDataList := make([]*models.SaveNationalStatisticsIndexAndDataReq, 0)
 		indexDataMap := make(map[string][]*models.BaseFromNationalStatisticsData)
@@ -500,21 +450,6 @@ func SyncXDateYQuotaZRegData(classifyId int, dbCode, classifyCode string, regLis
 			// dbcode+指标code+地区code
 			indexCode := fmt.Sprintf("%s%s%s", dbCode, q.Code, reg.Code)
 
-			// 数据去重
-			dataExistMap := make(map[string]bool)
-			dataOB := new(models.BaseFromNationalStatisticsData)
-			dataCond := ` AND index_code = ?`
-			dataPars := make([]interface{}, 0)
-			dataPars = append(dataPars, indexCode)
-			dataList, e := dataOB.GetItemsByCondition(dataCond, dataPars, []string{"index_code", "data_time"}, "")
-			if e != nil {
-				err = fmt.Errorf("获取指标数据列表失败, Err: %s", e.Error())
-				return
-			}
-			for _, v := range dataList {
-				dataExistMap[v.DataTime.Format(utils.FormatDate)] = true
-			}
-
 			// 指标
 			r := new(models.SaveNationalStatisticsIndexAndDataReq)
 			r.Index = &models.BaseFromNationalStatisticsIndex{
@@ -528,9 +463,6 @@ func SyncXDateYQuotaZRegData(classifyId int, dbCode, classifyCode string, regLis
 				CreateTime:                           time.Now().Local(),
 				ModifyTime:                           time.Now().Local(),
 			}
-			if indexExistMap[indexCode] {
-				r.IndexExist = true
-			}
 
 			// 数据
 			// zb.A01010201_reg.110000_sj.201608
@@ -541,14 +473,12 @@ func SyncXDateYQuotaZRegData(classifyId int, dbCode, classifyCode string, regLis
 					continue
 				}
 
+				// 日期处理
 				t, e := formatMonth2YearDateCode(d.Code)
 				if e != nil {
 					err = fmt.Errorf("格式化日期code失败, Err: %s", e.Error())
 					return
 				}
-				if dataExistMap[t.Format(utils.FormatDate)] {
-					continue
-				}
 
 				// 数据map
 				if indexDataMap[indexCode] == nil {
@@ -572,7 +502,7 @@ func SyncXDateYQuotaZRegData(classifyId int, dbCode, classifyCode string, regLis
 				continue
 			}
 			v.DataList = ds
-			if e := models.SaveNationalStatisticsIndexAndData(v); e != nil {
+			if e := models.SaveNationalStatisticsIndexAndData(v.Index, v.DataList); e != nil {
 				err = fmt.Errorf("保存指标和数据失败, Err: %s", e.Error())
 				return
 			}
@@ -751,21 +681,6 @@ func SyncXRegYDateZQuotaDbData(classifyId int, dbCode, classifyCode string) (err
 			}
 		}
 
-		// 指标编码去重, 指标编码+日期数据去重
-		indexOB := new(models.BaseFromNationalStatisticsIndex)
-		indexCond := ` AND dbcode = ?`
-		indexPars := make([]interface{}, 0)
-		indexPars = append(indexPars, dbCode)
-		indexList, e := indexOB.GetItemsByCondition(indexCond, indexPars, []string{"index_code"}, "")
-		if e != nil {
-			err = fmt.Errorf("获取指标列表失败, Err: %s", e.Error())
-			return
-		}
-		indexExistMap := make(map[string]bool)
-		for _, v := range indexList {
-			indexExistMap[v.IndexCode] = true
-		}
-
 		// 遍历XY轴
 		indexDataList := make([]*models.SaveNationalStatisticsIndexAndDataReq, 0)
 		indexDataMap := make(map[string][]*models.BaseFromNationalStatisticsData)
@@ -775,21 +690,6 @@ func SyncXRegYDateZQuotaDbData(classifyId int, dbCode, classifyCode string) (err
 			// 指标: dbcode+指标code+地区code
 			indexCode := fmt.Sprintf("%s%s%s", dbCode, quota.Code, reg.Code)
 
-			// 数据去重
-			dataExistMap := make(map[string]bool)
-			dataOB := new(models.BaseFromNationalStatisticsData)
-			dataCond := ` AND index_code = ?`
-			dataPars := make([]interface{}, 0)
-			dataPars = append(dataPars, indexCode)
-			dataList, e := dataOB.GetItemsByCondition(dataCond, dataPars, []string{"index_code", "data_time"}, "")
-			if e != nil {
-				err = fmt.Errorf("获取指标数据列表失败, Err: %s", e.Error())
-				return
-			}
-			for _, v := range dataList {
-				dataExistMap[v.DataTime.Format(utils.FormatDate)] = true
-			}
-
 			r := new(models.SaveNationalStatisticsIndexAndDataReq)
 			r.Index = &models.BaseFromNationalStatisticsIndex{
 				BaseFromNationalStatisticsClassifyId: classifyId,
@@ -802,9 +702,6 @@ func SyncXRegYDateZQuotaDbData(classifyId int, dbCode, classifyCode string) (err
 				CreateTime:                           time.Now().Local(),
 				ModifyTime:                           time.Now().Local(),
 			}
-			if indexExistMap[indexCode] {
-				r.IndexExist = true
-			}
 
 			// 遍历Y轴-日期
 			for _, d := range dateNodes {
@@ -820,9 +717,6 @@ func SyncXRegYDateZQuotaDbData(classifyId int, dbCode, classifyCode string) (err
 					err = fmt.Errorf("格式化日期code失败, Err: %s", e.Error())
 					return
 				}
-				if dataExistMap[t.Format(utils.FormatDate)] {
-					continue
-				}
 
 				// 数据map
 				if indexDataMap[indexCode] == nil {
@@ -846,7 +740,7 @@ func SyncXRegYDateZQuotaDbData(classifyId int, dbCode, classifyCode string) (err
 				continue
 			}
 			v.DataList = ds
-			if e := models.SaveNationalStatisticsIndexAndData(v); e != nil {
+			if e := models.SaveNationalStatisticsIndexAndData(v.Index, v.DataList); e != nil {
 				err = fmt.Errorf("保存指标和数据失败, Err: %s", e.Error())
 				return
 			}