add:上海有色爬虫入库

controllers/shanghai_smm.go

@@ -0,0 +1,331 @@
+package controllers
+
+import (
+	"encoding/json"
+	"eta/eta_index_lib/models"
+	shanghaismm "eta/eta_index_lib/services/shanghai_smm"
+	"eta/eta_index_lib/utils"
+	"fmt"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/mozillazg/go-pinyin"
+)
+
+type ShanghaiSmmController struct {
+	BaseAuthController
+}
+
+var ShanghaiSmmNameToIndexMap = make(map[string]*models.BaseFromSmmIndex)
+var ShanghaiSmmCodeToNameMap = make(map[string]string)
+var ShanghaiSmmNameToCodeMap = make(map[string]string)
+
+// @Title 刷新数据
+// @Description 刷新数据接口
+// @Param	request	body models.AddEdbClassifyReq true "type json string"
+// @Success 200 {object} models.EdbClassify
+// @router /refresh/list [post]
+func (this *ShanghaiSmmController) RefreshData() {
+	br := new(models.BaseResponse).Init()
+	defer func() {
+		this.Data["json"] = br
+		this.ServeJSON()
+	}()
+
+	var req []*shanghaismm.EdbInfoData
+	if err := json.Unmarshal(this.Ctx.Input.RequestBody, &req); err != nil {
+		br.Msg = "参数解析异常！"
+		br.ErrMsg = "参数解析失败，Err:" + err.Error()
+		return
+	}
+
+	allIndex, err := models.GetBaseFromSmmIndex()
+	if err != nil {
+		fmt.Println("select Code err:", err)
+		utils.FileLog.Info("GetBaseFromSci99Index err:", err)
+		return
+	}
+	for _, item := range allIndex {
+		ShanghaiSmmNameToIndexMap[item.IndexName] = item
+		ShanghaiSmmNameToCodeMap[item.IndexName] = item.IndexCode
+	}
+
+	// 检查指标是否存在，如果不存在添加指标
+	addIndexList := make([]*models.BaseFromSmmIndex, 0)
+	updateIndexList := make([]*models.BaseFromSmmIndex, 0)
+	addDateList := make([]*models.BaseFromSmmData, 0)
+	readyDateList := make([]*models.BaseFromSmmData, 0)
+	for _, v := range req {
+		indexInfo := ShanghaiSmmNameToIndexMap[v.IndexName]
+		if indexInfo == nil {
+			// 指标不存在，需要添加指标索引
+			indexCodeStr := "smm" + v.IndexName
+			indexCodeStr, needAdd := ShanghaiSmmIndexCodeGenerator(v.IndexName, indexCodeStr)
+			// 添加指标
+			if needAdd {
+				fmt.Printf("Name:%s, indexCode:%s \n", v.IndexName, indexCodeStr)
+				// 添加指标
+				item := new(models.BaseFromSmmIndex)
+				item.IndexCode = indexCodeStr
+				item.IndexName = v.IndexName
+				item.Frequency = v.Frequency
+				item.StartDate = v.RenewDate
+				item.EndDate = v.RenewDate
+				item.CreateTime = time.Now()
+				item.Unit = v.Unit
+				item.ModifyTime = time.Now()
+				addIndexList = append(addIndexList, item)
+
+				// 添加指标数据
+				itemDate := new(models.BaseFromSmmData)
+				itemDate.IndexCode = indexCodeStr
+				itemDate.DataTime = v.RenewDate
+				if v.Value == "highs" {
+					itemDate.Value = strconv.FormatFloat(v.Highs, 'f', -1, 64)
+				} else if v.Value == "low" {
+					itemDate.Value = strconv.FormatFloat(v.Low, 'f', -1, 64)
+				} else {
+					itemDate.Value = strconv.FormatFloat(v.Average, 'f', -1, 64)
+				}
+				itemDate.CreateTime = time.Now()
+				itemDate.ModifyTime = time.Now()
+				itemDate.DataTimestamp = time.Now().UnixMilli()
+				readyDateList = append(readyDateList, itemDate)
+			} else {
+				fmt.Printf("有重复的指标或指标名称 Name:%s, indexCode:%s \n", v.IndexName, indexCodeStr)
+				return
+			}
+		} else {
+			//存在指标，检查是否需要更新指标索引，及指标数据
+			if indexInfo.EndDate != v.RenewDate {
+				// 指标日期更新
+				t := new(models.BaseFromSmmIndex)
+				t.BaseFromSmmIndexId = indexInfo.BaseFromSmmIndexId
+				t.EndDate = v.RenewDate
+				updateIndexList = append(updateIndexList, t)
+
+				// 指标数据更新
+				item := new(models.BaseFromSmmData)
+				item.BaseFromSmmIndexId = int(indexInfo.BaseFromSmmIndexId)
+				item.IndexCode = indexInfo.IndexCode
+				item.DataTime = v.RenewDate
+				if v.Value == "highs" {
+					item.Value = strconv.FormatFloat(v.Highs, 'f', -1, 64)
+				} else if v.Value == "low" {
+					item.Value = strconv.FormatFloat(v.Low, 'f', -1, 64)
+				} else {
+					item.Value = strconv.FormatFloat(v.Average, 'f', -1, 64)
+				}
+				item.CreateTime = time.Now()
+				item.ModifyTime = time.Now()
+				item.DataTimestamp = time.Now().UnixMilli()
+				addDateList = append(addDateList, item)
+			}
+		}
+	}
+	if len(addIndexList) > 0 {
+		_, err = models.AddBaseFromSmmIndex(addIndexList)
+		if err != nil {
+			utils.FileLog.Info("Error 插入指标库失败:", err)
+			br.Msg = "插入指标库失败"
+			br.ErrMsg = "插入指标库失败,Err:" + err.Error()
+			return
+		}
+	}
+	if len(updateIndexList) > 0 {
+		for _, v := range updateIndexList {
+			err = models.ModifyBaseFromSmmIndexDate(v)
+			if err != nil {
+				br.Msg = "更新指标数据失败"
+				br.ErrMsg = "更新指标失败,Err:" + err.Error()
+				return
+			}
+		}
+	}
+	if len(addDateList) > 0 {
+		err = models.AddBaseFromSmmData(addDateList)
+		if err != nil {
+			utils.FileLog.Info("Error 插入指标数据失败:", err)
+			br.Msg = "插入指标数据失败"
+			br.ErrMsg = "插入指标数据失败,Err:" + err.Error()
+			return
+		}
+	}
+	allIndex, err = models.GetBaseFromSmmIndex()
+	if err != nil {
+		fmt.Println("select Code err:", err)
+		utils.FileLog.Info("GetBaseFromSci99Index err:", err)
+		return
+	}
+	// 获得更新后的数据
+	for _, item := range allIndex {
+		ShanghaiSmmNameToIndexMap[item.IndexName] = item
+		ShanghaiSmmNameToCodeMap[item.IndexName] = item.IndexCode
+	}
+	for _, v := range readyDateList {
+		indexName := ShanghaiSmmCodeToNameMap[v.IndexCode]
+		baseSmmIndex := ShanghaiSmmNameToIndexMap[indexName]
+		v.BaseFromSmmIndexId = int(baseSmmIndex.BaseFromSmmIndexId)
+	}
+	if len(readyDateList) > 0 {
+		err = models.AddBaseFromSmmData(readyDateList)
+		if err != nil {
+			utils.FileLog.Info("Error 插入指标数据失败:", err)
+			br.Msg = "插入指标数据失败"
+			br.ErrMsg = "插入指标数据失败,Err:" + err.Error()
+			return
+		}
+	}
+	br.Msg = "数据刷新成功"
+	br.Success = true
+	br.Ret = 200
+}
+
+// @Title 刷新数据
+// @Description 刷新数据接口
+// @Param	request	body models.AddEdbClassifyReq true "type json string"
+// @Success 200 {object} models.EdbClassify
+// @router /refresh/excel [post]
+func (this *ShanghaiSmmController) RefreshExcel() {
+	br := new(models.BaseResponse).Init()
+	defer func() {
+		this.Data["json"] = br
+		this.ServeJSON()
+	}()
+
+	var req []*shanghaismm.EdbInfoData
+	if err := json.Unmarshal(this.Ctx.Input.RequestBody, &req); err != nil {
+		br.Msg = "参数解析异常！"
+		br.ErrMsg = "参数解析失败，Err:" + err.Error()
+		return
+	}
+
+	allIndex, err := models.GetBaseFromSmmIndex()
+	if err != nil {
+		fmt.Println("select Code err:", err)
+		utils.FileLog.Info("GetBaseFromSci99Index err:", err)
+		return
+	}
+	ShanghaiSmmCodeToIndexMap := make(map[string]*models.BaseFromSmmIndex)
+	ShanghaiSmmCodeToIndexCount := make(map[string]int)
+	for _, item := range allIndex {
+		ShanghaiSmmCodeToIndexMap[item.IndexCode] = item
+		ShanghaiSmmCodeToIndexCount[item.IndexCode] = 0
+		ShanghaiSmmNameToCodeMap[item.IndexCode] = item.IndexName
+	}
+	updateIndexList := make([]*models.BaseFromSmmIndex, 0)
+	addDataList := make([]*models.BaseFromSmmData, 0)
+	for _, v := range req {
+		indexCode := strings.Replace(v.IndexCode, " ", "", -1)
+		indexInfo := ShanghaiSmmCodeToIndexMap[indexCode]
+		if indexInfo == nil {
+			fmt.Printf("指标名称或指标id有误v.IndexName:%s, v.IndexCode:%s", v.IndexName, v.IndexCode)
+			return
+		}
+		ShanghaiSmmCodeToIndexCount[v.IndexCode] += 1
+		if ShanghaiSmmCodeToIndexCount[v.IndexCode] > 1 {
+			fmt.Printf("指标名称或指标id有误v.IndexName:%s, v.IndexCode:%s", v.IndexName, v.IndexCode)
+			return
+		}
+		isAdd := false
+		indexItem := new(models.BaseFromSmmIndex)
+		indexItem.BaseFromSmmIndexId = indexInfo.BaseFromSmmIndexId
+		startDate, _ := time.Parse(utils.FormatDate, indexInfo.StartDate)
+		endDate, _ := time.Parse(utils.FormatDate, indexInfo.EndDate)
+		if v.LastDate.After(endDate) {
+			isAdd = true
+			indexItem.EndDate = v.LastDate.Format(utils.FormatDate)
+		} else {
+			indexItem.EndDate = indexInfo.EndDate
+		}
+		if v.OldDate.Before(startDate) {
+			isAdd = true
+			indexItem.StartDate = v.OldDate.Format(utils.FormatDate)
+		} else {
+			indexItem.StartDate = indexInfo.StartDate
+		}
+		// 如果指标index有更新，那么需要添加指标数据
+		if isAdd {
+			indexItem.IndexName = v.IndexName
+			indexItem.ModifyTime = time.Now()
+			updateIndexList = append(updateIndexList, indexItem)
+			for k, dv := range v.Data {
+				dataItem := new(models.BaseFromSmmData)
+				dataItem.BaseFromSmmIndexId = int(indexInfo.BaseFromSmmIndexId)
+				dataItem.IndexCode = indexInfo.IndexCode
+				dataItem.DataTime = k
+				dataItem.Value = strconv.FormatFloat(dv, 'f', -1, 64)
+				dataItem.CreateTime = time.Now()
+				dataItem.ModifyTime = time.Now()
+				dataItem.DataTimestamp = time.Now().UnixMilli()
+				addDataList = append(addDataList, dataItem)
+			}
+		}
+	}
+	if len(addDataList) > 0 {
+		err = models.AddBaseFromSmmData(addDataList)
+		if err != nil {
+			br.Msg = "插入指标数据失败"
+			br.ErrMsg = "插入指标数据失败,Err:" + err.Error()
+			return
+		}
+	}
+	if len(updateIndexList) > 0 {
+		for _, v := range updateIndexList {
+			err = v.UpdateCols([]string{"end_date", "start_date", "modify_time"})
+			if err != nil {
+				br.Msg = "更新指标数据失败"
+				br.ErrMsg = "更新指标失败,Err:" + err.Error()
+				return
+			}
+		}
+	}
+	br.Ret = 200
+	br.Success = true
+	br.Msg = "数据刷新成功"
+}
+
+func ShanghaiSmmIndexCodeGenerator(indexName, indexCodeStr string) (indexCode string, needAdd bool) {
+	strResult := ""
+	//首字母
+	a := pinyin.NewArgs()
+	a.Fallback = func(r rune, a pinyin.Args) []string {
+		return []string{string(r)}
+	}
+	rows := pinyin.Pinyin(indexCodeStr, a)
+	for i := 0; i < len(rows); i++ {
+		//strResult += rows[i][0]
+		if len(rows[i]) != 0 {
+			str := rows[i][0]
+			pi := str[0:1]
+			strResult += pi
+		}
+	}
+
+	// 去除特殊符号
+	strResult = strings.Replace(strResult, " ", "", -1)
+	strResult = strings.Replace(strResult, "-", "", -1)
+	strResult = strings.Replace(strResult, "/", "", -1)
+	strResult = strings.Replace(strResult, "#", "", -1)
+	strResult = strings.Replace(strResult, "：", "", -1)
+	strResult = strings.Replace(strResult, "(", "", -1)
+	strResult = strings.Replace(strResult, ")", "", -1)
+	strResult = strings.Replace(strResult, "%", "", -1)
+	strResult = strings.Replace(strResult, "<", "1", -1)
+	strResult = strings.Replace(strResult, "\xe2", "2", -1) // ≥
+	strResult = strings.Replace(strResult, ".", "", -1)
+	strResult = strings.Replace(strResult, ",", "", -1)
+	strResult = strings.Replace(strResult, ":", "", -1)
+
+	needAdd = true
+	indexCode = strings.Replace(strResult, " ", "", -1)
+	indexCode = strings.ToLower(indexCode)
+	if _, ok := ShanghaiSmmCodeToNameMap[indexCode]; !ok {
+		ShanghaiSmmCodeToNameMap[indexCode] = indexName
+		ShanghaiSmmNameToCodeMap[indexName] = indexCode
+	} else {
+		needAdd = false
+	}
+	return
+}

models/base_from_smm.go

@@ -23,7 +23,7 @@ type BaseFromSmmData struct {
 
 func AddBaseFromSmmData(item []*BaseFromSmmData) (err error) {
 	o := orm.NewOrm()
-	_, err = o.InsertMulti(len(item), item)
+	_, err = o.InsertMulti(500, item)
 	return
 }
 
@@ -440,6 +440,19 @@ func ModifyBaseFromSmmIndex(item *BaseFromSmmIndex) (err error) {
 	return
 }
 
+func (m *BaseFromSmmIndex) UpdateCols(cols []string) (err error) {
+	o := orm.NewOrm()
+	_, err = o.Update(m, cols...)
+	return
+}
+
+func ModifyBaseFromSmmIndexDate(item *BaseFromSmmIndex) (err error) {
+	o := orm.NewOrm()
+	sql := ` UPDATE base_from_smm_index SET end_date = ?, modify_time=NOW(), data_state= ? WHERE base_from_smm_index_id=?`
+	_, err = o.Raw(sql, item.EndDate, item.DataState, item.BaseFromSmmIndexId).Exec()
+	return
+}
+
 type SmmLatestDataResponse struct {
 	Code int           `json:"code"`
 	Msg  string        `json:"msg"`

routers/commentsRouter.go

@@ -1213,6 +1213,24 @@ func init() {
             Filters: nil,
             Params: nil})
 
+    beego.GlobalControllerRouter["eta/eta_index_lib/controllers:ShanghaiSmmController"] = append(beego.GlobalControllerRouter["eta/eta_index_lib/controllers:ShanghaiSmmController"],
+        beego.ControllerComments{
+            Method: "RefreshExcel",
+            Router: `/refresh/excel`,
+            AllowHTTPMethods: []string{"post"},
+            MethodParams: param.Make(),
+            Filters: nil,
+            Params: nil})
+
+    beego.GlobalControllerRouter["eta/eta_index_lib/controllers:ShanghaiSmmController"] = append(beego.GlobalControllerRouter["eta/eta_index_lib/controllers:ShanghaiSmmController"],
+        beego.ControllerComments{
+            Method: "RefreshData",
+            Router: `/refresh/list`,
+            AllowHTTPMethods: []string{"post"},
+            MethodParams: param.Make(),
+            Filters: nil,
+            Params: nil})
+
     beego.GlobalControllerRouter["eta/eta_index_lib/controllers:ShfeController"] = append(beego.GlobalControllerRouter["eta/eta_index_lib/controllers:ShfeController"],
         beego.ControllerComments{
             Method: "Add",

routers/router.go

@@ -12,6 +12,7 @@ import (
 	"eta/eta_index_lib/controllers/fix"
 	"eta/eta_index_lib/controllers/future_good"
 	"eta/eta_index_lib/controllers/open"
+
 	beego "github.com/beego/beego/v2/server/web"
 )
 
@@ -264,6 +265,11 @@ func init() {
 				&controllers.Sci99Crawler{},
 			),
 		),
+		beego.NSNamespace("/shanghai_smm",
+			beego.NSInclude(
+				&controllers.ShanghaiSmmController{},
+			),
+		),
 	)
 	beego.AddNamespace(ns)
 }

services/shanghai_smm/shanghai_smm.go

@@ -0,0 +1,31 @@
+package shanghaismm
+
+import "time"
+
+type SmmData struct {
+	Highs           float64 `json:"highs"`
+	Low             float64 `json:"low"`
+	Average         float64 `json:"average"`
+	VchangeRate     float64 `json:"vchange_rate"`
+	LowShow         string  `json:"low_show"`
+	HighShow        string  `json:"high_show"`
+	AverageShow     string  `json:"average_show"`
+	ProductId       string  `json:"product_id"`
+	Vchange         float64 `json:"vchange"`
+	RenewDate       string  `json:"renew_date"`
+	ChangeValueShow string  `json:"change_value_show"`
+	ChangeRateShow  string  `json:"change_rate_show"`
+}
+
+type EdbInfoData struct {
+	ClassifyName string
+	IndexName    string
+	IndexCode    string
+	Frequency    string
+	Unit         string
+	Value        string
+	LastDate     time.Time
+	OldDate      time.Time
+	Data         map[string]float64
+	SmmData
+}