改为lib接收信息并保存

controllers/exchange_crawler.go

@@ -4,7 +4,8 @@ import (
 	"encoding/json"
 	"eta/eta_index_lib/models"
 	"fmt"
-	"github.com/rdlucklib/rdluck_tools/http"
+	"strings"
+	"time"
 )
 
 // 交易所爬虫
@@ -16,51 +17,304 @@ type ExchangeCrawler struct {
 // @Description 刷新数据接口
 // @Param	request	body models.AddEdbClassifyReq true "type json string"
 // @Success 200 {object} models.EdbClassify
-// @router /refresh [post]
-func (this *ExchangeCrawler) GetOrAdd() {
+// @router /refresh/ine [post]
+func (this *ExchangeCrawler) RefreshIne() {
 	br := new(models.BaseResponse).Init()
 	defer func() {
 		this.Data["json"] = br
 		this.ServeJSON()
 	}()
-	var req models.RefreshExchangeoReq
+	var req models.RefreshINEExchangeReq
 	err := json.Unmarshal(this.Ctx.Input.RequestBody, &req)
 	if err != nil {
 		br.Msg = "参数解析异常！"
 		br.ErrMsg = "参数解析失败，Err:" + err.Error()
 		return
 	}
-	if req.Url == "" {
-		br.Msg = "交易所链接错误"
-		br.IsSendEmail = false
-		return
+
+	message := req.Data
+	var position = message.OCursor
+	var tradeDate = message.ReportDate
+	existIndexMap := make(map[string]*models.BaseFromTradeIneIndex)
+
+	//获取所有指标信息
+	allIndex, err := models.GetBaseFromTradeIneIndexAll(req.Date)
+	if err != nil {
+		fmt.Println("select err:", err)
+	}
+	for _, v := range allIndex {
+		indexKey := v.DealName + v.BuyName + v.SoldName
+		existIndexMap[indexKey] = v
+	}
+	var itemVerifyCode int
+	//处理指标
+	for _, p := range position {
+		var item = new(models.BaseFromTradeIneIndex)
+		if p.Rank > 0 && p.Rank < 40 && p.Participantabbr1 != "" {
+			//成交量
+			item.Rank = p.Rank
+			item.DealShortName = p.Participantabbr1
+			item.BuyShortName = p.Participantabbr2
+			item.SoldShortName = p.Participantabbr3
+			item.DealName = strings.Replace(fmt.Sprintf("%s", p.Participantabbr1+"_"+p.Instrumentid+"_成交量(手)"), " ", "", -1)
+			item.BuyName = strings.Replace(fmt.Sprintf("%s", p.Participantabbr2+"_"+p.Instrumentid+"_持买单量(手)"), " ", "", -1)
+			item.SoldName = strings.Replace(fmt.Sprintf("%s", p.Participantabbr3+"_"+p.Instrumentid+"_持卖单量(手)"), " ", "", -1)
+			item.DealCode = models.IneIndexCodeGenerator(item.DealShortName, item.DealName, p.Instrumentid, "deal")
+			item.BuyCode = models.IneIndexCodeGenerator(item.BuyShortName, item.BuyName, p.Instrumentid, "buy")
+			item.SoldCode = models.IneIndexCodeGenerator(item.SoldShortName, item.SoldName, p.Instrumentid, "sold")
+			item.ClassifyName = strings.Replace(p.Productname, " ", "", -1)
+			item.ClassifyType = strings.Replace(p.Instrumentid, " ", "", -1)
+			item.Frequency = "日度"
+			item.CreateTime = time.Now()
+			item.ModifyTime = time.Now()
+			item.DataTime = tradeDate
+
+			if deal, ok := p.Deal.(float64); ok {
+				item.DealValue = int(deal)
+			}
+			if change1, ok := p.Change1.(float64); ok {
+				item.DealChange = int(change1)
+			}
+			if buyIn, ok := p.BuyIn.(float64); ok {
+				item.BuyValue = int(buyIn)
+			}
+			if change2, ok := p.Change2.(float64); ok {
+				item.BuyChange = int(change2)
+			}
+			if soldOut, ok := p.SoldOut.(float64); ok {
+				item.SoldValue = int(soldOut)
+			}
+			if change3, ok := p.Change3.(float64); ok {
+				item.SoldChange = int(change3)
+			}
+
+			itemVerifyCode = item.BuyValue + item.DealValue + item.SoldValue
+			if existIndex, ok := existIndexMap[item.DealName+item.BuyName+item.SoldName]; !ok {
+				newID, err := models.AddBaseFromTradeIneIndex(item)
+				if err != nil {
+					fmt.Println("insert error:", err)
+				}
+				fmt.Println("insert new indexID:", newID)
+			} else if existIndex != nil && itemVerifyCode != (existIndex.DealValue+existIndex.BuyValue+existIndex.SoldValue) {
+				//更新
+				err := models.ModifyBaseFromTradeIneIndex(item.DealValue, item.BuyValue, item.SoldValue, existIndex.BaseFromTradeIneIndexId)
+				if err != nil {
+					fmt.Println("data update err:", err)
+				}
+			}
+		} else if p.Rank == 999 {
+			//Top 20
+			item.Rank = p.Rank
+			item.DealShortName = p.Participantabbr1
+			item.BuyShortName = p.Participantabbr2
+			item.SoldShortName = p.Participantabbr3
+			item.DealName = strings.Replace(fmt.Sprintf("%s", "top20_"+p.Instrumentid+"_成交量(手)"), " ", "", -1)
+			item.BuyName = strings.Replace(fmt.Sprintf("%s", "top20_"+p.Instrumentid+"_持买单量(手)"), " ", "", -1)
+			item.SoldName = strings.Replace(fmt.Sprintf("%s", "top20_"+p.Instrumentid+"_持卖单量(手)"), " ", "", -1)
+			item.DealCode = models.IneIndexCodeGenerator("top20", item.DealName, p.Instrumentid, "deal")
+			item.BuyCode = models.IneIndexCodeGenerator("top20", item.BuyName, p.Instrumentid, "buy")
+			item.SoldCode = models.IneIndexCodeGenerator("top20", item.SoldName, p.Instrumentid, "sold")
+			item.ClassifyName = strings.Replace(p.Productname, " ", "", -1)
+			item.ClassifyType = strings.Replace(p.Instrumentid, " ", "", -1)
+			item.Frequency = "日度"
+			item.CreateTime = time.Now()
+			item.ModifyTime = time.Now()
+			item.DataTime = tradeDate
+
+			if deal, ok := p.Deal.(float64); ok {
+				item.DealValue = int(deal)
+			}
+			if change1, ok := p.Change1.(float64); ok {
+				item.DealChange = int(change1)
+			}
+			if buyIn, ok := p.BuyIn.(float64); ok {
+				item.BuyValue = int(buyIn)
+			}
+			if change2, ok := p.Change2.(float64); ok {
+				item.BuyChange = int(change2)
+			}
+			if soldOut, ok := p.SoldOut.(float64); ok {
+				item.SoldValue = int(soldOut)
+			}
+			if change3, ok := p.Change3.(float64); ok {
+				item.SoldChange = int(change3)
+			}
+
+			itemVerifyCode = item.BuyValue + item.DealValue + item.SoldValue
+			if existIndex, ok := existIndexMap[item.DealName+item.BuyName+item.SoldName]; !ok {
+				newID, err := models.AddBaseFromTradeIneIndex(item)
+				if err != nil {
+					fmt.Println("insert error:", err)
+				}
+				fmt.Println("insert new indexID:", newID)
+			} else if existIndex != nil && itemVerifyCode != (existIndex.DealValue+existIndex.BuyValue+existIndex.SoldValue) {
+				//更新
+				err := models.ModifyBaseFromTradeIneIndex(item.DealValue, item.BuyValue, item.SoldValue, existIndex.BaseFromTradeIneIndexId)
+				if err != nil {
+					fmt.Println("data update err:", err)
+				}
+			}
+		}
 	}
 
-	body, e := http.Get(req.Url)
-	if e != nil {
-		err = e
-		fmt.Println("err:", err)
+	br.Ret = 200
+	br.Msg = "获取成功"
+	br.Success = true
+}
+
+// @Title 刷新数据
+// @Description 刷新数据接口
+// @Param	request	body models.AddEdbClassifyReq true "type json string"
+// @Success 200 {object} models.EdbClassify
+// @router /refresh/sh [post]
+func (this *ExchangeCrawler) RefreshSH() {
+	br := new(models.BaseResponse).Init()
+	defer func() {
+		this.Data["json"] = br
+		this.ServeJSON()
+	}()
+	var req models.RefreshSHExchangeReq
+	err := json.Unmarshal(this.Ctx.Input.RequestBody, &req)
+	if err != nil {
+		br.Msg = "参数解析异常！"
+		br.ErrMsg = "参数解析失败，Err:" + err.Error()
 		return
 	}
-	if req.Exchange == "ine" {
-		var resp models.IneJSONData
-		err = json.Unmarshal(body, &resp)
-		if err != nil {
-			fmt.Println(err)
-			return
-		}
-		br.Data = resp
-	} else {
-		var resp models.SHMessage
-		err = json.Unmarshal(body, &resp)
-		if err != nil {
-			fmt.Println(err)
-			return
+
+	message := req.Data
+	var position = message.Position
+	var tradeDate = message.ReportDate
+	existIndexMap := make(map[string]*models.BaseFromTradeShanghaiIndex)
+
+	//获取所有指标信息
+	allIndex, err := models.GetBaseFromTradeShangHaiIndexAll(req.Date)
+	if err != nil {
+		fmt.Println("select err:", err)
+	}
+	for _, v := range allIndex {
+		indexKey := v.DealName + v.BuyName + v.SoldName
+		existIndexMap[indexKey] = v
+	}
+	var itemVerifyCode int
+	//处理指标
+	for _, p := range position {
+		var item = new(models.BaseFromTradeShanghaiIndex)
+		if p.Rank > 0 && p.Rank < 40 && p.ParticipantName1 != "" {
+			if strings.Replace(p.ProductName, " ", "", -1) != "20号胶" && strings.Replace(p.ProductName, " ", "", -1) != "低硫燃料油" {
+				contractCode := strings.Replace(p.ContractCode, " ", "", -1)
+				//成交量
+				item.Rank = p.Rank
+				item.DealShortName = strings.Replace(p.ParticipantName1, " ", "", -1)
+				item.BuyShortName = strings.Replace(p.ParticipantName2, " ", "", -1)
+				item.SoldShortName = strings.Replace(p.ParticipantName3, " ", "", -1)
+				item.DealName = strings.Replace(fmt.Sprintf("%s", p.ParticipantName1+"_"+p.ContractCode+"_成交量(手)"), " ", "", -1)
+				item.BuyName = strings.Replace(fmt.Sprintf("%s", p.ParticipantName2+"_"+p.ContractCode+"_持买单量(手)"), " ", "", -1)
+				item.SoldName = strings.Replace(fmt.Sprintf("%s", p.ParticipantName3+"_"+p.ContractCode+"_持卖单量(手)"), " ", "", -1)
+				item.DealCode = models.ShIndexCodeGenerator(item.DealShortName, item.DealName, contractCode, "deal")
+				item.BuyCode = models.ShIndexCodeGenerator(item.BuyShortName, item.BuyName, contractCode, "buy")
+				item.SoldCode = models.ShIndexCodeGenerator(item.SoldShortName, item.SoldName, contractCode, "sold")
+				item.ClassifyName = strings.Replace(p.ProductName, " ", "", -1)
+				item.ClassifyType = strings.Replace(p.ContractCode, " ", "", -1)
+				item.Frequency = "日度"
+				item.CreateTime = time.Now()
+				item.ModifyTime = time.Now()
+				item.DataTime = tradeDate
+
+				if deal, ok := p.Deal.(float64); ok {
+					item.DealValue = int(deal)
+				}
+				if change1, ok := p.Change1.(float64); ok {
+					item.DealChange = int(change1)
+				}
+				if buyIn, ok := p.BuyIn.(float64); ok {
+					item.BuyValue = int(buyIn)
+				}
+				if change2, ok := p.Change2.(float64); ok {
+					item.BuyChange = int(change2)
+				}
+				if soldOut, ok := p.SoldOut.(float64); ok {
+					item.SoldValue = int(soldOut)
+				}
+				if change3, ok := p.Change3.(float64); ok {
+					item.SoldChange = int(change3)
+				}
+
+				itemVerifyCode = item.BuyValue + item.DealValue + item.SoldValue
+				if existIndex, ok := existIndexMap[item.DealName+item.BuyName+item.SoldName]; !ok {
+					newID, e := models.AddBaseFromTradeShangHaiIndex(item)
+					if e != nil {
+						err = e
+						fmt.Println("insert error:", err)
+					}
+					fmt.Println("insert new indexID:", newID)
+				} else if existIndex != nil && itemVerifyCode != (existIndex.DealValue+existIndex.BuyValue+existIndex.SoldValue) {
+					//更新
+					err = models.ModifyBaseFromTradeShangHaiIndex(item.DealValue, item.BuyValue, item.SoldValue, existIndex.BaseFromTradeShangHaiIndexId)
+					if err != nil {
+						fmt.Println("data update err:", err)
+					}
+				}
+			}
+		} else if p.Rank == 999 {
+			if strings.Replace(p.ProductName, " ", "", -1) != "20号胶" && strings.Replace(p.ProductName, " ", "", -1) != "低硫燃料油" {
+				contractCode := strings.Replace(p.ContractCode, " ", "", -1)
+				//Top 20
+				item.Rank = p.Rank
+				item.DealShortName = strings.Replace(p.ParticipantName1, " ", "", -1)
+				item.BuyShortName = strings.Replace(p.ParticipantName2, " ", "", -1)
+				item.SoldShortName = strings.Replace(p.ParticipantName3, " ", "", -1)
+				item.DealName = strings.Replace(fmt.Sprintf("%s", "top20_"+p.ContractCode+"_成交量(手)"), " ", "", -1)
+				item.BuyName = strings.Replace(fmt.Sprintf("%s", "top20_"+p.ContractCode+"_持买单量(手)"), " ", "", -1)
+				item.SoldName = strings.Replace(fmt.Sprintf("%s", "top20_"+p.ContractCode+"_持卖单量(手)"), " ", "", -1)
+				item.DealCode = models.ShIndexCodeGenerator("top20", item.DealName, contractCode, "deal")
+				item.BuyCode = models.ShIndexCodeGenerator("top20", item.BuyName, contractCode, "buy")
+				item.SoldCode = models.ShIndexCodeGenerator("top20", item.SoldName, contractCode, "sold")
+				item.ClassifyName = strings.Replace(p.ProductName, " ", "", -1)
+				item.ClassifyType = strings.Replace(p.ContractCode, " ", "", -1)
+				item.Frequency = "日度"
+				item.CreateTime = time.Now()
+				item.ModifyTime = time.Now()
+				item.DataTime = tradeDate
+
+				if deal, ok := p.Deal.(float64); ok {
+					item.DealValue = int(deal)
+				}
+				if change1, ok := p.Change1.(float64); ok {
+					item.DealChange = int(change1)
+				}
+				if buyIn, ok := p.BuyIn.(float64); ok {
+					item.BuyValue = int(buyIn)
+				}
+				if change2, ok := p.Change2.(float64); ok {
+					item.BuyChange = int(change2)
+				}
+				if soldOut, ok := p.SoldOut.(float64); ok {
+					item.SoldValue = int(soldOut)
+				}
+				if change3, ok := p.Change3.(float64); ok {
+					item.SoldChange = int(change3)
+				}
+
+				itemVerifyCode = item.BuyValue + item.DealValue + item.SoldValue
+				if existIndex, ok := existIndexMap[item.DealName+item.BuyName+item.SoldName]; !ok {
+					newID, e := models.AddBaseFromTradeShangHaiIndex(item)
+					if e != nil {
+						err = e
+						fmt.Println("insert error:", err)
+					}
+					fmt.Println("insert new indexID:", newID)
+				} else if existIndex != nil && itemVerifyCode != (existIndex.DealValue+existIndex.BuyValue+existIndex.SoldValue) {
+					//更新
+					err = models.ModifyBaseFromTradeShangHaiIndex(item.DealValue, item.BuyValue, item.SoldValue, existIndex.BaseFromTradeShangHaiIndexId)
+					if err != nil {
+						fmt.Println("data update err:", err)
+					}
+				}
+			}
 		}
-		br.Data = resp
 	}
 
 	br.Ret = 200
 	br.Msg = "获取成功"
 	br.Success = true
-}
+}

go.mod

@@ -8,6 +8,7 @@ require (
 	github.com/dgrijalva/jwt-go v3.2.0+incompatible
 	github.com/go-sql-driver/mysql v1.6.0
 	github.com/gonum/stat v0.0.0-20181125101827-41a0da705a5b
+	github.com/mozillazg/go-pinyin v0.20.0
 	github.com/nosixtools/solarlunar v0.0.0-20211112060703-1b6dea7b4a19
 	github.com/olivere/elastic/v7 v7.0.32
 	github.com/rdlucklib/rdluck_tools v1.0.3

go.sum

@@ -322,6 +322,8 @@ github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJ
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/mozillazg/go-pinyin v0.20.0 h1:BtR3DsxpApHfKReaPO1fCqF4pThRwH9uwvXzm+GnMFQ=
+github.com/mozillazg/go-pinyin v0.20.0/go.mod h1:iR4EnMMRXkfpFVV5FMi4FNB6wGq9NV6uDWbUuPhP4Yc=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/nats-io/jwt v0.3.0/go.mod h1:fRYCDE99xlTsqUzISS1Bi75UBJ6ljOJQOAAu5VglpSg=

models/base_from_sh.go

@@ -4,6 +4,7 @@ import (
 	"eta/eta_index_lib/utils"
 	"fmt"
 	"github.com/beego/beego/v2/client/orm"
+	"github.com/mozillazg/go-pinyin"
 	"strconv"
 	"strings"
 	"time"
@@ -263,35 +264,11 @@ func RefreshEdbDataFromSh(edbInfoId int, edbCode, startDate string) (err error)
 	return
 }
 
-type RefreshExchangeoReq struct {
+type RefreshSHExchangeReq struct {
 	Url      string `description:"交易所链接"`
 	Exchange string `description:"交易所"`
-}
-
-type IneJSONData struct {
-	OCursor    []OCursor   `json:"o_cursor"`
-	OCode      interface{} `json:"o_code"`
-	OMsg       string      `json:"o_msg"`
-	ReportDate string      `json:"report_date"`
-	UpdateDate string      `json:"update_date"`
-}
-type OCursor struct {
-	Instrumentid     string      `json:"INSTRUMENTID"`
-	Participantid3   string      `json:"PARTICIPANTID3"`
-	Participantid2   string      `json:"PARTICIPANTID2"`
-	Participantid1   string      `json:"PARTICIPANTID1"`
-	Participantabbr3 string      `json:"PARTICIPANTABBR3"`
-	Participantabbr2 string      `json:"PARTICIPANTABBR2"`
-	Rank             int         `json:"RANK"`
-	Participantabbr1 string      `json:"PARTICIPANTABBR1"`
-	BuyIn            interface{} `json:"CJ2"`
-	Deal             interface{} `json:"CJ1"`
-	Change1          interface{} `json:"CJ1_CHG"`
-	Change3          interface{} `json:"CJ3_CHG"`
-	Productname      string      `json:"Productname"`
-	Productsortno    interface{} `json:"PRODUCTSORTNO"`
-	SoldOut          interface{} `json:"CJ3"`
-	Change2          interface{} `json:"CJ2_CHG"`
+	Date     string `description:"日期"`
+	Data     SHMessage
 }
 
 type Position []struct {
@@ -321,4 +298,79 @@ type SHMessage struct {
 	ReportDate string   `json:"report_date"`
 	UpdateDate string   `json:"update_date"`
 	PrintDate  string   `json:"print_date"`
+}
+
+type BaseFromTradeShanghaiIndex struct {
+	BaseFromTradeShangHaiIndexId int `orm:"column(base_from_trade_shanghai_index_id);pk"`
+	Rank                         int
+	DealShortName                string
+	DealName                     string
+	DealCode                     string
+	DealValue                    int
+	DealChange                   int
+	BuyShortName                 string
+	BuyName                      string
+	BuyCode                      string
+	BuyValue                     int
+	BuyChange                    int
+	SoldShortName                string
+	SoldName                     string
+	SoldCode                     string
+	SoldValue                    int
+	SoldChange                   int
+	Frequency                    string
+	ClassifyName                 string
+	ClassifyType                 string
+	CreateTime                   time.Time
+	ModifyTime                   time.Time
+	DataTime                     string
+}
+
+func AddBaseFromTradeShangHaiIndex(item *BaseFromTradeShanghaiIndex) (lastId int64, err error) {
+	o := orm.NewOrmUsingDB("data")
+	lastId, err = o.Insert(item)
+	return
+}
+
+func GetBaseFromTradeShangHaiIndexAll(dateStr string) (list []*BaseFromTradeShanghaiIndex, err error) {
+	o := orm.NewOrmUsingDB("data")
+	sql := `SELECT * FROM base_from_trade_shanghai_index WHERE data_time=?`
+	_, err = o.Raw(sql, dateStr).QueryRows(&list)
+	return
+}
+
+func ModifyBaseFromTradeShangHaiIndex(dealValue, buyValue, soldValue int, dataId int) (err error) {
+	o := orm.NewOrmUsingDB("data")
+	sql := `UPDATE base_from_trade_shanghai_index SET deal_value=?,buy_value=?,sold_value=?,modify_time=NOW() WHERE base_from_trade_shanghai_index_id=? `
+	_, err = o.Raw(sql, dealValue, buyValue, soldValue, dataId).Exec()
+	return
+}
+
+func ShIndexCodeGenerator(shortName, indexName, contractCode, suffix string) string {
+	var indexCodeMap = make(map[string]string)
+	if shortName == "" {
+		return ""
+	}
+	strResult := ""
+	if shortName != "top20" {
+		//取公司全拼
+		a := pinyin.NewArgs()
+		rows := pinyin.LazyPinyin(shortName, a)
+		for i := 0; i < len(rows); i++ {
+			strResult += rows[i]
+		}
+	} else {
+		strResult = "top20"
+	}
+	indexCode, _ := indexCodeMap[indexName]
+	if indexCode == "" {
+		indexCode = strResult + contractCode + suffix
+		indexCode = strings.Replace(indexCode, " ", "", -1)
+		indexCodeMap[indexName] = indexCode
+		err := AddBaseFromTradeMapping(indexName, indexCode, "SH")
+		if err != nil {
+			fmt.Println("add Code err:", err)
+		}
+	}
+	return strings.Replace(indexCode, " ", "", -1)
 }

models/base_from_shfe.go

@@ -4,6 +4,7 @@ import (
 	"eta/eta_index_lib/utils"
 	"fmt"
 	"github.com/beego/beego/v2/client/orm"
+	"github.com/mozillazg/go-pinyin"
 	"strconv"
 	"strings"
 	"time"
@@ -260,3 +261,129 @@ func RefreshEdbDataFromShfe(edbInfoId int, edbCode, startDate string) (err error
 	}
 	return
 }
+
+type BaseFromTradeIneIndex struct {
+	BaseFromTradeIneIndexId int `orm:"column(base_from_trade_ine_index_id);pk"`
+	Rank                    int
+	DealShortName           string
+	DealName                string
+	DealCode                string
+	DealValue               int
+	DealChange              int
+	BuyShortName            string
+	BuyName                 string
+	BuyCode                 string
+	BuyValue                int
+	BuyChange               int
+	SoldShortName           string
+	SoldName                string
+	SoldCode                string
+	SoldValue               int
+	SoldChange              int
+	Frequency               string
+	ClassifyName            string
+	ClassifyType            string
+	CreateTime              time.Time
+	ModifyTime              time.Time
+	DataTime                string
+}
+
+type BaseFromTradeMapping struct {
+	BaseFromTradeMappingId int `orm:"column(base_from_trade_mapping_id);pk"`
+	IndexName              string
+	IndexCode              string
+	Exchange               string
+}
+func IneIndexCodeGenerator(shortName, indexName, Instrumentid, suffix string) string {
+	var ineIndexCodeMap = make(map[string]string)
+	if shortName == "" {
+		return ""
+	}
+	strResult := ""
+	if shortName != "top20" {
+		//取公司全拼
+		a := pinyin.NewArgs()
+		rows := pinyin.LazyPinyin(shortName, a)
+		for i := 0; i < len(rows); i++ {
+			strResult += rows[i]
+		}
+	} else {
+		strResult = "top20"
+	}
+	ineIndexCode, _ := ineIndexCodeMap[indexName]
+	if ineIndexCode == "" {
+		ineIndexCode = strResult + Instrumentid + suffix
+		ineIndexCode = strings.Replace(ineIndexCode, " ", "", -1)
+		ineIndexCodeMap[indexName] = ineIndexCode
+		err := AddBaseFromTradeMapping(indexName, ineIndexCode, "INE")
+		if err != nil {
+			fmt.Println("add Code err:", err)
+		}
+	}
+	return strings.Replace(ineIndexCode, " ", "", -1)
+}
+
+func AddBaseFromTradeIneIndex(item *BaseFromTradeIneIndex) (lastId int64, err error) {
+	o := orm.NewOrmUsingDB("data")
+	lastId, err = o.Insert(item)
+	return
+}
+
+func GetBaseFromTradeIneIndexAll(dateStr string) (list []*BaseFromTradeIneIndex, err error) {
+	o := orm.NewOrmUsingDB("data")
+	sql := `SELECT * FROM base_from_trade_ine_index where data_time=?`
+	_, err = o.Raw(sql, dateStr).QueryRows(&list)
+	return
+}
+
+func ModifyBaseFromTradeIneIndex(dealValue, buyValue, soldValue int, dataId int) (err error) {
+	o := orm.NewOrmUsingDB("data")
+	sql := `UPDATE base_from_trade_ine_index SET deal_value=?,buy_value=?,sold_value=?,modify_time=NOW() WHERE base_from_trade_ine_index_id=? `
+	_, err = o.Raw(sql, dealValue, buyValue, soldValue, dataId).Exec()
+	return
+}
+
+func GetIndexCodeFromMapping(exchange string) (list []*BaseFromTradeMapping, err error) {
+	o := orm.NewOrmUsingDB("data")
+	sql := `SELECT * FROM base_from_trade_mapping where exchange=?`
+	_, err = o.Raw(sql, exchange).QueryRows(&list)
+	return
+}
+
+func AddBaseFromTradeMapping(indexName, indexCode, exchange string) (err error) {
+	o := orm.NewOrmUsingDB("data")
+	sql := "Insert Into base_from_trade_mapping(index_name,index_code,exchange) Values('" + indexName + "','" + indexCode + "','" + exchange + "')"
+	_, err = o.Raw(sql).Exec()
+	return
+}
+
+type RefreshINEExchangeReq struct {
+	Date     string `description:"日期"`
+	Data     IneJSONData
+}
+
+type IneJSONData struct {
+	OCursor    []OCursor   `json:"o_cursor"`
+	OCode      interface{} `json:"o_code"`
+	OMsg       string      `json:"o_msg"`
+	ReportDate string      `json:"report_date"`
+	UpdateDate string      `json:"update_date"`
+}
+type OCursor struct {
+	Instrumentid     string      `json:"INSTRUMENTID"`
+	Participantid3   string      `json:"PARTICIPANTID3"`
+	Participantid2   string      `json:"PARTICIPANTID2"`
+	Participantid1   string      `json:"PARTICIPANTID1"`
+	Participantabbr3 string      `json:"PARTICIPANTABBR3"`
+	Participantabbr2 string      `json:"PARTICIPANTABBR2"`
+	Rank             int         `json:"RANK"`
+	Participantabbr1 string      `json:"PARTICIPANTABBR1"`
+	BuyIn            interface{} `json:"CJ2"`
+	Deal             interface{} `json:"CJ1"`
+	Change1          interface{} `json:"CJ1_CHG"`
+	Change3          interface{} `json:"CJ3_CHG"`
+	Productname      string      `json:"Productname"`
+	Productsortno    interface{} `json:"PRODUCTSORTNO"`
+	SoldOut          interface{} `json:"CJ3"`
+	Change2          interface{} `json:"CJ2_CHG"`
+}

routers/commentsRouter.go

@@ -288,8 +288,17 @@ func init() {
 
     beego.GlobalControllerRouter["eta/eta_index_lib/controllers:ExchangeCrawler"] = append(beego.GlobalControllerRouter["eta/eta_index_lib/controllers:ExchangeCrawler"],
         beego.ControllerComments{
-            Method: "GetOrAdd",
-            Router: `/refresh`,
+            Method: "RefreshIne",
+            Router: `/refresh/ine`,
+            AllowHTTPMethods: []string{"post"},
+            MethodParams: param.Make(),
+            Filters: nil,
+            Params: nil})
+
+    beego.GlobalControllerRouter["eta/eta_index_lib/controllers:ExchangeCrawler"] = append(beego.GlobalControllerRouter["eta/eta_index_lib/controllers:ExchangeCrawler"],
+        beego.ControllerComments{
+            Method: "RefreshSH",
+            Router: `/refresh/sh`,
             AllowHTTPMethods: []string{"post"},
             MethodParams: param.Make(),
             Filters: nil,