123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875 |
- package services
- import (
- "encoding/json"
- "eta/eta_crawler/models"
- "eta/eta_crawler/services/alarm_msg"
- "eta/eta_crawler/utils"
- "fmt"
- "github.com/mozillazg/go-pinyin"
- "strconv"
- "strings"
- "time"
- )
- // SyncRankingFromGuangzhou 广州交易所期货-日行情
- func CrawlerFromGuangzhouDayQuotes(date time.Time) {
- var err error
- defer func() {
- if err != nil {
- msg := "失败提醒" + "SyncRankingFromGuangzhou ErrMsg:" + err.Error()
- fmt.Println("msg:", msg)
- utils.FileLog.Info(msg)
- go alarm_msg.SendAlarmMsg(msg, 3)
- }
- }()
- now := date //time.Now()
- if now.Weekday() == time.Saturday || now.Weekday() == time.Sunday {
- return
- }
- dateStr := now.Format(utils.FormatDateUnSpace)
- zzUrl := `http://www.gfex.com.cn/u/interfacesWebTiDayQuotes/loadList`
- zzUrl = fmt.Sprintf(zzUrl)
- fmt.Println(zzUrl)
- // params := ""
- params := fmt.Sprintf("trade_date=%s&trade_type=0", dateStr)
- headerParams := make(map[string]string)
- //headerParams["Cookie"] = "CT6T=312900; SF_cookie_3=68941398"
- headerParams["User-Agent"] = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36"
- headerParams["Content-Type"] = "application/x-www-form-urlencoded;charset=utf-8"
- body, e := utils.HttpPost(zzUrl, params, headerParams)
- if e != nil {
- err = e
- fmt.Println("err:", err)
- return
- }
- fmt.Println("body :" + string(body))
- utils.FileLog.Info("body:" + string(body))
- if strings.Contains(string(body), "很抱歉! 因系统检测到您的请求可能对网站造成威胁") {
- return
- }
- item := new(models.TradeGuangZhou)
- err = json.Unmarshal(body, &item)
- if err != nil {
- fmt.Println("json.Unmarshal err:" + err.Error())
- return
- }
- gzObj := new(models.BaseFromTradeGuangzhouIndex)
- parentId := 1
- classifyList, err := gzObj.GetBaseFromTradeGuangzhouClassifyAll(parentId)
- if err != nil {
- fmt.Println("GetBaseFromTradeGuangzhouClassify Err:" + err.Error())
- return
- }
- classifyMap := make(map[string]*models.BaseFromTradeGuangzhouClassify)
- for _, cv := range classifyList {
- if cv.ClassifyCode != "" {
- classifyMap[cv.ClassifyCode] = cv
- }
- }
- //获取所有指标
- indexList, err := gzObj.GetBaseFromTradeGuangzhouIndexAll()
- if err != nil {
- fmt.Println("GetBaseFromTradeGuangzhouIndexAll Err:" + err.Error())
- return
- }
- indexMap := make(map[string]*models.BaseFromTradeGuangzhouIndex)
- for _, iv := range indexList {
- indexMap[iv.IndexCode] = iv
- }
- dataList, err := gzObj.GetBaseFromTradeGuangzhouDataByDate(now.Format(utils.FormatDate))
- dataMap := make(map[string]*models.BaseFromTradeGuangzhouData)
- for _, dv := range dataList {
- dataMap[dv.IndexCode] = dv
- }
- addDataList := make([]*models.BaseFromTradeGuangzhouData, 0)
- updateDataList := make([]*models.BaseFromTradeGuangzhouData, 0)
- for _, dv := range item.Data {
- if dv.Variety == "总计" {
- continue
- }
- var classifyId int
- if classifyItem, ok := classifyMap[dv.VarietyOrder]; !ok {
- classifyItem = new(models.BaseFromTradeGuangzhouClassify)
- classifyItem.ClassifyCode = dv.VarietyOrder
- classifyItem.ClassifyName = dv.Variety
- classifyItem.ParentId = 1
- classifyItem.ModifyTime = time.Now()
- classifyItem.CreateTime = time.Now()
- classifyLastId, err := gzObj.AddBaseFromTradeGuangzhouClassify(classifyItem)
- if err != nil {
- fmt.Println("AddBaseFromTradeGuangzhouClassify err:" + err.Error())
- return
- }
- classifyId = int(classifyLastId)
- } else {
- classifyId = classifyItem.BaseFromTradeGuangzhouClassifyId
- }
- if dv.DelivMonth == "" {
- continue
- }
- //指标编码
- //开盘价
- openIndexCode := dv.VarietyOrder + dv.DelivMonth + "open"
- //最高价
- highIndexCode := dv.VarietyOrder + dv.DelivMonth + "high"
- //最低价
- lowIndexCode := dv.VarietyOrder + dv.DelivMonth + "low"
- //收盘价
- closeIndexCode := dv.VarietyOrder + dv.DelivMonth + "close"
- //前结算价
- lastClearIndexCode := dv.VarietyOrder + dv.DelivMonth + "lastClear"
- //结算价
- clearPriceIndexCode := dv.VarietyOrder + dv.DelivMonth + "clearPrice"
- //涨跌
- diffIndexCode := dv.VarietyOrder + dv.DelivMonth + "diff"
- //涨跌1
- diff1IndexCode := dv.VarietyOrder + dv.DelivMonth + "diff1"
- //成交量
- volumeIndexCode := dv.VarietyOrder + dv.DelivMonth + "volumn"
- //持仓量
- openInterestIndexCode := dv.VarietyOrder + dv.DelivMonth + "openInterest"
- //持仓量变化
- diffIChangeIndexCode := dv.VarietyOrder + dv.DelivMonth + "diffI"
- //成交额
- turnoverIndexCode := dv.VarietyOrder + dv.DelivMonth + "turnover"
- handleIndexCodeArr := []string{openIndexCode, highIndexCode, lowIndexCode, closeIndexCode, lastClearIndexCode, clearPriceIndexCode, diffIndexCode, diff1IndexCode, volumeIndexCode, openInterestIndexCode, diffIChangeIndexCode, turnoverIndexCode}
- prefxCodeArr := []string{"open", "high", "low", "close", "lastClear", "clearPrice", "diff", "diff1", "volumn", "openInterest", "diffI", "turnover"}
- handleIndexNameArr := []string{"开盘价", "最高价", "最低价", "收盘价", "前结算价", "结算价", "涨跌", "涨跌1", "成交量", "持仓量", "持仓量变化", "成交额"}
- for ck, cv := range handleIndexCodeArr {
- var indexId int
- handleIndexCode := cv
- if indexItem, ok := indexMap[handleIndexCode]; !ok {
- indexName := dv.Variety + dv.DelivMonth + handleIndexNameArr[ck]
- var unit string
- if strings.Contains(indexName, "价") || strings.Contains(indexName, "涨跌") {
- unit = "元/吨"
- } else if strings.Contains(indexName, "量") {
- unit = "手"
- } else if strings.Contains(indexName, "成交额") {
- unit = "万元"
- }
- gzObj = &models.BaseFromTradeGuangzhouIndex{
- BaseFromTradeGuangzhouClassifyId: classifyId,
- IndexCode: handleIndexCode,
- IndexName: indexName,
- Frequency: "日度",
- Unit: unit,
- CreateTime: time.Now(),
- ModifyTime: time.Now(),
- }
- lastId, err := gzObj.AddBaseFromTradeGuangzhouIndex()
- if err != nil {
- fmt.Println("AddBaseFromTradeGuangzhouIndex Err:" + err.Error())
- return
- }
- indexId = int(lastId)
- } else {
- indexId = indexItem.BaseFromTradeGuangzhouIndexId
- }
- if dataItem, ok := dataMap[handleIndexCode]; !ok {
- dataItem = new(models.BaseFromTradeGuangzhouData)
- dataItem.BaseFromTradeGuangzhouIndexId = indexId
- dataItem.IndexCode = handleIndexCode
- dataItem.DataTime = dateStr
- switch prefxCodeArr[ck] {
- case "open":
- dataItem.Value = float64(dv.Open)
- case "high":
- dataItem.Value = float64(dv.High)
- case "low":
- dataItem.Value = float64(dv.Low)
- case "close":
- dataItem.Value = float64(dv.Close)
- case "lastClear":
- dataItem.Value = float64(dv.LastClear)
- case "clearPrice":
- dataItem.Value = float64(dv.ClearPrice)
- case "diff":
- dataItem.Value = float64(dv.Diff)
- case "diff1":
- dataItem.Value = float64(dv.Diff1)
- case "volumn":
- dataItem.Value = float64(dv.Volumn)
- case "openInterest":
- dataItem.Value = float64(dv.OpenInterest)
- case "diffI":
- dataItem.Value = float64(dv.DiffI)
- case "turnover":
- dataItem.Value = dv.Turnover
- }
- dataItem.CreateTime = time.Now()
- dataItem.ModifyTime = time.Now()
- addDataList = append(addDataList, dataItem)
- } else {
- var getValue float64
- switch prefxCodeArr[ck] {
- case "open":
- getValue = float64(dv.Open)
- case "high":
- getValue = float64(dv.High)
- case "low":
- getValue = float64(dv.Low)
- case "close":
- getValue = float64(dv.Close)
- case "lastClear":
- getValue = float64(dv.LastClear)
- case "clearPrice":
- getValue = float64(dv.ClearPrice)
- case "diff":
- getValue = float64(dv.Diff)
- case "diff1":
- getValue = float64(dv.Diff1)
- case "volumn":
- getValue = float64(dv.Volumn)
- case "openInterest":
- getValue = float64(dv.OpenInterest)
- case "diffI":
- getValue = float64(dv.DiffI)
- case "turnover":
- getValue = dv.Turnover
- }
- if dataItem.Value != getValue {
- dataItem.BaseFromTradeGuangzhouDataId = dataItem.BaseFromTradeGuangzhouDataId
- dataItem.BaseFromTradeGuangzhouIndexId = indexId
- dataItem.IndexCode = handleIndexCode
- dataItem.DataTime = dateStr
- dataItem.Value = getValue
- dataItem.ModifyTime = time.Now()
- updateDataList = append(updateDataList, dataItem)
- }
- }
- }
- }
- if len(addDataList) > 0 {
- err = gzObj.AddBaseFromTradeGuangzhouData(addDataList)
- if err != nil {
- fmt.Println("AddBaseFromTradeGuangzhouData Err:" + err.Error())
- return
- }
- }
- if len(updateDataList) > 0 {
- for _, v := range updateDataList {
- updateParams := make(map[string]interface{})
- whereParam := make(map[string]interface{})
- updateParams["value"] = v.Value
- updateParams["modify_time"] = time.Now()
- whereParam["base_from_trade_guangzhou_data_id"] = v.BaseFromTradeGuangzhouDataId
- err = gzObj.UpdateGuangzhouData(updateParams, whereParam)
- if err != nil {
- fmt.Println("UpdateGuangzhouData Err:" + err.Error())
- return
- }
- }
- }
- }
- // SyncRankingFromGuangzhou 广州交易所期货-合约
- func CrawlerFromGuangzhouContract(date time.Time) {
- // 准备: HTTP请求
- now := date //time.Now()
- if now.Weekday() == time.Saturday || now.Weekday() == time.Sunday {
- return
- }
- dateStr := now.Format(utils.FormatDateUnSpace)
- dateFormatStr := now.Format(utils.FormatDate)
- gzObj := new(models.BaseFromTradeGuangzhouIndex)
- parentId := 2
- classifyList, err := gzObj.GetBaseFromTradeGuangzhouClassifyAll(parentId)
- if err != nil {
- fmt.Println("GetBaseFromTradeGuangzhouClassify Err:" + err.Error())
- return
- }
- contractList, err := gzObj.GetBaseFromTradeGuangzhouContract("")
- if err != nil {
- fmt.Println("GetBaseFromTradeGuangzhouContract Err:" + err.Error())
- return
- }
- contractMap := make(map[string]*models.BaseFromTradeGuangzhouContract)
- for _, v := range contractList {
- contractMap[v.Contract+v.TradeDate] = v
- }
- for _, cv := range classifyList {
- if cv.ClassifyCode != "" {
- rn := utils.RangeRand(1, 30)
- fmt.Println(cv.ClassifyCode+";sleep:", strconv.Itoa(int(rn)))
- time.Sleep(time.Duration(rn) * time.Second)
- zzUrl := `http://www.gfex.com.cn/u/interfacesWebTiMemberDealPosiQuotes/loadListContract_id`
- zzUrl = fmt.Sprintf(zzUrl)
- fmt.Println(zzUrl)
- params := "variety=" + cv.ClassifyCode + "&trade_date=" + dateStr
- headerParams := make(map[string]string)
- headerParams["User-Agent"] = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36"
- headerParams["Content-Type"] = "application/x-www-form-urlencoded;charset=utf-8"
- body, err := utils.HttpPost(zzUrl, params, headerParams)
- if err != nil {
- fmt.Println("err:", err)
- return
- }
- fmt.Println("body :" + string(body))
- utils.FileLog.Info("body:" + string(body))
- contractItem := new(models.GuangzhouContract)
- err = json.Unmarshal(body, &contractItem)
- if err != nil {
- fmt.Println("CrawlerFromGuangzhouContract json.Unmarshal err:", err)
- return
- }
- if contractItem.Code == "0" {
- for _, dv := range contractItem.Data {
- if _, ok := contractMap[dv+dateFormatStr]; !ok {
- contractObj := new(models.BaseFromTradeGuangzhouContract)
- contractObj.BaseFromTradeGuangzhouClassifyId = cv.BaseFromTradeGuangzhouClassifyId
- contractObj.ClassifyCode = cv.ClassifyCode
- contractObj.Contract = dv
- contractObj.TradeDate = dateFormatStr
- err = gzObj.AddBaseFromTradeGuangzhouContract(contractObj)
- if err != nil {
- fmt.Println("AddBaseFromTradeGuangzhouContract err:", err)
- return
- }
- }
- }
- } else {
- utils.FileLog.Info("CrawlerFromGuangzhouContract Fail:" + string(body))
- }
- }
- }
- }
- // CrawlerFromGuangzhouDealPosiQuotes 广州交易所期货-日成交持仓排名
- func CrawlerFromGuangzhouDealPosiQuotes(date time.Time) {
- // 准备: HTTP请求
- now := date //time.Now()
- if now.Weekday() == time.Saturday || now.Weekday() == time.Sunday {
- return
- }
- dateStr := now.Format(utils.FormatDateUnSpace)
- dateStrFormat := now.Format(utils.FormatDate)
- gzObj := new(models.BaseFromTradeGuangzhouIndex)
- //获取所有指标
- indexList, err := gzObj.GetBaseFromTradeGuangzhouIndexAll()
- if err != nil {
- fmt.Println("GetBaseFromTradeGuangzhouIndexAll Err:" + err.Error())
- return
- }
- indexMap := make(map[string]*models.BaseFromTradeGuangzhouIndex)
- for _, iv := range indexList {
- indexMap[iv.IndexCode] = iv
- }
- dataList, err := gzObj.GetBaseFromTradeGuangzhouDataByDate(dateStrFormat)
- dataMap := make(map[string]*models.BaseFromTradeGuangzhouData)
- for _, dv := range dataList {
- dataMap[dv.IndexCode] = dv
- }
- contractList, err := gzObj.GetBaseFromTradeGuangzhouContract(dateStrFormat)
- if err != nil {
- fmt.Println("GetBaseFromTradeGuangzhouContract Err:" + err.Error())
- return
- }
- if len(contractList) <= 0 {
- fmt.Println("contractList is 0")
- return
- }
- zzUrl := `http://www.gfex.com.cn/u/interfacesWebTiMemberDealPosiQuotes/loadList`
- pages := []int{1, 2, 3}
- unit := "手"
- for _, cv := range contractList {
- for _, pv := range pages {
- rn := utils.RangeRand(1, 30)
- fmt.Println(cv.Contract+";sleep:", strconv.Itoa(int(rn)))
- time.Sleep(time.Duration(rn) * time.Second)
- params := "trade_date=" + dateStr + "&trade_type=0&variety=" + cv.ClassifyCode + "&contract_id=" + cv.Contract + "&data_type=" + strconv.Itoa(pv)
- //params = "trade_date=20231110&trade_type=0&variety=si&contract_id=si2312&data_type=1"
- //params = "trade_date=20231110&trade_type=0&variety=si&contract_id=si2402&data_type=1"
- fmt.Println("params:", params)
- headerParams := make(map[string]string)
- headerParams["User-Agent"] = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36"
- headerParams["Content-Type"] = "application/x-www-form-urlencoded;charset=utf-8"
- body, err := utils.HttpPost(zzUrl, params, headerParams)
- if err != nil {
- fmt.Println("err:", err)
- return
- }
- fmt.Println("body :" + string(body))
- utils.FileLog.Info("body:" + string(body))
- dealPosiQuotes := new(models.GuangzhouDealPosiQuotes)
- err = json.Unmarshal(body, dealPosiQuotes)
- if err != nil {
- fmt.Println("CrawlerFromGuangzhouDealPosiQuotes json.Unmarshal Err:" + err.Error())
- return
- }
- if dealPosiQuotes.Code == "0" {
- addDataList := make([]*models.BaseFromTradeGuangzhouData, 0)
- updateDataList := make([]*models.BaseFromTradeGuangzhouData, 0)
- for _, v := range dealPosiQuotes.Data {
- if v.Abbr == "总计" {
- continue
- }
- pinyinArg := pinyin.NewArgs()
- pinyinArr := pinyin.LazyPinyin(v.Abbr, pinyinArg)
- indexCode := strings.Join(pinyinArr, "")
- if indexCode == "" {
- continue
- }
- var indexName string
- switch pv {
- case 1:
- indexName = v.Abbr + "成交量"
- indexCode = indexCode + "deal"
- case 2:
- indexName = v.Abbr + "持买单量"
- indexCode = indexCode + "buy"
- case 3:
- indexName = v.Abbr + "持卖单量"
- indexCode = indexCode + "sold"
- }
- var indexId int
- if indexItem, ok := indexMap[indexCode]; !ok {
- gzObj = &models.BaseFromTradeGuangzhouIndex{
- BaseFromTradeGuangzhouClassifyId: cv.BaseFromTradeGuangzhouClassifyId,
- IndexCode: indexCode,
- IndexName: indexName,
- Frequency: "日度",
- Unit: unit,
- CreateTime: time.Now(),
- ModifyTime: time.Now(),
- }
- lastId, err := gzObj.AddBaseFromTradeGuangzhouIndex()
- if err != nil {
- fmt.Println("AddBaseFromTradeGuangzhouIndex Err:" + err.Error())
- return
- }
- indexId = int(lastId)
- } else {
- indexId = indexItem.BaseFromTradeGuangzhouIndexId
- }
- //判断数据是否存在
- if dataItem, ok := dataMap[indexCode]; !ok {
- dataItem = new(models.BaseFromTradeGuangzhouData)
- dataItem.BaseFromTradeGuangzhouIndexId = indexId
- dataItem.IndexCode = indexCode
- dataItem.DataTime = dateStr
- dataItem.Value = float64(v.TodayQty)
- dataItem.QtySub = float64(v.QtySub)
- dataItem.CreateTime = time.Now()
- dataItem.ModifyTime = time.Now()
- addDataList = append(addDataList, dataItem)
- } else {
- if dataItem.Value != float64(v.TodayQty) {
- dataItem.BaseFromTradeGuangzhouDataId = dataItem.BaseFromTradeGuangzhouDataId
- dataItem.BaseFromTradeGuangzhouIndexId = indexId
- dataItem.IndexCode = indexCode
- dataItem.DataTime = dateStr
- dataItem.Value = float64(v.TodayQty)
- dataItem.QtySub = float64(v.QtySub)
- dataItem.ModifyTime = time.Now()
- updateDataList = append(updateDataList, dataItem)
- }
- }
- }
- fmt.Println("addDataList:", len(addDataList))
- if len(addDataList) > 0 {
- err = gzObj.AddBaseFromTradeGuangzhouData(addDataList)
- if err != nil {
- fmt.Println("AddBaseFromTradeGuangzhouData Err:" + err.Error())
- return
- }
- }
- if len(updateDataList) > 0 {
- for _, v := range updateDataList {
- updateParams := make(map[string]interface{})
- whereParam := make(map[string]interface{})
- updateParams["value"] = v.Value
- updateParams["qty_sub"] = v.QtySub
- updateParams["modify_time"] = time.Now()
- whereParam["base_from_trade_guangzhou_data_id"] = v.BaseFromTradeGuangzhouDataId
- err = gzObj.UpdateGuangzhouData(updateParams, whereParam)
- if err != nil {
- fmt.Println("UpdateGuangzhouData Err:" + err.Error())
- return
- }
- }
- }
- }
- }
- }
- }
- // CrawlerFromGuangzhouWbillWeeklyQuotes 广州交易所期货-仓单日报
- func CrawlerFromGuangzhouWbillWeeklyQuotes(date time.Time) {
- now := date //time.Now()
- if now.Weekday() == time.Saturday || now.Weekday() == time.Sunday {
- return
- }
- dateStr := now.Format(utils.FormatDateUnSpace)
- dateStrFormat := now.Format(utils.FormatDate)
- gzObj := new(models.BaseFromTradeGuangzhouIndex)
- parentId := 3
- classifyList, err := gzObj.GetBaseFromTradeGuangzhouClassifyAll(parentId)
- if err != nil {
- fmt.Println("GetBaseFromTradeGuangzhouClassify Err:" + err.Error())
- return
- }
- classifyMap := make(map[string]*models.BaseFromTradeGuangzhouClassify)
- for _, cv := range classifyList {
- if cv.ClassifyCode != "" {
- classifyMap[cv.ClassifyCode] = cv
- }
- }
- //获取所有指标
- indexList, err := gzObj.GetBaseFromTradeGuangzhouIndexAll()
- if err != nil {
- fmt.Println("GetBaseFromTradeGuangzhouIndexAll Err:" + err.Error())
- return
- }
- indexMap := make(map[string]*models.BaseFromTradeGuangzhouIndex)
- for _, iv := range indexList {
- indexMap[iv.IndexCode] = iv
- }
- dataList, err := gzObj.GetBaseFromTradeGuangzhouDataByDate(dateStrFormat)
- dataMap := make(map[string]*models.BaseFromTradeGuangzhouData)
- for _, dv := range dataList {
- dataMap[dv.IndexCode] = dv
- }
- zzUrl := `http://www.gfex.com.cn/u/interfacesWebTdWbillWeeklyQuotes/loadList`
- params := "gen_date=" + dateStr
- fmt.Println("params:", params)
- headerParams := make(map[string]string)
- headerParams["User-Agent"] = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36"
- headerParams["Content-Type"] = "application/x-www-form-urlencoded;charset=utf-8"
- body, err := utils.HttpPost(zzUrl, params, headerParams)
- if err != nil {
- fmt.Println("err:", err)
- return
- }
- fmt.Println("body :" + string(body))
- utils.FileLog.Info("body:" + string(body))
- wbillQty := new(models.GuangzhouWbillQty)
- err = json.Unmarshal(body, wbillQty)
- if err != nil {
- fmt.Println("CrawlerFromGuangzhouWbillWeeklyQuotes json.Unmarshal Err:" + err.Error())
- return
- }
- if wbillQty.Code == "0" {
- addDataList := make([]*models.BaseFromTradeGuangzhouData, 0)
- updateDataList := make([]*models.BaseFromTradeGuangzhouData, 0)
- for _, v := range wbillQty.Data {
- if v.WhAbbr == "总计" {
- continue
- }
- pinyinArg := pinyin.NewArgs()
- pinyinArr := pinyin.LazyPinyin(v.WhAbbr, pinyinArg)
- indexCode := strings.Join(pinyinArr, "")
- if indexCode == "" {
- continue
- }
- indexCode += "wbill"
- indexName := v.WhAbbr + "仓单量"
- var indexId int
- if indexItem, ok := indexMap[indexCode]; !ok {
- var classifyId int
- if classifyItem, ok := classifyMap[v.VarietyOrder]; !ok {
- classifyItem = new(models.BaseFromTradeGuangzhouClassify)
- classifyItem.ClassifyCode = v.VarietyOrder
- classifyItem.ClassifyName = v.Variety
- classifyItem.ParentId = 3
- classifyItem.ModifyTime = time.Now()
- classifyItem.CreateTime = time.Now()
- classifyLastId, err := gzObj.AddBaseFromTradeGuangzhouClassify(classifyItem)
- if err != nil {
- fmt.Println("AddBaseFromTradeGuangzhouClassify err:" + err.Error())
- return
- }
- classifyId = int(classifyLastId)
- } else {
- classifyId = classifyItem.BaseFromTradeGuangzhouClassifyId
- }
- gzObj = &models.BaseFromTradeGuangzhouIndex{
- BaseFromTradeGuangzhouClassifyId: classifyId,
- IndexCode: indexCode,
- IndexName: indexName,
- Frequency: "日度",
- Unit: "手",
- CreateTime: time.Now(),
- ModifyTime: time.Now(),
- }
- lastId, err := gzObj.AddBaseFromTradeGuangzhouIndex()
- if err != nil {
- fmt.Println("AddBaseFromTradeGuangzhouIndex Err:" + err.Error())
- return
- }
- indexId = int(lastId)
- } else {
- indexId = indexItem.BaseFromTradeGuangzhouIndexId
- }
- //判断数据是否存在
- if dataItem, ok := dataMap[indexCode]; !ok {
- dataItem = new(models.BaseFromTradeGuangzhouData)
- dataItem.BaseFromTradeGuangzhouIndexId = indexId
- dataItem.IndexCode = indexCode
- dataItem.DataTime = dateStr
- dataItem.Value = float64(v.WbillQty)
- dataItem.QtySub = float64(v.RegWbillQty)
- dataItem.CreateTime = time.Now()
- dataItem.ModifyTime = time.Now()
- addDataList = append(addDataList, dataItem)
- } else {
- if dataItem.Value != float64(v.WbillQty) {
- dataItem.BaseFromTradeGuangzhouDataId = dataItem.BaseFromTradeGuangzhouDataId
- dataItem.BaseFromTradeGuangzhouIndexId = indexId
- dataItem.IndexCode = indexCode
- dataItem.DataTime = dateStr
- dataItem.Value = float64(v.WbillQty)
- dataItem.QtySub = float64(v.RegWbillQty)
- dataItem.ModifyTime = time.Now()
- updateDataList = append(updateDataList, dataItem)
- }
- }
- }
- fmt.Println("addDataList:", len(addDataList))
- if len(addDataList) > 0 {
- err = gzObj.AddBaseFromTradeGuangzhouData(addDataList)
- if err != nil {
- fmt.Println("AddBaseFromTradeGuangzhouData Err:" + err.Error())
- return
- }
- }
- if len(updateDataList) > 0 {
- for _, v := range updateDataList {
- updateParams := make(map[string]interface{})
- whereParam := make(map[string]interface{})
- updateParams["value"] = v.Value
- updateParams["qty_sub"] = v.QtySub
- updateParams["modify_time"] = time.Now()
- whereParam["base_from_trade_guangzhou_data_id"] = v.BaseFromTradeGuangzhouDataId
- err = gzObj.UpdateGuangzhouData(updateParams, whereParam)
- if err != nil {
- fmt.Println("UpdateGuangzhouData Err:" + err.Error())
- return
- }
- }
- }
- }
- }
- // CrawlerFromGuangzhouCongregateDeliveryQuotes 广州交易所期货-一次性交割卖方仓单查询
- func CrawlerFromGuangzhouCongregateDeliveryQuotes(date time.Time) {
- now := date //time.Now()
- dateStr := now.Format(utils.FormatMonthDateUnSpace)
- _, monthEndDay := utils.GetMonthStartAndEnd(strconv.Itoa(now.Year()), strconv.Itoa(int(now.Month())))
- gzObj := new(models.BaseFromTradeGuangzhouIndex)
- parentId := 4
- classifyList, err := gzObj.GetBaseFromTradeGuangzhouClassifyAll(parentId)
- if err != nil {
- fmt.Println("GetBaseFromTradeGuangzhouClassify Err:" + err.Error())
- return
- }
- classifyMap := make(map[string]*models.BaseFromTradeGuangzhouClassify)
- for _, cv := range classifyList {
- if cv.ClassifyCode != "" {
- classifyMap[cv.ClassifyCode] = cv
- }
- }
- //获取所有指标
- indexList, err := gzObj.GetBaseFromTradeGuangzhouIndexAll()
- if err != nil {
- fmt.Println("GetBaseFromTradeGuangzhouIndexAll Err:" + err.Error())
- return
- }
- indexMap := make(map[string]*models.BaseFromTradeGuangzhouIndex)
- for _, iv := range indexList {
- indexMap[iv.IndexCode] = iv
- }
- dataList, err := gzObj.GetBaseFromTradeGuangzhouDataByDate(monthEndDay)
- dataMap := make(map[string]*models.BaseFromTradeGuangzhouData)
- for _, dv := range dataList {
- dataMap[dv.IndexCode] = dv
- }
- zzUrl := `http://www.gfex.com.cn/u/interfacesWebTcCongregateDeliveryQuotes/loadList`
- params := "contract_month=" + dateStr
- fmt.Println("params:", params)
- headerParams := make(map[string]string)
- headerParams["User-Agent"] = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36"
- headerParams["Content-Type"] = "application/x-www-form-urlencoded;charset=utf-8"
- body, err := utils.HttpPost(zzUrl, params, headerParams)
- if err != nil {
- fmt.Println("err:", err)
- return
- }
- fmt.Println("body :" + string(body))
- utils.FileLog.Info("body:" + string(body))
- deliveryQuotes := new(models.GuangzhouCongregateDeliveryQuotes)
- err = json.Unmarshal(body, deliveryQuotes)
- if err != nil {
- fmt.Println("CrawlerFromGuangzhouCongregateDeliveryQuotes json.Unmarshal Err:" + err.Error())
- return
- }
- if deliveryQuotes.Code == "0" {
- addDataList := make([]*models.BaseFromTradeGuangzhouData, 0)
- updateDataList := make([]*models.BaseFromTradeGuangzhouData, 0)
- for _, v := range deliveryQuotes.Data {
- pinyinArg := pinyin.NewArgs()
- pinyinArr := pinyin.LazyPinyin(v.WhName, pinyinArg)
- indexCode := strings.Join(pinyinArr, "")
- if indexCode == "" {
- continue
- }
- indexCode += "wbillseller"
- indexName := v.WhName + "一次性交割卖方仓单量"
- var indexId int
- if indexItem, ok := indexMap[indexCode]; !ok {
- var classifyId int
- if classifyItem, ok := classifyMap[v.VarietyId]; !ok {
- classifyItem = new(models.BaseFromTradeGuangzhouClassify)
- classifyItem.ClassifyCode = v.VarietyId
- classifyItem.ClassifyName = v.VarietyName
- classifyItem.ParentId = 3
- classifyItem.ModifyTime = time.Now()
- classifyItem.CreateTime = time.Now()
- classifyLastId, err := gzObj.AddBaseFromTradeGuangzhouClassify(classifyItem)
- if err != nil {
- fmt.Println("AddBaseFromTradeGuangzhouClassify err:" + err.Error())
- return
- }
- classifyId = int(classifyLastId)
- } else {
- classifyId = classifyItem.BaseFromTradeGuangzhouClassifyId
- }
- gzObj = &models.BaseFromTradeGuangzhouIndex{
- BaseFromTradeGuangzhouClassifyId: classifyId,
- IndexCode: indexCode,
- IndexName: indexName,
- Frequency: "月度",
- Unit: "手",
- CreateTime: time.Now(),
- ModifyTime: time.Now(),
- }
- lastId, err := gzObj.AddBaseFromTradeGuangzhouIndex()
- if err != nil {
- fmt.Println("AddBaseFromTradeGuangzhouIndex Err:" + err.Error())
- return
- }
- indexId = int(lastId)
- } else {
- indexId = indexItem.BaseFromTradeGuangzhouIndexId
- }
- //判断数据是否存在
- if dataItem, ok := dataMap[indexCode]; !ok {
- dataItem = new(models.BaseFromTradeGuangzhouData)
- dataItem.BaseFromTradeGuangzhouIndexId = indexId
- dataItem.IndexCode = indexCode
- dataItem.DataTime = monthEndDay
- dataItem.Value = float64(v.WbillNum)
- dataItem.CreateTime = time.Now()
- dataItem.ModifyTime = time.Now()
- addDataList = append(addDataList, dataItem)
- } else {
- if dataItem.Value != float64(v.WbillNum) {
- dataItem.BaseFromTradeGuangzhouDataId = dataItem.BaseFromTradeGuangzhouDataId
- dataItem.BaseFromTradeGuangzhouIndexId = indexId
- dataItem.IndexCode = indexCode
- dataItem.DataTime = monthEndDay
- dataItem.Value = float64(v.WbillNum)
- dataItem.ModifyTime = time.Now()
- updateDataList = append(updateDataList, dataItem)
- }
- }
- }
- fmt.Println("addDataList:", len(addDataList))
- if len(addDataList) > 0 {
- err = gzObj.AddBaseFromTradeGuangzhouData(addDataList)
- if err != nil {
- fmt.Println("AddBaseFromTradeGuangzhouData Err:" + err.Error())
- return
- }
- }
- if len(updateDataList) > 0 {
- for _, v := range updateDataList {
- updateParams := make(map[string]interface{})
- whereParam := make(map[string]interface{})
- updateParams["value"] = v.Value
- updateParams["qty_sub"] = v.QtySub
- updateParams["modify_time"] = time.Now()
- whereParam["base_from_trade_guangzhou_data_id"] = v.BaseFromTradeGuangzhouDataId
- err = gzObj.UpdateGuangzhouData(updateParams, whereParam)
- if err != nil {
- fmt.Println("UpdateGuangzhouData Err:" + err.Error())
- return
- }
- }
- }
- }
- }
- //http://www.gfex.com.cn/u/interfacesWebTiMemberDealPosiQuotes/loadList
|