eta_crawler/services/trade_guangzhou.go

package services

import (
	"encoding/json"
	"eta/eta_crawler/models"
	"eta/eta_crawler/services/alarm_msg"
	"eta/eta_crawler/utils"
	"fmt"
	"github.com/mozillazg/go-pinyin"
	"strconv"
	"strings"
	"time"
)

// SyncRankingFromGuangzhou 广州交易所期货-日行情
func CrawlerFromGuangzhouDayQuotes(date time.Time) {
	var err error
	defer func() {
		if err != nil {
			msg := "失败提醒" + "SyncRankingFromGuangzhou ErrMsg:" + err.Error()
			fmt.Println("msg:", msg)
			utils.FileLog.Info(msg)
			go alarm_msg.SendAlarmMsg(msg, 3)
		}
	}()

	now := date //time.Now()

	if now.Weekday() == time.Saturday || now.Weekday() == time.Sunday {
		return
	}

	dateStr := now.Format(utils.FormatDateUnSpace)

	zzUrl := `http://www.gfex.com.cn/u/interfacesWebTiDayQuotes/loadList`

	zzUrl = fmt.Sprintf(zzUrl)
	fmt.Println(zzUrl)
	//	params := ""
	params := fmt.Sprintf("trade_date=%s&trade_type=0", dateStr)
	headerParams := make(map[string]string)
	//headerParams["Cookie"] = "CT6T=312900; SF_cookie_3=68941398"
	headerParams["User-Agent"] = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36"
	headerParams["Content-Type"] = "application/x-www-form-urlencoded;charset=utf-8"
	body, e := utils.HttpPost(zzUrl, params, headerParams)
	if e != nil {
		err = e
		fmt.Println("err:", err)
		return
	}
	fmt.Println("body :" + string(body))
	utils.FileLog.Info("body:" + string(body))
	if strings.Contains(string(body), "很抱歉！ 因系统检测到您的请求可能对网站造成威胁") {
		return
	}
	item := new(models.TradeGuangZhou)
	err = json.Unmarshal(body, &item)
	if err != nil {
		fmt.Println("json.Unmarshal err:" + err.Error())
		return
	}

	gzObj := new(models.BaseFromTradeGuangzhouIndex)

	parentId := 1
	classifyList, err := gzObj.GetBaseFromTradeGuangzhouClassifyAll(parentId)
	if err != nil {
		fmt.Println("GetBaseFromTradeGuangzhouClassify Err:" + err.Error())
		return
	}

	classifyMap := make(map[string]*models.BaseFromTradeGuangzhouClassify)
	for _, cv := range classifyList {
		if cv.ClassifyCode != "" {
			classifyMap[cv.ClassifyCode] = cv
		}
	}

	//获取所有指标
	indexList, err := gzObj.GetBaseFromTradeGuangzhouIndexAll()
	if err != nil {
		fmt.Println("GetBaseFromTradeGuangzhouIndexAll Err:" + err.Error())
		return
	}

	indexMap := make(map[string]*models.BaseFromTradeGuangzhouIndex)
	for _, iv := range indexList {
		indexMap[iv.IndexCode] = iv
	}

	dataList, err := gzObj.GetBaseFromTradeGuangzhouDataByDate(now.Format(utils.FormatDate))
	dataMap := make(map[string]*models.BaseFromTradeGuangzhouData)
	for _, dv := range dataList {
		dataMap[dv.IndexCode] = dv
	}

	addDataList := make([]*models.BaseFromTradeGuangzhouData, 0)
	updateDataList := make([]*models.BaseFromTradeGuangzhouData, 0)
	for _, dv := range item.Data {
		if dv.Variety == "总计" {
			continue
		}
		var classifyId int
		if classifyItem, ok := classifyMap[dv.VarietyOrder]; !ok {
			classifyItem = new(models.BaseFromTradeGuangzhouClassify)
			classifyItem.ClassifyCode = dv.VarietyOrder
			classifyItem.ClassifyName = dv.Variety
			classifyItem.ParentId = 1
			classifyItem.ModifyTime = time.Now()
			classifyItem.CreateTime = time.Now()
			classifyLastId, err := gzObj.AddBaseFromTradeGuangzhouClassify(classifyItem)
			if err != nil {
				fmt.Println("AddBaseFromTradeGuangzhouClassify err:" + err.Error())
				return
			}
			classifyId = int(classifyLastId)
		} else {
			classifyId = classifyItem.BaseFromTradeGuangzhouClassifyId
		}
		if dv.DelivMonth == "" {
			continue
		}
		//指标编码
		//开盘价
		openIndexCode := dv.VarietyOrder + dv.DelivMonth + "open"
		//最高价
		highIndexCode := dv.VarietyOrder + dv.DelivMonth + "high"
		//最低价
		lowIndexCode := dv.VarietyOrder + dv.DelivMonth + "low"
		//收盘价
		closeIndexCode := dv.VarietyOrder + dv.DelivMonth + "close"
		//前结算价
		lastClearIndexCode := dv.VarietyOrder + dv.DelivMonth + "lastClear"
		//结算价
		clearPriceIndexCode := dv.VarietyOrder + dv.DelivMonth + "clearPrice"
		//涨跌
		diffIndexCode := dv.VarietyOrder + dv.DelivMonth + "diff"
		//涨跌1
		diff1IndexCode := dv.VarietyOrder + dv.DelivMonth + "diff1"
		//成交量
		volumeIndexCode := dv.VarietyOrder + dv.DelivMonth + "volumn"
		//持仓量
		openInterestIndexCode := dv.VarietyOrder + dv.DelivMonth + "openInterest"
		//持仓量变化
		diffIChangeIndexCode := dv.VarietyOrder + dv.DelivMonth + "diffI"
		//成交额
		turnoverIndexCode := dv.VarietyOrder + dv.DelivMonth + "turnover"

		handleIndexCodeArr := []string{openIndexCode, highIndexCode, lowIndexCode, closeIndexCode, lastClearIndexCode, clearPriceIndexCode, diffIndexCode, diff1IndexCode, volumeIndexCode, openInterestIndexCode, diffIChangeIndexCode, turnoverIndexCode}
		prefxCodeArr := []string{"open", "high", "low", "close", "lastClear", "clearPrice", "diff", "diff1", "volumn", "openInterest", "diffI", "turnover"}
		handleIndexNameArr := []string{"开盘价", "最高价", "最低价", "收盘价", "前结算价", "结算价", "涨跌", "涨跌1", "成交量", "持仓量", "持仓量变化", "成交额"}

		for ck, cv := range handleIndexCodeArr {
			var indexId int
			handleIndexCode := cv

			if indexItem, ok := indexMap[handleIndexCode]; !ok {
				indexName := dv.Variety + dv.DelivMonth + handleIndexNameArr[ck]

				var unit string
				if strings.Contains(indexName, "价") || strings.Contains(indexName, "涨跌") {
					unit = "元/吨"
				} else if strings.Contains(indexName, "量") {
					unit = "手"
				} else if strings.Contains(indexName, "成交额") {
					unit = "万元"
				}

				gzObj = &models.BaseFromTradeGuangzhouIndex{
					BaseFromTradeGuangzhouClassifyId: classifyId,
					IndexCode:                        handleIndexCode,
					IndexName:                        indexName,
					Frequency:                        "日度",
					Unit:                             unit,
					CreateTime:                       time.Now(),
					ModifyTime:                       time.Now(),
				}
				lastId, err := gzObj.AddBaseFromTradeGuangzhouIndex()
				if err != nil {
					fmt.Println("AddBaseFromTradeGuangzhouIndex Err:" + err.Error())
					return
				}
				indexId = int(lastId)
			} else {
				indexId = indexItem.BaseFromTradeGuangzhouIndexId
			}

			if dataItem, ok := dataMap[handleIndexCode]; !ok {
				dataItem = new(models.BaseFromTradeGuangzhouData)
				dataItem.BaseFromTradeGuangzhouIndexId = indexId
				dataItem.IndexCode = handleIndexCode
				dataItem.DataTime = dateStr
				switch prefxCodeArr[ck] {
				case "open":
					dataItem.Value = float64(dv.Open)
				case "high":
					dataItem.Value = float64(dv.High)
				case "low":
					dataItem.Value = float64(dv.Low)
				case "close":
					dataItem.Value = float64(dv.Close)
				case "lastClear":
					dataItem.Value = float64(dv.LastClear)
				case "clearPrice":
					dataItem.Value = float64(dv.ClearPrice)
				case "diff":
					dataItem.Value = float64(dv.Diff)
				case "diff1":
					dataItem.Value = float64(dv.Diff1)
				case "volumn":
					dataItem.Value = float64(dv.Volumn)
				case "openInterest":
					dataItem.Value = float64(dv.OpenInterest)
				case "diffI":
					dataItem.Value = float64(dv.DiffI)
				case "turnover":
					dataItem.Value = dv.Turnover
				}
				dataItem.CreateTime = time.Now()
				dataItem.ModifyTime = time.Now()
				addDataList = append(addDataList, dataItem)
			} else {
				var getValue float64
				switch prefxCodeArr[ck] {
				case "open":
					getValue = float64(dv.Open)
				case "high":
					getValue = float64(dv.High)
				case "low":
					getValue = float64(dv.Low)
				case "close":
					getValue = float64(dv.Close)
				case "lastClear":
					getValue = float64(dv.LastClear)
				case "clearPrice":
					getValue = float64(dv.ClearPrice)
				case "diff":
					getValue = float64(dv.Diff)
				case "diff1":
					getValue = float64(dv.Diff1)
				case "volumn":
					getValue = float64(dv.Volumn)
				case "openInterest":
					getValue = float64(dv.OpenInterest)
				case "diffI":
					getValue = float64(dv.DiffI)
				case "turnover":
					getValue = dv.Turnover
				}
				if dataItem.Value != getValue {
					dataItem.BaseFromTradeGuangzhouDataId = dataItem.BaseFromTradeGuangzhouDataId
					dataItem.BaseFromTradeGuangzhouIndexId = indexId
					dataItem.IndexCode = handleIndexCode
					dataItem.DataTime = dateStr
					dataItem.Value = getValue
					dataItem.ModifyTime = time.Now()
					updateDataList = append(updateDataList, dataItem)
				}
			}
		}
	}

	if len(addDataList) > 0 {
		err = gzObj.AddBaseFromTradeGuangzhouData(addDataList)
		if err != nil {
			fmt.Println("AddBaseFromTradeGuangzhouData Err:" + err.Error())
			return
		}
	}
	if len(updateDataList) > 0 {
		for _, v := range updateDataList {
			updateParams := make(map[string]interface{})
			whereParam := make(map[string]interface{})
			updateParams["value"] = v.Value
			updateParams["modify_time"] = time.Now()
			whereParam["base_from_trade_guangzhou_data_id"] = v.BaseFromTradeGuangzhouDataId
			err = gzObj.UpdateGuangzhouData(updateParams, whereParam)
			if err != nil {
				fmt.Println("UpdateGuangzhouData Err:" + err.Error())
				return
			}
		}
	}
}

// SyncRankingFromGuangzhou 广州交易所期货-合约
func CrawlerFromGuangzhouContract(date time.Time) {
	// 准备: HTTP请求
	now := date //time.Now()
	if now.Weekday() == time.Saturday || now.Weekday() == time.Sunday {
		return
	}

	dateStr := now.Format(utils.FormatDateUnSpace)
	dateFormatStr := now.Format(utils.FormatDate)

	gzObj := new(models.BaseFromTradeGuangzhouIndex)

	parentId := 2
	classifyList, err := gzObj.GetBaseFromTradeGuangzhouClassifyAll(parentId)
	if err != nil {
		fmt.Println("GetBaseFromTradeGuangzhouClassify Err:" + err.Error())
		return
	}

	contractList, err := gzObj.GetBaseFromTradeGuangzhouContract("")
	if err != nil {
		fmt.Println("GetBaseFromTradeGuangzhouContract Err:" + err.Error())
		return
	}

	contractMap := make(map[string]*models.BaseFromTradeGuangzhouContract)
	for _, v := range contractList {
		contractMap[v.Contract+v.TradeDate] = v
	}
	for _, cv := range classifyList {
		if cv.ClassifyCode != "" {

			rn := utils.RangeRand(1, 30)
			fmt.Println(cv.ClassifyCode+";sleep:", strconv.Itoa(int(rn)))
			time.Sleep(time.Duration(rn) * time.Second)

			zzUrl := `http://www.gfex.com.cn/u/interfacesWebTiMemberDealPosiQuotes/loadListContract_id`

			zzUrl = fmt.Sprintf(zzUrl)
			fmt.Println(zzUrl)
			params := "variety=" + cv.ClassifyCode + "&trade_date=" + dateStr
			headerParams := make(map[string]string)
			headerParams["User-Agent"] = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36"
			headerParams["Content-Type"] = "application/x-www-form-urlencoded;charset=utf-8"
			body, err := utils.HttpPost(zzUrl, params, headerParams)
			if err != nil {
				fmt.Println("err:", err)
				return
			}
			fmt.Println("body :" + string(body))
			utils.FileLog.Info("body:" + string(body))
			contractItem := new(models.GuangzhouContract)
			err = json.Unmarshal(body, &contractItem)
			if err != nil {
				fmt.Println("CrawlerFromGuangzhouContract json.Unmarshal err:", err)
				return
			}
			if contractItem.Code == "0" {
				for _, dv := range contractItem.Data {
					if _, ok := contractMap[dv+dateFormatStr]; !ok {
						contractObj := new(models.BaseFromTradeGuangzhouContract)
						contractObj.BaseFromTradeGuangzhouClassifyId = cv.BaseFromTradeGuangzhouClassifyId
						contractObj.ClassifyCode = cv.ClassifyCode
						contractObj.Contract = dv
						contractObj.TradeDate = dateFormatStr
						err = gzObj.AddBaseFromTradeGuangzhouContract(contractObj)
						if err != nil {
							fmt.Println("AddBaseFromTradeGuangzhouContract err:", err)
							return
						}
					}
				}
			} else {
				utils.FileLog.Info("CrawlerFromGuangzhouContract Fail:" + string(body))
			}
		}
	}
}

// CrawlerFromGuangzhouDealPosiQuotes 广州交易所期货-日成交持仓排名
func CrawlerFromGuangzhouDealPosiQuotes(date time.Time) {
	// 准备: HTTP请求
	now := date //time.Now()
	if now.Weekday() == time.Saturday || now.Weekday() == time.Sunday {
		return
	}

	dateStr := now.Format(utils.FormatDateUnSpace)
	dateStrFormat := now.Format(utils.FormatDate)
	gzObj := new(models.BaseFromTradeGuangzhouIndex)

	//获取所有指标
	indexList, err := gzObj.GetBaseFromTradeGuangzhouIndexAll()
	if err != nil {
		fmt.Println("GetBaseFromTradeGuangzhouIndexAll Err:" + err.Error())
		return
	}

	indexMap := make(map[string]*models.BaseFromTradeGuangzhouIndex)
	for _, iv := range indexList {
		indexMap[iv.IndexCode] = iv
	}

	dataList, err := gzObj.GetBaseFromTradeGuangzhouDataByDate(dateStrFormat)
	dataMap := make(map[string]*models.BaseFromTradeGuangzhouData)
	for _, dv := range dataList {
		dataMap[dv.IndexCode] = dv
	}

	contractList, err := gzObj.GetBaseFromTradeGuangzhouContract(dateStrFormat)
	if err != nil {
		fmt.Println("GetBaseFromTradeGuangzhouContract Err:" + err.Error())
		return
	}

	if len(contractList) <= 0 {
		fmt.Println("contractList is 0")
		return
	}

	zzUrl := `http://www.gfex.com.cn/u/interfacesWebTiMemberDealPosiQuotes/loadList`

	pages := []int{1, 2, 3}
	unit := "手"

	for _, cv := range contractList {
		for _, pv := range pages {

			rn := utils.RangeRand(1, 30)
			fmt.Println(cv.Contract+";sleep:", strconv.Itoa(int(rn)))

			time.Sleep(time.Duration(rn) * time.Second)
			params := "trade_date=" + dateStr + "&trade_type=0&variety=" + cv.ClassifyCode + "&contract_id=" + cv.Contract + "&data_type=" + strconv.Itoa(pv)
			//params = "trade_date=20231110&trade_type=0&variety=si&contract_id=si2312&data_type=1"
			//params = "trade_date=20231110&trade_type=0&variety=si&contract_id=si2402&data_type=1"
			fmt.Println("params:", params)

			headerParams := make(map[string]string)
			headerParams["User-Agent"] = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36"
			headerParams["Content-Type"] = "application/x-www-form-urlencoded;charset=utf-8"
			body, err := utils.HttpPost(zzUrl, params, headerParams)
			if err != nil {
				fmt.Println("err:", err)
				return
			}
			fmt.Println("body :" + string(body))
			utils.FileLog.Info("body:" + string(body))

			dealPosiQuotes := new(models.GuangzhouDealPosiQuotes)
			err = json.Unmarshal(body, dealPosiQuotes)
			if err != nil {
				fmt.Println("CrawlerFromGuangzhouDealPosiQuotes json.Unmarshal Err:" + err.Error())
				return
			}

			if dealPosiQuotes.Code == "0" {
				addDataList := make([]*models.BaseFromTradeGuangzhouData, 0)
				updateDataList := make([]*models.BaseFromTradeGuangzhouData, 0)
				for _, v := range dealPosiQuotes.Data {
					if v.Abbr == "总计" {
						continue
					}
					pinyinArg := pinyin.NewArgs()
					pinyinArr := pinyin.LazyPinyin(v.Abbr, pinyinArg)
					indexCode := strings.Join(pinyinArr, "")
					if indexCode == "" {
						continue
					}
					var indexName string
					switch pv {
					case 1:
						indexName = v.Abbr + "成交量"
						indexCode = indexCode + "deal"
					case 2:
						indexName = v.Abbr + "持买单量"
						indexCode = indexCode + "buy"
					case 3:
						indexName = v.Abbr + "持卖单量"
						indexCode = indexCode + "sold"
					}

					var indexId int
					if indexItem, ok := indexMap[indexCode]; !ok {
						gzObj = &models.BaseFromTradeGuangzhouIndex{
							BaseFromTradeGuangzhouClassifyId: cv.BaseFromTradeGuangzhouClassifyId,
							IndexCode:                        indexCode,
							IndexName:                        indexName,
							Frequency:                        "日度",
							Unit:                             unit,
							CreateTime:                       time.Now(),
							ModifyTime:                       time.Now(),
						}
						lastId, err := gzObj.AddBaseFromTradeGuangzhouIndex()
						if err != nil {
							fmt.Println("AddBaseFromTradeGuangzhouIndex Err:" + err.Error())
							return
						}
						indexId = int(lastId)
					} else {
						indexId = indexItem.BaseFromTradeGuangzhouIndexId
					}
					//判断数据是否存在
					if dataItem, ok := dataMap[indexCode]; !ok {
						dataItem = new(models.BaseFromTradeGuangzhouData)
						dataItem.BaseFromTradeGuangzhouIndexId = indexId
						dataItem.IndexCode = indexCode
						dataItem.DataTime = dateStr
						dataItem.Value = float64(v.TodayQty)
						dataItem.QtySub = float64(v.QtySub)
						dataItem.CreateTime = time.Now()
						dataItem.ModifyTime = time.Now()
						addDataList = append(addDataList, dataItem)
					} else {
						if dataItem.Value != float64(v.TodayQty) {
							dataItem.BaseFromTradeGuangzhouDataId = dataItem.BaseFromTradeGuangzhouDataId
							dataItem.BaseFromTradeGuangzhouIndexId = indexId
							dataItem.IndexCode = indexCode
							dataItem.DataTime = dateStr
							dataItem.Value = float64(v.TodayQty)
							dataItem.QtySub = float64(v.QtySub)
							dataItem.ModifyTime = time.Now()
							updateDataList = append(updateDataList, dataItem)
						}
					}
				}
				fmt.Println("addDataList:", len(addDataList))
				if len(addDataList) > 0 {
					err = gzObj.AddBaseFromTradeGuangzhouData(addDataList)
					if err != nil {
						fmt.Println("AddBaseFromTradeGuangzhouData Err:" + err.Error())
						return
					}
				}

				if len(updateDataList) > 0 {
					for _, v := range updateDataList {
						updateParams := make(map[string]interface{})
						whereParam := make(map[string]interface{})
						updateParams["value"] = v.Value
						updateParams["qty_sub"] = v.QtySub
						updateParams["modify_time"] = time.Now()
						whereParam["base_from_trade_guangzhou_data_id"] = v.BaseFromTradeGuangzhouDataId
						err = gzObj.UpdateGuangzhouData(updateParams, whereParam)
						if err != nil {
							fmt.Println("UpdateGuangzhouData Err:" + err.Error())
							return
						}
					}
				}
			}
		}
	}

}

// CrawlerFromGuangzhouWbillWeeklyQuotes 广州交易所期货-仓单日报
func CrawlerFromGuangzhouWbillWeeklyQuotes(date time.Time) {
	now := date //time.Now()
	if now.Weekday() == time.Saturday || now.Weekday() == time.Sunday {
		return
	}

	dateStr := now.Format(utils.FormatDateUnSpace)
	dateStrFormat := now.Format(utils.FormatDate)
	gzObj := new(models.BaseFromTradeGuangzhouIndex)

	parentId := 3
	classifyList, err := gzObj.GetBaseFromTradeGuangzhouClassifyAll(parentId)
	if err != nil {
		fmt.Println("GetBaseFromTradeGuangzhouClassify Err:" + err.Error())
		return
	}

	classifyMap := make(map[string]*models.BaseFromTradeGuangzhouClassify)
	for _, cv := range classifyList {
		if cv.ClassifyCode != "" {
			classifyMap[cv.ClassifyCode] = cv
		}
	}

	//获取所有指标
	indexList, err := gzObj.GetBaseFromTradeGuangzhouIndexAll()
	if err != nil {
		fmt.Println("GetBaseFromTradeGuangzhouIndexAll Err:" + err.Error())
		return
	}

	indexMap := make(map[string]*models.BaseFromTradeGuangzhouIndex)
	for _, iv := range indexList {
		indexMap[iv.IndexCode] = iv
	}

	dataList, err := gzObj.GetBaseFromTradeGuangzhouDataByDate(dateStrFormat)
	dataMap := make(map[string]*models.BaseFromTradeGuangzhouData)
	for _, dv := range dataList {
		dataMap[dv.IndexCode] = dv
	}

	zzUrl := `http://www.gfex.com.cn/u/interfacesWebTdWbillWeeklyQuotes/loadList`

	params := "gen_date=" + dateStr
	fmt.Println("params:", params)

	headerParams := make(map[string]string)
	headerParams["User-Agent"] = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36"
	headerParams["Content-Type"] = "application/x-www-form-urlencoded;charset=utf-8"
	body, err := utils.HttpPost(zzUrl, params, headerParams)
	if err != nil {
		fmt.Println("err:", err)
		return
	}
	fmt.Println("body :" + string(body))
	utils.FileLog.Info("body:" + string(body))

	wbillQty := new(models.GuangzhouWbillQty)
	err = json.Unmarshal(body, wbillQty)
	if err != nil {
		fmt.Println("CrawlerFromGuangzhouWbillWeeklyQuotes json.Unmarshal Err:" + err.Error())
		return
	}

	if wbillQty.Code == "0" {
		addDataList := make([]*models.BaseFromTradeGuangzhouData, 0)
		updateDataList := make([]*models.BaseFromTradeGuangzhouData, 0)
		for _, v := range wbillQty.Data {
			if v.WhAbbr == "总计" {
				continue
			}
			pinyinArg := pinyin.NewArgs()
			pinyinArr := pinyin.LazyPinyin(v.WhAbbr, pinyinArg)
			indexCode := strings.Join(pinyinArr, "")
			if indexCode == "" {
				continue
			}
			indexCode += "wbill"
			indexName := v.WhAbbr + "仓单量"

			var indexId int
			if indexItem, ok := indexMap[indexCode]; !ok {
				var classifyId int
				if classifyItem, ok := classifyMap[v.VarietyOrder]; !ok {
					classifyItem = new(models.BaseFromTradeGuangzhouClassify)
					classifyItem.ClassifyCode = v.VarietyOrder
					classifyItem.ClassifyName = v.Variety
					classifyItem.ParentId = 3
					classifyItem.ModifyTime = time.Now()
					classifyItem.CreateTime = time.Now()
					classifyLastId, err := gzObj.AddBaseFromTradeGuangzhouClassify(classifyItem)
					if err != nil {
						fmt.Println("AddBaseFromTradeGuangzhouClassify err:" + err.Error())
						return
					}
					classifyId = int(classifyLastId)
				} else {
					classifyId = classifyItem.BaseFromTradeGuangzhouClassifyId
				}
				gzObj = &models.BaseFromTradeGuangzhouIndex{
					BaseFromTradeGuangzhouClassifyId: classifyId,
					IndexCode:                        indexCode,
					IndexName:                        indexName,
					Frequency:                        "日度",
					Unit:                             "手",
					CreateTime:                       time.Now(),
					ModifyTime:                       time.Now(),
				}
				lastId, err := gzObj.AddBaseFromTradeGuangzhouIndex()
				if err != nil {
					fmt.Println("AddBaseFromTradeGuangzhouIndex Err:" + err.Error())
					return
				}
				indexId = int(lastId)
			} else {
				indexId = indexItem.BaseFromTradeGuangzhouIndexId
			}
			//判断数据是否存在
			if dataItem, ok := dataMap[indexCode]; !ok {
				dataItem = new(models.BaseFromTradeGuangzhouData)
				dataItem.BaseFromTradeGuangzhouIndexId = indexId
				dataItem.IndexCode = indexCode
				dataItem.DataTime = dateStr
				dataItem.Value = float64(v.WbillQty)
				dataItem.QtySub = float64(v.RegWbillQty)
				dataItem.CreateTime = time.Now()
				dataItem.ModifyTime = time.Now()
				addDataList = append(addDataList, dataItem)
			} else {
				if dataItem.Value != float64(v.WbillQty) {
					dataItem.BaseFromTradeGuangzhouDataId = dataItem.BaseFromTradeGuangzhouDataId
					dataItem.BaseFromTradeGuangzhouIndexId = indexId
					dataItem.IndexCode = indexCode
					dataItem.DataTime = dateStr
					dataItem.Value = float64(v.WbillQty)
					dataItem.QtySub = float64(v.RegWbillQty)
					dataItem.ModifyTime = time.Now()
					updateDataList = append(updateDataList, dataItem)
				}
			}
		}
		fmt.Println("addDataList:", len(addDataList))
		if len(addDataList) > 0 {
			err = gzObj.AddBaseFromTradeGuangzhouData(addDataList)
			if err != nil {
				fmt.Println("AddBaseFromTradeGuangzhouData Err:" + err.Error())
				return
			}
		}

		if len(updateDataList) > 0 {
			for _, v := range updateDataList {
				updateParams := make(map[string]interface{})
				whereParam := make(map[string]interface{})
				updateParams["value"] = v.Value
				updateParams["qty_sub"] = v.QtySub
				updateParams["modify_time"] = time.Now()
				whereParam["base_from_trade_guangzhou_data_id"] = v.BaseFromTradeGuangzhouDataId
				err = gzObj.UpdateGuangzhouData(updateParams, whereParam)
				if err != nil {
					fmt.Println("UpdateGuangzhouData Err:" + err.Error())
					return
				}
			}
		}
	}
}

// CrawlerFromGuangzhouCongregateDeliveryQuotes 广州交易所期货-一次性交割卖方仓单查询
func CrawlerFromGuangzhouCongregateDeliveryQuotes(date time.Time) {
	now := date //time.Now()

	dateStr := now.Format(utils.FormatMonthDateUnSpace)
	_, monthEndDay := utils.GetMonthStartAndEnd(strconv.Itoa(now.Year()), strconv.Itoa(int(now.Month())))

	gzObj := new(models.BaseFromTradeGuangzhouIndex)

	parentId := 4
	classifyList, err := gzObj.GetBaseFromTradeGuangzhouClassifyAll(parentId)
	if err != nil {
		fmt.Println("GetBaseFromTradeGuangzhouClassify Err:" + err.Error())
		return
	}

	classifyMap := make(map[string]*models.BaseFromTradeGuangzhouClassify)
	for _, cv := range classifyList {
		if cv.ClassifyCode != "" {
			classifyMap[cv.ClassifyCode] = cv
		}
	}

	//获取所有指标
	indexList, err := gzObj.GetBaseFromTradeGuangzhouIndexAll()
	if err != nil {
		fmt.Println("GetBaseFromTradeGuangzhouIndexAll Err:" + err.Error())
		return
	}

	indexMap := make(map[string]*models.BaseFromTradeGuangzhouIndex)
	for _, iv := range indexList {
		indexMap[iv.IndexCode] = iv
	}

	dataList, err := gzObj.GetBaseFromTradeGuangzhouDataByDate(monthEndDay)
	dataMap := make(map[string]*models.BaseFromTradeGuangzhouData)
	for _, dv := range dataList {
		dataMap[dv.IndexCode] = dv
	}

	zzUrl := `http://www.gfex.com.cn/u/interfacesWebTcCongregateDeliveryQuotes/loadList`

	params := "contract_month=" + dateStr
	fmt.Println("params:", params)

	headerParams := make(map[string]string)
	headerParams["User-Agent"] = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36"
	headerParams["Content-Type"] = "application/x-www-form-urlencoded;charset=utf-8"
	body, err := utils.HttpPost(zzUrl, params, headerParams)
	if err != nil {
		fmt.Println("err:", err)
		return
	}
	fmt.Println("body :" + string(body))
	utils.FileLog.Info("body:" + string(body))

	deliveryQuotes := new(models.GuangzhouCongregateDeliveryQuotes)
	err = json.Unmarshal(body, deliveryQuotes)
	if err != nil {
		fmt.Println("CrawlerFromGuangzhouCongregateDeliveryQuotes json.Unmarshal Err:" + err.Error())
		return
	}

	if deliveryQuotes.Code == "0" {
		addDataList := make([]*models.BaseFromTradeGuangzhouData, 0)
		updateDataList := make([]*models.BaseFromTradeGuangzhouData, 0)
		for _, v := range deliveryQuotes.Data {
			pinyinArg := pinyin.NewArgs()
			pinyinArr := pinyin.LazyPinyin(v.WhName, pinyinArg)
			indexCode := strings.Join(pinyinArr, "")
			if indexCode == "" {
				continue
			}
			indexCode += "wbillseller"
			indexName := v.WhName + "一次性交割卖方仓单量"

			var indexId int
			if indexItem, ok := indexMap[indexCode]; !ok {
				var classifyId int
				if classifyItem, ok := classifyMap[v.VarietyId]; !ok {
					classifyItem = new(models.BaseFromTradeGuangzhouClassify)
					classifyItem.ClassifyCode = v.VarietyId
					classifyItem.ClassifyName = v.VarietyName
					classifyItem.ParentId = 3
					classifyItem.ModifyTime = time.Now()
					classifyItem.CreateTime = time.Now()
					classifyLastId, err := gzObj.AddBaseFromTradeGuangzhouClassify(classifyItem)
					if err != nil {
						fmt.Println("AddBaseFromTradeGuangzhouClassify err:" + err.Error())
						return
					}
					classifyId = int(classifyLastId)
				} else {
					classifyId = classifyItem.BaseFromTradeGuangzhouClassifyId
				}
				gzObj = &models.BaseFromTradeGuangzhouIndex{
					BaseFromTradeGuangzhouClassifyId: classifyId,
					IndexCode:                        indexCode,
					IndexName:                        indexName,
					Frequency:                        "月度",
					Unit:                             "手",
					CreateTime:                       time.Now(),
					ModifyTime:                       time.Now(),
				}
				lastId, err := gzObj.AddBaseFromTradeGuangzhouIndex()
				if err != nil {
					fmt.Println("AddBaseFromTradeGuangzhouIndex Err:" + err.Error())
					return
				}
				indexId = int(lastId)
			} else {
				indexId = indexItem.BaseFromTradeGuangzhouIndexId
			}
			//判断数据是否存在
			if dataItem, ok := dataMap[indexCode]; !ok {
				dataItem = new(models.BaseFromTradeGuangzhouData)
				dataItem.BaseFromTradeGuangzhouIndexId = indexId
				dataItem.IndexCode = indexCode
				dataItem.DataTime = monthEndDay
				dataItem.Value = float64(v.WbillNum)
				dataItem.CreateTime = time.Now()
				dataItem.ModifyTime = time.Now()
				addDataList = append(addDataList, dataItem)
			} else {
				if dataItem.Value != float64(v.WbillNum) {
					dataItem.BaseFromTradeGuangzhouDataId = dataItem.BaseFromTradeGuangzhouDataId
					dataItem.BaseFromTradeGuangzhouIndexId = indexId
					dataItem.IndexCode = indexCode
					dataItem.DataTime = monthEndDay
					dataItem.Value = float64(v.WbillNum)
					dataItem.ModifyTime = time.Now()
					updateDataList = append(updateDataList, dataItem)
				}
			}
		}
		fmt.Println("addDataList:", len(addDataList))
		if len(addDataList) > 0 {
			err = gzObj.AddBaseFromTradeGuangzhouData(addDataList)
			if err != nil {
				fmt.Println("AddBaseFromTradeGuangzhouData Err:" + err.Error())
				return
			}
		}

		if len(updateDataList) > 0 {
			for _, v := range updateDataList {
				updateParams := make(map[string]interface{})
				whereParam := make(map[string]interface{})
				updateParams["value"] = v.Value
				updateParams["qty_sub"] = v.QtySub
				updateParams["modify_time"] = time.Now()
				whereParam["base_from_trade_guangzhou_data_id"] = v.BaseFromTradeGuangzhouDataId
				err = gzObj.UpdateGuangzhouData(updateParams, whereParam)
				if err != nil {
					fmt.Println("UpdateGuangzhouData Err:" + err.Error())
					return
				}
			}
		}
	}
}

//http://www.gfex.com.cn/u/interfacesWebTiMemberDealPosiQuotes/loadList