eta_crawler/services/trade_guangzhou.go

package services

import (
	"encoding/json"
	"eta/eta_crawler/models"
	"eta/eta_crawler/services/alarm_msg"
	"eta/eta_crawler/utils"
	"fmt"
	"strconv"
	"strings"
	"time"
)

// SyncRankingFromGuangzhou 广州交易所期货-日行情
func CrawlerFromGuangzhouDayQuotes(date time.Time) {
	var err error
	defer func() {
		if err != nil {
			msg := "失败提醒" + "SyncRankingFromGuangzhou ErrMsg:" + err.Error()
			fmt.Println("msg:", msg)
			utils.FileLog.Info(msg)
			go alarm_msg.SendAlarmMsg(msg, 3)
		}
	}()

	now := date //time.Now()

	if now.Weekday() == time.Saturday || now.Weekday() == time.Sunday {
		return
	}

	dateStr := now.Format(utils.FormatDateUnSpace)

	zzUrl := `http://www.gfex.com.cn/u/interfacesWebTiDayQuotes/loadList`

	zzUrl = fmt.Sprintf(zzUrl)
	fmt.Println(zzUrl)
	//	params := ""
	params := fmt.Sprintf("trade_date=%s&trade_type=0", dateStr)
	headerParams := make(map[string]string)
	//headerParams["Cookie"] = "CT6T=312900; SF_cookie_3=68941398"
	headerParams["User-Agent"] = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36"
	headerParams["Content-Type"] = "application/x-www-form-urlencoded;charset=utf-8"
	body, e := utils.HttpPost(zzUrl, params, headerParams)
	if e != nil {
		err = e
		fmt.Println("err:", err)
		return
	}
	fmt.Println("body :" + string(body))
	utils.FileLog.Info("body:" + string(body))
	if strings.Contains(string(body), "很抱歉！ 因系统检测到您的请求可能对网站造成威胁") {
		return
	}
	item := new(models.TradeGuangZhou)
	err = json.Unmarshal(body, &item)
	if err != nil {
		fmt.Println("json.Unmarshal err:" + err.Error())
		return
	}

	gzObj := new(models.BaseFromTradeGuangzhouIndex)

	parentId := 1
	classifyList, err := gzObj.GetBaseFromTradeGuangzhouClassifyAll(parentId)
	if err != nil {
		fmt.Println("GetBaseFromTradeGuangzhouClassify Err:" + err.Error())
		return
	}

	classifyMap := make(map[string]*models.BaseFromTradeGuangzhouClassify)
	for _, cv := range classifyList {
		if cv.ClassifyCode != "" {
			classifyMap[cv.ClassifyCode] = cv
		}
	}

	//获取所有指标
	indexList, err := gzObj.GetBaseFromTradeGuangzhouIndexAll()
	if err != nil {
		fmt.Println("GetBaseFromTradeGuangzhouIndexAll Err:" + err.Error())
		return
	}

	indexMap := make(map[string]*models.BaseFromTradeGuangzhouIndex)
	for _, iv := range indexList {
		indexMap[iv.IndexCode] = iv
	}

	dataList, err := gzObj.GetBaseFromTradeGuangzhouDataByDate(now.Format(utils.FormatDate))
	dataMap := make(map[string]*models.BaseFromTradeGuangzhouData)
	for _, dv := range dataList {
		dataMap[dv.IndexCode] = dv
	}

	addDataList := make([]*models.BaseFromTradeGuangzhouData, 0)
	updateDataList := make([]*models.BaseFromTradeGuangzhouData, 0)
	for _, dv := range item.Data {
		if dv.Variety == "总计" {
			continue
		}
		var classifyId int
		if classifyItem, ok := classifyMap[dv.VarietyOrder]; !ok {
			classifyItem = new(models.BaseFromTradeGuangzhouClassify)
			classifyItem.ClassifyCode = dv.VarietyOrder
			classifyItem.ClassifyName = dv.Variety
			classifyItem.ParentId = 1
			classifyItem.ModifyTime = time.Now()
			classifyItem.CreateTime = time.Now()
			classifyLastId, err := gzObj.AddBaseFromTradeGuangzhouClassify(classifyItem)
			if err != nil {
				fmt.Println("AddBaseFromTradeGuangzhouClassify err:" + err.Error())
				return
			}
			classifyId = int(classifyLastId)
		} else {
			classifyId = classifyItem.BaseFromTradeGuangzhouClassifyId
		}
		if dv.DelivMonth == "" && !strings.Contains(dv.Variety, "小计") {
			continue
		}

		if !strings.Contains(dv.Variety, "小计") {
			//开盘价
			openIndexCode := dv.VarietyOrder + dv.DelivMonth + "open"
			//最高价
			highIndexCode := dv.VarietyOrder + dv.DelivMonth + "high"
			//最低价
			lowIndexCode := dv.VarietyOrder + dv.DelivMonth + "low"
			//收盘价
			closeIndexCode := dv.VarietyOrder + dv.DelivMonth + "close"
			//前结算价
			lastClearIndexCode := dv.VarietyOrder + dv.DelivMonth + "lastClear"
			//结算价
			clearPriceIndexCode := dv.VarietyOrder + dv.DelivMonth + "clearPrice"
			//涨跌
			diffIndexCode := dv.VarietyOrder + dv.DelivMonth + "diff"
			//涨跌1
			diff1IndexCode := dv.VarietyOrder + dv.DelivMonth + "diff1"
			//成交量
			volumeIndexCode := dv.VarietyOrder + dv.DelivMonth + "volumn"
			//持仓量
			openInterestIndexCode := dv.VarietyOrder + dv.DelivMonth + "openInterest"
			//持仓量变化
			diffIChangeIndexCode := dv.VarietyOrder + dv.DelivMonth + "diffI"
			//成交额
			turnoverIndexCode := dv.VarietyOrder + dv.DelivMonth + "turnover"

			handleIndexCodeArr := []string{openIndexCode, highIndexCode, lowIndexCode, closeIndexCode, lastClearIndexCode, clearPriceIndexCode, diffIndexCode, diff1IndexCode, volumeIndexCode, openInterestIndexCode, diffIChangeIndexCode, turnoverIndexCode}
			prefxCodeArr := []string{"open", "high", "low", "close", "lastClear", "clearPrice", "diff", "diff1", "volumn", "openInterest", "diffI", "turnover"}
			handleIndexNameArr := []string{"开盘价", "最高价", "最低价", "收盘价", "前结算价", "结算价", "涨跌", "涨跌1", "成交量", "持仓量", "持仓量变化", "成交额"}

			for ck, cv := range handleIndexCodeArr {
				var indexId int
				handleIndexCode := cv

				if indexItem, ok := indexMap[handleIndexCode]; !ok {
					indexName := dv.Variety + dv.DelivMonth + handleIndexNameArr[ck]

					var unit string
					if strings.Contains(indexName, "价") || strings.Contains(indexName, "涨跌") {
						unit = "元/吨"
					} else if strings.Contains(indexName, "量") {
						unit = "手"
					} else if strings.Contains(indexName, "成交额") {
						unit = "万元"
					}

					gzObj = &models.BaseFromTradeGuangzhouIndex{
						BaseFromTradeGuangzhouClassifyId: classifyId,
						IndexCode:                        handleIndexCode,
						IndexName:                        indexName,
						Frequency:                        "日度",
						Unit:                             unit,
						CreateTime:                       time.Now(),
						ModifyTime:                       time.Now(),
					}
					lastId, err := gzObj.AddBaseFromTradeGuangzhouIndex()
					if err != nil {
						fmt.Println("AddBaseFromTradeGuangzhouIndex Err:" + err.Error())
						return
					}
					indexId = int(lastId)
				} else {
					indexId = indexItem.BaseFromTradeGuangzhouIndexId
				}

				if dataItem, ok := dataMap[handleIndexCode]; !ok {
					dataItem = new(models.BaseFromTradeGuangzhouData)
					dataItem.BaseFromTradeGuangzhouIndexId = indexId
					dataItem.IndexCode = handleIndexCode
					dataItem.DataTime = dateStr
					switch prefxCodeArr[ck] {
					case "open":
						dataItem.Value = float64(dv.Open)
					case "high":
						dataItem.Value = float64(dv.High)
					case "low":
						dataItem.Value = float64(dv.Low)
					case "close":
						dataItem.Value = float64(dv.Close)
					case "lastClear":
						dataItem.Value = float64(dv.LastClear)
					case "clearPrice":
						dataItem.Value = float64(dv.ClearPrice)
					case "diff":
						dataItem.Value = float64(dv.Diff)
					case "diff1":
						dataItem.Value = float64(dv.Diff1)
					case "volumn":
						dataItem.Value = float64(dv.Volumn)
					case "openInterest":
						dataItem.Value = float64(dv.OpenInterest)
					case "diffI":
						dataItem.Value = float64(dv.DiffI)
					case "turnover":
						dataItem.Value = dv.Turnover
					}
					dataItem.CreateTime = time.Now()
					dataItem.ModifyTime = time.Now()
					addDataList = append(addDataList, dataItem)
				} else {
					var getValue float64
					switch prefxCodeArr[ck] {
					case "open":
						getValue = float64(dv.Open)
					case "high":
						getValue = float64(dv.High)
					case "low":
						getValue = float64(dv.Low)
					case "close":
						getValue = float64(dv.Close)
					case "lastClear":
						getValue = float64(dv.LastClear)
					case "clearPrice":
						getValue = float64(dv.ClearPrice)
					case "diff":
						getValue = float64(dv.Diff)
					case "diff1":
						getValue = float64(dv.Diff1)
					case "volumn":
						getValue = float64(dv.Volumn)
					case "openInterest":
						getValue = float64(dv.OpenInterest)
					case "diffI":
						getValue = float64(dv.DiffI)
					case "turnover":
						getValue = dv.Turnover
					}
					if dataItem.Value != getValue {
						dataItem.BaseFromTradeGuangzhouDataId = dataItem.BaseFromTradeGuangzhouDataId
						dataItem.BaseFromTradeGuangzhouIndexId = indexId
						dataItem.IndexCode = handleIndexCode
						dataItem.DataTime = dateStr
						dataItem.Value = getValue
						dataItem.ModifyTime = time.Now()
						updateDataList = append(updateDataList, dataItem)
					}
				}
			}
		} else {
			verietyPy := utils.ChineseToPinyinInitials("日行情" + dv.Variety)
			//成交量
			volumeIndexCode := dv.VarietyOrder + verietyPy + "volumn"
			//持仓量
			openInterestIndexCode := dv.VarietyOrder + verietyPy + "openInterest"
			//持仓量变化
			diffIChangeIndexCode := dv.VarietyOrder + verietyPy + "diffI"
			//成交额
			turnoverIndexCode := dv.VarietyOrder + verietyPy + "turnover"

			handleIndexCodeArr := []string{volumeIndexCode, openInterestIndexCode, diffIChangeIndexCode, turnoverIndexCode}
			prefxCodeArr := []string{"volumn", "openInterest", "diffI", "turnover"}
			handleIndexNameArr := []string{"成交量", "持仓量", "持仓量变化", "成交额"}

			for ck, cv := range handleIndexCodeArr {
				var indexId int
				handleIndexCode := cv

				if indexItem, ok := indexMap[handleIndexCode]; !ok {
					indexName := "日行情" + dv.Variety + handleIndexNameArr[ck]

					var unit string
					if strings.Contains(indexName, "价") || strings.Contains(indexName, "涨跌") {
						unit = "元/吨"
					} else if strings.Contains(indexName, "量") {
						unit = "手"
					} else if strings.Contains(indexName, "成交额") {
						unit = "万元"
					}

					gzObj = &models.BaseFromTradeGuangzhouIndex{
						BaseFromTradeGuangzhouClassifyId: classifyId,
						IndexCode:                        handleIndexCode,
						IndexName:                        indexName,
						Frequency:                        "日度",
						Unit:                             unit,
						CreateTime:                       time.Now(),
						ModifyTime:                       time.Now(),
					}
					lastId, err := gzObj.AddBaseFromTradeGuangzhouIndex()
					if err != nil {
						fmt.Println("AddBaseFromTradeGuangzhouIndex Err:" + err.Error())
						return
					}
					indexId = int(lastId)
				} else {
					indexId = indexItem.BaseFromTradeGuangzhouIndexId
				}

				if dataItem, ok := dataMap[handleIndexCode]; !ok {
					dataItem = new(models.BaseFromTradeGuangzhouData)
					dataItem.BaseFromTradeGuangzhouIndexId = indexId
					dataItem.IndexCode = handleIndexCode
					dataItem.DataTime = dateStr
					switch prefxCodeArr[ck] {
					case "volumn":
						dataItem.Value = float64(dv.Volumn)
					case "openInterest":
						dataItem.Value = float64(dv.OpenInterest)
					case "diffI":
						dataItem.Value = float64(dv.DiffI)
					case "turnover":
						dataItem.Value = dv.Turnover
					}
					dataItem.CreateTime = time.Now()
					dataItem.ModifyTime = time.Now()
					addDataList = append(addDataList, dataItem)
				} else {
					var getValue float64
					switch prefxCodeArr[ck] {
					case "volumn":
						getValue = float64(dv.Volumn)
					case "openInterest":
						getValue = float64(dv.OpenInterest)
					case "diffI":
						getValue = float64(dv.DiffI)
					case "turnover":
						getValue = dv.Turnover
					}
					if dataItem.Value != getValue {
						dataItem.BaseFromTradeGuangzhouDataId = dataItem.BaseFromTradeGuangzhouDataId
						dataItem.BaseFromTradeGuangzhouIndexId = indexId
						dataItem.IndexCode = handleIndexCode
						dataItem.DataTime = dateStr
						dataItem.Value = getValue
						dataItem.ModifyTime = time.Now()
						updateDataList = append(updateDataList, dataItem)
					}
				}
			}
		}
	}

	if len(addDataList) > 0 {
		err = gzObj.AddBaseFromTradeGuangzhouData(addDataList)
		if err != nil {
			fmt.Println("AddBaseFromTradeGuangzhouData Err:" + err.Error())
			return
		}
	}
	if len(updateDataList) > 0 {
		for _, v := range updateDataList {
			updateParams := make(map[string]interface{})
			whereParam := make(map[string]interface{})
			updateParams["value"] = v.Value
			updateParams["modify_time"] = time.Now()
			whereParam["base_from_trade_guangzhou_data_id"] = v.BaseFromTradeGuangzhouDataId
			err = gzObj.UpdateGuangzhouData(updateParams, whereParam)
			if err != nil {
				fmt.Println("UpdateGuangzhouData Err:" + err.Error())
				return
			}
		}
	}
}

// SyncRankingFromGuangzhou 广州交易所期货-合约
func CrawlerFromGuangzhouContract(date time.Time) {
	// 准备: HTTP请求
	now := date //time.Now()
	if now.Weekday() == time.Saturday || now.Weekday() == time.Sunday {
		return
	}

	dateStr := now.Format(utils.FormatDateUnSpace)
	dateFormatStr := now.Format(utils.FormatDate)

	gzObj := new(models.BaseFromTradeGuangzhouIndex)

	parentId := 2
	classifyList, err := gzObj.GetBaseFromTradeGuangzhouClassifyAll(parentId)
	if err != nil {
		fmt.Println("GetBaseFromTradeGuangzhouClassify Err:" + err.Error())
		return
	}

	contractList, err := gzObj.GetBaseFromTradeGuangzhouContract("")
	if err != nil {
		fmt.Println("GetBaseFromTradeGuangzhouContract Err:" + err.Error())
		return
	}

	contractMap := make(map[string]*models.BaseFromTradeGuangzhouContract)
	for _, v := range contractList {
		contractMap[v.Contract+v.TradeDate] = v
	}
	for _, cv := range classifyList {
		if cv.ClassifyCode != "" {

			rn := utils.RangeRand(1, 30)
			fmt.Println(cv.ClassifyCode+";sleep:", strconv.Itoa(int(rn)))
			time.Sleep(time.Duration(rn) * time.Second)

			zzUrl := `http://www.gfex.com.cn/u/interfacesWebTiMemberDealPosiQuotes/loadListContract_id`

			zzUrl = fmt.Sprintf(zzUrl)
			fmt.Println(zzUrl)
			params := "variety=" + cv.ClassifyCode + "&trade_date=" + dateStr
			headerParams := make(map[string]string)
			headerParams["User-Agent"] = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36"
			headerParams["Content-Type"] = "application/x-www-form-urlencoded;charset=utf-8"
			body, err := utils.HttpPost(zzUrl, params, headerParams)
			if err != nil {
				fmt.Println("err:", err)
				return
			}
			fmt.Println("body :" + string(body))
			utils.FileLog.Info("body:" + string(body))
			contractItem := new(models.GuangzhouContract)
			err = json.Unmarshal(body, &contractItem)
			if err != nil {
				fmt.Println("CrawlerFromGuangzhouContract json.Unmarshal err:", err)
				return
			}
			if contractItem.Code == "0" {
				for _, dv := range contractItem.Data {
					if _, ok := contractMap[dv+dateFormatStr]; !ok {
						contractObj := new(models.BaseFromTradeGuangzhouContract)
						contractObj.BaseFromTradeGuangzhouClassifyId = cv.BaseFromTradeGuangzhouClassifyId
						contractObj.ClassifyCode = cv.ClassifyCode
						contractObj.Contract = dv
						contractObj.TradeDate = dateFormatStr
						err = gzObj.AddBaseFromTradeGuangzhouContract(contractObj)
						if err != nil {
							fmt.Println("AddBaseFromTradeGuangzhouContract err:", err)
							return
						}
					}
				}
			} else {
				utils.FileLog.Info("CrawlerFromGuangzhouContract Fail:" + string(body))
			}
		}
	}
}

// CrawlerFromGuangzhouDealPosiQuotes 广州交易所期货-日成交持仓排名
func CrawlerFromGuangzhouDealPosiQuotes(date time.Time) {
	// 准备: HTTP请求
	now := date //time.Now()
	if now.Weekday() == time.Saturday || now.Weekday() == time.Sunday {
		return
	}

	dateStr := now.Format(utils.FormatDateUnSpace)
	dateStrFormat := now.Format(utils.FormatDate)
	gzObj := new(models.BaseFromTradeGuangzhouIndex)

	//获取所有指标
	indexList, err := gzObj.GetBaseFromTradeGuangzhouIndexAll()
	if err != nil {
		fmt.Println("GetBaseFromTradeGuangzhouIndexAll Err:" + err.Error())
		return
	}

	indexMap := make(map[string]*models.BaseFromTradeGuangzhouIndex)
	for _, iv := range indexList {
		indexMap[iv.IndexCode] = iv
	}

	dataList, err := gzObj.GetBaseFromTradeGuangzhouDataByDate(dateStrFormat)
	dataMap := make(map[string]*models.BaseFromTradeGuangzhouData)
	for _, dv := range dataList {
		dataMap[dv.IndexCode] = dv
	}

	contractList, err := gzObj.GetBaseFromTradeGuangzhouContract(dateStrFormat)
	if err != nil {
		fmt.Println("GetBaseFromTradeGuangzhouContract Err:" + err.Error())
		return
	}

	if len(contractList) <= 0 {
		fmt.Println("contractList is 0")
		return
	}

	zzUrl := `http://www.gfex.com.cn/u/interfacesWebTiMemberDealPosiQuotes/loadList`

	pages := []int{1, 2, 3}
	unit := "手"

	for _, cv := range contractList {
		for _, pv := range pages {

			rn := utils.RangeRand(1, 30)
			fmt.Println(cv.Contract+";sleep:", strconv.Itoa(int(rn)))

			time.Sleep(time.Duration(rn) * time.Second)
			params := "trade_date=" + dateStr + "&trade_type=0&variety=" + cv.ClassifyCode + "&contract_id=" + cv.Contract + "&data_type=" + strconv.Itoa(pv)
			//params = "trade_date=20231110&trade_type=0&variety=si&contract_id=si2312&data_type=1"
			//params = "trade_date=20231110&trade_type=0&variety=si&contract_id=si2402&data_type=1"
			fmt.Println("params:", params)

			headerParams := make(map[string]string)
			headerParams["User-Agent"] = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36"
			headerParams["Content-Type"] = "application/x-www-form-urlencoded;charset=utf-8"
			body, err := utils.HttpPost(zzUrl, params, headerParams)
			if err != nil {
				fmt.Println("err:", err)
				return
			}
			fmt.Println("body :" + string(body))
			utils.FileLog.Info("body:" + string(body))

			dealPosiQuotes := new(models.GuangzhouDealPosiQuotes)
			err = json.Unmarshal(body, dealPosiQuotes)
			if err != nil {
				fmt.Println("CrawlerFromGuangzhouDealPosiQuotes json.Unmarshal Err:" + err.Error())
				return
			}

			if dealPosiQuotes.Code == "0" {
				addDataList := make([]*models.BaseFromTradeGuangzhouData, 0)
				updateDataList := make([]*models.BaseFromTradeGuangzhouData, 0)
				for _, v := range dealPosiQuotes.Data {
					indexCode := utils.ChineseToPinyin(v.Abbr)
					var indexName string
					switch pv {
					case 1:
						indexName = v.Abbr + "_" + cv.Contract + "_" + "成交量"
						indexCode = indexCode + cv.Contract + "deal"
						if v.ContractId == "总计" {
							indexCode = utils.ChineseToPinyinInitials("日成交持仓排名总计") + cv.Contract + "deal"
							indexName = "日成交持仓排名_" + cv.Contract + "_成交量总计"
						}
					case 2:
						indexName = v.Abbr + "_" + cv.Contract + "_" + "持买单量"
						indexCode = indexCode + cv.Contract + "buy"
						if v.ContractId == "总计" {
							indexCode = utils.ChineseToPinyinInitials("日成交持仓排名总计") + cv.Contract + "buy"
							indexName = "日成交持仓排名_" + cv.Contract + "_持买单量总计"
						}
					case 3:
						indexName = v.Abbr + "_" + cv.Contract + "_" + "持卖单量"
						indexCode = indexCode + cv.Contract + "sold"
						if v.ContractId == "总计" {
							indexCode = utils.ChineseToPinyinInitials("日成交持仓排名总计") + cv.Contract + "sold"
							indexName = "日成交持仓排名_" + cv.Contract + "_持卖单量总计"
						}
					}

					var indexId int
					if indexItem, ok := indexMap[indexCode]; !ok {
						gzObj = &models.BaseFromTradeGuangzhouIndex{
							BaseFromTradeGuangzhouClassifyId: cv.BaseFromTradeGuangzhouClassifyId,
							BaseFromTradeGuangzhouContractId: cv.BaseFromTradeGuangzhouContractId,
							IndexCode:                        indexCode,
							IndexName:                        indexName,
							Frequency:                        "日度",
							Unit:                             unit,
							CreateTime:                       time.Now(),
							ModifyTime:                       time.Now(),
						}
						lastId, err := gzObj.AddBaseFromTradeGuangzhouIndex()
						if err != nil {
							fmt.Println("AddBaseFromTradeGuangzhouIndex Err:" + err.Error())
							return
						}
						indexId = int(lastId)
					} else {
						indexId = indexItem.BaseFromTradeGuangzhouIndexId
					}
					//判断数据是否存在
					if dataItem, ok := dataMap[indexCode]; !ok {
						dataItem = new(models.BaseFromTradeGuangzhouData)
						dataItem.BaseFromTradeGuangzhouIndexId = indexId
						dataItem.IndexCode = indexCode
						dataItem.DataTime = dateStr
						dataItem.Value = float64(v.TodayQty)
						dataItem.QtySub = float64(v.QtySub)
						dataItem.CreateTime = time.Now()
						dataItem.ModifyTime = time.Now()
						addDataList = append(addDataList, dataItem)
					} else {
						if dataItem.Value != float64(v.TodayQty) {
							dataItem.BaseFromTradeGuangzhouDataId = dataItem.BaseFromTradeGuangzhouDataId
							dataItem.BaseFromTradeGuangzhouIndexId = indexId
							dataItem.IndexCode = indexCode
							dataItem.DataTime = dateStr
							dataItem.Value = float64(v.TodayQty)
							dataItem.QtySub = float64(v.QtySub)
							dataItem.ModifyTime = time.Now()
							updateDataList = append(updateDataList, dataItem)
						}
					}
				}
				fmt.Println("addDataList:", len(addDataList))
				if len(addDataList) > 0 {
					err = gzObj.AddBaseFromTradeGuangzhouData(addDataList)
					if err != nil {
						fmt.Println("AddBaseFromTradeGuangzhouData Err:" + err.Error())
						return
					}
				}

				if len(updateDataList) > 0 {
					for _, v := range updateDataList {
						updateParams := make(map[string]interface{})
						whereParam := make(map[string]interface{})
						updateParams["value"] = v.Value
						updateParams["qty_sub"] = v.QtySub
						updateParams["modify_time"] = time.Now()
						whereParam["base_from_trade_guangzhou_data_id"] = v.BaseFromTradeGuangzhouDataId
						err = gzObj.UpdateGuangzhouData(updateParams, whereParam)
						if err != nil {
							fmt.Println("UpdateGuangzhouData Err:" + err.Error())
							return
						}
					}
				}
			}
		}
	}

}

// CrawlerFromGuangzhouWbillWeeklyQuotes 广州交易所期货-仓单日报
func CrawlerFromGuangzhouWbillWeeklyQuotes(date time.Time) {
	now := date //time.Now()
	if now.Weekday() == time.Saturday || now.Weekday() == time.Sunday {
		return
	}

	dateStr := now.Format(utils.FormatDateUnSpace)
	dateStrFormat := now.Format(utils.FormatDate)
	gzObj := new(models.BaseFromTradeGuangzhouIndex)

	parentId := 3
	classifyList, err := gzObj.GetBaseFromTradeGuangzhouClassifyAll(parentId)
	if err != nil {
		fmt.Println("GetBaseFromTradeGuangzhouClassify Err:" + err.Error())
		return
	}

	classifyMap := make(map[string]*models.BaseFromTradeGuangzhouClassify)
	for _, cv := range classifyList {
		if cv.ClassifyCode != "" {
			classifyMap[cv.ClassifyCode] = cv
		}
	}

	//获取所有指标
	indexList, err := gzObj.GetBaseFromTradeGuangzhouIndexAll()
	if err != nil {
		fmt.Println("GetBaseFromTradeGuangzhouIndexAll Err:" + err.Error())
		return
	}

	indexMap := make(map[string]*models.BaseFromTradeGuangzhouIndex)
	for _, iv := range indexList {
		indexMap[iv.IndexCode] = iv
	}

	dataList, err := gzObj.GetBaseFromTradeGuangzhouDataByDate(dateStrFormat)
	dataMap := make(map[string]*models.BaseFromTradeGuangzhouData)
	for _, dv := range dataList {
		dataMap[dv.IndexCode] = dv
	}

	zzUrl := `http://www.gfex.com.cn/u/interfacesWebTdWbillWeeklyQuotes/loadList`

	params := "gen_date=" + dateStr
	fmt.Println("params:", params)

	headerParams := make(map[string]string)
	headerParams["User-Agent"] = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36"
	headerParams["Content-Type"] = "application/x-www-form-urlencoded;charset=utf-8"
	body, err := utils.HttpPost(zzUrl, params, headerParams)
	if err != nil {
		fmt.Println("err:", err)
		return
	}
	fmt.Println("body :" + string(body))
	utils.FileLog.Info("body:" + string(body))

	wbillQty := new(models.GuangzhouWbillQty)
	err = json.Unmarshal(body, wbillQty)
	if err != nil {
		fmt.Println("CrawlerFromGuangzhouWbillWeeklyQuotes json.Unmarshal Err:" + err.Error())
		return
	}

	if wbillQty.Code == "0" {
		addDataList := make([]*models.BaseFromTradeGuangzhouData, 0)
		updateDataList := make([]*models.BaseFromTradeGuangzhouData, 0)
		for _, v := range wbillQty.Data {
			if v.Variety == "总计" {
				continue
			}
			if !strings.Contains(v.Variety, "小计") {
				indexCode := utils.ChineseToPinyinInitials(v.WhAbbr)
				if indexCode == "" {
					continue
				}
				indexCode = v.VarietyOrder + indexCode
				indexCode += "cdl"
				indexName := v.Variety + v.WhAbbr + "仓单量"

				var indexId int
				if indexItem, ok := indexMap[indexCode]; !ok {
					var classifyId int
					if classifyItem, ok := classifyMap[v.VarietyOrder]; !ok {
						classifyItem = new(models.BaseFromTradeGuangzhouClassify)
						classifyItem.ClassifyCode = v.VarietyOrder
						classifyItem.ClassifyName = v.Variety
						classifyItem.ParentId = 3
						classifyItem.ModifyTime = time.Now()
						classifyItem.CreateTime = time.Now()
						classifyLastId, err := gzObj.AddBaseFromTradeGuangzhouClassify(classifyItem)
						if err != nil {
							fmt.Println("AddBaseFromTradeGuangzhouClassify err:" + err.Error())
							return
						}
						classifyId = int(classifyLastId)
					} else {
						classifyId = classifyItem.BaseFromTradeGuangzhouClassifyId
					}
					gzObj = &models.BaseFromTradeGuangzhouIndex{
						BaseFromTradeGuangzhouClassifyId: classifyId,
						IndexCode:                        indexCode,
						IndexName:                        indexName,
						Frequency:                        "日度",
						Unit:                             "手",
						CreateTime:                       time.Now(),
						ModifyTime:                       time.Now(),
					}
					lastId, err := gzObj.AddBaseFromTradeGuangzhouIndex()
					if err != nil {
						fmt.Println("AddBaseFromTradeGuangzhouIndex Err:" + err.Error())
						return
					}
					indexId = int(lastId)
				} else {
					indexId = indexItem.BaseFromTradeGuangzhouIndexId
				}
				//判断数据是否存在
				if dataItem, ok := dataMap[indexCode]; !ok {
					dataItem = new(models.BaseFromTradeGuangzhouData)
					dataItem.BaseFromTradeGuangzhouIndexId = indexId
					dataItem.IndexCode = indexCode
					dataItem.DataTime = dateStr
					dataItem.Value = float64(v.WbillQty)
					dataItem.QtySub = float64(v.RegWbillQty)
					dataItem.CreateTime = time.Now()
					dataItem.ModifyTime = time.Now()
					addDataList = append(addDataList, dataItem)
				} else {
					if dataItem.Value != float64(v.WbillQty) {
						dataItem.BaseFromTradeGuangzhouDataId = dataItem.BaseFromTradeGuangzhouDataId
						dataItem.BaseFromTradeGuangzhouIndexId = indexId
						dataItem.IndexCode = indexCode
						dataItem.DataTime = dateStr
						dataItem.Value = float64(v.WbillQty)
						dataItem.QtySub = float64(v.RegWbillQty)
						dataItem.ModifyTime = time.Now()
						updateDataList = append(updateDataList, dataItem)
					}
				}
			} else {
				indexCode := utils.ChineseToPinyinInitials("仓单日报_" + v.Variety)
				indexCode = v.VarietyOrder + indexCode
				indexCode += "cdl"
				indexName := "仓单日报_" + v.Variety + "_仓单量"

				var indexId int
				if indexItem, ok := indexMap[indexCode]; !ok {
					var classifyId int
					if classifyItem, ok := classifyMap[v.VarietyOrder]; !ok {
						classifyItem = new(models.BaseFromTradeGuangzhouClassify)
						classifyItem.ClassifyCode = v.VarietyOrder
						classifyItem.ClassifyName = v.Variety
						classifyItem.ParentId = 3
						classifyItem.ModifyTime = time.Now()
						classifyItem.CreateTime = time.Now()
						classifyLastId, err := gzObj.AddBaseFromTradeGuangzhouClassify(classifyItem)
						if err != nil {
							fmt.Println("AddBaseFromTradeGuangzhouClassify err:" + err.Error())
							return
						}
						classifyId = int(classifyLastId)
					} else {
						classifyId = classifyItem.BaseFromTradeGuangzhouClassifyId
					}
					gzObj = &models.BaseFromTradeGuangzhouIndex{
						BaseFromTradeGuangzhouClassifyId: classifyId,
						IndexCode:                        indexCode,
						IndexName:                        indexName,
						Frequency:                        "日度",
						Unit:                             "手",
						CreateTime:                       time.Now(),
						ModifyTime:                       time.Now(),
					}
					lastId, err := gzObj.AddBaseFromTradeGuangzhouIndex()
					if err != nil {
						fmt.Println("AddBaseFromTradeGuangzhouIndex Err:" + err.Error())
						return
					}
					indexId = int(lastId)
				} else {
					indexId = indexItem.BaseFromTradeGuangzhouIndexId
				}
				//判断数据是否存在
				if dataItem, ok := dataMap[indexCode]; !ok {
					dataItem = new(models.BaseFromTradeGuangzhouData)
					dataItem.BaseFromTradeGuangzhouIndexId = indexId
					dataItem.IndexCode = indexCode
					dataItem.DataTime = dateStr
					dataItem.Value = float64(v.WbillQty)
					dataItem.QtySub = float64(v.RegWbillQty)
					dataItem.CreateTime = time.Now()
					dataItem.ModifyTime = time.Now()
					addDataList = append(addDataList, dataItem)
				} else {
					if dataItem.Value != float64(v.WbillQty) {
						dataItem.BaseFromTradeGuangzhouDataId = dataItem.BaseFromTradeGuangzhouDataId
						dataItem.BaseFromTradeGuangzhouIndexId = indexId
						dataItem.IndexCode = indexCode
						dataItem.DataTime = dateStr
						dataItem.Value = float64(v.WbillQty)
						dataItem.QtySub = float64(v.RegWbillQty)
						dataItem.ModifyTime = time.Now()
						updateDataList = append(updateDataList, dataItem)
					}
				}
			}
		}
		fmt.Println("addDataList:", len(addDataList))
		if len(addDataList) > 0 {
			err = gzObj.AddBaseFromTradeGuangzhouData(addDataList)
			if err != nil {
				fmt.Println("AddBaseFromTradeGuangzhouData Err:" + err.Error())
				return
			}
		}

		if len(updateDataList) > 0 {
			for _, v := range updateDataList {
				updateParams := make(map[string]interface{})
				whereParam := make(map[string]interface{})
				updateParams["value"] = v.Value
				updateParams["qty_sub"] = v.QtySub
				updateParams["modify_time"] = time.Now()
				whereParam["base_from_trade_guangzhou_data_id"] = v.BaseFromTradeGuangzhouDataId
				err = gzObj.UpdateGuangzhouData(updateParams, whereParam)
				if err != nil {
					fmt.Println("UpdateGuangzhouData Err:" + err.Error())
					return
				}
			}
		}
	}
}

// CrawlerFromGuangzhouCongregateDeliveryQuotes 广州交易所期货-一次性交割卖方仓单查询
func CrawlerFromGuangzhouCongregateDeliveryQuotes(date time.Time) {
	now := date //time.Now()

	dateStr := now.Format(utils.FormatMonthDateUnSpace)
	_, monthEndDay := utils.GetMonthStartAndEnd(strconv.Itoa(now.Year()), strconv.Itoa(int(now.Month())))

	gzObj := new(models.BaseFromTradeGuangzhouIndex)

	parentId := 4
	classifyList, err := gzObj.GetBaseFromTradeGuangzhouClassifyAll(parentId)
	if err != nil {
		fmt.Println("GetBaseFromTradeGuangzhouClassify Err:" + err.Error())
		return
	}

	classifyMap := make(map[string]*models.BaseFromTradeGuangzhouClassify)
	for _, cv := range classifyList {
		if cv.ClassifyCode != "" {
			classifyMap[cv.ClassifyCode] = cv
		}
	}

	//获取所有指标
	indexList, err := gzObj.GetBaseFromTradeGuangzhouIndexAll()
	if err != nil {
		fmt.Println("GetBaseFromTradeGuangzhouIndexAll Err:" + err.Error())
		return
	}

	indexMap := make(map[string]*models.BaseFromTradeGuangzhouIndex)
	for _, iv := range indexList {
		indexMap[iv.IndexCode] = iv
	}

	dataList, err := gzObj.GetBaseFromTradeGuangzhouDataByDate(monthEndDay)
	dataMap := make(map[string]*models.BaseFromTradeGuangzhouData)
	for _, dv := range dataList {
		dataMap[dv.IndexCode] = dv
	}

	zzUrl := `http://www.gfex.com.cn/u/interfacesWebTcCongregateDeliveryQuotes/loadList`

	params := "contract_month=" + dateStr
	fmt.Println("params:", params)

	headerParams := make(map[string]string)
	headerParams["User-Agent"] = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36"
	headerParams["Content-Type"] = "application/x-www-form-urlencoded;charset=utf-8"
	body, err := utils.HttpPost(zzUrl, params, headerParams)
	if err != nil {
		fmt.Println("err:", err)
		return
	}
	fmt.Println("body :" + string(body))
	utils.FileLog.Info("body:" + string(body))

	deliveryQuotes := new(models.GuangzhouCongregateDeliveryQuotes)
	err = json.Unmarshal(body, deliveryQuotes)
	if err != nil {
		fmt.Println("CrawlerFromGuangzhouCongregateDeliveryQuotes json.Unmarshal Err:" + err.Error())
		return
	}

	if deliveryQuotes.Code == "0" {
		addDataList := make([]*models.BaseFromTradeGuangzhouData, 0)
		updateDataList := make([]*models.BaseFromTradeGuangzhouData, 0)

		if len(deliveryQuotes.Data) <= 0 {
			return
		}

		varietyId := deliveryQuotes.Data[0].VarietyId
		varietyName := deliveryQuotes.Data[0].VarietyName

		totalIndexCode := varietyId + utils.ChineseToPinyinInitials("一次性交割卖方仓单合计")
		totalIndexName := varietyName + "一次性交割卖方仓单合计"

		total := 0
		for _, v := range deliveryQuotes.Data {
			total += v.WbillNum
		}

		var indexId int
		if indexItem, ok := indexMap[totalIndexCode]; !ok {
			var classifyId int
			if classifyItem, ok := classifyMap[varietyId]; !ok {
				classifyItem = new(models.BaseFromTradeGuangzhouClassify)
				classifyItem.ClassifyCode = varietyId
				classifyItem.ClassifyName = varietyName
				classifyItem.ParentId = parentId
				classifyItem.ModifyTime = time.Now()
				classifyItem.CreateTime = time.Now()
				classifyLastId, err := gzObj.AddBaseFromTradeGuangzhouClassify(classifyItem)
				if err != nil {
					fmt.Println("AddBaseFromTradeGuangzhouClassify err:" + err.Error())
					return
				}
				classifyId = int(classifyLastId)
			} else {
				classifyId = classifyItem.BaseFromTradeGuangzhouClassifyId
			}

			gzObj = &models.BaseFromTradeGuangzhouIndex{
				BaseFromTradeGuangzhouClassifyId: classifyId,
				IndexCode:                        totalIndexCode,
				IndexName:                        totalIndexName,
				Frequency:                        "月度",
				Unit:                             "手",
				CreateTime:                       time.Now(),
				ModifyTime:                       time.Now(),
			}
			lastId, err := gzObj.AddBaseFromTradeGuangzhouIndex()
			if err != nil {
				fmt.Println("AddBaseFromTradeGuangzhouIndex Err:" + err.Error())
				return
			}
			indexId = int(lastId)
		} else {
			indexId = indexItem.BaseFromTradeGuangzhouIndexId
		}

		//判断数据是否存在
		if dataItem, ok := dataMap[totalIndexCode]; !ok {
			dataItem = new(models.BaseFromTradeGuangzhouData)
			dataItem.BaseFromTradeGuangzhouIndexId = indexId
			dataItem.IndexCode = totalIndexCode
			dataItem.DataTime = monthEndDay
			dataItem.Value = float64(total)
			dataItem.CreateTime = time.Now()
			dataItem.ModifyTime = time.Now()
			addDataList = append(addDataList, dataItem)
		} else {
			if dataItem.Value != float64(total) {
				dataItem.BaseFromTradeGuangzhouDataId = dataItem.BaseFromTradeGuangzhouDataId
				dataItem.BaseFromTradeGuangzhouIndexId = indexId
				dataItem.IndexCode = totalIndexCode
				dataItem.DataTime = monthEndDay
				dataItem.Value = float64(total)
				dataItem.ModifyTime = time.Now()
				updateDataList = append(updateDataList, dataItem)
			}
		}
		fmt.Println("addDataList:", len(addDataList))
		if len(addDataList) > 0 {
			err = gzObj.AddBaseFromTradeGuangzhouData(addDataList)
			if err != nil {
				fmt.Println("AddBaseFromTradeGuangzhouData Err:" + err.Error())
				return
			}
		}

		if len(updateDataList) > 0 {
			for _, v := range updateDataList {
				updateParams := make(map[string]interface{})
				whereParam := make(map[string]interface{})
				updateParams["value"] = v.Value
				updateParams["qty_sub"] = v.QtySub
				updateParams["modify_time"] = time.Now()
				whereParam["base_from_trade_guangzhou_data_id"] = v.BaseFromTradeGuangzhouDataId
				err = gzObj.UpdateGuangzhouData(updateParams, whereParam)
				if err != nil {
					fmt.Println("UpdateGuangzhouData Err:" + err.Error())
					return
				}
			}
		}

		//处理明细
		addDataList = make([]*models.BaseFromTradeGuangzhouData, 0)
		updateDataList = make([]*models.BaseFromTradeGuangzhouData, 0)

		for _, v := range deliveryQuotes.Data {
			var subIndexId int
			indexCode = v.ContractId + utils.ChineseToPinyinInitials(v.WhName) + "cdl"
			if indexItem, ok := indexMap[indexCode]; !ok {
				gzObj = &models.BaseFromTradeGuangzhouIndex{
					BaseFromTradeGuangzhouClassifyId: indexId,
					IndexCode:                        indexCode,
					IndexName:                        v.WhName,
					Frequency:                        "月度",
					Unit:                             "手",
					CreateTime:                       time.Now(),
					ModifyTime:                       time.Now(),
					Value:                            float64(v.WbillNum),
					StartDate:                        monthEndDay,
					EndDate:                          monthEndDay,
				}
				lastId, err := gzObj.AddBaseFromTradeGuangzhouIndex()
				if err != nil {
					fmt.Println("AddBaseFromTradeGuangzhouIndex Err:" + err.Error())
					return
				}
				subIndexId = int(lastId)
			} else {
				subIndexId = indexItem.BaseFromTradeGuangzhouIndexId
			}
			//判断数据是否存在
			if dataItem, ok := dataMap[indexCode]; !ok {
				dataItem = new(models.BaseFromTradeGuangzhouData)
				dataItem.BaseFromTradeGuangzhouIndexId = subIndexId
				dataItem.IndexCode = indexCode
				dataItem.DataTime = monthEndDay
				dataItem.Value = float64(total)
				dataItem.CreateTime = time.Now()
				dataItem.ModifyTime = time.Now()
				addDataList = append(addDataList, dataItem)
			} else {
				if dataItem.Value != float64(total) {
					dataItem.BaseFromTradeGuangzhouDataId = dataItem.BaseFromTradeGuangzhouDataId
					dataItem.BaseFromTradeGuangzhouIndexId = subIndexId
					dataItem.IndexCode = indexCode
					dataItem.DataTime = monthEndDay
					dataItem.Value = float64(total)
					dataItem.ModifyTime = time.Now()
					updateDataList = append(updateDataList, dataItem)
				}
			}
		}

		if len(addDataList) > 0 {
			err = gzObj.AddBaseFromTradeGuangzhouData(addDataList)
			if err != nil {
				fmt.Println("AddBaseFromTradeGuangzhouData Err:" + err.Error())
				return
			}
		}

		if len(updateDataList) > 0 {
			for _, v := range updateDataList {
				updateParams := make(map[string]interface{})
				whereParam := make(map[string]interface{})
				updateParams["value"] = v.Value
				updateParams["qty_sub"] = v.QtySub
				updateParams["modify_time"] = time.Now()
				whereParam["base_from_trade_guangzhou_data_id"] = v.BaseFromTradeGuangzhouDataId
				err = gzObj.UpdateGuangzhouData(updateParams, whereParam)
				if err != nil {
					fmt.Println("UpdateGuangzhouData Err:" + err.Error())
					return
				}
			}
		}
	}
}

// CrawlerFromGuangzhouIndexMinAndMaxDate 广州交易所期货-处理指标开始日期和结束日期
func CrawlerFromGuangzhouIndexMinAndMaxDate() {
	gzObj := new(models.BaseFromTradeGuangzhouIndex)
	//获取所有指标
	indexList, err := gzObj.GetBaseFromTradeGuangzhouIndexAll()
	if err != nil {
		fmt.Println("GetBaseFromTradeGuangzhouIndexAll Err:" + err.Error())
		return
	}

	indexMap := make(map[int]*models.BaseFromTradeGuangzhouIndex)
	for _, iv := range indexList {
		indexMap[iv.BaseFromTradeGuangzhouIndexId] = iv
	}
	//获取数据开始和结束日期
	dataDateList, err := gzObj.GetBaseFromTradeGuangzhouDataMinAndMaxDate()
	if err != nil {
		fmt.Println("GetBaseFromTradeGuangzhouDataMinAndMaxDate Err:" + err.Error())
		return
	}
	for _, v := range dataDateList {
		if indexItem, ok := indexMap[v.BaseFromTradeGuangzhouIndexId]; ok {
			updateParams := make(map[string]interface{})
			whereParam := make(map[string]interface{})
			if indexItem.StartDate != v.StartDate {
				updateParams["start_date"] = v.StartDate
			}
			if indexItem.EndDate != v.EndDate {
				updateParams["end_date"] = v.EndDate
				updateParams["modify_time"] = time.Now()
				whereParam["base_from_trade_guangzhou_index_id"] = v.BaseFromTradeGuangzhouIndexId
				err = gzObj.UpdateGuangzhouIndex(updateParams, whereParam)
				if err != nil {
					fmt.Println("UpdateGuangzhouData Err:" + err.Error())
					return
				}
			}
		}
	}
}

//http://www.gfex.com.cn/u/interfacesWebTiMemberDealPosiQuotes/loadList