eta_index_lib/controllers/sci99_crawler.go

package controllers

import (
	"encoding/json"
	"eta/eta_index_lib/models"
	"eta/eta_index_lib/services/sci99"
	"eta/eta_index_lib/utils"
	"fmt"
	"github.com/mozillazg/go-pinyin"
	"strings"
	"time"
)

// 卓创资讯爬虫
type Sci99Crawler struct {
	BaseAuthController
}

var IndexCodeMap = make(map[string]string)
var IndexMap = make(map[string]*models.BaseFromSci99Index)
var ClassifyMap = make(map[string]*models.BaseFromSci99Classify)

// @Title 刷新数据
// @Description 刷新数据接口
// @Param	request	body models.AddEdbClassifyReq true "type json string"
// @Success 200 {object} models.EdbClassify
// @router /refresh/list [post]
func (this *Sci99Crawler) RefreshSci99Classify() {
	br := new(models.BaseResponse).Init()
	defer func() {
		this.Data["json"] = br
		this.ServeJSON()
	}()
	var req sci99.RefreshSci99CrawlerReq
	err := json.Unmarshal(this.Ctx.Input.RequestBody, &req)
	if err != nil {
		br.Msg = "参数解析异常！"
		br.ErrMsg = "参数解析失败，Err:" + err.Error()
		return
	}
	allClassify, err := models.GetBaseFromSci99Classify()
	if err != nil {
		fmt.Println("select Code err:", err)
		utils.FileLog.Info("GetBaseFromSci99Index err:", err)
		return
	}
	for _, item := range allClassify {
		ClassifyMap[item.ClassifyName] = item
	}

	if _, ok := ClassifyMap[req.ClassifyName]; !ok {
		classifyItem := models.BaseFromSci99Classify{
			ClassifyName: req.ClassifyName,
			CreateTime:   time.Now(),
			ModifyTime:   time.Now(),
		}

		id, e := models.AddBaseFromSci99Classify(&classifyItem)
		if e != nil {
			err = e
			fmt.Println("Error inserting into database:", err)
			utils.FileLog.Info("Error inserting into database:", err)
			return
		}
		fmt.Println("新增分类：", classifyItem.ClassifyName)
		classifyItem.BaseFromSciClassifyId = int(id)
		ClassifyMap[req.ClassifyName] = &classifyItem
	}

	allCode, e := models.GetBaseFromSci99Index()
	if e != nil {
		err = e
		fmt.Println("select Code err:", err)
		utils.FileLog.Info("GetBaseFromSci99Index err:", err)
		return
	}

	for _, item := range allCode {
		IndexCodeMap[item.IndexName] = item.IndexCode
		IndexMap[item.IndexName] = item
	}

	v := req.ListData
	// 组成指标
	indexName := req.IndexName
	fmt.Println("indexName:", indexName)
	indexCodeStr := "卓创资讯" + v.ProductName + v.PriceType
	indexCode, needAdd := Sci99IndexCodeGenerator(indexName, indexCodeStr, v.MarketSampleName, v.FactorySampleName+v.Model)
	if needAdd {
		index := models.BaseFromSci99Index{
			IndexCode:  indexCode,
			IndexName:  indexName,
			ClassifyId: ClassifyMap[req.ClassifyName].BaseFromSciClassifyId,
			Unit:       v.Unit,
			Frequency:  "日度",
			Describe:   v.DataItemName,
			CreateTime: time.Now(),
			ModifyTime: time.Now(),
		}
		id, e := models.AddBaseFromSci99Index(&index)
		if e != nil {
			err = e
			fmt.Println("Error inserting into database:", err)
			utils.FileLog.Info("Error inserting into database:", err)
			return
		}
		fmt.Println("新增指标：", index.IndexName)
		index.BaseFromSciIndexId = int(id)
		IndexMap[indexName] = &index
	}

	detailResponse := req.DetailData

	existDataMap := make(map[string]*models.BaseFromSci99Data)

	sic99Info, ok := IndexMap[indexName]
	if !ok {
		utils.FileLog.Info("指标异常，通过名称找不到指标")
		return
	}

	//获取所有指标信息
	allData, e := models.GetBaseFromTradeSci99DataAll(indexCode)
	if e != nil {
		err = e
		fmt.Println("select err:", err)
		utils.FileLog.Info("GetBaseFromTradeSci99IndexAll err:", err)
	}
	for _, vv := range allData {
		indexKey := vv.IndexCode + vv.DataTime
		existDataMap[indexKey] = vv
	}

	// 新增data数据
	addList := make([]*models.BaseFromSci99Data, 0)
	for _, v := range detailResponse.Data.List {
		if dataItem, ok := existDataMap[indexCode+v.RealDate]; ok {
			if dataItem.Value != v.MDataValue {
				// 更新
				fmt.Println("更新指标：", indexCode+v.RealDate)
				e = models.UpdateBaseFromSci99Data(dataItem.Value, indexCode, v.RealDate)
				if e != nil {
					err = e
					fmt.Println("Error update into database:", err)
					utils.FileLog.Info("Error update into database:", err)
					return
				}
			}
		} else {
			// 新增
			dataItem := models.BaseFromSci99Data{
				BaseFromSciIndexId: sic99Info.BaseFromSciIndexId,
				IndexCode:          indexCode,
				DataTime:           v.RealDate,
				Value:              v.MDataValue,
				CreateTime:         time.Now(),
				ModifyTime:         time.Now(),
			}
			addList = append(addList, &dataItem)
			fmt.Println("新增数据：", indexCode+v.RealDate)
		}
	}
	if len(addList) > 0 {
		e = models.AddBaseFromSci99DataMulti(addList)
		if e != nil {
			err = e
			fmt.Println("Error inserting into database:", err)
			utils.FileLog.Info("Error inserting into database:", err)
			return
		}
	}

	// 更新指标最新日期
	{
		go func() {
			var dateItem *models.EdbInfoMaxAndMinInfo
			dateItem, err = sic99Info.GetMaxAndMinDateByIndexCode(indexCode)
			if err != nil {
				err = fmt.Errorf("查询指标最新日期失败 Err：%s", err)
				return
			}
			sic99Info.ModifyIndexMaxAndMinDate(indexCode, dateItem)
		}()
	}

	time.Sleep(1 * time.Second)

	br.Ret = 200
	br.Msg = "获取成功"
	br.Success = true
}

func Sci99IndexCodeGenerator(indexName, indexCodeStr, marketSampleName, model string) (indexCode string, needAdd bool) {
	strResult := ""
	indexCode, _ = IndexCodeMap[indexName]
	if indexCode == "" {
		//首字母
		a := pinyin.NewArgs()
		a.Fallback = func(r rune, a pinyin.Args) []string {
			return []string{string(r)}
		}
		rows := pinyin.Pinyin(indexCodeStr, a)
		for i := 0; i < len(rows); i++ {
			//strResult += rows[i][0]
			if len(rows[i]) != 0 {
				str := rows[i][0]
				pi := str[0:1]
				strResult += pi
			}
		}

		// 处理市场名称
		if province, ok := ProvinceMap[marketSampleName]; ok {
			strResult += province
		} else {
			a := pinyin.NewArgs()
			rows := pinyin.LazyPinyin(marketSampleName, a)
			for i := 0; i < len(rows); i++ {
				strResult += rows[i]
			}
		}

		// 去除特殊符号
		model = strings.Replace(model, " ", "", -1)
		model = strings.Replace(model, "-", "", -1)
		model = strings.Replace(model, "/", "", -1)
		model = strings.Replace(model, "#", "", -1)
		model = strings.Replace(model, "：", "", -1)
		model = strings.Replace(model, "(", "", -1)
		model = strings.Replace(model, ")", "", -1)

		// 拼接型号
		modelRows := pinyin.Pinyin(model, a)
		for i := 0; i < len(modelRows); i++ {
			if len(modelRows[i]) != 0 {
				str := modelRows[i][0]
				pi := str[0:1]
				strResult += pi
			}
		}

		needAdd = true
		indexCode = strings.Replace(strResult, " ", "", -1)
		IndexCodeMap[indexName] = indexCode
	}
	return
}

var ProvinceMap = map[string]string{
	"上海":  "shanghai",
	"云南":  "yunnan",
	"内蒙古": "innermongolia",
	"北京":  "beijing",
	"台湾":  "taiwan",
	"吉林":  "jilin",
	"四川":  "sichuan",
	"天津":  "tianjin",
	"宁夏":  "ningxia",
	"安徽":  "anhui",
	"山东":  "shandong",
	"山西":  "shanxi",
	"广东":  "guangdong",
	"广西":  "guangxi",
	"新疆":  "xinjiang",
	"江苏":  "jiangsu",
	"江西":  "jiangxi",
	"河北":  "hebei",
	"河南":  "henan",
	"浙江":  "zhejiang",
	"海南":  "hainan",
	"湖北":  "hubei",
	"湖南":  "hunan",
	"澳门":  "macao",
	"甘肃":  "gansu",
	"福建":  "fujian",
	"西藏":  "tibet",
	"贵州":  "guizhou",
	"辽宁":  "liaoning",
	"重庆":  "chongqing",
	"陕西":  "shaanxi",
	"青海":  "qinhai",
	"香港":  "hongkong",
	"黑龙江": "heilongjiang",
}

var requestList = []sci99.ListRequestBody{
	{
		Ppname: "天然橡胶原料",
		PPIDs:  "13676",
		Navid:  "593",
	}, {
		Ppname: "混合胶",
		PPIDs:  "12973",
		Navid:  "591",
	}, {
		Ppname: "标准胶",
		PPIDs:  "12960",
		Navid:  "590",
	}, {
		Ppname: "烟片胶",
		PPIDs:  "12985",
		Navid:  "592",
	}, {
		Ppname: "天然乳胶",
		PPIDs:  "12947",
		Navid:  "595",
	}, {
		Ppname: "丁苯橡胶",
		PPIDs:  "12951",
		Navid:  "596",
	}, {
		Ppname: "顺丁橡胶",
		PPIDs:  "12964",
		Navid:  "597",
	}, {
		Ppname: "SBS",
		PPIDs:  "12948",
		Navid:  "598",
	}, {
		Ppname: "丁腈橡胶",
		PPIDs:  "12945",
		Navid:  "605",
	},
}