fix

controllers/sci99_crawler.go

@@ -0,0 +1,304 @@
+package controllers
+
+import (
+	"encoding/json"
+	"eta/eta_index_lib/models"
+	"eta/eta_index_lib/services/sci99"
+	"eta/eta_index_lib/utils"
+	"fmt"
+	"github.com/mozillazg/go-pinyin"
+	"strings"
+	"time"
+)
+
+// 卓创资讯爬虫
+type Sci99Crawler struct {
+	BaseAuthController
+}
+
+var IndexCodeMap = make(map[string]string)
+var IndexMap = make(map[string]*models.BaseFromSci99Index)
+var ClassifyMap = make(map[string]*models.BaseFromSci99Classify)
+
+// @Title 刷新数据
+// @Description 刷新数据接口
+// @Param	request	body models.AddEdbClassifyReq true "type json string"
+// @Success 200 {object} models.EdbClassify
+// @router /refresh/list [post]
+func (this *Sci99Crawler) RefreshSci99Classify() {
+	br := new(models.BaseResponse).Init()
+	defer func() {
+		this.Data["json"] = br
+		this.ServeJSON()
+	}()
+	var req sci99.RefreshSci99CrawlerReq
+	err := json.Unmarshal(this.Ctx.Input.RequestBody, &req)
+	if err != nil {
+		br.Msg = "参数解析异常！"
+		br.ErrMsg = "参数解析失败，Err:" + err.Error()
+		return
+	}
+	allClassify, err := models.GetBaseFromSci99Classify()
+	if err != nil {
+		fmt.Println("select Code err:", err)
+		utils.FileLog.Info("GetBaseFromSci99Index err:", err)
+		return
+	}
+	for _, item := range allClassify {
+		ClassifyMap[item.ClassifyName] = item
+	}
+
+	if _, ok := ClassifyMap[req.ClassifyName]; !ok {
+		classifyItem := models.BaseFromSci99Classify{
+			ClassifyName: req.ClassifyName,
+			CreateTime:   time.Now(),
+			ModifyTime:   time.Now(),
+		}
+
+		id, e := models.AddBaseFromSci99Classify(&classifyItem)
+		if e != nil {
+			err = e
+			fmt.Println("Error inserting into database:", err)
+			utils.FileLog.Info("Error inserting into database:", err)
+			return
+		}
+		fmt.Println("新增分类：", classifyItem.ClassifyName)
+		classifyItem.BaseFromSciClassifyId = int(id)
+		ClassifyMap[req.ClassifyName] = &classifyItem
+	}
+
+	allCode, e := models.GetBaseFromSci99Index()
+	if e != nil {
+		err = e
+		fmt.Println("select Code err:", err)
+		utils.FileLog.Info("GetBaseFromSci99Index err:", err)
+		return
+	}
+
+	for _, item := range allCode {
+		IndexCodeMap[item.IndexName] = item.IndexCode
+		IndexMap[item.IndexName] = item
+	}
+
+	v := req.ListData
+	// 组成指标
+	indexName := req.IndexName
+	fmt.Println("indexName:", indexName)
+	indexCodeStr := "卓创资讯" + v.ProductName + v.PriceType
+	indexCode, needAdd := Sci99IndexCodeGenerator(indexName, indexCodeStr, v.MarketSampleName, v.FactorySampleName+v.Model)
+	if needAdd {
+		index := models.BaseFromSci99Index{
+			IndexCode:  indexCode,
+			IndexName:  indexName,
+			ClassifyId: ClassifyMap[req.ClassifyName].BaseFromSciClassifyId,
+			Unit:       v.Unit,
+			Frequency:  "日度",
+			Describe:   v.DataItemName,
+			CreateTime: time.Now(),
+			ModifyTime: time.Now(),
+		}
+		id, e := models.AddBaseFromSci99Index(&index)
+		if e != nil {
+			err = e
+			fmt.Println("Error inserting into database:", err)
+			utils.FileLog.Info("Error inserting into database:", err)
+			return
+		}
+		fmt.Println("新增指标：", index.IndexName)
+		index.BaseFromSciIndexId = int(id)
+		IndexMap[indexName] = &index
+	}
+
+	detailResponse := req.DetailData
+
+	existDataMap := make(map[string]*models.BaseFromSci99Data)
+
+	//获取所有指标信息
+	allData, e := models.GetBaseFromTradeSci99DataAll(indexCode)
+	if e != nil {
+		err = e
+		fmt.Println("select err:", err)
+		utils.FileLog.Info("GetBaseFromTradeSci99IndexAll err:", err)
+	}
+	for _, vv := range allData {
+		indexKey := vv.IndexCode + vv.DataTime
+		existDataMap[indexKey] = vv
+	}
+
+	// 新增data数据
+	addList := make([]*models.BaseFromSci99Data, 0)
+	for _, v := range detailResponse.Data.List {
+		if dataItem, ok := existDataMap[indexCode+v.RealDate]; ok {
+			if dataItem.Value != v.MDataValue {
+				// 更新
+				fmt.Println("更新指标：", indexCode+v.RealDate)
+				e = models.UpdateBaseFromSci99Data(dataItem.Value, indexCode, v.RealDate)
+				if e != nil {
+					err = e
+					fmt.Println("Error update into database:", err)
+					utils.FileLog.Info("Error update into database:", err)
+					return
+				}
+			}
+		} else {
+			// 新增
+			dataItem := models.BaseFromSci99Data{
+				BaseFromSciIndexId: IndexMap[indexName].BaseFromSciIndexId,
+				IndexCode:          indexCode,
+				DataTime:           v.RealDate,
+				Value:              v.MDataValue,
+				CreateTime:         time.Now(),
+				ModifyTime:         time.Now(),
+			}
+			addList = append(addList, &dataItem)
+			fmt.Println("新增数据：", indexCode+v.RealDate)
+		}
+	}
+	if len(addList) > 0 {
+		e = models.AddBaseFromSci99DataMulti(addList)
+		if e != nil {
+			err = e
+			fmt.Println("Error inserting into database:", err)
+			utils.FileLog.Info("Error inserting into database:", err)
+			return
+		}
+	}
+	time.Sleep(1 * time.Second)
+
+	br.Ret = 200
+	br.Msg = "获取成功"
+	br.Success = true
+}
+
+func Sci99IndexCodeGenerator(indexName, indexCodeStr, marketSampleName, model string) (indexCode string, needAdd bool) {
+	strResult := ""
+	indexCode, _ = IndexCodeMap[indexName]
+	if indexCode == "" {
+		//首字母
+		a := pinyin.NewArgs()
+		a.Fallback = func(r rune, a pinyin.Args) []string {
+			return []string{string(r)}
+		}
+		rows := pinyin.Pinyin(indexCodeStr, a)
+		for i := 0; i < len(rows); i++ {
+			//strResult += rows[i][0]
+			if len(rows[i]) != 0 {
+				str := rows[i][0]
+				pi := str[0:1]
+				strResult += pi
+			}
+		}
+
+		// 处理市场名称
+		if province, ok := ProvinceMap[marketSampleName]; ok {
+			strResult += province
+		} else {
+			a := pinyin.NewArgs()
+			rows := pinyin.LazyPinyin(marketSampleName, a)
+			for i := 0; i < len(rows); i++ {
+				strResult += rows[i]
+			}
+		}
+
+		// 去除特殊符号
+		model = strings.Replace(model, " ", "", -1)
+		model = strings.Replace(model, "-", "", -1)
+		model = strings.Replace(model, "/", "", -1)
+		model = strings.Replace(model, "#", "", -1)
+		model = strings.Replace(model, "：", "", -1)
+		model = strings.Replace(model, "(", "", -1)
+		model = strings.Replace(model, ")", "", -1)
+
+		// 拼接型号
+		modelRows := pinyin.Pinyin(model, a)
+		for i := 0; i < len(modelRows); i++ {
+			if len(modelRows[i]) != 0 {
+				str := modelRows[i][0]
+				pi := str[0:1]
+				strResult += pi
+			}
+		}
+
+		needAdd = true
+		indexCode = strings.Replace(strResult, " ", "", -1)
+		IndexCodeMap[indexName] = indexCode
+	}
+	return
+}
+
+var ProvinceMap = map[string]string{
+	"上海":  "shanghai",
+	"云南":  "yunnan",
+	"内蒙古": "innermongolia",
+	"北京":  "beijing",
+	"台湾":  "taiwan",
+	"吉林":  "jilin",
+	"四川":  "sichuan",
+	"天津":  "tianjin",
+	"宁夏":  "ningxia",
+	"安徽":  "anhui",
+	"山东":  "shandong",
+	"山西":  "shanxi",
+	"广东":  "guangdong",
+	"广西":  "guangxi",
+	"新疆":  "xinjiang",
+	"江苏":  "jiangsu",
+	"江西":  "jiangxi",
+	"河北":  "hebei",
+	"河南":  "henan",
+	"浙江":  "zhejiang",
+	"海南":  "hainan",
+	"湖北":  "hubei",
+	"湖南":  "hunan",
+	"澳门":  "macao",
+	"甘肃":  "gansu",
+	"福建":  "fujian",
+	"西藏":  "tibet",
+	"贵州":  "guizhou",
+	"辽宁":  "liaoning",
+	"重庆":  "chongqing",
+	"陕西":  "shaanxi",
+	"青海":  "qinhai",
+	"香港":  "hongkong",
+	"黑龙江": "heilongjiang",
+}
+
+var requestList = []sci99.ListRequestBody{
+	{
+		Ppname: "天然橡胶原料",
+		PPIDs:  "13676",
+		Navid:  "593",
+	}, {
+		Ppname: "混合胶",
+		PPIDs:  "12973",
+		Navid:  "591",
+	}, {
+		Ppname: "标准胶",
+		PPIDs:  "12960",
+		Navid:  "590",
+	}, {
+		Ppname: "烟片胶",
+		PPIDs:  "12985",
+		Navid:  "592",
+	}, {
+		Ppname: "天然乳胶",
+		PPIDs:  "12947",
+		Navid:  "595",
+	}, {
+		Ppname: "丁苯橡胶",
+		PPIDs:  "12951",
+		Navid:  "596",
+	}, {
+		Ppname: "顺丁橡胶",
+		PPIDs:  "12964",
+		Navid:  "597",
+	}, {
+		Ppname: "SBS",
+		PPIDs:  "12948",
+		Navid:  "598",
+	}, {
+		Ppname: "丁腈橡胶",
+		PPIDs:  "12945",
+		Navid:  "605",
+	},
+}

models/base_from_sci99.go

@@ -23,7 +23,7 @@ type BaseFromSci99Index struct {
 }
 
 // BaseFromSci99Data 代表卓创资讯-原始指标数据表的结构
-type BaseFromSci99Data struct {
+type BaseFromSci99DataItem struct {
 	BaseFromSciDataId  int       `orm:"column(base_from_sci_data_id);pk"` // 主键，自动递增
 	BaseFromSciIndexId int       // 指标id
 	IndexCode          string    // 指标编码
@@ -33,6 +33,17 @@ type BaseFromSci99Data struct {
 	ModifyTime         time.Time // 修改时间
 }
 
+// BaseFromSci99Data 代表卓创资讯-原始指标数据表的结构
+type BaseFromSci99Data struct {
+	BaseFromSciDataId  int       `orm:"column(base_from_sci_data_id);pk"` // 主键，自动递增
+	BaseFromSciIndexId int       // 指标id
+	IndexCode          string    // 指标编码
+	DataTime           string    // 数据日期
+	Value              float64    // 数据值
+	CreateTime         time.Time // 创建时间
+	ModifyTime         time.Time // 修改时间
+}
+
 // BaseFromSci99Classify 代表卓创资讯-原始指标分类表的结构
 type BaseFromSci99Classify struct {
 	BaseFromSciClassifyId int       `orm:"column(base_from_sci_classify_id);pk"` // 主键，自动递增
@@ -55,31 +66,31 @@ func GetBaseFromSci99DataDataByCondition(condition string, pars []interface{}) (
 
 // 添加数据
 func AddBaseFromSci99Index(item *BaseFromSci99Index) (lastId int64, err error) {
-	o := orm.NewOrmUsingDB("data")
+	o := orm.NewOrm()
 	lastId, err = o.Insert(item)
 	return
 }
 
 func AddBaseFromSci99Classify(item *BaseFromSci99Classify) (lastId int64, err error) {
-	o := orm.NewOrmUsingDB("data")
+	o := orm.NewOrm()
 	lastId, err = o.Insert(item)
 	return
 }
 
 func AddBaseFromSci99DataMulti(item []*BaseFromSci99Data) (err error) {
-	o := orm.NewOrmUsingDB("data")
+	o := orm.NewOrm()
 	_, err = o.InsertMulti(1000, item)
 	return
 }
 
 func GetBaseFromSci99Index() (list []*BaseFromSci99Index, err error) {
-	o := orm.NewOrmUsingDB("data")
+	o := orm.NewOrm()
 	sql := `SELECT * FROM base_from_sci99_index `
 	_, err = o.Raw(sql).QueryRows(&list)
 	return
 }
 
-func GetBaseFromSci99DataByIndexCode(indexCode string) (items []*BaseFromSci99Data, err error) {
+func GetBaseFromSci99DataByIndexCode(indexCode string) (items []*BaseFromSci99DataItem, err error) {
 	o := orm.NewOrm()
 	sql := `SELECT * FROM base_from_sci99_data WHERE index_code=? `
 	_, err = o.Raw(sql, indexCode).QueryRows(&items)
@@ -240,3 +251,31 @@ func RefreshEdbDataFromSci99(edbInfoId int, edbCode, startDate string) (err erro
 	}
 	return
 }
+
+func GetBaseFromSci99Classify() (list []*BaseFromSci99Classify, err error) {
+	o := orm.NewOrm()
+	sql := `SELECT * FROM base_from_sci99_classify `
+	_, err = o.Raw(sql).QueryRows(&list)
+	return
+}
+
+func GetBaseFromTradeSci99DataAll(indexCode string) (list []*BaseFromSci99Data, err error) {
+	o := orm.NewOrm()
+	sql := `SELECT * FROM base_from_sci99_data where index_code=?`
+	_, err = o.Raw(sql, indexCode).QueryRows(&list)
+	return
+}
+func GetBaseFromTradeSci99LatestData(indexCode string) (date time.Time, err error) {
+	o := orm.NewOrm()
+	sql := `SELECT data_time FROM base_from_sci99_data where index_code=? ORDER BY data_time DESC limit 1 `
+	err = o.Raw(sql, indexCode).QueryRow(&date)
+	return
+}
+
+// UpdateBaseFromSci99Data
+func UpdateBaseFromSci99Data(value float64, indexCode, dataTime string) (err error) {
+	o := orm.NewOrm()
+	sql := `UPDATE base_from_sci99_data SET value=?,modify_time=NOW() WHERE index_code = ? AND data_time = ? `
+	_, err = o.Raw(sql, value, indexCode, dataTime).Exec()
+	return
+}

models/db.go

@@ -134,6 +134,9 @@ func initBaseIndex() {
 		new(BaseFromFenweiData),
 		new(BaseFromBloombergIndex),
 		new(BaseFromBloombergData),
+		new(BaseFromSci99Data),
+		new(BaseFromSci99Index),
+		new(BaseFromSci99Classify),
 	)
 }
 

routers/commentsRouter.go

@@ -1141,6 +1141,15 @@ func init() {
             Filters: nil,
             Params: nil})
 
+    beego.GlobalControllerRouter["eta/eta_index_lib/controllers:Sci99Crawler"] = append(beego.GlobalControllerRouter["eta/eta_index_lib/controllers:Sci99Crawler"],
+        beego.ControllerComments{
+            Method: "RefreshSci99Classify",
+            Router: `/refresh/list`,
+            AllowHTTPMethods: []string{"post"},
+            MethodParams: param.Make(),
+            Filters: nil,
+            Params: nil})
+
     beego.GlobalControllerRouter["eta/eta_index_lib/controllers:SciController"] = append(beego.GlobalControllerRouter["eta/eta_index_lib/controllers:SciController"],
         beego.ControllerComments{
             Method: "Add",

routers/router.go

@@ -259,6 +259,11 @@ func init() {
 				&controllers.Sci99Controller{},
 			),
 		),
+		beego.NSNamespace("/sci99_crawler",
+			beego.NSInclude(
+				&controllers.Sci99Crawler{},
+			),
+		),
 	)
 	beego.AddNamespace(ns)
 }

services/sci99/detail_struct.go

@@ -0,0 +1,140 @@
+package sci99
+
+// DataItem 定义了数据项的结构
+type DataItem struct {
+	DataTemplate        string          `json:"DataTemplate"`
+	ProductName         string          `json:"ProductName"`
+	ProductID           int             `json:"ProductID"`
+	Region              string          `json:"Region"`
+	Area                string          `json:"Area"`
+	MarketSampleName    string          `json:"MarketSampleName"`
+	Model               string          `json:"Model"`
+	FactorySampleName   string          `json:"FactorySampleName"`
+	DataTypeName        string          `json:"DataTypeName"`
+	DataTypeID          int             `json:"DataTypeID"`
+	Order               int             `json:"Order"`
+	Unit                string          `json:"Unit"`
+	DataName            string          `json:"DataName"`
+	DIID                int             `json:"DIID"`
+	Gid                 string          `json:"Gid"`
+	PriceProduct        *string         `json:"PriceProduct"` // 使用指针以处理null值
+	PriceProductID      int             `json:"PriceProductID"`
+	PriceType           string          `json:"PriceType"`
+	PriceTypeID         int             `json:"PriceTypeID"`
+	PriceCondition      string          `json:"PriceCondition"`
+	TradeTerms          string          `json:"TradeTerms"`
+	DataStatus          int             `json:"DataStatus"`
+	Cycle               string          `json:"Cycle"`
+	DataModelName       string          `json:"DataModelName"`
+	TimeMarkID          int             `json:"TimeMarkID"`
+	Digits              int             `json:"Digits"`
+	Model_zh            string          `json:"Model_zh"`
+	Province            string          `json:"Province"`
+	Payment             string          `json:"Payment"`
+	PickModel           string          `json:"PickModel"`
+	Tax                 string          `json:"Tax"`
+	PackModel           string          `json:"PackModel"`
+	TransModel          string          `json:"TransModel"`
+	BalanceModel        string          `json:"BalanceModel"`
+	EName               string          `json:"EName"`
+	GasRate             string          `json:"GasRate"`
+	FromArea            string          `json:"FromArea"`
+	DIHistory           []DIHistoryItem `json:"DIHistory"`
+	Purpose             string          `json:"Purpose"`
+	FuturesMark         string          `json:"FuturesMark"`
+	City                string          `json:"City"`
+	MainShippers        string          `json:"MainShippers"`
+	PriceClassification string          `json:"PriceClassification"`
+	FactoryID           int             `json:"FactoryID"`
+	Stand               string          `json:"Stand"`
+	Type                string          `json:"Type"`
+	DItemDTypeID        int             `json:"DItemDTypeID"`
+	Classify            string          `json:"Classify"`
+	LockDateName        string          `json:"LockDateName"`
+	DataItemName        string          `json:"DataItemName"`
+}
+
+// DIHistoryItem 定义了DIHistory中的每个项目的结构
+type DIHistoryItem struct {
+	TransModel          map[string]string `json:"TransModel"`
+	SpecialData         map[string]string `json:"SpecialData"`
+	LockTimeMarkID      int               `json:"LockTimeMarkID"`
+	DataModelName       string            `json:"DataModelName"`
+	Factory             map[string]string `json:"Factory"`
+	Tax                 map[string]string `json:"Tax"`
+	LockTimeMarkName    map[string]string `json:"LockTimeMarkName"`
+	StartDate           string            `json:"StartDate"`
+	NumeratorUnitID     int               `json:"NumeratorUnitID"`
+	DenominatorUnitID   int               `json:"DenominatorUnitID"`
+	FromArea            map[string]string `json:"FromArea"`
+	PushTimeMarkID      int               `json:"PushTimeMarkID"`
+	TradeTerms          map[string]string `json:"TradeTerms"`
+	TimeMarkName        map[string]string `json:"TimeMarkName"`
+	NumeratorUnitName   map[string]string `json:"NumeratorUnitName"`
+	PushTimeMarkName    map[string]string `json:"PushTimeMarkName"`
+	TimeMarkID          int               `json:"TimeMarkID"`
+	PickModel           map[string]string `json:"PickModel"`
+	EndDate             string            `json:"EndDate"`
+	Market              map[string]string `json:"Market"`
+	DenominatorUnitName map[string]string `json::"DenominatorUnitName"`
+	Area                map[string]string `json:"Area"`
+	Payment             map[string]string `json:"Payment"`
+	FactorySampleName   map[string]string `json:"FactorySampleName"`
+	PackModel           map[string]string `json:"PackModel"`
+	DataModel           int               `json:"DataModel"`
+	Model               map[string]string `json:"Model"`
+	DecimalPoint        int               `json:"DecimalPoint"`
+	BalanceModel        map[string]string `json:"BalanceModel"`
+}
+
+// List 定义了Data数组中的每个项目的结构
+type List struct {
+	DIID              int     `json:"DIID"`
+	DataTypeID        int     `json:"DataTypeID"`
+	InitialValue      float64 `json:"InitialValue"`
+	EndingValue       float64 `json:"EndingValue"`
+	LDataValue        float64 `json:"LDataValue"`
+	HDataValue        float64 `json:"HDataValue"`
+	MDataValue        float64 `json:"MDataValue"`
+	Change            float64 `json:"Change"`
+	ChangeRate        float64 `json:"ChangeRate"`
+	AmplitudeValue    float64 `json:"AmplitudeValue"`
+	Remark            *string `json:"Remark"` // 使用指针以处理null值
+	DataDate          string  `json:"DataDate"`
+	RealDate          string  `json:"realDate"`
+	MarketSampleName  string  `json:"MarketSampleName"`
+	FactorySampleName *string `json:"FactorySampleName"` // 使用指针以处理null值
+	Model             string  `json:"Model"`
+	Unit              string  `json:"Unit"`
+	PriceCondition    string  `json:"PriceCondition"`
+	Area              string  `json:"Area"`
+	Province          string  `json:"Province"`
+	City              string  `json:"City"`
+	Region            string  `json:"Region"`
+	LockState         string  `json:"LockState"`
+}
+
+// DetailResponse
+type DetailResponse struct {
+	Status int    `json:"status"`
+	Msg    string `json:"msg"`
+	Data   struct {
+		DataItem   DataItem  `json:"DataItem"`
+		UserPower  UserPower `json:"UserPower"`
+		List       []List    `json:"List"`
+		Privileged bool      `json:"Privileged"`
+		Choice     string    `json:"choice"`
+		LastDate   string    `json:"LastDate"`
+		IsCollect  bool      `json:"IsCollect"`
+		CollectId  string    `json:"collectId"`
+	} `json:"data"`
+}
+
+// UserPower 定义了用户权限的结构
+type UserPower struct {
+	Start     string `json:"Start"`
+	End       string `json:"End"`
+	PowerType int    `json:"PowerType"`
+	ProductID int    `json:"ProductID"`
+	Ppid      int    `json:"Ppid"`
+}

services/sci99/list_struct.go

@@ -0,0 +1,127 @@
+package sci99
+
+// Header 代表表头信息
+type Header struct {
+	Code        string  `json:"Code"`
+	DisplayName string  `json:"DisplayName"`
+	OrderNO     float64 `json:"OrderNO"`
+	DataType    int     `json:"DataType"`
+	DisplayType int     `json:"DisplayType"`
+}
+
+// Item 代表数据项
+type Item struct {
+	Area                string `json:"Area"`
+	DataModelName       string `json:"DataModelName"`
+	DataTypeID          int    `json:"DataTypeID"`
+	Digits              int    `json:"Digits"`
+	DIID                int    `json:"DIID"`
+	FactorySampleName   string `json:"FactorySampleName"`
+	MarketSampleName    string `json:"MarketSampleName"`
+	Model               string `json:"Model"`
+	TradeTerms          string `json:"TradeTerms"`
+	PriceType           string `json:"PriceType"`
+	PriceTypeID         int    `json:"PriceTypeID"`
+	ProductName         string `json:"ProductName"`
+	ProductID           int    `json:"ProductID"`
+	Province            string `json:"Province"`
+	Region              string `json:"Region"`
+	TimeMarkID          int    `json:"TimeMarkID"`
+	Unit                string `json:"Unit"`
+	Order               int    `json:"Order"`
+	DataName            string `json:"DataName"`
+	DataItemName        string `json:"DataItemName"`
+	DataTypeName        string `json:"DataTypeName"`
+	IsWarn              bool   `json:"IsWarn"`
+	EName               string `json:"EName"`
+	GasRate             string `json:"GasRate"`
+	FromArea            string `json:"FromArea"`
+	DataStatus          int    `json:"DataStatus"`
+	Purpose             string `json:"Purpose"`
+	FuturesMark         string `json:"FuturesMark"`
+	MainShippers        string `json:"MainShippers"`
+	PriceClassification string `json:"PriceClassification"`
+	Stand               string `json:"Stand"`
+	Type                string `json:"Type"`
+	DItemDTypeID        int    `json:"DItemDTypeID"`
+	Classification      string `json:"Classification"`
+	LockDateName        string `json:"LockDateName"`
+	PriceCondition      string `json:"PriceCondition"`
+	// 动态日期字段，需要特殊处理
+	//DateFields        map[string]string `json:"date_fields,inline"`
+	Change      string `json:"Change"`
+	ChangeRate  string `json:"ChangeRate"`
+	Remark      string `json:"Remark"`
+	LockState   string `json:"LockState"`
+	PowerStatus int    `json:"PowerStatus"`
+}
+
+// Data 代表数据部分
+type FirstData struct {
+	Headers    []Header   `json:"Headers"`
+	SecondData SecondData `json:"data"`
+}
+
+type SecondData struct {
+	LockTimeRange string  `json:"LockTimeRange"`
+	PageNO        int     `json:"PageNO"`
+	PageSize      int     `json:"PageSize"`
+	TotalItems    int     `json:"TotalItems"`
+	TotalPages    int     `json:"TotalPages"`
+	Items         []Item  `json:"Items"`
+	DiidDtypeids  *string `json:"DiidDtypeids"` // 使用指针以处理null值
+	TableHeader   *string `json:"TableHeader"`  // 使用指针以处理null值
+}
+
+// ListResponse 代表整个API响应
+type ListResponse struct {
+	Status    int       `json:"status"`
+	Msg       string    `json:"msg"`
+	FirstData FirstData `json:"data"`
+}
+
+// 创建请求体的数据结构
+type ListRequestBody struct {
+	Region           string `json:"region"`
+	Market           string `json:"market"`
+	Factory          string `json:"factory"`
+	Model            string `json:"model"`
+	Pname            string `json:"pname"`
+	CycleType        string `json:"cycletype"`
+	PriceCycle       string `json:"pricecycle"`
+	SpecialPriceType string `json:"specialpricetype"`
+	Groupname        string `json:"groupname"`
+	Ppname           string `json:"ppname"`
+	Province         string `json:"province"`
+	PriceTypeID      int    `json:"pricetypeid"`
+	PPIDs            string `json:"ppids"`
+	Navid            string `json:"navid"`
+	SiteType         int    `json:"sitetype"`
+	PageNo           int    `json:"pageno"`
+	PageSize         string `json:"pagesize"`
+	Purpose          string `json:"purpose"`
+	Stand            string `json:"stand"`
+	Type             string `json:"type"`
+	FromArea         string `json:"fromarea"`
+	Classification   string `json:"classification"`
+}
+
+// DetailRequest
+type DetailRequest struct {
+	Start        string `json:"start"`
+	End          string `json:"end"`
+	APIStart     string `json:"apiStart"`
+	APIEnd       string `json:"apiEnd"`
+	DIIDD        string `json:"diid"`
+	DataTypeID   string `json:"datatypeid"`
+	PPID         string `json:"ppid"`
+	CycleType    string `json:"cycletype"`
+	SelectConfig int    `json:"selectconfig"`
+}
+
+type RefreshSci99CrawlerReq struct {
+	ClassifyName string
+	IndexName    string
+	ListData     Item
+	DetailData   DetailResponse
+}

services/sci99/sci99.go

@@ -0,0 +1,146 @@
+package sci99
+
+import (
+	"eta/eta_index_lib/models"
+	"github.com/mozillazg/go-pinyin"
+	"strings"
+)
+
+var IndexCodeMap = make(map[string]string)
+var IndexMap = make(map[string]*models.BaseFromSci99Index)
+var ClassifyMap = make(map[string]*models.BaseFromSci99Classify)
+
+var PriceTypeIDs = []int{34320, 34318}
+var requestList = []ListRequestBody{
+	{
+		Ppname: "天然橡胶原料",
+		PPIDs:  "13676",
+		Navid:  "593",
+	}, {
+		Ppname: "混合胶",
+		PPIDs:  "12973",
+		Navid:  "591",
+	}, {
+		Ppname: "标准胶",
+		PPIDs:  "12960",
+		Navid:  "590",
+	}, {
+		Ppname: "烟片胶",
+		PPIDs:  "12985",
+		Navid:  "592",
+	}, {
+		Ppname: "天然乳胶",
+		PPIDs:  "12947",
+		Navid:  "595",
+	}, {
+		Ppname: "丁苯橡胶",
+		PPIDs:  "12951",
+		Navid:  "596",
+	}, {
+		Ppname: "顺丁橡胶",
+		PPIDs:  "12964",
+		Navid:  "597",
+	}, {
+		Ppname: "SBS",
+		PPIDs:  "12948",
+		Navid:  "598",
+	}, {
+		Ppname: "丁腈橡胶",
+		PPIDs:  "12945",
+		Navid:  "605",
+	},
+}
+
+
+func Sci99IndexCodeGenerator(indexName, indexCodeStr, marketSampleName, model string) (indexCode string, needAdd bool) {
+	strResult := ""
+	indexCode, _ = IndexCodeMap[indexName]
+	if indexCode == "" {
+		//首字母
+		a := pinyin.NewArgs()
+		a.Fallback = func(r rune, a pinyin.Args) []string {
+			return []string{string(r)}
+		}
+		rows := pinyin.Pinyin(indexCodeStr, a)
+		for i := 0; i < len(rows); i++ {
+			//strResult += rows[i][0]
+			if len(rows[i]) != 0 {
+				str := rows[i][0]
+				pi := str[0:1]
+				strResult += pi
+			}
+		}
+
+		// 处理市场名称
+		if province, ok := ProvinceMap[marketSampleName]; ok {
+			strResult += province
+		} else {
+			a := pinyin.NewArgs()
+			rows := pinyin.LazyPinyin(marketSampleName, a)
+			for i := 0; i < len(rows); i++ {
+				strResult += rows[i]
+			}
+		}
+
+		// 去除特殊符号
+		model = strings.Replace(model, " ", "", -1)
+		model = strings.Replace(model, "-", "", -1)
+		model = strings.Replace(model, "/", "", -1)
+		model = strings.Replace(model, "#", "", -1)
+		model = strings.Replace(model, "：", "", -1)
+		model = strings.Replace(model, "(", "", -1)
+		model = strings.Replace(model, ")", "", -1)
+
+		// 拼接型号
+		modelRows := pinyin.Pinyin(model, a)
+		for i := 0; i < len(modelRows); i++ {
+			if len(modelRows[i]) != 0 {
+				str := modelRows[i][0]
+				pi := str[0:1]
+				strResult += pi
+			}
+		}
+
+		needAdd = true
+		indexCode = strings.Replace(strResult, " ", "", -1)
+		IndexCodeMap[indexName] = indexCode
+	}
+	return
+}
+
+var ProvinceMap = map[string]string{
+	"上海":  "shanghai",
+	"云南":  "yunnan",
+	"内蒙古": "innermongolia",
+	"北京":  "beijing",
+	"台湾":  "taiwan",
+	"吉林":  "jilin",
+	"四川":  "sichuan",
+	"天津":  "tianjin",
+	"宁夏":  "ningxia",
+	"安徽":  "anhui",
+	"山东":  "shandong",
+	"山西":  "shanxi",
+	"广东":  "guangdong",
+	"广西":  "guangxi",
+	"新疆":  "xinjiang",
+	"江苏":  "jiangsu",
+	"江西":  "jiangxi",
+	"河北":  "hebei",
+	"河南":  "henan",
+	"浙江":  "zhejiang",
+	"海南":  "hainan",
+	"湖北":  "hubei",
+	"湖南":  "hunan",
+	"澳门":  "macao",
+	"甘肃":  "gansu",
+	"福建":  "fujian",
+	"西藏":  "tibet",
+	"贵州":  "guizhou",
+	"辽宁":  "liaoning",
+	"重庆":  "chongqing",
+	"陕西":  "shaanxi",
+	"青海":  "qinhai",
+	"香港":  "hongkong",
+	"黑龙江": "heilongjiang",
+}