Merge branch 'bzq/shanghai_smm' of eta_server/eta_crawler into debug

models/base_from_eia_steo.go

@@ -160,7 +160,8 @@ func HandleEiaSteoData(dataList map[string]interface{}, indexInfo *BaseFromEiaSt
 			//fmt.Println(date, "=========", nowVal.Equal(valDecimal))
 			if !nowVal.Equal(valDecimal) {
 				tmpBaseFromEiaSteoData.Value = val
-				_, err = to.Update(tmpBaseFromEiaSteoData, "Value")
+				tmpBaseFromEiaSteoData.ModifyTime = time.Now()
+				_, err = to.Update(tmpBaseFromEiaSteoData, "Value", "ModifyTime")
 				if err != nil {
 					return
 				}

models/base_from_sci99.go

@@ -0,0 +1,102 @@
+package models
+
+import (
+	"github.com/beego/beego/v2/client/orm"
+	"time"
+)
+
+// BaseFromSci99Index 代表卓创资讯-原始指标表的结构
+type BaseFromSci99Index struct {
+	BaseFromSciIndexId int       `orm:"column(base_from_sci_index_id);pk"` // 主键，自动递增
+	IndexCode          string    // 指标编码
+	IndexName          string    // 指标名称
+	ClassifyId         int       // 分类Id
+	Unit               string    // 单位
+	Frequency          string    // 频度
+	Describe           string    // 指标描述
+	CreateTime         time.Time // 创建时间
+	ModifyTime         time.Time // 修改时间
+}
+
+// BaseFromSci99Data 代表卓创资讯-原始指标数据表的结构
+type BaseFromSci99Data struct {
+	BaseFromSciDataId  int       `orm:"column(base_from_sci_data_id);pk"` // 主键，自动递增
+	BaseFromSciIndexId int       // 指标id
+	IndexCode          string    // 指标编码
+	DataTime           string    // 数据日期
+	Value              float64   // 数据值
+	CreateTime         time.Time // 创建时间
+	ModifyTime         time.Time // 修改时间
+}
+
+// BaseFromSci99Classify 代表卓创资讯-原始指标分类表的结构
+type BaseFromSci99Classify struct {
+	BaseFromSciClassifyId int       `orm:"column(base_from_sci_classify_id);pk"` // 主键，自动递增
+	ClassifyName          string    // 分类名称
+	Sort                  int       // 排序
+	CreateTime            time.Time // 创建时间
+	ModifyTime            time.Time // 修改时间
+}
+
+// 添加数据
+func AddBaseFromSci99Index(item *BaseFromSci99Index) (lastId int64, err error) {
+	o := orm.NewOrmUsingDB("data")
+	lastId, err = o.Insert(item)
+	return
+}
+
+func AddBaseFromSci99Classify(item *BaseFromSci99Classify) (lastId int64, err error) {
+	o := orm.NewOrmUsingDB("data")
+	lastId, err = o.Insert(item)
+	return
+}
+
+func AddBaseFromSci99DataMulti(item []*BaseFromSci99Data) (err error) {
+	o := orm.NewOrmUsingDB("data")
+	_, err = o.InsertMulti(1000, item)
+	return
+}
+
+func GetBaseFromSci99Index() (list []*BaseFromSci99Index, err error) {
+	o := orm.NewOrmUsingDB("data")
+	sql := `SELECT * FROM base_from_sci99_index `
+	_, err = o.Raw(sql).QueryRows(&list)
+	return
+}
+
+
+func GetBaseFromSci99Classify() (list []*BaseFromSci99Classify, err error) {
+	o := orm.NewOrmUsingDB("data")
+	sql := `SELECT * FROM base_from_sci99_classify `
+	_, err = o.Raw(sql).QueryRows(&list)
+	return
+}
+
+func GetBaseFromTradeSci99DataAll(dateStr string) (list []*BaseFromSci99Data, err error) {
+	o := orm.NewOrmUsingDB("data")
+	sql := `SELECT * FROM base_from_sci99_data where data_time=?`
+	_, err = o.Raw(sql, dateStr).QueryRows(&list)
+	return
+}
+
+func GetBaseFromTradeSci99IndexAll(indexCode string) (list []*BaseFromSci99Data, err error) {
+	o := orm.NewOrmUsingDB("data")
+	sql := `SELECT * FROM base_from_sci99_data where index_code=?`
+	_, err = o.Raw(sql, indexCode).QueryRows(&list)
+	return
+}
+
+func GetBaseFromTradeSci99LatestData(indexCode string) (date time.Time, err error) {
+	o := orm.NewOrmUsingDB("data")
+	sql := `SELECT data_time FROM base_from_sci99_data where index_code=? ORDER BY data_time DESC limit 1 `
+	err = o.Raw(sql, indexCode).QueryRow(&date)
+	return
+}
+
+// UpdateBaseFromSci99Data
+func UpdateBaseFromSci99Data(value float64, indexCode, dataTime string) (err error) {
+	o := orm.NewOrmUsingDB("data")
+	sql := `UPDATE base_from_sci99_data SET value=?,modify_time=NOW() WHERE index_code = ? AND data_time = ? `
+	_, err = o.Raw(sql, value, indexCode, dataTime).Exec()
+	return
+}

models/db.go

@@ -11,19 +11,21 @@ import (
 
 func init() {
 
-	_ = orm.RegisterDataBase("default", "mysql", utils.MYSQL_URL)
-	orm.SetMaxIdleConns("default", 50)
-	orm.SetMaxOpenConns("default", 100)
+	if utils.BusinessCode == utils.BusinessCodeRelease {
+		_ = orm.RegisterDataBase("default", "mysql", utils.MYSQL_URL)
+		orm.SetMaxIdleConns("default", 50)
+		orm.SetMaxOpenConns("default", 100)
 
-	db, _ := orm.GetDB("default")
-	db.SetConnMaxLifetime(10 * time.Minute)
+		db, _ := orm.GetDB("default")
+		db.SetConnMaxLifetime(10 * time.Minute)
 
-	_ = orm.RegisterDataBase("data", "mysql", utils.MYSQL_URL_DATA)
-	orm.SetMaxIdleConns("data", 50)
-	orm.SetMaxOpenConns("data", 100)
+		_ = orm.RegisterDataBase("data", "mysql", utils.MYSQL_URL_DATA)
+		orm.SetMaxIdleConns("data", 50)
+		orm.SetMaxOpenConns("data", 100)
 
-	data_db, _ := orm.GetDB("data")
-	data_db.SetConnMaxLifetime(10 * time.Minute)
+		data_db, _ := orm.GetDB("data")
+		data_db.SetConnMaxLifetime(10 * time.Minute)
+	}
 
 	orm.Debug = true
 	orm.DebugLog = orm.NewLog(utils.Binlog)
@@ -66,5 +68,8 @@ func init() {
 		new(BaseFromTradeGuangzhouContract),
 
 		new(BaseFromIcpiClassify),
+		new(BaseFromSci99Index),
+		new(BaseFromSci99Data),
+		new(BaseFromSci99Classify),
 	)
 }

services/commodity_trade_cffex.go

@@ -117,6 +117,7 @@ func SyncRankingFromCffex() {
 		for _, xmlItem := range xmlItems {
 			for _, i := range xmlItem.Data {
 				i.ShortName = strings.Replace(i.ShortName,"(经纪)","",-1)
+				i.ShortName = strings.Replace(i.ShortName,"(代客)","",-1)
 				var item = new(models.BaseFromTradeCffexIndex)
 				item.DealValue = -1
 				item.BuyValue = -1

services/commodity_trade_dalian.go

@@ -53,18 +53,18 @@ func SyncRankingFromDalianSearch(dayNum int) (err error) {
 	endDate := time.Now().AddDate(0, 0, -dayNum).Format(utils.FormatDateTime)
 	//endDate := time.Now().Format(utils.FormatDateTime)
 	timeDate := utils.StrTimeToTime(endDate)
-	currDate := timeDate.Format(utils.FormatDateUnSpace)
-	year := timeDate.Year()
-	month := timeDate.Format("01")
-	var dayStr string
-	day := timeDate.Day()
-	if day < 10 {
-		dayStr = "0" + strconv.Itoa(day)
-	} else {
-		dayStr = strconv.Itoa(day)
-	}
-	monthNum, _ := strconv.Atoi(month)
-	month = strconv.Itoa(monthNum - 1) //获取时月份需要减一
+	//currDate := timeDate.Format(utils.FormatDateUnSpace)
+	//year := timeDate.Year()
+	//month := timeDate.Format("01")
+	//var dayStr string
+	//day := timeDate.Day()
+	//if day < 10 {
+	//	dayStr = "0" + strconv.Itoa(day)
+	//} else {
+	//	dayStr = strconv.Itoa(day)
+	//}
+	//monthNum, _ := strconv.Atoi(month)
+	//month = strconv.Itoa(monthNum - 1) //获取时月份需要减一
 	list, err := models.GetBaseFromTradeDalianDataList(timeDate.Format(utils.FormatDate))
 	if err != nil {
 		fmt.Println(err)
@@ -76,24 +76,24 @@ func SyncRankingFromDalianSearch(dayNum int) (err error) {
 	}
 
 	var ContractId string
-	var CarietyCode string
+	//var CarietyCode string
 	var VarietyName string
 	//模拟form表单请求
 	url := "http://www.dce.com.cn/publicweb/quotesdata/memberDealPosiQuotes.html"
 	method := "POST"
 	payload := &bytes.Buffer{}
 	writer := multipart.NewWriter(payload)
-	_ = writer.WriteField("memberDealPosiQuotes.variety", CarietyCode)
+	_ = writer.WriteField("memberDealPosiQuotes.variety", "c")
 	_ = writer.WriteField("memberDealPosiQuotes.trade_type", "0")
-	_ = writer.WriteField("year", strconv.Itoa(year))
-	_ = writer.WriteField("month", month)
-	_ = writer.WriteField("day", dayStr)
-	_ = writer.WriteField("contract.contract_id", ContractId)
-	_ = writer.WriteField("contract.variety_id", CarietyCode)
-	_ = writer.WriteField("currDate", currDate)
+	_ = writer.WriteField("year", strconv.Itoa(2024))
+	_ = writer.WriteField("month", "4")
+	_ = writer.WriteField("day", "10")
+	_ = writer.WriteField("contract.contract_id", "c2405")
+	_ = writer.WriteField("contract.variety_id", "c")
+	//_ = writer.WriteField("currDate", currDate)
 	err = writer.Close()
 	if err != nil {
-		utils.FileLog.Info("获取指标失败：" + currDate + VarietyName + ContractId)
+		utils.FileLog.Info("获取指标失败："  + VarietyName + ContractId)
 		return err
 	}
 	client := &http.Client{}
@@ -112,7 +112,7 @@ func SyncRankingFromDalianSearch(dayNum int) (err error) {
 	defer res.Body.Close()
 	body, err := ioutil.ReadAll(res.Body)
 	if err != nil {
-		msg := "失败提醒" + "RefreshDataFromDalian ErrMsg:" + err.Error() + "获取指标失败：" + currDate + VarietyName + ContractId
+		msg := "失败提醒" + "RefreshDataFromDalian ErrMsg:" + err.Error() + "获取指标失败："  + VarietyName + ContractId
 		go alarm_msg.SendAlarmMsg(msg, 3)
 		//go utils.SendEmail(utils.APPNAME+"【"+utils.RunMode+"】"+"失败提醒", "RefreshDataFromDalian ErrMsg:"+err.Error()+"获取指标失败："+currDate+VarietyName+ContractId, utils.EmailSendToUsers)
 		return err
@@ -274,7 +274,9 @@ func DoHtml(body, name, contractId string, dateTime time.Time, listDataMap map[s
 				tdText := td.Text()
 				tdText = strings.Replace(tdText, "（代客）", "", -1)
 				if tk == 0 { //名次
-					rank = tdText
+					if tdText != " " {
+						rank = tdText
+					}
 				}
 				if tk == 1 { //会员简称
 					shortName = tdText
@@ -285,6 +287,11 @@ func DoHtml(body, name, contractId string, dateTime time.Time, listDataMap map[s
 				if tk == 3 { //增减
 					dealChange = strings.Replace(tdText, ",", "", -1)
 				}
+				if tk == 4 { //名次
+					if tdText != " " {
+						rank = tdText
+					}
+				}
 				if tk == 5 { //会员简称
 					buyName = tdText
 				}
@@ -295,6 +302,11 @@ func DoHtml(body, name, contractId string, dateTime time.Time, listDataMap map[s
 					buyChange = strings.Replace(tdText, ",", "", -1)
 				}
 
+				if tk == 8 { //名次
+					if tdText != " " {
+						rank = tdText
+					}
+				}
 				if tk == 9 { //会员简称
 					soldName = tdText
 				}

services/commodity_trade_zhengzhou.go

@@ -138,18 +138,21 @@ func SyncRankingFromZhengzhou() {
 
 					if tk == 1 {
 						if !strings.Contains(tdText, "会员简称") {
+							tdText = strings.Replace(tdText, "（代客）", "", -1)
 							memberShortNameArr = append(memberShortNameArr, tdText)
 						}
 					}
 
 					if tk == 4 {
 						if !strings.Contains(tdText, "会员简称") {
+							tdText = strings.Replace(tdText, "（代客）", "", -1)
 							memberShortNameArr = append(memberShortNameArr, tdText)
 						}
 					}
 
 					if tk == 7 {
 						if !strings.Contains(tdText, "会员简称") {
+							tdText = strings.Replace(tdText, "（代客）", "", -1)
 							memberShortNameArr = append(memberShortNameArr, tdText)
 						}
 					}

services/sci99/detail_struct.go

@@ -0,0 +1,140 @@
+package sci99
+
+// DataItem 定义了数据项的结构
+type DataItem struct {
+	DataTemplate        string          `json:"DataTemplate"`
+	ProductName         string          `json:"ProductName"`
+	ProductID           int             `json:"ProductID"`
+	Region              string          `json:"Region"`
+	Area                string          `json:"Area"`
+	MarketSampleName    string          `json:"MarketSampleName"`
+	Model               string          `json:"Model"`
+	FactorySampleName   string          `json:"FactorySampleName"`
+	DataTypeName        string          `json:"DataTypeName"`
+	DataTypeID          int             `json:"DataTypeID"`
+	Order               int             `json:"Order"`
+	Unit                string          `json:"Unit"`
+	DataName            string          `json:"DataName"`
+	DIID                int             `json:"DIID"`
+	Gid                 string          `json:"Gid"`
+	PriceProduct        *string         `json:"PriceProduct"` // 使用指针以处理null值
+	PriceProductID      int             `json:"PriceProductID"`
+	PriceType           string          `json:"PriceType"`
+	PriceTypeID         int             `json:"PriceTypeID"`
+	PriceCondition      string          `json:"PriceCondition"`
+	TradeTerms          string          `json:"TradeTerms"`
+	DataStatus          int             `json:"DataStatus"`
+	Cycle               string          `json:"Cycle"`
+	DataModelName       string          `json:"DataModelName"`
+	TimeMarkID          int             `json:"TimeMarkID"`
+	Digits              int             `json:"Digits"`
+	Model_zh            string          `json:"Model_zh"`
+	Province            string          `json:"Province"`
+	Payment             string          `json:"Payment"`
+	PickModel           string          `json:"PickModel"`
+	Tax                 string          `json:"Tax"`
+	PackModel           string          `json:"PackModel"`
+	TransModel          string          `json:"TransModel"`
+	BalanceModel        string          `json:"BalanceModel"`
+	EName               string          `json:"EName"`
+	GasRate             string          `json:"GasRate"`
+	FromArea            string          `json:"FromArea"`
+	DIHistory           []DIHistoryItem `json:"DIHistory"`
+	Purpose             string          `json:"Purpose"`
+	FuturesMark         string          `json:"FuturesMark"`
+	City                string          `json:"City"`
+	MainShippers        string          `json:"MainShippers"`
+	PriceClassification string          `json:"PriceClassification"`
+	FactoryID           int             `json:"FactoryID"`
+	Stand               string          `json:"Stand"`
+	Type                string          `json:"Type"`
+	DItemDTypeID        int             `json:"DItemDTypeID"`
+	Classify            string          `json:"Classify"`
+	LockDateName        string          `json:"LockDateName"`
+	DataItemName        string          `json:"DataItemName"`
+}
+
+// DIHistoryItem 定义了DIHistory中的每个项目的结构
+type DIHistoryItem struct {
+	TransModel          map[string]string `json:"TransModel"`
+	SpecialData         map[string]string `json:"SpecialData"`
+	LockTimeMarkID      int               `json:"LockTimeMarkID"`
+	DataModelName       string            `json:"DataModelName"`
+	Factory             map[string]string `json:"Factory"`
+	Tax                 map[string]string `json:"Tax"`
+	LockTimeMarkName    map[string]string `json:"LockTimeMarkName"`
+	StartDate           string            `json:"StartDate"`
+	NumeratorUnitID     int               `json:"NumeratorUnitID"`
+	DenominatorUnitID   int               `json:"DenominatorUnitID"`
+	FromArea            map[string]string `json:"FromArea"`
+	PushTimeMarkID      int               `json:"PushTimeMarkID"`
+	TradeTerms          map[string]string `json:"TradeTerms"`
+	TimeMarkName        map[string]string `json:"TimeMarkName"`
+	NumeratorUnitName   map[string]string `json:"NumeratorUnitName"`
+	PushTimeMarkName    map[string]string `json:"PushTimeMarkName"`
+	TimeMarkID          int               `json:"TimeMarkID"`
+	PickModel           map[string]string `json:"PickModel"`
+	EndDate             string            `json:"EndDate"`
+	Market              map[string]string `json:"Market"`
+	DenominatorUnitName map[string]string `json::"DenominatorUnitName"`
+	Area                map[string]string `json:"Area"`
+	Payment             map[string]string `json:"Payment"`
+	FactorySampleName   map[string]string `json:"FactorySampleName"`
+	PackModel           map[string]string `json:"PackModel"`
+	DataModel           int               `json:"DataModel"`
+	Model               map[string]string `json:"Model"`
+	DecimalPoint        int               `json:"DecimalPoint"`
+	BalanceModel        map[string]string `json:"BalanceModel"`
+}
+
+// List 定义了Data数组中的每个项目的结构
+type List struct {
+	DIID              int     `json:"DIID"`
+	DataTypeID        int     `json:"DataTypeID"`
+	InitialValue      float64 `json:"InitialValue"`
+	EndingValue       float64 `json:"EndingValue"`
+	LDataValue        float64 `json:"LDataValue"`
+	HDataValue        float64 `json:"HDataValue"`
+	MDataValue        float64 `json:"MDataValue"`
+	Change            float64 `json:"Change"`
+	ChangeRate        float64 `json:"ChangeRate"`
+	AmplitudeValue    float64 `json:"AmplitudeValue"`
+	Remark            *string `json:"Remark"` // 使用指针以处理null值
+	DataDate          string  `json:"DataDate"`
+	RealDate          string  `json:"realDate"`
+	MarketSampleName  string  `json:"MarketSampleName"`
+	FactorySampleName *string `json:"FactorySampleName"` // 使用指针以处理null值
+	Model             string  `json:"Model"`
+	Unit              string  `json:"Unit"`
+	PriceCondition    string  `json:"PriceCondition"`
+	Area              string  `json:"Area"`
+	Province          string  `json:"Province"`
+	City              string  `json:"City"`
+	Region            string  `json:"Region"`
+	LockState         string  `json:"LockState"`
+}
+
+// DetailResponse
+type DetailResponse struct {
+	Status int    `json:"status"`
+	Msg    string `json:"msg"`
+	Data   struct {
+		DataItem   DataItem  `json:"DataItem"`
+		UserPower  UserPower `json:"UserPower"`
+		List       []List    `json:"List"`
+		Privileged bool      `json:"Privileged"`
+		Choice     string    `json:"choice"`
+		LastDate   string    `json:"LastDate"`
+		IsCollect  bool      `json:"IsCollect"`
+		CollectId  string    `json:"collectId"`
+	} `json:"data"`
+}
+
+// UserPower 定义了用户权限的结构
+type UserPower struct {
+	Start     string `json:"Start"`
+	End       string `json:"End"`
+	PowerType int    `json:"PowerType"`
+	ProductID int    `json:"ProductID"`
+	Ppid      int    `json:"Ppid"`
+}

services/sci99/list_struct.go

@@ -0,0 +1,122 @@
+package sci99
+
+
+
+// Header 代表表头信息
+type Header struct {
+	Code        string  `json:"Code"`
+	DisplayName string  `json:"DisplayName"`
+	OrderNO     float64 `json:"OrderNO"`
+	DataType    int     `json:"DataType"`
+	DisplayType int     `json:"DisplayType"`
+}
+
+// Item 代表数据项
+type Item struct {
+	Area                string `json:"Area"`
+	DataModelName       string `json:"DataModelName"`
+	DataTypeID          int    `json:"DataTypeID"`
+	Digits              int    `json:"Digits"`
+	DIID                int    `json:"DIID"`
+	FactorySampleName   string `json:"FactorySampleName"`
+	MarketSampleName    string `json:"MarketSampleName"`
+	Model               string `json:"Model"`
+	TradeTerms          string `json:"TradeTerms"`
+	PriceType           string `json:"PriceType"`
+	PriceTypeID         int    `json:"PriceTypeID"`
+	ProductName         string `json:"ProductName"`
+	ProductID           int    `json:"ProductID"`
+	Province            string `json:"Province"`
+	Region              string `json:"Region"`
+	TimeMarkID          int    `json:"TimeMarkID"`
+	Unit                string `json:"Unit"`
+	Order               int    `json:"Order"`
+	DataName            string `json:"DataName"`
+	DataItemName        string `json:"DataItemName"`
+	DataTypeName        string `json:"DataTypeName"`
+	IsWarn              bool   `json:"IsWarn"`
+	EName               string `json:"EName"`
+	GasRate             string `json:"GasRate"`
+	FromArea            string `json:"FromArea"`
+	DataStatus          int    `json:"DataStatus"`
+	Purpose             string `json:"Purpose"`
+	FuturesMark         string `json:"FuturesMark"`
+	MainShippers        string `json:"MainShippers"`
+	PriceClassification string `json:"PriceClassification"`
+	Stand               string `json:"Stand"`
+	Type                string `json:"Type"`
+	DItemDTypeID        int    `json:"DItemDTypeID"`
+	Classification      string `json:"Classification"`
+	LockDateName        string `json:"LockDateName"`
+	PriceCondition      string `json:"PriceCondition"`
+	// 动态日期字段，需要特殊处理
+	//DateFields        map[string]string `json:"date_fields,inline"`
+	Change      string `json:"Change"`
+	ChangeRate  string `json:"ChangeRate"`
+	Remark      string `json:"Remark"`
+	LockState   string `json:"LockState"`
+	PowerStatus int    `json:"PowerStatus"`
+}
+
+// Data 代表数据部分
+type FirstData struct {
+	Headers    []Header   `json:"Headers"`
+	SecondData SecondData `json:"data"`
+}
+
+type SecondData struct {
+	LockTimeRange string  `json:"LockTimeRange"`
+	PageNO        int     `json:"PageNO"`
+	PageSize      int     `json:"PageSize"`
+	TotalItems    int     `json:"TotalItems"`
+	TotalPages    int     `json:"TotalPages"`
+	Items         []Item  `json:"Items"`
+	DiidDtypeids  *string `json:"DiidDtypeids"` // 使用指针以处理null值
+	TableHeader   *string `json:"TableHeader"`  // 使用指针以处理null值
+}
+
+// ListResponse 代表整个API响应
+type ListResponse struct {
+	Status    int       `json:"status"`
+	Msg       string    `json:"msg"`
+	FirstData FirstData `json:"data"`
+}
+
+// 创建请求体的数据结构
+type ListRequestBody struct {
+	Region           string `json:"region"`
+	Market           string `json:"market"`
+	Factory          string `json:"factory"`
+	Model            string `json:"model"`
+	Pname            string `json:"pname"`
+	CycleType        string `json:"cycletype"`
+	PriceCycle       string `json:"pricecycle"`
+	SpecialPriceType string `json:"specialpricetype"`
+	Groupname        string `json:"groupname"`
+	Ppname           string `json:"ppname"`
+	Province         string `json:"province"`
+	PriceTypeID      int    `json:"pricetypeid"`
+	PPIDs            string `json:"ppids"`
+	Navid            string `json:"navid"`
+	SiteType         int    `json:"sitetype"`
+	PageNo           int    `json:"pageno"`
+	PageSize         string `json:"pagesize"`
+	Purpose          string `json:"purpose"`
+	Stand            string `json:"stand"`
+	Type             string `json:"type"`
+	FromArea         string `json:"fromarea"`
+	Classification   string `json:"classification"`
+}
+
+// DetailRequest
+type DetailRequest struct {
+	Start        string `json:"start"`
+	End          string `json:"end"`
+	APIStart     string `json:"apiStart"`
+	APIEnd       string `json:"apiEnd"`
+	DIIDD        string `json:"diid"`
+	DataTypeID   string `json:"datatypeid"`
+	PPID         string `json:"ppid"`
+	CycleType    string `json:"cycletype"`
+	SelectConfig int    `json:"selectconfig"`
+}

services/sci99/sci99.go

@@ -0,0 +1,366 @@
+package sci99
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"eta/eta_crawler/utils"
+	"fmt"
+	"github.com/mozillazg/go-pinyin"
+	"io/ioutil"
+	"net/http"
+	"strconv"
+	"strings"
+	"time"
+)
+
+var IndexCodeMap = make(map[string]string)
+
+var PriceTypeIDs = []int{34320, 34318}
+var requestList = []ListRequestBody{
+	{
+		Ppname: "天然橡胶原料",
+		PPIDs:  "13676",
+		Navid:  "593",
+	}, {
+		Ppname: "混合胶",
+		PPIDs:  "12973",
+		Navid:  "591",
+	}, {
+		Ppname: "标准胶",
+		PPIDs:  "12960",
+		Navid:  "590",
+	}, {
+		Ppname: "烟片胶",
+		PPIDs:  "12985",
+		Navid:  "592",
+	}, {
+		Ppname: "天然乳胶",
+		PPIDs:  "12947",
+		Navid:  "595",
+	}, {
+		Ppname: "丁苯橡胶",
+		PPIDs:  "12951",
+		Navid:  "596",
+	}, {
+		Ppname: "顺丁橡胶",
+		PPIDs:  "12964",
+		Navid:  "597",
+	}, {
+		Ppname: "SBS",
+		PPIDs:  "12948",
+		Navid:  "598",
+	}, {
+		Ppname: "丁腈橡胶",
+		PPIDs:  "12945",
+		Navid:  "605",
+	},
+}
+
+func Scii99(cont context.Context) (err error) {
+	filePath := utils.SCI99_COOKIE_PATH
+
+	// 打开文件
+	file, err := ioutil.ReadFile(filePath)
+	if err != nil {
+		utils.FileLog.Info("ReadFile err:", err)
+		return
+	}
+
+	cookie := string(file)
+
+	// 定义请求的URL
+	listUrl := "https://prices.sci99.com/api/zh-cn/product/datavalue"
+	detailUrl := "https://prices.sci99.com/api/zh-cn/dataitem/datavalue"
+
+	for _, request := range requestList {
+		classifyName := request.Ppname
+
+		// 分别获取国内价格和国际价格
+		for _, priceTypeID := range PriceTypeIDs {
+			// 创建请求体实例并填充数据,这里的参数都是一样的，所以这边统一写
+			request.CycleType = "day"
+			request.SiteType = 1
+			request.PageNo = 1
+			request.PageSize = "1000"
+			request.PriceTypeID = priceTypeID
+
+			// 将请求体序列化为JSON
+			jsonData, e := json.Marshal(request)
+			if e != nil {
+				err = e
+				fmt.Println("Error marshalling JSON:", err)
+				utils.FileLog.Info("Error marshalling JSON:", err)
+				return
+			}
+
+			// 创建一个HTTP POST请求
+			req, e := http.NewRequest("POST", listUrl, bytes.NewBuffer(jsonData))
+			if e != nil {
+				err = e
+				fmt.Println("Error creating request:", err)
+				utils.FileLog.Info("Error creating request:", err)
+				return
+			}
+
+			// 设置请求头信息
+			req.Header.Set("Content-Type", "application/json")
+			req.Header.Set("Accept", "application/json")
+			req.Header.Set("Cookie", cookie)
+
+			// 发送请求
+			client := &http.Client{}
+			resp, e := client.Do(req)
+			if e != nil {
+				err = e
+				fmt.Println("Error sending request:", err)
+				utils.FileLog.Info("Error sending request:", err)
+				return
+			}
+			defer resp.Body.Close()
+
+			// 检查响应状态码
+			if resp.StatusCode != http.StatusOK {
+				fmt.Printf("Server returned non-200 status: %d\n", resp.StatusCode)
+				utils.FileLog.Info("Server returned non-200 status: %d", resp.StatusCode)
+				return
+			}
+			// 打印响应体
+			body, _ := ioutil.ReadAll(resp.Body)
+			//fmt.Println("Response body:", string(body))
+
+			var listResponse ListResponse
+
+			err = json.Unmarshal(body, &listResponse)
+			if err != nil {
+				fmt.Println(err)
+				utils.FileLog.Info("Error unmarshalling JSON:", err)
+				return
+			}
+
+			for _, v := range listResponse.FirstData.SecondData.Items {
+				if v.FromArea != "" && len(listResponse.FirstData.SecondData.Items) > 1 {
+					continue
+				}
+				// 组成指标
+				var indexName string
+				if v.FactorySampleName == "" || len(listResponse.FirstData.SecondData.Items) == 1 {
+					indexName = v.ProductName + "：" + v.PriceType + "：" + v.MarketSampleName + "：" + v.Model
+				} else {
+					indexName = v.ProductName + "：" + v.PriceType + "：" + v.MarketSampleName + "：" + v.FactorySampleName + "：" + v.Model
+				}
+
+				ppid := strconv.Itoa(v.ProductID)
+				dataTypeID := strconv.Itoa(v.DataTypeID)
+				diidd := strconv.Itoa(v.DIID)
+				deailReq := DetailRequest{
+					PPID:       ppid,
+					DataTypeID: dataTypeID,
+					DIIDD:      diidd,
+					APIStart:   "2021/01/01",
+					APIEnd:     time.Now().AddDate(1, 0, 0).Format(utils.FormatDate2),
+					CycleType:  "day",
+				}
+
+				// 初始化是不加日期获取全部，初始化后只拿前三天的
+				if utils.IS_INIT_SCI99 != "true" {
+					deailReq.Start = time.Now().AddDate(0,0,-3).Format(utils.FormatDate2)
+					deailReq.End = time.Now().Format(utils.FormatDate2)
+				}
+
+				// 将请求体序列化为JSON
+				reqJson, e := json.Marshal(deailReq)
+				if e != nil {
+					err = e
+					fmt.Println("Error marshalling JSON:", err)
+					utils.FileLog.Info("Error marshalling JSON:", err)
+					return
+				}
+
+				// 创建一个HTTP POST请求
+				req, err = http.NewRequest("POST", detailUrl, bytes.NewBuffer(reqJson))
+				if err != nil {
+					fmt.Println("Error creating request:", err)
+					utils.FileLog.Info("Error creating request:", err)
+					return
+				}
+
+				// 设置请求头信息
+				req.Header.Set("Content-Type", "application/json")
+				req.Header.Set("Accept", "application/json")
+				req.Header.Set("Cookie", cookie)
+
+				// 发送请求
+				resp, err = client.Do(req)
+				if err != nil {
+					fmt.Println("Error sending request:", err)
+					utils.FileLog.Info("Error sending request:", err)
+					return
+				}
+				defer resp.Body.Close()
+				// 检查响应状态码
+				if resp.StatusCode != http.StatusOK {
+					fmt.Printf("Server returned non-200 status: %d\n", resp.StatusCode)
+					utils.FileLog.Info("Server returned non-200 status: %d", resp.StatusCode)
+					return
+				}
+				// 打印响应体
+				body, _ = ioutil.ReadAll(resp.Body)
+				//fmt.Println("Response Detail body:", string(body))
+
+				var detailResponse DetailResponse
+
+				err = json.Unmarshal(body, &detailResponse)
+				if err != nil {
+					fmt.Println(err)
+					utils.FileLog.Info("Error unmarshalling JSON:", err)
+					return
+				}
+
+				param := make(map[string]interface{})
+				param["ListData"] = v
+				param["DetailData"] = detailResponse
+				param["IndexName"] = indexName
+				param["ClassifyName"] = classifyName
+				urlStr := `sci99_crawler/refresh/list`
+				postUrl := utils.EDB_LIB_URL + urlStr
+				postData, e := json.Marshal(param)
+				if e != nil {
+					err = e
+					utils.FileLog.Info("Marshal Err:" + err.Error())
+					fmt.Println(err)
+					return
+				}
+				result, e := HttpPost(postUrl, string(postData), "application/json")
+				if e != nil {
+					err = e
+					fmt.Println(err)
+					utils.FileLog.Info("HttpPost Err:" + err.Error())
+					return
+				}
+				utils.FileLog.Info("postRefreshEdbData:" + postUrl + ";" + string(postData) + ";result:" + string(result))
+				err = json.Unmarshal(result, &resp)
+				if err != nil {
+					fmt.Println(err)
+					utils.FileLog.Info("Unmarshal resp Err:" + err.Error())
+					return
+				}
+				time.Sleep(1 * time.Second)
+			}
+		}
+	}
+	return
+}
+
+func Sci99IndexCodeGenerator(indexName, indexCodeStr, marketSampleName, model string) (indexCode string, needAdd bool) {
+	strResult := ""
+	indexCode, _ = IndexCodeMap[indexName]
+	if indexCode == "" {
+		//首字母
+		a := pinyin.NewArgs()
+		a.Fallback = func(r rune, a pinyin.Args) []string {
+			return []string{string(r)}
+		}
+		rows := pinyin.Pinyin(indexCodeStr, a)
+		for i := 0; i < len(rows); i++ {
+			//strResult += rows[i][0]
+			if len(rows[i]) != 0 {
+				str := rows[i][0]
+				pi := str[0:1]
+				strResult += pi
+			}
+		}
+
+		// 处理市场名称
+		if province, ok := ProvinceMap[marketSampleName]; ok {
+			strResult += province
+		} else {
+			a := pinyin.NewArgs()
+			rows := pinyin.LazyPinyin(marketSampleName, a)
+			for i := 0; i < len(rows); i++ {
+				strResult += rows[i]
+			}
+		}
+
+		// 去除特殊符号
+		model = strings.Replace(model, " ", "", -1)
+		model = strings.Replace(model, "-", "", -1)
+		model = strings.Replace(model, "/", "", -1)
+		model = strings.Replace(model, "#", "", -1)
+		model = strings.Replace(model, "：", "", -1)
+		model = strings.Replace(model, "(", "", -1)
+		model = strings.Replace(model, ")", "", -1)
+
+		// 拼接型号
+		modelRows := pinyin.Pinyin(model, a)
+		for i := 0; i < len(modelRows); i++ {
+			if len(modelRows[i]) != 0 {
+				str := modelRows[i][0]
+				pi := str[0:1]
+				strResult += pi
+			}
+		}
+
+		needAdd = true
+		indexCode = strings.Replace(strResult, " ", "", -1)
+		IndexCodeMap[indexName] = indexCode
+	}
+	return
+}
+
+var ProvinceMap = map[string]string{
+	"上海":  "shanghai",
+	"云南":  "yunnan",
+	"内蒙古": "innermongolia",
+	"北京":  "beijing",
+	"台湾":  "taiwan",
+	"吉林":  "jilin",
+	"四川":  "sichuan",
+	"天津":  "tianjin",
+	"宁夏":  "ningxia",
+	"安徽":  "anhui",
+	"山东":  "shandong",
+	"山西":  "shanxi",
+	"广东":  "guangdong",
+	"广西":  "guangxi",
+	"新疆":  "xinjiang",
+	"江苏":  "jiangsu",
+	"江西":  "jiangxi",
+	"河北":  "hebei",
+	"河南":  "henan",
+	"浙江":  "zhejiang",
+	"海南":  "hainan",
+	"湖北":  "hubei",
+	"湖南":  "hunan",
+	"澳门":  "macao",
+	"甘肃":  "gansu",
+	"福建":  "fujian",
+	"西藏":  "tibet",
+	"贵州":  "guizhou",
+	"辽宁":  "liaoning",
+	"重庆":  "chongqing",
+	"陕西":  "shaanxi",
+	"青海":  "qinhai",
+	"香港":  "hongkong",
+	"黑龙江": "heilongjiang",
+}
+
+func HttpPost(url, postData string, params ...string) ([]byte, error) {
+	body := ioutil.NopCloser(strings.NewReader(postData))
+	client := &http.Client{}
+	req, err := http.NewRequest("POST", url, body)
+	if err != nil {
+		return nil, err
+	}
+	contentType := "application/x-www-form-urlencoded;charset=utf-8"
+	if len(params) > 0 && params[0] != "" {
+		contentType = params[0]
+	}
+	req.Header.Set("Content-Type", contentType)
+	req.Header.Set("authorization", utils.MD5(utils.APP_EDB_LIB_NAME_EN+utils.EDB_LIB_Md5_KEY))
+	resp, err := client.Do(req)
+	defer resp.Body.Close()
+	b, err := ioutil.ReadAll(resp.Body)
+	fmt.Println("HttpPost:" + string(b))
+	return b, err
+}

services/smm_shanghai.go

@@ -0,0 +1,550 @@
+package services
+
+import (
+	"context"
+	"encoding/json"
+	"eta/eta_crawler/models"
+	"eta/eta_crawler/services/sci99"
+	"eta/eta_crawler/utils"
+	"fmt"
+	"io"
+	"net/http"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/xuri/excelize/v2"
+)
+
+type SmmRequest struct {
+	Name      string
+	IndexCode string
+	Url       string
+	Frequency string
+	Unit      string
+	Value     string
+}
+
+var nickelReqs = []SmmRequest{
+	{
+		Name:      "SMM 1#电解镍",
+		IndexCode: "",
+		Url:       "https://hq.smm.cn/ajax/spot/history/201102250239",
+		Frequency: "日度",
+		Unit:      "元/吨",
+	},
+	{
+		Name:      "1#金川镍",
+		Url:       "https://hq.smm.cn/ajax/spot/history/201102250174",
+		Frequency: "日度",
+		Unit:      "元/吨",
+	},
+	{
+		Name:      "1#进口镍",
+		Url:       "https://hq.smm.cn/ajax/spot/history/201102250423",
+		Frequency: "日度",
+		Unit:      "元/吨",
+	},
+	{
+		Name:      "镍豆",
+		Url:       "https://hq.smm.cn/ajax/spot/history/202008270001",
+		Frequency: "日度",
+		Unit:      "元/吨",
+	},
+	{
+		Name:      `印尼内贸红土镍矿1.2%(到厂价)`,
+		Url:       "https://hq.smm.cn/ajax/spot/history/202311230010",
+		Frequency: "周度",
+		Unit:      "美元/湿吨",
+	},
+	{
+		Name:      `印尼内贸红土镍矿1.6%(到厂价)`,
+		Url:       "https://hq.smm.cn/ajax/spot/history/202311230011",
+		Frequency: "周度",
+		Unit:      "美元/湿吨",
+	},
+	{
+		Name:      `菲律宾红土镍矿0.9%,Al≥7%(CIF)`,
+		Url:       "https://hq.smm.cn/ajax/spot/history/201608170001",
+		Frequency: "日度",
+		Unit:      "美元/湿吨",
+	},
+	{
+		Name:      `菲律宾红土镍矿0.9%,Al<7%(CIF) `, // 重名
+		Url:       "https://hq.smm.cn/ajax/spot/history/202109140006",
+		Frequency: "日度",
+		Unit:      "美元/湿吨",
+	},
+
+	{
+		Name:      `菲律宾红土镍矿1.3%(CIF)`,
+		Url:       "https://hq.smm.cn/ajax/spot/history/202109140001",
+		Frequency: "日度",
+		Unit:      "美元/湿吨",
+	},
+	{
+		Name:      `菲律宾红土镍矿1.4%(CIF)`,
+		Url:       "https://hq.smm.cn/ajax/spot/history/202109140003",
+		Frequency: "日度",
+		Unit:      "美元/湿吨",
+	},
+	{
+		Name:      `菲律宾红土镍矿1.5%(CIF)`,
+		Url:       "https://hq.smm.cn/ajax/spot/history/201608170002",
+		Frequency: "日度",
+		Unit:      "美元/湿吨",
+	},
+	{
+		Name:      `红土镍矿1.8%(CIF)`,
+		Url:       "https://hq.smm.cn/ajax/spot/history/201608170003",
+		Frequency: "日度",
+		Unit:      "美元/湿吨",
+	},
+
+	{
+		Name:      `菲律宾红土镍矿0.9%,Al≥7%(FOB)`,
+		Url:       "https://hq.smm.cn/ajax/spot/history/201509220001",
+		Frequency: "日度",
+		Unit:      "美元/湿吨",
+	},
+	{
+		Name:      `菲律宾红土镍矿0.9%,Al<7%(FOB)`, //重名
+		Url:       "https://hq.smm.cn/ajax/spot/history/202109140005",
+		Frequency: "日度",
+		Unit:      "美元/湿吨",
+	},
+	{
+		Name:      `菲律宾红土镍矿1.3%(FOB)`,
+		Url:       "https://hq.smm.cn/ajax/spot/history/202109140002",
+		Frequency: "日度",
+		Unit:      "美元/湿吨",
+	},
+	{
+		Name:      `菲律宾红土镍矿1.4%(FOB)`,
+		Url:       "https://hq.smm.cn/ajax/spot/history/202109140004",
+		Frequency: "日度",
+		Unit:      "美元/湿吨",
+	},
+	{
+		Name:      `菲律宾红土镍矿1.5%(FOB)`,
+		Url:       "https://hq.smm.cn/ajax/spot/history/201509220002",
+		Frequency: "日度",
+		Unit:      "美元/湿吨",
+	},
+	{
+		Name:      `红土镍矿1.8%(FOB)`,
+		Url:       "https://hq.smm.cn/ajax/spot/history/201509220003",
+		Frequency: "日度",
+		Unit:      "美元/湿吨",
+	},
+	{
+		Name:      `8-12%高镍生铁(出厂价)`,
+		Url:       "https://hq.smm.cn/ajax/spot/history/201106150005",
+		Frequency: "日度",
+		Unit:      "元/镍点",
+	},
+	{
+		Name:      `1.5-1.7%镍生铁(出厂价)`,
+		Url:       "https://hq.smm.cn/ajax/spot/history/201106150007",
+		Frequency: "日度",
+		Unit:      "元/吨",
+	},
+}
+
+var copperReqs = []SmmRequest{
+	{
+		Name:      "SMM 1#电解铜",
+		Url:       "https://hq.smm.cn/ajax/spot/history/201102250376",
+		Unit:      "元/吨",
+		Frequency: "日度",
+	},
+	{
+		Name:      "SMM 1#电解铜升贴水最大值",
+		Url:       "https://hq.smm.cn/ajax/premium/history/201102250185",
+		Value:     "highs",
+		Frequency: "日度",
+	},
+	{
+		Name:      "SMM 1#电解铜升贴水最小值",
+		Url:       "https://hq.smm.cn/ajax/premium/history/201102250185",
+		Value:     "low",
+		Frequency: "日度",
+	},
+	{
+		Name:      "SMM 广东1#电解铜",
+		Url:       "https://hq.smm.cn/ajax/spot/history/201912300001",
+		Unit:      "元/吨",
+		Frequency: "日度",
+	},
+	{
+		Name:      "SMM 广东1#电解铜升贴水最大值",
+		Url:       "https://hq.smm.cn/ajax/premium/history/201912300002",
+		Value:     "highs",
+		Frequency: "日度",
+	},
+	{
+		Name:      "SMM 广东1#电解铜升贴水最小值",
+		Url:       "https://hq.smm.cn/ajax/premium/history/201912300002",
+		Value:     "low",
+		Frequency: "日度",
+	},
+	{
+		Name:      "进口铜精矿指数(周)",
+		Url:       "https://hq.smm.cn/ajax/spot/history/201910240001",
+		Unit:      "美元/吨",
+		Frequency: "周度",
+	},
+	{
+		Name:      "废铜:广东:1#光亮铜线",
+		Url:       "https://hq.smm.cn/ajax/spot/history/201108090036",
+		Unit:      "元/吨",
+		Frequency: "日度",
+	},
+}
+
+var zincReqs = []SmmRequest{
+	{
+		Name:      "SMM 0#锌锭",
+		Url:       "https://hq.smm.cn/ajax/spot/history/201102250173",
+		Unit:      "元/吨",
+		Frequency: "日度",
+	},
+	{
+		Name:      "SMM 1#锌锭",
+		Url:       "https://hq.smm.cn/ajax/spot/history/201102250418",
+		Unit:      "元/吨",
+		Frequency: "日度",
+	},
+	// {
+	// 	Name:      "SMM 0#锌锭溢价(广东)",
+	// 	Url:       "https://hq.smm.cn/ajax/spot/history/202010210003",
+	// 	Unit:      "元/吨",
+	// 	Frequency: "日度",
+	// },
+	{
+		Name:      "SMM 0#锌锭(广东)",
+		Url:       "https://hq.smm.cn/ajax/spot/history/201102250231",
+		Unit:      "元/吨",
+		Frequency: "日度",
+	},
+	// {
+	// 	Name:      "SMM 1#锌锭溢价(广东)",
+	// 	Url:       "https://hq.smm.cn/ajax/spot/history/202010210004",
+	// 	Unit:      "元/吨",
+	// 	Frequency: "日度",
+	// },
+	{
+		Name:      "SMM 1#锌锭(广东)",
+		Url:       "https://hq.smm.cn/ajax/spot/history/201102250069",
+		Unit:      "元/吨",
+		Frequency: "日度",
+	},
+	// {
+	// 	Name:      "SMM 0#锌锭溢价(天津)",
+	// 	Url:       "https://hq.smm.cn/ajax/spot/history/202010210001",
+	// 	Unit:      "元/吨",
+	// 	Frequency: "日度",
+	// },
+	{
+		Name:      "SMM 1#锌锭(天津)",
+		Url:       "https://hq.smm.cn/ajax/spot/history/201102250556",
+		Unit:      "元/吨",
+		Frequency: "日度",
+	},
+	{
+		Name:      "SMM 0#锌锭(天津)",
+		Url:       "https://hq.smm.cn/ajax/spot/history/201102250399",
+		Unit:      "元/吨",
+		Frequency: "日度",
+	},
+	// {
+	// 	Name:      "SMM 1#锌锭溢价(天津)",
+	// 	Url:       "https://hq.smm.cn/ajax/spot/history/202010210002",
+	// 	Unit:      "元/吨",
+	// 	Frequency: "日度",
+	// },
+	// {
+	// 	Name:      "SMM 0#锌锭溢价(宁波)",
+	// 	Url:       "https://hq.smm.cn/ajax/spot/history/202010210005",
+	// 	Unit:      "元/吨",
+	// 	Frequency: "日度",
+	// },
+	// {
+	// 	Name:      "SMM 0#锌锭(宁波)",
+	// 	Url:       "https://hq.smm.cn/ajax/spot/history/202004070006",
+	// 	Unit:      "元/吨",
+	// 	Frequency: "日度",
+	// },
+	{
+		Name:      "Zn50国产TC(月)",
+		Url:       "https://hq.smm.cn/ajax/spot/history/201312030008",
+		Unit:      "元/金属吨",
+		Frequency: "月度",
+	},
+	{
+		Name:      "Zn50进口TC(月)",
+		Url:       "https://hq.smm.cn/ajax/spot/history/201312030009",
+		Unit:      "美元/千吨",
+		Frequency: "月度",
+	},
+	{
+		Name:      "Zn50国产TC(周)",
+		Url:       "https://hq.smm.cn/ajax/spot/history/202004070002",
+		Unit:      "元/金属吨",
+		Frequency: "周度",
+	},
+	{
+		Name:      "Zn50进口TC(周)",
+		Url:       "https://hq.smm.cn/ajax/spot/history/202004070001",
+		Unit:      "美元/千吨",
+		Frequency: "周度",
+	},
+	{
+		Name:      "Zn50内蒙古国产TC(周)",
+		Url:       "https://hq.smm.cn/ajax/spot/history/201504080006",
+		Unit:      "元/金属吨",
+		Frequency: "周度",
+	},
+	{
+		Name:      "Zn50陕西国产TC(周)",
+		Url:       "https://hq.smm.cn/ajax/spot/history/201504080007",
+		Unit:      "元/金属吨",
+		Frequency: "周度",
+	},
+	{
+		Name:      "Zn50甘肃国产TC(周)",
+		Url:       "https://hq.smm.cn/ajax/spot/history/202108050001",
+		Unit:      "元/金属吨",
+		Frequency: "周度",
+	},
+	{
+		Name:      "Zn50四川国产TC(周)",
+		Url:       "https://hq.smm.cn/ajax/spot/history/201504080008",
+		Unit:      "元/金属吨",
+		Frequency: "周度",
+	},
+	{
+		Name:      "Zn50广西国产TC(周)",
+		Url:       "https://hq.smm.cn/ajax/spot/history/201504080009",
+		Unit:      "元/金属吨",
+		Frequency: "周度",
+	},
+	{
+		Name:      "Zn50云南国产TC(周)",
+		Url:       "https://hq.smm.cn/ajax/spot/history/201504080010",
+		Unit:      "元/金属吨",
+		Frequency: "周度",
+	},
+	{
+		Name:      "Zn50湖南国产TC(周)",
+		Url:       "https://hq.smm.cn/ajax/spot/history/201504080011",
+		Unit:      "元/金属吨",
+		Frequency: "周度",
+	},
+}
+
+type SmmResponse struct {
+	Code int
+	Data []*SmmData
+	Msg  string
+}
+
+type SmmData struct {
+	Highs           float64 `json:"highs"`
+	Low             float64 `json:"low"`
+	Average         float64 `json:"average"`
+	VchangeRate     float64 `json:"vchange_rate"`
+	LowShow         string  `json:"low_show"`
+	HighShow        string  `json:"high_show"`
+	AverageShow     string  `json:"average_show"`
+	ProductId       string  `json:"product_id"`
+	Vchange         float64 `json:"vchange"`
+	RenewDate       string  `json:"renew_date"`
+	ChangeValueShow string  `json:"change_value_show"`
+	ChangeRateShow  string  `json:"change_rate_show"`
+}
+
+type EdbInfoData struct {
+	ClassifyName string
+	IndexName    string
+	IndexCode    string
+	Frequency    string
+	Unit         string
+	Value        string
+	LastDate     time.Time
+	OldDate      time.Time
+	Data         map[string]float64
+	SmmData
+}
+
+func SyncShangHaiSmm(cont context.Context) (err error) {
+	upMonth := time.Now().AddDate(0, -1, 0).Format(utils.FormatDate)
+	curDate := time.Now().Format(utils.FormatDate)
+	reqList := make([]SmmRequest, 0)
+	reqList = append(reqList, zincReqs...)
+	reqList = append(reqList, copperReqs...)
+	reqList = append(reqList, nickelReqs...)
+
+	edbInfoList := make([]*EdbInfoData, 0)
+	for _, v := range reqList {
+		url := fmt.Sprintf("%s/%s/%s", v.Url, upMonth, curDate)
+		req, e := http.NewRequest("GET", url, nil)
+		if e != nil {
+			fmt.Println(e)
+			return
+		}
+		// 发送请求
+		client := &http.Client{}
+		resp, e := client.Do(req)
+		if e != nil {
+			fmt.Println(e)
+			return
+		}
+		var respData SmmResponse
+		body, er := io.ReadAll(resp.Body)
+		if er != nil {
+			fmt.Println(er)
+			return er
+		}
+		if err = json.Unmarshal(body, &respData); err != nil {
+			fmt.Println(err)
+			return
+		}
+		if respData.Code != 0 {
+			fmt.Printf("上海有色爬虫请求失败, 指标名称:%s, 指标地址:%s", v.Name, v.Url)
+			continue
+		}
+		data := respData.Data
+		edbInfoList = append(edbInfoList, &EdbInfoData{
+			IndexName: v.Name,
+			Frequency: v.Frequency,
+			Unit:      v.Unit,
+			Value:     v.Value,
+			SmmData:   *data[len(data)-1],
+		})
+	}
+
+	urlStr := `shanghai_smm/refresh/list`
+	postUrl := utils.EDB_LIB_URL + urlStr
+	postData, err := json.Marshal(edbInfoList)
+	if err != nil {
+		utils.FileLog.Info("Marshal Err:" + err.Error())
+		return
+	}
+	result, err := sci99.HttpPost(postUrl, string(postData), "application/json")
+	if err != nil {
+		utils.FileLog.Info("HttpPost Err:" + err.Error())
+		return
+	}
+	resp := new(models.BaseResponse)
+	err = json.Unmarshal(result, &resp)
+	if err != nil {
+		fmt.Println(err)
+		utils.FileLog.Info("Unmarshal resp Err:" + err.Error())
+		return
+	}
+	if resp.Ret != 200 {
+		fmt.Println("上海有色爬虫更新失败")
+		return
+	}
+	return
+}
+
+// 解析读取历史excel数据,将历史数据转化为Smm结构体
+func ExcelToSmm(f *excelize.File, s string) (edbInfoList []*EdbInfoData) {
+	cols, err := f.GetCols(s)
+	if err != nil {
+		fmt.Println(err)
+		return
+	}
+	date := cols[0][4:]
+	dateFormat := make([]string, 0)
+	for _, v := range date {
+		if v == "" {
+			continue
+		}
+		var parseDate time.Time
+		var err error
+		if strings.Contains(s, "锌") {
+			parseDate, err = time.Parse(utils.FormatDate, v)
+		} else {
+			parseDate, err = time.Parse("01-02-06", v)
+		}
+		if err != nil {
+			fmt.Println(err)
+			return
+		}
+		dateFormat = append(dateFormat, parseDate.Format(utils.FormatDate))
+	}
+
+	for i := 1; i < len(cols); i++ {
+		tmpEdbInfo := new(EdbInfoData)
+		tmpEdbInfo.IndexName = cols[i][0]
+		tmpEdbInfo.IndexCode = cols[i][1]
+		tmpEdbInfo.Frequency = cols[i][2]
+		tmpEdbInfo.Unit = cols[i][3]
+		tmpEdbInfo.Data = make(map[string]float64)
+		tmpEdbInfo.OldDate = time.Now()
+		for j := 4; j < len(cols[i]); j++ {
+			floatVal, err := strconv.ParseFloat(cols[i][j], 64)
+			if err != nil {
+				fmt.Println("无效字符", cols[i][j])
+				continue
+			}
+			tmpEdbInfo.Data[dateFormat[j-4]] = floatVal
+			curTime, _ := time.Parse(utils.FormatDate, dateFormat[j-4])
+			if curTime.After(tmpEdbInfo.LastDate) {
+				tmpEdbInfo.LastDate = curTime
+			}
+			if curTime.Before(tmpEdbInfo.OldDate) {
+				tmpEdbInfo.OldDate = curTime
+			}
+		}
+		edbInfoList = append(edbInfoList, tmpEdbInfo)
+	}
+	return
+}
+
+func OldExcel() {
+	excelPath := ""
+	f, err := excelize.OpenFile(excelPath)
+	if err != nil {
+		fmt.Print(err)
+		return
+	}
+	var edbClassify = []string{`镍`, `铜`, `锌`, `锌(日度)`}
+	for _, v := range edbClassify {
+		edbInfoList := ExcelToSmm(f, v)
+		_, err := json.Marshal(edbInfoList)
+		if err != nil {
+			fmt.Println(err)
+			return
+		}
+		fmt.Println("成功:", v)
+		urlStr := `shanghai_smm/refresh/excel`
+		postUrl := utils.EDB_LIB_URL + urlStr
+		postData, err := json.Marshal(edbInfoList)
+		if err != nil {
+			utils.FileLog.Info("Marshal Err:" + err.Error())
+			return
+		}
+		result, err := sci99.HttpPost(postUrl, string(postData), "application/json")
+		if err != nil {
+			utils.FileLog.Info("HttpPost Err:" + err.Error())
+			return
+		}
+		resp := new(models.BaseResponse)
+		err = json.Unmarshal(result, &resp)
+		if err != nil {
+			fmt.Println(err)
+			utils.FileLog.Info("Unmarshal resp Err:" + err.Error())
+			return
+		}
+		if resp.Ret != 200 {
+			fmt.Println("上海有色网excel历史有色数据更新失败")
+			utils.FileLog.Info("上海有色网excel历史有色数据更新失败")
+			return
+		}
+		time.Sleep(10 * time.Second)
+	}
+}

services/task.go

@@ -2,11 +2,13 @@ package services
 
 import (
 	"context"
+	"eta/eta_crawler/services/sci99"
 	"eta/eta_crawler/utils"
 	"fmt"
-	"github.com/beego/beego/v2/task"
 	"strconv"
 	"time"
+
+	"github.com/beego/beego/v2/task"
 )
 
 func Task() {
@@ -20,35 +22,46 @@ func Task() {
 	//SyncRankingFromIne()
 	//FileCoalCoastal()
 	//FileCoalInland()
-	refreshData := task.NewTask("refreshData", "0 0,30 16-18 * * *", RefreshData)
-	refreshEic := task.NewTask("RefreshEic", "0 0 2,6 * * *", RefreshEic)
-	refreshCoal := task.NewTask("RefreshCoal", "0 0,30 16-23 * * *", RefreshCoal)
-	refreshMeeting := task.NewTask("RefreshMeeting", "0 15 7-19/2 * * *", RefreshMeetingProbabilities)
-	//	refreshVisitors := task.NewTask("RefreshChangesVisitorsCovid", "0 30 2-22/10 * * *", RefreshChangesVisitorsCovid)
-	syncEiaSteoData := task.NewTask("SyncEiaSteoData", "0 0 22 * * *", SyncEiaSteoData)
-	syncYearComTrade := task.NewTask("SyncYearComTrade", "0 0 3 1 1 *", SyncYearComTrade)                // 每年一月一号同步
-	syncYearMonthComTrade := task.NewTask("SyncYearMonthComTrade", "0 0 3 1 * *", SyncYearMonthComTrade) // 每月1号同步
-
-	crawlerIcpi := task.NewTask("refreshData", "0 0,30 16-23 * * *", CrawlerIcpi) //居民消费价格指数
-
-	// 统计局-分月季年爬
-	//refreshNationalMonthA := task.NewTask("RefreshNationalMonthDbA", "0 15 2 10 * *", national_data.RefreshNationalMonthDbA)
-	//refreshNationalMonthB := task.NewTask("RefreshNationalMonthDbB", "0 15 2 16 * *", national_data.RefreshNationalMonthDbB)
-	//refreshNationalQuarter := task.NewTask("RefreshNationalQuarterDb", "0 25 1 15 * *", national_data.RefreshNationalQuarterDb)
-	//refreshNationalYearA := task.NewTask("RefreshNationalYearDbA", "0 45 1 20 * *", national_data.RefreshNationalYearDbA)
-	//refreshNationalYearB := task.NewTask("RefreshNationalYearDbB", "0 45 1 25 * *", national_data.RefreshNationalYearDbB)
-
-	task.AddTask("数据爬取", refreshData)
-	task.AddTask("欧洲天然气爬取", refreshEic)
-	task.AddTask("中国煤炭网爬取", refreshCoal)
-	task.AddTask("美联储加息概率爬取", refreshMeeting)
-	//task.AddTask("谷歌出行指数爬取", refreshVisitors)
-	task.AddTask("eia steo报告", syncEiaSteoData)   //每天22点爬一次
-	task.AddTask("UN年度数据", syncYearComTrade)      //每年一月一号的3点同步
-	task.AddTask("UN月度数据", syncYearMonthComTrade) //每月1号的3点同步
-
-	task.AddTask("居民消费价格指数", crawlerIcpi) //每月1号的3点同步
+	if utils.BusinessCode == utils.BusinessCodeRelease {
+		refreshData := task.NewTask("refreshData", "0 0,30 16-18 * * *", RefreshData)
+		refreshEic := task.NewTask("RefreshEic", "0 0 2,6 * * *", RefreshEic)
+		refreshCoal := task.NewTask("RefreshCoal", "0 0,30 16-23 * * *", RefreshCoal)
+		refreshMeeting := task.NewTask("RefreshMeeting", "0 15 7-19/2 * * *", RefreshMeetingProbabilities)
+		//	refreshVisitors := task.NewTask("RefreshChangesVisitorsCovid", "0 30 2-22/10 * * *", RefreshChangesVisitorsCovid)
+		syncEiaSteoData := task.NewTask("SyncEiaSteoData", "0 0 22 * * *", SyncEiaSteoData)
+		syncYearComTrade := task.NewTask("SyncYearComTrade", "0 0 3 1 1 *", SyncYearComTrade)                // 每年一月一号同步
+		syncYearMonthComTrade := task.NewTask("SyncYearMonthComTrade", "0 0 3 1 * *", SyncYearMonthComTrade) // 每月1号同步
+
+		crawlerIcpi := task.NewTask("refreshData", "0 0,30 16-23 * * *", CrawlerIcpi) //居民消费价格指数
+
+		// 统计局-分月季年爬
+		//refreshNationalMonthA := task.NewTask("RefreshNationalMonthDbA", "0 15 2 10 * *", national_data.RefreshNationalMonthDbA)
+		//refreshNationalMonthB := task.NewTask("RefreshNationalMonthDbB", "0 15 2 16 * *", national_data.RefreshNationalMonthDbB)
+		//refreshNationalQuarter := task.NewTask("RefreshNationalQuarterDb", "0 25 1 15 * *", national_data.RefreshNationalQuarterDb)
+		//refreshNationalYearA := task.NewTask("RefreshNationalYearDbA", "0 45 1 20 * *", national_data.RefreshNationalYearDbA)
+		//refreshNationalYearB := task.NewTask("RefreshNationalYearDbB", "0 45 1 25 * *", national_data.RefreshNationalYearDbB)
+
+		task.AddTask("数据爬取", refreshData)
+		task.AddTask("欧洲天然气爬取", refreshEic)
+		task.AddTask("中国煤炭网爬取", refreshCoal)
+		task.AddTask("美联储加息概率爬取", refreshMeeting)
+		//task.AddTask("谷歌出行指数爬取", refreshVisitors)
+		task.AddTask("eia steo报告", syncEiaSteoData)   //每天22点爬一次
+		task.AddTask("UN年度数据", syncYearComTrade)      //每年一月一号的3点同步
+		task.AddTask("UN月度数据", syncYearMonthComTrade) //每月1号的3点同步
+
+		task.AddTask("居民消费价格指数", crawlerIcpi) //每月1号的3点同步
+	}
+
+	if utils.BusinessCode == utils.BusinessCodeFuBang {
+		// 仅富邦
+		sci99 := task.NewTask("refreshSci99", "0 0,30 16-20 * * *", sci99.Scii99) //卓创资讯
+		task.AddTask("卓创资讯", sci99)
+	}
+
+	crawlerSmm := task.NewTask("refreshShangHaiSmmData", "0 0 10-18 * * *", SyncShangHaiSmm) // 上海有色网爬虫
 
+	task.AddTask("上海有色网指标爬取", crawlerSmm) //每天10-18点, 每小时爬一次
 	//task.AddTask("统计局数据爬取-月度A", refreshNationalMonthA) // 每月10号2:15执行
 	//task.AddTask("统计局数据爬取-月度B", refreshNationalMonthB) // 每月16号2:15执行
 	//task.AddTask("统计局数据爬取-季度", refreshNationalQuarter) // 每月15号1:25执行

utils/config.go

@@ -24,6 +24,20 @@ var (
 	LogMaxDays int //日志最大保留天数
 )
 
+var BusinessCode string //商户号
+
+// 卓创资讯
+var (
+	SCI99_COOKIE_PATH string
+	IS_INIT_SCI99     string
+)
+
+var (
+	EDB_LIB_URL         string
+	APP_EDB_LIB_NAME_EN string
+	EDB_LIB_Md5_KEY     string
+)
+
 func init() {
 	tmpRunMode, err := web.AppConfig.String("run_mode")
 	if err != nil {
@@ -50,8 +64,22 @@ func init() {
 		panic("配置文件读取错误 " + err.Error())
 	}
 	beeLogger.Log.Info(RunMode + " 模式")
-	MYSQL_URL = config["mysql_url"]
-	MYSQL_URL_DATA = config["mysql_url_data"]
+
+	// 商家编码
+	BusinessCode = config["business_code"]
+
+	if BusinessCode == BusinessCodeRelease {
+		MYSQL_URL = config["mysql_url"]
+		MYSQL_URL_DATA = config["mysql_url_data"]
+	}
+
+	if BusinessCode == BusinessCodeFuBang {
+		SCI99_COOKIE_PATH = config["sci99_cookie_path"]
+		IS_INIT_SCI99 = config["is_init_sci99"]
+		EDB_LIB_URL = config["edb_lib_url"]
+		APP_EDB_LIB_NAME_EN = config["app_edb_lib_name_en"]
+		EDB_LIB_Md5_KEY = config["edb_lib_md5_key"]
+	}
 
 	if RunMode == "release" {
 

utils/constants.go

@@ -4,10 +4,11 @@ const (
 	Md5Key = "Ks@h64WJ#tcVgG8$&WlNfqvLAtMgpxWN"
 )
 
-//常量定义
+// 常量定义
 const (
 	FormatTime             = "15:04:05"                //时间格式
 	FormatDate             = "2006-01-02"              //日期格式
+	FormatDate2            = "2006/01/02"              //日期格式
 	FormatDateUnSpace      = "20060102"                //日期格式
 	FormatDateTime         = "2006-01-02 15:04:05"     //完整时间格式
 	HlbFormatDateTime      = "2006-01-02_15:04:05.999" //完整时间格式
@@ -24,3 +25,8 @@ const (
 	APPNAME          = "弘则-数据爬虫"
 	EmailSendToUsers = "glji@hzinsights.com;pyan@hzinsights.com;cxzhang@hzinsights.com;zwxi@hzinsights.com;"
 )
+
+const (
+	BusinessCodeRelease = "E2023080900" // 生产环境
+	BusinessCodeFuBang  = "E2024020200"
+)