Merge branch 'feature/eta2.5.7_purang' of eta_server/eta_crawler into master

services/purang/model.go

@@ -0,0 +1,152 @@
+package purang
+
+import (
+	"crypto/md5"
+	"crypto/sha512"
+	"encoding/hex"
+	"fmt"
+	"sort"
+	"strings"
+	"time"
+)
+
+// Request 请求参数结构体
+type Request struct {
+	CompanyCode string    `json:"companyCode"` // 公司代码
+	StartTime   time.Time `json:"startTime"`   // 开始时间
+	EndTime     time.Time `json:"endTime"`     // 结束时间
+}
+
+// Response 响应结构体
+type Response struct {
+	Code    int         `json:"code"`    // 响应代码
+	Message string      `json:"message"` // 响应消息
+	Data    []DataItem  `json:"data"`    // 数据列表
+}
+
+// DataItem 数据项结构体
+type DataItem struct {
+	DeviceID    string    `json:"deviceId"`    // 设备ID
+	DeviceName  string    `json:"deviceName"`  // 设备名称
+	DataTime    time.Time `json:"dataTime"`    // 数据时间
+	Temperature float64   `json:"temperature"` // 温度值
+	Humidity    float64   `json:"humidity"`    // 湿度值
+	Status      int       `json:"status"`      // 设备状态
+}
+
+// DirectPasteRequest 直贴报价/成交行情请求参数
+type DirectPasteRequest struct {
+	StartDate string `json:"startDate"` // 开始日期，格式：yyyy-MM-dd
+	EndDate   string `json:"endDate"`   // 结束日期，格式：yyyy-MM-dd
+	PageNo    int    `json:"pageNo"`    // 页码，从1开始
+	PageSize  int    `json:"pageSize"`  // 每页记录数
+}
+
+// DirectPasteResponse 直贴报价/成交行情响应
+type DirectPasteResponse struct {
+	Code      int                `json:"code"`      // 响应代码
+	Message   string             `json:"message"`   // 响应消息
+	Data      []DirectPasteData  `json:"data"`      // 数据列表
+	Total     int               `json:"total"`     // 总记录数
+	PageNo    int               `json:"pageNo"`    // 当前页码
+	PageSize  int               `json:"pageSize"`  // 每页记录数
+}
+
+// DirectPasteData 直贴报价/成交行情数据项
+type DirectPasteData struct {
+	TradeDate     string  `json:"tradeDate"`     // 交易日期，格式：yyyy-MM-dd
+	BankName      string  `json:"bankName"`      // 银行名称
+	MinRate       float64 `json:"minRate"`       // 最低利率（%）
+	MaxRate       float64 `json:"maxRate"`       // 最高利率（%）
+	AverageRate   float64 `json:"averageRate"`   // 平均利率（%）
+	TotalAmount   float64 `json:"totalAmount"`   // 总金额（万元）
+	DealCount     int     `json:"dealCount"`     // 成交笔数
+	BusinessType  string  `json:"businessType"`  // 业务类型（报价/成交）
+	Deadline      int     `json:"deadline"`      // 期限（天）
+}
+
+// ErrorResponse 错误响应结构体
+type ErrorResponse struct {
+	Code    int    `json:"code"`
+	Message string `json:"message"`
+}
+
+// BaseRequest 基础请求结构
+type BaseRequest struct {
+	SpNo     string `json:"sp_no"`     // 用户编号（6位数字）
+	SignType string `json:"sign_type"` // 签名类型：1（MD5），2（SHA512）
+	ReqID    string `json:"req_id"`    // 平台流水号
+	Sign     string `json:"sign"`      // 签名
+}
+
+// BaseResponse 基础响应结构
+type BaseResponse struct {
+	Code     string      `json:"code"`      // 响应状态码
+	Message  string      `json:"message"`   // 响应状态描述
+	SignType string      `json:"sign_type"` // 签名类型
+	ReqID    string      `json:"req_id"`    // 平台流水号
+	Data     interface{} `json:"data"`      // 数据内容
+}
+
+// SignConfig 签名配置
+type SignConfig struct {
+	Key      string // API密钥
+	SignType string // 签名类型：1（MD5），2（SHA512）
+}
+
+// GenerateSign 生成签名
+func GenerateSign(params map[string]string, config SignConfig) (string, error) {
+	// 1. 按参数名称字符升序排列
+	var keys []string
+	for k := range params {
+		if k != "sign" { // 排除sign参数
+			keys = append(keys, k)
+		}
+	}
+	sort.Strings(keys)
+
+	// 2. 拼接待签名字符串
+	var signParts []string
+	for _, k := range keys {
+		if params[k] != "" { // 空值不参与签名
+			signParts = append(signParts, fmt.Sprintf("%s=%s", k, params[k]))
+		}
+	}
+	
+	// 3. 添加密钥
+	signParts = append(signParts, fmt.Sprintf("key=%s", config.Key))
+	signStr := strings.Join(signParts, "&")
+	//fmt.Println("生成明文",signStr)
+	// 4. 根据签名类型选择相应的摘要算法
+	var signBytes []byte
+	switch config.SignType {
+	case "1": // MD5
+		hash := md5.New()
+		hash.Write([]byte(signStr))
+		signBytes = hash.Sum(nil)
+	case "2": // SHA512
+		hash := sha512.New()
+		hash.Write([]byte(signStr))
+		signBytes = hash.Sum(nil)
+	default:
+		return "", fmt.Errorf("unsupported sign type: %s", config.SignType)
+	}
+
+	// 5. 转换为小写十六进制字符串
+	return hex.EncodeToString(signBytes), nil
+}
+
+// VerifySign 验证签名
+func VerifySign(params map[string]string, config SignConfig) (bool, error) {
+	originalSign := params["sign"]
+	if originalSign == "" {
+		return false, fmt.Errorf("sign parameter is missing")
+	}
+
+	calculatedSign, err := GenerateSign(params, config)
+	if err != nil {
+		return false, err
+	}
+
+	return strings.EqualFold(originalSign, calculatedSign), nil
+} 

services/purang/purang_data.go

@@ -0,0 +1,171 @@
+package purang
+
+import (
+	"context"
+	"encoding/json"
+	"eta/eta_crawler/services/alarm_msg"
+	"eta/eta_crawler/utils"
+	"fmt"
+	"os"
+	"time"
+)
+func GetPurangDataTask(ctx context.Context) (err error) {
+	beginDate := time.Now().AddDate(0, -1, 0).Format("20060102")
+	endDate := time.Now().Format("20060102")
+	err = GetPurangMarketData(beginDate, endDate)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func GetPurangMarketData(startDate, endDate string) (err error) {
+	utils.FileLog.Info(fmt.Sprintf("GetPurangMarketData startDate: %s, endDate: %s", startDate, endDate))
+	defer func() {
+		if err != nil {
+			msg := fmt.Sprintf("GetPurangMarketData failed Error: %v", err)
+			utils.FileLog.Info(msg)
+			go alarm_msg.SendAlarmMsg(msg, 3)
+		}
+	}()
+	if utils.PurangApiUrl == "" || utils.PurangUserCode == "" || utils.PurangApiKey == "" || utils.PurangSignType == "" {
+		return fmt.Errorf("PurangApiUrl, PurangUserCode, PurangApiKey, PurangSignType is empty")
+	}
+	// 创建客户端实例
+	client := NewClient(
+		utils.PurangApiUrl, // 测试环境URL（需要替换为实际的URL）
+		utils.PurangUserCode,                       // 用户编号（需要替换为实际的编号）
+		utils.PurangApiKey,           // API密钥（需要替换为实际的密钥）
+		utils.PurangSignType,                           // 签名类型：1（MD5）
+	)
+
+	// 示例：发送直贴报价/成交行情请求
+	params := map[string]string{
+		"beginDate": startDate,
+		"endDate": endDate,
+	}
+
+	//fmt.Println("\nSending custom request...")
+	resp, err := client.DoRequest("/info/bill/market/main-price", params)
+	if err != nil {
+		err = fmt.Errorf("Custom request failed: %v", err)
+		return 
+	}
+
+	// fmt.Printf("Response received:\n")
+	// fmt.Printf("Code: %s\n", resp.Code)
+	// fmt.Printf("Message: %s\n", resp.Message)
+	// fmt.Printf("ReqID: %s\n", resp.ReqID)
+		
+
+	// 将响应数据转换成 json 字符串
+	jsonData, err := json.Marshal(resp.Data)
+	if err != nil {
+		err = fmt.Errorf("Failed to marshal response data: %v", err)
+		return err
+	}
+	//fmt.Println("Response data as JSON:", string(jsonData))
+
+	// 将 json 字符串转换为 map
+	dataListMap := make([]map[string]interface{}, 0)
+	if err := json.Unmarshal(jsonData, &dataListMap); err != nil {
+		err = fmt.Errorf("Failed to unmarshal response data: %v", err)
+		return err
+	}
+	//fmt.Println("Response data as map:", dataListMap)
+
+	// 遍历 map 中的每个键值对
+	indexDataMap := make(map[string]map[string]string)
+	for _, dataItem := range dataListMap {
+		date := dataItem["date"].(string)
+		// 格式化date，改成yyyy-mm-dd
+		dateTime, err := time.Parse("20060102", date)
+		if err != nil {
+			err = fmt.Errorf("Failed to parse date: %v", err)
+			return err
+		}
+		date = dateTime.Format("2006-01-02")
+		for key, value := range dataItem {
+			if key == "date" {
+				continue
+			}
+			if _, ok := value.(string); ok {
+				if _, ok := indexDataMap[key]; !ok {
+					indexDataMap[key] = make(map[string]string)
+				}
+				indexDataMap[key][date] = value.(string)
+			}
+		}
+		//fmt.Printf("Key: %s, Value: %v\n", date, dataItem)
+	}
+
+	//根据purang.json 文件中的数据，创建一个指标列表
+	purangJson, err := os.ReadFile("static/purang.json")
+	if err != nil {
+		err = fmt.Errorf("Failed to read purang.json: %v", err)
+		return err
+	}
+	var purangDataList []*PurangDataIndex
+	if err := json.Unmarshal(purangJson, &purangDataList); err != nil {
+		err = fmt.Errorf("Failed to unmarshal purang.json: %v", err)
+		return err
+	}
+
+	indexList := make([]*PurangDataIndex, 0)
+	for k, purangData := range purangDataList {
+		if data, ok := indexDataMap[purangData.IndexCode]; ok {
+			purangData.ExcelDataMap = data
+		}
+		purangData.Sort = k
+		indexList = append(indexList, purangData)
+	}
+	err = addPurangData(indexList)
+	if err != nil {
+		return err
+	}
+	utils.FileLog.Info("GetPurangMarketData end ")
+	return nil	
+}
+
+
+// PurangDataIndex 普兰数据指标数据
+type PurangDataIndex struct {
+	ClassifyName       string `description:"指标目录"`
+	ParentClassifyName string `description:"父级指标目录"`
+	ClassifySort       int    `description:"指标目录排序号"`
+	IndexName          string `description:"指标名称"`
+	IndexCode          string `description:"指标编码"`
+	Unit               string `description:"单位"`
+	Sort               int    `description:"排序号"`
+	Frequency          string `description:"频度"`
+	TerminalCode       string `description:"编码"`
+	ExcelDataMap       map[string]string
+}
+
+func addPurangData(indexList []*PurangDataIndex) (err error) {
+	sheetName := "普兰数据指标"
+	if len(indexList) > 0 {
+		params := make(map[string]interface{})
+		params["List"] = indexList
+		params["TerminalCode"] = ""
+		result, e := utils.PostEdbLib(params, "purang/handle/excel_data")
+		if e != nil {
+			err = fmt.Errorf("sheet ：%s PostEdbLib err: %s", sheetName, e.Error())
+			b, _ := json.Marshal(params)
+			utils.FileLog.Info(fmt.Sprintf("sheet ：%s PostEdbLib err: %s, params: %s", sheetName, e.Error(), string(b)))
+			return
+		}
+		resp := new(utils.BaseEdbLibResponse)
+		if e := json.Unmarshal(result, &resp); e != nil {
+			err = fmt.Errorf("sheet ：%s json.Unmarshal err: %s", sheetName, e)
+			utils.FileLog.Info(fmt.Sprintf("sheet ：%s json.Unmarshal err: %s", sheetName, e))
+			return
+		}
+		if resp.Ret != 200 {
+			err = fmt.Errorf("sheet ：%s Msg: %s, ErrMsg: %s", sheetName, resp.Msg, resp.ErrMsg)
+			utils.FileLog.Info(fmt.Sprintf("sheet ：%s Msg: %s, ErrMsg: %s", sheetName, resp.Msg, resp.ErrMsg))
+			return
+		}
+	}
+	return
+}

services/task.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"eta/eta_crawler/services/email"
 	"eta/eta_crawler/services/liangyou"
+	"eta/eta_crawler/services/purang"
 	"eta/eta_crawler/services/sci99"
 	"eta/eta_crawler/utils"
 	"fmt"
@@ -40,6 +41,9 @@ func Task() {
 		crawlerIcpi := task.NewTask("refreshData", "0 0,30 16-23 * * *", CrawlerIcpi) //居民消费价格指数
 
 		downloadGPRRiskTask := task.NewTask("DownloadGPRRiskTask", "0 0,30 16-19 * * *", DownloadGPRRiskTask) //每天下午16点至19点，每隔半小时
+
+		purangDataTask := task.NewTask("GetPurangDataTask", "0 0 16 * * *", purang.GetPurangDataTask) //每天下午16点至19点，每隔半小时
+		
 		// 统计局-分月季年爬
 		//refreshNationalMonthA := task.NewTask("RefreshNationalMonthDbA", "0 15 2 10 * *", national_data.RefreshNationalMonthDbA)
 		//refreshNationalMonthB := task.NewTask("RefreshNationalMonthDbB", "0 15 2 16 * *", national_data.RefreshNationalMonthDbB)
@@ -60,7 +64,7 @@ func Task() {
 		task.AddTask("UN月度数据", syncYearMonthComTrade) //每月1号的3点同步
 
 		task.AddTask("居民消费价格指数", crawlerIcpi) //每月1号的3点同步
-
+		task.AddTask("Purang数据爬取", purangDataTask)	//每天下午16点更新数据
 	}
 
 	if utils.LY_OPEN == "1" {

static/purang.json

@@ -0,0 +1,192 @@
+[
+    {
+        "IndexName": "直贴3M国股利率",
+        "IndexCode": "ebaCountry3m",
+        "Unit": "%",
+        "Frequency": "日度",
+        "ClassifyName": "市场主流价格统计"
+    },
+    {
+        "IndexName": "直贴3M城商利率",
+        "IndexCode": "ebaUrban3m",
+        "Unit": "%",
+        "Frequency": "日度",
+        "ClassifyName": "市场主流价格统计"
+    },
+    {
+        "IndexName": "直贴3M三利率",
+        "IndexCode": "ebaRural3m",
+        "Unit": "%",
+        "Frequency": "日度",
+        "ClassifyName": "市场主流价格统计"
+    },
+    {
+        "IndexName": "直贴半年国股利率",
+        "IndexCode": "ebaCountry6m",
+        "Unit": "%",
+        "Frequency": "日度",
+        "ClassifyName": "市场主流价格统计"
+    },
+    {
+        "IndexName": "直贴半年城商利率",
+        "IndexCode": "ebaUrban6m",
+        "Unit": "%",
+        "Frequency": "日度",
+        "ClassifyName": "市场主流价格统计"
+    },
+    {
+        "IndexName": "直贴半年三农利率",
+        "IndexCode": "ebaRural6m",
+        "Unit": "%",
+        "Frequency": "日度",
+        "ClassifyName": "市场主流价格统计"
+    },
+    {
+        "IndexName": "直贴足年国股利率",
+        "IndexCode": "ebaCountry12m",
+        "Unit": "%",
+        "Frequency": "日度",
+        "ClassifyName": "市场主流价格统计"
+    },
+    {
+        "IndexName": "直贴足年城商利率",
+        "IndexCode": "ebaUrban12m",
+        "Unit": "%",
+        "Frequency": "日度",
+        "ClassifyName": "市场主流价格统计"
+    },
+    {
+        "IndexName": "直贴足年三农利率",
+        "IndexCode": "ebaRural12m",
+        "Unit": "%",
+        "Frequency": "日度",
+        "ClassifyName": "市场主流价格统计"
+    },
+    {
+        "IndexName": "转贴(国股贴)3M国股利率",
+        "IndexCode": "bbaCountry3m",
+        "Unit": "%",
+        "Frequency": "日度",
+        "ClassifyName": "市场主流价格统计"
+    },
+    {
+        "IndexName": "转贴(国股贴)3M城商利率",
+        "IndexCode": "bbaUrban3m",
+        "Unit": "%",
+        "Frequency": "日度",
+        "ClassifyName": "市场主流价格统计"
+    },
+    {
+        "IndexName": "转贴(国股贴)3M三农利率",
+        "IndexCode": "bbaRural3m",
+        "Unit": "%",
+        "Frequency": "日度",
+        "ClassifyName": "市场主流价格统计"
+    },
+    {
+        "IndexName": "转贴(国股贴)半年国股利率",
+        "IndexCode": "bbaCountry6m",
+        "Unit": "%",
+        "Frequency": "日度",
+        "ClassifyName": "市场主流价格统计"
+    },
+    {
+        "IndexName": "转贴(国股贴)半年城商利率",
+        "IndexCode": "bbaUrban6m",
+        "Unit": "%",
+        "Frequency": "日度",
+        "ClassifyName": "市场主流价格统计"
+    },
+    {
+        "IndexName": "转贴(国股贴)半年三农利率",
+        "IndexCode": "bbaRural6m",
+        "Unit": "%",
+        "Frequency": "日度",
+        "ClassifyName": "市场主流价格统计"
+    },
+    {
+        "IndexName": "转贴(国股贴)足年国股利率",
+        "IndexCode": "bbaCountry12m",
+        "Unit": "%",
+        "Frequency": "日度",
+        "ClassifyName": "市场主流价格统计"
+    },
+    {
+        "IndexName": "转贴(国股贴)足年城商利率",
+        "IndexCode": "bbaUrban12m",
+        "Unit": "%",
+        "Frequency": "日度",
+        "ClassifyName": "市场主流价格统计"
+    },
+    {
+        "IndexName": "转贴(国股贴)足年三农利率",
+        "IndexCode": "bbaRural12m",
+        "Unit": "%",
+        "Frequency": "日度",
+        "ClassifyName": "市场主流价格统计"
+    },
+    {
+        "IndexName": "转贴(城农贴)3M国股利率",
+        "IndexCode": "bbabCountry3m",
+        "Unit": "%",
+        "Frequency": "日度",
+        "ClassifyName": "市场主流价格统计"
+    },
+    {
+        "IndexName": "转贴(城农贴)3M城商利率",
+        "IndexCode": "bbabUrban3m",
+        "Unit": "%",
+        "Frequency": "日度",
+        "ClassifyName": "市场主流价格统计"
+    },
+    {
+        "IndexName": "转贴(城农贴)3M三农利率",
+        "IndexCode": "bbabRural3m",
+        "Unit": "%",
+        "Frequency": "日度",
+        "ClassifyName": "市场主流价格统计"
+    },
+    {
+        "IndexName": "转贴(城农贴)半年国股利率",
+        "IndexCode": "bbabCountry6m",
+        "Unit": "%",
+        "Frequency": "日度",
+        "ClassifyName": "市场主流价格统计"
+    },
+    {
+        "IndexName": "转贴(城农贴)半年城商利率",
+        "IndexCode": "bbabUrban6m",
+        "Unit": "%",
+        "Frequency": "日度",
+        "ClassifyName": "市场主流价格统计"
+    },
+    {
+        "IndexName": "转贴(城农贴)半年三农利率",
+        "IndexCode": "bbabRural6m",
+        "Unit": "%",
+        "Frequency": "日度",
+        "ClassifyName": "市场主流价格统计"
+    },
+    {
+        "IndexName": "转贴(城农贴)足年国股利率",
+        "IndexCode": "bbabCountry12m",
+        "Unit": "%",
+        "Frequency": "日度",
+        "ClassifyName": "市场主流价格统计"
+    },
+    {
+        "IndexName": "转贴(城农贴)足年城商利率",
+        "IndexCode": "bbabUrban12m",
+        "Unit": "%",
+        "Frequency": "日度",
+        "ClassifyName": "市场主流价格统计"
+    },
+    {
+        "IndexName": "转贴(城农贴)足年三农利率",
+        "IndexCode": "bbabRural12m",
+        "Unit": "%",
+        "Frequency": "日度",
+        "ClassifyName": "市场主流价格统计"
+    }
+]
+

utils/config.go

@@ -67,6 +67,15 @@ var (
 	CoalEmailNDay          string // 中国煤炭网监听取最近N天数据
 )
 
+// 普兰金融
+var (
+	PurangApiUrl string
+	PurangUserCode string
+	PurangApiKey string
+	PurangSignType string
+)
+
+// 配置文件初
 func init() {
 	tmpRunMode, err := web.AppConfig.String("run_mode")
 	if err != nil {
@@ -163,4 +172,11 @@ func init() {
 		CoalEmailNDay = config["coal_email_n_day"]
 	}
 
+	// 普兰金融
+	{
+		PurangApiUrl = config["purang_api_url"]
+		PurangUserCode = config["purang_user_code"]
+		PurangApiKey = config["purang_api_key"]
+		PurangSignType = config["purang_sign_type"]
+	}
 }