新增有色网渠道

models/data_manage/edb_data_table.go → models/data_manage/edb_data_base.go

@@ -2,7 +2,9 @@ package data_manage
 
 import (
 	"errors"
+	"fmt"
 	"hongze/hongze_task/utils"
+	"rdluck_tools/orm"
 	"strconv"
 	"time"
 )
@@ -29,6 +31,8 @@ func GetEdbDataTableName(source int) (tableName string) {
 		tableName = "edb_data_manual"
 	case utils.DATA_SOURCE_LZ:
 		tableName = "edb_data_lz"
+	case utils.DATA_SOURCE_YS:
+		tableName = "edb_data_ys"
 	default:
 		tableName = ""
 		errors.New("无效的渠道:" + strconv.Itoa(source))
@@ -65,4 +69,14 @@ func GetAddSql(edbInfoId, edbCode, dataTime, timestampStr,value string) (addSql
 	addSql += "," + "'" + timestampStr + "'"
 	addSql += "),"
 	return
+}
+
+func GetEdbDataBaseByCodeAndDate(source int, edbCode string, startDate string) (count int, err error) {
+	o := orm.NewOrm()
+	o.Using("data")
+	tableName := GetEdbDataTableName(source)
+	sql := ` SELECT COUNT(1) AS count FROM %s WHERE edb_code=? AND data_time=? `
+	sql = fmt.Sprintf(sql, tableName)
+	err = o.Raw(sql, edbCode, startDate).QueryRow(&count)
+	return
 }

models/data_manage/edb_data_ys.go

@@ -0,0 +1,258 @@
+/**
+* @Author: jgl
+* @Date: 2021/9/15 10:15
+ */
+
+package data_manage
+
+import (
+	"crypto/md5"
+	"encoding/hex"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"hongze/hongze_task/utils"
+	"io/ioutil"
+	"net/http"
+	"net/url"
+	"rdluck_tools/orm"
+	"strconv"
+	"strings"
+	"time"
+)
+
+//刷新有色指标数据
+func RefreshEdbDataByYs(edbInfoId int, edbCode, startDate, endDate string) (err error) {
+	o := orm.NewOrm()
+	o.Using("data")
+	o.Begin()
+	defer func() {
+		if err != nil {
+			o.Rollback()
+		} else {
+			o.Commit()
+		}
+	}()
+
+	token, err := getToken("pqian@hzinsights.com", "hz123456")
+	if err != nil {
+		fmt.Println(err)
+		return
+	}
+
+	ysResult, err := getApiData(token, edbCode, startDate, endDate)
+	if err != nil {
+		fmt.Println(err)
+		return
+	}
+	edbInfoIdStr := strconv.Itoa(edbInfoId)
+	if ysResult != nil && len(ysResult.Data.Content) > 0 {
+		var isAdd bool
+		addSql := ` INSERT INTO edb_data_ys(edb_info_id,edb_code,data_time,value,create_time,modify_time,status,data_timestamp) values `
+		existMap := make(map[string]string)
+		frequency := ysResult.Data.Frequency
+		for _, rv := range ysResult.Data.Content {
+			var eDate, sValue string
+			for ck, cv := range rv {
+				if ck == 0 {
+					eDate = cv
+				}
+				if ck == 1 {
+					sValue = strings.Replace(cv, ",", "", -1)
+				}
+			}
+			if sValue=="" {
+				continue
+			}
+			if frequency == "月" {
+				monthDate, err := time.Parse("2006-01", eDate)
+				if err != nil {
+					fmt.Println("time.Parse:" + err.Error())
+				}
+				lastTime := monthDate.AddDate(0, 1, -1)
+				lastYear, lastMonth, lastDay := lastTime.Date()
+				var lastDate string
+				if int(lastMonth) < 10 {
+					lastDate = strconv.Itoa(lastYear) + "-" + "0" + strconv.Itoa(int(lastMonth)) + "-" + strconv.Itoa(lastDay)
+				} else {
+					lastDate = strconv.Itoa(lastYear) + "-" + strconv.Itoa(int(lastMonth)) + "-" + strconv.Itoa(lastDay)
+				}
+				eDate = lastDate
+			}
+			count, err := GetEdbDataBaseByCodeAndDate(utils.DATA_SOURCE_YS, edbCode, eDate)
+			if err != nil && err.Error() != utils.ErrNoRow() {
+				return err
+			}
+			if count <= 0 {
+				dataTime, err := time.Parse(utils.FormatDate, eDate)
+				if err != nil {
+					return err
+				}
+				timestamp := dataTime.UnixNano() / 1e6
+				timeStr := fmt.Sprintf("%d", timestamp)
+				if _, ok := existMap[eDate]; !ok {
+					addSql += GetAddSql(edbInfoIdStr, edbCode, eDate, timeStr, sValue)
+					isAdd = true
+				}
+			} else {
+				sql := ` UPDATE edb_data_ys SET value=?,modify_time=NOW() WHERE edb_info_id=? AND data_time=? `
+				_, err = o.Raw(sql, sValue, edbInfoId, eDate).Exec()
+			}
+			existMap[eDate] = sValue
+		}
+		if isAdd {
+			addSql = strings.TrimRight(addSql, ",")
+			_, err = o.Raw(addSql).Exec()
+			if err != nil {
+				return err
+			}
+		}
+	}
+
+	maxAndMinItem, err := GetEdbInfoMaxAndMinInfo(utils.DATA_SOURCE_LZ, edbCode)
+	if err != nil {
+		return err
+	}
+	if maxAndMinItem != nil {
+		err = ModifyEdbInfoMaxAndMinInfo(edbInfoId, maxAndMinItem)
+		if err != nil {
+			return err
+		}
+	}
+	return
+}
+
+
+
+const (
+	dataUrl = "https://dataapi.smm.cn/GetData/" //data url (中文)
+	//dataUrl = "https://dataapi.smm.cn/GetDataEn/" //data url (english edition)
+	authUrl = "https://platform.smm.cn/usercenter/auth" // auth url (for all)
+)
+
+type TokenResp struct {
+	Code int       `json:"Code"`
+	Msg  string    `json:"Msg"`
+	Data TokenData `json:"Data"`
+}
+
+type TokenData struct {
+	Token string `json:"Token"`
+}
+
+//获取token
+func getToken(userName string, password string) (string, error) {
+	encryptAuth := md5.New()
+	encryptAuth.Write([]byte(password)) //encrypt password with md5
+	newPassword := hex.EncodeToString(encryptAuth.Sum(nil))
+
+	resp, err := http.PostForm(authUrl, url.Values{"user_name": {userName}, "password": {newPassword}})
+	if err != nil {
+		return "", err
+	}
+
+	defer resp.Body.Close()
+
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		fmt.Println("reponse error", err)
+		return "", err
+	}
+
+	var bodyJsonContent TokenResp
+
+	if err = json.Unmarshal([]byte(body), &bodyJsonContent); err != nil {
+		fmt.Println(err, "unmarsal failure")
+		return "", err
+	}
+
+	var token string
+	if bodyJsonContent.Code == 0 {
+		token = bodyJsonContent.Data.Token
+	}
+
+	//print(token)
+	return token, nil
+}
+
+//request response
+type DataResp struct {
+	Code int      `json:"Code"`
+	Msg  string   `json:"Msg"`
+	Data *ApiData `json:"Data"`
+}
+
+//api data response
+type ApiData struct {
+	Status  int        `json:"Status"` //0 no permission，1 ok
+	Field   []ApiField `json:"Field"`
+	Content [][]string `json:"Content"`
+}
+
+//api title
+type ApiField struct {
+	Unit       string `json:"Unit"`
+	Info       string `json:"Info"`
+	Name       string `json:"Name"`
+	ColumnType string `json:"ColumnType"`
+	ColIndex   uint   `json:"ColIndex"`
+	IsDate     string `json:"IsDate"`
+}
+
+type YsResult struct {
+	Code int64 `json:"Code"`
+	Data struct {
+		CompanyList []interface{} `json:"CompanyList"`
+		Content     [][]string    `json:"Content"`
+		Field       []struct {
+			ColIndex   int64  `json:"ColIndex"`
+			ColumnType string `json:"ColumnType"`
+			Info       string `json:"Info"`
+			IsDate     string `json:"IsDate"`
+			Name       string `json:"Name"`
+			Unit       string `json:"Unit"`
+			DBColName  string `json:"db_col_name"`
+		} `json:"Field"`
+		CountPage   int64  `json:"count_page"`
+		CurrentPage int64  `json:"current_page"`
+		Frequency   string `json:"frequency"`
+		Mindate     string `json:"mindate"`
+		PageNum     int64  `json:"page_num"`
+		Status      int64  `json:"status"`
+		TotalNum    int64  `json:"total_num"`
+	} `json:"Data"`
+	Msg string `json:"Msg"`
+}
+
+/*
+ * request data
+ * sdatetime,edatetime ==>format:yyyy-mm-dd,
+ * apiName ==> data.metal.com(for english)/data.smm.cn (for chinese)
+ */
+func getApiData(token string, apiName string, sdatetime string, edatetime string) (item *YsResult, err error) {
+	reqUrl := dataUrl + apiName
+	resp, err := http.PostForm(reqUrl, url.Values{"token": {token}, "sdatetime": {sdatetime}, "edatetime": {edatetime}})
+	if err != nil {
+		return nil, err
+	}
+
+	defer resp.Body.Close()
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		fmt.Println("response error")
+		return nil, err
+	}
+	utils.FileLog.Info("ys result:" + string(body))
+	dataJsonContent := new(YsResult)
+	if err = json.Unmarshal([]byte(body), &dataJsonContent); err != nil {
+		fmt.Println(err, "data unmarshal failure")
+		return nil, err
+	}
+
+	if dataJsonContent.Code == 200 && len(dataJsonContent.Data.Content) > 0 {
+		return dataJsonContent, nil
+	} else {
+		err = errors.New("code:" + strconv.Itoa(int(dataJsonContent.Code)) + "msg:" + dataJsonContent.Msg)
+	}
+	return nil, nil
+}

services/data/edb_info.go

@@ -452,3 +452,57 @@ func RefreshDataFromLz(wg *sync.WaitGroup) (err error) {
 	wg.Done()
 	return err
 }
+
+//刷新有色数据
+func RefreshDataFromYs(wg *sync.WaitGroup) (err error) {
+	defer func() {
+		if err != nil {
+			fmt.Println("RefreshDataFromYs Err:" + err.Error())
+			go utils.SendEmail(utils.APPNAME+"【"+utils.RunMode+"】"+"失败提醒", "RefreshDataFromYs ErrMsg:"+err.Error(), utils.EmailSendToUsers)
+		}
+	}()
+	var condition string
+	var pars []interface{}
+	condition += " AND source=? "
+	pars = append(pars, utils.DATA_SOURCE_YS)
+	items, err := data_manage.GetEdbInfoByCondition(condition, pars, 0)
+	if err != nil {
+		return errors.New("GetEdbInfoByCondition:" + err.Error())
+	}
+
+	var endDate string
+	endDate = time.Now().Format(utils.FormatDateUnSpace)
+	for _, v := range items {
+		startDate := ""
+		if v.Frequency == "日度" {
+			startDate = v.EndDate.AddDate(0, 0, -utils.DATA_REFRESH).Format(utils.FormatDate)
+		} else if v.Frequency == "周度" {
+			startDate = v.EndDate.AddDate(0, 0, -(utils.DATA_REFRESH * 7)).Format(utils.FormatDate)
+		} else if v.Frequency == "月度" {
+			startDate = v.EndDate.AddDate(0, -utils.DATA_REFRESH, 0).Format(utils.FormatDate)
+		} else if v.Frequency == "季度" {
+			startDate = v.EndDate.AddDate(0, -utils.DATA_REFRESH*3, 0).Format(utils.FormatDate)
+		} else if v.Frequency == "年度" {
+			startDate = v.EndDate.AddDate(-utils.DATA_REFRESH, 0, 0).Format(utils.FormatDate)
+		} else {
+			startDate = v.EndDate.AddDate(0, 0, -utils.DATA_REFRESH).Format(utils.FormatDate)
+		}
+		fmt.Println(startDate, endDate)
+		err = data_manage.RefreshEdbDataByYs(v.EdbInfoId, v.EdbCode, startDate, endDate)
+		if err != nil {
+			return errors.New("RefreshEdbDataByYs:" + err.Error())
+		}
+		item, err := data_manage.GetEdbInfoMaxAndMinInfo(v.Source, v.EdbCode)
+		if err != nil {
+			return err
+		}
+		if item != nil {
+			err = data_manage.ModifyEdbInfoMaxAndMinInfo(v.EdbInfoId, item)
+			if err != nil {
+				return err
+			}
+		}
+	}
+	wg.Done()
+	return err
+}

services/task.go

@@ -143,7 +143,7 @@ func OneMinute(cont context.Context) (err error) {
 
 func RefreshData(cont context.Context) (err error) {
 	wg := sync.WaitGroup{}
-	wg.Add(5)
+	wg.Add(6)
 	//wind
 	go data.RefreshDataFromWind(&wg)
 	//同花顺
@@ -154,6 +154,8 @@ func RefreshData(cont context.Context) (err error) {
 	go data.RefreshDataFromManual(&wg)
 	//隆众数据
 	go data.RefreshDataFromLz(&wg)
+	//有色
+	go data.RefreshDataFromYs(&wg)
 	wg.Wait()
 	data.RefreshDataFromCalculateAll()
 	time.Sleep(2 * time.Second)

utils/constants.go

@@ -73,6 +73,7 @@ const (
 	DATA_SOURCE_CALCULATE_NSZYDPJJS            //N数值移动平均计算
 	DATA_SOURCE_MANUAL                         //手工指标
 	DATA_SOURCE_LZ                             //隆众
+	DATA_SOURCE_YS                             //有色
 )
 
 //http://datawind.hzinsights.com:8040/hz_server