/** * @Author: jgl * @Date: 2021/9/15 10:15 */ package data_manage import ( "crypto/md5" "encoding/hex" "encoding/json" "errors" "fmt" "hongze/hongze_task/utils" "io/ioutil" "net/http" "net/url" "rdluck_tools/orm" "strconv" "strings" "time" ) //刷新有色指标数据 func RefreshEdbDataByYs(edbInfoId int, edbCode, startDate, endDate string) (err error) { o := orm.NewOrm() o.Using("data") o.Begin() defer func() { if err != nil { o.Rollback() } else { o.Commit() } }() token, err := getToken("pqian@hzinsights.com", "hz123456") if err != nil { fmt.Println(err) return } ysResult, err := getApiData(token, edbCode, startDate, endDate) if err != nil { fmt.Println(err) return } edbInfoIdStr := strconv.Itoa(edbInfoId) if ysResult != nil && len(ysResult.Data.Content) > 0 { var isAdd bool addSql := ` INSERT INTO edb_data_ys(edb_info_id,edb_code,data_time,value,create_time,modify_time,status,data_timestamp) values ` existMap := make(map[string]string) frequency := ysResult.Data.Frequency for _, rv := range ysResult.Data.Content { var eDate, sValue string for ck, cv := range rv { if ck == 0 { eDate = cv } if ck == 1 { sValue = strings.Replace(cv, ",", "", -1) } } if sValue=="" { continue } if frequency == "月" { monthDate, err := time.Parse("2006-01", eDate) if err != nil { fmt.Println("time.Parse:" + err.Error()) } lastTime := monthDate.AddDate(0, 1, -1) lastYear, lastMonth, lastDay := lastTime.Date() var lastDate string if int(lastMonth) < 10 { lastDate = strconv.Itoa(lastYear) + "-" + "0" + strconv.Itoa(int(lastMonth)) + "-" + strconv.Itoa(lastDay) } else { lastDate = strconv.Itoa(lastYear) + "-" + strconv.Itoa(int(lastMonth)) + "-" + strconv.Itoa(lastDay) } eDate = lastDate } count, err := GetEdbDataBaseByCodeAndDate(utils.DATA_SOURCE_YS, edbCode, eDate) if err != nil && err.Error() != utils.ErrNoRow() { return err } if count <= 0 { dataTime, err := time.Parse(utils.FormatDate, eDate) if err != nil { return err } timestamp := dataTime.UnixNano() / 1e6 timeStr := fmt.Sprintf("%d", timestamp) if _, ok := existMap[eDate]; !ok { addSql += GetAddSql(edbInfoIdStr, edbCode, eDate, timeStr, sValue) isAdd = true } } else { sql := ` UPDATE edb_data_ys SET value=?,modify_time=NOW() WHERE edb_info_id=? AND data_time=? ` _, err = o.Raw(sql, sValue, edbInfoId, eDate).Exec() } existMap[eDate] = sValue } if isAdd { addSql = strings.TrimRight(addSql, ",") _, err = o.Raw(addSql).Exec() if err != nil { return err } } } maxAndMinItem, err := GetEdbInfoMaxAndMinInfo(utils.DATA_SOURCE_LZ, edbCode) if err != nil { return err } if maxAndMinItem != nil { err = ModifyEdbInfoMaxAndMinInfo(edbInfoId, maxAndMinItem) if err != nil { return err } } return } const ( dataUrl = "https://dataapi.smm.cn/GetData/" //data url (中文) //dataUrl = "https://dataapi.smm.cn/GetDataEn/" //data url (english edition) authUrl = "https://platform.smm.cn/usercenter/auth" // auth url (for all) ) type TokenResp struct { Code int `json:"Code"` Msg string `json:"Msg"` Data TokenData `json:"Data"` } type TokenData struct { Token string `json:"Token"` } //获取token func getToken(userName string, password string) (string, error) { encryptAuth := md5.New() encryptAuth.Write([]byte(password)) //encrypt password with md5 newPassword := hex.EncodeToString(encryptAuth.Sum(nil)) resp, err := http.PostForm(authUrl, url.Values{"user_name": {userName}, "password": {newPassword}}) if err != nil { return "", err } defer resp.Body.Close() body, err := ioutil.ReadAll(resp.Body) if err != nil { fmt.Println("reponse error", err) return "", err } var bodyJsonContent TokenResp if err = json.Unmarshal([]byte(body), &bodyJsonContent); err != nil { fmt.Println(err, "unmarsal failure") return "", err } var token string if bodyJsonContent.Code == 0 { token = bodyJsonContent.Data.Token } //print(token) return token, nil } //request response type DataResp struct { Code int `json:"Code"` Msg string `json:"Msg"` Data *ApiData `json:"Data"` } //api data response type ApiData struct { Status int `json:"Status"` //0 no permission，1 ok Field []ApiField `json:"Field"` Content [][]string `json:"Content"` } //api title type ApiField struct { Unit string `json:"Unit"` Info string `json:"Info"` Name string `json:"Name"` ColumnType string `json:"ColumnType"` ColIndex uint `json:"ColIndex"` IsDate string `json:"IsDate"` } type YsResult struct { Code int64 `json:"Code"` Data struct { CompanyList []interface{} `json:"CompanyList"` Content [][]string `json:"Content"` Field []struct { ColIndex int64 `json:"ColIndex"` ColumnType string `json:"ColumnType"` Info string `json:"Info"` IsDate string `json:"IsDate"` Name string `json:"Name"` Unit string `json:"Unit"` DBColName string `json:"db_col_name"` } `json:"Field"` CountPage int64 `json:"count_page"` CurrentPage int64 `json:"current_page"` Frequency string `json:"frequency"` Mindate string `json:"mindate"` PageNum int64 `json:"page_num"` Status int64 `json:"status"` TotalNum int64 `json:"total_num"` } `json:"Data"` Msg string `json:"Msg"` } /* * request data * sdatetime,edatetime ==>format:yyyy-mm-dd, * apiName ==> data.metal.com(for english)/data.smm.cn (for chinese) */ func getApiData(token string, apiName string, sdatetime string, edatetime string) (item *YsResult, err error) { reqUrl := dataUrl + apiName resp, err := http.PostForm(reqUrl, url.Values{"token": {token}, "sdatetime": {sdatetime}, "edatetime": {edatetime}}) if err != nil { return nil, err } defer resp.Body.Close() body, err := ioutil.ReadAll(resp.Body) if err != nil { fmt.Println("response error") return nil, err } utils.FileLog.Info("ys result:" + string(body)) dataJsonContent := new(YsResult) if err = json.Unmarshal([]byte(body), &dataJsonContent); err != nil { fmt.Println(err, "data unmarshal failure") return nil, err } if dataJsonContent.Code == 200 && len(dataJsonContent.Data.Content) > 0 { return dataJsonContent, nil } else { err = errors.New("code:" + strconv.Itoa(int(dataJsonContent.Code)) + "msg:" + dataJsonContent.Msg) } return nil, nil }