package services import ( "context" "fmt" "github.com/pkg/sftp" "golang.org/x/crypto/ssh" "hongze/hz_sync_logs/utils" "io" "log" "os" "strings" "time" ) func SyncYesterdayLogs(cont context.Context) (err error) { defer func() { if err != nil { //fmt.Println(utils.APPNAME+"【"+utils.RunMode+"】"+"失败提醒", "发送消息至同花顺失败 ErrMsg:"+err.Error(), utils.EmailSendToUsers) log.Printf(utils.APPNAME+"【"+utils.RunMode+"】"+"失败提醒", "同步昨天日志失败 ErrMsg:"+err.Error()) } }() serviceMap := make(map[string]string) for host, pwd := range serviceMap { service(host, pwd) } return } func service(host, pwd string) { projectNameArr := []string{ "eta_api", "eta_chart_lib", "eta_crawler", "eta_data_publish", "eta_data_push", "eta_index_lib", "eta_mobile", "eta_pub", "eta_task", "eta_forum", "eta_forum_admin", "eta_forum_hub", } sshHost := host //ssh的地址和端口(测试) sshConfig := &ssh.ClientConfig{ User: "root", Auth: []ssh.AuthMethod{ ssh.Password(pwd), }, HostKeyCallback: ssh.InsecureIgnoreHostKey(), ClientVersion: "", Timeout: 10 * time.Second, } //建立与SSH服务器的链接 sshClient, err := ssh.Dial("tcp", sshHost, sshConfig) if err != nil { fmt.Println("ssh.Dial Err:" + err.Error()) return } defer sshClient.Close() sftpClient, err := sftp.NewClient(sshClient) if err != nil { fmt.Println("sftp.NewClient Err:" + err.Error()) return } defer sftpClient.Close() //获取当前目录 cwd, err := sftpClient.Getwd() if err != nil { fmt.Println("Getwd Err:" + err.Error()) return } fmt.Println("当前目录:" + cwd) //显示文件/目录详情 fi, err := sftpClient.Lstat(cwd) if err != nil { fmt.Println("sftpClient.Lstat Err:" + err.Error()) return } fmt.Println(fi) prefixDir := `/data/etalogs/` //preSaveDirPath := `/Users/roc/go/src/hongze/hz_sync_logs/test/` //dirCut := `/` //linux的目录分割符 preSaveDirPath := `F:\\etalogs\\` if strings.Contains(host, "75") { preSaveDirPath += "75" + "\\" } else { preSaveDirPath += "55" + "\\" } dirCut := `\\` //windows的目录分割符 yesterdayTime := time.Now().AddDate(0, 0, -1) logNameArr := []string{ "apilog", "binlog", "datalog", "filelog", "mongolog", } for _, projectName := range projectNameArr { for _, logFileName := range logNameArr { for i := 1; i < 2; i++ { serverFilePath := prefixDir + projectName + "/" + logFileName localFilePath := preSaveDirPath + projectName + "\\" + logFileName //下载文件 err = syncLog(sftpClient, yesterdayTime, serverFilePath, localFilePath, dirCut, logFileName, i) if err != nil { fmt.Println(serverFilePath+"下载失败,ERR:", err) } } } } return } func syncLog(sftpClient *sftp.Client, dateTime time.Time, dataDirPath, saveDirPath, dirCut, logFileName string, i int) (err error) { fileExt := `.log` date := dateTime.Format(utils.FormatDate) //服务器的文件路径 remoteFileName := dataDirPath + "/" + logFileName + `.` + date + "." + fmt.Sprintf("%03d", i) + fileExt remoteFile, err := sftpClient.Open(remoteFileName) if err != nil { if err.Error() == "file does not exist" { fmt.Println(remoteFileName + " file does not exist") err = nil } else { fmt.Println("sftpClient.Open err:" + err.Error() + " ;remoteFileName:" + remoteFileName) } return } defer remoteFile.Close() //本地文件路径 saveFileName := saveDirPath + dirCut fmt.Println("saveFileName:" + saveFileName) localFileName := logFileName + `.` + date + "." + fmt.Sprintf("%03d", i) + fileExt if !utils.FileIsExist(saveFileName) { err = os.MkdirAll(saveFileName, 0766) if err != nil { fmt.Println("创建目录失败,Err:" + err.Error()) return } } localFilePath := saveFileName + localFileName fmt.Println("remoteFileName:" + remoteFileName) fmt.Println("localFilePath:" + localFilePath) localFile, err := os.Create(localFilePath) if err != nil { fmt.Println("create file err:" + err.Error()) return } defer localFile.Close() n, err := io.Copy(localFile, remoteFile) if err != nil { fmt.Println("copy err:" + err.Error()) return } //获取远程文件大小 remoteFileInfo, err := sftpClient.Stat(remoteFileName) if err != nil { fmt.Println("sftpClient.Stat Err:" + err.Error()) log.Fatalln(err.Error()) } log.Printf("文件下载成功[%s->%s]远程文件大小:%s,下载文件大小:%s", remoteFileName, localFilePath, formatFileSize(remoteFileInfo.Size()), formatFileSize(n)) //删除远程目录文件 err = sftpClient.Remove(remoteFileName) if err != nil { fmt.Println("sftpClient.Remove Err:" + err.Error()) return } return } //func SyncHistoryLogs() { // projectNameArr := []string{ // "equity_admin", // "hongze_admin", // "hongze_advisory", // "hongze_api", // "hongze_cygx", // "hongze_mobile_admin", // "hongze_open_api", // } // sshHost := `` //ssh的地址和端口(测试) // pwd := `` //ssh密码(测试) // // sshConfig := &ssh.ClientConfig{ // User: "root", // Auth: []ssh.AuthMethod{ // ssh.Password(pwd), // }, // HostKeyCallback: ssh.InsecureIgnoreHostKey(), // ClientVersion: "", // Timeout: 10 * time.Second, // } // // //建立与SSH服务器的链接 // sshClient, err := ssh.Dial("tcp", sshHost, sshConfig) // if err != nil { // fmt.Println("ssh.Dial Err:" + err.Error()) // return // } // defer sshClient.Close() // sftpClient, err := sftp.NewClient(sshClient) // if err != nil { // fmt.Println("sftp.NewClient Err:" + err.Error()) // return // } // defer sftpClient.Close() // //获取当前目录 // cwd, err := sftpClient.Getwd() // if err != nil { // fmt.Println("Getwd Err:" + err.Error()) // return // } // fmt.Println("当前目录:" + cwd) // // //显示文件/目录详情 // fi, err := sftpClient.Lstat(cwd) // if err != nil { // fmt.Println("sftpClient.Lstat Err:" + err.Error()) // return // } // fmt.Println(fi) // prefixDir := `/data/rdlucklog/` // //preSaveDirPath := `/Users/roc/go/src/hongze/hz_sync_logs/test/` // //dirCut := `/` //linux的目录分割符 // preSaveDirPath := `E:\\hongze_logs\\` // dirCut := `\\` //windows的目录分割符 // // yesterdayTime := time.Now().AddDate(0, 0, -1) // // for _, projectName := range projectNameArr { // serverFilePath := prefixDir + projectName // localFilePath := preSaveDirPath + projectName // //下载文件 // for i := 0; i < 22; i++ { // dateTime := yesterdayTime.AddDate(0, 0, -i) // err = syncLog(sftpClient, dateTime, serverFilePath, localFilePath, dirCut) // if err != nil { // fmt.Println(serverFilePath+"下载失败,ERR:", err) // } // } // // } // // return // { // dataDirPath := `/data/rdlucklog` // saveDirPath := `E:\\log\\` // fileExt := `.api.log` // // logPath := time.Now().AddDate(0, -1, 0).Format("200601") // dateObj, _ := time.Parse("2006-01-02", "2021-10-01") // for _, proName := range projectNameArr { // for i := 0; i < 31; i++ { // date := dateObj.AddDate(0, 0, i).Format("20060102") // remoteDirName := dataDirPath + `/` + proName + `/` + logPath + `/` + date + fileExt // remoteFile, err := sftpClient.Open(remoteDirName) // if err != nil { // fmt.Println("sftpClient.Open err:" + err.Error() + " ;remoteDirName:" + remoteDirName) // if err.Error() == "file does not exist" { // continue // } else { // fmt.Println("sftpClient.Open err:" + err.Error() + " ;remoteDirName:" + remoteDirName) // return // } // } // defer remoteFile.Close() // saveFileName := saveDirPath + proName + `\\202110\\` // localFileName := date + fileExt // fmt.Println("saveFileName:", saveFileName) // if !utils.FileIsExist(saveFileName) { // err = os.MkdirAll(saveFileName, 777) // if err != nil { // fmt.Println("创建目录失败,Err:" + err.Error()) // return // } // } // localFilePath := saveFileName + localFileName // fmt.Println("localFilePath:" + localFilePath) // localFile, err := os.Create(localFilePath) // if err != nil { // fmt.Println("create file err:" + err.Error()) // return // } // defer localFile.Close() // n, err := io.Copy(localFile, remoteFile) // if err != nil { // fmt.Println("copy err:" + err.Error()) // return // } // //获取远程文件大小 // remoteFileInfo, err := sftpClient.Stat(remoteDirName) // if err != nil { // fmt.Println("sftpClient.Stat Err:" + err.Error()) // log.Fatalln(err.Error()) // } // log.Printf("文件下载成功[%s->%s]远程文件大小:%s,下载文件大小:%s", remoteDirName, localFileName, formatFileSize(remoteFileInfo.Size()), formatFileSize(n)) // //删除远程目录文件 // //err = sftpClient.Remove(remoteDirName) // //if err != nil { // // fmt.Println("sftpClient.Remove Err:" + err.Error()) // // return // //} // } // } // } //} func SyncLogs2() { projectNameArr := []string{ "equity_admin", //"hongze_admin", //"hongze_api", //"hongze_cygx", } sshConfig := &ssh.ClientConfig{ User: "root", Auth: []ssh.AuthMethod{ ssh.Password("pwd"), }, HostKeyCallback: ssh.InsecureIgnoreHostKey(), ClientVersion: "", Timeout: 10 * time.Second, } //建立与SSH服务器的链接 sshClient, err := ssh.Dial("tcp", "ip:port", sshConfig) if err != nil { fmt.Println("ssh.Dial Err:" + err.Error()) return } defer sshClient.Close() sftpClient, err := sftp.NewClient(sshClient) if err != nil { fmt.Println("sftp.NewClient Err:" + err.Error()) return } defer sftpClient.Close() //获取当前目录 cwd, err := sftpClient.Getwd() if err != nil { fmt.Println("Getwd Err:" + err.Error()) return } fmt.Println("当前目录:" + cwd) //显示文件/目录详情 fi, err := sftpClient.Lstat(cwd) if err != nil { fmt.Println("sftpClient.Lstat Err:" + err.Error()) return } fmt.Println(fi) //下载文件 { dataDirPath := `/data/rdlucklog` saveDirPath := `E:\\log\\` fileExt := `.api.log` logPath := time.Now().AddDate(0, -1, 0).Format("200601") dateObj, _ := time.Parse("2006-01-02", "2021-10-01") for _, proName := range projectNameArr { for i := 0; i < 31; i++ { date := dateObj.AddDate(0, 0, i).Format("20060102") remoteDirName := dataDirPath + `/` + proName + `/` + logPath + `/` + date + fileExt remoteFile, err := sftpClient.Open(remoteDirName) if err != nil { fmt.Println("sftpClient.Open err:" + err.Error() + " ;remoteDirName:" + remoteDirName) if err.Error() == "file does not exist" { continue } else { fmt.Println("sftpClient.Open err:" + err.Error() + " ;remoteDirName:" + remoteDirName) return } } defer remoteFile.Close() saveFileName := saveDirPath + proName + `\\202110\\` localFileName := date + fileExt fmt.Println("saveFileName:", saveFileName) if !utils.FileIsExist(saveFileName) { err = os.MkdirAll(saveFileName, 777) if err != nil { fmt.Println("创建目录失败,Err:" + err.Error()) return } } localFilePath := saveFileName + localFileName fmt.Println("localFilePath:" + localFilePath) localFile, err := os.Create(localFilePath) if err != nil { fmt.Println("create file err:" + err.Error()) return } defer localFile.Close() n, err := io.Copy(localFile, remoteFile) if err != nil { fmt.Println("copy err:" + err.Error()) return } //获取远程文件大小 remoteFileInfo, err := sftpClient.Stat(remoteDirName) if err != nil { fmt.Println("sftpClient.Stat Err:" + err.Error()) log.Fatalln(err.Error()) } log.Printf("文件下载成功[%s->%s]远程文件大小:%s,下载文件大小:%s", remoteDirName, localFileName, formatFileSize(remoteFileInfo.Size()), formatFileSize(n)) //删除远程目录文件 //err = sftpClient.Remove(remoteDirName) //if err != nil { // fmt.Println("sftpClient.Remove Err:" + err.Error()) // return //} } } } } // 字节的单位转换 保留两位小数 func formatFileSize(s int64) (size string) { if s < 1024 { return fmt.Sprintf("%.2fB", float64(s)/float64(1)) } else if s < (1024 * 1024) { return fmt.Sprintf("%.2fKB", float64(s)/float64(1024)) } else if s < (1024 * 1024 * 1024) { return fmt.Sprintf("%.2fMB", float64(s)/float64(1024*1024)) } else if s < (1024 * 1024 * 1024 * 1024) { return fmt.Sprintf("%.2fGB", float64(s)/float64(1024*1024*1024)) } else if s < (1024 * 1024 * 1024 * 1024 * 1024) { return fmt.Sprintf("%.2fTB", float64(s)/float64(1024*1024*1024*1024)) } else { //if s < (1024 * 1024 * 1024 * 1024 * 1024 * 1024) return fmt.Sprintf("%.2fEB", float64(s)/float64(1024*1024*1024*1024*1024)) } }