hz_sync_logs/services/sync_log.go

package services

import (
	"context"
	"fmt"
	"github.com/pkg/sftp"
	"golang.org/x/crypto/ssh"
	"hongze/hz_sync_logs/utils"
	"io"
	"log"
	"os"
	"strings"
	"time"
)

func SyncYesterdayLogs(cont context.Context) (err error) {
	defer func() {
		if err != nil {

			//fmt.Println(utils.APPNAME+"【"+utils.RunMode+"】"+"失败提醒", "发送消息至同花顺失败 ErrMsg:"+err.Error(), utils.EmailSendToUsers)
			log.Printf(utils.APPNAME+"【"+utils.RunMode+"】"+"失败提醒", "同步昨天日志失败 ErrMsg:"+err.Error())
		}
	}()
	serviceMap := make(map[string]string)
	for host, pwd := range serviceMap {
		service(host, pwd)
	}
	return
}

func service(host, pwd string) {
	projectNameArr := []string{
		"eta_api",
		"eta_chart_lib",
		"eta_crawler",
		"eta_data_publish",
		"eta_data_push",
		"eta_index_lib",
		"eta_mobile",
		"eta_pub",
		"eta_task",
		"eta_forum",
		"eta_forum_admin",
		"eta_forum_hub",
	}
	sshHost := host //ssh的地址和端口（测试）

	sshConfig := &ssh.ClientConfig{
		User: "root",
		Auth: []ssh.AuthMethod{
			ssh.Password(pwd),
		},
		HostKeyCallback: ssh.InsecureIgnoreHostKey(),
		ClientVersion:   "",
		Timeout:         10 * time.Second,
	}

	//建立与SSH服务器的链接
	sshClient, err := ssh.Dial("tcp", sshHost, sshConfig)
	if err != nil {
		fmt.Println("ssh.Dial Err:" + err.Error())
		return
	}
	defer sshClient.Close()
	sftpClient, err := sftp.NewClient(sshClient)
	if err != nil {
		fmt.Println("sftp.NewClient Err:" + err.Error())
		return
	}
	defer sftpClient.Close()
	//获取当前目录
	cwd, err := sftpClient.Getwd()
	if err != nil {
		fmt.Println("Getwd Err:" + err.Error())
		return
	}
	fmt.Println("当前目录:" + cwd)

	//显示文件/目录详情
	fi, err := sftpClient.Lstat(cwd)
	if err != nil {
		fmt.Println("sftpClient.Lstat Err:" + err.Error())
		return
	}
	fmt.Println(fi)
	prefixDir := `/data/etalogs/`
	//preSaveDirPath := `/Users/roc/go/src/hongze/hz_sync_logs/test/`
	//dirCut := `/` //linux的目录分割符
	preSaveDirPath := `F:\\etalogs\\`
	if strings.Contains(host, "75") {
		preSaveDirPath += "75" + "\\"
	} else {
		preSaveDirPath += "55" + "\\"
	}
	dirCut := `\\` //windows的目录分割符

	yesterdayTime := time.Now().AddDate(0, 0, -1)

	logNameArr := []string{
		"apilog",
		"binlog",
		"datalog",
		"filelog",
		"mongolog",
	}

	for _, projectName := range projectNameArr {
		for _, logFileName := range logNameArr {
			for i := 1; i < 2; i++ {
				serverFilePath := prefixDir + projectName + "/" + logFileName
				localFilePath := preSaveDirPath + projectName + "\\" + logFileName
				//下载文件
				err = syncLog(sftpClient, yesterdayTime, serverFilePath, localFilePath, dirCut, logFileName, i)
				if err != nil {
					fmt.Println(serverFilePath+"下载失败，ERR：", err)
				}
			}
		}
	}
	return
}

func syncLog(sftpClient *sftp.Client, dateTime time.Time, dataDirPath, saveDirPath, dirCut, logFileName string, i int) (err error) {
	fileExt := `.log`

	date := dateTime.Format(utils.FormatDate)

	//服务器的文件路径
	remoteFileName := dataDirPath + "/" + logFileName + `.` + date + "." + fmt.Sprintf("%03d", i) + fileExt

	remoteFile, err := sftpClient.Open(remoteFileName)
	if err != nil {
		if err.Error() == "file does not exist" {
			fmt.Println(remoteFileName + " file does not exist")
			err = nil
		} else {
			fmt.Println("sftpClient.Open err:" + err.Error() + " ;remoteFileName:" + remoteFileName)
		}
		return
	}
	defer remoteFile.Close()

	//本地文件路径
	saveFileName := saveDirPath + dirCut
	fmt.Println("saveFileName:" + saveFileName)

	localFileName := logFileName + `.` + date + "." + fmt.Sprintf("%03d", i) + fileExt
	if !utils.FileIsExist(saveFileName) {
		err = os.MkdirAll(saveFileName, 0766)
		if err != nil {
			fmt.Println("创建目录失败,Err:" + err.Error())
			return
		}
	}
	localFilePath := saveFileName + localFileName
	fmt.Println("remoteFileName:" + remoteFileName)
	fmt.Println("localFilePath:" + localFilePath)

	localFile, err := os.Create(localFilePath)
	if err != nil {
		fmt.Println("create file err:" + err.Error())
		return
	}
	defer localFile.Close()
	n, err := io.Copy(localFile, remoteFile)
	if err != nil {
		fmt.Println("copy err:" + err.Error())
		return
	}
	//获取远程文件大小
	remoteFileInfo, err := sftpClient.Stat(remoteFileName)
	if err != nil {
		fmt.Println("sftpClient.Stat Err:" + err.Error())
		log.Fatalln(err.Error())
	}
	log.Printf("文件下载成功[%s->%s]远程文件大小：%s，下载文件大小：%s", remoteFileName, localFilePath, formatFileSize(remoteFileInfo.Size()), formatFileSize(n))
	//删除远程目录文件
	err = sftpClient.Remove(remoteFileName)
	if err != nil {
		fmt.Println("sftpClient.Remove Err:" + err.Error())
		return
	}
	return
}

//func SyncHistoryLogs() {
//	projectNameArr := []string{
//		"equity_admin",
//		"hongze_admin",
//		"hongze_advisory",
//		"hongze_api",
//		"hongze_cygx",
//		"hongze_mobile_admin",
//		"hongze_open_api",
//	}
//	sshHost := `` //ssh的地址和端口（测试）
//	pwd := ``     //ssh密码（测试）
//
//	sshConfig := &ssh.ClientConfig{
//		User: "root",
//		Auth: []ssh.AuthMethod{
//			ssh.Password(pwd),
//		},
//		HostKeyCallback: ssh.InsecureIgnoreHostKey(),
//		ClientVersion:   "",
//		Timeout:         10 * time.Second,
//	}
//
//	//建立与SSH服务器的链接
//	sshClient, err := ssh.Dial("tcp", sshHost, sshConfig)
//	if err != nil {
//		fmt.Println("ssh.Dial Err:" + err.Error())
//		return
//	}
//	defer sshClient.Close()
//	sftpClient, err := sftp.NewClient(sshClient)
//	if err != nil {
//		fmt.Println("sftp.NewClient Err:" + err.Error())
//		return
//	}
//	defer sftpClient.Close()
//	//获取当前目录
//	cwd, err := sftpClient.Getwd()
//	if err != nil {
//		fmt.Println("Getwd Err:" + err.Error())
//		return
//	}
//	fmt.Println("当前目录:" + cwd)
//
//	//显示文件/目录详情
//	fi, err := sftpClient.Lstat(cwd)
//	if err != nil {
//		fmt.Println("sftpClient.Lstat Err:" + err.Error())
//		return
//	}
//	fmt.Println(fi)
//	prefixDir := `/data/rdlucklog/`
//	//preSaveDirPath := `/Users/roc/go/src/hongze/hz_sync_logs/test/`
//	//dirCut := `/` //linux的目录分割符
//	preSaveDirPath := `E:\\hongze_logs\\`
//	dirCut := `\\` //windows的目录分割符
//
//	yesterdayTime := time.Now().AddDate(0, 0, -1)
//
//	for _, projectName := range projectNameArr {
//		serverFilePath := prefixDir + projectName
//		localFilePath := preSaveDirPath + projectName
//		//下载文件
//		for i := 0; i < 22; i++ {
//			dateTime := yesterdayTime.AddDate(0, 0, -i)
//			err = syncLog(sftpClient, dateTime, serverFilePath, localFilePath, dirCut)
//			if err != nil {
//				fmt.Println(serverFilePath+"下载失败，ERR：", err)
//			}
//		}
//
//	}
//
//	return
//	{
//		dataDirPath := `/data/rdlucklog`
//		saveDirPath := `E:\\log\\`
//		fileExt := `.api.log`
//
//		logPath := time.Now().AddDate(0, -1, 0).Format("200601")
//		dateObj, _ := time.Parse("2006-01-02", "2021-10-01")
//		for _, proName := range projectNameArr {
//			for i := 0; i < 31; i++ {
//				date := dateObj.AddDate(0, 0, i).Format("20060102")
//				remoteDirName := dataDirPath + `/` + proName + `/` + logPath + `/` + date + fileExt
//				remoteFile, err := sftpClient.Open(remoteDirName)
//				if err != nil {
//					fmt.Println("sftpClient.Open err:" + err.Error() + " ;remoteDirName:" + remoteDirName)
//					if err.Error() == "file does not exist" {
//						continue
//					} else {
//						fmt.Println("sftpClient.Open err:" + err.Error() + " ;remoteDirName:" + remoteDirName)
//						return
//					}
//				}
//				defer remoteFile.Close()
//				saveFileName := saveDirPath + proName + `\\202110\\`
//				localFileName := date + fileExt
//				fmt.Println("saveFileName:", saveFileName)
//				if !utils.FileIsExist(saveFileName) {
//					err = os.MkdirAll(saveFileName, 777)
//					if err != nil {
//						fmt.Println("创建目录失败,Err:" + err.Error())
//						return
//					}
//				}
//				localFilePath := saveFileName + localFileName
//				fmt.Println("localFilePath:" + localFilePath)
//				localFile, err := os.Create(localFilePath)
//				if err != nil {
//					fmt.Println("create file err:" + err.Error())
//					return
//				}
//				defer localFile.Close()
//				n, err := io.Copy(localFile, remoteFile)
//				if err != nil {
//					fmt.Println("copy err:" + err.Error())
//					return
//				}
//				//获取远程文件大小
//				remoteFileInfo, err := sftpClient.Stat(remoteDirName)
//				if err != nil {
//					fmt.Println("sftpClient.Stat Err:" + err.Error())
//					log.Fatalln(err.Error())
//				}
//				log.Printf("文件下载成功[%s->%s]远程文件大小：%s，下载文件大小：%s", remoteDirName, localFileName, formatFileSize(remoteFileInfo.Size()), formatFileSize(n))
//				//删除远程目录文件
//				//err = sftpClient.Remove(remoteDirName)
//				//if err != nil {
//				//	fmt.Println("sftpClient.Remove Err:" + err.Error())
//				//	return
//				//}
//			}
//		}
//	}
//}

func SyncLogs2() {
	projectNameArr := []string{
		"equity_admin",
		//"hongze_admin",
		//"hongze_api",
		//"hongze_cygx",
	}
	sshConfig := &ssh.ClientConfig{
		User: "root",
		Auth: []ssh.AuthMethod{
			ssh.Password("pwd"),
		},
		HostKeyCallback: ssh.InsecureIgnoreHostKey(),
		ClientVersion:   "",
		Timeout:         10 * time.Second,
	}

	//建立与SSH服务器的链接
	sshClient, err := ssh.Dial("tcp", "ip:port", sshConfig)
	if err != nil {
		fmt.Println("ssh.Dial Err:" + err.Error())
		return
	}
	defer sshClient.Close()
	sftpClient, err := sftp.NewClient(sshClient)
	if err != nil {
		fmt.Println("sftp.NewClient Err:" + err.Error())
		return
	}
	defer sftpClient.Close()
	//获取当前目录
	cwd, err := sftpClient.Getwd()
	if err != nil {
		fmt.Println("Getwd Err:" + err.Error())
		return
	}
	fmt.Println("当前目录:" + cwd)
	//显示文件/目录详情
	fi, err := sftpClient.Lstat(cwd)
	if err != nil {
		fmt.Println("sftpClient.Lstat Err:" + err.Error())
		return
	}
	fmt.Println(fi)
	//下载文件
	{
		dataDirPath := `/data/rdlucklog`
		saveDirPath := `E:\\log\\`
		fileExt := `.api.log`

		logPath := time.Now().AddDate(0, -1, 0).Format("200601")
		dateObj, _ := time.Parse("2006-01-02", "2021-10-01")
		for _, proName := range projectNameArr {
			for i := 0; i < 31; i++ {
				date := dateObj.AddDate(0, 0, i).Format("20060102")
				remoteDirName := dataDirPath + `/` + proName + `/` + logPath + `/` + date + fileExt
				remoteFile, err := sftpClient.Open(remoteDirName)
				if err != nil {
					fmt.Println("sftpClient.Open err:" + err.Error() + " ;remoteDirName:" + remoteDirName)
					if err.Error() == "file does not exist" {
						continue
					} else {
						fmt.Println("sftpClient.Open err:" + err.Error() + " ;remoteDirName:" + remoteDirName)
						return
					}
				}
				defer remoteFile.Close()
				saveFileName := saveDirPath + proName + `\\202110\\`
				localFileName := date + fileExt
				fmt.Println("saveFileName:", saveFileName)
				if !utils.FileIsExist(saveFileName) {
					err = os.MkdirAll(saveFileName, 777)
					if err != nil {
						fmt.Println("创建目录失败,Err:" + err.Error())
						return
					}
				}
				localFilePath := saveFileName + localFileName
				fmt.Println("localFilePath:" + localFilePath)
				localFile, err := os.Create(localFilePath)
				if err != nil {
					fmt.Println("create file err:" + err.Error())
					return
				}
				defer localFile.Close()
				n, err := io.Copy(localFile, remoteFile)
				if err != nil {
					fmt.Println("copy err:" + err.Error())
					return
				}
				//获取远程文件大小
				remoteFileInfo, err := sftpClient.Stat(remoteDirName)
				if err != nil {
					fmt.Println("sftpClient.Stat Err:" + err.Error())
					log.Fatalln(err.Error())
				}
				log.Printf("文件下载成功[%s->%s]远程文件大小：%s，下载文件大小：%s", remoteDirName, localFileName, formatFileSize(remoteFileInfo.Size()), formatFileSize(n))
				//删除远程目录文件
				//err = sftpClient.Remove(remoteDirName)
				//if err != nil {
				//	fmt.Println("sftpClient.Remove Err:" + err.Error())
				//	return
				//}
			}
		}
	}
}

// 字节的单位转换 保留两位小数
func formatFileSize(s int64) (size string) {
	if s < 1024 {
		return fmt.Sprintf("%.2fB", float64(s)/float64(1))
	} else if s < (1024 * 1024) {
		return fmt.Sprintf("%.2fKB", float64(s)/float64(1024))
	} else if s < (1024 * 1024 * 1024) {
		return fmt.Sprintf("%.2fMB", float64(s)/float64(1024*1024))
	} else if s < (1024 * 1024 * 1024 * 1024) {
		return fmt.Sprintf("%.2fGB", float64(s)/float64(1024*1024*1024))
	} else if s < (1024 * 1024 * 1024 * 1024 * 1024) {
		return fmt.Sprintf("%.2fTB", float64(s)/float64(1024*1024*1024*1024))
	} else { //if s < (1024 * 1024 * 1024 * 1024 * 1024 * 1024)
		return fmt.Sprintf("%.2fEB", float64(s)/float64(1024*1024*1024*1024*1024))
	}
}