修改数据同步策略

conf/app.conf

@@ -1,5 +1,5 @@
 appname = hz_sync_log
-httpport = 8080
+httpport = 8089
 runmode = dev
 autorender = false
 copyrequestbody = true

services/sync_log.go

@@ -9,10 +9,11 @@ import (
 	"io"
 	"log"
 	"os"
+	"strings"
 	"time"
 )
 
-func SyncYesterdayLogs(cont context.Context) (err error){
+func SyncYesterdayLogs(cont context.Context) (err error) {
 	defer func() {
 		if err != nil {
 
@@ -20,17 +21,30 @@ func SyncYesterdayLogs(cont context.Context) (err error){
 			log.Printf(utils.APPNAME+"【"+utils.RunMode+"】"+"失败提醒", "同步昨天日志失败 ErrMsg:"+err.Error())
 		}
 	}()
+	serviceMap := make(map[string]string)
+	
+	for host, pwd := range serviceMap {
+		service(host, pwd)
+	}
+	return
+}
+
+func service(host, pwd string) {
 	projectNameArr := []string{
-		"equity_admin",
-		"hongze_admin",
-		"hongze_advisory",
-		"hongze_api",
-		"hongze_cygx",
-		"hongze_mobile_admin",
-		"hongze_open_api",
+		"eta_api",
+		"eta_chart_lib",
+		"eta_crawler",
+		"eta_data_publish",
+		"eta_data_push",
+		"eta_index_lib",
+		"eta_mobile",
+		"eta_pub",
+		"eta_task",
+		"eta_forum",
+		"eta_forum_admin",
+		"eta_forum_hub",
 	}
-	sshHost := ``	//ssh的地址和端口（测试）
-	pwd := ``	//ssh密码（测试）
+	sshHost := host //ssh的地址和端口（测试）
 
 	sshConfig := &ssh.ClientConfig{
 		User: "root",
@@ -70,38 +84,53 @@ func SyncYesterdayLogs(cont context.Context) (err error){
 		return
 	}
 	fmt.Println(fi)
-	prefixDir := `/data/rdlucklog/`
+	prefixDir := `/data/etalogs/`
 	//preSaveDirPath := `/Users/roc/go/src/hongze/hz_sync_logs/test/`
 	//dirCut := `/` //linux的目录分割符
-	preSaveDirPath := `E:\\hongze_logs\\`
-	dirCut := `\\`	//windows的目录分割符
+	preSaveDirPath := `F:\\etalogs\\`
+	if strings.Contains(host, "75") {
+		preSaveDirPath += "75" + "\\"
+	} else {
+		preSaveDirPath += "55" + "\\"
+	}
+	dirCut := `\\` //windows的目录分割符
 
-	yesterdayTime := time.Now().AddDate(0,0,-1)
+	yesterdayTime := time.Now().AddDate(0, 0, -1)
 
-	for _,projectName := range projectNameArr{
-		serverFilePath := prefixDir + projectName
-		localFilePath := preSaveDirPath + projectName
-		//下载文件
-		err = syncLog(sftpClient, yesterdayTime, serverFilePath, localFilePath, dirCut)
-		if err != nil {
-			fmt.Println(serverFilePath+"下载失败，ERR：", err)
-		}
+	logNameArr := []string{
+		"apilog",
+		"binlog",
+		"datalog",
+		"filelog",
+		"mongolog",
+	}
 
+	for _, projectName := range projectNameArr {
+		for _, logFileName := range logNameArr {
+			for i := 1; i < 2; i++ {
+				serverFilePath := prefixDir + projectName + "/" + logFileName
+				localFilePath := preSaveDirPath + projectName + "\\" + logFileName
+				//下载文件
+				err = syncLog(sftpClient, yesterdayTime, serverFilePath, localFilePath, dirCut, logFileName, i)
+				if err != nil {
+					fmt.Println(serverFilePath+"下载失败，ERR：", err)
+				}
+			}
+		}
 	}
 	return
 }
 
-func syncLog(sftpClient *sftp.Client, dateTime time.Time, dataDirPath, saveDirPath, dirCut string) (err error) {
-	fileExt := `.api.log`
+func syncLog(sftpClient *sftp.Client, dateTime time.Time, dataDirPath, saveDirPath, dirCut, logFileName string, i int) (err error) {
+	fileExt := `.log`
 
-	date := dateTime.Format(utils.FormatDateUnSpace)
-	logPath := dateTime.Format(utils.FormatDateUnSpaceMonth)
+	date := dateTime.Format(utils.FormatDate)
 
 	//服务器的文件路径
-	remoteFileName := dataDirPath + "/" + logPath + `/` + date + fileExt
+	remoteFileName := dataDirPath + "/" + logFileName + `.` + date + "." + fmt.Sprintf("%03d", i) + fileExt
+
 	remoteFile, err := sftpClient.Open(remoteFileName)
 	if err != nil {
-		fmt.Println("sftpClient.Open err:" + err.Error() + " ;remoteDirName:" + remoteFileName)
 		if err.Error() == "file does not exist" {
 			fmt.Println(remoteFileName + " file does not exist")
 			err = nil
@@ -113,9 +142,10 @@ func syncLog(sftpClient *sftp.Client, dateTime time.Time, dataDirPath, saveDirPa
 	defer remoteFile.Close()
 
 	//本地文件路径
-	saveFileName := saveDirPath + dirCut  + logPath + dirCut
+	saveFileName := saveDirPath + dirCut
+	fmt.Println("saveFileName:" + saveFileName)
 
-	localFileName := date + fileExt
+	localFileName := logFileName + `.` + date + "." + fmt.Sprintf("%03d", i) + fileExt
 	if !utils.FileIsExist(saveFileName) {
 		err = os.MkdirAll(saveFileName, 0766)
 		if err != nil {
@@ -154,143 +184,142 @@ func syncLog(sftpClient *sftp.Client, dateTime time.Time, dataDirPath, saveDirPa
 	return
 }
 
-func SyncHistoryLogs() {
-	projectNameArr := []string{
-		"equity_admin",
-		"hongze_admin",
-		"hongze_advisory",
-		"hongze_api",
-		"hongze_cygx",
-		"hongze_mobile_admin",
-		"hongze_open_api",
-	}
-	sshHost := ``	//ssh的地址和端口（测试）
-	pwd := ``	//ssh密码（测试）
-
-	sshConfig := &ssh.ClientConfig{
-		User: "root",
-		Auth: []ssh.AuthMethod{
-			ssh.Password(pwd),
-		},
-		HostKeyCallback: ssh.InsecureIgnoreHostKey(),
-		ClientVersion:   "",
-		Timeout:         10 * time.Second,
-	}
-
-	//建立与SSH服务器的链接
-	sshClient, err := ssh.Dial("tcp", sshHost, sshConfig)
-	if err != nil {
-		fmt.Println("ssh.Dial Err:" + err.Error())
-		return
-	}
-	defer sshClient.Close()
-	sftpClient, err := sftp.NewClient(sshClient)
-	if err != nil {
-		fmt.Println("sftp.NewClient Err:" + err.Error())
-		return
-	}
-	defer sftpClient.Close()
-	//获取当前目录
-	cwd, err := sftpClient.Getwd()
-	if err != nil {
-		fmt.Println("Getwd Err:" + err.Error())
-		return
-	}
-	fmt.Println("当前目录:" + cwd)
-
-	//显示文件/目录详情
-	fi, err := sftpClient.Lstat(cwd)
-	if err != nil {
-		fmt.Println("sftpClient.Lstat Err:" + err.Error())
-		return
-	}
-	fmt.Println(fi)
-	prefixDir := `/data/rdlucklog/`
-	//preSaveDirPath := `/Users/roc/go/src/hongze/hz_sync_logs/test/`
-	//dirCut := `/` //linux的目录分割符
-	preSaveDirPath := `E:\\hongze_logs\\`
-	dirCut := `\\`	//windows的目录分割符
-
-	yesterdayTime := time.Now().AddDate(0,0,-1)
-
-	for _,projectName := range projectNameArr{
-		serverFilePath := prefixDir + projectName
-		localFilePath := preSaveDirPath + projectName
-		//下载文件
-		for i := 0; i < 22; i++ {
-			dateTime := yesterdayTime.AddDate(0, 0, -i)
-			err = syncLog(sftpClient, dateTime, serverFilePath, localFilePath, dirCut)
-			if err != nil {
-				fmt.Println(serverFilePath+"下载失败，ERR：", err)
-			}
-		}
-
-	}
-
-
-	return
-	{
-		dataDirPath := `/data/rdlucklog`
-		saveDirPath := `E:\\log\\`
-		fileExt := `.api.log`
-
-		logPath := time.Now().AddDate(0, -1, 0).Format("200601")
-		dateObj, _ := time.Parse("2006-01-02", "2021-10-01")
-		for _, proName := range projectNameArr {
-			for i := 0; i < 31; i++ {
-				date := dateObj.AddDate(0, 0, i).Format("20060102")
-				remoteDirName := dataDirPath + `/` + proName + `/` + logPath + `/` + date + fileExt
-				remoteFile, err := sftpClient.Open(remoteDirName)
-				if err != nil {
-					fmt.Println("sftpClient.Open err:" + err.Error() + " ;remoteDirName:" + remoteDirName)
-					if err.Error() == "file does not exist" {
-						continue
-					} else {
-						fmt.Println("sftpClient.Open err:" + err.Error() + " ;remoteDirName:" + remoteDirName)
-						return
-					}
-				}
-				defer remoteFile.Close()
-				saveFileName := saveDirPath + proName + `\\202110\\`
-				localFileName := date + fileExt
-				fmt.Println("saveFileName:", saveFileName)
-				if !utils.FileIsExist(saveFileName) {
-					err = os.MkdirAll(saveFileName, 777)
-					if err != nil {
-						fmt.Println("创建目录失败,Err:" + err.Error())
-						return
-					}
-				}
-				localFilePath := saveFileName + localFileName
-				fmt.Println("localFilePath:" + localFilePath)
-				localFile, err := os.Create(localFilePath)
-				if err != nil {
-					fmt.Println("create file err:" + err.Error())
-					return
-				}
-				defer localFile.Close()
-				n, err := io.Copy(localFile, remoteFile)
-				if err != nil {
-					fmt.Println("copy err:" + err.Error())
-					return
-				}
-				//获取远程文件大小
-				remoteFileInfo, err := sftpClient.Stat(remoteDirName)
-				if err != nil {
-					fmt.Println("sftpClient.Stat Err:" + err.Error())
-					log.Fatalln(err.Error())
-				}
-				log.Printf("文件下载成功[%s->%s]远程文件大小：%s，下载文件大小：%s", remoteDirName, localFileName, formatFileSize(remoteFileInfo.Size()), formatFileSize(n))
-				//删除远程目录文件
-				//err = sftpClient.Remove(remoteDirName)
-				//if err != nil {
-				//	fmt.Println("sftpClient.Remove Err:" + err.Error())
-				//	return
-				//}
-			}
-		}
-	}
-}
+//func SyncHistoryLogs() {
+//	projectNameArr := []string{
+//		"equity_admin",
+//		"hongze_admin",
+//		"hongze_advisory",
+//		"hongze_api",
+//		"hongze_cygx",
+//		"hongze_mobile_admin",
+//		"hongze_open_api",
+//	}
+//	sshHost := `` //ssh的地址和端口（测试）
+//	pwd := ``     //ssh密码（测试）
+//
+//	sshConfig := &ssh.ClientConfig{
+//		User: "root",
+//		Auth: []ssh.AuthMethod{
+//			ssh.Password(pwd),
+//		},
+//		HostKeyCallback: ssh.InsecureIgnoreHostKey(),
+//		ClientVersion:   "",
+//		Timeout:         10 * time.Second,
+//	}
+//
+//	//建立与SSH服务器的链接
+//	sshClient, err := ssh.Dial("tcp", sshHost, sshConfig)
+//	if err != nil {
+//		fmt.Println("ssh.Dial Err:" + err.Error())
+//		return
+//	}
+//	defer sshClient.Close()
+//	sftpClient, err := sftp.NewClient(sshClient)
+//	if err != nil {
+//		fmt.Println("sftp.NewClient Err:" + err.Error())
+//		return
+//	}
+//	defer sftpClient.Close()
+//	//获取当前目录
+//	cwd, err := sftpClient.Getwd()
+//	if err != nil {
+//		fmt.Println("Getwd Err:" + err.Error())
+//		return
+//	}
+//	fmt.Println("当前目录:" + cwd)
+//
+//	//显示文件/目录详情
+//	fi, err := sftpClient.Lstat(cwd)
+//	if err != nil {
+//		fmt.Println("sftpClient.Lstat Err:" + err.Error())
+//		return
+//	}
+//	fmt.Println(fi)
+//	prefixDir := `/data/rdlucklog/`
+//	//preSaveDirPath := `/Users/roc/go/src/hongze/hz_sync_logs/test/`
+//	//dirCut := `/` //linux的目录分割符
+//	preSaveDirPath := `E:\\hongze_logs\\`
+//	dirCut := `\\` //windows的目录分割符
+//
+//	yesterdayTime := time.Now().AddDate(0, 0, -1)
+//
+//	for _, projectName := range projectNameArr {
+//		serverFilePath := prefixDir + projectName
+//		localFilePath := preSaveDirPath + projectName
+//		//下载文件
+//		for i := 0; i < 22; i++ {
+//			dateTime := yesterdayTime.AddDate(0, 0, -i)
+//			err = syncLog(sftpClient, dateTime, serverFilePath, localFilePath, dirCut)
+//			if err != nil {
+//				fmt.Println(serverFilePath+"下载失败，ERR：", err)
+//			}
+//		}
+//
+//	}
+//
+//	return
+//	{
+//		dataDirPath := `/data/rdlucklog`
+//		saveDirPath := `E:\\log\\`
+//		fileExt := `.api.log`
+//
+//		logPath := time.Now().AddDate(0, -1, 0).Format("200601")
+//		dateObj, _ := time.Parse("2006-01-02", "2021-10-01")
+//		for _, proName := range projectNameArr {
+//			for i := 0; i < 31; i++ {
+//				date := dateObj.AddDate(0, 0, i).Format("20060102")
+//				remoteDirName := dataDirPath + `/` + proName + `/` + logPath + `/` + date + fileExt
+//				remoteFile, err := sftpClient.Open(remoteDirName)
+//				if err != nil {
+//					fmt.Println("sftpClient.Open err:" + err.Error() + " ;remoteDirName:" + remoteDirName)
+//					if err.Error() == "file does not exist" {
+//						continue
+//					} else {
+//						fmt.Println("sftpClient.Open err:" + err.Error() + " ;remoteDirName:" + remoteDirName)
+//						return
+//					}
+//				}
+//				defer remoteFile.Close()
+//				saveFileName := saveDirPath + proName + `\\202110\\`
+//				localFileName := date + fileExt
+//				fmt.Println("saveFileName:", saveFileName)
+//				if !utils.FileIsExist(saveFileName) {
+//					err = os.MkdirAll(saveFileName, 777)
+//					if err != nil {
+//						fmt.Println("创建目录失败,Err:" + err.Error())
+//						return
+//					}
+//				}
+//				localFilePath := saveFileName + localFileName
+//				fmt.Println("localFilePath:" + localFilePath)
+//				localFile, err := os.Create(localFilePath)
+//				if err != nil {
+//					fmt.Println("create file err:" + err.Error())
+//					return
+//				}
+//				defer localFile.Close()
+//				n, err := io.Copy(localFile, remoteFile)
+//				if err != nil {
+//					fmt.Println("copy err:" + err.Error())
+//					return
+//				}
+//				//获取远程文件大小
+//				remoteFileInfo, err := sftpClient.Stat(remoteDirName)
+//				if err != nil {
+//					fmt.Println("sftpClient.Stat Err:" + err.Error())
+//					log.Fatalln(err.Error())
+//				}
+//				log.Printf("文件下载成功[%s->%s]远程文件大小：%s，下载文件大小：%s", remoteDirName, localFileName, formatFileSize(remoteFileInfo.Size()), formatFileSize(n))
+//				//删除远程目录文件
+//				//err = sftpClient.Remove(remoteDirName)
+//				//if err != nil {
+//				//	fmt.Println("sftpClient.Remove Err:" + err.Error())
+//				//	return
+//				//}
+//			}
+//		}
+//	}
+//}
 
 func SyncLogs2() {
 	projectNameArr := []string{