启动处理历史研究员头像压缩

controllers/analyst.go

@@ -89,7 +89,6 @@ func (this *AnalystController) UploadImage() {
 		return
 	}
 	//新增上传缩略图
-
 	thumbnailUrl, err := ossClient.UploadFile("", thumbnailPath, saveOssPath+"_thumbnail")
 	if err != nil {
 		br.Msg = "头像上传失败"

go.mod

@@ -31,6 +31,7 @@ require (
 	github.com/disintegration/imaging v1.6.2 // indirect
 	github.com/dustin/go-humanize v1.0.1 // indirect
 	github.com/fatih/structs v1.1.0 // indirect
+	github.com/gabriel-vasile/mimetype v1.4.7 // indirect
 	github.com/go-ini/ini v1.67.0 // indirect
 	github.com/goccy/go-json v0.10.3 // indirect
 	github.com/google/uuid v1.6.0 // indirect
@@ -64,10 +65,10 @@ require (
 	github.com/valyala/bytebufferpool v1.0.0 // indirect
 	github.com/xuri/efp v0.0.0-20240408161823-9ad904a10d6d // indirect
 	github.com/xuri/nfp v0.0.0-20240318013403-ab9948c2c4a7 // indirect
-	golang.org/x/crypto v0.28.0 // indirect
+	golang.org/x/crypto v0.29.0 // indirect
 	golang.org/x/image v0.23.0 // indirect
-	golang.org/x/net v0.30.0 // indirect
-	golang.org/x/sys v0.26.0 // indirect
+	golang.org/x/net v0.31.0 // indirect
+	golang.org/x/sys v0.27.0 // indirect
 	golang.org/x/text v0.21.0 // indirect
 	golang.org/x/time v0.6.0 // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20240820151423-278611b39280 // indirect

main.go

@@ -3,6 +3,7 @@ package main
 import (
 	"eta/eta_mini_crm_ht/controllers"
 	_ "eta/eta_mini_crm_ht/routers"
+	"eta/eta_mini_crm_ht/services"
 	"eta/eta_mini_crm_ht/utils"
 	"fmt"
 	"github.com/beego/beego/v2/core/logs"
@@ -23,6 +24,10 @@ func main() {
 	// 内存调整
 	web.BConfig.MaxMemory = 1024 * 1024 * 128
 	web.BConfig.RecoverFunc = Recover
+
+	if utils.MinioClient != nil {
+		services.RepairImageData()
+	}
 	web.Run()
 }
 

services/analyst.go

@@ -0,0 +1,84 @@
+package services
+
+import (
+	"eta/eta_mini_crm_ht/models"
+	"eta/eta_mini_crm_ht/utils"
+	"fmt"
+	"os"
+	"sync"
+	"time"
+)
+
+func RepairImageData() {
+	sql := ` and (head_origin_img_url ='' or head_origin_img_url is null) `
+	count, err := models.GetAnalystCount(sql)
+	if err != nil {
+		fmt.Println("修复研究员头像数据失败")
+		return
+	}
+	list, err := models.GetAnalystList(sql, 0, count)
+	if err != nil {
+		fmt.Println("修复研究员头像数据失败")
+		return
+	}
+	ossClient := NewOssClient()
+	if ossClient == nil {
+		fmt.Println("初始化OSS服务失败")
+		return
+	}
+	var wg sync.WaitGroup
+	wg.Add(len(list))
+	for _, v := range list {
+		go func(analyst *models.CrmFinancialAnalyst) {
+			defer wg.Done()
+			if analyst.HeadImgUrl == "" {
+				return
+			}
+			analyst.HeadOriginImgUrl = analyst.HeadImgUrl
+			dateDir := time.Now().Format(time.DateOnly)
+			downloadDir := utils.STATIC_DIR + "ht/analyst/repair/" + dateDir
+			if info, existErr := os.Stat(downloadDir); existErr != nil || !info.IsDir() {
+				err = os.MkdirAll(downloadDir, utils.DIR_MOD)
+				if err != nil {
+					fmt.Println("存储目录创建失败")
+					return
+				}
+			}
+			fileName := analyst.Name + "-头像"
+			savePath := downloadDir + "/" + fileName
+			savePath, err = ossClient.GetFile(analyst.HeadImgUrl, savePath)
+			if err != nil {
+				fmt.Printf("下载图片失败: %s,src:%s\n", err, analyst.HeadImgUrl)
+				return
+			}
+			var thumbnailPath string
+			thumbnailPath, err = utils.ImageResize(savePath, 400)
+			if err != nil {
+				fmt.Printf("图片压缩失败: %s,src:%s\n", err, analyst.HeadImgUrl)
+			}
+			if thumbnailPath != "" {
+				ossUploadDir := utils.RESOURCE_DIR + "analyst/"
+				saveOssPath := ossUploadDir + time.Now().Format("200601/20060102/")
+				imgName := utils.GetRandStringNoSpecialChar(28)
+				saveOssPath += imgName
+				var url string
+				url, err = ossClient.UploadFile("", thumbnailPath, saveOssPath)
+				if err != nil {
+					fmt.Printf("图片上传失败: %s,src:%s\n", err, analyst.HeadImgUrl)
+					return
+				}
+				analyst.HeadImgUrl = url
+				err = analyst.Update([]string{"head_img_url", "head_origin_img_url"})
+				if err != nil {
+					return
+				}
+			}
+			defer func() {
+				_ = os.Remove(savePath)
+				_ = os.Remove(thumbnailPath)
+			}()
+		}(v)
+	}
+	wg.Wait()
+	fmt.Println("修复研究员头像数据完成")
+}

services/minio.go

@@ -58,6 +58,45 @@ func (m *MinioOss) UploadFile(fileName, filePath, savePath string) (string, erro
 	return resourceUrl, err
 }
 
+func (m *MinioOss) GetExt(string) (ext string, err error) {
+	if utils.MinIoAccessKeyId == `` || utils.MinIoAccessKeySecret == `` {
+		err = errors.New("MinIo信息未配置")
+		return
+	}
+	ctx := context.Background()
+	// 此处兼容一下前后端endpoint不一致的情况, 前端用minio_endpoint后端用minio_back_endpoint, minio_back_endpoint为空则都取前者
+	endpoint := utils.MinIoEndpoint
+	if utils.MinIoBackEndpoint != "" {
+		endpoint = utils.MinIoBackEndpoint
+	}
+	accessKeyID := utils.MinIoAccessKeyId
+	secretAccessKey := utils.MinIoAccessKeySecret
+	useSSL := false
+	if utils.MinIoUseSSL == "true" {
+		useSSL = true
+	}
+	minioClient, err := minio.New(endpoint, &minio.Options{
+		Creds:  credentials.NewStaticV4(accessKeyID, secretAccessKey, ""),
+		Secure: useSSL,
+	})
+	if err != nil {
+		utils.FileLog.Error(err.Error())
+		return
+	}
+	bucketName := utils.MinIoBucketname
+	exists, errBucketExists := minioClient.BucketExists(ctx, bucketName)
+	if errBucketExists != nil || !exists {
+		err = fmt.Errorf("BucketExists: %v; err: %v", exists, errBucketExists)
+		return
+	}
+	info, minioErr := minioClient.StatObject(ctx, bucketName, "test.txt", minio.GetObjectOptions{})
+	if minioErr != nil {
+		err = fmt.Errorf("StatObject: %v; err: %v", info, minioErr)
+		return
+	}
+	ext = GetFileExtensionFromContentType(info.ContentType)
+	return
+}
 func (m *MinioOss) GetUploadToken() (token OssToken, err error) {
 	token.AccessKeyId = utils.MinIoAccessKeyId
 	token.SecretKeyId = utils.MinIoAccessKeySecret
@@ -70,7 +109,7 @@ func (m *MinioOss) GetUploadToken() (token OssToken, err error) {
 	return
 }
 
-func (m *MinioOss) GetFile(filePath, savePath string) (err error) {
+func (m *MinioOss) GetFile(filePath, savePath string) (path string, err error) {
 	if utils.MinIoAccessKeyId == `` || utils.MinIoAccessKeySecret == `` {
 		err = errors.New("MinIo信息未配置")
 		return
@@ -99,9 +138,17 @@ func (m *MinioOss) GetFile(filePath, savePath string) (err error) {
 	exists, errBucketExists := minioClient.BucketExists(ctx, bucketName)
 	if errBucketExists != nil || !exists {
 		err = fmt.Errorf("BucketExists: %v; err: %v", exists, errBucketExists)
+		utils.FileLog.Error(err.Error())
+		return
+	}
+	ext, err := m.GetExt(filePath)
+	if err != nil {
+		err = fmt.Errorf("GetExt: %v; err: %v", ext, err)
+		utils.FileLog.Error(err.Error())
 		return
 	}
-	err = minioClient.FGetObject(ctx, bucketName, filePath, savePath, minio.GetObjectOptions{})
+	path = savePath + ext
+	err = minioClient.FGetObject(ctx, bucketName, filePath, path, minio.GetObjectOptions{})
 	if err != nil {
 		utils.FileLog.Error(err.Error())
 		return

services/oss.go

@@ -5,11 +5,13 @@ import (
 	"errors"
 	"eta/eta_mini_crm_ht/utils"
 	"fmt"
-	"time"
-
-	"github.com/aliyun/aliyun-oss-go-sdk/oss"
-
 	"github.com/aliyun/alibaba-cloud-sdk-go/services/sts"
+	"github.com/aliyun/aliyun-oss-go-sdk/oss"
+	"github.com/gabriel-vasile/mimetype"
+	"io"
+	"os"
+	"strings"
+	"time"
 )
 
 type STSToken struct {
@@ -26,8 +28,8 @@ type STSToken struct {
 type OssClient interface {
 	UploadFile(string, string, string) (string, error)
 	GetUploadToken() (OssToken, error)
-	GetFile(filePath, savePath string) (err error)
-
+	GetFile(filePath, savePath string) (path string, err error)
+	GetExt(string) (string, error)
 	MultiUploadFile() error
 }
 
@@ -163,6 +165,27 @@ func NewSTSToken() (item *STSToken, err error) {
 
 type AliOss struct{}
 
+func (m *AliOss) GetExt(ossPath string) (ext string, err error) {
+	if utils.AccessKeyId == `` {
+		return "0", errors.New("阿里云信息未配置")
+	}
+	client, err := oss.New(utils.Endpoint, utils.AccessKeyId, utils.AccessKeySecret)
+	if err != nil {
+		return
+	}
+	bucket, err := client.Bucket(utils.Bucketname)
+	if err != nil {
+		return
+	}
+	objectKey := ossPath[strings.Index(ossPath, utils.RESOURCE_DIR):]
+	metaInfo, err := bucket.GetObjectMeta(objectKey)
+	if err != nil {
+		return
+	}
+	ext = GetFileExtensionFromContentType(metaInfo["Content-Type"][0])
+	return
+}
+
 // UploadFile 上传文件
 func (m *AliOss) UploadFile(fileName, filePath, savePath string) (string, error) {
 	if utils.AccessKeyId == `` {
@@ -208,6 +231,71 @@ func (m *AliOss) GetUploadToken() (token OssToken, err error) {
 func (m *AliOss) MultiUploadFile() (err error) {
 	return
 }
-func (m *AliOss) GetFile(filePath, savePath string) (err error) {
+func (m *AliOss) GetFile(filePath, savePath string) (path string, err error) {
+	if utils.AccessKeyId == `` {
+		return "", errors.New("阿里云信息未配置")
+	}
+	client, err := oss.New(utils.Endpoint, utils.AccessKeyId, utils.AccessKeySecret)
+	if err != nil {
+		return
+	}
+	bucket, err := client.Bucket(utils.Bucketname)
+	if err != nil {
+		return
+	}
+	objectKey := filePath[strings.Index(filePath, utils.RESOURCE_DIR):]
+	reader, err := bucket.GetObject(objectKey)
+	if err != nil {
+		return
+	}
+	buffer := make([]byte, 512)
+	n, err := reader.Read(buffer)
+	if err != nil && err != io.EOF {
+		return "", fmt.Errorf("Error reading object: %s", err)
+	}
+	mime := mimetype.Detect(buffer[:n])
+	fileExt := mime.Extension()
+	path = savePath + fileExt
+	defer reader.Close()
+	// 打开或创建本地文件
+	localFile, err := os.Create(path)
+	if err != nil {
+		utils.FileLog.Info("处理原始图片信息失败:Error create local dir: %s", err)
+		return
+	}
+	defer localFile.Close()
+	// 将io.ReadCloser中的数据复制到本地文件
+	_, err = localFile.Write(buffer[:n])
+	if err != nil {
+		utils.FileLog.Info("处理原始图片信息失败:Error writing to local file: %s", err)
+		return
+	}
+	_, err = io.Copy(localFile, reader)
 	return
 }
+func GetFileExtensionFromContentType(contentType string) string {
+	switch contentType {
+	case "image/jpeg":
+		return ".jpg"
+	case "image/png":
+		return ".png"
+	case "application/pdf":
+		return ".pdf"
+	case "application/msword":
+		return ".doc"
+	case "application/vnd.openxmlformats-officedocument.wordprocessingml.document":
+		return ".docx"
+	case "application/vnd.ms-excel":
+		return ".xls"
+	case "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet":
+		return ".xlsx"
+	case "text/plain":
+		return ".txt"
+	case "application/json":
+		return ".json"
+	case "application/octet-stream":
+		return ".bin"
+	default:
+		return ""
+	}
+}

utils/config.go

@@ -15,6 +15,7 @@ var (
 	REDIS_CACHE string      //缓存地址
 	Rc          RedisClient //redis缓存
 	PRCPool     ClientPool  //rpc连接池
+	MinioClient *MinioOss
 )
 var ObjectStorageClient string // 目前有oss minio，默认oss
 // 推送模版消息
@@ -255,4 +256,5 @@ func init() {
 
 	Rc = redisClient
 
+	MinioClient = DefaultClient()
 }

utils/minio.go

@@ -0,0 +1,125 @@
+package utils
+
+import (
+	"context"
+	"github.com/minio/minio-go/v7"
+	"github.com/minio/minio-go/v7/pkg/credentials"
+	"sync"
+	"time"
+)
+
+const (
+	DefaultExpiredTime = 30 * time.Minute
+)
+
+var (
+	once        sync.Once
+	minioClient *MinioOss
+)
+
+type MinioOss struct {
+	bucketName  string
+	ObjectName  string
+	minioClient *minio.Client
+	ExpiredTime time.Duration
+}
+
+func (mo *MinioOss) oss() *minio.Client {
+	return mo.minioClient
+}
+func (mo *MinioOss) GetObject(objectName string, filePath string) (err error) {
+	err = minioClient.oss().FGetObject(context.Background(), mo.GetBucketName(), objectName, filePath, minio.GetObjectOptions{})
+	return
+}
+func (mo *MinioOss) SaveObject(objectName string, filePath string) (err error) {
+	_, err = minioClient.oss().FPutObject(context.Background(), mo.GetBucketName(), objectName, filePath, minio.PutObjectOptions{})
+	return
+}
+func (mo *MinioOss) GetState(objectName string) (state minio.ObjectInfo, err error) {
+	state, err = minioClient.oss().StatObject(context.Background(), mo.GetBucketName(), objectName, minio.GetObjectOptions{})
+	return
+}
+func (mo *MinioOss) GetObjectExt(contentType string) (ext string) {
+	return getFileExtensionFromContentType(contentType)
+}
+func getFileExtensionFromContentType(contentType string) string {
+	switch contentType {
+	case "image/jpeg":
+		return ".jpg"
+	case "image/png":
+		return ".png"
+	case "application/pdf":
+		return ".pdf"
+	case "application/msword":
+		return ".doc"
+	case "application/vnd.openxmlformats-officedocument.wordprocessingml.document":
+		return ".docx"
+	case "application/vnd.ms-excel":
+		return ".xls"
+	case "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet":
+		return ".xlsx"
+	case "text/plain":
+		return ".txt"
+	case "application/json":
+		return ".json"
+	case "application/octet-stream":
+		return ".bin"
+	default:
+		return ""
+	}
+}
+
+func DefaultClient() *MinioOss {
+	once.Do(func() {
+		if minioClient == nil {
+			// 此处兼容一下前后端endpoint不一致的情况, 前端用minio_endpoint后端用minio_back_endpoint, minio_back_endpoint为空则都取前者
+			endpoint := MinIoEndpoint
+			if MinIoBackEndpoint != "" {
+				endpoint = MinIoBackEndpoint
+			}
+			accessKeyID := MinIoAccessKeyId
+			secretAccessKey := MinIoAccessKeySecret
+			useSSL := false
+			if MinIoUseSSL == "true" {
+				useSSL = true
+			}
+			client, err := minio.New(endpoint, &minio.Options{
+				Creds:  credentials.NewStaticV4(accessKeyID, secretAccessKey, ""),
+				Secure: useSSL,
+			})
+			if err != nil {
+				FileLog.Error("初始化minio客户端失败: %s", err)
+			}
+			bucketName := MinIoBucketname
+			exists, errBucketExists := client.BucketExists(context.Background(), bucketName)
+			if errBucketExists != nil || !exists {
+				FileLog.Error("BucketExists: %v; err: %v", exists, errBucketExists)
+			}
+			minioClient = &MinioOss{
+				minioClient: client,
+				bucketName:  bucketName,
+				ExpiredTime: DefaultExpiredTime,
+			}
+		}
+	})
+	return minioClient
+}
+
+func (mo *MinioOss) GetBucketName() string {
+	return mo.bucketName
+}
+func (mo *MinioOss) GetExpiredTime() time.Duration {
+	return mo.ExpiredTime
+}
+
+// 生成预签名的PUT URL
+func (mo *MinioOss) GeneratePreSignedPutURLHandler(objectName string) (url string, err error) {
+	// 生成预签名的PUT URL
+	PreSignedURL, err := minioClient.oss().PresignedPutObject(context.Background(), minioClient.GetBucketName(), objectName, minioClient.GetExpiredTime())
+	if err != nil {
+		FileLog.Error("生成预签名的PUT URL失败: %s", err)
+		return
+	}
+	url = PreSignedURL.String()
+	return
+}