From 0033a1cdbd9bc1a209dbdcc266924f2d3fcc55c6 Mon Sep 17 00:00:00 2001 From: Michael Li Date: Tue, 26 Jul 2022 00:00:25 +0800 Subject: [PATCH] optimize create post/comment logic that tweet type is media contents --- README.md | 6 ++-- config.yaml.sample | 4 ++- internal/conf/conf.go | 2 ++ internal/conf/settting.go | 4 +++ internal/core/storage.go | 3 +- internal/dao/storage/alioss.go | 28 ++++++++++++--- internal/dao/storage/cos.go | 9 +++-- internal/dao/storage/huaweiobs.go | 26 +++++++++++--- internal/dao/storage/localoss.go | 9 +++-- internal/dao/storage/minio.go | 31 +++++++++++++---- internal/dao/storage/storage.go | 56 ++++++++++++++++++++---------- internal/routers/api/attachment.go | 2 +- internal/service/comment.go | 16 +++++++-- internal/service/post.go | 22 ++++-------- internal/service/service.go | 25 +++++++++++++ internal/service/user.go | 15 +++++++- 16 files changed, 196 insertions(+), 62 deletions(-) diff --git a/README.md b/README.md index 5d0eb35c..f8b795c0 100644 --- a/README.md +++ b/README.md @@ -311,6 +311,7 @@ release/paopao-ce --no-default-features --features sqlite3,localoss,loggerfile,r `COS` 腾讯云对象存储服务； `HuaweiOBS` 华为云对象存储服务； `MinIO` [MinIO](https://github.com/minio/minio)对象存储服务； + `S3` AWS S3兼容的对象存储服务； `LocalOSS` 提供使用本地目录文件作为对象存储的功能，仅用于开发调试环境； * 缓存: Redis/SimpleCacheIndex/BigCacheIndex `SimpleCacheIndex` 提供简单的 广场推文列表 的缓存功能； @@ -326,8 +327,9 @@ release/paopao-ce --no-default-features --features sqlite3,localoss,loggerfile,r `Alipay` 开启基于[支付宝开放平台](https://open.alipay.com/)的钱包功能； * 短信验证码: SmsJuhe(需要开启sms) `Sms` 开启短信验证码功能，用于手机绑定验证手机是否注册者的；功能如果没有开启，手机绑定时任意短信验证码都可以绑定手机； -* 其他: PhoneBind - `PhoneBind` 开启手机绑定功能； +* 其他: PhoneBind/PersistObjct + `PhoneBind` 开启手机绑定功能； + `PersistObject` 开启对象存储的持久化对象功能，允许先创建临时对象然后再持久化； ### 搭建依赖环境 #### [Zinc](https://github.com/zinclabs/zinc) 搜索引擎: diff --git a/config.yaml.sample b/config.yaml.sample index 17c72053..9b8e1adb 100644 --- a/config.yaml.sample +++ b/config.yaml.sample @@ -13,7 +13,7 @@ Server: # 服务设置 Features: Default: ["Base", "MySQL", "Option", "Zinc", "LocalOSS", "LoggerFile"] Develop: ["Base", "MySQL", "BigCacheIndex", "Meili", "Sms", "AliOSS", "LoggerMeili", "Migration"] - Demo: ["Base", "MySQL", "Option", "Zinc", "Sms", "MinIO", "LoggerZinc"] + Demo: ["Base", "MySQL", "Option", "Zinc", "Sms", "MinIO", "LoggerZinc", "PersistObject"] Slim: ["Base", "Sqlite3", "LocalOSS", "LoggerFile"] Base: ["Redis", "Alipay", "PhoneBind"] Option: ["SimpleCacheIndex"] @@ -73,6 +73,8 @@ Meili: # Meili搜索配置 Index: paopao-data ApiKey: paopao-meilisearch Secure: False +ObjectStorage: # 对象存储通用配置 + RetainInDays: 2 # 临时对象过期时间多少天 AliOSS: # 阿里云OSS存储配置 Endpoint: AccessKeyID: diff --git a/internal/conf/conf.go b/internal/conf/conf.go index 4545e996..76a2536d 100644 --- a/internal/conf/conf.go +++ b/internal/conf/conf.go @@ -28,6 +28,7 @@ var ( TweetSearchSetting *TweetSearchS ZincSetting *ZincSettingS MeiliSetting *MeiliSettingS + ObjectStorage *ObjectStorageS AliOSSSetting *AliOSSSettingS COSSetting *COSSettingS HuaweiOBSSetting *HuaweiOBSSettingS @@ -72,6 +73,7 @@ func setupSetting(suite []string, noDefault bool) error { "Meili": &MeiliSetting, "Redis": &redisSetting, "JWT": &JWTSetting, + "ObjectStorage": &ObjectStorage, "AliOSS": &AliOSSSetting, "COS": &COSSetting, "HuaweiOBS": &HuaweiOBSSetting, diff --git a/internal/conf/settting.go b/internal/conf/settting.go index d6c66a8d..e5d0d00e 100644 --- a/internal/conf/settting.go +++ b/internal/conf/settting.go @@ -134,6 +134,10 @@ type Sqlite3SettingS struct { Path string } +type ObjectStorageS struct { + RetainInDays int +} + type MinIOSettingS struct { AccessKey string SecretKey string diff --git a/internal/core/storage.go b/internal/core/storage.go index 2203c6f4..dd74778a 100644 --- a/internal/core/storage.go +++ b/internal/core/storage.go @@ -6,7 +6,8 @@ import ( // ObjectStorageService storage service interface that implement base AliOSS、MINIO or other type ObjectStorageService interface { - PutObject(objectKey string, reader io.Reader, objectSize int64, contentType string) (string, error) + PutObject(objectKey string, reader io.Reader, objectSize int64, contentType string, persistance bool) (string, error) + PersistObject(objectKey string) error DeleteObject(objectKey string) error DeleteObjects(objectKeys []string) error IsObjectExist(objectKey string) (bool, error) diff --git a/internal/dao/storage/alioss.go b/internal/dao/storage/alioss.go index ff2f0a2d..0a560095 100644 --- a/internal/dao/storage/alioss.go +++ b/internal/dao/storage/alioss.go @@ -4,6 +4,7 @@ import ( "io" "net/url" "strings" + "time" "github.com/Masterminds/semver/v3" "github.com/aliyun/aliyun-oss-go-sdk/oss" @@ -17,8 +18,11 @@ var ( ) type aliossServant struct { - bucket *oss.Bucket - domain string + bucket *oss.Bucket + domain string + retainInDays time.Duration + retainUntilDate time.Time + allowPersistObject bool } func (s *aliossServant) Name() string { @@ -26,17 +30,31 @@ func (s *aliossServant) Name() string { } func (s *aliossServant) Version() *semver.Version { - return semver.MustParse("v0.1.0") + return semver.MustParse("v0.2.0") } -func (s *aliossServant) PutObject(objectKey string, reader io.Reader, objectSize int64, contentType string) (string, error) { - err := s.bucket.PutObject(objectKey, reader, oss.ContentLength(objectSize), oss.ContentType(contentType)) +func (s *aliossServant) PutObject(objectKey string, reader io.Reader, objectSize int64, contentType string, persistance bool) (string, error) { + options := []oss.Option{ + oss.ContentLength(objectSize), + oss.ContentType(contentType), + } + if s.allowPersistObject && !persistance { + options = append(options, oss.Expires(time.Now().Add(s.retainInDays))) + } + err := s.bucket.PutObject(objectKey, reader, options...) if err != nil { return "", err } return s.domain + objectKey, nil } +func (s *aliossServant) PersistObject(objectKey string) error { + if !s.allowPersistObject { + return nil + } + return s.bucket.SetObjectMeta(objectKey, oss.Expires(s.retainUntilDate)) +} + func (s *aliossServant) DeleteObject(objectKey string) error { return s.bucket.DeleteObject(objectKey) } diff --git a/internal/dao/storage/cos.go b/internal/dao/storage/cos.go index b429fb4c..aaf63fd4 100644 --- a/internal/dao/storage/cos.go +++ b/internal/dao/storage/cos.go @@ -30,10 +30,10 @@ func (s *cosServant) Name() string { } func (s *cosServant) Version() *semver.Version { - return semver.MustParse("v0.1.0") + return semver.MustParse("v0.2.0") } -func (s *cosServant) PutObject(objectKey string, reader io.Reader, objectSize int64, contentType string) (string, error) { +func (s *cosServant) PutObject(objectKey string, reader io.Reader, objectSize int64, contentType string, _persistance bool) (string, error) { _, err := s.client.Object.Put(context.Background(), objectKey, reader, &cos.ObjectPutOptions{ ObjectPutHeaderOptions: &cos.ObjectPutHeaderOptions{ ContentType: contentType, @@ -45,6 +45,11 @@ func (s *cosServant) PutObject(objectKey string, reader io.Reader, objectSize in return s.domain + objectKey, nil } +func (s *cosServant) PersistObject(objectKey string) error { + // empty + return nil +} + func (s *cosServant) DeleteObject(objectKey string) error { _, err := s.client.Object.Delete(context.Background(), objectKey) return err diff --git a/internal/dao/storage/huaweiobs.go b/internal/dao/storage/huaweiobs.go index f9d572dc..913e7108 100644 --- a/internal/dao/storage/huaweiobs.go +++ b/internal/dao/storage/huaweiobs.go @@ -17,9 +17,12 @@ var ( ) type huaweiobsServant struct { - client *obs.ObsClient - bucket string - domain string + client *obs.ObsClient + bucket string + domain string + retainInDays int64 + retainUntilDays string + allowPersistObject bool } func (s *huaweiobsServant) Name() string { @@ -27,13 +30,16 @@ func (s *huaweiobsServant) Name() string { } func (s *huaweiobsServant) Version() *semver.Version { - return semver.MustParse("v0.1.0") + return semver.MustParse("v0.2.0") } -func (s *huaweiobsServant) PutObject(objectKey string, reader io.Reader, objectSize int64, contentType string) (string, error) { +func (s *huaweiobsServant) PutObject(objectKey string, reader io.Reader, objectSize int64, contentType string, persistance bool) (string, error) { input := &obs.PutObjectInput{} input.Bucket, input.Key, input.Body = s.bucket, objectKey, reader input.ContentType, input.ContentLength = contentType, objectSize + if s.allowPersistObject && !persistance { + input.Expires = s.retainInDays + } _, err := s.client.PutObject(input) if err != nil { return "", err @@ -41,6 +47,16 @@ func (s *huaweiobsServant) PutObject(objectKey string, reader io.Reader, objectS return s.domain + objectKey, nil } +func (s *huaweiobsServant) PersistObject(objectKey string) error { + if !s.allowPersistObject { + return nil + } + _, err := s.client.SetObjectMetadata(&obs.SetObjectMetadataInput{ + Expires: s.retainUntilDays, + }) + return err +} + func (s *huaweiobsServant) DeleteObject(objectKey string) error { _, err := s.client.DeleteObject(&obs.DeleteObjectInput{ Bucket: s.bucket, diff --git a/internal/dao/storage/localoss.go b/internal/dao/storage/localoss.go index 00f7d378..c2792bb0 100644 --- a/internal/dao/storage/localoss.go +++ b/internal/dao/storage/localoss.go @@ -28,10 +28,10 @@ func (s *localossServant) Name() string { } func (s *localossServant) Version() *semver.Version { - return semver.MustParse("v0.1.0") + return semver.MustParse("v0.2.0") } -func (s *localossServant) PutObject(objectKey string, reader io.Reader, objectSize int64, contentType string) (string, error) { +func (s *localossServant) PutObject(objectKey string, reader io.Reader, objectSize int64, contentType string, _persistance bool) (string, error) { saveDir := s.savePath + filepath.Dir(objectKey) err := os.MkdirAll(saveDir, 0750) if err != nil && !os.IsExist(err) { @@ -57,6 +57,11 @@ func (s *localossServant) PutObject(objectKey string, reader io.Reader, objectSi return s.domain + objectKey, nil } +func (s *localossServant) PersistObject(objectKey string) error { + // empty + return nil +} + func (s *localossServant) DeleteObject(objectKey string) error { return os.Remove(s.savePath + objectKey) } diff --git a/internal/dao/storage/minio.go b/internal/dao/storage/minio.go index 4a23b71b..a73bea9e 100644 --- a/internal/dao/storage/minio.go +++ b/internal/dao/storage/minio.go @@ -19,9 +19,12 @@ var ( ) type minioServant struct { - client *minio.Client - bucket string - domain string + client *minio.Client + bucket string + domain string + retainInDays time.Duration + retainUntilDate time.Time + allowPersistObject bool } type s3Servant = minioServant @@ -31,11 +34,16 @@ func (s *minioServant) Name() string { } func (s *minioServant) Version() *semver.Version { - return semver.MustParse("v0.1.0") + return semver.MustParse("v0.2.0") } -func (s *minioServant) PutObject(objectKey string, reader io.Reader, objectSize int64, contentType string) (string, error) { - uploadInfo, err := s.client.PutObject(context.Background(), s.bucket, objectKey, reader, objectSize, minio.PutObjectOptions{ContentType: contentType}) +func (s *minioServant) PutObject(objectKey string, reader io.Reader, objectSize int64, contentType string, persistance bool) (string, error) { + opts := minio.PutObjectOptions{ContentType: contentType} + if s.allowPersistObject && !persistance { + opts.Mode = minio.Governance + opts.RetainUntilDate = time.Now().Add(s.retainInDays) + } + uploadInfo, err := s.client.PutObject(context.Background(), s.bucket, objectKey, reader, objectSize, opts) if err != nil { return "", err } @@ -43,6 +51,17 @@ func (s *minioServant) PutObject(objectKey string, reader io.Reader, objectSize return s.domain + objectKey, nil } +func (s *minioServant) PersistObject(objectKey string) error { + if !s.allowPersistObject { + return nil + } + retentionMode := minio.Governance + return s.client.PutObjectRetention(context.Background(), s.bucket, objectKey, minio.PutObjectRetentionOptions{ + Mode: &retentionMode, + RetainUntilDate: &s.retainUntilDate, + }) +} + func (s *minioServant) DeleteObject(objectKey string) error { return s.client.RemoveObject(context.Background(), s.bucket, objectKey, minio.RemoveObjectOptions{ForceDelete: true}) } diff --git a/internal/dao/storage/storage.go b/internal/dao/storage/storage.go index a1d0a440..a33c5107 100644 --- a/internal/dao/storage/storage.go +++ b/internal/dao/storage/storage.go @@ -5,6 +5,8 @@ import ( "net/http" "net/url" "path/filepath" + "strconv" + "time" "github.com/aliyun/aliyun-oss-go-sdk/oss" "github.com/huaweicloud/huaweicloud-sdk-go-obs/obs" @@ -19,17 +21,20 @@ import ( func MustAliossService() (core.ObjectStorageService, core.VersionInfo) { client, err := oss.New(conf.AliOSSSetting.Endpoint, conf.AliOSSSetting.AccessKeyID, conf.AliOSSSetting.AccessKeySecret) if err != nil { - logrus.Fatalf("alioss.New err: %v", err) + logrus.Fatalf("storage.MustAliossService create client err: %s", err) } bucket, err := client.Bucket(conf.AliOSSSetting.Bucket) if err != nil { - logrus.Fatalf("client.Bucket err: %v", err) + logrus.Fatalf("storage.MustAliossService create bucket err: %v", err) } obj := &aliossServant{ - bucket: bucket, - domain: conf.GetOssDomain(), + bucket: bucket, + domain: conf.GetOssDomain(), + retainInDays: time.Duration(conf.ObjectStorage.RetainInDays) * time.Hour * 24, + retainUntilDate: time.Date(2049, time.December, 1, 12, 0, 0, 0, time.UTC), + allowPersistObject: conf.CfgIf("PersistObject"), } return obj, obj } @@ -56,12 +61,17 @@ func MustHuaweiobsService() (core.ObjectStorageService, core.VersionInfo) { s := conf.HuaweiOBSSetting client, err := obs.New(s.AccessKey, s.SecretKey, s.Endpoint) if err != nil { - logrus.Fatalf("create huawei obs client failed: %s", err) + logrus.Fatalf("storage.MustHuaweiobsService create huawei obs client failed: %s", err) } + + retainUntilDays := time.Until(time.Date(2049, time.December, 1, 12, 0, 0, 0, time.UTC)) / (24 * time.Hour) obj := &huaweiobsServant{ - client: client, - bucket: s.Bucket, - domain: conf.GetOssDomain(), + client: client, + bucket: s.Bucket, + domain: conf.GetOssDomain(), + retainInDays: int64(conf.ObjectStorage.RetainInDays), + retainUntilDays: strconv.FormatInt(int64(retainUntilDays), 10), + allowPersistObject: conf.CfgIf("PersistObject"), } return obj, obj } @@ -69,7 +79,7 @@ func MustHuaweiobsService() (core.ObjectStorageService, core.VersionInfo) { func MustLocalossService() (core.ObjectStorageService, core.VersionInfo) { savePath, err := filepath.Abs(conf.LocalOSSSetting.SavePath) if err != nil { - logrus.Fatalf("get localOSS save path err: %v", err) + logrus.Fatalf("storage.MustLocalossService get localOSS save path err: %v", err) } obj := &localossServant{ @@ -86,14 +96,18 @@ func MustMinioService() (core.ObjectStorageService, core.VersionInfo) { Secure: conf.MinIOSetting.Secure, }) if err != nil { - logrus.Fatalf("minio.New err: %v", err) + logrus.Fatalf("storage.MustMinioService create client failed: %s", err) } ms := &minioServant{ - client: client, - bucket: conf.MinIOSetting.Bucket, - domain: conf.GetOssDomain(), + client: client, + bucket: conf.MinIOSetting.Bucket, + domain: conf.GetOssDomain(), + retainInDays: time.Duration(conf.ObjectStorage.RetainInDays) * time.Hour * 24, + retainUntilDate: time.Date(2049, time.December, 1, 12, 0, 0, 0, time.UTC), + allowPersistObject: conf.CfgIf("PersistObject"), } + return ms, ms } @@ -104,13 +118,17 @@ func MustS3Service() (core.ObjectStorageService, core.VersionInfo) { Secure: conf.S3Setting.Secure, }) if err != nil { - logrus.Fatalf("s3.New err: %v", err) + logrus.Fatalf("storage.MustS3Service create client failed: %s", err) } - obj := &s3Servant{ - client: client, - bucket: conf.MinIOSetting.Bucket, - domain: conf.GetOssDomain(), + s3 := &s3Servant{ + client: client, + bucket: conf.MinIOSetting.Bucket, + domain: conf.GetOssDomain(), + retainInDays: time.Duration(conf.ObjectStorage.RetainInDays) * time.Hour * 24, + retainUntilDate: time.Date(2049, time.December, 1, 12, 0, 0, 0, time.UTC), + allowPersistObject: conf.CfgIf("PersistObject"), } - return obj, obj + + return s3, s3 } diff --git a/internal/routers/api/attachment.go b/internal/routers/api/attachment.go index 79e03584..905b2c03 100644 --- a/internal/routers/api/attachment.go +++ b/internal/routers/api/attachment.go @@ -102,7 +102,7 @@ func UploadAttachment(c *gin.Context) { randomPath := uuid.Must(uuid.NewV4()).String() ossSavePath := uploadType + "/" + GeneratePath(randomPath[:8]) + "/" + randomPath[9:] + fileExt - objectUrl, err := objectStorage.PutObject(ossSavePath, file, fileHeader.Size, contentType) + objectUrl, err := objectStorage.PutObject(ossSavePath, file, fileHeader.Size, contentType, false) if err != nil { logrus.Errorf("putObject err: %v", err) response.ToErrorResponse(errcode.FileUploadFailed) diff --git a/internal/service/comment.go b/internal/service/comment.go index fa2864db..f5165d71 100644 --- a/internal/service/comment.go +++ b/internal/service/comment.go @@ -89,7 +89,19 @@ func GetPostComments(postID int64, sort string, offset, limit int) ([]*model.Com return commentsFormated, totalRows, nil } -func CreatePostComment(ctx *gin.Context, userID int64, param CommentCreationReq) (*model.Comment, error) { +func CreatePostComment(ctx *gin.Context, userID int64, param CommentCreationReq) (comment *model.Comment, err error) { + var mediaContents []string + + defer func() { + if err != nil { + deleteOssObjects(mediaContents) + } + }() + + if mediaContents, err = persistMediaContents(param.Contents); err != nil { + return + } + // 加载Post post, err := ds.GetPostByID(param.PostID) @@ -101,7 +113,7 @@ func CreatePostComment(ctx *gin.Context, userID int64, param CommentCreationReq) return nil, errcode.MaxCommentCount } ip := ctx.ClientIP() - comment := &model.Comment{ + comment = &model.Comment{ PostID: post.ID, UserID: userID, IP: ip, diff --git a/internal/service/post.go b/internal/service/post.go index a0228044..37d1334d 100644 --- a/internal/service/post.go +++ b/internal/service/post.go @@ -104,24 +104,18 @@ func tagsFrom(originTags []string) []string { // CreatePost 创建文章 // TODO: 推文+推文内容需要在一个事务中添加，后续优化 func CreatePost(c *gin.Context, userID int64, param PostCreationReq) (post *model.Post, err error) { - // 获取媒体内容 - mediaContents := make([]string, 0, len(param.Contents)) - for _, item := range param.Contents { - switch item.Type { - case model.CONTENT_TYPE_IMAGE, - model.CONTENT_TYPE_VIDEO, - model.CONTENT_TYPE_AUDIO, - model.CONTENT_TYPE_ATTACHMENT, - model.CONTENT_TYPE_CHARGE_ATTACHMENT: - mediaContents = append(mediaContents, item.Content) - } - } + var mediaContents []string + defer func() { if err != nil { deleteOssObjects(mediaContents) } }() + if mediaContents, err = persistMediaContents(param.Contents); err != nil { + return + } + ip := c.ClientIP() tags := tagsFrom(param.Tags) post = &model.Post{ @@ -168,9 +162,7 @@ func CreatePost(c *gin.Context, userID int64, param PostCreationReq) (post *mode UserID: userID, Tag: t, } - if _, err := ds.CreateTag(tag); err != nil { - return nil, err - } + ds.CreateTag(tag) } // 创建用户消息提醒 diff --git a/internal/service/service.go b/internal/service/service.go index c14c3153..7fd91ec8 100644 --- a/internal/service/service.go +++ b/internal/service/service.go @@ -4,6 +4,8 @@ import ( "github.com/rocboss/paopao-ce/internal/conf" "github.com/rocboss/paopao-ce/internal/core" "github.com/rocboss/paopao-ce/internal/dao" + "github.com/rocboss/paopao-ce/internal/model" + "github.com/sirupsen/logrus" ) var ( @@ -19,3 +21,26 @@ func Initialize() { oss = dao.ObjectStorageService() DisablePhoneVerify = !conf.CfgIf("Sms") } + +// persistMediaContents 获取媒体内容并持久化 +func persistMediaContents(contents []*PostContentItem) (items []string, err error) { + items = make([]string, 0, len(contents)) + for _, item := range contents { + switch item.Type { + case model.CONTENT_TYPE_IMAGE, + model.CONTENT_TYPE_VIDEO, + model.CONTENT_TYPE_AUDIO, + model.CONTENT_TYPE_ATTACHMENT, + model.CONTENT_TYPE_CHARGE_ATTACHMENT: + items = append(items, item.Content) + if err != nil { + continue + } + if err = oss.PersistObject(oss.ObjectKey(item.Content)); err != nil { + logrus.Errorf("service.persistMediaContents failed: %s", err) + continue + } + } + } + return +} diff --git a/internal/service/user.go b/internal/service/user.go index d64d512e..e3faf1d3 100644 --- a/internal/service/user.go +++ b/internal/service/user.go @@ -14,6 +14,7 @@ import ( "github.com/rocboss/paopao-ce/pkg/convert" "github.com/rocboss/paopao-ce/pkg/errcode" "github.com/rocboss/paopao-ce/pkg/util" + "github.com/sirupsen/logrus" ) const MAX_CAPTCHA_TIMES = 2 @@ -267,10 +268,22 @@ func UpdateUserInfo(user *model.User) *errcode.Error { return nil } -func ChangeUserAvatar(user *model.User, avatar string) *errcode.Error { +func ChangeUserAvatar(user *model.User, avatar string) (err *errcode.Error) { + defer func() { + if err != nil { + deleteOssObjects([]string{avatar}) + } + }() + if err := ds.CheckAttachment(avatar); err != nil { return errcode.InvalidParams } + + if err := oss.PersistObject(oss.ObjectKey(avatar)); err != nil { + logrus.Errorf("service.ChangeUserAvatar persist object failed: %s", err) + return errcode.ServerError + } + user.Avatar = avatar return UpdateUserInfo(user) }