optimize create post/comment logic that tweet type is media contents

README.md

@@ -311,6 +311,7 @@ release/paopao-ce --no-default-features --features sqlite3,localoss,loggerfile,r
   `COS` 腾讯云对象存储服务；  
   `HuaweiOBS` 华为云对象存储服务；  
   `MinIO` [MinIO](https://github.com/minio/minio)对象存储服务；  
+  `S3` AWS S3兼容的对象存储服务；    
   `LocalOSS` 提供使用本地目录文件作为对象存储的功能，仅用于开发调试环境；  
 * 缓存: Redis/SimpleCacheIndex/BigCacheIndex     
   `SimpleCacheIndex` 提供简单的 广场推文列表 的缓存功能；   
@@ -326,8 +327,9 @@ release/paopao-ce --no-default-features --features sqlite3,localoss,loggerfile,r
   `Alipay`  开启基于[支付宝开放平台](https://open.alipay.com/)的钱包功能；   
 * 短信验证码: SmsJuhe(需要开启sms)    
   `Sms` 开启短信验证码功能，用于手机绑定验证手机是否注册者的；功能如果没有开启，手机绑定时任意短信验证码都可以绑定手机；     
-* 其他: PhoneBind   
-  `PhoneBind` 开启手机绑定功能；    
+* 其他: PhoneBind/PersistObjct
+  `PhoneBind` 开启手机绑定功能；
+  `PersistObject` 开启对象存储的持久化对象功能，允许先创建临时对象然后再持久化；     
 
 ### 搭建依赖环境
 #### [Zinc](https://github.com/zinclabs/zinc) 搜索引擎:

config.yaml.sample

@@ -13,7 +13,7 @@ Server: # 服务设置
 Features:
   Default: ["Base", "MySQL", "Option", "Zinc", "LocalOSS", "LoggerFile"]
   Develop: ["Base", "MySQL", "BigCacheIndex", "Meili", "Sms", "AliOSS", "LoggerMeili", "Migration"]
-  Demo: ["Base", "MySQL", "Option", "Zinc", "Sms", "MinIO", "LoggerZinc"]
+  Demo: ["Base", "MySQL", "Option", "Zinc", "Sms", "MinIO", "LoggerZinc", "PersistObject"]
   Slim: ["Base", "Sqlite3", "LocalOSS", "LoggerFile"]
   Base: ["Redis", "Alipay", "PhoneBind"]
   Option: ["SimpleCacheIndex"]
@@ -73,6 +73,8 @@ Meili: # Meili搜索配置
   Index: paopao-data
   ApiKey: paopao-meilisearch
   Secure: False
+ObjectStorage: # 对象存储通用配置
+  RetainInDays: 2   # 临时对象过期时间多少天
 AliOSS: # 阿里云OSS存储配置
   Endpoint: 
   AccessKeyID: 

internal/conf/conf.go

@@ -28,6 +28,7 @@ var (
 	TweetSearchSetting      *TweetSearchS
 	ZincSetting             *ZincSettingS
 	MeiliSetting            *MeiliSettingS
+	ObjectStorage           *ObjectStorageS
 	AliOSSSetting           *AliOSSSettingS
 	COSSetting              *COSSettingS
 	HuaweiOBSSetting        *HuaweiOBSSettingS
@@ -72,6 +73,7 @@ func setupSetting(suite []string, noDefault bool) error {
 		"Meili":            &MeiliSetting,
 		"Redis":            &redisSetting,
 		"JWT":              &JWTSetting,
+		"ObjectStorage":    &ObjectStorage,
 		"AliOSS":           &AliOSSSetting,
 		"COS":              &COSSetting,
 		"HuaweiOBS":        &HuaweiOBSSetting,

internal/conf/settting.go

@@ -134,6 +134,10 @@ type Sqlite3SettingS struct {
 	Path string
 }
 
+type ObjectStorageS struct {
+	RetainInDays int
+}
+
 type MinIOSettingS struct {
 	AccessKey string
 	SecretKey string

internal/core/storage.go

@@ -6,7 +6,8 @@ import (
 
 // ObjectStorageService storage service interface that implement base AliOSS、MINIO or other
 type ObjectStorageService interface {
-	PutObject(objectKey string, reader io.Reader, objectSize int64, contentType string) (string, error)
+	PutObject(objectKey string, reader io.Reader, objectSize int64, contentType string, persistance bool) (string, error)
+	PersistObject(objectKey string) error
 	DeleteObject(objectKey string) error
 	DeleteObjects(objectKeys []string) error
 	IsObjectExist(objectKey string) (bool, error)

internal/dao/storage/alioss.go

@@ -4,6 +4,7 @@ import (
 	"io"
 	"net/url"
 	"strings"
+	"time"
 
 	"github.com/Masterminds/semver/v3"
 	"github.com/aliyun/aliyun-oss-go-sdk/oss"
@@ -17,8 +18,11 @@ var (
 )
 
 type aliossServant struct {
-	bucket *oss.Bucket
-	domain string
+	bucket             *oss.Bucket
+	domain             string
+	retainInDays       time.Duration
+	retainUntilDate    time.Time
+	allowPersistObject bool
 }
 
 func (s *aliossServant) Name() string {
@@ -26,17 +30,31 @@ func (s *aliossServant) Name() string {
 }
 
 func (s *aliossServant) Version() *semver.Version {
-	return semver.MustParse("v0.1.0")
+	return semver.MustParse("v0.2.0")
 }
 
-func (s *aliossServant) PutObject(objectKey string, reader io.Reader, objectSize int64, contentType string) (string, error) {
-	err := s.bucket.PutObject(objectKey, reader, oss.ContentLength(objectSize), oss.ContentType(contentType))
+func (s *aliossServant) PutObject(objectKey string, reader io.Reader, objectSize int64, contentType string, persistance bool) (string, error) {
+	options := []oss.Option{
+		oss.ContentLength(objectSize),
+		oss.ContentType(contentType),
+	}
+	if s.allowPersistObject && !persistance {
+		options = append(options, oss.Expires(time.Now().Add(s.retainInDays)))
+	}
+	err := s.bucket.PutObject(objectKey, reader, options...)
 	if err != nil {
 		return "", err
 	}
 	return s.domain + objectKey, nil
 }
 
+func (s *aliossServant) PersistObject(objectKey string) error {
+	if !s.allowPersistObject {
+		return nil
+	}
+	return s.bucket.SetObjectMeta(objectKey, oss.Expires(s.retainUntilDate))
+}
+
 func (s *aliossServant) DeleteObject(objectKey string) error {
 	return s.bucket.DeleteObject(objectKey)
 }

internal/dao/storage/cos.go

@@ -30,10 +30,10 @@ func (s *cosServant) Name() string {
 }
 
 func (s *cosServant) Version() *semver.Version {
-	return semver.MustParse("v0.1.0")
+	return semver.MustParse("v0.2.0")
 }
 
-func (s *cosServant) PutObject(objectKey string, reader io.Reader, objectSize int64, contentType string) (string, error) {
+func (s *cosServant) PutObject(objectKey string, reader io.Reader, objectSize int64, contentType string, _persistance bool) (string, error) {
 	_, err := s.client.Object.Put(context.Background(), objectKey, reader, &cos.ObjectPutOptions{
 		ObjectPutHeaderOptions: &cos.ObjectPutHeaderOptions{
 			ContentType: contentType,
@@ -45,6 +45,11 @@ func (s *cosServant) PutObject(objectKey string, reader io.Reader, objectSize in
 	return s.domain + objectKey, nil
 }
 
+func (s *cosServant) PersistObject(objectKey string) error {
+	// empty
+	return nil
+}
+
 func (s *cosServant) DeleteObject(objectKey string) error {
 	_, err := s.client.Object.Delete(context.Background(), objectKey)
 	return err

internal/dao/storage/huaweiobs.go

@@ -17,9 +17,12 @@ var (
 )
 
 type huaweiobsServant struct {
-	client *obs.ObsClient
-	bucket string
-	domain string
+	client             *obs.ObsClient
+	bucket             string
+	domain             string
+	retainInDays       int64
+	retainUntilDays    string
+	allowPersistObject bool
 }
 
 func (s *huaweiobsServant) Name() string {
@@ -27,13 +30,16 @@ func (s *huaweiobsServant) Name() string {
 }
 
 func (s *huaweiobsServant) Version() *semver.Version {
-	return semver.MustParse("v0.1.0")
+	return semver.MustParse("v0.2.0")
 }
 
-func (s *huaweiobsServant) PutObject(objectKey string, reader io.Reader, objectSize int64, contentType string) (string, error) {
+func (s *huaweiobsServant) PutObject(objectKey string, reader io.Reader, objectSize int64, contentType string, persistance bool) (string, error) {
 	input := &obs.PutObjectInput{}
 	input.Bucket, input.Key, input.Body = s.bucket, objectKey, reader
 	input.ContentType, input.ContentLength = contentType, objectSize
+	if s.allowPersistObject && !persistance {
+		input.Expires = s.retainInDays
+	}
 	_, err := s.client.PutObject(input)
 	if err != nil {
 		return "", err
@@ -41,6 +47,16 @@ func (s *huaweiobsServant) PutObject(objectKey string, reader io.Reader, objectS
 	return s.domain + objectKey, nil
 }
 
+func (s *huaweiobsServant) PersistObject(objectKey string) error {
+	if !s.allowPersistObject {
+		return nil
+	}
+	_, err := s.client.SetObjectMetadata(&obs.SetObjectMetadataInput{
+		Expires: s.retainUntilDays,
+	})
+	return err
+}
+
 func (s *huaweiobsServant) DeleteObject(objectKey string) error {
 	_, err := s.client.DeleteObject(&obs.DeleteObjectInput{
 		Bucket: s.bucket,

internal/dao/storage/localoss.go

@@ -28,10 +28,10 @@ func (s *localossServant) Name() string {
 }
 
 func (s *localossServant) Version() *semver.Version {
-	return semver.MustParse("v0.1.0")
+	return semver.MustParse("v0.2.0")
 }
 
-func (s *localossServant) PutObject(objectKey string, reader io.Reader, objectSize int64, contentType string) (string, error) {
+func (s *localossServant) PutObject(objectKey string, reader io.Reader, objectSize int64, contentType string, _persistance bool) (string, error) {
 	saveDir := s.savePath + filepath.Dir(objectKey)
 	err := os.MkdirAll(saveDir, 0750)
 	if err != nil && !os.IsExist(err) {
@@ -57,6 +57,11 @@ func (s *localossServant) PutObject(objectKey string, reader io.Reader, objectSi
 	return s.domain + objectKey, nil
 }
 
+func (s *localossServant) PersistObject(objectKey string) error {
+	// empty
+	return nil
+}
+
 func (s *localossServant) DeleteObject(objectKey string) error {
 	return os.Remove(s.savePath + objectKey)
 }

internal/dao/storage/minio.go

@@ -19,9 +19,12 @@ var (
 )
 
 type minioServant struct {
-	client *minio.Client
-	bucket string
-	domain string
+	client             *minio.Client
+	bucket             string
+	domain             string
+	retainInDays       time.Duration
+	retainUntilDate    time.Time
+	allowPersistObject bool
 }
 
 type s3Servant = minioServant
@@ -31,11 +34,16 @@ func (s *minioServant) Name() string {
 }
 
 func (s *minioServant) Version() *semver.Version {
-	return semver.MustParse("v0.1.0")
+	return semver.MustParse("v0.2.0")
 }
 
-func (s *minioServant) PutObject(objectKey string, reader io.Reader, objectSize int64, contentType string) (string, error) {
-	uploadInfo, err := s.client.PutObject(context.Background(), s.bucket, objectKey, reader, objectSize, minio.PutObjectOptions{ContentType: contentType})
+func (s *minioServant) PutObject(objectKey string, reader io.Reader, objectSize int64, contentType string, persistance bool) (string, error) {
+	opts := minio.PutObjectOptions{ContentType: contentType}
+	if s.allowPersistObject && !persistance {
+		opts.Mode = minio.Governance
+		opts.RetainUntilDate = time.Now().Add(s.retainInDays)
+	}
+	uploadInfo, err := s.client.PutObject(context.Background(), s.bucket, objectKey, reader, objectSize, opts)
 	if err != nil {
 		return "", err
 	}
@@ -43,6 +51,17 @@ func (s *minioServant) PutObject(objectKey string, reader io.Reader, objectSize
 	return s.domain + objectKey, nil
 }
 
+func (s *minioServant) PersistObject(objectKey string) error {
+	if !s.allowPersistObject {
+		return nil
+	}
+	retentionMode := minio.Governance
+	return s.client.PutObjectRetention(context.Background(), s.bucket, objectKey, minio.PutObjectRetentionOptions{
+		Mode:            &retentionMode,
+		RetainUntilDate: &s.retainUntilDate,
+	})
+}
+
 func (s *minioServant) DeleteObject(objectKey string) error {
 	return s.client.RemoveObject(context.Background(), s.bucket, objectKey, minio.RemoveObjectOptions{ForceDelete: true})
 }

internal/dao/storage/storage.go

@@ -5,6 +5,8 @@ import (
 	"net/http"
 	"net/url"
 	"path/filepath"
+	"strconv"
+	"time"
 
 	"github.com/aliyun/aliyun-oss-go-sdk/oss"
 	"github.com/huaweicloud/huaweicloud-sdk-go-obs/obs"
@@ -19,17 +21,20 @@ import (
 func MustAliossService() (core.ObjectStorageService, core.VersionInfo) {
 	client, err := oss.New(conf.AliOSSSetting.Endpoint, conf.AliOSSSetting.AccessKeyID, conf.AliOSSSetting.AccessKeySecret)
 	if err != nil {
-		logrus.Fatalf("alioss.New err: %v", err)
+		logrus.Fatalf("storage.MustAliossService create client err: %s", err)
 	}
 
 	bucket, err := client.Bucket(conf.AliOSSSetting.Bucket)
 	if err != nil {
-		logrus.Fatalf("client.Bucket err: %v", err)
+		logrus.Fatalf("storage.MustAliossService create bucket err: %v", err)
 	}
 
 	obj := &aliossServant{
-		bucket: bucket,
-		domain: conf.GetOssDomain(),
+		bucket:             bucket,
+		domain:             conf.GetOssDomain(),
+		retainInDays:       time.Duration(conf.ObjectStorage.RetainInDays) * time.Hour * 24,
+		retainUntilDate:    time.Date(2049, time.December, 1, 12, 0, 0, 0, time.UTC),
+		allowPersistObject: conf.CfgIf("PersistObject"),
 	}
 	return obj, obj
 }
@@ -56,12 +61,17 @@ func MustHuaweiobsService() (core.ObjectStorageService, core.VersionInfo) {
 	s := conf.HuaweiOBSSetting
 	client, err := obs.New(s.AccessKey, s.SecretKey, s.Endpoint)
 	if err != nil {
-		logrus.Fatalf("create huawei obs client failed: %s", err)
+		logrus.Fatalf("storage.MustHuaweiobsService create huawei obs client failed: %s", err)
 	}
+
+	retainUntilDays := time.Until(time.Date(2049, time.December, 1, 12, 0, 0, 0, time.UTC)) / (24 * time.Hour)
 	obj := &huaweiobsServant{
-		client: client,
-		bucket: s.Bucket,
-		domain: conf.GetOssDomain(),
+		client:             client,
+		bucket:             s.Bucket,
+		domain:             conf.GetOssDomain(),
+		retainInDays:       int64(conf.ObjectStorage.RetainInDays),
+		retainUntilDays:    strconv.FormatInt(int64(retainUntilDays), 10),
+		allowPersistObject: conf.CfgIf("PersistObject"),
 	}
 	return obj, obj
 }
@@ -69,7 +79,7 @@ func MustHuaweiobsService() (core.ObjectStorageService, core.VersionInfo) {
 func MustLocalossService() (core.ObjectStorageService, core.VersionInfo) {
 	savePath, err := filepath.Abs(conf.LocalOSSSetting.SavePath)
 	if err != nil {
-		logrus.Fatalf("get localOSS save path err: %v", err)
+		logrus.Fatalf("storage.MustLocalossService get localOSS save path err: %v", err)
 	}
 
 	obj := &localossServant{
@@ -86,14 +96,18 @@ func MustMinioService() (core.ObjectStorageService, core.VersionInfo) {
 		Secure: conf.MinIOSetting.Secure,
 	})
 	if err != nil {
-		logrus.Fatalf("minio.New err: %v", err)
+		logrus.Fatalf("storage.MustMinioService create client failed: %s", err)
 	}
 
 	ms := &minioServant{
-		client: client,
-		bucket: conf.MinIOSetting.Bucket,
-		domain: conf.GetOssDomain(),
+		client:             client,
+		bucket:             conf.MinIOSetting.Bucket,
+		domain:             conf.GetOssDomain(),
+		retainInDays:       time.Duration(conf.ObjectStorage.RetainInDays) * time.Hour * 24,
+		retainUntilDate:    time.Date(2049, time.December, 1, 12, 0, 0, 0, time.UTC),
+		allowPersistObject: conf.CfgIf("PersistObject"),
 	}
+
 	return ms, ms
 }
 
@@ -104,13 +118,17 @@ func MustS3Service() (core.ObjectStorageService, core.VersionInfo) {
 		Secure: conf.S3Setting.Secure,
 	})
 	if err != nil {
-		logrus.Fatalf("s3.New err: %v", err)
+		logrus.Fatalf("storage.MustS3Service create client failed: %s", err)
 	}
 
-	obj := &s3Servant{
-		client: client,
-		bucket: conf.MinIOSetting.Bucket,
-		domain: conf.GetOssDomain(),
+	s3 := &s3Servant{
+		client:             client,
+		bucket:             conf.MinIOSetting.Bucket,
+		domain:             conf.GetOssDomain(),
+		retainInDays:       time.Duration(conf.ObjectStorage.RetainInDays) * time.Hour * 24,
+		retainUntilDate:    time.Date(2049, time.December, 1, 12, 0, 0, 0, time.UTC),
+		allowPersistObject: conf.CfgIf("PersistObject"),
 	}
-	return obj, obj
+
+	return s3, s3
 }

internal/routers/api/attachment.go

@@ -102,7 +102,7 @@ func UploadAttachment(c *gin.Context) {
 	randomPath := uuid.Must(uuid.NewV4()).String()
 	ossSavePath := uploadType + "/" + GeneratePath(randomPath[:8]) + "/" + randomPath[9:] + fileExt
 
-	objectUrl, err := objectStorage.PutObject(ossSavePath, file, fileHeader.Size, contentType)
+	objectUrl, err := objectStorage.PutObject(ossSavePath, file, fileHeader.Size, contentType, false)
 	if err != nil {
 		logrus.Errorf("putObject err: %v", err)
 		response.ToErrorResponse(errcode.FileUploadFailed)

internal/service/comment.go

@@ -89,7 +89,19 @@ func GetPostComments(postID int64, sort string, offset, limit int) ([]*model.Com
 	return commentsFormated, totalRows, nil
 }
 
-func CreatePostComment(ctx *gin.Context, userID int64, param CommentCreationReq) (*model.Comment, error) {
+func CreatePostComment(ctx *gin.Context, userID int64, param CommentCreationReq) (comment *model.Comment, err error) {
+	var mediaContents []string
+
+	defer func() {
+		if err != nil {
+			deleteOssObjects(mediaContents)
+		}
+	}()
+
+	if mediaContents, err = persistMediaContents(param.Contents); err != nil {
+		return
+	}
+
 	// 加载Post
 	post, err := ds.GetPostByID(param.PostID)
 
@@ -101,7 +113,7 @@ func CreatePostComment(ctx *gin.Context, userID int64, param CommentCreationReq)
 		return nil, errcode.MaxCommentCount
 	}
 	ip := ctx.ClientIP()
-	comment := &model.Comment{
+	comment = &model.Comment{
 		PostID: post.ID,
 		UserID: userID,
 		IP:     ip,

internal/service/post.go

@@ -104,24 +104,18 @@ func tagsFrom(originTags []string) []string {
 // CreatePost 创建文章
 // TODO: 推文+推文内容需要在一个事务中添加，后续优化
 func CreatePost(c *gin.Context, userID int64, param PostCreationReq) (post *model.Post, err error) {
-	// 获取媒体内容
-	mediaContents := make([]string, 0, len(param.Contents))
-	for _, item := range param.Contents {
-		switch item.Type {
-		case model.CONTENT_TYPE_IMAGE,
-			model.CONTENT_TYPE_VIDEO,
-			model.CONTENT_TYPE_AUDIO,
-			model.CONTENT_TYPE_ATTACHMENT,
-			model.CONTENT_TYPE_CHARGE_ATTACHMENT:
-			mediaContents = append(mediaContents, item.Content)
-		}
-	}
+	var mediaContents []string
+
 	defer func() {
 		if err != nil {
 			deleteOssObjects(mediaContents)
 		}
 	}()
 
+	if mediaContents, err = persistMediaContents(param.Contents); err != nil {
+		return
+	}
+
 	ip := c.ClientIP()
 	tags := tagsFrom(param.Tags)
 	post = &model.Post{
@@ -168,9 +162,7 @@ func CreatePost(c *gin.Context, userID int64, param PostCreationReq) (post *mode
 				UserID: userID,
 				Tag:    t,
 			}
-			if _, err := ds.CreateTag(tag); err != nil {
-				return nil, err
-			}
+			ds.CreateTag(tag)
 		}
 
 		// 创建用户消息提醒

internal/service/service.go

@@ -4,6 +4,8 @@ import (
 	"github.com/rocboss/paopao-ce/internal/conf"
 	"github.com/rocboss/paopao-ce/internal/core"
 	"github.com/rocboss/paopao-ce/internal/dao"
+	"github.com/rocboss/paopao-ce/internal/model"
+	"github.com/sirupsen/logrus"
 )
 
 var (
@@ -19,3 +21,26 @@ func Initialize() {
 	oss = dao.ObjectStorageService()
 	DisablePhoneVerify = !conf.CfgIf("Sms")
 }
+
+// persistMediaContents 获取媒体内容并持久化
+func persistMediaContents(contents []*PostContentItem) (items []string, err error) {
+	items = make([]string, 0, len(contents))
+	for _, item := range contents {
+		switch item.Type {
+		case model.CONTENT_TYPE_IMAGE,
+			model.CONTENT_TYPE_VIDEO,
+			model.CONTENT_TYPE_AUDIO,
+			model.CONTENT_TYPE_ATTACHMENT,
+			model.CONTENT_TYPE_CHARGE_ATTACHMENT:
+			items = append(items, item.Content)
+			if err != nil {
+				continue
+			}
+			if err = oss.PersistObject(oss.ObjectKey(item.Content)); err != nil {
+				logrus.Errorf("service.persistMediaContents failed: %s", err)
+				continue
+			}
+		}
+	}
+	return
+}

internal/service/user.go

@@ -14,6 +14,7 @@ import (
 	"github.com/rocboss/paopao-ce/pkg/convert"
 	"github.com/rocboss/paopao-ce/pkg/errcode"
 	"github.com/rocboss/paopao-ce/pkg/util"
+	"github.com/sirupsen/logrus"
 )
 
 const MAX_CAPTCHA_TIMES = 2
@@ -267,10 +268,22 @@ func UpdateUserInfo(user *model.User) *errcode.Error {
 	return nil
 }
 
-func ChangeUserAvatar(user *model.User, avatar string) *errcode.Error {
+func ChangeUserAvatar(user *model.User, avatar string) (err *errcode.Error) {
+	defer func() {
+		if err != nil {
+			deleteOssObjects([]string{avatar})
+		}
+	}()
+
 	if err := ds.CheckAttachment(avatar); err != nil {
 		return errcode.InvalidParams
 	}
+
+	if err := oss.PersistObject(oss.ObjectKey(avatar)); err != nil {
+		logrus.Errorf("service.ChangeUserAvatar persist object failed: %s", err)
+		return errcode.ServerError
+	}
+
 	user.Avatar = avatar
 	return UpdateUserInfo(user)
 }