diff --git a/README.md b/README.md index 21522de1..59aad303 100644 --- a/README.md +++ b/README.md @@ -327,9 +327,10 @@ release/paopao-ce --no-default-features --features sqlite3,localoss,loggerfile,r `Alipay` 开启基于[支付宝开放平台](https://open.alipay.com/)的钱包功能； * 短信验证码: SmsJuhe(需要开启sms) `Sms` 开启短信验证码功能，用于手机绑定验证手机是否注册者的；功能如果没有开启，手机绑定时任意短信验证码都可以绑定手机； -* 其他: PhoneBind/PersistObjct - `PhoneBind` 开启手机绑定功能； - `PersistObject` 开启对象存储的持久化对象功能，允许先创建临时对象然后再持久化； +* 其他: PhoneBind/OSS:Retention/OSS:TempDir + `PhoneBind` 手机绑定功能； + `OSS:Retention` 基于对象存储系统的对象过期自动删除特性实现 先创建临时对象再持久化的功能(目前状态: 内测阶段)； + `OSS:TempDir` 基于对象存储系统的对象拷贝/移动特性实现 先创建临时对象再持久化的功能(目前状态: 内测阶段)； ### 搭建依赖环境 #### [Zinc](https://github.com/zinclabs/zinc) 搜索引擎: diff --git a/config.yaml.sample b/config.yaml.sample index 9b8e1adb..f381149d 100644 --- a/config.yaml.sample +++ b/config.yaml.sample @@ -12,9 +12,9 @@ Server: # 服务设置 WriteTimeout: 60 Features: Default: ["Base", "MySQL", "Option", "Zinc", "LocalOSS", "LoggerFile"] - Develop: ["Base", "MySQL", "BigCacheIndex", "Meili", "Sms", "AliOSS", "LoggerMeili", "Migration"] - Demo: ["Base", "MySQL", "Option", "Zinc", "Sms", "MinIO", "LoggerZinc", "PersistObject"] - Slim: ["Base", "Sqlite3", "LocalOSS", "LoggerFile"] + Develop: ["Base", "MySQL", "BigCacheIndex", "Meili", "Sms", "AliOSS", "LoggerMeili", "OSS:Retention"] + Demo: ["Base", "MySQL", "Option", "Zinc", "Sms", "MinIO", "LoggerZinc", "Migration"] + Slim: ["Base", "Sqlite3", "LocalOSS", "LoggerFile", "OSS:TempDir"] Base: ["Redis", "Alipay", "PhoneBind"] Option: ["SimpleCacheIndex"] Sms: "SmsJuhe" diff --git a/internal/conf/settting.go b/internal/conf/settting.go index e5d0d00e..e644fe89 100644 --- a/internal/conf/settting.go +++ b/internal/conf/settting.go @@ -136,6 +136,7 @@ type Sqlite3SettingS struct { type ObjectStorageS struct { RetainInDays int + TempDir string } type MinIOSettingS struct { @@ -407,6 +408,10 @@ func (s *LoggerMeiliSettingS) maxLogBuffer() int { return s.MaxLogBuffer } +func (s *ObjectStorageS) TempDirSlash() string { + return strings.Trim(s.TempDir, " /") + "/" +} + func (s *ZincSettingS) Endpoint() string { return endpoint(s.Host, s.Secure) } diff --git a/internal/core/storage.go b/internal/core/storage.go index dd74778a..2a241bca 100644 --- a/internal/core/storage.go +++ b/internal/core/storage.go @@ -6,12 +6,23 @@ import ( // ObjectStorageService storage service interface that implement base AliOSS、MINIO or other type ObjectStorageService interface { + OssCreateService + OssDeleteService + + SignURL(objectKey string, expiredInSec int64) (string, error) + ObjectURL(objetKey string) string + ObjectKey(cUrl string) string +} + +// OssCreateService Object Storage System Object Create service +type OssCreateService interface { PutObject(objectKey string, reader io.Reader, objectSize int64, contentType string, persistance bool) (string, error) PersistObject(objectKey string) error +} + +// OssCreateService Object Storage System Object Delete service +type OssDeleteService interface { DeleteObject(objectKey string) error DeleteObjects(objectKeys []string) error IsObjectExist(objectKey string) (bool, error) - SignURL(objectKey string, expiredInSec int64) (string, error) - ObjectURL(objetKey string) string - ObjectKey(cUrl string) string } diff --git a/internal/dao/storage/alioss.go b/internal/dao/storage/alioss.go index 0a560095..8baf3ed3 100644 --- a/internal/dao/storage/alioss.go +++ b/internal/dao/storage/alioss.go @@ -14,31 +14,60 @@ import ( var ( _ core.ObjectStorageService = (*aliossServant)(nil) + _ core.OssCreateService = (*aliossCreateServant)(nil) + _ core.OssCreateService = (*aliossCreateRetentionServant)(nil) + _ core.OssCreateService = (*aliossCreateTempDirServant)(nil) _ core.VersionInfo = (*aliossServant)(nil) ) +type aliossCreateServant struct { + bucket *oss.Bucket + domain string +} + +type aliossCreateRetentionServant struct { + bucket *oss.Bucket + domain string + retainInDays time.Duration + retainUntilDate time.Time +} + +type aliossCreateTempDirServant struct { + bucket *oss.Bucket + domain string + tempDir string +} + type aliossServant struct { - bucket *oss.Bucket - domain string - retainInDays time.Duration - retainUntilDate time.Time - allowPersistObject bool + core.OssCreateService + + bucket *oss.Bucket + domain string } -func (s *aliossServant) Name() string { - return "AliOSS" +func (s *aliossCreateServant) PutObject(objectKey string, reader io.Reader, objectSize int64, contentType string, _persistance bool) (string, error) { + options := []oss.Option{ + oss.ContentLength(objectSize), + oss.ContentType(contentType), + } + err := s.bucket.PutObject(objectKey, reader, options...) + if err != nil { + return "", err + } + return s.domain + objectKey, nil } -func (s *aliossServant) Version() *semver.Version { - return semver.MustParse("v0.2.0") +func (s *aliossCreateServant) PersistObject(_objectKey string) error { + // empty + return nil } -func (s *aliossServant) PutObject(objectKey string, reader io.Reader, objectSize int64, contentType string, persistance bool) (string, error) { +func (s *aliossCreateRetentionServant) PutObject(objectKey string, reader io.Reader, objectSize int64, contentType string, persistance bool) (string, error) { options := []oss.Option{ oss.ContentLength(objectSize), oss.ContentType(contentType), } - if s.allowPersistObject && !persistance { + if !persistance { options = append(options, oss.Expires(time.Now().Add(s.retainInDays))) } err := s.bucket.PutObject(objectKey, reader, options...) @@ -48,11 +77,37 @@ func (s *aliossServant) PutObject(objectKey string, reader io.Reader, objectSize return s.domain + objectKey, nil } -func (s *aliossServant) PersistObject(objectKey string) error { - if !s.allowPersistObject { +func (s *aliossCreateRetentionServant) PersistObject(objectKey string) error { + return s.bucket.SetObjectMeta(objectKey, oss.Expires(s.retainUntilDate)) +} + +func (s *aliossCreateTempDirServant) PutObject(objectKey string, reader io.Reader, objectSize int64, contentType string, persistance bool) (string, error) { + if !persistance { + objectKey = s.tempDir + objectKey + } + options := []oss.Option{ + oss.ContentLength(objectSize), + oss.ContentType(contentType), + } + err := s.bucket.PutObject(objectKey, reader, options...) + if err != nil { + return "", err + } + return s.domain + objectKey, nil +} + +func (s *aliossCreateTempDirServant) PersistObject(objectKey string) error { + exsit, err := s.bucket.IsObjectExist(objectKey) + if err != nil { + return err + } + if exsit { return nil } - return s.bucket.SetObjectMeta(objectKey, oss.Expires(s.retainUntilDate)) + if _, err := s.bucket.CopyObject(s.tempDir+objectKey, objectKey); err != nil { + return err + } + return s.bucket.DeleteObject(s.tempDir + objectKey) } func (s *aliossServant) DeleteObject(objectKey string) error { @@ -98,3 +153,11 @@ func (s *aliossServant) ObjectURL(objetKey string) string { func (s *aliossServant) ObjectKey(objectUrl string) string { return strings.Replace(objectUrl, s.domain, "", -1) } + +func (s *aliossServant) Name() string { + return "AliOSS" +} + +func (s *aliossServant) Version() *semver.Version { + return semver.MustParse("v0.2.0") +} diff --git a/internal/dao/storage/cos.go b/internal/dao/storage/cos.go index aaf63fd4..8023e3a6 100644 --- a/internal/dao/storage/cos.go +++ b/internal/dao/storage/cos.go @@ -17,26 +17,35 @@ import ( var ( _ core.ObjectStorageService = (*cosServant)(nil) + _ core.OssCreateService = (*cosCreateServant)(nil) + _ core.OssCreateService = (*cosCreateTempDirServant)(nil) _ core.VersionInfo = (*cosServant)(nil) ) -type cosServant struct { +type cosCreateServant struct { client *cos.Client domain string } -func (s *cosServant) Name() string { - return "COS" +type cosCreateTempDirServant struct { + client *cos.Client + domain string + bucketUrl string + tempDir string } -func (s *cosServant) Version() *semver.Version { - return semver.MustParse("v0.2.0") +type cosServant struct { + core.OssCreateService + + client *cos.Client + domain string } -func (s *cosServant) PutObject(objectKey string, reader io.Reader, objectSize int64, contentType string, _persistance bool) (string, error) { +func (s *cosCreateServant) PutObject(objectKey string, reader io.Reader, objectSize int64, contentType string, _persistance bool) (string, error) { _, err := s.client.Object.Put(context.Background(), objectKey, reader, &cos.ObjectPutOptions{ ObjectPutHeaderOptions: &cos.ObjectPutHeaderOptions{ - ContentType: contentType, + ContentType: contentType, + ContentLength: objectSize, }, }) if err != nil { @@ -45,11 +54,43 @@ func (s *cosServant) PutObject(objectKey string, reader io.Reader, objectSize in return s.domain + objectKey, nil } -func (s *cosServant) PersistObject(objectKey string) error { +func (s *cosCreateServant) PersistObject(_objectKey string) error { // empty return nil } +func (s *cosCreateTempDirServant) PutObject(objectKey string, reader io.Reader, objectSize int64, contentType string, persistance bool) (string, error) { + if !persistance { + objectKey = s.tempDir + objectKey + } + _, err := s.client.Object.Put(context.Background(), objectKey, reader, &cos.ObjectPutOptions{ + ObjectPutHeaderOptions: &cos.ObjectPutHeaderOptions{ + ContentType: contentType, + ContentLength: objectSize, + }, + }) + if err != nil { + return "", err + } + return s.domain + objectKey, nil +} + +func (s *cosCreateTempDirServant) PersistObject(objectKey string) error { + exsit, err := s.client.Object.IsExist(context.Background(), objectKey) + if err != nil { + return err + } + if exsit { + return nil + } + _, _, err = s.client.Object.Copy(context.Background(), objectKey, s.bucketUrl+s.tempDir+objectKey, nil) + if err != nil { + return err + } + _, err = s.client.Object.Delete(context.Background(), s.tempDir+objectKey, nil) + return err +} + func (s *cosServant) DeleteObject(objectKey string) error { _, err := s.client.Object.Delete(context.Background(), objectKey) return err @@ -96,3 +137,11 @@ func (s *cosServant) ObjectURL(objetKey string) string { func (s *cosServant) ObjectKey(objectUrl string) string { return strings.Replace(objectUrl, s.domain, "", -1) } + +func (s *cosServant) Name() string { + return "COS" +} + +func (s *cosServant) Version() *semver.Version { + return semver.MustParse("v0.2.0") +} diff --git a/internal/dao/storage/huaweiobs.go b/internal/dao/storage/huaweiobs.go index 913e7108..e8302f2d 100644 --- a/internal/dao/storage/huaweiobs.go +++ b/internal/dao/storage/huaweiobs.go @@ -13,31 +13,62 @@ import ( var ( _ core.ObjectStorageService = (*huaweiobsServant)(nil) + _ core.OssCreateService = (*hwobsCreateServant)(nil) + _ core.OssCreateService = (*hwobsCreateRetentionServant)(nil) + _ core.OssCreateService = (*hwobsCreateTempDirServant)(nil) _ core.VersionInfo = (*huaweiobsServant)(nil) ) +type hwobsCreateServant struct { + client *obs.ObsClient + bucket string + domain string +} + +type hwobsCreateRetentionServant struct { + client *obs.ObsClient + bucket string + domain string + retainInDays int64 + retainUntilDays string +} + +type hwobsCreateTempDirServant struct { + client *obs.ObsClient + bucket string + domain string + tempDir string +} + type huaweiobsServant struct { - client *obs.ObsClient - bucket string - domain string - retainInDays int64 - retainUntilDays string - allowPersistObject bool + core.OssCreateService + + client *obs.ObsClient + bucket string + domain string } -func (s *huaweiobsServant) Name() string { - return "HuaweiOBS" +func (s *hwobsCreateServant) PutObject(objectKey string, reader io.Reader, objectSize int64, contentType string, _persistance bool) (string, error) { + input := &obs.PutObjectInput{} + input.Bucket, input.Key, input.Body = s.bucket, objectKey, reader + input.ContentType, input.ContentLength = contentType, objectSize + _, err := s.client.PutObject(input) + if err != nil { + return "", err + } + return s.domain + objectKey, nil } -func (s *huaweiobsServant) Version() *semver.Version { - return semver.MustParse("v0.2.0") +func (s *hwobsCreateServant) PersistObject(_objectKey string) error { + // empty + return nil } -func (s *huaweiobsServant) PutObject(objectKey string, reader io.Reader, objectSize int64, contentType string, persistance bool) (string, error) { +func (s *hwobsCreateRetentionServant) PutObject(objectKey string, reader io.Reader, objectSize int64, contentType string, persistance bool) (string, error) { input := &obs.PutObjectInput{} input.Bucket, input.Key, input.Body = s.bucket, objectKey, reader input.ContentType, input.ContentLength = contentType, objectSize - if s.allowPersistObject && !persistance { + if !persistance { input.Expires = s.retainInDays } _, err := s.client.PutObject(input) @@ -47,16 +78,56 @@ func (s *huaweiobsServant) PutObject(objectKey string, reader io.Reader, objectS return s.domain + objectKey, nil } -func (s *huaweiobsServant) PersistObject(objectKey string) error { - if !s.allowPersistObject { - return nil - } +func (s *hwobsCreateRetentionServant) PersistObject(objectKey string) error { _, err := s.client.SetObjectMetadata(&obs.SetObjectMetadataInput{ Expires: s.retainUntilDays, }) return err } +func (s *hwobsCreateTempDirServant) PutObject(objectKey string, reader io.Reader, objectSize int64, contentType string, persistance bool) (string, error) { + if !persistance { + objectKey = s.tempDir + objectKey + } + input := &obs.PutObjectInput{} + input.Bucket, input.Key, input.Body = s.bucket, objectKey, reader + input.ContentType, input.ContentLength = contentType, objectSize + _, err := s.client.PutObject(input) + if err != nil { + return "", err + } + return s.domain + objectKey, nil +} + +func (s *hwobsCreateTempDirServant) PersistObject(objectKey string) error { + // is object exist that key equel objectKey + input := &obs.GetObjectMetadataInput{ + Bucket: s.bucket, + Key: objectKey, + } + _, err := s.client.GetObjectMetadata(input) + if err == nil { + // do nothing if object exist + return nil + } + + tmpObjKey := s.tempDir + objectKey + copyInput := &obs.CopyObjectInput{ + CopySourceBucket: s.bucket, + CopySourceKey: tmpObjKey, + } + copyInput.Bucket, copyInput.Key = s.bucket, objectKey + if _, err = s.client.CopyObject(copyInput); err != nil { + return err + } + + _, err = s.client.DeleteObject(&obs.DeleteObjectInput{ + Bucket: s.bucket, + Key: tmpObjKey, + }) + return err +} + func (s *huaweiobsServant) DeleteObject(objectKey string) error { _, err := s.client.DeleteObject(&obs.DeleteObjectInput{ Bucket: s.bucket, @@ -126,3 +197,11 @@ func (s *huaweiobsServant) ObjectURL(objetKey string) string { func (s *huaweiobsServant) ObjectKey(objectUrl string) string { return strings.Replace(objectUrl, s.domain, "", -1) } + +func (s *huaweiobsServant) Name() string { + return "HuaweiOBS" +} + +func (s *huaweiobsServant) Version() *semver.Version { + return semver.MustParse("v0.2.0") +} diff --git a/internal/dao/storage/localoss.go b/internal/dao/storage/localoss.go index c2792bb0..31d52a79 100644 --- a/internal/dao/storage/localoss.go +++ b/internal/dao/storage/localoss.go @@ -15,23 +15,30 @@ import ( var ( _ core.ObjectStorageService = (*localossServant)(nil) + _ core.OssCreateService = (*localossCreateServant)(nil) + _ core.OssCreateService = (*localossCreateTempDirServant)(nil) _ core.VersionInfo = (*localossServant)(nil) ) -type localossServant struct { +type localossCreateServant struct { savePath string domain string } -func (s *localossServant) Name() string { - return "LocalOSS" +type localossCreateTempDirServant struct { + savePath string + domain string + tempDir string } -func (s *localossServant) Version() *semver.Version { - return semver.MustParse("v0.2.0") +type localossServant struct { + core.OssCreateService + + savePath string + domain string } -func (s *localossServant) PutObject(objectKey string, reader io.Reader, objectSize int64, contentType string, _persistance bool) (string, error) { +func (s *localossCreateServant) PutObject(objectKey string, reader io.Reader, objectSize int64, contentType string, _persistance bool) (string, error) { saveDir := s.savePath + filepath.Dir(objectKey) err := os.MkdirAll(saveDir, 0750) if err != nil && !os.IsExist(err) { @@ -57,11 +64,78 @@ func (s *localossServant) PutObject(objectKey string, reader io.Reader, objectSi return s.domain + objectKey, nil } -func (s *localossServant) PersistObject(objectKey string) error { +func (s *localossCreateServant) PersistObject(_objectKey string) error { // empty return nil } +func (s *localossCreateTempDirServant) PutObject(objectKey string, reader io.Reader, objectSize int64, contentType string, persistance bool) (string, error) { + if !persistance { + objectKey = s.tempDir + objectKey + } + saveDir := s.savePath + filepath.Dir(objectKey) + err := os.MkdirAll(saveDir, 0750) + if err != nil && !os.IsExist(err) { + return "", err + } + + savePath := s.savePath + objectKey + writer, err := os.Create(savePath) + if err != nil { + return "", err + } + defer writer.Close() + + written, err := io.Copy(writer, reader) + if err != nil { + return "", err + } + if written != objectSize { + os.Remove(savePath) + return "", errors.New("put object not complete") + } + + return s.domain + objectKey, nil +} + +func (s *localossCreateTempDirServant) PersistObject(objectKey string) error { + fi, err := os.Stat(s.savePath + objectKey) + if err != nil { + return err + } + if !fi.IsDir() { + // do nothing if object is exsit + return nil + } + + saveDir := s.savePath + filepath.Dir(objectKey) + if err = os.MkdirAll(saveDir, 0750); err != nil && !os.IsExist(err) { + return err + } + + tmpObjPath := s.savePath + s.tempDir + objectKey + reader, err := os.Open(tmpObjPath) + if err != nil { + return err + } + writer, err := os.Create(s.savePath + objectKey) + if err != nil { + return err + } + defer writer.Close() + if _, err = io.Copy(writer, reader); err != nil { + reader.Close() + return err + } + reader.Close() + + if err = os.Remove(tmpObjPath); err != nil { + return err + } + + return nil +} + func (s *localossServant) DeleteObject(objectKey string) error { return os.Remove(s.savePath + objectKey) } @@ -103,3 +177,11 @@ func (s *localossServant) ObjectURL(objetKey string) string { func (s *localossServant) ObjectKey(objectUrl string) string { return strings.Replace(objectUrl, s.domain, "", -1) } + +func (s *localossServant) Name() string { + return "LocalOSS" +} + +func (s *localossServant) Version() *semver.Version { + return semver.MustParse("v0.2.0") +} diff --git a/internal/dao/storage/minio.go b/internal/dao/storage/minio.go index 6140bfdc..dae6d0ad 100644 --- a/internal/dao/storage/minio.go +++ b/internal/dao/storage/minio.go @@ -14,31 +14,60 @@ import ( var ( _ core.ObjectStorageService = (*minioServant)(nil) + _ core.OssCreateService = (*minioCreateServant)(nil) + _ core.OssCreateService = (*minioCreateRetentionServant)(nil) + _ core.OssCreateService = (*minioCreateTempDirServant)(nil) _ core.VersionInfo = (*minioServant)(nil) ) +type minioCreateServant struct { + client *minio.Client + bucket string + domain string +} + +type minioCreateRetentionServant struct { + client *minio.Client + bucket string + domain string + retainInDays time.Duration + retainUntilDate time.Time +} + +type minioCreateTempDirServant struct { + client *minio.Client + bucket string + domain string + tempDir string +} + type minioServant struct { - client *minio.Client - bucket string - domain string - retainInDays time.Duration - retainUntilDate time.Time - allowPersistObject bool + core.OssCreateService + + client *minio.Client + bucket string + domain string } type s3Servant = minioServant -func (s *minioServant) Name() string { - return "MinIO" +func (s *minioCreateServant) PutObject(objectKey string, reader io.Reader, objectSize int64, contentType string, _persistance bool) (string, error) { + opts := minio.PutObjectOptions{ContentType: contentType} + _, err := s.client.PutObject(context.Background(), s.bucket, objectKey, reader, objectSize, opts) + if err != nil { + return "", err + } + return s.domain + objectKey, nil } -func (s *minioServant) Version() *semver.Version { - return semver.MustParse("v0.2.0") +func (s *minioCreateServant) PersistObject(_objectKey string) error { + // empty + return nil } -func (s *minioServant) PutObject(objectKey string, reader io.Reader, objectSize int64, contentType string, persistance bool) (string, error) { +func (s *minioCreateRetentionServant) PutObject(objectKey string, reader io.Reader, objectSize int64, contentType string, persistance bool) (string, error) { opts := minio.PutObjectOptions{ContentType: contentType} - if s.allowPersistObject && !persistance { + if !persistance { opts.Mode = minio.Governance opts.RetainUntilDate = time.Now().Add(s.retainInDays) } @@ -49,10 +78,7 @@ func (s *minioServant) PutObject(objectKey string, reader io.Reader, objectSize return s.domain + objectKey, nil } -func (s *minioServant) PersistObject(objectKey string) error { - if !s.allowPersistObject { - return nil - } +func (s *minioCreateRetentionServant) PersistObject(objectKey string) error { retentionMode := minio.Governance return s.client.PutObjectRetention(context.Background(), s.bucket, objectKey, minio.PutObjectRetentionOptions{ Mode: &retentionMode, @@ -60,6 +86,41 @@ func (s *minioServant) PersistObject(objectKey string) error { }) } +func (s *minioCreateTempDirServant) PutObject(objectKey string, reader io.Reader, objectSize int64, contentType string, persistance bool) (string, error) { + opts := minio.PutObjectOptions{ContentType: contentType} + if !persistance { + objectKey = s.tempDir + objectKey + } + _, err := s.client.PutObject(context.Background(), s.bucket, objectKey, reader, objectSize, opts) + if err != nil { + return "", err + } + return s.domain + objectKey, nil +} + +func (s *minioCreateTempDirServant) PersistObject(objectKey string) error { + _, err := s.client.StatObject(context.Background(), s.bucket, objectKey, minio.StatObjectOptions{}) + if err == nil { + // do nothing if object exist + return nil + } + + tmpObjKey := s.tempDir + objectKey + src := minio.CopySrcOptions{ + Bucket: s.bucket, + Object: tmpObjKey, + } + dst := minio.CopyDestOptions{ + Bucket: s.bucket, + Object: objectKey, + } + if _, err = s.client.CopyObject(context.Background(), dst, src); err != nil { + return err + } + + return s.client.RemoveObject(context.Background(), s.bucket, tmpObjKey, minio.RemoveObjectOptions{ForceDelete: true}) +} + func (s *minioServant) DeleteObject(objectKey string) error { return s.client.RemoveObject(context.Background(), s.bucket, objectKey, minio.RemoveObjectOptions{ForceDelete: true}) } @@ -108,3 +169,11 @@ func (s *minioServant) ObjectURL(objetKey string) string { func (s *minioServant) ObjectKey(objectUrl string) string { return strings.Replace(objectUrl, s.domain, "", -1) } + +func (s *minioServant) Name() string { + return "MinIO" +} + +func (s *minioServant) Version() *semver.Version { + return semver.MustParse("v0.2.0") +} diff --git a/internal/dao/storage/storage.go b/internal/dao/storage/storage.go index a33c5107..63506bab 100644 --- a/internal/dao/storage/storage.go +++ b/internal/dao/storage/storage.go @@ -26,15 +26,38 @@ func MustAliossService() (core.ObjectStorageService, core.VersionInfo) { bucket, err := client.Bucket(conf.AliOSSSetting.Bucket) if err != nil { - logrus.Fatalf("storage.MustAliossService create bucket err: %v", err) + logrus.Fatalf("storage.MustAliossService create bucket err: %s", err) + } + + domain := conf.GetOssDomain() + var cs core.OssCreateService + if conf.CfgIf("OSS:TempDir") { + cs = &aliossCreateTempDirServant{ + bucket: bucket, + domain: domain, + tempDir: conf.ObjectStorage.TempDirSlash(), + } + logrus.Debugln("use OSS:TempDir feature") + } else if conf.CfgIf("OSS:Retention") { + cs = &aliossCreateRetentionServant{ + bucket: bucket, + domain: domain, + retainInDays: time.Duration(conf.ObjectStorage.RetainInDays) * time.Hour * 24, + retainUntilDate: time.Date(2049, time.December, 1, 12, 0, 0, 0, time.UTC), + } + logrus.Debugln("use OSS:Retention feature") + } else { + cs = &aliossCreateServant{ + bucket: bucket, + domain: domain, + } + logrus.Debugln("use OSS:Direct feature") } obj := &aliossServant{ - bucket: bucket, - domain: conf.GetOssDomain(), - retainInDays: time.Duration(conf.ObjectStorage.RetainInDays) * time.Hour * 24, - retainUntilDate: time.Date(2049, time.December, 1, 12, 0, 0, 0, time.UTC), - allowPersistObject: conf.CfgIf("PersistObject"), + OssCreateService: cs, + bucket: bucket, + domain: conf.GetOssDomain(), } return obj, obj } @@ -50,9 +73,28 @@ func NewCosService() (core.ObjectStorageService, core.VersionInfo) { }, }) + domain := conf.GetOssDomain() + var cs core.OssCreateService + if conf.CfgIf("OSS:TempDir") { + cs = &cosCreateTempDirServant{ + client: client, + domain: domain, + bucketUrl: fmt.Sprintf("%s.cos.%s.myqcloud.com/", conf.COSSetting.Bucket, conf.COSSetting.Region), + tempDir: conf.ObjectStorage.TempDirSlash(), + } + logrus.Debugln("use OSS:TempDir feature") + } else { + cs = &cosCreateServant{ + client: client, + domain: domain, + } + logrus.Debugln("use OSS:Direct feature") + } + obj := &cosServant{ - client: client, - domain: conf.GetOssDomain(), + OssCreateService: cs, + client: client, + domain: domain, } return obj, obj } @@ -64,14 +106,40 @@ func MustHuaweiobsService() (core.ObjectStorageService, core.VersionInfo) { logrus.Fatalf("storage.MustHuaweiobsService create huawei obs client failed: %s", err) } - retainUntilDays := time.Until(time.Date(2049, time.December, 1, 12, 0, 0, 0, time.UTC)) / (24 * time.Hour) + domain := conf.GetOssDomain() + var cs core.OssCreateService + if conf.CfgIf("OSS:TempDir") { + cs = &hwobsCreateTempDirServant{ + client: client, + bucket: s.Bucket, + domain: domain, + tempDir: conf.ObjectStorage.TempDirSlash(), + } + logrus.Debugln("use OSS:TempDir feature") + } else if conf.CfgIf("OSS:Retention") { + retainUntilDays := time.Until(time.Date(2049, time.December, 1, 12, 0, 0, 0, time.UTC)) / (24 * time.Hour) + cs = &hwobsCreateRetentionServant{ + client: client, + bucket: s.Bucket, + domain: domain, + retainInDays: int64(conf.ObjectStorage.RetainInDays), + retainUntilDays: strconv.FormatInt(int64(retainUntilDays), 10), + } + logrus.Debugln("use OSS:Retention feature") + } else { + cs = &hwobsCreateServant{ + client: client, + bucket: s.Bucket, + domain: domain, + } + logrus.Debugln("use OSS:Direct feature") + } + obj := &huaweiobsServant{ - client: client, - bucket: s.Bucket, - domain: conf.GetOssDomain(), - retainInDays: int64(conf.ObjectStorage.RetainInDays), - retainUntilDays: strconv.FormatInt(int64(retainUntilDays), 10), - allowPersistObject: conf.CfgIf("PersistObject"), + OssCreateService: cs, + client: client, + bucket: s.Bucket, + domain: domain, } return obj, obj } @@ -79,12 +147,31 @@ func MustHuaweiobsService() (core.ObjectStorageService, core.VersionInfo) { func MustLocalossService() (core.ObjectStorageService, core.VersionInfo) { savePath, err := filepath.Abs(conf.LocalOSSSetting.SavePath) if err != nil { - logrus.Fatalf("storage.MustLocalossService get localOSS save path err: %v", err) + logrus.Fatalf("storage.MustLocalossService get localOSS save path err: %s", err) + } + + domain := conf.GetOssDomain() + savePath = savePath + "/" + conf.LocalOSSSetting.Bucket + "/" + var cs core.OssCreateService + if conf.CfgIf("OSS:TempDir") { + cs = &localossCreateTempDirServant{ + savePath: savePath, + domain: domain, + tempDir: conf.ObjectStorage.TempDirSlash(), + } + logrus.Debugln("use OSS:TempDir feature") + } else { + cs = &localossCreateServant{ + savePath: savePath, + domain: domain, + } + logrus.Debugln("use OSS:Direct feature") } obj := &localossServant{ - savePath: savePath + "/" + conf.LocalOSSSetting.Bucket + "/", - domain: conf.GetOssDomain(), + OssCreateService: cs, + savePath: savePath, + domain: domain, } return obj, obj } @@ -99,16 +186,41 @@ func MustMinioService() (core.ObjectStorageService, core.VersionInfo) { logrus.Fatalf("storage.MustMinioService create client failed: %s", err) } - ms := &minioServant{ - client: client, - bucket: conf.MinIOSetting.Bucket, - domain: conf.GetOssDomain(), - retainInDays: time.Duration(conf.ObjectStorage.RetainInDays) * time.Hour * 24, - retainUntilDate: time.Date(2049, time.December, 1, 12, 0, 0, 0, time.UTC), - allowPersistObject: conf.CfgIf("PersistObject"), + domain := conf.GetOssDomain() + var cs core.OssCreateService + if conf.CfgIf("OSS:TempDir") { + cs = &minioCreateTempDirServant{ + client: client, + bucket: conf.MinIOSetting.Bucket, + domain: domain, + tempDir: conf.ObjectStorage.TempDirSlash(), + } + logrus.Debugln("use OSS:TempDir feature") + } else if conf.CfgIf("OSS:Retention") { + cs = &minioCreateRetentionServant{ + client: client, + bucket: conf.MinIOSetting.Bucket, + domain: domain, + retainInDays: time.Duration(conf.ObjectStorage.RetainInDays) * time.Hour * 24, + retainUntilDate: time.Date(2049, time.December, 1, 12, 0, 0, 0, time.UTC), + } + logrus.Debugln("use OSS:Retention feature") + } else { + cs = &minioCreateServant{ + client: client, + bucket: conf.MinIOSetting.Bucket, + domain: domain, + } + logrus.Debugln("use OSS:Direct feature") } - return ms, ms + obj := &minioServant{ + OssCreateService: cs, + client: client, + bucket: conf.MinIOSetting.Bucket, + domain: domain, + } + return obj, obj } func MustS3Service() (core.ObjectStorageService, core.VersionInfo) { @@ -121,14 +233,39 @@ func MustS3Service() (core.ObjectStorageService, core.VersionInfo) { logrus.Fatalf("storage.MustS3Service create client failed: %s", err) } - s3 := &s3Servant{ - client: client, - bucket: conf.MinIOSetting.Bucket, - domain: conf.GetOssDomain(), - retainInDays: time.Duration(conf.ObjectStorage.RetainInDays) * time.Hour * 24, - retainUntilDate: time.Date(2049, time.December, 1, 12, 0, 0, 0, time.UTC), - allowPersistObject: conf.CfgIf("PersistObject"), + domain := conf.GetOssDomain() + var cs core.OssCreateService + if conf.CfgIf("OSS:TempDir") { + cs = &minioCreateTempDirServant{ + client: client, + bucket: conf.MinIOSetting.Bucket, + domain: domain, + tempDir: conf.ObjectStorage.TempDirSlash(), + } + logrus.Debugln("use OSS:TempDir feature") + } else if conf.CfgIf("OSS:Retention") { + cs = &minioCreateRetentionServant{ + client: client, + bucket: conf.MinIOSetting.Bucket, + domain: domain, + retainInDays: time.Duration(conf.ObjectStorage.RetainInDays) * time.Hour * 24, + retainUntilDate: time.Date(2049, time.December, 1, 12, 0, 0, 0, time.UTC), + } + logrus.Debugln("use OSS:Retention feature") + } else { + cs = &minioCreateServant{ + client: client, + bucket: conf.MinIOSetting.Bucket, + domain: domain, + } + logrus.Debugln("use OSS:Direct feature") } - return s3, s3 + obj := &s3Servant{ + OssCreateService: cs, + client: client, + bucket: conf.MinIOSetting.Bucket, + domain: domain, + } + return obj, obj }