From 4232099036e9bb787942d515bb5b53d9082e0caa Mon Sep 17 00:00:00 2001 From: Michael Li Date: Sat, 23 Jul 2022 17:19:35 +0800 Subject: [PATCH] support Huawei Cloud OBS as OSS service --- Makefile | 5 +- README.md | 7 +- config.yaml.sample | 6 ++ go.mod | 3 +- go.sum | 3 +- internal/conf/conf.go | 4 ++ internal/conf/settting.go | 10 +++ internal/dao/dao.go | 14 ++-- internal/dao/storage/huaweiobs.go | 112 ++++++++++++++++++++++++++++++ internal/dao/storage/storage.go | 25 +++++-- 10 files changed, 172 insertions(+), 17 deletions(-) create mode 100644 internal/dao/storage/huaweiobs.go diff --git a/Makefile b/Makefile index 148d1377..4d0b94a2 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: all build run test clean fmt help +.PHONY: all build run test clean fmt pre-commit help TARGET = paopao-ce ifeq ($(OS),Windows_NT) @@ -82,6 +82,9 @@ fmt: test: @go test ./... +pre-commit: fmt + go mod tidy + help: @echo "make: make" @echo "make run: start api server" diff --git a/README.md b/README.md index 43c48c01..5d0eb35c 100644 --- a/README.md +++ b/README.md @@ -306,9 +306,10 @@ release/paopao-ce --no-default-features --features sqlite3,localoss,loggerfile,r * 数据库: MySQL/Sqlite3/PostgreSQL `Gorm` + `MySQL`/`Sqlite3`/`PostgreSQL` 使用[gorm](https://github.com/go-gorm/gorm)作为数据库的ORM，默认使用 `Grom` + `MySQL`组合(目前状态：稳定，默认，推荐使用)； `Sqlx` + `MySQL`/`PostgreSQL` 使用[sqlx](https://github.com/jmoiron/sqlx)作为数据库的ORM(目前状态：WIP)； -* 对象存储: AliOSS/COS/MinIO/LocalOSS - `AliOSS` 阿里云对象存储服务； - `COS` 腾讯云对象存储服务 +* 对象存储: AliOSS/COS/HuaweiOBS/MinIO/LocalOSS + `AliOSS` 阿里云对象存储服务； + `COS` 腾讯云对象存储服务； + `HuaweiOBS` 华为云对象存储服务； `MinIO` [MinIO](https://github.com/minio/minio)对象存储服务； `LocalOSS` 提供使用本地目录文件作为对象存储的功能，仅用于开发调试环境； * 缓存: Redis/SimpleCacheIndex/BigCacheIndex diff --git a/config.yaml.sample b/config.yaml.sample index e41acbb0..17c72053 100644 --- a/config.yaml.sample +++ b/config.yaml.sample @@ -85,6 +85,12 @@ COS: # 腾讯云COS存储配置 Region: ap-shanghai Bucket: demo-1888888888 Domain: +HuaweiOBS: # 华为云OBS存储配置 + AccessKey: + SecretKey: + Endpoint: + Bucket: paopao + Domain: MinIO: # MinIO 存储配置 AccessKey: Q3AM3UQ867SPQQA43P2F SecretKey: zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG diff --git a/go.mod b/go.mod index f90895f8..faacfb23 100644 --- a/go.mod +++ b/go.mod @@ -22,6 +22,7 @@ require ( github.com/gofrs/uuid v4.0.0+incompatible github.com/golang-migrate/migrate/v4 v4.15.2 github.com/google/go-cmp v0.5.7 // indirect + github.com/huaweicloud/huaweicloud-sdk-go-obs v3.21.12+incompatible github.com/json-iterator/go v1.1.12 github.com/meilisearch/meilisearch-go v0.19.1 github.com/minio/minio-go/v7 v7.0.27 @@ -30,7 +31,7 @@ require ( github.com/sirupsen/logrus v1.8.1 github.com/smartwalle/alipay/v3 v3.1.7 github.com/spf13/viper v1.10.1 - github.com/tencentyun/cos-go-sdk-v5 v0.7.35 // indirect + github.com/tencentyun/cos-go-sdk-v5 v0.7.35 github.com/ugorji/go v1.2.7 // indirect github.com/yinheli/mahonia v0.0.0-20131226213531-0eef680515cc golang.org/x/crypto v0.0.0-20220307211146-efcb8507fb70 // indirect diff --git a/go.sum b/go.sum index 1cd5a3e5..abd6b0f1 100644 --- a/go.sum +++ b/go.sum @@ -823,6 +823,8 @@ github.com/hashicorp/serf v0.9.6/go.mod h1:TXZNMjZQijwlDvp+r0b63xZ45H7JmCmgg4gpT github.com/holiman/bloomfilter/v2 v2.0.3/go.mod h1:zpoh+gs7qcpqrHr3dB55AMiJwo0iURXE7ZOP9L9hSkA= github.com/holiman/uint256 v1.2.0/go.mod h1:y4ga/t+u+Xwd7CpDgZESaRcWy0I7XMlTMA25ApIH5Jw= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/huaweicloud/huaweicloud-sdk-go-obs v3.21.12+incompatible h1:tANYIteuFrosKbRYUk1Yo/OGJjbt4x3OVg211Qc60M0= +github.com/huaweicloud/huaweicloud-sdk-go-obs v3.21.12+incompatible/go.mod h1:l7VUhRbTKCzdOacdT4oWCwATKyvZqUOlOqr0Ous3k4s= github.com/huin/goupnp v1.0.2/go.mod h1:0dxJBVBHqTMjIUMkESDTNgOOx/Mw5wYIfyFmdzSamkM= github.com/huin/goutil v0.0.0-20170803182201-1ca381bf3150/go.mod h1:PpLOETDnJ0o3iZrZfqZzyLl6l7F3c6L1oWn7OICBi6o= github.com/iancoleman/strcase v0.2.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho= @@ -1104,7 +1106,6 @@ github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:F github.com/mitchellh/mapstructure v0.0.0-20180220230111-00c29f56e238/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= -github.com/mitchellh/mapstructure v1.4.3 h1:OVowDSCllw/YjdLkam3/sm7wEtOy59d8ndGgCcyj8cs= github.com/mitchellh/mapstructure v1.4.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= diff --git a/internal/conf/conf.go b/internal/conf/conf.go index 270fe0c1..4545e996 100644 --- a/internal/conf/conf.go +++ b/internal/conf/conf.go @@ -30,6 +30,7 @@ var ( MeiliSetting *MeiliSettingS AliOSSSetting *AliOSSSettingS COSSetting *COSSettingS + HuaweiOBSSetting *HuaweiOBSSettingS MinIOSetting *MinIOSettingS S3Setting *S3SettingS LocalOSSSetting *LocalOSSSettingS @@ -73,6 +74,7 @@ func setupSetting(suite []string, noDefault bool) error { "JWT": &JWTSetting, "AliOSS": &AliOSSSetting, "COS": &COSSetting, + "HuaweiOBS": &HuaweiOBSSetting, "MinIO": &MinIOSetting, "LocalOSS": &LocalOSSSetting, "S3": &S3Setting, @@ -121,6 +123,8 @@ func GetOssDomain() string { return uri + AliOSSSetting.Domain + "/" } else if CfgIf("COS") { return uri + COSSetting.Domain + "/" + } else if CfgIf("HuaweiOBS") { + return uri + HuaweiOBSSetting.Domain + "/" } else if CfgIf("MinIO") { if !MinIOSetting.Secure { uri = "http://" diff --git a/internal/conf/settting.go b/internal/conf/settting.go index 29e88557..d6c66a8d 100644 --- a/internal/conf/settting.go +++ b/internal/conf/settting.go @@ -159,6 +159,7 @@ type AliOSSSettingS struct { Bucket string Domain string } + type COSSettingS struct { SecretID string SecretKey string @@ -166,6 +167,15 @@ type COSSettingS struct { Bucket string Domain string } + +type HuaweiOBSSettingS struct { + AccessKey string + SecretKey string + Endpoint string + Bucket string + Domain string +} + type LocalOSSSettingS struct { SavePath string Secure bool diff --git a/internal/dao/dao.go b/internal/dao/dao.go index 9890a90c..9a1ad705 100644 --- a/internal/dao/dao.go +++ b/internal/dao/dao.go @@ -43,20 +43,22 @@ func ObjectStorageService() core.ObjectStorageService { onceOss.Do(func() { var v core.VersionInfo if conf.CfgIf("AliOSS") { - oss, v = storage.NewAliossService() + oss, v = storage.MustAliossService() } else if conf.CfgIf("COS") { - oss, v = storage.NewCosServent() + oss, v = storage.NewCosService() + } else if conf.CfgIf("HuaweiOBS") { + oss, v = storage.MustHuaweiobsService() } else if conf.CfgIf("MinIO") { - oss, v = storage.NewMinioService() + oss, v = storage.MustMinioService() } else if conf.CfgIf("S3") { - oss, v = storage.NewS3Service() + oss, v = storage.MustS3Service() logrus.Infof("use S3 as object storage by version %s", v.Version()) return } else if conf.CfgIf("LocalOSS") { - oss, v = storage.NewLocalossService() + oss, v = storage.MustLocalossService() } else { // default use AliOSS as object storage service - oss, v = storage.NewAliossService() + oss, v = storage.MustAliossService() logrus.Infof("use default AliOSS as object storage by version %s", v.Version()) return } diff --git a/internal/dao/storage/huaweiobs.go b/internal/dao/storage/huaweiobs.go new file mode 100644 index 00000000..f9d572dc --- /dev/null +++ b/internal/dao/storage/huaweiobs.go @@ -0,0 +1,112 @@ +package storage + +import ( + "io" + "net/url" + "strings" + + "github.com/Masterminds/semver/v3" + "github.com/huaweicloud/huaweicloud-sdk-go-obs/obs" + "github.com/rocboss/paopao-ce/internal/core" + "github.com/sirupsen/logrus" +) + +var ( + _ core.ObjectStorageService = (*huaweiobsServant)(nil) + _ core.VersionInfo = (*huaweiobsServant)(nil) +) + +type huaweiobsServant struct { + client *obs.ObsClient + bucket string + domain string +} + +func (s *huaweiobsServant) Name() string { + return "HuaweiOBS" +} + +func (s *huaweiobsServant) Version() *semver.Version { + return semver.MustParse("v0.1.0") +} + +func (s *huaweiobsServant) PutObject(objectKey string, reader io.Reader, objectSize int64, contentType string) (string, error) { + input := &obs.PutObjectInput{} + input.Bucket, input.Key, input.Body = s.bucket, objectKey, reader + input.ContentType, input.ContentLength = contentType, objectSize + _, err := s.client.PutObject(input) + if err != nil { + return "", err + } + return s.domain + objectKey, nil +} + +func (s *huaweiobsServant) DeleteObject(objectKey string) error { + _, err := s.client.DeleteObject(&obs.DeleteObjectInput{ + Bucket: s.bucket, + Key: objectKey, + }) + return err +} + +func (s *huaweiobsServant) DeleteObjects(objectKeys []string) error { + input := &obs.DeleteObjectsInput{ + Bucket: s.bucket, + Objects: make([]obs.ObjectToDelete, 0, len(objectKeys)), + } + + for _, key := range objectKeys { + input.Objects = append(input.Objects, obs.ObjectToDelete{Key: key}) + } + + _, err := s.client.DeleteObjects(input) + return err +} + +func (s *huaweiobsServant) IsObjectExist(objectKey string) (bool, error) { + input := &obs.GetObjectMetadataInput{ + Bucket: s.bucket, + Key: objectKey, + } + if _, err := s.client.GetObjectMetadata(input); err != nil { + return false, err + } + return true, nil +} + +func (s *huaweiobsServant) SignURL(objectKey string, expiredInSec int64) (string, error) { + input := &obs.CreateSignedUrlInput{ + Method: obs.HttpMethodGet, + Bucket: s.bucket, + Key: objectKey, + Expires: int(expiredInSec), + } + out, err := s.client.CreateSignedUrl(input) + if err != nil { + logrus.Errorf("huaweiobsServant.SignURL err: %v", err) + return "", err + } + + ur, err := url.Parse(out.SignedUrl) + if err != nil { + logrus.Errorf("url.Parse err: %v", err) + return "", err + } + + epath, err := url.PathUnescape(ur.Path) + if err != nil { + logrus.Errorf("url.PathUnescape err: %v", err) + return "", err + } + + ur.Path, ur.RawPath = epath, epath + return ur.String(), nil +} + +func (s *huaweiobsServant) ObjectURL(objetKey string) string { + return s.domain + objetKey +} + +func (s *huaweiobsServant) ObjectKey(objectUrl string) string { + return strings.Replace(objectUrl, s.domain, "", -1) +} diff --git a/internal/dao/storage/storage.go b/internal/dao/storage/storage.go index dcd9ee5d..a1d0a440 100644 --- a/internal/dao/storage/storage.go +++ b/internal/dao/storage/storage.go @@ -7,6 +7,7 @@ import ( "path/filepath" "github.com/aliyun/aliyun-oss-go-sdk/oss" + "github.com/huaweicloud/huaweicloud-sdk-go-obs/obs" "github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7/pkg/credentials" "github.com/rocboss/paopao-ce/internal/conf" @@ -15,7 +16,7 @@ import ( "github.com/tencentyun/cos-go-sdk-v5" ) -func NewAliossService() (core.ObjectStorageService, core.VersionInfo) { +func MustAliossService() (core.ObjectStorageService, core.VersionInfo) { client, err := oss.New(conf.AliOSSSetting.Endpoint, conf.AliOSSSetting.AccessKeyID, conf.AliOSSSetting.AccessKeySecret) if err != nil { logrus.Fatalf("alioss.New err: %v", err) @@ -33,7 +34,7 @@ func NewAliossService() (core.ObjectStorageService, core.VersionInfo) { return obj, obj } -func NewCosServent() (core.ObjectStorageService, core.VersionInfo) { +func NewCosService() (core.ObjectStorageService, core.VersionInfo) { u, _ := url.Parse(fmt.Sprintf("https://%s.cos.%s.myqcloud.com", conf.COSSetting.Bucket, conf.COSSetting.Region)) su, _ := url.Parse(fmt.Sprintf("https://cos.%s.myqcloud.com", conf.COSSetting.Region)) @@ -51,7 +52,21 @@ func NewCosServent() (core.ObjectStorageService, core.VersionInfo) { return obj, obj } -func NewLocalossService() (core.ObjectStorageService, core.VersionInfo) { +func MustHuaweiobsService() (core.ObjectStorageService, core.VersionInfo) { + s := conf.HuaweiOBSSetting + client, err := obs.New(s.AccessKey, s.SecretKey, s.Endpoint) + if err != nil { + logrus.Fatalf("create huawei obs client failed: %s", err) + } + obj := &huaweiobsServant{ + client: client, + bucket: s.Bucket, + domain: conf.GetOssDomain(), + } + return obj, obj +} + +func MustLocalossService() (core.ObjectStorageService, core.VersionInfo) { savePath, err := filepath.Abs(conf.LocalOSSSetting.SavePath) if err != nil { logrus.Fatalf("get localOSS save path err: %v", err) @@ -64,7 +79,7 @@ func NewLocalossService() (core.ObjectStorageService, core.VersionInfo) { return obj, obj } -func NewMinioService() (core.ObjectStorageService, core.VersionInfo) { +func MustMinioService() (core.ObjectStorageService, core.VersionInfo) { // Initialize minio client object. client, err := minio.New(conf.MinIOSetting.Endpoint, &minio.Options{ Creds: credentials.NewStaticV4(conf.MinIOSetting.AccessKey, conf.MinIOSetting.SecretKey, ""), @@ -82,7 +97,7 @@ func NewMinioService() (core.ObjectStorageService, core.VersionInfo) { return ms, ms } -func NewS3Service() (core.ObjectStorageService, core.VersionInfo) { +func MustS3Service() (core.ObjectStorageService, core.VersionInfo) { // Initialize s3 client object use minio-go. client, err := minio.New(conf.S3Setting.Endpoint, &minio.Options{ Creds: credentials.NewStaticV4(conf.S3Setting.AccessKey, conf.S3Setting.SecretKey, ""),