support Huawei Cloud OBS as OSS service

pull/167/head
Michael Li 2 years ago
parent 500adeb6af
commit 4232099036

@ -1,4 +1,4 @@
.PHONY: all build run test clean fmt help
.PHONY: all build run test clean fmt pre-commit help
TARGET = paopao-ce
ifeq ($(OS),Windows_NT)
@ -82,6 +82,9 @@ fmt:
test:
@go test ./...
pre-commit: fmt
go mod tidy
help:
@echo "make: make"
@echo "make run: start api server"

@ -306,9 +306,10 @@ release/paopao-ce --no-default-features --features sqlite3,localoss,loggerfile,r
* 数据库: MySQL/Sqlite3/PostgreSQL
`Gorm` + `MySQL`/`Sqlite3`/`PostgreSQL` 使用[gorm](https://github.com/go-gorm/gorm)作为数据库的ORM默认使用 `Grom` + `MySQL`组合(目前状态:稳定,默认,推荐使用)
`Sqlx` + `MySQL`/`PostgreSQL` 使用[sqlx](https://github.com/jmoiron/sqlx)作为数据库的ORM(目前状态WIP)
* 对象存储: AliOSS/COS/MinIO/LocalOSS
`AliOSS` 阿里云对象存储服务;
`COS` 腾讯云对象存储服务
* 对象存储: AliOSS/COS/HuaweiOBS/MinIO/LocalOSS
`AliOSS` 阿里云对象存储服务;
`COS` 腾讯云对象存储服务;
`HuaweiOBS` 华为云对象存储服务;
`MinIO` [MinIO](https://github.com/minio/minio)对象存储服务;
`LocalOSS` 提供使用本地目录文件作为对象存储的功能,仅用于开发调试环境;
* 缓存: Redis/SimpleCacheIndex/BigCacheIndex

@ -85,6 +85,12 @@ COS: # 腾讯云COS存储配置
Region: ap-shanghai
Bucket: demo-1888888888
Domain:
HuaweiOBS: # 华为云OBS存储配置
AccessKey:
SecretKey:
Endpoint:
Bucket: paopao
Domain:
MinIO: # MinIO 存储配置
AccessKey: Q3AM3UQ867SPQQA43P2F
SecretKey: zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG

@ -22,6 +22,7 @@ require (
github.com/gofrs/uuid v4.0.0+incompatible
github.com/golang-migrate/migrate/v4 v4.15.2
github.com/google/go-cmp v0.5.7 // indirect
github.com/huaweicloud/huaweicloud-sdk-go-obs v3.21.12+incompatible
github.com/json-iterator/go v1.1.12
github.com/meilisearch/meilisearch-go v0.19.1
github.com/minio/minio-go/v7 v7.0.27
@ -30,7 +31,7 @@ require (
github.com/sirupsen/logrus v1.8.1
github.com/smartwalle/alipay/v3 v3.1.7
github.com/spf13/viper v1.10.1
github.com/tencentyun/cos-go-sdk-v5 v0.7.35 // indirect
github.com/tencentyun/cos-go-sdk-v5 v0.7.35
github.com/ugorji/go v1.2.7 // indirect
github.com/yinheli/mahonia v0.0.0-20131226213531-0eef680515cc
golang.org/x/crypto v0.0.0-20220307211146-efcb8507fb70 // indirect

@ -823,6 +823,8 @@ github.com/hashicorp/serf v0.9.6/go.mod h1:TXZNMjZQijwlDvp+r0b63xZ45H7JmCmgg4gpT
github.com/holiman/bloomfilter/v2 v2.0.3/go.mod h1:zpoh+gs7qcpqrHr3dB55AMiJwo0iURXE7ZOP9L9hSkA=
github.com/holiman/uint256 v1.2.0/go.mod h1:y4ga/t+u+Xwd7CpDgZESaRcWy0I7XMlTMA25ApIH5Jw=
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
github.com/huaweicloud/huaweicloud-sdk-go-obs v3.21.12+incompatible h1:tANYIteuFrosKbRYUk1Yo/OGJjbt4x3OVg211Qc60M0=
github.com/huaweicloud/huaweicloud-sdk-go-obs v3.21.12+incompatible/go.mod h1:l7VUhRbTKCzdOacdT4oWCwATKyvZqUOlOqr0Ous3k4s=
github.com/huin/goupnp v1.0.2/go.mod h1:0dxJBVBHqTMjIUMkESDTNgOOx/Mw5wYIfyFmdzSamkM=
github.com/huin/goutil v0.0.0-20170803182201-1ca381bf3150/go.mod h1:PpLOETDnJ0o3iZrZfqZzyLl6l7F3c6L1oWn7OICBi6o=
github.com/iancoleman/strcase v0.2.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho=
@ -1104,7 +1106,6 @@ github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:F
github.com/mitchellh/mapstructure v0.0.0-20180220230111-00c29f56e238/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
github.com/mitchellh/mapstructure v1.4.3 h1:OVowDSCllw/YjdLkam3/sm7wEtOy59d8ndGgCcyj8cs=
github.com/mitchellh/mapstructure v1.4.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=

@ -30,6 +30,7 @@ var (
MeiliSetting *MeiliSettingS
AliOSSSetting *AliOSSSettingS
COSSetting *COSSettingS
HuaweiOBSSetting *HuaweiOBSSettingS
MinIOSetting *MinIOSettingS
S3Setting *S3SettingS
LocalOSSSetting *LocalOSSSettingS
@ -73,6 +74,7 @@ func setupSetting(suite []string, noDefault bool) error {
"JWT": &JWTSetting,
"AliOSS": &AliOSSSetting,
"COS": &COSSetting,
"HuaweiOBS": &HuaweiOBSSetting,
"MinIO": &MinIOSetting,
"LocalOSS": &LocalOSSSetting,
"S3": &S3Setting,
@ -121,6 +123,8 @@ func GetOssDomain() string {
return uri + AliOSSSetting.Domain + "/"
} else if CfgIf("COS") {
return uri + COSSetting.Domain + "/"
} else if CfgIf("HuaweiOBS") {
return uri + HuaweiOBSSetting.Domain + "/"
} else if CfgIf("MinIO") {
if !MinIOSetting.Secure {
uri = "http://"

@ -159,6 +159,7 @@ type AliOSSSettingS struct {
Bucket string
Domain string
}
type COSSettingS struct {
SecretID string
SecretKey string
@ -166,6 +167,15 @@ type COSSettingS struct {
Bucket string
Domain string
}
type HuaweiOBSSettingS struct {
AccessKey string
SecretKey string
Endpoint string
Bucket string
Domain string
}
type LocalOSSSettingS struct {
SavePath string
Secure bool

@ -43,20 +43,22 @@ func ObjectStorageService() core.ObjectStorageService {
onceOss.Do(func() {
var v core.VersionInfo
if conf.CfgIf("AliOSS") {
oss, v = storage.NewAliossService()
oss, v = storage.MustAliossService()
} else if conf.CfgIf("COS") {
oss, v = storage.NewCosServent()
oss, v = storage.NewCosService()
} else if conf.CfgIf("HuaweiOBS") {
oss, v = storage.MustHuaweiobsService()
} else if conf.CfgIf("MinIO") {
oss, v = storage.NewMinioService()
oss, v = storage.MustMinioService()
} else if conf.CfgIf("S3") {
oss, v = storage.NewS3Service()
oss, v = storage.MustS3Service()
logrus.Infof("use S3 as object storage by version %s", v.Version())
return
} else if conf.CfgIf("LocalOSS") {
oss, v = storage.NewLocalossService()
oss, v = storage.MustLocalossService()
} else {
// default use AliOSS as object storage service
oss, v = storage.NewAliossService()
oss, v = storage.MustAliossService()
logrus.Infof("use default AliOSS as object storage by version %s", v.Version())
return
}

@ -0,0 +1,112 @@
package storage
import (
"io"
"net/url"
"strings"
"github.com/Masterminds/semver/v3"
"github.com/huaweicloud/huaweicloud-sdk-go-obs/obs"
"github.com/rocboss/paopao-ce/internal/core"
"github.com/sirupsen/logrus"
)
var (
_ core.ObjectStorageService = (*huaweiobsServant)(nil)
_ core.VersionInfo = (*huaweiobsServant)(nil)
)
type huaweiobsServant struct {
client *obs.ObsClient
bucket string
domain string
}
func (s *huaweiobsServant) Name() string {
return "HuaweiOBS"
}
func (s *huaweiobsServant) Version() *semver.Version {
return semver.MustParse("v0.1.0")
}
func (s *huaweiobsServant) PutObject(objectKey string, reader io.Reader, objectSize int64, contentType string) (string, error) {
input := &obs.PutObjectInput{}
input.Bucket, input.Key, input.Body = s.bucket, objectKey, reader
input.ContentType, input.ContentLength = contentType, objectSize
_, err := s.client.PutObject(input)
if err != nil {
return "", err
}
return s.domain + objectKey, nil
}
func (s *huaweiobsServant) DeleteObject(objectKey string) error {
_, err := s.client.DeleteObject(&obs.DeleteObjectInput{
Bucket: s.bucket,
Key: objectKey,
})
return err
}
func (s *huaweiobsServant) DeleteObjects(objectKeys []string) error {
input := &obs.DeleteObjectsInput{
Bucket: s.bucket,
Objects: make([]obs.ObjectToDelete, 0, len(objectKeys)),
}
for _, key := range objectKeys {
input.Objects = append(input.Objects, obs.ObjectToDelete{Key: key})
}
_, err := s.client.DeleteObjects(input)
return err
}
func (s *huaweiobsServant) IsObjectExist(objectKey string) (bool, error) {
input := &obs.GetObjectMetadataInput{
Bucket: s.bucket,
Key: objectKey,
}
if _, err := s.client.GetObjectMetadata(input); err != nil {
return false, err
}
return true, nil
}
func (s *huaweiobsServant) SignURL(objectKey string, expiredInSec int64) (string, error) {
input := &obs.CreateSignedUrlInput{
Method: obs.HttpMethodGet,
Bucket: s.bucket,
Key: objectKey,
Expires: int(expiredInSec),
}
out, err := s.client.CreateSignedUrl(input)
if err != nil {
logrus.Errorf("huaweiobsServant.SignURL err: %v", err)
return "", err
}
ur, err := url.Parse(out.SignedUrl)
if err != nil {
logrus.Errorf("url.Parse err: %v", err)
return "", err
}
epath, err := url.PathUnescape(ur.Path)
if err != nil {
logrus.Errorf("url.PathUnescape err: %v", err)
return "", err
}
ur.Path, ur.RawPath = epath, epath
return ur.String(), nil
}
func (s *huaweiobsServant) ObjectURL(objetKey string) string {
return s.domain + objetKey
}
func (s *huaweiobsServant) ObjectKey(objectUrl string) string {
return strings.Replace(objectUrl, s.domain, "", -1)
}

@ -7,6 +7,7 @@ import (
"path/filepath"
"github.com/aliyun/aliyun-oss-go-sdk/oss"
"github.com/huaweicloud/huaweicloud-sdk-go-obs/obs"
"github.com/minio/minio-go/v7"
"github.com/minio/minio-go/v7/pkg/credentials"
"github.com/rocboss/paopao-ce/internal/conf"
@ -15,7 +16,7 @@ import (
"github.com/tencentyun/cos-go-sdk-v5"
)
func NewAliossService() (core.ObjectStorageService, core.VersionInfo) {
func MustAliossService() (core.ObjectStorageService, core.VersionInfo) {
client, err := oss.New(conf.AliOSSSetting.Endpoint, conf.AliOSSSetting.AccessKeyID, conf.AliOSSSetting.AccessKeySecret)
if err != nil {
logrus.Fatalf("alioss.New err: %v", err)
@ -33,7 +34,7 @@ func NewAliossService() (core.ObjectStorageService, core.VersionInfo) {
return obj, obj
}
func NewCosServent() (core.ObjectStorageService, core.VersionInfo) {
func NewCosService() (core.ObjectStorageService, core.VersionInfo) {
u, _ := url.Parse(fmt.Sprintf("https://%s.cos.%s.myqcloud.com", conf.COSSetting.Bucket, conf.COSSetting.Region))
su, _ := url.Parse(fmt.Sprintf("https://cos.%s.myqcloud.com", conf.COSSetting.Region))
@ -51,7 +52,21 @@ func NewCosServent() (core.ObjectStorageService, core.VersionInfo) {
return obj, obj
}
func NewLocalossService() (core.ObjectStorageService, core.VersionInfo) {
func MustHuaweiobsService() (core.ObjectStorageService, core.VersionInfo) {
s := conf.HuaweiOBSSetting
client, err := obs.New(s.AccessKey, s.SecretKey, s.Endpoint)
if err != nil {
logrus.Fatalf("create huawei obs client failed: %s", err)
}
obj := &huaweiobsServant{
client: client,
bucket: s.Bucket,
domain: conf.GetOssDomain(),
}
return obj, obj
}
func MustLocalossService() (core.ObjectStorageService, core.VersionInfo) {
savePath, err := filepath.Abs(conf.LocalOSSSetting.SavePath)
if err != nil {
logrus.Fatalf("get localOSS save path err: %v", err)
@ -64,7 +79,7 @@ func NewLocalossService() (core.ObjectStorageService, core.VersionInfo) {
return obj, obj
}
func NewMinioService() (core.ObjectStorageService, core.VersionInfo) {
func MustMinioService() (core.ObjectStorageService, core.VersionInfo) {
// Initialize minio client object.
client, err := minio.New(conf.MinIOSetting.Endpoint, &minio.Options{
Creds: credentials.NewStaticV4(conf.MinIOSetting.AccessKey, conf.MinIOSetting.SecretKey, ""),
@ -82,7 +97,7 @@ func NewMinioService() (core.ObjectStorageService, core.VersionInfo) {
return ms, ms
}
func NewS3Service() (core.ObjectStorageService, core.VersionInfo) {
func MustS3Service() (core.ObjectStorageService, core.VersionInfo) {
// Initialize s3 client object use minio-go.
client, err := minio.New(conf.S3Setting.Endpoint, &minio.Options{
Creds: credentials.NewStaticV4(conf.S3Setting.AccessKey, conf.S3Setting.SecretKey, ""),

Loading…
Cancel
Save