feat: tencent cos support

pull/167/head
ROC 2 years ago
parent aa34122a91
commit 986e80af32

@ -74,11 +74,17 @@ Meili: # Meili搜索配置
ApiKey: paopao-meilisearch
Secure: False
AliOSS: # 阿里云OSS存储配置
Endpoint:
Endpoint:
AccessKeyID:
AccessKeySecret:
Bucket:
Domain:
COS: # 腾讯云COS存储配置
SecretID:
SecretKey:
Region:
Bucket:
Domain:
MinIO: # MinIO 存储配置
AccessKey: Q3AM3UQ867SPQQA43P2F
SecretKey: zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG

@ -25,9 +25,12 @@ require (
github.com/json-iterator/go v1.1.12
github.com/meilisearch/meilisearch-go v0.19.1
github.com/minio/minio-go/v7 v7.0.27
github.com/mitchellh/mapstructure v1.5.0 // indirect
github.com/mozillazg/go-httpheader v0.3.1 // indirect
github.com/sirupsen/logrus v1.8.1
github.com/smartwalle/alipay/v3 v3.1.7
github.com/spf13/viper v1.10.1
github.com/tencentyun/cos-go-sdk-v5 v0.7.35 // indirect
github.com/ugorji/go v1.2.7 // indirect
github.com/yinheli/mahonia v0.0.0-20131226213531-0eef680515cc
golang.org/x/crypto v0.0.0-20220307211146-efcb8507fb70 // indirect

@ -129,6 +129,7 @@ github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbt
github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
github.com/QcloudApi/qcloud_sign_golang v0.0.0-20141224014652-e4130a326409/go.mod h1:1pk82RBxDY/JZnPQrtqHlUFfCctgdorsd9M06fMynOM=
github.com/Shopify/logrus-bugsnag v0.0.0-20171204204709-577dee27f20d/go.mod h1:HI8ITrYtUY+O+ZhtlqUnD8+KwNPOyugEhfP9fdUIaEQ=
github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg=
github.com/VictoriaMetrics/fastcache v1.6.0/go.mod h1:0qHz5QP0GMX4pfmMA/zt5RgfNuXJrTP0zS7DqpHGGTw=
@ -266,6 +267,8 @@ github.com/cilium/ebpf v0.6.2/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJ
github.com/cilium/ebpf v0.7.0/go.mod h1:/oI2+1shJiTGAMgl6/RgJr36Eo1jzrRcAWbcXO2usCA=
github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag=
github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I=
github.com/clbanning/mxj v1.8.4 h1:HuhwZtbyvyOw+3Z1AowPkU87JkJUSv751ELWaiTpj8I=
github.com/clbanning/mxj v1.8.4/go.mod h1:BVjHeAH+rl9rs6f+QIpeRl0tfu10SXn1pUSa5PVGJng=
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
github.com/cloudflare/cloudflare-go v0.14.0/go.mod h1:EnwdgGMaFOruiPZRFSgn+TsQ3hQ7C/YWzIGLeu5c304=
github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58/go.mod h1:EOBUe0h4xcZ5GoxqC5SDxFQ8gwyZPKQoEzownBlhI80=
@ -714,6 +717,7 @@ github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8
github.com/google/go-containerregistry v0.5.1/go.mod h1:Ct15B4yir3PLOP5jsy0GNeYVaIZs/MK/Jz5any1wFW0=
github.com/google/go-github/v39 v39.2.0/go.mod h1:C1s8C5aCC9L+JXIYpJM5GYytdX52vC1bLvHEF1IhBrE=
github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8=
github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU=
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
@ -1102,6 +1106,8 @@ github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh
github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
github.com/mitchellh/mapstructure v1.4.3 h1:OVowDSCllw/YjdLkam3/sm7wEtOy59d8ndGgCcyj8cs=
github.com/mitchellh/mapstructure v1.4.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
github.com/mitchellh/osext v0.0.0-20151018003038-5e2d6d41470f/go.mod h1:OkQIRizQZAeMln+1tSwduZz7+Af5oFlKirV/MSYes2A=
github.com/mitchellh/pointerstructure v1.2.0/go.mod h1:BRAsLI5zgXmw97Lf6s25bs8ohIXc3tViBH44KcwB2g4=
github.com/moby/locker v1.0.1/go.mod h1:S7SDdo5zpBK84bzzVlKr2V0hz+7x9hWbYC/kq7oQppc=
@ -1126,6 +1132,9 @@ github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjY
github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc=
github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A=
github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc=
github.com/mozillazg/go-httpheader v0.2.1/go.mod h1:jJ8xECTlalr6ValeXYdOF8fFUISeBAdw6E61aqQma60=
github.com/mozillazg/go-httpheader v0.3.1 h1:IRP+HFrMX2SlwY9riuio7raffXUpzAosHtZu25BSJok=
github.com/mozillazg/go-httpheader v0.3.1/go.mod h1:PuT8h0pw6efvp8ZeUec1Rs7dwjK08bt6gKSReGMqtdA=
github.com/mrunalp/fileutils v0.5.0/go.mod h1:M1WthSahJixYnrXQl/DFQuteStB1weuxD2QJNHXfbSQ=
github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae/go.mod h1:qAyveg+e4CE+eKJXWVjKXM4ck2QobLqTDytGJbLLhJg=
github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
@ -1385,6 +1394,10 @@ github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2/go.mod h1:hkRG
github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww=
github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7/go.mod h1:q4W45IWZaF22tdD+VEXcAWRA037jwmWEB5VWYORlTpc=
github.com/tchap/go-patricia v2.2.6+incompatible/go.mod h1:bmLyhP68RS6kStMGxByiQ23RP/odRBOTVjwp2cDyi6I=
github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common v1.0.194/go.mod h1:7sCQWVkxcsR38nffDW057DRGk8mUjK1Ing/EFOK8s8Y=
github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/kms v1.0.194/go.mod h1:yrBKWhChnDqNz1xuXdSbWXG56XawEq0G5j1lg4VwBD4=
github.com/tencentyun/cos-go-sdk-v5 v0.7.35 h1:XVk5GQ4eH1q+DBUJfpaMMdU9TJZWMjwNNwv0PG5nbLQ=
github.com/tencentyun/cos-go-sdk-v5 v0.7.35/go.mod h1:4dCEtLHGh8QPxHEkgq+nFaky7yZxQuYwgSJM87icDaw=
github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
github.com/tinylib/msgp v1.0.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE=
github.com/tklauser/go-sysconf v0.3.5/go.mod h1:MkWzOF4RMCshBAMXuhXJs64Rte09mITnppBXY/rYEFI=

@ -29,6 +29,7 @@ var (
ZincSetting *ZincSettingS
MeiliSetting *MeiliSettingS
AliOSSSetting *AliOSSSettingS
COSSetting *COSSettingS
MinIOSetting *MinIOSettingS
S3Setting *S3SettingS
LocalOSSSetting *LocalOSSSettingS
@ -71,6 +72,7 @@ func setupSetting(suite []string, noDefault bool) error {
"Redis": &redisSetting,
"JWT": &JWTSetting,
"AliOSS": &AliOSSSetting,
"COS": &COSSetting,
"MinIO": &MinIOSetting,
"LocalOSS": &LocalOSSSetting,
"S3": &S3Setting,
@ -117,6 +119,8 @@ func GetOssDomain() string {
uri := "https://"
if CfgIf("AliOSS") {
return uri + AliOSSSetting.Domain + "/"
} else if CfgIf("COS") {
return uri + COSSetting.Domain + "/"
} else if CfgIf("MinIO") {
if !MinIOSetting.Secure {
uri = "http://"

@ -159,7 +159,13 @@ type AliOSSSettingS struct {
Bucket string
Domain string
}
type COSSettingS struct {
SecretID string
SecretKey string
Region string
Bucket string
Domain string
}
type LocalOSSSettingS struct {
SavePath string
Secure bool

@ -8,7 +8,7 @@ import (
type ObjectStorageService interface {
PutObject(objectKey string, reader io.Reader, objectSize int64, contentType string) (string, error)
DeleteObject(objectKey string) error
DeleteObjcets(objectKeys []string) error
DeleteObjects(objectKeys []string) error
IsObjectExist(objectKey string) (bool, error)
SignURL(objectKey string, expiredInSec int64) (string, error)
ObjectURL(objetKey string) string

@ -44,6 +44,8 @@ func ObjectStorageService() core.ObjectStorageService {
var v core.VersionInfo
if conf.CfgIf("AliOSS") {
oss, v = storage.NewAliossService()
} else if conf.CfgIf("COS") {
oss, v = storage.NewCosServent()
} else if conf.CfgIf("MinIO") {
oss, v = storage.NewMinioService()
} else if conf.CfgIf("S3") {

@ -41,7 +41,7 @@ func (s *aliossServant) DeleteObject(objectKey string) error {
return s.bucket.DeleteObject(objectKey)
}
func (s *aliossServant) DeleteObjcets(objectKeys []string) error {
func (s *aliossServant) DeleteObjects(objectKeys []string) error {
_, err := s.bucket.DeleteObjects(objectKeys)
return err
}

@ -0,0 +1,93 @@
package storage
import (
"context"
"io"
"net/http"
"net/url"
"strings"
"time"
"github.com/Masterminds/semver/v3"
"github.com/rocboss/paopao-ce/internal/conf"
"github.com/rocboss/paopao-ce/internal/core"
"github.com/sirupsen/logrus"
"github.com/tencentyun/cos-go-sdk-v5"
)
var (
_ core.ObjectStorageService = (*cosServant)(nil)
_ core.VersionInfo = (*cosServant)(nil)
)
type cosServant struct {
client *cos.Client
domain string
}
func (s *cosServant) Name() string {
return "COS"
}
func (s *cosServant) Version() *semver.Version {
return semver.MustParse("v0.1.0")
}
func (s *cosServant) PutObject(objectKey string, reader io.Reader, objectSize int64, contentType string) (string, error) {
_, err := s.client.Object.Put(context.Background(), objectKey, reader, &cos.ObjectPutOptions{
ObjectPutHeaderOptions: &cos.ObjectPutHeaderOptions{
ContentType: contentType,
},
})
if err != nil {
return "", err
}
return s.domain + objectKey, nil
}
func (s *cosServant) DeleteObject(objectKey string) error {
_, err := s.client.Object.Delete(context.Background(), objectKey)
return err
}
func (s *cosServant) DeleteObjects(objectKeys []string) error {
obs := []cos.Object{}
for _, v := range objectKeys {
obs = append(obs, cos.Object{Key: v})
}
_, _, err := s.client.Object.DeleteMulti(context.Background(), &cos.ObjectDeleteMultiOptions{
Objects: obs,
})
return err
}
func (s *cosServant) IsObjectExist(objectKey string) (bool, error) {
return s.client.Object.IsExist(context.Background(), objectKey)
}
func (s *cosServant) SignURL(objectKey string, expiredInSec int64) (string, error) {
signedURL, err := s.client.Object.GetPresignedURL(context.Background(),
http.MethodGet, objectKey, conf.COSSetting.SecretID, conf.COSSetting.SecretKey, time.Second*time.Duration(expiredInSec), nil)
if err != nil {
logrus.Errorf("client.SignURL err: %v", err)
return "", err
}
epath, err := url.PathUnescape(signedURL.Path)
if err != nil {
logrus.Errorf("url.PathUnescape err: %v", err)
return "", err
}
signedURL.Path, signedURL.RawPath = epath, epath
return signedURL.String(), nil
}
func (s *cosServant) ObjectURL(objetKey string) string {
return s.domain + objetKey
}
func (s *cosServant) ObjectKey(objectUrl string) string {
return strings.Replace(objectUrl, s.domain, "", -1)
}

@ -61,7 +61,7 @@ func (s *localossServant) DeleteObject(objectKey string) error {
return os.Remove(s.savePath + objectKey)
}
func (s *localossServant) DeleteObjcets(objectKeys []string) (err error) {
func (s *localossServant) DeleteObjects(objectKeys []string) (err error) {
// 宽松处理删除动作尽可能删除所有objectKey如果出错只返回最后一个错误
for _, objectKey := range objectKeys {
if e := os.Remove(s.savePath + objectKey); e != nil {

@ -47,7 +47,7 @@ func (s *minioServant) DeleteObject(objectKey string) error {
return s.client.RemoveObject(context.Background(), s.bucket, objectKey, minio.RemoveObjectOptions{ForceDelete: true})
}
func (s *minioServant) DeleteObjcets(objectKeys []string) (err error) {
func (s *minioServant) DeleteObjects(objectKeys []string) (err error) {
objectsCh := make(chan minio.ObjectInfo, len(objectKeys))
resCh := s.client.RemoveObjects(context.Background(), s.bucket, objectsCh, minio.RemoveObjectsOptions{})

@ -1,6 +1,9 @@
package storage
import (
"fmt"
"net/http"
"net/url"
"path/filepath"
"github.com/aliyun/aliyun-oss-go-sdk/oss"
@ -9,6 +12,7 @@ import (
"github.com/rocboss/paopao-ce/internal/conf"
"github.com/rocboss/paopao-ce/internal/core"
"github.com/sirupsen/logrus"
"github.com/tencentyun/cos-go-sdk-v5"
)
func NewAliossService() (core.ObjectStorageService, core.VersionInfo) {
@ -29,6 +33,24 @@ func NewAliossService() (core.ObjectStorageService, core.VersionInfo) {
return obj, obj
}
func NewCosServent() (core.ObjectStorageService, core.VersionInfo) {
u, _ := url.Parse(fmt.Sprintf("https://%s.cos.%s.myqcloud.com", conf.COSSetting.Bucket, conf.COSSetting.Region))
su, _ := url.Parse(fmt.Sprintf("https://cos.%s.myqcloud.com", conf.COSSetting.Region))
client := cos.NewClient(&cos.BaseURL{BucketURL: u, ServiceURL: su}, &http.Client{
Transport: &cos.AuthorizationTransport{
SecretID: conf.COSSetting.SecretID,
SecretKey: conf.COSSetting.SecretKey,
},
})
obj := &cosServant{
client: client,
domain: conf.GetOssDomain(),
}
return obj, obj
}
func NewLocalossService() (core.ObjectStorageService, core.VersionInfo) {
savePath, err := filepath.Abs(conf.LocalOSSSetting.SavePath)
if err != nil {

@ -235,7 +235,7 @@ func deleteOssObjects(mediaContents []string) {
objectKeys = append(objectKeys, oss.ObjectKey(cUrl))
}
// TODO: 优化处理尽量使用channel传递objectKeys使用可控数量的Goroutine集中处理object删除动作后续完善
go oss.DeleteObjcets(objectKeys)
go oss.DeleteObjects(objectKeys)
} else if mediaContentsSize == 1 {
oss.DeleteObject(oss.ObjectKey(mediaContents[0]))
}

Loading…
Cancel
Save