From 986e80af3285eb13614092ee4ce1b7cfad6875f5 Mon Sep 17 00:00:00 2001 From: ROC Date: Sat, 23 Jul 2022 14:00:46 +0800 Subject: [PATCH] feat: tencent cos support --- config.yaml.sample | 8 ++- go.mod | 3 ++ go.sum | 13 +++++ internal/conf/conf.go | 4 ++ internal/conf/settting.go | 8 ++- internal/core/storage.go | 2 +- internal/dao/dao.go | 2 + internal/dao/storage/alioss.go | 2 +- internal/dao/storage/cos.go | 93 ++++++++++++++++++++++++++++++++ internal/dao/storage/localoss.go | 2 +- internal/dao/storage/minio.go | 2 +- internal/dao/storage/storage.go | 22 ++++++++ internal/service/post.go | 2 +- 13 files changed, 156 insertions(+), 7 deletions(-) create mode 100644 internal/dao/storage/cos.go diff --git a/config.yaml.sample b/config.yaml.sample index 262ced2b..89cb81f9 100644 --- a/config.yaml.sample +++ b/config.yaml.sample @@ -74,11 +74,17 @@ Meili: # Meili搜索配置 ApiKey: paopao-meilisearch Secure: False AliOSS: # 阿里云OSS存储配置 - Endpoint: + Endpoint: AccessKeyID: AccessKeySecret: Bucket: Domain: +COS: # 腾讯云COS存储配置 + SecretID: + SecretKey: + Region: + Bucket: + Domain: MinIO: # MinIO 存储配置 AccessKey: Q3AM3UQ867SPQQA43P2F SecretKey: zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG diff --git a/go.mod b/go.mod index 7dc9755b..f90895f8 100644 --- a/go.mod +++ b/go.mod @@ -25,9 +25,12 @@ require ( github.com/json-iterator/go v1.1.12 github.com/meilisearch/meilisearch-go v0.19.1 github.com/minio/minio-go/v7 v7.0.27 + github.com/mitchellh/mapstructure v1.5.0 // indirect + github.com/mozillazg/go-httpheader v0.3.1 // indirect github.com/sirupsen/logrus v1.8.1 github.com/smartwalle/alipay/v3 v3.1.7 github.com/spf13/viper v1.10.1 + github.com/tencentyun/cos-go-sdk-v5 v0.7.35 // indirect github.com/ugorji/go v1.2.7 // indirect github.com/yinheli/mahonia v0.0.0-20131226213531-0eef680515cc golang.org/x/crypto v0.0.0-20220307211146-efcb8507fb70 // indirect diff --git a/go.sum b/go.sum index f9d52126..1cd5a3e5 100644 --- a/go.sum +++ b/go.sum @@ -129,6 +129,7 @@ github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbt github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= +github.com/QcloudApi/qcloud_sign_golang v0.0.0-20141224014652-e4130a326409/go.mod h1:1pk82RBxDY/JZnPQrtqHlUFfCctgdorsd9M06fMynOM= github.com/Shopify/logrus-bugsnag v0.0.0-20171204204709-577dee27f20d/go.mod h1:HI8ITrYtUY+O+ZhtlqUnD8+KwNPOyugEhfP9fdUIaEQ= github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= github.com/VictoriaMetrics/fastcache v1.6.0/go.mod h1:0qHz5QP0GMX4pfmMA/zt5RgfNuXJrTP0zS7DqpHGGTw= @@ -266,6 +267,8 @@ github.com/cilium/ebpf v0.6.2/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJ github.com/cilium/ebpf v0.7.0/go.mod h1:/oI2+1shJiTGAMgl6/RgJr36Eo1jzrRcAWbcXO2usCA= github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag= github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I= +github.com/clbanning/mxj v1.8.4 h1:HuhwZtbyvyOw+3Z1AowPkU87JkJUSv751ELWaiTpj8I= +github.com/clbanning/mxj v1.8.4/go.mod h1:BVjHeAH+rl9rs6f+QIpeRl0tfu10SXn1pUSa5PVGJng= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cloudflare/cloudflare-go v0.14.0/go.mod h1:EnwdgGMaFOruiPZRFSgn+TsQ3hQ7C/YWzIGLeu5c304= github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58/go.mod h1:EOBUe0h4xcZ5GoxqC5SDxFQ8gwyZPKQoEzownBlhI80= @@ -714,6 +717,7 @@ github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8 github.com/google/go-containerregistry v0.5.1/go.mod h1:Ct15B4yir3PLOP5jsy0GNeYVaIZs/MK/Jz5any1wFW0= github.com/google/go-github/v39 v39.2.0/go.mod h1:C1s8C5aCC9L+JXIYpJM5GYytdX52vC1bLvHEF1IhBrE= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= +github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= @@ -1102,6 +1106,8 @@ github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/mapstructure v1.4.3 h1:OVowDSCllw/YjdLkam3/sm7wEtOy59d8ndGgCcyj8cs= github.com/mitchellh/mapstructure v1.4.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= +github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/osext v0.0.0-20151018003038-5e2d6d41470f/go.mod h1:OkQIRizQZAeMln+1tSwduZz7+Af5oFlKirV/MSYes2A= github.com/mitchellh/pointerstructure v1.2.0/go.mod h1:BRAsLI5zgXmw97Lf6s25bs8ohIXc3tViBH44KcwB2g4= github.com/moby/locker v1.0.1/go.mod h1:S7SDdo5zpBK84bzzVlKr2V0hz+7x9hWbYC/kq7oQppc= @@ -1126,6 +1132,9 @@ github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjY github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= +github.com/mozillazg/go-httpheader v0.2.1/go.mod h1:jJ8xECTlalr6ValeXYdOF8fFUISeBAdw6E61aqQma60= +github.com/mozillazg/go-httpheader v0.3.1 h1:IRP+HFrMX2SlwY9riuio7raffXUpzAosHtZu25BSJok= +github.com/mozillazg/go-httpheader v0.3.1/go.mod h1:PuT8h0pw6efvp8ZeUec1Rs7dwjK08bt6gKSReGMqtdA= github.com/mrunalp/fileutils v0.5.0/go.mod h1:M1WthSahJixYnrXQl/DFQuteStB1weuxD2QJNHXfbSQ= github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae/go.mod h1:qAyveg+e4CE+eKJXWVjKXM4ck2QobLqTDytGJbLLhJg= github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= @@ -1385,6 +1394,10 @@ github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2/go.mod h1:hkRG github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7/go.mod h1:q4W45IWZaF22tdD+VEXcAWRA037jwmWEB5VWYORlTpc= github.com/tchap/go-patricia v2.2.6+incompatible/go.mod h1:bmLyhP68RS6kStMGxByiQ23RP/odRBOTVjwp2cDyi6I= +github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common v1.0.194/go.mod h1:7sCQWVkxcsR38nffDW057DRGk8mUjK1Ing/EFOK8s8Y= +github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/kms v1.0.194/go.mod h1:yrBKWhChnDqNz1xuXdSbWXG56XawEq0G5j1lg4VwBD4= +github.com/tencentyun/cos-go-sdk-v5 v0.7.35 h1:XVk5GQ4eH1q+DBUJfpaMMdU9TJZWMjwNNwv0PG5nbLQ= +github.com/tencentyun/cos-go-sdk-v5 v0.7.35/go.mod h1:4dCEtLHGh8QPxHEkgq+nFaky7yZxQuYwgSJM87icDaw= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/tinylib/msgp v1.0.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= github.com/tklauser/go-sysconf v0.3.5/go.mod h1:MkWzOF4RMCshBAMXuhXJs64Rte09mITnppBXY/rYEFI= diff --git a/internal/conf/conf.go b/internal/conf/conf.go index 5314add7..270fe0c1 100644 --- a/internal/conf/conf.go +++ b/internal/conf/conf.go @@ -29,6 +29,7 @@ var ( ZincSetting *ZincSettingS MeiliSetting *MeiliSettingS AliOSSSetting *AliOSSSettingS + COSSetting *COSSettingS MinIOSetting *MinIOSettingS S3Setting *S3SettingS LocalOSSSetting *LocalOSSSettingS @@ -71,6 +72,7 @@ func setupSetting(suite []string, noDefault bool) error { "Redis": &redisSetting, "JWT": &JWTSetting, "AliOSS": &AliOSSSetting, + "COS": &COSSetting, "MinIO": &MinIOSetting, "LocalOSS": &LocalOSSSetting, "S3": &S3Setting, @@ -117,6 +119,8 @@ func GetOssDomain() string { uri := "https://" if CfgIf("AliOSS") { return uri + AliOSSSetting.Domain + "/" + } else if CfgIf("COS") { + return uri + COSSetting.Domain + "/" } else if CfgIf("MinIO") { if !MinIOSetting.Secure { uri = "http://" diff --git a/internal/conf/settting.go b/internal/conf/settting.go index 55f7eb3e..29e88557 100644 --- a/internal/conf/settting.go +++ b/internal/conf/settting.go @@ -159,7 +159,13 @@ type AliOSSSettingS struct { Bucket string Domain string } - +type COSSettingS struct { + SecretID string + SecretKey string + Region string + Bucket string + Domain string +} type LocalOSSSettingS struct { SavePath string Secure bool diff --git a/internal/core/storage.go b/internal/core/storage.go index e158c675..2203c6f4 100644 --- a/internal/core/storage.go +++ b/internal/core/storage.go @@ -8,7 +8,7 @@ import ( type ObjectStorageService interface { PutObject(objectKey string, reader io.Reader, objectSize int64, contentType string) (string, error) DeleteObject(objectKey string) error - DeleteObjcets(objectKeys []string) error + DeleteObjects(objectKeys []string) error IsObjectExist(objectKey string) (bool, error) SignURL(objectKey string, expiredInSec int64) (string, error) ObjectURL(objetKey string) string diff --git a/internal/dao/dao.go b/internal/dao/dao.go index 10e7cc61..9890a90c 100644 --- a/internal/dao/dao.go +++ b/internal/dao/dao.go @@ -44,6 +44,8 @@ func ObjectStorageService() core.ObjectStorageService { var v core.VersionInfo if conf.CfgIf("AliOSS") { oss, v = storage.NewAliossService() + } else if conf.CfgIf("COS") { + oss, v = storage.NewCosServent() } else if conf.CfgIf("MinIO") { oss, v = storage.NewMinioService() } else if conf.CfgIf("S3") { diff --git a/internal/dao/storage/alioss.go b/internal/dao/storage/alioss.go index 48cdcec0..ff2f0a2d 100644 --- a/internal/dao/storage/alioss.go +++ b/internal/dao/storage/alioss.go @@ -41,7 +41,7 @@ func (s *aliossServant) DeleteObject(objectKey string) error { return s.bucket.DeleteObject(objectKey) } -func (s *aliossServant) DeleteObjcets(objectKeys []string) error { +func (s *aliossServant) DeleteObjects(objectKeys []string) error { _, err := s.bucket.DeleteObjects(objectKeys) return err } diff --git a/internal/dao/storage/cos.go b/internal/dao/storage/cos.go new file mode 100644 index 00000000..b429fb4c --- /dev/null +++ b/internal/dao/storage/cos.go @@ -0,0 +1,93 @@ +package storage + +import ( + "context" + "io" + "net/http" + "net/url" + "strings" + "time" + + "github.com/Masterminds/semver/v3" + "github.com/rocboss/paopao-ce/internal/conf" + "github.com/rocboss/paopao-ce/internal/core" + "github.com/sirupsen/logrus" + "github.com/tencentyun/cos-go-sdk-v5" +) + +var ( + _ core.ObjectStorageService = (*cosServant)(nil) + _ core.VersionInfo = (*cosServant)(nil) +) + +type cosServant struct { + client *cos.Client + domain string +} + +func (s *cosServant) Name() string { + return "COS" +} + +func (s *cosServant) Version() *semver.Version { + return semver.MustParse("v0.1.0") +} + +func (s *cosServant) PutObject(objectKey string, reader io.Reader, objectSize int64, contentType string) (string, error) { + _, err := s.client.Object.Put(context.Background(), objectKey, reader, &cos.ObjectPutOptions{ + ObjectPutHeaderOptions: &cos.ObjectPutHeaderOptions{ + ContentType: contentType, + }, + }) + if err != nil { + return "", err + } + return s.domain + objectKey, nil +} + +func (s *cosServant) DeleteObject(objectKey string) error { + _, err := s.client.Object.Delete(context.Background(), objectKey) + return err +} + +func (s *cosServant) DeleteObjects(objectKeys []string) error { + obs := []cos.Object{} + for _, v := range objectKeys { + obs = append(obs, cos.Object{Key: v}) + } + _, _, err := s.client.Object.DeleteMulti(context.Background(), &cos.ObjectDeleteMultiOptions{ + Objects: obs, + }) + return err +} + +func (s *cosServant) IsObjectExist(objectKey string) (bool, error) { + return s.client.Object.IsExist(context.Background(), objectKey) +} + +func (s *cosServant) SignURL(objectKey string, expiredInSec int64) (string, error) { + signedURL, err := s.client.Object.GetPresignedURL(context.Background(), + http.MethodGet, objectKey, conf.COSSetting.SecretID, conf.COSSetting.SecretKey, time.Second*time.Duration(expiredInSec), nil) + + if err != nil { + logrus.Errorf("client.SignURL err: %v", err) + return "", err + } + + epath, err := url.PathUnescape(signedURL.Path) + if err != nil { + logrus.Errorf("url.PathUnescape err: %v", err) + return "", err + } + + signedURL.Path, signedURL.RawPath = epath, epath + return signedURL.String(), nil +} + +func (s *cosServant) ObjectURL(objetKey string) string { + return s.domain + objetKey +} + +func (s *cosServant) ObjectKey(objectUrl string) string { + return strings.Replace(objectUrl, s.domain, "", -1) +} diff --git a/internal/dao/storage/localoss.go b/internal/dao/storage/localoss.go index 6ed4a165..00f7d378 100644 --- a/internal/dao/storage/localoss.go +++ b/internal/dao/storage/localoss.go @@ -61,7 +61,7 @@ func (s *localossServant) DeleteObject(objectKey string) error { return os.Remove(s.savePath + objectKey) } -func (s *localossServant) DeleteObjcets(objectKeys []string) (err error) { +func (s *localossServant) DeleteObjects(objectKeys []string) (err error) { // 宽松处理删除动作，尽可能删除所有objectKey，如果出错，只返回最后一个错误 for _, objectKey := range objectKeys { if e := os.Remove(s.savePath + objectKey); e != nil { diff --git a/internal/dao/storage/minio.go b/internal/dao/storage/minio.go index e767a356..4a23b71b 100644 --- a/internal/dao/storage/minio.go +++ b/internal/dao/storage/minio.go @@ -47,7 +47,7 @@ func (s *minioServant) DeleteObject(objectKey string) error { return s.client.RemoveObject(context.Background(), s.bucket, objectKey, minio.RemoveObjectOptions{ForceDelete: true}) } -func (s *minioServant) DeleteObjcets(objectKeys []string) (err error) { +func (s *minioServant) DeleteObjects(objectKeys []string) (err error) { objectsCh := make(chan minio.ObjectInfo, len(objectKeys)) resCh := s.client.RemoveObjects(context.Background(), s.bucket, objectsCh, minio.RemoveObjectsOptions{}) diff --git a/internal/dao/storage/storage.go b/internal/dao/storage/storage.go index 78bdb0f4..dcd9ee5d 100644 --- a/internal/dao/storage/storage.go +++ b/internal/dao/storage/storage.go @@ -1,6 +1,9 @@ package storage import ( + "fmt" + "net/http" + "net/url" "path/filepath" "github.com/aliyun/aliyun-oss-go-sdk/oss" @@ -9,6 +12,7 @@ import ( "github.com/rocboss/paopao-ce/internal/conf" "github.com/rocboss/paopao-ce/internal/core" "github.com/sirupsen/logrus" + "github.com/tencentyun/cos-go-sdk-v5" ) func NewAliossService() (core.ObjectStorageService, core.VersionInfo) { @@ -29,6 +33,24 @@ func NewAliossService() (core.ObjectStorageService, core.VersionInfo) { return obj, obj } +func NewCosServent() (core.ObjectStorageService, core.VersionInfo) { + u, _ := url.Parse(fmt.Sprintf("https://%s.cos.%s.myqcloud.com", conf.COSSetting.Bucket, conf.COSSetting.Region)) + su, _ := url.Parse(fmt.Sprintf("https://cos.%s.myqcloud.com", conf.COSSetting.Region)) + + client := cos.NewClient(&cos.BaseURL{BucketURL: u, ServiceURL: su}, &http.Client{ + Transport: &cos.AuthorizationTransport{ + SecretID: conf.COSSetting.SecretID, + SecretKey: conf.COSSetting.SecretKey, + }, + }) + + obj := &cosServant{ + client: client, + domain: conf.GetOssDomain(), + } + return obj, obj +} + func NewLocalossService() (core.ObjectStorageService, core.VersionInfo) { savePath, err := filepath.Abs(conf.LocalOSSSetting.SavePath) if err != nil { diff --git a/internal/service/post.go b/internal/service/post.go index 661b70a0..a0228044 100644 --- a/internal/service/post.go +++ b/internal/service/post.go @@ -235,7 +235,7 @@ func deleteOssObjects(mediaContents []string) { objectKeys = append(objectKeys, oss.ObjectKey(cUrl)) } // TODO: 优化处理尽量使用channel传递objectKeys使用可控数量的Goroutine集中处理object删除动作，后续完善 - go oss.DeleteObjcets(objectKeys) + go oss.DeleteObjects(objectKeys) } else if mediaContentsSize == 1 { oss.DeleteObject(oss.ObjectKey(mediaContents[0])) }