From c825e2db4323d6c8c44c0a86c6e42d7bab128a3c Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Thu, 14 Sep 2023 18:30:25 +0800 Subject: [PATCH] s3 public read --- pkg/common/db/s3/cos/doc.go | 15 ----------- pkg/common/db/s3/cos/internal_test.go | 10 ------- pkg/common/db/s3/minio/doc.go | 15 ----------- pkg/common/db/s3/minio/internal_test.go | 19 ------------- pkg/common/db/s3/minio/minio.go | 6 +++++ pkg/common/db/s3/oss/doc.go | 15 ----------- pkg/common/db/s3/oss/{sign.go => internal.go} | 12 ++++++++- pkg/common/db/s3/oss/oss.go | 17 ++++++------ pkg/common/db/s3/oss/sign_test.go | 27 ------------------- 9 files changed, 26 insertions(+), 110 deletions(-) delete mode 100644 pkg/common/db/s3/cos/doc.go delete mode 100644 pkg/common/db/s3/cos/internal_test.go delete mode 100644 pkg/common/db/s3/minio/doc.go delete mode 100644 pkg/common/db/s3/minio/internal_test.go delete mode 100644 pkg/common/db/s3/oss/doc.go rename pkg/common/db/s3/oss/{sign.go => internal.go} (75%) delete mode 100644 pkg/common/db/s3/oss/sign_test.go diff --git a/pkg/common/db/s3/cos/doc.go b/pkg/common/db/s3/cos/doc.go deleted file mode 100644 index 592f90ee2..000000000 --- a/pkg/common/db/s3/cos/doc.go +++ /dev/null @@ -1,15 +0,0 @@ -// Copyright © 2023 OpenIM. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package cos // import "github.com/openimsdk/open-im-server/v3/pkg/common/db/s3/cos" diff --git a/pkg/common/db/s3/cos/internal_test.go b/pkg/common/db/s3/cos/internal_test.go deleted file mode 100644 index 3ea449911..000000000 --- a/pkg/common/db/s3/cos/internal_test.go +++ /dev/null @@ -1,10 +0,0 @@ -package cos - -import ( - "testing" -) - -func TestName(t *testing.T) { - newRequest(nil, nil, nil, "", "", nil, nil, nil) - -} diff --git a/pkg/common/db/s3/minio/doc.go b/pkg/common/db/s3/minio/doc.go deleted file mode 100644 index 42e898c15..000000000 --- a/pkg/common/db/s3/minio/doc.go +++ /dev/null @@ -1,15 +0,0 @@ -// Copyright © 2023 OpenIM. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package minio // import "github.com/openimsdk/open-im-server/v3/pkg/common/db/s3/minio" diff --git a/pkg/common/db/s3/minio/internal_test.go b/pkg/common/db/s3/minio/internal_test.go deleted file mode 100644 index 4a20b589b..000000000 --- a/pkg/common/db/s3/minio/internal_test.go +++ /dev/null @@ -1,19 +0,0 @@ -package minio - -import ( - "testing" -) - -func TestName(t *testing.T) { - //u, err := makeTargetURL(&minio.Client{}, "openim", "test.png", "", false, nil) - //if err != nil { - // panic(err) - //} - //u.String() - //t.Log(percentEncodeSlash("1234")) - // - //t.Log(FastRand()) - t.Log(makeTargetURL(nil, "", "", "", false, nil)) - //t.Log(privateNew("", nil)) - -} diff --git a/pkg/common/db/s3/minio/minio.go b/pkg/common/db/s3/minio/minio.go index abf3eeff6..bb9d593ab 100644 --- a/pkg/common/db/s3/minio/minio.go +++ b/pkg/common/db/s3/minio/minio.go @@ -139,6 +139,12 @@ func (m *Minio) initMinio(ctx context.Context) error { return fmt.Errorf("make bucket error: %w", err) } } + if conf.PublicRead { + policy := fmt.Sprintf(`{"Version": "2012-10-17","Statement": [{"Action": ["s3:GetObject","s3:PutObject"],"Effect": "Allow","Principal": {"AWS": ["*"]},"Resource": ["arn:aws:s3:::%s/*"],"Sid": ""}]}`, conf.Bucket) + if err := m.core.Client.SetBucketPolicy(ctx, conf.Bucket, policy); err != nil { + return err + } + } m.location, err = m.core.Client.GetBucketLocation(ctx, conf.Bucket) if err != nil { return err diff --git a/pkg/common/db/s3/oss/doc.go b/pkg/common/db/s3/oss/doc.go deleted file mode 100644 index d2e2c91af..000000000 --- a/pkg/common/db/s3/oss/doc.go +++ /dev/null @@ -1,15 +0,0 @@ -// Copyright © 2023 OpenIM. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package oss // import "github.com/openimsdk/open-im-server/v3/pkg/common/db/s3/oss" diff --git a/pkg/common/db/s3/oss/sign.go b/pkg/common/db/s3/oss/internal.go similarity index 75% rename from pkg/common/db/s3/oss/sign.go rename to pkg/common/db/s3/oss/internal.go index b296b236d..d7913b902 100644 --- a/pkg/common/db/s3/oss/sign.go +++ b/pkg/common/db/s3/oss/internal.go @@ -25,5 +25,15 @@ import ( //go:linkname ossSignHeader github.com/aliyun/aliyun-oss-go-sdk/oss.Conn.signHeader func ossSignHeader(c *oss.Conn, req *http.Request, canonicalizedResource string) +//go:linkname getURLParams github.com/aliyun/aliyun-oss-go-sdk/oss.Conn.getURLParams +func getURLParams(c *oss.Conn, params map[string]interface{}) string + //go:linkname getURL github.com/aliyun/aliyun-oss-go-sdk/oss.urlMaker.getURL -func getURL(ptr any, bucket, object, params string) *url.URL +func getURL(um urlMaker, bucket, object, params string) *url.URL + +type urlMaker struct { + Scheme string + NetLoc string + Type int + IsProxy bool +} diff --git a/pkg/common/db/s3/oss/oss.go b/pkg/common/db/s3/oss/oss.go index 1dbc2d14b..a0a23ce3e 100644 --- a/pkg/common/db/s3/oss/oss.go +++ b/pkg/common/db/s3/oss/oss.go @@ -18,14 +18,14 @@ import ( "context" "errors" "fmt" + "github.com/aliyun/aliyun-oss-go-sdk/oss" "net/http" "net/url" + "reflect" "strconv" "strings" "time" - "github.com/aliyun/aliyun-oss-go-sdk/oss" - "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/open-im-server/v3/pkg/common/db/s3" ) @@ -69,6 +69,7 @@ func NewOSS() (s3.Interface, error) { bucketURL: conf.BucketURL, bucket: bucket, credentials: client.Config.GetCredentials(), + um: *(*urlMaker)(reflect.ValueOf(bucket.Client.Conn).Elem().FieldByName("url").UnsafePointer()), }, nil } @@ -76,6 +77,7 @@ type OSS struct { bucketURL string bucket *oss.Bucket credentials oss.Credentials + um urlMaker } func (o *OSS) Engine() string { @@ -314,10 +316,9 @@ func (o *OSS) AccessURL(ctx context.Context, name string, expire time.Duration, if !config.Config.Object.Oss.PublicRead { return o.bucket.SignURL(name, http.MethodGet, int64(expire/time.Second), opts...) } - //params, err := oss.GetRawParams(opts) - //if err != nil { - // return "", err - //} - - return o.bucket.SignURL(name, http.MethodGet, int64(expire/time.Second), opts...) + params, err := oss.GetRawParams(opts) + if err != nil { + return "", err + } + return getURL(o.um, o.bucket.BucketName, name, getURLParams(o.bucket.Client.Conn, params)).String(), nil } diff --git a/pkg/common/db/s3/oss/sign_test.go b/pkg/common/db/s3/oss/sign_test.go deleted file mode 100644 index 82717173c..000000000 --- a/pkg/common/db/s3/oss/sign_test.go +++ /dev/null @@ -1,27 +0,0 @@ -package oss - -import ( - "testing" -) - -func TestName(t *testing.T) { - //ossSignHeader(nil, nil, "") - //t.Log("ossSignHeader") - - //var c oss.Conn - //blc := reflect.ValueOf(&c).Elem().FieldByName("url") // *urlMaker - // - //urlPtr := reflect.New(blc.Type().Elem()).Addr() // *urlMaker - // - //vblc := reflect.New(reflect.PtrTo(blc.Type())) - //*(*unsafe.Pointer)(vblc.UnsafePointer()) = unsafe.Pointer(blc.UnsafeAddr()) - //vblc.Elem().Elem().Interface().(interface{ Set(string, string) }).Set(conf.Bucket, m.location) - // - // - // - // - //fmt.Println(inter) - - //getURL(nil, "", "", "") - -}