feat: s3 public read (#1080)

* fix: repeated modification session notification

* fix: repeated modification session notification

* fix: jpush return a nil pointer panic

* fix: push redis pkg

* fix: OANotification

* feat: add rpc GetConversationNeedOfflinePushUserIDs

* update pkg

* cicd: robot automated Change

* offlinePushMsg

* conversation

* conversation

* cicd: robot automated Change

* conversation

* cicd: robot automated Change

* conversation

* url 2 im s3

* url 2 im s3

* cicd: robot automated Change

* url 2 im s3

* s3 public read

* cicd: robot automated Change

* s3 public read

* cicd: robot automated Change

* s3 public read

* s3 public read

* s3 public read

* s3 public read

* s3 public read

* cicd: robot automated Change

* s3 public read

* s3 public read

* fix: SendMsg api

* config scripts

* config scripts

---------

Co-authored-by: withchao <withchao@users.noreply.github.com>

config/config.yaml

@@ -140,11 +140,13 @@ object:
     secretAccessKey: "openIM123"
     sessionToken: ''
     signEndpoint: "http://127.0.0.1:10005"
+    publicRead: false
   cos:
     bucketURL: https://temp-1252357374.cos.ap-chengdu.myqcloud.com
     secretID: ''
     secretKey: ''
     sessionToken: ''
+    publicRead: false
   oss:
     endpoint: "https://oss-cn-chengdu.aliyuncs.com"
     bucket: "demo-9999999"
@@ -152,6 +154,7 @@ object:
     accessKeyID: ''
     accessKeySecret: ''
     sessionToken: ''
+    publicRead: false
 
 
 ###################### RPC Port Configuration ######################

deployments/templates/openim.yaml

@@ -140,11 +140,13 @@ object:
     secretAccessKey: "${MINIO_SECRET_KEY}"
     sessionToken: ${MINIO_SESSION_TOKEN}
     signEndpoint: "${MINIO_SIGN_ENDPOINT}"
+    publicRead: ${MINIO_PUBLIC_READ}
   cos:
     bucketURL: ${COS_BUCKET_URL}
     secretID: ${COS_SECRET_ID}
     secretKey: ${COS_SECRET_KEY}
     sessionToken: ${COS_SESSION_TOKEN}
+    publicRead: ${COS_PUBLIC_READ}
   oss:
     endpoint: "${OSS_ENDPOINT}"
     bucket: "${OSS_BUCKET}"
@@ -152,7 +154,7 @@ object:
     accessKeyID: ${OSS_ACCESS_KEY_ID}
     accessKeySecret: ${OSS_ACCESS_KEY_SECRET}
     sessionToken: ${OSS_SESSION_TOKEN}
-
+    publicRead: ${OSS_PUBLIC_READ}
 
 ###################### RPC Port Configuration ######################
 # RPC service ports

internal/api/msg.go

@@ -58,7 +58,7 @@ func (m MessageApi) newUserSendMsgReq(c *gin.Context, params *apistruct.SendMsg)
 	options := make(map[string]bool, 5)
 	switch params.ContentType {
 	case constant.Text:
-		newContent = params.Content["text"].(string)
+		fallthrough
 	case constant.Picture:
 		fallthrough
 	case constant.Custom:

pkg/common/config/config.go

@@ -128,12 +128,14 @@ type configStruct struct {
 			SecretAccessKey string `yaml:"secretAccessKey"`
 			SessionToken    string `yaml:"sessionToken"`
 			SignEndpoint    string `yaml:"signEndpoint"`
+			PublicRead      bool   `yaml:"publicRead"`
 		} `yaml:"minio"`
 		Cos struct {
 			BucketURL    string `yaml:"bucketURL"`
 			SecretID     string `yaml:"secretID"`
 			SecretKey    string `yaml:"secretKey"`
 			SessionToken string `yaml:"sessionToken"`
+			PublicRead   bool   `yaml:"publicRead"`
 		} `yaml:"cos"`
 		Oss struct {
 			Endpoint        string `yaml:"endpoint"`
@@ -142,6 +144,7 @@ type configStruct struct {
 			AccessKeyID     string `yaml:"accessKeyID"`
 			AccessKeySecret string `yaml:"accessKeySecret"`
 			SessionToken    string `yaml:"sessionToken"`
+			PublicRead      bool   `yaml:"publicRead"`
 		} `yaml:"oss"`
 	} `yaml:"object"`
 

pkg/common/db/s3/cos/cos.go

@@ -288,7 +288,7 @@ func (c *Cos) AccessURL(ctx context.Context, name string, expire time.Duration,
 				style = append(style, "format/"+opt.Image.Format)
 			}
 			if len(style) > 0 {
-				imageMogr = "&imageMogr2/thumbnail/" + strings.Join(style, "/") + "/ignore-error/1"
+				imageMogr = "imageMogr2/thumbnail/" + strings.Join(style, "/") + "/ignore-error/1"
 			}
 		}
 		if opt.ContentType != "" {
@@ -306,13 +306,23 @@ func (c *Cos) AccessURL(ctx context.Context, name string, expire time.Duration,
 	} else if expire < time.Second {
 		expire = time.Second
 	}
-	rawURL, err := c.client.Object.GetPresignedURL(ctx, http.MethodGet, name, c.credential.SecretID, c.credential.SecretKey, expire, &option)
+	rawURL, err := c.getPresignedURL(ctx, name, expire, &option)
 	if err != nil {
 		return "", err
 	}
-	urlStr := rawURL.String()
 	if imageMogr != "" {
-		urlStr += imageMogr
+		if rawURL.RawQuery == "" {
+			rawURL.RawQuery = imageMogr
+		} else {
+			rawURL.RawQuery = rawURL.RawQuery + "&" + imageMogr
+		}
+	}
+	return rawURL.String(), nil
+}
+
+func (c *Cos) getPresignedURL(ctx context.Context, name string, expire time.Duration, opt *cos.PresignedURLOptions) (*url.URL, error) {
+	if !config.Config.Object.Cos.PublicRead {
+		return c.client.Object.GetPresignedURL(ctx, http.MethodGet, name, c.credential.SecretID, c.credential.SecretKey, expire, opt)
 	}
-	return urlStr, nil
+	return c.client.Object.GetObjectURL(name), nil
 }

pkg/common/db/s3/cos/doc.go

@@ -1,15 +0,0 @@
-// Copyright © 2023 OpenIM. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package cos // import "github.com/openimsdk/open-im-server/v3/pkg/common/db/s3/cos"

pkg/common/db/s3/cos/internal.go

@@ -0,0 +1,13 @@
+package cos
+
+import (
+	"context"
+	"net/http"
+	"net/url"
+	_ "unsafe"
+
+	"github.com/tencentyun/cos-go-sdk-v5"
+)
+
+//go:linkname newRequest github.com/tencentyun/cos-go-sdk-v5.(*Client).newRequest
+func newRequest(c *cos.Client, ctx context.Context, baseURL *url.URL, uri, method string, body interface{}, optQuery interface{}, optHeader interface{}) (req *http.Request, err error)

pkg/common/db/s3/minio/doc.go

@@ -1,15 +0,0 @@
-// Copyright © 2023 OpenIM. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package minio // import "github.com/openimsdk/open-im-server/v3/pkg/common/db/s3/minio"

pkg/common/db/s3/minio/internal.go

@@ -0,0 +1,11 @@
+package minio
+
+import (
+	"net/url"
+	_ "unsafe"
+
+	"github.com/minio/minio-go/v7"
+)
+
+//go:linkname makeTargetURL github.com/minio/minio-go/v7.(*Client).makeTargetURL
+func makeTargetURL(client *minio.Client, bucketName, objectName, bucketLocation string, isVirtualHostStyle bool, queryValues url.Values) (*url.URL, error)

pkg/common/db/s3/minio/minio.go

@@ -139,6 +139,15 @@ func (m *Minio) initMinio(ctx context.Context) error {
 			return fmt.Errorf("make bucket error: %w", err)
 		}
 	}
+	if conf.PublicRead {
+		policy := fmt.Sprintf(
+			`{"Version": "2012-10-17","Statement": [{"Action": ["s3:GetObject","s3:PutObject"],"Effect": "Allow","Principal": {"AWS": ["*"]},"Resource": ["arn:aws:s3:::%s/*"],"Sid": ""}]}`,
+			conf.Bucket,
+		)
+		if err := m.core.Client.SetBucketPolicy(ctx, conf.Bucket, policy); err != nil {
+			return err
+		}
+	}
 	m.location, err = m.core.Client.GetBucketLocation(ctx, conf.Bucket)
 	if err != nil {
 		return err
@@ -375,7 +384,15 @@ func (m *Minio) presignedGetObject(ctx context.Context, name string, expire time
 	} else if expire < time.Second {
 		expire = time.Second
 	}
-	rawURL, err := m.sign.PresignedGetObject(ctx, m.bucket, name, expire, query)
+	var (
+		rawURL *url.URL
+		err    error
+	)
+	if config.Config.Object.Minio.PublicRead {
+		rawURL, err = makeTargetURL(m.sign, m.bucket, name, m.location, false, query)
+	} else {
+		rawURL, err = m.sign.PresignedGetObject(ctx, m.bucket, name, expire, query)
+	}
 	if err != nil {
 		return "", err
 	}

pkg/common/db/s3/oss/doc.go

@@ -1,15 +0,0 @@
-// Copyright © 2023 OpenIM. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package oss // import "github.com/openimsdk/open-im-server/v3/pkg/common/db/s3/oss"

pkg/common/db/s3/oss/internal.go

@@ -16,10 +16,24 @@ package oss
 
 import (
 	"net/http"
+	"net/url"
 	_ "unsafe"
 
 	"github.com/aliyun/aliyun-oss-go-sdk/oss"
 )
 
-//go:linkname ossSignHeader github.com/aliyun/aliyun-oss-go-sdk/oss.(*Conn).signHeader
+//go:linkname signHeader github.com/aliyun/aliyun-oss-go-sdk/oss.Conn.signHeader
-func ossSignHeader(c *oss.Conn, req *http.Request, canonicalizedResource string)
+func signHeader(c oss.Conn, req *http.Request, canonicalizedResource string)
+
+//go:linkname getURLParams github.com/aliyun/aliyun-oss-go-sdk/oss.Conn.getURLParams
+func getURLParams(c oss.Conn, params map[string]interface{}) string
+
+//go:linkname getURL github.com/aliyun/aliyun-oss-go-sdk/oss.urlMaker.getURL
+func getURL(um urlMaker, bucket, object, params string) *url.URL
+
+type urlMaker struct {
+	Scheme  string
+	NetLoc  string
+	Type    int
+	IsProxy bool
+}

pkg/common/db/s3/oss/oss.go

@@ -20,6 +20,7 @@ import (
 	"fmt"
 	"net/http"
 	"net/url"
+	"reflect"
 	"strconv"
 	"strings"
 	"time"
@@ -69,6 +70,7 @@ func NewOSS() (s3.Interface, error) {
 		bucketURL:   conf.BucketURL,
 		bucket:      bucket,
 		credentials: client.Config.GetCredentials(),
+		um:          *(*urlMaker)(reflect.ValueOf(bucket.Client.Conn).Elem().FieldByName("url").UnsafePointer()),
 	}, nil
 }
 
@@ -76,6 +78,7 @@ type OSS struct {
 	bucketURL   string
 	bucket      *oss.Bucket
 	credentials oss.Credentials
+	um          urlMaker
 }
 
 func (o *OSS) Engine() string {
@@ -163,7 +166,7 @@ func (o *OSS) AuthSign(ctx context.Context, uploadID string, name string, expire
 		request.Header.Set(oss.HTTPHeaderHost, request.Host)
 		request.Header.Set(oss.HTTPHeaderDate, now)
 		request.Header.Set(oss.HttpHeaderOssDate, now)
-		ossSignHeader(o.bucket.Client.Conn, request, fmt.Sprintf(`/%s/%s?partNumber=%d&uploadId=%s`, o.bucket.BucketName, name, partNumber, uploadID))
+		signHeader(*o.bucket.Client.Conn, request, fmt.Sprintf(`/%s/%s?partNumber=%d&uploadId=%s`, o.bucket.BucketName, name, partNumber, uploadID))
 		delete(request.Header, oss.HTTPHeaderDate)
 		result.Parts[i] = s3.SignPart{
 			PartNumber: partNumber,
@@ -272,6 +275,7 @@ func (o *OSS) ListUploadedParts(ctx context.Context, uploadID string, name strin
 }
 
 func (o *OSS) AccessURL(ctx context.Context, name string, expire time.Duration, opt *s3.AccessURLOption) (string, error) {
+	publicRead := config.Config.Object.Oss.PublicRead
 	var opts []oss.Option
 	if opt != nil {
 		if opt.Image != nil {
@@ -299,6 +303,7 @@ func (o *OSS) AccessURL(ctx context.Context, name string, expire time.Duration,
 			process += ",format," + format
 			opts = append(opts, oss.Process(process))
 		}
+		if !publicRead {
 			if opt.ContentType != "" {
 				opts = append(opts, oss.ResponseContentType(opt.ContentType))
 			}
@@ -306,10 +311,19 @@ func (o *OSS) AccessURL(ctx context.Context, name string, expire time.Duration,
 				opts = append(opts, oss.ResponseContentDisposition(`attachment; filename=`+strconv.Quote(opt.Filename)))
 			}
 		}
+	}
 	if expire <= 0 {
 		expire = time.Hour * 24 * 365 * 99 // 99 years
 	} else if expire < time.Second {
 		expire = time.Second
 	}
+	if !publicRead {
 		return o.bucket.SignURL(name, http.MethodGet, int64(expire/time.Second), opts...)
 	}
+	rawParams, err := oss.GetRawParams(opts)
+	if err != nil {
+		return "", err
+	}
+	params := getURLParams(*o.bucket.Client.Conn, rawParams)
+	return getURL(o.um, o.bucket.BucketName, name, params).String(), nil
+}

scripts/install/environment.sh

@@ -186,17 +186,21 @@ def "MINIO_ACCESS_KEY" "${USER}"
 def "MINIO_SECRET_KEY" "${PASSWORD}"                                              # MinIO的密钥
 def "MINIO_SESSION_TOKEN"                                                         # MinIO的会话令牌
 readonly MINIO_SIGN_ENDPOINT=${MINIO_SIGN_ENDPOINT:-"http://${IP}:${MINIO_PORT}"} # signEndpoint为minio公网地址
+def "MINIO_PUBLIC_READ" "false"                                                   # 公有读
+
 # 腾讯云COS的存储桶URL
 def "COS_BUCKET_URL" "https://temp-1252357374.cos.ap-chengdu.myqcloud.com"
 def "COS_SECRET_ID"                                                     # 腾讯云COS的密钥ID
 def "COS_SECRET_KEY"                                                    # 腾讯云COS的密钥
 def "COS_SESSION_TOKEN"                                                 # 腾讯云COS的会话令牌
+def "COS_PUBLIC_READ" "false"                                           # 公有读
 def "OSS_ENDPOINT" "https://oss-cn-chengdu.aliyuncs.com"                # 阿里云OSS的端点URL
 def "OSS_BUCKET" "demo-9999999"                                         # 阿里云OSS的存储桶名称
 def "OSS_BUCKET_URL" "https://demo-9999999.oss-cn-chengdu.aliyuncs.com" # 阿里云OSS的存储桶URL
 def "OSS_ACCESS_KEY_ID"                                                 # 阿里云OSS的访问密钥ID
 def "OSS_ACCESS_KEY_SECRET"                                             # 阿里云OSS的密钥
 def "OSS_SESSION_TOKEN"                                                 # 阿里云OSS的会话令牌
+def "OSS_PUBLIC_READ" "false"                                           # 公有读
 
 ###################### Redis 配置信息 ######################
 def "REDIS_PORT" "16379"                                    # Redis的端口