feat: s3 public read (#1080)

* fix: repeated modification session notification

* fix: repeated modification session notification

* fix: jpush return a nil pointer panic

* fix: push redis pkg

* fix: OANotification

* feat: add rpc GetConversationNeedOfflinePushUserIDs

* update pkg

* cicd: robot automated Change

* offlinePushMsg

* conversation

* conversation

* cicd: robot automated Change

* conversation

* cicd: robot automated Change

* conversation

* url 2 im s3

* url 2 im s3

* cicd: robot automated Change

* url 2 im s3

* s3 public read

* cicd: robot automated Change

* s3 public read

* cicd: robot automated Change

* s3 public read

* s3 public read

* s3 public read

* s3 public read

* s3 public read

* cicd: robot automated Change

* s3 public read

* s3 public read

* fix: SendMsg api

* config scripts

* config scripts

---------

Co-authored-by: withchao <withchao@users.noreply.github.com>
pull/1102/head
withchao 1 year ago committed by GitHub
parent bba662f404
commit 5c31d12253
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23

@ -140,11 +140,13 @@ object:
secretAccessKey: "openIM123" secretAccessKey: "openIM123"
sessionToken: '' sessionToken: ''
signEndpoint: "http://127.0.0.1:10005" signEndpoint: "http://127.0.0.1:10005"
publicRead: false
cos: cos:
bucketURL: https://temp-1252357374.cos.ap-chengdu.myqcloud.com bucketURL: https://temp-1252357374.cos.ap-chengdu.myqcloud.com
secretID: '' secretID: ''
secretKey: '' secretKey: ''
sessionToken: '' sessionToken: ''
publicRead: false
oss: oss:
endpoint: "https://oss-cn-chengdu.aliyuncs.com" endpoint: "https://oss-cn-chengdu.aliyuncs.com"
bucket: "demo-9999999" bucket: "demo-9999999"
@ -152,6 +154,7 @@ object:
accessKeyID: '' accessKeyID: ''
accessKeySecret: '' accessKeySecret: ''
sessionToken: '' sessionToken: ''
publicRead: false
###################### RPC Port Configuration ###################### ###################### RPC Port Configuration ######################

@ -140,11 +140,13 @@ object:
secretAccessKey: "${MINIO_SECRET_KEY}" secretAccessKey: "${MINIO_SECRET_KEY}"
sessionToken: ${MINIO_SESSION_TOKEN} sessionToken: ${MINIO_SESSION_TOKEN}
signEndpoint: "${MINIO_SIGN_ENDPOINT}" signEndpoint: "${MINIO_SIGN_ENDPOINT}"
publicRead: ${MINIO_PUBLIC_READ}
cos: cos:
bucketURL: ${COS_BUCKET_URL} bucketURL: ${COS_BUCKET_URL}
secretID: ${COS_SECRET_ID} secretID: ${COS_SECRET_ID}
secretKey: ${COS_SECRET_KEY} secretKey: ${COS_SECRET_KEY}
sessionToken: ${COS_SESSION_TOKEN} sessionToken: ${COS_SESSION_TOKEN}
publicRead: ${COS_PUBLIC_READ}
oss: oss:
endpoint: "${OSS_ENDPOINT}" endpoint: "${OSS_ENDPOINT}"
bucket: "${OSS_BUCKET}" bucket: "${OSS_BUCKET}"
@ -152,7 +154,7 @@ object:
accessKeyID: ${OSS_ACCESS_KEY_ID} accessKeyID: ${OSS_ACCESS_KEY_ID}
accessKeySecret: ${OSS_ACCESS_KEY_SECRET} accessKeySecret: ${OSS_ACCESS_KEY_SECRET}
sessionToken: ${OSS_SESSION_TOKEN} sessionToken: ${OSS_SESSION_TOKEN}
publicRead: ${OSS_PUBLIC_READ}
###################### RPC Port Configuration ###################### ###################### RPC Port Configuration ######################
# RPC service ports # RPC service ports

@ -58,7 +58,7 @@ func (m MessageApi) newUserSendMsgReq(c *gin.Context, params *apistruct.SendMsg)
options := make(map[string]bool, 5) options := make(map[string]bool, 5)
switch params.ContentType { switch params.ContentType {
case constant.Text: case constant.Text:
newContent = params.Content["text"].(string) fallthrough
case constant.Picture: case constant.Picture:
fallthrough fallthrough
case constant.Custom: case constant.Custom:

@ -128,12 +128,14 @@ type configStruct struct {
SecretAccessKey string `yaml:"secretAccessKey"` SecretAccessKey string `yaml:"secretAccessKey"`
SessionToken string `yaml:"sessionToken"` SessionToken string `yaml:"sessionToken"`
SignEndpoint string `yaml:"signEndpoint"` SignEndpoint string `yaml:"signEndpoint"`
PublicRead bool `yaml:"publicRead"`
} `yaml:"minio"` } `yaml:"minio"`
Cos struct { Cos struct {
BucketURL string `yaml:"bucketURL"` BucketURL string `yaml:"bucketURL"`
SecretID string `yaml:"secretID"` SecretID string `yaml:"secretID"`
SecretKey string `yaml:"secretKey"` SecretKey string `yaml:"secretKey"`
SessionToken string `yaml:"sessionToken"` SessionToken string `yaml:"sessionToken"`
PublicRead bool `yaml:"publicRead"`
} `yaml:"cos"` } `yaml:"cos"`
Oss struct { Oss struct {
Endpoint string `yaml:"endpoint"` Endpoint string `yaml:"endpoint"`
@ -142,6 +144,7 @@ type configStruct struct {
AccessKeyID string `yaml:"accessKeyID"` AccessKeyID string `yaml:"accessKeyID"`
AccessKeySecret string `yaml:"accessKeySecret"` AccessKeySecret string `yaml:"accessKeySecret"`
SessionToken string `yaml:"sessionToken"` SessionToken string `yaml:"sessionToken"`
PublicRead bool `yaml:"publicRead"`
} `yaml:"oss"` } `yaml:"oss"`
} `yaml:"object"` } `yaml:"object"`

@ -288,7 +288,7 @@ func (c *Cos) AccessURL(ctx context.Context, name string, expire time.Duration,
style = append(style, "format/"+opt.Image.Format) style = append(style, "format/"+opt.Image.Format)
} }
if len(style) > 0 { if len(style) > 0 {
imageMogr = "&imageMogr2/thumbnail/" + strings.Join(style, "/") + "/ignore-error/1" imageMogr = "imageMogr2/thumbnail/" + strings.Join(style, "/") + "/ignore-error/1"
} }
} }
if opt.ContentType != "" { if opt.ContentType != "" {
@ -306,13 +306,23 @@ func (c *Cos) AccessURL(ctx context.Context, name string, expire time.Duration,
} else if expire < time.Second { } else if expire < time.Second {
expire = time.Second expire = time.Second
} }
rawURL, err := c.client.Object.GetPresignedURL(ctx, http.MethodGet, name, c.credential.SecretID, c.credential.SecretKey, expire, &option) rawURL, err := c.getPresignedURL(ctx, name, expire, &option)
if err != nil { if err != nil {
return "", err return "", err
} }
urlStr := rawURL.String()
if imageMogr != "" { if imageMogr != "" {
urlStr += imageMogr if rawURL.RawQuery == "" {
rawURL.RawQuery = imageMogr
} else {
rawURL.RawQuery = rawURL.RawQuery + "&" + imageMogr
}
}
return rawURL.String(), nil
}
func (c *Cos) getPresignedURL(ctx context.Context, name string, expire time.Duration, opt *cos.PresignedURLOptions) (*url.URL, error) {
if !config.Config.Object.Cos.PublicRead {
return c.client.Object.GetPresignedURL(ctx, http.MethodGet, name, c.credential.SecretID, c.credential.SecretKey, expire, opt)
} }
return urlStr, nil return c.client.Object.GetObjectURL(name), nil
} }

@ -1,15 +0,0 @@
// Copyright © 2023 OpenIM. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package cos // import "github.com/openimsdk/open-im-server/v3/pkg/common/db/s3/cos"

@ -0,0 +1,13 @@
package cos
import (
"context"
"net/http"
"net/url"
_ "unsafe"
"github.com/tencentyun/cos-go-sdk-v5"
)
//go:linkname newRequest github.com/tencentyun/cos-go-sdk-v5.(*Client).newRequest
func newRequest(c *cos.Client, ctx context.Context, baseURL *url.URL, uri, method string, body interface{}, optQuery interface{}, optHeader interface{}) (req *http.Request, err error)

@ -1,15 +0,0 @@
// Copyright © 2023 OpenIM. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package minio // import "github.com/openimsdk/open-im-server/v3/pkg/common/db/s3/minio"

@ -0,0 +1,11 @@
package minio
import (
"net/url"
_ "unsafe"
"github.com/minio/minio-go/v7"
)
//go:linkname makeTargetURL github.com/minio/minio-go/v7.(*Client).makeTargetURL
func makeTargetURL(client *minio.Client, bucketName, objectName, bucketLocation string, isVirtualHostStyle bool, queryValues url.Values) (*url.URL, error)

@ -139,6 +139,15 @@ func (m *Minio) initMinio(ctx context.Context) error {
return fmt.Errorf("make bucket error: %w", err) return fmt.Errorf("make bucket error: %w", err)
} }
} }
if conf.PublicRead {
policy := fmt.Sprintf(
`{"Version": "2012-10-17","Statement": [{"Action": ["s3:GetObject","s3:PutObject"],"Effect": "Allow","Principal": {"AWS": ["*"]},"Resource": ["arn:aws:s3:::%s/*"],"Sid": ""}]}`,
conf.Bucket,
)
if err := m.core.Client.SetBucketPolicy(ctx, conf.Bucket, policy); err != nil {
return err
}
}
m.location, err = m.core.Client.GetBucketLocation(ctx, conf.Bucket) m.location, err = m.core.Client.GetBucketLocation(ctx, conf.Bucket)
if err != nil { if err != nil {
return err return err
@ -375,7 +384,15 @@ func (m *Minio) presignedGetObject(ctx context.Context, name string, expire time
} else if expire < time.Second { } else if expire < time.Second {
expire = time.Second expire = time.Second
} }
rawURL, err := m.sign.PresignedGetObject(ctx, m.bucket, name, expire, query) var (
rawURL *url.URL
err error
)
if config.Config.Object.Minio.PublicRead {
rawURL, err = makeTargetURL(m.sign, m.bucket, name, m.location, false, query)
} else {
rawURL, err = m.sign.PresignedGetObject(ctx, m.bucket, name, expire, query)
}
if err != nil { if err != nil {
return "", err return "", err
} }

@ -1,15 +0,0 @@
// Copyright © 2023 OpenIM. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package oss // import "github.com/openimsdk/open-im-server/v3/pkg/common/db/s3/oss"

@ -16,10 +16,24 @@ package oss
import ( import (
"net/http" "net/http"
"net/url"
_ "unsafe" _ "unsafe"
"github.com/aliyun/aliyun-oss-go-sdk/oss" "github.com/aliyun/aliyun-oss-go-sdk/oss"
) )
//go:linkname ossSignHeader github.com/aliyun/aliyun-oss-go-sdk/oss.(*Conn).signHeader //go:linkname signHeader github.com/aliyun/aliyun-oss-go-sdk/oss.Conn.signHeader
func ossSignHeader(c *oss.Conn, req *http.Request, canonicalizedResource string) func signHeader(c oss.Conn, req *http.Request, canonicalizedResource string)
//go:linkname getURLParams github.com/aliyun/aliyun-oss-go-sdk/oss.Conn.getURLParams
func getURLParams(c oss.Conn, params map[string]interface{}) string
//go:linkname getURL github.com/aliyun/aliyun-oss-go-sdk/oss.urlMaker.getURL
func getURL(um urlMaker, bucket, object, params string) *url.URL
type urlMaker struct {
Scheme string
NetLoc string
Type int
IsProxy bool
}

@ -20,6 +20,7 @@ import (
"fmt" "fmt"
"net/http" "net/http"
"net/url" "net/url"
"reflect"
"strconv" "strconv"
"strings" "strings"
"time" "time"
@ -69,6 +70,7 @@ func NewOSS() (s3.Interface, error) {
bucketURL: conf.BucketURL, bucketURL: conf.BucketURL,
bucket: bucket, bucket: bucket,
credentials: client.Config.GetCredentials(), credentials: client.Config.GetCredentials(),
um: *(*urlMaker)(reflect.ValueOf(bucket.Client.Conn).Elem().FieldByName("url").UnsafePointer()),
}, nil }, nil
} }
@ -76,6 +78,7 @@ type OSS struct {
bucketURL string bucketURL string
bucket *oss.Bucket bucket *oss.Bucket
credentials oss.Credentials credentials oss.Credentials
um urlMaker
} }
func (o *OSS) Engine() string { func (o *OSS) Engine() string {
@ -163,7 +166,7 @@ func (o *OSS) AuthSign(ctx context.Context, uploadID string, name string, expire
request.Header.Set(oss.HTTPHeaderHost, request.Host) request.Header.Set(oss.HTTPHeaderHost, request.Host)
request.Header.Set(oss.HTTPHeaderDate, now) request.Header.Set(oss.HTTPHeaderDate, now)
request.Header.Set(oss.HttpHeaderOssDate, now) request.Header.Set(oss.HttpHeaderOssDate, now)
ossSignHeader(o.bucket.Client.Conn, request, fmt.Sprintf(`/%s/%s?partNumber=%d&uploadId=%s`, o.bucket.BucketName, name, partNumber, uploadID)) signHeader(*o.bucket.Client.Conn, request, fmt.Sprintf(`/%s/%s?partNumber=%d&uploadId=%s`, o.bucket.BucketName, name, partNumber, uploadID))
delete(request.Header, oss.HTTPHeaderDate) delete(request.Header, oss.HTTPHeaderDate)
result.Parts[i] = s3.SignPart{ result.Parts[i] = s3.SignPart{
PartNumber: partNumber, PartNumber: partNumber,
@ -272,6 +275,7 @@ func (o *OSS) ListUploadedParts(ctx context.Context, uploadID string, name strin
} }
func (o *OSS) AccessURL(ctx context.Context, name string, expire time.Duration, opt *s3.AccessURLOption) (string, error) { func (o *OSS) AccessURL(ctx context.Context, name string, expire time.Duration, opt *s3.AccessURLOption) (string, error) {
publicRead := config.Config.Object.Oss.PublicRead
var opts []oss.Option var opts []oss.Option
if opt != nil { if opt != nil {
if opt.Image != nil { if opt.Image != nil {
@ -299,11 +303,13 @@ func (o *OSS) AccessURL(ctx context.Context, name string, expire time.Duration,
process += ",format," + format process += ",format," + format
opts = append(opts, oss.Process(process)) opts = append(opts, oss.Process(process))
} }
if opt.ContentType != "" { if !publicRead {
opts = append(opts, oss.ResponseContentType(opt.ContentType)) if opt.ContentType != "" {
} opts = append(opts, oss.ResponseContentType(opt.ContentType))
if opt.Filename != "" { }
opts = append(opts, oss.ResponseContentDisposition(`attachment; filename=`+strconv.Quote(opt.Filename))) if opt.Filename != "" {
opts = append(opts, oss.ResponseContentDisposition(`attachment; filename=`+strconv.Quote(opt.Filename)))
}
} }
} }
if expire <= 0 { if expire <= 0 {
@ -311,5 +317,13 @@ func (o *OSS) AccessURL(ctx context.Context, name string, expire time.Duration,
} else if expire < time.Second { } else if expire < time.Second {
expire = time.Second expire = time.Second
} }
return o.bucket.SignURL(name, http.MethodGet, int64(expire/time.Second), opts...) if !publicRead {
return o.bucket.SignURL(name, http.MethodGet, int64(expire/time.Second), opts...)
}
rawParams, err := oss.GetRawParams(opts)
if err != nil {
return "", err
}
params := getURLParams(*o.bucket.Client.Conn, rawParams)
return getURL(o.um, o.bucket.BucketName, name, params).String(), nil
} }

@ -186,17 +186,21 @@ def "MINIO_ACCESS_KEY" "${USER}"
def "MINIO_SECRET_KEY" "${PASSWORD}" # MinIO的密钥 def "MINIO_SECRET_KEY" "${PASSWORD}" # MinIO的密钥
def "MINIO_SESSION_TOKEN" # MinIO的会话令牌 def "MINIO_SESSION_TOKEN" # MinIO的会话令牌
readonly MINIO_SIGN_ENDPOINT=${MINIO_SIGN_ENDPOINT:-"http://${IP}:${MINIO_PORT}"} # signEndpoint为minio公网地址 readonly MINIO_SIGN_ENDPOINT=${MINIO_SIGN_ENDPOINT:-"http://${IP}:${MINIO_PORT}"} # signEndpoint为minio公网地址
def "MINIO_PUBLIC_READ" "false" # 公有读
# 腾讯云COS的存储桶URL # 腾讯云COS的存储桶URL
def "COS_BUCKET_URL" "https://temp-1252357374.cos.ap-chengdu.myqcloud.com" def "COS_BUCKET_URL" "https://temp-1252357374.cos.ap-chengdu.myqcloud.com"
def "COS_SECRET_ID" # 腾讯云COS的密钥ID def "COS_SECRET_ID" # 腾讯云COS的密钥ID
def "COS_SECRET_KEY" # 腾讯云COS的密钥 def "COS_SECRET_KEY" # 腾讯云COS的密钥
def "COS_SESSION_TOKEN" # 腾讯云COS的会话令牌 def "COS_SESSION_TOKEN" # 腾讯云COS的会话令牌
def "COS_PUBLIC_READ" "false" # 公有读
def "OSS_ENDPOINT" "https://oss-cn-chengdu.aliyuncs.com" # 阿里云OSS的端点URL def "OSS_ENDPOINT" "https://oss-cn-chengdu.aliyuncs.com" # 阿里云OSS的端点URL
def "OSS_BUCKET" "demo-9999999" # 阿里云OSS的存储桶名称 def "OSS_BUCKET" "demo-9999999" # 阿里云OSS的存储桶名称
def "OSS_BUCKET_URL" "https://demo-9999999.oss-cn-chengdu.aliyuncs.com" # 阿里云OSS的存储桶URL def "OSS_BUCKET_URL" "https://demo-9999999.oss-cn-chengdu.aliyuncs.com" # 阿里云OSS的存储桶URL
def "OSS_ACCESS_KEY_ID" # 阿里云OSS的访问密钥ID def "OSS_ACCESS_KEY_ID" # 阿里云OSS的访问密钥ID
def "OSS_ACCESS_KEY_SECRET" # 阿里云OSS的密钥 def "OSS_ACCESS_KEY_SECRET" # 阿里云OSS的密钥
def "OSS_SESSION_TOKEN" # 阿里云OSS的会话令牌 def "OSS_SESSION_TOKEN" # 阿里云OSS的会话令牌
def "OSS_PUBLIC_READ" "false" # 公有读
###################### Redis 配置信息 ###################### ###################### Redis 配置信息 ######################
def "REDIS_PORT" "16379" # Redis的端口 def "REDIS_PORT" "16379" # Redis的端口

Loading…
Cancel
Save