diff --git a/config/config.yaml b/config/config.yaml index 4127b2f76..ec813d69c 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -24,10 +24,10 @@ # Zookeeper username # Zookeeper password zookeeper: - schema: openim - address: [ 127.0.0.1:2181 ] - username: - password: + schema: openim + address: [ 127.0.0.1:2181 ] + username: + password: ###################### Mysql ###################### # MySQL configuration @@ -42,12 +42,12 @@ mysql: address: [ 127.0.0.1:13306 ] username: root password: openIM123 - database: openIM_v3 - maxOpenConn: 1000 - maxIdleConn: 100 - maxLifeTime: 60 - logLevel: 4 - slowThreshold: 500 + database: openIM_v3 + maxOpenConn: 1000 + maxIdleConn: 100 + maxLifeTime: 60 + logLevel: 4 + slowThreshold: 500 ###################### Mongo ###################### # MongoDB configuration @@ -62,7 +62,7 @@ mongo: database: openIM_v3 username: root password: openIM123 - maxPoolSize: 100 + maxPoolSize: 100 ###################### Redis ###################### # Redis configuration @@ -70,7 +70,7 @@ mongo: # Username is required only for Redis version 6.0+ redis: address: [ 127.0.0.1:16379 ] - username: + username: password: openIM123 ###################### Kafka ###################### @@ -81,13 +81,13 @@ redis: # It's not recommended to modify this topic name # Consumer group ID, it's not recommended to modify kafka: - username: - password: + username: + password: addr: [ 127.0.0.1:9092 ] latestMsgToRedis: - topic: "latestMsgToRedis" + topic: "latestMsgToRedis" offlineMsgToMongo: - topic: "offlineMsgToMongoMysql" + topic: "offlineMsgToMongoMysql" msgToPush: topic: "msgToPush" consumerGroupID: @@ -111,8 +111,8 @@ rpc: # API service port # Default listen IP is 0.0.0.0 api: - openImApiPort: [ 10002 ] - listenIP: 0.0.0.0 + openImApiPort: [ 10002 ] + listenIP: 0.0.0.0 ###################### Gateway ###################### # Object storage configuration @@ -125,21 +125,22 @@ api: # Configuration for Tencent COS # Configuration for Aliyun OSS object: - enable: "minio" + enable: "minio" apiURL: http://127.0.0.1:10002 minio: - bucket: "openim" + bucket: "openim" endpoint: http://127.0.0.1:10005 accessKeyID: root secretAccessKey: openIM123 sessionToken: "" + signEndpoint: "http://127.0.0.1:10005" thumbnailApi: "http://127.0.0.1:10003" - cos: + cos: bucketURL: "https://temp-1252357374.cos.ap-chengdu.myqcloud.com" secretID: "" secretKey: "" sessionToken: "" - oss: + oss: endpoint: "https://oss-cn-chengdu.aliyuncs.com" bucket: "demo-9999999" bucketURL: "https://demo-9999999.oss-cn-chengdu.aliyuncs.com" @@ -151,7 +152,7 @@ object: # These ports are passed into the program by the script and are not recommended to modify # For launching multiple programs, just fill in multiple ports separated by commas # For example, [10110, 10111] -rpcPort: +rpcPort: openImUserPort: [ 10110 ] openImFriendPort: [ 10120 ] openImMessagePort: [ 10130 ] @@ -184,12 +185,12 @@ rpcRegisterName: # Whether to output in json format # Whether to include stack trace in logs log: - storageLocation: ../../../../../logs/ - rotationTime: 24 - remainRotationCount: 2 - remainLogLevel: 6 - isStdout: false - isJson: false + storageLocation: ../../../../../logs/ + rotationTime: 24 + remainRotationCount: 2 + remainLogLevel: 6 + isStdout: false + isJson: false withStack: false # Long connection server configuration @@ -199,10 +200,10 @@ log: # Maximum length of websocket request package # Websocket connection handshake timeout longConnSvr: - openImWsPort: [ 10001 ] - websocketMaxConnNum: 100000 - websocketMaxMsgLen: 4096 - websocketTimeout: 10 + openImWsPort: [ 10001 ] + websocketMaxConnNum: 100000 + websocketMaxMsgLen: 4096 + websocketTimeout: 10 # Push notification service configuration # diff --git a/pkg/common/config/config.go b/pkg/common/config/config.go index e6b6cde0f..da7942cf8 100644 --- a/pkg/common/config/config.go +++ b/pkg/common/config/config.go @@ -120,6 +120,7 @@ type configStruct struct { SecretAccessKey string `yaml:"secretAccessKey"` SessionToken string `yaml:"sessionToken"` ThumbnailApi string `yaml:"thumbnailApi"` + SignEndpoint string `yaml:"signEndpoint"` } `yaml:"minio"` Cos struct { BucketURL string `yaml:"bucketURL"` diff --git a/pkg/common/db/s3/minio/minio.go b/pkg/common/db/s3/minio/minio.go index dbc0d502e..5773c5d04 100644 --- a/pkg/common/db/s3/minio/minio.go +++ b/pkg/common/db/s3/minio/minio.go @@ -19,15 +19,18 @@ import ( "errors" "fmt" "github.com/OpenIMSDK/tools/errs" + "github.com/OpenIMSDK/tools/log" "github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7/pkg/credentials" "github.com/minio/minio-go/v7/pkg/signer" "net/http" "net/url" + "reflect" "strconv" "strings" "sync" "time" + "unsafe" "github.com/OpenIMSDK/Open-IM-Server/pkg/common/config" "github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/s3" @@ -68,11 +71,26 @@ func NewMinio() (s3.Interface, error) { bucket: conf.Bucket, bucketURL: conf.Endpoint + "/" + conf.Bucket + "/", imageApi: imageApi, - opts: opts, core: &minio.Core{Client: client}, lock: &sync.Mutex{}, init: false, } + if conf.SignEndpoint == "" { + m.sign = m.core.Client + } else { + su, err := url.Parse(conf.SignEndpoint) + if err != nil { + return nil, err + } + m.opts = &minio.Options{ + Creds: credentials.NewStaticV4(conf.AccessKeyID, conf.SecretAccessKey, conf.SessionToken), + Secure: su.Scheme == "https", + } + m.sign, err = minio.New(su.Host, m.opts) + if err != nil { + return nil, err + } + } ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() if err := m.initMinio(ctx); err != nil { @@ -85,8 +103,10 @@ type Minio struct { bucket string bucketURL string imageApi string + location string opts *minio.Options core *minio.Core + sign *minio.Client lock sync.Locker init bool } @@ -100,15 +120,34 @@ func (m *Minio) initMinio(ctx context.Context) error { if m.init { return nil } - exists, err := m.core.Client.BucketExists(ctx, config.Config.Object.Minio.Bucket) + conf := config.Config.Object.Minio + exists, err := m.core.Client.BucketExists(ctx, conf.Bucket) if err != nil { return fmt.Errorf("check bucket exists error: %w", err) } if !exists { - if err := m.core.Client.MakeBucket(ctx, config.Config.Object.Minio.Bucket, minio.MakeBucketOptions{}); err != nil { + if err := m.core.Client.MakeBucket(ctx, conf.Bucket, minio.MakeBucketOptions{}); err != nil { return fmt.Errorf("make bucket error: %w", err) } } + m.location, err = m.core.Client.GetBucketLocation(ctx, conf.Bucket) + if err != nil { + return err + } + func() { + if conf.SignEndpoint == "" { + return + } + defer func() { + if r := recover(); r != nil { + log.ZWarn(context.Background(), "set sign bucket location cache panic", errors.New("failed to get private field value"), "recover", fmt.Sprintf("%+v", r), "development version", "github.com/minio/minio-go/v7 v7.0.61") + } + }() + filed := reflect.ValueOf(m.sign).Elem().FieldByName("bucketLocCache") + zero := reflect.New(reflect.PtrTo(filed.Type())) + *(*unsafe.Pointer)(zero.UnsafePointer()) = unsafe.Pointer(filed.UnsafeAddr()) + zero.Elem().Elem().Interface().(interface{ Set(string, string) }).Set(conf.Bucket, m.location) + }() m.init = true return nil } @@ -200,7 +239,7 @@ func (m *Minio) AuthSign(ctx context.Context, uploadID string, name string, expi return nil, err } request.Header.Set("X-Amz-Content-Sha256", unsignedPayload) - request = signer.SignV4Trailer(*request, creds.AccessKeyID, creds.SecretAccessKey, creds.SessionToken, "us-east-1", nil) + request = signer.SignV4Trailer(*request, creds.AccessKeyID, creds.SecretAccessKey, creds.SessionToken, m.location, nil) result.Parts[i] = s3.SignPart{ PartNumber: partNumber, URL: request.URL.String(), @@ -215,7 +254,7 @@ func (m *Minio) PresignedPutObject(ctx context.Context, name string, expire time if err := m.initMinio(ctx); err != nil { return "", err } - rawURL, err := m.core.Client.PresignedPutObject(ctx, m.bucket, name, expire) + rawURL, err := m.sign.PresignedPutObject(ctx, m.bucket, name, expire) if err != nil { return "", err } @@ -330,7 +369,7 @@ func (m *Minio) AccessURL(ctx context.Context, name string, expire time.Duration } else if expire < time.Second { expire = time.Second } - u, err := m.core.Client.PresignedGetObject(ctx, m.bucket, name, expire, reqParams) + u, err := m.sign.PresignedGetObject(ctx, m.bucket, name, expire, reqParams) if err != nil { return "", err }