From c25352b6609e04db97ec64e442467946e75bf52c Mon Sep 17 00:00:00 2001 From: wangchuxiao Date: Fri, 29 Jul 2022 20:49:27 +0800 Subject: [PATCH 01/11] fix bug --- internal/rpc/group/super_group.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/internal/rpc/group/super_group.go b/internal/rpc/group/super_group.go index 9b50c7955..b667abe76 100644 --- a/internal/rpc/group/super_group.go +++ b/internal/rpc/group/super_group.go @@ -1,7 +1,6 @@ package group import ( - "Open_IM/pkg/common/constant" rocksCache "Open_IM/pkg/common/db/rocks_cache" "Open_IM/pkg/common/log" cp "Open_IM/pkg/common/utils" @@ -15,7 +14,6 @@ import ( func (s *groupServer) GetJoinedSuperGroupList(ctx context.Context, req *pbGroup.GetJoinedSuperGroupListReq) (*pbGroup.GetJoinedSuperGroupListResp, error) { log.NewInfo(req.OperationID, utils.GetSelfFuncName(), "req: ", req.String()) resp := &pbGroup.GetJoinedSuperGroupListResp{CommonResp: &pbGroup.CommonResp{}} - //userToSuperGroup, err := db.DB.GetSuperGroupByUserID(req.UserID) groupIDList, err := rocksCache.GetJoinedSuperGroupListFromCache(req.UserID) if err != nil { if err == redis.Nil { @@ -23,8 +21,8 @@ func (s *groupServer) GetJoinedSuperGroupList(ctx context.Context, req *pbGroup. return resp, nil } log.NewError(req.OperationID, utils.GetSelfFuncName(), "GetSuperGroupByUserID failed ", err.Error(), req.UserID) - resp.CommonResp.ErrCode = constant.ErrDB.ErrCode - resp.CommonResp.ErrMsg = constant.ErrDB.ErrMsg + //resp.CommonResp.ErrCode = constant.ErrDB.ErrCode + //resp.CommonResp.ErrMsg = constant.ErrDB.ErrMsg return resp, nil } for _, groupID := range groupIDList { From 0a8a24834960f5e1f7447ca51492e389fef95743 Mon Sep 17 00:00:00 2001 From: wangchuxiao Date: Fri, 29 Jul 2022 21:53:53 +0800 Subject: [PATCH 02/11] fix bug --- pkg/common/db/rocks_cache/rocks_cache.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pkg/common/db/rocks_cache/rocks_cache.go b/pkg/common/db/rocks_cache/rocks_cache.go index 86ce3dea6..033731e8d 100644 --- a/pkg/common/db/rocks_cache/rocks_cache.go +++ b/pkg/common/db/rocks_cache/rocks_cache.go @@ -8,6 +8,7 @@ import ( "Open_IM/pkg/utils" "context" "encoding/json" + "errors" "fmt" "time" ) @@ -323,6 +324,9 @@ func GetJoinedSuperGroupListFromCache(userID string) ([]string, error) { if err != nil { return "", utils.Wrap(err, "") } + if len(userToSuperGroup.GroupIDList) == 0 { + return "", errors.New("GroupIDList == 0") + } bytes, err := json.Marshal(userToSuperGroup.GroupIDList) return string(bytes), utils.Wrap(err, "") } From c39df537b40de8ee9cd48a4f95599e4cc5094db2 Mon Sep 17 00:00:00 2001 From: wangchuxiao Date: Mon, 1 Aug 2022 10:58:29 +0800 Subject: [PATCH 03/11] minio obj locking --- internal/api/third/minio_init.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/api/third/minio_init.go b/internal/api/third/minio_init.go index 393122858..1511bca46 100644 --- a/internal/api/third/minio_init.go +++ b/internal/api/third/minio_init.go @@ -47,7 +47,7 @@ func MinioInit() { } opt := minio.MakeBucketOptions{ Region: config.Config.Credential.Minio.Location, - ObjectLocking: false, + ObjectLocking: true, } err = MinioClient.MakeBucket(context.Background(), config.Config.Credential.Minio.Bucket, opt) if err != nil { From 4930906b621906f24c5c597609de684c0193ccdf Mon Sep 17 00:00:00 2001 From: wangchuxiao Date: Mon, 1 Aug 2022 11:47:24 +0800 Subject: [PATCH 04/11] fix bug --- internal/api/third/minio_init.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/internal/api/third/minio_init.go b/internal/api/third/minio_init.go index 1511bca46..398072c6c 100644 --- a/internal/api/third/minio_init.go +++ b/internal/api/third/minio_init.go @@ -31,7 +31,8 @@ func MinioInit() { return } opts := &minio.Options{ - Creds: credentials.NewStaticV4(config.Config.Credential.Minio.AccessKeyID, config.Config.Credential.Minio.SecretAccessKey, ""), + Creds: credentials.NewStaticV4(config.Config.Credential.Minio.AccessKeyID, config.Config.Credential.Minio.SecretAccessKey, ""), + Region: config.Config.Credential.Minio.Location, } if minioUrl.Scheme == "http" { opts.Secure = false From 6f968342dc3d2502665f2ea27a18c37310fa6c9c Mon Sep 17 00:00:00 2001 From: wangchuxiao Date: Mon, 1 Aug 2022 11:49:50 +0800 Subject: [PATCH 05/11] fix bug --- internal/api/third/minio_init.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/internal/api/third/minio_init.go b/internal/api/third/minio_init.go index 398072c6c..8c1038e86 100644 --- a/internal/api/third/minio_init.go +++ b/internal/api/third/minio_init.go @@ -31,8 +31,8 @@ func MinioInit() { return } opts := &minio.Options{ - Creds: credentials.NewStaticV4(config.Config.Credential.Minio.AccessKeyID, config.Config.Credential.Minio.SecretAccessKey, ""), - Region: config.Config.Credential.Minio.Location, + Creds: credentials.NewStaticV4(config.Config.Credential.Minio.AccessKeyID, config.Config.Credential.Minio.SecretAccessKey, ""), + //Region: config.Config.Credential.Minio.Location, } if minioUrl.Scheme == "http" { opts.Secure = false @@ -48,7 +48,7 @@ func MinioInit() { } opt := minio.MakeBucketOptions{ Region: config.Config.Credential.Minio.Location, - ObjectLocking: true, + ObjectLocking: false, } err = MinioClient.MakeBucket(context.Background(), config.Config.Credential.Minio.Bucket, opt) if err != nil { From ef9c291d9a1e30aeefa8f52f689e46f60e2dc00c Mon Sep 17 00:00:00 2001 From: wangchuxiao Date: Mon, 1 Aug 2022 11:52:43 +0800 Subject: [PATCH 06/11] fix bug --- internal/rpc/group/group.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/internal/rpc/group/group.go b/internal/rpc/group/group.go index 1f728d6d7..a36da065b 100644 --- a/internal/rpc/group/group.go +++ b/internal/rpc/group/group.go @@ -61,7 +61,13 @@ func (s *groupServer) Run() { log.NewInfo("", "listen network success, ", address, listener) defer listener.Close() //grpc server - srv := grpc.NewServer() + recvSize := 1024 * 1024 * 30 + sendSize := 1024 * 1024 * 30 + var options = []grpc.ServerOption{ + grpc.MaxRecvMsgSize(recvSize), + grpc.MaxSendMsgSize(sendSize), + } + srv := grpc.NewServer(options...) defer srv.GracefulStop() //Service registers with etcd pbGroup.RegisterGroupServer(srv, s) From 9136838b63ddca0ccd0e59d9fa2113761351e5ac Mon Sep 17 00:00:00 2001 From: wangchuxiao Date: Mon, 1 Aug 2022 14:33:01 +0800 Subject: [PATCH 07/11] fix bug --- internal/api/third/minio_init.go | 6 ++++-- internal/api/third/minio_storage_credential.go | 1 + pkg/base_info/third_api_struct.go | 13 +++++++------ pkg/common/config/config.go | 1 + 4 files changed, 13 insertions(+), 8 deletions(-) diff --git a/internal/api/third/minio_init.go b/internal/api/third/minio_init.go index 8c1038e86..4580a6304 100644 --- a/internal/api/third/minio_init.go +++ b/internal/api/third/minio_init.go @@ -47,8 +47,10 @@ func MinioInit() { return } opt := minio.MakeBucketOptions{ - Region: config.Config.Credential.Minio.Location, - ObjectLocking: false, + Region: config.Config.Credential.Minio.Location, + } + if config.Config.Credential.Minio.IsDistributedMod == true { + opt.ObjectLocking = true } err = MinioClient.MakeBucket(context.Background(), config.Config.Credential.Minio.Bucket, opt) if err != nil { diff --git a/internal/api/third/minio_storage_credential.go b/internal/api/third/minio_storage_credential.go index 7f6c81638..4c6fad1f2 100644 --- a/internal/api/third/minio_storage_credential.go +++ b/internal/api/third/minio_storage_credential.go @@ -161,6 +161,7 @@ func MinioStorageCredential(c *gin.Context) { resp.BucketName = config.Config.Credential.Minio.Bucket resp.StsEndpointURL = config.Config.Credential.Minio.Endpoint resp.StorageTime = config.Config.Credential.Minio.StorageTime + resp.IsDistributedMod = config.Config.Credential.Minio.IsDistributedMod c.JSON(http.StatusOK, gin.H{"errCode": 0, "errMsg": "", "data": resp}) } diff --git a/pkg/base_info/third_api_struct.go b/pkg/base_info/third_api_struct.go index 8785ecd37..ddf9291ec 100644 --- a/pkg/base_info/third_api_struct.go +++ b/pkg/base_info/third_api_struct.go @@ -7,12 +7,13 @@ type MinioStorageCredentialReq struct { } type MiniostorageCredentialResp struct { - SecretAccessKey string `json:"secretAccessKey"` - AccessKeyID string `json:"accessKeyID"` - SessionToken string `json:"sessionToken"` - BucketName string `json:"bucketName"` - StsEndpointURL string `json:"stsEndpointURL"` - StorageTime int `json:"storageTime"` + SecretAccessKey string `json:"secretAccessKey"` + AccessKeyID string `json:"accessKeyID"` + SessionToken string `json:"sessionToken"` + BucketName string `json:"bucketName"` + StsEndpointURL string `json:"stsEndpointURL"` + StorageTime int `json:"storageTime"` + IsDistributedMod bool `json:"isDistributedMod"` } type MinioUploadFileReq struct { diff --git a/pkg/common/config/config.go b/pkg/common/config/config.go index 6c0e1a499..dc73cf25a 100644 --- a/pkg/common/config/config.go +++ b/pkg/common/config/config.go @@ -72,6 +72,7 @@ type config struct { EndpointInner string `yaml:"endpointInner"` EndpointInnerEnable bool `yaml:"endpointInnerEnable"` StorageTime int `yaml:"storageTime"` + IsDistributedMod bool `yaml:"isDistributedMod"` } `yaml:"minio"` } From a8a91fe945dd4f9f82ff36a4f7afec26aa2c0531 Mon Sep 17 00:00:00 2001 From: wangchuxiao Date: Mon, 1 Aug 2022 14:33:50 +0800 Subject: [PATCH 08/11] fix bug --- config/config.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/config/config.yaml b/config/config.yaml index 680747d01..30e0b8b4f 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -119,6 +119,7 @@ credential: #腾讯cos，发送图片、视频、文件时需要，请自行申 accessKeyID: user12345 secretAccessKey: key12345 storageTime: 50 #文件在minio中保存的时间 + isDistributedMod: false # 是否分布式多硬盘部署 默认docker-compose中为false ali: # ali oss regionID: "oss-cn-beijing" accessKeyID: "" From e1a941b99823bcb1e16f06921ec6db7d96658458 Mon Sep 17 00:00:00 2001 From: wangchuxiao Date: Mon, 1 Aug 2022 14:44:29 +0800 Subject: [PATCH 09/11] fix bug --- internal/api/third/minio_init.go | 27 +++++++++++++-------------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/internal/api/third/minio_init.go b/internal/api/third/minio_init.go index 4580a6304..424208038 100644 --- a/internal/api/third/minio_init.go +++ b/internal/api/third/minio_init.go @@ -5,7 +5,6 @@ import ( "Open_IM/pkg/common/log" "Open_IM/pkg/utils" "context" - "fmt" "github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7/pkg/credentials" url2 "net/url" @@ -82,17 +81,17 @@ func MinioInit() { } } // 自动化桶public的代码 - policyJsonString := fmt.Sprintf(`{"Version": "2012-10-17","Statement": [{"Action": ["s3:GetObject","s3:PutObject"], - "Effect": "Allow","Principal": {"AWS": ["*"]},"Resource": ["arn:aws:s3:::%s/*"],"Sid": ""}]}`, config.Config.Credential.Minio.Bucket) - err = MinioClient.SetBucketPolicy(context.Background(), config.Config.Credential.Minio.Bucket, policyJsonString) - if err != nil { - log.NewInfo("", utils.GetSelfFuncName(), "SetBucketPolicy failed please set in web", err.Error()) - } - policyJsonString = fmt.Sprintf(`{"Version": "2012-10-17","Statement": [{"Action": ["s3:GetObject","s3:PutObject"], - "Effect": "Allow","Principal": {"AWS": ["*"]},"Resource": ["arn:aws:s3:::%s/*"],"Sid": ""}]}`, config.Config.Credential.Minio.AppBucket) - err = MinioClient.SetBucketPolicy(context.Background(), config.Config.Credential.Minio.AppBucket, policyJsonString) - if err != nil { - log.NewInfo("", utils.GetSelfFuncName(), "SetBucketPolicy failed please set in web", err.Error()) - } - log.NewInfo(operationID, utils.GetSelfFuncName(), "minio create and set policy success") + //policyJsonString := fmt.Sprintf(`{"Version": "2012-10-17","Statement": [{"Action": ["s3:GetObject","s3:PutObject"], + // "Effect": "Allow","Principal": {"AWS": ["*"]},"Resource": ["arn:aws:s3:::%s/*"],"Sid": ""}]}`, config.Config.Credential.Minio.Bucket) + //err = MinioClient.SetBucketPolicy(context.Background(), config.Config.Credential.Minio.Bucket, policyJsonString) + //if err != nil { + // log.NewInfo("", utils.GetSelfFuncName(), "SetBucketPolicy failed please set in web", err.Error()) + //} + //policyJsonString = fmt.Sprintf(`{"Version": "2012-10-17","Statement": [{"Action": ["s3:GetObject","s3:PutObject"], + // "Effect": "Allow","Principal": {"AWS": ["*"]},"Resource": ["arn:aws:s3:::%s/*"],"Sid": ""}]}`, config.Config.Credential.Minio.AppBucket) + //err = MinioClient.SetBucketPolicy(context.Background(), config.Config.Credential.Minio.AppBucket, policyJsonString) + //if err != nil { + // log.NewInfo("", utils.GetSelfFuncName(), "SetBucketPolicy failed please set in web", err.Error()) + //} + //log.NewInfo(operationID, utils.GetSelfFuncName(), "minio create and set policy success") } From 4c610abd44f25bdd33077eaf9463cb6a4c428600 Mon Sep 17 00:00:00 2001 From: wangchuxiao Date: Mon, 1 Aug 2022 14:56:20 +0800 Subject: [PATCH 10/11] fix bug --- internal/api/third/minio_init.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/internal/api/third/minio_init.go b/internal/api/third/minio_init.go index 424208038..04cc53bc1 100644 --- a/internal/api/third/minio_init.go +++ b/internal/api/third/minio_init.go @@ -80,6 +80,9 @@ func MinioInit() { return } } + policy, err := MinioClient.GetBucketPolicy(context.Background(), config.Config.Credential.Minio.Bucket) + log.NewInfo("", utils.GetSelfFuncName(), policy) + // 自动化桶public的代码 //policyJsonString := fmt.Sprintf(`{"Version": "2012-10-17","Statement": [{"Action": ["s3:GetObject","s3:PutObject"], // "Effect": "Allow","Principal": {"AWS": ["*"]},"Resource": ["arn:aws:s3:::%s/*"],"Sid": ""}]}`, config.Config.Credential.Minio.Bucket) From d99375a536bcd80f68991fd6014d110cb379a8cd Mon Sep 17 00:00:00 2001 From: wangchuxiao Date: Mon, 1 Aug 2022 15:27:51 +0800 Subject: [PATCH 11/11] fix bug --- internal/api/third/minio_init.go | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/internal/api/third/minio_init.go b/internal/api/third/minio_init.go index 04cc53bc1..b1088df50 100644 --- a/internal/api/third/minio_init.go +++ b/internal/api/third/minio_init.go @@ -5,6 +5,7 @@ import ( "Open_IM/pkg/common/log" "Open_IM/pkg/utils" "context" + "fmt" "github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7/pkg/credentials" url2 "net/url" @@ -84,17 +85,17 @@ func MinioInit() { log.NewInfo("", utils.GetSelfFuncName(), policy) // 自动化桶public的代码 - //policyJsonString := fmt.Sprintf(`{"Version": "2012-10-17","Statement": [{"Action": ["s3:GetObject","s3:PutObject"], - // "Effect": "Allow","Principal": {"AWS": ["*"]},"Resource": ["arn:aws:s3:::%s/*"],"Sid": ""}]}`, config.Config.Credential.Minio.Bucket) - //err = MinioClient.SetBucketPolicy(context.Background(), config.Config.Credential.Minio.Bucket, policyJsonString) - //if err != nil { - // log.NewInfo("", utils.GetSelfFuncName(), "SetBucketPolicy failed please set in web", err.Error()) - //} - //policyJsonString = fmt.Sprintf(`{"Version": "2012-10-17","Statement": [{"Action": ["s3:GetObject","s3:PutObject"], - // "Effect": "Allow","Principal": {"AWS": ["*"]},"Resource": ["arn:aws:s3:::%s/*"],"Sid": ""}]}`, config.Config.Credential.Minio.AppBucket) - //err = MinioClient.SetBucketPolicy(context.Background(), config.Config.Credential.Minio.AppBucket, policyJsonString) - //if err != nil { - // log.NewInfo("", utils.GetSelfFuncName(), "SetBucketPolicy failed please set in web", err.Error()) - //} - //log.NewInfo(operationID, utils.GetSelfFuncName(), "minio create and set policy success") + policyJsonString := fmt.Sprintf(`{"Version": "2012-10-17","Statement": [{"Action": ["s3:GetObject","s3:PutObject"], + "Effect": "Allow","Principal": {"AWS": ["*"]},"Resource": ["arn:aws:s3:::%s/*"],"Sid": ""}]}`, config.Config.Credential.Minio.Bucket) + err = MinioClient.SetBucketPolicy(context.Background(), config.Config.Credential.Minio.Bucket, policyJsonString) + if err != nil { + log.NewInfo("", utils.GetSelfFuncName(), "SetBucketPolicy failed please set in web", err.Error()) + } + policyJsonString = fmt.Sprintf(`{"Version": "2012-10-17","Statement": [{"Action": ["s3:GetObject","s3:PutObject"], + "Effect": "Allow","Principal": {"AWS": ["*"]},"Resource": ["arn:aws:s3:::%s/*"],"Sid": ""}]}`, config.Config.Credential.Minio.AppBucket) + err = MinioClient.SetBucketPolicy(context.Background(), config.Config.Credential.Minio.AppBucket, policyJsonString) + if err != nil { + log.NewInfo("", utils.GetSelfFuncName(), "SetBucketPolicy failed please set in web", err.Error()) + } + log.NewInfo(operationID, utils.GetSelfFuncName(), "minio create and set policy success") }