From 67f30199e329dbd4e7398b09aea99a61f702e952 Mon Sep 17 00:00:00 2001 From: chao <48119764+withchao@users.noreply.github.com> Date: Mon, 23 Sep 2024 12:16:18 +0800 Subject: [PATCH 1/5] fix: GetMsgBySeqs boundary issues (#2647) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * update gomake version * update gomake version * fix: seq conversion bug * fix: redis pipe exec * fix: ImportFriends * fix: A large number of logs keysAndValues ​​length is not even * feat: mark read aggregate write * feat: online status supports redis cluster * feat: online status supports redis cluster * feat: online status supports redis cluster * merge * merge * read seq is written to mongo * read seq is written to mongo * fix: invitation to join group notification * fix: friend op_user_id * feat: optimizing asynchronous context * feat: optimizing memamq size * feat: add GetSeqMessage * feat: GroupApplicationAgreeMemberEnterNotification * feat: GroupApplicationAgreeMemberEnterNotification * feat: go.mod * feat: go.mod * feat: join group notification and get seq * feat: join group notification and get seq * feat: avoid pulling messages from sessions with a large number of max seq values of 0 * feat: API supports gzip * go.mod * fix: nil pointer error on close * fix: listen error * fix: listen error * update go.mod * feat: add log * fix: token parse token value * fix: GetMsgBySeqs boundary issues --------- Co-authored-by: withchao --- pkg/common/storage/cache/redis/token.go | 9 ++++++--- pkg/common/storage/controller/msg.go | 19 ++++++++++++++++--- 2 files changed, 22 insertions(+), 6 deletions(-) diff --git a/pkg/common/storage/cache/redis/token.go b/pkg/common/storage/cache/redis/token.go index b82259658..24e9c3005 100644 --- a/pkg/common/storage/cache/redis/token.go +++ b/pkg/common/storage/cache/redis/token.go @@ -19,8 +19,8 @@ import ( "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/cachekey" "github.com/openimsdk/tools/errs" - "github.com/openimsdk/tools/utils/stringutil" "github.com/redis/go-redis/v9" + "strconv" "time" ) @@ -58,9 +58,12 @@ func (c *tokenCache) GetTokensWithoutError(ctx context.Context, userID string, p } mm := make(map[string]int) for k, v := range m { - mm[k] = stringutil.StringToInt(v) + state, err := strconv.Atoi(v) + if err != nil { + return nil, errs.WrapMsg(err, "redis token value is not int", "value", v, "userID", userID, "platformID", platformID) + } + mm[k] = state } - return mm, nil } diff --git a/pkg/common/storage/controller/msg.go b/pkg/common/storage/controller/msg.go index ba3d7b9e8..16a7b1c9b 100644 --- a/pkg/common/storage/controller/msg.go +++ b/pkg/common/storage/controller/msg.go @@ -446,7 +446,7 @@ func (db *commonMsgDatabase) GetMsgBySeqsRange(ctx context.Context, userID strin func (db *commonMsgDatabase) GetMsgBySeqs(ctx context.Context, userID string, conversationID string, seqs []int64) (int64, int64, []*sdkws.MsgData, error) { userMinSeq, err := db.seqUser.GetUserMinSeq(ctx, conversationID, userID) - if err != nil && errs.Unwrap(err) != redis.Nil { + if err != nil { return 0, 0, nil, err } minSeq, err := db.seqConversation.GetMinSeq(ctx, conversationID) @@ -457,15 +457,28 @@ func (db *commonMsgDatabase) GetMsgBySeqs(ctx context.Context, userID string, co if err != nil { return 0, 0, nil, err } - if userMinSeq < minSeq { + userMaxSeq, err := db.seqUser.GetUserMaxSeq(ctx, conversationID, userID) + if err != nil { + return 0, 0, nil, err + } + if userMinSeq > minSeq { minSeq = userMinSeq } - var newSeqs []int64 + if userMaxSeq > 0 && userMaxSeq < maxSeq { + maxSeq = userMaxSeq + } + newSeqs := make([]int64, 0, len(seqs)) for _, seq := range seqs { + if seq <= 0 { + continue + } if seq >= minSeq && seq <= maxSeq { newSeqs = append(newSeqs, seq) } } + if len(newSeqs) == 0 { + return minSeq, maxSeq, nil, nil + } successMsgs, failedSeqs, err := db.msg.GetMessagesBySeq(ctx, conversationID, newSeqs) if err != nil { if err != redis.Nil { From 5b2b2c1fc68b2deb89e84e3b592bc93ad34b1c32 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=94=A1=E7=9B=B8=E8=B7=83?= Date: Mon, 23 Sep 2024 15:04:25 +0800 Subject: [PATCH 2/5] fix: the attribute version is obsolete, remove it (#2644) --- docker-compose.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 512f951db..edac65b13 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,5 +1,3 @@ -version: '3' - networks: openim: driver: bridge From 79bf21453a19d83c0401ac11b9eaae847fe72727 Mon Sep 17 00:00:00 2001 From: Monet Lee Date: Mon, 23 Sep 2024 15:06:11 +0800 Subject: [PATCH 3/5] refactor: update Userregister request field. (#2650) --- go.mod | 2 +- go.sum | 4 ++-- internal/rpc/user/callback.go | 3 +-- internal/rpc/user/user.go | 8 ++++---- pkg/callbackstruct/user.go | 2 -- test/e2e/api/token/token.go | 6 ++---- 6 files changed, 10 insertions(+), 15 deletions(-) diff --git a/go.mod b/go.mod index 1ee4c7968..5fab50991 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/gorilla/websocket v1.5.1 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/mitchellh/mapstructure v1.5.0 - github.com/openimsdk/protocol v0.0.72-alpha.24 + github.com/openimsdk/protocol v0.0.72-alpha.25 github.com/openimsdk/tools v0.0.50-alpha.12 github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 diff --git a/go.sum b/go.sum index 52d7b5202..4c3b4aa21 100644 --- a/go.sum +++ b/go.sum @@ -322,8 +322,8 @@ github.com/onsi/gomega v1.25.0 h1:Vw7br2PCDYijJHSfBOWhov+8cAnUf8MfMaIOV323l6Y= github.com/onsi/gomega v1.25.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdMPGhM= github.com/openimsdk/gomake v0.0.14-alpha.5 h1:VY9c5x515lTfmdhhPjMvR3BBRrRquAUCFsz7t7vbv7Y= github.com/openimsdk/gomake v0.0.14-alpha.5/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= -github.com/openimsdk/protocol v0.0.72-alpha.24 h1:1Dl8TEZVXhdmve+ewoLkJa2wbFBIHqPgjvr9u/J66JM= -github.com/openimsdk/protocol v0.0.72-alpha.24/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= +github.com/openimsdk/protocol v0.0.72-alpha.25 h1:W8E6gnwt5V6anr/8lYOf5v/Lcsggf7gIAzJbw7YU6So= +github.com/openimsdk/protocol v0.0.72-alpha.25/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= github.com/openimsdk/tools v0.0.50-alpha.12 h1:rV3BxgqN+F79vZvdoQ+97Eob8ScsRVEM8D+Wrcl23uo= github.com/openimsdk/tools v0.0.50-alpha.12/go.mod h1:h1cYmfyaVtgFbKmb1Cfsl8XwUOMTt8ubVUQrdGtsUh4= github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= diff --git a/internal/rpc/user/callback.go b/internal/rpc/user/callback.go index 1bdf399d2..b58053c5a 100644 --- a/internal/rpc/user/callback.go +++ b/internal/rpc/user/callback.go @@ -16,6 +16,7 @@ package user import ( "context" + "github.com/openimsdk/open-im-server/v3/pkg/common/webhook" "github.com/openimsdk/tools/utils/datautil" @@ -88,7 +89,6 @@ func (s *userServer) webhookBeforeUserRegister(ctx context.Context, before *conf return webhook.WithCondition(ctx, before, func(ctx context.Context) error { cbReq := &cbapi.CallbackBeforeUserRegisterReq{ CallbackCommand: cbapi.CallbackBeforeUserRegisterCommand, - Secret: req.Secret, Users: req.Users, } @@ -108,7 +108,6 @@ func (s *userServer) webhookBeforeUserRegister(ctx context.Context, before *conf func (s *userServer) webhookAfterUserRegister(ctx context.Context, after *config.AfterConfig, req *pbuser.UserRegisterReq) { cbReq := &cbapi.CallbackAfterUserRegisterReq{ CallbackCommand: cbapi.CallbackAfterUserRegisterCommand, - Secret: req.Secret, Users: req.Users, } diff --git a/internal/rpc/user/user.go b/internal/rpc/user/user.go index 8b22c8f9b..4669ed513 100644 --- a/internal/rpc/user/user.go +++ b/internal/rpc/user/user.go @@ -47,7 +47,6 @@ import ( "github.com/openimsdk/tools/db/pagination" registry "github.com/openimsdk/tools/discovery" "github.com/openimsdk/tools/errs" - "github.com/openimsdk/tools/log" "github.com/openimsdk/tools/utils/datautil" "google.golang.org/grpc" ) @@ -263,10 +262,11 @@ func (s *userServer) UserRegister(ctx context.Context, req *pbuser.UserRegisterR if len(req.Users) == 0 { return nil, errs.ErrArgs.WrapMsg("users is empty") } - if req.Secret != s.config.Share.Secret { - log.ZDebug(ctx, "UserRegister", s.config.Share.Secret, req.Secret) - return nil, errs.ErrNoPermission.WrapMsg("secret invalid") + + if err = authverify.CheckAdmin(ctx, s.config.Share.IMAdminUserID); err != nil { + return nil, err } + if datautil.DuplicateAny(req.Users, func(e *sdkws.UserInfo) string { return e.UserID }) { return nil, errs.ErrArgs.WrapMsg("userID repeated") } diff --git a/pkg/callbackstruct/user.go b/pkg/callbackstruct/user.go index 504c7ffb7..d8640da29 100644 --- a/pkg/callbackstruct/user.go +++ b/pkg/callbackstruct/user.go @@ -72,7 +72,6 @@ type CallbackAfterUpdateUserInfoExResp struct { type CallbackBeforeUserRegisterReq struct { CallbackCommand `json:"callbackCommand"` - Secret string `json:"secret"` Users []*sdkws.UserInfo `json:"users"` } @@ -83,7 +82,6 @@ type CallbackBeforeUserRegisterResp struct { type CallbackAfterUserRegisterReq struct { CallbackCommand `json:"callbackCommand"` - Secret string `json:"secret"` Users []*sdkws.UserInfo `json:"users"` } diff --git a/test/e2e/api/token/token.go b/test/e2e/api/token/token.go index 908f9b8a0..c862dc6db 100644 --- a/test/e2e/api/token/token.go +++ b/test/e2e/api/token/token.go @@ -53,8 +53,7 @@ type User struct { // UserRegisterRequest represents a request to register a user. type UserRegisterRequest struct { - Secret string `json:"secret"` - Users []User `json:"users"` + Users []User `json:"users"` } /* func main() { @@ -109,8 +108,7 @@ func RegisterUser(token, userID, nickname, faceURL string) error { FaceURL: faceURL, } reqBody := UserRegisterRequest{ - Secret: SecretKey, - Users: []User{user}, + Users: []User{user}, } reqBytes, err := json.Marshal(reqBody) if err != nil { From c0eaa1c16672d1894d870933663738a330af427e Mon Sep 17 00:00:00 2001 From: icey-yu <119291641+icey-yu@users.noreply.github.com> Date: Mon, 23 Sep 2024 17:10:35 +0800 Subject: [PATCH 4/5] Test Workflow (#2640) * feat: cicd * feat: cicd * fix: cicd * fix: cicd * fix: cicd * fix: cicd --- .github/workflows/go-build-test.yml | 59 +++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) diff --git a/.github/workflows/go-build-test.yml b/.github/workflows/go-build-test.yml index 5341c919d..1ed8f0397 100644 --- a/.github/workflows/go-build-test.yml +++ b/.github/workflows/go-build-test.yml @@ -89,6 +89,65 @@ jobs: mage start mage check + go-test: + name: Benchmark Test with go ${{ matrix.go_version }} on ${{ matrix.os }} + runs-on: ${{ matrix.os }} + permissions: + contents: write + env: + SDK_DIR: openim-sdk-core + CONFIG_PATH: config/notification.yml + # pull-requests: write + strategy: + matrix: + os: [ ubuntu-latest ] + go_version: [ "1.22.x" ] + + steps: + - name: Checkout Server repository + uses: actions/checkout@v4 + + - name: Checkout SDK repository + uses: actions/checkout@v4 + with: + repository: 'openimsdk/openim-sdk-core' + path: ${{ env.SDK_DIR }} + + - name: Set up Go ${{ matrix.go_version }} + uses: actions/setup-go@v5 + with: + go-version: ${{ matrix.go_version }} + + - name: Get Server dependencies + run: | + go install github.com/magefile/mage@latest + go mod download + + - name: Install yq + run: | + sudo wget https://github.com/mikefarah/yq/releases/download/v4.34.1/yq_linux_amd64 -O /usr/bin/yq + sudo chmod +x /usr/bin/yq + + - name: Modify Server Configuration + run: | + yq e '.groupCreated.unreadCount = true' -i ${{ env.CONFIG_PATH }} + yq e '.friendApplicationApproved.unreadCount = true' -i ${{ env.CONFIG_PATH }} + + - name: Start Server Services + run: | + docker compose up -d + mage build + mage start + mage check + + - name: Build test SDK core + run: | + cd ${{ env.SDK_DIR }} + go mod tidy + cd integration_test + mkdir data + go run main.go -lgr 0.8 -imf -crg -ckgn -ckcon -sem -ckmsn -u 20 -su 5 -lg 2 -cg 2 -cgm 3 -sm 10 -gm 10 -reg + dockerfile-test: name: Build and Test Dockerfile runs-on: ubuntu-latest From 6add09d47620612e8e087cf8665264544614ea4d Mon Sep 17 00:00:00 2001 From: icey-yu <119291641+icey-yu@users.noreply.github.com> Date: Mon, 23 Sep 2024 17:12:11 +0800 Subject: [PATCH 5/5] fix: kick group member callback (#2643) * fix: kill group member callback * fix: change port * fix: change port --- config/openim-msgtransfer.yml | 2 +- config/openim-push.yml | 2 +- config/prometheus.yml | 2 +- internal/rpc/group/callback.go | 1 + 4 files changed, 4 insertions(+), 3 deletions(-) diff --git a/config/openim-msgtransfer.yml b/config/openim-msgtransfer.yml index 753ac10bc..94ed073d8 100644 --- a/config/openim-msgtransfer.yml +++ b/config/openim-msgtransfer.yml @@ -3,4 +3,4 @@ prometheus: enable: true # List of ports that Prometheus listens on; each port corresponds to an instance of monitoring. Ensure these are managed accordingly # Because four instances have been launched, four ports need to be specified - ports: [ 12020, 12021, 12022, 12023, 12024, 12025, 12026, 12027 ] + ports: [ 12020, 12021, 12022, 12023, 12024, 12025, 12026, 12027, 12028, 12029, 12030, 12031, 12032, 12033, 12034, 12035 ] diff --git a/config/openim-push.yml b/config/openim-push.yml index 6df2a62b7..4d2aaca6b 100644 --- a/config/openim-push.yml +++ b/config/openim-push.yml @@ -10,7 +10,7 @@ prometheus: # Enable or disable Prometheus monitoring enable: true # List of ports that Prometheus listens on; these must match the number of rpc.ports to ensure correct monitoring setup - ports: [ 12170, 12171, 12172, 12173, 12174, 12175, 12176, 12177, 12178, 12179, 12180, 12181, 12182, 12183, 12184, 12185 ] + ports: [ 12170, 12171, 12172, 12173, 12174, 12175, 12176, 12177, 12178, 12179, 12180, 12182, 12183, 12184, 12185, 12186 ] maxConcurrentWorkers: 3 #Use geTui for offline push notifications, or choose fcm or jpns; corresponding configuration settings must be specified. diff --git a/config/prometheus.yml b/config/prometheus.yml index 4f0f7e32c..ab427ee82 100644 --- a/config/prometheus.yml +++ b/config/prometheus.yml @@ -46,7 +46,7 @@ scrape_configs: - job_name: openimserver-openim-push static_configs: - targets: [ internal_ip:12170, internal_ip:12171, internal_ip:12172, internal_ip:12173, internal_ip:12174, internal_ip:12175, internal_ip:12176, internal_ip:12177 ] -# - targets: [ internal_ip:12170, internal_ip:12171, internal_ip:12172, internal_ip:12173, internal_ip:12174, internal_ip:12175, internal_ip:12176, internal_ip:12177, internal_ip:12178, internal_ip:12179, internal_ip:12180, internal_ip:12181, internal_ip:12182, internal_ip:12183, internal_ip:12184, internal_ip:12185 ] +# - targets: [ internal_ip:12170, internal_ip:12171, internal_ip:12172, internal_ip:12173, internal_ip:12174, internal_ip:12175, internal_ip:12176, internal_ip:12177, internal_ip:12178, internal_ip:12179, internal_ip:12180, internal_ip:12182, internal_ip:12183, internal_ip:12184, internal_ip:12185, internal_ip:12186 ] labels: namespace: default - job_name: openimserver-openim-rpc-auth diff --git a/internal/rpc/group/callback.go b/internal/rpc/group/callback.go index ef391bd70..e748c66dc 100644 --- a/internal/rpc/group/callback.go +++ b/internal/rpc/group/callback.go @@ -218,6 +218,7 @@ func (s *groupServer) webhookAfterKickGroupMember(ctx context.Context, after *co CallbackCommand: callbackstruct.CallbackAfterKickGroupCommand, GroupID: req.GroupID, KickedUserIDs: req.KickedUserIDs, + Reason: req.Reason, } s.webhookClient.AsyncPost(ctx, cbReq.GetCallbackCommand(), cbReq, &callbackstruct.CallbackKillGroupMemberResp{}, after) }