diff --git a/tools/codescan/config.yaml b/.github/code-language-detector.yml similarity index 100% rename from tools/codescan/config.yaml rename to .github/code-language-detector.yml diff --git a/.github/workflows/bot-auto-cherry-pick.yml b/.github/workflows/bot-auto-cherry-pick.yml index baafecad7..cdd7241e2 100644 --- a/.github/workflows/bot-auto-cherry-pick.yml +++ b/.github/workflows/bot-auto-cherry-pick.yml @@ -25,6 +25,7 @@ jobs: - name: Comment cherry-pick command uses: actions/github-script@v7 with: + github-token: ${{ secrets.BOT_GITHUB_TOKEN }} script: | const pr = context.payload.pull_request; if (!pr.merged) { @@ -63,5 +64,4 @@ jobs: repo: context.repo.repo, issue_number: pr.number, body: cherryPickCmd - }); - github-token: ${{ secrets.BOT_GITHUB_TOKEN }} \ No newline at end of file + }); \ No newline at end of file diff --git a/.github/workflows/bot-cherry-pick.yml b/.github/workflows/bot-cherry-pick.yml index e488e97de..71597189c 100644 --- a/.github/workflows/bot-cherry-pick.yml +++ b/.github/workflows/bot-cherry-pick.yml @@ -12,23 +12,57 @@ # See the License for the specific language governing permissions and # limitations under the License. -name: Github Rebot for Cherry Pick On Comment +name: Github Robot for Cherry Pick On Comment + on: issue_comment: types: [created] + jobs: cherry-pick: name: Cherry Pick - # && github.event.comment.user.login=='kubbot' if: github.event.issue.pull_request != '' && contains(github.event.comment.body, '/cherry-pick') runs-on: ubuntu-latest + steps: - name: Checkout the latest code uses: actions/checkout@v4 with: token: ${{ secrets.BOT_GITHUB_TOKEN }} - fetch-depth: 0 # otherwise, you will fail to push refs to dest repo + fetch-depth: 0 # To ensure all history is available for cherry-picking + - name: Automatic Cherry Pick uses: vendoo/gha-cherry-pick@v1 + with: + # Assuming the cherry-pick commit SHA is passed in the comment like '/cherry-pick sha' + commit-sha: ${{ github.event.comment.body }} + env: + GITHUB_TOKEN: ${{ secrets.BOT_GITHUB_TOKEN }} + + - name: Create a new branch for PR + run: | + PR_BRANCH="cherry-pick-${GITHUB_SHA}-to-${{ github.base_ref }}" + git checkout -b $PR_BRANCH + git push origin $PR_BRANCH + env: + GITHUB_TOKEN: ${{ secrets.BOT_GITHUB_TOKEN }} + + - name: Create Pull Request + uses: actions/github-script@v5 + with: + script: | + const prTitle = "Cherry-pick to ${{ github.base_ref }}" + const prBody = "Automated cherry-pick of ${{ github.event.comment.body }}\n\n/cc @kubbot" + const base = "${{ github.base_ref }}" + const head = "cherry-pick-${{ github.sha }}-to-${{ github.base_ref }}" + const createPr = await github.rest.pulls.create({ + owner: context.repo.owner, + repo: context.repo.repo, + title: prTitle, + body: prBody, + head: head, + base: base, + maintainer_can_modify: true, // Allows maintainers to edit the PR + }) env: - GITHUB_TOKEN: ${{ secrets.BOT_GITHUB_TOKEN }} \ No newline at end of file + GITHUB_TOKEN: ${{ secrets.BOT_GITHUB_TOKEN }} diff --git a/.github/workflows/code-language-detector.yml b/.github/workflows/code-language-detector.yml new file mode 100644 index 000000000..80ec94733 --- /dev/null +++ b/.github/workflows/code-language-detector.yml @@ -0,0 +1,27 @@ +# Copyright © 2024 OpenIM. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: Language Check Workflow Test + +on: [pull_request] + +jobs: + comment-language-detector: + runs-on: ubuntu-latest + steps: + - name: Checkout Repository + uses: actions/checkout@v4 + + - name: Code Language Detector + uses: kubecub/comment-lang-detector@v1.0.0 \ No newline at end of file diff --git a/.github/workflows/e2e-test.yml b/.github/workflows/e2e-test.yml index b5f901d25..f98221e41 100644 --- a/.github/workflows/e2e-test.yml +++ b/.github/workflows/e2e-test.yml @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -name: OpenIM Linux System E2E Test +name: OpenIM E2E And API Test on: workflow_dispatch: @@ -82,7 +82,7 @@ jobs: sudo make tidy sudo make tools.verify.go-gitlint - - name: Build, Start + - name: Build, Start(make build && make start) run: | sudo ./scripts/install/install.sh -i @@ -90,9 +90,8 @@ jobs: run: | sudo ./scripts/install/install.sh -s - - name: Exec OpenIM API test + - name: Exec OpenIM API test (make test-api) run: | - sudo make test-api mkdir -p ./tmp touch ./tmp/test.md echo "# OpenIM Test" >> ./tmp/test.md @@ -103,9 +102,10 @@ jobs: echo "" >> ./tmp/test.md echo "" >> ./tmp/test.md - - name: Exec OpenIM E2E Test + sudo make test-api + + - name: Exec OpenIM E2E Test (make test-e2e) run: | - sudo make test-e2e echo "" >> ./tmp/test.md echo "## OpenIM E2E Test" >> ./tmp/test.md echo "
Command Output for OpenIM E2E Test" >> ./tmp/test.md @@ -114,6 +114,8 @@ jobs: echo "" >> ./tmp/test.md echo "
" >> ./tmp/test.md + sudo make test-e2e + - name: Comment PR with file uses: thollander/actions-comment-pull-request@v2 with: @@ -143,4 +145,4 @@ jobs: PUBLISH_BRANCH: gh-pages env: GITHUB_TOKEN: ${{ secrets.BOT_GITHUB_TOKEN }} - continue-on-error: true \ No newline at end of file + continue-on-error: true diff --git a/.github/workflows/milestone.yml b/.github/workflows/milestone.yml index ecd64656a..c74e7074a 100644 --- a/.github/workflows/milestone.yml +++ b/.github/workflows/milestone.yml @@ -41,6 +41,7 @@ jobs: steps: - uses: actions/github-script@v7 # v6 with: + github-token: ${{ secrets.BOT_GITHUB_TOKEN }} script: | if (!context.payload.pull_request.merged) { console.log('PR was not merged, skipping.'); @@ -56,9 +57,10 @@ jobs: owner: context.repo.owner, repo: context.repo.repo, state: 'open', - sort: 'due_on', - direction: 'asc' + sort: 'title', + direction: 'desc' }) + if (milestones.data.length === 0) { console.log('There are no milestones, skipping.'); return; diff --git a/.github/workflows/openimci.yml b/.github/workflows/openimci.yml index d10033a1c..5aeddd09a 100644 --- a/.github/workflows/openimci.yml +++ b/.github/workflows/openimci.yml @@ -57,7 +57,7 @@ jobs: steps: - name: Setup uses: actions/checkout@v4 - + - name: Set up Go ${{ matrix.go_version }} uses: actions/setup-go@v5 with: @@ -70,6 +70,9 @@ jobs: version: '3.x' # If available, use the latest major version that's compatible repo-token: ${{ secrets.GITHUB_TOKEN }} + - name: Code Typecheck Detector + uses: kubecub/typecheck@main + - name: Module Operations run: | sudo make tidy diff --git a/Dockerfile b/Dockerfile index 228c09c93..9b1021216 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,6 +10,7 @@ ENV GOPROXY=$GOPROXY # Set up the working directory WORKDIR /openim/openim-server + # Copy all files to the container ADD . . diff --git a/deployments/templates/env-template.yaml b/deployments/templates/env-template.yaml index 85619b422..b1fb8b78b 100644 --- a/deployments/templates/env-template.yaml +++ b/deployments/templates/env-template.yaml @@ -143,7 +143,7 @@ KAFKA_LATESTMSG_REDIS_TOPIC=${KAFKA_LATESTMSG_REDIS_TOPIC} # MINIO_PORT # ---------- # MINIO_PORT sets the port for the MinIO object storage service. -# Upon changing this port, the MinIO endpoint URLs in the `config/config.yaml` file must be updated +# Upon changing this port, the MinIO endpoint URLs in the config/config.yaml file must be updated # to reflect this change. The endpoints include both the 'endpoint' and 'signEndpoint' # under the MinIO configuration. # diff --git a/docker-compose.yml b/docker-compose.yml index 8538eec83..ef0714329 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -186,23 +186,6 @@ services: # server: # ipv4_address: ${OPENIM_SERVER_NETWORK_ADDRESS:-172.28.0.8} -### TODO: mysql is required to deploy the openim-chat component - # mysql: - # image: mysql:${MYSQL_IMAGE_VERSION:-5.7} - # platform: linux/amd64 - # ports: - # - "${MYSQL_PORT:-13306}:3306" - # container_name: mysql - # volumes: - # - "${DATA_DIR:-./}/components/mysql/data:/var/lib/mysql" - # - "/etc/localtime:/etc/localtime" - # environment: - # MYSQL_ROOT_PASSWORD: "${MYSQL_PASSWORD:-openIM123}" - # restart: always - # networks: - # server: - # ipv4_address: ${MYSQL_NETWORK_ADDRESS:-172.28.0.15} - # openim-chat: # image: ${IMAGE_REGISTRY:-ghcr.io/openimsdk}/openim-chat:${CHAT_IMAGE_VERSION:-main} # container_name: openim-chat diff --git a/docs/contrib/version.md b/docs/contrib/version.md index 574badf59..337aa4f7d 100644 --- a/docs/contrib/version.md +++ b/docs/contrib/version.md @@ -96,7 +96,6 @@ We reinforce our approach to branch management and versioning with stringent tes This document describes the maximum version skew supported between various openim components. Specific cluster deployment tools may place additional restrictions on version skew. - ### Supported version skew In highly-available (HA) clusters, the newest and oldest `openim-api` instances must be within one minor version. @@ -210,6 +209,7 @@ git merge release-v3.1 # Push the updates to the main branch git push origin main ``` + ## Release Process ``` @@ -232,5 +232,7 @@ For more details on managing Docker image versions, visit [OpenIM Docker Images More on multi-branch version management design and version management design at helm charts: -+ https://github.com/openimsdk/open-im-server/issues/1695 -+ https://github.com/openimsdk/open-im-server/issues/1662 \ No newline at end of file +About Helm's version management strategy for Multiple Apps and multiple Services: + ++ [中文版本管理文档](https://github.com/openimsdk/helm-charts/blob/main/docs/contrib/version-zh.md) ++ [English version management documents](https://github.com/openimsdk/helm-charts/blob/main/docs/contrib/version.md) diff --git a/go.sum b/go.sum index ff81ceecc..2e1db8322 100644 --- a/go.sum +++ b/go.sum @@ -16,8 +16,12 @@ cloud.google.com/go/storage v1.36.0/go.mod h1:M6M/3V/D3KpzMTJyPOR/HU6n2Si5QdaXYE firebase.google.com/go v3.13.0+incompatible h1:3TdYC3DDi6aHn20qoRkxwGqNgdjtblwVAyRLQwGn/+4= firebase.google.com/go v3.13.0+incompatible/go.mod h1:xlah6XbEyW6tbfSklcfe5FHJIwjt8toICdV5Wh9ptHs= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/IBM/sarama v1.43.0 h1:YFFDn8mMI2QL0wOrG0J2sFoVIAFl7hS9JQi2YZsXtJc= -github.com/IBM/sarama v1.43.0/go.mod h1:zlE6HEbC/SMQ9mhEYaF7nNLYOUyrs0obySKCckWP9BM= +github.com/IBM/sarama v1.42.2 h1:VoY4hVIZ+WQJ8G9KNY/SQlWguBQXQ9uvFPOnrcu8hEw= +github.com/IBM/sarama v1.42.2/go.mod h1:FLPGUGwYqEs62hq2bVG6Io2+5n+pS6s/WOXVKWSLFtE= +github.com/OpenIMSDK/protocol v0.0.56 h1:mbVFyDBachEsmJLfYW5AU1z2KL8AUEpoHG8RPCIxjgg= +github.com/OpenIMSDK/protocol v0.0.56/go.mod h1:F25dFrwrIx3lkNoiuf6FkCfxuwf8L4Z8UIsdTHP/r0Y= +github.com/OpenIMSDK/tools v0.0.37 h1:qvDqmA4RbEJtPjZouWCkVuf/pjm6Y8nUrG5iH2gcnOg= +github.com/OpenIMSDK/tools v0.0.37/go.mod h1:wBfR5CYmEyvxl03QJbTkhz1CluK6J4/lX0lviu8JAjE= github.com/QcloudApi/qcloud_sign_golang v0.0.0-20141224014652-e4130a326409/go.mod h1:1pk82RBxDY/JZnPQrtqHlUFfCctgdorsd9M06fMynOM= github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7 h1:uSoVVbwJiQipAclBbw+8quDsfcvFjOpI5iCf4p/cqCs= github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7/go.mod h1:6zEj6s6u/ghQa61ZWa/C2Aw3RkjiTBOix7dkqa1VLIs= diff --git a/go.work b/go.work index 8dc91a631..02e4154d3 100644 --- a/go.work +++ b/go.work @@ -2,8 +2,6 @@ go 1.19 use ( . - ./test/typecheck - ./tools/codescan ./tools/changelog ./tools/component ./tools/formitychecker diff --git a/internal/api/msg.go b/internal/api/msg.go index dd8ec2926..92060fd12 100644 --- a/internal/api/msg.go +++ b/internal/api/msg.go @@ -223,25 +223,22 @@ func (m *MessageApi) SendMessage(c *gin.Context) { // Set the receiver ID in the message data. sendMsgReq.MsgData.RecvID = req.RecvID - // Declare a variable to store the message sending status. - var status int - // Attempt to send the message using the client. respPb, err := m.Client.SendMsg(c, sendMsgReq) if err != nil { // Set the status to failed and respond with an error if sending fails. - status = constant.MsgSendFailed apiresp.GinError(c, err) return } // Set the status to successful if the message is sent. - status = constant.MsgSendSuccessed + var status int = constant.MsgSendSuccessed // Attempt to update the message sending status in the system. _, err = m.Client.SetSendMsgStatus(c, &msg.SetSendMsgStatusReq{ Status: int32(status), }) + if err != nil { // Log the error if updating the status fails. apiresp.GinError(c, err) diff --git a/internal/msggateway/user_map.go b/internal/msggateway/user_map.go index 11d9ebaf6..ffaff75d9 100644 --- a/internal/msggateway/user_map.go +++ b/internal/msggateway/user_map.go @@ -72,24 +72,32 @@ func (u *UserMap) Set(key string, v *Client) { } func (u *UserMap) delete(key string, connRemoteAddr string) (isDeleteUser bool) { + // Attempt to load the clients associated with the key. allClients, existed := u.m.Load(key) - if existed { - oldClients := allClients.([]*Client) - var a []*Client - for _, client := range oldClients { - if client.ctx.GetRemoteAddr() != connRemoteAddr { - a = append(a, client) - } - } - if len(a) == 0 { - u.m.Delete(key) - return true - } else { - u.m.Store(key, a) - return false + if !existed { + // Return false immediately if the key does not exist. + return false + } + + // Convert allClients to a slice of *Client. + oldClients := allClients.([]*Client) + var remainingClients []*Client + for _, client := range oldClients { + // Keep clients that do not match the connRemoteAddr. + if client.ctx.GetRemoteAddr() != connRemoteAddr { + remainingClients = append(remainingClients, client) } } - return existed + + // If no clients remain after filtering, delete the key from the map. + if len(remainingClients) == 0 { + u.m.Delete(key) + return true + } + + // Otherwise, update the key with the remaining clients. + u.m.Store(key, remainingClients) + return false } func (u *UserMap) deleteClients(key string, clients []*Client) (isDeleteUser bool) { @@ -97,23 +105,28 @@ func (u *UserMap) deleteClients(key string, clients []*Client) (isDeleteUser boo return c.ctx.GetRemoteAddr(), struct{}{} }) allClients, existed := u.m.Load(key) - if existed { - oldClients := allClients.([]*Client) - var a []*Client - for _, client := range oldClients { - if _, ok := m[client.ctx.GetRemoteAddr()]; !ok { - a = append(a, client) - } - } - if len(a) == 0 { - u.m.Delete(key) - return true - } else { - u.m.Store(key, a) - return false + if !existed { + // If the key doesn't exist, return false. + return false + } + + // Filter out clients that are in the deleteMap. + oldClients := allClients.([]*Client) + var remainingClients []*Client + for _, client := range oldClients { + if _, shouldBeDeleted := deleteMap[client.ctx.GetRemoteAddr()]; !shouldBeDeleted { + remainingClients = append(remainingClients, client) } } - return existed + + // Update or delete the key based on the remaining clients. + if len(remainingClients) == 0 { + u.m.Delete(key) + return true + } + + u.m.Store(key, remainingClients) + return false } func (u *UserMap) DeleteAll(key string) { diff --git a/internal/msgtransfer/online_history_msg_handler.go b/internal/msgtransfer/online_history_msg_handler.go index 40e4c6e35..0e06c9624 100644 --- a/internal/msgtransfer/online_history_msg_handler.go +++ b/internal/msgtransfer/online_history_msg_handler.go @@ -159,12 +159,11 @@ func (och *OnlineHistoryRedisConsumerHandler) getPushStorageMsgList( options2 := msgprocessor.Options(msg.Options) if options2.IsHistory() { return true - } else { - // if !(!options2.IsSenderSync() && conversationID == msg.MsgData.SendID) { - // return false - // } - return false } + // if !(!options2.IsSenderSync() && conversationID == msg.MsgData.SendID) { + // return false + // } + return false } for _, v := range totalMsgs { options := msgprocessor.Options(v.message.Options) diff --git a/internal/push/offlinepush/getui/push.go b/internal/push/offlinepush/getui/push.go index c8e546388..d9beadb9c 100644 --- a/internal/push/offlinepush/getui/push.go +++ b/internal/push/offlinepush/getui/push.go @@ -90,9 +90,8 @@ func (g *Client) Push(ctx context.Context, userIDs []string, title, content stri for i, v := range s.GetSplitResult() { go func(index int, userIDs []string) { defer wg.Done() - if err := g.batchPush(ctx, token, userIDs, pushReq); err != nil { + if err = g.batchPush(ctx, token, userIDs, pushReq); err != nil { log.ZError(ctx, "batchPush failed", err, "index", index, "token", token, "req", pushReq) - err = err } }(i, v.Item) } diff --git a/internal/push/push_rpc_server.go b/internal/push/push_rpc_server.go index 6f5a28b18..c56e8ba78 100644 --- a/internal/push/push_rpc_server.go +++ b/internal/push/push_rpc_server.go @@ -91,9 +91,8 @@ func (r *pushServer) PushMsg(ctx context.Context, pbData *pbpush.PushMsgReq) (re if err != nil { if err != errNoOfflinePusher { return nil, err - } else { - log.ZWarn(ctx, "offline push failed", err, "msg", pbData.String()) } + log.ZWarn(ctx, "offline push failed", err, "msg", pbData.String()) } return &pbpush.PushMsgResp{}, nil } diff --git a/internal/rpc/conversation/conversaion.go b/internal/rpc/conversation/conversaion.go index 36b5712cf..22d0ac8e3 100644 --- a/internal/rpc/conversation/conversaion.go +++ b/internal/rpc/conversation/conversaion.go @@ -499,19 +499,27 @@ func (c *conversationServer) getConversationInfo( switch chatLog.SessionType { case constant.SingleChatType: if chatLog.SendID == userID { - msgInfo.FaceURL = sendMap[chatLog.RecvID].FaceURL - msgInfo.SenderName = sendMap[chatLog.RecvID].Nickname + if recv, ok := sendMap[chatLog.RecvID]; ok { + msgInfo.FaceURL = recv.FaceURL + msgInfo.SenderName = recv.Nickname + } break } - msgInfo.FaceURL = sendMap[chatLog.SendID].FaceURL - msgInfo.SenderName = sendMap[chatLog.SendID].Nickname + if send, ok := sendMap[chatLog.SendID]; ok { + msgInfo.FaceURL = send.FaceURL + msgInfo.SenderName = send.Nickname + } case constant.GroupChatType, constant.SuperGroupChatType: - msgInfo.GroupName = groupMap[chatLog.GroupID].GroupName - msgInfo.GroupFaceURL = groupMap[chatLog.GroupID].FaceURL - msgInfo.GroupMemberCount = groupMap[chatLog.GroupID].MemberCount msgInfo.GroupID = chatLog.GroupID - msgInfo.GroupType = groupMap[chatLog.GroupID].GroupType - msgInfo.SenderName = sendMap[chatLog.SendID].Nickname + if group, ok := groupMap[chatLog.GroupID]; ok { + msgInfo.GroupName = group.GroupName + msgInfo.GroupFaceURL = group.FaceURL + msgInfo.GroupMemberCount = group.MemberCount + msgInfo.GroupType = group.GroupType + } + if send, ok := sendMap[chatLog.SendID]; ok { + msgInfo.SenderName = send.Nickname + } } pbchatLog.ConversationID = conversationID msgInfo.LatestMsgRecvTime = chatLog.SendTime diff --git a/internal/rpc/friend/black.go b/internal/rpc/friend/black.go index 1d3aa1e11..ad63445c4 100644 --- a/internal/rpc/friend/black.go +++ b/internal/rpc/friend/black.go @@ -82,9 +82,14 @@ func (s *friendServer) AddBlack(ctx context.Context, req *pbfriend.AddBlackReq) CreateTime: time.Now(), Ex: req.Ex, } + if err := s.blackDatabase.Create(ctx, []*relation.BlackModel{&black}); err != nil { return nil, err } - s.notificationSender.BlackAddedNotification(ctx, req) + + if err := s.notificationSender.BlackAddedNotification(ctx, req); err != nil { + return nil, err + } + return &pbfriend.AddBlackResp{}, nil } diff --git a/internal/rpc/friend/friend.go b/internal/rpc/friend/friend.go index 57c0d331d..372f089e6 100644 --- a/internal/rpc/friend/friend.go +++ b/internal/rpc/friend/friend.go @@ -114,26 +114,35 @@ func (s *friendServer) ApplyToAddFriend(ctx context.Context, req *pbfriend.Apply if err := authverify.CheckAccessV3(ctx, req.FromUserID, &s.config.Manager, &s.config.IMAdmin); err != nil { return nil, err } + if req.ToUserID == req.FromUserID { return nil, errs.ErrCanNotAddYourself.WrapMsg("req.ToUserID", req.ToUserID) } if err = CallbackBeforeAddFriend(ctx, &s.config.Callback, req); err != nil && err != errs.ErrCallbackContinue { return nil, err } + if _, err := s.userRpcClient.GetUsersInfoMap(ctx, []string{req.ToUserID, req.FromUserID}); err != nil { return nil, err } + in1, in2, err := s.friendDatabase.CheckIn(ctx, req.FromUserID, req.ToUserID) if err != nil { return nil, err } + if in1 && in2 { return nil, errs.ErrRelationshipAlready.WrapMsg("already friends has f") } + if err = s.friendDatabase.AddFriendRequest(ctx, req.FromUserID, req.ToUserID, req.ReqMsg, req.Ex); err != nil { return nil, err } - s.notificationSender.FriendApplicationAddNotification(ctx, req) + + if err = s.notificationSender.FriendApplicationAddNotification(ctx, req); err != nil { + return nil, err + } + if err = CallbackAfterAddFriend(ctx, &s.config.Callback, req); err != nil && err != errs.ErrCallbackContinue { return nil, err } @@ -196,7 +205,9 @@ func (s *friendServer) RespondFriendApply(ctx context.Context, req *pbfriend.Res if err != nil { return nil, err } - s.notificationSender.FriendApplicationAgreedNotification(ctx, req) + if err := s.notificationSender.FriendApplicationAgreedNotification(ctx, req); err != nil { + return nil, err + } return resp, nil } if req.HandleResult == constant.FriendResponseRefuse { diff --git a/internal/rpc/group/group.go b/internal/rpc/group/group.go index ab3b8c270..e01ff6fad 100644 --- a/internal/rpc/group/group.go +++ b/internal/rpc/group/group.go @@ -679,6 +679,7 @@ func (s *groupServer) GetGroupMembersInfo(ctx context.Context, req *pbgroup.GetG return resp, nil } +// GetGroupApplicationList handles functions that get a list of group requests. func (s *groupServer) GetGroupApplicationList(ctx context.Context, req *pbgroup.GetGroupApplicationListReq) (*pbgroup.GetGroupApplicationListResp, error) { groupIDs, err := s.db.FindUserManagedGroupID(ctx, req.FromUserID) if err != nil { @@ -1014,6 +1015,7 @@ func (s *groupServer) SetGroupInfo(ctx context.Context, req *pbgroup.SetGroupInf return nil, errs.ErrDismissedAlready.Wrap() } resp := &pbgroup.SetGroupInfoResp{} + count, err := s.db.FindGroupMemberNum(ctx, group.GroupID) if err != nil { return nil, err @@ -1147,6 +1149,7 @@ func (s *groupServer) GetGroups(ctx context.Context, req *pbgroup.GetGroupsReq) total, group, err = s.db.SearchGroup(ctx, req.GroupName, req.Pagination) resp.Total = uint32(total) } + if err != nil { return nil, err } @@ -1154,10 +1157,12 @@ func (s *groupServer) GetGroups(ctx context.Context, req *pbgroup.GetGroupsReq) groupIDs := utils.Slice(group, func(e *relationtb.GroupModel) string { return e.GroupID }) + ownerMembers, err := s.db.FindGroupsOwner(ctx, groupIDs) if err != nil { return nil, err } + ownerMemberMap := utils.SliceToMap(ownerMembers, func(e *relationtb.GroupMemberModel) string { return e.GroupID }) diff --git a/internal/rpc/msg/callback.go b/internal/rpc/msg/callback.go index dc9e38311..a56a793e1 100644 --- a/internal/rpc/msg/callback.go +++ b/internal/rpc/msg/callback.go @@ -157,27 +157,27 @@ func callbackMsgModify(ctx context.Context, globalConfig *config.GlobalConfig, m } func CallbackGroupMsgRead(ctx context.Context, globalConfig *config.GlobalConfig, req *cbapi.CallbackGroupMsgReadReq) error { - if !globalConfig.Callback.CallbackGroupMsgRead.Enable || req.ContentType != constant.Text { + if !globalConfig.Callback.CallbackGroupMsgRead.Enable { return nil } req.CallbackCommand = cbapi.CallbackGroupMsgReadCommand resp := &cbapi.CallbackGroupMsgReadResp{} - if err := http.CallBackPostReturn(ctx, globalConfig.Callback.CallbackUrl, req, resp, globalConfig.Callback.CallbackMsgModify); err != nil { + if err := http.CallBackPostReturn(ctx, globalConfig.Callback.CallbackUrl, req, resp, globalConfig.Callback.CallbackGroupMsgRead); err != nil { return err } return nil } func CallbackSingleMsgRead(ctx context.Context, globalConfig *config.GlobalConfig, req *cbapi.CallbackSingleMsgReadReq) error { - if !globalConfig.Callback.CallbackSingleMsgRead.Enable || req.ContentType != constant.Text { + if !globalConfig.Callback.CallbackSingleMsgRead.Enable { return nil } req.CallbackCommand = cbapi.CallbackSingleMsgRead resp := &cbapi.CallbackSingleMsgReadResp{} - if err := http.CallBackPostReturn(ctx, globalConfig.Callback.CallbackUrl, req, resp, globalConfig.Callback.CallbackMsgModify); err != nil { + if err := http.CallBackPostReturn(ctx, globalConfig.Callback.CallbackUrl, req, resp, globalConfig.Callback.CallbackSingleMsgRead); err != nil { return err } return nil diff --git a/internal/rpc/msg/sync_msg.go b/internal/rpc/msg/sync_msg.go index 517ee23c5..0fadce781 100644 --- a/internal/rpc/msg/sync_msg.go +++ b/internal/rpc/msg/sync_msg.go @@ -134,6 +134,7 @@ func (m *msgServer) SearchMessage(ctx context.Context, req *msg.SearchMessageReq groupIDs = append(groupIDs, chatLog.GroupID) } } + // Retrieve sender and receiver information if len(sendIDs) != 0 { sendInfos, err := m.UserLocalCache.GetUsersInfo(ctx, sendIDs) if err != nil { @@ -152,6 +153,8 @@ func (m *msgServer) SearchMessage(ctx context.Context, req *msg.SearchMessageReq recvMap[recvInfo.UserID] = recvInfo.Nickname } } + + // Retrieve group information including member counts if len(groupIDs) != 0 { groupInfos, err := m.GroupLocalCache.GetGroupInfos(ctx, groupIDs) if err != nil { @@ -159,8 +162,14 @@ func (m *msgServer) SearchMessage(ctx context.Context, req *msg.SearchMessageReq } for _, groupInfo := range groupInfos { groupMap[groupInfo.GroupID] = groupInfo + // Get actual member count + memberIDs, err := m.GroupLocalCache.GetGroupMemberIDs(ctx, groupInfo.GroupID) + if err == nil { + groupInfo.MemberCount = uint32(len(memberIDs)) // Update the member count with actual number + } } } + // Construct response with updated information for _, chatLog := range chatLogs { pbchatLog := &msg.ChatLog{} utils.CopyStructFields(pbchatLog, chatLog) @@ -172,14 +181,14 @@ func (m *msgServer) SearchMessage(ctx context.Context, req *msg.SearchMessageReq switch chatLog.SessionType { case constant.SingleChatType, constant.NotificationChatType: pbchatLog.RecvNickname = recvMap[chatLog.RecvID] - case constant.GroupChatType, constant.SuperGroupChatType: - pbchatLog.SenderFaceURL = groupMap[chatLog.GroupID].FaceURL - pbchatLog.GroupMemberCount = groupMap[chatLog.GroupID].MemberCount - pbchatLog.RecvID = groupMap[chatLog.GroupID].GroupID - pbchatLog.GroupName = groupMap[chatLog.GroupID].GroupName - pbchatLog.GroupOwner = groupMap[chatLog.GroupID].OwnerUserID - pbchatLog.GroupType = groupMap[chatLog.GroupID].GroupType + groupInfo := groupMap[chatLog.GroupID] + pbchatLog.SenderFaceURL = groupInfo.FaceURL + pbchatLog.GroupMemberCount = groupInfo.MemberCount // Reflects actual member count + pbchatLog.RecvID = groupInfo.GroupID + pbchatLog.GroupName = groupInfo.GroupName + pbchatLog.GroupOwner = groupInfo.OwnerUserID + pbchatLog.GroupType = groupInfo.GroupType } resp.ChatLogs = append(resp.ChatLogs, pbchatLog) } diff --git a/pkg/common/db/mgo/group.go b/pkg/common/db/mgo/group.go index 0b8505a2f..430c5bcc8 100644 --- a/pkg/common/db/mgo/group.go +++ b/pkg/common/db/mgo/group.go @@ -70,10 +70,17 @@ func (g *GroupMgo) Take(ctx context.Context, groupID string) (group *relation.Gr } func (g *GroupMgo) Search(ctx context.Context, keyword string, pagination pagination.Pagination) (total int64, groups []*relation.GroupModel, err error) { - return mongoutil.FindPage[*relation.GroupModel](ctx, g.coll, bson.M{"group_name": bson.M{"$regex": keyword}, - "status": bson.M{"$ne": constant.GroupStatusDismissed}}, pagination) + // Define the sorting options + opts := options.Find().SetSort(bson.D{{Key: "created_at", Value: -1}}) + + // Perform the search with pagination and sorting + return mongoutil.FindPage[*relation.GroupModel](ctx, g.coll, bson.M{ + "group_name": bson.M{"$regex": keyword}, + "status": bson.M{"$ne": constant.GroupStatusDismissed}, + }, pagination, opts) } + func (g *GroupMgo) CountTotal(ctx context.Context, before *time.Time) (count int64, err error) { if before == nil { return mongoutil.Count(ctx, g.coll, bson.M{}) diff --git a/pkg/rpcclient/group.go b/pkg/rpcclient/group.go index adc5d2d45..1756211db 100644 --- a/pkg/rpcclient/group.go +++ b/pkg/rpcclient/group.go @@ -47,11 +47,7 @@ func NewGroupRpcClient(discov discoveryregistry.SvcDiscoveryRegistry, rpcRegiste return GroupRpcClient(*NewGroup(discov, rpcRegisterName)) } -func (g *GroupRpcClient) GetGroupInfos( - ctx context.Context, - groupIDs []string, - complete bool, -) ([]*sdkws.GroupInfo, error) { +func (g *GroupRpcClient) GetGroupInfos(ctx context.Context, groupIDs []string, complete bool) ([]*sdkws.GroupInfo, error) { resp, err := g.Client.GetGroupsInfo(ctx, &group.GetGroupsInfoReq{ GroupIDs: groupIDs, }) @@ -182,11 +178,7 @@ func (g *GroupRpcClient) GetGroupInfoCache(ctx context.Context, groupID string) return resp.GroupInfo, nil } -func (g *GroupRpcClient) GetGroupMemberCache( - ctx context.Context, - groupID string, - groupMemberID string, -) (*sdkws.GroupMemberFullInfo, error) { +func (g *GroupRpcClient) GetGroupMemberCache(ctx context.Context, groupID string, groupMemberID string) (*sdkws.GroupMemberFullInfo, error) { resp, err := g.Client.GetGroupMemberCache(ctx, &group.GetGroupMemberCacheReq{ GroupID: groupID, GroupMemberID: groupMemberID, diff --git a/pkg/rpcclient/notification/friend.go b/pkg/rpcclient/notification/friend.go index 590b0427c..76438023b 100644 --- a/pkg/rpcclient/notification/friend.go +++ b/pkg/rpcclient/notification/friend.go @@ -126,10 +126,7 @@ func (f *FriendNotificationSender) UserInfoUpdatedNotification(ctx context.Conte return f.Notification(ctx, mcontext.GetOpUserID(ctx), changedUserID, constant.UserInfoUpdatedNotification, &tips) } -func (f *FriendNotificationSender) FriendApplicationAddNotification( - ctx context.Context, - req *pbfriend.ApplyToAddFriendReq, -) error { +func (f *FriendNotificationSender) FriendApplicationAddNotification(ctx context.Context, req *pbfriend.ApplyToAddFriendReq) error { tips := sdkws.FriendApplicationTips{FromToUserID: &sdkws.FromToUserID{ FromUserID: req.FromUserID, ToUserID: req.ToUserID, diff --git a/scripts/install/test.sh b/scripts/install/test.sh index 326307570..51d541ece 100755 --- a/scripts/install/test.sh +++ b/scripts/install/test.sh @@ -932,7 +932,7 @@ openim::test::set_group_info() { { "groupInfoForSet": { "groupID": "${1}", - "groupName": "new-name", + "groupName": "new group name", "notification": "new notification", "introduction": "new introduction", "faceURL": "www.newfaceURL.com", @@ -1076,6 +1076,7 @@ function openim::test::group() { local GROUP_ID=$RANDOM local GROUP_ID2=$RANDOM + # Assumes that TEST_GROUP_ID, USER_ID, and other necessary IDs are set as environment variables before running this suite. # 0. Register a friend user. openim::test::user_register "${USER_ID}" "group00" "new_face_url" diff --git a/scripts/lib/util.sh b/scripts/lib/util.sh index db0d4226b..13bd263ca 100755 --- a/scripts/lib/util.sh +++ b/scripts/lib/util.sh @@ -413,7 +413,7 @@ openim::util::check_process_names() { else # If there are PIDs, loop through each one for pid in "${pids[@]}"; do - local command=$(ps -p $pid -o cmd=) + local command=$(ps -p $pid -o comm=) local start_time=$(ps -p $pid -o lstart=) local port=$(get_port $pid) @@ -489,7 +489,7 @@ openim::util::check_process_names_for_stop() { else # If there are PIDs, loop through each one for pid in "${pids[@]}"; do - local command=$(ps -p $pid -o cmd=) + local command=$(ps -p $pid -o comm=) local start_time=$(ps -p $pid -o lstart=) local port=$(get_port $pid) diff --git a/scripts/make-rules/tools.mk b/scripts/make-rules/tools.mk index 5d39258ea..917c18cfe 100644 --- a/scripts/make-rules/tools.mk +++ b/scripts/make-rules/tools.mk @@ -237,6 +237,17 @@ install.richgo: install.rts: @$(GO) install github.com/galeone/rts/cmd/rts@latest +# ================= kubecub openim tools ========================================= +## install.typecheck: install kubecub typecheck check for go code +.PHONY: install.typecheck +install.typecheck: + @$(GO) install github.com/kubecub/typecheck@latest + +## install.comment-lang-detector: install kubecub comment-lang-detector check for go code comment language +.PHONY: install.comment-lang-detector +install.comment-lang-detector: + @$(GO) install github.com/kubecub/comment-lang-detector/cmd/cld@latest + ## tools.help: Display help information about the tools package .PHONY: tools.help tools.help: scripts/make-rules/tools.mk diff --git a/scripts/run-in-gopath.sh b/scripts/run-in-gopath.sh index 6af986975..6d8b7943b 100755 --- a/scripts/run-in-gopath.sh +++ b/scripts/run-in-gopath.sh @@ -20,10 +20,6 @@ # the project. # Usage: `scripts/run-in-gopath.sh `. - - - - OPENIM_ROOT=$(dirname "${BASH_SOURCE[0]}")/.. source "${OPENIM_ROOT}/scripts/lib/init.sh" diff --git a/scripts/update-generated-docs.sh b/scripts/update-generated-docs.sh index d48a4067b..4c1fbfccc 100755 --- a/scripts/update-generated-docs.sh +++ b/scripts/update-generated-docs.sh @@ -18,10 +18,6 @@ # immediately before exporting docs. We do not want to check these documents in # by default. - - - - OPENIM_ROOT=$(dirname "${BASH_SOURCE[0]}")/.. source "${OPENIM_ROOT}/scripts/lib/init.sh" @@ -33,7 +29,7 @@ BINS=( genman genyaml ) -make -C "${OPENIM_ROOT}" WHAT="${BINS[*]}" +make -C "${OPENIM_ROOT}" BINS="${BINS[*]}" openim::util::ensure-temp-dir diff --git a/scripts/update-yamlfmt.sh b/scripts/update-yamlfmt.sh index 24ec60de9..8de0cc84c 100755 --- a/scripts/update-yamlfmt.sh +++ b/scripts/update-yamlfmt.sh @@ -13,11 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. - - - - - OPENIM_ROOT=$(dirname "${BASH_SOURCE[0]}")/.. source "${OPENIM_ROOT}/scripts/lib/init.sh" diff --git a/scripts/verify-annotation-language.sh b/scripts/verify-annotation-language.sh deleted file mode 100755 index 6b863776c..000000000 --- a/scripts/verify-annotation-language.sh +++ /dev/null @@ -1,46 +0,0 @@ -#!/usr/bin/env bash -# Copyright © 2023 OpenIM. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -# This script verifies whether codes follow golang convention. -# Usage: `scripts/verify-pkg-names.sh`. - -set -o errexit - -OPENIM_ROOT=$(dirname "${BASH_SOURCE[0]}")/.. -source "${OPENIM_ROOT}/scripts/lib/init.sh" - -openim::golang::verify_go_version - -openim::golang::verify_go_version - -OPENIM_OUTPUT_HOSTBIN_TOOLS="${OPENIM_ROOT}/_output/bin/tools/linux/amd64" -CODESCAN_BINARY="${OPENIM_OUTPUT_HOSTBIN_TOOLS}/codescan" - -if [[ ! -f "${CODESCAN_BINARY}" ]]; then - echo "codescan binary not found, building..." - pushd "${OPENIM_ROOT}" >/dev/null - make build BINS="codescan" - popd >/dev/null -fi - -if [[ ! -f "${CODESCAN_BINARY}" ]]; then - echo "Failed to build codescan binary." - exit 1 -fi - -CONFIG_PATH="${OPENIM_ROOT}/tools/codescan/config.yaml" - -"${CODESCAN_BINARY}" -config "${CONFIG_PATH}" \ No newline at end of file diff --git a/scripts/verify-shellcheck.sh b/scripts/verify-shellcheck.sh index 0c4f165bf..3e56038dd 100755 --- a/scripts/verify-shellcheck.sh +++ b/scripts/verify-shellcheck.sh @@ -17,10 +17,6 @@ # This script lints each shell script by `shellcheck`. # Usage: `scripts/verify-shellcheck.sh`. - - - - OPENIM_ROOT=$(dirname "${BASH_SOURCE[0]}")/.. source "${OPENIM_ROOT}/scripts/lib/init.sh" diff --git a/scripts/verify-spelling.sh b/scripts/verify-spelling.sh index fa0852866..c718c1ad1 100755 --- a/scripts/verify-spelling.sh +++ b/scripts/verify-spelling.sh @@ -17,10 +17,6 @@ # working directory by client9/misspell package. # Usage: `scripts/verify-spelling.sh`. - - - - OPENIM_ROOT=$(dirname "${BASH_SOURCE[0]}")/.. export OPENIM_ROOT source "${OPENIM_ROOT}/scripts/lib/init.sh" diff --git a/scripts/verify-typecheck.sh b/scripts/verify-typecheck.sh index c9b2aaf30..f6c14844f 100755 --- a/scripts/verify-typecheck.sh +++ b/scripts/verify-typecheck.sh @@ -16,26 +16,19 @@ # This script does a fast type check of script srnetes code for all platforms. # Usage: `scripts/verify-typecheck.sh`. - - - - OPENIM_ROOT=$(dirname "${BASH_SOURCE[0]}")/.. source "${OPENIM_ROOT}/scripts/lib/init.sh" openim::golang::verify_go_version cd "${OPENIM_ROOT}" - -# As of June, 2020 the typecheck tool is written in terms of go/packages, but -# that library doesn't work well with multiple modules. Until that is done, -# force this tooling to run in a fake GOPATH. ret=0 TYPECHECK_SERIAL="${TYPECHECK_SERIAL:-false}" scripts/run-in-gopath.sh \ -go run test/typecheck/typecheck.go "$@" "--serial=$TYPECHECK_SERIAL" || ret=$? +make tools.verify.typecheck +${OPENIM_ROOT}/_output/tools/typecheck "$@" "--serial=$TYPECHECK_SERIAL" || ret=$? if [[ $ret -ne 0 ]]; then openim::log::error "Type Check has failed. This may cause cross platform build failures." >&2 - openim::log::error "Please see https://github.com/openimsdk/open-im-server/tree/main/test/typecheck for more information." >&2 + openim::log::error "Please see https://github.com/kubecub/typecheck for more information." >&2 exit 1 fi diff --git a/scripts/verify-yamlfmt.sh b/scripts/verify-yamlfmt.sh index 3acbf457c..a0aa583a8 100755 --- a/scripts/verify-yamlfmt.sh +++ b/scripts/verify-yamlfmt.sh @@ -19,10 +19,6 @@ # # Usage: `scripts/verify-yamlfmt.sh`. - - - - OPENIM_ROOT=$(dirname "${BASH_SOURCE[0]}")/.. source "${OPENIM_ROOT}/scripts/lib/init.sh" diff --git a/test/typecheck/README.md b/test/typecheck/README.md deleted file mode 100644 index e5b76d4c6..000000000 --- a/test/typecheck/README.md +++ /dev/null @@ -1,52 +0,0 @@ -# OpenIM Typecheck: Cross-Platform Source Code Type Checking for Go - -## Introduction - -OpenIM Typecheck is a robust tool designed for cross-platform source code type checking across all Go build platforms. This utility leverages Go’s built-in parsing and type-check libraries (`go/parser` and `go/types`) to deliver efficient and reliable code analysis. - -## Advantages - -- **Speed**: A complete compilation with OpenIM can take approximately 3 minutes. In contrast, OpenIM Typecheck achieves this in mere seconds, significantly enhancing productivity. -- **Resource Efficiency**: Unlike the typical requirement of over 40GB of RAM for standard processes, Typecheck operates effectively with less than 8GB of RAM. This reduction in resource consumption makes it highly suitable for a variety of systems, reducing overheads and facilitating smoother operations. - -## Implementation - -OpenIM Typecheck employs Go's native parsing and type-checking libraries (`go/parser` and `go/types`). However, it's important to note that these libraries aren't identical to those used by the Go compiler. While occasional mismatches may occur, these libraries generally provide close approximations to the compiler's functionality, offering a reliable basis for type checking. - -## Error Handling - -Typecheck's approach to error handling is pragmatic, focusing on practicality and build continuity. - -**Errors reported by `go/types` but not by `go build`**: -- **Actual Errors** (as per the specification): - - These should ideally be rectified. If rectification is not feasible, such as in cases of ongoing work or external dependencies in the code, these errors can be overlooked. - - Example: Unused variables within a closure. -- **False Positives**: - - These errors should be ignored and, where appropriate, reported upstream for resolution. - - Example: Type mismatches between staging and generated types. - -**Errors reported by `go build` but not by us**: -- CGo-related errors, including both syntax and linker issues, are outside our scope. - -## Usage - -### Locally - -To run Typecheck locally, simply use the following command: - -```bash -make verify -``` - -### Continuous Integration (CI) - -In CI environments, Typecheck can be integrated into the workflow as follows: - -```yaml -- name: Typecheck - run: make verify -``` - -This streamlined process facilitates efficient error detection and resolution, ensuring a robust and reliable build pipeline. - -More to learn about typecheck [share blog](https://nsddd.top/posts/concurrent-type-checking-and-cross-platform-development-in-go/) \ No newline at end of file diff --git a/test/typecheck/go.mod b/test/typecheck/go.mod deleted file mode 100644 index 9ef1b1da7..000000000 --- a/test/typecheck/go.mod +++ /dev/null @@ -1,10 +0,0 @@ -module github.com/openimsdk/open-im-server/test/typecheck - -go 1.19 - -require golang.org/x/tools v0.12.0 - -require ( - golang.org/x/mod v0.12.0 // indirect - golang.org/x/sys v0.11.0 // indirect -) diff --git a/test/typecheck/go.sum b/test/typecheck/go.sum deleted file mode 100644 index 14a66101b..000000000 --- a/test/typecheck/go.sum +++ /dev/null @@ -1,7 +0,0 @@ -golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc= -golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= -golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM= -golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/tools v0.12.0 h1:YW6HUoUmYBpwSgyaGaZq1fHjrBjX1rlpZ54T6mu2kss= -golang.org/x/tools v0.12.0/go.mod h1:Sc0INKfu04TlqNoRA1hgpFZbhYXHPr4V5DzpSBTPqQM= diff --git a/test/typecheck/typecheck.go b/test/typecheck/typecheck.go deleted file mode 100644 index 975ce988d..000000000 --- a/test/typecheck/typecheck.go +++ /dev/null @@ -1,319 +0,0 @@ -// Copyright © 2023 OpenIM. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// do a fast type check of openim code, for all platforms. -package main - -import ( - "flag" - "fmt" - "io" - "log" - "os" - "path/filepath" - "sort" - "strings" - "sync" - "time" - - "golang.org/x/tools/go/packages" -) - -var ( - verbose = flag.Bool("verbose", false, "print more information") - cross = flag.Bool("cross", true, "build for all platforms") - platforms = flag.String("platform", "", "comma-separated list of platforms to typecheck") - timings = flag.Bool("time", false, "output times taken for each phase") - defuses = flag.Bool("defuse", false, "output defs/uses") - serial = flag.Bool("serial", false, "don't type check platforms in parallel (equivalent to --parallel=1)") - parallel = flag.Int("parallel", 2, "limits how many platforms can be checked in parallel. 0 means no limit.") - skipTest = flag.Bool("skip-test", false, "don't type check test code") - tags = flag.String("tags", "", "comma-separated list of build tags to apply in addition to go's defaults") - ignoreDirs = flag.String("ignore-dirs", "", "comma-separated list of directories to ignore in addition to the default hardcoded list including staging, vendor, and hidden dirs") - - // When processed in order, windows and darwin are early to make - // interesting OS-based errors happen earlier. - crossPlatforms = []string{ - "linux/amd64", "windows/386", - "darwin/amd64", "darwin/arm64", - "linux/386", "linux/arm", - "windows/amd64", "linux/arm64", - "linux/ppc64le", "linux/s390x", - "windows/arm64", - } - - // directories we always ignore - standardIgnoreDirs = []string{ - // Staging code is symlinked from vendor/k8s.io, and uses import - // paths as if it were inside of vendor/. It fails typechecking - // inside of staging/, but works when typechecked as part of vendor/. - "staging", - "components", - "logs", - // OS-specific vendor code tends to be imported by OS-specific - // packages. We recursively typecheck imported vendored packages for - // each OS, but don't typecheck everything for every OS. - "vendor", - "test", - "_output", - "*/mw/rpc_server_interceptor.go", - // Tools we use for maintaining the code base but not necessarily - // ship as part of the release - "sopenim::golang::setup_env:tools/yamlfmt/yamlfmt.go:tools", - } -) - -func newConfig(platform string) *packages.Config { - platSplit := strings.Split(platform, "/") - goos, goarch := platSplit[0], platSplit[1] - mode := packages.NeedName | packages.NeedFiles | packages.NeedTypes | packages.NeedSyntax | packages.NeedDeps | packages.NeedImports | packages.NeedModule - if *defuses { - mode = mode | packages.NeedTypesInfo - } - env := append(os.Environ(), - "CGO_ENABLED=1", - fmt.Sprintf("GOOS=%s", goos), - fmt.Sprintf("GOARCH=%s", goarch)) - tagstr := "selinux" - if *tags != "" { - tagstr = tagstr + "," + *tags - } - flags := []string{"-tags", tagstr} - - return &packages.Config{ - Mode: mode, - Env: env, - BuildFlags: flags, - Tests: !(*skipTest), - } -} - -type collector struct { - dirs []string - ignoreDirs []string -} - -func newCollector(ignoreDirs string) collector { - c := collector{ - ignoreDirs: append([]string(nil), standardIgnoreDirs...), - } - if ignoreDirs != "" { - c.ignoreDirs = append(c.ignoreDirs, strings.Split(ignoreDirs, ",")...) - } - return c -} - -func (c *collector) walk(roots []string) error { - for _, root := range roots { - err := filepath.Walk(root, c.handlePath) - if err != nil { - return err - } - } - sort.Strings(c.dirs) - return nil -} - -// handlePath walks the filesystem recursively, collecting directories, -// ignoring some unneeded directories (hidden/vendored) that are handled -// specially later. -func (c *collector) handlePath(path string, info os.FileInfo, err error) error { - if err != nil { - return err - } - if info.IsDir() { - name := info.Name() - // Ignore hidden directories (.git, .cache, etc) - if (len(name) > 1 && (name[0] == '.' || name[0] == '_')) || name == "testdata" { - if *verbose { - fmt.Printf("DBG: skipping dir %s\n", path) - } - return filepath.SkipDir - } - for _, dir := range c.ignoreDirs { - if path == dir { - if *verbose { - fmt.Printf("DBG: ignoring dir %s\n", path) - } - return filepath.SkipDir - } - } - // Make dirs into relative pkg names. - // NOTE: can't use filepath.Join because it elides the leading "./" - pkg := path - if !strings.HasPrefix(pkg, "./") { - pkg = "./" + pkg - } - c.dirs = append(c.dirs, pkg) - if *verbose { - fmt.Printf("DBG: added dir %s\n", path) - } - } - return nil -} - -func (c *collector) verify(plat string) ([]string, error) { - errors := []packages.Error{} - start := time.Now() - config := newConfig(plat) - - rootPkgs, err := packages.Load(config, c.dirs...) - if err != nil { - return nil, err - } - - // Recursively import all deps and flatten to one list. - allMap := map[string]*packages.Package{} - for _, pkg := range rootPkgs { - if *verbose { - serialFprintf(os.Stdout, "pkg %q has %d GoFiles\n", pkg.PkgPath, len(pkg.GoFiles)) - } - allMap[pkg.PkgPath] = pkg - if len(pkg.Imports) > 0 { - for _, imp := range pkg.Imports { - if *verbose { - serialFprintf(os.Stdout, "pkg %q imports %q\n", pkg.PkgPath, imp.PkgPath) - } - allMap[imp.PkgPath] = imp - } - } - } - keys := make([]string, 0, len(allMap)) - for k := range allMap { - keys = append(keys, k) - } - sort.Strings(keys) - allList := make([]*packages.Package, 0, len(keys)) - for _, k := range keys { - allList = append(allList, allMap[k]) - } - - for _, pkg := range allList { - if len(pkg.GoFiles) > 0 { - if len(pkg.Errors) > 0 && (pkg.PkgPath == "main" || strings.Contains(pkg.PkgPath, ".")) { - errors = append(errors, pkg.Errors...) - } - } - if *defuses { - for id, obj := range pkg.TypesInfo.Defs { - serialFprintf(os.Stdout, "%s: %q defines %v\n", - pkg.Fset.Position(id.Pos()), id.Name, obj) - } - for id, obj := range pkg.TypesInfo.Uses { - serialFprintf(os.Stdout, "%s: %q uses %v\n", - pkg.Fset.Position(id.Pos()), id.Name, obj) - } - } - } - if *timings { - serialFprintf(os.Stdout, "%s took %.1fs\n", plat, time.Since(start).Seconds()) - } - return dedup(errors), nil -} - -func dedup(errors []packages.Error) []string { - ret := []string{} - - m := map[string]bool{} - for _, e := range errors { - es := e.Error() - if !m[es] { - ret = append(ret, es) - m[es] = true - } - } - return ret -} - -var outMu sync.Mutex - -func serialFprintf(w io.Writer, format string, a ...any) (n int, err error) { - outMu.Lock() - defer outMu.Unlock() - return fmt.Fprintf(w, format, a...) -} - -func main() { - flag.Parse() - args := flag.Args() - - if *verbose { - *serial = true // to avoid confusing interleaved logs - } - - if len(args) == 0 { - args = append(args, ".") - } - - c := newCollector(*ignoreDirs) - - if err := c.walk(args); err != nil { - log.Fatalf("Error walking: %v", err) - } - - plats := crossPlatforms[:] - if *platforms != "" { - plats = strings.Split(*platforms, ",") - } else if !*cross { - plats = plats[:1] - } - - var wg sync.WaitGroup - var failMu sync.Mutex - failed := false - - if *serial { - *parallel = 1 - } else if *parallel == 0 { - *parallel = len(plats) - } - throttle := make(chan int, *parallel) - - for _, plat := range plats { - wg.Add(1) - go func(plat string) { - // block until there's room for this task - throttle <- 1 - defer func() { - // indicate this task is done - <-throttle - }() - - f := false - serialFprintf(os.Stdout, "type-checking %s\n", plat) - errors, err := c.verify(plat) - if err != nil { - serialFprintf(os.Stderr, "ERROR(%s): failed to verify: %v\n", plat, err) - f = true - } else if len(errors) > 0 { - for _, e := range errors { - // Special case CGo errors which may depend on headers we - // don't have. - if !strings.HasSuffix(e, "could not import C (no metadata for C)") { - f = true - serialFprintf(os.Stderr, "ERROR(%s): %s\n", plat, e) - } - } - } - failMu.Lock() - failed = failed || f - failMu.Unlock() - wg.Done() - }(plat) - } - wg.Wait() - if failed { - os.Exit(1) - } -} diff --git a/test/typecheck/typecheck_test.go b/test/typecheck/typecheck_test.go deleted file mode 100644 index 3f6924cbd..000000000 --- a/test/typecheck/typecheck_test.go +++ /dev/null @@ -1,121 +0,0 @@ -// Copyright © 2023 OpenIM. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package main - -import ( - "errors" - "flag" - "os" - "path/filepath" - "testing" - - "golang.org/x/tools/go/packages" -) - -// This exists because `go` is not always in the PATH when running CI. -var goBinary = flag.String("go", "", "path to a `go` binary") - -func TestVerify(t *testing.T) { - // x/tools/packages is going to literally exec `go`, so it needs some - // setup. - setEnvVars(t) - - tcs := []struct { - path string - expect int - }{ - // {"./testdata/good", 0}, - // {"./testdata/bad", 18}, - } - - for _, tc := range tcs { - c := newCollector("") - if err := c.walk([]string{tc.path}); err != nil { - t.Fatalf("error walking %s: %v", tc.path, err) - } - - errs, err := c.verify("linux/amd64") - if err != nil { - t.Errorf("unexpected error: %v", err) - } else if len(errs) != tc.expect { - t.Errorf("Expected %d errors, got %d: %v", tc.expect, len(errs), errs) - } - } -} - -func setEnvVars(t testing.TB) { - t.Helper() - if *goBinary != "" { - newPath := filepath.Dir(*goBinary) - curPath := os.Getenv("PATH") - if curPath != "" { - newPath = newPath + ":" + curPath - } - t.Setenv("PATH", newPath) - } - if os.Getenv("HOME") == "" { - t.Setenv("HOME", "/tmp") - } -} - -func TestHandlePath(t *testing.T) { - c := collector{ - ignoreDirs: standardIgnoreDirs, - } - e := errors.New("ex") - i, _ := os.Stat(".") // i.IsDir() == true - if c.handlePath("foo", nil, e) != e { - t.Error("handlePath not returning errors") - } - if c.handlePath("vendor", i, nil) != filepath.SkipDir { - t.Error("should skip vendor") - } -} - -func TestDedup(t *testing.T) { - testcases := []struct { - input []packages.Error - expected int - }{{ - input: nil, - expected: 0, - }, { - input: []packages.Error{ - {Pos: "file:7", Msg: "message", Kind: packages.ParseError}, - }, - expected: 1, - }, { - input: []packages.Error{ - {Pos: "file:7", Msg: "message1", Kind: packages.ParseError}, - {Pos: "file:8", Msg: "message2", Kind: packages.ParseError}, - }, - expected: 2, - }, { - input: []packages.Error{ - {Pos: "file:7", Msg: "message1", Kind: packages.ParseError}, - {Pos: "file:8", Msg: "message2", Kind: packages.ParseError}, - {Pos: "file:7", Msg: "message1", Kind: packages.ParseError}, - }, - expected: 2, - }} - - for i, tc := range testcases { - out := dedup(tc.input) - if len(out) != tc.expected { - t.Errorf("[%d] dedup(%v) = '%v', expected %d", - i, tc.input, out, tc.expected) - } - } -} diff --git a/tools/codescan/checker/checker.go b/tools/codescan/checker/checker.go deleted file mode 100644 index ad724dd5b..000000000 --- a/tools/codescan/checker/checker.go +++ /dev/null @@ -1,104 +0,0 @@ -// Copyright © 2024 OpenIM. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package checker - -import ( - "bufio" - "fmt" - "os" - "path/filepath" - "regexp" - "strings" - - "github.com/openimsdk/open-im-server/tools/codescan/config" -) - -type CheckResult struct { - FilePath string - Lines []int -} - -func checkFileForChineseComments(filePath string) ([]CheckResult, error) { - file, err := os.Open(filePath) - if err != nil { - return nil, err - } - defer file.Close() - - var results []CheckResult - scanner := bufio.NewScanner(file) - reg := regexp.MustCompile(`[\p{Han}]+`) - lineNumber := 0 - - var linesWithChinese []int - for scanner.Scan() { - lineNumber++ - if reg.FindString(scanner.Text()) != "" { - linesWithChinese = append(linesWithChinese, lineNumber) - } - } - - if len(linesWithChinese) > 0 { - results = append(results, CheckResult{ - FilePath: filePath, - Lines: linesWithChinese, - }) - } - - if err := scanner.Err(); err != nil { - return nil, err - } - - return results, nil -} - -func WalkDirAndCheckComments(cfg config.Config) error { - var allResults []CheckResult - err := filepath.Walk(cfg.Directory, func(path string, info os.FileInfo, err error) error { - if err != nil { - return err - } - if info.IsDir() { - return nil - } - for _, fileType := range cfg.FileTypes { - if filepath.Ext(path) == fileType { - results, err := checkFileForChineseComments(path) - if err != nil { - return err - } - if len(results) > 0 { - allResults = append(allResults, results...) - } - } - } - return nil - }) - - if err != nil { - return err - } - - if len(allResults) > 0 { - var errMsg strings.Builder - errMsg.WriteString("Files containing Chinese comments:\n") - for _, result := range allResults { - errMsg.WriteString(fmt.Sprintf("%s: Lines %v\n", result.FilePath, result.Lines)) - } - return fmt.Errorf(errMsg.String()) - } - - return nil -} diff --git a/tools/codescan/codescan.go b/tools/codescan/codescan.go deleted file mode 100644 index a83e895fc..000000000 --- a/tools/codescan/codescan.go +++ /dev/null @@ -1,34 +0,0 @@ -// Copyright © 2024 OpenIM. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package main - -import ( - "log" - - "github.com/openimsdk/open-im-server/tools/codescan/checker" - "github.com/openimsdk/open-im-server/tools/codescan/config" -) - -func main() { - cfg, err := config.ParseConfig() - if err != nil { - log.Fatalf("Error parsing config: %v", err) - } - - err = checker.WalkDirAndCheckComments(cfg) - if err != nil { - panic(err) - } -} diff --git a/tools/codescan/config/config.go b/tools/codescan/config/config.go deleted file mode 100644 index aebf0d4f7..000000000 --- a/tools/codescan/config/config.go +++ /dev/null @@ -1,49 +0,0 @@ -// Copyright © 2024 OpenIM. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package config - -import ( - "flag" - "log" - "os" - - "gopkg.in/yaml.v2" -) - -type Config struct { - Directory string `yaml:"directory"` - FileTypes []string `yaml:"file_types"` - Languages []string `yaml:"languages"` -} - -func ParseConfig() (Config, error) { - var configPath string - flag.StringVar(&configPath, "config", "./", "Path to config file") - flag.Parse() - - var config Config - if configPath != "" { - configFile, err := os.ReadFile(configPath) - if err != nil { - return Config{}, err - } - if err := yaml.Unmarshal(configFile, &config); err != nil { - return Config{}, err - } - } else { - log.Fatal("Config file must be provided") - } - return config, nil -} diff --git a/tools/codescan/go.mod b/tools/codescan/go.mod deleted file mode 100644 index 2ad132101..000000000 --- a/tools/codescan/go.mod +++ /dev/null @@ -1,3 +0,0 @@ -module github.com/openimsdk/open-im-server/tools/codescan - -go 1.19 diff --git a/tools/component/component.go b/tools/component/component.go index be913d0be..485573837 100644 --- a/tools/component/component.go +++ b/tools/component/component.go @@ -96,13 +96,14 @@ func main() { } checks := []checkFunc{ - //{name: "Mysql", function: checkMysql}, {name: "Mongo", function: checkMongo, config: conf}, {name: "Redis", function: checkRedis, config: conf}, - {name: "Minio", function: checkMinio, config: conf}, {name: "Zookeeper", function: checkZookeeper, config: conf}, {name: "Kafka", function: checkKafka, config: conf}, } + if conf.Object.Enable == "minio" { + checks = append(checks, checkFunc{name: "Minio", function: checkMinio, config: conf}) + } for i := 0; i < maxRetry; i++ { if i != 0 {