feat: improve searchMsg implement. (#2614)

* refactor: refactor workflows contents.

* add tool workflows.

* update field.

* fix: remove chat error.

* Fix err.

* fix error.

* remove cn comment.

* update workflows files.

* update infra config.

* move workflows.

* feat: update bot.

* fix: solve uncorrect outdated msg get.

* update get docIDs logic.

* update

* update skip logic.

* fix

* update.

* fix: delay deleteObject func.

* remove unused content.

* update log type.

* feat: implement request batch count limit.

* update

* update

* remove unused script.

* feat: improve searchMsg implement.

* update mongo config.
pull/2638/head
Monet Lee 3 months ago committed by GitHub
parent 12b284d840
commit 9424e3ed5f
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194

@ -12,7 +12,7 @@ require (
github.com/gorilla/websocket v1.5.1
github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0
github.com/mitchellh/mapstructure v1.5.0
github.com/openimsdk/protocol v0.0.72-alpha.18
github.com/openimsdk/protocol v0.0.72-alpha.20
github.com/openimsdk/tools v0.0.50-alpha.12
github.com/pkg/errors v0.9.1 // indirect
github.com/prometheus/client_golang v1.18.0

@ -322,8 +322,8 @@ github.com/onsi/gomega v1.25.0 h1:Vw7br2PCDYijJHSfBOWhov+8cAnUf8MfMaIOV323l6Y=
github.com/onsi/gomega v1.25.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdMPGhM=
github.com/openimsdk/gomake v0.0.14-alpha.5 h1:VY9c5x515lTfmdhhPjMvR3BBRrRquAUCFsz7t7vbv7Y=
github.com/openimsdk/gomake v0.0.14-alpha.5/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI=
github.com/openimsdk/protocol v0.0.72-alpha.18 h1:EytTtgZuXMG1cgTlJryqXXSO1J3t3wrLIn3Os2PRBEE=
github.com/openimsdk/protocol v0.0.72-alpha.18/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8=
github.com/openimsdk/protocol v0.0.72-alpha.20 h1:kfSYOnWRp9KKkwGelR9Zo20TdjMq5LLzfYKyVqUaolo=
github.com/openimsdk/protocol v0.0.72-alpha.20/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8=
github.com/openimsdk/tools v0.0.50-alpha.12 h1:rV3BxgqN+F79vZvdoQ+97Eob8ScsRVEM8D+Wrcl23uo=
github.com/openimsdk/tools v0.0.50-alpha.12/go.mod h1:h1cYmfyaVtgFbKmb1Cfsl8XwUOMTt8ubVUQrdGtsUh4=
github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM=

@ -16,6 +16,7 @@ package msg
import (
"context"
"github.com/openimsdk/open-im-server/v3/pkg/authverify"
"github.com/openimsdk/open-im-server/v3/pkg/msgprocessor"
"github.com/openimsdk/open-im-server/v3/pkg/util/conversationutil"
@ -144,7 +145,8 @@ func (m *msgServer) GetMaxSeq(ctx context.Context, req *sdkws.GetMaxSeqReq) (*sd
}
func (m *msgServer) SearchMessage(ctx context.Context, req *msg.SearchMessageReq) (resp *msg.SearchMessageResp, err error) {
var chatLogs []*sdkws.MsgData
// var chatLogs []*sdkws.MsgData
var chatLogs []*msg.SearchedMsgData
var total int64
resp = &msg.SearchMessageResp{}
if total, chatLogs, err = m.MsgDatabase.SearchMessage(ctx, req); err != nil {
@ -159,17 +161,19 @@ func (m *msgServer) SearchMessage(ctx context.Context, req *msg.SearchMessageReq
recvMap = make(map[string]string)
groupMap = make(map[string]*sdkws.GroupInfo)
)
for _, chatLog := range chatLogs {
if chatLog.SenderNickname == "" {
sendIDs = append(sendIDs, chatLog.SendID)
if chatLog.MsgData.SenderNickname == "" {
sendIDs = append(sendIDs, chatLog.MsgData.SendID)
}
switch chatLog.SessionType {
switch chatLog.MsgData.SessionType {
case constant.SingleChatType, constant.NotificationChatType:
recvIDs = append(recvIDs, chatLog.RecvID)
recvIDs = append(recvIDs, chatLog.MsgData.RecvID)
case constant.WriteGroupChatType, constant.ReadGroupChatType:
groupIDs = append(groupIDs, chatLog.GroupID)
groupIDs = append(groupIDs, chatLog.MsgData.GroupID)
}
}
// Retrieve sender and receiver information
if len(sendIDs) != 0 {
sendInfos, err := m.UserLocalCache.GetUsersInfo(ctx, sendIDs)
@ -180,6 +184,7 @@ func (m *msgServer) SearchMessage(ctx context.Context, req *msg.SearchMessageReq
sendMap[sendInfo.UserID] = sendInfo.Nickname
}
}
if len(recvIDs) != 0 {
recvInfos, err := m.UserLocalCache.GetUsersInfo(ctx, recvIDs)
if err != nil {
@ -205,20 +210,21 @@ func (m *msgServer) SearchMessage(ctx context.Context, req *msg.SearchMessageReq
}
}
}
// Construct response with updated information
for _, chatLog := range chatLogs {
pbchatLog := &msg.ChatLog{}
datautil.CopyStructFields(pbchatLog, chatLog)
pbchatLog.SendTime = chatLog.SendTime
pbchatLog.CreateTime = chatLog.CreateTime
if chatLog.SenderNickname == "" {
pbchatLog.SenderNickname = sendMap[chatLog.SendID]
datautil.CopyStructFields(pbchatLog, chatLog.MsgData)
pbchatLog.SendTime = chatLog.MsgData.SendTime
pbchatLog.CreateTime = chatLog.MsgData.CreateTime
if chatLog.MsgData.SenderNickname == "" {
pbchatLog.SenderNickname = sendMap[chatLog.MsgData.SendID]
}
switch chatLog.SessionType {
switch chatLog.MsgData.SessionType {
case constant.SingleChatType, constant.NotificationChatType:
pbchatLog.RecvNickname = recvMap[chatLog.RecvID]
case constant.WriteGroupChatType, constant.ReadGroupChatType:
groupInfo := groupMap[chatLog.GroupID]
pbchatLog.RecvNickname = recvMap[chatLog.MsgData.RecvID]
case constant.ReadGroupChatType:
groupInfo := groupMap[chatLog.MsgData.GroupID]
pbchatLog.SenderFaceURL = groupInfo.FaceURL
pbchatLog.GroupMemberCount = groupInfo.MemberCount // Reflects actual member count
pbchatLog.RecvID = groupInfo.GroupID
@ -226,7 +232,9 @@ func (m *msgServer) SearchMessage(ctx context.Context, req *msg.SearchMessageReq
pbchatLog.GroupOwner = groupInfo.OwnerUserID
pbchatLog.GroupType = groupInfo.GroupType
}
resp.ChatLogs = append(resp.ChatLogs, pbchatLog)
searchChatLog := &msg.SearchChatLog{ChatLog: pbchatLog, IsRevoked: chatLog.IsRevoked}
resp.ChatLogs = append(resp.ChatLogs, searchChatLog)
}
resp.ChatLogsNum = int32(total)
return resp, nil

@ -17,9 +17,10 @@ package third
import (
"context"
"crypto/rand"
relationtb "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model"
"time"
relationtb "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model"
"github.com/openimsdk/open-im-server/v3/pkg/authverify"
"github.com/openimsdk/open-im-server/v3/pkg/common/servererrs"
"github.com/openimsdk/protocol/constant"

@ -78,7 +78,7 @@ type CommonMsgDatabase interface {
//GetConversationMinMaxSeqInMongoAndCache(ctx context.Context, conversationID string) (minSeqMongo, maxSeqMongo, minSeqCache, maxSeqCache int64, err error)
SetSendMsgStatus(ctx context.Context, id string, status int32) error
GetSendMsgStatus(ctx context.Context, id string) (int32, error)
SearchMessage(ctx context.Context, req *pbmsg.SearchMessageReq) (total int64, msgData []*sdkws.MsgData, err error)
SearchMessage(ctx context.Context, req *pbmsg.SearchMessageReq) (total int64, msgData []*pbmsg.SearchedMsgData, err error)
FindOneByDocIDs(ctx context.Context, docIDs []string, seqs map[string]int64) (map[string]*sdkws.MsgData, error)
// to mq
@ -747,8 +747,8 @@ func (db *commonMsgDatabase) RangeGroupSendCount(
return db.msgDocDatabase.RangeGroupSendCount(ctx, start, end, ase, pageNumber, showNumber)
}
func (db *commonMsgDatabase) SearchMessage(ctx context.Context, req *pbmsg.SearchMessageReq) (total int64, msgData []*sdkws.MsgData, err error) {
var totalMsgs []*sdkws.MsgData
func (db *commonMsgDatabase) SearchMessage(ctx context.Context, req *pbmsg.SearchMessageReq) (total int64, msgData []*pbmsg.SearchedMsgData, err error) {
var totalMsgs []*pbmsg.SearchedMsgData
total, msgs, err := db.msgDocDatabase.SearchMessage(ctx, req)
if err != nil {
return 0, nil, err
@ -757,7 +757,13 @@ func (db *commonMsgDatabase) SearchMessage(ctx context.Context, req *pbmsg.Searc
if msg.IsRead {
msg.Msg.IsRead = true
}
totalMsgs = append(totalMsgs, convert.MsgDB2Pb(msg.Msg))
searchedMsgData := &pbmsg.SearchedMsgData{MsgData: convert.MsgDB2Pb(msg.Msg)}
if msg.Revoke != nil {
searchedMsgData.IsRevoked = true
}
totalMsgs = append(totalMsgs, searchedMsgData)
}
return total, totalMsgs, nil
}

@ -16,12 +16,13 @@ package controller
import (
"context"
"time"
"github.com/openimsdk/open-im-server/v3/pkg/common/storage/database"
"github.com/openimsdk/open-im-server/v3/pkg/common/storage/model"
"github.com/openimsdk/tools/db/pagination"
"github.com/openimsdk/tools/db/tx"
"github.com/openimsdk/tools/utils/datautil"
"time"
"github.com/openimsdk/protocol/user"
"github.com/openimsdk/tools/errs"
@ -111,10 +112,14 @@ func (u *userDatabase) InitOnce(ctx context.Context, users []*model.User) error
// FindWithError Get the information of the specified user and return an error if the userID is not found.
func (u *userDatabase) FindWithError(ctx context.Context, userIDs []string) (users []*model.User, err error) {
userIDs = datautil.Distinct(userIDs)
// TODO: Add logic to identify which user IDs are distinct and which user IDs were not found.
users, err = u.cache.GetUsersInfo(ctx, userIDs)
if err != nil {
return
}
if len(users) != len(userIDs) {
err = errs.ErrRecordNotFound.WrapMsg("userID not found")
}

@ -1,92 +0,0 @@
#!/usr/bin/env bash
# Copyright © 2023 OpenIMSDK.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# ==============================================================================
#
# Store this file as .git/hooks/commit-msg in your repository in order to
# enforce checking for proper commit message format before actual commits.
# You may need to make the scripts executable by 'chmod +x .git/hooks/commit-msg'.
# commit-msg use go-gitlint tool, install go-gitlint via `go get github.com/llorllale/go-gitlint/cmd/go-gitlint`
# go-gitlint --msg-file="$1"
# An example hook scripts to check the commit log message.
# Called by "git commit" with one argument, the name of the file
# that has the commit message. The hook should exit with non-zero
# status after issuing an appropriate message if it wants to stop the
# commit. The hook is allowed to edit the commit message file.
YELLOW="\e[93m"
GREEN="\e[32m"
RED="\e[31m"
ENDCOLOR="\e[0m"
printMessage() {
printf "${YELLOW}OpenIM : $1${ENDCOLOR}\n"
}
printSuccess() {
printf "${GREEN}OpenIM : $1${ENDCOLOR}\n"
}
printError() {
printf "${RED}OpenIM : $1${ENDCOLOR}\n"
}
printMessage "Running the OpenIM commit-msg hook."
# This example catches duplicate Signed-off-by lines.
test "" = "$(grep '^Signed-off-by: ' "$1" |
sort | uniq -c | sed -e '/^[ ]*1[ ]/d')" || {
echo >&2 Duplicate Signed-off-by lines.
exit 1
}
# TODO: go-gitlint dir set
OPENIM_ROOT=$(dirname "${BASH_SOURCE[0]}")/../..
GITLINT_DIR="$OPENIM_ROOT/_output/tools/go-gitlint"
$GITLINT_DIR \
--msg-file=$1 \
--subject-regex="^(build|chore|ci|docs|feat|feature|fix|perf|refactor|revert|style|bot|test)(.*)?:\s?.*" \
--subject-maxlen=150 \
--subject-minlen=10 \
--body-regex=".*" \
--max-parents=1
if [ $? -ne 0 ]
then
if ! command -v $GITLINT_DIR &>/dev/null; then
printError "$GITLINT_DIR not found. Please run 'make tools' OR 'make tools.verify.go-gitlint' make verto install it."
fi
printError "Please fix your commit message to match kubecub coding standards"
printError "https://gist.github.com/cubxxw/126b72104ac0b0ca484c9db09c3e5694#file-githook-md"
exit 1
fi
### Add Sign-off-by line to the end of the commit message
# Get local git config
NAME=$(git config user.name)
EMAIL=$(git config user.email)
# Check if the commit message contains a sign-off line
grep -qs "^Signed-off-by: " "$1"
SIGNED_OFF_BY_EXISTS=$?
# Add "Signed-off-by" line if it doesn't exist
if [ $SIGNED_OFF_BY_EXISTS -ne 0 ]; then
echo -e "\nSigned-off-by: $NAME <$EMAIL>" >> "$1"
fi

@ -1,111 +0,0 @@
#!/usr/bin/env bash
# Copyright © 2023 OpenIMSDK.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# ==============================================================================
# This is a pre-commit hook that ensures attempts to commit files that are
# are larger than $limit to your _local_ repo fail, with a helpful error message.
# You can override the default limit of 2MB by supplying the environment variable:
# GIT_FILE_SIZE_LIMIT=50000000 git commit -m "test: this commit is allowed file sizes up to 50MB"
#
# ==============================================================================
#
LC_ALL=C
local_branch="$(git rev-parse --abbrev-ref HEAD)"
valid_branch_regex="^(main|master|develop|release(-[a-zA-Z0-9._-]+)?)$|(feature|feat|openim|hotfix|test|bug|bot|refactor|revert|ci|cicd|style|)\/[a-z0-9._-]+$|^HEAD$"
YELLOW="\e[93m"
GREEN="\e[32m"
RED="\e[31m"
ENDCOLOR="\e[0m"
printMessage() {
printf "${YELLOW}openim : $1${ENDCOLOR}\n"
}
printSuccess() {
printf "${GREEN}openim : $1${ENDCOLOR}\n"
}
printError() {
printf "${RED}openim : $1${ENDCOLOR}\n"
}
printMessage "Running local openim pre-commit hook."
# flutter format .
# https://gist.github.com/cubxxw/126b72104ac0b0ca484c9db09c3e5694#file-githook-md
# TODO! GIT_FILE_SIZE_LIMIT=50000000 git commit -m "test: this commit is allowed file sizes up to 50MB"
# Maximum file size limit in bytes
limit=${GIT_FILE_SIZE_LIMIT:-2000000} # Default 2MB
limitInMB=$(( $limit / 1000000 ))
function file_too_large(){
filename=$0
filesize=$(( $1 / 2**20 ))
cat <<HEREDOC
File $filename is $filesize MB, which is larger than github's maximum
file size (2 MB). We will not be able to push this file to GitHub.
Commit aborted
HEREDOC
git status
}
# Move to the repo root so git files paths make sense
repo_root=$( git rev-parse --show-toplevel )
cd $repo_root
empty_tree=$( git hash-object -t tree /dev/null )
if git rev-parse --verify HEAD > /dev/null 2>&1
then
against=HEAD
else
against="$empty_tree"
fi
# Set split so that for loop below can handle spaces in file names by splitting on line breaks
IFS='
'
shouldFail=false
for file in $( git diff-index --cached --name-only $against ); do
file_size=$(([ ! -f $file ] && echo 0) || (ls -la $file | awk '{ print $5 }'))
if [ "$file_size" -gt "$limit" ]; then
printError "File $file is $(( $file_size / 10**6 )) MB, which is larger than our configured limit of $limitInMB MB"
shouldFail=true
fi
done
if $shouldFail
then
printMessage "If you really need to commit this file, you can override the size limit by setting the GIT_FILE_SIZE_LIMIT environment variable, e.g. GIT_FILE_SIZE_LIMIT=42000000 for 42MB. Or, commit with the --no-verify switch to skip the check entirely."
printError "Commit aborted"
exit 1;
fi
if [[ ! $local_branch =~ $valid_branch_regex ]]
then
printError "There is something wrong with your branch name. Branch names in this project must adhere to this contract: $valid_branch_regex.
Your commit will be rejected. You should rename your branch to a valid name(feat/name OR fix/name) and try again."
printError "For more on this, read on: https://gist.github.com/cubxxw/126b72104ac0b0ca484c9db09c3e5694"
exit 1
fi

@ -1,119 +0,0 @@
#!/usr/bin/env bash
# Copyright © 2023 OpenIMSDK.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# ==============================================================================
#
YELLOW="\e[93m"
GREEN="\e[32m"
RED="\e[31m"
ENDCOLOR="\e[0m"
local_branch="$(git rev-parse --abbrev-ref HEAD)"
valid_branch_regex="^(main|master|develop|release(-[a-zA-Z0-9._-]+)?)$|(feature|feat|openim|hotfix|test|bug|ci|cicd|style|)\/[a-z0-9._-]+$|^HEAD$"
printMessage() {
printf "${YELLOW}OpenIM : $1${ENDCOLOR}\n"
}
printSuccess() {
printf "${GREEN}OpenIM : $1${ENDCOLOR}\n"
}
printError() {
printf "${RED}OpenIM : $1${ENDCOLOR}\n"
}
printMessage "Running local OpenIM pre-push hook."
if [[ $(git status --porcelain) ]]; then
printError "This scripts needs to run against committed code only. Please commit or stash you changes."
exit 1
fi
COLOR_SUFFIX="\033[0m"
BLACK_PREFIX="\033[30m"
RED_PREFIX="\033[31m"
GREEN_PREFIX="\033[32m"
BACKGROUND_GREEN="\033[33m"
BLUE_PREFIX="\033[34m"
PURPLE_PREFIX="\033[35m"
SKY_BLUE_PREFIX="\033[36m"
WHITE_PREFIX="\033[37m"
BOLD_PREFIX="\033[1m"
UNDERLINE_PREFIX="\033[4m"
ITALIC_PREFIX="\033[3m"
# Function to print colored text
print_color() {
local text=$1
local color=$2
echo -e "${color}${text}${COLOR_SUFFIX}"
}
# Function to print section separator
print_separator() {
print_color "==========================================================" ${PURPLE_PREFIX}
}
# Get current time
time=$(date +"%Y-%m-%d %H:%M:%S")
# Print section separator
print_separator
# Print time of submission
print_color "PTIME: ${time}" "${BOLD_PREFIX}${CYAN_PREFIX}"
echo ""
author=$(git config user.name)
repository=$(basename -s .git $(git config --get remote.origin.url))
# Print additional information if needed
print_color "Repository: ${repository}" "${BLUE_PREFIX}"
echo ""
print_color "Author: ${author}" "${PURPLE_PREFIX}"
# Print section separator
print_separator
file_list=$(git diff --name-status HEAD @{u})
added_files=$(grep -c '^A' <<< "$file_list")
modified_files=$(grep -c '^M' <<< "$file_list")
deleted_files=$(grep -c '^D' <<< "$file_list")
print_color "Added Files: ${added_files}" "${BACKGROUND_GREEN}"
print_color "Modified Files: ${modified_files}" "${BACKGROUND_GREEN}"
print_color "Deleted Files: ${deleted_files}" "${BACKGROUND_GREEN}"
if [[ ! $local_branch =~ $valid_branch_regex ]]
then
printError "There is something wrong with your branch name. Branch names in this project must adhere to this contract: $valid_branch_regex.
Your commit will be rejected. You should rename your branch to a valid name(feat/name OR fix/name) and try again."
printError "For more on this, read on: https://gist.github.com/cubxxw/126b72104ac0b0ca484c9db09c3e5694"
exit 1
fi
#
#printMessage "Running the Flutter analyzer"
#flutter analyze
#
#if [ $? -ne 0 ]; then
# printError "Flutter analyzer error"
# exit 1
#fi
#
#printMessage "Finished running the Flutter analyzer"
Loading…
Cancel
Save