From d2b02dbabecc9934271f4a7cc5b7f3e5cbfd5cd9 Mon Sep 17 00:00:00 2001 From: chao <48119764+withchao@users.noreply.github.com> Date: Wed, 31 Jul 2024 10:50:55 +0800 Subject: [PATCH 1/3] feat: add some logs (#2461) * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * new mongo * new mongo * new mongo * new mongo * new mongo * new mongo * new mongo * new mongo * friend incr sync * friend incr sync * friend incr sync * friend incr sync * friend incr sync * mage * optimization version log * optimization version log * sync * sync * sync * group sync * sync option * sync option * refactor: replace `friend` package with `realtion`. * refactor: update lastest commit to relation. * sync option * sync option * sync option * sync * sync * go.mod * seq * update: go mod * refactor: change incremental to full * feat: get full friend user ids * feat: api and config * seq * group version * merge * seq * seq * seq * fix: sort by id avoid unstable sort friends. * group * group * group * fix: sort by id avoid unstable sort friends. * fix: sort by id avoid unstable sort friends. * fix: sort by id avoid unstable sort friends. * user version * seq * seq * seq user * user online * implement minio expire delete. * user online * config * fix * fix * implement minio expire delete logic. * online cache * online cache * online cache * online cache * online cache * online cache * online cache * online cache * online cache * online cache * online cache * online cache * feat: implement scheduled delete outdated object in minio. * update gomake version * update gomake version * implement FindExpires pagination. * remove unnesseary incr. * fix uncorrect args call. * online push * online push * online push * resolving conflicts * resolving conflicts * test * api prommetrics * api prommetrics * api prommetrics * api prommetrics * api prommetrics * rpc prommetrics * rpc prommetrics * online status * online status * online status * online status * sub * conversation version incremental * merge seq * merge online * merge online * merge online * merge seq * GetOwnerConversation * fix: change incremental syncer router name. * rockscache batch get * rockscache seq batch get * fix: GetMsgDocModelByIndex bug * update go.mod * update go.mod * merge * feat: prometheus * feat: prometheus * group member sort * sub * sub * fix: seq conversion bug * fix: redis pipe exec * sort version * sort version * sort version * remove old version online subscription * remove old version online subscription * version log index * version log index * batch push * batch push * seq void filling * fix: batchGetMaxSeq * fix: batchGetMaxSeq * cache db error log * 111 * fix bug * fix: ImportFriends * add online cache * add some logs --------- Co-authored-by: withchao Co-authored-by: Monet Lee Co-authored-by: OpenIM-Gordon <46924906+FGadvancer@users.noreply.github.com> Co-authored-by: icey-yu <1186114839@qq.com> --- internal/push/push_handler.go | 1 + pkg/common/storage/cache/redis/online.go | 6 +- pkg/rpccache/online.go | 76 +++++++++++++----------- 3 files changed, 47 insertions(+), 36 deletions(-) diff --git a/internal/push/push_handler.go b/internal/push/push_handler.go index ed87b3929..c3b27372b 100644 --- a/internal/push/push_handler.go +++ b/internal/push/push_handler.go @@ -198,6 +198,7 @@ func (c *ConsumerHandler) GetConnsAndOnlinePush(ctx context.Context, msg *sdkws. offlineUserIDs = append(offlineUserIDs, userID) } } + log.ZDebug(ctx, "GetConnsAndOnlinePush online cache", "sendID", msg.SendID, "recvID", msg.RecvID, "groupID", msg.GroupID, "sessionType", msg.SessionType, "clientMsgID", msg.ClientMsgID, "serverMsgID", msg.ServerMsgID, "offlineUserIDs", offlineUserIDs, "onlineUserIDs", onlineUserIDs) var result []*msggateway.SingleMsgToUserResults if len(onlineUserIDs) > 0 { var err error diff --git a/pkg/common/storage/cache/redis/online.go b/pkg/common/storage/cache/redis/online.go index dc6a5f775..a012e1cd2 100644 --- a/pkg/common/storage/cache/redis/online.go +++ b/pkg/common/storage/cache/redis/online.go @@ -5,6 +5,7 @@ import ( "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/cachekey" "github.com/openimsdk/tools/errs" + "github.com/openimsdk/tools/log" "github.com/redis/go-redis/v9" "strconv" "time" @@ -82,8 +83,11 @@ func (s *userOnline) SetUserOnline(ctx context.Context, userID string, online, o argv = append(argv, platformID) } keys := []string{s.getUserOnlineKey(userID), userID, s.channelName} - if err := s.rdb.Eval(ctx, script, keys, argv).Err(); err != nil { + status, err := s.rdb.Eval(ctx, script, keys, argv).Result() + if err != nil { + log.ZError(ctx, "redis SetUserOnline", err, "userID", userID, "online", online, "offline", offline) return err } + log.ZDebug(ctx, "redis SetUserOnline", "userID", userID, "online", online, "offline", offline, "status", status) return nil } diff --git a/pkg/rpccache/online.go b/pkg/rpccache/online.go index 5db68d198..372a7efe8 100644 --- a/pkg/rpccache/online.go +++ b/pkg/rpccache/online.go @@ -28,7 +28,7 @@ func NewOnlineCache(user rpcclient.UserRpcClient, group *GroupLocalCache, rdb re for message := range rdb.Subscribe(ctx, cachekey.OnlineChannel).Channel() { userID, platformIDs, err := useronline.ParseUserOnlineStatus(message.Payload) if err != nil { - log.ZError(ctx, "OnlineCache redis subscribe parseUserOnlineStatus", err, "payload", message.Payload, "channel", message.Channel) + log.ZError(ctx, "OnlineCache setUserOnline redis subscribe parseUserOnlineStatus", err, "payload", message.Payload, "channel", message.Channel) continue } storageCache := x.setUserOnline(userID, platformIDs) @@ -48,9 +48,15 @@ type OnlineCache struct { } func (o *OnlineCache) GetUserOnlinePlatform(ctx context.Context, userID string) ([]int32, error) { - return o.local.Get(userID, func() ([]int32, error) { + platformIDs, err := o.local.Get(userID, func() ([]int32, error) { return o.user.GetUserOnlinePlatform(ctx, userID) }) + if err != nil { + log.ZError(ctx, "OnlineCache GetUserOnlinePlatform", err, "userID", userID) + return nil, err + } + log.ZDebug(ctx, "OnlineCache GetUserOnlinePlatform", "userID", userID, "platformIDs", platformIDs) + return nil, err } func (o *OnlineCache) GetUserOnline(ctx context.Context, userID string) (bool, error) { @@ -61,39 +67,39 @@ func (o *OnlineCache) GetUserOnline(ctx context.Context, userID string) (bool, e return len(platformIDs) > 0, nil } -func (o *OnlineCache) GetUsersOnline(ctx context.Context, userIDs []string) ([]string, error) { - onlineUserIDs := make([]string, 0, len(userIDs)) - for _, userID := range userIDs { - online, err := o.GetUserOnline(ctx, userID) - if err != nil { - return nil, err - } - if online { - onlineUserIDs = append(onlineUserIDs, userID) - } - } - log.ZDebug(ctx, "OnlineCache GetUsersOnline", "userIDs", userIDs, "onlineUserIDs", onlineUserIDs) - return onlineUserIDs, nil -} - -func (o *OnlineCache) GetGroupOnline(ctx context.Context, groupID string) ([]string, error) { - userIDs, err := o.group.GetGroupMemberIDs(ctx, groupID) - if err != nil { - return nil, err - } - var onlineUserIDs []string - for _, userID := range userIDs { - online, err := o.GetUserOnline(ctx, userID) - if err != nil { - return nil, err - } - if online { - onlineUserIDs = append(onlineUserIDs, userID) - } - } - log.ZDebug(ctx, "OnlineCache GetGroupOnline", "groupID", groupID, "onlineUserIDs", onlineUserIDs, "allUserID", userIDs) - return onlineUserIDs, nil -} +//func (o *OnlineCache) GetUsersOnline(ctx context.Context, userIDs []string) ([]string, error) { +// onlineUserIDs := make([]string, 0, len(userIDs)) +// for _, userID := range userIDs { +// online, err := o.GetUserOnline(ctx, userID) +// if err != nil { +// return nil, err +// } +// if online { +// onlineUserIDs = append(onlineUserIDs, userID) +// } +// } +// log.ZDebug(ctx, "OnlineCache GetUsersOnline", "userIDs", userIDs, "onlineUserIDs", onlineUserIDs) +// return onlineUserIDs, nil +//} +// +//func (o *OnlineCache) GetGroupOnline(ctx context.Context, groupID string) ([]string, error) { +// userIDs, err := o.group.GetGroupMemberIDs(ctx, groupID) +// if err != nil { +// return nil, err +// } +// var onlineUserIDs []string +// for _, userID := range userIDs { +// online, err := o.GetUserOnline(ctx, userID) +// if err != nil { +// return nil, err +// } +// if online { +// onlineUserIDs = append(onlineUserIDs, userID) +// } +// } +// log.ZDebug(ctx, "OnlineCache GetGroupOnline", "groupID", groupID, "onlineUserIDs", onlineUserIDs, "allUserID", userIDs) +// return onlineUserIDs, nil +//} func (o *OnlineCache) setUserOnline(userID string, platformIDs []int32) bool { return o.local.SetHas(userID, platformIDs) From 9cfbc3aaffa0ce5b81c2f8701b8205f30be74eb5 Mon Sep 17 00:00:00 2001 From: chao <48119764+withchao@users.noreply.github.com> Date: Wed, 31 Jul 2024 14:21:19 +0800 Subject: [PATCH 2/3] fix: return value (#2462) * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * new mongo * new mongo * new mongo * new mongo * new mongo * new mongo * new mongo * new mongo * friend incr sync * friend incr sync * friend incr sync * friend incr sync * friend incr sync * mage * optimization version log * optimization version log * sync * sync * sync * group sync * sync option * sync option * refactor: replace `friend` package with `realtion`. * refactor: update lastest commit to relation. * sync option * sync option * sync option * sync * sync * go.mod * seq * update: go mod * refactor: change incremental to full * feat: get full friend user ids * feat: api and config * seq * group version * merge * seq * seq * seq * fix: sort by id avoid unstable sort friends. * group * group * group * fix: sort by id avoid unstable sort friends. * fix: sort by id avoid unstable sort friends. * fix: sort by id avoid unstable sort friends. * user version * seq * seq * seq user * user online * implement minio expire delete. * user online * config * fix * fix * implement minio expire delete logic. * online cache * online cache * online cache * online cache * online cache * online cache * online cache * online cache * online cache * online cache * online cache * online cache * feat: implement scheduled delete outdated object in minio. * update gomake version * update gomake version * implement FindExpires pagination. * remove unnesseary incr. * fix uncorrect args call. * online push * online push * online push * resolving conflicts * resolving conflicts * test * api prommetrics * api prommetrics * api prommetrics * api prommetrics * api prommetrics * rpc prommetrics * rpc prommetrics * online status * online status * online status * online status * sub * conversation version incremental * merge seq * merge online * merge online * merge online * merge seq * GetOwnerConversation * fix: change incremental syncer router name. * rockscache batch get * rockscache seq batch get * fix: GetMsgDocModelByIndex bug * update go.mod * update go.mod * merge * feat: prometheus * feat: prometheus * group member sort * sub * sub * fix: seq conversion bug * fix: redis pipe exec * sort version * sort version * sort version * remove old version online subscription * remove old version online subscription * version log index * version log index * batch push * batch push * seq void filling * fix: batchGetMaxSeq * fix: batchGetMaxSeq * cache db error log * 111 * fix bug * fix: ImportFriends * add online cache * add some logs * add some logs --------- Co-authored-by: withchao Co-authored-by: Monet Lee Co-authored-by: OpenIM-Gordon <46924906+FGadvancer@users.noreply.github.com> Co-authored-by: icey-yu <1186114839@qq.com> --- pkg/rpccache/online.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/rpccache/online.go b/pkg/rpccache/online.go index 372a7efe8..13578a7df 100644 --- a/pkg/rpccache/online.go +++ b/pkg/rpccache/online.go @@ -56,7 +56,7 @@ func (o *OnlineCache) GetUserOnlinePlatform(ctx context.Context, userID string) return nil, err } log.ZDebug(ctx, "OnlineCache GetUserOnlinePlatform", "userID", userID, "platformIDs", platformIDs) - return nil, err + return platformIDs, nil } func (o *OnlineCache) GetUserOnline(ctx context.Context, userID string) (bool, error) { From 5dac91569de904efd619f165ef87089c428ed1c1 Mon Sep 17 00:00:00 2001 From: Monet Lee Date: Wed, 31 Jul 2024 18:12:19 +0800 Subject: [PATCH 3/3] refactor: rename friend module to relation. (#2463) * update protocol in go mod. * add debug log in writePongMsg. * update log level. * add Warn log in writePongMsg. * add debug log. * feat: update webhookBeforeMemberJoinGroup to batch method. * feat: update version field implement. * update webhook implement contents. * update method field and contents. * update callbackCommand field. * fix: add correct fields. * update struct tags. * refactor: rename friend module to relation. --- internal/rpc/{friend => relation}/black.go | 2 +- internal/rpc/{friend => relation}/callback.go | 2 +- internal/rpc/{friend => relation}/friend.go | 3 +- .../rpc/{friend => relation}/notification.go | 2 +- internal/rpc/{friend => relation}/sync.go | 2 +- internal/rpc/user/user.go | 6 ++-- pkg/common/cmd/friend.go | 34 +++++++++---------- 7 files changed, 26 insertions(+), 25 deletions(-) rename internal/rpc/{friend => relation}/black.go (99%) rename internal/rpc/{friend => relation}/callback.go (99%) rename internal/rpc/{friend => relation}/friend.go (99%) rename internal/rpc/{friend => relation}/notification.go (99%) rename internal/rpc/{friend => relation}/sync.go (99%) diff --git a/internal/rpc/friend/black.go b/internal/rpc/relation/black.go similarity index 99% rename from internal/rpc/friend/black.go rename to internal/rpc/relation/black.go index 218d1e7f8..e149e3165 100644 --- a/internal/rpc/friend/black.go +++ b/internal/rpc/relation/black.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package friend +package relation import ( "context" diff --git a/internal/rpc/friend/callback.go b/internal/rpc/relation/callback.go similarity index 99% rename from internal/rpc/friend/callback.go rename to internal/rpc/relation/callback.go index 746ad21fa..69c4c9e0e 100644 --- a/internal/rpc/friend/callback.go +++ b/internal/rpc/relation/callback.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package friend +package relation import ( "context" diff --git a/internal/rpc/friend/friend.go b/internal/rpc/relation/friend.go similarity index 99% rename from internal/rpc/friend/friend.go rename to internal/rpc/relation/friend.go index bdb786bca..3d29ad337 100644 --- a/internal/rpc/friend/friend.go +++ b/internal/rpc/relation/friend.go @@ -12,10 +12,11 @@ // See the License for the specific language governing permissions and // limitations under the License. -package friend +package relation import ( "context" + "github.com/openimsdk/tools/mq/memamq" "github.com/openimsdk/open-im-server/v3/pkg/common/config" diff --git a/internal/rpc/friend/notification.go b/internal/rpc/relation/notification.go similarity index 99% rename from internal/rpc/friend/notification.go rename to internal/rpc/relation/notification.go index 5fb34577f..83c5d2ca9 100644 --- a/internal/rpc/friend/notification.go +++ b/internal/rpc/relation/notification.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package friend +package relation import ( "context" diff --git a/internal/rpc/friend/sync.go b/internal/rpc/relation/sync.go similarity index 99% rename from internal/rpc/friend/sync.go rename to internal/rpc/relation/sync.go index 902cc7303..0ad94fe82 100644 --- a/internal/rpc/friend/sync.go +++ b/internal/rpc/relation/sync.go @@ -1,4 +1,4 @@ -package friend +package relation import ( "context" diff --git a/internal/rpc/user/user.go b/internal/rpc/user/user.go index 1e534437d..a6952bd6d 100644 --- a/internal/rpc/user/user.go +++ b/internal/rpc/user/user.go @@ -17,7 +17,7 @@ package user import ( "context" "errors" - "github.com/openimsdk/open-im-server/v3/internal/rpc/friend" + "github.com/openimsdk/open-im-server/v3/internal/rpc/relation" "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/open-im-server/v3/pkg/common/prommetrics" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache" @@ -54,7 +54,7 @@ import ( type userServer struct { online cache.OnlineCache db controller.UserDatabase - friendNotificationSender *friend.FriendNotificationSender + friendNotificationSender *relation.FriendNotificationSender userNotificationSender *UserNotificationSender friendRpcClient *rpcclient.FriendRpcClient groupRpcClient *rpcclient.GroupRpcClient @@ -105,7 +105,7 @@ func Start(ctx context.Context, config *Config, client registry.SvcDiscoveryRegi RegisterCenter: client, friendRpcClient: &friendRpcClient, groupRpcClient: &groupRpcClient, - friendNotificationSender: friend.NewFriendNotificationSender(&config.NotificationConfig, &msgRpcClient, friend.WithDBFunc(database.FindWithError)), + friendNotificationSender: relation.NewFriendNotificationSender(&config.NotificationConfig, &msgRpcClient, relation.WithDBFunc(database.FindWithError)), userNotificationSender: NewUserNotificationSender(config, &msgRpcClient, WithUserFunc(database.FindWithError)), config: config, webhookClient: webhook.NewWebhookClient(config.WebhooksConfig.URL), diff --git a/pkg/common/cmd/friend.go b/pkg/common/cmd/friend.go index 1bc9e6c54..a564facd0 100644 --- a/pkg/common/cmd/friend.go +++ b/pkg/common/cmd/friend.go @@ -17,7 +17,7 @@ package cmd import ( "context" - "github.com/openimsdk/open-im-server/v3/internal/rpc/friend" + "github.com/openimsdk/open-im-server/v3/internal/rpc/relation" "github.com/openimsdk/open-im-server/v3/pkg/common/startrpc" "github.com/openimsdk/open-im-server/v3/version" "github.com/openimsdk/tools/system/program" @@ -26,23 +26,23 @@ import ( type FriendRpcCmd struct { *RootCmd - ctx context.Context - configMap map[string]any - friendConfig *friend.Config + ctx context.Context + configMap map[string]any + relationConfig *relation.Config } func NewFriendRpcCmd() *FriendRpcCmd { - var friendConfig friend.Config - ret := &FriendRpcCmd{friendConfig: &friendConfig} + var relationConfig relation.Config + ret := &FriendRpcCmd{relationConfig: &relationConfig} ret.configMap = map[string]any{ - OpenIMRPCFriendCfgFileName: &friendConfig.RpcConfig, - RedisConfigFileName: &friendConfig.RedisConfig, - MongodbConfigFileName: &friendConfig.MongodbConfig, - ShareFileName: &friendConfig.Share, - NotificationFileName: &friendConfig.NotificationConfig, - WebhooksConfigFileName: &friendConfig.WebhooksConfig, - LocalCacheConfigFileName: &friendConfig.LocalCacheConfig, - DiscoveryConfigFilename: &friendConfig.Discovery, + OpenIMRPCFriendCfgFileName: &relationConfig.RpcConfig, + RedisConfigFileName: &relationConfig.RedisConfig, + MongodbConfigFileName: &relationConfig.MongodbConfig, + ShareFileName: &relationConfig.Share, + NotificationFileName: &relationConfig.NotificationConfig, + WebhooksConfigFileName: &relationConfig.WebhooksConfig, + LocalCacheConfigFileName: &relationConfig.LocalCacheConfig, + DiscoveryConfigFilename: &relationConfig.Discovery, } ret.RootCmd = NewRootCmd(program.GetProcessName(), WithConfigMap(ret.configMap)) ret.ctx = context.WithValue(context.Background(), "version", version.Version) @@ -57,7 +57,7 @@ func (a *FriendRpcCmd) Exec() error { } func (a *FriendRpcCmd) runE() error { - return startrpc.Start(a.ctx, &a.friendConfig.Discovery, &a.friendConfig.RpcConfig.Prometheus, a.friendConfig.RpcConfig.RPC.ListenIP, - a.friendConfig.RpcConfig.RPC.RegisterIP, a.friendConfig.RpcConfig.RPC.Ports, - a.Index(), a.friendConfig.Share.RpcRegisterName.Friend, &a.friendConfig.Share, a.friendConfig, friend.Start) + return startrpc.Start(a.ctx, &a.relationConfig.Discovery, &a.relationConfig.RpcConfig.Prometheus, a.relationConfig.RpcConfig.RPC.ListenIP, + a.relationConfig.RpcConfig.RPC.RegisterIP, a.relationConfig.RpcConfig.RPC.Ports, + a.Index(), a.relationConfig.Share.RpcRegisterName.Friend, &a.relationConfig.Share, a.relationConfig, relation.Start) }