From 8f0403e67c2415d0cc0760e695ba663caecfca76 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Wed, 26 Jun 2024 18:19:38 +0800 Subject: [PATCH 01/27] user online --- .../openim-rpc-conversation/main.go | 4 ++ cmd/openim-rpc/openim-rpc-group/main.go | 4 ++ config/discovery.yml | 4 +- config/kafka.yml | 2 +- config/minio.yml | 4 +- config/mongodb.yml | 2 +- config/redis.yml | 2 +- internal/msggateway/n_ws_server.go | 6 +- internal/msggateway/user_map.go | 4 +- internal/msggateway/user_map2.go | 32 +++++++++ pkg/common/storage/cache/redis/online.go | 58 +++++++++++++++ .../storage/cache/redis/seq_user_test.go | 70 +++++++++++++++++++ pkg/common/storage/cache/redis/user.go | 9 +++ 13 files changed, 189 insertions(+), 12 deletions(-) create mode 100644 internal/msggateway/user_map2.go create mode 100644 pkg/common/storage/cache/redis/online.go create mode 100644 pkg/common/storage/cache/redis/seq_user_test.go diff --git a/cmd/openim-rpc/openim-rpc-conversation/main.go b/cmd/openim-rpc/openim-rpc-conversation/main.go index 5b2e66c95..5b3077ccb 100644 --- a/cmd/openim-rpc/openim-rpc-conversation/main.go +++ b/cmd/openim-rpc/openim-rpc-conversation/main.go @@ -17,9 +17,13 @@ package main import ( "github.com/openimsdk/open-im-server/v3/pkg/common/cmd" "github.com/openimsdk/tools/system/program" + "os" ) func main() { + if len(os.Args) == 1 { + os.Args = []string{os.Args[0], "-i", "0", "-c", "/Users/chao/Desktop/project/open-im-server/config"} + } if err := cmd.NewConversationRpcCmd().Exec(); err != nil { program.ExitWithError(err) } diff --git a/cmd/openim-rpc/openim-rpc-group/main.go b/cmd/openim-rpc/openim-rpc-group/main.go index 5badf934e..44e5509df 100644 --- a/cmd/openim-rpc/openim-rpc-group/main.go +++ b/cmd/openim-rpc/openim-rpc-group/main.go @@ -17,9 +17,13 @@ package main import ( "github.com/openimsdk/open-im-server/v3/pkg/common/cmd" "github.com/openimsdk/tools/system/program" + "os" ) func main() { + if len(os.Args) == 1 { + os.Args = []string{os.Args[0], "-i", "0", "-c", "/Users/chao/Desktop/project/open-im-server/config"} + } if err := cmd.NewGroupRpcCmd().Exec(); err != nil { program.ExitWithError(err) } diff --git a/config/discovery.yml b/config/discovery.yml index 3d96ff9b6..9cd23c590 100644 --- a/config/discovery.yml +++ b/config/discovery.yml @@ -1,13 +1,13 @@ enable: "etcd" etcd: rootDirectory: openim - address: [ localhost:12379 ] + address: [ 172.16.8.48:12379 ] username: '' password: '' zookeeper: schema: openim - address: [ localhost:12181 ] + address: [ 172.16.8.48:12181 ] username: '' password: '' diff --git a/config/kafka.yml b/config/kafka.yml index d412e1be0..d9b7ffa3c 100644 --- a/config/kafka.yml +++ b/config/kafka.yml @@ -7,7 +7,7 @@ producerAck: "" # Compression type to use (e.g., none, gzip, snappy) compressType: "none" # List of Kafka broker addresses -address: [ localhost:19094 ] +address: [ 172.16.8.48:19094 ] # Kafka topic for Redis integration toRedisTopic: "toRedis" # Kafka topic for MongoDB integration diff --git a/config/minio.yml b/config/minio.yml index 11a9ace35..d143a1da3 100644 --- a/config/minio.yml +++ b/config/minio.yml @@ -7,9 +7,9 @@ secretAccessKey: "openIM123" # Session token for MinIO authentication (optional) sessionToken: '' # Internal address of the MinIO server -internalAddress: "localhost:10005" +internalAddress: "172.16.8.48:10005" # External address of the MinIO server, accessible from outside. Supports both HTTP and HTTPS using a domain name -externalAddress: "http://external_ip:10005" +externalAddress: "http://172.16.8.48:10005" # Flag to enable or disable public read access to the bucket publicRead: false diff --git a/config/mongodb.yml b/config/mongodb.yml index 98f5694e4..53969298b 100644 --- a/config/mongodb.yml +++ b/config/mongodb.yml @@ -1,7 +1,7 @@ # URI for database connection, leave empty if using address and credential settings directly uri: '' # List of MongoDB server addresses -address: [ localhost:37017 ] +address: [ 172.16.8.48:37017 ] # Name of the database database: openim_v3 # Username for database authentication diff --git a/config/redis.yml b/config/redis.yml index 87abed0e1..404d18953 100644 --- a/config/redis.yml +++ b/config/redis.yml @@ -1,4 +1,4 @@ -address: [ localhost:16379 ] +address: [ 172.16.8.48:16379 ] username: '' password: openIM123 clusterMode: false diff --git a/internal/msggateway/n_ws_server.go b/internal/msggateway/n_ws_server.go index defec16df..17e550fef 100644 --- a/internal/msggateway/n_ws_server.go +++ b/internal/msggateway/n_ws_server.go @@ -309,7 +309,7 @@ func getRemoteAdders(client []*Client) string { } func (ws *WsServer) KickUserConn(client *Client) error { - ws.clients.deleteClients(client.UserID, []*Client{client}) + ws.clients.DeleteClients(client.UserID, []*Client{client}) return client.KickOnlineMessage() } @@ -325,7 +325,7 @@ func (ws *WsServer) multiTerminalLoginChecker(clientOK bool, oldClients []*Clien if !clientOK { return } - ws.clients.deleteClients(newClient.UserID, oldClients) + ws.clients.DeleteClients(newClient.UserID, oldClients) for _, c := range oldClients { err := c.KickOnlineMessage() if err != nil { @@ -345,7 +345,7 @@ func (ws *WsServer) multiTerminalLoginChecker(clientOK bool, oldClients []*Clien func (ws *WsServer) unregisterClient(client *Client) { defer ws.clientPool.Put(client) - isDeleteUser := ws.clients.delete(client.UserID, client.ctx.GetRemoteAddr()) + isDeleteUser := ws.clients.Delete(client.UserID, client.ctx.GetRemoteAddr()) if isDeleteUser { ws.onlineUserNum.Add(-1) prommetrics.OnlineUserGauge.Dec() diff --git a/internal/msggateway/user_map.go b/internal/msggateway/user_map.go index 79cc53d1b..f8bf69f9a 100644 --- a/internal/msggateway/user_map.go +++ b/internal/msggateway/user_map.go @@ -72,7 +72,7 @@ func (u *UserMap) Set(key string, v *Client) { } } -func (u *UserMap) delete(key string, connRemoteAddr string) (isDeleteUser bool) { +func (u *UserMap) Delete(key string, connRemoteAddr string) (isDeleteUser bool) { // Attempt to load the clients associated with the key. allClients, existed := u.m.Load(key) if !existed { @@ -101,7 +101,7 @@ func (u *UserMap) delete(key string, connRemoteAddr string) (isDeleteUser bool) return false } -func (u *UserMap) deleteClients(key string, clients []*Client) (isDeleteUser bool) { +func (u *UserMap) DeleteClients(key string, clients []*Client) (isDeleteUser bool) { m := datautil.SliceToMapAny(clients, func(c *Client) (string, struct{}) { return c.ctx.GetRemoteAddr(), struct{}{} }) diff --git a/internal/msggateway/user_map2.go b/internal/msggateway/user_map2.go new file mode 100644 index 000000000..1c9a91c6a --- /dev/null +++ b/internal/msggateway/user_map2.go @@ -0,0 +1,32 @@ +package msggateway + +/* + +sorted set + +userID: 10000 + +USER_ONLINE:10000 + + + + + +platformID: 1 + + + + + +key score + +E1 123456789 +O1 234567895 + + + + + + + +*/ diff --git a/pkg/common/storage/cache/redis/online.go b/pkg/common/storage/cache/redis/online.go new file mode 100644 index 000000000..138f9a573 --- /dev/null +++ b/pkg/common/storage/cache/redis/online.go @@ -0,0 +1,58 @@ +package redis + +import ( + "context" + "github.com/redis/go-redis/v9" + "time" +) + +type userOnline struct { + rdb redis.UniversalClient + expire time.Duration + channelName string +} + +func (s *userOnline) getUserOnlineKey(userID string) string { + return "USER_ONLINE:" + userID +} + +func (s *userOnline) SetUserOnline(ctx context.Context, userID string, online, offline []int32) error { + script := ` + local key = KEYS[1] + local score = ARGV[3] + local num1 = redis.call("ZCARD", key) + redis.call("ZREMRANGEBYSCORE", key, "-inf", ARGV[2]) + for i = 5, tonumber(ARGV[4])+4 do + redis.call("ZREM", key, ARGV[i]) + end + local num2 = redis.call("ZCARD", key) + for i = 5+tonumber(ARGV[4]), #ARGV do + redis.call("ZADD", key, score, ARGV[i]) + end + redis.call("EXPIRE", key, ARGV[1]) + local num3 = redis.call("ZCARD", key) + local change = (num1 ~= num2) or (num2 ~= num3) + if change then + local members = redis.call("ZRANGE", key, 0, -1) + table.insert(members, KEYS[2]) + redis.call("PUBLISH", KEYS[3], table.concat(members, ":")) + return 1 + else + return 0 + end +` + now := time.Now() + argv := make([]any, 0, 2+len(online)+len(offline)) + argv = append(argv, int32(s.expire/time.Second), now.Unix(), now.Add(s.expire).Unix(), int32(len(offline))) + for _, platformID := range offline { + argv = append(argv, platformID) + } + for _, platformID := range online { + argv = append(argv, platformID) + } + keys := []string{s.getUserOnlineKey(userID), userID, s.channelName} + if err := s.rdb.Eval(ctx, script, keys, argv).Err(); err != nil { + return err + } + return nil +} diff --git a/pkg/common/storage/cache/redis/seq_user_test.go b/pkg/common/storage/cache/redis/seq_user_test.go new file mode 100644 index 000000000..04a5d49cb --- /dev/null +++ b/pkg/common/storage/cache/redis/seq_user_test.go @@ -0,0 +1,70 @@ +package redis + +import ( + "context" + "fmt" + "github.com/redis/go-redis/v9" + "log" + "testing" + "time" +) + +func newTestOnline() *userOnline { + opt := &redis.Options{ + Addr: "172.16.8.48:16379", + Password: "openIM123", + DB: 1, + } + rdb := redis.NewClient(opt) + if err := rdb.Ping(context.Background()).Err(); err != nil { + panic(err) + } + return &userOnline{rdb: rdb, expire: time.Hour, channelName: "user_online"} +} + +func TestOnline(t *testing.T) { + ts := newTestOnline() + + //err := ts.SetUserOnline(context.Background(), "1000", []int32{1, 2, 3}, []int32{4, 5, 6}) + err := ts.SetUserOnline(context.Background(), "1000", nil, []int32{1, 2, 3}) + + t.Log(err) + +} + +/* + +local function tableToString(tbl, separator) + local result = {} + for _, v in ipairs(tbl) do + table.insert(result, tostring(v)) + end + return table.concat(result, separator) +end + +local myTable = {"one", "two", "three"} +local result = tableToString(myTable, ":") + +print(result) + +*/ + +func TestRecvOnline(t *testing.T) { + ts := newTestOnline() + ctx := context.Background() + pubsub := ts.rdb.Subscribe(ctx, "user_online") + + // 等待订阅确认 + _, err := pubsub.Receive(ctx) + if err != nil { + log.Fatalf("Could not subscribe: %v", err) + } + + // 创建一个通道来接收消息 + ch := pubsub.Channel() + + // 处理接收到的消息 + for msg := range ch { + fmt.Printf("Received message from channel %s: %s\n", msg.Channel, msg.Payload) + } +} diff --git a/pkg/common/storage/cache/redis/user.go b/pkg/common/storage/cache/redis/user.go index 3de01563b..c05cd3895 100644 --- a/pkg/common/storage/cache/redis/user.go +++ b/pkg/common/storage/cache/redis/user.go @@ -131,6 +131,15 @@ func (u *UserCacheRedis) DelUsersGlobalRecvMsgOpt(userIDs ...string) cache.UserC return cache } +/* + + */ + +type RedisUserOnline struct { + // 平台id, 平台更新时间 + PlatformIDs map[int32]int64 +} + // GetUserStatus get user status. func (u *UserCacheRedis) GetUserStatus(ctx context.Context, userIDs []string) ([]*user.OnlineStatus, error) { userStatus := make([]*user.OnlineStatus, 0, len(userIDs)) From 8f86049599afe62a85a612ac85dee72dcad583d8 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Fri, 28 Jun 2024 16:18:21 +0800 Subject: [PATCH 02/27] user online --- go.mod | 2 +- go.sum | 4 +- internal/msggateway/init.go | 7 +- internal/msggateway/n_ws_server.go | 42 +-- internal/msggateway/online.go | 106 ++++++++ internal/msggateway/user_map.go | 239 +++++++++--------- internal/msggateway/user_map2.go | 208 ++++++++++++--- internal/rpc/user/online.go | 122 +++++++++ internal/rpc/user/user.go | 73 +----- internal/tools/cron_task.go | 6 + pkg/common/config/config.go | 1 + pkg/common/storage/cache/cachekey/online.go | 13 + pkg/common/storage/cache/cachekey/user.go | 5 - pkg/common/storage/cache/online.go | 8 + pkg/common/storage/cache/redis/online.go | 33 ++- .../storage/cache/redis/seq_user_test.go | 53 ++-- pkg/common/storage/cache/redis/user.go | 183 +++++++------- pkg/common/storage/cache/user.go | 5 +- pkg/common/storage/controller/user.go | 15 -- 19 files changed, 740 insertions(+), 385 deletions(-) create mode 100644 internal/msggateway/online.go create mode 100644 internal/rpc/user/online.go create mode 100644 pkg/common/storage/cache/cachekey/online.go create mode 100644 pkg/common/storage/cache/online.go diff --git a/go.mod b/go.mod index 2614e0f32..e7ed446f4 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/lestrrat-go/file-rotatelogs v2.4.0+incompatible // indirect github.com/mitchellh/mapstructure v1.5.0 - github.com/openimsdk/protocol v0.0.69-alpha.17 + github.com/openimsdk/protocol v0.0.69-alpha.20 github.com/openimsdk/tools v0.0.49-alpha.25 github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 diff --git a/go.sum b/go.sum index fe4f0c390..adde3aa82 100644 --- a/go.sum +++ b/go.sum @@ -270,8 +270,8 @@ github.com/onsi/gomega v1.25.0 h1:Vw7br2PCDYijJHSfBOWhov+8cAnUf8MfMaIOV323l6Y= github.com/onsi/gomega v1.25.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdMPGhM= github.com/openimsdk/gomake v0.0.13 h1:xLDe/moqgWpRoptHzI4packAWzs4C16b+sVY+txNJp0= github.com/openimsdk/gomake v0.0.13/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= -github.com/openimsdk/protocol v0.0.69-alpha.17 h1:pEag4ZdlovE+AyLsw1VYFU/3sk6ayvGdPzgufQfKf9M= -github.com/openimsdk/protocol v0.0.69-alpha.17/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= +github.com/openimsdk/protocol v0.0.69-alpha.20 h1:skZu82sqoMhiQVEZgrRsjcfI3Grp1IpThx1LJPqETWs= +github.com/openimsdk/protocol v0.0.69-alpha.20/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= github.com/openimsdk/tools v0.0.49-alpha.25 h1:OpRPwDZ2xWX7Zj5kyfZhryu/NfZTrsRVr2GFwu1HQHI= github.com/openimsdk/tools v0.0.49-alpha.25/go.mod h1:rwsFI1G/nBHNfiNapbven41akRDPBbH4df0Cgy6xueU= github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= diff --git a/internal/msggateway/init.go b/internal/msggateway/init.go index f4d8b0381..815ec8ca6 100644 --- a/internal/msggateway/init.go +++ b/internal/msggateway/init.go @@ -42,16 +42,15 @@ func Start(ctx context.Context, index int, conf *Config) error { if err != nil { return err } - longServer, err := NewWsServer( + longServer := NewWsServer( conf, WithPort(wsPort), WithMaxConnNum(int64(conf.MsgGateway.LongConnSvr.WebsocketMaxConnNum)), WithHandshakeTimeout(time.Duration(conf.MsgGateway.LongConnSvr.WebsocketTimeout)*time.Second), WithMessageMaxMsgLength(conf.MsgGateway.LongConnSvr.WebsocketMaxMsgLen), ) - if err != nil { - return err - } + + go longServer.ChangeOnlineStatus(4) hubServer := NewServer(rpcPort, longServer, conf) netDone := make(chan error) diff --git a/internal/msggateway/n_ws_server.go b/internal/msggateway/n_ws_server.go index 17e550fef..ff93ce6bc 100644 --- a/internal/msggateway/n_ws_server.go +++ b/internal/msggateway/n_ws_server.go @@ -60,7 +60,7 @@ type WsServer struct { registerChan chan *Client unregisterChan chan *Client kickHandlerChan chan *kickHandler - clients *UserMap + clients UMap clientPool sync.Pool onlineUserNum atomic.Int64 onlineUserConnNum atomic.Int64 @@ -90,18 +90,18 @@ func (ws *WsServer) SetDiscoveryRegistry(disCov discovery.SvcDiscoveryRegistry, ws.disCov = disCov } -func (ws *WsServer) SetUserOnlineStatus(ctx context.Context, client *Client, status int32) { - err := ws.userClient.SetUserStatus(ctx, client.UserID, status, client.PlatformID) - if err != nil { - log.ZWarn(ctx, "SetUserStatus err", err) - } - switch status { - case constant.Online: - ws.webhookAfterUserOnline(ctx, &ws.msgGatewayConfig.WebhooksConfig.AfterUserOnline, client.UserID, client.PlatformID, client.IsBackground, client.ctx.GetConnID()) - case constant.Offline: - ws.webhookAfterUserOffline(ctx, &ws.msgGatewayConfig.WebhooksConfig.AfterUserOffline, client.UserID, client.PlatformID, client.ctx.GetConnID()) - } -} +//func (ws *WsServer) SetUserOnlineStatus(ctx context.Context, client *Client, status int32) { +// err := ws.userClient.SetUserStatus(ctx, client.UserID, status, client.PlatformID) +// if err != nil { +// log.ZWarn(ctx, "SetUserStatus err", err) +// } +// switch status { +// case constant.Online: +// ws.webhookAfterUserOnline(ctx, &ws.msgGatewayConfig.WebhooksConfig.AfterUserOnline, client.UserID, client.PlatformID, client.IsBackground, client.ctx.GetConnID()) +// case constant.Offline: +// ws.webhookAfterUserOffline(ctx, &ws.msgGatewayConfig.WebhooksConfig.AfterUserOffline, client.UserID, client.PlatformID, client.ctx.GetConnID()) +// } +//} func (ws *WsServer) UnRegister(c *Client) { ws.unregisterChan <- c @@ -119,7 +119,7 @@ func (ws *WsServer) GetUserPlatformCons(userID string, platform int) ([]*Client, return ws.clients.Get(userID, platform) } -func NewWsServer(msgGatewayConfig *Config, opts ...Option) (*WsServer, error) { +func NewWsServer(msgGatewayConfig *Config, opts ...Option) *WsServer { var config configs for _, o := range opts { o(&config) @@ -144,7 +144,7 @@ func NewWsServer(msgGatewayConfig *Config, opts ...Option) (*WsServer, error) { Compressor: NewGzipCompressor(), Encoder: NewGobEncoder(), webhookClient: webhook.NewWebhookClient(msgGatewayConfig.WebhooksConfig.URL), - }, nil + } } func (ws *WsServer) Run(done chan error) error { @@ -278,11 +278,11 @@ func (ws *WsServer) registerClient(client *Client) { }() } - wg.Add(1) - go func() { - defer wg.Done() - ws.SetUserOnlineStatus(client.ctx, client, constant.Online) - }() + //wg.Add(1) + //go func() { + // defer wg.Done() + // ws.SetUserOnlineStatus(client.ctx, client, constant.Online) + //}() wg.Wait() @@ -351,7 +351,7 @@ func (ws *WsServer) unregisterClient(client *Client) { prommetrics.OnlineUserGauge.Dec() } ws.onlineUserConnNum.Add(-1) - ws.SetUserOnlineStatus(client.ctx, client, constant.Offline) + //ws.SetUserOnlineStatus(client.ctx, client, constant.Offline) log.ZInfo(client.ctx, "user offline", "close reason", client.closedErr, "online user Num", ws.onlineUserNum.Load(), "online user conn Num", ws.onlineUserConnNum.Load(), diff --git a/internal/msggateway/online.go b/internal/msggateway/online.go new file mode 100644 index 000000000..611ecbd79 --- /dev/null +++ b/internal/msggateway/online.go @@ -0,0 +1,106 @@ +package msggateway + +import ( + "context" + "crypto/md5" + "encoding/binary" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/cachekey" + pbuser "github.com/openimsdk/protocol/user" + "github.com/openimsdk/tools/log" + "github.com/openimsdk/tools/mcontext" + "github.com/openimsdk/tools/utils/datautil" + "math/rand" + "strconv" + "time" +) + +func (ws *WsServer) ChangeOnlineStatus(concurrent int) { + if concurrent < 1 { + concurrent = 1 + } + scanTicker := time.NewTicker(time.Minute * 3) + + requestChs := make([]chan *pbuser.SetUserOnlineStatusReq, concurrent) + changeStatus := make([][]UserState, concurrent) + + for i := 0; i < concurrent; i++ { + requestChs[i] = make(chan *pbuser.SetUserOnlineStatusReq, 64) + changeStatus[i] = make([]UserState, 100) + } + + mergeTicker := time.NewTicker(time.Second) + + local2pb := func(u UserState) *pbuser.UserOnlineStatus { + return &pbuser.UserOnlineStatus{ + UserID: u.UserID, + Online: u.Online, + Offline: u.Offline, + } + } + + rNum := rand.Uint64() + pushUserState := func(us ...UserState) { + for _, u := range us { + sum := md5.Sum([]byte(u.UserID)) + i := (binary.BigEndian.Uint64(sum[:]) + rNum) % uint64(concurrent) + changeStatus[i] = append(changeStatus[i], u) + status := changeStatus[i] + if len(status) == cap(status) { + req := &pbuser.SetUserOnlineStatusReq{ + Status: datautil.Slice(status, local2pb), + } + changeStatus[i] = status[0:] + select { + case requestChs[i] <- req: + default: + log.ZError(context.Background(), "user online processing is too slow", nil) + } + } + } + } + + pushAllUserState := func() { + for i, status := range changeStatus { + if len(status) == 0 { + continue + } + req := &pbuser.SetUserOnlineStatusReq{ + Status: datautil.Slice(status, local2pb), + } + changeStatus[i] = status[0:] + select { + case requestChs[i] <- req: + default: + log.ZError(context.Background(), "user online processing is too slow", nil) + } + } + } + + opIdCtx := mcontext.SetOperationID(context.Background(), "r"+strconv.FormatUint(rNum, 10)) + doRequest := func(req *pbuser.SetUserOnlineStatusReq) { + ctx, cancel := context.WithTimeout(opIdCtx, time.Second*5) + defer cancel() + if _, err := ws.userClient.Client.SetUserOnlineStatus(ctx, req); err != nil { + log.ZError(ctx, "update user online status", err) + } + } + + for i := 0; i < concurrent; i++ { + go func(ch <-chan *pbuser.SetUserOnlineStatusReq) { + for req := range ch { + doRequest(req) + } + }(requestChs[i]) + } + + for { + select { + case <-mergeTicker.C: + pushAllUserState() + case now := <-scanTicker.C: + pushUserState(ws.clients.GetAllUserStatus(now.Add(-cachekey.OnlineExpire/3), now)...) + case state := <-ws.clients.UserState(): + pushUserState(state) + } + } +} diff --git a/internal/msggateway/user_map.go b/internal/msggateway/user_map.go index f8bf69f9a..dcab066f8 100644 --- a/internal/msggateway/user_map.go +++ b/internal/msggateway/user_map.go @@ -14,122 +14,123 @@ package msggateway -import ( - "context" - "sync" - - "github.com/openimsdk/tools/log" - "github.com/openimsdk/tools/utils/datautil" -) - -type UserMap struct { - m sync.Map -} - -func newUserMap() *UserMap { - return &UserMap{} -} - -func (u *UserMap) GetAll(key string) ([]*Client, bool) { - allClients, ok := u.m.Load(key) - if ok { - return allClients.([]*Client), ok - } - return nil, ok -} - -func (u *UserMap) Get(key string, platformID int) ([]*Client, bool, bool) { - allClients, userExisted := u.m.Load(key) - if userExisted { - var clients []*Client - for _, client := range allClients.([]*Client) { - if client.PlatformID == platformID { - clients = append(clients, client) - } - } - if len(clients) > 0 { - return clients, userExisted, true - } - return clients, userExisted, false - } - return nil, userExisted, false -} - -// Set adds a client to the map. -func (u *UserMap) Set(key string, v *Client) { - allClients, existed := u.m.Load(key) - if existed { - log.ZDebug(context.Background(), "Set existed", "user_id", key, "client_user_id", v.UserID) - oldClients := allClients.([]*Client) - oldClients = append(oldClients, v) - u.m.Store(key, oldClients) - } else { - log.ZDebug(context.Background(), "Set not existed", "user_id", key, "client_user_id", v.UserID) - - var clients []*Client - clients = append(clients, v) - u.m.Store(key, clients) - } -} - -func (u *UserMap) Delete(key string, connRemoteAddr string) (isDeleteUser bool) { - // Attempt to load the clients associated with the key. - allClients, existed := u.m.Load(key) - if !existed { - // Return false immediately if the key does not exist. - return false - } - - // Convert allClients to a slice of *Client. - oldClients := allClients.([]*Client) - var remainingClients []*Client - for _, client := range oldClients { - // Keep clients that do not match the connRemoteAddr. - if client.ctx.GetRemoteAddr() != connRemoteAddr { - remainingClients = append(remainingClients, client) - } - } - - // If no clients remain after filtering, delete the key from the map. - if len(remainingClients) == 0 { - u.m.Delete(key) - return true - } - - // Otherwise, update the key with the remaining clients. - u.m.Store(key, remainingClients) - return false -} - -func (u *UserMap) DeleteClients(key string, clients []*Client) (isDeleteUser bool) { - m := datautil.SliceToMapAny(clients, func(c *Client) (string, struct{}) { - return c.ctx.GetRemoteAddr(), struct{}{} - }) - allClients, existed := u.m.Load(key) - if !existed { - // If the key doesn't exist, return false. - return false - } - - // Filter out clients that are in the deleteMap. - oldClients := allClients.([]*Client) - var remainingClients []*Client - for _, client := range oldClients { - if _, shouldBeDeleted := m[client.ctx.GetRemoteAddr()]; !shouldBeDeleted { - remainingClients = append(remainingClients, client) - } - } - - // Update or delete the key based on the remaining clients. - if len(remainingClients) == 0 { - u.m.Delete(key) - return true - } - - u.m.Store(key, remainingClients) - return false -} - -func (u *UserMap) DeleteAll(key string) { - u.m.Delete(key) -} +// +//import ( +// "context" +// "sync" +// +// "github.com/openimsdk/tools/log" +// "github.com/openimsdk/tools/utils/datautil" +//) +// +//type UserMap struct { +// m sync.Map +//} +// +//func newUserMap() UMap { +// return &UserMap{} +//} +// +//func (u *UserMap) GetAll(key string) ([]*Client, bool) { +// allClients, ok := u.m.Load(key) +// if ok { +// return allClients.([]*Client), ok +// } +// return nil, ok +//} +// +//func (u *UserMap) Get(key string, platformID int) ([]*Client, bool, bool) { +// allClients, userExisted := u.m.Load(key) +// if userExisted { +// var clients []*Client +// for _, client := range allClients.([]*Client) { +// if client.PlatformID == platformID { +// clients = append(clients, client) +// } +// } +// if len(clients) > 0 { +// return clients, userExisted, true +// } +// return clients, userExisted, false +// } +// return nil, userExisted, false +//} +// +//// Set adds a client to the map. +//func (u *UserMap) Set(key string, v *Client) { +// allClients, existed := u.m.Load(key) +// if existed { +// log.ZDebug(context.Background(), "Set existed", "user_id", key, "client_user_id", v.UserID) +// oldClients := allClients.([]*Client) +// oldClients = append(oldClients, v) +// u.m.Store(key, oldClients) +// } else { +// log.ZDebug(context.Background(), "Set not existed", "user_id", key, "client_user_id", v.UserID) +// +// var clients []*Client +// clients = append(clients, v) +// u.m.Store(key, clients) +// } +//} +// +//func (u *UserMap) Delete(key string, connRemoteAddr string) (isDeleteUser bool) { +// // Attempt to load the clients associated with the key. +// allClients, existed := u.m.Load(key) +// if !existed { +// // Return false immediately if the key does not exist. +// return false +// } +// +// // Convert allClients to a slice of *Client. +// oldClients := allClients.([]*Client) +// var remainingClients []*Client +// for _, client := range oldClients { +// // Keep clients that do not match the connRemoteAddr. +// if client.ctx.GetRemoteAddr() != connRemoteAddr { +// remainingClients = append(remainingClients, client) +// } +// } +// +// // If no clients remain after filtering, delete the key from the map. +// if len(remainingClients) == 0 { +// u.m.Delete(key) +// return true +// } +// +// // Otherwise, update the key with the remaining clients. +// u.m.Store(key, remainingClients) +// return false +//} +// +//func (u *UserMap) DeleteClients(key string, clients []*Client) (isDeleteUser bool) { +// m := datautil.SliceToMapAny(clients, func(c *Client) (string, struct{}) { +// return c.ctx.GetRemoteAddr(), struct{}{} +// }) +// allClients, existed := u.m.Load(key) +// if !existed { +// // If the key doesn't exist, return false. +// return false +// } +// +// // Filter out clients that are in the deleteMap. +// oldClients := allClients.([]*Client) +// var remainingClients []*Client +// for _, client := range oldClients { +// if _, shouldBeDeleted := m[client.ctx.GetRemoteAddr()]; !shouldBeDeleted { +// remainingClients = append(remainingClients, client) +// } +// } +// +// // Update or delete the key based on the remaining clients. +// if len(remainingClients) == 0 { +// u.m.Delete(key) +// return true +// } +// +// u.m.Store(key, remainingClients) +// return false +//} +// +//func (u *UserMap) DeleteAll(key string) { +// u.m.Delete(key) +//} diff --git a/internal/msggateway/user_map2.go b/internal/msggateway/user_map2.go index 1c9a91c6a..b6ed40373 100644 --- a/internal/msggateway/user_map2.go +++ b/internal/msggateway/user_map2.go @@ -1,32 +1,180 @@ package msggateway -/* - -sorted set - -userID: 10000 - -USER_ONLINE:10000 - - - - - -platformID: 1 - - - - - -key score - -E1 123456789 -O1 234567895 - - - - - - - -*/ +import ( + "sync" + "time" +) + +type UMap interface { + GetAll(userID string) ([]*Client, bool) + Get(userID string, platformID int) ([]*Client, bool, bool) + Set(userID string, v *Client) + Delete(userID string, connRemoteAddr string) (isDeleteUser bool) + DeleteClients(userID string, clients []*Client) (isDeleteUser bool) + UserState() <-chan UserState + GetAllUserStatus(deadline, nowtime time.Time) []UserState +} + +var _ UMap = (*UserMap2)(nil) + +type UserPlatform struct { + Time time.Time + Clients map[string]*Client +} + +func (u *UserPlatform) PlatformIDs() []int32 { + if len(u.Clients) == 0 { + return nil + } + platformIDs := make([]int32, 0, len(u.Clients)) + for _, client := range u.Clients { + platformIDs = append(platformIDs, int32(client.PlatformID)) + } + return platformIDs +} + +func newUserMap() UMap { + return &UserMap2{ + data: make(map[string]*UserPlatform), + ch: make(chan UserState, 10000), + } +} + +type UserMap2 struct { + lock sync.RWMutex + data map[string]*UserPlatform + ch chan UserState +} + +func (u *UserMap2) push(userID string, userPlatform *UserPlatform, offline []int32) bool { + select { + case u.ch <- UserState{UserID: userID, Online: userPlatform.PlatformIDs(), Offline: offline}: + userPlatform.Time = time.Now() + return true + default: + return false + } +} + +func (u *UserMap2) GetAll(userID string) ([]*Client, bool) { + u.lock.RLock() + defer u.lock.RUnlock() + result, ok := u.data[userID] + if !ok { + return nil, false + } + clients := make([]*Client, 0, len(result.Clients)) + for _, client := range result.Clients { + clients = append(clients, client) + } + return clients, true +} + +func (u *UserMap2) Get(userID string, platformID int) ([]*Client, bool, bool) { + u.lock.RLock() + defer u.lock.RUnlock() + result, ok := u.data[userID] + if !ok { + return nil, false, false + } + var clients []*Client + for _, client := range result.Clients { + if client.PlatformID == platformID { + clients = append(clients, client) + } + } + return clients, true, len(clients) > 0 +} + +func (u *UserMap2) Set(userID string, client *Client) { + u.lock.Lock() + defer u.lock.Unlock() + result, ok := u.data[userID] + if ok { + result.Clients[client.ctx.GetRemoteAddr()] = client + } else { + result = &UserPlatform{ + Clients: map[string]*Client{ + client.ctx.GetRemoteAddr(): client, + }, + } + } + u.push(client.UserID, result, nil) +} + +func (u *UserMap2) Delete(userID string, connRemoteAddr string) (isDeleteUser bool) { + u.lock.Lock() + defer u.lock.Unlock() + result, ok := u.data[userID] + if !ok { + return false + } + client, ok := result.Clients[connRemoteAddr] + if !ok { + return false + } + delete(result.Clients, connRemoteAddr) + defer u.push(userID, result, []int32{int32(client.PlatformID)}) + if len(result.Clients) > 0 { + return false + } + delete(u.data, userID) + return true +} + +func (u *UserMap2) DeleteClients(userID string, clients []*Client) (isDeleteUser bool) { + if len(clients) == 0 { + return false + } + u.lock.Lock() + defer u.lock.Unlock() + result, ok := u.data[userID] + if !ok { + return false + } + offline := make([]int32, 0, len(clients)) + for _, client := range clients { + offline = append(offline, int32(client.PlatformID)) + delete(result.Clients, client.ctx.GetRemoteAddr()) + } + defer u.push(userID, result, offline) + if len(result.Clients) > 0 { + return false + } + delete(u.data, userID) + return true +} + +func (u *UserMap2) GetAllUserStatus(deadline, nowtime time.Time) []UserState { + u.lock.RLock() + defer u.lock.RUnlock() + if len(u.data) == 0 { + return nil + } + result := make([]UserState, 0, len(u.data)) + for userID, p := range u.data { + if len(result) == cap(result) { + break + } + if p.Time.Before(deadline) { + continue + } + p.Time = nowtime + online := make([]int32, 0, len(p.Clients)) + for _, client := range p.Clients { + online = append(online, int32(client.PlatformID)) + } + result = append(result, UserState{UserID: userID, Online: online}) + } + return result +} + +func (u *UserMap2) UserState() <-chan UserState { + return u.ch +} + +type UserState struct { + UserID string + Online []int32 + Offline []int32 +} diff --git a/internal/rpc/user/online.go b/internal/rpc/user/online.go new file mode 100644 index 000000000..e853ceae2 --- /dev/null +++ b/internal/rpc/user/online.go @@ -0,0 +1,122 @@ +package user + +import ( + "context" + "github.com/openimsdk/protocol/constant" + "github.com/openimsdk/protocol/sdkws" + pbuser "github.com/openimsdk/protocol/user" +) + +func (s *userServer) getUserOnlineStatus(ctx context.Context, userID string) (*pbuser.OnlineStatus, error) { + platformIDs, err := s.online.GetOnline(ctx, userID) + if err != nil { + return nil, err + } + status := pbuser.OnlineStatus{ + UserID: userID, + PlatformIDs: platformIDs, + } + if len(platformIDs) > 0 { + status.Status = constant.Online + } else { + status.Status = constant.Offline + } + return &status, nil +} + +func (s *userServer) getUsersOnlineStatus(ctx context.Context, userIDs []string) ([]*pbuser.OnlineStatus, error) { + res := make([]*pbuser.OnlineStatus, 0, len(userIDs)) + for _, userID := range userIDs { + status, err := s.getUserOnlineStatus(ctx, userID) + if err != nil { + return nil, err + } + res = append(res, status) + } + return res, nil +} + +// SubscribeOrCancelUsersStatus Subscribe online or cancel online users. +func (s *userServer) SubscribeOrCancelUsersStatus(ctx context.Context, req *pbuser.SubscribeOrCancelUsersStatusReq) (*pbuser.SubscribeOrCancelUsersStatusResp, error) { + if req.Genre == constant.SubscriberUser { + err := s.db.SubscribeUsersStatus(ctx, req.UserID, req.UserIDs) + if err != nil { + return nil, err + } + var status []*pbuser.OnlineStatus + status, err = s.getUsersOnlineStatus(ctx, req.UserIDs) + if err != nil { + return nil, err + } + return &pbuser.SubscribeOrCancelUsersStatusResp{StatusList: status}, nil + } else if req.Genre == constant.Unsubscribe { + err := s.db.UnsubscribeUsersStatus(ctx, req.UserID, req.UserIDs) + if err != nil { + return nil, err + } + } + return &pbuser.SubscribeOrCancelUsersStatusResp{}, nil +} + +// GetUserStatus Get the online status of the user. +func (s *userServer) GetUserStatus(ctx context.Context, req *pbuser.GetUserStatusReq) (*pbuser.GetUserStatusResp, error) { + res, err := s.getUsersOnlineStatus(ctx, req.UserIDs) + if err != nil { + return nil, err + } + return &pbuser.GetUserStatusResp{StatusList: res}, nil +} + +// SetUserStatus Synchronize user's online status. +func (s *userServer) SetUserStatus(ctx context.Context, req *pbuser.SetUserStatusReq) (*pbuser.SetUserStatusResp, error) { + var ( + online []int32 + offline []int32 + ) + switch req.Status { + case constant.Online: + online = []int32{req.PlatformID} + case constant.Offline: + online = []int32{req.PlatformID} + } + if err := s.online.SetUserOnline(ctx, req.UserID, online, offline); err != nil { + return nil, err + } + list, err := s.db.GetSubscribedList(ctx, req.UserID) + if err != nil { + return nil, err + } + for _, userID := range list { + tips := &sdkws.UserStatusChangeTips{ + FromUserID: req.UserID, + ToUserID: userID, + Status: req.Status, + PlatformID: req.PlatformID, + } + s.userNotificationSender.UserStatusChangeNotification(ctx, tips) + } + + return &pbuser.SetUserStatusResp{}, nil +} + +// GetSubscribeUsersStatus Get the online status of subscribers. +func (s *userServer) GetSubscribeUsersStatus(ctx context.Context, req *pbuser.GetSubscribeUsersStatusReq) (*pbuser.GetSubscribeUsersStatusResp, error) { + userList, err := s.db.GetAllSubscribeList(ctx, req.UserID) + if err != nil { + return nil, err + } + onlineStatusList, err := s.getUsersOnlineStatus(ctx, userList) + if err != nil { + return nil, err + } + return &pbuser.GetSubscribeUsersStatusResp{StatusList: onlineStatusList}, nil +} + +func (s *userServer) SetUserOnlineStatus(ctx context.Context, req *pbuser.SetUserOnlineStatusReq) (*pbuser.SetUserOnlineStatusResp, error) { + for _, status := range req.Status { + if err := s.online.SetUserOnline(ctx, status.UserID, status.Online, status.Offline); err != nil { + return nil, err + } + } + return &pbuser.SetUserOnlineStatusResp{}, nil +} diff --git a/internal/rpc/user/user.go b/internal/rpc/user/user.go index 211b360b7..0b96077ec 100644 --- a/internal/rpc/user/user.go +++ b/internal/rpc/user/user.go @@ -19,6 +19,7 @@ import ( "errors" "github.com/openimsdk/open-im-server/v3/internal/rpc/friend" "github.com/openimsdk/open-im-server/v3/pkg/common/config" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/redis" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database/mgo" tablerelation "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" @@ -50,6 +51,7 @@ import ( ) type userServer struct { + online cache.OnlineCache db controller.UserDatabase friendNotificationSender *friend.FriendNotificationSender userNotificationSender *UserNotificationSender @@ -98,6 +100,7 @@ func Start(ctx context.Context, config *Config, client registry.SvcDiscoveryRegi msgRpcClient := rpcclient.NewMessageRpcClient(client, config.Share.RpcRegisterName.Msg) localcache.InitLocalCache(&config.LocalCacheConfig) u := &userServer{ + online: redis.NewUserOnline(rdb), db: database, RegisterCenter: client, friendRpcClient: &friendRpcClient, @@ -329,76 +332,6 @@ func (s *userServer) GetAllUserID(ctx context.Context, req *pbuser.GetAllUserIDR return &pbuser.GetAllUserIDResp{Total: int32(total), UserIDs: userIDs}, nil } -// SubscribeOrCancelUsersStatus Subscribe online or cancel online users. -func (s *userServer) SubscribeOrCancelUsersStatus(ctx context.Context, req *pbuser.SubscribeOrCancelUsersStatusReq) (resp *pbuser.SubscribeOrCancelUsersStatusResp, err error) { - if req.Genre == constant.SubscriberUser { - err = s.db.SubscribeUsersStatus(ctx, req.UserID, req.UserIDs) - if err != nil { - return nil, err - } - var status []*pbuser.OnlineStatus - status, err = s.db.GetUserStatus(ctx, req.UserIDs) - if err != nil { - return nil, err - } - return &pbuser.SubscribeOrCancelUsersStatusResp{StatusList: status}, nil - } else if req.Genre == constant.Unsubscribe { - err = s.db.UnsubscribeUsersStatus(ctx, req.UserID, req.UserIDs) - if err != nil { - return nil, err - } - } - return &pbuser.SubscribeOrCancelUsersStatusResp{}, nil -} - -// GetUserStatus Get the online status of the user. -func (s *userServer) GetUserStatus(ctx context.Context, req *pbuser.GetUserStatusReq) (resp *pbuser.GetUserStatusResp, - err error) { - onlineStatusList, err := s.db.GetUserStatus(ctx, req.UserIDs) - if err != nil { - return nil, err - } - return &pbuser.GetUserStatusResp{StatusList: onlineStatusList}, nil -} - -// SetUserStatus Synchronize user's online status. -func (s *userServer) SetUserStatus(ctx context.Context, req *pbuser.SetUserStatusReq) (resp *pbuser.SetUserStatusResp, - err error) { - err = s.db.SetUserStatus(ctx, req.UserID, req.Status, req.PlatformID) - if err != nil { - return nil, err - } - list, err := s.db.GetSubscribedList(ctx, req.UserID) - if err != nil { - return nil, err - } - for _, userID := range list { - tips := &sdkws.UserStatusChangeTips{ - FromUserID: req.UserID, - ToUserID: userID, - Status: req.Status, - PlatformID: req.PlatformID, - } - s.userNotificationSender.UserStatusChangeNotification(ctx, tips) - } - - return &pbuser.SetUserStatusResp{}, nil -} - -// GetSubscribeUsersStatus Get the online status of subscribers. -func (s *userServer) GetSubscribeUsersStatus(ctx context.Context, - req *pbuser.GetSubscribeUsersStatusReq) (*pbuser.GetSubscribeUsersStatusResp, error) { - userList, err := s.db.GetAllSubscribeList(ctx, req.UserID) - if err != nil { - return nil, err - } - onlineStatusList, err := s.db.GetUserStatus(ctx, userList) - if err != nil { - return nil, err - } - return &pbuser.GetSubscribeUsersStatusResp{StatusList: onlineStatusList}, nil -} - // ProcessUserCommandAdd user general function add. func (s *userServer) ProcessUserCommandAdd(ctx context.Context, req *pbuser.ProcessUserCommandAddReq) (*pbuser.ProcessUserCommandAddResp, error) { err := authverify.CheckAccessV3(ctx, req.UserID, s.config.Share.IMAdminUserID) diff --git a/internal/tools/cron_task.go b/internal/tools/cron_task.go index bf037b694..50f328b55 100644 --- a/internal/tools/cron_task.go +++ b/internal/tools/cron_task.go @@ -65,10 +65,16 @@ func Start(ctx context.Context, config *CronTaskConfig) error { return } log.ZInfo(ctx, "cron clear chat records success", "deltime", deltime, "cont", time.Since(now)) + } + delFile := func() { + } if _, err := crontab.AddFunc(config.CronTask.ChatRecordsClearTime, clearFunc); err != nil { return errs.Wrap(err) } + if _, err := crontab.AddFunc(config.CronTask.ChatRecordsClearTime, delFile); err != nil { + return errs.Wrap(err) + } log.ZInfo(ctx, "start cron task", "chatRecordsClearTime", config.CronTask.ChatRecordsClearTime) crontab.Start() <-ctx.Done() diff --git a/pkg/common/config/config.go b/pkg/common/config/config.go index 6260dc00f..a4e613560 100644 --- a/pkg/common/config/config.go +++ b/pkg/common/config/config.go @@ -107,6 +107,7 @@ type API struct { type CronTask struct { ChatRecordsClearTime string `mapstructure:"chatRecordsClearTime"` RetainChatRecords int `mapstructure:"retainChatRecords"` + FileTime int } type OfflinePushConfig struct { diff --git a/pkg/common/storage/cache/cachekey/online.go b/pkg/common/storage/cache/cachekey/online.go new file mode 100644 index 000000000..164e5f2f4 --- /dev/null +++ b/pkg/common/storage/cache/cachekey/online.go @@ -0,0 +1,13 @@ +package cachekey + +import "time" + +const ( + OnlineKey = "ONLINE:" + OnlineChannel = "online_change" + OnlineExpire = time.Hour / 2 +) + +func GetOnlineKey(userID string) string { + return OnlineKey + userID +} diff --git a/pkg/common/storage/cache/cachekey/user.go b/pkg/common/storage/cache/cachekey/user.go index 7d06d4f75..473ca1b12 100644 --- a/pkg/common/storage/cache/cachekey/user.go +++ b/pkg/common/storage/cache/cachekey/user.go @@ -17,7 +17,6 @@ package cachekey const ( UserInfoKey = "USER_INFO:" UserGlobalRecvMsgOptKey = "USER_GLOBAL_RECV_MSG_OPT_KEY:" - olineStatusKey = "ONLINE_STATUS:" ) func GetUserInfoKey(userID string) string { @@ -27,7 +26,3 @@ func GetUserInfoKey(userID string) string { func GetUserGlobalRecvMsgOptKey(userID string) string { return UserGlobalRecvMsgOptKey + userID } - -func GetOnlineStatusKey(modKey string) string { - return olineStatusKey + modKey -} diff --git a/pkg/common/storage/cache/online.go b/pkg/common/storage/cache/online.go new file mode 100644 index 000000000..7669c8a11 --- /dev/null +++ b/pkg/common/storage/cache/online.go @@ -0,0 +1,8 @@ +package cache + +import "context" + +type OnlineCache interface { + GetOnline(ctx context.Context, userID string) ([]int32, error) + SetUserOnline(ctx context.Context, userID string, online, offline []int32) error +} diff --git a/pkg/common/storage/cache/redis/online.go b/pkg/common/storage/cache/redis/online.go index 138f9a573..dc6a5f775 100644 --- a/pkg/common/storage/cache/redis/online.go +++ b/pkg/common/storage/cache/redis/online.go @@ -2,10 +2,22 @@ package redis import ( "context" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/cachekey" + "github.com/openimsdk/tools/errs" "github.com/redis/go-redis/v9" + "strconv" "time" ) +func NewUserOnline(rdb redis.UniversalClient) cache.OnlineCache { + return &userOnline{ + rdb: rdb, + expire: cachekey.OnlineExpire, + channelName: cachekey.OnlineChannel, + } +} + type userOnline struct { rdb redis.UniversalClient expire time.Duration @@ -13,7 +25,26 @@ type userOnline struct { } func (s *userOnline) getUserOnlineKey(userID string) string { - return "USER_ONLINE:" + userID + return cachekey.GetOnlineKey(userID) +} + +func (s *userOnline) GetOnline(ctx context.Context, userID string) ([]int32, error) { + members, err := s.rdb.ZRangeByScore(ctx, s.getUserOnlineKey(userID), &redis.ZRangeBy{ + Min: strconv.FormatInt(time.Now().Unix(), 10), + Max: "+inf", + }).Result() + if err != nil { + return nil, errs.Wrap(err) + } + platformIDs := make([]int32, 0, len(members)) + for _, member := range members { + val, err := strconv.Atoi(member) + if err != nil { + return nil, errs.Wrap(err) + } + platformIDs = append(platformIDs, int32(val)) + } + return platformIDs, nil } func (s *userOnline) SetUserOnline(ctx context.Context, userID string, online, offline []int32) error { diff --git a/pkg/common/storage/cache/redis/seq_user_test.go b/pkg/common/storage/cache/redis/seq_user_test.go index 04a5d49cb..b4c852cd0 100644 --- a/pkg/common/storage/cache/redis/seq_user_test.go +++ b/pkg/common/storage/cache/redis/seq_user_test.go @@ -5,6 +5,8 @@ import ( "fmt" "github.com/redis/go-redis/v9" "log" + "strconv" + "sync/atomic" "testing" "time" ) @@ -24,30 +26,39 @@ func newTestOnline() *userOnline { func TestOnline(t *testing.T) { ts := newTestOnline() + var count atomic.Int64 + for i := 0; i < 64; i++ { + go func(userID string) { + var err error + for i := 0; ; i++ { + if i%2 == 0 { + err = ts.SetUserOnline(context.Background(), userID, []int32{5, 6}, []int32{7, 8, 9}) + } else { + err = ts.SetUserOnline(context.Background(), userID, []int32{1, 2, 3}, []int32{4, 5, 6}) + } + if err != nil { + panic(err) + } + count.Add(1) + } + }(strconv.Itoa(10000 + i)) + } - //err := ts.SetUserOnline(context.Background(), "1000", []int32{1, 2, 3}, []int32{4, 5, 6}) - err := ts.SetUserOnline(context.Background(), "1000", nil, []int32{1, 2, 3}) - - t.Log(err) - + ticker := time.NewTicker(time.Second) + for range ticker.C { + t.Log(count.Swap(0)) + } } -/* - -local function tableToString(tbl, separator) - local result = {} - for _, v in ipairs(tbl) do - table.insert(result, tostring(v)) - end - return table.concat(result, separator) -end - -local myTable = {"one", "two", "three"} -local result = tableToString(myTable, ":") - -print(result) - -*/ +func TestGetOnline(t *testing.T) { + ts := newTestOnline() + ctx := context.Background() + pIDs, err := ts.GetOnline(ctx, "10000") + if err != nil { + panic(err) + } + t.Log(pIDs) +} func TestRecvOnline(t *testing.T) { ts := newTestOnline() diff --git a/pkg/common/storage/cache/redis/user.go b/pkg/common/storage/cache/redis/user.go index c05cd3895..b846bf398 100644 --- a/pkg/common/storage/cache/redis/user.go +++ b/pkg/common/storage/cache/redis/user.go @@ -17,7 +17,6 @@ package redis import ( "context" "encoding/json" - "errors" "github.com/dtm-labs/rockscache" "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache" @@ -29,8 +28,6 @@ import ( "github.com/openimsdk/tools/errs" "github.com/openimsdk/tools/log" "github.com/redis/go-redis/v9" - "hash/crc32" - "strconv" "time" ) @@ -61,9 +58,9 @@ func NewUserCacheRedis(rdb redis.UniversalClient, localCache *config.LocalCache, } } -func (u *UserCacheRedis) getOnlineStatusKey(modKey string) string { - return cachekey.GetOnlineStatusKey(modKey) -} +//func (u *UserCacheRedis) getOnlineStatusKey(modKey string) string { +// return cachekey.GetOnlineStatusKey(modKey) +//} func (u *UserCacheRedis) CloneUserCache() cache.UserCache { return &UserCacheRedis{ @@ -141,95 +138,95 @@ type RedisUserOnline struct { } // GetUserStatus get user status. -func (u *UserCacheRedis) GetUserStatus(ctx context.Context, userIDs []string) ([]*user.OnlineStatus, error) { - userStatus := make([]*user.OnlineStatus, 0, len(userIDs)) - for _, userID := range userIDs { - UserIDNum := crc32.ChecksumIEEE([]byte(userID)) - modKey := strconv.Itoa(int(UserIDNum % statusMod)) - var onlineStatus user.OnlineStatus - key := u.getOnlineStatusKey(modKey) - result, err := u.rdb.HGet(ctx, key, userID).Result() - if err != nil { - if errors.Is(err, redis.Nil) { - // key or field does not exist - userStatus = append(userStatus, &user.OnlineStatus{ - UserID: userID, - Status: constant.Offline, - PlatformIDs: nil, - }) - - continue - } else { - return nil, errs.Wrap(err) - } - } - err = json.Unmarshal([]byte(result), &onlineStatus) - if err != nil { - return nil, errs.Wrap(err) - } - onlineStatus.UserID = userID - onlineStatus.Status = constant.Online - userStatus = append(userStatus, &onlineStatus) - } - - return userStatus, nil -} +//func (u *UserCacheRedis) GetUserStatus(ctx context.Context, userIDs []string) ([]*user.OnlineStatus, error) { +// userStatus := make([]*user.OnlineStatus, 0, len(userIDs)) +// for _, userID := range userIDs { +// UserIDNum := crc32.ChecksumIEEE([]byte(userID)) +// modKey := strconv.Itoa(int(UserIDNum % statusMod)) +// var onlineStatus user.OnlineStatus +// key := u.getOnlineStatusKey(modKey) +// result, err := u.rdb.HGet(ctx, key, userID).Result() +// if err != nil { +// if errors.Is(err, redis.Nil) { +// // key or field does not exist +// userStatus = append(userStatus, &user.OnlineStatus{ +// UserID: userID, +// Status: constant.Offline, +// PlatformIDs: nil, +// }) +// +// continue +// } else { +// return nil, errs.Wrap(err) +// } +// } +// err = json.Unmarshal([]byte(result), &onlineStatus) +// if err != nil { +// return nil, errs.Wrap(err) +// } +// onlineStatus.UserID = userID +// onlineStatus.Status = constant.Online +// userStatus = append(userStatus, &onlineStatus) +// } +// +// return userStatus, nil +//} // SetUserStatus Set the user status and save it in redis. -func (u *UserCacheRedis) SetUserStatus(ctx context.Context, userID string, status, platformID int32) error { - UserIDNum := crc32.ChecksumIEEE([]byte(userID)) - modKey := strconv.Itoa(int(UserIDNum % statusMod)) - key := u.getOnlineStatusKey(modKey) - log.ZDebug(ctx, "SetUserStatus args", "userID", userID, "status", status, "platformID", platformID, "modKey", modKey, "key", key) - isNewKey, err := u.rdb.Exists(ctx, key).Result() - if err != nil { - return errs.Wrap(err) - } - if isNewKey == 0 { - if status == constant.Online { - onlineStatus := user.OnlineStatus{ - UserID: userID, - Status: constant.Online, - PlatformIDs: []int32{platformID}, - } - jsonData, err := json.Marshal(&onlineStatus) - if err != nil { - return errs.Wrap(err) - } - _, err = u.rdb.HSet(ctx, key, userID, string(jsonData)).Result() - if err != nil { - return errs.Wrap(err) - } - u.rdb.Expire(ctx, key, userOlineStatusExpireTime) - - return nil - } - } - - isNil := false - result, err := u.rdb.HGet(ctx, key, userID).Result() - if err != nil { - if errors.Is(err, redis.Nil) { - isNil = true - } else { - return errs.Wrap(err) - } - } - - if status == constant.Offline { - err = u.refreshStatusOffline(ctx, userID, status, platformID, isNil, err, result, key) - if err != nil { - return err - } - } else { - err = u.refreshStatusOnline(ctx, userID, platformID, isNil, err, result, key) - if err != nil { - return errs.Wrap(err) - } - } - - return nil -} +//func (u *UserCacheRedis) SetUserStatus(ctx context.Context, userID string, status, platformID int32) error { +// UserIDNum := crc32.ChecksumIEEE([]byte(userID)) +// modKey := strconv.Itoa(int(UserIDNum % statusMod)) +// key := u.getOnlineStatusKey(modKey) +// log.ZDebug(ctx, "SetUserStatus args", "userID", userID, "status", status, "platformID", platformID, "modKey", modKey, "key", key) +// isNewKey, err := u.rdb.Exists(ctx, key).Result() +// if err != nil { +// return errs.Wrap(err) +// } +// if isNewKey == 0 { +// if status == constant.Online { +// onlineStatus := user.OnlineStatus{ +// UserID: userID, +// Status: constant.Online, +// PlatformIDs: []int32{platformID}, +// } +// jsonData, err := json.Marshal(&onlineStatus) +// if err != nil { +// return errs.Wrap(err) +// } +// _, err = u.rdb.HSet(ctx, key, userID, string(jsonData)).Result() +// if err != nil { +// return errs.Wrap(err) +// } +// u.rdb.Expire(ctx, key, userOlineStatusExpireTime) +// +// return nil +// } +// } +// +// isNil := false +// result, err := u.rdb.HGet(ctx, key, userID).Result() +// if err != nil { +// if errors.Is(err, redis.Nil) { +// isNil = true +// } else { +// return errs.Wrap(err) +// } +// } +// +// if status == constant.Offline { +// err = u.refreshStatusOffline(ctx, userID, status, platformID, isNil, err, result, key) +// if err != nil { +// return err +// } +// } else { +// err = u.refreshStatusOnline(ctx, userID, platformID, isNil, err, result, key) +// if err != nil { +// return errs.Wrap(err) +// } +// } +// +// return nil +//} func (u *UserCacheRedis) refreshStatusOffline(ctx context.Context, userID string, status, platformID int32, isNil bool, err error, result, key string) error { if isNil { diff --git a/pkg/common/storage/cache/user.go b/pkg/common/storage/cache/user.go index 5101c0b6c..69a11635c 100644 --- a/pkg/common/storage/cache/user.go +++ b/pkg/common/storage/cache/user.go @@ -17,7 +17,6 @@ package cache import ( "context" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" - "github.com/openimsdk/protocol/user" ) type UserCache interface { @@ -28,6 +27,6 @@ type UserCache interface { DelUsersInfo(userIDs ...string) UserCache GetUserGlobalRecvMsgOpt(ctx context.Context, userID string) (opt int, err error) DelUsersGlobalRecvMsgOpt(userIDs ...string) UserCache - GetUserStatus(ctx context.Context, userIDs []string) ([]*user.OnlineStatus, error) - SetUserStatus(ctx context.Context, userID string, status, platformID int32) error + //GetUserStatus(ctx context.Context, userIDs []string) ([]*user.OnlineStatus, error) + //SetUserStatus(ctx context.Context, userID string, status, platformID int32) error } diff --git a/pkg/common/storage/controller/user.go b/pkg/common/storage/controller/user.go index 9efe535c0..321eff03c 100644 --- a/pkg/common/storage/controller/user.go +++ b/pkg/common/storage/controller/user.go @@ -70,10 +70,6 @@ type UserDatabase interface { GetAllSubscribeList(ctx context.Context, userID string) ([]string, error) // GetSubscribedList Get all subscribed lists GetSubscribedList(ctx context.Context, userID string) ([]string, error) - // GetUserStatus Get the online status of the user - GetUserStatus(ctx context.Context, userIDs []string) ([]*user.OnlineStatus, error) - // SetUserStatus Set the user status and store the user status in redis - SetUserStatus(ctx context.Context, userID string, status, platformID int32) error // CRUD user command AddUserCommand(ctx context.Context, userID string, Type int32, UUID string, value string, ex string) error @@ -246,17 +242,6 @@ func (u *userDatabase) GetSubscribedList(ctx context.Context, userID string) ([] return list, nil } -// GetUserStatus get user status. -func (u *userDatabase) GetUserStatus(ctx context.Context, userIDs []string) ([]*user.OnlineStatus, error) { - onlineStatusList, err := u.cache.GetUserStatus(ctx, userIDs) - return onlineStatusList, err -} - -// SetUserStatus Set the user status and save it in redis. -func (u *userDatabase) SetUserStatus(ctx context.Context, userID string, status, platformID int32) error { - return u.cache.SetUserStatus(ctx, userID, status, platformID) -} - func (u *userDatabase) AddUserCommand(ctx context.Context, userID string, Type int32, UUID string, value string, ex string) error { return u.userDB.AddUserCommand(ctx, userID, Type, UUID, value, ex) } From 3960d28a106c52c7cd402adf2418ed0c25d8b8a1 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Fri, 28 Jun 2024 16:22:20 +0800 Subject: [PATCH 03/27] config --- config/discovery.yml | 4 ++-- config/kafka.yml | 2 +- config/minio.yml | 4 ++-- config/mongodb.yml | 2 +- config/redis.yml | 2 +- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/config/discovery.yml b/config/discovery.yml index 9cd23c590..3d96ff9b6 100644 --- a/config/discovery.yml +++ b/config/discovery.yml @@ -1,13 +1,13 @@ enable: "etcd" etcd: rootDirectory: openim - address: [ 172.16.8.48:12379 ] + address: [ localhost:12379 ] username: '' password: '' zookeeper: schema: openim - address: [ 172.16.8.48:12181 ] + address: [ localhost:12181 ] username: '' password: '' diff --git a/config/kafka.yml b/config/kafka.yml index d9b7ffa3c..d412e1be0 100644 --- a/config/kafka.yml +++ b/config/kafka.yml @@ -7,7 +7,7 @@ producerAck: "" # Compression type to use (e.g., none, gzip, snappy) compressType: "none" # List of Kafka broker addresses -address: [ 172.16.8.48:19094 ] +address: [ localhost:19094 ] # Kafka topic for Redis integration toRedisTopic: "toRedis" # Kafka topic for MongoDB integration diff --git a/config/minio.yml b/config/minio.yml index d143a1da3..11a9ace35 100644 --- a/config/minio.yml +++ b/config/minio.yml @@ -7,9 +7,9 @@ secretAccessKey: "openIM123" # Session token for MinIO authentication (optional) sessionToken: '' # Internal address of the MinIO server -internalAddress: "172.16.8.48:10005" +internalAddress: "localhost:10005" # External address of the MinIO server, accessible from outside. Supports both HTTP and HTTPS using a domain name -externalAddress: "http://172.16.8.48:10005" +externalAddress: "http://external_ip:10005" # Flag to enable or disable public read access to the bucket publicRead: false diff --git a/config/mongodb.yml b/config/mongodb.yml index 53969298b..98f5694e4 100644 --- a/config/mongodb.yml +++ b/config/mongodb.yml @@ -1,7 +1,7 @@ # URI for database connection, leave empty if using address and credential settings directly uri: '' # List of MongoDB server addresses -address: [ 172.16.8.48:37017 ] +address: [ localhost:37017 ] # Name of the database database: openim_v3 # Username for database authentication diff --git a/config/redis.yml b/config/redis.yml index 404d18953..87abed0e1 100644 --- a/config/redis.yml +++ b/config/redis.yml @@ -1,4 +1,4 @@ -address: [ 172.16.8.48:16379 ] +address: [ localhost:16379 ] username: '' password: openIM123 clusterMode: false From affa909e175c4f1b800d0e6dec7e73fcf50b351e Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Fri, 28 Jun 2024 16:31:53 +0800 Subject: [PATCH 04/27] fix --- internal/msggateway/online.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/msggateway/online.go b/internal/msggateway/online.go index 611ecbd79..a4a1c1f15 100644 --- a/internal/msggateway/online.go +++ b/internal/msggateway/online.go @@ -25,7 +25,7 @@ func (ws *WsServer) ChangeOnlineStatus(concurrent int) { for i := 0; i < concurrent; i++ { requestChs[i] = make(chan *pbuser.SetUserOnlineStatusReq, 64) - changeStatus[i] = make([]UserState, 100) + changeStatus[i] = make([]UserState, 0, 100) } mergeTicker := time.NewTicker(time.Second) From 3ef62d4aa5b32df5324cd0dfd02b4344844b3802 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Fri, 28 Jun 2024 16:57:01 +0800 Subject: [PATCH 05/27] fix --- internal/msggateway/online.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/internal/msggateway/online.go b/internal/msggateway/online.go index a4a1c1f15..61549083a 100644 --- a/internal/msggateway/online.go +++ b/internal/msggateway/online.go @@ -49,7 +49,7 @@ func (ws *WsServer) ChangeOnlineStatus(concurrent int) { req := &pbuser.SetUserOnlineStatusReq{ Status: datautil.Slice(status, local2pb), } - changeStatus[i] = status[0:] + changeStatus[i] = status[:0] select { case requestChs[i] <- req: default: @@ -67,7 +67,7 @@ func (ws *WsServer) ChangeOnlineStatus(concurrent int) { req := &pbuser.SetUserOnlineStatusReq{ Status: datautil.Slice(status, local2pb), } - changeStatus[i] = status[0:] + changeStatus[i] = status[:0] select { case requestChs[i] <- req: default: @@ -100,6 +100,7 @@ func (ws *WsServer) ChangeOnlineStatus(concurrent int) { case now := <-scanTicker.C: pushUserState(ws.clients.GetAllUserStatus(now.Add(-cachekey.OnlineExpire/3), now)...) case state := <-ws.clients.UserState(): + log.ZDebug(context.Background(), "user online change", "userID", state.UserID, "online", state.Online, "offline", state.Offline) pushUserState(state) } } From 31677806207b394bc86608dc48de3bdd88ae304c Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Mon, 1 Jul 2024 16:29:13 +0800 Subject: [PATCH 06/27] online cache --- internal/push/push_handler.go | 17 ++- .../storage/cache/redis/seq_user_test.go | 5 +- pkg/localcache/cache.go | 12 +- pkg/localcache/lru/lru.go | 1 + pkg/localcache/lru/lru_expiration.go | 10 ++ pkg/localcache/lru/lru_lazy.go | 22 ++++ pkg/localcache/lru/lru_slot.go | 4 + pkg/localcache/option.go | 14 +- pkg/rpccache/online.go | 121 ++++++++++++++++++ pkg/rpccache/user.go | 15 +++ pkg/rpcclient/user.go | 22 ++++ 11 files changed, 223 insertions(+), 20 deletions(-) create mode 100644 pkg/rpccache/online.go diff --git a/internal/push/push_handler.go b/internal/push/push_handler.go index 03c299b7a..029330f1a 100644 --- a/internal/push/push_handler.go +++ b/internal/push/push_handler.go @@ -45,6 +45,7 @@ type ConsumerHandler struct { pushConsumerGroup *kafka.MConsumerGroup offlinePusher offlinepush.OfflinePusher onlinePusher OnlinePusher + onlineCache *rpccache.OnlineCache groupLocalCache *rpccache.GroupLocalCache conversationLocalCache *rpccache.ConversationLocalCache msgRpcClient rpcclient.MessageRpcClient @@ -63,16 +64,17 @@ func NewConsumerHandler(config *Config, offlinePusher offlinepush.OfflinePusher, if err != nil { return nil, err } + userRpcClient := rpcclient.NewUserRpcClient(client, config.Share.RpcRegisterName.User, config.Share.IMAdminUserID) consumerHandler.offlinePusher = offlinePusher consumerHandler.onlinePusher = NewOnlinePusher(client, config) consumerHandler.groupRpcClient = rpcclient.NewGroupRpcClient(client, config.Share.RpcRegisterName.Group) consumerHandler.groupLocalCache = rpccache.NewGroupLocalCache(consumerHandler.groupRpcClient, &config.LocalCacheConfig, rdb) consumerHandler.msgRpcClient = rpcclient.NewMessageRpcClient(client, config.Share.RpcRegisterName.Msg) consumerHandler.conversationRpcClient = rpcclient.NewConversationRpcClient(client, config.Share.RpcRegisterName.Conversation) - consumerHandler.conversationLocalCache = rpccache.NewConversationLocalCache(consumerHandler.conversationRpcClient, - &config.LocalCacheConfig, rdb) + consumerHandler.conversationLocalCache = rpccache.NewConversationLocalCache(consumerHandler.conversationRpcClient, &config.LocalCacheConfig, rdb) consumerHandler.webhookClient = webhook.NewWebhookClient(config.WebhooksConfig.URL) consumerHandler.config = config + consumerHandler.onlineCache = rpccache.NewOnlineCache(userRpcClient, consumerHandler.groupLocalCache, rdb) return &consumerHandler, nil } @@ -100,11 +102,13 @@ func (c *ConsumerHandler) handleMs2PsChat(ctx context.Context, msg []byte) { var pushUserIDList []string isSenderSync := datautil.GetSwitchFromOptions(pbData.MsgData.Options, constant.IsSenderSync) if !isSenderSync || pbData.MsgData.SendID == pbData.MsgData.RecvID { - pushUserIDList = append(pushUserIDList, pbData.MsgData.RecvID) + pushUserIDList, err = c.onlineCache.GetUsersOnline(ctx, []string{pbData.MsgData.RecvID}) } else { - pushUserIDList = append(pushUserIDList, pbData.MsgData.RecvID, pbData.MsgData.SendID) + pushUserIDList, err = c.onlineCache.GetUsersOnline(ctx, []string{pbData.MsgData.RecvID, pbData.MsgData.SendID}) + } + if err == nil { + err = c.Push2User(ctx, pushUserIDList, pbData.MsgData) } - err = c.Push2User(ctx, pushUserIDList, pbData.MsgData) } if err != nil { log.ZWarn(ctx, "push failed", err, "msg", pbData.String()) @@ -233,7 +237,8 @@ func (c *ConsumerHandler) Push2Group(ctx context.Context, groupID string, msg *s } func (c *ConsumerHandler) groupMessagesHandler(ctx context.Context, groupID string, pushToUserIDs *[]string, msg *sdkws.MsgData) (err error) { if len(*pushToUserIDs) == 0 { - *pushToUserIDs, err = c.groupLocalCache.GetGroupMemberIDs(ctx, groupID) + //*pushToUserIDs, err = c.groupLocalCache.GetGroupMemberIDs(ctx, groupID) + *pushToUserIDs, err = c.onlineCache.GetGroupOnline(ctx, groupID) // if err != nil { return err } diff --git a/pkg/common/storage/cache/redis/seq_user_test.go b/pkg/common/storage/cache/redis/seq_user_test.go index b4c852cd0..cfbea004c 100644 --- a/pkg/common/storage/cache/redis/seq_user_test.go +++ b/pkg/common/storage/cache/redis/seq_user_test.go @@ -3,6 +3,7 @@ package redis import ( "context" "fmt" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/cachekey" "github.com/redis/go-redis/v9" "log" "strconv" @@ -15,7 +16,7 @@ func newTestOnline() *userOnline { opt := &redis.Options{ Addr: "172.16.8.48:16379", Password: "openIM123", - DB: 1, + DB: 0, } rdb := redis.NewClient(opt) if err := rdb.Ping(context.Background()).Err(); err != nil { @@ -63,7 +64,7 @@ func TestGetOnline(t *testing.T) { func TestRecvOnline(t *testing.T) { ts := newTestOnline() ctx := context.Background() - pubsub := ts.rdb.Subscribe(ctx, "user_online") + pubsub := ts.rdb.Subscribe(ctx, cachekey.OnlineChannel) // 等待订阅确认 _, err := pubsub.Receive(ctx) diff --git a/pkg/localcache/cache.go b/pkg/localcache/cache.go index 0e040ad38..ba849f892 100644 --- a/pkg/localcache/cache.go +++ b/pkg/localcache/cache.go @@ -31,6 +31,12 @@ type Cache[V any] interface { Stop() } +func LRUStringHash(key string) uint64 { + h := fnv.New64a() + h.Write(*(*[]byte)(unsafe.Pointer(&key))) + return h.Sum64() +} + func New[V any](opts ...Option) Cache[V] { opt := defaultOption() for _, o := range opts { @@ -49,11 +55,7 @@ func New[V any](opts ...Option) Cache[V] { if opt.localSlotNum == 1 { c.local = createSimpleLRU() } else { - c.local = lru.NewSlotLRU[string, V](opt.localSlotNum, func(key string) uint64 { - h := fnv.New64a() - h.Write(*(*[]byte)(unsafe.Pointer(&key))) - return h.Sum64() - }, createSimpleLRU) + c.local = lru.NewSlotLRU[string, V](opt.localSlotNum, LRUStringHash, createSimpleLRU) } if opt.linkSlotNum > 0 { c.link = link.New(opt.linkSlotNum) diff --git a/pkg/localcache/lru/lru.go b/pkg/localcache/lru/lru.go index 64280f238..2fedffc48 100644 --- a/pkg/localcache/lru/lru.go +++ b/pkg/localcache/lru/lru.go @@ -20,6 +20,7 @@ type EvictCallback[K comparable, V any] simplelru.EvictCallback[K, V] type LRU[K comparable, V any] interface { Get(key K, fetch func() (V, error)) (V, error) + SetHas(key K, value V) bool Del(key K) bool Stop() } diff --git a/pkg/localcache/lru/lru_expiration.go b/pkg/localcache/lru/lru_expiration.go index 970ac083e..d27e67057 100644 --- a/pkg/localcache/lru/lru_expiration.go +++ b/pkg/localcache/lru/lru_expiration.go @@ -89,5 +89,15 @@ func (x *ExpirationLRU[K, V]) Del(key K) bool { return ok } +func (x *ExpirationLRU[K, V]) SetHas(key K, value V) bool { + x.lock.Lock() + defer x.lock.Unlock() + if x.core.Contains(key) { + x.core.Add(key, &expirationLruItem[V]{value: value}) + return true + } + return false +} + func (x *ExpirationLRU[K, V]) Stop() { } diff --git a/pkg/localcache/lru/lru_lazy.go b/pkg/localcache/lru/lru_lazy.go index d6e64aae4..e935c687c 100644 --- a/pkg/localcache/lru/lru_lazy.go +++ b/pkg/localcache/lru/lru_lazy.go @@ -88,6 +88,28 @@ func (x *LayLRU[K, V]) Get(key K, fetch func() (V, error)) (V, error) { return v.value, v.err } +//func (x *LayLRU[K, V]) Set(key K, value V) { +// x.lock.Lock() +// x.core.Add(key, &layLruItem[V]{value: value, expires: time.Now().Add(x.successTTL).UnixMilli()}) +// x.lock.Unlock() +//} +// +//func (x *LayLRU[K, V]) Has(key K) bool { +// x.lock.Lock() +// defer x.lock.Unlock() +// return x.core.Contains(key) +//} + +func (x *LayLRU[K, V]) SetHas(key K, value V) bool { + x.lock.Lock() + defer x.lock.Unlock() + if x.core.Contains(key) { + x.core.Add(key, &layLruItem[V]{value: value, expires: time.Now().Add(x.successTTL).UnixMilli()}) + return true + } + return false +} + func (x *LayLRU[K, V]) Del(key K) bool { x.lock.Lock() ok := x.core.Remove(key) diff --git a/pkg/localcache/lru/lru_slot.go b/pkg/localcache/lru/lru_slot.go index d034e94d3..4538ca20e 100644 --- a/pkg/localcache/lru/lru_slot.go +++ b/pkg/localcache/lru/lru_slot.go @@ -40,6 +40,10 @@ func (x *slotLRU[K, V]) Get(key K, fetch func() (V, error)) (V, error) { return x.slots[x.getIndex(key)].Get(key, fetch) } +func (x *slotLRU[K, V]) SetHas(key K, value V) bool { + return x.slots[x.getIndex(key)].SetHas(key, value) +} + func (x *slotLRU[K, V]) Del(key K) bool { return x.slots[x.getIndex(key)].Del(key) } diff --git a/pkg/localcache/option.go b/pkg/localcache/option.go index 00bb9d044..7d91aba6c 100644 --- a/pkg/localcache/option.go +++ b/pkg/localcache/option.go @@ -30,7 +30,7 @@ func defaultOption() *option { localSuccessTTL: time.Minute, localFailedTTL: time.Second * 5, delFn: make([]func(ctx context.Context, key ...string), 0, 2), - target: emptyTarget{}, + target: EmptyTarget{}, } } @@ -123,14 +123,14 @@ func WithDeleteKeyBefore(fn func(ctx context.Context, key ...string)) Option { } } -type emptyTarget struct{} +type EmptyTarget struct{} -func (e emptyTarget) IncrGetHit() {} +func (e EmptyTarget) IncrGetHit() {} -func (e emptyTarget) IncrGetSuccess() {} +func (e EmptyTarget) IncrGetSuccess() {} -func (e emptyTarget) IncrGetFailed() {} +func (e EmptyTarget) IncrGetFailed() {} -func (e emptyTarget) IncrDelHit() {} +func (e EmptyTarget) IncrDelHit() {} -func (e emptyTarget) IncrDelNotFound() {} +func (e EmptyTarget) IncrDelNotFound() {} diff --git a/pkg/rpccache/online.go b/pkg/rpccache/online.go new file mode 100644 index 000000000..79d799bbe --- /dev/null +++ b/pkg/rpccache/online.go @@ -0,0 +1,121 @@ +package rpccache + +import ( + "context" + "errors" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/cachekey" + "github.com/openimsdk/open-im-server/v3/pkg/localcache" + "github.com/openimsdk/open-im-server/v3/pkg/localcache/lru" + "github.com/openimsdk/open-im-server/v3/pkg/rpcclient" + "github.com/openimsdk/tools/log" + "github.com/openimsdk/tools/mcontext" + "github.com/redis/go-redis/v9" + "math/rand" + "strconv" + "strings" + "time" +) + +func NewOnlineCache(user rpcclient.UserRpcClient, group *GroupLocalCache, rdb redis.UniversalClient) *OnlineCache { + x := &OnlineCache{ + user: user, + group: group, + local: lru.NewSlotLRU(1024, localcache.LRUStringHash, func() lru.LRU[string, []int32] { + return lru.NewLayLRU[string, []int32](2048, cachekey.OnlineExpire, time.Second*3, localcache.EmptyTarget{}, func(key string, value []int32) {}) + }), + } + go func() { + parseUserOnlineStatus := func(payload string) (string, []int32, error) { + arr := strings.Split(payload, ":") + if len(arr) == 0 { + return "", nil, errors.New("invalid data") + } + userID := arr[len(arr)-1] + if userID == "" { + return "", nil, errors.New("userID is empty") + } + platformIDs := make([]int32, len(arr)-1) + for i := range platformIDs { + platformID, err := strconv.Atoi(arr[i]) + if err != nil { + return "", nil, err + } + platformIDs[i] = int32(platformID) + } + return userID, platformIDs, nil + } + ctx := mcontext.SetOperationID(context.Background(), cachekey.OnlineChannel+strconv.FormatUint(rand.Uint64(), 10)) + for message := range rdb.Subscribe(ctx, cachekey.OnlineChannel).Channel() { + userID, platformIDs, err := parseUserOnlineStatus(message.Payload) + if err != nil { + log.ZError(ctx, "redis subscribe parseUserOnlineStatus", err, "payload", message.Payload, "channel", message.Channel) + continue + } + x.setUserOnline(userID, platformIDs) + //if err := x.setUserOnline(ctx, userID, platformIDs); err != nil { + // log.ZError(ctx, "redis subscribe setUserOnline", err, "payload", message.Payload, "channel", message.Channel) + //} + } + }() + return x +} + +type OnlineCache struct { + user rpcclient.UserRpcClient + group *GroupLocalCache + local lru.LRU[string, []int32] +} + +func (o *OnlineCache) getUserOnlineKey(userID string) string { + return "" + userID +} + +func (o *OnlineCache) GetUserOnlinePlatform(ctx context.Context, userID string) ([]int32, error) { + return o.local.Get(userID, func() ([]int32, error) { + return o.user.GetUserOnlinePlatform(ctx, userID) + }) +} + +func (o *OnlineCache) GetUserOnline(ctx context.Context, userID string) (bool, error) { + platformIDs, err := o.GetUserOnlinePlatform(ctx, userID) + if err != nil { + return false, err + } + return len(platformIDs) > 0, nil +} + +func (o *OnlineCache) GetUsersOnline(ctx context.Context, userIDs []string) ([]string, error) { + onlineUserIDs := make([]string, 0, len(userIDs)) + for _, userID := range userIDs { + online, err := o.GetUserOnline(ctx, userID) + if err != nil { + return nil, err + } + if online { + onlineUserIDs = append(onlineUserIDs, userID) + } + } + return onlineUserIDs, nil +} + +func (o *OnlineCache) GetGroupOnline(ctx context.Context, groupID string) ([]string, error) { + userIDs, err := o.group.GetGroupMemberIDs(ctx, groupID) + if err != nil { + return nil, err + } + var onlineUserIDs []string + for _, userID := range userIDs { + online, err := o.GetUserOnline(ctx, userID) + if err != nil { + return nil, err + } + if online { + onlineUserIDs = append(onlineUserIDs, userID) + } + } + return onlineUserIDs, nil +} + +func (o *OnlineCache) setUserOnline(userID string, platformIDs []int32) { + o.local.SetHas(o.getUserOnlineKey(userID), platformIDs) +} diff --git a/pkg/rpccache/user.go b/pkg/rpccache/user.go index 25a8eb20d..6126f5891 100644 --- a/pkg/rpccache/user.go +++ b/pkg/rpccache/user.go @@ -110,3 +110,18 @@ func (u *UserLocalCache) GetUsersInfoMap(ctx context.Context, userIDs []string) } return users, nil } + +//func (u *UserLocalCache) GetUserOnlinePlatform(ctx context.Context, userID string) (val []int32, err error) { +// log.ZDebug(ctx, "UserLocalCache GetUserOnlinePlatform req", "userID", userID) +// defer func() { +// if err == nil { +// log.ZDebug(ctx, "UserLocalCache GetUserOnlinePlatform return", "value", val) +// } else { +// log.ZError(ctx, "UserLocalCache GetUserOnlinePlatform return", err) +// } +// }() +// return localcache.AnyValue[[]int32](u.local.Get(ctx, cachekey.GetOnlineKey(userID), func(ctx context.Context) (any, error) { +// log.ZDebug(ctx, "UserLocalCache GetUserGlobalMsgRecvOpt rpc", "userID", userID) +// return u.client.GetUserGlobalMsgRecvOpt(ctx, userID) +// })) +//} diff --git a/pkg/rpcclient/user.go b/pkg/rpcclient/user.go index aab96603e..cac44fce0 100644 --- a/pkg/rpcclient/user.go +++ b/pkg/rpcclient/user.go @@ -193,3 +193,25 @@ func (u *UserRpcClient) GetNotificationByID(ctx context.Context, userID string) }) return err } + +func (u *UserRpcClient) GetUsersOnlinePlatform(ctx context.Context, userIDs []string) ([]*user.OnlineStatus, error) { + if len(userIDs) == 0 { + return nil, nil + } + resp, err := u.Client.GetUserStatus(ctx, &user.GetUserStatusReq{UserIDs: userIDs}) + if err != nil { + return nil, err + } + return resp.StatusList, nil +} + +func (u *UserRpcClient) GetUserOnlinePlatform(ctx context.Context, userID string) ([]int32, error) { + resp, err := u.GetUsersOnlinePlatform(ctx, []string{userID}) + if err != nil { + return nil, err + } + if len(resp) == 0 { + return nil, nil + } + return resp[0].PlatformIDs, nil +} From d06c323f115fecc9b67fa66a226465251dc86202 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Mon, 1 Jul 2024 16:43:19 +0800 Subject: [PATCH 07/27] online cache --- pkg/rpcclient/user.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/rpcclient/user.go b/pkg/rpcclient/user.go index cac44fce0..eabe77b94 100644 --- a/pkg/rpcclient/user.go +++ b/pkg/rpcclient/user.go @@ -198,7 +198,7 @@ func (u *UserRpcClient) GetUsersOnlinePlatform(ctx context.Context, userIDs []st if len(userIDs) == 0 { return nil, nil } - resp, err := u.Client.GetUserStatus(ctx, &user.GetUserStatusReq{UserIDs: userIDs}) + resp, err := u.Client.GetUserStatus(ctx, &user.GetUserStatusReq{UserIDs: userIDs, UserID: u.imAdminUserID[0]}) if err != nil { return nil, err } From 45b07bc3cc0076d117bb12861dc76881f6b99969 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Mon, 1 Jul 2024 17:23:35 +0800 Subject: [PATCH 08/27] online cache --- internal/push/push_handler.go | 15 +++++++++------ pkg/rpccache/online.go | 5 ++++- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/internal/push/push_handler.go b/internal/push/push_handler.go index 029330f1a..33f987528 100644 --- a/internal/push/push_handler.go +++ b/internal/push/push_handler.go @@ -102,13 +102,11 @@ func (c *ConsumerHandler) handleMs2PsChat(ctx context.Context, msg []byte) { var pushUserIDList []string isSenderSync := datautil.GetSwitchFromOptions(pbData.MsgData.Options, constant.IsSenderSync) if !isSenderSync || pbData.MsgData.SendID == pbData.MsgData.RecvID { - pushUserIDList, err = c.onlineCache.GetUsersOnline(ctx, []string{pbData.MsgData.RecvID}) + pushUserIDList = append(pushUserIDList, pbData.MsgData.RecvID) } else { - pushUserIDList, err = c.onlineCache.GetUsersOnline(ctx, []string{pbData.MsgData.RecvID, pbData.MsgData.SendID}) - } - if err == nil { - err = c.Push2User(ctx, pushUserIDList, pbData.MsgData) + pushUserIDList = append(pushUserIDList, pbData.MsgData.RecvID, pbData.MsgData.SendID) } + err = c.Push2User(ctx, pushUserIDList, pbData.MsgData) } if err != nil { log.ZWarn(ctx, "push failed", err, "msg", pbData.String()) @@ -129,7 +127,12 @@ func (c *ConsumerHandler) ConsumeClaim(sess sarama.ConsumerGroupSession, claim s } // Push2User Suitable for two types of conversations, one is SingleChatType and the other is NotificationChatType. -func (c *ConsumerHandler) Push2User(ctx context.Context, userIDs []string, msg *sdkws.MsgData) error { +func (c *ConsumerHandler) Push2User(ctx context.Context, userIDs []string, msg *sdkws.MsgData) (err error) { + userIDs, err = c.onlineCache.GetUsersOnline(ctx, userIDs) + if err != nil { + return err + } + log.ZDebug(ctx, "Get msg from msg_transfer And push msg", "userIDs", userIDs, "msg", msg.String()) if err := c.webhookBeforeOnlinePush(ctx, &c.config.WebhooksConfig.BeforeOnlinePush, userIDs, msg); err != nil { return err diff --git a/pkg/rpccache/online.go b/pkg/rpccache/online.go index 79d799bbe..aeeafcf2b 100644 --- a/pkg/rpccache/online.go +++ b/pkg/rpccache/online.go @@ -48,9 +48,10 @@ func NewOnlineCache(user rpcclient.UserRpcClient, group *GroupLocalCache, rdb re for message := range rdb.Subscribe(ctx, cachekey.OnlineChannel).Channel() { userID, platformIDs, err := parseUserOnlineStatus(message.Payload) if err != nil { - log.ZError(ctx, "redis subscribe parseUserOnlineStatus", err, "payload", message.Payload, "channel", message.Channel) + log.ZError(ctx, "OnlineCache redis subscribe parseUserOnlineStatus", err, "payload", message.Payload, "channel", message.Channel) continue } + log.ZDebug(ctx, "OnlineCache setUserOnline", "userID", userID, "platformIDs", platformIDs, "payload", message.Payload) x.setUserOnline(userID, platformIDs) //if err := x.setUserOnline(ctx, userID, platformIDs); err != nil { // log.ZError(ctx, "redis subscribe setUserOnline", err, "payload", message.Payload, "channel", message.Channel) @@ -95,6 +96,7 @@ func (o *OnlineCache) GetUsersOnline(ctx context.Context, userIDs []string) ([]s onlineUserIDs = append(onlineUserIDs, userID) } } + log.ZDebug(ctx, "OnlineCache GetUsersOnline", "userIDs", userIDs, "onlineUserIDs", onlineUserIDs) return onlineUserIDs, nil } @@ -113,6 +115,7 @@ func (o *OnlineCache) GetGroupOnline(ctx context.Context, groupID string) ([]str onlineUserIDs = append(onlineUserIDs, userID) } } + log.ZDebug(ctx, "OnlineCache GetGroupOnline", "groupID", groupID, "onlineUserIDs", onlineUserIDs) return onlineUserIDs, nil } From 32c5f65d2fd35f501a0db7a9a322e61fa7ad2800 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Tue, 2 Jul 2024 14:32:58 +0800 Subject: [PATCH 09/27] online cache --- .../openim-rpc-conversation/main.go | 4 - cmd/openim-rpc/openim-rpc-group/main.go | 4 - internal/msggateway/n_ws_server.go | 4 +- internal/msggateway/online.go | 2 +- internal/msggateway/user_map.go | 270 ++++++++++-------- internal/msggateway/user_map2.go | 27 +- 6 files changed, 156 insertions(+), 155 deletions(-) diff --git a/cmd/openim-rpc/openim-rpc-conversation/main.go b/cmd/openim-rpc/openim-rpc-conversation/main.go index 5b3077ccb..5b2e66c95 100644 --- a/cmd/openim-rpc/openim-rpc-conversation/main.go +++ b/cmd/openim-rpc/openim-rpc-conversation/main.go @@ -17,13 +17,9 @@ package main import ( "github.com/openimsdk/open-im-server/v3/pkg/common/cmd" "github.com/openimsdk/tools/system/program" - "os" ) func main() { - if len(os.Args) == 1 { - os.Args = []string{os.Args[0], "-i", "0", "-c", "/Users/chao/Desktop/project/open-im-server/config"} - } if err := cmd.NewConversationRpcCmd().Exec(); err != nil { program.ExitWithError(err) } diff --git a/cmd/openim-rpc/openim-rpc-group/main.go b/cmd/openim-rpc/openim-rpc-group/main.go index 44e5509df..5badf934e 100644 --- a/cmd/openim-rpc/openim-rpc-group/main.go +++ b/cmd/openim-rpc/openim-rpc-group/main.go @@ -17,13 +17,9 @@ package main import ( "github.com/openimsdk/open-im-server/v3/pkg/common/cmd" "github.com/openimsdk/tools/system/program" - "os" ) func main() { - if len(os.Args) == 1 { - os.Args = []string{os.Args[0], "-i", "0", "-c", "/Users/chao/Desktop/project/open-im-server/config"} - } if err := cmd.NewGroupRpcCmd().Exec(); err != nil { program.ExitWithError(err) } diff --git a/internal/msggateway/n_ws_server.go b/internal/msggateway/n_ws_server.go index ff93ce6bc..b87f93ab6 100644 --- a/internal/msggateway/n_ws_server.go +++ b/internal/msggateway/n_ws_server.go @@ -140,7 +140,7 @@ func NewWsServer(msgGatewayConfig *Config, opts ...Option) *WsServer { unregisterChan: make(chan *Client, 1000), kickHandlerChan: make(chan *kickHandler, 1000), validate: v, - clients: newUserMap(), + clients: newUserMap1(), Compressor: NewGzipCompressor(), Encoder: NewGobEncoder(), webhookClient: webhook.NewWebhookClient(msgGatewayConfig.WebhooksConfig.URL), @@ -345,7 +345,7 @@ func (ws *WsServer) multiTerminalLoginChecker(clientOK bool, oldClients []*Clien func (ws *WsServer) unregisterClient(client *Client) { defer ws.clientPool.Put(client) - isDeleteUser := ws.clients.Delete(client.UserID, client.ctx.GetRemoteAddr()) + isDeleteUser := ws.clients.DeleteClients(client.UserID, []*Client{client}) if isDeleteUser { ws.onlineUserNum.Add(-1) prommetrics.OnlineUserGauge.Dec() diff --git a/internal/msggateway/online.go b/internal/msggateway/online.go index 61549083a..b640cdd80 100644 --- a/internal/msggateway/online.go +++ b/internal/msggateway/online.go @@ -98,7 +98,7 @@ func (ws *WsServer) ChangeOnlineStatus(concurrent int) { case <-mergeTicker.C: pushAllUserState() case now := <-scanTicker.C: - pushUserState(ws.clients.GetAllUserStatus(now.Add(-cachekey.OnlineExpire/3), now)...) + pushUserState(ws.clients.GetAllUserStatus(now.Add(-cachekey.OnlineExpire / 3))...) case state := <-ws.clients.UserState(): log.ZDebug(context.Background(), "user online change", "userID", state.UserID, "online", state.Online, "offline", state.Offline) pushUserState(state) diff --git a/internal/msggateway/user_map.go b/internal/msggateway/user_map.go index dcab066f8..119b07ae5 100644 --- a/internal/msggateway/user_map.go +++ b/internal/msggateway/user_map.go @@ -14,123 +14,153 @@ package msggateway -// -//import ( -// "context" -// "sync" -// -// "github.com/openimsdk/tools/log" -// "github.com/openimsdk/tools/utils/datautil" -//) -// -//type UserMap struct { -// m sync.Map -//} -// -//func newUserMap() UMap { -// return &UserMap{} -//} -// -//func (u *UserMap) GetAll(key string) ([]*Client, bool) { -// allClients, ok := u.m.Load(key) -// if ok { -// return allClients.([]*Client), ok -// } -// return nil, ok -//} -// -//func (u *UserMap) Get(key string, platformID int) ([]*Client, bool, bool) { -// allClients, userExisted := u.m.Load(key) -// if userExisted { -// var clients []*Client -// for _, client := range allClients.([]*Client) { -// if client.PlatformID == platformID { -// clients = append(clients, client) -// } -// } -// if len(clients) > 0 { -// return clients, userExisted, true -// } -// return clients, userExisted, false -// } -// return nil, userExisted, false -//} -// -//// Set adds a client to the map. -//func (u *UserMap) Set(key string, v *Client) { -// allClients, existed := u.m.Load(key) -// if existed { -// log.ZDebug(context.Background(), "Set existed", "user_id", key, "client_user_id", v.UserID) -// oldClients := allClients.([]*Client) -// oldClients = append(oldClients, v) -// u.m.Store(key, oldClients) -// } else { -// log.ZDebug(context.Background(), "Set not existed", "user_id", key, "client_user_id", v.UserID) -// -// var clients []*Client -// clients = append(clients, v) -// u.m.Store(key, clients) -// } -//} -// -//func (u *UserMap) Delete(key string, connRemoteAddr string) (isDeleteUser bool) { -// // Attempt to load the clients associated with the key. -// allClients, existed := u.m.Load(key) -// if !existed { -// // Return false immediately if the key does not exist. -// return false -// } -// -// // Convert allClients to a slice of *Client. -// oldClients := allClients.([]*Client) -// var remainingClients []*Client -// for _, client := range oldClients { -// // Keep clients that do not match the connRemoteAddr. -// if client.ctx.GetRemoteAddr() != connRemoteAddr { -// remainingClients = append(remainingClients, client) -// } -// } -// -// // If no clients remain after filtering, delete the key from the map. -// if len(remainingClients) == 0 { -// u.m.Delete(key) -// return true -// } -// -// // Otherwise, update the key with the remaining clients. -// u.m.Store(key, remainingClients) -// return false -//} -// -//func (u *UserMap) DeleteClients(key string, clients []*Client) (isDeleteUser bool) { -// m := datautil.SliceToMapAny(clients, func(c *Client) (string, struct{}) { -// return c.ctx.GetRemoteAddr(), struct{}{} -// }) -// allClients, existed := u.m.Load(key) -// if !existed { -// // If the key doesn't exist, return false. -// return false -// } -// -// // Filter out clients that are in the deleteMap. -// oldClients := allClients.([]*Client) -// var remainingClients []*Client -// for _, client := range oldClients { -// if _, shouldBeDeleted := m[client.ctx.GetRemoteAddr()]; !shouldBeDeleted { -// remainingClients = append(remainingClients, client) -// } -// } -// -// // Update or delete the key based on the remaining clients. -// if len(remainingClients) == 0 { -// u.m.Delete(key) -// return true -// } -// -// u.m.Store(key, remainingClients) -// return false -//} -// -//func (u *UserMap) DeleteAll(key string) { -// u.m.Delete(key) -//} +import ( + "context" + "sync" + "time" + + "github.com/openimsdk/tools/log" + "github.com/openimsdk/tools/utils/datautil" +) + +func newUserMap1() UMap { + return &UserMap{ + ch: make(chan UserState, 1024), + } +} + +type UserPlatform1 struct { + Time time.Time + Clients []*Client +} + +func (u *UserPlatform1) PlatformIDs() []int32 { + if len(u.Clients) == 0 { + return nil + } + platformIDs := make([]int32, 0, len(u.Clients)) + for _, client := range u.Clients { + platformIDs = append(platformIDs, int32(client.PlatformID)) + } + return platformIDs +} + +type UserMap struct { + m sync.Map + ch chan UserState +} + +func (u *UserMap) UserState() <-chan UserState { + return u.ch +} + +func (u *UserMap) GetAllUserStatus(deadline time.Time) []UserState { + var result []UserState + u.m.Range(func(key, value any) bool { + client := value.(*UserPlatform1) + if client.Time.Before(deadline) { + return true + } + client.Time = time.Now() + us := UserState{ + UserID: key.(string), + Online: make([]int32, 0, len(client.Clients)), + } + for _, c := range client.Clients { + us.Online = append(us.Online, int32(c.PlatformID)) + } + return true + }) + return result +} + +func (u *UserMap) push(userID string, userPlatform *UserPlatform1, offline []int32) bool { + select { + case u.ch <- UserState{UserID: userID, Online: userPlatform.PlatformIDs(), Offline: offline}: + userPlatform.Time = time.Now() + return true + default: + return false + } +} + +func (u *UserMap) GetAll(key string) ([]*Client, bool) { + allClients, ok := u.m.Load(key) + if ok { + return allClients.(*UserPlatform1).Clients, ok + } + return nil, ok +} + +func (u *UserMap) Get(key string, platformID int) ([]*Client, bool, bool) { + allClients, userExisted := u.m.Load(key) + if userExisted { + var clients []*Client + for _, client := range allClients.(*UserPlatform1).Clients { + if client.PlatformID == platformID { + clients = append(clients, client) + } + } + if len(clients) > 0 { + return clients, true, true + } + return clients, true, false + } + return nil, false, false +} + +// Set adds a client to the map. +func (u *UserMap) Set(key string, v *Client) { + allClients, existed := u.m.Load(key) + if existed { + log.ZDebug(context.Background(), "Set existed", "user_id", key, "client_user_id", v.UserID) + oldClients := allClients.(*UserPlatform1) + oldClients.Time = time.Now() + oldClients.Clients = append(oldClients.Clients, v) + u.push(key, oldClients, nil) + } else { + log.ZDebug(context.Background(), "Set not existed", "user_id", key, "client_user_id", v.UserID) + cli := &UserPlatform1{ + Time: time.Now(), + Clients: []*Client{v}, + } + u.m.Store(key, cli) + u.push(key, cli, nil) + } + +} + +func (u *UserMap) DeleteClients(key string, clients []*Client) (isDeleteUser bool) { + m := datautil.SliceToMapAny(clients, func(c *Client) (string, struct{}) { + return c.ctx.GetRemoteAddr(), struct{}{} + }) + allClients, existed := u.m.Load(key) + if !existed { + // If the key doesn't exist, return false. + return false + } + + // Filter out clients that are in the deleteMap. + oldClients := allClients.(*UserPlatform1) + var ( + remainingClients []*Client + offline []int32 + ) + for _, client := range oldClients.Clients { + if _, shouldBeDeleted := m[client.ctx.GetRemoteAddr()]; !shouldBeDeleted { + remainingClients = append(remainingClients, client) + } else { + offline = append(offline, int32(client.PlatformID)) + } + } + + oldClients.Clients = remainingClients + defer u.push(key, oldClients, offline) + // Update or delete the key based on the remaining clients. + if len(remainingClients) == 0 { + u.m.Delete(key) + return true + } + + return false +} diff --git a/internal/msggateway/user_map2.go b/internal/msggateway/user_map2.go index b6ed40373..8a064ab83 100644 --- a/internal/msggateway/user_map2.go +++ b/internal/msggateway/user_map2.go @@ -9,10 +9,9 @@ type UMap interface { GetAll(userID string) ([]*Client, bool) Get(userID string, platformID int) ([]*Client, bool, bool) Set(userID string, v *Client) - Delete(userID string, connRemoteAddr string) (isDeleteUser bool) DeleteClients(userID string, clients []*Client) (isDeleteUser bool) UserState() <-chan UserState - GetAllUserStatus(deadline, nowtime time.Time) []UserState + GetAllUserStatus(deadline time.Time) []UserState } var _ UMap = (*UserMap2)(nil) @@ -102,26 +101,6 @@ func (u *UserMap2) Set(userID string, client *Client) { u.push(client.UserID, result, nil) } -func (u *UserMap2) Delete(userID string, connRemoteAddr string) (isDeleteUser bool) { - u.lock.Lock() - defer u.lock.Unlock() - result, ok := u.data[userID] - if !ok { - return false - } - client, ok := result.Clients[connRemoteAddr] - if !ok { - return false - } - delete(result.Clients, connRemoteAddr) - defer u.push(userID, result, []int32{int32(client.PlatformID)}) - if len(result.Clients) > 0 { - return false - } - delete(u.data, userID) - return true -} - func (u *UserMap2) DeleteClients(userID string, clients []*Client) (isDeleteUser bool) { if len(clients) == 0 { return false @@ -145,7 +124,7 @@ func (u *UserMap2) DeleteClients(userID string, clients []*Client) (isDeleteUser return true } -func (u *UserMap2) GetAllUserStatus(deadline, nowtime time.Time) []UserState { +func (u *UserMap2) GetAllUserStatus(deadline time.Time) []UserState { u.lock.RLock() defer u.lock.RUnlock() if len(u.data) == 0 { @@ -159,7 +138,7 @@ func (u *UserMap2) GetAllUserStatus(deadline, nowtime time.Time) []UserState { if p.Time.Before(deadline) { continue } - p.Time = nowtime + p.Time = time.Now() online := make([]int32, 0, len(p.Clients)) for _, client := range p.Clients { online = append(online, int32(client.PlatformID)) From 9388cb61e8661b48726a6deb2e68de248438e6f8 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Tue, 2 Jul 2024 15:25:43 +0800 Subject: [PATCH 10/27] online cache --- internal/msggateway/n_ws_server.go | 2 +- internal/msggateway/online.go | 2 +- internal/msggateway/user_map.go | 4 +- internal/msggateway/user_map2.go | 66 ++++++++++++++---------------- 4 files changed, 35 insertions(+), 39 deletions(-) diff --git a/internal/msggateway/n_ws_server.go b/internal/msggateway/n_ws_server.go index b87f93ab6..58408e1ef 100644 --- a/internal/msggateway/n_ws_server.go +++ b/internal/msggateway/n_ws_server.go @@ -140,7 +140,7 @@ func NewWsServer(msgGatewayConfig *Config, opts ...Option) *WsServer { unregisterChan: make(chan *Client, 1000), kickHandlerChan: make(chan *kickHandler, 1000), validate: v, - clients: newUserMap1(), + clients: newUserMap(), Compressor: NewGzipCompressor(), Encoder: NewGobEncoder(), webhookClient: webhook.NewWebhookClient(msgGatewayConfig.WebhooksConfig.URL), diff --git a/internal/msggateway/online.go b/internal/msggateway/online.go index b640cdd80..61549083a 100644 --- a/internal/msggateway/online.go +++ b/internal/msggateway/online.go @@ -98,7 +98,7 @@ func (ws *WsServer) ChangeOnlineStatus(concurrent int) { case <-mergeTicker.C: pushAllUserState() case now := <-scanTicker.C: - pushUserState(ws.clients.GetAllUserStatus(now.Add(-cachekey.OnlineExpire / 3))...) + pushUserState(ws.clients.GetAllUserStatus(now.Add(-cachekey.OnlineExpire/3), now)...) case state := <-ws.clients.UserState(): log.ZDebug(context.Background(), "user online change", "userID", state.UserID, "online", state.Online, "offline", state.Offline) pushUserState(state) diff --git a/internal/msggateway/user_map.go b/internal/msggateway/user_map.go index 119b07ae5..e1388d588 100644 --- a/internal/msggateway/user_map.go +++ b/internal/msggateway/user_map.go @@ -54,14 +54,14 @@ func (u *UserMap) UserState() <-chan UserState { return u.ch } -func (u *UserMap) GetAllUserStatus(deadline time.Time) []UserState { +func (u *UserMap) GetAllUserStatus(deadline time.Time, nowtime time.Time) []UserState { var result []UserState u.m.Range(func(key, value any) bool { client := value.(*UserPlatform1) if client.Time.Before(deadline) { return true } - client.Time = time.Now() + client.Time = nowtime us := UserState{ UserID: key.(string), Online: make([]int32, 0, len(client.Clients)), diff --git a/internal/msggateway/user_map2.go b/internal/msggateway/user_map2.go index 8a064ab83..74b49d9d4 100644 --- a/internal/msggateway/user_map2.go +++ b/internal/msggateway/user_map2.go @@ -1,6 +1,7 @@ package msggateway import ( + "github.com/openimsdk/tools/utils/datautil" "sync" "time" ) @@ -11,14 +12,14 @@ type UMap interface { Set(userID string, v *Client) DeleteClients(userID string, clients []*Client) (isDeleteUser bool) UserState() <-chan UserState - GetAllUserStatus(deadline time.Time) []UserState + GetAllUserStatus(deadline time.Time, nowtime time.Time) []UserState } -var _ UMap = (*UserMap2)(nil) +var _ UMap = (*userMap)(nil) type UserPlatform struct { Time time.Time - Clients map[string]*Client + Clients []*Client } func (u *UserPlatform) PlatformIDs() []int32 { @@ -33,19 +34,19 @@ func (u *UserPlatform) PlatformIDs() []int32 { } func newUserMap() UMap { - return &UserMap2{ + return &userMap{ data: make(map[string]*UserPlatform), ch: make(chan UserState, 10000), } } -type UserMap2 struct { +type userMap struct { lock sync.RWMutex data map[string]*UserPlatform ch chan UserState } -func (u *UserMap2) push(userID string, userPlatform *UserPlatform, offline []int32) bool { +func (u *userMap) push(userID string, userPlatform *UserPlatform, offline []int32) bool { select { case u.ch <- UserState{UserID: userID, Online: userPlatform.PlatformIDs(), Offline: offline}: userPlatform.Time = time.Now() @@ -55,21 +56,17 @@ func (u *UserMap2) push(userID string, userPlatform *UserPlatform, offline []int } } -func (u *UserMap2) GetAll(userID string) ([]*Client, bool) { +func (u *userMap) GetAll(userID string) ([]*Client, bool) { u.lock.RLock() defer u.lock.RUnlock() result, ok := u.data[userID] if !ok { return nil, false } - clients := make([]*Client, 0, len(result.Clients)) - for _, client := range result.Clients { - clients = append(clients, client) - } - return clients, true + return result.Clients, true } -func (u *UserMap2) Get(userID string, platformID int) ([]*Client, bool, bool) { +func (u *userMap) Get(userID string, platformID int) ([]*Client, bool, bool) { u.lock.RLock() defer u.lock.RUnlock() result, ok := u.data[userID] @@ -85,23 +82,21 @@ func (u *UserMap2) Get(userID string, platformID int) ([]*Client, bool, bool) { return clients, true, len(clients) > 0 } -func (u *UserMap2) Set(userID string, client *Client) { +func (u *userMap) Set(userID string, client *Client) { u.lock.Lock() defer u.lock.Unlock() result, ok := u.data[userID] if ok { - result.Clients[client.ctx.GetRemoteAddr()] = client + result.Clients = append(result.Clients, client) } else { result = &UserPlatform{ - Clients: map[string]*Client{ - client.ctx.GetRemoteAddr(): client, - }, + Clients: []*Client{client}, } } u.push(client.UserID, result, nil) } -func (u *UserMap2) DeleteClients(userID string, clients []*Client) (isDeleteUser bool) { +func (u *userMap) DeleteClients(userID string, clients []*Client) (isDeleteUser bool) { if len(clients) == 0 { return false } @@ -112,9 +107,16 @@ func (u *UserMap2) DeleteClients(userID string, clients []*Client) (isDeleteUser return false } offline := make([]int32, 0, len(clients)) - for _, client := range clients { - offline = append(offline, int32(client.PlatformID)) - delete(result.Clients, client.ctx.GetRemoteAddr()) + deleteAddr := datautil.SliceSetAny(clients, func(client *Client) string { + return client.ctx.GetRemoteAddr() + }) + tmp := result.Clients + result.Clients = result.Clients[:0] + for _, client := range tmp { + if _, ok := deleteAddr[client.ctx.GetRemoteAddr()]; ok { + continue + } + result.Clients = append(result.Clients, client) } defer u.push(userID, result, offline) if len(result.Clients) > 0 { @@ -124,23 +126,17 @@ func (u *UserMap2) DeleteClients(userID string, clients []*Client) (isDeleteUser return true } -func (u *UserMap2) GetAllUserStatus(deadline time.Time) []UserState { +func (u *userMap) GetAllUserStatus(deadline time.Time, nowtime time.Time) []UserState { u.lock.RLock() defer u.lock.RUnlock() - if len(u.data) == 0 { - return nil - } result := make([]UserState, 0, len(u.data)) - for userID, p := range u.data { - if len(result) == cap(result) { - break - } - if p.Time.Before(deadline) { + for userID, userPlatform := range u.data { + if userPlatform.Time.Before(deadline) { continue } - p.Time = time.Now() - online := make([]int32, 0, len(p.Clients)) - for _, client := range p.Clients { + userPlatform.Time = time.Now() + online := make([]int32, 0, len(userPlatform.Clients)) + for _, client := range userPlatform.Clients { online = append(online, int32(client.PlatformID)) } result = append(result, UserState{UserID: userID, Online: online}) @@ -148,7 +144,7 @@ func (u *UserMap2) GetAllUserStatus(deadline time.Time) []UserState { return result } -func (u *UserMap2) UserState() <-chan UserState { +func (u *userMap) UserState() <-chan UserState { return u.ch } From 28c8b78b311a52a226d7819c4e826c7e8693bece Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Tue, 2 Jul 2024 15:33:31 +0800 Subject: [PATCH 11/27] online cache --- internal/msggateway/user_map2.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/internal/msggateway/user_map2.go b/internal/msggateway/user_map2.go index 74b49d9d4..c913f49cc 100644 --- a/internal/msggateway/user_map2.go +++ b/internal/msggateway/user_map2.go @@ -1,6 +1,8 @@ package msggateway import ( + "context" + "github.com/openimsdk/tools/log" "github.com/openimsdk/tools/utils/datautil" "sync" "time" @@ -57,6 +59,7 @@ func (u *userMap) push(userID string, userPlatform *UserPlatform, offline []int3 } func (u *userMap) GetAll(userID string) ([]*Client, bool) { + log.ZInfo(context.Background(), "UserMap GetAll", "userID", userID) u.lock.RLock() defer u.lock.RUnlock() result, ok := u.data[userID] @@ -67,6 +70,7 @@ func (u *userMap) GetAll(userID string) ([]*Client, bool) { } func (u *userMap) Get(userID string, platformID int) ([]*Client, bool, bool) { + log.ZInfo(context.Background(), "UserMap Get", "userID", userID, "platformID", platformID) u.lock.RLock() defer u.lock.RUnlock() result, ok := u.data[userID] @@ -83,6 +87,7 @@ func (u *userMap) Get(userID string, platformID int) ([]*Client, bool, bool) { } func (u *userMap) Set(userID string, client *Client) { + log.ZInfo(context.Background(), "UserMap Set", "userID", userID, "client", client.ctx.GetRemoteAddr()) u.lock.Lock() defer u.lock.Unlock() result, ok := u.data[userID] @@ -97,6 +102,7 @@ func (u *userMap) Set(userID string, client *Client) { } func (u *userMap) DeleteClients(userID string, clients []*Client) (isDeleteUser bool) { + log.ZInfo(context.Background(), "UserMap DeleteClients", "userID", userID, "client", len(clients)) if len(clients) == 0 { return false } @@ -134,7 +140,7 @@ func (u *userMap) GetAllUserStatus(deadline time.Time, nowtime time.Time) []User if userPlatform.Time.Before(deadline) { continue } - userPlatform.Time = time.Now() + userPlatform.Time = nowtime online := make([]int32, 0, len(userPlatform.Clients)) for _, client := range userPlatform.Clients { online = append(online, int32(client.PlatformID)) From 6e2659c64a7f91faefb804af8c1084913647c2e5 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Tue, 2 Jul 2024 15:46:57 +0800 Subject: [PATCH 12/27] online cache --- internal/msggateway/n_ws_server.go | 2 +- internal/msggateway/user_map.go | 208 ++++++++++++++--------------- internal/msggateway/user_map2.go | 161 ---------------------- 3 files changed, 99 insertions(+), 272 deletions(-) delete mode 100644 internal/msggateway/user_map2.go diff --git a/internal/msggateway/n_ws_server.go b/internal/msggateway/n_ws_server.go index 58408e1ef..1af9f014f 100644 --- a/internal/msggateway/n_ws_server.go +++ b/internal/msggateway/n_ws_server.go @@ -60,7 +60,7 @@ type WsServer struct { registerChan chan *Client unregisterChan chan *Client kickHandlerChan chan *kickHandler - clients UMap + clients UserMap clientPool sync.Pool onlineUserNum atomic.Int64 onlineUserConnNum atomic.Int64 diff --git a/internal/msggateway/user_map.go b/internal/msggateway/user_map.go index e1388d588..eb03123a5 100644 --- a/internal/msggateway/user_map.go +++ b/internal/msggateway/user_map.go @@ -1,40 +1,32 @@ -// Copyright © 2023 OpenIM. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - package msggateway import ( - "context" + "github.com/openimsdk/tools/utils/datautil" "sync" "time" - - "github.com/openimsdk/tools/log" - "github.com/openimsdk/tools/utils/datautil" ) -func newUserMap1() UMap { - return &UserMap{ - ch: make(chan UserState, 1024), - } +type UserMap interface { + GetAll(userID string) ([]*Client, bool) + Get(userID string, platformID int) ([]*Client, bool, bool) + Set(userID string, v *Client) + DeleteClients(userID string, clients []*Client) (isDeleteUser bool) + UserState() <-chan UserState + GetAllUserStatus(deadline time.Time, nowtime time.Time) []UserState } -type UserPlatform1 struct { +type UserState struct { + UserID string + Online []int32 + Offline []int32 +} + +type UserPlatform struct { Time time.Time Clients []*Client } -func (u *UserPlatform1) PlatformIDs() []int32 { +func (u *UserPlatform) PlatformIDs() []int32 { if len(u.Clients) == 0 { return nil } @@ -45,36 +37,20 @@ func (u *UserPlatform1) PlatformIDs() []int32 { return platformIDs } -type UserMap struct { - m sync.Map - ch chan UserState -} - -func (u *UserMap) UserState() <-chan UserState { - return u.ch +func newUserMap() UserMap { + return &userMap{ + data: make(map[string]*UserPlatform), + ch: make(chan UserState, 10000), + } } -func (u *UserMap) GetAllUserStatus(deadline time.Time, nowtime time.Time) []UserState { - var result []UserState - u.m.Range(func(key, value any) bool { - client := value.(*UserPlatform1) - if client.Time.Before(deadline) { - return true - } - client.Time = nowtime - us := UserState{ - UserID: key.(string), - Online: make([]int32, 0, len(client.Clients)), - } - for _, c := range client.Clients { - us.Online = append(us.Online, int32(c.PlatformID)) - } - return true - }) - return result +type userMap struct { + lock sync.RWMutex + data map[string]*UserPlatform + ch chan UserState } -func (u *UserMap) push(userID string, userPlatform *UserPlatform1, offline []int32) bool { +func (u *userMap) push(userID string, userPlatform *UserPlatform, offline []int32) bool { select { case u.ch <- UserState{UserID: userID, Online: userPlatform.PlatformIDs(), Offline: offline}: userPlatform.Time = time.Now() @@ -84,83 +60,95 @@ func (u *UserMap) push(userID string, userPlatform *UserPlatform1, offline []int } } -func (u *UserMap) GetAll(key string) ([]*Client, bool) { - allClients, ok := u.m.Load(key) - if ok { - return allClients.(*UserPlatform1).Clients, ok +func (u *userMap) GetAll(userID string) ([]*Client, bool) { + u.lock.RLock() + defer u.lock.RUnlock() + result, ok := u.data[userID] + if !ok { + return nil, false } - return nil, ok + return result.Clients, true } -func (u *UserMap) Get(key string, platformID int) ([]*Client, bool, bool) { - allClients, userExisted := u.m.Load(key) - if userExisted { - var clients []*Client - for _, client := range allClients.(*UserPlatform1).Clients { - if client.PlatformID == platformID { - clients = append(clients, client) - } - } - if len(clients) > 0 { - return clients, true, true +func (u *userMap) Get(userID string, platformID int) ([]*Client, bool, bool) { + u.lock.RLock() + defer u.lock.RUnlock() + result, ok := u.data[userID] + if !ok { + return nil, false, false + } + var clients []*Client + for _, client := range result.Clients { + if client.PlatformID == platformID { + clients = append(clients, client) } - return clients, true, false } - return nil, false, false + return clients, true, len(clients) > 0 } -// Set adds a client to the map. -func (u *UserMap) Set(key string, v *Client) { - allClients, existed := u.m.Load(key) - if existed { - log.ZDebug(context.Background(), "Set existed", "user_id", key, "client_user_id", v.UserID) - oldClients := allClients.(*UserPlatform1) - oldClients.Time = time.Now() - oldClients.Clients = append(oldClients.Clients, v) - u.push(key, oldClients, nil) +func (u *userMap) Set(userID string, client *Client) { + u.lock.Lock() + defer u.lock.Unlock() + result, ok := u.data[userID] + if ok { + result.Clients = append(result.Clients, client) } else { - log.ZDebug(context.Background(), "Set not existed", "user_id", key, "client_user_id", v.UserID) - cli := &UserPlatform1{ - Time: time.Now(), - Clients: []*Client{v}, + result = &UserPlatform{ + Clients: []*Client{client}, } - u.m.Store(key, cli) - u.push(key, cli, nil) + u.data[userID] = result } - + u.push(client.UserID, result, nil) } -func (u *UserMap) DeleteClients(key string, clients []*Client) (isDeleteUser bool) { - m := datautil.SliceToMapAny(clients, func(c *Client) (string, struct{}) { - return c.ctx.GetRemoteAddr(), struct{}{} - }) - allClients, existed := u.m.Load(key) - if !existed { - // If the key doesn't exist, return false. +func (u *userMap) DeleteClients(userID string, clients []*Client) (isDeleteUser bool) { + if len(clients) == 0 { return false } - - // Filter out clients that are in the deleteMap. - oldClients := allClients.(*UserPlatform1) - var ( - remainingClients []*Client - offline []int32 - ) - for _, client := range oldClients.Clients { - if _, shouldBeDeleted := m[client.ctx.GetRemoteAddr()]; !shouldBeDeleted { - remainingClients = append(remainingClients, client) - } else { - offline = append(offline, int32(client.PlatformID)) + u.lock.Lock() + defer u.lock.Unlock() + result, ok := u.data[userID] + if !ok { + return false + } + offline := make([]int32, 0, len(clients)) + deleteAddr := datautil.SliceSetAny(clients, func(client *Client) string { + return client.ctx.GetRemoteAddr() + }) + tmp := result.Clients + result.Clients = result.Clients[:0] + for _, client := range tmp { + if _, ok := deleteAddr[client.ctx.GetRemoteAddr()]; ok { + continue } + result.Clients = append(result.Clients, client) } + defer u.push(userID, result, offline) + if len(result.Clients) > 0 { + return false + } + delete(u.data, userID) + return true +} - oldClients.Clients = remainingClients - defer u.push(key, oldClients, offline) - // Update or delete the key based on the remaining clients. - if len(remainingClients) == 0 { - u.m.Delete(key) - return true +func (u *userMap) GetAllUserStatus(deadline time.Time, nowtime time.Time) []UserState { + u.lock.RLock() + defer u.lock.RUnlock() + result := make([]UserState, 0, len(u.data)) + for userID, userPlatform := range u.data { + if userPlatform.Time.Before(deadline) { + continue + } + userPlatform.Time = nowtime + online := make([]int32, 0, len(userPlatform.Clients)) + for _, client := range userPlatform.Clients { + online = append(online, int32(client.PlatformID)) + } + result = append(result, UserState{UserID: userID, Online: online}) } + return result +} - return false +func (u *userMap) UserState() <-chan UserState { + return u.ch } diff --git a/internal/msggateway/user_map2.go b/internal/msggateway/user_map2.go deleted file mode 100644 index c913f49cc..000000000 --- a/internal/msggateway/user_map2.go +++ /dev/null @@ -1,161 +0,0 @@ -package msggateway - -import ( - "context" - "github.com/openimsdk/tools/log" - "github.com/openimsdk/tools/utils/datautil" - "sync" - "time" -) - -type UMap interface { - GetAll(userID string) ([]*Client, bool) - Get(userID string, platformID int) ([]*Client, bool, bool) - Set(userID string, v *Client) - DeleteClients(userID string, clients []*Client) (isDeleteUser bool) - UserState() <-chan UserState - GetAllUserStatus(deadline time.Time, nowtime time.Time) []UserState -} - -var _ UMap = (*userMap)(nil) - -type UserPlatform struct { - Time time.Time - Clients []*Client -} - -func (u *UserPlatform) PlatformIDs() []int32 { - if len(u.Clients) == 0 { - return nil - } - platformIDs := make([]int32, 0, len(u.Clients)) - for _, client := range u.Clients { - platformIDs = append(platformIDs, int32(client.PlatformID)) - } - return platformIDs -} - -func newUserMap() UMap { - return &userMap{ - data: make(map[string]*UserPlatform), - ch: make(chan UserState, 10000), - } -} - -type userMap struct { - lock sync.RWMutex - data map[string]*UserPlatform - ch chan UserState -} - -func (u *userMap) push(userID string, userPlatform *UserPlatform, offline []int32) bool { - select { - case u.ch <- UserState{UserID: userID, Online: userPlatform.PlatformIDs(), Offline: offline}: - userPlatform.Time = time.Now() - return true - default: - return false - } -} - -func (u *userMap) GetAll(userID string) ([]*Client, bool) { - log.ZInfo(context.Background(), "UserMap GetAll", "userID", userID) - u.lock.RLock() - defer u.lock.RUnlock() - result, ok := u.data[userID] - if !ok { - return nil, false - } - return result.Clients, true -} - -func (u *userMap) Get(userID string, platformID int) ([]*Client, bool, bool) { - log.ZInfo(context.Background(), "UserMap Get", "userID", userID, "platformID", platformID) - u.lock.RLock() - defer u.lock.RUnlock() - result, ok := u.data[userID] - if !ok { - return nil, false, false - } - var clients []*Client - for _, client := range result.Clients { - if client.PlatformID == platformID { - clients = append(clients, client) - } - } - return clients, true, len(clients) > 0 -} - -func (u *userMap) Set(userID string, client *Client) { - log.ZInfo(context.Background(), "UserMap Set", "userID", userID, "client", client.ctx.GetRemoteAddr()) - u.lock.Lock() - defer u.lock.Unlock() - result, ok := u.data[userID] - if ok { - result.Clients = append(result.Clients, client) - } else { - result = &UserPlatform{ - Clients: []*Client{client}, - } - } - u.push(client.UserID, result, nil) -} - -func (u *userMap) DeleteClients(userID string, clients []*Client) (isDeleteUser bool) { - log.ZInfo(context.Background(), "UserMap DeleteClients", "userID", userID, "client", len(clients)) - if len(clients) == 0 { - return false - } - u.lock.Lock() - defer u.lock.Unlock() - result, ok := u.data[userID] - if !ok { - return false - } - offline := make([]int32, 0, len(clients)) - deleteAddr := datautil.SliceSetAny(clients, func(client *Client) string { - return client.ctx.GetRemoteAddr() - }) - tmp := result.Clients - result.Clients = result.Clients[:0] - for _, client := range tmp { - if _, ok := deleteAddr[client.ctx.GetRemoteAddr()]; ok { - continue - } - result.Clients = append(result.Clients, client) - } - defer u.push(userID, result, offline) - if len(result.Clients) > 0 { - return false - } - delete(u.data, userID) - return true -} - -func (u *userMap) GetAllUserStatus(deadline time.Time, nowtime time.Time) []UserState { - u.lock.RLock() - defer u.lock.RUnlock() - result := make([]UserState, 0, len(u.data)) - for userID, userPlatform := range u.data { - if userPlatform.Time.Before(deadline) { - continue - } - userPlatform.Time = nowtime - online := make([]int32, 0, len(userPlatform.Clients)) - for _, client := range userPlatform.Clients { - online = append(online, int32(client.PlatformID)) - } - result = append(result, UserState{UserID: userID, Online: online}) - } - return result -} - -func (u *userMap) UserState() <-chan UserState { - return u.ch -} - -type UserState struct { - UserID string - Online []int32 - Offline []int32 -} From f87ee44459682a1517f01b1552f0e79537cc8784 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Tue, 2 Jul 2024 15:52:57 +0800 Subject: [PATCH 13/27] online cache --- internal/msggateway/online.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/msggateway/online.go b/internal/msggateway/online.go index 61549083a..ec03fc803 100644 --- a/internal/msggateway/online.go +++ b/internal/msggateway/online.go @@ -100,7 +100,7 @@ func (ws *WsServer) ChangeOnlineStatus(concurrent int) { case now := <-scanTicker.C: pushUserState(ws.clients.GetAllUserStatus(now.Add(-cachekey.OnlineExpire/3), now)...) case state := <-ws.clients.UserState(): - log.ZDebug(context.Background(), "user online change", "userID", state.UserID, "online", state.Online, "offline", state.Offline) + log.ZDebug(context.Background(), "OnlineCache user online change", "userID", state.UserID, "online", state.Online, "offline", state.Offline) pushUserState(state) } } From 1bfaf3e2d911eddc106d3eee58e4262dfc624d57 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Tue, 2 Jul 2024 15:56:20 +0800 Subject: [PATCH 14/27] online cache --- internal/msggateway/user_map.go | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/msggateway/user_map.go b/internal/msggateway/user_map.go index eb03123a5..a5a4a2de5 100644 --- a/internal/msggateway/user_map.go +++ b/internal/msggateway/user_map.go @@ -121,6 +121,7 @@ func (u *userMap) DeleteClients(userID string, clients []*Client) (isDeleteUser if _, ok := deleteAddr[client.ctx.GetRemoteAddr()]; ok { continue } + offline = append(offline, int32(client.PlatformID)) result.Clients = append(result.Clients, client) } defer u.push(userID, result, offline) From dcd874979e261bec3115a94ebed00c829d37cab6 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Tue, 2 Jul 2024 16:16:11 +0800 Subject: [PATCH 15/27] online cache --- internal/msggateway/user_map.go | 8 ++++---- pkg/rpccache/online.go | 13 +++++-------- 2 files changed, 9 insertions(+), 12 deletions(-) diff --git a/internal/msggateway/user_map.go b/internal/msggateway/user_map.go index a5a4a2de5..57fa78087 100644 --- a/internal/msggateway/user_map.go +++ b/internal/msggateway/user_map.go @@ -118,11 +118,11 @@ func (u *userMap) DeleteClients(userID string, clients []*Client) (isDeleteUser tmp := result.Clients result.Clients = result.Clients[:0] for _, client := range tmp { - if _, ok := deleteAddr[client.ctx.GetRemoteAddr()]; ok { - continue + if _, delCli := deleteAddr[client.ctx.GetRemoteAddr()]; delCli { + offline = append(offline, int32(client.PlatformID)) + } else { + result.Clients = append(result.Clients, client) } - offline = append(offline, int32(client.PlatformID)) - result.Clients = append(result.Clients, client) } defer u.push(userID, result, offline) if len(result.Clients) > 0 { diff --git a/pkg/rpccache/online.go b/pkg/rpccache/online.go index aeeafcf2b..5072984b0 100644 --- a/pkg/rpccache/online.go +++ b/pkg/rpccache/online.go @@ -51,11 +51,8 @@ func NewOnlineCache(user rpcclient.UserRpcClient, group *GroupLocalCache, rdb re log.ZError(ctx, "OnlineCache redis subscribe parseUserOnlineStatus", err, "payload", message.Payload, "channel", message.Channel) continue } - log.ZDebug(ctx, "OnlineCache setUserOnline", "userID", userID, "platformIDs", platformIDs, "payload", message.Payload) - x.setUserOnline(userID, platformIDs) - //if err := x.setUserOnline(ctx, userID, platformIDs); err != nil { - // log.ZError(ctx, "redis subscribe setUserOnline", err, "payload", message.Payload, "channel", message.Channel) - //} + storageCache := x.setUserOnline(userID, platformIDs) + log.ZDebug(ctx, "OnlineCache setUserOnline", "userID", userID, "platformIDs", platformIDs, "payload", message.Payload, "storageCache", storageCache) } }() return x @@ -115,10 +112,10 @@ func (o *OnlineCache) GetGroupOnline(ctx context.Context, groupID string) ([]str onlineUserIDs = append(onlineUserIDs, userID) } } - log.ZDebug(ctx, "OnlineCache GetGroupOnline", "groupID", groupID, "onlineUserIDs", onlineUserIDs) + log.ZDebug(ctx, "OnlineCache GetGroupOnline", "groupID", groupID, "onlineUserIDs", onlineUserIDs, "allUserID", userIDs) return onlineUserIDs, nil } -func (o *OnlineCache) setUserOnline(userID string, platformIDs []int32) { - o.local.SetHas(o.getUserOnlineKey(userID), platformIDs) +func (o *OnlineCache) setUserOnline(userID string, platformIDs []int32) bool { + return o.local.SetHas(o.getUserOnlineKey(userID), platformIDs) } From 14aba3bb8976f20651246fd49a0367e9e401ae41 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Tue, 2 Jul 2024 16:26:24 +0800 Subject: [PATCH 16/27] online cache --- pkg/rpccache/online.go | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/pkg/rpccache/online.go b/pkg/rpccache/online.go index 5072984b0..5bacd647d 100644 --- a/pkg/rpccache/online.go +++ b/pkg/rpccache/online.go @@ -64,10 +64,6 @@ type OnlineCache struct { local lru.LRU[string, []int32] } -func (o *OnlineCache) getUserOnlineKey(userID string) string { - return "" + userID -} - func (o *OnlineCache) GetUserOnlinePlatform(ctx context.Context, userID string) ([]int32, error) { return o.local.Get(userID, func() ([]int32, error) { return o.user.GetUserOnlinePlatform(ctx, userID) @@ -117,5 +113,5 @@ func (o *OnlineCache) GetGroupOnline(ctx context.Context, groupID string) ([]str } func (o *OnlineCache) setUserOnline(userID string, platformIDs []int32) bool { - return o.local.SetHas(o.getUserOnlineKey(userID), platformIDs) + return o.local.SetHas(userID, platformIDs) } From 3df39a8382c16866540456868cbd12b1bc41213a Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Tue, 2 Jul 2024 16:43:43 +0800 Subject: [PATCH 17/27] online cache --- internal/msggateway/online.go | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/internal/msggateway/online.go b/internal/msggateway/online.go index ec03fc803..b50608f93 100644 --- a/internal/msggateway/online.go +++ b/internal/msggateway/online.go @@ -18,7 +18,9 @@ func (ws *WsServer) ChangeOnlineStatus(concurrent int) { if concurrent < 1 { concurrent = 1 } - scanTicker := time.NewTicker(time.Minute * 3) + const renewalTime = cachekey.OnlineExpire / 3 + //const renewalTime = time.Second * 10 + renewalTicker := time.NewTicker(renewalTime) requestChs := make([]chan *pbuser.SetUserOnlineStatusReq, concurrent) changeStatus := make([][]UserState, concurrent) @@ -97,8 +99,11 @@ func (ws *WsServer) ChangeOnlineStatus(concurrent int) { select { case <-mergeTicker.C: pushAllUserState() - case now := <-scanTicker.C: - pushUserState(ws.clients.GetAllUserStatus(now.Add(-cachekey.OnlineExpire/3), now)...) + case now := <-renewalTicker.C: + deadline := now.Add(-cachekey.OnlineExpire / 3) + users := ws.clients.GetAllUserStatus(deadline, now) + log.ZDebug(context.Background(), "renewal ticker", "deadline", deadline, "nowtime", now, "num", len(users)) + pushUserState(users...) case state := <-ws.clients.UserState(): log.ZDebug(context.Background(), "OnlineCache user online change", "userID", state.UserID, "online", state.Online, "offline", state.Offline) pushUserState(state) From e791cbde969715d918b1451ea275a2e3b297980b Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Wed, 3 Jul 2024 15:06:09 +0800 Subject: [PATCH 18/27] online push --- internal/push/push_handler.go | 45 ++++++++++++++++++++++++++++------- 1 file changed, 36 insertions(+), 9 deletions(-) diff --git a/internal/push/push_handler.go b/internal/push/push_handler.go index 33f987528..3f3ca11e8 100644 --- a/internal/push/push_handler.go +++ b/internal/push/push_handler.go @@ -28,6 +28,7 @@ import ( "github.com/openimsdk/open-im-server/v3/pkg/util/conversationutil" "github.com/openimsdk/protocol/constant" pbchat "github.com/openimsdk/protocol/msg" + "github.com/openimsdk/protocol/msggateway" pbpush "github.com/openimsdk/protocol/push" "github.com/openimsdk/protocol/sdkws" "github.com/openimsdk/tools/discovery" @@ -128,16 +129,11 @@ func (c *ConsumerHandler) ConsumeClaim(sess sarama.ConsumerGroupSession, claim s // Push2User Suitable for two types of conversations, one is SingleChatType and the other is NotificationChatType. func (c *ConsumerHandler) Push2User(ctx context.Context, userIDs []string, msg *sdkws.MsgData) (err error) { - userIDs, err = c.onlineCache.GetUsersOnline(ctx, userIDs) - if err != nil { - return err - } - log.ZDebug(ctx, "Get msg from msg_transfer And push msg", "userIDs", userIDs, "msg", msg.String()) if err := c.webhookBeforeOnlinePush(ctx, &c.config.WebhooksConfig.BeforeOnlinePush, userIDs, msg); err != nil { return err } - wsResults, err := c.onlinePusher.GetConnsAndOnlinePush(ctx, msg, userIDs) + wsResults, err := c.GetConnsAndOnlinePush(ctx, msg, userIDs) if err != nil { return err } @@ -186,6 +182,38 @@ func (c *ConsumerHandler) shouldPushOffline(_ context.Context, msg *sdkws.MsgDat return true } +func (c *ConsumerHandler) GetConnsAndOnlinePush(ctx context.Context, msg *sdkws.MsgData, pushToUserIDs []string) ([]*msggateway.SingleMsgToUserResults, error) { + var ( + onlineUserIDs []string + offlineUserIDs []string + ) + for _, userID := range pushToUserIDs { + online, err := c.onlineCache.GetUserOnline(ctx, userID) + if err != nil { + return nil, err + } + if online { + onlineUserIDs = append(onlineUserIDs, userID) + } else { + offlineUserIDs = append(offlineUserIDs, userID) + } + } + var result []*msggateway.SingleMsgToUserResults + if len(onlineUserIDs) > 0 { + var err error + result, err = c.onlinePusher.GetConnsAndOnlinePush(ctx, msg, pushToUserIDs) + if err != nil { + return nil, err + } + } + for _, userID := range offlineUserIDs { + result = append(result, &msggateway.SingleMsgToUserResults{ + UserID: userID, + }) + } + return result, nil +} + func (c *ConsumerHandler) Push2Group(ctx context.Context, groupID string, msg *sdkws.MsgData) (err error) { log.ZDebug(ctx, "Get super group msg from msg_transfer and push msg", "msg", msg.String(), "groupID", groupID) var pushToUserIDs []string @@ -199,7 +227,7 @@ func (c *ConsumerHandler) Push2Group(ctx context.Context, groupID string, msg *s return err } - wsResults, err := c.onlinePusher.GetConnsAndOnlinePush(ctx, msg, pushToUserIDs) + wsResults, err := c.GetConnsAndOnlinePush(ctx, msg, pushToUserIDs) if err != nil { return err } @@ -240,8 +268,7 @@ func (c *ConsumerHandler) Push2Group(ctx context.Context, groupID string, msg *s } func (c *ConsumerHandler) groupMessagesHandler(ctx context.Context, groupID string, pushToUserIDs *[]string, msg *sdkws.MsgData) (err error) { if len(*pushToUserIDs) == 0 { - //*pushToUserIDs, err = c.groupLocalCache.GetGroupMemberIDs(ctx, groupID) - *pushToUserIDs, err = c.onlineCache.GetGroupOnline(ctx, groupID) // + *pushToUserIDs, err = c.groupLocalCache.GetGroupMemberIDs(ctx, groupID) if err != nil { return err } From 006766cd14a274ea51da6d6565d88eb59d2ff9db Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Wed, 3 Jul 2024 15:21:34 +0800 Subject: [PATCH 19/27] online push --- pkg/rpccache/online.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/rpccache/online.go b/pkg/rpccache/online.go index 5bacd647d..eac2f43af 100644 --- a/pkg/rpccache/online.go +++ b/pkg/rpccache/online.go @@ -21,7 +21,7 @@ func NewOnlineCache(user rpcclient.UserRpcClient, group *GroupLocalCache, rdb re user: user, group: group, local: lru.NewSlotLRU(1024, localcache.LRUStringHash, func() lru.LRU[string, []int32] { - return lru.NewLayLRU[string, []int32](2048, cachekey.OnlineExpire, time.Second*3, localcache.EmptyTarget{}, func(key string, value []int32) {}) + return lru.NewLayLRU[string, []int32](2048, cachekey.OnlineExpire/2, time.Second*3, localcache.EmptyTarget{}, func(key string, value []int32) {}) }), } go func() { From fcda73f4bc0eb3f48b74d5567ff25364d853fb4e Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Thu, 4 Jul 2024 10:16:31 +0800 Subject: [PATCH 20/27] online push --- cmd/openim-api/main.go | 1 - go.mod | 2 +- go.sum | 4 +- internal/msggateway/client.go | 2 + internal/msggateway/n_ws_server.go | 22 +++-- internal/msggateway/subscription.go | 148 ++++++++++++++++++++++++++++ 6 files changed, 165 insertions(+), 14 deletions(-) create mode 100644 internal/msggateway/subscription.go diff --git a/cmd/openim-api/main.go b/cmd/openim-api/main.go index 58e540c05..e29ed2a59 100644 --- a/cmd/openim-api/main.go +++ b/cmd/openim-api/main.go @@ -25,5 +25,4 @@ func main() { if err := cmd.NewApiCmd().Exec(); err != nil { program.ExitWithError(err) } - } diff --git a/go.mod b/go.mod index e7ed446f4..e4f45e963 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/lestrrat-go/file-rotatelogs v2.4.0+incompatible // indirect github.com/mitchellh/mapstructure v1.5.0 - github.com/openimsdk/protocol v0.0.69-alpha.20 + github.com/openimsdk/protocol v0.0.69-alpha.24 github.com/openimsdk/tools v0.0.49-alpha.25 github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 diff --git a/go.sum b/go.sum index adde3aa82..b0f495e82 100644 --- a/go.sum +++ b/go.sum @@ -270,8 +270,8 @@ github.com/onsi/gomega v1.25.0 h1:Vw7br2PCDYijJHSfBOWhov+8cAnUf8MfMaIOV323l6Y= github.com/onsi/gomega v1.25.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdMPGhM= github.com/openimsdk/gomake v0.0.13 h1:xLDe/moqgWpRoptHzI4packAWzs4C16b+sVY+txNJp0= github.com/openimsdk/gomake v0.0.13/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= -github.com/openimsdk/protocol v0.0.69-alpha.20 h1:skZu82sqoMhiQVEZgrRsjcfI3Grp1IpThx1LJPqETWs= -github.com/openimsdk/protocol v0.0.69-alpha.20/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= +github.com/openimsdk/protocol v0.0.69-alpha.24 h1:TYcNJeWOTuE40UQ54eNPdDdy0KTOh9rAOgax8lCyhDc= +github.com/openimsdk/protocol v0.0.69-alpha.24/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= github.com/openimsdk/tools v0.0.49-alpha.25 h1:OpRPwDZ2xWX7Zj5kyfZhryu/NfZTrsRVr2GFwu1HQHI= github.com/openimsdk/tools v0.0.49-alpha.25/go.mod h1:rwsFI1G/nBHNfiNapbven41akRDPBbH4df0Cgy6xueU= github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= diff --git a/internal/msggateway/client.go b/internal/msggateway/client.go index 0581a025b..62c71d5be 100644 --- a/internal/msggateway/client.go +++ b/internal/msggateway/client.go @@ -72,6 +72,8 @@ type Client struct { closed atomic.Bool closedErr error token string + //subLock sync.Mutex + //subUserIDs map[string]struct{} } // ResetClient updates the client's state with new connection and context information. diff --git a/internal/msggateway/n_ws_server.go b/internal/msggateway/n_ws_server.go index 1af9f014f..8a79151fe 100644 --- a/internal/msggateway/n_ws_server.go +++ b/internal/msggateway/n_ws_server.go @@ -54,13 +54,14 @@ type LongConnServer interface { } type WsServer struct { - msgGatewayConfig *Config - port int - wsMaxConnNum int64 - registerChan chan *Client - unregisterChan chan *Client - kickHandlerChan chan *kickHandler - clients UserMap + msgGatewayConfig *Config + port int + wsMaxConnNum int64 + registerChan chan *Client + unregisterChan chan *Client + kickHandlerChan chan *kickHandler + clients UserMap + //subscription *Subscription clientPool sync.Pool onlineUserNum atomic.Int64 onlineUserConnNum atomic.Int64 @@ -141,9 +142,10 @@ func NewWsServer(msgGatewayConfig *Config, opts ...Option) *WsServer { kickHandlerChan: make(chan *kickHandler, 1000), validate: v, clients: newUserMap(), - Compressor: NewGzipCompressor(), - Encoder: NewGobEncoder(), - webhookClient: webhook.NewWebhookClient(msgGatewayConfig.WebhooksConfig.URL), + //subscription: newSubscription(), + Compressor: NewGzipCompressor(), + Encoder: NewGobEncoder(), + webhookClient: webhook.NewWebhookClient(msgGatewayConfig.WebhooksConfig.URL), } } diff --git a/internal/msggateway/subscription.go b/internal/msggateway/subscription.go new file mode 100644 index 000000000..d7e606037 --- /dev/null +++ b/internal/msggateway/subscription.go @@ -0,0 +1,148 @@ +package msggateway + +//import ( +// "context" +// "encoding/json" +// "github.com/openimsdk/protocol/constant" +// "github.com/openimsdk/protocol/sdkws" +// "github.com/openimsdk/tools/log" +// "github.com/openimsdk/tools/utils/datautil" +// "github.com/openimsdk/tools/utils/idutil" +// "sync" +// "time" +//) +// +//type subClient struct { +// clients map[string]*Client +//} +// +//func newSubscription() *Subscription { +// return &Subscription{ +// userIDs: make(map[string]*subClient), +// } +//} +// +//type Subscription struct { +// lock sync.RWMutex +// userIDs map[string]*subClient +//} +// +//func (s *Subscription) GetClient(userID string) []*Client { +// s.lock.RLock() +// defer s.lock.RUnlock() +// cs, ok := s.userIDs[userID] +// if !ok { +// return nil +// } +// clients := make([]*Client, 0, len(cs.clients)) +// for _, client := range cs.clients { +// clients = append(clients, client) +// } +// return clients +//} +// +//func (s *Subscription) DelClient(client *Client) { +// client.subLock.Lock() +// userIDs := datautil.Keys(client.subUserIDs) +// for _, userID := range userIDs { +// delete(client.subUserIDs, userID) +// } +// client.subLock.Unlock() +// if len(userIDs) == 0 { +// return +// } +// addr := client.ctx.GetRemoteAddr() +// s.lock.Lock() +// defer s.lock.Unlock() +// for _, userID := range userIDs { +// sub, ok := s.userIDs[userID] +// if !ok { +// continue +// } +// delete(sub.clients, addr) +// if len(sub.clients) == 0 { +// delete(s.userIDs, userID) +// } +// } +//} +// +//func (s *Subscription) Sub(client *Client, addUserIDs, delUserIDs []string) { +// if len(addUserIDs)+len(delUserIDs) == 0 { +// return +// } +// var ( +// del = make(map[string]struct{}) +// add = make(map[string]struct{}) +// ) +// client.subLock.Lock() +// for _, userID := range delUserIDs { +// if _, ok := client.subUserIDs[userID]; !ok { +// continue +// } +// del[userID] = struct{}{} +// delete(client.subUserIDs, userID) +// } +// for _, userID := range addUserIDs { +// delete(del, userID) +// if _, ok := client.subUserIDs[userID]; ok { +// continue +// } +// client.subUserIDs[userID] = struct{}{} +// } +// client.subLock.Unlock() +// if len(del)+len(add) == 0 { +// return +// } +// addr := client.ctx.GetRemoteAddr() +// s.lock.Lock() +// defer s.lock.Unlock() +// for userID := range del { +// sub, ok := s.userIDs[userID] +// if !ok { +// continue +// } +// delete(sub.clients, addr) +// if len(sub.clients) == 0 { +// delete(s.userIDs, userID) +// } +// } +// for userID := range add { +// sub, ok := s.userIDs[userID] +// if !ok { +// sub = &subClient{clients: make(map[string]*Client)} +// s.userIDs[userID] = sub +// } +// sub.clients[addr] = client +// } +//} +// +//func (ws *WsServer) pushUserIDOnlineStatus(ctx context.Context, userID string, platformIDs []int32) { +// clients := ws.subscription.GetClient(userID) +// if len(clients) == 0 { +// return +// } +// msgContent, err := json.Marshal(platformIDs) +// if err != nil { +// log.ZError(ctx, "pushUserIDOnlineStatus json.Marshal", err) +// return +// } +// now := time.Now().UnixMilli() +// msgID := idutil.GetMsgIDByMD5(userID) +// msg := &sdkws.MsgData{ +// SendID: userID, +// ClientMsgID: msgID, +// ServerMsgID: msgID, +// SenderPlatformID: constant.AdminPlatformID, +// SessionType: constant.NotificationChatType, +// ContentType: constant.UserSubscribeOnlineStatusNotification, +// Content: msgContent, +// SendTime: now, +// CreateTime: now, +// } +// for _, client := range clients { +// msg.RecvID = client.UserID +// if err := client.PushMessage(ctx, msg); err != nil { +// log.ZError(ctx, "UserSubscribeOnlineStatusNotification push failed", err, "userID", client.UserID, "platformID", client.PlatformID, "changeUserID", userID, "content", msgContent) +// } +// } +//} From a9ab9baa2300c3e603ec4c1b11294ca8d9697401 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Thu, 4 Jul 2024 10:36:18 +0800 Subject: [PATCH 21/27] resolving conflicts --- go.sum | 4 ++-- internal/rpc/user/user.go | 5 ----- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/go.sum b/go.sum index f452bcd87..8d2eb40bb 100644 --- a/go.sum +++ b/go.sum @@ -270,8 +270,8 @@ github.com/onsi/gomega v1.25.0 h1:Vw7br2PCDYijJHSfBOWhov+8cAnUf8MfMaIOV323l6Y= github.com/onsi/gomega v1.25.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdMPGhM= github.com/openimsdk/gomake v0.0.14-alpha.5 h1:VY9c5x515lTfmdhhPjMvR3BBRrRquAUCFsz7t7vbv7Y= github.com/openimsdk/gomake v0.0.14-alpha.5/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= -github.com/openimsdk/protocol v0.0.69-alpha.22 h1:kifZWVNDkg9diXFJUJ/Q9xFc80cveBhc+1dUXcE9xHQ= -github.com/openimsdk/protocol v0.0.69-alpha.22/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= +github.com/openimsdk/protocol v0.0.69-alpha.24 h1:TYcNJeWOTuE40UQ54eNPdDdy0KTOh9rAOgax8lCyhDc= +github.com/openimsdk/protocol v0.0.69-alpha.24/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= github.com/openimsdk/tools v0.0.49-alpha.39 h1:bl5+q7xHsc/j1NnkN8/gYmn23RsNNbRizDY58d2EY1w= github.com/openimsdk/tools v0.0.49-alpha.39/go.mod h1:zc0maZ2ohXlHd0ylY5JnCE8uqq/hslhcfcKa6iO5PCU= github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= diff --git a/internal/rpc/user/user.go b/internal/rpc/user/user.go index 1da923e95..0b96077ec 100644 --- a/internal/rpc/user/user.go +++ b/internal/rpc/user/user.go @@ -62,11 +62,6 @@ type userServer struct { webhookClient *webhook.Client } -func (s *userServer) SetUserOnlineStatus(ctx context.Context, req *pbuser.SetUserOnlineStatusReq) (*pbuser.SetUserOnlineStatusResp, error) { - //TODO implement me - panic("implement me") -} - type Config struct { RpcConfig config.User RedisConfig config.Redis From 97636c4c7adc9d07871f8eb9ac9a402a043a1d38 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Thu, 4 Jul 2024 10:51:26 +0800 Subject: [PATCH 22/27] test --- pkg/common/storage/cache/redis/seq_user_test.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/pkg/common/storage/cache/redis/seq_user_test.go b/pkg/common/storage/cache/redis/seq_user_test.go index cfbea004c..e4fd95922 100644 --- a/pkg/common/storage/cache/redis/seq_user_test.go +++ b/pkg/common/storage/cache/redis/seq_user_test.go @@ -66,16 +66,13 @@ func TestRecvOnline(t *testing.T) { ctx := context.Background() pubsub := ts.rdb.Subscribe(ctx, cachekey.OnlineChannel) - // 等待订阅确认 _, err := pubsub.Receive(ctx) if err != nil { log.Fatalf("Could not subscribe: %v", err) } - // 创建一个通道来接收消息 ch := pubsub.Channel() - // 处理接收到的消息 for msg := range ch { fmt.Printf("Received message from channel %s: %s\n", msg.Channel, msg.Payload) } From c1967a63caf1af009c3312c407f667c87e093c5a Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Fri, 5 Jul 2024 18:47:48 +0800 Subject: [PATCH 23/27] online status --- internal/msggateway/init.go | 8 ++++++++ internal/msggateway/subscription.go | 25 ++++++++++++++++++++++++ internal/msggateway/user_map.go | 30 +++++++++++++++++++++++++++++ pkg/common/cmd/msg_gateway.go | 1 + pkg/rpccache/online.go | 24 ++--------------------- pkg/util/useronline/split.go | 27 ++++++++++++++++++++++++++ 6 files changed, 93 insertions(+), 22 deletions(-) create mode 100644 pkg/util/useronline/split.go diff --git a/internal/msggateway/init.go b/internal/msggateway/init.go index 815ec8ca6..739a71ecb 100644 --- a/internal/msggateway/init.go +++ b/internal/msggateway/init.go @@ -17,6 +17,7 @@ package msggateway import ( "context" "github.com/openimsdk/open-im-server/v3/pkg/common/config" + "github.com/openimsdk/tools/db/redisutil" "github.com/openimsdk/tools/utils/datautil" "time" @@ -26,6 +27,7 @@ import ( type Config struct { MsgGateway config.MsgGateway Share config.Share + RedisConfig config.Redis WebhooksConfig config.Webhooks Discovery config.Discovery } @@ -42,6 +44,10 @@ func Start(ctx context.Context, index int, conf *Config) error { if err != nil { return err } + rdb, err := redisutil.NewRedisClient(ctx, conf.RedisConfig.Build()) + if err != nil { + return err + } longServer := NewWsServer( conf, WithPort(wsPort), @@ -52,6 +58,8 @@ func Start(ctx context.Context, index int, conf *Config) error { go longServer.ChangeOnlineStatus(4) + go longServer.SubscriberUserOnlineStatusChanges(rdb) + hubServer := NewServer(rpcPort, longServer, conf) netDone := make(chan error) go func() { diff --git a/internal/msggateway/subscription.go b/internal/msggateway/subscription.go index d7e606037..98cf41366 100644 --- a/internal/msggateway/subscription.go +++ b/internal/msggateway/subscription.go @@ -1,5 +1,30 @@ package msggateway +import ( + "context" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/cachekey" + "github.com/openimsdk/open-im-server/v3/pkg/util/useronline" + "github.com/openimsdk/tools/log" + "github.com/openimsdk/tools/mcontext" + "github.com/redis/go-redis/v9" + "math/rand" + "strconv" +) + +func (ws *WsServer) SubscriberUserOnlineStatusChanges(rdb redis.UniversalClient) { + ctx := mcontext.SetOperationID(context.Background(), cachekey.OnlineChannel+strconv.FormatUint(rand.Uint64(), 10)) + for message := range rdb.Subscribe(ctx, cachekey.OnlineChannel).Channel() { + userID, platformIDs, err := useronline.ParseUserOnlineStatus(message.Payload) + if err != nil { + log.ZError(ctx, "OnlineCache redis subscribe parseUserOnlineStatus", err, "payload", message.Payload, "channel", message.Channel) + continue + } + if ws.clients.RecvSubChange(userID, platformIDs) { + log.ZDebug(ctx, "receive subscription message and go back online", "userID", userID) + } + } +} + //import ( // "context" // "encoding/json" diff --git a/internal/msggateway/user_map.go b/internal/msggateway/user_map.go index 57fa78087..bd1f19728 100644 --- a/internal/msggateway/user_map.go +++ b/internal/msggateway/user_map.go @@ -13,6 +13,7 @@ type UserMap interface { DeleteClients(userID string, clients []*Client) (isDeleteUser bool) UserState() <-chan UserState GetAllUserStatus(deadline time.Time, nowtime time.Time) []UserState + RecvSubChange(userID string, platformIDs []int32) bool } type UserState struct { @@ -37,6 +38,17 @@ func (u *UserPlatform) PlatformIDs() []int32 { return platformIDs } +func (u *UserPlatform) PlatformIDSet() map[int32]struct{} { + if len(u.Clients) == 0 { + return nil + } + platformIDs := make(map[int32]struct{}) + for _, client := range u.Clients { + platformIDs[int32(client.PlatformID)] = struct{}{} + } + return platformIDs +} + func newUserMap() UserMap { return &userMap{ data: make(map[string]*UserPlatform), @@ -50,6 +62,24 @@ type userMap struct { ch chan UserState } +func (u *userMap) RecvSubChange(userID string, platformIDs []int32) bool { + u.lock.RLock() + defer u.lock.RUnlock() + result, ok := u.data[userID] + if !ok { + return false + } + localPlatformIDs := result.PlatformIDSet() + for _, platformID := range platformIDs { + delete(localPlatformIDs, platformID) + } + if len(localPlatformIDs) == 0 { + return false + } + u.push(userID, result, nil) + return true +} + func (u *userMap) push(userID string, userPlatform *UserPlatform, offline []int32) bool { select { case u.ch <- UserState{UserID: userID, Online: userPlatform.PlatformIDs(), Offline: offline}: diff --git a/pkg/common/cmd/msg_gateway.go b/pkg/common/cmd/msg_gateway.go index 78004094c..29d3fba33 100644 --- a/pkg/common/cmd/msg_gateway.go +++ b/pkg/common/cmd/msg_gateway.go @@ -37,6 +37,7 @@ func NewMsgGatewayCmd() *MsgGatewayCmd { ret.configMap = map[string]any{ OpenIMMsgGatewayCfgFileName: &msgGatewayConfig.MsgGateway, ShareFileName: &msgGatewayConfig.Share, + RedisConfigFileName: &msgGatewayConfig.RedisConfig, WebhooksConfigFileName: &msgGatewayConfig.WebhooksConfig, DiscoveryConfigFilename: &msgGatewayConfig.Discovery, } diff --git a/pkg/rpccache/online.go b/pkg/rpccache/online.go index eac2f43af..d986651c7 100644 --- a/pkg/rpccache/online.go +++ b/pkg/rpccache/online.go @@ -2,17 +2,16 @@ package rpccache import ( "context" - "errors" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/cachekey" "github.com/openimsdk/open-im-server/v3/pkg/localcache" "github.com/openimsdk/open-im-server/v3/pkg/localcache/lru" "github.com/openimsdk/open-im-server/v3/pkg/rpcclient" + "github.com/openimsdk/open-im-server/v3/pkg/util/useronline" "github.com/openimsdk/tools/log" "github.com/openimsdk/tools/mcontext" "github.com/redis/go-redis/v9" "math/rand" "strconv" - "strings" "time" ) @@ -25,28 +24,9 @@ func NewOnlineCache(user rpcclient.UserRpcClient, group *GroupLocalCache, rdb re }), } go func() { - parseUserOnlineStatus := func(payload string) (string, []int32, error) { - arr := strings.Split(payload, ":") - if len(arr) == 0 { - return "", nil, errors.New("invalid data") - } - userID := arr[len(arr)-1] - if userID == "" { - return "", nil, errors.New("userID is empty") - } - platformIDs := make([]int32, len(arr)-1) - for i := range platformIDs { - platformID, err := strconv.Atoi(arr[i]) - if err != nil { - return "", nil, err - } - platformIDs[i] = int32(platformID) - } - return userID, platformIDs, nil - } ctx := mcontext.SetOperationID(context.Background(), cachekey.OnlineChannel+strconv.FormatUint(rand.Uint64(), 10)) for message := range rdb.Subscribe(ctx, cachekey.OnlineChannel).Channel() { - userID, platformIDs, err := parseUserOnlineStatus(message.Payload) + userID, platformIDs, err := useronline.ParseUserOnlineStatus(message.Payload) if err != nil { log.ZError(ctx, "OnlineCache redis subscribe parseUserOnlineStatus", err, "payload", message.Payload, "channel", message.Channel) continue diff --git a/pkg/util/useronline/split.go b/pkg/util/useronline/split.go new file mode 100644 index 000000000..c39d31d15 --- /dev/null +++ b/pkg/util/useronline/split.go @@ -0,0 +1,27 @@ +package useronline + +import ( + "errors" + "strconv" + "strings" +) + +func ParseUserOnlineStatus(payload string) (string, []int32, error) { + arr := strings.Split(payload, ":") + if len(arr) == 0 { + return "", nil, errors.New("invalid data") + } + userID := arr[len(arr)-1] + if userID == "" { + return "", nil, errors.New("userID is empty") + } + platformIDs := make([]int32, len(arr)-1) + for i := range platformIDs { + platformID, err := strconv.Atoi(arr[i]) + if err != nil { + return "", nil, err + } + platformIDs[i] = int32(platformID) + } + return userID, platformIDs, nil +} From b53a26d2cadf6eab3b2ef7e834c145c0dcd63489 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Mon, 8 Jul 2024 10:04:31 +0800 Subject: [PATCH 24/27] online status --- internal/msggateway/subscription.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/internal/msggateway/subscription.go b/internal/msggateway/subscription.go index 98cf41366..8df548f54 100644 --- a/internal/msggateway/subscription.go +++ b/internal/msggateway/subscription.go @@ -20,7 +20,9 @@ func (ws *WsServer) SubscriberUserOnlineStatusChanges(rdb redis.UniversalClient) continue } if ws.clients.RecvSubChange(userID, platformIDs) { - log.ZDebug(ctx, "receive subscription message and go back online", "userID", userID) + log.ZDebug(ctx, "gateway receive subscription message and go back online", "userID", userID, "platformIDs", platformIDs) + } else { + log.ZDebug(ctx, "gateway ignore user online status changes", "userID", userID, "platformIDs", platformIDs) } } } From fd2b0b32843b6e1316cf6507e90fd19293be0a8e Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Mon, 8 Jul 2024 10:11:40 +0800 Subject: [PATCH 25/27] online status --- go.sum | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/go.sum b/go.sum index 5bc9f6d98..577154a1b 100644 --- a/go.sum +++ b/go.sum @@ -262,8 +262,8 @@ github.com/onsi/gomega v1.25.0 h1:Vw7br2PCDYijJHSfBOWhov+8cAnUf8MfMaIOV323l6Y= github.com/onsi/gomega v1.25.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdMPGhM= github.com/openimsdk/gomake v0.0.14-alpha.5 h1:VY9c5x515lTfmdhhPjMvR3BBRrRquAUCFsz7t7vbv7Y= github.com/openimsdk/gomake v0.0.14-alpha.5/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= -github.com/openimsdk/protocol v0.0.69-alpha.17 h1:pEag4ZdlovE+AyLsw1VYFU/3sk6ayvGdPzgufQfKf9M= -github.com/openimsdk/protocol v0.0.69-alpha.17/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= +github.com/openimsdk/protocol v0.0.69-alpha.24 h1:TYcNJeWOTuE40UQ54eNPdDdy0KTOh9rAOgax8lCyhDc= +github.com/openimsdk/protocol v0.0.69-alpha.24/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= github.com/openimsdk/tools v0.0.49-alpha.45 h1:XIzCoef4myybOiIlGuRY9FTtGBisZFC4Uy4PhG0ZWQ0= github.com/openimsdk/tools v0.0.49-alpha.45/go.mod h1:HtSRjPTL8PsuZ+PhR5noqzrYBF0sdwW3/O/sWVucWg8= github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= From 67d17c82d519b9372136ed46e5ea0a4cedc9b301 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Mon, 8 Jul 2024 11:09:03 +0800 Subject: [PATCH 26/27] online status --- pkg/common/storage/cache/redis/user.go | 100 ------------------------- 1 file changed, 100 deletions(-) diff --git a/pkg/common/storage/cache/redis/user.go b/pkg/common/storage/cache/redis/user.go index b846bf398..c3accd2c3 100644 --- a/pkg/common/storage/cache/redis/user.go +++ b/pkg/common/storage/cache/redis/user.go @@ -128,106 +128,6 @@ func (u *UserCacheRedis) DelUsersGlobalRecvMsgOpt(userIDs ...string) cache.UserC return cache } -/* - - */ - -type RedisUserOnline struct { - // 平台id, 平台更新时间 - PlatformIDs map[int32]int64 -} - -// GetUserStatus get user status. -//func (u *UserCacheRedis) GetUserStatus(ctx context.Context, userIDs []string) ([]*user.OnlineStatus, error) { -// userStatus := make([]*user.OnlineStatus, 0, len(userIDs)) -// for _, userID := range userIDs { -// UserIDNum := crc32.ChecksumIEEE([]byte(userID)) -// modKey := strconv.Itoa(int(UserIDNum % statusMod)) -// var onlineStatus user.OnlineStatus -// key := u.getOnlineStatusKey(modKey) -// result, err := u.rdb.HGet(ctx, key, userID).Result() -// if err != nil { -// if errors.Is(err, redis.Nil) { -// // key or field does not exist -// userStatus = append(userStatus, &user.OnlineStatus{ -// UserID: userID, -// Status: constant.Offline, -// PlatformIDs: nil, -// }) -// -// continue -// } else { -// return nil, errs.Wrap(err) -// } -// } -// err = json.Unmarshal([]byte(result), &onlineStatus) -// if err != nil { -// return nil, errs.Wrap(err) -// } -// onlineStatus.UserID = userID -// onlineStatus.Status = constant.Online -// userStatus = append(userStatus, &onlineStatus) -// } -// -// return userStatus, nil -//} - -// SetUserStatus Set the user status and save it in redis. -//func (u *UserCacheRedis) SetUserStatus(ctx context.Context, userID string, status, platformID int32) error { -// UserIDNum := crc32.ChecksumIEEE([]byte(userID)) -// modKey := strconv.Itoa(int(UserIDNum % statusMod)) -// key := u.getOnlineStatusKey(modKey) -// log.ZDebug(ctx, "SetUserStatus args", "userID", userID, "status", status, "platformID", platformID, "modKey", modKey, "key", key) -// isNewKey, err := u.rdb.Exists(ctx, key).Result() -// if err != nil { -// return errs.Wrap(err) -// } -// if isNewKey == 0 { -// if status == constant.Online { -// onlineStatus := user.OnlineStatus{ -// UserID: userID, -// Status: constant.Online, -// PlatformIDs: []int32{platformID}, -// } -// jsonData, err := json.Marshal(&onlineStatus) -// if err != nil { -// return errs.Wrap(err) -// } -// _, err = u.rdb.HSet(ctx, key, userID, string(jsonData)).Result() -// if err != nil { -// return errs.Wrap(err) -// } -// u.rdb.Expire(ctx, key, userOlineStatusExpireTime) -// -// return nil -// } -// } -// -// isNil := false -// result, err := u.rdb.HGet(ctx, key, userID).Result() -// if err != nil { -// if errors.Is(err, redis.Nil) { -// isNil = true -// } else { -// return errs.Wrap(err) -// } -// } -// -// if status == constant.Offline { -// err = u.refreshStatusOffline(ctx, userID, status, platformID, isNil, err, result, key) -// if err != nil { -// return err -// } -// } else { -// err = u.refreshStatusOnline(ctx, userID, platformID, isNil, err, result, key) -// if err != nil { -// return errs.Wrap(err) -// } -// } -// -// return nil -//} - func (u *UserCacheRedis) refreshStatusOffline(ctx context.Context, userID string, status, platformID int32, isNil bool, err error, result, key string) error { if isNil { log.ZWarn(ctx, "this user not online,maybe trigger order not right", From 9a4f5f78cb03fa16261059a44d3d95d348121d41 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Tue, 9 Jul 2024 14:26:55 +0800 Subject: [PATCH 27/27] sub --- go.mod | 2 + go.sum | 2 - internal/msggateway/client.go | 6 +- internal/msggateway/compressor_test.go | 9 +- internal/msggateway/constant.go | 1 + internal/msggateway/hub_server.go | 10 +- internal/msggateway/init.go | 9 +- internal/msggateway/n_ws_server.go | 32 ++- internal/msggateway/subscription.go | 330 +++++++++++++------------ internal/push/push_handler.go | 2 +- pkg/rpccache/online.go | 5 +- 11 files changed, 222 insertions(+), 186 deletions(-) diff --git a/go.mod b/go.mod index 6450f3615..41ab0a322 100644 --- a/go.mod +++ b/go.mod @@ -175,3 +175,5 @@ require ( golang.org/x/crypto v0.21.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect ) + +replace github.com/openimsdk/protocol => /Users/chao/Desktop/project/protocol diff --git a/go.sum b/go.sum index 577154a1b..92350d745 100644 --- a/go.sum +++ b/go.sum @@ -262,8 +262,6 @@ github.com/onsi/gomega v1.25.0 h1:Vw7br2PCDYijJHSfBOWhov+8cAnUf8MfMaIOV323l6Y= github.com/onsi/gomega v1.25.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdMPGhM= github.com/openimsdk/gomake v0.0.14-alpha.5 h1:VY9c5x515lTfmdhhPjMvR3BBRrRquAUCFsz7t7vbv7Y= github.com/openimsdk/gomake v0.0.14-alpha.5/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= -github.com/openimsdk/protocol v0.0.69-alpha.24 h1:TYcNJeWOTuE40UQ54eNPdDdy0KTOh9rAOgax8lCyhDc= -github.com/openimsdk/protocol v0.0.69-alpha.24/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= github.com/openimsdk/tools v0.0.49-alpha.45 h1:XIzCoef4myybOiIlGuRY9FTtGBisZFC4Uy4PhG0ZWQ0= github.com/openimsdk/tools v0.0.49-alpha.45/go.mod h1:HtSRjPTL8PsuZ+PhR5noqzrYBF0sdwW3/O/sWVucWg8= github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= diff --git a/internal/msggateway/client.go b/internal/msggateway/client.go index 62c71d5be..bed454e49 100644 --- a/internal/msggateway/client.go +++ b/internal/msggateway/client.go @@ -72,8 +72,8 @@ type Client struct { closed atomic.Bool closedErr error token string - //subLock sync.Mutex - //subUserIDs map[string]struct{} + subLock sync.Mutex + subUserIDs map[string]struct{} } // ResetClient updates the client's state with new connection and context information. @@ -204,6 +204,8 @@ func (c *Client) handleMessage(message []byte) error { resp, messageErr = c.longConnServer.UserLogout(ctx, binaryReq) case WsSetBackgroundStatus: resp, messageErr = c.setAppBackgroundStatus(ctx, binaryReq) + case WsSubUserOnlineStatus: + resp, messageErr = c.longConnServer.SubUserOnlineStatus(ctx, c, binaryReq) default: return fmt.Errorf( "ReqIdentifier failed,sendID:%s,msgIncr:%s,reqIdentifier:%d", diff --git a/internal/msggateway/compressor_test.go b/internal/msggateway/compressor_test.go index 173c9bb20..952bd4d95 100644 --- a/internal/msggateway/compressor_test.go +++ b/internal/msggateway/compressor_test.go @@ -16,10 +16,10 @@ package msggateway import ( "crypto/rand" + "github.com/stretchr/testify/assert" "sync" "testing" - - "github.com/stretchr/testify/assert" + "unsafe" ) func mockRandom() []byte { @@ -132,3 +132,8 @@ func BenchmarkDecompressWithSyncPool(b *testing.B) { assert.Equal(b, nil, err) } } + +func TestName(t *testing.T) { + t.Log(unsafe.Sizeof(Client{})) + +} diff --git a/internal/msggateway/constant.go b/internal/msggateway/constant.go index 64664ac0a..bc74ed583 100644 --- a/internal/msggateway/constant.go +++ b/internal/msggateway/constant.go @@ -43,6 +43,7 @@ const ( WSKickOnlineMsg = 2002 WsLogoutMsg = 2003 WsSetBackgroundStatus = 2004 + WsSubUserOnlineStatus = 2005 WSDataError = 3001 ) diff --git a/internal/msggateway/hub_server.go b/internal/msggateway/hub_server.go index 8ff6d1001..3891aa532 100644 --- a/internal/msggateway/hub_server.go +++ b/internal/msggateway/hub_server.go @@ -19,6 +19,7 @@ import ( "github.com/openimsdk/open-im-server/v3/pkg/authverify" "github.com/openimsdk/open-im-server/v3/pkg/common/servererrs" "github.com/openimsdk/open-im-server/v3/pkg/common/startrpc" + "github.com/openimsdk/open-im-server/v3/pkg/rpcclient" "github.com/openimsdk/protocol/constant" "github.com/openimsdk/protocol/msggateway" "github.com/openimsdk/tools/discovery" @@ -31,6 +32,10 @@ import ( func (s *Server) InitServer(ctx context.Context, config *Config, disCov discovery.SvcDiscoveryRegistry, server *grpc.Server) error { s.LongConnServer.SetDiscoveryRegistry(disCov, config) msggateway.RegisterMsgGatewayServer(server, s) + s.userRcp = rpcclient.NewUserRpcClient(disCov, config.Share.RpcRegisterName.User, config.Share.IMAdminUserID) + if s.ready != nil { + return s.ready(s) + } return nil } @@ -50,18 +55,21 @@ type Server struct { LongConnServer LongConnServer config *Config pushTerminal map[int]struct{} + ready func(srv *Server) error + userRcp rpcclient.UserRpcClient } func (s *Server) SetLongConnServer(LongConnServer LongConnServer) { s.LongConnServer = LongConnServer } -func NewServer(rpcPort int, longConnServer LongConnServer, conf *Config) *Server { +func NewServer(rpcPort int, longConnServer LongConnServer, conf *Config, ready func(srv *Server) error) *Server { s := &Server{ rpcPort: rpcPort, LongConnServer: longConnServer, pushTerminal: make(map[int]struct{}), config: conf, + ready: ready, } s.pushTerminal[constant.IOSPlatformID] = struct{}{} s.pushTerminal[constant.AndroidPlatformID] = struct{}{} diff --git a/internal/msggateway/init.go b/internal/msggateway/init.go index 739a71ecb..44e79e412 100644 --- a/internal/msggateway/init.go +++ b/internal/msggateway/init.go @@ -17,6 +17,7 @@ package msggateway import ( "context" "github.com/openimsdk/open-im-server/v3/pkg/common/config" + "github.com/openimsdk/open-im-server/v3/pkg/rpccache" "github.com/openimsdk/tools/db/redisutil" "github.com/openimsdk/tools/utils/datautil" "time" @@ -56,11 +57,13 @@ func Start(ctx context.Context, index int, conf *Config) error { WithMessageMaxMsgLength(conf.MsgGateway.LongConnSvr.WebsocketMaxMsgLen), ) - go longServer.ChangeOnlineStatus(4) + hubServer := NewServer(rpcPort, longServer, conf, func(srv *Server) error { + longServer.online = rpccache.NewOnlineCache(srv.userRcp, nil, rdb, longServer.subscriberUserOnlineStatusChanges) + return nil + }) - go longServer.SubscriberUserOnlineStatusChanges(rdb) + go longServer.ChangeOnlineStatus(4) - hubServer := NewServer(rpcPort, longServer, conf) netDone := make(chan error) go func() { err = hubServer.Start(ctx, index, conf) diff --git a/internal/msggateway/n_ws_server.go b/internal/msggateway/n_ws_server.go index 8a79151fe..e903084a9 100644 --- a/internal/msggateway/n_ws_server.go +++ b/internal/msggateway/n_ws_server.go @@ -18,6 +18,7 @@ import ( "context" "fmt" "github.com/openimsdk/open-im-server/v3/pkg/common/webhook" + "github.com/openimsdk/open-im-server/v3/pkg/rpccache" pbAuth "github.com/openimsdk/protocol/auth" "github.com/openimsdk/tools/mcontext" "net/http" @@ -48,20 +49,22 @@ type LongConnServer interface { KickUserConn(client *Client) error UnRegister(c *Client) SetKickHandlerInfo(i *kickHandler) + SubUserOnlineStatus(ctx context.Context, client *Client, data *Req) ([]byte, error) Compressor Encoder MessageHandler } type WsServer struct { - msgGatewayConfig *Config - port int - wsMaxConnNum int64 - registerChan chan *Client - unregisterChan chan *Client - kickHandlerChan chan *kickHandler - clients UserMap - //subscription *Subscription + msgGatewayConfig *Config + port int + wsMaxConnNum int64 + registerChan chan *Client + unregisterChan chan *Client + kickHandlerChan chan *kickHandler + clients UserMap + online *rpccache.OnlineCache + subscription *Subscription clientPool sync.Pool onlineUserNum atomic.Int64 onlineUserConnNum atomic.Int64 @@ -125,6 +128,8 @@ func NewWsServer(msgGatewayConfig *Config, opts ...Option) *WsServer { for _, o := range opts { o(&config) } + //userRpcClient := rpcclient.NewUserRpcClient(client, config.Share.RpcRegisterName.User, config.Share.IMAdminUserID) + v := validator.New() return &WsServer{ msgGatewayConfig: msgGatewayConfig, @@ -142,10 +147,10 @@ func NewWsServer(msgGatewayConfig *Config, opts ...Option) *WsServer { kickHandlerChan: make(chan *kickHandler, 1000), validate: v, clients: newUserMap(), - //subscription: newSubscription(), - Compressor: NewGzipCompressor(), - Encoder: NewGobEncoder(), - webhookClient: webhook.NewWebhookClient(msgGatewayConfig.WebhooksConfig.URL), + subscription: newSubscription(), + Compressor: NewGzipCompressor(), + Encoder: NewGobEncoder(), + webhookClient: webhook.NewWebhookClient(msgGatewayConfig.WebhooksConfig.URL), } } @@ -353,6 +358,9 @@ func (ws *WsServer) unregisterClient(client *Client) { prommetrics.OnlineUserGauge.Dec() } ws.onlineUserConnNum.Add(-1) + client.subLock.Lock() + clear(client.subUserIDs) + client.subLock.Unlock() //ws.SetUserOnlineStatus(client.ctx, client, constant.Offline) log.ZInfo(client.ctx, "user offline", "close reason", client.closedErr, "online user Num", ws.onlineUserNum.Load(), "online user conn Num", diff --git a/internal/msggateway/subscription.go b/internal/msggateway/subscription.go index 8df548f54..9460f5dbf 100644 --- a/internal/msggateway/subscription.go +++ b/internal/msggateway/subscription.go @@ -2,174 +2,180 @@ package msggateway import ( "context" - "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/cachekey" - "github.com/openimsdk/open-im-server/v3/pkg/util/useronline" + "encoding/json" + "github.com/openimsdk/protocol/constant" + "github.com/openimsdk/protocol/sdkws" "github.com/openimsdk/tools/log" - "github.com/openimsdk/tools/mcontext" - "github.com/redis/go-redis/v9" - "math/rand" - "strconv" + "github.com/openimsdk/tools/utils/datautil" + "github.com/openimsdk/tools/utils/idutil" + "google.golang.org/protobuf/proto" + "sync" + "time" ) -func (ws *WsServer) SubscriberUserOnlineStatusChanges(rdb redis.UniversalClient) { - ctx := mcontext.SetOperationID(context.Background(), cachekey.OnlineChannel+strconv.FormatUint(rand.Uint64(), 10)) - for message := range rdb.Subscribe(ctx, cachekey.OnlineChannel).Channel() { - userID, platformIDs, err := useronline.ParseUserOnlineStatus(message.Payload) - if err != nil { - log.ZError(ctx, "OnlineCache redis subscribe parseUserOnlineStatus", err, "payload", message.Payload, "channel", message.Channel) +func (ws *WsServer) subscriberUserOnlineStatusChanges(ctx context.Context, userID string, platformIDs []int32) { + if ws.clients.RecvSubChange(userID, platformIDs) { + log.ZDebug(ctx, "gateway receive subscription message and go back online", "userID", userID, "platformIDs", platformIDs) + } else { + log.ZDebug(ctx, "gateway ignore user online status changes", "userID", userID, "platformIDs", platformIDs) + } + ws.pushUserIDOnlineStatus(ctx, userID, platformIDs) +} + +func (ws *WsServer) SubUserOnlineStatus(ctx context.Context, client *Client, data *Req) ([]byte, error) { + var sub sdkws.SubUserOnlineStatus + if err := proto.Unmarshal(data.Data, &sub); err != nil { + return nil, err + } + ws.subscription.Sub(client, sub.SubscribeUserID, sub.UnsubscribeUserID) + var resp sdkws.SubUserOnlineStatusTips + if len(sub.SubscribeUserID) > 0 { + resp.Subscribers = make([]*sdkws.SubUserOnlineStatusElem, 0, len(sub.SubscribeUserID)) + for _, userID := range sub.SubscribeUserID { + platformIDs, err := ws.online.GetUserOnlinePlatform(ctx, userID) + if err != nil { + return nil, err + } + resp.Subscribers = append(resp.Subscribers, &sdkws.SubUserOnlineStatusElem{ + UserID: userID, + OnlinePlatformIDs: platformIDs, + }) + } + } + return proto.Marshal(&resp) +} + +type subClient struct { + clients map[string]*Client +} + +func newSubscription() *Subscription { + return &Subscription{ + userIDs: make(map[string]*subClient), + } +} + +type Subscription struct { + lock sync.RWMutex + userIDs map[string]*subClient +} + +func (s *Subscription) GetClient(userID string) []*Client { + s.lock.RLock() + defer s.lock.RUnlock() + cs, ok := s.userIDs[userID] + if !ok { + return nil + } + clients := make([]*Client, 0, len(cs.clients)) + for _, client := range cs.clients { + clients = append(clients, client) + } + return clients +} + +func (s *Subscription) DelClient(client *Client) { + client.subLock.Lock() + userIDs := datautil.Keys(client.subUserIDs) + for _, userID := range userIDs { + delete(client.subUserIDs, userID) + } + client.subLock.Unlock() + if len(userIDs) == 0 { + return + } + addr := client.ctx.GetRemoteAddr() + s.lock.Lock() + defer s.lock.Unlock() + for _, userID := range userIDs { + sub, ok := s.userIDs[userID] + if !ok { continue } - if ws.clients.RecvSubChange(userID, platformIDs) { - log.ZDebug(ctx, "gateway receive subscription message and go back online", "userID", userID, "platformIDs", platformIDs) - } else { - log.ZDebug(ctx, "gateway ignore user online status changes", "userID", userID, "platformIDs", platformIDs) + delete(sub.clients, addr) + if len(sub.clients) == 0 { + delete(s.userIDs, userID) } } } -//import ( -// "context" -// "encoding/json" -// "github.com/openimsdk/protocol/constant" -// "github.com/openimsdk/protocol/sdkws" -// "github.com/openimsdk/tools/log" -// "github.com/openimsdk/tools/utils/datautil" -// "github.com/openimsdk/tools/utils/idutil" -// "sync" -// "time" -//) -// -//type subClient struct { -// clients map[string]*Client -//} -// -//func newSubscription() *Subscription { -// return &Subscription{ -// userIDs: make(map[string]*subClient), -// } -//} -// -//type Subscription struct { -// lock sync.RWMutex -// userIDs map[string]*subClient -//} -// -//func (s *Subscription) GetClient(userID string) []*Client { -// s.lock.RLock() -// defer s.lock.RUnlock() -// cs, ok := s.userIDs[userID] -// if !ok { -// return nil -// } -// clients := make([]*Client, 0, len(cs.clients)) -// for _, client := range cs.clients { -// clients = append(clients, client) -// } -// return clients -//} -// -//func (s *Subscription) DelClient(client *Client) { -// client.subLock.Lock() -// userIDs := datautil.Keys(client.subUserIDs) -// for _, userID := range userIDs { -// delete(client.subUserIDs, userID) -// } -// client.subLock.Unlock() -// if len(userIDs) == 0 { -// return -// } -// addr := client.ctx.GetRemoteAddr() -// s.lock.Lock() -// defer s.lock.Unlock() -// for _, userID := range userIDs { -// sub, ok := s.userIDs[userID] -// if !ok { -// continue -// } -// delete(sub.clients, addr) -// if len(sub.clients) == 0 { -// delete(s.userIDs, userID) -// } -// } -//} -// -//func (s *Subscription) Sub(client *Client, addUserIDs, delUserIDs []string) { -// if len(addUserIDs)+len(delUserIDs) == 0 { -// return -// } -// var ( -// del = make(map[string]struct{}) -// add = make(map[string]struct{}) -// ) -// client.subLock.Lock() -// for _, userID := range delUserIDs { -// if _, ok := client.subUserIDs[userID]; !ok { -// continue -// } -// del[userID] = struct{}{} -// delete(client.subUserIDs, userID) -// } -// for _, userID := range addUserIDs { -// delete(del, userID) -// if _, ok := client.subUserIDs[userID]; ok { -// continue -// } -// client.subUserIDs[userID] = struct{}{} -// } -// client.subLock.Unlock() -// if len(del)+len(add) == 0 { -// return -// } -// addr := client.ctx.GetRemoteAddr() -// s.lock.Lock() -// defer s.lock.Unlock() -// for userID := range del { -// sub, ok := s.userIDs[userID] -// if !ok { -// continue -// } -// delete(sub.clients, addr) -// if len(sub.clients) == 0 { -// delete(s.userIDs, userID) -// } -// } -// for userID := range add { -// sub, ok := s.userIDs[userID] -// if !ok { -// sub = &subClient{clients: make(map[string]*Client)} -// s.userIDs[userID] = sub -// } -// sub.clients[addr] = client -// } -//} -// -//func (ws *WsServer) pushUserIDOnlineStatus(ctx context.Context, userID string, platformIDs []int32) { -// clients := ws.subscription.GetClient(userID) -// if len(clients) == 0 { -// return -// } -// msgContent, err := json.Marshal(platformIDs) -// if err != nil { -// log.ZError(ctx, "pushUserIDOnlineStatus json.Marshal", err) -// return -// } -// now := time.Now().UnixMilli() -// msgID := idutil.GetMsgIDByMD5(userID) -// msg := &sdkws.MsgData{ -// SendID: userID, -// ClientMsgID: msgID, -// ServerMsgID: msgID, -// SenderPlatformID: constant.AdminPlatformID, -// SessionType: constant.NotificationChatType, -// ContentType: constant.UserSubscribeOnlineStatusNotification, -// Content: msgContent, -// SendTime: now, -// CreateTime: now, -// } -// for _, client := range clients { -// msg.RecvID = client.UserID -// if err := client.PushMessage(ctx, msg); err != nil { -// log.ZError(ctx, "UserSubscribeOnlineStatusNotification push failed", err, "userID", client.UserID, "platformID", client.PlatformID, "changeUserID", userID, "content", msgContent) -// } -// } -//} +func (s *Subscription) Sub(client *Client, addUserIDs, delUserIDs []string) { + if len(addUserIDs)+len(delUserIDs) == 0 { + return + } + var ( + del = make(map[string]struct{}) + add = make(map[string]struct{}) + ) + client.subLock.Lock() + for _, userID := range delUserIDs { + if _, ok := client.subUserIDs[userID]; !ok { + continue + } + del[userID] = struct{}{} + delete(client.subUserIDs, userID) + } + for _, userID := range addUserIDs { + delete(del, userID) + if _, ok := client.subUserIDs[userID]; ok { + continue + } + client.subUserIDs[userID] = struct{}{} + } + client.subLock.Unlock() + if len(del)+len(add) == 0 { + return + } + addr := client.ctx.GetRemoteAddr() + s.lock.Lock() + defer s.lock.Unlock() + for userID := range del { + sub, ok := s.userIDs[userID] + if !ok { + continue + } + delete(sub.clients, addr) + if len(sub.clients) == 0 { + delete(s.userIDs, userID) + } + } + for userID := range add { + sub, ok := s.userIDs[userID] + if !ok { + sub = &subClient{clients: make(map[string]*Client)} + s.userIDs[userID] = sub + } + sub.clients[addr] = client + } +} + +func (ws *WsServer) pushUserIDOnlineStatus(ctx context.Context, userID string, platformIDs []int32) { + clients := ws.subscription.GetClient(userID) + if len(clients) == 0 { + return + } + msgContent, err := json.Marshal(platformIDs) + if err != nil { + log.ZError(ctx, "pushUserIDOnlineStatus json.Marshal", err) + return + } + now := time.Now().UnixMilli() + msgID := idutil.GetMsgIDByMD5(userID) + msg := &sdkws.MsgData{ + SendID: userID, + ClientMsgID: msgID, + ServerMsgID: msgID, + SenderPlatformID: constant.AdminPlatformID, + SessionType: constant.NotificationChatType, + ContentType: constant.UserSubscribeOnlineStatusNotification, + Content: msgContent, + SendTime: now, + CreateTime: now, + } + for _, client := range clients { + msg.RecvID = client.UserID + if err := client.PushMessage(ctx, msg); err != nil { + log.ZError(ctx, "UserSubscribeOnlineStatusNotification push failed", err, "userID", client.UserID, "platformID", client.PlatformID, "changeUserID", userID, "content", msgContent) + } + } +} diff --git a/internal/push/push_handler.go b/internal/push/push_handler.go index e942f40ae..ed87b3929 100644 --- a/internal/push/push_handler.go +++ b/internal/push/push_handler.go @@ -75,7 +75,7 @@ func NewConsumerHandler(config *Config, offlinePusher offlinepush.OfflinePusher, consumerHandler.conversationLocalCache = rpccache.NewConversationLocalCache(consumerHandler.conversationRpcClient, &config.LocalCacheConfig, rdb) consumerHandler.webhookClient = webhook.NewWebhookClient(config.WebhooksConfig.URL) consumerHandler.config = config - consumerHandler.onlineCache = rpccache.NewOnlineCache(userRpcClient, consumerHandler.groupLocalCache, rdb) + consumerHandler.onlineCache = rpccache.NewOnlineCache(userRpcClient, consumerHandler.groupLocalCache, rdb, nil) return &consumerHandler, nil } diff --git a/pkg/rpccache/online.go b/pkg/rpccache/online.go index d986651c7..5db68d198 100644 --- a/pkg/rpccache/online.go +++ b/pkg/rpccache/online.go @@ -15,7 +15,7 @@ import ( "time" ) -func NewOnlineCache(user rpcclient.UserRpcClient, group *GroupLocalCache, rdb redis.UniversalClient) *OnlineCache { +func NewOnlineCache(user rpcclient.UserRpcClient, group *GroupLocalCache, rdb redis.UniversalClient, fn func(ctx context.Context, userID string, platformIDs []int32)) *OnlineCache { x := &OnlineCache{ user: user, group: group, @@ -33,6 +33,9 @@ func NewOnlineCache(user rpcclient.UserRpcClient, group *GroupLocalCache, rdb re } storageCache := x.setUserOnline(userID, platformIDs) log.ZDebug(ctx, "OnlineCache setUserOnline", "userID", userID, "platformIDs", platformIDs, "payload", message.Payload, "storageCache", storageCache) + if fn != nil { + fn(ctx, userID, platformIDs) + } } }() return x