From 7f44319b9b882a3559dc9315b271bb8989d56e51 Mon Sep 17 00:00:00 2001 From: chao <48119764+withchao@users.noreply.github.com> Date: Mon, 14 Oct 2024 09:31:44 +0800 Subject: [PATCH 001/199] feat: provide the interface required by js sdk (#2712) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * update gomake version * update gomake version * fix: seq conversion bug * fix: redis pipe exec * fix: ImportFriends * fix: A large number of logs keysAndValues ​​length is not even * feat: mark read aggregate write * feat: online status supports redis cluster * feat: online status supports redis cluster * feat: online status supports redis cluster * merge * merge * read seq is written to mongo * read seq is written to mongo * fix: invitation to join group notification * fix: friend op_user_id * feat: optimizing asynchronous context * feat: optimizing memamq size * feat: add GetSeqMessage * feat: GroupApplicationAgreeMemberEnterNotification * feat: GroupApplicationAgreeMemberEnterNotification * feat: go.mod * feat: go.mod * feat: join group notification and get seq * feat: join group notification and get seq * feat: avoid pulling messages from sessions with a large number of max seq values of 0 * feat: API supports gzip * go.mod * fix: nil pointer error on close * fix: listen error * fix: listen error * update go.mod * feat: add log * fix: token parse token value * fix: GetMsgBySeqs boundary issues * fix: sn_ not sort * fix: sn_ not sort * fix: sn_ not sort * fix: jssdk add * fix: jssdk support * fix: jssdk support * fix: jssdk support --------- Co-authored-by: withchao --- cmd/openim-api/main.go | 3 +- go.mod | 2 +- go.sum | 4 +- internal/api/jssdk/jssdk.go | 122 +++++++++++++++++++++++--------- internal/api/jssdk/stu.go | 22 ------ internal/api/jssdk/tools.go | 57 ++++++++++++++- internal/api/jssdk_test.go | 37 ---------- internal/api/router.go | 2 +- internal/rpc/relation/friend.go | 9 ++- pkg/common/convert/friend.go | 25 ++++--- 10 files changed, 174 insertions(+), 109 deletions(-) delete mode 100644 internal/api/jssdk/stu.go delete mode 100644 internal/api/jssdk_test.go diff --git a/cmd/openim-api/main.go b/cmd/openim-api/main.go index e29ed2a59..3690cfc99 100644 --- a/cmd/openim-api/main.go +++ b/cmd/openim-api/main.go @@ -15,10 +15,9 @@ package main import ( - _ "net/http/pprof" - "github.com/openimsdk/open-im-server/v3/pkg/common/cmd" "github.com/openimsdk/tools/system/program" + _ "net/http/pprof" ) func main() { diff --git a/go.mod b/go.mod index 09c626bc7..b6baca2a1 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/gorilla/websocket v1.5.1 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/mitchellh/mapstructure v1.5.0 - github.com/openimsdk/protocol v0.0.72 + github.com/openimsdk/protocol v0.0.72-alpha.41 github.com/openimsdk/tools v0.0.50-alpha.16 github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 diff --git a/go.sum b/go.sum index 00ecc7ed7..6f5475274 100644 --- a/go.sum +++ b/go.sum @@ -319,8 +319,8 @@ github.com/onsi/gomega v1.25.0 h1:Vw7br2PCDYijJHSfBOWhov+8cAnUf8MfMaIOV323l6Y= github.com/onsi/gomega v1.25.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdMPGhM= github.com/openimsdk/gomake v0.0.14-alpha.5 h1:VY9c5x515lTfmdhhPjMvR3BBRrRquAUCFsz7t7vbv7Y= github.com/openimsdk/gomake v0.0.14-alpha.5/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= -github.com/openimsdk/protocol v0.0.72 h1:K+vslwaR7lDXyBzb07UuEQITaqsgighz7NyXVIWsu6A= -github.com/openimsdk/protocol v0.0.72/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= +github.com/openimsdk/protocol v0.0.72-alpha.41 h1:SMMoTc1iu+wtRqUqmIgqPJFejLgPeauOwoJ4VVG4iMQ= +github.com/openimsdk/protocol v0.0.72-alpha.41/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= github.com/openimsdk/tools v0.0.50-alpha.16 h1:bC1AQvJMuOHtZm8LZRvN8L5mH1Ws2VYdL+TLTs1iGSc= github.com/openimsdk/tools v0.0.50-alpha.16/go.mod h1:h1cYmfyaVtgFbKmb1Cfsl8XwUOMTt8ubVUQrdGtsUh4= github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= diff --git a/internal/api/jssdk/jssdk.go b/internal/api/jssdk/jssdk.go index 7f136c74c..2fcbf5ec6 100644 --- a/internal/api/jssdk/jssdk.go +++ b/internal/api/jssdk/jssdk.go @@ -1,11 +1,15 @@ package jssdk import ( + "context" "github.com/gin-gonic/gin" "github.com/openimsdk/protocol/conversation" + "github.com/openimsdk/protocol/group" + "github.com/openimsdk/protocol/jssdk" "github.com/openimsdk/protocol/msg" + "github.com/openimsdk/protocol/relation" "github.com/openimsdk/protocol/sdkws" - "github.com/openimsdk/tools/a2r" + "github.com/openimsdk/protocol/user" "github.com/openimsdk/tools/mcontext" "github.com/openimsdk/tools/utils/datautil" "sort" @@ -16,16 +20,22 @@ const ( defaultGetActiveConversation = 100 ) -func NewJSSdkApi(msg msg.MsgClient, conv conversation.ConversationClient) *JSSdk { +func NewJSSdkApi(user user.UserClient, friend relation.FriendClient, group group.GroupClient, msg msg.MsgClient, conv conversation.ConversationClient) *JSSdk { return &JSSdk{ - msg: msg, - conv: conv, + user: user, + friend: friend, + group: group, + msg: msg, + conv: conv, } } type JSSdk struct { - msg msg.MsgClient - conv conversation.ConversationClient + user user.UserClient + friend relation.FriendClient + group group.GroupClient + msg msg.MsgClient + conv conversation.ConversationClient } func (x *JSSdk) GetActiveConversations(c *gin.Context) { @@ -36,25 +46,71 @@ func (x *JSSdk) GetConversations(c *gin.Context) { call(c, x.getConversations) } -func (x *JSSdk) getActiveConversations(ctx *gin.Context) (*ConversationsResp, error) { - req, err := a2r.ParseRequest[ActiveConversationsReq](ctx) - if err != nil { - return nil, err +func (x *JSSdk) fillConversations(ctx context.Context, conversations []*jssdk.ConversationMsg) error { + if len(conversations) == 0 { + return nil + } + var ( + userIDs []string + groupIDs []string + ) + for _, c := range conversations { + if c.Conversation.GroupID == "" { + userIDs = append(userIDs, c.Conversation.UserID) + } else { + groupIDs = append(groupIDs, c.Conversation.GroupID) + } + } + var ( + userMap map[string]*sdkws.UserInfo + friendMap map[string]*relation.FriendInfoOnly + groupMap map[string]*sdkws.GroupInfo + ) + if len(userIDs) > 0 { + users, err := field(ctx, x.user.GetDesignateUsers, &user.GetDesignateUsersReq{UserIDs: userIDs}, (*user.GetDesignateUsersResp).GetUsersInfo) + if err != nil { + return err + } + friends, err := field(ctx, x.friend.GetFriendInfo, &relation.GetFriendInfoReq{OwnerUserID: conversations[0].Conversation.OwnerUserID, FriendUserIDs: userIDs}, (*relation.GetFriendInfoResp).GetFriendInfos) + if err != nil { + return err + } + userMap = datautil.SliceToMap(users, (*sdkws.UserInfo).GetUserID) + friendMap = datautil.SliceToMap(friends, (*relation.FriendInfoOnly).GetFriendUserID) + } + if len(groupIDs) > 0 { + resp, err := x.group.GetGroupsInfo(ctx, &group.GetGroupsInfoReq{GroupIDs: groupIDs}) + if err != nil { + return err + } + groupMap = datautil.SliceToMap(resp.GroupInfos, (*sdkws.GroupInfo).GetGroupID) } + for _, c := range conversations { + if c.Conversation.GroupID == "" { + c.User = userMap[c.Conversation.UserID] + c.Friend = friendMap[c.Conversation.UserID] + } else { + c.Group = groupMap[c.Conversation.GroupID] + } + } + return nil +} + +func (x *JSSdk) getActiveConversations(ctx context.Context, req *jssdk.GetActiveConversationsReq) (*jssdk.GetActiveConversationsResp, error) { if req.Count <= 0 || req.Count > maxGetActiveConversation { req.Count = defaultGetActiveConversation } - opUserID := mcontext.GetOpUserID(ctx) + req.OwnerUserID = mcontext.GetOpUserID(ctx) conversationIDs, err := field(ctx, x.conv.GetConversationIDs, - &conversation.GetConversationIDsReq{UserID: opUserID}, (*conversation.GetConversationIDsResp).GetConversationIDs) + &conversation.GetConversationIDsReq{UserID: req.OwnerUserID}, (*conversation.GetConversationIDsResp).GetConversationIDs) if err != nil { return nil, err } if len(conversationIDs) == 0 { - return &ConversationsResp{}, nil + return &jssdk.GetActiveConversationsResp{}, nil } readSeq, err := field(ctx, x.msg.GetHasReadSeqs, - &msg.GetHasReadSeqsReq{UserID: opUserID, ConversationIDs: conversationIDs}, (*msg.SeqsInfoResp).GetMaxSeqs) + &msg.GetHasReadSeqsReq{UserID: req.OwnerUserID, ConversationIDs: conversationIDs}, (*msg.SeqsInfoResp).GetMaxSeqs) if err != nil { return nil, err } @@ -64,24 +120,24 @@ func (x *JSSdk) getActiveConversations(ctx *gin.Context) (*ConversationsResp, er return nil, err } if len(activeConversation) == 0 { - return &ConversationsResp{}, nil + return &jssdk.GetActiveConversationsResp{}, nil } sortConversations := sortActiveConversations{ Conversation: activeConversation, } if len(activeConversation) > 1 { pinnedConversationIDs, err := field(ctx, x.conv.GetPinnedConversationIDs, - &conversation.GetPinnedConversationIDsReq{UserID: opUserID}, (*conversation.GetPinnedConversationIDsResp).GetConversationIDs) + &conversation.GetPinnedConversationIDsReq{UserID: req.OwnerUserID}, (*conversation.GetPinnedConversationIDsResp).GetConversationIDs) if err != nil { return nil, err } sortConversations.PinnedConversationIDs = datautil.SliceSet(pinnedConversationIDs) } sort.Sort(&sortConversations) - sortList := sortConversations.Top(req.Count) + sortList := sortConversations.Top(int(req.Count)) conversations, err := field(ctx, x.conv.GetConversations, &conversation.GetConversationsReq{ - OwnerUserID: opUserID, + OwnerUserID: req.OwnerUserID, ConversationIDs: datautil.Slice(sortList, func(c *msg.ActiveConversation) string { return c.ConversationID })}, (*conversation.GetConversationsResp).GetConversations) @@ -90,7 +146,7 @@ func (x *JSSdk) getActiveConversations(ctx *gin.Context) (*ConversationsResp, er } msgs, err := field(ctx, x.msg.GetSeqMessage, &msg.GetSeqMessageReq{ - UserID: opUserID, + UserID: req.OwnerUserID, Conversations: datautil.Slice(sortList, func(c *msg.ActiveConversation) *msg.ConversationSeqs { return &msg.ConversationSeqs{ ConversationID: c.ConversationID, @@ -104,7 +160,7 @@ func (x *JSSdk) getActiveConversations(ctx *gin.Context) (*ConversationsResp, er conversationMap := datautil.SliceToMap(conversations, func(c *conversation.Conversation) string { return c.ConversationID }) - resp := make([]ConversationMsg, 0, len(sortList)) + resp := make([]*jssdk.ConversationMsg, 0, len(sortList)) for _, c := range sortList { conv, ok := conversationMap[c.ConversationID] if !ok { @@ -114,13 +170,16 @@ func (x *JSSdk) getActiveConversations(ctx *gin.Context) (*ConversationsResp, er if msgList, ok := msgs[c.ConversationID]; ok && len(msgList.Msgs) > 0 { lastMsg = msgList.Msgs[0] } - resp = append(resp, ConversationMsg{ + resp = append(resp, &jssdk.ConversationMsg{ Conversation: conv, LastMsg: lastMsg, MaxSeq: c.MaxSeq, ReadSeq: readSeq[c.ConversationID], }) } + if err := x.fillConversations(ctx, resp); err != nil { + return nil, err + } var unreadCount int64 for _, c := range activeConversation { count := c.MaxSeq - readSeq[c.ConversationID] @@ -128,24 +187,20 @@ func (x *JSSdk) getActiveConversations(ctx *gin.Context) (*ConversationsResp, er unreadCount += count } } - return &ConversationsResp{ + return &jssdk.GetActiveConversationsResp{ Conversations: resp, UnreadCount: unreadCount, }, nil } -func (x *JSSdk) getConversations(ctx *gin.Context) (*ConversationsResp, error) { - req, err := a2r.ParseRequest[conversation.GetConversationsReq](ctx) - if err != nil { - return nil, err - } +func (x *JSSdk) getConversations(ctx context.Context, req *jssdk.GetConversationsReq) (*jssdk.GetConversationsResp, error) { req.OwnerUserID = mcontext.GetOpUserID(ctx) - conversations, err := field(ctx, x.conv.GetConversations, req, (*conversation.GetConversationsResp).GetConversations) + conversations, err := field(ctx, x.conv.GetConversations, &conversation.GetConversationsReq{OwnerUserID: req.OwnerUserID, ConversationIDs: req.ConversationIDs}, (*conversation.GetConversationsResp).GetConversations) if err != nil { return nil, err } if len(conversations) == 0 { - return &ConversationsResp{}, nil + return &jssdk.GetConversationsResp{}, nil } req.ConversationIDs = datautil.Slice(conversations, func(c *conversation.Conversation) string { return c.ConversationID @@ -177,19 +232,22 @@ func (x *JSSdk) getConversations(ctx *gin.Context) (*ConversationsResp, error) { return nil, err } } - resp := make([]ConversationMsg, 0, len(conversations)) + resp := make([]*jssdk.ConversationMsg, 0, len(conversations)) for _, c := range conversations { var lastMsg *sdkws.MsgData if msgList, ok := msgs[c.ConversationID]; ok && len(msgList.Msgs) > 0 { lastMsg = msgList.Msgs[0] } - resp = append(resp, ConversationMsg{ + resp = append(resp, &jssdk.ConversationMsg{ Conversation: c, LastMsg: lastMsg, MaxSeq: maxSeqs[c.ConversationID], ReadSeq: readSeqs[c.ConversationID], }) } + if err := x.fillConversations(ctx, resp); err != nil { + return nil, err + } var unreadCount int64 for conversationID, maxSeq := range maxSeqs { count := maxSeq - readSeqs[conversationID] @@ -197,7 +255,7 @@ func (x *JSSdk) getConversations(ctx *gin.Context) (*ConversationsResp, error) { unreadCount += count } } - return &ConversationsResp{ + return &jssdk.GetConversationsResp{ Conversations: resp, UnreadCount: unreadCount, }, nil diff --git a/internal/api/jssdk/stu.go b/internal/api/jssdk/stu.go deleted file mode 100644 index 2f63975b3..000000000 --- a/internal/api/jssdk/stu.go +++ /dev/null @@ -1,22 +0,0 @@ -package jssdk - -import ( - "github.com/openimsdk/protocol/conversation" - "github.com/openimsdk/protocol/sdkws" -) - -type ActiveConversationsReq struct { - Count int `json:"count"` -} - -type ConversationMsg struct { - Conversation *conversation.Conversation `json:"conversation"` - LastMsg *sdkws.MsgData `json:"lastMsg"` - MaxSeq int64 `json:"maxSeq"` - ReadSeq int64 `json:"readSeq"` -} - -type ConversationsResp struct { - UnreadCount int64 `json:"unreadCount"` - Conversations []ConversationMsg `json:"conversations"` -} diff --git a/internal/api/jssdk/tools.go b/internal/api/jssdk/tools.go index c57457d9f..c19d8970b 100644 --- a/internal/api/jssdk/tools.go +++ b/internal/api/jssdk/tools.go @@ -3,8 +3,14 @@ package jssdk import ( "context" "github.com/gin-gonic/gin" + "github.com/openimsdk/tools/a2r" "github.com/openimsdk/tools/apiresp" + "github.com/openimsdk/tools/checker" + "github.com/openimsdk/tools/errs" "google.golang.org/grpc" + "google.golang.org/protobuf/proto" + "io" + "strings" ) func field[A, B, C any](ctx context.Context, fn func(ctx context.Context, req *A, opts ...grpc.CallOption) (*B, error), req *A, get func(*B) C) (C, error) { @@ -16,11 +22,56 @@ func field[A, B, C any](ctx context.Context, fn func(ctx context.Context, req *A return get(resp), nil } -func call[R any](c *gin.Context, fn func(ctx *gin.Context) (R, error)) { - resp, err := fn(c) +func call[A, B any](c *gin.Context, fn func(ctx context.Context, req *A) (*B, error)) { + var isJSON bool + switch contentType := c.GetHeader("Content-Type"); { + case contentType == "": + isJSON = true + case strings.Contains(contentType, "application/json"): + isJSON = true + case strings.Contains(contentType, "application/protobuf"): + case strings.Contains(contentType, "application/x-protobuf"): + default: + apiresp.GinError(c, errs.ErrArgs.WrapMsg("unsupported content type")) + return + } + var req *A + if isJSON { + var err error + req, err = a2r.ParseRequest[A](c) + if err != nil { + apiresp.GinError(c, err) + return + } + } else { + body, err := io.ReadAll(c.Request.Body) + if err != nil { + apiresp.GinError(c, err) + return + } + req = new(A) + if err := proto.Unmarshal(body, any(req).(proto.Message)); err != nil { + apiresp.GinError(c, err) + return + } + if err := checker.Validate(&req); err != nil { + apiresp.GinError(c, err) + return + } + } + resp, err := fn(c, req) + if err != nil { + apiresp.GinError(c, err) + return + } + if isJSON { + apiresp.GinSuccess(c, resp) + return + } + body, err := proto.Marshal(any(resp).(proto.Message)) if err != nil { apiresp.GinError(c, err) return } - apiresp.GinSuccess(c, resp) + apiresp.GinSuccess(c, body) } diff --git a/internal/api/jssdk_test.go b/internal/api/jssdk_test.go deleted file mode 100644 index 472ca56b5..000000000 --- a/internal/api/jssdk_test.go +++ /dev/null @@ -1,37 +0,0 @@ -package api - -import ( - "github.com/openimsdk/protocol/msg" - "sort" - "testing" -) - -func TestName(t *testing.T) { - val := sortActiveConversations{ - Conversation: []*msg.ActiveConversation{ - { - ConversationID: "100", - LastTime: 100, - }, - { - ConversationID: "200", - LastTime: 200, - }, - { - ConversationID: "300", - LastTime: 300, - }, - { - ConversationID: "400", - LastTime: 400, - }, - }, - //PinnedConversationIDs: map[string]struct{}{ - // "100": {}, - // "300": {}, - //}, - } - sort.Sort(&val) - t.Log(val) - -} diff --git a/internal/api/router.go b/internal/api/router.go index f87ec526c..560516d30 100644 --- a/internal/api/router.go +++ b/internal/api/router.go @@ -77,7 +77,7 @@ func newGinRouter(disCov discovery.SvcDiscoveryRegistry, config *Config) *gin.En r.Use(prommetricsGin(), gin.Recovery(), mw.CorsHandler(), mw.GinParseOperationID(), GinParseToken(authRpc)) u := NewUserApi(*userRpc) m := NewMessageApi(messageRpc, userRpc, config.Share.IMAdminUserID) - j := jssdk.NewJSSdkApi(messageRpc.Client, conversationRpc.Client) + j := jssdk.NewJSSdkApi(userRpc.Client, friendRpc.Client, groupRpc.Client, messageRpc.Client, conversationRpc.Client) userRouterGroup := r.Group("/user") { userRouterGroup.POST("/user_register", u.UserRegister) diff --git a/internal/rpc/relation/friend.go b/internal/rpc/relation/friend.go index 9d55ba4d9..2f4843a8e 100644 --- a/internal/rpc/relation/friend.go +++ b/internal/rpc/relation/friend.go @@ -273,7 +273,14 @@ func (s *friendServer) SetFriendRemark(ctx context.Context, req *relation.SetFri return &relation.SetFriendRemarkResp{}, nil } -// ok. +func (s *friendServer) GetFriendInfo(ctx context.Context, req *relation.GetFriendInfoReq) (*relation.GetFriendInfoResp, error) { + friends, err := s.db.FindFriendsWithError(ctx, req.OwnerUserID, req.FriendUserIDs) + if err != nil { + return nil, err + } + return &relation.GetFriendInfoResp{FriendInfos: convert.FriendOnlyDB2PbOnly(friends)}, nil +} + func (s *friendServer) GetDesignatedFriends(ctx context.Context, req *relation.GetDesignatedFriendsReq) (resp *relation.GetDesignatedFriendsResp, err error) { resp = &relation.GetDesignatedFriendsResp{} if datautil.Duplicate(req.FriendUserIDs) { diff --git a/pkg/common/convert/friend.go b/pkg/common/convert/friend.go index 8d6cfad18..6d346b0f4 100644 --- a/pkg/common/convert/friend.go +++ b/pkg/common/convert/friend.go @@ -18,6 +18,7 @@ import ( "context" "fmt" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" + "github.com/openimsdk/protocol/relation" "github.com/openimsdk/protocol/sdkws" "github.com/openimsdk/tools/utils/datautil" @@ -35,9 +36,7 @@ func FriendPb2DB(friend *sdkws.FriendInfo) *model.Friend { return dbFriend } -func FriendDB2Pb(ctx context.Context, friendDB *model.Friend, - getUsers func(ctx context.Context, userIDs []string) (map[string]*sdkws.UserInfo, error), -) (*sdkws.FriendInfo, error) { +func FriendDB2Pb(ctx context.Context, friendDB *model.Friend, getUsers func(ctx context.Context, userIDs []string) (map[string]*sdkws.UserInfo, error)) (*sdkws.FriendInfo, error) { users, err := getUsers(ctx, []string{friendDB.FriendUserID}) if err != nil { return nil, err @@ -53,11 +52,7 @@ func FriendDB2Pb(ctx context.Context, friendDB *model.Friend, }, nil } -func FriendsDB2Pb( - ctx context.Context, - friendsDB []*model.Friend, - getUsers func(ctx context.Context, userIDs []string) (map[string]*sdkws.UserInfo, error), -) (friendsPb []*sdkws.FriendInfo, err error) { +func FriendsDB2Pb(ctx context.Context, friendsDB []*model.Friend, getUsers func(ctx context.Context, userIDs []string) (map[string]*sdkws.UserInfo, error)) (friendsPb []*sdkws.FriendInfo, err error) { if len(friendsDB) == 0 { return nil, nil } @@ -86,7 +81,21 @@ func FriendsDB2Pb( friendsPb = append(friendsPb, friendPb) } return friendsPb, nil +} +func FriendOnlyDB2PbOnly(friendsDB []*model.Friend) []*relation.FriendInfoOnly { + return datautil.Slice(friendsDB, func(f *model.Friend) *relation.FriendInfoOnly { + return &relation.FriendInfoOnly{ + OwnerUserID: f.OwnerUserID, + FriendUserID: f.FriendUserID, + Remark: f.Remark, + CreateTime: f.CreateTime.UnixMilli(), + AddSource: f.AddSource, + OperatorUserID: f.OperatorUserID, + Ex: f.Ex, + IsPinned: f.IsPinned, + } + }) } func FriendRequestDB2Pb(ctx context.Context, friendRequests []*model.FriendRequest, getUsers func(ctx context.Context, userIDs []string) (map[string]*sdkws.UserInfo, error)) ([]*sdkws.FriendRequest, error) { From 87f79d3cee0a3b73491386b3a4bf6252f9a2326a Mon Sep 17 00:00:00 2001 From: icey-yu <119291641+icey-yu@users.noreply.github.com> Date: Mon, 14 Oct 2024 11:02:56 +0800 Subject: [PATCH 002/199] Line webhook (#2716) * feat: online and offline webhook * feat: online and offline webhook * feat: remove zk --- config/discovery.yml | 5 ----- docker-compose.yml | 14 +------------- internal/msggateway/online.go | 13 +++++++++++++ 3 files changed, 14 insertions(+), 18 deletions(-) diff --git a/config/discovery.yml b/config/discovery.yml index 78a36f3d1..6e68cbff4 100644 --- a/config/discovery.yml +++ b/config/discovery.yml @@ -5,9 +5,4 @@ etcd: username: '' password: '' -zookeeper: - schema: openim - address: [ localhost:12181 ] - username: '' - password: '' diff --git a/docker-compose.yml b/docker-compose.yml index edac65b13..6d88bac10 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -43,19 +43,6 @@ services: networks: - openim - zookeeper: - image: "${ZOOKEEPER_IMAGE}" - container_name: zookeeper - ports: - - "12181:2181" - environment: - #JVMFLAGS: "-Xms32m -Xmx128m" - TZ: "Asia/Shanghai" - ALLOW_ANONYMOUS_LOGIN: "yes" - restart: always - networks: - - openim - etcd: image: "${ETCD_IMAGE}" container_name: etcd @@ -142,6 +129,7 @@ services: # image: ${PROMETHEUS_IMAGE} # container_name: prometheus # restart: always +# user: root # volumes: # - ./config/prometheus.yml:/etc/prometheus/prometheus.yml # - ./config/instance-down-rules.yml:/etc/prometheus/instance-down-rules.yml diff --git a/internal/msggateway/online.go b/internal/msggateway/online.go index 27b4544aa..f29869b6e 100644 --- a/internal/msggateway/online.go +++ b/internal/msggateway/online.go @@ -90,6 +90,19 @@ func (ws *WsServer) ChangeOnlineStatus(concurrent int) { if _, err := ws.userClient.Client.SetUserOnlineStatus(ctx, req); err != nil { log.ZError(ctx, "update user online status", err) } + for _, ss := range req.Status { + for _, online := range ss.Online { + client, _, _ := ws.clients.Get(ss.UserID, int(online)) + back := false + if len(client) > 0 { + back = client[0].IsBackground + } + ws.webhookAfterUserOnline(ctx, &ws.msgGatewayConfig.WebhooksConfig.AfterUserOnline, ss.UserID, int(online), back, ss.ConnID) + } + for _, offline := range ss.Offline { + ws.webhookAfterUserOffline(ctx, &ws.msgGatewayConfig.WebhooksConfig.AfterUserOffline, ss.UserID, int(offline), ss.ConnID) + } + } } for i := 0; i < concurrent; i++ { From 598750e8c7f14b235d0a3ba9c582c4b3cb4994d2 Mon Sep 17 00:00:00 2001 From: chao <48119764+withchao@users.noreply.github.com> Date: Mon, 14 Oct 2024 11:08:00 +0800 Subject: [PATCH 003/199] fix: the message I sent is not set to read seq in mongodb (#2718) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * update gomake version * update gomake version * fix: seq conversion bug * fix: redis pipe exec * fix: ImportFriends * fix: A large number of logs keysAndValues ​​length is not even * feat: mark read aggregate write * feat: online status supports redis cluster * feat: online status supports redis cluster * feat: online status supports redis cluster * merge * merge * read seq is written to mongo * read seq is written to mongo * fix: invitation to join group notification * fix: friend op_user_id * feat: optimizing asynchronous context * feat: optimizing memamq size * feat: add GetSeqMessage * feat: GroupApplicationAgreeMemberEnterNotification * feat: GroupApplicationAgreeMemberEnterNotification * feat: go.mod * feat: go.mod * feat: join group notification and get seq * feat: join group notification and get seq * feat: avoid pulling messages from sessions with a large number of max seq values of 0 * feat: API supports gzip * go.mod * fix: nil pointer error on close * fix: listen error * fix: listen error * update go.mod * feat: add log * fix: token parse token value * fix: GetMsgBySeqs boundary issues * fix: sn_ not sort * fix: sn_ not sort * fix: sn_ not sort * fix: jssdk add * fix: jssdk support * fix: jssdk support * fix: jssdk support * fix: the message I sent is not set to read seq in mongodb --------- Co-authored-by: withchao --- internal/push/a_test.go | 29 ------------------- pkg/common/storage/controller/msg_transfer.go | 2 +- 2 files changed, 1 insertion(+), 30 deletions(-) delete mode 100644 internal/push/a_test.go diff --git a/internal/push/a_test.go b/internal/push/a_test.go deleted file mode 100644 index 8b2d86407..000000000 --- a/internal/push/a_test.go +++ /dev/null @@ -1,29 +0,0 @@ -package push - -import ( - "github.com/openimsdk/protocol/sdkws" - "testing" -) - -func TestName(t *testing.T) { - var c ConsumerHandler - c.readCh = make(chan *sdkws.MarkAsReadTips) - - go c.loopRead() - - go func() { - for i := 0; ; i++ { - seq := int64(i + 1) - if seq%3 == 0 { - seq = 1 - } - c.readCh <- &sdkws.MarkAsReadTips{ - ConversationID: "c100", - MarkAsReadUserID: "u100", - HasReadSeq: seq, - } - } - }() - - select {} -} diff --git a/pkg/common/storage/controller/msg_transfer.go b/pkg/common/storage/controller/msg_transfer.go index 5e540a2c3..c5dfd011f 100644 --- a/pkg/common/storage/controller/msg_transfer.go +++ b/pkg/common/storage/controller/msg_transfer.go @@ -254,7 +254,7 @@ func (db *msgTransferDatabase) BatchInsertChat2Cache(ctx context.Context, conver func (db *msgTransferDatabase) setHasReadSeqs(ctx context.Context, conversationID string, userSeqMap map[string]int64) error { for userID, seq := range userSeqMap { - if err := db.seqUser.SetUserReadSeq(ctx, conversationID, userID, seq); err != nil { + if err := db.seqUser.SetUserReadSeqToDB(ctx, conversationID, userID, seq); err != nil { return err } } From 3167f9943f699e0418c36d87a5fb9e2432ff08d6 Mon Sep 17 00:00:00 2001 From: chao <48119764+withchao@users.noreply.github.com> Date: Mon, 14 Oct 2024 11:43:51 +0800 Subject: [PATCH 004/199] fix: cannot modify group member avatars (#2719) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * update gomake version * update gomake version * fix: seq conversion bug * fix: redis pipe exec * fix: ImportFriends * fix: A large number of logs keysAndValues ​​length is not even * feat: mark read aggregate write * feat: online status supports redis cluster * feat: online status supports redis cluster * feat: online status supports redis cluster * merge * merge * read seq is written to mongo * read seq is written to mongo * fix: invitation to join group notification * fix: friend op_user_id * feat: optimizing asynchronous context * feat: optimizing memamq size * feat: add GetSeqMessage * feat: GroupApplicationAgreeMemberEnterNotification * feat: GroupApplicationAgreeMemberEnterNotification * feat: go.mod * feat: go.mod * feat: join group notification and get seq * feat: join group notification and get seq * feat: avoid pulling messages from sessions with a large number of max seq values of 0 * feat: API supports gzip * go.mod * fix: nil pointer error on close * fix: listen error * fix: listen error * update go.mod * feat: add log * fix: token parse token value * fix: GetMsgBySeqs boundary issues * fix: sn_ not sort * fix: sn_ not sort * fix: sn_ not sort * fix: jssdk add * fix: jssdk support * fix: jssdk support * fix: jssdk support * fix: the message I sent is not set to read seq in mongodb * fix: cannot modify group member avatars --------- Co-authored-by: withchao --- internal/rpc/group/group.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/internal/rpc/group/group.go b/internal/rpc/group/group.go index 80d1c9b2f..fce33de6f 100644 --- a/internal/rpc/group/group.go +++ b/internal/rpc/group/group.go @@ -1485,9 +1485,6 @@ func (g *groupServer) SetGroupMemberInfo(ctx context.Context, req *pbgroup.SetGr return nil, errs.ErrNoPermission.WrapMsg("no op user id") } isAppManagerUid := authverify.IsAppManagerUid(ctx, g.config.Share.IMAdminUserID) - for i := range req.Members { - req.Members[i].FaceURL = nil - } groupMembers := make(map[string][]*pbgroup.SetGroupMemberInfo) for i, member := range req.Members { if member.RoleLevel != nil { From e7c7bf3bd1c055455910584927cde18bd04b3a08 Mon Sep 17 00:00:00 2001 From: liangkai Date: Tue, 15 Oct 2024 14:47:34 +0800 Subject: [PATCH 005/199] fix: auth package import twice (#2724) --- pkg/rpcclient/auth.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pkg/rpcclient/auth.go b/pkg/rpcclient/auth.go index 6665936bd..fead624a3 100644 --- a/pkg/rpcclient/auth.go +++ b/pkg/rpcclient/auth.go @@ -16,8 +16,8 @@ package rpcclient import ( "context" + "github.com/openimsdk/protocol/auth" - pbAuth "github.com/openimsdk/protocol/auth" "github.com/openimsdk/tools/discovery" "github.com/openimsdk/tools/system/program" "google.golang.org/grpc" @@ -38,8 +38,8 @@ type Auth struct { discov discovery.SvcDiscoveryRegistry } -func (a *Auth) ParseToken(ctx context.Context, token string) (*pbAuth.ParseTokenResp, error) { - req := pbAuth.ParseTokenReq{ +func (a *Auth) ParseToken(ctx context.Context, token string) (*auth.ParseTokenResp, error) { + req := auth.ParseTokenReq{ Token: token, } resp, err := a.Client.ParseToken(ctx, &req) @@ -49,8 +49,8 @@ func (a *Auth) ParseToken(ctx context.Context, token string) (*pbAuth.ParseToken return resp, err } -func (a *Auth) InvalidateToken(ctx context.Context, preservedToken, userID string, platformID int) (*pbAuth.InvalidateTokenResp, error) { - req := pbAuth.InvalidateTokenReq{ +func (a *Auth) InvalidateToken(ctx context.Context, preservedToken, userID string, platformID int) (*auth.InvalidateTokenResp, error) { + req := auth.InvalidateTokenReq{ PreservedToken: preservedToken, UserID: userID, PlatformID: int32(platformID), From 0b612c13c64e6f983758c056892cc5a6d0d8a171 Mon Sep 17 00:00:00 2001 From: chao <48119764+withchao@users.noreply.github.com> Date: Wed, 23 Oct 2024 17:59:37 +0800 Subject: [PATCH 006/199] fix: join the group chat directly, notification type error (#2772) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * update gomake version * update gomake version * fix: seq conversion bug * fix: redis pipe exec * fix: ImportFriends * fix: A large number of logs keysAndValues ​​length is not even * feat: mark read aggregate write * feat: online status supports redis cluster * feat: online status supports redis cluster * feat: online status supports redis cluster * merge * merge * read seq is written to mongo * read seq is written to mongo * fix: invitation to join group notification * fix: friend op_user_id * feat: optimizing asynchronous context * feat: optimizing memamq size * feat: add GetSeqMessage * feat: GroupApplicationAgreeMemberEnterNotification * feat: GroupApplicationAgreeMemberEnterNotification * feat: go.mod * feat: go.mod * feat: join group notification and get seq * feat: join group notification and get seq * feat: avoid pulling messages from sessions with a large number of max seq values of 0 * feat: API supports gzip * go.mod * fix: nil pointer error on close * fix: listen error * fix: listen error * update go.mod * feat: add log * fix: token parse token value * fix: GetMsgBySeqs boundary issues * fix: sn_ not sort * fix: sn_ not sort * fix: sn_ not sort * fix: jssdk add * fix: jssdk support * fix: jssdk support * fix: jssdk support * fix: the message I sent is not set to read seq in mongodb * fix: cannot modify group member avatars * fix: MemberEnterNotification * fix: MemberEnterNotification --------- Co-authored-by: withchao --- internal/rpc/group/group.go | 2 +- internal/rpc/group/notification.go | 48 ++++++++++++++++++++++++++++-- 2 files changed, 47 insertions(+), 3 deletions(-) diff --git a/internal/rpc/group/group.go b/internal/rpc/group/group.go index fce33de6f..c3ee0d3d5 100644 --- a/internal/rpc/group/group.go +++ b/internal/rpc/group/group.go @@ -465,7 +465,7 @@ func (g *groupServer) InviteUserToGroup(ctx context.Context, req *pbgroup.Invite return nil, err } - if err = g.notification.MemberEnterNotification(ctx, req.GroupID, req.InvitedUserIDs...); err != nil { + if err = g.notification.GroupApplicationAgreeMemberEnterNotification(ctx, req.GroupID, opUserID, req.InvitedUserIDs...); err != nil { return nil, err } return &pbgroup.InviteUserToGroupResp{}, nil diff --git a/internal/rpc/group/notification.go b/internal/rpc/group/notification.go index 64e922fe2..54a6146f5 100644 --- a/internal/rpc/group/notification.go +++ b/internal/rpc/group/notification.go @@ -38,6 +38,7 @@ import ( "github.com/openimsdk/tools/utils/datautil" "github.com/openimsdk/tools/utils/stringutil" "go.mongodb.org/mongo-driver/mongo" + "time" ) // GroupApplicationReceiver @@ -572,8 +573,51 @@ func (g *GroupNotificationSender) GroupApplicationAgreeMemberEnterNotification(c return nil } -func (g *GroupNotificationSender) MemberEnterNotification(ctx context.Context, groupID string, entrantUserID ...string) error { - return g.GroupApplicationAgreeMemberEnterNotification(ctx, groupID, "", entrantUserID...) +func (g *GroupNotificationSender) MemberEnterNotification(ctx context.Context, groupID string, entrantUserID string) error { + var err error + defer func() { + if err != nil { + log.ZError(ctx, stringutil.GetFuncName(1)+" failed", err) + } + }() + + if !g.config.RpcConfig.EnableHistoryForNewMembers { + conversationID := msgprocessor.GetConversationIDBySessionType(constant.ReadGroupChatType, groupID) + maxSeq, err := g.msgRpcClient.GetConversationMaxSeq(ctx, conversationID) + if err != nil { + return err + } + if _, err = g.msgRpcClient.SetUserConversationsMinSeq(ctx, &msg.SetUserConversationsMinSeqReq{ + UserIDs: []string{entrantUserID}, + ConversationID: conversationID, + Seq: maxSeq, + }); err != nil { + return err + } + } + + if err := g.conversationRpcClient.GroupChatFirstCreateConversation(ctx, groupID, []string{entrantUserID}); err != nil { + return err + } + + var group *sdkws.GroupInfo + group, err = g.getGroupInfo(ctx, groupID) + if err != nil { + return err + } + user, err := g.getGroupMember(ctx, groupID, entrantUserID) + if err != nil { + return err + } + + tips := &sdkws.MemberEnterTips{ + Group: group, + EntrantUser: user, + OperationTime: time.Now().UnixMilli(), + } + g.setVersion(ctx, &tips.GroupMemberVersion, &tips.GroupMemberVersionID, database.GroupMemberVersionName, tips.Group.GroupID) + g.Notification(ctx, mcontext.GetOpUserID(ctx), group.GroupID, constant.MemberEnterNotification, tips) + return nil } func (g *GroupNotificationSender) GroupDismissedNotification(ctx context.Context, tips *sdkws.GroupDismissedTips) { From a2110e416aff254bcca6f8b74f86ee59e348529c Mon Sep 17 00:00:00 2001 From: icey-yu <119291641+icey-yu@users.noreply.github.com> Date: Thu, 24 Oct 2024 14:59:33 +0800 Subject: [PATCH 007/199] fix: group level change logic (#2730) --- internal/rpc/group/group.go | 72 ++++++++++++++++++++++++++----------- 1 file changed, 52 insertions(+), 20 deletions(-) diff --git a/internal/rpc/group/group.go b/internal/rpc/group/group.go index c3ee0d3d5..57560a28e 100644 --- a/internal/rpc/group/group.go +++ b/internal/rpc/group/group.go @@ -1526,29 +1526,61 @@ func (g *groupServer) SetGroupMemberInfo(ctx context.Context, req *pbgroup.SetGr case 0: if !isAppManagerUid { roleLevel := dbMembers[opUserIndex].RoleLevel - if roleLevel != constant.GroupOwner { - switch roleLevel { - case constant.GroupAdmin: - for _, member := range dbMembers { - if member.RoleLevel == constant.GroupOwner { - return nil, errs.ErrNoPermission.WrapMsg("admin can not change group owner") - } - if member.RoleLevel == constant.GroupAdmin && member.UserID != opUserID { - return nil, errs.ErrNoPermission.WrapMsg("admin can not change other group admin") - } + var ( + dbSelf = &model.GroupMember{} + reqSelf *pbgroup.SetGroupMemberInfo + ) + switch roleLevel { + case constant.GroupOwner: + for _, member := range dbMembers { + if member.UserID == opUserID { + dbSelf = member + break } - case constant.GroupOrdinaryUsers: - for _, member := range dbMembers { - if !(member.RoleLevel == constant.GroupOrdinaryUsers && member.UserID == opUserID) { - return nil, errs.ErrNoPermission.WrapMsg("ordinary users can not change other role level") - } + } + case constant.GroupAdmin: + for _, member := range dbMembers { + if member.UserID == opUserID { + dbSelf = member + } + if member.RoleLevel == constant.GroupOwner { + return nil, errs.ErrNoPermission.WrapMsg("admin can not change group owner") } - default: - for _, member := range dbMembers { - if member.RoleLevel >= roleLevel { - return nil, errs.ErrNoPermission.WrapMsg("can not change higher role level") - } + if member.RoleLevel == constant.GroupAdmin && member.UserID != opUserID { + return nil, errs.ErrNoPermission.WrapMsg("admin can not change other group admin") + } + } + case constant.GroupOrdinaryUsers: + for _, member := range dbMembers { + if member.UserID == opUserID { + dbSelf = member + } + if !(member.RoleLevel == constant.GroupOrdinaryUsers && member.UserID == opUserID) { + return nil, errs.ErrNoPermission.WrapMsg("ordinary users can not change other role level") + } + } + default: + for _, member := range dbMembers { + if member.UserID == opUserID { + dbSelf = member } + if member.RoleLevel >= roleLevel { + return nil, errs.ErrNoPermission.WrapMsg("can not change higher role level") + } + } + } + for _, member := range req.Members { + if member.UserID == opUserID { + reqSelf = member + break + } + } + if reqSelf != nil && reqSelf.RoleLevel != nil { + if reqSelf.RoleLevel.GetValue() > dbSelf.RoleLevel { + return nil, errs.ErrNoPermission.WrapMsg("can not improve role level by self") + } + if roleLevel == constant.GroupOwner { + return nil, errs.ErrArgs.WrapMsg("group owner can not change own role level") // Prevent the absence of a group owner } } } From 9e8a389698e3c5973fe04a78cb9467aab7878983 Mon Sep 17 00:00:00 2001 From: icey-yu <119291641+icey-yu@users.noreply.github.com> Date: Thu, 24 Oct 2024 15:02:44 +0800 Subject: [PATCH 008/199] feat: Add More Multi Login Policy (#2770) * feat: multiLogin * feat: change config --- config/share.yml | 15 +- go.mod | 2 +- go.sum | 4 +- internal/msggateway/ws_server.go | 70 ++++-- internal/rpc/auth/auth.go | 9 +- pkg/common/config/config.go | 26 ++- pkg/common/storage/cache/cachekey/token.go | 19 +- pkg/common/storage/cache/redis/token.go | 64 ++++-- pkg/common/storage/cache/token.go | 2 + pkg/common/storage/controller/auth.go | 241 ++++++++++++++++----- pkg/rpcclient/auth.go | 11 + 11 files changed, 370 insertions(+), 93 deletions(-) diff --git a/config/share.yml b/config/share.yml index 5f8521eaa..7d977ae15 100644 --- a/config/share.yml +++ b/config/share.yml @@ -13,4 +13,17 @@ rpcRegisterName: imAdminUserID: [ imAdmin ] # 1: For Android, iOS, Windows, Mac, and web platforms, only one instance can be online at a time -multiLoginPolicy: 1 +multiLogin: + policy: 1 + maxNumOneEnd: 30 + customizeLoginNum: + ios: 1 + android: 1 + windows: 1 + osx: 1 + web: 1 + miniWeb: 1 + linux: 1 + aPad: 1 + iPad: 1 + admin: 1 diff --git a/go.mod b/go.mod index b6baca2a1..e54abee38 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/gorilla/websocket v1.5.1 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/mitchellh/mapstructure v1.5.0 - github.com/openimsdk/protocol v0.0.72-alpha.41 + github.com/openimsdk/protocol v0.0.72-alpha.44 github.com/openimsdk/tools v0.0.50-alpha.16 github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 diff --git a/go.sum b/go.sum index 6f5475274..1dbd81a13 100644 --- a/go.sum +++ b/go.sum @@ -319,8 +319,8 @@ github.com/onsi/gomega v1.25.0 h1:Vw7br2PCDYijJHSfBOWhov+8cAnUf8MfMaIOV323l6Y= github.com/onsi/gomega v1.25.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdMPGhM= github.com/openimsdk/gomake v0.0.14-alpha.5 h1:VY9c5x515lTfmdhhPjMvR3BBRrRquAUCFsz7t7vbv7Y= github.com/openimsdk/gomake v0.0.14-alpha.5/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= -github.com/openimsdk/protocol v0.0.72-alpha.41 h1:SMMoTc1iu+wtRqUqmIgqPJFejLgPeauOwoJ4VVG4iMQ= -github.com/openimsdk/protocol v0.0.72-alpha.41/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= +github.com/openimsdk/protocol v0.0.72-alpha.44 h1:TFFzkasClsYkYFKB7aZsaskO1BVJV88yshRhyxQE+/c= +github.com/openimsdk/protocol v0.0.72-alpha.44/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= github.com/openimsdk/tools v0.0.50-alpha.16 h1:bC1AQvJMuOHtZm8LZRvN8L5mH1Ws2VYdL+TLTs1iGSc= github.com/openimsdk/tools v0.0.50-alpha.16/go.mod h1:h1cYmfyaVtgFbKmb1Cfsl8XwUOMTt8ubVUQrdGtsUh4= github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= diff --git a/internal/msggateway/ws_server.go b/internal/msggateway/ws_server.go index 7df297488..b92d7eb44 100644 --- a/internal/msggateway/ws_server.go +++ b/internal/msggateway/ws_server.go @@ -1,17 +1,3 @@ -// Copyright © 2023 OpenIM. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - package msggateway import ( @@ -212,7 +198,6 @@ func (ws *WsServer) sendUserOnlineInfoToOtherNode(ctx context.Context, client *C if err != nil { return err } - wg := errgroup.Group{} wg.SetLimit(concurrentRequest) @@ -321,8 +306,32 @@ func (ws *WsServer) KickUserConn(client *Client) error { } func (ws *WsServer) multiTerminalLoginChecker(clientOK bool, oldClients []*Client, newClient *Client) { - switch ws.msgGatewayConfig.Share.MultiLoginPolicy { + kickTokenFunc := func(kickClients []*Client) { + var kickTokens []string + ws.clients.DeleteClients(newClient.UserID, kickClients) + for _, c := range kickClients { + kickTokens = append(kickTokens, c.token) + err := c.KickOnlineMessage() + if err != nil { + log.ZWarn(c.ctx, "KickOnlineMessage", err) + } + } + ctx := mcontext.WithMustInfoCtx( + []string{newClient.ctx.GetOperationID(), newClient.ctx.GetUserID(), + constant.PlatformIDToName(newClient.PlatformID), newClient.ctx.GetConnID()}, + ) + if _, err := ws.authClient.KickTokens(ctx, kickTokens); err != nil { + log.ZWarn(newClient.ctx, "kickTokens err", err) + } + } + + switch ws.msgGatewayConfig.Share.MultiLogin.Policy { case constant.DefalutNotKick: + case constant.WebAndOther: + if constant.PlatformIDToClass(newClient.PlatformID) == constant.WebPlatformStr { + return + } + fallthrough case constant.PCAndOther: if constant.PlatformIDToClass(newClient.PlatformID) == constant.TerminalPC { return @@ -347,6 +356,35 @@ func (ws *WsServer) multiTerminalLoginChecker(clientOK bool, oldClients []*Clien log.ZWarn(newClient.ctx, "InvalidateToken err", err, "userID", newClient.UserID, "platformID", newClient.PlatformID) } + case constant.PcMobileAndWeb: + clients, ok := ws.clients.GetAll(newClient.UserID) + if !ok { + return + } + var ( + kickClients []*Client + ) + for _, client := range clients { + if constant.PlatformIDToClass(client.PlatformID) == constant.PlatformIDToClass(newClient.PlatformID) { + kickClients = append(kickClients, client) + } + } + kickTokenFunc(kickClients) + + case constant.SingleTerminalLogin: + clients, ok := ws.clients.GetAll(newClient.UserID) + if !ok { + return + } + var ( + kickClients []*Client + ) + for _, client := range clients { + kickClients = append(kickClients, client) + } + kickTokenFunc(kickClients) + case constant.Customize: + // todo } } diff --git a/internal/rpc/auth/auth.go b/internal/rpc/auth/auth.go index 06ae89d97..62df74d21 100644 --- a/internal/rpc/auth/auth.go +++ b/internal/rpc/auth/auth.go @@ -65,7 +65,7 @@ func Start(ctx context.Context, config *Config, client discovery.SvcDiscoveryReg redis2.NewTokenCacheModel(rdb, config.RpcConfig.TokenPolicy.Expire), config.Share.Secret, config.RpcConfig.TokenPolicy.Expire, - config.Share.MultiLoginPolicy, + config.Share.MultiLogin, ), config: config, }) @@ -230,3 +230,10 @@ func (s *authServer) InvalidateToken(ctx context.Context, req *pbauth.Invalidate } return &pbauth.InvalidateTokenResp{}, nil } + +func (s *authServer) KickTokens(ctx context.Context, req *pbauth.KickTokensReq) (*pbauth.KickTokensResp, error) { + if err := s.authDatabase.BatchSetTokenMapByUidPid(ctx, req.Tokens); err != nil { + return nil, err + } + return &pbauth.KickTokensResp{}, nil +} diff --git a/pkg/common/config/config.go b/pkg/common/config/config.go index 77fcbb8aa..da6c63d60 100644 --- a/pkg/common/config/config.go +++ b/pkg/common/config/config.go @@ -361,11 +361,29 @@ type AfterConfig struct { } type Share struct { - Secret string `mapstructure:"secret"` - RpcRegisterName RpcRegisterName `mapstructure:"rpcRegisterName"` - IMAdminUserID []string `mapstructure:"imAdminUserID"` - MultiLoginPolicy int `mapstructure:"multiLoginPolicy"` + Secret string `mapstructure:"secret"` + RpcRegisterName RpcRegisterName `mapstructure:"rpcRegisterName"` + IMAdminUserID []string `mapstructure:"imAdminUserID"` + MultiLogin MultiLogin `mapstructure:"multiLogin"` +} + +type MultiLogin struct { + Policy int `mapstructure:"policy"` + MaxNumOneEnd int `mapstructure:"maxNumOneEnd"` + CustomizeLoginNum struct { + IOS int `mapstructure:"ios"` + Android int `mapstructure:"android"` + Windows int `mapstructure:"windows"` + OSX int `mapstructure:"osx"` + Web int `mapstructure:"web"` + MiniWeb int `mapstructure:"miniWeb"` + Linux int `mapstructure:"linux"` + APad int `mapstructure:"aPad"` + IPad int `mapstructure:"iPad"` + Admin int `mapstructure:"admin"` + } `mapstructure:"customizeLoginNum"` } + type RpcRegisterName struct { User string `mapstructure:"user"` Friend string `mapstructure:"friend"` diff --git a/pkg/common/storage/cache/cachekey/token.go b/pkg/common/storage/cache/cachekey/token.go index 94468dc31..83ba2f211 100644 --- a/pkg/common/storage/cache/cachekey/token.go +++ b/pkg/common/storage/cache/cachekey/token.go @@ -1,6 +1,9 @@ package cachekey -import "github.com/openimsdk/protocol/constant" +import ( + "github.com/openimsdk/protocol/constant" + "strings" +) const ( UidPidToken = "UID_PID_TOKEN_STATUS:" @@ -9,3 +12,17 @@ const ( func GetTokenKey(userID string, platformID int) string { return UidPidToken + userID + ":" + constant.PlatformIDToName(platformID) } + +func GetAllPlatformTokenKey(userID string) []string { + res := make([]string, len(constant.PlatformID2Name)) + for k := range constant.PlatformID2Name { + res[k-1] = GetTokenKey(userID, k) + } + return res +} + +func GetPlatformIDByTokenKey(key string) int { + splitKey := strings.Split(key, ":") + platform := splitKey[len(splitKey)-1] + return constant.PlatformNameToID(platform) +} diff --git a/pkg/common/storage/cache/redis/token.go b/pkg/common/storage/cache/redis/token.go index 24e9c3005..998b4f1c9 100644 --- a/pkg/common/storage/cache/redis/token.go +++ b/pkg/common/storage/cache/redis/token.go @@ -1,17 +1,3 @@ -// Copyright © 2023 OpenIM. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - package redis import ( @@ -21,6 +7,7 @@ import ( "github.com/openimsdk/tools/errs" "github.com/redis/go-redis/v9" "strconv" + "sync" "time" ) @@ -67,6 +54,43 @@ func (c *tokenCache) GetTokensWithoutError(ctx context.Context, userID string, p return mm, nil } +func (c *tokenCache) GetAllTokensWithoutError(ctx context.Context, userID string) (map[int]map[string]int, error) { + var ( + res = make(map[int]map[string]int) + resLock = sync.Mutex{} + ) + + keys := cachekey.GetAllPlatformTokenKey(userID) + if err := ProcessKeysBySlot(ctx, c.rdb, keys, func(ctx context.Context, slot int64, keys []string) error { + pipe := c.rdb.Pipeline() + mapRes := make([]*redis.MapStringStringCmd, len(keys)) + for i, key := range keys { + mapRes[i] = pipe.HGetAll(ctx, key) + } + _, err := pipe.Exec(ctx) + if err != nil { + return err + } + for i, m := range mapRes { + mm := make(map[string]int) + for k, v := range m.Val() { + state, err := strconv.Atoi(v) + if err != nil { + return errs.WrapMsg(err, "redis token value is not int", "value", v, "userID", userID) + } + mm[k] = state + } + resLock.Lock() + res[cachekey.GetPlatformIDByTokenKey(keys[i])] = mm + resLock.Unlock() + } + return nil + }); err != nil { + return nil, err + } + return res, nil +} + func (c *tokenCache) SetTokenMapByUidPid(ctx context.Context, userID string, platformID int, m map[string]int) error { mm := make(map[string]any) for k, v := range m { @@ -75,6 +99,18 @@ func (c *tokenCache) SetTokenMapByUidPid(ctx context.Context, userID string, pla return errs.Wrap(c.rdb.HSet(ctx, cachekey.GetTokenKey(userID, platformID), mm).Err()) } +func (c *tokenCache) BatchSetTokenMapByUidPid(ctx context.Context, tokens map[string]map[string]int) error { + pipe := c.rdb.Pipeline() + for k, v := range tokens { + pipe.HSet(ctx, k, v) + } + _, err := pipe.Exec(ctx) + if err != nil { + return errs.Wrap(err) + } + return nil +} + func (c *tokenCache) DeleteTokenByUidPid(ctx context.Context, userID string, platformID int, fields []string) error { return errs.Wrap(c.rdb.HDel(ctx, cachekey.GetTokenKey(userID, platformID), fields...).Err()) } diff --git a/pkg/common/storage/cache/token.go b/pkg/common/storage/cache/token.go index 4a0fee087..ee0004d7f 100644 --- a/pkg/common/storage/cache/token.go +++ b/pkg/common/storage/cache/token.go @@ -9,6 +9,8 @@ type TokenModel interface { // SetTokenFlagEx set token and flag with expire time SetTokenFlagEx(ctx context.Context, userID string, platformID int, token string, flag int) error GetTokensWithoutError(ctx context.Context, userID string, platformID int) (map[string]int, error) + GetAllTokensWithoutError(ctx context.Context, userID string) (map[int]map[string]int, error) SetTokenMapByUidPid(ctx context.Context, userID string, platformID int, m map[string]int) error + BatchSetTokenMapByUidPid(ctx context.Context, tokens map[string]map[string]int) error DeleteTokenByUidPid(ctx context.Context, userID string, platformID int, fields []string) error } diff --git a/pkg/common/storage/controller/auth.go b/pkg/common/storage/controller/auth.go index 94f18b3ae..de8f93462 100644 --- a/pkg/common/storage/controller/auth.go +++ b/pkg/common/storage/controller/auth.go @@ -1,21 +1,9 @@ -// Copyright © 2023 OpenIM. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - package controller import ( "context" + "github.com/openimsdk/open-im-server/v3/pkg/common/config" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/cachekey" "github.com/openimsdk/tools/log" "github.com/golang-jwt/jwt/v4" @@ -32,18 +20,41 @@ type AuthDatabase interface { // Create token CreateToken(ctx context.Context, userID string, platformID int) (string, error) + BatchSetTokenMapByUidPid(ctx context.Context, tokens []string) error + SetTokenMapByUidPid(ctx context.Context, userID string, platformID int, m map[string]int) error } +type multiLoginConfig struct { + Policy int + MaxNumOneEnd int + CustomizeLoginNum map[int]int +} + type authDatabase struct { - cache cache.TokenModel - accessSecret string - accessExpire int64 - multiLoginPolicy int + cache cache.TokenModel + accessSecret string + accessExpire int64 + multiLogin multiLoginConfig } -func NewAuthDatabase(cache cache.TokenModel, accessSecret string, accessExpire int64, policy int) AuthDatabase { - return &authDatabase{cache: cache, accessSecret: accessSecret, accessExpire: accessExpire, multiLoginPolicy: policy} +func NewAuthDatabase(cache cache.TokenModel, accessSecret string, accessExpire int64, multiLogin config.MultiLogin) AuthDatabase { + return &authDatabase{cache: cache, accessSecret: accessSecret, accessExpire: accessExpire, multiLogin: multiLoginConfig{ + Policy: multiLogin.Policy, + MaxNumOneEnd: multiLogin.MaxNumOneEnd, + CustomizeLoginNum: map[int]int{ + constant.IOSPlatformID: multiLogin.CustomizeLoginNum.IOS, + constant.AndroidPlatformID: multiLogin.CustomizeLoginNum.Android, + constant.WindowsPlatformID: multiLogin.CustomizeLoginNum.Windows, + constant.OSXPlatformID: multiLogin.CustomizeLoginNum.OSX, + constant.WebPlatformID: multiLogin.CustomizeLoginNum.Web, + constant.MiniWebPlatformID: multiLogin.CustomizeLoginNum.MiniWeb, + constant.LinuxPlatformID: multiLogin.CustomizeLoginNum.Linux, + constant.AndroidPadPlatformID: multiLogin.CustomizeLoginNum.APad, + constant.IPadPlatformID: multiLogin.CustomizeLoginNum.IPad, + constant.AdminPlatformID: multiLogin.CustomizeLoginNum.Admin, + }, + }} } // If the result is empty. @@ -55,22 +66,38 @@ func (a *authDatabase) SetTokenMapByUidPid(ctx context.Context, userID string, p return a.cache.SetTokenMapByUidPid(ctx, userID, platformID, m) } +func (a *authDatabase) BatchSetTokenMapByUidPid(ctx context.Context, tokens []string) error { + setMap := make(map[string]map[string]int) + for _, token := range tokens { + claims, err := tokenverify.GetClaimFromToken(token, authverify.Secret(a.accessSecret)) + key := cachekey.GetTokenKey(claims.UserID, claims.PlatformID) + if err != nil { + continue + } else { + if v, ok := setMap[key]; ok { + v[token] = constant.KickedToken + } else { + setMap[key] = map[string]int{ + token: constant.KickedToken, + } + } + } + } + if err := a.cache.BatchSetTokenMapByUidPid(ctx, setMap); err != nil { + return err + } + return nil +} + // Create Token. func (a *authDatabase) CreateToken(ctx context.Context, userID string, platformID int) (string, error) { - // todo: get all platform token - tokens, err := a.cache.GetTokensWithoutError(ctx, userID, platformID) + tokens, err := a.cache.GetAllTokensWithoutError(ctx, userID) if err != nil { return "", err } - var deleteTokenKey []string - var kickedTokenKey []string - for k, v := range tokens { - t, err := tokenverify.GetClaimFromToken(k, authverify.Secret(a.accessSecret)) - if err != nil || v != constant.NormalToken { - deleteTokenKey = append(deleteTokenKey, k) - } else if a.checkKickToken(ctx, platformID, t) { - kickedTokenKey = append(kickedTokenKey, k) - } + deleteTokenKey, kickedTokenKey, err := a.checkToken(ctx, tokens, platformID) + if err != nil { + return "", err } if len(deleteTokenKey) != 0 { err = a.cache.DeleteTokenByUidPid(ctx, userID, platformID, deleteTokenKey) @@ -78,16 +105,6 @@ func (a *authDatabase) CreateToken(ctx context.Context, userID string, platformI return "", err } } - - const adminTokenMaxNum = 30 - if platformID == constant.AdminPlatformID { - if len(kickedTokenKey) > adminTokenMaxNum { - kickedTokenKey = kickedTokenKey[:len(kickedTokenKey)-adminTokenMaxNum] - } else { - kickedTokenKey = nil - } - } - if len(kickedTokenKey) != 0 { for _, k := range kickedTokenKey { err := a.cache.SetTokenFlagEx(ctx, userID, platformID, k, constant.KickedToken) @@ -111,22 +128,140 @@ func (a *authDatabase) CreateToken(ctx context.Context, userID string, platformI return tokenString, nil } -func (a *authDatabase) checkKickToken(ctx context.Context, platformID int, token *tokenverify.Claims) bool { - switch a.multiLoginPolicy { +func (a *authDatabase) checkToken(ctx context.Context, tokens map[int]map[string]int, platformID int) ([]string, []string, error) { + // todo: Move the logic for handling old data to another location. + var ( + loginTokenMap = make(map[int][]string) // The length of the value of the map must be greater than 0 + deleteToken = make([]string, 0) + kickToken = make([]string, 0) + adminToken = make([]string, 0) + unkickTerminal = "" + ) + + for plfID, tks := range tokens { + for k, v := range tks { + _, err := tokenverify.GetClaimFromToken(k, authverify.Secret(a.accessSecret)) + if err != nil || v != constant.NormalToken { + deleteToken = append(deleteToken, k) + } else { + if plfID != constant.AdminPlatformID { + loginTokenMap[plfID] = append(loginTokenMap[plfID], k) + } else { + adminToken = append(adminToken, k) + } + } + } + } + + switch a.multiLogin.Policy { case constant.DefalutNotKick: - return false - case constant.PCAndOther: - if constant.PlatformIDToClass(platformID) == constant.TerminalPC || - constant.PlatformIDToClass(token.PlatformID) == constant.TerminalPC { - return false + for plt, ts := range loginTokenMap { + l := len(ts) + if platformID == plt { + l++ + } + limit := a.multiLogin.MaxNumOneEnd + if l > limit { + kickToken = append(kickToken, ts[:l-limit]...) + } } - return true case constant.AllLoginButSameTermKick: - if platformID == token.PlatformID { - return true + for plt, ts := range loginTokenMap { + kickToken = append(kickToken, ts[:len(ts)-1]...) + if plt == platformID { + kickToken = append(kickToken, ts[len(ts)-1]) + } + } + case constant.SingleTerminalLogin: + for _, ts := range loginTokenMap { + kickToken = append(kickToken, ts...) + } + case constant.WebAndOther: + unkickTerminal = constant.WebPlatformStr + fallthrough + case constant.PCAndOther: + if unkickTerminal == "" { + unkickTerminal = constant.TerminalPC + } + if constant.PlatformIDToClass(platformID) != unkickTerminal { + for plt, ts := range loginTokenMap { + if constant.PlatformIDToClass(plt) != unkickTerminal { + kickToken = append(kickToken, ts...) + } + } + } else { + var ( + preKick []string + isReserve = true + ) + for plt, ts := range loginTokenMap { + if constant.PlatformIDToClass(plt) != unkickTerminal { + // Keep a token from another end + if isReserve { + isReserve = false + kickToken = append(kickToken, ts[:len(ts)-1]...) + preKick = append(preKick, ts[len(ts)-1]) + continue + } else { + // Prioritize keeping Android + if plt == constant.AndroidPlatformID { + kickToken = append(kickToken, preKick...) + kickToken = append(kickToken, ts[:len(ts)-1]...) + } else { + kickToken = append(kickToken, ts...) + } + } + } + } + } + case constant.PcMobileAndWeb: + var ( + reserved = make(map[string]bool) + ) + + for plt, ts := range loginTokenMap { + if constant.PlatformIDToClass(plt) == constant.PlatformIDToClass(platformID) { + kickToken = append(kickToken, ts...) + } else { + if !reserved[constant.PlatformIDToClass(plt)] { + reserved[constant.PlatformIDToClass(plt)] = true + kickToken = append(kickToken, ts[:len(ts)-1]...) + continue + } else { + kickToken = append(kickToken, ts...) + } + } + } + + case constant.Customize: + if a.multiLogin.CustomizeLoginNum[platformID] <= 0 { + return nil, nil, errs.New("Do not allow login on this end").Wrap() + } + for plt, ts := range loginTokenMap { + l := len(ts) + if platformID == plt { + l++ + } + // a.multiLogin.CustomizeLoginNum[platformID] must > 0 + limit := min(a.multiLogin.CustomizeLoginNum[plt], a.multiLogin.MaxNumOneEnd) + if l > limit { + kickToken = append(kickToken, ts[:l-limit]...) + } } - return false default: - return false + return nil, nil, errs.New("unknown multiLogin policy").Wrap() + } + + var adminTokenMaxNum = a.multiLogin.MaxNumOneEnd + if a.multiLogin.Policy == constant.Customize { + adminTokenMaxNum = a.multiLogin.CustomizeLoginNum[constant.AdminPlatformID] + } + l := len(adminToken) + if platformID == constant.AdminPlatformID { + l++ + } + if l > adminTokenMaxNum { + kickToken = append(kickToken, adminToken[:l-adminTokenMaxNum]...) } + return deleteToken, kickToken, nil } diff --git a/pkg/rpcclient/auth.go b/pkg/rpcclient/auth.go index fead624a3..05fec35a0 100644 --- a/pkg/rpcclient/auth.go +++ b/pkg/rpcclient/auth.go @@ -61,3 +61,14 @@ func (a *Auth) InvalidateToken(ctx context.Context, preservedToken, userID strin } return resp, err } + +func (a *Auth) KickTokens(ctx context.Context, tokens []string) (*auth.KickTokensResp, error) { + req := auth.KickTokensReq{ + Tokens: tokens, + } + resp, err := a.Client.KickTokens(ctx, &req) + if err != nil { + return nil, err + } + return resp, err +} From a84f7bd217268c465c9d71b6f06dfeeb57bbf21b Mon Sep 17 00:00:00 2001 From: Alilestera <75608652+alilestera@users.noreply.github.com> Date: Thu, 24 Oct 2024 15:21:51 +0800 Subject: [PATCH 009/199] fix: joinSource check args error. (#2773) Co-authored-by: Monet Lee --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index e54abee38..aea5360d7 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/gorilla/websocket v1.5.1 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/mitchellh/mapstructure v1.5.0 - github.com/openimsdk/protocol v0.0.72-alpha.44 + github.com/openimsdk/protocol v0.0.72-alpha.45 github.com/openimsdk/tools v0.0.50-alpha.16 github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 diff --git a/go.sum b/go.sum index 1dbd81a13..473272660 100644 --- a/go.sum +++ b/go.sum @@ -319,8 +319,8 @@ github.com/onsi/gomega v1.25.0 h1:Vw7br2PCDYijJHSfBOWhov+8cAnUf8MfMaIOV323l6Y= github.com/onsi/gomega v1.25.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdMPGhM= github.com/openimsdk/gomake v0.0.14-alpha.5 h1:VY9c5x515lTfmdhhPjMvR3BBRrRquAUCFsz7t7vbv7Y= github.com/openimsdk/gomake v0.0.14-alpha.5/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= -github.com/openimsdk/protocol v0.0.72-alpha.44 h1:TFFzkasClsYkYFKB7aZsaskO1BVJV88yshRhyxQE+/c= -github.com/openimsdk/protocol v0.0.72-alpha.44/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= +github.com/openimsdk/protocol v0.0.72-alpha.45 h1:xTxEG/NzBw/ZxLggqz76l7rl9HUfg7Kb2xS+jU0G2E4= +github.com/openimsdk/protocol v0.0.72-alpha.45/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= github.com/openimsdk/tools v0.0.50-alpha.16 h1:bC1AQvJMuOHtZm8LZRvN8L5mH1Ws2VYdL+TLTs1iGSc= github.com/openimsdk/tools v0.0.50-alpha.16/go.mod h1:h1cYmfyaVtgFbKmb1Cfsl8XwUOMTt8ubVUQrdGtsUh4= github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= From 0d03b46ac8d61c1ff5ff9cccc19e6df36ec7289f Mon Sep 17 00:00:00 2001 From: icey-yu <119291641+icey-yu@users.noreply.github.com> Date: Thu, 24 Oct 2024 16:47:06 +0800 Subject: [PATCH 010/199] feat: change push config (#2775) --- internal/push/offlinepush/offlinepusher.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/internal/push/offlinepush/offlinepusher.go b/internal/push/offlinepush/offlinepusher.go index 9aa6625de..d655a924a 100644 --- a/internal/push/offlinepush/offlinepusher.go +++ b/internal/push/offlinepush/offlinepusher.go @@ -23,10 +23,13 @@ import ( "github.com/openimsdk/open-im-server/v3/internal/push/offlinepush/options" "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache" + "github.com/openimsdk/tools/log" + "github.com/openimsdk/tools/mcontext" + "strings" ) const ( - geTUI = "geTui" + geTUI = "getui" firebase = "fcm" jPush = "jpush" ) @@ -38,6 +41,7 @@ type OfflinePusher interface { func NewOfflinePusher(pushConf *config.Push, cache cache.ThirdCache, fcmConfigPath string) (OfflinePusher, error) { var offlinePusher OfflinePusher + pushConf.Enable = strings.ToLower(pushConf.Enable) switch pushConf.Enable { case geTUI: offlinePusher = getui.NewClient(pushConf, cache) @@ -47,6 +51,7 @@ func NewOfflinePusher(pushConf *config.Push, cache cache.ThirdCache, fcmConfigPa offlinePusher = jpush.NewClient(pushConf) default: offlinePusher = dummy.NewClient() + log.ZWarn(mcontext.WithMustInfoCtx([]string{"push start", "admin", "admin", ""}), "Unknown push config", nil) } return offlinePusher, nil } From 43919bc5fea503fdc175442139563d7f20c1e562 Mon Sep 17 00:00:00 2001 From: icey-yu <119291641+icey-yu@users.noreply.github.com> Date: Thu, 24 Oct 2024 16:48:46 +0800 Subject: [PATCH 011/199] fix: change group member info send notification (#2777) --- internal/rpc/group/group.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/rpc/group/group.go b/internal/rpc/group/group.go index 57560a28e..b8e6c2aca 100644 --- a/internal/rpc/group/group.go +++ b/internal/rpc/group/group.go @@ -1621,7 +1621,7 @@ func (g *groupServer) SetGroupMemberInfo(ctx context.Context, req *pbgroup.SetGr g.notification.GroupMemberSetToOrdinaryUserNotification(ctx, member.GroupID, member.UserID) } } - if member.Nickname != nil || member.FaceURL != nil || member.Ex != nil { + if member.Nickname != nil || member.FaceURL != nil || member.Ex != nil || member.RoleLevel != nil { g.notification.GroupMemberInfoSetNotification(ctx, member.GroupID, member.UserID) } } From b5ef71f5c2b62aca1fa555726404832709924a70 Mon Sep 17 00:00:00 2001 From: chao <48119764+withchao@users.noreply.github.com> Date: Thu, 24 Oct 2024 17:33:31 +0800 Subject: [PATCH 012/199] fix: client sends message status error to server (#2779) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * update gomake version * update gomake version * fix: seq conversion bug * fix: redis pipe exec * fix: ImportFriends * fix: A large number of logs keysAndValues ​​length is not even * feat: mark read aggregate write * feat: online status supports redis cluster * feat: online status supports redis cluster * feat: online status supports redis cluster * merge * merge * read seq is written to mongo * read seq is written to mongo * fix: invitation to join group notification * fix: friend op_user_id * feat: optimizing asynchronous context * feat: optimizing memamq size * feat: add GetSeqMessage * feat: GroupApplicationAgreeMemberEnterNotification * feat: GroupApplicationAgreeMemberEnterNotification * feat: go.mod * feat: go.mod * feat: join group notification and get seq * feat: join group notification and get seq * feat: avoid pulling messages from sessions with a large number of max seq values of 0 * feat: API supports gzip * go.mod * fix: nil pointer error on close * fix: listen error * fix: listen error * update go.mod * feat: add log * fix: token parse token value * fix: GetMsgBySeqs boundary issues * fix: sn_ not sort * fix: sn_ not sort * fix: sn_ not sort * fix: jssdk add * fix: jssdk support * fix: jssdk support * fix: jssdk support * fix: the message I sent is not set to read seq in mongodb * fix: cannot modify group member avatars * fix: MemberEnterNotification * fix: MemberEnterNotification * fix: MsgData status --------- Co-authored-by: withchao --- go.mod | 2 +- go.sum | 4 ++-- internal/push/offlinepush_handler.go | 3 +++ internal/push/push_handler.go | 3 +++ pkg/common/storage/controller/msg_transfer.go | 4 ++++ 5 files changed, 13 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index aea5360d7..b982bc7d0 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/gorilla/websocket v1.5.1 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/mitchellh/mapstructure v1.5.0 - github.com/openimsdk/protocol v0.0.72-alpha.45 + github.com/openimsdk/protocol v0.0.72-alpha.46 github.com/openimsdk/tools v0.0.50-alpha.16 github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 diff --git a/go.sum b/go.sum index 473272660..df9cf5194 100644 --- a/go.sum +++ b/go.sum @@ -319,8 +319,8 @@ github.com/onsi/gomega v1.25.0 h1:Vw7br2PCDYijJHSfBOWhov+8cAnUf8MfMaIOV323l6Y= github.com/onsi/gomega v1.25.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdMPGhM= github.com/openimsdk/gomake v0.0.14-alpha.5 h1:VY9c5x515lTfmdhhPjMvR3BBRrRquAUCFsz7t7vbv7Y= github.com/openimsdk/gomake v0.0.14-alpha.5/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= -github.com/openimsdk/protocol v0.0.72-alpha.45 h1:xTxEG/NzBw/ZxLggqz76l7rl9HUfg7Kb2xS+jU0G2E4= -github.com/openimsdk/protocol v0.0.72-alpha.45/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= +github.com/openimsdk/protocol v0.0.72-alpha.46 h1:1LZlfEHLzw1F4afFmqBczmXKJWm5rUQ+yr8rJ4oyEAc= +github.com/openimsdk/protocol v0.0.72-alpha.46/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= github.com/openimsdk/tools v0.0.50-alpha.16 h1:bC1AQvJMuOHtZm8LZRvN8L5mH1Ws2VYdL+TLTs1iGSc= github.com/openimsdk/tools v0.0.50-alpha.16/go.mod h1:h1cYmfyaVtgFbKmb1Cfsl8XwUOMTt8ubVUQrdGtsUh4= github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= diff --git a/internal/push/offlinepush_handler.go b/internal/push/offlinepush_handler.go index bf69aed3e..a80c147f4 100644 --- a/internal/push/offlinepush_handler.go +++ b/internal/push/offlinepush_handler.go @@ -55,6 +55,9 @@ func (o *OfflinePushConsumerHandler) handleMsg2OfflinePush(ctx context.Context, log.ZError(ctx, "offline push msg is empty", errs.New("offlinePushMsg is empty"), "userIDs", offlinePushMsg.UserIDs, "msg", offlinePushMsg.MsgData) return } + if offlinePushMsg.MsgData.Status == constant.MsgStatusSending { + offlinePushMsg.MsgData.Status = constant.MsgStatusSendSuccess + } log.ZInfo(ctx, "receive to OfflinePush MQ", "userIDs", offlinePushMsg.UserIDs, "msg", offlinePushMsg.MsgData) err := o.offlinePushMsg(ctx, offlinePushMsg.MsgData, offlinePushMsg.UserIDs) diff --git a/internal/push/push_handler.go b/internal/push/push_handler.go index 4ecf20de5..41ad5962a 100644 --- a/internal/push/push_handler.go +++ b/internal/push/push_handler.go @@ -194,6 +194,9 @@ func (c *ConsumerHandler) shouldPushOffline(_ context.Context, msg *sdkws.MsgDat } func (c *ConsumerHandler) GetConnsAndOnlinePush(ctx context.Context, msg *sdkws.MsgData, pushToUserIDs []string) ([]*msggateway.SingleMsgToUserResults, error) { + if msg != nil && msg.Status == constant.MsgStatusSending { + msg.Status = constant.MsgStatusSendSuccess + } onlineUserIDs, offlineUserIDs, err := c.onlineCache.GetUsersOnline(ctx, pushToUserIDs) if err != nil { return nil, err diff --git a/pkg/common/storage/controller/msg_transfer.go b/pkg/common/storage/controller/msg_transfer.go index c5dfd011f..c6013dbc1 100644 --- a/pkg/common/storage/controller/msg_transfer.go +++ b/pkg/common/storage/controller/msg_transfer.go @@ -2,6 +2,7 @@ package controller import ( "context" + "github.com/openimsdk/protocol/constant" "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/open-im-server/v3/pkg/common/prommetrics" @@ -83,6 +84,9 @@ func (db *msgTransferDatabase) BatchInsertChat2DB(ctx context.Context, conversat IOSBadgeCount: msg.OfflinePushInfo.IOSBadgeCount, } } + if msg.Status == constant.MsgStatusSending { + msg.Status = constant.MsgStatusSendSuccess + } msgs[i] = &model.MsgDataModel{ SendID: msg.SendID, RecvID: msg.RecvID, From 9baf1ffe82a26c92f418a3ac10dd44142e299a72 Mon Sep 17 00:00:00 2001 From: OpenIM-Gordon <46924906+FGadvancer@users.noreply.github.com> Date: Thu, 24 Oct 2024 20:59:56 +0800 Subject: [PATCH 013/199] fix: del UserB's conversation version cache when userA set conversation's isPrivateChat to true. (#2785) --- pkg/common/storage/controller/conversation.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pkg/common/storage/controller/conversation.go b/pkg/common/storage/controller/conversation.go index 06a073365..f0b7d70db 100644 --- a/pkg/common/storage/controller/conversation.go +++ b/pkg/common/storage/controller/conversation.go @@ -16,9 +16,10 @@ package controller import ( "context" + "time" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" relationtb "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" - "time" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache" "github.com/openimsdk/open-im-server/v3/pkg/msgprocessor" @@ -194,7 +195,7 @@ func (c *conversationDatabase) SyncPeerUserPrivateConversationTx(ctx context.Con return c.tx.Transaction(ctx, func(ctx context.Context) error { cache := c.cache.CloneConversationCache() for _, conversation := range conversations { - cache = cache.DelConversationVersionUserIDs(conversation.OwnerUserID) + cache = cache.DelConversationVersionUserIDs(conversation.OwnerUserID, conversation.UserID) for _, v := range [][2]string{{conversation.OwnerUserID, conversation.UserID}, {conversation.UserID, conversation.OwnerUserID}} { ownerUserID := v[0] userID := v[1] From bbac036d6c40cbebc7520b3bc04d19f84f9a60dc Mon Sep 17 00:00:00 2001 From: Alilestera <75608652+alilestera@users.noreply.github.com> Date: Fri, 25 Oct 2024 10:31:43 +0800 Subject: [PATCH 014/199] chore: remove unused .chglog and unnecessary content in goreleaser (#2786) --- CHANGELOG/.chglog/CHANGELOG.tpl.md | 62 -------------------- CHANGELOG/.chglog/config.yml | 67 ---------------------- build/goreleaser.yaml | 91 ------------------------------ 3 files changed, 220 deletions(-) delete mode 100644 CHANGELOG/.chglog/CHANGELOG.tpl.md delete mode 100644 CHANGELOG/.chglog/config.yml diff --git a/CHANGELOG/.chglog/CHANGELOG.tpl.md b/CHANGELOG/.chglog/CHANGELOG.tpl.md deleted file mode 100644 index 100a29ed8..000000000 --- a/CHANGELOG/.chglog/CHANGELOG.tpl.md +++ /dev/null @@ -1,62 +0,0 @@ -# Version logging for OpenIM - - - - - -{{ if .Versions -}} - -## [Unreleased] - -{{ if .Unreleased.CommitGroups -}} -{{ range .Unreleased.CommitGroups -}} -### {{ .Title }} -{{ range .Commits -}} -- {{ if .Scope }}**{{ .Scope }}:** {{ end }}{{ .Subject }} -{{ end }} -{{ end -}} -{{ end -}} -{{ end -}} - -{{ range .Versions }} - -## {{ if .Tag.Previous }}[{{ .Tag.Name }}]{{ else }}{{ .Tag.Name }}{{ end }} - {{ datetime "2006-01-02" .Tag.Date }} -{{ range .CommitGroups -}} -### {{ .Title }} -{{ range .Commits -}} -- {{ if .Scope }}**{{ .Scope }}:** {{ end }}{{ .Subject }} -{{ end }} -{{ end -}} - -{{- if .RevertCommits -}} -### Reverts -{{ range .RevertCommits -}} -- {{ .Revert.Header }} -{{ end }} -{{ end -}} - -{{- if .MergeCommits -}} -### Pull Requests -{{ range .MergeCommits -}} -- {{ .Header }} -{{ end }} -{{ end -}} - -{{- if .NoteGroups -}} -{{ range .NoteGroups -}} -### {{ .Title }} -{{ range .Notes }} -{{ .Body }} -{{ end }} -{{ end -}} -{{ end -}} -{{ end -}} - -{{- if .Versions }} -[Unreleased]: {{ .Info.RepositoryURL }}/compare/{{ $latest := index .Versions 0 }}{{ $latest.Tag.Name }}...HEAD -{{ range .Versions -}} -{{ if .Tag.Previous -}} -[{{ .Tag.Name }}]: {{ $.Info.RepositoryURL }}/compare/{{ .Tag.Previous.Name }}...{{ .Tag.Name }} -{{ end -}} -{{ end -}} -{{ end -}} \ No newline at end of file diff --git a/CHANGELOG/.chglog/config.yml b/CHANGELOG/.chglog/config.yml deleted file mode 100644 index 2a45bc26a..000000000 --- a/CHANGELOG/.chglog/config.yml +++ /dev/null @@ -1,67 +0,0 @@ -bin: git -style: github -template: CHANGELOG.tpl.md -info: - title: CHANGELOG - repository_url: https://github.com/openimsdk/open-im-server -options: - tag_filter_pattern: '^v' - sort: "date" - - commits: - filters: - Type: - - feat - - fix - - perf - - refactor - - docs - - test - - chore - - ci - - build - sort_by: Scope - - commit_groups: - group_by: Type - sort_by: Title - title_order: - - feat - - fix - - perf - - refactor - - docs - - test - - chore - - ci - - build - title_maps: - feat: Features - - header: - pattern: "" - pattern_maps: - - PropName - - issues: - prefix: - - # - - refs: - actions: - - Closes - - Fixes - - merges: - pattern: "^Merge branch '(\\w+)'$" - pattern_maps: - - Source - - reverts: - pattern: "^Revert \"([\\s\\S]*)\"$" - pattern_maps: - - Header - - notes: - keywords: - - BREAKING CHANGE \ No newline at end of file diff --git a/build/goreleaser.yaml b/build/goreleaser.yaml index 93fe9f4c8..c24fb65da 100644 --- a/build/goreleaser.yaml +++ b/build/goreleaser.yaml @@ -53,15 +53,8 @@ builds: - windows - linux goarch: - - s390x - - mips64 - - mips64le - amd64 - - ppc64le - arm64 - goarm: - - "6" - - "7" - binary: openim-cmdutils id: openim-cmdutils @@ -71,15 +64,8 @@ builds: - windows - linux goarch: - - s390x - - mips64 - - mips64le - amd64 - - ppc64le - arm64 - goarm: - - "6" - - "7" - binary: openim-crontask id: openim-crontask @@ -89,15 +75,8 @@ builds: - windows - linux goarch: - - s390x - - mips64 - - mips64le - amd64 - - ppc64le - arm64 - goarm: - - "6" - - "7" - binary: openim-msggateway id: openim-msggateway @@ -107,15 +86,8 @@ builds: - windows - linux goarch: - - s390x - - mips64 - - mips64le - amd64 - - ppc64le - arm64 - goarm: - - "6" - - "7" - binary: openim-msgtransfer id: openim-msgtransfer @@ -125,15 +97,8 @@ builds: - windows - linux goarch: - - s390x - - mips64 - - mips64le - amd64 - - ppc64le - arm64 - goarm: - - "6" - - "7" - binary: openim-push id: openim-push @@ -143,15 +108,8 @@ builds: - windows - linux goarch: - - s390x - - mips64 - - mips64le - amd64 - - ppc64le - arm64 - goarm: - - "6" - - "7" - binary: openim-rpc-auth id: openim-rpc-auth @@ -161,15 +119,8 @@ builds: - windows - linux goarch: - - s390x - - mips64 - - mips64le - amd64 - - ppc64le - arm64 - goarm: - - "6" - - "7" - binary: openim-rpc-conversation id: openim-rpc-conversation @@ -179,15 +130,8 @@ builds: - windows - linux goarch: - - s390x - - mips64 - - mips64le - amd64 - - ppc64le - arm64 - goarm: - - "6" - - "7" - binary: openim-rpc-friend id: openim-rpc-friend @@ -197,15 +141,8 @@ builds: - windows - linux goarch: - - s390x - - mips64 - - mips64le - amd64 - - ppc64le - arm64 - goarm: - - "6" - - "7" - binary: openim-rpc-group id: openim-rpc-group @@ -215,15 +152,8 @@ builds: - windows - linux goarch: - - s390x - - mips64 - - mips64le - amd64 - - ppc64le - arm64 - goarm: - - "6" - - "7" - binary: openim-rpc-msg id: openim-rpc-msg @@ -233,15 +163,8 @@ builds: - windows - linux goarch: - - s390x - - mips64 - - mips64le - amd64 - - ppc64le - arm64 - goarm: - - "6" - - "7" - binary: openim-rpc-third id: openim-rpc-third @@ -251,15 +174,8 @@ builds: - windows - linux goarch: - - s390x - - mips64 - - mips64le - amd64 - - ppc64le - arm64 - goarm: - - "6" - - "7" - binary: openim-rpc-user id: openim-rpc-user @@ -269,15 +185,8 @@ builds: - windows - linux goarch: - - s390x - - mips64 - - mips64le - amd64 - - ppc64le - arm64 - goarm: - - "6" - - "7" # TODO：Need a script, such as the init - release to help binary to find the right directory From 0207f1dab140f05284a02b74705f6425d15143bb Mon Sep 17 00:00:00 2001 From: Monet Lee Date: Fri, 25 Oct 2024 10:34:17 +0800 Subject: [PATCH 015/199] fix: improve setConversationAtInfo logic. (#2782) * fix: improve ConversationATInfo logic. * fix logic err. --- internal/rpc/msg/send.go | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/internal/rpc/msg/send.go b/internal/rpc/msg/send.go index a16ca5665..2c3f8c0a3 100644 --- a/internal/rpc/msg/send.go +++ b/internal/rpc/msg/send.go @@ -29,7 +29,6 @@ import ( "github.com/openimsdk/tools/log" "github.com/openimsdk/tools/mcontext" "github.com/openimsdk/tools/utils/datautil" - "github.com/openimsdk/tools/utils/stringutil" ) func (m *msgServer) SendMsg(ctx context.Context, req *pbmsg.SendMsgReq) (*pbmsg.SendMsgResp, error) { @@ -80,13 +79,17 @@ func (m *msgServer) sendMsgGroupChat(ctx context.Context, req *pbmsg.SendMsgReq) func (m *msgServer) setConversationAtInfo(nctx context.Context, msg *sdkws.MsgData) { log.ZDebug(nctx, "setConversationAtInfo", "msg", msg) + ctx := mcontext.NewCtx("@@@" + mcontext.GetOperationID(nctx)) + var atUserID []string + conversation := &pbconversation.ConversationReq{ ConversationID: msgprocessor.GetConversationIDByMsg(msg), ConversationType: msg.SessionType, GroupID: msg.GroupID, } + tagAll := datautil.Contain(constant.AtAllString, msg.AtUserIDList...) if tagAll { memberUserIDList, err := m.GroupLocalCache.GetGroupMemberIDs(ctx, msg.GroupID) @@ -94,25 +97,35 @@ func (m *msgServer) setConversationAtInfo(nctx context.Context, msg *sdkws.MsgDa log.ZWarn(ctx, "GetGroupMemberIDs", err) return } - atUserID = stringutil.DifferenceString([]string{constant.AtAllString}, msg.AtUserIDList) + + memberUserIDList = datautil.DeleteElems(memberUserIDList, msg.SendID) + + atUserID = datautil.Single([]string{constant.AtAllString}, msg.AtUserIDList) + if len(atUserID) == 0 { // just @everyone conversation.GroupAtType = &wrapperspb.Int32Value{Value: constant.AtAll} } else { // @Everyone and @other people conversation.GroupAtType = &wrapperspb.Int32Value{Value: constant.AtAllAtMe} + err = m.Conversation.SetConversations(ctx, atUserID, conversation) if err != nil { log.ZWarn(ctx, "SetConversations", err, "userID", atUserID, "conversation", conversation) } - memberUserIDList = stringutil.DifferenceString(atUserID, memberUserIDList) + + memberUserIDList = datautil.Single(atUserID, memberUserIDList) } + conversation.GroupAtType = &wrapperspb.Int32Value{Value: constant.AtAll} + err = m.Conversation.SetConversations(ctx, memberUserIDList, conversation) if err != nil { log.ZWarn(ctx, "SetConversations", err, "userID", memberUserIDList, "conversation", conversation) } + return } conversation.GroupAtType = &wrapperspb.Int32Value{Value: constant.AtMe} + err := m.Conversation.SetConversations(ctx, msg.AtUserIDList, conversation) if err != nil { log.ZWarn(ctx, "SetConversations", err, msg.AtUserIDList, conversation) From 4be508a640405159cc0bda960507b30700a49b29 Mon Sep 17 00:00:00 2001 From: icey-yu <119291641+icey-yu@users.noreply.github.com> Date: Fri, 25 Oct 2024 11:58:09 +0800 Subject: [PATCH 016/199] Revert: Change group member roleLevel can`t send notification (#2789) * fix: change group member info send notification * fix: change group member info send notification * fix: group * fix: group * fix: group --- internal/rpc/group/group.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/rpc/group/group.go b/internal/rpc/group/group.go index b8e6c2aca..57560a28e 100644 --- a/internal/rpc/group/group.go +++ b/internal/rpc/group/group.go @@ -1621,7 +1621,7 @@ func (g *groupServer) SetGroupMemberInfo(ctx context.Context, req *pbgroup.SetGr g.notification.GroupMemberSetToOrdinaryUserNotification(ctx, member.GroupID, member.UserID) } } - if member.Nickname != nil || member.FaceURL != nil || member.Ex != nil || member.RoleLevel != nil { + if member.Nickname != nil || member.FaceURL != nil || member.Ex != nil { g.notification.GroupMemberInfoSetNotification(ctx, member.GroupID, member.UserID) } } From b36b69506f759ea2575c50425bf9c45926803b99 Mon Sep 17 00:00:00 2001 From: Monet Lee Date: Fri, 25 Oct 2024 14:25:56 +0800 Subject: [PATCH 017/199] fix: improve transfer Owner logic when newOwner is mute. (#2790) * fix: improve ConversationATInfo logic. * fix logic err. * fix: improve transfer Owner logic when newOwner is mute. --- internal/rpc/group/group.go | 30 +++++++++++++++++++++++++++--- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/internal/rpc/group/group.go b/internal/rpc/group/group.go index 57560a28e..ef917d539 100644 --- a/internal/rpc/group/group.go +++ b/internal/rpc/group/group.go @@ -1180,36 +1180,53 @@ func (g *groupServer) TransferGroupOwner(ctx context.Context, req *pbgroup.Trans if err != nil { return nil, err } + if group.Status == constant.GroupStatusDismissed { return nil, servererrs.ErrDismissedAlready.Wrap() } + if req.OldOwnerUserID == req.NewOwnerUserID { return nil, errs.ErrArgs.WrapMsg("OldOwnerUserID == NewOwnerUserID") } + members, err := g.db.FindGroupMembers(ctx, req.GroupID, []string{req.OldOwnerUserID, req.NewOwnerUserID}) if err != nil { return nil, err } + if err := g.PopulateGroupMember(ctx, members...); err != nil { return nil, err } + memberMap := datautil.SliceToMap(members, func(e *model.GroupMember) string { return e.UserID }) if ids := datautil.Single([]string{req.OldOwnerUserID, req.NewOwnerUserID}, datautil.Keys(memberMap)); len(ids) > 0 { return nil, errs.ErrArgs.WrapMsg("user not in group " + strings.Join(ids, ",")) } + oldOwner := memberMap[req.OldOwnerUserID] if oldOwner == nil { return nil, errs.ErrArgs.WrapMsg("OldOwnerUserID not in group " + req.NewOwnerUserID) } + newOwner := memberMap[req.NewOwnerUserID] if newOwner == nil { return nil, errs.ErrArgs.WrapMsg("NewOwnerUser not in group " + req.NewOwnerUserID) } + if !authverify.IsAppManagerUid(ctx, g.config.Share.IMAdminUserID) { if !(mcontext.GetOpUserID(ctx) == oldOwner.UserID && oldOwner.RoleLevel == constant.GroupOwner) { return nil, errs.ErrNoPermission.WrapMsg("no permission transfer group owner") } } + + if newOwner.MuteEndTime != time.Unix(0, 0) { + if _, err := g.CancelMuteGroupMember(ctx, &pbgroup.CancelMuteGroupMemberReq{ + GroupID: group.GroupID, + UserID: req.NewOwnerUserID}); err != nil { + return nil, err + } + } + if err := g.db.TransferGroupOwner(ctx, req.GroupID, req.OldOwnerUserID, req.NewOwnerUserID, newOwner.RoleLevel); err != nil { return nil, err } @@ -1217,6 +1234,7 @@ func (g *groupServer) TransferGroupOwner(ctx context.Context, req *pbgroup.Trans g.webhookAfterTransferGroupOwner(ctx, &g.config.WebhooksConfig.AfterTransferGroupOwner, req) g.notification.GroupOwnerTransferredNotification(ctx, req) + return &pbgroup.TransferGroupOwnerResp{}, nil } @@ -1425,32 +1443,38 @@ func (g *groupServer) CancelMuteGroupMember(ctx context.Context, req *pbgroup.Ca if err != nil { return nil, err } + if err := g.PopulateGroupMember(ctx, member); err != nil { return nil, err } + if !authverify.IsAppManagerUid(ctx, g.config.Share.IMAdminUserID) { opMember, err := g.db.TakeGroupMember(ctx, req.GroupID, mcontext.GetOpUserID(ctx)) if err != nil { return nil, err } + switch member.RoleLevel { case constant.GroupOwner: - return nil, errs.ErrNoPermission.WrapMsg("set group owner mute") + return nil, errs.ErrNoPermission.WrapMsg("Can not set group owner unmute") case constant.GroupAdmin: if opMember.RoleLevel != constant.GroupOwner { - return nil, errs.ErrNoPermission.WrapMsg("set group admin mute") + return nil, errs.ErrNoPermission.WrapMsg("Can not set group admin unmute") } case constant.GroupOrdinaryUsers: if !(opMember.RoleLevel == constant.GroupAdmin || opMember.RoleLevel == constant.GroupOwner) { - return nil, errs.ErrNoPermission.WrapMsg("set group ordinary users mute") + return nil, errs.ErrNoPermission.WrapMsg("Can not set group ordinary users unmute") } } } + data := UpdateGroupMemberMutedTimeMap(time.Unix(0, 0)) if err := g.db.UpdateGroupMember(ctx, member.GroupID, member.UserID, data); err != nil { return nil, err } + g.notification.GroupMemberCancelMutedNotification(ctx, req.GroupID, req.UserID) + return &pbgroup.CancelMuteGroupMemberResp{}, nil } From 312c8ba9d60df39417f4eedbcd50b211b1417181 Mon Sep 17 00:00:00 2001 From: Monet Lee Date: Fri, 25 Oct 2024 16:23:21 +0800 Subject: [PATCH 018/199] fix: improve getUserInfo logic. (#2792) * fix: improve ConversationATInfo logic. * fix logic err. * fix: improve transfer Owner logic when newOwner is mute. * fix: improve getUserInfo logic. --- internal/rpc/user/user.go | 7 +++---- pkg/rpccache/user.go | 2 ++ 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/internal/rpc/user/user.go b/internal/rpc/user/user.go index 4669ed513..b47c516d9 100644 --- a/internal/rpc/user/user.go +++ b/internal/rpc/user/user.go @@ -116,18 +116,17 @@ func Start(ctx context.Context, config *Config, client registry.SvcDiscoveryRegi func (s *userServer) GetDesignateUsers(ctx context.Context, req *pbuser.GetDesignateUsersReq) (resp *pbuser.GetDesignateUsersResp, err error) { resp = &pbuser.GetDesignateUsersResp{} - users, err := s.db.FindWithError(ctx, req.UserIDs) + users, err := s.db.Find(ctx, req.UserIDs) if err != nil { return nil, err } + resp.UsersInfo = convert.UsersDB2Pb(users) return resp, nil } // deprecated: - -//UpdateUserInfo - +// UpdateUserInfo func (s *userServer) UpdateUserInfo(ctx context.Context, req *pbuser.UpdateUserInfoReq) (resp *pbuser.UpdateUserInfoResp, err error) { resp = &pbuser.UpdateUserInfoResp{} err = authverify.CheckAccessV3(ctx, req.UserInfo.UserID, s.config.Share.IMAdminUserID) diff --git a/pkg/rpccache/user.go b/pkg/rpccache/user.go index 7c676f30a..79a768597 100644 --- a/pkg/rpccache/user.go +++ b/pkg/rpccache/user.go @@ -16,6 +16,7 @@ package rpccache import ( "context" + "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/cachekey" "github.com/openimsdk/open-im-server/v3/pkg/localcache" @@ -97,6 +98,7 @@ func (u *UserLocalCache) GetUsersInfo(ctx context.Context, userIDs []string) ([] user, err := u.GetUserInfo(ctx, userID) if err != nil { if errs.ErrRecordNotFound.Is(err) { + log.ZWarn(ctx, "User info notFound", err, "userID", userID) continue } return nil, err From 649250b7c644fd904be4e6e8ec7e997def26092a Mon Sep 17 00:00:00 2001 From: chao <48119764+withchao@users.noreply.github.com> Date: Fri, 25 Oct 2024 16:50:58 +0800 Subject: [PATCH 019/199] feat: support app update service (#2794) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * update gomake version * update gomake version * fix: seq conversion bug * fix: redis pipe exec * fix: ImportFriends * fix: A large number of logs keysAndValues ​​length is not even * feat: mark read aggregate write * feat: online status supports redis cluster * feat: online status supports redis cluster * feat: online status supports redis cluster * merge * merge * read seq is written to mongo * read seq is written to mongo * fix: invitation to join group notification * fix: friend op_user_id * feat: optimizing asynchronous context * feat: optimizing memamq size * feat: add GetSeqMessage * feat: GroupApplicationAgreeMemberEnterNotification * feat: GroupApplicationAgreeMemberEnterNotification * feat: go.mod * feat: go.mod * feat: join group notification and get seq * feat: join group notification and get seq * feat: avoid pulling messages from sessions with a large number of max seq values of 0 * feat: API supports gzip * go.mod * fix: nil pointer error on close * fix: listen error * fix: listen error * update go.mod * feat: add log * fix: token parse token value * fix: GetMsgBySeqs boundary issues * fix: sn_ not sort * fix: sn_ not sort * fix: sn_ not sort * fix: jssdk add * fix: jssdk support * fix: jssdk support * fix: jssdk support * fix: the message I sent is not set to read seq in mongodb * fix: cannot modify group member avatars * fix: MemberEnterNotification * fix: MemberEnterNotification * fix: MsgData status * feat: add ApplicationVersion --------- Co-authored-by: withchao --- go.mod | 2 +- go.sum | 4 +- internal/api/router.go | 9 ++ internal/api/third.go | 20 +++ internal/rpc/third/application.go | 117 ++++++++++++++++++ internal/rpc/third/third.go | 31 +++-- internal/rpc/third/tool.go | 8 ++ pkg/common/storage/cache/application.go | 11 ++ .../storage/cache/cachekey/application.go | 9 ++ pkg/common/storage/cache/redis/application.go | 43 +++++++ pkg/common/storage/controller/application.go | 69 +++++++++++ pkg/common/storage/database/application.go | 17 +++ .../storage/database/mgo/application.go | 82 ++++++++++++ pkg/common/storage/model/application.go | 17 +++ 14 files changed, 424 insertions(+), 15 deletions(-) create mode 100644 internal/rpc/third/application.go create mode 100644 pkg/common/storage/cache/application.go create mode 100644 pkg/common/storage/cache/cachekey/application.go create mode 100644 pkg/common/storage/cache/redis/application.go create mode 100644 pkg/common/storage/controller/application.go create mode 100644 pkg/common/storage/database/application.go create mode 100644 pkg/common/storage/database/mgo/application.go create mode 100644 pkg/common/storage/model/application.go diff --git a/go.mod b/go.mod index b982bc7d0..5e5a8b5be 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/gorilla/websocket v1.5.1 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/mitchellh/mapstructure v1.5.0 - github.com/openimsdk/protocol v0.0.72-alpha.46 + github.com/openimsdk/protocol v0.0.72-alpha.47 github.com/openimsdk/tools v0.0.50-alpha.16 github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 diff --git a/go.sum b/go.sum index df9cf5194..53109c890 100644 --- a/go.sum +++ b/go.sum @@ -319,8 +319,8 @@ github.com/onsi/gomega v1.25.0 h1:Vw7br2PCDYijJHSfBOWhov+8cAnUf8MfMaIOV323l6Y= github.com/onsi/gomega v1.25.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdMPGhM= github.com/openimsdk/gomake v0.0.14-alpha.5 h1:VY9c5x515lTfmdhhPjMvR3BBRrRquAUCFsz7t7vbv7Y= github.com/openimsdk/gomake v0.0.14-alpha.5/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= -github.com/openimsdk/protocol v0.0.72-alpha.46 h1:1LZlfEHLzw1F4afFmqBczmXKJWm5rUQ+yr8rJ4oyEAc= -github.com/openimsdk/protocol v0.0.72-alpha.46/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= +github.com/openimsdk/protocol v0.0.72-alpha.47 h1:FGHnEwsA05GxT3vnz7YH3fbVkuoO3P71ZZgkQQ71MjA= +github.com/openimsdk/protocol v0.0.72-alpha.47/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= github.com/openimsdk/tools v0.0.50-alpha.16 h1:bC1AQvJMuOHtZm8LZRvN8L5mH1Ws2VYdL+TLTs1iGSc= github.com/openimsdk/tools v0.0.50-alpha.16/go.mod h1:h1cYmfyaVtgFbKmb1Cfsl8XwUOMTt8ubVUQrdGtsUh4= github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= diff --git a/internal/api/router.go b/internal/api/router.go index 560516d30..17c998912 100644 --- a/internal/api/router.go +++ b/internal/api/router.go @@ -198,6 +198,13 @@ func newGinRouter(disCov discovery.SvcDiscoveryRegistry, config *Config) *gin.En objectGroup.POST("/initiate_form_data", t.InitiateFormData) objectGroup.POST("/complete_form_data", t.CompleteFormData) objectGroup.GET("/*name", t.ObjectRedirect) + + applicationGroup := r.Group("application") + applicationGroup.POST("/add_version", t.AddApplicationVersion) + applicationGroup.POST("/update_version", t.UpdateApplicationVersion) + applicationGroup.POST("/delete_version", t.DeleteApplicationVersion) + applicationGroup.POST("/latest_version", t.LatestApplicationVersion) + applicationGroup.POST("/page_versions", t.PageApplicationVersion) } // Message msgGroup := r.Group("/msg") @@ -290,4 +297,6 @@ func GinParseToken(authRPC *rpcclient.Auth) gin.HandlerFunc { var Whitelist = []string{ "/auth/get_admin_token", "/auth/parse_token", + "/application/latest_version", + "/application/page_versions", } diff --git a/internal/api/third.go b/internal/api/third.go index 6baa70ee5..56661ba89 100644 --- a/internal/api/third.go +++ b/internal/api/third.go @@ -170,3 +170,23 @@ func (o *ThirdApi) SearchLogs(c *gin.Context) { func (o *ThirdApi) GetPrometheus(c *gin.Context) { c.Redirect(http.StatusFound, o.GrafanaUrl) } + +func (o *ThirdApi) LatestApplicationVersion(c *gin.Context) { + a2r.Call(third.ThirdClient.LatestApplicationVersion, o.Client, c) +} + +func (o *ThirdApi) AddApplicationVersion(c *gin.Context) { + a2r.Call(third.ThirdClient.AddApplicationVersion, o.Client, c) +} + +func (o *ThirdApi) UpdateApplicationVersion(c *gin.Context) { + a2r.Call(third.ThirdClient.UpdateApplicationVersion, o.Client, c) +} + +func (o *ThirdApi) DeleteApplicationVersion(c *gin.Context) { + a2r.Call(third.ThirdClient.DeleteApplicationVersion, o.Client, c) +} + +func (o *ThirdApi) PageApplicationVersion(c *gin.Context) { + a2r.Call(third.ThirdClient.PageApplicationVersion, o.Client, c) +} diff --git a/internal/rpc/third/application.go b/internal/rpc/third/application.go new file mode 100644 index 000000000..a6556055c --- /dev/null +++ b/internal/rpc/third/application.go @@ -0,0 +1,117 @@ +package third + +import ( + "context" + "github.com/openimsdk/open-im-server/v3/pkg/authverify" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" + "github.com/openimsdk/protocol/third" + "github.com/openimsdk/tools/errs" + "github.com/openimsdk/tools/utils/datautil" + "github.com/redis/go-redis/v9" + "go.mongodb.org/mongo-driver/bson/primitive" + "go.mongodb.org/mongo-driver/mongo" + "time" +) + +func IsNotFound(err error) bool { + switch errs.Unwrap(err) { + case redis.Nil, mongo.ErrNoDocuments: + return true + default: + return false + } +} + +func (t *thirdServer) db2pbApplication(val *model.Application) *third.ApplicationVersion { + return &third.ApplicationVersion{ + Id: val.ID.Hex(), + Platform: val.Platform, + Version: val.Version, + Url: val.Url, + Text: val.Text, + Force: val.Force, + Latest: val.Latest, + CreateTime: val.CreateTime.UnixMilli(), + } +} + +func (t *thirdServer) LatestApplicationVersion(ctx context.Context, req *third.LatestApplicationVersionReq) (*third.LatestApplicationVersionResp, error) { + res, err := t.applicationDatabase.LatestVersion(ctx, req.Platform) + if err == nil { + return &third.LatestApplicationVersionResp{Version: t.db2pbApplication(res)}, nil + } else if IsNotFound(err) { + return &third.LatestApplicationVersionResp{}, nil + } else { + return nil, err + } +} + +func (t *thirdServer) AddApplicationVersion(ctx context.Context, req *third.AddApplicationVersionReq) (*third.AddApplicationVersionResp, error) { + if err := authverify.CheckAdmin(ctx, t.config.Share.IMAdminUserID); err != nil { + return nil, err + } + val := &model.Application{ + ID: primitive.NewObjectID(), + Platform: req.Platform, + Version: req.Version, + Url: req.Url, + Text: req.Text, + Force: req.Force, + Latest: req.Latest, + CreateTime: time.Now(), + } + if err := t.applicationDatabase.AddVersion(ctx, val); err != nil { + return nil, err + } + return &third.AddApplicationVersionResp{}, nil +} + +func (t *thirdServer) UpdateApplicationVersion(ctx context.Context, req *third.UpdateApplicationVersionReq) (*third.UpdateApplicationVersionResp, error) { + if err := authverify.CheckAdmin(ctx, t.config.Share.IMAdminUserID); err != nil { + return nil, err + } + oid, err := primitive.ObjectIDFromHex(req.Id) + if err != nil { + return nil, errs.ErrArgs.WrapMsg("invalid id " + err.Error()) + } + update := make(map[string]any) + putUpdate(update, "platform", req.Platform) + putUpdate(update, "version", req.Version) + putUpdate(update, "url", req.Url) + putUpdate(update, "text", req.Text) + putUpdate(update, "force", req.Force) + putUpdate(update, "latest", req.Latest) + if err := t.applicationDatabase.UpdateVersion(ctx, oid, update); err != nil { + return nil, err + } + return &third.UpdateApplicationVersionResp{}, nil +} + +func (t *thirdServer) DeleteApplicationVersion(ctx context.Context, req *third.DeleteApplicationVersionReq) (*third.DeleteApplicationVersionResp, error) { + if err := authverify.CheckAdmin(ctx, t.config.Share.IMAdminUserID); err != nil { + return nil, err + } + ids := make([]primitive.ObjectID, 0, len(req.Id)) + for _, id := range req.Id { + oid, err := primitive.ObjectIDFromHex(id) + if err != nil { + return nil, errs.ErrArgs.WrapMsg("invalid id " + err.Error()) + } + ids = append(ids, oid) + } + if err := t.applicationDatabase.DeleteVersion(ctx, ids); err != nil { + return nil, err + } + return &third.DeleteApplicationVersionResp{}, nil +} + +func (t *thirdServer) PageApplicationVersion(ctx context.Context, req *third.PageApplicationVersionReq) (*third.PageApplicationVersionResp, error) { + total, res, err := t.applicationDatabase.PageVersion(ctx, req.Platform, req.Pagination) + if err != nil { + return nil, err + } + return &third.PageApplicationVersionResp{ + Total: total, + Versions: datautil.Slice(res, t.db2pbApplication), + }, nil +} diff --git a/internal/rpc/third/third.go b/internal/rpc/third/third.go index 0eeaaa314..c6b588d8d 100644 --- a/internal/rpc/third/third.go +++ b/internal/rpc/third/third.go @@ -38,12 +38,13 @@ import ( ) type thirdServer struct { - thirdDatabase controller.ThirdDatabase - s3dataBase controller.S3Database - userRpcClient rpcclient.UserRpcClient - defaultExpire time.Duration - config *Config - minio *minio.Minio + thirdDatabase controller.ThirdDatabase + s3dataBase controller.S3Database + userRpcClient rpcclient.UserRpcClient + defaultExpire time.Duration + config *Config + minio *minio.Minio + applicationDatabase controller.ApplicationDatabase } type Config struct { @@ -74,6 +75,11 @@ func Start(ctx context.Context, config *Config, client discovery.SvcDiscoveryReg if err != nil { return err } + applicationMgo, err := mgo.NewApplicationMgo(mgocli.GetDB()) + if err != nil { + return err + } + // Select the oss method according to the profile policy enable := config.RpcConfig.Object.Enable var ( @@ -98,12 +104,13 @@ func Start(ctx context.Context, config *Config, client discovery.SvcDiscoveryReg } localcache.InitLocalCache(&config.LocalCacheConfig) third.RegisterThirdServer(server, &thirdServer{ - thirdDatabase: controller.NewThirdDatabase(redis.NewThirdCache(rdb), logdb), - userRpcClient: rpcclient.NewUserRpcClient(client, config.Share.RpcRegisterName.User, config.Share.IMAdminUserID), - s3dataBase: controller.NewS3Database(rdb, o, s3db), - defaultExpire: time.Hour * 24 * 7, - config: config, - minio: minioCli, + thirdDatabase: controller.NewThirdDatabase(redis.NewThirdCache(rdb), logdb), + userRpcClient: rpcclient.NewUserRpcClient(client, config.Share.RpcRegisterName.User, config.Share.IMAdminUserID), + s3dataBase: controller.NewS3Database(rdb, o, s3db), + defaultExpire: time.Hour * 24 * 7, + config: config, + minio: minioCli, + applicationDatabase: controller.NewApplicationDatabase(applicationMgo, redis.NewApplicationRedisCache(applicationMgo, rdb)), }) return nil } diff --git a/internal/rpc/third/tool.go b/internal/rpc/third/tool.go index ac4be3968..4e22ffbf9 100644 --- a/internal/rpc/third/tool.go +++ b/internal/rpc/third/tool.go @@ -82,3 +82,11 @@ func checkValidObjectName(objectName string) error { func (t *thirdServer) IsManagerUserID(opUserID string) bool { return authverify.IsManagerUserID(opUserID, t.config.Share.IMAdminUserID) } + +func putUpdate[T any](update map[string]any, name string, val interface{ GetValuePtr() *T }) { + ptrVal := val.GetValuePtr() + if ptrVal == nil { + return + } + update[name] = *ptrVal +} diff --git a/pkg/common/storage/cache/application.go b/pkg/common/storage/cache/application.go new file mode 100644 index 000000000..588732ec8 --- /dev/null +++ b/pkg/common/storage/cache/application.go @@ -0,0 +1,11 @@ +package cache + +import ( + "context" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" +) + +type ApplicationCache interface { + LatestVersion(ctx context.Context, platform string) (*model.Application, error) + DeleteCache(ctx context.Context, platforms []string) error +} diff --git a/pkg/common/storage/cache/cachekey/application.go b/pkg/common/storage/cache/cachekey/application.go new file mode 100644 index 000000000..032adba3c --- /dev/null +++ b/pkg/common/storage/cache/cachekey/application.go @@ -0,0 +1,9 @@ +package cachekey + +const ( + ApplicationLatestVersion = "APPLICATION_LATEST_VERSION:" +) + +func GetApplicationLatestVersionKey(platform string) string { + return ApplicationLatestVersion + platform +} diff --git a/pkg/common/storage/cache/redis/application.go b/pkg/common/storage/cache/redis/application.go new file mode 100644 index 000000000..4a7a4ced6 --- /dev/null +++ b/pkg/common/storage/cache/redis/application.go @@ -0,0 +1,43 @@ +package redis + +import ( + "context" + "github.com/dtm-labs/rockscache" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/cachekey" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" + "github.com/openimsdk/tools/utils/datautil" + "github.com/redis/go-redis/v9" + "time" +) + +func NewApplicationRedisCache(db database.Application, rdb redis.UniversalClient) *ApplicationRedisCache { + return &ApplicationRedisCache{ + db: db, + rcClient: rockscache.NewClient(rdb, *GetRocksCacheOptions()), + deleter: NewBatchDeleterRedis(rdb, GetRocksCacheOptions(), nil), + expireTime: time.Hour * 24 * 7, + } +} + +type ApplicationRedisCache struct { + db database.Application + rcClient *rockscache.Client + deleter *BatchDeleterRedis + expireTime time.Duration +} + +func (a *ApplicationRedisCache) LatestVersion(ctx context.Context, platform string) (*model.Application, error) { + return getCache(ctx, a.rcClient, cachekey.GetApplicationLatestVersionKey(platform), a.expireTime, func(ctx context.Context) (*model.Application, error) { + return a.db.LatestVersion(ctx, platform) + }) +} + +func (a *ApplicationRedisCache) DeleteCache(ctx context.Context, platforms []string) error { + if len(platforms) == 0 { + return nil + } + return a.deleter.ExecDelWithKeys(ctx, datautil.Slice(platforms, func(platform string) string { + return cachekey.GetApplicationLatestVersionKey(platform) + })) +} diff --git a/pkg/common/storage/controller/application.go b/pkg/common/storage/controller/application.go new file mode 100644 index 000000000..72bca07ef --- /dev/null +++ b/pkg/common/storage/controller/application.go @@ -0,0 +1,69 @@ +package controller + +import ( + "context" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" + "github.com/openimsdk/tools/db/pagination" + "go.mongodb.org/mongo-driver/bson/primitive" +) + +type ApplicationDatabase interface { + LatestVersion(ctx context.Context, platform string) (*model.Application, error) + AddVersion(ctx context.Context, val *model.Application) error + UpdateVersion(ctx context.Context, id primitive.ObjectID, update map[string]any) error + DeleteVersion(ctx context.Context, id []primitive.ObjectID) error + PageVersion(ctx context.Context, platforms []string, page pagination.Pagination) (int64, []*model.Application, error) +} + +func NewApplicationDatabase(db database.Application, cache cache.ApplicationCache) ApplicationDatabase { + return &applicationDatabase{db: db, cache: cache} +} + +type applicationDatabase struct { + db database.Application + cache cache.ApplicationCache +} + +func (a *applicationDatabase) LatestVersion(ctx context.Context, platform string) (*model.Application, error) { + return a.cache.LatestVersion(ctx, platform) +} + +func (a *applicationDatabase) AddVersion(ctx context.Context, val *model.Application) error { + if err := a.db.AddVersion(ctx, val); err != nil { + return err + } + return a.cache.DeleteCache(ctx, []string{val.Platform}) +} + +func (a *applicationDatabase) UpdateVersion(ctx context.Context, id primitive.ObjectID, update map[string]any) error { + platforms, err := a.db.FindPlatform(ctx, []primitive.ObjectID{id}) + if err != nil { + return err + } + if err := a.db.UpdateVersion(ctx, id, update); err != nil { + return err + } + if p, ok := update["platform"]; ok { + if val, ok := p.(string); ok { + platforms = append(platforms, val) + } + } + return a.cache.DeleteCache(ctx, platforms) +} + +func (a *applicationDatabase) DeleteVersion(ctx context.Context, id []primitive.ObjectID) error { + platforms, err := a.db.FindPlatform(ctx, id) + if err != nil { + return err + } + if err := a.db.DeleteVersion(ctx, id); err != nil { + return err + } + return a.cache.DeleteCache(ctx, platforms) +} + +func (a *applicationDatabase) PageVersion(ctx context.Context, platforms []string, page pagination.Pagination) (int64, []*model.Application, error) { + return a.db.PageVersion(ctx, platforms, page) +} diff --git a/pkg/common/storage/database/application.go b/pkg/common/storage/database/application.go new file mode 100644 index 000000000..c98ae74c8 --- /dev/null +++ b/pkg/common/storage/database/application.go @@ -0,0 +1,17 @@ +package database + +import ( + "context" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" + "github.com/openimsdk/tools/db/pagination" + "go.mongodb.org/mongo-driver/bson/primitive" +) + +type Application interface { + LatestVersion(ctx context.Context, platform string) (*model.Application, error) + AddVersion(ctx context.Context, val *model.Application) error + UpdateVersion(ctx context.Context, id primitive.ObjectID, update map[string]any) error + DeleteVersion(ctx context.Context, id []primitive.ObjectID) error + PageVersion(ctx context.Context, platforms []string, page pagination.Pagination) (int64, []*model.Application, error) + FindPlatform(ctx context.Context, id []primitive.ObjectID) ([]string, error) +} diff --git a/pkg/common/storage/database/mgo/application.go b/pkg/common/storage/database/mgo/application.go new file mode 100644 index 000000000..e59c0560a --- /dev/null +++ b/pkg/common/storage/database/mgo/application.go @@ -0,0 +1,82 @@ +package mgo + +import ( + "context" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" + "github.com/openimsdk/tools/db/mongoutil" + "github.com/openimsdk/tools/db/pagination" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" +) + +func NewApplicationMgo(db *mongo.Database) (*ApplicationMgo, error) { + coll := db.Collection("application") + _, err := coll.Indexes().CreateMany(context.Background(), []mongo.IndexModel{ + { + Keys: bson.D{ + {Key: "platform", Value: 1}, + {Key: "version", Value: 1}, + }, + Options: options.Index().SetUnique(true), + }, + { + Keys: bson.D{ + {Key: "latest", Value: -1}, + }, + }, + }) + if err != nil { + return nil, err + } + return &ApplicationMgo{coll: coll}, nil +} + +type ApplicationMgo struct { + coll *mongo.Collection +} + +func (a *ApplicationMgo) sort() any { + return bson.D{{"latest", -1}, {"_id", -1}} +} + +func (a *ApplicationMgo) LatestVersion(ctx context.Context, platform string) (*model.Application, error) { + return mongoutil.FindOne[*model.Application](ctx, a.coll, bson.M{"platform": platform}, options.FindOne().SetSort(a.sort())) +} + +func (a *ApplicationMgo) AddVersion(ctx context.Context, val *model.Application) error { + if val.ID.IsZero() { + val.ID = primitive.NewObjectID() + } + return mongoutil.InsertMany(ctx, a.coll, []*model.Application{val}) +} + +func (a *ApplicationMgo) UpdateVersion(ctx context.Context, id primitive.ObjectID, update map[string]any) error { + if len(update) == 0 { + return nil + } + return mongoutil.UpdateOne(ctx, a.coll, bson.M{"_id": id}, bson.M{"$set": update}, true) +} + +func (a *ApplicationMgo) DeleteVersion(ctx context.Context, id []primitive.ObjectID) error { + if len(id) == 0 { + return nil + } + return mongoutil.DeleteMany(ctx, a.coll, bson.M{"_id": bson.M{"$in": id}}) +} + +func (a *ApplicationMgo) PageVersion(ctx context.Context, platforms []string, page pagination.Pagination) (int64, []*model.Application, error) { + filter := bson.M{} + if len(platforms) > 0 { + filter["platform"] = bson.M{"$in": platforms} + } + return mongoutil.FindPage[*model.Application](ctx, a.coll, filter, page, options.Find().SetSort(a.sort())) +} + +func (a *ApplicationMgo) FindPlatform(ctx context.Context, id []primitive.ObjectID) ([]string, error) { + if len(id) == 0 { + return nil, nil + } + return mongoutil.Find[string](ctx, a.coll, bson.M{"_id": bson.M{"$in": id}}, options.Find().SetProjection(bson.M{"_id": 0, "platform": 1})) +} diff --git a/pkg/common/storage/model/application.go b/pkg/common/storage/model/application.go new file mode 100644 index 000000000..f5bae2be6 --- /dev/null +++ b/pkg/common/storage/model/application.go @@ -0,0 +1,17 @@ +package model + +import ( + "go.mongodb.org/mongo-driver/bson/primitive" + "time" +) + +type Application struct { + ID primitive.ObjectID `bson:"_id"` + Platform string `bson:"platform"` + Version string `bson:"version"` + Url string `bson:"url"` + Text string `bson:"text"` + Force bool `bson:"force"` + Latest bool `bson:"latest"` + CreateTime time.Time `bson:"create_time"` +} From 4a8abfa21d662a42fca9630ce890a8bae35796f4 Mon Sep 17 00:00:00 2001 From: Monet Lee Date: Fri, 25 Oct 2024 18:12:29 +0800 Subject: [PATCH 020/199] feat: implement merge milestone PR to target-branch. (#2796) --- .../cleanup-after-milestone-prs-merged.yml | 65 ++++++ .github/workflows/merge-from-milestone.yml | 221 ++++++++++++++++++ 2 files changed, 286 insertions(+) create mode 100644 .github/workflows/cleanup-after-milestone-prs-merged.yml create mode 100644 .github/workflows/merge-from-milestone.yml diff --git a/.github/workflows/cleanup-after-milestone-prs-merged.yml b/.github/workflows/cleanup-after-milestone-prs-merged.yml new file mode 100644 index 000000000..ff4e8029d --- /dev/null +++ b/.github/workflows/cleanup-after-milestone-prs-merged.yml @@ -0,0 +1,65 @@ +name: Cleanup After Milestone PRs Merged + +on: + pull_request: + types: + - closed + +jobs: + handle_pr: + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Get the PR title and extract PR numbers + id: extract_pr_numbers + run: | + # Get the PR title + PR_TITLE="${{ github.event.pull_request.title }}" + + echo "PR Title: $PR_TITLE" + + # Extract PR numbers from the title + PR_NUMBERS=$(echo "$PR_TITLE" | grep -oE "#[0-9]+" | tr -d '#' | tr '\n' ' ') + echo "Extracted PR Numbers: $PR_NUMBERS" + + # Save PR numbers to a file + echo "$PR_NUMBERS" > pr_numbers.txt + echo "Saved PR Numbers to pr_numbers.txt" + + # Check if the title matches a specific pattern + if echo "$PR_TITLE" | grep -qE "^deps: Merge( #[0-9]+)+ PRs into .+"; then + echo "proceed=true" >> $GITHUB_OUTPUT + else + echo "proceed=false" >> $GITHUB_OUTPUT + fi + + - name: Use extracted PR numbers and label PRs + if: (steps.extract_pr_numbers.outputs.proceed == 'true' || contains(github.event.pull_request.labels.*.name, 'milestone-merge')) && github.event.pull_request.merged == true + run: | + # Read the previously saved PR numbers + PR_NUMBERS=$(cat pr_numbers.txt) + echo "Using extracted PR Numbers: $PR_NUMBERS" + + # Loop through each PR number and add label + for PR_NUMBER in $PR_NUMBERS; do + echo "Adding 'cherry-picked' label to PR #$PR_NUMBER" + curl -X POST \ + -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ + -H "Accept: application/vnd.github+json" \ + https://api.github.com/repos/${{ github.repository }}/issues/$PR_NUMBER/labels \ + -d '{"labels":["cherry-picked"]}' + done + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Delete branch after PR close + if: steps.extract_pr_numbers.outputs.proceed == 'true' || contains(github.event.pull_request.labels.*.name, 'milestone-merge') + run: | + BRANCH_NAME="${{ github.event.pull_request.head.ref }}" + echo "Branch to delete: $BRANCH_NAME" + git push origin --delete "$BRANCH_NAME" + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file diff --git a/.github/workflows/merge-from-milestone.yml b/.github/workflows/merge-from-milestone.yml new file mode 100644 index 000000000..d9110b0c3 --- /dev/null +++ b/.github/workflows/merge-from-milestone.yml @@ -0,0 +1,221 @@ +name: Create Pre-Release PR from Milestone + +permissions: + contents: write + pull-requests: write + issues: write + +on: + workflow_dispatch: + inputs: + milestone_name: + description: 'Milestone name to collect closed PRs from' + required: true + default: 'v3.8.2' + target_branch: + description: 'Target branch to merge the consolidated PR' + required: true + default: 'pre-release-v3.8.2' + + schedule: + - cron: '0 2 * * 0' + +env: + MILESTONE_NAME: ${{ github.event.inputs.milestone_name || 'v3.8.2' }} + TARGET_BRANCH: ${{ github.event.inputs.target_branch || 'pre-release-v3.8.2' }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + BOT_TOKEN: ${{ secrets.BOT_TOKEN }} + LABEL_NAME: cherry-picked + TEMP_DIR: /tmp # Using /tmp as the temporary directory + +jobs: + cherry_pick_milestone_prs: + runs-on: ubuntu-latest + steps: + - name: Setup temp directory + run: | + # Create the temporary directory and initialize necessary files + mkdir -p ${{ env.TEMP_DIR }} + touch ${{ env.TEMP_DIR }}/pr_numbers.txt + touch ${{ env.TEMP_DIR }}/commit_hashes.txt + touch ${{ env.TEMP_DIR }}/pr_title.txt + touch ${{ env.TEMP_DIR }}/pr_body.txt + touch ${{ env.TEMP_DIR }}/created_pr_number.txt + + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + token: ${{ secrets.BOT_TOKEN }} + + - name: Setup Git User for OpenIM-Robot + run: | + # Set up Git credentials for the bot + git config --global user.email "OpenIM-Robot@users.noreply.github.com" + git config --global user.name "OpenIM-Robot" + + - name: Fetch Milestone ID and Filter PR Numbers + env: + MILESTONE_NAME: ${{ env.MILESTONE_NAME }} + run: | + # Fetch milestone details and extract milestone ID + milestones=$(curl -s -H "Authorization: token $BOT_TOKEN" \ + -H "Accept: application/vnd.github+json" \ + "https://api.github.com/repos/${{ github.repository }}/milestones") + milestone_id=$(echo "$milestones" | grep -B3 "\"title\": \"$MILESTONE_NAME\"" | grep '"number":' | head -n1 | grep -o '[0-9]\+') + if [ -z "$milestone_id" ]; then + echo "Milestone '$MILESTONE_NAME' not found. Exiting." + exit 1 + fi + echo "Milestone ID: $milestone_id" + echo "MILESTONE_ID=$milestone_id" >> $GITHUB_ENV + + # Fetch issues for the milestone + issues=$(curl -s -H "Authorization: token $BOT_TOKEN" \ + -H "Accept: application/vnd.github+json" \ + "https://api.github.com/repos/${{ github.repository }}/issues?milestone=$milestone_id&state=closed&per_page=100") + + > ${{ env.TEMP_DIR }}/pr_numbers.txt + + # Filter PRs that do not have the 'cherry-picked' label + for pr_number in $(echo "$issues" | jq -r '.[] | select(.pull_request != null) | .number'); do + labels=$(curl -s -H "Authorization: token $BOT_TOKEN" \ + -H "Accept: application/vnd.github+json" \ + "https://api.github.com/repos/${{ github.repository }}/issues/$pr_number/labels" | jq -r '.[].name') + + if ! echo "$labels" | grep -q "${LABEL_NAME}"; then + echo "PR #$pr_number does not have the 'cherry-picked' label. Adding to the list." + echo "$pr_number" >> ${{ env.TEMP_DIR }}/pr_numbers.txt + else + echo "PR #$pr_number already has the 'cherry-picked' label. Skipping." + fi + done + + # Sort the filtered PR numbers + sort -n ${{ env.TEMP_DIR }}/pr_numbers.txt -o ${{ env.TEMP_DIR }}/pr_numbers.txt + + echo "Filtered and sorted PR numbers:" + cat ${{ env.TEMP_DIR }}/pr_numbers.txt || echo "No closed PR numbers found for milestone." + + - name: Fetch Merge Commits for PRs and Generate Title and Body + run: | + # Ensure the files are initialized + > ${{ env.TEMP_DIR }}/commit_hashes.txt + > ${{ env.TEMP_DIR }}/pr_title.txt + > ${{ env.TEMP_DIR }}/pr_body.txt + + # Write description to the PR body + echo "### Description:" >> ${{ env.TEMP_DIR }}/pr_body.txt + echo "Merging PRs from milestone \`$MILESTONE_NAME\` into target branch \`$TARGET_BRANCH\`." >> ${{ env.TEMP_DIR }}/pr_body.txt + echo "" >> ${{ env.TEMP_DIR }}/pr_body.txt + echo "### Need Merge PRs:" >> ${{ env.TEMP_DIR }}/pr_body.txt + + pr_numbers_in_title="" + + # Process sorted PR numbers and generate commit hashes + for pr_number in $(cat ${{ env.TEMP_DIR }}/pr_numbers.txt); do + echo "Processing PR #$pr_number" + pr_details=$(curl -s -H "Authorization: token $BOT_TOKEN" \ + -H "Accept: application/vnd.github+json" \ + "https://api.github.com/repos/${{ github.repository }}/pulls/$pr_number") + pr_title=$(echo "$pr_details" | jq -r '.title') + merge_commit=$(echo "$pr_details" | jq -r '.merge_commit_sha') + short_commit_hash=$(echo "$merge_commit" | cut -c 1-7) + + # Append PR details to the body + echo "- $pr_title: (#$pr_number) ($short_commit_hash)" >> ${{ env.TEMP_DIR }}/pr_body.txt + + if [ "$merge_commit" != "null" ];then + echo "$merge_commit" >> ${{ env.TEMP_DIR }}/commit_hashes.txt + echo "#$pr_number" >> ${{ env.TEMP_DIR }}/pr_title.txt + pr_numbers_in_title="$pr_numbers_in_title #$pr_number" + fi + done + + commit_hashes=$(cat ${{ env.TEMP_DIR }}/commit_hashes.txt | tr '\n' ' ') + first_commit_hash=$(head -n 1 ${{ env.TEMP_DIR }}/commit_hashes.txt) + cherry_pick_branch="cherry-pick-${first_commit_hash:0:7}" + echo "COMMIT_HASHES=$commit_hashes" >> $GITHUB_ENV + echo "CHERRY_PICK_BRANCH=$cherry_pick_branch" >> $GITHUB_ENV + echo "pr_numbers_in_title=$pr_numbers_in_title" >> $GITHUB_ENV + + - name: Pull and Cherry-pick Commits, Then Push + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + BOT_TOKEN: ${{ secrets.BOT_TOKEN }} + run: | + # Fetch and pull the latest changes from the target branch + git fetch origin + git checkout $TARGET_BRANCH + git pull origin $TARGET_BRANCH + + # Create a new branch for cherry-picking + git checkout -b $CHERRY_PICK_BRANCH + + # Cherry-pick the commits and handle conflicts + for commit_hash in $COMMIT_HASHES; do + echo "Attempting to cherry-pick commit $commit_hash" + if ! git cherry-pick "$commit_hash" --strategy=recursive -X theirs; then + echo "Conflict detected for $commit_hash. Resolving with incoming changes." + conflict_files=$(git diff --name-only --diff-filter=U) + echo "Conflicting files:" + echo "$conflict_files" + + for file in $conflict_files; do + if [ -f "$file" ]; then + echo "Resolving conflict for $file" + git add "$file" + else + echo "File $file has been deleted. Skipping." + git rm "$file" + fi + done + + echo "Conflicts resolved. Continuing cherry-pick." + git cherry-pick --continue + else + echo "Cherry-pick successful for commit $commit_hash." + fi + done + + # Push the cherry-pick branch to the repository + git remote set-url origin "https://${BOT_TOKEN}@github.com/${{ github.repository }}.git" + git push origin $CHERRY_PICK_BRANCH --force + + - name: Create Pull Request + run: | + # Prepare and create the PR + pr_title="deps: Merge ${{ env.pr_numbers_in_title }} PRs into $TARGET_BRANCH" + pr_body=$(cat ${{ env.TEMP_DIR }}/pr_body.txt) + + echo "Prepared PR title:" + echo "$pr_title" + echo "Prepared PR body:" + echo "$pr_body" + + # Create the PR using the GitHub API + response=$(curl -s -X POST -H "Authorization: token $BOT_TOKEN" \ + -H "Accept: application/vnd.github+json" \ + https://api.github.com/repos/${{ github.repository }}/pulls \ + -d "$(jq -n --arg title "$pr_title" \ + --arg head "$CHERRY_PICK_BRANCH" \ + --arg base "$TARGET_BRANCH" \ + --arg body "$pr_body" \ + '{title: $title, head: $head, base: $base, body: $body}')") + + pr_number=$(echo "$response" | jq -r '.number') + echo "$pr_number" > ${{ env.TEMP_DIR }}/created_pr_number.txt + echo "Created PR #$pr_number" + + - name: Add Label to Created Pull Request + run: | + # Add 'milestone-merge' label to the created PR + pr_number=$(cat ${{ env.TEMP_DIR }}/created_pr_number.txt) + echo "Adding label to PR #$pr_number" + + curl -s -X POST -H "Authorization: token $GITHUB_TOKEN" \ + -H "Accept: application/vnd.github+json" \ + -d '{"labels": ["milestone-merge"]}' \ + "https://api.github.com/repos/${{ github.repository }}/issues/$pr_number/labels" + + echo "Added 'milestone-merge' label to PR #$pr_number." From 4de3befd60e5ce6a0ea48826ebc07158e371cbf3 Mon Sep 17 00:00:00 2001 From: Monet Lee Date: Mon, 28 Oct 2024 14:05:45 +0800 Subject: [PATCH 021/199] build: improve workflows logic. (#2801) --- .github/workflows/go-build-test.yml | 4 ---- .github/workflows/merge-from-milestone.yml | 3 --- 2 files changed, 7 deletions(-) diff --git a/.github/workflows/go-build-test.yml b/.github/workflows/go-build-test.yml index 1ed8f0397..5b37bf47d 100644 --- a/.github/workflows/go-build-test.yml +++ b/.github/workflows/go-build-test.yml @@ -2,11 +2,7 @@ name: Go Build Test on: push: - branches: - - main pull_request: - branches: - - main paths-ignore: - '**/*.md' diff --git a/.github/workflows/merge-from-milestone.yml b/.github/workflows/merge-from-milestone.yml index d9110b0c3..44b4f81f4 100644 --- a/.github/workflows/merge-from-milestone.yml +++ b/.github/workflows/merge-from-milestone.yml @@ -17,9 +17,6 @@ on: required: true default: 'pre-release-v3.8.2' - schedule: - - cron: '0 2 * * 0' - env: MILESTONE_NAME: ${{ github.event.inputs.milestone_name || 'v3.8.2' }} TARGET_BRANCH: ${{ github.event.inputs.target_branch || 'pre-release-v3.8.2' }} From 53cf2c0540cf7009100fedc3b10e6b7676458962 Mon Sep 17 00:00:00 2001 From: Monet Lee Date: Tue, 29 Oct 2024 10:42:04 +0800 Subject: [PATCH 022/199] fix: improve time condition check mehtod. (#2804) * fix: improve time condition check mehtod. * fix --- internal/rpc/group/group.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/rpc/group/group.go b/internal/rpc/group/group.go index ef917d539..c59d9d3f4 100644 --- a/internal/rpc/group/group.go +++ b/internal/rpc/group/group.go @@ -1219,7 +1219,7 @@ func (g *groupServer) TransferGroupOwner(ctx context.Context, req *pbgroup.Trans } } - if newOwner.MuteEndTime != time.Unix(0, 0) { + if newOwner.MuteEndTime.After(time.Now()) { if _, err := g.CancelMuteGroupMember(ctx, &pbgroup.CancelMuteGroupMemberReq{ GroupID: group.GroupID, UserID: req.NewOwnerUserID}); err != nil { From a4eae351185e78a116242f4fe53c66f4a4187344 Mon Sep 17 00:00:00 2001 From: icey-yu <119291641+icey-yu@users.noreply.github.com> Date: Tue, 29 Oct 2024 12:29:25 +0800 Subject: [PATCH 023/199] fix: webhook before online push (#2805) --- internal/push/callback.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/internal/push/callback.go b/internal/push/callback.go index 889729582..f8e17bb8c 100644 --- a/internal/push/callback.go +++ b/internal/push/callback.go @@ -24,7 +24,6 @@ import ( "github.com/openimsdk/protocol/constant" "github.com/openimsdk/protocol/sdkws" "github.com/openimsdk/tools/mcontext" - "github.com/openimsdk/tools/utils/datautil" ) func (c *ConsumerHandler) webhookBeforeOfflinePush(ctx context.Context, before *config.BeforeConfig, userIDs []string, msg *sdkws.MsgData, offlinePushUserIDs *[]string) error { @@ -70,7 +69,7 @@ func (c *ConsumerHandler) webhookBeforeOfflinePush(ctx context.Context, before * func (c *ConsumerHandler) webhookBeforeOnlinePush(ctx context.Context, before *config.BeforeConfig, userIDs []string, msg *sdkws.MsgData) error { return webhook.WithCondition(ctx, before, func(ctx context.Context) error { - if datautil.Contain(msg.SendID, userIDs...) || msg.ContentType == constant.Typing { + if msg.ContentType == constant.Typing { return nil } req := callbackstruct.CallbackBeforePushReq{ From 07d30a645bd3ddab5d173df312e7ae4985570067 Mon Sep 17 00:00:00 2001 From: OpenIM-Gordon <46924906+FGadvancer@users.noreply.github.com> Date: Wed, 30 Oct 2024 10:02:03 +0800 Subject: [PATCH 024/199] fix: set own read seq in MongoDB when sender send a message. (#2808) --- internal/msgtransfer/init.go | 3 + .../msgtransfer/online_history_msg_handler.go | 99 +++++++++++++------ pkg/common/storage/controller/msg_transfer.go | 35 ++++--- 3 files changed, 90 insertions(+), 47 deletions(-) diff --git a/internal/msgtransfer/init.go b/internal/msgtransfer/init.go index 7dc2ebeea..f11cfde1a 100644 --- a/internal/msgtransfer/init.go +++ b/internal/msgtransfer/init.go @@ -128,6 +128,7 @@ func (m *MsgTransfer) Start(index int, config *Config) error { go m.historyCH.historyConsumerGroup.RegisterHandleAndConsumer(m.ctx, m.historyCH) go m.historyMongoCH.historyConsumerGroup.RegisterHandleAndConsumer(m.ctx, m.historyMongoCH) + go m.historyCH.HandleUserHasReadSeqMessages(m.ctx) err := m.historyCH.redisMessageBatches.Start() if err != nil { return err @@ -157,12 +158,14 @@ func (m *MsgTransfer) Start(index int, config *Config) error { // graceful close kafka client. m.cancel() m.historyCH.redisMessageBatches.Close() + m.historyCH.Close() m.historyCH.historyConsumerGroup.Close() m.historyMongoCH.historyConsumerGroup.Close() return nil case <-netDone: m.cancel() m.historyCH.redisMessageBatches.Close() + m.historyCH.Close() m.historyCH.historyConsumerGroup.Close() m.historyMongoCH.historyConsumerGroup.Close() close(netDone) diff --git a/internal/msgtransfer/online_history_msg_handler.go b/internal/msgtransfer/online_history_msg_handler.go index b0078649c..84453c8df 100644 --- a/internal/msgtransfer/online_history_msg_handler.go +++ b/internal/msgtransfer/online_history_msg_handler.go @@ -18,8 +18,10 @@ import ( "context" "encoding/json" "errors" + "github.com/openimsdk/open-im-server/v3/pkg/common/prommetrics" "strconv" "strings" + "sync" "time" "github.com/IBM/sarama" @@ -40,11 +42,12 @@ import ( ) const ( - size = 500 - mainDataBuffer = 500 - subChanBuffer = 50 - worker = 50 - interval = 100 * time.Millisecond + size = 500 + mainDataBuffer = 500 + subChanBuffer = 50 + worker = 50 + interval = 100 * time.Millisecond + hasReadChanBuffer = 1000 ) type ContextMsg struct { @@ -52,14 +55,23 @@ type ContextMsg struct { ctx context.Context } +// This structure is used for asynchronously writing the sender’s read sequence (seq) regarding a message into MongoDB. +// For example, if the sender sends a message with a seq of 10, then their own read seq for this conversation should be set to 10. +type userHasReadSeq struct { + conversationID string + userHasReadMap map[string]int64 +} + type OnlineHistoryRedisConsumerHandler struct { historyConsumerGroup *kafka.MConsumerGroup redisMessageBatches *batcher.Batcher[sarama.ConsumerMessage] - msgTransferDatabase controller.MsgTransferDatabase - conversationRpcClient *rpcclient.ConversationRpcClient - groupRpcClient *rpcclient.GroupRpcClient + msgTransferDatabase controller.MsgTransferDatabase + conversationRpcClient *rpcclient.ConversationRpcClient + groupRpcClient *rpcclient.GroupRpcClient + conversationUserHasReadChan chan *userHasReadSeq + wg sync.WaitGroup } func NewOnlineHistoryRedisConsumerHandler(kafkaConf *config.Kafka, database controller.MsgTransferDatabase, @@ -70,6 +82,8 @@ func NewOnlineHistoryRedisConsumerHandler(kafkaConf *config.Kafka, database cont } var och OnlineHistoryRedisConsumerHandler och.msgTransferDatabase = database + och.conversationUserHasReadChan = make(chan *userHasReadSeq, hasReadChanBuffer) + och.wg.Add(1) b := batcher.New[sarama.ConsumerMessage]( batcher.WithSize(size), @@ -115,25 +129,25 @@ func (och *OnlineHistoryRedisConsumerHandler) do(ctx context.Context, channelID } func (och *OnlineHistoryRedisConsumerHandler) doSetReadSeq(ctx context.Context, msgs []*ContextMsg) { - type seqKey struct { - conversationID string - userID string - } - var readSeq map[seqKey]int64 + + var conversationID string + var userSeqMap map[string]int64 for _, msg := range msgs { if msg.message.ContentType != constant.HasReadReceipt { continue } var elem sdkws.NotificationElem if err := json.Unmarshal(msg.message.Content, &elem); err != nil { - log.ZError(ctx, "handlerConversationRead Unmarshal NotificationElem msg err", err, "msg", msg) + log.ZWarn(ctx, "handlerConversationRead Unmarshal NotificationElem msg err", err, "msg", msg) continue } var tips sdkws.MarkAsReadTips if err := json.Unmarshal([]byte(elem.Detail), &tips); err != nil { - log.ZError(ctx, "handlerConversationRead Unmarshal MarkAsReadTips msg err", err, "msg", msg) + log.ZWarn(ctx, "handlerConversationRead Unmarshal MarkAsReadTips msg err", err, "msg", msg) continue } + //The conversation ID for each batch of messages processed by the batcher is the same. + conversationID = tips.ConversationID if len(tips.Seqs) > 0 { for _, seq := range tips.Seqs { if tips.HasReadSeq < seq { @@ -146,26 +160,25 @@ func (och *OnlineHistoryRedisConsumerHandler) doSetReadSeq(ctx context.Context, if tips.HasReadSeq < 0 { continue } - if readSeq == nil { - readSeq = make(map[seqKey]int64) - } - key := seqKey{ - conversationID: tips.ConversationID, - userID: tips.MarkAsReadUserID, + if userSeqMap == nil { + userSeqMap = make(map[string]int64) } - if readSeq[key] > tips.HasReadSeq { + + if userSeqMap[tips.MarkAsReadUserID] > tips.HasReadSeq { continue } - readSeq[key] = tips.HasReadSeq + userSeqMap[tips.MarkAsReadUserID] = tips.HasReadSeq } - if readSeq == nil { + if userSeqMap == nil { return } - for key, seq := range readSeq { - if err := och.msgTransferDatabase.SetHasReadSeqToDB(ctx, key.userID, key.conversationID, seq); err != nil { - log.ZError(ctx, "set read seq to db error", err, "userID", key.userID, "conversationID", key.conversationID, "seq", seq) - } + if len(conversationID) == 0 { + log.ZWarn(ctx, "conversation err", nil, "conversationID", conversationID) } + if err := och.msgTransferDatabase.SetHasReadSeqToDB(ctx, conversationID, userSeqMap); err != nil { + log.ZWarn(ctx, "set read seq to db error", err, "conversationID", conversationID, "userSeqMap", userSeqMap) + } + } func (och *OnlineHistoryRedisConsumerHandler) parseConsumerMessages(ctx context.Context, consumerMessages []*sarama.ConsumerMessage) []*ContextMsg { @@ -250,12 +263,21 @@ func (och *OnlineHistoryRedisConsumerHandler) handleMsg(ctx context.Context, key } if len(storageMessageList) > 0 { msg := storageMessageList[0] - lastSeq, isNewConversation, err := och.msgTransferDatabase.BatchInsertChat2Cache(ctx, conversationID, storageMessageList) + lastSeq, isNewConversation, userSeqMap, err := och.msgTransferDatabase.BatchInsertChat2Cache(ctx, conversationID, storageMessageList) if err != nil && !errors.Is(errs.Unwrap(err), redis.Nil) { - log.ZError(ctx, "batch data insert to redis err", err, "storageMsgList", storageMessageList) + log.ZWarn(ctx, "batch data insert to redis err", err, "storageMsgList", storageMessageList) return } log.ZInfo(ctx, "BatchInsertChat2Cache end") + err = och.msgTransferDatabase.SetHasReadSeqs(ctx, conversationID, userSeqMap) + if err != nil { + log.ZWarn(ctx, "SetHasReadSeqs error", err, "userSeqMap", userSeqMap, "conversationID", conversationID) + prommetrics.SeqSetFailedCounter.Inc() + } + och.conversationUserHasReadChan <- &userHasReadSeq{ + conversationID: conversationID, + userHasReadMap: userSeqMap, + } if isNewConversation { switch msg.SessionType { @@ -308,7 +330,7 @@ func (och *OnlineHistoryRedisConsumerHandler) handleNotification(ctx context.Con storageMessageList = append(storageMessageList, msg.message) } if len(storageMessageList) > 0 { - lastSeq, _, err := och.msgTransferDatabase.BatchInsertChat2Cache(ctx, conversationID, storageMessageList) + lastSeq, _, _, err := och.msgTransferDatabase.BatchInsertChat2Cache(ctx, conversationID, storageMessageList) if err != nil { log.ZError(ctx, "notification batch insert to redis error", err, "conversationID", conversationID, "storageList", storageMessageList) @@ -323,6 +345,21 @@ func (och *OnlineHistoryRedisConsumerHandler) handleNotification(ctx context.Con och.toPushTopic(ctx, key, conversationID, storageList) } } +func (och *OnlineHistoryRedisConsumerHandler) HandleUserHasReadSeqMessages(ctx context.Context) { + defer och.wg.Done() + + for msg := range och.conversationUserHasReadChan { + if err := och.msgTransferDatabase.SetHasReadSeqToDB(ctx, msg.conversationID, msg.userHasReadMap); err != nil { + log.ZWarn(ctx, "set read seq to db error", err, "conversationID", msg.conversationID, "userSeqMap", msg.userHasReadMap) + } + } + + log.ZInfo(ctx, "Channel closed, exiting handleUserHasReadSeqMessages") +} +func (och *OnlineHistoryRedisConsumerHandler) Close() { + close(och.conversationUserHasReadChan) + och.wg.Wait() +} func (och *OnlineHistoryRedisConsumerHandler) toPushTopic(ctx context.Context, key, conversationID string, msgs []*ContextMsg) { for _, v := range msgs { diff --git a/pkg/common/storage/controller/msg_transfer.go b/pkg/common/storage/controller/msg_transfer.go index c6013dbc1..1ecd786aa 100644 --- a/pkg/common/storage/controller/msg_transfer.go +++ b/pkg/common/storage/controller/msg_transfer.go @@ -24,8 +24,11 @@ type MsgTransferDatabase interface { DeleteMessagesFromCache(ctx context.Context, conversationID string, seqs []int64) error // BatchInsertChat2Cache increments the sequence number and then batch inserts messages into the cache. - BatchInsertChat2Cache(ctx context.Context, conversationID string, msgs []*sdkws.MsgData) (seq int64, isNewConversation bool, err error) - SetHasReadSeqToDB(ctx context.Context, userID string, conversationID string, hasReadSeq int64) error + BatchInsertChat2Cache(ctx context.Context, conversationID string, msgs []*sdkws.MsgData) (seq int64, isNewConversation bool, userHasReadMap map[string]int64, err error) + + SetHasReadSeqs(ctx context.Context, conversationID string, userSeqMap map[string]int64) error + + SetHasReadSeqToDB(ctx context.Context, conversationID string, userSeqMap map[string]int64) error // to mq MsgToPushMQ(ctx context.Context, key, conversationID string, msg2mq *sdkws.MsgData) (int32, int64, error) @@ -219,18 +222,18 @@ func (db *msgTransferDatabase) DeleteMessagesFromCache(ctx context.Context, conv return db.msg.DeleteMessagesFromCache(ctx, conversationID, seqs) } -func (db *msgTransferDatabase) BatchInsertChat2Cache(ctx context.Context, conversationID string, msgs []*sdkws.MsgData) (seq int64, isNew bool, err error) { +func (db *msgTransferDatabase) BatchInsertChat2Cache(ctx context.Context, conversationID string, msgs []*sdkws.MsgData) (seq int64, isNew bool, userHasReadMap map[string]int64, err error) { lenList := len(msgs) if int64(lenList) > db.msgTable.GetSingleGocMsgNum() { - return 0, false, errs.New("message count exceeds limit", "limit", db.msgTable.GetSingleGocMsgNum()).Wrap() + return 0, false, nil, errs.New("message count exceeds limit", "limit", db.msgTable.GetSingleGocMsgNum()).Wrap() } if lenList < 1 { - return 0, false, errs.New("no messages to insert", "minCount", 1).Wrap() + return 0, false, nil, errs.New("no messages to insert", "minCount", 1).Wrap() } currentMaxSeq, err := db.seqConversation.Malloc(ctx, conversationID, int64(len(msgs))) if err != nil { log.ZError(ctx, "storage.seq.Malloc", err) - return 0, false, err + return 0, false, nil, err } isNew = currentMaxSeq == 0 lastMaxSeq := currentMaxSeq @@ -248,25 +251,25 @@ func (db *msgTransferDatabase) BatchInsertChat2Cache(ctx context.Context, conver } else { prommetrics.MsgInsertRedisSuccessCounter.Inc() } - err = db.setHasReadSeqs(ctx, conversationID, userSeqMap) - if err != nil { - log.ZError(ctx, "SetHasReadSeqs error", err, "userSeqMap", userSeqMap, "conversationID", conversationID) - prommetrics.SeqSetFailedCounter.Inc() - } - return lastMaxSeq, isNew, errs.Wrap(err) + return lastMaxSeq, isNew, userSeqMap, errs.Wrap(err) } -func (db *msgTransferDatabase) setHasReadSeqs(ctx context.Context, conversationID string, userSeqMap map[string]int64) error { +func (db *msgTransferDatabase) SetHasReadSeqs(ctx context.Context, conversationID string, userSeqMap map[string]int64) error { for userID, seq := range userSeqMap { - if err := db.seqUser.SetUserReadSeqToDB(ctx, conversationID, userID, seq); err != nil { + if err := db.seqUser.SetUserReadSeq(ctx, conversationID, userID, seq); err != nil { return err } } return nil } -func (db *msgTransferDatabase) SetHasReadSeqToDB(ctx context.Context, userID string, conversationID string, hasReadSeq int64) error { - return db.seqUser.SetUserReadSeqToDB(ctx, conversationID, userID, hasReadSeq) +func (db *msgTransferDatabase) SetHasReadSeqToDB(ctx context.Context, conversationID string, userSeqMap map[string]int64) error { + for userID, seq := range userSeqMap { + if err := db.seqUser.SetUserReadSeqToDB(ctx, conversationID, userID, seq); err != nil { + return err + } + } + return nil } func (db *msgTransferDatabase) MsgToPushMQ(ctx context.Context, key, conversationID string, msg2mq *sdkws.MsgData) (int32, int64, error) { From 5af41e83ce2ccae406b8359a73598f17086eaff4 Mon Sep 17 00:00:00 2001 From: Monet Lee Date: Wed, 30 Oct 2024 10:07:20 +0800 Subject: [PATCH 025/199] fix: solve err Notification when setGroupInfo. (#2806) * fix: solve err Notification when setGroupInfo. * build: update checkout version. * fix: update notification contents. --- .../cleanup-after-milestone-prs-merged.yml | 2 +- config/notification.yml | 23 +++---------------- internal/rpc/group/group.go | 5 ++-- 3 files changed, 7 insertions(+), 23 deletions(-) diff --git a/.github/workflows/cleanup-after-milestone-prs-merged.yml b/.github/workflows/cleanup-after-milestone-prs-merged.yml index ff4e8029d..8a3e381d6 100644 --- a/.github/workflows/cleanup-after-milestone-prs-merged.yml +++ b/.github/workflows/cleanup-after-milestone-prs-merged.yml @@ -11,7 +11,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4.2.0 - name: Get the PR title and extract PR numbers id: extract_pr_numbers diff --git a/config/notification.yml b/config/notification.yml index 85ca91af1..ba5ca1c21 100644 --- a/config/notification.yml +++ b/config/notification.yml @@ -1,20 +1,3 @@ -# Copyright © 2023 OpenIM. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Determines if a message should be sent. If set to false, it triggers a silent sync without a message. If true, it requires triggering a conversation. -# For rpc notification, send twice: once as a message and once as a notification. -# The options field 'isNotification' indicates if it's a notification. groupCreated: isSendMsg: true # Reliability level of the message sending. @@ -309,9 +292,9 @@ userInfoUpdated: unreadCount: false offlinePush: enable: true - title: Remove a blocked user - desc: Remove a blocked user - ext: Remove a blocked user + title: userInfo updated + desc: userInfo updated + ext: userInfo updated userStatusChanged: isSendMsg: false diff --git a/internal/rpc/group/group.go b/internal/rpc/group/group.go index c59d9d3f4..30ac19a76 100644 --- a/internal/rpc/group/group.go +++ b/internal/rpc/group/group.go @@ -1026,7 +1026,7 @@ func (g *groupServer) SetGroupInfo(ctx context.Context, req *pbgroup.SetGroupInf } num := len(update) if req.GroupInfoForSet.Notification != "" { - num-- + num -= 3 func() { conversation := &pbconversation.ConversationReq{ ConversationID: msgprocessor.GetConversationIDBySessionType(constant.ReadGroupChatType, req.GroupInfoForSet.GroupID), @@ -1133,8 +1133,9 @@ func (g *groupServer) SetGroupInfoEx(ctx context.Context, req *pbgroup.SetGroupI } num := len(updatedData) + if req.Notification != nil { - num-- + num -= 3 if req.Notification.Value != "" { func() { From b85e36e97c27c3f8b432fb96cc820e21178abc74 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?K=C3=BCr=C5=9Fat=20Akta=C5=9F?= Date: Wed, 30 Oct 2024 06:39:17 +0300 Subject: [PATCH 026/199] Introducing OpenIM Guru on Gurubase.io (#2788) --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index a99559cdb..6b1779b25 100644 --- a/README.md +++ b/README.md @@ -16,6 +16,7 @@ [![Best Practices](https://img.shields.io/badge/Best%20Practices-purple?style=for-the-badge)](https://www.bestpractices.dev/projects/8045) [![Good First Issues](https://img.shields.io/github/issues/openimsdk/open-im-server/good%20first%20issue?style=for-the-badge&logo=github)](https://github.com/openimsdk/open-im-server/issues?q=is%3Aissue+is%3Aopen+sort%3Aupdated-desc+label%3A%22good+first+issue%22) [![Language](https://img.shields.io/badge/Language-Go-blue.svg?style=for-the-badge&logo=go&logoColor=white)](https://golang.org/) +[![Gurubase](https://img.shields.io/badge/Gurubase-Ask%20OpenIM%20Guru-006BFF?style=for-the-badge)](https://gurubase.io/g/openim)

From ea477fd91f1c3285f0819d2940ca6855a272f79c Mon Sep 17 00:00:00 2001 From: chao <48119764+withchao@users.noreply.github.com> Date: Wed, 30 Oct 2024 14:38:09 +0800 Subject: [PATCH 027/199] feat: support app update service (#2811) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * update gomake version * update gomake version * fix: seq conversion bug * fix: redis pipe exec * fix: ImportFriends * fix: A large number of logs keysAndValues ​​length is not even * feat: mark read aggregate write * feat: online status supports redis cluster * feat: online status supports redis cluster * feat: online status supports redis cluster * merge * merge * read seq is written to mongo * read seq is written to mongo * fix: invitation to join group notification * fix: friend op_user_id * feat: optimizing asynchronous context * feat: optimizing memamq size * feat: add GetSeqMessage * feat: GroupApplicationAgreeMemberEnterNotification * feat: GroupApplicationAgreeMemberEnterNotification * feat: go.mod * feat: go.mod * feat: join group notification and get seq * feat: join group notification and get seq * feat: avoid pulling messages from sessions with a large number of max seq values of 0 * feat: API supports gzip * go.mod * fix: nil pointer error on close * fix: listen error * fix: listen error * update go.mod * feat: add log * fix: token parse token value * fix: GetMsgBySeqs boundary issues * fix: sn_ not sort * fix: sn_ not sort * fix: sn_ not sort * fix: jssdk add * fix: jssdk support * fix: jssdk support * fix: jssdk support * fix: the message I sent is not set to read seq in mongodb * fix: cannot modify group member avatars * fix: MemberEnterNotification * fix: MemberEnterNotification * fix: MsgData status * feat: add ApplicationVersion * feat: add ApplicationVersion * feat: add ApplicationVersion --------- Co-authored-by: withchao --- go.mod | 2 +- go.sum | 4 +-- internal/rpc/third/application.go | 27 ++++++++++++++++--- pkg/common/storage/cache/application.go | 2 +- .../storage/cache/cachekey/application.go | 10 +++++-- pkg/common/storage/cache/redis/application.go | 15 ++++++----- pkg/common/storage/controller/application.go | 6 ++--- pkg/common/storage/database/application.go | 2 +- .../storage/database/mgo/application.go | 5 ++-- pkg/common/storage/model/application.go | 1 + 10 files changed, 51 insertions(+), 23 deletions(-) diff --git a/go.mod b/go.mod index 5e5a8b5be..57d01c38f 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/gorilla/websocket v1.5.1 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/mitchellh/mapstructure v1.5.0 - github.com/openimsdk/protocol v0.0.72-alpha.47 + github.com/openimsdk/protocol v0.0.72-alpha.48 github.com/openimsdk/tools v0.0.50-alpha.16 github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 diff --git a/go.sum b/go.sum index 53109c890..94553cedd 100644 --- a/go.sum +++ b/go.sum @@ -319,8 +319,8 @@ github.com/onsi/gomega v1.25.0 h1:Vw7br2PCDYijJHSfBOWhov+8cAnUf8MfMaIOV323l6Y= github.com/onsi/gomega v1.25.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdMPGhM= github.com/openimsdk/gomake v0.0.14-alpha.5 h1:VY9c5x515lTfmdhhPjMvR3BBRrRquAUCFsz7t7vbv7Y= github.com/openimsdk/gomake v0.0.14-alpha.5/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= -github.com/openimsdk/protocol v0.0.72-alpha.47 h1:FGHnEwsA05GxT3vnz7YH3fbVkuoO3P71ZZgkQQ71MjA= -github.com/openimsdk/protocol v0.0.72-alpha.47/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= +github.com/openimsdk/protocol v0.0.72-alpha.48 h1:DVeT8Kej6OjB9bsxQ0q6FU160anwfPuVmAL/1J6VzqM= +github.com/openimsdk/protocol v0.0.72-alpha.48/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= github.com/openimsdk/tools v0.0.50-alpha.16 h1:bC1AQvJMuOHtZm8LZRvN8L5mH1Ws2VYdL+TLTs1iGSc= github.com/openimsdk/tools v0.0.50-alpha.16/go.mod h1:h1cYmfyaVtgFbKmb1Cfsl8XwUOMTt8ubVUQrdGtsUh4= github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= diff --git a/internal/rpc/third/application.go b/internal/rpc/third/application.go index a6556055c..fd58e4071 100644 --- a/internal/rpc/third/application.go +++ b/internal/rpc/third/application.go @@ -31,21 +31,38 @@ func (t *thirdServer) db2pbApplication(val *model.Application) *third.Applicatio Text: val.Text, Force: val.Force, Latest: val.Latest, + Hot: val.Hot, CreateTime: val.CreateTime.UnixMilli(), } } -func (t *thirdServer) LatestApplicationVersion(ctx context.Context, req *third.LatestApplicationVersionReq) (*third.LatestApplicationVersionResp, error) { - res, err := t.applicationDatabase.LatestVersion(ctx, req.Platform) +func (t *thirdServer) getLatestApplicationVersion(ctx context.Context, platform string, hot bool) (*third.ApplicationVersion, error) { + res, err := t.applicationDatabase.LatestVersion(ctx, platform, hot) if err == nil { - return &third.LatestApplicationVersionResp{Version: t.db2pbApplication(res)}, nil + return t.db2pbApplication(res), nil } else if IsNotFound(err) { - return &third.LatestApplicationVersionResp{}, nil + return nil, nil } else { return nil, err } } +func (t *thirdServer) LatestApplicationVersion(ctx context.Context, req *third.LatestApplicationVersionReq) (*third.LatestApplicationVersionResp, error) { + var ( + resp third.LatestApplicationVersionResp + err error + ) + resp.Version, err = t.getLatestApplicationVersion(ctx, req.Platform, false) + if err != nil { + return nil, err + } + resp.Hot, err = t.getLatestApplicationVersion(ctx, req.Platform, true) + if err != nil { + return nil, err + } + return &resp, nil +} + func (t *thirdServer) AddApplicationVersion(ctx context.Context, req *third.AddApplicationVersionReq) (*third.AddApplicationVersionResp, error) { if err := authverify.CheckAdmin(ctx, t.config.Share.IMAdminUserID); err != nil { return nil, err @@ -58,6 +75,7 @@ func (t *thirdServer) AddApplicationVersion(ctx context.Context, req *third.AddA Text: req.Text, Force: req.Force, Latest: req.Latest, + Hot: req.Hot, CreateTime: time.Now(), } if err := t.applicationDatabase.AddVersion(ctx, val); err != nil { @@ -81,6 +99,7 @@ func (t *thirdServer) UpdateApplicationVersion(ctx context.Context, req *third.U putUpdate(update, "text", req.Text) putUpdate(update, "force", req.Force) putUpdate(update, "latest", req.Latest) + putUpdate(update, "hot", req.Hot) if err := t.applicationDatabase.UpdateVersion(ctx, oid, update); err != nil { return nil, err } diff --git a/pkg/common/storage/cache/application.go b/pkg/common/storage/cache/application.go index 588732ec8..bb23a3a17 100644 --- a/pkg/common/storage/cache/application.go +++ b/pkg/common/storage/cache/application.go @@ -6,6 +6,6 @@ import ( ) type ApplicationCache interface { - LatestVersion(ctx context.Context, platform string) (*model.Application, error) + LatestVersion(ctx context.Context, platform string, hot bool) (*model.Application, error) DeleteCache(ctx context.Context, platforms []string) error } diff --git a/pkg/common/storage/cache/cachekey/application.go b/pkg/common/storage/cache/cachekey/application.go index 032adba3c..d7835b289 100644 --- a/pkg/common/storage/cache/cachekey/application.go +++ b/pkg/common/storage/cache/cachekey/application.go @@ -4,6 +4,12 @@ const ( ApplicationLatestVersion = "APPLICATION_LATEST_VERSION:" ) -func GetApplicationLatestVersionKey(platform string) string { - return ApplicationLatestVersion + platform +func GetApplicationLatestVersionKey(platform string, hot bool) string { + var hotStr string + if hot { + hotStr = "1:" + } else { + hotStr = "0:" + } + return ApplicationLatestVersion + hotStr + platform } diff --git a/pkg/common/storage/cache/redis/application.go b/pkg/common/storage/cache/redis/application.go index 4a7a4ced6..a858fa25b 100644 --- a/pkg/common/storage/cache/redis/application.go +++ b/pkg/common/storage/cache/redis/application.go @@ -6,7 +6,6 @@ import ( "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/cachekey" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" - "github.com/openimsdk/tools/utils/datautil" "github.com/redis/go-redis/v9" "time" ) @@ -27,9 +26,9 @@ type ApplicationRedisCache struct { expireTime time.Duration } -func (a *ApplicationRedisCache) LatestVersion(ctx context.Context, platform string) (*model.Application, error) { - return getCache(ctx, a.rcClient, cachekey.GetApplicationLatestVersionKey(platform), a.expireTime, func(ctx context.Context) (*model.Application, error) { - return a.db.LatestVersion(ctx, platform) +func (a *ApplicationRedisCache) LatestVersion(ctx context.Context, platform string, hot bool) (*model.Application, error) { + return getCache(ctx, a.rcClient, cachekey.GetApplicationLatestVersionKey(platform, hot), a.expireTime, func(ctx context.Context) (*model.Application, error) { + return a.db.LatestVersion(ctx, platform, hot) }) } @@ -37,7 +36,9 @@ func (a *ApplicationRedisCache) DeleteCache(ctx context.Context, platforms []str if len(platforms) == 0 { return nil } - return a.deleter.ExecDelWithKeys(ctx, datautil.Slice(platforms, func(platform string) string { - return cachekey.GetApplicationLatestVersionKey(platform) - })) + keys := make([]string, 0, len(platforms)*2) + for _, platform := range platforms { + keys = append(keys, cachekey.GetApplicationLatestVersionKey(platform, true), cachekey.GetApplicationLatestVersionKey(platform, false)) + } + return a.deleter.ExecDelWithKeys(ctx, keys) } diff --git a/pkg/common/storage/controller/application.go b/pkg/common/storage/controller/application.go index 72bca07ef..388805772 100644 --- a/pkg/common/storage/controller/application.go +++ b/pkg/common/storage/controller/application.go @@ -10,7 +10,7 @@ import ( ) type ApplicationDatabase interface { - LatestVersion(ctx context.Context, platform string) (*model.Application, error) + LatestVersion(ctx context.Context, platform string, hot bool) (*model.Application, error) AddVersion(ctx context.Context, val *model.Application) error UpdateVersion(ctx context.Context, id primitive.ObjectID, update map[string]any) error DeleteVersion(ctx context.Context, id []primitive.ObjectID) error @@ -26,8 +26,8 @@ type applicationDatabase struct { cache cache.ApplicationCache } -func (a *applicationDatabase) LatestVersion(ctx context.Context, platform string) (*model.Application, error) { - return a.cache.LatestVersion(ctx, platform) +func (a *applicationDatabase) LatestVersion(ctx context.Context, platform string, hot bool) (*model.Application, error) { + return a.cache.LatestVersion(ctx, platform, hot) } func (a *applicationDatabase) AddVersion(ctx context.Context, val *model.Application) error { diff --git a/pkg/common/storage/database/application.go b/pkg/common/storage/database/application.go index c98ae74c8..50796be62 100644 --- a/pkg/common/storage/database/application.go +++ b/pkg/common/storage/database/application.go @@ -8,7 +8,7 @@ import ( ) type Application interface { - LatestVersion(ctx context.Context, platform string) (*model.Application, error) + LatestVersion(ctx context.Context, platform string, hot bool) (*model.Application, error) AddVersion(ctx context.Context, val *model.Application) error UpdateVersion(ctx context.Context, id primitive.ObjectID, update map[string]any) error DeleteVersion(ctx context.Context, id []primitive.ObjectID) error diff --git a/pkg/common/storage/database/mgo/application.go b/pkg/common/storage/database/mgo/application.go index e59c0560a..e28965247 100644 --- a/pkg/common/storage/database/mgo/application.go +++ b/pkg/common/storage/database/mgo/application.go @@ -18,6 +18,7 @@ func NewApplicationMgo(db *mongo.Database) (*ApplicationMgo, error) { Keys: bson.D{ {Key: "platform", Value: 1}, {Key: "version", Value: 1}, + {Key: "hot", Value: 1}, }, Options: options.Index().SetUnique(true), }, @@ -41,8 +42,8 @@ func (a *ApplicationMgo) sort() any { return bson.D{{"latest", -1}, {"_id", -1}} } -func (a *ApplicationMgo) LatestVersion(ctx context.Context, platform string) (*model.Application, error) { - return mongoutil.FindOne[*model.Application](ctx, a.coll, bson.M{"platform": platform}, options.FindOne().SetSort(a.sort())) +func (a *ApplicationMgo) LatestVersion(ctx context.Context, platform string, hot bool) (*model.Application, error) { + return mongoutil.FindOne[*model.Application](ctx, a.coll, bson.M{"platform": platform, "hot": hot}, options.FindOne().SetSort(a.sort())) } func (a *ApplicationMgo) AddVersion(ctx context.Context, val *model.Application) error { diff --git a/pkg/common/storage/model/application.go b/pkg/common/storage/model/application.go index f5bae2be6..b09b0e894 100644 --- a/pkg/common/storage/model/application.go +++ b/pkg/common/storage/model/application.go @@ -8,6 +8,7 @@ import ( type Application struct { ID primitive.ObjectID `bson:"_id"` Platform string `bson:"platform"` + Hot bool `bson:"hot"` Version string `bson:"version"` Url string `bson:"url"` Text string `bson:"text"` From 61fc9bffec59da129b2b8cfe9274e0a423da8c8b Mon Sep 17 00:00:00 2001 From: chao <48119764+withchao@users.noreply.github.com> Date: Wed, 30 Oct 2024 17:12:33 +0800 Subject: [PATCH 028/199] feat: ApplicationVersion move chat (#2813) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * update gomake version * update gomake version * fix: seq conversion bug * fix: redis pipe exec * fix: ImportFriends * fix: A large number of logs keysAndValues ​​length is not even * feat: mark read aggregate write * feat: online status supports redis cluster * feat: online status supports redis cluster * feat: online status supports redis cluster * merge * merge * read seq is written to mongo * read seq is written to mongo * fix: invitation to join group notification * fix: friend op_user_id * feat: optimizing asynchronous context * feat: optimizing memamq size * feat: add GetSeqMessage * feat: GroupApplicationAgreeMemberEnterNotification * feat: GroupApplicationAgreeMemberEnterNotification * feat: go.mod * feat: go.mod * feat: join group notification and get seq * feat: join group notification and get seq * feat: avoid pulling messages from sessions with a large number of max seq values of 0 * feat: API supports gzip * go.mod * fix: nil pointer error on close * fix: listen error * fix: listen error * update go.mod * feat: add log * fix: token parse token value * fix: GetMsgBySeqs boundary issues * fix: sn_ not sort * fix: sn_ not sort * fix: sn_ not sort * fix: jssdk add * fix: jssdk support * fix: jssdk support * fix: jssdk support * fix: the message I sent is not set to read seq in mongodb * fix: cannot modify group member avatars * fix: MemberEnterNotification * fix: MemberEnterNotification * fix: MsgData status * feat: add ApplicationVersion * feat: ApplicationVersion move chat --------- Co-authored-by: withchao --- go.mod | 2 +- go.sum | 4 +- internal/api/router.go | 9 -- internal/api/third.go | 20 --- internal/rpc/third/application.go | 136 ------------------ internal/rpc/third/third.go | 30 ++-- pkg/common/storage/cache/application.go | 11 -- .../storage/cache/cachekey/application.go | 15 -- pkg/common/storage/cache/redis/application.go | 44 ------ pkg/common/storage/controller/application.go | 69 --------- pkg/common/storage/database/application.go | 17 --- .../storage/database/mgo/application.go | 83 ----------- 12 files changed, 15 insertions(+), 425 deletions(-) delete mode 100644 internal/rpc/third/application.go delete mode 100644 pkg/common/storage/cache/application.go delete mode 100644 pkg/common/storage/cache/cachekey/application.go delete mode 100644 pkg/common/storage/cache/redis/application.go delete mode 100644 pkg/common/storage/controller/application.go delete mode 100644 pkg/common/storage/database/application.go delete mode 100644 pkg/common/storage/database/mgo/application.go diff --git a/go.mod b/go.mod index 57d01c38f..7cd4e4550 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/gorilla/websocket v1.5.1 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/mitchellh/mapstructure v1.5.0 - github.com/openimsdk/protocol v0.0.72-alpha.48 + github.com/openimsdk/protocol v0.0.72-alpha.51 github.com/openimsdk/tools v0.0.50-alpha.16 github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 diff --git a/go.sum b/go.sum index 94553cedd..7cc45616f 100644 --- a/go.sum +++ b/go.sum @@ -319,8 +319,8 @@ github.com/onsi/gomega v1.25.0 h1:Vw7br2PCDYijJHSfBOWhov+8cAnUf8MfMaIOV323l6Y= github.com/onsi/gomega v1.25.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdMPGhM= github.com/openimsdk/gomake v0.0.14-alpha.5 h1:VY9c5x515lTfmdhhPjMvR3BBRrRquAUCFsz7t7vbv7Y= github.com/openimsdk/gomake v0.0.14-alpha.5/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= -github.com/openimsdk/protocol v0.0.72-alpha.48 h1:DVeT8Kej6OjB9bsxQ0q6FU160anwfPuVmAL/1J6VzqM= -github.com/openimsdk/protocol v0.0.72-alpha.48/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= +github.com/openimsdk/protocol v0.0.72-alpha.51 h1:G5Yjndp/FRyOJWhoQcSF2x2GvYiAIlqN0vjkvjUPycU= +github.com/openimsdk/protocol v0.0.72-alpha.51/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= github.com/openimsdk/tools v0.0.50-alpha.16 h1:bC1AQvJMuOHtZm8LZRvN8L5mH1Ws2VYdL+TLTs1iGSc= github.com/openimsdk/tools v0.0.50-alpha.16/go.mod h1:h1cYmfyaVtgFbKmb1Cfsl8XwUOMTt8ubVUQrdGtsUh4= github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= diff --git a/internal/api/router.go b/internal/api/router.go index 17c998912..560516d30 100644 --- a/internal/api/router.go +++ b/internal/api/router.go @@ -198,13 +198,6 @@ func newGinRouter(disCov discovery.SvcDiscoveryRegistry, config *Config) *gin.En objectGroup.POST("/initiate_form_data", t.InitiateFormData) objectGroup.POST("/complete_form_data", t.CompleteFormData) objectGroup.GET("/*name", t.ObjectRedirect) - - applicationGroup := r.Group("application") - applicationGroup.POST("/add_version", t.AddApplicationVersion) - applicationGroup.POST("/update_version", t.UpdateApplicationVersion) - applicationGroup.POST("/delete_version", t.DeleteApplicationVersion) - applicationGroup.POST("/latest_version", t.LatestApplicationVersion) - applicationGroup.POST("/page_versions", t.PageApplicationVersion) } // Message msgGroup := r.Group("/msg") @@ -297,6 +290,4 @@ func GinParseToken(authRPC *rpcclient.Auth) gin.HandlerFunc { var Whitelist = []string{ "/auth/get_admin_token", "/auth/parse_token", - "/application/latest_version", - "/application/page_versions", } diff --git a/internal/api/third.go b/internal/api/third.go index 56661ba89..6baa70ee5 100644 --- a/internal/api/third.go +++ b/internal/api/third.go @@ -170,23 +170,3 @@ func (o *ThirdApi) SearchLogs(c *gin.Context) { func (o *ThirdApi) GetPrometheus(c *gin.Context) { c.Redirect(http.StatusFound, o.GrafanaUrl) } - -func (o *ThirdApi) LatestApplicationVersion(c *gin.Context) { - a2r.Call(third.ThirdClient.LatestApplicationVersion, o.Client, c) -} - -func (o *ThirdApi) AddApplicationVersion(c *gin.Context) { - a2r.Call(third.ThirdClient.AddApplicationVersion, o.Client, c) -} - -func (o *ThirdApi) UpdateApplicationVersion(c *gin.Context) { - a2r.Call(third.ThirdClient.UpdateApplicationVersion, o.Client, c) -} - -func (o *ThirdApi) DeleteApplicationVersion(c *gin.Context) { - a2r.Call(third.ThirdClient.DeleteApplicationVersion, o.Client, c) -} - -func (o *ThirdApi) PageApplicationVersion(c *gin.Context) { - a2r.Call(third.ThirdClient.PageApplicationVersion, o.Client, c) -} diff --git a/internal/rpc/third/application.go b/internal/rpc/third/application.go deleted file mode 100644 index fd58e4071..000000000 --- a/internal/rpc/third/application.go +++ /dev/null @@ -1,136 +0,0 @@ -package third - -import ( - "context" - "github.com/openimsdk/open-im-server/v3/pkg/authverify" - "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" - "github.com/openimsdk/protocol/third" - "github.com/openimsdk/tools/errs" - "github.com/openimsdk/tools/utils/datautil" - "github.com/redis/go-redis/v9" - "go.mongodb.org/mongo-driver/bson/primitive" - "go.mongodb.org/mongo-driver/mongo" - "time" -) - -func IsNotFound(err error) bool { - switch errs.Unwrap(err) { - case redis.Nil, mongo.ErrNoDocuments: - return true - default: - return false - } -} - -func (t *thirdServer) db2pbApplication(val *model.Application) *third.ApplicationVersion { - return &third.ApplicationVersion{ - Id: val.ID.Hex(), - Platform: val.Platform, - Version: val.Version, - Url: val.Url, - Text: val.Text, - Force: val.Force, - Latest: val.Latest, - Hot: val.Hot, - CreateTime: val.CreateTime.UnixMilli(), - } -} - -func (t *thirdServer) getLatestApplicationVersion(ctx context.Context, platform string, hot bool) (*third.ApplicationVersion, error) { - res, err := t.applicationDatabase.LatestVersion(ctx, platform, hot) - if err == nil { - return t.db2pbApplication(res), nil - } else if IsNotFound(err) { - return nil, nil - } else { - return nil, err - } -} - -func (t *thirdServer) LatestApplicationVersion(ctx context.Context, req *third.LatestApplicationVersionReq) (*third.LatestApplicationVersionResp, error) { - var ( - resp third.LatestApplicationVersionResp - err error - ) - resp.Version, err = t.getLatestApplicationVersion(ctx, req.Platform, false) - if err != nil { - return nil, err - } - resp.Hot, err = t.getLatestApplicationVersion(ctx, req.Platform, true) - if err != nil { - return nil, err - } - return &resp, nil -} - -func (t *thirdServer) AddApplicationVersion(ctx context.Context, req *third.AddApplicationVersionReq) (*third.AddApplicationVersionResp, error) { - if err := authverify.CheckAdmin(ctx, t.config.Share.IMAdminUserID); err != nil { - return nil, err - } - val := &model.Application{ - ID: primitive.NewObjectID(), - Platform: req.Platform, - Version: req.Version, - Url: req.Url, - Text: req.Text, - Force: req.Force, - Latest: req.Latest, - Hot: req.Hot, - CreateTime: time.Now(), - } - if err := t.applicationDatabase.AddVersion(ctx, val); err != nil { - return nil, err - } - return &third.AddApplicationVersionResp{}, nil -} - -func (t *thirdServer) UpdateApplicationVersion(ctx context.Context, req *third.UpdateApplicationVersionReq) (*third.UpdateApplicationVersionResp, error) { - if err := authverify.CheckAdmin(ctx, t.config.Share.IMAdminUserID); err != nil { - return nil, err - } - oid, err := primitive.ObjectIDFromHex(req.Id) - if err != nil { - return nil, errs.ErrArgs.WrapMsg("invalid id " + err.Error()) - } - update := make(map[string]any) - putUpdate(update, "platform", req.Platform) - putUpdate(update, "version", req.Version) - putUpdate(update, "url", req.Url) - putUpdate(update, "text", req.Text) - putUpdate(update, "force", req.Force) - putUpdate(update, "latest", req.Latest) - putUpdate(update, "hot", req.Hot) - if err := t.applicationDatabase.UpdateVersion(ctx, oid, update); err != nil { - return nil, err - } - return &third.UpdateApplicationVersionResp{}, nil -} - -func (t *thirdServer) DeleteApplicationVersion(ctx context.Context, req *third.DeleteApplicationVersionReq) (*third.DeleteApplicationVersionResp, error) { - if err := authverify.CheckAdmin(ctx, t.config.Share.IMAdminUserID); err != nil { - return nil, err - } - ids := make([]primitive.ObjectID, 0, len(req.Id)) - for _, id := range req.Id { - oid, err := primitive.ObjectIDFromHex(id) - if err != nil { - return nil, errs.ErrArgs.WrapMsg("invalid id " + err.Error()) - } - ids = append(ids, oid) - } - if err := t.applicationDatabase.DeleteVersion(ctx, ids); err != nil { - return nil, err - } - return &third.DeleteApplicationVersionResp{}, nil -} - -func (t *thirdServer) PageApplicationVersion(ctx context.Context, req *third.PageApplicationVersionReq) (*third.PageApplicationVersionResp, error) { - total, res, err := t.applicationDatabase.PageVersion(ctx, req.Platform, req.Pagination) - if err != nil { - return nil, err - } - return &third.PageApplicationVersionResp{ - Total: total, - Versions: datautil.Slice(res, t.db2pbApplication), - }, nil -} diff --git a/internal/rpc/third/third.go b/internal/rpc/third/third.go index c6b588d8d..d37689b31 100644 --- a/internal/rpc/third/third.go +++ b/internal/rpc/third/third.go @@ -38,13 +38,12 @@ import ( ) type thirdServer struct { - thirdDatabase controller.ThirdDatabase - s3dataBase controller.S3Database - userRpcClient rpcclient.UserRpcClient - defaultExpire time.Duration - config *Config - minio *minio.Minio - applicationDatabase controller.ApplicationDatabase + thirdDatabase controller.ThirdDatabase + s3dataBase controller.S3Database + userRpcClient rpcclient.UserRpcClient + defaultExpire time.Duration + config *Config + minio *minio.Minio } type Config struct { @@ -75,10 +74,6 @@ func Start(ctx context.Context, config *Config, client discovery.SvcDiscoveryReg if err != nil { return err } - applicationMgo, err := mgo.NewApplicationMgo(mgocli.GetDB()) - if err != nil { - return err - } // Select the oss method according to the profile policy enable := config.RpcConfig.Object.Enable @@ -104,13 +99,12 @@ func Start(ctx context.Context, config *Config, client discovery.SvcDiscoveryReg } localcache.InitLocalCache(&config.LocalCacheConfig) third.RegisterThirdServer(server, &thirdServer{ - thirdDatabase: controller.NewThirdDatabase(redis.NewThirdCache(rdb), logdb), - userRpcClient: rpcclient.NewUserRpcClient(client, config.Share.RpcRegisterName.User, config.Share.IMAdminUserID), - s3dataBase: controller.NewS3Database(rdb, o, s3db), - defaultExpire: time.Hour * 24 * 7, - config: config, - minio: minioCli, - applicationDatabase: controller.NewApplicationDatabase(applicationMgo, redis.NewApplicationRedisCache(applicationMgo, rdb)), + thirdDatabase: controller.NewThirdDatabase(redis.NewThirdCache(rdb), logdb), + userRpcClient: rpcclient.NewUserRpcClient(client, config.Share.RpcRegisterName.User, config.Share.IMAdminUserID), + s3dataBase: controller.NewS3Database(rdb, o, s3db), + defaultExpire: time.Hour * 24 * 7, + config: config, + minio: minioCli, }) return nil } diff --git a/pkg/common/storage/cache/application.go b/pkg/common/storage/cache/application.go deleted file mode 100644 index bb23a3a17..000000000 --- a/pkg/common/storage/cache/application.go +++ /dev/null @@ -1,11 +0,0 @@ -package cache - -import ( - "context" - "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" -) - -type ApplicationCache interface { - LatestVersion(ctx context.Context, platform string, hot bool) (*model.Application, error) - DeleteCache(ctx context.Context, platforms []string) error -} diff --git a/pkg/common/storage/cache/cachekey/application.go b/pkg/common/storage/cache/cachekey/application.go deleted file mode 100644 index d7835b289..000000000 --- a/pkg/common/storage/cache/cachekey/application.go +++ /dev/null @@ -1,15 +0,0 @@ -package cachekey - -const ( - ApplicationLatestVersion = "APPLICATION_LATEST_VERSION:" -) - -func GetApplicationLatestVersionKey(platform string, hot bool) string { - var hotStr string - if hot { - hotStr = "1:" - } else { - hotStr = "0:" - } - return ApplicationLatestVersion + hotStr + platform -} diff --git a/pkg/common/storage/cache/redis/application.go b/pkg/common/storage/cache/redis/application.go deleted file mode 100644 index a858fa25b..000000000 --- a/pkg/common/storage/cache/redis/application.go +++ /dev/null @@ -1,44 +0,0 @@ -package redis - -import ( - "context" - "github.com/dtm-labs/rockscache" - "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/cachekey" - "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" - "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" - "github.com/redis/go-redis/v9" - "time" -) - -func NewApplicationRedisCache(db database.Application, rdb redis.UniversalClient) *ApplicationRedisCache { - return &ApplicationRedisCache{ - db: db, - rcClient: rockscache.NewClient(rdb, *GetRocksCacheOptions()), - deleter: NewBatchDeleterRedis(rdb, GetRocksCacheOptions(), nil), - expireTime: time.Hour * 24 * 7, - } -} - -type ApplicationRedisCache struct { - db database.Application - rcClient *rockscache.Client - deleter *BatchDeleterRedis - expireTime time.Duration -} - -func (a *ApplicationRedisCache) LatestVersion(ctx context.Context, platform string, hot bool) (*model.Application, error) { - return getCache(ctx, a.rcClient, cachekey.GetApplicationLatestVersionKey(platform, hot), a.expireTime, func(ctx context.Context) (*model.Application, error) { - return a.db.LatestVersion(ctx, platform, hot) - }) -} - -func (a *ApplicationRedisCache) DeleteCache(ctx context.Context, platforms []string) error { - if len(platforms) == 0 { - return nil - } - keys := make([]string, 0, len(platforms)*2) - for _, platform := range platforms { - keys = append(keys, cachekey.GetApplicationLatestVersionKey(platform, true), cachekey.GetApplicationLatestVersionKey(platform, false)) - } - return a.deleter.ExecDelWithKeys(ctx, keys) -} diff --git a/pkg/common/storage/controller/application.go b/pkg/common/storage/controller/application.go deleted file mode 100644 index 388805772..000000000 --- a/pkg/common/storage/controller/application.go +++ /dev/null @@ -1,69 +0,0 @@ -package controller - -import ( - "context" - "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache" - "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" - "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" - "github.com/openimsdk/tools/db/pagination" - "go.mongodb.org/mongo-driver/bson/primitive" -) - -type ApplicationDatabase interface { - LatestVersion(ctx context.Context, platform string, hot bool) (*model.Application, error) - AddVersion(ctx context.Context, val *model.Application) error - UpdateVersion(ctx context.Context, id primitive.ObjectID, update map[string]any) error - DeleteVersion(ctx context.Context, id []primitive.ObjectID) error - PageVersion(ctx context.Context, platforms []string, page pagination.Pagination) (int64, []*model.Application, error) -} - -func NewApplicationDatabase(db database.Application, cache cache.ApplicationCache) ApplicationDatabase { - return &applicationDatabase{db: db, cache: cache} -} - -type applicationDatabase struct { - db database.Application - cache cache.ApplicationCache -} - -func (a *applicationDatabase) LatestVersion(ctx context.Context, platform string, hot bool) (*model.Application, error) { - return a.cache.LatestVersion(ctx, platform, hot) -} - -func (a *applicationDatabase) AddVersion(ctx context.Context, val *model.Application) error { - if err := a.db.AddVersion(ctx, val); err != nil { - return err - } - return a.cache.DeleteCache(ctx, []string{val.Platform}) -} - -func (a *applicationDatabase) UpdateVersion(ctx context.Context, id primitive.ObjectID, update map[string]any) error { - platforms, err := a.db.FindPlatform(ctx, []primitive.ObjectID{id}) - if err != nil { - return err - } - if err := a.db.UpdateVersion(ctx, id, update); err != nil { - return err - } - if p, ok := update["platform"]; ok { - if val, ok := p.(string); ok { - platforms = append(platforms, val) - } - } - return a.cache.DeleteCache(ctx, platforms) -} - -func (a *applicationDatabase) DeleteVersion(ctx context.Context, id []primitive.ObjectID) error { - platforms, err := a.db.FindPlatform(ctx, id) - if err != nil { - return err - } - if err := a.db.DeleteVersion(ctx, id); err != nil { - return err - } - return a.cache.DeleteCache(ctx, platforms) -} - -func (a *applicationDatabase) PageVersion(ctx context.Context, platforms []string, page pagination.Pagination) (int64, []*model.Application, error) { - return a.db.PageVersion(ctx, platforms, page) -} diff --git a/pkg/common/storage/database/application.go b/pkg/common/storage/database/application.go deleted file mode 100644 index 50796be62..000000000 --- a/pkg/common/storage/database/application.go +++ /dev/null @@ -1,17 +0,0 @@ -package database - -import ( - "context" - "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" - "github.com/openimsdk/tools/db/pagination" - "go.mongodb.org/mongo-driver/bson/primitive" -) - -type Application interface { - LatestVersion(ctx context.Context, platform string, hot bool) (*model.Application, error) - AddVersion(ctx context.Context, val *model.Application) error - UpdateVersion(ctx context.Context, id primitive.ObjectID, update map[string]any) error - DeleteVersion(ctx context.Context, id []primitive.ObjectID) error - PageVersion(ctx context.Context, platforms []string, page pagination.Pagination) (int64, []*model.Application, error) - FindPlatform(ctx context.Context, id []primitive.ObjectID) ([]string, error) -} diff --git a/pkg/common/storage/database/mgo/application.go b/pkg/common/storage/database/mgo/application.go deleted file mode 100644 index e28965247..000000000 --- a/pkg/common/storage/database/mgo/application.go +++ /dev/null @@ -1,83 +0,0 @@ -package mgo - -import ( - "context" - "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" - "github.com/openimsdk/tools/db/mongoutil" - "github.com/openimsdk/tools/db/pagination" - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/bson/primitive" - "go.mongodb.org/mongo-driver/mongo" - "go.mongodb.org/mongo-driver/mongo/options" -) - -func NewApplicationMgo(db *mongo.Database) (*ApplicationMgo, error) { - coll := db.Collection("application") - _, err := coll.Indexes().CreateMany(context.Background(), []mongo.IndexModel{ - { - Keys: bson.D{ - {Key: "platform", Value: 1}, - {Key: "version", Value: 1}, - {Key: "hot", Value: 1}, - }, - Options: options.Index().SetUnique(true), - }, - { - Keys: bson.D{ - {Key: "latest", Value: -1}, - }, - }, - }) - if err != nil { - return nil, err - } - return &ApplicationMgo{coll: coll}, nil -} - -type ApplicationMgo struct { - coll *mongo.Collection -} - -func (a *ApplicationMgo) sort() any { - return bson.D{{"latest", -1}, {"_id", -1}} -} - -func (a *ApplicationMgo) LatestVersion(ctx context.Context, platform string, hot bool) (*model.Application, error) { - return mongoutil.FindOne[*model.Application](ctx, a.coll, bson.M{"platform": platform, "hot": hot}, options.FindOne().SetSort(a.sort())) -} - -func (a *ApplicationMgo) AddVersion(ctx context.Context, val *model.Application) error { - if val.ID.IsZero() { - val.ID = primitive.NewObjectID() - } - return mongoutil.InsertMany(ctx, a.coll, []*model.Application{val}) -} - -func (a *ApplicationMgo) UpdateVersion(ctx context.Context, id primitive.ObjectID, update map[string]any) error { - if len(update) == 0 { - return nil - } - return mongoutil.UpdateOne(ctx, a.coll, bson.M{"_id": id}, bson.M{"$set": update}, true) -} - -func (a *ApplicationMgo) DeleteVersion(ctx context.Context, id []primitive.ObjectID) error { - if len(id) == 0 { - return nil - } - return mongoutil.DeleteMany(ctx, a.coll, bson.M{"_id": bson.M{"$in": id}}) -} - -func (a *ApplicationMgo) PageVersion(ctx context.Context, platforms []string, page pagination.Pagination) (int64, []*model.Application, error) { - filter := bson.M{} - if len(platforms) > 0 { - filter["platform"] = bson.M{"$in": platforms} - } - return mongoutil.FindPage[*model.Application](ctx, a.coll, filter, page, options.Find().SetSort(a.sort())) -} - -func (a *ApplicationMgo) FindPlatform(ctx context.Context, id []primitive.ObjectID) ([]string, error) { - if len(id) == 0 { - return nil, nil - } - return mongoutil.Find[string](ctx, a.coll, bson.M{"_id": bson.M{"$in": id}}, options.Find().SetProjection(bson.M{"_id": 0, "platform": 1})) -} From 269dd7180f8159569e264fbcdc0a06b0000c28bb Mon Sep 17 00:00:00 2001 From: Monet Lee Date: Thu, 31 Oct 2024 11:21:14 +0800 Subject: [PATCH 029/199] fix: improve condition check. (#2815) --- internal/rpc/group/group.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/rpc/group/group.go b/internal/rpc/group/group.go index 30ac19a76..b5ab1b209 100644 --- a/internal/rpc/group/group.go +++ b/internal/rpc/group/group.go @@ -1811,7 +1811,6 @@ func (g *groupServer) GetSpecifiedUserGroupRequestInfo(ctx context.Context, req } if req.UserID != opUserID { - req.UserID = mcontext.GetOpUserID(ctx) adminIDs, err := g.db.GetGroupRoleLevelMemberIDs(ctx, req.GroupID, constant.GroupAdmin) if err != nil { return nil, err @@ -1820,10 +1819,11 @@ func (g *groupServer) GetSpecifiedUserGroupRequestInfo(ctx context.Context, req adminIDs = append(adminIDs, owners[0].UserID) adminIDs = append(adminIDs, g.config.Share.IMAdminUserID...) - if !datautil.Contain(req.UserID, adminIDs...) { + if !datautil.Contain(opUserID, adminIDs...) { return nil, errs.ErrNoPermission.WrapMsg("opUser no permission") } } + requests, err := g.db.FindGroupRequests(ctx, req.GroupID, []string{req.UserID}) if err != nil { return nil, err From 41d5688de620ee9671b3babd5c589d042c722175 Mon Sep 17 00:00:00 2001 From: icey-yu <119291641+icey-yu@users.noreply.github.com> Date: Fri, 1 Nov 2024 14:49:31 +0800 Subject: [PATCH 030/199] Update login policy (#2822) * fix: login Policy * fix: del login Policy --- go.mod | 2 +- go.sum | 4 +-- internal/msggateway/ws_server.go | 22 +---------------- pkg/common/storage/controller/auth.go | 35 ++++----------------------- 4 files changed, 9 insertions(+), 54 deletions(-) diff --git a/go.mod b/go.mod index 7cd4e4550..96a06729f 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/gorilla/websocket v1.5.1 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/mitchellh/mapstructure v1.5.0 - github.com/openimsdk/protocol v0.0.72-alpha.51 + github.com/openimsdk/protocol v0.0.72-alpha.53 github.com/openimsdk/tools v0.0.50-alpha.16 github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 diff --git a/go.sum b/go.sum index 7cc45616f..92a98110b 100644 --- a/go.sum +++ b/go.sum @@ -319,8 +319,8 @@ github.com/onsi/gomega v1.25.0 h1:Vw7br2PCDYijJHSfBOWhov+8cAnUf8MfMaIOV323l6Y= github.com/onsi/gomega v1.25.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdMPGhM= github.com/openimsdk/gomake v0.0.14-alpha.5 h1:VY9c5x515lTfmdhhPjMvR3BBRrRquAUCFsz7t7vbv7Y= github.com/openimsdk/gomake v0.0.14-alpha.5/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= -github.com/openimsdk/protocol v0.0.72-alpha.51 h1:G5Yjndp/FRyOJWhoQcSF2x2GvYiAIlqN0vjkvjUPycU= -github.com/openimsdk/protocol v0.0.72-alpha.51/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= +github.com/openimsdk/protocol v0.0.72-alpha.53 h1:DMzvDd418GaJJLT2Iw+AX+oNc41DROWErXDkZxB+MMM= +github.com/openimsdk/protocol v0.0.72-alpha.53/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= github.com/openimsdk/tools v0.0.50-alpha.16 h1:bC1AQvJMuOHtZm8LZRvN8L5mH1Ws2VYdL+TLTs1iGSc= github.com/openimsdk/tools v0.0.50-alpha.16/go.mod h1:h1cYmfyaVtgFbKmb1Cfsl8XwUOMTt8ubVUQrdGtsUh4= github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= diff --git a/internal/msggateway/ws_server.go b/internal/msggateway/ws_server.go index b92d7eb44..48e4b5cee 100644 --- a/internal/msggateway/ws_server.go +++ b/internal/msggateway/ws_server.go @@ -327,11 +327,6 @@ func (ws *WsServer) multiTerminalLoginChecker(clientOK bool, oldClients []*Clien switch ws.msgGatewayConfig.Share.MultiLogin.Policy { case constant.DefalutNotKick: - case constant.WebAndOther: - if constant.PlatformIDToClass(newClient.PlatformID) == constant.WebPlatformStr { - return - } - fallthrough case constant.PCAndOther: if constant.PlatformIDToClass(newClient.PlatformID) == constant.TerminalPC { return @@ -356,7 +351,7 @@ func (ws *WsServer) multiTerminalLoginChecker(clientOK bool, oldClients []*Clien log.ZWarn(newClient.ctx, "InvalidateToken err", err, "userID", newClient.UserID, "platformID", newClient.PlatformID) } - case constant.PcMobileAndWeb: + case constant.AllLoginButSameClassKick: clients, ok := ws.clients.GetAll(newClient.UserID) if !ok { return @@ -370,21 +365,6 @@ func (ws *WsServer) multiTerminalLoginChecker(clientOK bool, oldClients []*Clien } } kickTokenFunc(kickClients) - - case constant.SingleTerminalLogin: - clients, ok := ws.clients.GetAll(newClient.UserID) - if !ok { - return - } - var ( - kickClients []*Client - ) - for _, client := range clients { - kickClients = append(kickClients, client) - } - kickTokenFunc(kickClients) - case constant.Customize: - // todo } } diff --git a/pkg/common/storage/controller/auth.go b/pkg/common/storage/controller/auth.go index de8f93462..82d6f9476 100644 --- a/pkg/common/storage/controller/auth.go +++ b/pkg/common/storage/controller/auth.go @@ -172,17 +172,8 @@ func (a *authDatabase) checkToken(ctx context.Context, tokens map[int]map[string kickToken = append(kickToken, ts[len(ts)-1]) } } - case constant.SingleTerminalLogin: - for _, ts := range loginTokenMap { - kickToken = append(kickToken, ts...) - } - case constant.WebAndOther: - unkickTerminal = constant.WebPlatformStr - fallthrough case constant.PCAndOther: - if unkickTerminal == "" { - unkickTerminal = constant.TerminalPC - } + unkickTerminal = constant.TerminalPC if constant.PlatformIDToClass(platformID) != unkickTerminal { for plt, ts := range loginTokenMap { if constant.PlatformIDToClass(plt) != unkickTerminal { @@ -214,17 +205,17 @@ func (a *authDatabase) checkToken(ctx context.Context, tokens map[int]map[string } } } - case constant.PcMobileAndWeb: + case constant.AllLoginButSameClassKick: var ( - reserved = make(map[string]bool) + reserved = make(map[string]struct{}) ) for plt, ts := range loginTokenMap { if constant.PlatformIDToClass(plt) == constant.PlatformIDToClass(platformID) { kickToken = append(kickToken, ts...) } else { - if !reserved[constant.PlatformIDToClass(plt)] { - reserved[constant.PlatformIDToClass(plt)] = true + if _, ok := reserved[constant.PlatformIDToClass(plt)]; !ok { + reserved[constant.PlatformIDToClass(plt)] = struct{}{} kickToken = append(kickToken, ts[:len(ts)-1]...) continue } else { @@ -232,22 +223,6 @@ func (a *authDatabase) checkToken(ctx context.Context, tokens map[int]map[string } } } - - case constant.Customize: - if a.multiLogin.CustomizeLoginNum[platformID] <= 0 { - return nil, nil, errs.New("Do not allow login on this end").Wrap() - } - for plt, ts := range loginTokenMap { - l := len(ts) - if platformID == plt { - l++ - } - // a.multiLogin.CustomizeLoginNum[platformID] must > 0 - limit := min(a.multiLogin.CustomizeLoginNum[plt], a.multiLogin.MaxNumOneEnd) - if l > limit { - kickToken = append(kickToken, ts[:l-limit]...) - } - } default: return nil, nil, errs.New("unknown multiLogin policy").Wrap() } From 1516e9cb3af33634b53b2fb703ed3380c4976ccf Mon Sep 17 00:00:00 2001 From: Monet Lee Date: Mon, 4 Nov 2024 14:50:07 +0800 Subject: [PATCH 031/199] build: implement version file update when release. (#2826) --- .../update-version-file-on-release.yml | 84 +++++++++++++++++++ 1 file changed, 84 insertions(+) create mode 100644 .github/workflows/.github/workflows/update-version-file-on-release.yml diff --git a/.github/workflows/.github/workflows/update-version-file-on-release.yml b/.github/workflows/.github/workflows/update-version-file-on-release.yml new file mode 100644 index 000000000..113537fd9 --- /dev/null +++ b/.github/workflows/.github/workflows/update-version-file-on-release.yml @@ -0,0 +1,84 @@ +name: Update Version File on Release + +on: + release: + types: [created] + +jobs: + update-version: + runs-on: ubuntu-latest + env: + TAG_VERSION: ${{ github.event.release.tag_name }} + steps: + # Step 1: Checkout the original repository's code + - name: Checkout code + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + # Step 2: Set up Git with official account + - name: Set up Git + run: | + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + + # Step 3: Check and delete existing tag + - name: Check and delete existing tag + run: | + if git rev-parse ${{ env.TAG_VERSION }} >/dev/null 2>&1; then + git tag -d ${{ env.TAG_VERSION }} + git push --delete origin ${{ env.TAG_VERSION }} + fi + + # Step 4: Update version file + - name: Update version file + run: | + echo "${{ env.TAG_VERSION }}" > version/version + + # Step 5: Commit and push changes + - name: Commit and push changes + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + git add version/version + git commit -m "Update version to ${{ env.TAG_VERSION }}" + git push origin HEAD:${{ github.ref }} + + # Step 6: Create and push tag + - name: Create and push tag + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + git tag ${{ env.TAG_VERSION }} + git push origin ${{ env.TAG_VERSION }} + + # Step 7: Find and Publish Draft Release + - name: Find and Publish Draft Release + uses: actions/github-script@v6 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + // Get the list of releases + const releases = await github.rest.repos.listReleases({ + owner: context.repo.owner, + repo: context.repo.repo + }); + + // Find the draft release where the title and tag_name are the same + const draftRelease = releases.data.find(release => + release.draft && release.name === release.tag_name + ); + + if (draftRelease) { + // Publish the draft release using the release_id + await github.rest.repos.updateRelease({ + owner: context.repo.owner, + repo: context.repo.repo, + release_id: draftRelease.id, // Use release_id + draft: false + }); + + core.info(`Draft Release ${draftRelease.tag_name} published successfully.`); + } else { + core.info("No matching draft release found."); + } \ No newline at end of file From 8d308a41634b312c480a916cd91db233826e3924 Mon Sep 17 00:00:00 2001 From: chao <48119764+withchao@users.noreply.github.com> Date: Mon, 4 Nov 2024 16:40:39 +0800 Subject: [PATCH 032/199] feat: support stream message (#2824) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * update gomake version * update gomake version * fix: seq conversion bug * fix: redis pipe exec * fix: ImportFriends * fix: A large number of logs keysAndValues ​​length is not even * feat: mark read aggregate write * feat: online status supports redis cluster * feat: online status supports redis cluster * feat: online status supports redis cluster * merge * merge * read seq is written to mongo * read seq is written to mongo * fix: invitation to join group notification * fix: friend op_user_id * feat: optimizing asynchronous context * feat: optimizing memamq size * feat: add GetSeqMessage * feat: GroupApplicationAgreeMemberEnterNotification * feat: GroupApplicationAgreeMemberEnterNotification * feat: go.mod * feat: go.mod * feat: join group notification and get seq * feat: join group notification and get seq * feat: avoid pulling messages from sessions with a large number of max seq values of 0 * feat: API supports gzip * go.mod * fix: nil pointer error on close * fix: listen error * fix: listen error * update go.mod * feat: add log * fix: token parse token value * fix: GetMsgBySeqs boundary issues * fix: sn_ not sort * fix: sn_ not sort * fix: sn_ not sort * fix: jssdk add * fix: jssdk support * fix: jssdk support * fix: jssdk support * fix: the message I sent is not set to read seq in mongodb * fix: cannot modify group member avatars * fix: MemberEnterNotification * fix: MemberEnterNotification * fix: MsgData status * feat: stream msg * feat: support stream messages --------- Co-authored-by: withchao --- go.mod | 2 +- go.sum | 4 +- internal/api/msg.go | 10 ++ internal/api/router.go | 2 + internal/rpc/msg/notification.go | 4 + internal/rpc/msg/send.go | 5 + internal/rpc/msg/server.go | 10 +- internal/rpc/msg/stream_msg.go | 114 +++++++++++++ pkg/apistruct/msg.go | 5 + pkg/common/storage/controller/stream_msg.go | 34 ++++ pkg/common/storage/database/mgo/stream_msg.go | 60 +++++++ pkg/common/storage/database/name.go | 1 + pkg/common/storage/database/stream_msg.go | 13 ++ pkg/common/storage/model/stream_msg.go | 21 +++ tools/streammsg/main.go | 161 ++++++++++++++++++ 15 files changed, 441 insertions(+), 5 deletions(-) create mode 100644 internal/rpc/msg/stream_msg.go create mode 100644 pkg/common/storage/controller/stream_msg.go create mode 100644 pkg/common/storage/database/mgo/stream_msg.go create mode 100644 pkg/common/storage/database/stream_msg.go create mode 100644 pkg/common/storage/model/stream_msg.go create mode 100644 tools/streammsg/main.go diff --git a/go.mod b/go.mod index 96a06729f..051beb403 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/gorilla/websocket v1.5.1 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/mitchellh/mapstructure v1.5.0 - github.com/openimsdk/protocol v0.0.72-alpha.53 + github.com/openimsdk/protocol v0.0.72-alpha.54 github.com/openimsdk/tools v0.0.50-alpha.16 github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 diff --git a/go.sum b/go.sum index 92a98110b..816c82094 100644 --- a/go.sum +++ b/go.sum @@ -319,8 +319,8 @@ github.com/onsi/gomega v1.25.0 h1:Vw7br2PCDYijJHSfBOWhov+8cAnUf8MfMaIOV323l6Y= github.com/onsi/gomega v1.25.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdMPGhM= github.com/openimsdk/gomake v0.0.14-alpha.5 h1:VY9c5x515lTfmdhhPjMvR3BBRrRquAUCFsz7t7vbv7Y= github.com/openimsdk/gomake v0.0.14-alpha.5/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= -github.com/openimsdk/protocol v0.0.72-alpha.53 h1:DMzvDd418GaJJLT2Iw+AX+oNc41DROWErXDkZxB+MMM= -github.com/openimsdk/protocol v0.0.72-alpha.53/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= +github.com/openimsdk/protocol v0.0.72-alpha.54 h1:opato7N4QjjRq/SHD54bDSVBpOEEDp1VLWVk5Os2A9s= +github.com/openimsdk/protocol v0.0.72-alpha.54/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= github.com/openimsdk/tools v0.0.50-alpha.16 h1:bC1AQvJMuOHtZm8LZRvN8L5mH1Ws2VYdL+TLTs1iGSc= github.com/openimsdk/tools v0.0.50-alpha.16/go.mod h1:h1cYmfyaVtgFbKmb1Cfsl8XwUOMTt8ubVUQrdGtsUh4= github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= diff --git a/internal/api/msg.go b/internal/api/msg.go index bf7cb83a4..ce94b5f4f 100644 --- a/internal/api/msg.go +++ b/internal/api/msg.go @@ -173,6 +173,8 @@ func (m *MessageApi) getSendMsgReq(c *gin.Context, req apistruct.SendMsg) (sendM data = apistruct.AtElem{} case constant.Custom: data = apistruct.CustomElem{} + case constant.Stream: + data = apistruct.StreamMsgElem{} case constant.OANotification: data = apistruct.OANotificationElem{} req.SessionType = constant.NotificationChatType @@ -373,3 +375,11 @@ func (m *MessageApi) SearchMsg(c *gin.Context) { func (m *MessageApi) GetServerTime(c *gin.Context) { a2r.Call(msg.MsgClient.GetServerTime, m.Client, c) } + +func (m *MessageApi) GetStreamMsg(c *gin.Context) { + a2r.Call(msg.MsgClient.GetStreamMsg, m.Client, c) +} + +func (m *MessageApi) AppendStreamMsg(c *gin.Context) { + a2r.Call(msg.MsgClient.AppendStreamMsg, m.Client, c) +} diff --git a/internal/api/router.go b/internal/api/router.go index 560516d30..4ac301b07 100644 --- a/internal/api/router.go +++ b/internal/api/router.go @@ -222,6 +222,8 @@ func newGinRouter(disCov discovery.SvcDiscoveryRegistry, config *Config) *gin.En msgGroup.POST("/batch_send_msg", m.BatchSendMsg) msgGroup.POST("/check_msg_is_send_success", m.CheckMsgIsSendSuccess) msgGroup.POST("/get_server_time", m.GetServerTime) + msgGroup.POST("/get_stream_msg", m.GetStreamMsg) + msgGroup.POST("/append_stream_msg", m.AppendStreamMsg) } // Conversation conversationGroup := r.Group("/conversation") diff --git a/internal/rpc/msg/notification.go b/internal/rpc/msg/notification.go index 3b13676bf..26e3c7f46 100644 --- a/internal/rpc/msg/notification.go +++ b/internal/rpc/msg/notification.go @@ -48,3 +48,7 @@ func (m *MsgNotificationSender) MarkAsReadNotification(ctx context.Context, conv } m.NotificationWithSessionType(ctx, sendID, recvID, constant.HasReadReceipt, sessionType, tips) } + +func (m *MsgNotificationSender) StreamMsgNotification(ctx context.Context, sendID string, recvID string, sessionType int32, tips *sdkws.StreamMsgTips) { + m.NotificationWithSessionType(ctx, sendID, recvID, constant.StreamMsgNotification, sessionType, tips) +} diff --git a/internal/rpc/msg/send.go b/internal/rpc/msg/send.go index 2c3f8c0a3..4762f24de 100644 --- a/internal/rpc/msg/send.go +++ b/internal/rpc/msg/send.go @@ -34,6 +34,11 @@ import ( func (m *msgServer) SendMsg(ctx context.Context, req *pbmsg.SendMsgReq) (*pbmsg.SendMsgResp, error) { if req.MsgData != nil { m.encapsulateMsgData(req.MsgData) + if req.MsgData.ContentType == constant.Stream { + if err := m.handlerStreamMsg(ctx, req.MsgData); err != nil { + return nil, err + } + } switch req.MsgData.SessionType { case constant.SingleChatType: return m.sendMsgSingleChat(ctx, req) diff --git a/internal/rpc/msg/server.go b/internal/rpc/msg/server.go index 91f41f1b1..bf8781747 100644 --- a/internal/rpc/msg/server.go +++ b/internal/rpc/msg/server.go @@ -42,8 +42,9 @@ type ( // MsgServer encapsulates dependencies required for message handling. msgServer struct { - RegisterCenter discovery.SvcDiscoveryRegistry // Service discovery registry for service registration. - MsgDatabase controller.CommonMsgDatabase // Interface for message database operations. + RegisterCenter discovery.SvcDiscoveryRegistry // Service discovery registry for service registration. + MsgDatabase controller.CommonMsgDatabase // Interface for message database operations. + StreamMsgDatabase controller.StreamMsgDatabase Conversation *rpcclient.ConversationRpcClient // RPC client for conversation service. UserLocalCache *rpccache.UserLocalCache // Local cache for user data. FriendLocalCache *rpccache.FriendLocalCache // Local cache for friend data. @@ -101,6 +102,10 @@ func Start(ctx context.Context, config *Config, client discovery.SvcDiscoveryReg if err != nil { return err } + streamMsg, err := mgo.NewStreamMsgMongo(mgocli.GetDB()) + if err != nil { + return err + } seqUserCache := redis.NewSeqUserCacheRedis(rdb, seqUser) msgDatabase, err := controller.NewCommonMsgDatabase(msgDocModel, msgModel, seqUserCache, seqConversationCache, &config.KafkaConfig) if err != nil { @@ -109,6 +114,7 @@ func Start(ctx context.Context, config *Config, client discovery.SvcDiscoveryReg s := &msgServer{ Conversation: &conversationClient, MsgDatabase: msgDatabase, + StreamMsgDatabase: controller.NewStreamMsgDatabase(streamMsg), RegisterCenter: client, UserLocalCache: rpccache.NewUserLocalCache(userRpcClient, &config.LocalCacheConfig, rdb), GroupLocalCache: rpccache.NewGroupLocalCache(groupRpcClient, &config.LocalCacheConfig, rdb), diff --git a/internal/rpc/msg/stream_msg.go b/internal/rpc/msg/stream_msg.go new file mode 100644 index 000000000..5db2aad48 --- /dev/null +++ b/internal/rpc/msg/stream_msg.go @@ -0,0 +1,114 @@ +package msg + +import ( + "context" + "fmt" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" + "github.com/openimsdk/open-im-server/v3/pkg/msgprocessor" + "github.com/openimsdk/protocol/constant" + "github.com/openimsdk/protocol/msg" + "github.com/openimsdk/protocol/sdkws" + "github.com/openimsdk/tools/errs" + "time" +) + +const StreamDeadlineTime = time.Second * 60 * 10 + +func (m *msgServer) handlerStreamMsg(ctx context.Context, msgData *sdkws.MsgData) error { + now := time.Now() + val := &model.StreamMsg{ + ClientMsgID: msgData.ClientMsgID, + ConversationID: msgprocessor.GetConversationIDByMsg(msgData), + UserID: msgData.SendID, + CreateTime: now, + DeadlineTime: now.Add(StreamDeadlineTime), + } + return m.StreamMsgDatabase.CreateStreamMsg(ctx, val) +} + +func (m *msgServer) getStreamMsg(ctx context.Context, clientMsgID string) (*model.StreamMsg, error) { + res, err := m.StreamMsgDatabase.GetStreamMsg(ctx, clientMsgID) + if err != nil { + return nil, err + } + now := time.Now() + if !res.End && res.DeadlineTime.Before(now) { + res.End = true + res.DeadlineTime = now + _ = m.StreamMsgDatabase.AppendStreamMsg(ctx, res.ClientMsgID, 0, nil, true, now) + } + return res, nil +} + +func (m *msgServer) AppendStreamMsg(ctx context.Context, req *msg.AppendStreamMsgReq) (*msg.AppendStreamMsgResp, error) { + res, err := m.getStreamMsg(ctx, req.ClientMsgID) + if err != nil { + return nil, err + } + if res.End { + return nil, errs.ErrNoPermission.WrapMsg("stream msg is end") + } + if len(res.Packets) < int(req.StartIndex) { + return nil, errs.ErrNoPermission.WrapMsg("start index is invalid") + } + if val := len(res.Packets) - int(req.StartIndex); val > 0 { + exist := res.Packets[int(req.StartIndex):] + for i, s := range exist { + if len(req.Packets) == 0 { + break + } + if s != req.Packets[i] { + return nil, errs.ErrNoPermission.WrapMsg(fmt.Sprintf("packet %d has been written and is inconsistent", i)) + } + req.StartIndex++ + req.Packets = req.Packets[1:] + } + } + if len(req.Packets) == 0 && res.End == req.End { + return &msg.AppendStreamMsgResp{}, nil + } + deadlineTime := time.Now().Add(StreamDeadlineTime) + if err := m.StreamMsgDatabase.AppendStreamMsg(ctx, req.ClientMsgID, int(req.StartIndex), req.Packets, req.End, deadlineTime); err != nil { + return nil, err + } + conversation, err := m.Conversation.GetConversation(ctx, res.UserID, res.ConversationID) + if err != nil { + return nil, err + } + tips := &sdkws.StreamMsgTips{ + ConversationID: res.ConversationID, + ClientMsgID: res.ClientMsgID, + StartIndex: req.StartIndex, + Packets: req.Packets, + End: req.End, + } + var ( + recvID string + sessionType int32 + ) + if conversation.GroupID == "" { + sessionType = constant.SingleChatType + recvID = conversation.UserID + } else { + sessionType = constant.ReadGroupChatType + recvID = conversation.GroupID + } + m.msgNotificationSender.StreamMsgNotification(ctx, res.UserID, recvID, sessionType, tips) + return &msg.AppendStreamMsgResp{}, nil +} + +func (m *msgServer) GetStreamMsg(ctx context.Context, req *msg.GetStreamMsgReq) (*msg.GetStreamMsgResp, error) { + res, err := m.getStreamMsg(ctx, req.ClientMsgID) + if err != nil { + return nil, err + } + return &msg.GetStreamMsgResp{ + ClientMsgID: res.ClientMsgID, + ConversationID: res.ConversationID, + UserID: res.UserID, + Packets: res.Packets, + End: res.End, + CreateTime: res.CreateTime.UnixMilli(), + DeadlineTime: res.DeadlineTime.UnixMilli(), + }, nil +} diff --git a/pkg/apistruct/msg.go b/pkg/apistruct/msg.go index dc20b5104..dda3ff317 100644 --- a/pkg/apistruct/msg.go +++ b/pkg/apistruct/msg.go @@ -81,6 +81,11 @@ type TextElem struct { Content string `json:"content" validate:"required"` } +type StreamMsgElem struct { + Type string `mapstructure:"type" validate:"required"` + Content string `mapstructure:"content" validate:"required"` +} + type RevokeElem struct { RevokeMsgClientID string `mapstructure:"revokeMsgClientID" validate:"required"` } diff --git a/pkg/common/storage/controller/stream_msg.go b/pkg/common/storage/controller/stream_msg.go new file mode 100644 index 000000000..3409ccd93 --- /dev/null +++ b/pkg/common/storage/controller/stream_msg.go @@ -0,0 +1,34 @@ +package controller + +import ( + "context" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" + "time" +) + +type StreamMsgDatabase interface { + CreateStreamMsg(ctx context.Context, model *model.StreamMsg) error + AppendStreamMsg(ctx context.Context, clientMsgID string, startIndex int, packets []string, end bool, deadlineTime time.Time) error + GetStreamMsg(ctx context.Context, clientMsgID string) (*model.StreamMsg, error) +} + +func NewStreamMsgDatabase(db database.StreamMsg) StreamMsgDatabase { + return &streamMsgDatabase{db: db} +} + +type streamMsgDatabase struct { + db database.StreamMsg +} + +func (m *streamMsgDatabase) CreateStreamMsg(ctx context.Context, model *model.StreamMsg) error { + return m.db.CreateStreamMsg(ctx, model) +} + +func (m *streamMsgDatabase) AppendStreamMsg(ctx context.Context, clientMsgID string, startIndex int, packets []string, end bool, deadlineTime time.Time) error { + return m.db.AppendStreamMsg(ctx, clientMsgID, startIndex, packets, end, deadlineTime) +} + +func (m *streamMsgDatabase) GetStreamMsg(ctx context.Context, clientMsgID string) (*model.StreamMsg, error) { + return m.db.GetStreamMsg(ctx, clientMsgID) +} diff --git a/pkg/common/storage/database/mgo/stream_msg.go b/pkg/common/storage/database/mgo/stream_msg.go new file mode 100644 index 000000000..c57798daa --- /dev/null +++ b/pkg/common/storage/database/mgo/stream_msg.go @@ -0,0 +1,60 @@ +package mgo + +import ( + "context" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" + "github.com/openimsdk/tools/db/mongoutil" + "github.com/openimsdk/tools/errs" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" + "time" +) + +func NewStreamMsgMongo(db *mongo.Database) (*StreamMsgMongo, error) { + coll := db.Collection(database.StreamMsgName) + _, err := coll.Indexes().CreateOne(context.Background(), mongo.IndexModel{ + Keys: bson.D{ + {Key: "client_msg_id", Value: 1}, + }, + Options: options.Index().SetUnique(true), + }) + if err != nil { + return nil, errs.Wrap(err) + } + return &StreamMsgMongo{coll: coll}, nil +} + +type StreamMsgMongo struct { + coll *mongo.Collection +} + +func (m *StreamMsgMongo) CreateStreamMsg(ctx context.Context, val *model.StreamMsg) error { + if val.Packets == nil { + val.Packets = []string{} + } + return mongoutil.InsertMany(ctx, m.coll, []*model.StreamMsg{val}) +} + +func (m *StreamMsgMongo) AppendStreamMsg(ctx context.Context, clientMsgID string, startIndex int, packets []string, end bool, deadlineTime time.Time) error { + update := bson.M{ + "$set": bson.M{ + "end": end, + "deadline_time": deadlineTime, + }, + } + if len(packets) > 0 { + update["$push"] = bson.M{ + "packets": bson.M{ + "$each": packets, + "$position": startIndex, + }, + } + } + return mongoutil.UpdateOne(ctx, m.coll, bson.M{"client_msg_id": clientMsgID, "end": false}, update, true) +} + +func (m *StreamMsgMongo) GetStreamMsg(ctx context.Context, clientMsgID string) (*model.StreamMsg, error) { + return mongoutil.FindOne[*model.StreamMsg](ctx, m.coll, bson.M{"client_msg_id": clientMsgID}) +} diff --git a/pkg/common/storage/database/name.go b/pkg/common/storage/database/name.go index 748bd844d..9742f933f 100644 --- a/pkg/common/storage/database/name.go +++ b/pkg/common/storage/database/name.go @@ -17,4 +17,5 @@ const ( UserName = "user" SeqConversationName = "seq" SeqUserName = "seq_user" + StreamMsgName = "stream_msg" ) diff --git a/pkg/common/storage/database/stream_msg.go b/pkg/common/storage/database/stream_msg.go new file mode 100644 index 000000000..e83fffbaa --- /dev/null +++ b/pkg/common/storage/database/stream_msg.go @@ -0,0 +1,13 @@ +package database + +import ( + "context" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" + "time" +) + +type StreamMsg interface { + CreateStreamMsg(ctx context.Context, model *model.StreamMsg) error + AppendStreamMsg(ctx context.Context, clientMsgID string, startIndex int, packets []string, end bool, deadlineTime time.Time) error + GetStreamMsg(ctx context.Context, clientMsgID string) (*model.StreamMsg, error) +} diff --git a/pkg/common/storage/model/stream_msg.go b/pkg/common/storage/model/stream_msg.go new file mode 100644 index 000000000..c040426a4 --- /dev/null +++ b/pkg/common/storage/model/stream_msg.go @@ -0,0 +1,21 @@ +package model + +import ( + "time" +) + +const ( + StreamMsgStatusWait = 0 + StreamMsgStatusDone = 1 + StreamMsgStatusFail = 2 +) + +type StreamMsg struct { + ClientMsgID string `bson:"client_msg_id"` + ConversationID string `bson:"conversation_id"` + UserID string `bson:"user_id"` + Packets []string `bson:"packets"` + End bool `bson:"end"` + CreateTime time.Time `bson:"create_time"` + DeadlineTime time.Time `bson:"deadline_time"` +} diff --git a/tools/streammsg/main.go b/tools/streammsg/main.go new file mode 100644 index 000000000..bb567e233 --- /dev/null +++ b/tools/streammsg/main.go @@ -0,0 +1,161 @@ +package main + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "github.com/gin-gonic/gin" + "github.com/google/uuid" + "github.com/openimsdk/open-im-server/v3/pkg/apistruct" + cbapi "github.com/openimsdk/open-im-server/v3/pkg/callbackstruct" + "github.com/openimsdk/protocol/auth" + "github.com/openimsdk/protocol/constant" + "github.com/openimsdk/protocol/msg" + "github.com/openimsdk/tools/apiresp" + "github.com/openimsdk/tools/errs" + "io" + "net/http" + "strings" + "time" +) + +const ( + getAdminToken = "/auth/get_admin_token" + sendMsgApi = "/msg/send_msg" + appendStreamMsg = "/msg/append_stream_msg" +) + +var ( + ApiAddr = "http://127.0.0.1:10002" + Token string +) + +func ApiCall[R any](api string, req any) (*R, error) { + data, err := json.Marshal(req) + if err != nil { + return nil, err + } + ctx, cancel := context.WithTimeout(context.Background(), time.Second*3) + defer cancel() + request, err := http.NewRequestWithContext(ctx, http.MethodPost, ApiAddr+api, bytes.NewBuffer(data)) + if err != nil { + return nil, err + } + if Token != "" { + request.Header.Set("token", Token) + } + request.Header.Set(constant.OperationID, uuid.New().String()) + response, err := http.DefaultClient.Do(request) + if err != nil { + return nil, err + } + defer response.Body.Close() + var resp R + apiResponse := apiresp.ApiResponse{ + Data: &resp, + } + if err := json.NewDecoder(response.Body).Decode(&apiResponse); err != nil { + return nil, err + } + if apiResponse.ErrCode != 0 { + return nil, errs.NewCodeError(apiResponse.ErrCode, apiResponse.ErrMsg) + } + return &resp, nil +} + +func main() { + resp, err := ApiCall[auth.GetAdminTokenResp](getAdminToken, &auth.GetAdminTokenReq{ + Secret: "openIM123", + UserID: "imAdmin", + }) + if err != nil { + fmt.Println("get admin token failed", err) + return + } + Token = resp.Token + g := gin.Default() + g.POST("/callbackExample/callbackAfterSendSingleMsgCommand", toGin(handlerUserMsg)) + if err := g.Run(":10006"); err != nil { + panic(err) + } +} + +func toGin[R any](fn func(c *gin.Context, req *R) error) gin.HandlerFunc { + return func(c *gin.Context) { + body, err := io.ReadAll(c.Request.Body) + if err != nil { + c.String(http.StatusInternalServerError, err.Error()) + return + } + fmt.Printf("HTTP %s %s %s\n", c.Request.Method, c.Request.URL, body) + var req R + if err := json.Unmarshal(body, &req); err != nil { + c.String(http.StatusInternalServerError, err.Error()) + return + } + if err := fn(c, &req); err != nil { + c.String(http.StatusInternalServerError, err.Error()) + return + } + c.String(http.StatusOK, "{}") + } +} + +func handlerUserMsg(c *gin.Context, req *cbapi.CallbackAfterSendSingleMsgReq) error { + if req.ContentType != constant.Text { + return nil + } + if !strings.Contains(req.Content, "stream") { + return nil + } + apiReq := apistruct.SendMsgReq{ + RecvID: req.SendID, + SendMsg: apistruct.SendMsg{ + SendID: req.RecvID, + SenderNickname: "xxx", + SenderFaceURL: "", + SenderPlatformID: constant.AdminPlatformID, + ContentType: constant.Stream, + SessionType: req.SessionType, + SendTime: time.Now().UnixMilli(), + Content: map[string]any{ + "type": "xxx", + "content": "server test stream msg", + }, + }, + } + go func() { + if err := doPushStreamMsg(&apiReq); err != nil { + fmt.Println("doPushStreamMsg failed", err) + return + } + fmt.Println("doPushStreamMsg success") + }() + return nil +} + +func doPushStreamMsg(sendReq *apistruct.SendMsgReq) error { + resp, err := ApiCall[msg.SendMsgResp](sendMsgApi, sendReq) + if err != nil { + return err + } + const num = 5 + for i := 1; i <= num; i++ { + _, err := ApiCall[msg.AppendStreamMsgResp](appendStreamMsg, &msg.AppendStreamMsgReq{ + ClientMsgID: resp.ClientMsgID, + StartIndex: int64(i - 1), + Packets: []string{ + fmt.Sprintf("stream_msg_packet_%03d", i), + }, + End: i == num, + }) + if err != nil { + fmt.Println("append stream msg failed", "clientMsgID", resp.ClientMsgID, "index", fmt.Sprintf("%d/%d", i, num), "error", err) + return err + } + fmt.Println("append stream msg success", "clientMsgID", resp.ClientMsgID, "index", fmt.Sprintf("%d/%d", i, num)) + time.Sleep(time.Second * 10) + } + return nil +} From 3ebd1bbaa4d8b72667dea395a9df2428587bbc48 Mon Sep 17 00:00:00 2001 From: icey-yu <119291641+icey-yu@users.noreply.github.com> Date: Tue, 5 Nov 2024 15:33:15 +0800 Subject: [PATCH 033/199] fix: write msg to redis (#2836) --- pkg/common/storage/cache/redis/msg.go | 14 -------------- pkg/common/storage/controller/msg.go | 10 ++++++++++ 2 files changed, 10 insertions(+), 14 deletions(-) diff --git a/pkg/common/storage/cache/redis/msg.go b/pkg/common/storage/cache/redis/msg.go index 30f367bb7..b04bc5c35 100644 --- a/pkg/common/storage/cache/redis/msg.go +++ b/pkg/common/storage/cache/redis/msg.go @@ -1,17 +1,3 @@ -// Copyright © 2023 OpenIM. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - package redis import ( diff --git a/pkg/common/storage/controller/msg.go b/pkg/common/storage/controller/msg.go index d579069b6..90b479064 100644 --- a/pkg/common/storage/controller/msg.go +++ b/pkg/common/storage/controller/msg.go @@ -443,6 +443,11 @@ func (db *commonMsgDatabase) GetMsgBySeqsRange(ctx context.Context, userID strin return 0, 0, nil, err } successMsgs = append(mongoMsgs, successMsgs...) + + _, err = db.msg.SetMessagesToCache(ctx, conversationID, mongoMsgs) + if err != nil { + return 0, 0, nil, err + } } return minSeq, maxSeq, successMsgs, nil @@ -500,6 +505,11 @@ func (db *commonMsgDatabase) GetMsgBySeqs(ctx context.Context, userID string, co } successMsgs = append(successMsgs, mongoMsgs...) + + _, err = db.msg.SetMessagesToCache(ctx, conversationID, mongoMsgs) + if err != nil { + return 0, 0, nil, err + } } return minSeq, maxSeq, successMsgs, nil } From ad8ae5f5f0faf643ba022991f964fe31544c3979 Mon Sep 17 00:00:00 2001 From: icey-yu <119291641+icey-yu@users.noreply.github.com> Date: Thu, 7 Nov 2024 14:59:31 +0800 Subject: [PATCH 034/199] fix: get group return repeated result (#2842) --- pkg/common/storage/database/mgo/group.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/common/storage/database/mgo/group.go b/pkg/common/storage/database/mgo/group.go index 3be7883af..620269b43 100644 --- a/pkg/common/storage/database/mgo/group.go +++ b/pkg/common/storage/database/mgo/group.go @@ -76,7 +76,7 @@ func (g *GroupMgo) Take(ctx context.Context, groupID string) (group *model.Group func (g *GroupMgo) Search(ctx context.Context, keyword string, pagination pagination.Pagination) (total int64, groups []*model.Group, err error) { // Define the sorting options - opts := options.Find().SetSort(bson.D{{Key: "created_at", Value: -1}}) + opts := options.Find().SetSort(bson.D{{Key: "create_time", Value: -1}}) // Perform the search with pagination and sorting return mongoutil.FindPage[*model.Group](ctx, g.coll, bson.M{ From 68698961f18a2aac751d5422f5924f7a637d7aff Mon Sep 17 00:00:00 2001 From: icey-yu <119291641+icey-yu@users.noreply.github.com> Date: Thu, 7 Nov 2024 18:11:43 +0800 Subject: [PATCH 035/199] fix: SetConversations can update new conversation (#2838) --- internal/rpc/conversation/conversation.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/internal/rpc/conversation/conversation.go b/internal/rpc/conversation/conversation.go index 6f6ca1f67..0c8a6fd85 100644 --- a/internal/rpc/conversation/conversation.go +++ b/internal/rpc/conversation/conversation.go @@ -261,27 +261,35 @@ func (c *conversationServer) SetConversations(ctx context.Context, req *pbconver setConversationFieldsFunc := func() { if req.Conversation.RecvMsgOpt != nil { + conversation.RecvMsgOpt = req.Conversation.RecvMsgOpt.Value m["recv_msg_opt"] = req.Conversation.RecvMsgOpt.Value } if req.Conversation.AttachedInfo != nil { + conversation.AttachedInfo = req.Conversation.AttachedInfo.Value m["attached_info"] = req.Conversation.AttachedInfo.Value } if req.Conversation.Ex != nil { + conversation.Ex = req.Conversation.Ex.Value m["ex"] = req.Conversation.Ex.Value } if req.Conversation.IsPinned != nil { + conversation.IsPinned = req.Conversation.IsPinned.Value m["is_pinned"] = req.Conversation.IsPinned.Value } if req.Conversation.GroupAtType != nil { + conversation.GroupAtType = req.Conversation.GroupAtType.Value m["group_at_type"] = req.Conversation.GroupAtType.Value } if req.Conversation.MsgDestructTime != nil { + conversation.MsgDestructTime = req.Conversation.MsgDestructTime.Value m["msg_destruct_time"] = req.Conversation.MsgDestructTime.Value } if req.Conversation.IsMsgDestruct != nil { + conversation.IsMsgDestruct = req.Conversation.IsMsgDestruct.Value m["is_msg_destruct"] = req.Conversation.IsMsgDestruct.Value } if req.Conversation.BurnDuration != nil { + conversation.BurnDuration = req.Conversation.BurnDuration.Value m["burn_duration"] = req.Conversation.BurnDuration.Value } } From 12790e141d95bdea42875b74e8ae4447581f21ed Mon Sep 17 00:00:00 2001 From: chao <48119764+withchao@users.noreply.github.com> Date: Thu, 14 Nov 2024 14:35:18 +0800 Subject: [PATCH 036/199] feat: merge js sdk (#2856) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * update gomake version * update gomake version * fix: seq conversion bug * fix: redis pipe exec * fix: ImportFriends * fix: A large number of logs keysAndValues ​​length is not even * feat: mark read aggregate write * feat: online status supports redis cluster * feat: online status supports redis cluster * feat: online status supports redis cluster * merge * merge * read seq is written to mongo * read seq is written to mongo * fix: invitation to join group notification * fix: friend op_user_id * feat: optimizing asynchronous context * feat: optimizing memamq size * feat: add GetSeqMessage * feat: implement no gob encoder. * update unitTest content. * Update hub_server.go * feat: GroupApplicationAgreeMemberEnterNotification * fix: encoder replace to json encoder. * feat: GroupApplicationAgreeMemberEnterNotification * feat: go.mod * feat: go.mod * feat: join group notification and get seq * feat: join group notification and get seq * feat: avoid pulling messages from sessions with a large number of max seq values of 0 * feat: API supports gzip * go.mod * fix: nil pointer error on close * fix: listen error * fix: listen error * update go.mod * feat: add log * fix: token parse token value * fix: GetMsgBySeqs boundary issues * merge: merge main code into js branch. (#2648) * feat: update group notification when set to null. (#2590) * refactor: refactor workflows contents. * add tool workflows. * update field. * fix: remove chat error. * Fix err. * fix error. * remove cn comment. * update workflows files. * update infra config. * move workflows. * feat: update bot. * fix: solve uncorrect outdated msg get. * update get docIDs logic. * update * update skip logic. * fix * update. * fix: delay deleteObject func. * remove unused content. * feat: update group notification when set to null. * update log standard. * feat: add long time push msg in prometheus (#2584) * feat: add long time push msg in prometheus * fix: log print * fix: go mod * fix: log msg * fix: log init * feat: push msg * feat: go mod ,remove cgo package * feat: remove error log * feat: test dummy push * feat:redis pool config * feat: push to kafka log * feat: supports getting messages based on session ID and seq (#2582) * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * update gomake version * update gomake version * fix: seq conversion bug * fix: redis pipe exec * fix: ImportFriends * fix: A large number of logs keysAndValues ​​length is not even * feat: mark read aggregate write * feat: online status supports redis cluster * feat: online status supports redis cluster * feat: online status supports redis cluster * merge * merge * read seq is written to mongo * read seq is written to mongo * fix: invitation to join group notification * fix: friend op_user_id * feat: optimizing asynchronous context * feat: optimizing memamq size * feat: add GetSeqMessage * feat: GroupApplicationAgreeMemberEnterNotification * feat: GroupApplicationAgreeMemberEnterNotification * feat: go.mod * feat: go.mod * feat: join group notification and get seq --------- Co-authored-by: withchao * feat: implement request batch count limit. (#2591) * refactor: refactor workflows contents. * add tool workflows. * update field. * fix: remove chat error. * Fix err. * fix error. * remove cn comment. * update workflows files. * update infra config. * move workflows. * feat: update bot. * fix: solve uncorrect outdated msg get. * update get docIDs logic. * update * update skip logic. * fix * update. * fix: delay deleteObject func. * remove unused content. * update log type. * feat: implement request batch count limit. * update * update * fix: getting messages based on session ID and seq (#2595) * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * update gomake version * update gomake version * fix: seq conversion bug * fix: redis pipe exec * fix: ImportFriends * fix: A large number of logs keysAndValues ​​length is not even * feat: mark read aggregate write * feat: online status supports redis cluster * feat: online status supports redis cluster * feat: online status supports redis cluster * merge * merge * read seq is written to mongo * read seq is written to mongo * fix: invitation to join group notification * fix: friend op_user_id * feat: optimizing asynchronous context * feat: optimizing memamq size * feat: add GetSeqMessage * feat: GroupApplicationAgreeMemberEnterNotification * feat: GroupApplicationAgreeMemberEnterNotification * feat: go.mod * feat: go.mod * feat: join group notification and get seq * feat: join group notification and get seq --------- Co-authored-by: withchao * feat: avoid pulling messages from sessions with a large number of max seq values of 0 (#2602) * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * update gomake version * update gomake version * fix: seq conversion bug * fix: redis pipe exec * fix: ImportFriends * fix: A large number of logs keysAndValues ​​length is not even * feat: mark read aggregate write * feat: online status supports redis cluster * feat: online status supports redis cluster * feat: online status supports redis cluster * merge * merge * read seq is written to mongo * read seq is written to mongo * fix: invitation to join group notification * fix: friend op_user_id * feat: optimizing asynchronous context * feat: optimizing memamq size * feat: add GetSeqMessage * feat: GroupApplicationAgreeMemberEnterNotification * feat: GroupApplicationAgreeMemberEnterNotification * feat: go.mod * feat: go.mod * feat: join group notification and get seq * feat: join group notification and get seq * feat: avoid pulling messages from sessions with a large number of max seq values of 0 --------- Co-authored-by: withchao * refactor: improve db structure in `storage/controller` (#2604) * refactor: refactor workflows contents. * add tool workflows. * update field. * fix: remove chat error. * Fix err. * fix error. * remove cn comment. * update workflows files. * update infra config. * move workflows. * feat: update bot. * fix: solve uncorrect outdated msg get. * update get docIDs logic. * update * update skip logic. * fix * update. * fix: delay deleteObject func. * remove unused content. * update log type. * feat: implement request batch count limit. * update * update * refactor: improve db structure in `storage/controller` * feat: implement offline push using kafka (#2600) * refactor: refactor workflows contents. * add tool workflows. * update field. * fix: remove chat error. * Fix err. * fix error. * remove cn comment. * update workflows files. * update infra config. * move workflows. * feat: update bot. * fix: solve uncorrect outdated msg get. * update get docIDs logic. * update * update skip logic. * fix * update. * fix: delay deleteObject func. * remove unused content. * update log type. * feat: implement request batch count limit. * update * update * feat: implement offline push. * feat: implement batch Push spilt * update go mod * feat: implement kafka producer and consumer. * update format, * add PushMQ log. * feat: update Handler logic. * update MQ logic. * update * update * fix: update OfflinePushConsumerHandler. * feat: API supports gzip (#2609) * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * update gomake version * update gomake version * fix: seq conversion bug * fix: redis pipe exec * fix: ImportFriends * fix: A large number of logs keysAndValues ​​length is not even * feat: mark read aggregate write * feat: online status supports redis cluster * feat: online status supports redis cluster * feat: online status supports redis cluster * merge * merge * read seq is written to mongo * read seq is written to mongo * fix: invitation to join group notification * fix: friend op_user_id * feat: optimizing asynchronous context * feat: optimizing memamq size * feat: add GetSeqMessage * feat: GroupApplicationAgreeMemberEnterNotification * feat: GroupApplicationAgreeMemberEnterNotification * feat: go.mod * feat: go.mod * feat: join group notification and get seq * feat: join group notification and get seq * feat: avoid pulling messages from sessions with a large number of max seq values of 0 * feat: API supports gzip --------- Co-authored-by: withchao * Fix err (#2608) * refactor: refactor workflows contents. * add tool workflows. * update field. * fix: remove chat error. * Fix err. * fix error. * remove cn comment. * update workflows files. * update infra config. * move workflows. * feat: update bot. * fix: solve uncorrect outdated msg get. * update get docIDs logic. * update * update skip logic. * fix * update. * fix: delay deleteObject func. * remove unused content. * update log type. * feat: implement request batch count limit. * update * update * feat: add rocksTimeout * feat: wrap logs * feat: add logs * feat: listen config * feat: enable listen TIME_WAIT port * feat: add logs * feat: cache batch * chore: enable fullUserCache * feat: push rpc num * feat: push err * feat: with operationID * feat: sleep * feat: change 1s * feat: change log * feat: implement Getbatch in rpcCache. * feat: print getOnline cost * feat: change log * feat: change kafka and push config * feat: del interface * feat: fix err * feat: change config * feat: go mod * feat: change config * feat: change config * feat: add sleep in push * feat: warn logs * feat: logs * feat: logs * feat: change port * feat: start config * feat: remove port reuse * feat: prometheus config * feat: prometheus config * feat: prometheus config * feat: add long time send msg to grafana * feat: init * feat: init * feat: implement offline push. * feat: batch get user online * feat: implement batch Push spilt * update go mod * Revert "feat: change port" This reverts commit 06d5e944 * feat: change port * feat: change config * feat: implement kafka producer and consumer. * update format, * add PushMQ log. * feat: get all online users and init push * feat: lock in online cache * feat: config * fix: init online status * fix: add logs * fix: userIDs * fix: add logs * feat: update Handler logic. * update MQ logic. * update * update * fix: method name * fix: update OfflinePushConsumerHandler. * fix: prommetrics * fix: add logs * fix: ctx * fix: log * fix: config * feat: change port * fix: atomic online cache status --------- Co-authored-by: Monet Lee * feature: add GetConversationsHasReadAndMaxSeq interface to the WebSocket API. (#2611) * fix: lru lock (#2613) * fix: lru lock * fix: lru lock * fix: lru lock * fix: nil pointer error on close (#2618) * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * update gomake version * update gomake version * fix: seq conversion bug * fix: redis pipe exec * fix: ImportFriends * fix: A large number of logs keysAndValues ​​length is not even * feat: mark read aggregate write * feat: online status supports redis cluster * feat: online status supports redis cluster * feat: online status supports redis cluster * merge * merge * read seq is written to mongo * read seq is written to mongo * fix: invitation to join group notification * fix: friend op_user_id * feat: optimizing asynchronous context * feat: optimizing memamq size * feat: add GetSeqMessage * feat: GroupApplicationAgreeMemberEnterNotification * feat: GroupApplicationAgreeMemberEnterNotification * feat: go.mod * feat: go.mod * feat: join group notification and get seq * feat: join group notification and get seq * feat: avoid pulling messages from sessions with a large number of max seq values of 0 * feat: API supports gzip * go.mod * fix: nil pointer error on close --------- Co-authored-by: withchao * feat: create group can push notification (#2617) * fix: blockage caused by listen error (#2620) * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * update gomake version * update gomake version * fix: seq conversion bug * fix: redis pipe exec * fix: ImportFriends * fix: A large number of logs keysAndValues ​​length is not even * feat: mark read aggregate write * feat: online status supports redis cluster * feat: online status supports redis cluster * feat: online status supports redis cluster * merge * merge * read seq is written to mongo * read seq is written to mongo * fix: invitation to join group notification * fix: friend op_user_id * feat: optimizing asynchronous context * feat: optimizing memamq size * feat: add GetSeqMessage * feat: GroupApplicationAgreeMemberEnterNotification * feat: GroupApplicationAgreeMemberEnterNotification * feat: go.mod * feat: go.mod * feat: join group notification and get seq * feat: join group notification and get seq * feat: avoid pulling messages from sessions with a large number of max seq values of 0 * feat: API supports gzip * go.mod * fix: nil pointer error on close * fix: listen error --------- Co-authored-by: withchao * fix: go.mod (#2621) * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * update gomake version * update gomake version * fix: seq conversion bug * fix: redis pipe exec * fix: ImportFriends * fix: A large number of logs keysAndValues ​​length is not even * feat: mark read aggregate write * feat: online status supports redis cluster * feat: online status supports redis cluster * feat: online status supports redis cluster * merge * merge * read seq is written to mongo * read seq is written to mongo * fix: invitation to join group notification * fix: friend op_user_id * feat: optimizing asynchronous context * feat: optimizing memamq size * feat: add GetSeqMessage * feat: GroupApplicationAgreeMemberEnterNotification * feat: GroupApplicationAgreeMemberEnterNotification * feat: go.mod * feat: go.mod * feat: join group notification and get seq * feat: join group notification and get seq * feat: avoid pulling messages from sessions with a large number of max seq values of 0 * feat: API supports gzip * go.mod * fix: nil pointer error on close * fix: listen error * fix: listen error * update go.mod --------- Co-authored-by: withchao * feat: improve searchMsg implement. (#2614) * refactor: refactor workflows contents. * add tool workflows. * update field. * fix: remove chat error. * Fix err. * fix error. * remove cn comment. * update workflows files. * update infra config. * move workflows. * feat: update bot. * fix: solve uncorrect outdated msg get. * update get docIDs logic. * update * update skip logic. * fix * update. * fix: delay deleteObject func. * remove unused content. * update log type. * feat: implement request batch count limit. * update * update * remove unused script. * feat: improve searchMsg implement. * update mongo config. * Fix lock (#2622) * fix:log * fix: lock * fix: update setGroupInfoEX field name. (#2625) * refactor: refactor workflows contents. * add tool workflows. * update field. * fix: remove chat error. * Fix err. * fix error. * remove cn comment. * update workflows files. * update infra config. * move workflows. * feat: update bot. * fix: solve uncorrect outdated msg get. * update get docIDs logic. * update * update skip logic. * fix * update. * fix: delay deleteObject func. * remove unused content. * update log type. * feat: implement request batch count limit. * update * update * fix: update setGroupInfoEX field name. * fix: update setGroupInfoEX field name (#2626) * refactor: refactor workflows contents. * add tool workflows. * update field. * fix: remove chat error. * Fix err. * fix error. * remove cn comment. * update workflows files. * update infra config. * move workflows. * feat: update bot. * fix: solve uncorrect outdated msg get. * update get docIDs logic. * update * update skip logic. * fix * update. * fix: delay deleteObject func. * remove unused content. * update log type. * feat: implement request batch count limit. * update * update * fix: update setGroupInfoEX field name. * fix: update setGroupInfoEX field name * feat: msg gateway add log (#2631) * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * update gomake version * update gomake version * fix: seq conversion bug * fix: redis pipe exec * fix: ImportFriends * fix: A large number of logs keysAndValues ​​length is not even * feat: mark read aggregate write * feat: online status supports redis cluster * feat: online status supports redis cluster * feat: online status supports redis cluster * merge * merge * read seq is written to mongo * read seq is written to mongo * fix: invitation to join group notification * fix: friend op_user_id * feat: optimizing asynchronous context * feat: optimizing memamq size * feat: add GetSeqMessage * feat: GroupApplicationAgreeMemberEnterNotification * feat: GroupApplicationAgreeMemberEnterNotification * feat: go.mod * feat: go.mod * feat: join group notification and get seq * feat: join group notification and get seq * feat: avoid pulling messages from sessions with a large number of max seq values of 0 * feat: API supports gzip * go.mod * fix: nil pointer error on close * fix: listen error * fix: listen error * update go.mod * feat: add log --------- Co-authored-by: withchao * fix: update setGroupInfoEx func name and field. (#2634) * refactor: refactor workflows contents. * add tool workflows. * update field. * fix: remove chat error. * Fix err. * fix error. * remove cn comment. * update workflows files. * update infra config. * move workflows. * feat: update bot. * fix: solve uncorrect outdated msg get. * update get docIDs logic. * update * update skip logic. * fix * update. * fix: delay deleteObject func. * remove unused content. * update log type. * feat: implement request batch count limit. * update * update * fix: update setGroupInfoEx func name and field. * refactor: update groupinfoEx field. * refactor: update database name in mongodb.yml * add groupName Condition * fix: fix setConversations req fill. (#2645) * refactor: refactor workflows contents. * add tool workflows. * update field. * fix: remove chat error. * Fix err. * fix error. * remove cn comment. * update workflows files. * update infra config. * move workflows. * feat: update bot. * fix: solve uncorrect outdated msg get. * update get docIDs logic. * update * update skip logic. * fix * update. * fix: delay deleteObject func. * remove unused content. * update log type. * feat: implement request batch count limit. * update * update * fix: fix setConversations req fill. * fix: GetMsgBySeqs boundary issues (#2647) * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * update gomake version * update gomake version * fix: seq conversion bug * fix: redis pipe exec * fix: ImportFriends * fix: A large number of logs keysAndValues ​​length is not even * feat: mark read aggregate write * feat: online status supports redis cluster * feat: online status supports redis cluster * feat: online status supports redis cluster * merge * merge * read seq is written to mongo * read seq is written to mongo * fix: invitation to join group notification * fix: friend op_user_id * feat: optimizing asynchronous context * feat: optimizing memamq size * feat: add GetSeqMessage * feat: GroupApplicationAgreeMemberEnterNotification * feat: GroupApplicationAgreeMemberEnterNotification * feat: go.mod * feat: go.mod * feat: join group notification and get seq * feat: join group notification and get seq * feat: avoid pulling messages from sessions with a large number of max seq values of 0 * feat: API supports gzip * go.mod * fix: nil pointer error on close * fix: listen error * fix: listen error * update go.mod * feat: add log * fix: token parse token value * fix: GetMsgBySeqs boundary issues --------- Co-authored-by: withchao * fix: the attribute version is obsolete, remove it (#2644) * refactor: update Userregister request field. (#2650) --------- Co-authored-by: Monet Lee Co-authored-by: icey-yu <119291641+icey-yu@users.noreply.github.com> Co-authored-by: chao <48119764+withchao@users.noreply.github.com> Co-authored-by: withchao Co-authored-by: 蔡相跃 * update go mod * fix: sn_ not sort * fix: sn_ not sort * fix: sn_ not sort * fix: jssdk add * fix: jssdk support * fix: jssdk support * fix: jssdk support * fix: the message I sent is not set to read seq in mongodb * fix: cannot modify group member avatars * merge: update code from main to v3.8-js-sdk-only. (#2720) * fix: fix update groupName invalid. (#2673) * refactor: change platform to platformID (#2670) * feat: don`t return nil data (#2675) Co-authored-by: Monet Lee * refactor: update fields type in userStatus and check registered. (#2676) * fix: usertoken auth. (#2677) * refactor: update fields type in userStatus and check registered. * fix: usertoken auth. * update contents. * update content. * update * fix * update pb file. * feat: add friend agree after callback (#2680) * fix: sn not sort (#2682) * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * update gomake version * update gomake version * fix: seq conversion bug * fix: redis pipe exec * fix: ImportFriends * fix: A large number of logs keysAndValues ​​length is not even * feat: mark read aggregate write * feat: online status supports redis cluster * feat: online status supports redis cluster * feat: online status supports redis cluster * merge * merge * read seq is written to mongo * read seq is written to mongo * fix: invitation to join group notification * fix: friend op_user_id * feat: optimizing asynchronous context * feat: optimizing memamq size * feat: add GetSeqMessage * feat: GroupApplicationAgreeMemberEnterNotification * feat: GroupApplicationAgreeMemberEnterNotification * feat: go.mod * feat: go.mod * feat: join group notification and get seq * feat: join group notification and get seq * feat: avoid pulling messages from sessions with a large number of max seq values of 0 * feat: API supports gzip * go.mod * fix: nil pointer error on close * fix: listen error * fix: listen error * update go.mod * feat: add log * fix: token parse token value * fix: GetMsgBySeqs boundary issues * fix: sn_ not sort * fix: sn_ not sort * fix: sn_ not sort --------- Co-authored-by: withchao * refactor: add GetAdminToken interface. (#2684) * refactor: add GetAdminToken interface. * update config. * fix: admin token (#2686) * fix: update workflows logic. (#2688) * refactor: add GetAdminToken interface. * update config. * update workflows logic. * fix: admin token (#2687) * update the front image (#2692) * update the front image * update version * feat: improve publish docker image workflows (#2697) * refactor: add GetAdminToken interface. * update config. * update workflows logic. * feat: improve publish docker image workflows * update condition logic. * fix: update load file logic. (#2700) * refactor: add GetAdminToken interface. * update config. * update workflows logic. * feat: improve publish docker image workflows * update condition logic. * fix: update load file logic. * feat: Msg filter (#2703) * feat: msg filter * feat: msg filter * feat: msg filter * feat: provide the interface required by js sdk (#2712) * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * update gomake version * update gomake version * fix: seq conversion bug * fix: redis pipe exec * fix: ImportFriends * fix: A large number of logs keysAndValues ​​length is not even * feat: mark read aggregate write * feat: online status supports redis cluster * feat: online status supports redis cluster * feat: online status supports redis cluster * merge * merge * read seq is written to mongo * read seq is written to mongo * fix: invitation to join group notification * fix: friend op_user_id * feat: optimizing asynchronous context * feat: optimizing memamq size * feat: add GetSeqMessage * feat: GroupApplicationAgreeMemberEnterNotification * feat: GroupApplicationAgreeMemberEnterNotification * feat: go.mod * feat: go.mod * feat: join group notification and get seq * feat: join group notification and get seq * feat: avoid pulling messages from sessions with a large number of max seq values of 0 * feat: API supports gzip * go.mod * fix: nil pointer error on close * fix: listen error * fix: listen error * update go.mod * feat: add log * fix: token parse token value * fix: GetMsgBySeqs boundary issues * fix: sn_ not sort * fix: sn_ not sort * fix: sn_ not sort * fix: jssdk add * fix: jssdk support * fix: jssdk support * fix: jssdk support --------- Co-authored-by: withchao * Line webhook (#2716) * feat: online and offline webhook * feat: online and offline webhook * feat: remove zk * fix: the message I sent is not set to read seq in mongodb (#2718) * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * update gomake version * update gomake version * fix: seq conversion bug * fix: redis pipe exec * fix: ImportFriends * fix: A large number of logs keysAndValues ​​length is not even * feat: mark read aggregate write * feat: online status supports redis cluster * feat: online status supports redis cluster * feat: online status supports redis cluster * merge * merge * read seq is written to mongo * read seq is written to mongo * fix: invitation to join group notification * fix: friend op_user_id * feat: optimizing asynchronous context * feat: optimizing memamq size * feat: add GetSeqMessage * feat: GroupApplicationAgreeMemberEnterNotification * feat: GroupApplicationAgreeMemberEnterNotification * feat: go.mod * feat: go.mod * feat: join group notification and get seq * feat: join group notification and get seq * feat: avoid pulling messages from sessions with a large number of max seq values of 0 * feat: API supports gzip * go.mod * fix: nil pointer error on close * fix: listen error * fix: listen error * update go.mod * feat: add log * fix: token parse token value * fix: GetMsgBySeqs boundary issues * fix: sn_ not sort * fix: sn_ not sort * fix: sn_ not sort * fix: jssdk add * fix: jssdk support * fix: jssdk support * fix: jssdk support * fix: the message I sent is not set to read seq in mongodb --------- Co-authored-by: withchao * fix: cannot modify group member avatars (#2719) * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * update gomake version * update gomake version * fix: seq conversion bug * fix: redis pipe exec * fix: ImportFriends * fix: A large number of logs keysAndValues ​​length is not even * feat: mark read aggregate write * feat: online status supports redis cluster * feat: online status supports redis cluster * feat: online status supports redis cluster * merge * merge * read seq is written to mongo * read seq is written to mongo * fix: invitation to join group notification * fix: friend op_user_id * feat: optimizing asynchronous context * feat: optimizing memamq size * feat: add GetSeqMessage * feat: GroupApplicationAgreeMemberEnterNotification * feat: GroupApplicationAgreeMemberEnterNotification * feat: go.mod * feat: go.mod * feat: join group notification and get seq * feat: join group notification and get seq * feat: avoid pulling messages from sessions with a large number of max seq values of 0 * feat: API supports gzip * go.mod * fix: nil pointer error on close * fix: listen error * fix: listen error * update go.mod * feat: add log * fix: token parse token value * fix: GetMsgBySeqs boundary issues * fix: sn_ not sort * fix: sn_ not sort * fix: sn_ not sort * fix: jssdk add * fix: jssdk support * fix: jssdk support * fix: jssdk support * fix: the message I sent is not set to read seq in mongodb * fix: cannot modify group member avatars --------- Co-authored-by: withchao --------- Co-authored-by: Monet Lee Co-authored-by: icey-yu <119291641+icey-yu@users.noreply.github.com> Co-authored-by: chao <48119764+withchao@users.noreply.github.com> Co-authored-by: withchao Co-authored-by: skiffer-git <72860476+skiffer-git@users.noreply.github.com> * fix: MemberEnterNotification * fix: MemberEnterNotification * fix: MsgData status * merge * merge: update code from main to v3.8-js-sdk-only. (#2818) * feat: implement merge milestone PR to target-branch. (#2796) * build: improve workflows logic. (#2801) * fix: improve time condition check mehtod. (#2804) * fix: improve time condition check mehtod. * fix * fix: webhook before online push (#2805) * fix: set own read seq in MongoDB when sender send a message. (#2808) * fix: solve err Notification when setGroupInfo. (#2806) * fix: solve err Notification when setGroupInfo. * build: update checkout version. * fix: update notification contents. * Introducing OpenIM Guru on Gurubase.io (#2788) * feat: support app update service (#2811) * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * update gomake version * update gomake version * fix: seq conversion bug * fix: redis pipe exec * fix: ImportFriends * fix: A large number of logs keysAndValues ​​length is not even * feat: mark read aggregate write * feat: online status supports redis cluster * feat: online status supports redis cluster * feat: online status supports redis cluster * merge * merge * read seq is written to mongo * read seq is written to mongo * fix: invitation to join group notification * fix: friend op_user_id * feat: optimizing asynchronous context * feat: optimizing memamq size * feat: add GetSeqMessage * feat: GroupApplicationAgreeMemberEnterNotification * feat: GroupApplicationAgreeMemberEnterNotification * feat: go.mod * feat: go.mod * feat: join group notification and get seq * feat: join group notification and get seq * feat: avoid pulling messages from sessions with a large number of max seq values of 0 * feat: API supports gzip * go.mod * fix: nil pointer error on close * fix: listen error * fix: listen error * update go.mod * feat: add log * fix: token parse token value * fix: GetMsgBySeqs boundary issues * fix: sn_ not sort * fix: sn_ not sort * fix: sn_ not sort * fix: jssdk add * fix: jssdk support * fix: jssdk support * fix: jssdk support * fix: the message I sent is not set to read seq in mongodb * fix: cannot modify group member avatars * fix: MemberEnterNotification * fix: MemberEnterNotification * fix: MsgData status * feat: add ApplicationVersion * feat: add ApplicationVersion * feat: add ApplicationVersion --------- Co-authored-by: withchao * feat: ApplicationVersion move chat (#2813) * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * update gomake version * update gomake version * fix: seq conversion bug * fix: redis pipe exec * fix: ImportFriends * fix: A large number of logs keysAndValues ​​length is not even * feat: mark read aggregate write * feat: online status supports redis cluster * feat: online status supports redis cluster * feat: online status supports redis cluster * merge * merge * read seq is written to mongo * read seq is written to mongo * fix: invitation to join group notification * fix: friend op_user_id * feat: optimizing asynchronous context * feat: optimizing memamq size * feat: add GetSeqMessage * feat: GroupApplicationAgreeMemberEnterNotification * feat: GroupApplicationAgreeMemberEnterNotification * feat: go.mod * feat: go.mod * feat: join group notification and get seq * feat: join group notification and get seq * feat: avoid pulling messages from sessions with a large number of max seq values of 0 * feat: API supports gzip * go.mod * fix: nil pointer error on close * fix: listen error * fix: listen error * update go.mod * feat: add log * fix: token parse token value * fix: GetMsgBySeqs boundary issues * fix: sn_ not sort * fix: sn_ not sort * fix: sn_ not sort * fix: jssdk add * fix: jssdk support * fix: jssdk support * fix: jssdk support * fix: the message I sent is not set to read seq in mongodb * fix: cannot modify group member avatars * fix: MemberEnterNotification * fix: MemberEnterNotification * fix: MsgData status * feat: add ApplicationVersion * feat: ApplicationVersion move chat --------- Co-authored-by: withchao * fix: improve condition check. (#2815) --------- Co-authored-by: Monet Lee Co-authored-by: icey-yu <119291641+icey-yu@users.noreply.github.com> Co-authored-by: Kürşat Aktaş Co-authored-by: chao <48119764+withchao@users.noreply.github.com> Co-authored-by: withchao * feat: support text ping pong * feat: support text ping pong * feat: gob json encoder * feat: gob json encoder * feat: gob json encoder * feat: gob json encoder * feat: gob json encoder * feat: gob json encoder --------- Co-authored-by: withchao Co-authored-by: Monet Lee Co-authored-by: OpenIM-Gordon <46924906+FGadvancer@users.noreply.github.com> Co-authored-by: icey-yu <119291641+icey-yu@users.noreply.github.com> Co-authored-by: 蔡相跃 Co-authored-by: skiffer-git <72860476+skiffer-git@users.noreply.github.com> Co-authored-by: Kürşat Aktaş --- config/openim-msggateway.yml | 2 -- config/openim-push.yml | 26 +++++++-------- internal/msggateway/client.go | 45 +++++++++++++++++++++++--- internal/msggateway/constant.go | 6 ++++ internal/msggateway/context.go | 14 +++++++- internal/msggateway/encoder.go | 33 ++++++++++++++++--- internal/msggateway/hub_server.go | 16 +++------ internal/msggateway/message_handler.go | 11 +++++++ internal/msggateway/ws_server.go | 13 ++------ pkg/common/storage/controller/auth.go | 7 ++-- 10 files changed, 120 insertions(+), 53 deletions(-) diff --git a/config/openim-msggateway.yml b/config/openim-msggateway.yml index 5659c6f9b..6c46b52a8 100644 --- a/config/openim-msggateway.yml +++ b/config/openim-msggateway.yml @@ -22,5 +22,3 @@ longConnSvr: websocketMaxMsgLen: 4096 # WebSocket connection handshake timeout in seconds websocketTimeout: 10 - - diff --git a/config/openim-push.yml b/config/openim-push.yml index 4d2aaca6b..2d54bb9e1 100644 --- a/config/openim-push.yml +++ b/config/openim-push.yml @@ -17,25 +17,25 @@ maxConcurrentWorkers: 3 enable: geTui geTui: pushUrl: https://restapi.getui.com/v2/$appId - masterSecret: - appKey: - intent: - channelID: - channelName: + masterSecret: + appKey: + intent: + channelID: + channelName: fcm: # Prioritize using file paths. If the file path is empty, use URL - filePath: # File path is concatenated with the parameters passed in through - c(`mage` default pass in `config/`) and filePath. + filePath: # File path is concatenated with the parameters passed in through - c(`mage` default pass in `config/`) and filePath. authURL: # Must start with https or http. jpns: - appKey: - masterSecret: - pushURL: - pushIntent: + appKey: + masterSecret: + pushURL: + pushIntent: # iOS system push sound and badge count iosPush: - pushSound: xxx - badgeCount: true - production: false + pushSound: xxx + badgeCount: true + production: false fullUserCache: true diff --git a/internal/msggateway/client.go b/internal/msggateway/client.go index bc06fa950..af96e7d46 100644 --- a/internal/msggateway/client.go +++ b/internal/msggateway/client.go @@ -16,6 +16,7 @@ package msggateway import ( "context" + "encoding/json" "fmt" "runtime/debug" "sync" @@ -69,6 +70,8 @@ type Client struct { IsCompress bool `json:"isCompress"` UserID string `json:"userID"` IsBackground bool `json:"isBackground"` + SDKType string `json:"sdkType"` + Encoder Encoder ctx *UserConnContext longConnServer LongConnServer closed atomic.Bool @@ -94,11 +97,17 @@ func (c *Client) ResetClient(ctx *UserConnContext, conn LongConn, longConnServer c.closed.Store(false) c.closedErr = nil c.token = ctx.GetToken() + c.SDKType = ctx.GetSDKType() c.hbCtx, c.hbCancel = context.WithCancel(c.ctx) c.subLock = new(sync.Mutex) if c.subUserIDs != nil { clear(c.subUserIDs) } + if c.SDKType == GoSDK { + c.Encoder = NewGobEncoder() + } else { + c.Encoder = NewJsonEncoder() + } c.subUserIDs = make(map[string]struct{}) } @@ -159,9 +168,12 @@ func (c *Client) readMessage() { return } case MessageText: - c.closedErr = ErrNotSupportMessageProtocol - return - + _ = c.conn.SetReadDeadline(pongWait) + parseDataErr := c.handlerTextMessage(message) + if parseDataErr != nil { + c.closedErr = parseDataErr + return + } case PingMessage: err := c.writePongMsg("") log.ZError(c.ctx, "writePongMsg", err) @@ -188,7 +200,7 @@ func (c *Client) handleMessage(message []byte) error { var binaryReq = getReq() defer freeReq(binaryReq) - err := c.longConnServer.Decode(message, binaryReq) + err := c.Encoder.Decode(message, binaryReq) if err != nil { return err } @@ -335,7 +347,7 @@ func (c *Client) writeBinaryMsg(resp Resp) error { return nil } - encodedBuf, err := c.longConnServer.Encode(resp) + encodedBuf, err := c.Encoder.Encode(resp) if err != nil { return err } @@ -419,3 +431,26 @@ func (c *Client) writePongMsg(appData string) error { return errs.Wrap(err) } + +func (c *Client) handlerTextMessage(b []byte) error { + var msg TextMessage + if err := json.Unmarshal(b, &msg); err != nil { + return err + } + switch msg.Type { + case TextPong: + return nil + case TextPing: + msg.Type = TextPong + msgData, err := json.Marshal(msg) + if err != nil { + return err + } + if err := c.conn.SetWriteDeadline(writeWait); err != nil { + return err + } + return c.conn.WriteMessage(MessageText, msgData) + default: + return fmt.Errorf("not support message type %s", msg.Type) + } +} diff --git a/internal/msggateway/constant.go b/internal/msggateway/constant.go index 584cebe1e..a825c0519 100644 --- a/internal/msggateway/constant.go +++ b/internal/msggateway/constant.go @@ -27,6 +27,12 @@ const ( GzipCompressionProtocol = "gzip" BackgroundStatus = "isBackground" SendResponse = "isMsgResp" + SDKType = "sdkType" +) + +const ( + GoSDK = "go" + JsSDK = "js" ) const ( diff --git a/internal/msggateway/context.go b/internal/msggateway/context.go index 3909766b1..d73a96df4 100644 --- a/internal/msggateway/context.go +++ b/internal/msggateway/context.go @@ -153,6 +153,14 @@ func (c *UserConnContext) GetCompression() bool { return false } +func (c *UserConnContext) GetSDKType() string { + sdkType := c.Req.URL.Query().Get(SDKType) + if sdkType == "" { + sdkType = GoSDK + } + return sdkType +} + func (c *UserConnContext) ShouldSendResp() bool { errResp, exists := c.Query(SendResponse) if exists { @@ -193,7 +201,11 @@ func (c *UserConnContext) ParseEssentialArgs() error { _, err := strconv.Atoi(platformIDStr) if err != nil { return servererrs.ErrConnArgsErr.WrapMsg("platformID is not int") - + } + switch sdkType, _ := c.Query(SDKType); sdkType { + case "", GoSDK, JsSDK: + default: + return servererrs.ErrConnArgsErr.WrapMsg("sdkType is not go or js") } return nil } diff --git a/internal/msggateway/encoder.go b/internal/msggateway/encoder.go index 3af266374..6a5936d6d 100644 --- a/internal/msggateway/encoder.go +++ b/internal/msggateway/encoder.go @@ -17,6 +17,7 @@ package msggateway import ( "bytes" "encoding/gob" + "encoding/json" "github.com/openimsdk/tools/errs" ) @@ -28,12 +29,12 @@ type Encoder interface { type GobEncoder struct{} -func NewGobEncoder() *GobEncoder { - return &GobEncoder{} +func NewGobEncoder() Encoder { + return GobEncoder{} } -func (g *GobEncoder) Encode(data any) ([]byte, error) { - buff := bytes.Buffer{} +func (g GobEncoder) Encode(data any) ([]byte, error) { + var buff bytes.Buffer enc := gob.NewEncoder(&buff) if err := enc.Encode(data); err != nil { return nil, errs.WrapMsg(err, "GobEncoder.Encode failed", "action", "encode") @@ -41,7 +42,7 @@ func (g *GobEncoder) Encode(data any) ([]byte, error) { return buff.Bytes(), nil } -func (g *GobEncoder) Decode(encodeData []byte, decodeData any) error { +func (g GobEncoder) Decode(encodeData []byte, decodeData any) error { buff := bytes.NewBuffer(encodeData) dec := gob.NewDecoder(buff) if err := dec.Decode(decodeData); err != nil { @@ -49,3 +50,25 @@ func (g *GobEncoder) Decode(encodeData []byte, decodeData any) error { } return nil } + +type JsonEncoder struct{} + +func NewJsonEncoder() Encoder { + return JsonEncoder{} +} + +func (g JsonEncoder) Encode(data any) ([]byte, error) { + b, err := json.Marshal(data) + if err != nil { + return nil, errs.New("JsonEncoder.Encode failed", "action", "encode") + } + return b, nil +} + +func (g JsonEncoder) Decode(encodeData []byte, decodeData any) error { + err := json.Unmarshal(encodeData, decodeData) + if err != nil { + return errs.New("JsonEncoder.Decode failed", "action", "decode") + } + return nil +} diff --git a/internal/msggateway/hub_server.go b/internal/msggateway/hub_server.go index e96ab4b0d..23d915013 100644 --- a/internal/msggateway/hub_server.go +++ b/internal/msggateway/hub_server.go @@ -83,17 +83,11 @@ func NewServer(rpcPort int, longConnServer LongConnServer, conf *Config, ready f return s } -func (s *Server) OnlinePushMsg( - context context.Context, - req *msggateway.OnlinePushMsgReq, -) (*msggateway.OnlinePushMsgResp, error) { +func (s *Server) OnlinePushMsg(context context.Context, req *msggateway.OnlinePushMsgReq) (*msggateway.OnlinePushMsgResp, error) { panic("implement me") } -func (s *Server) GetUsersOnlineStatus( - ctx context.Context, - req *msggateway.GetUsersOnlineStatusReq, -) (*msggateway.GetUsersOnlineStatusResp, error) { +func (s *Server) GetUsersOnlineStatus(ctx context.Context, req *msggateway.GetUsersOnlineStatusReq) (*msggateway.GetUsersOnlineStatusResp, error) { if !authverify.IsAppManagerUid(ctx, s.config.Share.IMAdminUserID) { return nil, errs.ErrNoPermission.WrapMsg("only app manager") } @@ -155,6 +149,7 @@ func (s *Server) pushToUser(ctx context.Context, userID string, msgData *sdkws.M (client.IsBackground && client.PlatformID != constant.IOSPlatformID) { err := client.PushMessage(ctx, msgData) if err != nil { + log.ZWarn(ctx, "online push msg failed", err, "userID", userID, "platformID", client.PlatformID) userPlatform.ResultCode = int64(servererrs.ErrPushMsgErr.Code()) } else { if _, ok := s.pushTerminal[client.PlatformID]; ok { @@ -220,10 +215,7 @@ func (s *Server) SuperGroupOnlineBatchPushOneMsg(ctx context.Context, req *msgga } } -func (s *Server) KickUserOffline( - ctx context.Context, - req *msggateway.KickUserOfflineReq, -) (*msggateway.KickUserOfflineResp, error) { +func (s *Server) KickUserOffline(ctx context.Context, req *msggateway.KickUserOfflineReq) (*msggateway.KickUserOfflineResp, error) { for _, v := range req.KickUserIDList { clients, _, ok := s.LongConnServer.GetUserPlatformCons(v, int(req.PlatformID)) if !ok { diff --git a/internal/msggateway/message_handler.go b/internal/msggateway/message_handler.go index 4b78c1004..5407ba90c 100644 --- a/internal/msggateway/message_handler.go +++ b/internal/msggateway/message_handler.go @@ -16,6 +16,7 @@ package msggateway import ( "context" + "encoding/json" "sync" "github.com/go-playground/validator/v10" @@ -31,6 +32,16 @@ import ( "github.com/openimsdk/tools/utils/jsonutil" ) +const ( + TextPing = "ping" + TextPong = "pong" +) + +type TextMessage struct { + Type string `json:"type"` + Body json.RawMessage `json:"body"` +} + type Req struct { ReqIdentifier int32 `json:"reqIdentifier" validate:"required"` Token string `json:"token"` diff --git a/internal/msggateway/ws_server.go b/internal/msggateway/ws_server.go index 48e4b5cee..e6b4f3fa4 100644 --- a/internal/msggateway/ws_server.go +++ b/internal/msggateway/ws_server.go @@ -37,7 +37,6 @@ type LongConnServer interface { SetKickHandlerInfo(i *kickHandler) SubUserOnlineStatus(ctx context.Context, client *Client, data *Req) ([]byte, error) Compressor - Encoder MessageHandler } @@ -61,7 +60,7 @@ type WsServer struct { authClient *rpcclient.Auth disCov discovery.SvcDiscoveryRegistry Compressor - Encoder + //Encoder MessageHandler webhookClient *webhook.Client } @@ -135,7 +134,6 @@ func NewWsServer(msgGatewayConfig *Config, opts ...Option) *WsServer { clients: newUserMap(), subscription: newSubscription(), Compressor: NewGzipCompressor(), - Encoder: NewGobEncoder(), webhookClient: webhook.NewWebhookClient(msgGatewayConfig.WebhooksConfig.URL), } } @@ -278,14 +276,7 @@ func (ws *WsServer) registerClient(client *Client) { wg.Wait() - log.ZDebug( - client.ctx, - "user online", - "online user Num", - ws.onlineUserNum.Load(), - "online user conn Num", - ws.onlineUserConnNum.Load(), - ) + log.ZDebug(client.ctx, "user online", "online user Num", ws.onlineUserNum.Load(), "online user conn Num", ws.onlineUserConnNum.Load()) } func getRemoteAdders(client []*Client) string { diff --git a/pkg/common/storage/controller/auth.go b/pkg/common/storage/controller/auth.go index 82d6f9476..df1274967 100644 --- a/pkg/common/storage/controller/auth.go +++ b/pkg/common/storage/controller/auth.go @@ -2,15 +2,14 @@ package controller import ( "context" - "github.com/openimsdk/open-im-server/v3/pkg/common/config" - "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/cachekey" - "github.com/openimsdk/tools/log" - "github.com/golang-jwt/jwt/v4" "github.com/openimsdk/open-im-server/v3/pkg/authverify" + "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/cachekey" "github.com/openimsdk/protocol/constant" "github.com/openimsdk/tools/errs" + "github.com/openimsdk/tools/log" "github.com/openimsdk/tools/tokenverify" ) From e0284724b5e4dafef311e1e513a90e8615d4c4f4 Mon Sep 17 00:00:00 2001 From: Monet Lee Date: Mon, 18 Nov 2024 11:32:47 +0800 Subject: [PATCH 037/199] build: update mongo and kafka start logic. (#2858) * build: update mongo and kafka start logic. * build: update go version image in dockerfile. * build: remove zookeeper image. * add authSource comment. * update tools version. * add created sucess print. * remove unused script. * format. --- .env | 4 +-- Dockerfile | 6 ++-- config/mongodb.yml | 2 ++ docker-compose.yml | 36 ++++++++++++++----- go.mod | 8 +++-- go.sum | 12 +++---- pkg/common/config/config.go | 2 ++ scripts/create-topic.sh | 55 ----------------------------- scripts/mongo-init.sh | 66 ----------------------------------- tools/check-component/main.go | 2 +- 10 files changed, 48 insertions(+), 145 deletions(-) delete mode 100755 scripts/create-topic.sh delete mode 100755 scripts/mongo-init.sh diff --git a/.env b/.env index 0c02bf9f6..2b903c949 100644 --- a/.env +++ b/.env @@ -1,6 +1,5 @@ -MONGO_IMAGE=mongo:6.0.2 +MONGO_IMAGE=mongo:7.0 REDIS_IMAGE=redis:7.0.0 -ZOOKEEPER_IMAGE=bitnami/zookeeper:3.8 KAFKA_IMAGE=bitnami/kafka:3.5.1 MINIO_IMAGE=minio/minio:RELEASE.2024-01-11T07-46-16Z ETCD_IMAGE=quay.io/coreos/etcd:v3.5.13 @@ -16,4 +15,3 @@ OPENIM_ADMIN_FRONT_IMAGE=openim/openim-admin-front:release-v1.8.2 #OPENIM_ADMIN_FRONT_IMAGE=registry.cn-hangzhou.aliyuncs.com/openimsdk/openim-admin-front:release-v1.8.2 DATA_DIR=./ - diff --git a/Dockerfile b/Dockerfile index f8cfbda9e..4b38d711b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ -# Use Go 1.21 Alpine as the base image for building the application -FROM golang:1.21-alpine AS builder +# Use Go 1.22 Alpine as the base image for building the application +FROM golang:1.22-alpine AS builder # Define the base directory for the application as an environment variable ENV SERVER_DIR=/openim-server @@ -22,7 +22,7 @@ RUN go install github.com/magefile/mage@v1.15.0 RUN mage build # Using Alpine Linux with Go environment for the final image -FROM golang:1.21-alpine +FROM golang:1.22-alpine # Install necessary packages, such as bash RUN apk add --no-cache bash diff --git a/config/mongodb.yml b/config/mongodb.yml index 78f85992c..072cb4b8f 100644 --- a/config/mongodb.yml +++ b/config/mongodb.yml @@ -8,6 +8,8 @@ database: openim_v3 username: openIM # Password for database authentication password: openIM123 +# Authentication source for database authentication, if use root user, set it to admin +authSource: openim_v3 # Maximum number of connections in the connection pool maxPoolSize: 100 # Maximum number of retry attempts for a failed database connection diff --git a/docker-compose.yml b/docker-compose.yml index 6d88bac10..8d25383bc 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -8,12 +8,35 @@ services: ports: - "37017:27017" container_name: mongo - command: ["/bin/bash", "-c", "/docker-entrypoint-initdb.d/mongo-init.sh; docker-entrypoint.sh mongod --wiredTigerCacheSizeGB 1 --auth"] + command: > + bash -c ' + docker-entrypoint.sh mongod --wiredTigerCacheSizeGB $$wiredTigerCacheSizeGB --auth & + until mongosh -u $$MONGO_INITDB_ROOT_USERNAME -p $$MONGO_INITDB_ROOT_PASSWORD --authenticationDatabase admin --eval "db.runCommand({ ping: 1 })" &>/dev/null; do + echo "Waiting for MongoDB to start..." + sleep 1 + done && + mongosh -u $$MONGO_INITDB_ROOT_USERNAME -p $$MONGO_INITDB_ROOT_PASSWORD --authenticationDatabase admin --eval " + db = db.getSiblingDB(\"$$MONGO_INITDB_DATABASE\"); + if (!db.getUser(\"$$MONGO_OPENIM_USERNAME\")) { + db.createUser({ + user: \"$$MONGO_OPENIM_USERNAME\", + pwd: \"$$MONGO_OPENIM_PASSWORD\", + roles: [{role: \"readWrite\", db: \"$$MONGO_INITDB_DATABASE\"}] + }); + print(\"User created successfully: \"); + print(\"Username: $$MONGO_OPENIM_USERNAME\"); + print(\"Password: $$MONGO_OPENIM_PASSWORD\"); + print(\"Database: $$MONGO_INITDB_DATABASE\"); + } else { + print(\"User already exists in database: $$MONGO_INITDB_DATABASE, Username: $$MONGO_OPENIM_USERNAME\"); + } + " && + tail -f /dev/null + ' volumes: - "${DATA_DIR}/components/mongodb/data/db:/data/db" - "${DATA_DIR}/components/mongodb/data/logs:/data/logs" - "${DATA_DIR}/components/mongodb/data/conf:/etc/mongo" - - "./scripts/mongo-init.sh:/docker-entrypoint-initdb.d/mongo-init.sh:ro" environment: - TZ=Asia/Shanghai - wiredTigerCacheSizeGB=1 @@ -71,10 +94,7 @@ services: ports: - "19094:9094" volumes: - - ./scripts/create-topic.sh:/opt/bitnami/kafka/create-topic.sh - "${DATA_DIR}/components/kafka:/bitnami/kafka" - command: > - bash -c "/opt/bitnami/scripts/kafka/run.sh & /opt/bitnami/kafka/create-topic.sh; wait" environment: #KAFKA_HEAP_OPTS: "-Xms128m -Xmx256m" TZ: Asia/Shanghai @@ -85,10 +105,11 @@ services: KAFKA_CFG_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092,EXTERNAL://localhost:19094 KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,EXTERNAL:PLAINTEXT,PLAINTEXT:PLAINTEXT KAFKA_CFG_CONTROLLER_LISTENER_NAMES: CONTROLLER + KAFKA_NUM_PARTITIONS: 8 + KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE: "true" networks: - openim - minio: image: "${MINIO_IMAGE}" ports: @@ -124,7 +145,7 @@ services: - "11002:80" networks: - openim - + # prometheus: # image: ${PROMETHEUS_IMAGE} # container_name: prometheus @@ -171,4 +192,3 @@ services: # - ${DATA_DIR:-./}/components/grafana:/var/lib/grafana # networks: # - openim - diff --git a/go.mod b/go.mod index 051beb403..8a05b016c 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,8 @@ module github.com/openimsdk/open-im-server/v3 -go 1.21.2 +go 1.22.0 + +toolchain go1.23.2 require ( firebase.google.com/go/v4 v4.14.1 @@ -13,7 +15,7 @@ require ( github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/mitchellh/mapstructure v1.5.0 github.com/openimsdk/protocol v0.0.72-alpha.54 - github.com/openimsdk/tools v0.0.50-alpha.16 + github.com/openimsdk/tools v0.0.50-alpha.32 github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 github.com/stretchr/testify v1.9.0 @@ -92,7 +94,7 @@ require ( github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/gabriel-vasile/mimetype v1.4.3 // indirect github.com/gin-contrib/sse v0.1.0 // indirect - github.com/go-logr/logr v1.4.1 // indirect + github.com/go-logr/logr v1.4.2 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-ole/go-ole v1.2.6 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect diff --git a/go.sum b/go.sum index 816c82094..3cae53b0d 100644 --- a/go.sum +++ b/go.sum @@ -126,8 +126,8 @@ github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm github.com/gin-gonic/gin v1.9.1 h1:4idEAncQnU5cB7BeOkPtxjfCSye0AAm1R0RVIqJ+Jmg= github.com/gin-gonic/gin v1.9.1/go.mod h1:hPrL7YrpYKXt5YId3A/Tnip5kqbEAP+KLuI3SUcPTeU= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= -github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= @@ -321,8 +321,8 @@ github.com/openimsdk/gomake v0.0.14-alpha.5 h1:VY9c5x515lTfmdhhPjMvR3BBRrRquAUCF github.com/openimsdk/gomake v0.0.14-alpha.5/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= github.com/openimsdk/protocol v0.0.72-alpha.54 h1:opato7N4QjjRq/SHD54bDSVBpOEEDp1VLWVk5Os2A9s= github.com/openimsdk/protocol v0.0.72-alpha.54/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= -github.com/openimsdk/tools v0.0.50-alpha.16 h1:bC1AQvJMuOHtZm8LZRvN8L5mH1Ws2VYdL+TLTs1iGSc= -github.com/openimsdk/tools v0.0.50-alpha.16/go.mod h1:h1cYmfyaVtgFbKmb1Cfsl8XwUOMTt8ubVUQrdGtsUh4= +github.com/openimsdk/tools v0.0.50-alpha.32 h1:JEsUFHFnaYg230TG+Ke3SUnaA2h44t4kABAzEdv5VZw= +github.com/openimsdk/tools v0.0.50-alpha.32/go.mod h1:r5U6RbxcR4xhKb2fhTmKGC9Yt5LcErHBVt3lhXQIHSo= github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= @@ -356,8 +356,8 @@ github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE= -github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= -github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= +github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= +github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc= github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= diff --git a/pkg/common/config/config.go b/pkg/common/config/config.go index da6c63d60..10285b565 100644 --- a/pkg/common/config/config.go +++ b/pkg/common/config/config.go @@ -69,6 +69,7 @@ type Mongo struct { Database string `mapstructure:"database"` Username string `mapstructure:"username"` Password string `mapstructure:"password"` + AuthSource string `mapstructure:"authSource"` MaxPoolSize int `mapstructure:"maxPoolSize"` MaxRetry int `mapstructure:"maxRetry"` } @@ -490,6 +491,7 @@ func (m *Mongo) Build() *mongoutil.Config { Database: m.Database, Username: m.Username, Password: m.Password, + AuthSource: m.AuthSource, MaxPoolSize: m.MaxPoolSize, MaxRetry: m.MaxRetry, } diff --git a/scripts/create-topic.sh b/scripts/create-topic.sh deleted file mode 100755 index bbc739287..000000000 --- a/scripts/create-topic.sh +++ /dev/null @@ -1,55 +0,0 @@ -#!/usr/bin/env bash -# Copyright © 2023 OpenIM. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Wait for Kafka to be ready - -KAFKA_SERVER=localhost:9092 - -MAX_ATTEMPTS=300 -attempt_num=1 - -echo "Waiting for Kafka to be ready..." - -until /opt/bitnami/kafka/bin/kafka-topics.sh --list --bootstrap-server $KAFKA_SERVER; do - echo "Attempt $attempt_num of $MAX_ATTEMPTS: Kafka not ready yet..." - if [ $attempt_num -eq $MAX_ATTEMPTS ]; then - echo "Kafka not ready after $MAX_ATTEMPTS attempts, exiting" - exit 1 - fi - attempt_num=$((attempt_num+1)) - sleep 1 -done - -echo "Kafka is ready. Creating topics..." - - -topics=("toRedis" "toMongo" "toPush" "toOfflinePush") -partitions=8 -replicationFactor=1 - -for topic in "${topics[@]}"; do - if /opt/bitnami/kafka/bin/kafka-topics.sh --create \ - --bootstrap-server $KAFKA_SERVER \ - --replication-factor $replicationFactor \ - --partitions $partitions \ - --topic $topic - then - echo "Topic $topic created." - else - echo "Failed to create topic $topic." - fi -done - -echo "All topics created." diff --git a/scripts/mongo-init.sh b/scripts/mongo-init.sh deleted file mode 100755 index 25bb2d654..000000000 --- a/scripts/mongo-init.sh +++ /dev/null @@ -1,66 +0,0 @@ -# Copyright © 2023 OpenIM. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -mongosh < 0) { - try { - db = connect('mongodb://127.0.0.1:27017/admin'); - var authResult = db.auth(rootUsername, rootPassword); - if (authResult) { - print('Authentication successful for root user: ' + rootUsername); - connected = true; - } else { - print('Authentication failed for root user: ' + rootUsername + ' with password: ' + rootPassword); - quit(1); - } - } catch (e) { - maxRetries--; - print('Connection failed, retrying... Remaining attempts: ' + maxRetries); - sleep(1000); // Sleep for 1 second - } -} - -if (connected) { - db = db.getSiblingDB(dbName); - var createUserResult = db.createUser({ - user: openimUsername, - pwd: openimPassword, - roles: [{ - role: 'readWrite', - db: dbName - }] - }); - - if (createUserResult.ok == 1) { - print('User creation successful. User: ' + openimUsername + ', Database: ' + dbName); - } else { - print('User creation failed for user: ' + openimUsername + ' in database: ' + dbName); - quit(1); - } -} else { - print('Failed to connect to MongoDB after 300 retries.'); - quit(1); -} -EOF - - - - diff --git a/tools/check-component/main.go b/tools/check-component/main.go index 4f4c08c16..94dbd613c 100644 --- a/tools/check-component/main.go +++ b/tools/check-component/main.go @@ -66,7 +66,7 @@ func CheckMinIO(ctx context.Context, config *config.Minio) error { } func CheckKafka(ctx context.Context, conf *config.Kafka) error { - return kafka.Check(ctx, conf.Build(), []string{conf.ToMongoTopic, conf.ToRedisTopic, conf.ToPushTopic, conf.ToOfflinePushTopic}) + return kafka.CheckHealth(ctx, conf.Build()) } func initConfig(configDir string) (*config.Mongo, *config.Redis, *config.Kafka, *config.Minio, *config.Discovery, error) { From 8471a0ef3bcd92567e210667bacb5c15b9ecc921 Mon Sep 17 00:00:00 2001 From: yoyoIU Date: Mon, 18 Nov 2024 16:25:46 +0800 Subject: [PATCH 038/199] fix(push): push content with jpush (#2844) * fix(push): push content with jpush * docs: fix push enable example value * fix(push): jpush error response --- config/openim-push.yml | 2 +- .../offlinepush/jpush/body/notification.go | 33 +++++++++++-------- internal/push/offlinepush/jpush/push.go | 29 +++++++++++----- internal/push/offlinepush_handler.go | 2 +- internal/push/push_handler.go | 10 +++--- 5 files changed, 49 insertions(+), 27 deletions(-) diff --git a/config/openim-push.yml b/config/openim-push.yml index 2d54bb9e1..dd2d6ce9c 100644 --- a/config/openim-push.yml +++ b/config/openim-push.yml @@ -13,7 +13,7 @@ prometheus: ports: [ 12170, 12171, 12172, 12173, 12174, 12175, 12176, 12177, 12178, 12179, 12180, 12182, 12183, 12184, 12185, 12186 ] maxConcurrentWorkers: 3 -#Use geTui for offline push notifications, or choose fcm or jpns; corresponding configuration settings must be specified. +#Use geTui for offline push notifications, or choose fcm or jpush; corresponding configuration settings must be specified. enable: geTui geTui: pushUrl: https://restapi.getui.com/v2/$appId diff --git a/internal/push/offlinepush/jpush/body/notification.go b/internal/push/offlinepush/jpush/body/notification.go index 42e59c46c..a482a9439 100644 --- a/internal/push/offlinepush/jpush/body/notification.go +++ b/internal/push/offlinepush/jpush/body/notification.go @@ -15,6 +15,7 @@ package body import ( + "github.com/openimsdk/open-im-server/v3/internal/push/offlinepush/options" "github.com/openimsdk/open-im-server/v3/pkg/common/config" ) @@ -26,32 +27,38 @@ type Notification struct { type Android struct { Alert string `json:"alert,omitempty"` + Title string `json:"title,omitempty"` Intent struct { URL string `json:"url,omitempty"` } `json:"intent,omitempty"` - Extras Extras `json:"extras"` + Extras map[string]string `json:"extras,omitempty"` } type Ios struct { - Alert string `json:"alert,omitempty"` - Sound string `json:"sound,omitempty"` - Badge string `json:"badge,omitempty"` - Extras Extras `json:"extras"` - MutableContent bool `json:"mutable-content"` + Alert IosAlert `json:"alert,omitempty"` + Sound string `json:"sound,omitempty"` + Badge string `json:"badge,omitempty"` + Extras map[string]string `json:"extras,omitempty"` + MutableContent bool `json:"mutable-content"` } -type Extras struct { - ClientMsgID string `json:"clientMsgID"` +type IosAlert struct { + Title string `json:"title,omitempty"` + Body string `json:"body,omitempty"` } -func (n *Notification) SetAlert(alert string) { +func (n *Notification) SetAlert(alert string, title string, opts *options.Opts) { n.Alert = alert n.Android.Alert = alert - n.IOS.Alert = alert - n.IOS.Sound = "default" - n.IOS.Badge = "+1" + n.Android.Title = title + n.IOS.Alert.Body = alert + n.IOS.Alert.Title = title + n.IOS.Sound = opts.IOSPushSound + if opts.IOSBadgeCount { + n.IOS.Badge = "+1" + } } -func (n *Notification) SetExtras(extras Extras) { +func (n *Notification) SetExtras(extras map[string]string) { n.IOS.Extras = extras n.Android.Extras = extras } diff --git a/internal/push/offlinepush/jpush/push.go b/internal/push/offlinepush/jpush/push.go index dac52597f..33c8602c9 100644 --- a/internal/push/offlinepush/jpush/push.go +++ b/internal/push/offlinepush/jpush/push.go @@ -18,9 +18,9 @@ import ( "context" "encoding/base64" "fmt" - "github.com/openimsdk/open-im-server/v3/internal/push/offlinepush/options" "github.com/openimsdk/open-im-server/v3/internal/push/offlinepush/jpush/body" + "github.com/openimsdk/open-im-server/v3/internal/push/offlinepush/options" "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/tools/utils/httputil" ) @@ -57,17 +57,23 @@ func (j *JPush) Push(ctx context.Context, userIDs []string, title, content strin var au body.Audience au.SetAlias(userIDs) var no body.Notification - var extras body.Extras + extras := make(map[string]string) + extras["ex"] = opts.Ex if opts.Signal.ClientMsgID != "" { - extras.ClientMsgID = opts.Signal.ClientMsgID + extras["ClientMsgID"] = opts.Signal.ClientMsgID } no.IOSEnableMutableContent() no.SetExtras(extras) - no.SetAlert(title) + no.SetAlert(content, title, opts) no.SetAndroidIntent(j.pushConf) var msg body.Message msg.SetMsgContent(content) + msg.SetTitle(title) + if opts.Signal.ClientMsgID != "" { + msg.SetExtras("ClientMsgID", opts.Signal.ClientMsgID) + } + msg.SetExtras("ex", opts.Ex) var opt body.Options opt.SetApnsProduction(j.pushConf.IOSPush.Production) var pushObj body.PushObj @@ -76,12 +82,12 @@ func (j *JPush) Push(ctx context.Context, userIDs []string, title, content strin pushObj.SetNotification(&no) pushObj.SetMessage(&msg) pushObj.SetOptions(&opt) - var resp any - return j.request(ctx, pushObj, resp, 5) + var resp map[string]any + return j.request(ctx, pushObj, &resp, 5) } -func (j *JPush) request(ctx context.Context, po body.PushObj, resp any, timeout int) error { - return j.httpClient.PostReturn( +func (j *JPush) request(ctx context.Context, po body.PushObj, resp *map[string]any, timeout int) error { + err := j.httpClient.PostReturn( ctx, j.pushConf.JPNS.PushURL, map[string]string{ @@ -91,4 +97,11 @@ func (j *JPush) request(ctx context.Context, po body.PushObj, resp any, timeout resp, timeout, ) + if err != nil { + return err + } + if (*resp)["sendno"] != "0" { + return fmt.Errorf("jpush push failed %v", resp) + } + return nil } diff --git a/internal/push/offlinepush_handler.go b/internal/push/offlinepush_handler.go index a80c147f4..5c69da005 100644 --- a/internal/push/offlinepush_handler.go +++ b/internal/push/offlinepush_handler.go @@ -73,7 +73,7 @@ func (o *OfflinePushConsumerHandler) getOfflinePushInfos(msg *sdkws.MsgData) (ti IsAtSelf bool `json:"isAtSelf"` } - opts = &options.Opts{Signal: &options.Signal{}} + opts = &options.Opts{Signal: &options.Signal{ClientMsgID: msg.ClientMsgID}} if msg.OfflinePushInfo != nil { opts.IOSBadgeCount = msg.OfflinePushInfo.IOSBadgeCount opts.IOSPushSound = msg.OfflinePushInfo.IOSPushSound diff --git a/internal/push/push_handler.go b/internal/push/push_handler.go index 41ad5962a..ae308dfe9 100644 --- a/internal/push/push_handler.go +++ b/internal/push/push_handler.go @@ -4,6 +4,10 @@ import ( "context" "encoding/json" + "math/rand" + "strconv" + "time" + "github.com/IBM/sarama" "github.com/openimsdk/open-im-server/v3/internal/push/offlinepush" "github.com/openimsdk/open-im-server/v3/internal/push/offlinepush/options" @@ -27,9 +31,6 @@ import ( "github.com/openimsdk/tools/utils/timeutil" "github.com/redis/go-redis/v9" "google.golang.org/protobuf/proto" - "math/rand" - "strconv" - "time" ) type ConsumerHandler struct { @@ -335,6 +336,7 @@ func (c *ConsumerHandler) groupMessagesHandler(ctx context.Context, groupID stri func (c *ConsumerHandler) offlinePushMsg(ctx context.Context, msg *sdkws.MsgData, offlinePushUserIDs []string) error { title, content, opts, err := c.getOfflinePushInfos(msg) if err != nil { + log.ZError(ctx, "getOfflinePushInfos failed", err, "msg", msg) return err } err = c.offlinePusher.Push(ctx, offlinePushUserIDs, title, content, opts) @@ -364,7 +366,7 @@ func (c *ConsumerHandler) getOfflinePushInfos(msg *sdkws.MsgData) (title, conten IsAtSelf bool `json:"isAtSelf"` } - opts = &options.Opts{Signal: &options.Signal{}} + opts = &options.Opts{Signal: &options.Signal{ClientMsgID: msg.ClientMsgID}} if msg.OfflinePushInfo != nil { opts.IOSBadgeCount = msg.OfflinePushInfo.IOSBadgeCount opts.IOSPushSound = msg.OfflinePushInfo.IOSPushSound From 1df49ee7186ba04e741a3e04d3c4ebfa1d3abf5e Mon Sep 17 00:00:00 2001 From: Wiky Lyu <65841899+wikylyu@users.noreply.github.com> Date: Tue, 19 Nov 2024 09:59:21 +0800 Subject: [PATCH 039/199] fix #2860 migrate jpns to jpush (#2861) --- config/openim-push.yml | 2 +- deployments/templates/config.yaml | 10 +++++----- docs/contrib/environment.md | 8 ++++---- internal/push/offlinepush/jpush/body/notification.go | 2 +- internal/push/offlinepush/jpush/push.go | 4 ++-- pkg/common/config/config.go | 4 ++-- 6 files changed, 15 insertions(+), 15 deletions(-) diff --git a/config/openim-push.yml b/config/openim-push.yml index dd2d6ce9c..70e67add2 100644 --- a/config/openim-push.yml +++ b/config/openim-push.yml @@ -26,7 +26,7 @@ fcm: # Prioritize using file paths. If the file path is empty, use URL filePath: # File path is concatenated with the parameters passed in through - c(`mage` default pass in `config/`) and filePath. authURL: # Must start with https or http. -jpns: +jpush: appKey: masterSecret: pushURL: diff --git a/deployments/templates/config.yaml b/deployments/templates/config.yaml index fee0bf90a..c108de5e8 100644 --- a/deployments/templates/config.yaml +++ b/deployments/templates/config.yaml @@ -240,11 +240,11 @@ push: channelName: ${GETUI_CHANNEL_NAME} fcm: serviceAccount: "${FCM_SERVICE_ACCOUNT}" - jpns: - appKey: ${JPNS_APP_KEY} - masterSecret: ${JPNS_MASTER_SECRET} - pushUrl: ${JPNS_PUSH_URL} - pushIntent: ${JPNS_PUSH_INTENT} + jpush: + appKey: ${JPUSH_APP_KEY} + masterSecret: ${JPUSH_MASTER_SECRET} + pushUrl: ${JPUSH_PUSH_URL} + pushIntent: ${JPUSH_PUSH_INTENT} # App manager configuration # diff --git a/docs/contrib/environment.md b/docs/contrib/environment.md index d2db7cbf3..0b10abc96 100644 --- a/docs/contrib/environment.md +++ b/docs/contrib/environment.md @@ -474,10 +474,10 @@ This section involves setting up additional configuration variables for Websocke | GETUI_CHANNEL_ID | [User Defined] | GeTui Channel ID | | GETUI_CHANNEL_NAME | [User Defined] | GeTui Channel Name | | FCM_SERVICE_ACCOUNT | "x.json" | FCM Service Account | -| JPNS_APP_KEY | [User Defined] | JPNS Application Key | -| JPNS_MASTER_SECRET | [User Defined] | JPNS Master Secret | -| JPNS_PUSH_URL | [User Defined] | JPNS Push Notification URL | -| JPNS_PUSH_INTENT | [User Defined] | JPNS Push Intent | +| JPUSH_APP_KEY | [User Defined] | JPUSH Application Key | +| JPUSH_MASTER_SECRET | [User Defined] | JPUSH Master Secret | +| JPUSH_PUSH_URL | [User Defined] | JPUSH Push Notification URL | +| JPUSH_PUSH_INTENT | [User Defined] | JPUSH Push Intent | | IM_ADMIN_USERID | "imAdmin" | IM Administrator ID | | IM_ADMIN_NAME | "imAdmin" | IM Administrator Nickname | | MULTILOGIN_POLICY | "1" | Multi-login Policy | diff --git a/internal/push/offlinepush/jpush/body/notification.go b/internal/push/offlinepush/jpush/body/notification.go index a482a9439..383b3fb26 100644 --- a/internal/push/offlinepush/jpush/body/notification.go +++ b/internal/push/offlinepush/jpush/body/notification.go @@ -64,7 +64,7 @@ func (n *Notification) SetExtras(extras map[string]string) { } func (n *Notification) SetAndroidIntent(pushConf *config.Push) { - n.Android.Intent.URL = pushConf.JPNS.PushIntent + n.Android.Intent.URL = pushConf.JPush.PushIntent } func (n *Notification) IOSEnableMutableContent() { diff --git a/internal/push/offlinepush/jpush/push.go b/internal/push/offlinepush/jpush/push.go index 33c8602c9..2694902f2 100644 --- a/internal/push/offlinepush/jpush/push.go +++ b/internal/push/offlinepush/jpush/push.go @@ -89,9 +89,9 @@ func (j *JPush) Push(ctx context.Context, userIDs []string, title, content strin func (j *JPush) request(ctx context.Context, po body.PushObj, resp *map[string]any, timeout int) error { err := j.httpClient.PostReturn( ctx, - j.pushConf.JPNS.PushURL, + j.pushConf.JPush.PushURL, map[string]string{ - "Authorization": j.getAuthorization(j.pushConf.JPNS.AppKey, j.pushConf.JPNS.MasterSecret), + "Authorization": j.getAuthorization(j.pushConf.JPush.AppKey, j.pushConf.JPush.MasterSecret), }, po, resp, diff --git a/pkg/common/config/config.go b/pkg/common/config/config.go index 10285b565..c40018cbc 100644 --- a/pkg/common/config/config.go +++ b/pkg/common/config/config.go @@ -213,12 +213,12 @@ type Push struct { FilePath string `mapstructure:"filePath"` AuthURL string `mapstructure:"authURL"` } `mapstructure:"fcm"` - JPNS struct { + JPush struct { AppKey string `mapstructure:"appKey"` MasterSecret string `mapstructure:"masterSecret"` PushURL string `mapstructure:"pushURL"` PushIntent string `mapstructure:"pushIntent"` - } `mapstructure:"jpns"` + } `mapstructure:"jpush"` IOSPush struct { PushSound string `mapstructure:"pushSound"` BadgeCount bool `mapstructure:"badgeCount"` From 15648cd44d39f7db4bd474a5ccff4cdc66f0bcc6 Mon Sep 17 00:00:00 2001 From: chao <48119764+withchao@users.noreply.github.com> Date: Thu, 21 Nov 2024 17:35:39 +0800 Subject: [PATCH 040/199] fix: concurrent write to websocket connection (#2866) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * update gomake version * update gomake version * fix: seq conversion bug * fix: redis pipe exec * fix: ImportFriends * fix: A large number of logs keysAndValues ​​length is not even * feat: mark read aggregate write * feat: online status supports redis cluster * feat: online status supports redis cluster * feat: online status supports redis cluster * merge * merge * read seq is written to mongo * read seq is written to mongo * fix: invitation to join group notification * fix: friend op_user_id * feat: optimizing asynchronous context * feat: optimizing memamq size * feat: add GetSeqMessage * feat: implement no gob encoder. * update unitTest content. * Update hub_server.go * feat: GroupApplicationAgreeMemberEnterNotification * fix: encoder replace to json encoder. * feat: GroupApplicationAgreeMemberEnterNotification * feat: go.mod * feat: go.mod * feat: join group notification and get seq * feat: join group notification and get seq * feat: avoid pulling messages from sessions with a large number of max seq values of 0 * feat: API supports gzip * go.mod * fix: nil pointer error on close * fix: listen error * fix: listen error * update go.mod * feat: add log * fix: token parse token value * fix: GetMsgBySeqs boundary issues * merge: merge main code into js branch. (#2648) * feat: update group notification when set to null. (#2590) * refactor: refactor workflows contents. * add tool workflows. * update field. * fix: remove chat error. * Fix err. * fix error. * remove cn comment. * update workflows files. * update infra config. * move workflows. * feat: update bot. * fix: solve uncorrect outdated msg get. * update get docIDs logic. * update * update skip logic. * fix * update. * fix: delay deleteObject func. * remove unused content. * feat: update group notification when set to null. * update log standard. * feat: add long time push msg in prometheus (#2584) * feat: add long time push msg in prometheus * fix: log print * fix: go mod * fix: log msg * fix: log init * feat: push msg * feat: go mod ,remove cgo package * feat: remove error log * feat: test dummy push * feat:redis pool config * feat: push to kafka log * feat: supports getting messages based on session ID and seq (#2582) * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * update gomake version * update gomake version * fix: seq conversion bug * fix: redis pipe exec * fix: ImportFriends * fix: A large number of logs keysAndValues ​​length is not even * feat: mark read aggregate write * feat: online status supports redis cluster * feat: online status supports redis cluster * feat: online status supports redis cluster * merge * merge * read seq is written to mongo * read seq is written to mongo * fix: invitation to join group notification * fix: friend op_user_id * feat: optimizing asynchronous context * feat: optimizing memamq size * feat: add GetSeqMessage * feat: GroupApplicationAgreeMemberEnterNotification * feat: GroupApplicationAgreeMemberEnterNotification * feat: go.mod * feat: go.mod * feat: join group notification and get seq --------- Co-authored-by: withchao * feat: implement request batch count limit. (#2591) * refactor: refactor workflows contents. * add tool workflows. * update field. * fix: remove chat error. * Fix err. * fix error. * remove cn comment. * update workflows files. * update infra config. * move workflows. * feat: update bot. * fix: solve uncorrect outdated msg get. * update get docIDs logic. * update * update skip logic. * fix * update. * fix: delay deleteObject func. * remove unused content. * update log type. * feat: implement request batch count limit. * update * update * fix: getting messages based on session ID and seq (#2595) * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * update gomake version * update gomake version * fix: seq conversion bug * fix: redis pipe exec * fix: ImportFriends * fix: A large number of logs keysAndValues ​​length is not even * feat: mark read aggregate write * feat: online status supports redis cluster * feat: online status supports redis cluster * feat: online status supports redis cluster * merge * merge * read seq is written to mongo * read seq is written to mongo * fix: invitation to join group notification * fix: friend op_user_id * feat: optimizing asynchronous context * feat: optimizing memamq size * feat: add GetSeqMessage * feat: GroupApplicationAgreeMemberEnterNotification * feat: GroupApplicationAgreeMemberEnterNotification * feat: go.mod * feat: go.mod * feat: join group notification and get seq * feat: join group notification and get seq --------- Co-authored-by: withchao * feat: avoid pulling messages from sessions with a large number of max seq values of 0 (#2602) * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * update gomake version * update gomake version * fix: seq conversion bug * fix: redis pipe exec * fix: ImportFriends * fix: A large number of logs keysAndValues ​​length is not even * feat: mark read aggregate write * feat: online status supports redis cluster * feat: online status supports redis cluster * feat: online status supports redis cluster * merge * merge * read seq is written to mongo * read seq is written to mongo * fix: invitation to join group notification * fix: friend op_user_id * feat: optimizing asynchronous context * feat: optimizing memamq size * feat: add GetSeqMessage * feat: GroupApplicationAgreeMemberEnterNotification * feat: GroupApplicationAgreeMemberEnterNotification * feat: go.mod * feat: go.mod * feat: join group notification and get seq * feat: join group notification and get seq * feat: avoid pulling messages from sessions with a large number of max seq values of 0 --------- Co-authored-by: withchao * refactor: improve db structure in `storage/controller` (#2604) * refactor: refactor workflows contents. * add tool workflows. * update field. * fix: remove chat error. * Fix err. * fix error. * remove cn comment. * update workflows files. * update infra config. * move workflows. * feat: update bot. * fix: solve uncorrect outdated msg get. * update get docIDs logic. * update * update skip logic. * fix * update. * fix: delay deleteObject func. * remove unused content. * update log type. * feat: implement request batch count limit. * update * update * refactor: improve db structure in `storage/controller` * feat: implement offline push using kafka (#2600) * refactor: refactor workflows contents. * add tool workflows. * update field. * fix: remove chat error. * Fix err. * fix error. * remove cn comment. * update workflows files. * update infra config. * move workflows. * feat: update bot. * fix: solve uncorrect outdated msg get. * update get docIDs logic. * update * update skip logic. * fix * update. * fix: delay deleteObject func. * remove unused content. * update log type. * feat: implement request batch count limit. * update * update * feat: implement offline push. * feat: implement batch Push spilt * update go mod * feat: implement kafka producer and consumer. * update format, * add PushMQ log. * feat: update Handler logic. * update MQ logic. * update * update * fix: update OfflinePushConsumerHandler. * feat: API supports gzip (#2609) * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * update gomake version * update gomake version * fix: seq conversion bug * fix: redis pipe exec * fix: ImportFriends * fix: A large number of logs keysAndValues ​​length is not even * feat: mark read aggregate write * feat: online status supports redis cluster * feat: online status supports redis cluster * feat: online status supports redis cluster * merge * merge * read seq is written to mongo * read seq is written to mongo * fix: invitation to join group notification * fix: friend op_user_id * feat: optimizing asynchronous context * feat: optimizing memamq size * feat: add GetSeqMessage * feat: GroupApplicationAgreeMemberEnterNotification * feat: GroupApplicationAgreeMemberEnterNotification * feat: go.mod * feat: go.mod * feat: join group notification and get seq * feat: join group notification and get seq * feat: avoid pulling messages from sessions with a large number of max seq values of 0 * feat: API supports gzip --------- Co-authored-by: withchao * Fix err (#2608) * refactor: refactor workflows contents. * add tool workflows. * update field. * fix: remove chat error. * Fix err. * fix error. * remove cn comment. * update workflows files. * update infra config. * move workflows. * feat: update bot. * fix: solve uncorrect outdated msg get. * update get docIDs logic. * update * update skip logic. * fix * update. * fix: delay deleteObject func. * remove unused content. * update log type. * feat: implement request batch count limit. * update * update * feat: add rocksTimeout * feat: wrap logs * feat: add logs * feat: listen config * feat: enable listen TIME_WAIT port * feat: add logs * feat: cache batch * chore: enable fullUserCache * feat: push rpc num * feat: push err * feat: with operationID * feat: sleep * feat: change 1s * feat: change log * feat: implement Getbatch in rpcCache. * feat: print getOnline cost * feat: change log * feat: change kafka and push config * feat: del interface * feat: fix err * feat: change config * feat: go mod * feat: change config * feat: change config * feat: add sleep in push * feat: warn logs * feat: logs * feat: logs * feat: change port * feat: start config * feat: remove port reuse * feat: prometheus config * feat: prometheus config * feat: prometheus config * feat: add long time send msg to grafana * feat: init * feat: init * feat: implement offline push. * feat: batch get user online * feat: implement batch Push spilt * update go mod * Revert "feat: change port" This reverts commit 06d5e944 * feat: change port * feat: change config * feat: implement kafka producer and consumer. * update format, * add PushMQ log. * feat: get all online users and init push * feat: lock in online cache * feat: config * fix: init online status * fix: add logs * fix: userIDs * fix: add logs * feat: update Handler logic. * update MQ logic. * update * update * fix: method name * fix: update OfflinePushConsumerHandler. * fix: prommetrics * fix: add logs * fix: ctx * fix: log * fix: config * feat: change port * fix: atomic online cache status --------- Co-authored-by: Monet Lee * feature: add GetConversationsHasReadAndMaxSeq interface to the WebSocket API. (#2611) * fix: lru lock (#2613) * fix: lru lock * fix: lru lock * fix: lru lock * fix: nil pointer error on close (#2618) * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * update gomake version * update gomake version * fix: seq conversion bug * fix: redis pipe exec * fix: ImportFriends * fix: A large number of logs keysAndValues ​​length is not even * feat: mark read aggregate write * feat: online status supports redis cluster * feat: online status supports redis cluster * feat: online status supports redis cluster * merge * merge * read seq is written to mongo * read seq is written to mongo * fix: invitation to join group notification * fix: friend op_user_id * feat: optimizing asynchronous context * feat: optimizing memamq size * feat: add GetSeqMessage * feat: GroupApplicationAgreeMemberEnterNotification * feat: GroupApplicationAgreeMemberEnterNotification * feat: go.mod * feat: go.mod * feat: join group notification and get seq * feat: join group notification and get seq * feat: avoid pulling messages from sessions with a large number of max seq values of 0 * feat: API supports gzip * go.mod * fix: nil pointer error on close --------- Co-authored-by: withchao * feat: create group can push notification (#2617) * fix: blockage caused by listen error (#2620) * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * update gomake version * update gomake version * fix: seq conversion bug * fix: redis pipe exec * fix: ImportFriends * fix: A large number of logs keysAndValues ​​length is not even * feat: mark read aggregate write * feat: online status supports redis cluster * feat: online status supports redis cluster * feat: online status supports redis cluster * merge * merge * read seq is written to mongo * read seq is written to mongo * fix: invitation to join group notification * fix: friend op_user_id * feat: optimizing asynchronous context * feat: optimizing memamq size * feat: add GetSeqMessage * feat: GroupApplicationAgreeMemberEnterNotification * feat: GroupApplicationAgreeMemberEnterNotification * feat: go.mod * feat: go.mod * feat: join group notification and get seq * feat: join group notification and get seq * feat: avoid pulling messages from sessions with a large number of max seq values of 0 * feat: API supports gzip * go.mod * fix: nil pointer error on close * fix: listen error --------- Co-authored-by: withchao * fix: go.mod (#2621) * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * update gomake version * update gomake version * fix: seq conversion bug * fix: redis pipe exec * fix: ImportFriends * fix: A large number of logs keysAndValues ​​length is not even * feat: mark read aggregate write * feat: online status supports redis cluster * feat: online status supports redis cluster * feat: online status supports redis cluster * merge * merge * read seq is written to mongo * read seq is written to mongo * fix: invitation to join group notification * fix: friend op_user_id * feat: optimizing asynchronous context * feat: optimizing memamq size * feat: add GetSeqMessage * feat: GroupApplicationAgreeMemberEnterNotification * feat: GroupApplicationAgreeMemberEnterNotification * feat: go.mod * feat: go.mod * feat: join group notification and get seq * feat: join group notification and get seq * feat: avoid pulling messages from sessions with a large number of max seq values of 0 * feat: API supports gzip * go.mod * fix: nil pointer error on close * fix: listen error * fix: listen error * update go.mod --------- Co-authored-by: withchao * feat: improve searchMsg implement. (#2614) * refactor: refactor workflows contents. * add tool workflows. * update field. * fix: remove chat error. * Fix err. * fix error. * remove cn comment. * update workflows files. * update infra config. * move workflows. * feat: update bot. * fix: solve uncorrect outdated msg get. * update get docIDs logic. * update * update skip logic. * fix * update. * fix: delay deleteObject func. * remove unused content. * update log type. * feat: implement request batch count limit. * update * update * remove unused script. * feat: improve searchMsg implement. * update mongo config. * Fix lock (#2622) * fix:log * fix: lock * fix: update setGroupInfoEX field name. (#2625) * refactor: refactor workflows contents. * add tool workflows. * update field. * fix: remove chat error. * Fix err. * fix error. * remove cn comment. * update workflows files. * update infra config. * move workflows. * feat: update bot. * fix: solve uncorrect outdated msg get. * update get docIDs logic. * update * update skip logic. * fix * update. * fix: delay deleteObject func. * remove unused content. * update log type. * feat: implement request batch count limit. * update * update * fix: update setGroupInfoEX field name. * fix: update setGroupInfoEX field name (#2626) * refactor: refactor workflows contents. * add tool workflows. * update field. * fix: remove chat error. * Fix err. * fix error. * remove cn comment. * update workflows files. * update infra config. * move workflows. * feat: update bot. * fix: solve uncorrect outdated msg get. * update get docIDs logic. * update * update skip logic. * fix * update. * fix: delay deleteObject func. * remove unused content. * update log type. * feat: implement request batch count limit. * update * update * fix: update setGroupInfoEX field name. * fix: update setGroupInfoEX field name * feat: msg gateway add log (#2631) * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * update gomake version * update gomake version * fix: seq conversion bug * fix: redis pipe exec * fix: ImportFriends * fix: A large number of logs keysAndValues ​​length is not even * feat: mark read aggregate write * feat: online status supports redis cluster * feat: online status supports redis cluster * feat: online status supports redis cluster * merge * merge * read seq is written to mongo * read seq is written to mongo * fix: invitation to join group notification * fix: friend op_user_id * feat: optimizing asynchronous context * feat: optimizing memamq size * feat: add GetSeqMessage * feat: GroupApplicationAgreeMemberEnterNotification * feat: GroupApplicationAgreeMemberEnterNotification * feat: go.mod * feat: go.mod * feat: join group notification and get seq * feat: join group notification and get seq * feat: avoid pulling messages from sessions with a large number of max seq values of 0 * feat: API supports gzip * go.mod * fix: nil pointer error on close * fix: listen error * fix: listen error * update go.mod * feat: add log --------- Co-authored-by: withchao * fix: update setGroupInfoEx func name and field. (#2634) * refactor: refactor workflows contents. * add tool workflows. * update field. * fix: remove chat error. * Fix err. * fix error. * remove cn comment. * update workflows files. * update infra config. * move workflows. * feat: update bot. * fix: solve uncorrect outdated msg get. * update get docIDs logic. * update * update skip logic. * fix * update. * fix: delay deleteObject func. * remove unused content. * update log type. * feat: implement request batch count limit. * update * update * fix: update setGroupInfoEx func name and field. * refactor: update groupinfoEx field. * refactor: update database name in mongodb.yml * add groupName Condition * fix: fix setConversations req fill. (#2645) * refactor: refactor workflows contents. * add tool workflows. * update field. * fix: remove chat error. * Fix err. * fix error. * remove cn comment. * update workflows files. * update infra config. * move workflows. * feat: update bot. * fix: solve uncorrect outdated msg get. * update get docIDs logic. * update * update skip logic. * fix * update. * fix: delay deleteObject func. * remove unused content. * update log type. * feat: implement request batch count limit. * update * update * fix: fix setConversations req fill. * fix: GetMsgBySeqs boundary issues (#2647) * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * update gomake version * update gomake version * fix: seq conversion bug * fix: redis pipe exec * fix: ImportFriends * fix: A large number of logs keysAndValues ​​length is not even * feat: mark read aggregate write * feat: online status supports redis cluster * feat: online status supports redis cluster * feat: online status supports redis cluster * merge * merge * read seq is written to mongo * read seq is written to mongo * fix: invitation to join group notification * fix: friend op_user_id * feat: optimizing asynchronous context * feat: optimizing memamq size * feat: add GetSeqMessage * feat: GroupApplicationAgreeMemberEnterNotification * feat: GroupApplicationAgreeMemberEnterNotification * feat: go.mod * feat: go.mod * feat: join group notification and get seq * feat: join group notification and get seq * feat: avoid pulling messages from sessions with a large number of max seq values of 0 * feat: API supports gzip * go.mod * fix: nil pointer error on close * fix: listen error * fix: listen error * update go.mod * feat: add log * fix: token parse token value * fix: GetMsgBySeqs boundary issues --------- Co-authored-by: withchao * fix: the attribute version is obsolete, remove it (#2644) * refactor: update Userregister request field. (#2650) --------- Co-authored-by: Monet Lee Co-authored-by: icey-yu <119291641+icey-yu@users.noreply.github.com> Co-authored-by: chao <48119764+withchao@users.noreply.github.com> Co-authored-by: withchao Co-authored-by: 蔡相跃 * update go mod * fix: sn_ not sort * fix: sn_ not sort * fix: sn_ not sort * fix: jssdk add * fix: jssdk support * fix: jssdk support * fix: jssdk support * fix: the message I sent is not set to read seq in mongodb * fix: cannot modify group member avatars * merge: update code from main to v3.8-js-sdk-only. (#2720) * fix: fix update groupName invalid. (#2673) * refactor: change platform to platformID (#2670) * feat: don`t return nil data (#2675) Co-authored-by: Monet Lee * refactor: update fields type in userStatus and check registered. (#2676) * fix: usertoken auth. (#2677) * refactor: update fields type in userStatus and check registered. * fix: usertoken auth. * update contents. * update content. * update * fix * update pb file. * feat: add friend agree after callback (#2680) * fix: sn not sort (#2682) * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * update gomake version * update gomake version * fix: seq conversion bug * fix: redis pipe exec * fix: ImportFriends * fix: A large number of logs keysAndValues ​​length is not even * feat: mark read aggregate write * feat: online status supports redis cluster * feat: online status supports redis cluster * feat: online status supports redis cluster * merge * merge * read seq is written to mongo * read seq is written to mongo * fix: invitation to join group notification * fix: friend op_user_id * feat: optimizing asynchronous context * feat: optimizing memamq size * feat: add GetSeqMessage * feat: GroupApplicationAgreeMemberEnterNotification * feat: GroupApplicationAgreeMemberEnterNotification * feat: go.mod * feat: go.mod * feat: join group notification and get seq * feat: join group notification and get seq * feat: avoid pulling messages from sessions with a large number of max seq values of 0 * feat: API supports gzip * go.mod * fix: nil pointer error on close * fix: listen error * fix: listen error * update go.mod * feat: add log * fix: token parse token value * fix: GetMsgBySeqs boundary issues * fix: sn_ not sort * fix: sn_ not sort * fix: sn_ not sort --------- Co-authored-by: withchao * refactor: add GetAdminToken interface. (#2684) * refactor: add GetAdminToken interface. * update config. * fix: admin token (#2686) * fix: update workflows logic. (#2688) * refactor: add GetAdminToken interface. * update config. * update workflows logic. * fix: admin token (#2687) * update the front image (#2692) * update the front image * update version * feat: improve publish docker image workflows (#2697) * refactor: add GetAdminToken interface. * update config. * update workflows logic. * feat: improve publish docker image workflows * update condition logic. * fix: update load file logic. (#2700) * refactor: add GetAdminToken interface. * update config. * update workflows logic. * feat: improve publish docker image workflows * update condition logic. * fix: update load file logic. * feat: Msg filter (#2703) * feat: msg filter * feat: msg filter * feat: msg filter * feat: provide the interface required by js sdk (#2712) * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * update gomake version * update gomake version * fix: seq conversion bug * fix: redis pipe exec * fix: ImportFriends * fix: A large number of logs keysAndValues ​​length is not even * feat: mark read aggregate write * feat: online status supports redis cluster * feat: online status supports redis cluster * feat: online status supports redis cluster * merge * merge * read seq is written to mongo * read seq is written to mongo * fix: invitation to join group notification * fix: friend op_user_id * feat: optimizing asynchronous context * feat: optimizing memamq size * feat: add GetSeqMessage * feat: GroupApplicationAgreeMemberEnterNotification * feat: GroupApplicationAgreeMemberEnterNotification * feat: go.mod * feat: go.mod * feat: join group notification and get seq * feat: join group notification and get seq * feat: avoid pulling messages from sessions with a large number of max seq values of 0 * feat: API supports gzip * go.mod * fix: nil pointer error on close * fix: listen error * fix: listen error * update go.mod * feat: add log * fix: token parse token value * fix: GetMsgBySeqs boundary issues * fix: sn_ not sort * fix: sn_ not sort * fix: sn_ not sort * fix: jssdk add * fix: jssdk support * fix: jssdk support * fix: jssdk support --------- Co-authored-by: withchao * Line webhook (#2716) * feat: online and offline webhook * feat: online and offline webhook * feat: remove zk * fix: the message I sent is not set to read seq in mongodb (#2718) * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * update gomake version * update gomake version * fix: seq conversion bug * fix: redis pipe exec * fix: ImportFriends * fix: A large number of logs keysAndValues ​​length is not even * feat: mark read aggregate write * feat: online status supports redis cluster * feat: online status supports redis cluster * feat: online status supports redis cluster * merge * merge * read seq is written to mongo * read seq is written to mongo * fix: invitation to join group notification * fix: friend op_user_id * feat: optimizing asynchronous context * feat: optimizing memamq size * feat: add GetSeqMessage * feat: GroupApplicationAgreeMemberEnterNotification * feat: GroupApplicationAgreeMemberEnterNotification * feat: go.mod * feat: go.mod * feat: join group notification and get seq * feat: join group notification and get seq * feat: avoid pulling messages from sessions with a large number of max seq values of 0 * feat: API supports gzip * go.mod * fix: nil pointer error on close * fix: listen error * fix: listen error * update go.mod * feat: add log * fix: token parse token value * fix: GetMsgBySeqs boundary issues * fix: sn_ not sort * fix: sn_ not sort * fix: sn_ not sort * fix: jssdk add * fix: jssdk support * fix: jssdk support * fix: jssdk support * fix: the message I sent is not set to read seq in mongodb --------- Co-authored-by: withchao * fix: cannot modify group member avatars (#2719) * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * update gomake version * update gomake version * fix: seq conversion bug * fix: redis pipe exec * fix: ImportFriends * fix: A large number of logs keysAndValues ​​length is not even * feat: mark read aggregate write * feat: online status supports redis cluster * feat: online status supports redis cluster * feat: online status supports redis cluster * merge * merge * read seq is written to mongo * read seq is written to mongo * fix: invitation to join group notification * fix: friend op_user_id * feat: optimizing asynchronous context * feat: optimizing memamq size * feat: add GetSeqMessage * feat: GroupApplicationAgreeMemberEnterNotification * feat: GroupApplicationAgreeMemberEnterNotification * feat: go.mod * feat: go.mod * feat: join group notification and get seq * feat: join group notification and get seq * feat: avoid pulling messages from sessions with a large number of max seq values of 0 * feat: API supports gzip * go.mod * fix: nil pointer error on close * fix: listen error * fix: listen error * update go.mod * feat: add log * fix: token parse token value * fix: GetMsgBySeqs boundary issues * fix: sn_ not sort * fix: sn_ not sort * fix: sn_ not sort * fix: jssdk add * fix: jssdk support * fix: jssdk support * fix: jssdk support * fix: the message I sent is not set to read seq in mongodb * fix: cannot modify group member avatars --------- Co-authored-by: withchao --------- Co-authored-by: Monet Lee Co-authored-by: icey-yu <119291641+icey-yu@users.noreply.github.com> Co-authored-by: chao <48119764+withchao@users.noreply.github.com> Co-authored-by: withchao Co-authored-by: skiffer-git <72860476+skiffer-git@users.noreply.github.com> * fix: MemberEnterNotification * fix: MemberEnterNotification * fix: MsgData status * merge * merge: update code from main to v3.8-js-sdk-only. (#2818) * feat: implement merge milestone PR to target-branch. (#2796) * build: improve workflows logic. (#2801) * fix: improve time condition check mehtod. (#2804) * fix: improve time condition check mehtod. * fix * fix: webhook before online push (#2805) * fix: set own read seq in MongoDB when sender send a message. (#2808) * fix: solve err Notification when setGroupInfo. (#2806) * fix: solve err Notification when setGroupInfo. * build: update checkout version. * fix: update notification contents. * Introducing OpenIM Guru on Gurubase.io (#2788) * feat: support app update service (#2811) * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * update gomake version * update gomake version * fix: seq conversion bug * fix: redis pipe exec * fix: ImportFriends * fix: A large number of logs keysAndValues ​​length is not even * feat: mark read aggregate write * feat: online status supports redis cluster * feat: online status supports redis cluster * feat: online status supports redis cluster * merge * merge * read seq is written to mongo * read seq is written to mongo * fix: invitation to join group notification * fix: friend op_user_id * feat: optimizing asynchronous context * feat: optimizing memamq size * feat: add GetSeqMessage * feat: GroupApplicationAgreeMemberEnterNotification * feat: GroupApplicationAgreeMemberEnterNotification * feat: go.mod * feat: go.mod * feat: join group notification and get seq * feat: join group notification and get seq * feat: avoid pulling messages from sessions with a large number of max seq values of 0 * feat: API supports gzip * go.mod * fix: nil pointer error on close * fix: listen error * fix: listen error * update go.mod * feat: add log * fix: token parse token value * fix: GetMsgBySeqs boundary issues * fix: sn_ not sort * fix: sn_ not sort * fix: sn_ not sort * fix: jssdk add * fix: jssdk support * fix: jssdk support * fix: jssdk support * fix: the message I sent is not set to read seq in mongodb * fix: cannot modify group member avatars * fix: MemberEnterNotification * fix: MemberEnterNotification * fix: MsgData status * feat: add ApplicationVersion * feat: add ApplicationVersion * feat: add ApplicationVersion --------- Co-authored-by: withchao * feat: ApplicationVersion move chat (#2813) * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * update gomake version * update gomake version * fix: seq conversion bug * fix: redis pipe exec * fix: ImportFriends * fix: A large number of logs keysAndValues ​​length is not even * feat: mark read aggregate write * feat: online status supports redis cluster * feat: online status supports redis cluster * feat: online status supports redis cluster * merge * merge * read seq is written to mongo * read seq is written to mongo * fix: invitation to join group notification * fix: friend op_user_id * feat: optimizing asynchronous context * feat: optimizing memamq size * feat: add GetSeqMessage * feat: GroupApplicationAgreeMemberEnterNotification * feat: GroupApplicationAgreeMemberEnterNotification * feat: go.mod * feat: go.mod * feat: join group notification and get seq * feat: join group notification and get seq * feat: avoid pulling messages from sessions with a large number of max seq values of 0 * feat: API supports gzip * go.mod * fix: nil pointer error on close * fix: listen error * fix: listen error * update go.mod * feat: add log * fix: token parse token value * fix: GetMsgBySeqs boundary issues * fix: sn_ not sort * fix: sn_ not sort * fix: sn_ not sort * fix: jssdk add * fix: jssdk support * fix: jssdk support * fix: jssdk support * fix: the message I sent is not set to read seq in mongodb * fix: cannot modify group member avatars * fix: MemberEnterNotification * fix: MemberEnterNotification * fix: MsgData status * feat: add ApplicationVersion * feat: ApplicationVersion move chat --------- Co-authored-by: withchao * fix: improve condition check. (#2815) --------- Co-authored-by: Monet Lee Co-authored-by: icey-yu <119291641+icey-yu@users.noreply.github.com> Co-authored-by: Kürşat Aktaş Co-authored-by: chao <48119764+withchao@users.noreply.github.com> Co-authored-by: withchao * feat: support text ping pong * feat: support text ping pong * feat: gob json encoder * feat: gob json encoder * feat: gob json encoder * feat: gob json encoder * feat: gob json encoder * feat: gob json encoder * fix: concurrent write to websocket connection * fix: concurrent write to websocket connection * fix: concurrent write to websocket connection --------- Co-authored-by: withchao Co-authored-by: Monet Lee Co-authored-by: OpenIM-Gordon <46924906+FGadvancer@users.noreply.github.com> Co-authored-by: icey-yu <119291641+icey-yu@users.noreply.github.com> Co-authored-by: 蔡相跃 Co-authored-by: skiffer-git <72860476+skiffer-git@users.noreply.github.com> Co-authored-by: Kürşat Aktaş --- internal/msggateway/client.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/internal/msggateway/client.go b/internal/msggateway/client.go index af96e7d46..6e31751dc 100644 --- a/internal/msggateway/client.go +++ b/internal/msggateway/client.go @@ -446,6 +446,8 @@ func (c *Client) handlerTextMessage(b []byte) error { if err != nil { return err } + c.w.Lock() + defer c.w.Unlock() if err := c.conn.SetWriteDeadline(writeWait); err != nil { return err } From daba153d27f86af34630804d5d53144e67afa93d Mon Sep 17 00:00:00 2001 From: icey-yu <119291641+icey-yu@users.noreply.github.com> Date: Fri, 22 Nov 2024 12:25:28 +0800 Subject: [PATCH 041/199] fix: admin token limit (#2871) --- internal/rpc/auth/auth.go | 10 +++- pkg/common/storage/controller/auth.go | 69 +++++++++++++++------------ pkg/common/storage/controller/msg.go | 2 +- 3 files changed, 48 insertions(+), 33 deletions(-) diff --git a/internal/rpc/auth/auth.go b/internal/rpc/auth/auth.go index 62df74d21..a1acfd931 100644 --- a/internal/rpc/auth/auth.go +++ b/internal/rpc/auth/auth.go @@ -16,6 +16,7 @@ package auth import ( "context" + "errors" "github.com/openimsdk/open-im-server/v3/pkg/common/config" redis2 "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/redis" @@ -66,6 +67,7 @@ func Start(ctx context.Context, config *Config, client discovery.SvcDiscoveryReg config.Share.Secret, config.RpcConfig.TokenPolicy.Expire, config.Share.MultiLogin, + config.Share.IMAdminUserID, ), config: config, }) @@ -129,6 +131,10 @@ func (s *authServer) parseToken(ctx context.Context, tokensString string) (claim if err != nil { return nil, errs.Wrap(err) } + isAdmin := authverify.IsManagerUserID(claims.UserID, s.config.Share.IMAdminUserID) + if isAdmin { + return claims, nil + } m, err := s.authDatabase.GetTokensWithoutError(ctx, claims.UserID, claims.PlatformID) if err != nil { return nil, err @@ -190,7 +196,7 @@ func (s *authServer) forceKickOff(ctx context.Context, userID string, platformID } m, err := s.authDatabase.GetTokensWithoutError(ctx, userID, int(platformID)) - if err != nil && err != redis.Nil { + if err != nil && errors.Is(err, redis.Nil) { return err } for k := range m { @@ -208,7 +214,7 @@ func (s *authServer) forceKickOff(ctx context.Context, userID string, platformID func (s *authServer) InvalidateToken(ctx context.Context, req *pbauth.InvalidateTokenReq) (*pbauth.InvalidateTokenResp, error) { m, err := s.authDatabase.GetTokensWithoutError(ctx, req.UserID, int(req.PlatformID)) - if err != nil && err != redis.Nil { + if err != nil && errors.Is(err, redis.Nil) { return nil, err } if m == nil { diff --git a/pkg/common/storage/controller/auth.go b/pkg/common/storage/controller/auth.go index df1274967..e7b5bc297 100644 --- a/pkg/common/storage/controller/auth.go +++ b/pkg/common/storage/controller/auth.go @@ -35,9 +35,10 @@ type authDatabase struct { accessSecret string accessExpire int64 multiLogin multiLoginConfig + adminUserIDs []string } -func NewAuthDatabase(cache cache.TokenModel, accessSecret string, accessExpire int64, multiLogin config.MultiLogin) AuthDatabase { +func NewAuthDatabase(cache cache.TokenModel, accessSecret string, accessExpire int64, multiLogin config.MultiLogin, adminUserIDs []string) AuthDatabase { return &authDatabase{cache: cache, accessSecret: accessSecret, accessExpire: accessExpire, multiLogin: multiLoginConfig{ Policy: multiLogin.Policy, MaxNumOneEnd: multiLogin.MaxNumOneEnd, @@ -53,7 +54,8 @@ func NewAuthDatabase(cache cache.TokenModel, accessSecret string, accessExpire i constant.IPadPlatformID: multiLogin.CustomizeLoginNum.IPad, constant.AdminPlatformID: multiLogin.CustomizeLoginNum.Admin, }, - }} + }, adminUserIDs: adminUserIDs, + } } // If the result is empty. @@ -90,27 +92,31 @@ func (a *authDatabase) BatchSetTokenMapByUidPid(ctx context.Context, tokens []st // Create Token. func (a *authDatabase) CreateToken(ctx context.Context, userID string, platformID int) (string, error) { - tokens, err := a.cache.GetAllTokensWithoutError(ctx, userID) - if err != nil { - return "", err - } - deleteTokenKey, kickedTokenKey, err := a.checkToken(ctx, tokens, platformID) - if err != nil { - return "", err - } - if len(deleteTokenKey) != 0 { - err = a.cache.DeleteTokenByUidPid(ctx, userID, platformID, deleteTokenKey) + isAdmin := authverify.IsManagerUserID(userID, a.adminUserIDs) + if !isAdmin { + tokens, err := a.cache.GetAllTokensWithoutError(ctx, userID) if err != nil { return "", err } - } - if len(kickedTokenKey) != 0 { - for _, k := range kickedTokenKey { - err := a.cache.SetTokenFlagEx(ctx, userID, platformID, k, constant.KickedToken) + + deleteTokenKey, kickedTokenKey, err := a.checkToken(ctx, tokens, platformID) + if err != nil { + return "", err + } + if len(deleteTokenKey) != 0 { + err = a.cache.DeleteTokenByUidPid(ctx, userID, platformID, deleteTokenKey) if err != nil { return "", err } - log.ZDebug(ctx, "kicked token in create token", "token", k) + } + if len(kickedTokenKey) != 0 { + for _, k := range kickedTokenKey { + err := a.cache.SetTokenFlagEx(ctx, userID, platformID, k, constant.KickedToken) + if err != nil { + return "", err + } + log.ZDebug(ctx, "kicked token in create token", "token", k) + } } } @@ -121,9 +127,12 @@ func (a *authDatabase) CreateToken(ctx context.Context, userID string, platformI return "", errs.WrapMsg(err, "token.SignedString") } - if err = a.cache.SetTokenFlagEx(ctx, userID, platformID, tokenString, constant.NormalToken); err != nil { - return "", err + if !isAdmin { + if err = a.cache.SetTokenFlagEx(ctx, userID, platformID, tokenString, constant.NormalToken); err != nil { + return "", err + } } + return tokenString, nil } @@ -226,16 +235,16 @@ func (a *authDatabase) checkToken(ctx context.Context, tokens map[int]map[string return nil, nil, errs.New("unknown multiLogin policy").Wrap() } - var adminTokenMaxNum = a.multiLogin.MaxNumOneEnd - if a.multiLogin.Policy == constant.Customize { - adminTokenMaxNum = a.multiLogin.CustomizeLoginNum[constant.AdminPlatformID] - } - l := len(adminToken) - if platformID == constant.AdminPlatformID { - l++ - } - if l > adminTokenMaxNum { - kickToken = append(kickToken, adminToken[:l-adminTokenMaxNum]...) - } + //var adminTokenMaxNum = a.multiLogin.MaxNumOneEnd + //if a.multiLogin.Policy == constant.Customize { + // adminTokenMaxNum = a.multiLogin.CustomizeLoginNum[constant.AdminPlatformID] + //} + //l := len(adminToken) + //if platformID == constant.AdminPlatformID { + // l++ + //} + //if l > adminTokenMaxNum { + // kickToken = append(kickToken, adminToken[:l-adminTokenMaxNum]...) + //} return deleteToken, kickToken, nil } diff --git a/pkg/common/storage/controller/msg.go b/pkg/common/storage/controller/msg.go index 90b479064..9636f7a15 100644 --- a/pkg/common/storage/controller/msg.go +++ b/pkg/common/storage/controller/msg.go @@ -490,7 +490,7 @@ func (db *commonMsgDatabase) GetMsgBySeqs(ctx context.Context, userID string, co } successMsgs, failedSeqs, err := db.msg.GetMessagesBySeq(ctx, conversationID, newSeqs) if err != nil { - if err != redis.Nil { + if errors.Is(err, redis.Nil) { log.ZError(ctx, "get message from redis exception", err, "failedSeqs", failedSeqs, "conversationID", conversationID) } } From a37eb5ea0dc123ef4175ef3ad52bfdb62f5564a6 Mon Sep 17 00:00:00 2001 From: Monet Lee Date: Fri, 22 Nov 2024 12:26:29 +0800 Subject: [PATCH 042/199] build: create changelog tool and workflows. (#2869) --- .github/workflows/changelog.yml | 78 ++++++++ tools/changelog/changelog.go | 198 ++++++++++++++++++++ tools/changelog/main.go | 308 -------------------------------- 3 files changed, 276 insertions(+), 308 deletions(-) create mode 100644 .github/workflows/changelog.yml create mode 100644 tools/changelog/changelog.go delete mode 100644 tools/changelog/main.go diff --git a/.github/workflows/changelog.yml b/.github/workflows/changelog.yml new file mode 100644 index 000000000..b97036d91 --- /dev/null +++ b/.github/workflows/changelog.yml @@ -0,0 +1,78 @@ +name: Release Changelog + +on: + release: + types: [released] + +permissions: + contents: write + pull-requests: write + +jobs: + update-changelog: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Run Go Changelog Generator + run: | + # Run the Go changelog generator, passing the release tag if available + if [ "${{ github.event.release.tag_name }}" = "latest" ]; then + go run tools/changelog/changelog.go > "${{ github.event.release.tag_name }}-changelog.md" + else + go run tools/changelog/changelog.go "${{ github.event.release.tag_name }}" > "${{ github.event.release.tag_name }}-changelog.md" + fi + + - name: Handle changelog files + run: | + # Ensure that the CHANGELOG directory exists + mkdir -p CHANGELOG + + # Extract Major.Minor version by removing the 'v' prefix from the tag name + TAG_NAME=${{ github.event.release.tag_name }} + CHANGELOG_VERSION_NUMBER=$(echo "$TAG_NAME" | sed 's/^v//' | grep -oP '^\d+\.\d+') + + # Define the new changelog file path + CHANGELOG_FILENAME="CHANGELOG-$CHANGELOG_VERSION_NUMBER.md" + CHANGELOG_PATH="CHANGELOG/$CHANGELOG_FILENAME" + + # Check if the changelog file for the current release already exists + if [ -f "$CHANGELOG_PATH" ]; then + # If the file exists, append the new changelog to the existing one + cat "$CHANGELOG_PATH" >> "${TAG_NAME}-changelog.md" + # Overwrite the existing changelog with the updated content + mv "${TAG_NAME}-changelog.md" "$CHANGELOG_PATH" + else + # If the changelog file doesn't exist, rename the temp changelog file to the new changelog file + mv "${TAG_NAME}-changelog.md" "$CHANGELOG_PATH" + + # Ensure that README.md exists + if [ ! -f "CHANGELOG/README.md" ]; then + echo -e "# CHANGELOGs\n\n" > CHANGELOG/README.md + fi + + # Add the new changelog entry at the top of the README.md + if ! grep -q "\[$CHANGELOG_FILENAME\]" CHANGELOG/README.md; then + sed -i "3i- [$CHANGELOG_FILENAME](./$CHANGELOG_FILENAME)" CHANGELOG/README.md + # Remove the extra newline character added by sed + # sed -i '4d' CHANGELOG/README.md + fi + fi + + - name: Clean up + run: | + # Remove any temporary files that were created during the process + rm -f "${{ github.event.release.tag_name }}-changelog.md" + + - name: Create Pull Request + uses: peter-evans/create-pull-request@v7.0.5 + with: + token: ${{ secrets.GITHUB_TOKEN }} + commit-message: "Update CHANGELOG for release ${{ github.event.release.tag_name }}" + title: "Update CHANGELOG for release ${{ github.event.release.tag_name }}" + body: "This PR updates the CHANGELOG files for release ${{ github.event.release.tag_name }}" + branch: changelog-${{ github.event.release.tag_name }} + base: main + delete-branch: true + labels: changelog diff --git a/tools/changelog/changelog.go b/tools/changelog/changelog.go new file mode 100644 index 000000000..75d914a27 --- /dev/null +++ b/tools/changelog/changelog.go @@ -0,0 +1,198 @@ +package main + +import ( + "encoding/json" + "fmt" + "io" + "net/http" + "os" + "regexp" + "strings" +) + +// You can specify a tag as a command line argument to generate the changelog for a specific version. +// Example: go run tools/changelog/changelog.go v0.0.33 +// If no tag is provided, the latest release will be used. + +// Setting repo owner and repo name by generate changelog +const ( + repoOwner = "openimsdk" + repoName = "open-im-server" +) + +// GitHubRepo struct represents the repo details. +type GitHubRepo struct { + Owner string + Repo string + FullChangelog string +} + +// ReleaseData represents the JSON structure for release data. +type ReleaseData struct { + TagName string `json:"tag_name"` + Body string `json:"body"` + HtmlUrl string `json:"html_url"` + Published string `json:"published_at"` +} + +// Method to classify and format release notes. +func (g *GitHubRepo) classifyReleaseNotes(body string) map[string][]string { + result := map[string][]string{ + "feat": {}, + "fix": {}, + "chore": {}, + "refactor": {}, + "build": {}, + "other": {}, + } + + // Regular expression to extract PR number and URL (case insensitive) + rePR := regexp.MustCompile(`(?i)in (https://github\.com/[^\s]+/pull/(\d+))`) + + // Split the body into individual lines. + lines := strings.Split(body, "\n") + + for _, line := range lines { + // Skip lines that contain "deps: Merge" + if strings.Contains(strings.ToLower(line), "deps: merge #") { + continue + } + + // Use a regular expression to extract Full Changelog link and its title (case insensitive). + if strings.Contains(strings.ToLower(line), "**full changelog**") { + matches := regexp.MustCompile(`(?i)\*\*full changelog\*\*: (https://github\.com/[^\s]+/compare/([^\s]+))`).FindStringSubmatch(line) + if len(matches) > 2 { + // Format the Full Changelog link with title + g.FullChangelog = fmt.Sprintf("[%s](%s)", matches[2], matches[1]) + } + continue // Skip further processing for this line. + } + + if strings.HasPrefix(line, "*") { + var category string + + // Use strings.ToLower to make the matching case insensitive + lowerLine := strings.ToLower(line) + + // Determine the category based on the prefix (case insensitive). + if strings.HasPrefix(lowerLine, "* feat") { + category = "feat" + } else if strings.HasPrefix(lowerLine, "* fix") { + category = "fix" + } else if strings.HasPrefix(lowerLine, "* chore") { + category = "chore" + } else if strings.HasPrefix(lowerLine, "* refactor") { + category = "refactor" + } else if strings.HasPrefix(lowerLine, "* build") { + category = "build" + } else { + category = "other" + } + + // Extract PR number and URL (case insensitive) + matches := rePR.FindStringSubmatch(line) + if len(matches) == 3 { + prURL := matches[1] + prNumber := matches[2] + // Format the line with the PR link and use original content for the final result + formattedLine := fmt.Sprintf("* %s [#%s](%s)", strings.Split(line, " by ")[0][2:], prNumber, prURL) + result[category] = append(result[category], formattedLine) + } else { + // If no PR link is found, just add the line as is + result[category] = append(result[category], line) + } + } + } + + return result +} + +// Method to generate the final changelog. +func (g *GitHubRepo) generateChangelog(tag, date, htmlURL, body string) string { + sections := g.classifyReleaseNotes(body) + + // Convert ISO 8601 date to simpler format (YYYY-MM-DD) + formattedDate := date[:10] + + // Changelog header with tag, date, and links. + changelog := fmt.Sprintf("## [%s](%s) \t(%s)\n\n", tag, htmlURL, formattedDate) + + if len(sections["feat"]) > 0 { + changelog += "### New Features\n" + strings.Join(sections["feat"], "\n") + "\n\n" + } + if len(sections["fix"]) > 0 { + changelog += "### Bug Fixes\n" + strings.Join(sections["fix"], "\n") + "\n\n" + } + if len(sections["chore"]) > 0 { + changelog += "### Chores\n" + strings.Join(sections["chore"], "\n") + "\n\n" + } + if len(sections["refactor"]) > 0 { + changelog += "### Refactors\n" + strings.Join(sections["refactor"], "\n") + "\n\n" + } + if len(sections["build"]) > 0 { + changelog += "### Builds\n" + strings.Join(sections["build"], "\n") + "\n\n" + } + if len(sections["other"]) > 0 { + changelog += "### Others\n" + strings.Join(sections["other"], "\n") + "\n\n" + } + + if g.FullChangelog != "" { + changelog += fmt.Sprintf("**Full Changelog**: %s\n", g.FullChangelog) + } + + return changelog +} + +// Method to fetch release data from GitHub API. +func (g *GitHubRepo) fetchReleaseData(version string) (*ReleaseData, error) { + var apiURL string + + if version == "" { + // Fetch the latest release. + apiURL = fmt.Sprintf("https://api.github.com/repos/%s/%s/releases/latest", g.Owner, g.Repo) + } else { + // Fetch a specific version. + apiURL = fmt.Sprintf("https://api.github.com/repos/%s/%s/releases/tags/%s", g.Owner, g.Repo, version) + } + + resp, err := http.Get(apiURL) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, err + } + + var releaseData ReleaseData + err = json.Unmarshal(body, &releaseData) + if err != nil { + return nil, err + } + + return &releaseData, nil +} + +func main() { + repo := &GitHubRepo{Owner: repoOwner, Repo: repoName} + + // Get the version from command line arguments, if provided + var version string // Default is use latest + + if len(os.Args) > 1 { + version = os.Args[1] // Use the provided version + } + + // Fetch release data (either for latest or specific version) + releaseData, err := repo.fetchReleaseData(version) + if err != nil { + fmt.Println("Error fetching release data:", err) + return + } + + // Generate and print the formatted changelog + changelog := repo.generateChangelog(releaseData.TagName, releaseData.Published, releaseData.HtmlUrl, releaseData.Body) + fmt.Println(changelog) +} diff --git a/tools/changelog/main.go b/tools/changelog/main.go deleted file mode 100644 index ff9a7eab9..000000000 --- a/tools/changelog/main.go +++ /dev/null @@ -1,308 +0,0 @@ -// Copyright © 2023 OpenIM. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package main - -import ( - "fmt" - "log" - "os" - "os/exec" - "regexp" - "sort" - "strings" -) - -var ( - mergeRequest = regexp.MustCompile(`Merge pull request #([\d]+)`) - webconsoleBump = regexp.MustCompile(regexp.QuoteMeta("bump(github.com/openshift/origin-web-console): ") + `([\w]+)`) - upstreamKube = regexp.MustCompile(`^UPSTREAM: (\d+)+:(.+)`) - upstreamRepo = regexp.MustCompile(`^UPSTREAM: ([\w/-]+): (\d+)+:(.+)`) - prefix = regexp.MustCompile(`^[\w-]: `) - - assignments = []prefixAssignment{ - {"cluster up", "cluster"}, - {" pv ", "storage"}, - {"haproxy", "router"}, - {"router", "router"}, - {"route", "route"}, - {"authoriz", "auth"}, - {"rbac", "auth"}, - {"authent", "auth"}, - {"reconcil", "auth"}, - {"auth", "auth"}, - {"role", "auth"}, - {" dc ", "deploy"}, - {"deployment", "deploy"}, - {"rolling", "deploy"}, - {"security context constr", "security"}, - {"scc", "security"}, - {"pipeline", "build"}, - {"build", "build"}, - {"registry", "registry"}, - {"registries", "image"}, - {"image", "image"}, - {" arp ", "network"}, - {" cni ", "network"}, - {"egress", "network"}, - {"network", "network"}, - {"oc ", "cli"}, - {"template", "template"}, - {"etcd", "server"}, - {"pod", "node"}, - {"scripts/", "hack"}, - {"e2e", "test"}, - {"integration", "test"}, - {"cluster", "cluster"}, - {"master", "server"}, - {"packages", "hack"}, - {"api", "server"}, - } -) - -type prefixAssignment struct { - term string - prefix string -} - -type commit struct { - short string - parents []string - message string -} - -func contains(arr []string, value string) bool { - for _, s := range arr { - if s == value { - return true - } - } - return false -} - -func main() { - log.SetFlags(0) - if len(os.Args) != 3 { - log.Fatalf("Must specify two arguments, FROM and TO") - } - from := os.Args[1] - to := os.Args[2] - - out, err := exec.Command("git", "log", "--topo-order", "--pretty=tformat:%h %p|%s", "--reverse", fmt.Sprintf("%s..%s", from, to)).CombinedOutput() - if err != nil { - log.Fatal(err) - } - - hide := make(map[string]struct{}) - var apiChanges []string - var webconsole []string - var commits []commit - var upstreams []commit - var bumps []commit - for _, line := range strings.Split(string(out), "\n") { - if len(strings.TrimSpace(line)) == 0 { - continue - } - parts := strings.SplitN(line, "|", 2) - hashes := strings.Split(parts[0], " ") - c := commit{short: hashes[0], parents: hashes[1:], message: parts[1]} - - if strings.HasPrefix(c.message, "UPSTREAM: ") { - hide[c.short] = struct{}{} - upstreams = append(upstreams, c) - } - if strings.HasPrefix(c.message, "bump(") { - hide[c.short] = struct{}{} - bumps = append(bumps, c) - } - - if len(c.parents) == 1 { - commits = append(commits, c) - continue - } - - matches := mergeRequest.FindStringSubmatch(line) - if len(matches) == 0 { - // this may have been a human pressing the merge button, we'll just record this as a direct push - continue - } - - // split the accumulated commits into any that are force merges (assumed to be the initial set due - // to --topo-order) from the PR commits as soon as we see any of our merge parents. Then print - // any of the force merges - var first int - for i := range commits { - first = i - if contains(c.parents, commits[i].short) { - first++ - break - } - } - individual := commits[:first] - merged := commits[first:] - for _, commit := range individual { - if len(commit.parents) > 1 { - continue - } - if _, ok := hide[commit.short]; ok { - continue - } - fmt.Printf("force-merge: %s %s\n", commit.message, commit.short) - } - - // try to find either the PR title or the first commit title from the merge commit - out, err := exec.Command("git", "show", "--pretty=tformat:%b", c.short).CombinedOutput() - if err != nil { - log.Fatal(err) - } - var message string - para := strings.Split(string(out), "\n\n") - if len(para) > 0 && strings.HasPrefix(para[0], "Automatic merge from submit-queue") { - para = para[1:] - } - // this is no longer necessary with the submit queue in place - if len(para) > 0 && strings.HasPrefix(para[0], "Merged by ") { - para = para[1:] - } - // post submit-queue, the merge bot will add the PR title, which is usually pretty good - if len(para) > 0 { - message = strings.Split(para[0], "\n")[0] - } - if len(message) == 0 && len(merged) > 0 { - message = merged[0].message - } - if len(message) > 0 && len(merged) == 1 && message == merged[0].message { - merged = nil - } - - // try to calculate a prefix based on the diff - if len(message) > 0 && !prefix.MatchString(message) { - prefix, ok := findPrefixFor(message, merged) - if ok { - message = prefix + ": " + message - } - } - - // github merge - - // has api changes - display := fmt.Sprintf("%s [\\#%s](https://github.com/openimsdk/Open-IM-Server/pull/%s)", message, matches[1], matches[1]) - if hasFileChanges(c.short, "pkg/apistruct/") { - apiChanges = append(apiChanges, display) - } - - var filtered []commit - for _, commit := range merged { - if _, ok := hide[commit.short]; ok { - continue - } - filtered = append(filtered, commit) - } - if len(filtered) > 0 { - fmt.Printf("- %s\n", display) - for _, commit := range filtered { - fmt.Printf(" - %s (%s)\n", commit.message, commit.short) - } - } - - // stick the merge commit in at the beginning of the next list so we can anchor the previous parent - commits = []commit{c} - } - - // chunk the bumps - var lines []string - for _, commit := range bumps { - if m := webconsoleBump.FindStringSubmatch(commit.message); len(m) > 0 { - webconsole = append(webconsole, m[1]) - continue - } - lines = append(lines, commit.message) - } - lines = sortAndUniq(lines) - for _, line := range lines { - fmt.Printf("- %s\n", line) - } - - // chunk the upstreams - lines = nil - for _, commit := range upstreams { - lines = append(lines, commit.message) - } - lines = sortAndUniq(lines) - for _, line := range lines { - fmt.Printf("- %s\n", upstreamLinkify(line)) - } - - if len(webconsole) > 0 { - fmt.Printf("- web: from %s^..%s\n", webconsole[0], webconsole[len(webconsole)-1]) - } - - for _, apiChange := range apiChanges { - fmt.Printf(" - %s\n", apiChange) - } -} - -func findPrefixFor(message string, commits []commit) (string, bool) { - message = strings.ToLower(message) - for _, m := range assignments { - if strings.Contains(message, m.term) { - return m.prefix, true - } - } - for _, c := range commits { - if prefix, ok := findPrefixFor(c.message, nil); ok { - return prefix, ok - } - } - return "", false -} - -func hasFileChanges(commit string, prefixes ...string) bool { - out, err := exec.Command("git", "diff", "--name-only", fmt.Sprintf("%s^..%s", commit, commit)).CombinedOutput() - if err != nil { - log.Fatal(err) - } - for _, file := range strings.Split(string(out), "\n") { - for _, prefix := range prefixes { - if strings.HasPrefix(file, prefix) { - return true - } - } - } - return false -} - -func sortAndUniq(lines []string) []string { - sort.Strings(lines) - out := make([]string, 0, len(lines)) - last := "" - for _, s := range lines { - if last == s { - continue - } - last = s - out = append(out, s) - } - return out -} - -func upstreamLinkify(line string) string { - if m := upstreamKube.FindStringSubmatch(line); len(m) > 0 { - return fmt.Sprintf("UPSTREAM: [#%s](https://github.com/openimsdk/open-im-server/pull/%s):%s", m[1], m[1], m[2]) - } - if m := upstreamRepo.FindStringSubmatch(line); len(m) > 0 { - return fmt.Sprintf("UPSTREAM: [%s#%s](https://github.com/%s/pull/%s):%s", m[1], m[2], m[1], m[2], m[3]) - } - return line -} From bf8853580024e22aa48fb1b6731415cbdbe63f94 Mon Sep 17 00:00:00 2001 From: Sasaya Date: Fri, 22 Nov 2024 14:45:24 +0800 Subject: [PATCH 043/199] fix: single chat offline push webhook (#2862) --- internal/push/push_handler.go | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/internal/push/push_handler.go b/internal/push/push_handler.go index ae308dfe9..b5bbed3ed 100644 --- a/internal/push/push_handler.go +++ b/internal/push/push_handler.go @@ -166,17 +166,21 @@ func (c *ConsumerHandler) Push2User(ctx context.Context, userIDs []string, msg * return nil } } - offlinePushUserID := []string{msg.RecvID} + needOfflinePushUserID := []string{msg.RecvID} + var offlinePushUserID []string //receiver offline push - if err = c.webhookBeforeOfflinePush(ctx, &c.config.WebhooksConfig.BeforeOfflinePush, - offlinePushUserID, msg, nil); err != nil { + if err = c.webhookBeforeOfflinePush(ctx, &c.config.WebhooksConfig.BeforeOfflinePush, needOfflinePushUserID, msg, &offlinePushUserID); err != nil { return err } log.ZInfo(ctx, "webhookBeforeOfflinePush end") - err = c.offlinePushMsg(ctx, msg, offlinePushUserID) + + if len(offlinePushUserID) > 0 { + needOfflinePushUserID = offlinePushUserID + } + err = c.offlinePushMsg(ctx, msg, needOfflinePushUserID) if err != nil { - log.ZWarn(ctx, "offlinePushMsg failed", err, "offlinePushUserID", offlinePushUserID, "msg", msg) + log.ZWarn(ctx, "offlinePushMsg failed", err, "needOfflinePushUserID", needOfflinePushUserID, "msg", msg) return nil } From 334749c5a2549f5f5619ace17274cf76925bc0e2 Mon Sep 17 00:00:00 2001 From: icey-yu <119291641+icey-yu@users.noreply.github.com> Date: Fri, 22 Nov 2024 15:05:39 +0800 Subject: [PATCH 044/199] feat: Print Panic Log (#2850) * feat: catch panic * feat: docker file * feat: cicd * feat: dockerfile --------- Co-authored-by: Monet Lee --- .github/workflows/go-build-test.yml | 2 +- internal/api/router.go | 2 +- internal/msggateway/client.go | 6 ++++++ internal/msgtransfer/init.go | 5 +++++ internal/msgtransfer/online_history_msg_handler.go | 7 +++++++ internal/rpc/msg/send.go | 8 ++++++++ pkg/rpccache/subscriber.go | 6 ++++++ 7 files changed, 34 insertions(+), 2 deletions(-) diff --git a/.github/workflows/go-build-test.yml b/.github/workflows/go-build-test.yml index 5b37bf47d..2ca960cc9 100644 --- a/.github/workflows/go-build-test.yml +++ b/.github/workflows/go-build-test.yml @@ -149,7 +149,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - go_version: ["1.21"] + go_version: ["1.22"] steps: - name: Checkout Repository diff --git a/internal/api/router.go b/internal/api/router.go index 4ac301b07..8e4d17ef1 100644 --- a/internal/api/router.go +++ b/internal/api/router.go @@ -74,7 +74,7 @@ func newGinRouter(disCov discovery.SvcDiscoveryRegistry, config *Config) *gin.En case BestSpeed: r.Use(gzip.Gzip(gzip.BestSpeed)) } - r.Use(prommetricsGin(), gin.Recovery(), mw.CorsHandler(), mw.GinParseOperationID(), GinParseToken(authRpc)) + r.Use(prommetricsGin(), gin.RecoveryWithWriter(gin.DefaultErrorWriter, mw.GinPanicErr), mw.CorsHandler(), mw.GinParseOperationID(), GinParseToken(authRpc)) u := NewUserApi(*userRpc) m := NewMessageApi(messageRpc, userRpc, config.Share.IMAdminUserID) j := jssdk.NewJSSdkApi(userRpc.Client, friendRpc.Client, groupRpc.Client, messageRpc.Client, conversationRpc.Client) diff --git a/internal/msggateway/client.go b/internal/msggateway/client.go index 6e31751dc..0da7d7220 100644 --- a/internal/msggateway/client.go +++ b/internal/msggateway/client.go @@ -18,6 +18,7 @@ import ( "context" "encoding/json" "fmt" + "github.com/openimsdk/tools/mw" "runtime/debug" "sync" "sync/atomic" @@ -375,6 +376,11 @@ func (c *Client) writeBinaryMsg(resp Resp) error { func (c *Client) activeHeartbeat(ctx context.Context) { if c.PlatformID == constant.WebPlatformID { go func() { + defer func() { + if r := recover(); r != nil { + mw.PanicStackToLog(ctx, r) + } + }() log.ZDebug(ctx, "server initiative send heartbeat start.") ticker := time.NewTicker(pingPeriod) defer ticker.Stop() diff --git a/internal/msgtransfer/init.go b/internal/msgtransfer/init.go index f11cfde1a..92053931c 100644 --- a/internal/msgtransfer/init.go +++ b/internal/msgtransfer/init.go @@ -136,6 +136,11 @@ func (m *MsgTransfer) Start(index int, config *Config) error { if config.MsgTransfer.Prometheus.Enable { go func() { + defer func() { + if r := recover(); r != nil { + mw.PanicStackToLog(m.ctx, r) + } + }() prometheusPort, err := datautil.GetElemByIndex(config.MsgTransfer.Prometheus.Ports, index) if err != nil { netErr = err diff --git a/internal/msgtransfer/online_history_msg_handler.go b/internal/msgtransfer/online_history_msg_handler.go index 84453c8df..4a5d5ba89 100644 --- a/internal/msgtransfer/online_history_msg_handler.go +++ b/internal/msgtransfer/online_history_msg_handler.go @@ -19,6 +19,7 @@ import ( "encoding/json" "errors" "github.com/openimsdk/open-im-server/v3/pkg/common/prommetrics" + "github.com/openimsdk/tools/mw" "strconv" "strings" "sync" @@ -346,6 +347,12 @@ func (och *OnlineHistoryRedisConsumerHandler) handleNotification(ctx context.Con } } func (och *OnlineHistoryRedisConsumerHandler) HandleUserHasReadSeqMessages(ctx context.Context) { + defer func() { + if r := recover(); r != nil { + mw.PanicStackToLog(ctx, r) + } + }() + defer och.wg.Done() for msg := range och.conversationUserHasReadChan { diff --git a/internal/rpc/msg/send.go b/internal/rpc/msg/send.go index 4762f24de..5c5d081c3 100644 --- a/internal/rpc/msg/send.go +++ b/internal/rpc/msg/send.go @@ -16,6 +16,7 @@ package msg import ( "context" + "github.com/openimsdk/tools/mw" "github.com/openimsdk/open-im-server/v3/pkg/common/prommetrics" "github.com/openimsdk/open-im-server/v3/pkg/msgprocessor" @@ -83,8 +84,15 @@ func (m *msgServer) sendMsgGroupChat(ctx context.Context, req *pbmsg.SendMsgReq) } func (m *msgServer) setConversationAtInfo(nctx context.Context, msg *sdkws.MsgData) { + log.ZDebug(nctx, "setConversationAtInfo", "msg", msg) + defer func() { + if r := recover(); r != nil { + mw.PanicStackToLog(nctx, r) + } + }() + ctx := mcontext.NewCtx("@@@" + mcontext.GetOperationID(nctx)) var atUserID []string diff --git a/pkg/rpccache/subscriber.go b/pkg/rpccache/subscriber.go index 3046f84b1..3c73ef449 100644 --- a/pkg/rpccache/subscriber.go +++ b/pkg/rpccache/subscriber.go @@ -17,12 +17,18 @@ package rpccache import ( "context" "encoding/json" + "github.com/openimsdk/tools/mw" "github.com/openimsdk/tools/log" "github.com/redis/go-redis/v9" ) func subscriberRedisDeleteCache(ctx context.Context, client redis.UniversalClient, channel string, del func(ctx context.Context, key ...string)) { + defer func() { + if r := recover(); r != nil { + mw.PanicStackToLog(ctx, r) + } + }() for message := range client.Subscribe(ctx, channel).Channel() { log.ZDebug(ctx, "subscriberRedisDeleteCache", "channel", channel, "payload", message.Payload) var keys []string From b17f4f15e853fde7e7c574e671cdc0f4eb4f5fa3 Mon Sep 17 00:00:00 2001 From: Monet Lee Date: Fri, 22 Nov 2024 15:06:28 +0800 Subject: [PATCH 045/199] fix: move workflow to correct path (#2837) * build: implement version file update when release. * build: move file to correct path. * remove. --- .../{.github/workflows => }/update-version-file-on-release.yml | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{.github/workflows => }/update-version-file-on-release.yml (100%) diff --git a/.github/workflows/.github/workflows/update-version-file-on-release.yml b/.github/workflows/update-version-file-on-release.yml similarity index 100% rename from .github/workflows/.github/workflows/update-version-file-on-release.yml rename to .github/workflows/update-version-file-on-release.yml From c2fd0061b042fe5153e885722374cd75f212d6c0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 22 Nov 2024 15:48:47 +0800 Subject: [PATCH 046/199] build(deps): bump github.com/golang-jwt/jwt/v4 from 4.5.0 to 4.5.1 (#2851) Bumps [github.com/golang-jwt/jwt/v4](https://github.com/golang-jwt/jwt) from 4.5.0 to 4.5.1. - [Release notes](https://github.com/golang-jwt/jwt/releases) - [Changelog](https://github.com/golang-jwt/jwt/blob/main/VERSION_HISTORY.md) - [Commits](https://github.com/golang-jwt/jwt/compare/v4.5.0...v4.5.1) --- updated-dependencies: - dependency-name: github.com/golang-jwt/jwt/v4 dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 8a05b016c..d39ad1d34 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,7 @@ require ( github.com/gin-gonic/gin v1.9.1 github.com/go-playground/validator/v10 v10.20.0 github.com/gogo/protobuf v1.3.2 // indirect - github.com/golang-jwt/jwt/v4 v4.5.0 + github.com/golang-jwt/jwt/v4 v4.5.1 github.com/gorilla/websocket v1.5.1 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/mitchellh/mapstructure v1.5.0 diff --git a/go.sum b/go.sum index 3cae53b0d..df5a34516 100644 --- a/go.sum +++ b/go.sum @@ -158,8 +158,8 @@ github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5x github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang-jwt/jwt/v4 v4.4.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= -github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg= -github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= +github.com/golang-jwt/jwt/v4 v4.5.1 h1:JdqV9zKUdtaa9gdPlywC3aeoEsR681PlKC+4F5gQgeo= +github.com/golang-jwt/jwt/v4 v4.5.1/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= From 2c612ba6d0b8a8d81d47929eb6013881b07c9dcd Mon Sep 17 00:00:00 2001 From: icey-yu <119291641+icey-yu@users.noreply.github.com> Date: Fri, 22 Nov 2024 15:49:16 +0800 Subject: [PATCH 047/199] fix: del login Policy (#2825) * fix: del login Policy * feat: offline push * feat: offline push * fix: err * fix: err --- config/openim-push.yml | 4 ++-- config/share.yml | 13 +------------ go.mod | 2 +- go.sum | 4 ++-- internal/push/offlinepush/dummy/push.go | 1 + internal/push/offlinepush/offlinepusher.go | 3 --- internal/rpc/msg/as_read.go | 5 +++-- internal/rpc/msg/seq.go | 4 ++-- pkg/common/config/config.go | 16 ++-------------- pkg/common/storage/controller/auth.go | 20 ++++---------------- pkg/common/storage/controller/msg.go | 2 +- 11 files changed, 19 insertions(+), 55 deletions(-) diff --git a/config/openim-push.yml b/config/openim-push.yml index 70e67add2..92f716ba2 100644 --- a/config/openim-push.yml +++ b/config/openim-push.yml @@ -13,8 +13,8 @@ prometheus: ports: [ 12170, 12171, 12172, 12173, 12174, 12175, 12176, 12177, 12178, 12179, 12180, 12182, 12183, 12184, 12185, 12186 ] maxConcurrentWorkers: 3 -#Use geTui for offline push notifications, or choose fcm or jpush; corresponding configuration settings must be specified. -enable: geTui +#Use geTui for offline push notifications, or choose fcm or jpns; corresponding configuration settings must be specified. +enable: geTui: pushUrl: https://restapi.getui.com/v2/$appId masterSecret: diff --git a/config/share.yml b/config/share.yml index 7d977ae15..1726af2dc 100644 --- a/config/share.yml +++ b/config/share.yml @@ -15,15 +15,4 @@ imAdminUserID: [ imAdmin ] # 1: For Android, iOS, Windows, Mac, and web platforms, only one instance can be online at a time multiLogin: policy: 1 - maxNumOneEnd: 30 - customizeLoginNum: - ios: 1 - android: 1 - windows: 1 - osx: 1 - web: 1 - miniWeb: 1 - linux: 1 - aPad: 1 - iPad: 1 - admin: 1 + maxNumOneEnd: 30 \ No newline at end of file diff --git a/go.mod b/go.mod index d39ad1d34..96c229166 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( github.com/gorilla/websocket v1.5.1 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/mitchellh/mapstructure v1.5.0 - github.com/openimsdk/protocol v0.0.72-alpha.54 + github.com/openimsdk/protocol v0.0.72-alpha.55 github.com/openimsdk/tools v0.0.50-alpha.32 github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 diff --git a/go.sum b/go.sum index df5a34516..36f3d1615 100644 --- a/go.sum +++ b/go.sum @@ -319,8 +319,8 @@ github.com/onsi/gomega v1.25.0 h1:Vw7br2PCDYijJHSfBOWhov+8cAnUf8MfMaIOV323l6Y= github.com/onsi/gomega v1.25.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdMPGhM= github.com/openimsdk/gomake v0.0.14-alpha.5 h1:VY9c5x515lTfmdhhPjMvR3BBRrRquAUCFsz7t7vbv7Y= github.com/openimsdk/gomake v0.0.14-alpha.5/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= -github.com/openimsdk/protocol v0.0.72-alpha.54 h1:opato7N4QjjRq/SHD54bDSVBpOEEDp1VLWVk5Os2A9s= -github.com/openimsdk/protocol v0.0.72-alpha.54/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= +github.com/openimsdk/protocol v0.0.72-alpha.55 h1:9PPWPHvkFk3neBSbNr+IoOdKIFjxTvEqUfMK/TEq1+8= +github.com/openimsdk/protocol v0.0.72-alpha.55/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= github.com/openimsdk/tools v0.0.50-alpha.32 h1:JEsUFHFnaYg230TG+Ke3SUnaA2h44t4kABAzEdv5VZw= github.com/openimsdk/tools v0.0.50-alpha.32/go.mod h1:r5U6RbxcR4xhKb2fhTmKGC9Yt5LcErHBVt3lhXQIHSo= github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= diff --git a/internal/push/offlinepush/dummy/push.go b/internal/push/offlinepush/dummy/push.go index 09831cabf..0bccaf4a4 100644 --- a/internal/push/offlinepush/dummy/push.go +++ b/internal/push/offlinepush/dummy/push.go @@ -29,5 +29,6 @@ type Dummy struct { func (d *Dummy) Push(ctx context.Context, userIDs []string, title, content string, opts *options.Opts) error { log.ZDebug(ctx, "dummy push") + log.ZWarn(ctx, "Dummy push", nil, "ps", "The offline push is not configured. To configure it, please go to config/openim-push.yml.") return nil } diff --git a/internal/push/offlinepush/offlinepusher.go b/internal/push/offlinepush/offlinepusher.go index d655a924a..17d5d7071 100644 --- a/internal/push/offlinepush/offlinepusher.go +++ b/internal/push/offlinepush/offlinepusher.go @@ -23,8 +23,6 @@ import ( "github.com/openimsdk/open-im-server/v3/internal/push/offlinepush/options" "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache" - "github.com/openimsdk/tools/log" - "github.com/openimsdk/tools/mcontext" "strings" ) @@ -51,7 +49,6 @@ func NewOfflinePusher(pushConf *config.Push, cache cache.ThirdCache, fcmConfigPa offlinePusher = jpush.NewClient(pushConf) default: offlinePusher = dummy.NewClient() - log.ZWarn(mcontext.WithMustInfoCtx([]string{"push start", "admin", "admin", ""}), "Unknown push config", nil) } return offlinePusher, nil } diff --git a/internal/rpc/msg/as_read.go b/internal/rpc/msg/as_read.go index 03f35b42d..312c4d556 100644 --- a/internal/rpc/msg/as_read.go +++ b/internal/rpc/msg/as_read.go @@ -16,6 +16,7 @@ package msg import ( "context" + "errors" cbapi "github.com/openimsdk/open-im-server/v3/pkg/callbackstruct" "github.com/openimsdk/protocol/constant" @@ -108,7 +109,7 @@ func (m *msgServer) MarkMsgsAsRead(ctx context.Context, req *msg.MarkMsgsAsReadR return nil, err } currentHasReadSeq, err := m.MsgDatabase.GetHasReadSeq(ctx, req.UserID, req.ConversationID) - if err != nil && errs.Unwrap(err) != redis.Nil { + if err != nil && errors.Is(err, redis.Nil) { return nil, err } if hasReadSeq > currentHasReadSeq { @@ -136,7 +137,7 @@ func (m *msgServer) MarkConversationAsRead(ctx context.Context, req *msg.MarkCon return nil, err } hasReadSeq, err := m.MsgDatabase.GetHasReadSeq(ctx, req.UserID, req.ConversationID) - if err != nil && errs.Unwrap(err) != redis.Nil { + if err != nil && errors.Is(err, redis.Nil) { return nil, err } var seqs []int64 diff --git a/internal/rpc/msg/seq.go b/internal/rpc/msg/seq.go index 5d40160de..ddf84a267 100644 --- a/internal/rpc/msg/seq.go +++ b/internal/rpc/msg/seq.go @@ -16,15 +16,15 @@ package msg import ( "context" + "errors" pbmsg "github.com/openimsdk/protocol/msg" - "github.com/openimsdk/tools/errs" "github.com/redis/go-redis/v9" "sort" ) func (m *msgServer) GetConversationMaxSeq(ctx context.Context, req *pbmsg.GetConversationMaxSeqReq) (*pbmsg.GetConversationMaxSeqResp, error) { maxSeq, err := m.MsgDatabase.GetMaxSeq(ctx, req.ConversationID) - if err != nil && errs.Unwrap(err) != redis.Nil { + if err != nil && errors.Is(err, redis.Nil) { return nil, err } return &pbmsg.GetConversationMaxSeqResp{MaxSeq: maxSeq}, nil diff --git a/pkg/common/config/config.go b/pkg/common/config/config.go index c40018cbc..468a150e8 100644 --- a/pkg/common/config/config.go +++ b/pkg/common/config/config.go @@ -369,20 +369,8 @@ type Share struct { } type MultiLogin struct { - Policy int `mapstructure:"policy"` - MaxNumOneEnd int `mapstructure:"maxNumOneEnd"` - CustomizeLoginNum struct { - IOS int `mapstructure:"ios"` - Android int `mapstructure:"android"` - Windows int `mapstructure:"windows"` - OSX int `mapstructure:"osx"` - Web int `mapstructure:"web"` - MiniWeb int `mapstructure:"miniWeb"` - Linux int `mapstructure:"linux"` - APad int `mapstructure:"aPad"` - IPad int `mapstructure:"iPad"` - Admin int `mapstructure:"admin"` - } `mapstructure:"customizeLoginNum"` + Policy int `mapstructure:"policy"` + MaxNumOneEnd int `mapstructure:"maxNumOneEnd"` } type RpcRegisterName struct { diff --git a/pkg/common/storage/controller/auth.go b/pkg/common/storage/controller/auth.go index e7b5bc297..0a7029662 100644 --- a/pkg/common/storage/controller/auth.go +++ b/pkg/common/storage/controller/auth.go @@ -25,9 +25,8 @@ type AuthDatabase interface { } type multiLoginConfig struct { - Policy int - MaxNumOneEnd int - CustomizeLoginNum map[int]int + Policy int + MaxNumOneEnd int } type authDatabase struct { @@ -42,19 +41,8 @@ func NewAuthDatabase(cache cache.TokenModel, accessSecret string, accessExpire i return &authDatabase{cache: cache, accessSecret: accessSecret, accessExpire: accessExpire, multiLogin: multiLoginConfig{ Policy: multiLogin.Policy, MaxNumOneEnd: multiLogin.MaxNumOneEnd, - CustomizeLoginNum: map[int]int{ - constant.IOSPlatformID: multiLogin.CustomizeLoginNum.IOS, - constant.AndroidPlatformID: multiLogin.CustomizeLoginNum.Android, - constant.WindowsPlatformID: multiLogin.CustomizeLoginNum.Windows, - constant.OSXPlatformID: multiLogin.CustomizeLoginNum.OSX, - constant.WebPlatformID: multiLogin.CustomizeLoginNum.Web, - constant.MiniWebPlatformID: multiLogin.CustomizeLoginNum.MiniWeb, - constant.LinuxPlatformID: multiLogin.CustomizeLoginNum.Linux, - constant.AndroidPadPlatformID: multiLogin.CustomizeLoginNum.APad, - constant.IPadPlatformID: multiLogin.CustomizeLoginNum.IPad, - constant.AdminPlatformID: multiLogin.CustomizeLoginNum.Admin, - }, - }, adminUserIDs: adminUserIDs, + }, + adminUserIDs: adminUserIDs, } } diff --git a/pkg/common/storage/controller/msg.go b/pkg/common/storage/controller/msg.go index 9636f7a15..789adb1f6 100644 --- a/pkg/common/storage/controller/msg.go +++ b/pkg/common/storage/controller/msg.go @@ -372,7 +372,7 @@ func (db *commonMsgDatabase) getMsgBySeqsRange(ctx context.Context, userID strin // This ensures that their message retrieval starts from the point they joined. func (db *commonMsgDatabase) GetMsgBySeqsRange(ctx context.Context, userID string, conversationID string, begin, end, num, userMaxSeq int64) (int64, int64, []*sdkws.MsgData, error) { userMinSeq, err := db.seqUser.GetUserMinSeq(ctx, conversationID, userID) - if err != nil && errs.Unwrap(err) != redis.Nil { + if err != nil && errors.Is(err, redis.Nil) { return 0, 0, nil, err } minSeq, err := db.seqConversation.GetMinSeq(ctx, conversationID) From bfc4684a36bd80f8faeceb33d3d01525a394e38c Mon Sep 17 00:00:00 2001 From: icey-yu <119291641+icey-yu@users.noreply.github.com> Date: Fri, 22 Nov 2024 17:46:02 +0800 Subject: [PATCH 048/199] fix: err (#2876) --- internal/rpc/msg/as_read.go | 2 +- internal/rpc/msg/seq.go | 2 +- pkg/common/storage/controller/msg.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/internal/rpc/msg/as_read.go b/internal/rpc/msg/as_read.go index 312c4d556..e13e017e5 100644 --- a/internal/rpc/msg/as_read.go +++ b/internal/rpc/msg/as_read.go @@ -109,7 +109,7 @@ func (m *msgServer) MarkMsgsAsRead(ctx context.Context, req *msg.MarkMsgsAsReadR return nil, err } currentHasReadSeq, err := m.MsgDatabase.GetHasReadSeq(ctx, req.UserID, req.ConversationID) - if err != nil && errors.Is(err, redis.Nil) { + if err != nil && !errors.Is(err, redis.Nil) { return nil, err } if hasReadSeq > currentHasReadSeq { diff --git a/internal/rpc/msg/seq.go b/internal/rpc/msg/seq.go index ddf84a267..7f5fa1adb 100644 --- a/internal/rpc/msg/seq.go +++ b/internal/rpc/msg/seq.go @@ -24,7 +24,7 @@ import ( func (m *msgServer) GetConversationMaxSeq(ctx context.Context, req *pbmsg.GetConversationMaxSeqReq) (*pbmsg.GetConversationMaxSeqResp, error) { maxSeq, err := m.MsgDatabase.GetMaxSeq(ctx, req.ConversationID) - if err != nil && errors.Is(err, redis.Nil) { + if err != nil && !errors.Is(err, redis.Nil) { return nil, err } return &pbmsg.GetConversationMaxSeqResp{MaxSeq: maxSeq}, nil diff --git a/pkg/common/storage/controller/msg.go b/pkg/common/storage/controller/msg.go index 789adb1f6..cdc8fd888 100644 --- a/pkg/common/storage/controller/msg.go +++ b/pkg/common/storage/controller/msg.go @@ -372,7 +372,7 @@ func (db *commonMsgDatabase) getMsgBySeqsRange(ctx context.Context, userID strin // This ensures that their message retrieval starts from the point they joined. func (db *commonMsgDatabase) GetMsgBySeqsRange(ctx context.Context, userID string, conversationID string, begin, end, num, userMaxSeq int64) (int64, int64, []*sdkws.MsgData, error) { userMinSeq, err := db.seqUser.GetUserMinSeq(ctx, conversationID, userID) - if err != nil && errors.Is(err, redis.Nil) { + if err != nil && !errors.Is(err, redis.Nil) { return 0, 0, nil, err } minSeq, err := db.seqConversation.GetMinSeq(ctx, conversationID) From cbade46ae0c40ecd47e9177c20862d68df46ef71 Mon Sep 17 00:00:00 2001 From: Morya Date: Mon, 25 Nov 2024 10:49:31 +0800 Subject: [PATCH 049/199] fix: minor log typo (#2881) --- pkg/common/storage/cache/redis/conversation.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/common/storage/cache/redis/conversation.go b/pkg/common/storage/cache/redis/conversation.go index 326f60b96..91d8ed69d 100644 --- a/pkg/common/storage/cache/redis/conversation.go +++ b/pkg/common/storage/cache/redis/conversation.go @@ -38,7 +38,7 @@ const ( func NewConversationRedis(rdb redis.UniversalClient, localCache *config.LocalCache, opts *rockscache.Options, db database.Conversation) cache.ConversationCache { batchHandler := NewBatchDeleterRedis(rdb, opts, []string{localCache.Conversation.Topic}) c := localCache.Conversation - log.ZDebug(context.Background(), "black local cache init", "Topic", c.Topic, "SlotNum", c.SlotNum, "SlotSize", c.SlotSize, "enable", c.Enable()) + log.ZDebug(context.Background(), "conversation local cache init", "Topic", c.Topic, "SlotNum", c.SlotNum, "SlotSize", c.SlotSize, "enable", c.Enable()) return &ConversationRedisCache{ BatchDeleter: batchHandler, rcClient: rockscache.NewClient(rdb, *opts), From 38cfa4a9c41c0d52aef9c63a914dcbf9a77bda3d Mon Sep 17 00:00:00 2001 From: icey-yu <119291641+icey-yu@users.noreply.github.com> Date: Mon, 25 Nov 2024 14:11:34 +0800 Subject: [PATCH 050/199] fix: webhookAfterSingleMsgRead (#2884) --- internal/rpc/msg/as_read.go | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/internal/rpc/msg/as_read.go b/internal/rpc/msg/as_read.go index e13e017e5..c9610969d 100644 --- a/internal/rpc/msg/as_read.go +++ b/internal/rpc/msg/as_read.go @@ -181,14 +181,23 @@ func (m *msgServer) MarkConversationAsRead(ctx context.Context, req *msg.MarkCon req.UserID, seqs, hasReadSeq) } - reqCall := &cbapi.CallbackGroupMsgReadReq{ - SendID: conversation.OwnerUserID, - ReceiveID: req.UserID, - UnreadMsgNum: req.HasReadSeq, - ContentType: int64(conversation.ConversationType), + if conversation.ConversationType == constant.SingleChatType { + reqCall := &cbapi.CallbackSingleMsgReadReq{ + ConversationID: conversation.ConversationID, + UserID: conversation.OwnerUserID, + Seqs: req.Seqs, + ContentType: conversation.ConversationType, + } + m.webhookAfterSingleMsgRead(ctx, &m.config.WebhooksConfig.AfterSingleMsgRead, reqCall) + } else if conversation.ConversationType == constant.ReadGroupChatType { + reqCall := &cbapi.CallbackGroupMsgReadReq{ + SendID: conversation.OwnerUserID, + ReceiveID: req.UserID, + UnreadMsgNum: req.HasReadSeq, + ContentType: int64(conversation.ConversationType), + } + m.webhookAfterGroupMsgRead(ctx, &m.config.WebhooksConfig.AfterGroupMsgRead, reqCall) } - - m.webhookAfterGroupMsgRead(ctx, &m.config.WebhooksConfig.AfterGroupMsgRead, reqCall) return &msg.MarkConversationAsReadResp{}, nil } From f0a60a71aea9ec1627704731a67bfb6a2f70c7e3 Mon Sep 17 00:00:00 2001 From: icey-yu <119291641+icey-yu@users.noreply.github.com> Date: Mon, 25 Nov 2024 14:12:03 +0800 Subject: [PATCH 051/199] fix: webhookBeforeSendSingleMsg will call before black and friend check (#2885) --- internal/rpc/msg/send.go | 3 --- internal/rpc/msg/verify.go | 3 +++ 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/internal/rpc/msg/send.go b/internal/rpc/msg/send.go index 5c5d081c3..2cbbcd1fc 100644 --- a/internal/rpc/msg/send.go +++ b/internal/rpc/msg/send.go @@ -179,9 +179,6 @@ func (m *msgServer) sendMsgSingleChat(ctx context.Context, req *pbmsg.SendMsgReq prommetrics.SingleChatMsgProcessFailedCounter.Inc() return nil, nil } else { - if err = m.webhookBeforeSendSingleMsg(ctx, &m.config.WebhooksConfig.BeforeSendSingleMsg, req); err != nil { - return nil, err - } if err := m.webhookBeforeMsgModify(ctx, &m.config.WebhooksConfig.BeforeMsgModify, req); err != nil { return nil, err } diff --git a/internal/rpc/msg/verify.go b/internal/rpc/msg/verify.go index 33879bfe7..f6c3147ba 100644 --- a/internal/rpc/msg/verify.go +++ b/internal/rpc/msg/verify.go @@ -59,6 +59,9 @@ func (m *msgServer) messageVerification(ctx context.Context, data *msg.SendMsgRe data.MsgData.ContentType >= constant.NotificationBegin { return nil } + if err := m.webhookBeforeSendSingleMsg(ctx, &m.config.WebhooksConfig.BeforeSendSingleMsg, data); err != nil { + return err + } black, err := m.FriendLocalCache.IsBlack(ctx, data.MsgData.SendID, data.MsgData.RecvID) if err != nil { return err From 2b8b8b32047df66cbbecde0bfff8afc65c62f0ea Mon Sep 17 00:00:00 2001 From: icey-yu <119291641+icey-yu@users.noreply.github.com> Date: Mon, 25 Nov 2024 18:32:04 +0800 Subject: [PATCH 052/199] revert: write msg to redis (#2883) --- pkg/common/storage/controller/msg.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/pkg/common/storage/controller/msg.go b/pkg/common/storage/controller/msg.go index cdc8fd888..59718e7b9 100644 --- a/pkg/common/storage/controller/msg.go +++ b/pkg/common/storage/controller/msg.go @@ -444,10 +444,10 @@ func (db *commonMsgDatabase) GetMsgBySeqsRange(ctx context.Context, userID strin } successMsgs = append(mongoMsgs, successMsgs...) - _, err = db.msg.SetMessagesToCache(ctx, conversationID, mongoMsgs) - if err != nil { - return 0, 0, nil, err - } + //_, err = db.msg.SetMessagesToCache(ctx, conversationID, mongoMsgs) + //if err != nil { + // return 0, 0, nil, err + //} } return minSeq, maxSeq, successMsgs, nil @@ -506,10 +506,10 @@ func (db *commonMsgDatabase) GetMsgBySeqs(ctx context.Context, userID string, co successMsgs = append(successMsgs, mongoMsgs...) - _, err = db.msg.SetMessagesToCache(ctx, conversationID, mongoMsgs) - if err != nil { - return 0, 0, nil, err - } + //_, err = db.msg.SetMessagesToCache(ctx, conversationID, mongoMsgs) + //if err != nil { + // return 0, 0, nil, err + //} } return minSeq, maxSeq, successMsgs, nil } From 181d30449681fdcc2e0c0528bc4ada2017742437 Mon Sep 17 00:00:00 2001 From: Monet Lee Date: Mon, 25 Nov 2024 18:38:43 +0800 Subject: [PATCH 053/199] build: update Server version. (#2887) * build: update Server version. * update version. --- version/version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version/version b/version/version index aaaff9192..88d050b19 100644 --- a/version/version +++ b/version/version @@ -1 +1 @@ -3.8.1 \ No newline at end of file +main \ No newline at end of file From c353f557b76a033f8679a4e1612118d8920aa293 Mon Sep 17 00:00:00 2001 From: Kevin Lee <59052025+lgz5689@users.noreply.github.com> Date: Tue, 26 Nov 2024 15:59:05 +0800 Subject: [PATCH 054/199] chore: update admin front image version (#2893) --- .env | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.env b/.env index 2b903c949..87f4badba 100644 --- a/.env +++ b/.env @@ -8,10 +8,10 @@ ALERTMANAGER_IMAGE=prom/alertmanager:v0.27.0 GRAFANA_IMAGE=grafana/grafana:11.0.1 OPENIM_WEB_FRONT_IMAGE=openim/openim-web-front:release-v3.8.1 -OPENIM_ADMIN_FRONT_IMAGE=openim/openim-admin-front:release-v1.8.2 +OPENIM_ADMIN_FRONT_IMAGE=openim/openim-admin-front:release-v1.8.3 #FRONT_IMAGE: use aliyun images #OPENIM_WEB_FRONT_IMAGE=registry.cn-hangzhou.aliyuncs.com/openimsdk/openim-web-front:release-v3.8.1 -#OPENIM_ADMIN_FRONT_IMAGE=registry.cn-hangzhou.aliyuncs.com/openimsdk/openim-admin-front:release-v1.8.2 +#OPENIM_ADMIN_FRONT_IMAGE=registry.cn-hangzhou.aliyuncs.com/openimsdk/openim-admin-front:release-v1.8.3 DATA_DIR=./ From cf740a1b1a81bfe8ae11adc36629e3e0cff72f62 Mon Sep 17 00:00:00 2001 From: icey-yu <119291641+icey-yu@users.noreply.github.com> Date: Fri, 29 Nov 2024 15:50:22 +0800 Subject: [PATCH 055/199] fix: Wrong Redis Error Check (#2891) * fix: redis err check * fix: log err --- internal/rpc/auth/auth.go | 4 ++-- internal/rpc/msg/as_read.go | 2 +- pkg/common/storage/controller/msg.go | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/internal/rpc/auth/auth.go b/internal/rpc/auth/auth.go index a1acfd931..fe81bb45f 100644 --- a/internal/rpc/auth/auth.go +++ b/internal/rpc/auth/auth.go @@ -196,7 +196,7 @@ func (s *authServer) forceKickOff(ctx context.Context, userID string, platformID } m, err := s.authDatabase.GetTokensWithoutError(ctx, userID, int(platformID)) - if err != nil && errors.Is(err, redis.Nil) { + if err != nil && !errors.Is(err, redis.Nil) { return err } for k := range m { @@ -214,7 +214,7 @@ func (s *authServer) forceKickOff(ctx context.Context, userID string, platformID func (s *authServer) InvalidateToken(ctx context.Context, req *pbauth.InvalidateTokenReq) (*pbauth.InvalidateTokenResp, error) { m, err := s.authDatabase.GetTokensWithoutError(ctx, req.UserID, int(req.PlatformID)) - if err != nil && errors.Is(err, redis.Nil) { + if err != nil && !errors.Is(err, redis.Nil) { return nil, err } if m == nil { diff --git a/internal/rpc/msg/as_read.go b/internal/rpc/msg/as_read.go index c9610969d..de1879438 100644 --- a/internal/rpc/msg/as_read.go +++ b/internal/rpc/msg/as_read.go @@ -137,7 +137,7 @@ func (m *msgServer) MarkConversationAsRead(ctx context.Context, req *msg.MarkCon return nil, err } hasReadSeq, err := m.MsgDatabase.GetHasReadSeq(ctx, req.UserID, req.ConversationID) - if err != nil && errors.Is(err, redis.Nil) { + if err != nil && !errors.Is(err, redis.Nil) { return nil, err } var seqs []int64 diff --git a/pkg/common/storage/controller/msg.go b/pkg/common/storage/controller/msg.go index 59718e7b9..3cd706458 100644 --- a/pkg/common/storage/controller/msg.go +++ b/pkg/common/storage/controller/msg.go @@ -490,8 +490,8 @@ func (db *commonMsgDatabase) GetMsgBySeqs(ctx context.Context, userID string, co } successMsgs, failedSeqs, err := db.msg.GetMessagesBySeq(ctx, conversationID, newSeqs) if err != nil { - if errors.Is(err, redis.Nil) { - log.ZError(ctx, "get message from redis exception", err, "failedSeqs", failedSeqs, "conversationID", conversationID) + if !errors.Is(err, redis.Nil) { + log.ZWarn(ctx, "get message from redis exception", err, "failedSeqs", failedSeqs, "conversationID", conversationID) } } log.ZDebug(ctx, "db.seq.GetMessagesBySeq", "userID", userID, "conversationID", conversationID, "seqs", From 83ac4e83b98824f620c06b11a46bbdadee908e71 Mon Sep 17 00:00:00 2001 From: Monet Lee Date: Fri, 29 Nov 2024 16:59:24 +0800 Subject: [PATCH 056/199] fix: improve crontask delete outdated Data. (#2901) * fix: improve crontask delete outdated Data. * remove comments. * update go pkg * revert go pkg. * update gRPC Implement. * feat: support use config filter in deleteOutdatedData. * update go pkg. * update pkg. * comment GOProxy. --- Dockerfile | 2 +- config/openim-crontask.yml | 3 +- go.mod | 53 ++++++---- go.sum | 122 +++++++++++++--------- internal/msggateway/hub_server.go | 1 + internal/push/push.go | 1 + internal/rpc/auth/auth.go | 1 + internal/rpc/conversation/conversation.go | 1 + internal/rpc/group/group.go | 1 + internal/rpc/msg/server.go | 1 + internal/rpc/relation/friend.go | 1 + internal/rpc/third/s3.go | 89 +++++++++++----- internal/rpc/third/third.go | 1 + internal/rpc/user/user.go | 1 + internal/tools/cron_task.go | 57 ++++++---- pkg/common/config/config.go | 7 +- pkg/common/storage/controller/s3.go | 13 ++- pkg/common/storage/database/mgo/object.go | 37 ++++++- pkg/common/storage/database/object.go | 4 +- 19 files changed, 259 insertions(+), 137 deletions(-) diff --git a/Dockerfile b/Dockerfile index 4b38d711b..f6a2ee9fe 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,7 +8,7 @@ ENV SERVER_DIR=/openim-server WORKDIR $SERVER_DIR # Set the Go proxy to improve dependency resolution speed -ENV GOPROXY=https://goproxy.io,direct +# ENV GOPROXY=https://goproxy.io,direct # Copy all files from the current directory into the container COPY . . diff --git a/config/openim-crontask.yml b/config/openim-crontask.yml index c05bd2485..ff69c7dc8 100644 --- a/config/openim-crontask.yml +++ b/config/openim-crontask.yml @@ -1,3 +1,4 @@ cronExecuteTime: 0 2 * * * retainChatRecords: 365 -fileExpireTime: 90 +fileExpireTime: 180 +deleteObjectType: ["msg-picture","msg-file", "msg-voice","msg-video","msg-video-snapshot","sdklog"] \ No newline at end of file diff --git a/go.mod b/go.mod index 96c229166..a8dfa7891 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/openimsdk/open-im-server/v3 -go 1.22.0 +go 1.22.7 toolchain go1.23.2 @@ -14,15 +14,15 @@ require ( github.com/gorilla/websocket v1.5.1 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/mitchellh/mapstructure v1.5.0 - github.com/openimsdk/protocol v0.0.72-alpha.55 - github.com/openimsdk/tools v0.0.50-alpha.32 + github.com/openimsdk/protocol v0.0.72-alpha.57 + github.com/openimsdk/tools v0.0.50-alpha.37 github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 github.com/stretchr/testify v1.9.0 go.mongodb.org/mongo-driver v1.14.0 google.golang.org/api v0.170.0 - google.golang.org/grpc v1.66.2 - google.golang.org/protobuf v1.34.2 + google.golang.org/grpc v1.68.0 + google.golang.org/protobuf v1.35.1 gopkg.in/yaml.v3 v3.0.1 ) @@ -50,31 +50,33 @@ require ( require ( cloud.google.com/go v0.112.1 // indirect - cloud.google.com/go/compute/metadata v0.3.0 // indirect + cloud.google.com/go/compute/metadata v0.5.0 // indirect cloud.google.com/go/firestore v1.15.0 // indirect cloud.google.com/go/iam v1.1.7 // indirect cloud.google.com/go/longrunning v0.5.5 // indirect cloud.google.com/go/storage v1.40.0 // indirect + github.com/BurntSushi/toml v1.3.2 // indirect github.com/MicahParks/keyfunc v1.9.0 // indirect + github.com/alex-ant/gomath v0.0.0-20160516115720-89013a210a82 // indirect github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible // indirect - github.com/aws/aws-sdk-go-v2 v1.23.1 // indirect + github.com/aws/aws-sdk-go-v2 v1.32.5 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.1 // indirect - github.com/aws/aws-sdk-go-v2/config v1.25.4 // indirect - github.com/aws/aws-sdk-go-v2/credentials v1.16.3 // indirect - github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.5 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.4 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.4 // indirect - github.com/aws/aws-sdk-go-v2/internal/ini v1.7.1 // indirect + github.com/aws/aws-sdk-go-v2/config v1.28.5 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.17.46 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.20 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.24 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.24 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 // indirect github.com/aws/aws-sdk-go-v2/internal/v4a v1.2.4 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.1 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.1 // indirect github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.2.4 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.4 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.5 // indirect github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.4 // indirect github.com/aws/aws-sdk-go-v2/service/s3 v1.43.1 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.17.3 // indirect - github.com/aws/aws-sdk-go-v2/service/ssooidc v1.20.1 // indirect - github.com/aws/aws-sdk-go-v2/service/sts v1.25.4 // indirect - github.com/aws/smithy-go v1.17.0 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.24.6 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.5 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.33.1 // indirect + github.com/aws/smithy-go v1.22.1 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/bytedance/sonic v1.11.6 // indirect github.com/bytedance/sonic/loader v0.1.1 // indirect @@ -90,15 +92,19 @@ require ( github.com/eapache/go-resiliency v1.6.0 // indirect github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 // indirect github.com/eapache/queue v1.1.0 // indirect + github.com/elastic/go-sysinfo v1.0.2 // indirect + github.com/elastic/go-windows v1.0.0 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/gabriel-vasile/mimetype v1.4.3 // indirect + github.com/gammazero/toposort v0.1.1 // indirect github.com/gin-contrib/sse v0.1.0 // indirect github.com/go-logr/logr v1.4.2 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-ole/go-ole v1.2.6 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect github.com/go-zookeeper/zk v1.0.3 // indirect + github.com/gofrs/flock v0.8.1 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/golang/snappy v0.0.4 // indirect @@ -119,6 +125,7 @@ require ( github.com/jinzhu/copier v0.4.0 // indirect github.com/jinzhu/inflection v1.0.0 // indirect github.com/jinzhu/now v1.1.5 // indirect + github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/kelindar/simd v1.1.2 // indirect github.com/klauspost/compress v1.17.7 // indirect @@ -143,7 +150,7 @@ require ( github.com/prometheus/client_model v0.5.0 // indirect github.com/prometheus/common v0.45.0 // indirect github.com/prometheus/procfs v0.12.0 // indirect - github.com/qiniu/go-sdk/v7 v7.18.2 // indirect + github.com/qiniu/go-sdk/v7 v7.25.0 // indirect github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect github.com/rs/xid v1.5.0 // indirect github.com/sagikazarmark/locafero v0.4.0 // indirect @@ -177,15 +184,17 @@ require ( golang.org/x/arch v0.7.0 // indirect golang.org/x/image v0.15.0 // indirect golang.org/x/net v0.29.0 // indirect - golang.org/x/oauth2 v0.21.0 // indirect + golang.org/x/oauth2 v0.23.0 // indirect golang.org/x/sys v0.25.0 // indirect golang.org/x/text v0.18.0 // indirect golang.org/x/time v0.5.0 // indirect google.golang.org/appengine/v2 v2.0.2 // indirect google.golang.org/genproto v0.0.0-20240213162025-012b6fc9bca9 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20240604185151-ef581f913117 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20240903143218-8af14fe29dc1 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 // indirect gorm.io/gorm v1.25.8 // indirect + howett.net/plist v0.0.0-20181124034731-591f970eefbb // indirect + modernc.org/fileutil v1.0.0 // indirect stathat.com/c/consistent v1.0.0 // indirect ) diff --git a/go.sum b/go.sum index 36f3d1615..f13f71f5f 100644 --- a/go.sum +++ b/go.sum @@ -1,8 +1,8 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.112.1 h1:uJSeirPke5UNZHIb4SxfZklVSiWWVqW4oXlETwZziwM= cloud.google.com/go v0.112.1/go.mod h1:+Vbu+Y1UU+I1rjmzeMOb/8RfkKJK2Gyxi1X6jJCZLo4= -cloud.google.com/go/compute/metadata v0.3.0 h1:Tz+eQXMEqDIKRsmY3cHTL6FVaynIjX2QxYC4trgAKZc= -cloud.google.com/go/compute/metadata v0.3.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k= +cloud.google.com/go/compute/metadata v0.5.0 h1:Zr0eK8JbFv6+Wi4ilXAR8FJ3wyNdpxHKJNPos6LTZOY= +cloud.google.com/go/compute/metadata v0.5.0/go.mod h1:aHnloV2TPI38yx4s9+wAZhHykWvVCfu7hQbF+9CWoiY= cloud.google.com/go/firestore v1.15.0 h1:/k8ppuWOtNuDHt2tsRV42yI21uaGnKDEQnRFeBpbFF8= cloud.google.com/go/firestore v1.15.0/go.mod h1:GWOxFXcv8GZUtYpWHw/w6IuYNux/BtmeVTMmjrm4yhk= cloud.google.com/go/iam v1.1.7 h1:z4VHOhwKLF/+UYXAJDFwGtNF0b6gjsW1Pk9Ml0U/IoM= @@ -14,49 +14,53 @@ cloud.google.com/go/storage v1.40.0/go.mod h1:Rrj7/hKlG87BLqDJYtwR0fbPld8uJPbQ2u firebase.google.com/go/v4 v4.14.1 h1:4qiUETaFRWoFGE1XP5VbcEdtPX93Qs+8B/7KvP2825g= firebase.google.com/go/v4 v4.14.1/go.mod h1:fgk2XshgNDEKaioKco+AouiegSI9oTWVqRaBdTTGBoM= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/toml v1.3.2 h1:o7IhLm0Msx3BaB+n3Ag7L8EVlByGnpq14C4YWiu/gL8= +github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/IBM/sarama v1.43.0 h1:YFFDn8mMI2QL0wOrG0J2sFoVIAFl7hS9JQi2YZsXtJc= github.com/IBM/sarama v1.43.0/go.mod h1:zlE6HEbC/SMQ9mhEYaF7nNLYOUyrs0obySKCckWP9BM= github.com/MicahParks/keyfunc v1.9.0 h1:lhKd5xrFHLNOWrDc4Tyb/Q1AJ4LCzQ48GVJyVIID3+o= github.com/MicahParks/keyfunc v1.9.0/go.mod h1:IdnCilugA0O/99dW+/MkvlyrsX8+L8+x95xuVNtM5jw= github.com/QcloudApi/qcloud_sign_golang v0.0.0-20141224014652-e4130a326409/go.mod h1:1pk82RBxDY/JZnPQrtqHlUFfCctgdorsd9M06fMynOM= +github.com/alex-ant/gomath v0.0.0-20160516115720-89013a210a82 h1:7dONQ3WNZ1zy960TmkxJPuwoolZwL7xKtpcM04MBnt4= +github.com/alex-ant/gomath v0.0.0-20160516115720-89013a210a82/go.mod h1:nLnM0KdK1CmygvjpDUO6m1TjSsiQtL61juhNsvV/JVI= github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible h1:8psS8a+wKfiLt1iVDX79F7Y6wUM49Lcha2FMXt4UM8g= github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8= -github.com/aws/aws-sdk-go-v2 v1.23.1 h1:qXaFsOOMA+HsZtX8WoCa+gJnbyW7qyFFBlPqvTSzbaI= -github.com/aws/aws-sdk-go-v2 v1.23.1/go.mod h1:i1XDttT4rnf6vxc9AuskLc6s7XBee8rlLilKlc03uAA= +github.com/aws/aws-sdk-go-v2 v1.32.5 h1:U8vdWJuY7ruAkzaOdD7guwJjD06YSKmnKCJs7s3IkIo= +github.com/aws/aws-sdk-go-v2 v1.32.5/go.mod h1:P5WJBrYqqbWVaOxgH0X/FYYD47/nooaPOZPlQdmiN2U= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.1 h1:ZY3108YtBNq96jNZTICHxN1gSBSbnvIdYwwqnvCV4Mc= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.1/go.mod h1:t8PYl/6LzdAqsU4/9tz28V/kU+asFePvpOMkdul0gEQ= -github.com/aws/aws-sdk-go-v2/config v1.25.4 h1:r+X1x8QI6FEPdJDWCNBDZHyAcyFwSjHN8q8uuus+Axs= -github.com/aws/aws-sdk-go-v2/config v1.25.4/go.mod h1:8GTjImECskr7D88P/Nn9uM4M4rLY9i77hLJZgkZEWV8= -github.com/aws/aws-sdk-go-v2/credentials v1.16.3 h1:8PeI2krzzjDJ5etmgaMiD1JswsrLrWvKKu/uBUtNy1g= -github.com/aws/aws-sdk-go-v2/credentials v1.16.3/go.mod h1:Kdh/okh+//vQ/AjEt81CjvkTo64+/zIE4OewP7RpfXk= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.5 h1:KehRNiVzIfAcj6gw98zotVbb/K67taJE0fkfgM6vzqU= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.5/go.mod h1:VhnExhw6uXy9QzetvpXDolo1/hjhx4u9qukBGkuUwjs= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.4 h1:LAm3Ycm9HJfbSCd5I+wqC2S9Ej7FPrgr5CQoOljJZcE= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.4/go.mod h1:xEhvbJcyUf/31yfGSQBe01fukXwXJ0gxDp7rLfymWE0= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.4 h1:4GV0kKZzUxiWxSVpn/9gwR0g21NF1Jsyduzo9rHgC/Q= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.4/go.mod h1:dYvTNAggxDZy6y1AF7YDwXsPuHFy/VNEpEI/2dWK9IU= -github.com/aws/aws-sdk-go-v2/internal/ini v1.7.1 h1:uR9lXYjdPX0xY+NhvaJ4dD8rpSRz5VY81ccIIoNG+lw= -github.com/aws/aws-sdk-go-v2/internal/ini v1.7.1/go.mod h1:6fQQgfuGmw8Al/3M2IgIllycxV7ZW7WCdVSqfBeUiCY= +github.com/aws/aws-sdk-go-v2/config v1.28.5 h1:Za41twdCXbuyyWv9LndXxZZv3QhTG1DinqlFsSuvtI0= +github.com/aws/aws-sdk-go-v2/config v1.28.5/go.mod h1:4VsPbHP8JdcdUDmbTVgNL/8w9SqOkM5jyY8ljIxLO3o= +github.com/aws/aws-sdk-go-v2/credentials v1.17.46 h1:AU7RcriIo2lXjUfHFnFKYsLCwgbz1E7Mm95ieIRDNUg= +github.com/aws/aws-sdk-go-v2/credentials v1.17.46/go.mod h1:1FmYyLGL08KQXQ6mcTlifyFXfJVCNJTVGuQP4m0d/UA= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.20 h1:sDSXIrlsFSFJtWKLQS4PUWRvrT580rrnuLydJrCQ/yA= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.20/go.mod h1:WZ/c+w0ofps+/OUqMwWgnfrgzZH1DZO1RIkktICsqnY= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.24 h1:4usbeaes3yJnCFC7kfeyhkdkPtoRYPa/hTmCqMpKpLI= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.24/go.mod h1:5CI1JemjVwde8m2WG3cz23qHKPOxbpkq0HaoreEgLIY= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.24 h1:N1zsICrQglfzaBnrfM0Ys00860C+QFwu6u/5+LomP+o= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.24/go.mod h1:dCn9HbJ8+K31i8IQ8EWmWj0EiIk0+vKiHNMxTTYveAg= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 h1:VaRN3TlFdd6KxX1x3ILT5ynH6HvKgqdiXoTxAF4HQcQ= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1/go.mod h1:FbtygfRFze9usAadmnGJNc8KsP346kEe+y2/oyhGAGc= github.com/aws/aws-sdk-go-v2/internal/v4a v1.2.4 h1:40Q4X5ebZruRtknEZH/bg91sT5pR853F7/1X9QRbI54= github.com/aws/aws-sdk-go-v2/internal/v4a v1.2.4/go.mod h1:u77N7eEECzUv7F0xl2gcfK/vzc8wcjWobpy+DcrLJ5E= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.1 h1:rpkF4n0CyFcrJUG/rNNohoTmhtWlFTRI4BsZOh9PvLs= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.1/go.mod h1:l9ymW25HOqymeU2m1gbUQ3rUIsTwKs8gYHXkqDQUhiI= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.1 h1:iXtILhvDxB6kPvEXgsDhGaZCSC6LQET5ZHSdJozeI0Y= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.1/go.mod h1:9nu0fVANtYiAePIBh2/pFUSwtJ402hLnp854CNoDOeE= github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.2.4 h1:6DRKQc+9cChgzL5gplRGusI5dBGeiEod4m/pmGbcX48= github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.2.4/go.mod h1:s8ORvrW4g4v7IvYKIAoBg17w3GQ+XuwXDXYrQ5SkzU0= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.4 h1:rdovz3rEu0vZKbzoMYPTehp0E8veoE9AyfzqCr5Eeao= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.4/go.mod h1:aYCGNjyUCUelhofxlZyj63srdxWUSsBSGg5l6MCuXuE= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.5 h1:wtpJ4zcwrSbwhECWQoI/g6WM9zqCcSpHDJIWSbMLOu4= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.5/go.mod h1:qu/W9HXQbbQ4+1+JcZp0ZNPV31ym537ZJN+fiS7Ti8E= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.4 h1:o3DcfCxGDIT20pTbVKVhp3vWXOj/VvgazNJvumWeYW0= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.4/go.mod h1:Uy0KVOxuTK2ne+/PKQ+VvEeWmjMMksE17k/2RK/r5oM= github.com/aws/aws-sdk-go-v2/service/s3 v1.43.1 h1:1w11lfXOa8HoHoSlNtt4mqv/N3HmDOa+OnUH3Y9DHm8= github.com/aws/aws-sdk-go-v2/service/s3 v1.43.1/go.mod h1:dqJ5JBL0clzgHriH35Amx3LRFY6wNIPUX7QO/BerSBo= -github.com/aws/aws-sdk-go-v2/service/sso v1.17.3 h1:CdsSOGlFF3Pn+koXOIpTtvX7st0IuGsZ8kJqcWMlX54= -github.com/aws/aws-sdk-go-v2/service/sso v1.17.3/go.mod h1:oA6VjNsLll2eVuUoF2D+CMyORgNzPEW/3PyUdq6WQjI= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.20.1 h1:cbRqFTVnJV+KRpwFl76GJdIZJKKCdTPnjUZ7uWh3pIU= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.20.1/go.mod h1:hHL974p5auvXlZPIjJTblXJpbkfK4klBczlsEaMCGVY= -github.com/aws/aws-sdk-go-v2/service/sts v1.25.4 h1:yEvZ4neOQ/KpUqyR+X0ycUTW/kVRNR4nDZ38wStHGAA= -github.com/aws/aws-sdk-go-v2/service/sts v1.25.4/go.mod h1:feTnm2Tk/pJxdX+eooEsxvlvTWBvDm6CasRZ+JOs2IY= -github.com/aws/smithy-go v1.17.0 h1:wWJD7LX6PBV6etBUwO0zElG0nWN9rUhp0WdYeHSHAaI= -github.com/aws/smithy-go v1.17.0/go.mod h1:NukqUGpCZIILqqiV0NIjeFh24kd/FAa4beRb6nbIUPE= +github.com/aws/aws-sdk-go-v2/service/sso v1.24.6 h1:3zu537oLmsPfDMyjnUS2g+F2vITgy5pB74tHI+JBNoM= +github.com/aws/aws-sdk-go-v2/service/sso v1.24.6/go.mod h1:WJSZH2ZvepM6t6jwu4w/Z45Eoi75lPN7DcydSRtJg6Y= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.5 h1:K0OQAsDywb0ltlFrZm0JHPY3yZp/S9OaoLU33S7vPS8= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.5/go.mod h1:ORITg+fyuMoeiQFiVGoqB3OydVTLkClw/ljbblMq6Cc= +github.com/aws/aws-sdk-go-v2/service/sts v1.33.1 h1:6SZUVRQNvExYlMLbHdlKB48x0fLbc2iVROyaNEwBHbU= +github.com/aws/aws-sdk-go-v2/service/sts v1.33.1/go.mod h1:GqWyYCwLXnlUB1lOAXQyNSPqPLQJvmo8J0DWBzp9mtg= +github.com/aws/smithy-go v1.22.1 h1:/HPHZQ0g7f4eUeK6HKglFz8uwVfZKgoI25rb/J+dnro= +github.com/aws/smithy-go v1.22.1/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= @@ -86,6 +90,7 @@ github.com/coreos/go-systemd/v22 v22.3.2 h1:D9/bQk5vlXQFZ6Kwuu6zaiXJ9oTPe68++AzA github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/dave/jennifer v1.6.1/go.mod h1:nXbxhEmQfOZhWml3D1cDK5M1FLnMSozpbFN/m3RmGZc= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= @@ -103,6 +108,10 @@ github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 h1:Oy0F4A github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3/go.mod h1:YvSRo5mw33fLEx1+DlK6L2VV43tJt5Eyel9n9XBcR+0= github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc= github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= +github.com/elastic/go-sysinfo v1.0.2 h1:Wq1bOgnSz7Obl7DbMjbn0tzx1bE5G8Cfy3MVFa6C1Cc= +github.com/elastic/go-sysinfo v1.0.2/go.mod h1:O/D5m1VpYLwGjCYzEt63g3Z1uO3jXfwyzzjiW90t8cY= +github.com/elastic/go-windows v1.0.0 h1:qLURgZFkkrYyTTkvYpsZIgf83AUsdIHfvlJaqaZ7aSY= +github.com/elastic/go-windows v1.0.0/go.mod h1:TsU0Nrp7/y3+VwE82FoZF8gC/XFg/Elz6CcloAxnPgU= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= @@ -119,6 +128,8 @@ github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nos github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0= github.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk= +github.com/gammazero/toposort v0.1.1 h1:OivGxsWxF3U3+U80VoLJ+f50HcPU1MIqE1JlKzoJ2Eg= +github.com/gammazero/toposort v0.1.1/go.mod h1:H2cozTnNpMw0hg2VHAYsAxmkHXBYroNangj2NTBQDvw= github.com/gin-contrib/gzip v1.0.1 h1:HQ8ENHODeLY7a4g1Au/46Z92bdGFl74OhxcZble9WJE= github.com/gin-contrib/gzip v1.0.1/go.mod h1:njt428fdUNRvjuJf16tZMYZ2Yl+WQB53X5wmhDwXvC4= github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= @@ -143,7 +154,7 @@ github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+ github.com/go-playground/universal-translator v0.18.0/go.mod h1:UvRDBj+xPUEGrFYl+lu/H90nyDXpg0fqeB/AQUGNTVA= github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= -github.com/go-playground/validator/v10 v10.8.0/go.mod h1:9JhgTzTaE31GZDpH/HSvHiRJrJ3iKAgqqH0Bl/Ocjdk= +github.com/go-playground/validator/v10 v10.7.0/go.mod h1:xm76BBt941f7yWdGnI2DVPFFg1UK3YY04qifoXU3lOk= github.com/go-playground/validator/v10 v10.20.0 h1:K9ISHbSaI0lyB2eWMPJo+kOS/FBExVwjEviJTixqxL8= github.com/go-playground/validator/v10 v10.20.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM= github.com/go-redis/redis v6.15.9+incompatible h1:K0pv1D7EQUjfyoMql+r/jZqCLizCGKFlFgcHWWmHQjg= @@ -155,6 +166,8 @@ github.com/go-zookeeper/zk v1.0.3/go.mod h1:nOB03cncLtlp4t+UAkGSV+9beXP/akpekBwL github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/gofrs/flock v0.8.1 h1:+gYjHKf32LDeiEEFhQaotPbLuUXjY5ZqxKgXy7n59aw= +github.com/gofrs/flock v0.8.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang-jwt/jwt/v4 v4.4.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= @@ -204,6 +217,8 @@ github.com/googleapis/enterprise-certificate-proxy v0.3.2 h1:Vie5ybvEvT75RniqhfF github.com/googleapis/enterprise-certificate-proxy v0.3.2/go.mod h1:VLSiSSBs/ksPL8kq3OBOQ6WRI2QnaFynd1DCjZ62+V0= github.com/googleapis/gax-go/v2 v2.12.3 h1:5/zPPDvw8Q1SuXjrqrZslrqT7dL/uJT2CQii/cLCKqA= github.com/googleapis/gax-go/v2 v2.12.3/go.mod h1:AKloxT6GtNbaLm8QTNSidHUVsHYcBHwWRvkNFJUQcS4= +github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY= +github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ= github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4= github.com/gorilla/sessions v1.2.1/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM= github.com/gorilla/websocket v1.5.1 h1:gmztn0JnHVt9JZquRuzLw3g4wouNVzKL15iLr/zn/QY= @@ -222,6 +237,7 @@ github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/iancoleman/strcase v0.3.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/jcmturner/aescts/v2 v2.0.0 h1:9YKLH6ey7H4eDBXW8khjYslgyqG2xZikXP0EQFKrle8= @@ -236,12 +252,15 @@ github.com/jcmturner/gokrb5/v8 v8.4.4 h1:x1Sv4HaTpepFkXbt2IkL29DXRf8sOfZXo8eRKh6 github.com/jcmturner/gokrb5/v8 v8.4.4/go.mod h1:1btQEpgT6k+unzCwX1KdWMEwPPkkgBtP+F6aCACiMrs= github.com/jcmturner/rpc/v2 v2.0.3 h1:7FXXj8Ti1IaVFpSAziCZWNzbNuZmnvw/i6CqLNdWfZY= github.com/jcmturner/rpc/v2 v2.0.3/go.mod h1:VUJYCIDm3PVOEHw8sgt091/20OJjskO/YJki3ELg/Hc= +github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jinzhu/copier v0.4.0 h1:w3ciUoD19shMCRargcpm0cm91ytaBhDvuRpz1ODO/U8= github.com/jinzhu/copier v0.4.0/go.mod h1:DfbEm0FYsaqBcKcFuvmOZb218JkPGtvSHsKg8S8hyyg= github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= +github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901 h1:rp+c0RAYOWj8l6qbCUTSiRLG/iKnW3K3/QfPPuSsBt4= +github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901/go.mod h1:Z86h9688Y0wesXCyonoVr47MasHilkuLMqGhRZ4Hpak= github.com/jonboulle/clockwork v0.4.0 h1:p4Cf1aMWXnXAUh8lVfewRBx1zaTSYKrKMF2g3ST4RZ4= github.com/jonboulle/clockwork v0.4.0/go.mod h1:xgRqUGwRcjKCO1vbZUEtSLrqKoPSsUpK7fnezOII0kc= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= @@ -268,6 +287,7 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY= github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= @@ -319,15 +339,16 @@ github.com/onsi/gomega v1.25.0 h1:Vw7br2PCDYijJHSfBOWhov+8cAnUf8MfMaIOV323l6Y= github.com/onsi/gomega v1.25.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdMPGhM= github.com/openimsdk/gomake v0.0.14-alpha.5 h1:VY9c5x515lTfmdhhPjMvR3BBRrRquAUCFsz7t7vbv7Y= github.com/openimsdk/gomake v0.0.14-alpha.5/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= -github.com/openimsdk/protocol v0.0.72-alpha.55 h1:9PPWPHvkFk3neBSbNr+IoOdKIFjxTvEqUfMK/TEq1+8= -github.com/openimsdk/protocol v0.0.72-alpha.55/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= -github.com/openimsdk/tools v0.0.50-alpha.32 h1:JEsUFHFnaYg230TG+Ke3SUnaA2h44t4kABAzEdv5VZw= -github.com/openimsdk/tools v0.0.50-alpha.32/go.mod h1:r5U6RbxcR4xhKb2fhTmKGC9Yt5LcErHBVt3lhXQIHSo= +github.com/openimsdk/protocol v0.0.72-alpha.57 h1:oAVg0SJkDK15L8yDrL0KPG32f3iB/vjEpfpX577p5n4= +github.com/openimsdk/protocol v0.0.72-alpha.57/go.mod h1:Iet+piS/jaS+kWWyj6EEr36mk4ISzIRYjoMSVA4dq2M= +github.com/openimsdk/tools v0.0.50-alpha.37 h1:1souET6XA8uQ9BrLVWkP2pVGOYzATYE2Ig1pC6Kp9Hc= +github.com/openimsdk/tools v0.0.50-alpha.37/go.mod h1:FJCH5AvOzqHXdxe1Lrm7v3aVppAPSeVir3LwDAevF5A= github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= @@ -342,11 +363,12 @@ github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cY github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI= github.com/prometheus/common v0.45.0 h1:2BGz0eBc2hdMDLnO/8n0jeB3oPrt2D08CekT0lneoxM= github.com/prometheus/common v0.45.0/go.mod h1:YJmSTw9BoKxJplESWWxlbyttQR4uaEcGyv9MZjVOJsY= +github.com/prometheus/procfs v0.0.0-20190425082905-87a4384529e0/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= github.com/qiniu/dyn v1.3.0/go.mod h1:E8oERcm8TtwJiZvkQPbcAh0RL8jO1G0VXJMW3FAWdkk= -github.com/qiniu/go-sdk/v7 v7.18.2 h1:vk9eo5OO7aqgAOPF0Ytik/gt7CMKuNgzC/IPkhda6rk= -github.com/qiniu/go-sdk/v7 v7.18.2/go.mod h1:nqoYCNo53ZlGA521RvRethvxUDvXKt4gtYXOwye868w= +github.com/qiniu/go-sdk/v7 v7.25.0 h1:Roi4XMxRly9K4wb87DhQOKaQylyiphEXC7/l8uqJZaQ= +github.com/qiniu/go-sdk/v7 v7.25.0/go.mod h1:uZE85Pi0ftIHT/UNLShosdzwsovqpdas0LwAGO7cPao= github.com/qiniu/x v1.10.5/go.mod h1:03Ni9tj+N2h2aKnAz+6N0Xfl8FwMEDRC2PAlxekASDs= github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:N/ElC8H3+5XpJzTSTfLsJV/mx9Q9g7kxmchpfZyxgzM= github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= @@ -461,9 +483,7 @@ golang.org/x/arch v0.7.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= golang.org/x/crypto v0.27.0 h1:GXm2NjJrPaiv/h1tb2UH8QfgC/hOf/+z0p6PT8o1w7A= golang.org/x/crypto v0.27.0/go.mod h1:1Xngt8kV6Dvbssa53Ziq6Eqn0HqbZi5Z6R0ZpwQzt70= @@ -491,16 +511,16 @@ golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwY golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20220708220712-1185a9018129/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.29.0 h1:5ORfpBpCs4HzDYoodCDBbwHzdR5UrLBZ3sOnUJmFoHo= golang.org/x/net v0.29.0/go.mod h1:gLkgy8jTGERgjzMic6DS9+SP0ajcu6Xu3Orq/SpETg0= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.21.0 h1:tsimM75w1tF/uws5rbeHzIWxEqElMehnc+iW793zsZs= -golang.org/x/oauth2 v0.21.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= +golang.org/x/oauth2 v0.23.0 h1:PbgcYx2W7i4LvjJWEbf0ngHV6qJYr86PkAV3bXdLEbs= +golang.org/x/oauth2 v0.23.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -510,22 +530,20 @@ golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190425145619-16072639606e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34= golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= @@ -533,7 +551,6 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= -golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224= golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= @@ -565,8 +582,8 @@ google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98 google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= google.golang.org/genproto v0.0.0-20240213162025-012b6fc9bca9 h1:9+tzLLstTlPTRyJTh+ah5wIMsBW5c4tQwGTN3thOW9Y= google.golang.org/genproto v0.0.0-20240213162025-012b6fc9bca9/go.mod h1:mqHbVIp48Muh7Ywss/AD6I5kNVKZMmAa/QEW58Gxp2s= -google.golang.org/genproto/googleapis/api v0.0.0-20240604185151-ef581f913117 h1:+rdxYoE3E5htTEWIe15GlN6IfvbURM//Jt0mmkmm6ZU= -google.golang.org/genproto/googleapis/api v0.0.0-20240604185151-ef581f913117/go.mod h1:OimBR/bc1wPO9iV4NC2bpyjy3VnAwZh5EBPQdtaE5oo= +google.golang.org/genproto/googleapis/api v0.0.0-20240903143218-8af14fe29dc1 h1:hjSy6tcFQZ171igDaN5QHOw2n6vx40juYbC/x67CEhc= +google.golang.org/genproto/googleapis/api v0.0.0-20240903143218-8af14fe29dc1/go.mod h1:qpvKtACPCQhAdu3PyQgV4l3LMXZEtft7y8QcarRsp9I= google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 h1:pPJltXNxVzT4pK9yD8vR9X75DaWYYmLGMsEvBfFQZzQ= google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1/go.mod h1:UqMtugtsSgubUsoxbuAoiCXvqvErP7Gf0so0mK9tHxU= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= @@ -574,8 +591,8 @@ google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyac google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= -google.golang.org/grpc v1.66.2 h1:3QdXkuq3Bkh7w+ywLdLvM56cmGvQHUMZpiCzt6Rqaoo= -google.golang.org/grpc v1.66.2/go.mod h1:s3/l6xSSCURdVfAnL+TqCNMyTDAGN6+lZeVxnZR128Y= +google.golang.org/grpc v1.68.0 h1:aHQeeJbo8zAkAa3pRzrVjZlbz6uSfeOXlJNQM0RAbz0= +google.golang.org/grpc v1.68.0/go.mod h1:fmSPC5AsjSBCK54MyHRx48kpOti1/jRfOlwEWywNjWA= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -585,8 +602,8 @@ google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2 google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= -google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= -google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= +google.golang.org/protobuf v1.35.1 h1:m3LfL6/Ca+fqnjnlqQXNpFPABW1UD7mjh8KO2mKFytA= +google.golang.org/protobuf v1.35.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= @@ -596,6 +613,7 @@ gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= @@ -607,6 +625,10 @@ gorm.io/gorm v1.25.8 h1:WAGEZ/aEcznN4D03laj8DKnehe1e9gYQAjW8xyPRdeo= gorm.io/gorm v1.25.8/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +howett.net/plist v0.0.0-20181124034731-591f970eefbb h1:jhnBjNi9UFpfpl8YZhA9CrOqpnJdvzuiHsl/dnxl11M= +howett.net/plist v0.0.0-20181124034731-591f970eefbb/go.mod h1:vMygbs4qMhSZSc4lCUl2OEE+rDiIIJAIdR4m7MiMcm0= +modernc.org/fileutil v1.0.0 h1:Z1AFLZwl6BO8A5NldQg/xTSjGLetp+1Ubvl4alfGx8w= +modernc.org/fileutil v1.0.0/go.mod h1:JHsWpkrk/CnVV1H/eGlFf85BEpfkrp56ro8nojIq9Q8= nullprogram.com/x/optparse v1.0.0/go.mod h1:KdyPE+Igbe0jQUrVfMqDMeJQIJZEuyV7pjYmp6pbG50= rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= stathat.com/c/consistent v1.0.0 h1:ezyc51EGcRPJUxfHGSgJjWzJdj3NiMU9pNfLNGiXV0c= diff --git a/internal/msggateway/hub_server.go b/internal/msggateway/hub_server.go index 23d915013..8eff32899 100644 --- a/internal/msggateway/hub_server.go +++ b/internal/msggateway/hub_server.go @@ -56,6 +56,7 @@ func (s *Server) Start(ctx context.Context, index int, conf *Config) error { } type Server struct { + msggateway.UnimplementedMsgGatewayServer rpcPort int LongConnServer LongConnServer config *Config diff --git a/internal/push/push.go b/internal/push/push.go index 850f91d22..7f14bced7 100644 --- a/internal/push/push.go +++ b/internal/push/push.go @@ -14,6 +14,7 @@ import ( ) type pushServer struct { + pbpush.UnimplementedPushMsgServiceServer database controller.PushDatabase disCov discovery.SvcDiscoveryRegistry offlinePusher offlinepush.OfflinePusher diff --git a/internal/rpc/auth/auth.go b/internal/rpc/auth/auth.go index fe81bb45f..f0f4a022d 100644 --- a/internal/rpc/auth/auth.go +++ b/internal/rpc/auth/auth.go @@ -40,6 +40,7 @@ import ( ) type authServer struct { + pbauth.UnimplementedAuthServer authDatabase controller.AuthDatabase userRpcClient *rpcclient.UserRpcClient RegisterCenter discovery.SvcDiscoveryRegistry diff --git a/internal/rpc/conversation/conversation.go b/internal/rpc/conversation/conversation.go index 0c8a6fd85..ce720e640 100644 --- a/internal/rpc/conversation/conversation.go +++ b/internal/rpc/conversation/conversation.go @@ -43,6 +43,7 @@ import ( ) type conversationServer struct { + pbconversation.UnimplementedConversationServer msgRpcClient *rpcclient.MessageRpcClient user *rpcclient.UserRpcClient groupRpcClient *rpcclient.GroupRpcClient diff --git a/internal/rpc/group/group.go b/internal/rpc/group/group.go index b5ab1b209..9e610df0f 100644 --- a/internal/rpc/group/group.go +++ b/internal/rpc/group/group.go @@ -57,6 +57,7 @@ import ( ) type groupServer struct { + pbgroup.UnimplementedGroupServer db controller.GroupDatabase user rpcclient.UserRpcClient notification *GroupNotificationSender diff --git a/internal/rpc/msg/server.go b/internal/rpc/msg/server.go index bf8781747..6d5922ce3 100644 --- a/internal/rpc/msg/server.go +++ b/internal/rpc/msg/server.go @@ -55,6 +55,7 @@ type ( msgNotificationSender *MsgNotificationSender // RPC client for sending msg notifications. config *Config // Global configuration settings. webhookClient *webhook.Client + msg.UnimplementedMsgServer } Config struct { diff --git a/internal/rpc/relation/friend.go b/internal/rpc/relation/friend.go index 2f4843a8e..036c7aff5 100644 --- a/internal/rpc/relation/friend.go +++ b/internal/rpc/relation/friend.go @@ -43,6 +43,7 @@ import ( ) type friendServer struct { + relation.UnimplementedFriendServer db controller.FriendDatabase blackDatabase controller.BlackDatabase userRpcClient *rpcclient.UserRpcClient diff --git a/internal/rpc/third/s3.go b/internal/rpc/third/s3.go index fb6a1157e..b4b064d7f 100644 --- a/internal/rpc/third/s3.go +++ b/internal/rpc/third/s3.go @@ -290,48 +290,85 @@ func (t *thirdServer) apiAddress(prefix, name string) string { func (t *thirdServer) DeleteOutdatedData(ctx context.Context, req *third.DeleteOutdatedDataReq) (*third.DeleteOutdatedDataResp, error) { var conf config.Third expireTime := time.UnixMilli(req.ExpireTime) - var deltotal int + findPagination := &sdkws.RequestPagination{ PageNumber: 1, - ShowNumber: 1000, + ShowNumber: 500, + } + + // Find all expired data in S3 database + total, models, err := t.s3dataBase.FindNeedDeleteObjectByDB(ctx, expireTime, req.ObjectGroup, findPagination) + if err != nil && errs.Unwrap(err) != mongo.ErrNoDocuments { + return nil, errs.Wrap(err) + } + + if total == 0 { + log.ZDebug(ctx, "Not have OutdatedData", "delete Total", total) + return &third.DeleteOutdatedDataResp{Count: int32(total)}, nil } - for { - total, models, err := t.s3dataBase.FindByExpires(ctx, expireTime, findPagination) + + needDelObjectKeys := make([]string, len(models)) + for _, model := range models { + needDelObjectKeys = append(needDelObjectKeys, model.Key) + } + + // Remove duplicate keys, have the same key use in different models + needDelObjectKeys = datautil.Distinct(needDelObjectKeys) + + for _, key := range needDelObjectKeys { + // Find all models by key + keyModels, err := t.s3dataBase.FindModelsByKey(ctx, key) if err != nil && errs.Unwrap(err) != mongo.ErrNoDocuments { return nil, errs.Wrap(err) } - needDelObjectKeys := make([]string, 0) - for _, model := range models { - needDelObjectKeys = append(needDelObjectKeys, model.Key) + + // check keyModels, if all keyModels. + needDelKey := true // Default can delete + for _, keymodel := range keyModels { + // If group is empty or CreateTime is after expireTime, can't delete this key + if keymodel.Group == "" || keymodel.CreateTime.After(expireTime) { + needDelKey = false + break + } } - needDelObjectKeys = datautil.Distinct(needDelObjectKeys) - for _, key := range needDelObjectKeys { - count, err := t.s3dataBase.FindNotDelByS3(ctx, key, expireTime) - if err != nil && errs.Unwrap(err) != mongo.ErrNoDocuments { - return nil, errs.Wrap(err) + // If this object is not referenced by not expire data, delete it + if needDelKey && t.minio != nil { + // If have a thumbnail, delete it + thumbnailKey, _ := t.getMinioImageThumbnailKey(ctx, key) + if thumbnailKey != "" { + err := t.s3dataBase.DeleteObject(ctx, thumbnailKey) + if err != nil { + log.ZWarn(ctx, "Delete thumbnail object is error:", errs.Wrap(err), "thumbnailKey", thumbnailKey) + } } - if int(count) < 1 && t.minio != nil { - thumbnailKey, _ := t.getMinioImageThumbnailKey(ctx, key) - t.s3dataBase.DeleteObject(ctx, thumbnailKey) - t.s3dataBase.DelS3Key(ctx, conf.Object.Enable, needDelObjectKeys...) - t.s3dataBase.DeleteObject(ctx, key) + // Delete object + err = t.s3dataBase.DeleteObject(ctx, key) + if err != nil { + log.ZWarn(ctx, "Delete object is error", errs.Wrap(err), "object key", key) } - } - for _, model := range models { - err := t.s3dataBase.DeleteSpecifiedData(ctx, model.Engine, model.Name) + + // Delete cache key + err = t.s3dataBase.DelS3Key(ctx, conf.Object.Enable, key) if err != nil { - return nil, errs.Wrap(err) + log.ZWarn(ctx, "Delete cache key is error:", errs.Wrap(err), "cache S3 key:", key) } } - if total < int64(findPagination.ShowNumber) { - break + } + + // handle delete data in S3 database + for _, model := range models { + // Delete all expired data row in S3 database + err := t.s3dataBase.DeleteSpecifiedData(ctx, model.Engine, model.Name) + if err != nil { + return nil, errs.Wrap(err) } - deltotal += int(total) } - log.ZDebug(ctx, "DeleteOutdatedData", "delete Total", deltotal) - return &third.DeleteOutdatedDataResp{}, nil + + log.ZDebug(ctx, "DeleteOutdatedData", "delete Total", total) + + return &third.DeleteOutdatedDataResp{Count: int32(total)}, nil } type FormDataMate struct { diff --git a/internal/rpc/third/third.go b/internal/rpc/third/third.go index d37689b31..4206a2d6f 100644 --- a/internal/rpc/third/third.go +++ b/internal/rpc/third/third.go @@ -38,6 +38,7 @@ import ( ) type thirdServer struct { + third.UnimplementedThirdServer thirdDatabase controller.ThirdDatabase s3dataBase controller.S3Database userRpcClient rpcclient.UserRpcClient diff --git a/internal/rpc/user/user.go b/internal/rpc/user/user.go index b47c516d9..2dfbb01df 100644 --- a/internal/rpc/user/user.go +++ b/internal/rpc/user/user.go @@ -52,6 +52,7 @@ import ( ) type userServer struct { + pbuser.UnimplementedUserServer online cache.OnlineCache db controller.UserDatabase friendNotificationSender *relation.FriendNotificationSender diff --git a/internal/tools/cron_task.go b/internal/tools/cron_task.go index dbb4e34f6..484786880 100644 --- a/internal/tools/cron_task.go +++ b/internal/tools/cron_task.go @@ -24,6 +24,7 @@ import ( kdisc "github.com/openimsdk/open-im-server/v3/pkg/common/discoveryregister" pbconversation "github.com/openimsdk/protocol/conversation" "github.com/openimsdk/protocol/msg" + "github.com/openimsdk/protocol/third" "github.com/openimsdk/tools/mcontext" "github.com/openimsdk/tools/mw" @@ -58,10 +59,10 @@ func Start(ctx context.Context, config *CronTaskConfig) error { return err } - // thirdConn, err := client.GetConn(ctx, config.Share.RpcRegisterName.Third) - // if err != nil { - // return err - // } + thirdConn, err := client.GetConn(ctx, config.Share.RpcRegisterName.Third) + if err != nil { + return err + } conversationConn, err := client.GetConn(ctx, config.Share.RpcRegisterName.Conversation) if err != nil { @@ -70,7 +71,7 @@ func Start(ctx context.Context, config *CronTaskConfig) error { msgClient := msg.NewMsgClient(msgConn) conversationClient := pbconversation.NewConversationClient(conversationConn) - // thirdClient := third.NewThirdClient(thirdConn) + thirdClient := third.NewThirdClient(thirdConn) crontab := cron.New() @@ -114,21 +115,37 @@ func Start(ctx context.Context, config *CronTaskConfig) error { return errs.Wrap(err) } - // // scheduled delete outdated file Objects and their datas in specific time. - // deleteObjectFunc := func() { - // now := time.Now() - // deleteTime := now.Add(-time.Hour * 24 * time.Duration(config.CronTask.FileExpireTime)) - // ctx := mcontext.SetOperationID(ctx, fmt.Sprintf("cron_%d_%d", os.Getpid(), deleteTime.UnixMilli())) - // log.ZDebug(ctx, "deleteoutDatedData ", "deletetime", deleteTime, "timestamp", deleteTime.UnixMilli()) - // if _, err := thirdClient.DeleteOutdatedData(ctx, &third.DeleteOutdatedDataReq{ExpireTime: deleteTime.UnixMilli()}); err != nil { - // log.ZError(ctx, "cron deleteoutDatedData failed", err, "deleteTime", deleteTime, "cont", time.Since(now)) - // return - // } - // log.ZDebug(ctx, "cron deleteoutDatedData success", "deltime", deleteTime, "cont", time.Since(now)) - // } - // if _, err := crontab.AddFunc(config.CronTask.CronExecuteTime, deleteObjectFunc); err != nil { - // return errs.Wrap(err) - // } + // scheduled delete outdated file Objects and their datas in specific time. + deleteObjectFunc := func() { + now := time.Now() + executeNum := 5 + // number of pagination. if need modify, need update value in third.DeleteOutdatedData + pageShowNumber := 500 + deleteTime := now.Add(-time.Hour * 24 * time.Duration(config.CronTask.FileExpireTime)) + ctx := mcontext.SetOperationID(ctx, fmt.Sprintf("cron_%d_%d", os.Getpid(), deleteTime.UnixMilli())) + log.ZDebug(ctx, "deleteoutDatedData", "deletetime", deleteTime, "timestamp", deleteTime.UnixMilli()) + + if len(config.CronTask.DeleteObjectType) == 0 { + log.ZDebug(ctx, "cron deleteoutDatedData not type need delete", "deletetime", deleteTime, "DeleteObjectType", config.CronTask.DeleteObjectType, "cont", time.Since(now)) + return + } + + for i := 0; i < executeNum; i++ { + resp, err := thirdClient.DeleteOutdatedData(ctx, &third.DeleteOutdatedDataReq{ExpireTime: deleteTime.UnixMilli(), ObjectGroup: config.CronTask.DeleteObjectType}) + if err != nil { + log.ZError(ctx, "cron deleteoutDatedData failed", err, "deleteTime", deleteTime, "cont", time.Since(now)) + return + } + if resp.Count == 0 || resp.Count < int32(pageShowNumber) { + break + } + } + + log.ZDebug(ctx, "cron deleteoutDatedData success", "deltime", deleteTime, "cont", time.Since(now)) + } + if _, err := crontab.AddFunc(config.CronTask.CronExecuteTime, deleteObjectFunc); err != nil { + return errs.Wrap(err) + } log.ZDebug(ctx, "start cron task", "CronExecuteTime", config.CronTask.CronExecuteTime) crontab.Start() diff --git a/pkg/common/config/config.go b/pkg/common/config/config.go index 468a150e8..fa571e9f2 100644 --- a/pkg/common/config/config.go +++ b/pkg/common/config/config.go @@ -113,9 +113,10 @@ type API struct { } type CronTask struct { - CronExecuteTime string `mapstructure:"cronExecuteTime"` - RetainChatRecords int `mapstructure:"retainChatRecords"` - FileExpireTime int `mapstructure:"fileExpireTime"` + CronExecuteTime string `mapstructure:"cronExecuteTime"` + RetainChatRecords int `mapstructure:"retainChatRecords"` + FileExpireTime int `mapstructure:"fileExpireTime"` + DeleteObjectType []string `mapstructure:"deleteObjectType"` } type OfflinePushConfig struct { diff --git a/pkg/common/storage/controller/s3.go b/pkg/common/storage/controller/s3.go index 9b56661a5..4e5ad18b6 100644 --- a/pkg/common/storage/controller/s3.go +++ b/pkg/common/storage/controller/s3.go @@ -40,10 +40,10 @@ type S3Database interface { SetObject(ctx context.Context, info *model.Object) error StatObject(ctx context.Context, name string) (*s3.ObjectInfo, error) FormData(ctx context.Context, name string, size int64, contentType string, duration time.Duration) (*s3.FormData, error) - FindByExpires(ctx context.Context, duration time.Time, pagination pagination.Pagination) (total int64, objects []*model.Object, err error) + FindNeedDeleteObjectByDB(ctx context.Context, duration time.Time, needDelType []string, pagination pagination.Pagination) (total int64, objects []*model.Object, err error) DeleteObject(ctx context.Context, name string) error DeleteSpecifiedData(ctx context.Context, engine string, name string) error - FindNotDelByS3(ctx context.Context, key string, duration time.Time) (int64, error) + FindModelsByKey(ctx context.Context, key string) (objects []*model.Object, err error) DelS3Key(ctx context.Context, engine string, keys ...string) error } @@ -120,9 +120,8 @@ func (s *s3Database) StatObject(ctx context.Context, name string) (*s3.ObjectInf func (s *s3Database) FormData(ctx context.Context, name string, size int64, contentType string, duration time.Duration) (*s3.FormData, error) { return s.s3.FormData(ctx, name, size, contentType, duration) } -func (s *s3Database) FindByExpires(ctx context.Context, duration time.Time, pagination pagination.Pagination) (total int64, objects []*model.Object, err error) { - - return s.db.FindByExpires(ctx, duration, pagination) +func (s *s3Database) FindNeedDeleteObjectByDB(ctx context.Context, duration time.Time, needDelType []string, pagination pagination.Pagination) (total int64, objects []*model.Object, err error) { + return s.db.FindNeedDeleteObjectByDB(ctx, duration, needDelType, pagination) } func (s *s3Database) DeleteObject(ctx context.Context, name string) error { @@ -132,8 +131,8 @@ func (s *s3Database) DeleteSpecifiedData(ctx context.Context, engine string, nam return s.db.Delete(ctx, engine, name) } -func (s *s3Database) FindNotDelByS3(ctx context.Context, key string, duration time.Time) (int64, error) { - return s.db.FindNotDelByS3(ctx, key, duration) +func (s *s3Database) FindModelsByKey(ctx context.Context, key string) (objects []*model.Object, err error) { + return s.db.FindModelsByKey(ctx, key) } func (s *s3Database) DelS3Key(ctx context.Context, engine string, keys ...string) error { diff --git a/pkg/common/storage/database/mgo/object.go b/pkg/common/storage/database/mgo/object.go index 4242fbb53..5bc329d33 100644 --- a/pkg/common/storage/database/mgo/object.go +++ b/pkg/common/storage/database/mgo/object.go @@ -31,6 +31,8 @@ import ( func NewS3Mongo(db *mongo.Database) (database.ObjectInfo, error) { coll := db.Collection(database.ObjectName) + + // Create index for name _, err := coll.Indexes().CreateOne(context.Background(), mongo.IndexModel{ Keys: bson.D{ {Key: "name", Value: 1}, @@ -40,6 +42,27 @@ func NewS3Mongo(db *mongo.Database) (database.ObjectInfo, error) { if err != nil { return nil, errs.Wrap(err) } + + // Create index for create_time + _, err = coll.Indexes().CreateOne(context.Background(), mongo.IndexModel{ + Keys: bson.D{ + {Key: "create_time", Value: 1}, + }, + }) + if err != nil { + return nil, errs.Wrap(err) + } + + // Create index for key + _, err = coll.Indexes().CreateOne(context.Background(), mongo.IndexModel{ + Keys: bson.D{ + {Key: "key", Value: 1}, + }, + }) + if err != nil { + return nil, errs.Wrap(err) + } + return &S3Mongo{coll: coll}, nil } @@ -71,14 +94,18 @@ func (o *S3Mongo) Take(ctx context.Context, engine string, name string) (*model. func (o *S3Mongo) Delete(ctx context.Context, engine string, name string) error { return mongoutil.DeleteOne(ctx, o.coll, bson.M{"name": name, "engine": engine}) } -func (o *S3Mongo) FindByExpires(ctx context.Context, duration time.Time, pagination pagination.Pagination) (total int64, objects []*model.Object, err error) { + +// Find Expires object +func (o *S3Mongo) FindNeedDeleteObjectByDB(ctx context.Context, duration time.Time, needDelType []string, pagination pagination.Pagination) (total int64, objects []*model.Object, err error) { return mongoutil.FindPage[*model.Object](ctx, o.coll, bson.M{ "create_time": bson.M{"$lt": duration}, + "group": bson.M{"$in": needDelType}, }, pagination) } -func (o *S3Mongo) FindNotDelByS3(ctx context.Context, key string, duration time.Time) (int64, error) { - return mongoutil.Count(ctx, o.coll, bson.M{ - "key": key, - "create_time": bson.M{"$gt": duration}, + +// Find object by key +func (o *S3Mongo) FindModelsByKey(ctx context.Context, key string) (objects []*model.Object, err error) { + return mongoutil.Find[*model.Object](ctx, o.coll, bson.M{ + "key": key, }) } diff --git a/pkg/common/storage/database/object.go b/pkg/common/storage/database/object.go index 8292006a0..c741e39a6 100644 --- a/pkg/common/storage/database/object.go +++ b/pkg/common/storage/database/object.go @@ -26,6 +26,6 @@ type ObjectInfo interface { SetObject(ctx context.Context, obj *model.Object) error Take(ctx context.Context, engine string, name string) (*model.Object, error) Delete(ctx context.Context, engine string, name string) error - FindByExpires(ctx context.Context, duration time.Time, pagination pagination.Pagination) (total int64, objects []*model.Object, err error) - FindNotDelByS3(ctx context.Context, key string, duration time.Time) (int64, error) + FindNeedDeleteObjectByDB(ctx context.Context, duration time.Time, needDelType []string, pagination pagination.Pagination) (total int64, objects []*model.Object, err error) + FindModelsByKey(ctx context.Context, key string) (objects []*model.Object, err error) } From de57e2044689fe5398c38c5fcfe45b4bba2feffc Mon Sep 17 00:00:00 2001 From: icey-yu <119291641+icey-yu@users.noreply.github.com> Date: Mon, 2 Dec 2024 09:44:31 +0800 Subject: [PATCH 057/199] fix: go mod (#2906) --- go.mod | 13 ++----------- go.sum | 46 ++++++++++++---------------------------------- 2 files changed, 14 insertions(+), 45 deletions(-) diff --git a/go.mod b/go.mod index a8dfa7891..375c0406c 100644 --- a/go.mod +++ b/go.mod @@ -15,7 +15,7 @@ require ( github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/mitchellh/mapstructure v1.5.0 github.com/openimsdk/protocol v0.0.72-alpha.57 - github.com/openimsdk/tools v0.0.50-alpha.37 + github.com/openimsdk/tools v0.0.50-alpha.38 github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 github.com/stretchr/testify v1.9.0 @@ -55,9 +55,7 @@ require ( cloud.google.com/go/iam v1.1.7 // indirect cloud.google.com/go/longrunning v0.5.5 // indirect cloud.google.com/go/storage v1.40.0 // indirect - github.com/BurntSushi/toml v1.3.2 // indirect github.com/MicahParks/keyfunc v1.9.0 // indirect - github.com/alex-ant/gomath v0.0.0-20160516115720-89013a210a82 // indirect github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible // indirect github.com/aws/aws-sdk-go-v2 v1.32.5 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.1 // indirect @@ -92,19 +90,15 @@ require ( github.com/eapache/go-resiliency v1.6.0 // indirect github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 // indirect github.com/eapache/queue v1.1.0 // indirect - github.com/elastic/go-sysinfo v1.0.2 // indirect - github.com/elastic/go-windows v1.0.0 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/gabriel-vasile/mimetype v1.4.3 // indirect - github.com/gammazero/toposort v0.1.1 // indirect github.com/gin-contrib/sse v0.1.0 // indirect github.com/go-logr/logr v1.4.2 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-ole/go-ole v1.2.6 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect github.com/go-zookeeper/zk v1.0.3 // indirect - github.com/gofrs/flock v0.8.1 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/golang/snappy v0.0.4 // indirect @@ -125,7 +119,6 @@ require ( github.com/jinzhu/copier v0.4.0 // indirect github.com/jinzhu/inflection v1.0.0 // indirect github.com/jinzhu/now v1.1.5 // indirect - github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/kelindar/simd v1.1.2 // indirect github.com/klauspost/compress v1.17.7 // indirect @@ -150,7 +143,7 @@ require ( github.com/prometheus/client_model v0.5.0 // indirect github.com/prometheus/common v0.45.0 // indirect github.com/prometheus/procfs v0.12.0 // indirect - github.com/qiniu/go-sdk/v7 v7.25.0 // indirect + github.com/qiniu/go-sdk/v7 v7.18.2 // indirect github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect github.com/rs/xid v1.5.0 // indirect github.com/sagikazarmark/locafero v0.4.0 // indirect @@ -193,8 +186,6 @@ require ( google.golang.org/genproto/googleapis/api v0.0.0-20240903143218-8af14fe29dc1 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 // indirect gorm.io/gorm v1.25.8 // indirect - howett.net/plist v0.0.0-20181124034731-591f970eefbb // indirect - modernc.org/fileutil v1.0.0 // indirect stathat.com/c/consistent v1.0.0 // indirect ) diff --git a/go.sum b/go.sum index f13f71f5f..74ee1c2c6 100644 --- a/go.sum +++ b/go.sum @@ -14,15 +14,11 @@ cloud.google.com/go/storage v1.40.0/go.mod h1:Rrj7/hKlG87BLqDJYtwR0fbPld8uJPbQ2u firebase.google.com/go/v4 v4.14.1 h1:4qiUETaFRWoFGE1XP5VbcEdtPX93Qs+8B/7KvP2825g= firebase.google.com/go/v4 v4.14.1/go.mod h1:fgk2XshgNDEKaioKco+AouiegSI9oTWVqRaBdTTGBoM= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/BurntSushi/toml v1.3.2 h1:o7IhLm0Msx3BaB+n3Ag7L8EVlByGnpq14C4YWiu/gL8= -github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/IBM/sarama v1.43.0 h1:YFFDn8mMI2QL0wOrG0J2sFoVIAFl7hS9JQi2YZsXtJc= github.com/IBM/sarama v1.43.0/go.mod h1:zlE6HEbC/SMQ9mhEYaF7nNLYOUyrs0obySKCckWP9BM= github.com/MicahParks/keyfunc v1.9.0 h1:lhKd5xrFHLNOWrDc4Tyb/Q1AJ4LCzQ48GVJyVIID3+o= github.com/MicahParks/keyfunc v1.9.0/go.mod h1:IdnCilugA0O/99dW+/MkvlyrsX8+L8+x95xuVNtM5jw= github.com/QcloudApi/qcloud_sign_golang v0.0.0-20141224014652-e4130a326409/go.mod h1:1pk82RBxDY/JZnPQrtqHlUFfCctgdorsd9M06fMynOM= -github.com/alex-ant/gomath v0.0.0-20160516115720-89013a210a82 h1:7dONQ3WNZ1zy960TmkxJPuwoolZwL7xKtpcM04MBnt4= -github.com/alex-ant/gomath v0.0.0-20160516115720-89013a210a82/go.mod h1:nLnM0KdK1CmygvjpDUO6m1TjSsiQtL61juhNsvV/JVI= github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible h1:8psS8a+wKfiLt1iVDX79F7Y6wUM49Lcha2FMXt4UM8g= github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8= github.com/aws/aws-sdk-go-v2 v1.32.5 h1:U8vdWJuY7ruAkzaOdD7guwJjD06YSKmnKCJs7s3IkIo= @@ -90,7 +86,6 @@ github.com/coreos/go-systemd/v22 v22.3.2 h1:D9/bQk5vlXQFZ6Kwuu6zaiXJ9oTPe68++AzA github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/dave/jennifer v1.6.1/go.mod h1:nXbxhEmQfOZhWml3D1cDK5M1FLnMSozpbFN/m3RmGZc= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= @@ -108,10 +103,6 @@ github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 h1:Oy0F4A github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3/go.mod h1:YvSRo5mw33fLEx1+DlK6L2VV43tJt5Eyel9n9XBcR+0= github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc= github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= -github.com/elastic/go-sysinfo v1.0.2 h1:Wq1bOgnSz7Obl7DbMjbn0tzx1bE5G8Cfy3MVFa6C1Cc= -github.com/elastic/go-sysinfo v1.0.2/go.mod h1:O/D5m1VpYLwGjCYzEt63g3Z1uO3jXfwyzzjiW90t8cY= -github.com/elastic/go-windows v1.0.0 h1:qLURgZFkkrYyTTkvYpsZIgf83AUsdIHfvlJaqaZ7aSY= -github.com/elastic/go-windows v1.0.0/go.mod h1:TsU0Nrp7/y3+VwE82FoZF8gC/XFg/Elz6CcloAxnPgU= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= @@ -128,8 +119,6 @@ github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nos github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0= github.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk= -github.com/gammazero/toposort v0.1.1 h1:OivGxsWxF3U3+U80VoLJ+f50HcPU1MIqE1JlKzoJ2Eg= -github.com/gammazero/toposort v0.1.1/go.mod h1:H2cozTnNpMw0hg2VHAYsAxmkHXBYroNangj2NTBQDvw= github.com/gin-contrib/gzip v1.0.1 h1:HQ8ENHODeLY7a4g1Au/46Z92bdGFl74OhxcZble9WJE= github.com/gin-contrib/gzip v1.0.1/go.mod h1:njt428fdUNRvjuJf16tZMYZ2Yl+WQB53X5wmhDwXvC4= github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= @@ -154,7 +143,7 @@ github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+ github.com/go-playground/universal-translator v0.18.0/go.mod h1:UvRDBj+xPUEGrFYl+lu/H90nyDXpg0fqeB/AQUGNTVA= github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= -github.com/go-playground/validator/v10 v10.7.0/go.mod h1:xm76BBt941f7yWdGnI2DVPFFg1UK3YY04qifoXU3lOk= +github.com/go-playground/validator/v10 v10.8.0/go.mod h1:9JhgTzTaE31GZDpH/HSvHiRJrJ3iKAgqqH0Bl/Ocjdk= github.com/go-playground/validator/v10 v10.20.0 h1:K9ISHbSaI0lyB2eWMPJo+kOS/FBExVwjEviJTixqxL8= github.com/go-playground/validator/v10 v10.20.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM= github.com/go-redis/redis v6.15.9+incompatible h1:K0pv1D7EQUjfyoMql+r/jZqCLizCGKFlFgcHWWmHQjg= @@ -166,8 +155,6 @@ github.com/go-zookeeper/zk v1.0.3/go.mod h1:nOB03cncLtlp4t+UAkGSV+9beXP/akpekBwL github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= -github.com/gofrs/flock v0.8.1 h1:+gYjHKf32LDeiEEFhQaotPbLuUXjY5ZqxKgXy7n59aw= -github.com/gofrs/flock v0.8.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang-jwt/jwt/v4 v4.4.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= @@ -217,8 +204,6 @@ github.com/googleapis/enterprise-certificate-proxy v0.3.2 h1:Vie5ybvEvT75RniqhfF github.com/googleapis/enterprise-certificate-proxy v0.3.2/go.mod h1:VLSiSSBs/ksPL8kq3OBOQ6WRI2QnaFynd1DCjZ62+V0= github.com/googleapis/gax-go/v2 v2.12.3 h1:5/zPPDvw8Q1SuXjrqrZslrqT7dL/uJT2CQii/cLCKqA= github.com/googleapis/gax-go/v2 v2.12.3/go.mod h1:AKloxT6GtNbaLm8QTNSidHUVsHYcBHwWRvkNFJUQcS4= -github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY= -github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ= github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4= github.com/gorilla/sessions v1.2.1/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM= github.com/gorilla/websocket v1.5.1 h1:gmztn0JnHVt9JZquRuzLw3g4wouNVzKL15iLr/zn/QY= @@ -237,7 +222,6 @@ github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= -github.com/iancoleman/strcase v0.3.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/jcmturner/aescts/v2 v2.0.0 h1:9YKLH6ey7H4eDBXW8khjYslgyqG2xZikXP0EQFKrle8= @@ -252,15 +236,12 @@ github.com/jcmturner/gokrb5/v8 v8.4.4 h1:x1Sv4HaTpepFkXbt2IkL29DXRf8sOfZXo8eRKh6 github.com/jcmturner/gokrb5/v8 v8.4.4/go.mod h1:1btQEpgT6k+unzCwX1KdWMEwPPkkgBtP+F6aCACiMrs= github.com/jcmturner/rpc/v2 v2.0.3 h1:7FXXj8Ti1IaVFpSAziCZWNzbNuZmnvw/i6CqLNdWfZY= github.com/jcmturner/rpc/v2 v2.0.3/go.mod h1:VUJYCIDm3PVOEHw8sgt091/20OJjskO/YJki3ELg/Hc= -github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jinzhu/copier v0.4.0 h1:w3ciUoD19shMCRargcpm0cm91ytaBhDvuRpz1ODO/U8= github.com/jinzhu/copier v0.4.0/go.mod h1:DfbEm0FYsaqBcKcFuvmOZb218JkPGtvSHsKg8S8hyyg= github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= -github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901 h1:rp+c0RAYOWj8l6qbCUTSiRLG/iKnW3K3/QfPPuSsBt4= -github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901/go.mod h1:Z86h9688Y0wesXCyonoVr47MasHilkuLMqGhRZ4Hpak= github.com/jonboulle/clockwork v0.4.0 h1:p4Cf1aMWXnXAUh8lVfewRBx1zaTSYKrKMF2g3ST4RZ4= github.com/jonboulle/clockwork v0.4.0/go.mod h1:xgRqUGwRcjKCO1vbZUEtSLrqKoPSsUpK7fnezOII0kc= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= @@ -287,7 +268,6 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY= github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= @@ -341,14 +321,13 @@ github.com/openimsdk/gomake v0.0.14-alpha.5 h1:VY9c5x515lTfmdhhPjMvR3BBRrRquAUCF github.com/openimsdk/gomake v0.0.14-alpha.5/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= github.com/openimsdk/protocol v0.0.72-alpha.57 h1:oAVg0SJkDK15L8yDrL0KPG32f3iB/vjEpfpX577p5n4= github.com/openimsdk/protocol v0.0.72-alpha.57/go.mod h1:Iet+piS/jaS+kWWyj6EEr36mk4ISzIRYjoMSVA4dq2M= -github.com/openimsdk/tools v0.0.50-alpha.37 h1:1souET6XA8uQ9BrLVWkP2pVGOYzATYE2Ig1pC6Kp9Hc= -github.com/openimsdk/tools v0.0.50-alpha.37/go.mod h1:FJCH5AvOzqHXdxe1Lrm7v3aVppAPSeVir3LwDAevF5A= +github.com/openimsdk/tools v0.0.50-alpha.38 h1:AU6/cvDfN4ciIOwAj8IWEwze3DeEp2cHYPgW3y0OlbU= +github.com/openimsdk/tools v0.0.50-alpha.38/go.mod h1:/Em/fQH46CuWf60+hcmvZyboGCQpSDEb2MdQ4nmQRAk= github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= -github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= @@ -363,12 +342,11 @@ github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cY github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI= github.com/prometheus/common v0.45.0 h1:2BGz0eBc2hdMDLnO/8n0jeB3oPrt2D08CekT0lneoxM= github.com/prometheus/common v0.45.0/go.mod h1:YJmSTw9BoKxJplESWWxlbyttQR4uaEcGyv9MZjVOJsY= -github.com/prometheus/procfs v0.0.0-20190425082905-87a4384529e0/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= github.com/qiniu/dyn v1.3.0/go.mod h1:E8oERcm8TtwJiZvkQPbcAh0RL8jO1G0VXJMW3FAWdkk= -github.com/qiniu/go-sdk/v7 v7.25.0 h1:Roi4XMxRly9K4wb87DhQOKaQylyiphEXC7/l8uqJZaQ= -github.com/qiniu/go-sdk/v7 v7.25.0/go.mod h1:uZE85Pi0ftIHT/UNLShosdzwsovqpdas0LwAGO7cPao= +github.com/qiniu/go-sdk/v7 v7.18.2 h1:vk9eo5OO7aqgAOPF0Ytik/gt7CMKuNgzC/IPkhda6rk= +github.com/qiniu/go-sdk/v7 v7.18.2/go.mod h1:nqoYCNo53ZlGA521RvRethvxUDvXKt4gtYXOwye868w= github.com/qiniu/x v1.10.5/go.mod h1:03Ni9tj+N2h2aKnAz+6N0Xfl8FwMEDRC2PAlxekASDs= github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:N/ElC8H3+5XpJzTSTfLsJV/mx9Q9g7kxmchpfZyxgzM= github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= @@ -483,7 +461,9 @@ golang.org/x/arch v0.7.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= golang.org/x/crypto v0.27.0 h1:GXm2NjJrPaiv/h1tb2UH8QfgC/hOf/+z0p6PT8o1w7A= golang.org/x/crypto v0.27.0/go.mod h1:1Xngt8kV6Dvbssa53Ziq6Eqn0HqbZi5Z6R0ZpwQzt70= @@ -511,6 +491,7 @@ golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwY golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20220708220712-1185a9018129/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.29.0 h1:5ORfpBpCs4HzDYoodCDBbwHzdR5UrLBZ3sOnUJmFoHo= @@ -520,7 +501,6 @@ golang.org/x/oauth2 v0.23.0 h1:PbgcYx2W7i4LvjJWEbf0ngHV6qJYr86PkAV3bXdLEbs= golang.org/x/oauth2 v0.23.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -530,20 +510,22 @@ golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190425145619-16072639606e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34= golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= @@ -551,6 +533,7 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= +golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224= golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= @@ -613,7 +596,6 @@ gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= -gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= @@ -625,10 +607,6 @@ gorm.io/gorm v1.25.8 h1:WAGEZ/aEcznN4D03laj8DKnehe1e9gYQAjW8xyPRdeo= gorm.io/gorm v1.25.8/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -howett.net/plist v0.0.0-20181124034731-591f970eefbb h1:jhnBjNi9UFpfpl8YZhA9CrOqpnJdvzuiHsl/dnxl11M= -howett.net/plist v0.0.0-20181124034731-591f970eefbb/go.mod h1:vMygbs4qMhSZSc4lCUl2OEE+rDiIIJAIdR4m7MiMcm0= -modernc.org/fileutil v1.0.0 h1:Z1AFLZwl6BO8A5NldQg/xTSjGLetp+1Ubvl4alfGx8w= -modernc.org/fileutil v1.0.0/go.mod h1:JHsWpkrk/CnVV1H/eGlFf85BEpfkrp56ro8nojIq9Q8= nullprogram.com/x/optparse v1.0.0/go.mod h1:KdyPE+Igbe0jQUrVfMqDMeJQIJZEuyV7pjYmp6pbG50= rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= stathat.com/c/consistent v1.0.0 h1:ezyc51EGcRPJUxfHGSgJjWzJdj3NiMU9pNfLNGiXV0c= From 1534575be5943a13680b405347f042c17a55162b Mon Sep 17 00:00:00 2001 From: chao <48119764+withchao@users.noreply.github.com> Date: Mon, 2 Dec 2024 16:15:22 +0800 Subject: [PATCH 058/199] fix: group member update face_url (#2910) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * update gomake version * update gomake version * fix: seq conversion bug * fix: redis pipe exec * fix: ImportFriends * fix: A large number of logs keysAndValues ​​length is not even * feat: mark read aggregate write * feat: online status supports redis cluster * feat: online status supports redis cluster * feat: online status supports redis cluster * merge * merge * read seq is written to mongo * read seq is written to mongo * fix: invitation to join group notification * fix: friend op_user_id * feat: optimizing asynchronous context * feat: optimizing memamq size * feat: add GetSeqMessage * feat: implement no gob encoder. * update unitTest content. * Update hub_server.go * feat: GroupApplicationAgreeMemberEnterNotification * fix: encoder replace to json encoder. * feat: GroupApplicationAgreeMemberEnterNotification * feat: go.mod * feat: go.mod * feat: join group notification and get seq * feat: join group notification and get seq * feat: avoid pulling messages from sessions with a large number of max seq values of 0 * feat: API supports gzip * go.mod * fix: nil pointer error on close * fix: listen error * fix: listen error * update go.mod * feat: add log * fix: token parse token value * fix: GetMsgBySeqs boundary issues * merge: merge main code into js branch. (#2648) * feat: update group notification when set to null. (#2590) * refactor: refactor workflows contents. * add tool workflows. * update field. * fix: remove chat error. * Fix err. * fix error. * remove cn comment. * update workflows files. * update infra config. * move workflows. * feat: update bot. * fix: solve uncorrect outdated msg get. * update get docIDs logic. * update * update skip logic. * fix * update. * fix: delay deleteObject func. * remove unused content. * feat: update group notification when set to null. * update log standard. * feat: add long time push msg in prometheus (#2584) * feat: add long time push msg in prometheus * fix: log print * fix: go mod * fix: log msg * fix: log init * feat: push msg * feat: go mod ,remove cgo package * feat: remove error log * feat: test dummy push * feat:redis pool config * feat: push to kafka log * feat: supports getting messages based on session ID and seq (#2582) * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * update gomake version * update gomake version * fix: seq conversion bug * fix: redis pipe exec * fix: ImportFriends * fix: A large number of logs keysAndValues ​​length is not even * feat: mark read aggregate write * feat: online status supports redis cluster * feat: online status supports redis cluster * feat: online status supports redis cluster * merge * merge * read seq is written to mongo * read seq is written to mongo * fix: invitation to join group notification * fix: friend op_user_id * feat: optimizing asynchronous context * feat: optimizing memamq size * feat: add GetSeqMessage * feat: GroupApplicationAgreeMemberEnterNotification * feat: GroupApplicationAgreeMemberEnterNotification * feat: go.mod * feat: go.mod * feat: join group notification and get seq --------- Co-authored-by: withchao * feat: implement request batch count limit. (#2591) * refactor: refactor workflows contents. * add tool workflows. * update field. * fix: remove chat error. * Fix err. * fix error. * remove cn comment. * update workflows files. * update infra config. * move workflows. * feat: update bot. * fix: solve uncorrect outdated msg get. * update get docIDs logic. * update * update skip logic. * fix * update. * fix: delay deleteObject func. * remove unused content. * update log type. * feat: implement request batch count limit. * update * update * fix: getting messages based on session ID and seq (#2595) * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * update gomake version * update gomake version * fix: seq conversion bug * fix: redis pipe exec * fix: ImportFriends * fix: A large number of logs keysAndValues ​​length is not even * feat: mark read aggregate write * feat: online status supports redis cluster * feat: online status supports redis cluster * feat: online status supports redis cluster * merge * merge * read seq is written to mongo * read seq is written to mongo * fix: invitation to join group notification * fix: friend op_user_id * feat: optimizing asynchronous context * feat: optimizing memamq size * feat: add GetSeqMessage * feat: GroupApplicationAgreeMemberEnterNotification * feat: GroupApplicationAgreeMemberEnterNotification * feat: go.mod * feat: go.mod * feat: join group notification and get seq * feat: join group notification and get seq --------- Co-authored-by: withchao * feat: avoid pulling messages from sessions with a large number of max seq values of 0 (#2602) * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * update gomake version * update gomake version * fix: seq conversion bug * fix: redis pipe exec * fix: ImportFriends * fix: A large number of logs keysAndValues ​​length is not even * feat: mark read aggregate write * feat: online status supports redis cluster * feat: online status supports redis cluster * feat: online status supports redis cluster * merge * merge * read seq is written to mongo * read seq is written to mongo * fix: invitation to join group notification * fix: friend op_user_id * feat: optimizing asynchronous context * feat: optimizing memamq size * feat: add GetSeqMessage * feat: GroupApplicationAgreeMemberEnterNotification * feat: GroupApplicationAgreeMemberEnterNotification * feat: go.mod * feat: go.mod * feat: join group notification and get seq * feat: join group notification and get seq * feat: avoid pulling messages from sessions with a large number of max seq values of 0 --------- Co-authored-by: withchao * refactor: improve db structure in `storage/controller` (#2604) * refactor: refactor workflows contents. * add tool workflows. * update field. * fix: remove chat error. * Fix err. * fix error. * remove cn comment. * update workflows files. * update infra config. * move workflows. * feat: update bot. * fix: solve uncorrect outdated msg get. * update get docIDs logic. * update * update skip logic. * fix * update. * fix: delay deleteObject func. * remove unused content. * update log type. * feat: implement request batch count limit. * update * update * refactor: improve db structure in `storage/controller` * feat: implement offline push using kafka (#2600) * refactor: refactor workflows contents. * add tool workflows. * update field. * fix: remove chat error. * Fix err. * fix error. * remove cn comment. * update workflows files. * update infra config. * move workflows. * feat: update bot. * fix: solve uncorrect outdated msg get. * update get docIDs logic. * update * update skip logic. * fix * update. * fix: delay deleteObject func. * remove unused content. * update log type. * feat: implement request batch count limit. * update * update * feat: implement offline push. * feat: implement batch Push spilt * update go mod * feat: implement kafka producer and consumer. * update format, * add PushMQ log. * feat: update Handler logic. * update MQ logic. * update * update * fix: update OfflinePushConsumerHandler. * feat: API supports gzip (#2609) * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * update gomake version * update gomake version * fix: seq conversion bug * fix: redis pipe exec * fix: ImportFriends * fix: A large number of logs keysAndValues ​​length is not even * feat: mark read aggregate write * feat: online status supports redis cluster * feat: online status supports redis cluster * feat: online status supports redis cluster * merge * merge * read seq is written to mongo * read seq is written to mongo * fix: invitation to join group notification * fix: friend op_user_id * feat: optimizing asynchronous context * feat: optimizing memamq size * feat: add GetSeqMessage * feat: GroupApplicationAgreeMemberEnterNotification * feat: GroupApplicationAgreeMemberEnterNotification * feat: go.mod * feat: go.mod * feat: join group notification and get seq * feat: join group notification and get seq * feat: avoid pulling messages from sessions with a large number of max seq values of 0 * feat: API supports gzip --------- Co-authored-by: withchao * Fix err (#2608) * refactor: refactor workflows contents. * add tool workflows. * update field. * fix: remove chat error. * Fix err. * fix error. * remove cn comment. * update workflows files. * update infra config. * move workflows. * feat: update bot. * fix: solve uncorrect outdated msg get. * update get docIDs logic. * update * update skip logic. * fix * update. * fix: delay deleteObject func. * remove unused content. * update log type. * feat: implement request batch count limit. * update * update * feat: add rocksTimeout * feat: wrap logs * feat: add logs * feat: listen config * feat: enable listen TIME_WAIT port * feat: add logs * feat: cache batch * chore: enable fullUserCache * feat: push rpc num * feat: push err * feat: with operationID * feat: sleep * feat: change 1s * feat: change log * feat: implement Getbatch in rpcCache. * feat: print getOnline cost * feat: change log * feat: change kafka and push config * feat: del interface * feat: fix err * feat: change config * feat: go mod * feat: change config * feat: change config * feat: add sleep in push * feat: warn logs * feat: logs * feat: logs * feat: change port * feat: start config * feat: remove port reuse * feat: prometheus config * feat: prometheus config * feat: prometheus config * feat: add long time send msg to grafana * feat: init * feat: init * feat: implement offline push. * feat: batch get user online * feat: implement batch Push spilt * update go mod * Revert "feat: change port" This reverts commit 06d5e944 * feat: change port * feat: change config * feat: implement kafka producer and consumer. * update format, * add PushMQ log. * feat: get all online users and init push * feat: lock in online cache * feat: config * fix: init online status * fix: add logs * fix: userIDs * fix: add logs * feat: update Handler logic. * update MQ logic. * update * update * fix: method name * fix: update OfflinePushConsumerHandler. * fix: prommetrics * fix: add logs * fix: ctx * fix: log * fix: config * feat: change port * fix: atomic online cache status --------- Co-authored-by: Monet Lee * feature: add GetConversationsHasReadAndMaxSeq interface to the WebSocket API. (#2611) * fix: lru lock (#2613) * fix: lru lock * fix: lru lock * fix: lru lock * fix: nil pointer error on close (#2618) * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * update gomake version * update gomake version * fix: seq conversion bug * fix: redis pipe exec * fix: ImportFriends * fix: A large number of logs keysAndValues ​​length is not even * feat: mark read aggregate write * feat: online status supports redis cluster * feat: online status supports redis cluster * feat: online status supports redis cluster * merge * merge * read seq is written to mongo * read seq is written to mongo * fix: invitation to join group notification * fix: friend op_user_id * feat: optimizing asynchronous context * feat: optimizing memamq size * feat: add GetSeqMessage * feat: GroupApplicationAgreeMemberEnterNotification * feat: GroupApplicationAgreeMemberEnterNotification * feat: go.mod * feat: go.mod * feat: join group notification and get seq * feat: join group notification and get seq * feat: avoid pulling messages from sessions with a large number of max seq values of 0 * feat: API supports gzip * go.mod * fix: nil pointer error on close --------- Co-authored-by: withchao * feat: create group can push notification (#2617) * fix: blockage caused by listen error (#2620) * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * update gomake version * update gomake version * fix: seq conversion bug * fix: redis pipe exec * fix: ImportFriends * fix: A large number of logs keysAndValues ​​length is not even * feat: mark read aggregate write * feat: online status supports redis cluster * feat: online status supports redis cluster * feat: online status supports redis cluster * merge * merge * read seq is written to mongo * read seq is written to mongo * fix: invitation to join group notification * fix: friend op_user_id * feat: optimizing asynchronous context * feat: optimizing memamq size * feat: add GetSeqMessage * feat: GroupApplicationAgreeMemberEnterNotification * feat: GroupApplicationAgreeMemberEnterNotification * feat: go.mod * feat: go.mod * feat: join group notification and get seq * feat: join group notification and get seq * feat: avoid pulling messages from sessions with a large number of max seq values of 0 * feat: API supports gzip * go.mod * fix: nil pointer error on close * fix: listen error --------- Co-authored-by: withchao * fix: go.mod (#2621) * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * update gomake version * update gomake version * fix: seq conversion bug * fix: redis pipe exec * fix: ImportFriends * fix: A large number of logs keysAndValues ​​length is not even * feat: mark read aggregate write * feat: online status supports redis cluster * feat: online status supports redis cluster * feat: online status supports redis cluster * merge * merge * read seq is written to mongo * read seq is written to mongo * fix: invitation to join group notification * fix: friend op_user_id * feat: optimizing asynchronous context * feat: optimizing memamq size * feat: add GetSeqMessage * feat: GroupApplicationAgreeMemberEnterNotification * feat: GroupApplicationAgreeMemberEnterNotification * feat: go.mod * feat: go.mod * feat: join group notification and get seq * feat: join group notification and get seq * feat: avoid pulling messages from sessions with a large number of max seq values of 0 * feat: API supports gzip * go.mod * fix: nil pointer error on close * fix: listen error * fix: listen error * update go.mod --------- Co-authored-by: withchao * feat: improve searchMsg implement. (#2614) * refactor: refactor workflows contents. * add tool workflows. * update field. * fix: remove chat error. * Fix err. * fix error. * remove cn comment. * update workflows files. * update infra config. * move workflows. * feat: update bot. * fix: solve uncorrect outdated msg get. * update get docIDs logic. * update * update skip logic. * fix * update. * fix: delay deleteObject func. * remove unused content. * update log type. * feat: implement request batch count limit. * update * update * remove unused script. * feat: improve searchMsg implement. * update mongo config. * Fix lock (#2622) * fix:log * fix: lock * fix: update setGroupInfoEX field name. (#2625) * refactor: refactor workflows contents. * add tool workflows. * update field. * fix: remove chat error. * Fix err. * fix error. * remove cn comment. * update workflows files. * update infra config. * move workflows. * feat: update bot. * fix: solve uncorrect outdated msg get. * update get docIDs logic. * update * update skip logic. * fix * update. * fix: delay deleteObject func. * remove unused content. * update log type. * feat: implement request batch count limit. * update * update * fix: update setGroupInfoEX field name. * fix: update setGroupInfoEX field name (#2626) * refactor: refactor workflows contents. * add tool workflows. * update field. * fix: remove chat error. * Fix err. * fix error. * remove cn comment. * update workflows files. * update infra config. * move workflows. * feat: update bot. * fix: solve uncorrect outdated msg get. * update get docIDs logic. * update * update skip logic. * fix * update. * fix: delay deleteObject func. * remove unused content. * update log type. * feat: implement request batch count limit. * update * update * fix: update setGroupInfoEX field name. * fix: update setGroupInfoEX field name * feat: msg gateway add log (#2631) * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * update gomake version * update gomake version * fix: seq conversion bug * fix: redis pipe exec * fix: ImportFriends * fix: A large number of logs keysAndValues ​​length is not even * feat: mark read aggregate write * feat: online status supports redis cluster * feat: online status supports redis cluster * feat: online status supports redis cluster * merge * merge * read seq is written to mongo * read seq is written to mongo * fix: invitation to join group notification * fix: friend op_user_id * feat: optimizing asynchronous context * feat: optimizing memamq size * feat: add GetSeqMessage * feat: GroupApplicationAgreeMemberEnterNotification * feat: GroupApplicationAgreeMemberEnterNotification * feat: go.mod * feat: go.mod * feat: join group notification and get seq * feat: join group notification and get seq * feat: avoid pulling messages from sessions with a large number of max seq values of 0 * feat: API supports gzip * go.mod * fix: nil pointer error on close * fix: listen error * fix: listen error * update go.mod * feat: add log --------- Co-authored-by: withchao * fix: update setGroupInfoEx func name and field. (#2634) * refactor: refactor workflows contents. * add tool workflows. * update field. * fix: remove chat error. * Fix err. * fix error. * remove cn comment. * update workflows files. * update infra config. * move workflows. * feat: update bot. * fix: solve uncorrect outdated msg get. * update get docIDs logic. * update * update skip logic. * fix * update. * fix: delay deleteObject func. * remove unused content. * update log type. * feat: implement request batch count limit. * update * update * fix: update setGroupInfoEx func name and field. * refactor: update groupinfoEx field. * refactor: update database name in mongodb.yml * add groupName Condition * fix: fix setConversations req fill. (#2645) * refactor: refactor workflows contents. * add tool workflows. * update field. * fix: remove chat error. * Fix err. * fix error. * remove cn comment. * update workflows files. * update infra config. * move workflows. * feat: update bot. * fix: solve uncorrect outdated msg get. * update get docIDs logic. * update * update skip logic. * fix * update. * fix: delay deleteObject func. * remove unused content. * update log type. * feat: implement request batch count limit. * update * update * fix: fix setConversations req fill. * fix: GetMsgBySeqs boundary issues (#2647) * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * update gomake version * update gomake version * fix: seq conversion bug * fix: redis pipe exec * fix: ImportFriends * fix: A large number of logs keysAndValues ​​length is not even * feat: mark read aggregate write * feat: online status supports redis cluster * feat: online status supports redis cluster * feat: online status supports redis cluster * merge * merge * read seq is written to mongo * read seq is written to mongo * fix: invitation to join group notification * fix: friend op_user_id * feat: optimizing asynchronous context * feat: optimizing memamq size * feat: add GetSeqMessage * feat: GroupApplicationAgreeMemberEnterNotification * feat: GroupApplicationAgreeMemberEnterNotification * feat: go.mod * feat: go.mod * feat: join group notification and get seq * feat: join group notification and get seq * feat: avoid pulling messages from sessions with a large number of max seq values of 0 * feat: API supports gzip * go.mod * fix: nil pointer error on close * fix: listen error * fix: listen error * update go.mod * feat: add log * fix: token parse token value * fix: GetMsgBySeqs boundary issues --------- Co-authored-by: withchao * fix: the attribute version is obsolete, remove it (#2644) * refactor: update Userregister request field. (#2650) --------- Co-authored-by: Monet Lee Co-authored-by: icey-yu <119291641+icey-yu@users.noreply.github.com> Co-authored-by: chao <48119764+withchao@users.noreply.github.com> Co-authored-by: withchao Co-authored-by: 蔡相跃 * update go mod * fix: sn_ not sort * fix: sn_ not sort * fix: sn_ not sort * fix: jssdk add * fix: jssdk support * fix: jssdk support * fix: jssdk support * fix: the message I sent is not set to read seq in mongodb * fix: cannot modify group member avatars * merge: update code from main to v3.8-js-sdk-only. (#2720) * fix: fix update groupName invalid. (#2673) * refactor: change platform to platformID (#2670) * feat: don`t return nil data (#2675) Co-authored-by: Monet Lee * refactor: update fields type in userStatus and check registered. (#2676) * fix: usertoken auth. (#2677) * refactor: update fields type in userStatus and check registered. * fix: usertoken auth. * update contents. * update content. * update * fix * update pb file. * feat: add friend agree after callback (#2680) * fix: sn not sort (#2682) * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * update gomake version * update gomake version * fix: seq conversion bug * fix: redis pipe exec * fix: ImportFriends * fix: A large number of logs keysAndValues ​​length is not even * feat: mark read aggregate write * feat: online status supports redis cluster * feat: online status supports redis cluster * feat: online status supports redis cluster * merge * merge * read seq is written to mongo * read seq is written to mongo * fix: invitation to join group notification * fix: friend op_user_id * feat: optimizing asynchronous context * feat: optimizing memamq size * feat: add GetSeqMessage * feat: GroupApplicationAgreeMemberEnterNotification * feat: GroupApplicationAgreeMemberEnterNotification * feat: go.mod * feat: go.mod * feat: join group notification and get seq * feat: join group notification and get seq * feat: avoid pulling messages from sessions with a large number of max seq values of 0 * feat: API supports gzip * go.mod * fix: nil pointer error on close * fix: listen error * fix: listen error * update go.mod * feat: add log * fix: token parse token value * fix: GetMsgBySeqs boundary issues * fix: sn_ not sort * fix: sn_ not sort * fix: sn_ not sort --------- Co-authored-by: withchao * refactor: add GetAdminToken interface. (#2684) * refactor: add GetAdminToken interface. * update config. * fix: admin token (#2686) * fix: update workflows logic. (#2688) * refactor: add GetAdminToken interface. * update config. * update workflows logic. * fix: admin token (#2687) * update the front image (#2692) * update the front image * update version * feat: improve publish docker image workflows (#2697) * refactor: add GetAdminToken interface. * update config. * update workflows logic. * feat: improve publish docker image workflows * update condition logic. * fix: update load file logic. (#2700) * refactor: add GetAdminToken interface. * update config. * update workflows logic. * feat: improve publish docker image workflows * update condition logic. * fix: update load file logic. * feat: Msg filter (#2703) * feat: msg filter * feat: msg filter * feat: msg filter * feat: provide the interface required by js sdk (#2712) * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * update gomake version * update gomake version * fix: seq conversion bug * fix: redis pipe exec * fix: ImportFriends * fix: A large number of logs keysAndValues ​​length is not even * feat: mark read aggregate write * feat: online status supports redis cluster * feat: online status supports redis cluster * feat: online status supports redis cluster * merge * merge * read seq is written to mongo * read seq is written to mongo * fix: invitation to join group notification * fix: friend op_user_id * feat: optimizing asynchronous context * feat: optimizing memamq size * feat: add GetSeqMessage * feat: GroupApplicationAgreeMemberEnterNotification * feat: GroupApplicationAgreeMemberEnterNotification * feat: go.mod * feat: go.mod * feat: join group notification and get seq * feat: join group notification and get seq * feat: avoid pulling messages from sessions with a large number of max seq values of 0 * feat: API supports gzip * go.mod * fix: nil pointer error on close * fix: listen error * fix: listen error * update go.mod * feat: add log * fix: token parse token value * fix: GetMsgBySeqs boundary issues * fix: sn_ not sort * fix: sn_ not sort * fix: sn_ not sort * fix: jssdk add * fix: jssdk support * fix: jssdk support * fix: jssdk support --------- Co-authored-by: withchao * Line webhook (#2716) * feat: online and offline webhook * feat: online and offline webhook * feat: remove zk * fix: the message I sent is not set to read seq in mongodb (#2718) * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * update gomake version * update gomake version * fix: seq conversion bug * fix: redis pipe exec * fix: ImportFriends * fix: A large number of logs keysAndValues ​​length is not even * feat: mark read aggregate write * feat: online status supports redis cluster * feat: online status supports redis cluster * feat: online status supports redis cluster * merge * merge * read seq is written to mongo * read seq is written to mongo * fix: invitation to join group notification * fix: friend op_user_id * feat: optimizing asynchronous context * feat: optimizing memamq size * feat: add GetSeqMessage * feat: GroupApplicationAgreeMemberEnterNotification * feat: GroupApplicationAgreeMemberEnterNotification * feat: go.mod * feat: go.mod * feat: join group notification and get seq * feat: join group notification and get seq * feat: avoid pulling messages from sessions with a large number of max seq values of 0 * feat: API supports gzip * go.mod * fix: nil pointer error on close * fix: listen error * fix: listen error * update go.mod * feat: add log * fix: token parse token value * fix: GetMsgBySeqs boundary issues * fix: sn_ not sort * fix: sn_ not sort * fix: sn_ not sort * fix: jssdk add * fix: jssdk support * fix: jssdk support * fix: jssdk support * fix: the message I sent is not set to read seq in mongodb --------- Co-authored-by: withchao * fix: cannot modify group member avatars (#2719) * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * update gomake version * update gomake version * fix: seq conversion bug * fix: redis pipe exec * fix: ImportFriends * fix: A large number of logs keysAndValues ​​length is not even * feat: mark read aggregate write * feat: online status supports redis cluster * feat: online status supports redis cluster * feat: online status supports redis cluster * merge * merge * read seq is written to mongo * read seq is written to mongo * fix: invitation to join group notification * fix: friend op_user_id * feat: optimizing asynchronous context * feat: optimizing memamq size * feat: add GetSeqMessage * feat: GroupApplicationAgreeMemberEnterNotification * feat: GroupApplicationAgreeMemberEnterNotification * feat: go.mod * feat: go.mod * feat: join group notification and get seq * feat: join group notification and get seq * feat: avoid pulling messages from sessions with a large number of max seq values of 0 * feat: API supports gzip * go.mod * fix: nil pointer error on close * fix: listen error * fix: listen error * update go.mod * feat: add log * fix: token parse token value * fix: GetMsgBySeqs boundary issues * fix: sn_ not sort * fix: sn_ not sort * fix: sn_ not sort * fix: jssdk add * fix: jssdk support * fix: jssdk support * fix: jssdk support * fix: the message I sent is not set to read seq in mongodb * fix: cannot modify group member avatars --------- Co-authored-by: withchao --------- Co-authored-by: Monet Lee Co-authored-by: icey-yu <119291641+icey-yu@users.noreply.github.com> Co-authored-by: chao <48119764+withchao@users.noreply.github.com> Co-authored-by: withchao Co-authored-by: skiffer-git <72860476+skiffer-git@users.noreply.github.com> * fix: MemberEnterNotification * fix: MemberEnterNotification * fix: MsgData status * merge * merge: update code from main to v3.8-js-sdk-only. (#2818) * feat: implement merge milestone PR to target-branch. (#2796) * build: improve workflows logic. (#2801) * fix: improve time condition check mehtod. (#2804) * fix: improve time condition check mehtod. * fix * fix: webhook before online push (#2805) * fix: set own read seq in MongoDB when sender send a message. (#2808) * fix: solve err Notification when setGroupInfo. (#2806) * fix: solve err Notification when setGroupInfo. * build: update checkout version. * fix: update notification contents. * Introducing OpenIM Guru on Gurubase.io (#2788) * feat: support app update service (#2811) * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * update gomake version * update gomake version * fix: seq conversion bug * fix: redis pipe exec * fix: ImportFriends * fix: A large number of logs keysAndValues ​​length is not even * feat: mark read aggregate write * feat: online status supports redis cluster * feat: online status supports redis cluster * feat: online status supports redis cluster * merge * merge * read seq is written to mongo * read seq is written to mongo * fix: invitation to join group notification * fix: friend op_user_id * feat: optimizing asynchronous context * feat: optimizing memamq size * feat: add GetSeqMessage * feat: GroupApplicationAgreeMemberEnterNotification * feat: GroupApplicationAgreeMemberEnterNotification * feat: go.mod * feat: go.mod * feat: join group notification and get seq * feat: join group notification and get seq * feat: avoid pulling messages from sessions with a large number of max seq values of 0 * feat: API supports gzip * go.mod * fix: nil pointer error on close * fix: listen error * fix: listen error * update go.mod * feat: add log * fix: token parse token value * fix: GetMsgBySeqs boundary issues * fix: sn_ not sort * fix: sn_ not sort * fix: sn_ not sort * fix: jssdk add * fix: jssdk support * fix: jssdk support * fix: jssdk support * fix: the message I sent is not set to read seq in mongodb * fix: cannot modify group member avatars * fix: MemberEnterNotification * fix: MemberEnterNotification * fix: MsgData status * feat: add ApplicationVersion * feat: add ApplicationVersion * feat: add ApplicationVersion --------- Co-authored-by: withchao * feat: ApplicationVersion move chat (#2813) * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * update gomake version * update gomake version * fix: seq conversion bug * fix: redis pipe exec * fix: ImportFriends * fix: A large number of logs keysAndValues ​​length is not even * feat: mark read aggregate write * feat: online status supports redis cluster * feat: online status supports redis cluster * feat: online status supports redis cluster * merge * merge * read seq is written to mongo * read seq is written to mongo * fix: invitation to join group notification * fix: friend op_user_id * feat: optimizing asynchronous context * feat: optimizing memamq size * feat: add GetSeqMessage * feat: GroupApplicationAgreeMemberEnterNotification * feat: GroupApplicationAgreeMemberEnterNotification * feat: go.mod * feat: go.mod * feat: join group notification and get seq * feat: join group notification and get seq * feat: avoid pulling messages from sessions with a large number of max seq values of 0 * feat: API supports gzip * go.mod * fix: nil pointer error on close * fix: listen error * fix: listen error * update go.mod * feat: add log * fix: token parse token value * fix: GetMsgBySeqs boundary issues * fix: sn_ not sort * fix: sn_ not sort * fix: sn_ not sort * fix: jssdk add * fix: jssdk support * fix: jssdk support * fix: jssdk support * fix: the message I sent is not set to read seq in mongodb * fix: cannot modify group member avatars * fix: MemberEnterNotification * fix: MemberEnterNotification * fix: MsgData status * feat: add ApplicationVersion * feat: ApplicationVersion move chat --------- Co-authored-by: withchao * fix: improve condition check. (#2815) --------- Co-authored-by: Monet Lee Co-authored-by: icey-yu <119291641+icey-yu@users.noreply.github.com> Co-authored-by: Kürşat Aktaş Co-authored-by: chao <48119764+withchao@users.noreply.github.com> Co-authored-by: withchao * feat: support text ping pong * feat: support text ping pong * feat: gob json encoder * feat: gob json encoder * feat: gob json encoder * feat: gob json encoder * feat: gob json encoder * feat: gob json encoder * fix: concurrent write to websocket connection * fix: concurrent write to websocket connection * fix: concurrent write to websocket connection * fix: group member update face_url --------- Co-authored-by: withchao Co-authored-by: Monet Lee Co-authored-by: OpenIM-Gordon <46924906+FGadvancer@users.noreply.github.com> Co-authored-by: icey-yu <119291641+icey-yu@users.noreply.github.com> Co-authored-by: 蔡相跃 Co-authored-by: skiffer-git <72860476+skiffer-git@users.noreply.github.com> Co-authored-by: Kürşat Aktaş --- internal/rpc/group/db_map.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/rpc/group/db_map.go b/internal/rpc/group/db_map.go index 26c9a4625..e99f9e772 100644 --- a/internal/rpc/group/db_map.go +++ b/internal/rpc/group/db_map.go @@ -110,7 +110,7 @@ func UpdateGroupMemberMap(req *pbgroup.SetGroupMemberInfo) map[string]any { m["nickname"] = req.Nickname.Value } if req.FaceURL != nil { - m["user_group_face_url"] = req.FaceURL.Value + m["face_url"] = req.FaceURL.Value } if req.RoleLevel != nil { m["role_level"] = req.RoleLevel.Value From 14477321fd23e8b5992a9ab944a266fc5fa79173 Mon Sep 17 00:00:00 2001 From: Monet Lee Date: Tue, 3 Dec 2024 14:12:47 +0800 Subject: [PATCH 059/199] fix: update set seq implement. (#2911) * update set seq implement. * revert seq logic. * Update func logic. * add log print. --- go.mod | 2 +- go.sum | 4 +-- internal/push/push_handler.go | 1 + internal/rpc/conversation/conversation.go | 7 ++-- internal/rpc/group/group.go | 1 + internal/rpc/msg/clear.go | 35 ++++++++++++------- internal/tools/cron_task.go | 35 ++++++++++--------- pkg/common/storage/controller/conversation.go | 2 +- pkg/common/storage/controller/msg.go | 31 ++++++++++------ pkg/common/storage/database/conversation.go | 1 + .../storage/database/mgo/conversation.go | 3 +- .../storage/database/mgo/group_member.go | 2 +- pkg/rpcclient/conversation.go | 4 +-- pkg/rpcclient/msg.go | 4 +-- 14 files changed, 79 insertions(+), 53 deletions(-) diff --git a/go.mod b/go.mod index 375c0406c..8eae1edb2 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( github.com/gorilla/websocket v1.5.1 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/mitchellh/mapstructure v1.5.0 - github.com/openimsdk/protocol v0.0.72-alpha.57 + github.com/openimsdk/protocol v0.0.72-alpha.59 github.com/openimsdk/tools v0.0.50-alpha.38 github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 diff --git a/go.sum b/go.sum index 74ee1c2c6..e572f0b47 100644 --- a/go.sum +++ b/go.sum @@ -319,8 +319,8 @@ github.com/onsi/gomega v1.25.0 h1:Vw7br2PCDYijJHSfBOWhov+8cAnUf8MfMaIOV323l6Y= github.com/onsi/gomega v1.25.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdMPGhM= github.com/openimsdk/gomake v0.0.14-alpha.5 h1:VY9c5x515lTfmdhhPjMvR3BBRrRquAUCFsz7t7vbv7Y= github.com/openimsdk/gomake v0.0.14-alpha.5/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= -github.com/openimsdk/protocol v0.0.72-alpha.57 h1:oAVg0SJkDK15L8yDrL0KPG32f3iB/vjEpfpX577p5n4= -github.com/openimsdk/protocol v0.0.72-alpha.57/go.mod h1:Iet+piS/jaS+kWWyj6EEr36mk4ISzIRYjoMSVA4dq2M= +github.com/openimsdk/protocol v0.0.72-alpha.59 h1:+ycb2+68mLKPIo7VrxF0id/GXP6OqZ2/nBM1YZQr7qY= +github.com/openimsdk/protocol v0.0.72-alpha.59/go.mod h1:Iet+piS/jaS+kWWyj6EEr36mk4ISzIRYjoMSVA4dq2M= github.com/openimsdk/tools v0.0.50-alpha.38 h1:AU6/cvDfN4ciIOwAj8IWEwze3DeEp2cHYPgW3y0OlbU= github.com/openimsdk/tools v0.0.50-alpha.38/go.mod h1:/Em/fQH46CuWf60+hcmvZyboGCQpSDEb2MdQ4nmQRAk= github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= diff --git a/internal/push/push_handler.go b/internal/push/push_handler.go index b5bbed3ed..c1d1ac2f9 100644 --- a/internal/push/push_handler.go +++ b/internal/push/push_handler.go @@ -414,6 +414,7 @@ func (c *ConsumerHandler) DeleteMemberAndSetConversationSeq(ctx context.Context, if err != nil { return err } + return c.conversationRpcClient.SetConversationMaxSeq(ctx, userIDs, conversationID, maxSeq) } diff --git a/internal/rpc/conversation/conversation.go b/internal/rpc/conversation/conversation.go index ce720e640..0b6b656a4 100644 --- a/internal/rpc/conversation/conversation.go +++ b/internal/rpc/conversation/conversation.go @@ -441,6 +441,7 @@ func (c *conversationServer) SetConversationMaxSeq(ctx context.Context, req *pbc map[string]any{"max_seq": req.MaxSeq}); err != nil { return nil, err } + return &pbconversation.SetConversationMaxSeqResp{}, nil } @@ -670,7 +671,7 @@ func (c *conversationServer) GetOwnerConversation(ctx context.Context, req *pbco }, nil } -func (c *conversationServer) GetConversationsNeedDestructMsgs(ctx context.Context, _ *pbconversation.GetConversationsNeedDestructMsgsReq) (*pbconversation.GetConversationsNeedDestructMsgsResp, error) { +func (c *conversationServer) GetConversationsNeedClearMsg(ctx context.Context, _ *pbconversation.GetConversationsNeedClearMsgReq) (*pbconversation.GetConversationsNeedClearMsgResp, error) { num, err := c.conversationDatabase.GetAllConversationIDsNumber(ctx) if err != nil { log.ZError(ctx, "GetAllConversationIDsNumber failed", err) @@ -694,7 +695,7 @@ func (c *conversationServer) GetConversationsNeedDestructMsgs(ctx context.Contex conversationIDs, err := c.conversationDatabase.PageConversationIDs(ctx, pagination) if err != nil { - // log.ZError(ctx, "PageConversationIDs failed", err, "pageNumber", pageNumber) + log.ZError(ctx, "PageConversationIDs failed", err, "pageNumber", pageNumber) continue } @@ -717,7 +718,7 @@ func (c *conversationServer) GetConversationsNeedDestructMsgs(ctx context.Contex } } - return &pbconversation.GetConversationsNeedDestructMsgsResp{Conversations: convert.ConversationsDB2Pb(temp)}, nil + return &pbconversation.GetConversationsNeedClearMsgResp{Conversations: convert.ConversationsDB2Pb(temp)}, nil } func (c *conversationServer) GetNotNotifyConversationIDs(ctx context.Context, req *pbconversation.GetNotNotifyConversationIDsReq) (*pbconversation.GetNotNotifyConversationIDsResp, error) { diff --git a/internal/rpc/group/group.go b/internal/rpc/group/group.go index 9e610df0f..ba0c8a42d 100644 --- a/internal/rpc/group/group.go +++ b/internal/rpc/group/group.go @@ -964,6 +964,7 @@ func (g *groupServer) deleteMemberAndSetConversationSeq(ctx context.Context, gro if err != nil { return err } + return g.conversationRpcClient.SetConversationMaxSeq(ctx, userIDs, conevrsationID, maxSeq) } diff --git a/internal/rpc/msg/clear.go b/internal/rpc/msg/clear.go index c5bd36b44..ff732136a 100644 --- a/internal/rpc/msg/clear.go +++ b/internal/rpc/msg/clear.go @@ -12,13 +12,14 @@ import ( "github.com/openimsdk/tools/errs" "github.com/openimsdk/tools/log" "github.com/openimsdk/tools/mcontext" + "github.com/openimsdk/tools/utils/datautil" "github.com/openimsdk/tools/utils/idutil" "github.com/openimsdk/tools/utils/stringutil" "golang.org/x/sync/errgroup" ) // hard delete in Database. -func (m *msgServer) ClearMsg(ctx context.Context, req *msg.ClearMsgReq) (_ *msg.ClearMsgResp, err error) { +func (m *msgServer) DestructMsgs(ctx context.Context, req *msg.DestructMsgsReq) (_ *msg.DestructMsgsResp, err error) { if err := authverify.CheckAdmin(ctx, m.config.Share.IMAdminUserID); err != nil { return nil, err } @@ -26,18 +27,19 @@ func (m *msgServer) ClearMsg(ctx context.Context, req *msg.ClearMsgReq) (_ *msg. return nil, errs.ErrArgs.WrapMsg("request millisecond timestamp error") } var ( - docNum int - msgNum int - start = time.Now() + docNum int + msgNum int + start = time.Now() + getLimit = 5000 ) - clearMsg := func(ctx context.Context) (bool, error) { + destructMsg := func(ctx context.Context) (bool, error) { docIDs, err := m.MsgDatabase.GetDocIDs(ctx) if err != nil { return false, err } - msgs, err := m.MsgDatabase.GetBeforeMsg(ctx, req.Timestamp, docIDs, 5000) + msgs, err := m.MsgDatabase.GetBeforeMsg(ctx, req.Timestamp, docIDs, getLimit) if err != nil { return false, err } @@ -61,7 +63,7 @@ func (m *msgServer) ClearMsg(ctx context.Context, req *msg.ClearMsgReq) (_ *msg. return true, nil } - _, err = clearMsg(ctx) + _, err = destructMsg(ctx) if err != nil { log.ZError(ctx, "clear msg failed", err, "docNum", docNum, "msgNum", msgNum, "cost", time.Since(start)) return nil, err @@ -69,11 +71,11 @@ func (m *msgServer) ClearMsg(ctx context.Context, req *msg.ClearMsgReq) (_ *msg. log.ZDebug(ctx, "clearing message", "docNum", docNum, "msgNum", msgNum, "cost", time.Since(start)) - return &msg.ClearMsgResp{}, nil + return &msg.DestructMsgsResp{}, nil } -// soft delete for self -func (m *msgServer) DestructMsgs(ctx context.Context, req *msg.DestructMsgsReq) (_ *msg.DestructMsgsResp, err error) { +// soft delete for user self +func (m *msgServer) ClearMsg(ctx context.Context, req *msg.ClearMsgReq) (_ *msg.ClearMsgResp, err error) { temp := convert.ConversationsPb2DB(req.Conversations) batchNum := 100 @@ -93,22 +95,31 @@ func (m *msgServer) DestructMsgs(ctx context.Context, req *msg.DestructMsgsReq) "msgDestructTime", conversation.MsgDestructTime, "lastMsgDestructTime", conversation.LatestMsgDestructTime) - seqs, err := m.MsgDatabase.UserMsgsDestruct(handleCtx, conversation.OwnerUserID, conversation.ConversationID, conversation.MsgDestructTime, conversation.LatestMsgDestructTime) + seqs, err := m.MsgDatabase.ClearUserMsgs(handleCtx, conversation.OwnerUserID, conversation.ConversationID, conversation.MsgDestructTime, conversation.LatestMsgDestructTime) if err != nil { log.ZError(handleCtx, "user msg destruct failed", err, "conversationID", conversation.ConversationID, "ownerUserID", conversation.OwnerUserID) continue } if len(seqs) > 0 { + minseq := datautil.Max(seqs...) + + // update if err := m.Conversation.UpdateConversation(handleCtx, &pbconversation.UpdateConversationReq{ UserIDs: []string{conversation.OwnerUserID}, ConversationID: conversation.ConversationID, - LatestMsgDestructTime: wrapperspb.Int64(time.Now().UnixMilli())}); err != nil { + LatestMsgDestructTime: wrapperspb.Int64(time.Now().UnixMilli()), + MinSeq: wrapperspb.Int64(minseq), + }); err != nil { log.ZError(handleCtx, "updateUsersConversationField failed", err, "conversationID", conversation.ConversationID, "ownerUserID", conversation.OwnerUserID) continue } + if err := m.Conversation.SetConversationMinSeq(handleCtx, []string{conversation.OwnerUserID}, conversation.ConversationID, minseq); err != nil { + return err + } + // if you need Notify SDK client userseq is update. // m.msgNotificationSender.UserDeleteMsgsNotification(handleCtx, conversation.OwnerUserID, conversation.ConversationID, seqs) } diff --git a/internal/tools/cron_task.go b/internal/tools/cron_task.go index 484786880..4f87036c7 100644 --- a/internal/tools/cron_task.go +++ b/internal/tools/cron_task.go @@ -76,42 +76,43 @@ func Start(ctx context.Context, config *CronTaskConfig) error { crontab := cron.New() // scheduled hard delete outdated Msgs in specific time. - clearMsgFunc := func() { + destructMsgsFunc := func() { now := time.Now() deltime := now.Add(-time.Hour * 24 * time.Duration(config.CronTask.RetainChatRecords)) ctx := mcontext.SetOperationID(ctx, fmt.Sprintf("cron_%d_%d", os.Getpid(), deltime.UnixMilli())) - log.ZDebug(ctx, "clear chat records", "deltime", deltime, "timestamp", deltime.UnixMilli()) + log.ZDebug(ctx, "Destruct chat records", "deltime", deltime, "timestamp", deltime.UnixMilli()) - if _, err := msgClient.ClearMsg(ctx, &msg.ClearMsgReq{Timestamp: deltime.UnixMilli()}); err != nil { - log.ZError(ctx, "cron clear chat records failed", err, "deltime", deltime, "cont", time.Since(now)) + if _, err := msgClient.DestructMsgs(ctx, &msg.DestructMsgsReq{Timestamp: deltime.UnixMilli()}); err != nil { + log.ZError(ctx, "cron destruct chat records failed", err, "deltime", deltime, "cont", time.Since(now)) return } - log.ZDebug(ctx, "cron clear chat records success", "deltime", deltime, "cont", time.Since(now)) + log.ZDebug(ctx, "cron destruct chat records success", "deltime", deltime, "cont", time.Since(now)) } - if _, err := crontab.AddFunc(config.CronTask.CronExecuteTime, clearMsgFunc); err != nil { + if _, err := crontab.AddFunc(config.CronTask.CronExecuteTime, destructMsgsFunc); err != nil { return errs.Wrap(err) } // scheduled soft delete outdated Msgs in specific time when user set `is_msg_destruct` feature. - msgDestructFunc := func() { + clearMsgFunc := func() { now := time.Now() ctx := mcontext.SetOperationID(ctx, fmt.Sprintf("cron_%d_%d", os.Getpid(), now.UnixMilli())) - log.ZDebug(ctx, "msg destruct cron start", "now", now) + log.ZDebug(ctx, "clear msg cron start", "now", now) - conversations, err := conversationClient.GetConversationsNeedDestructMsgs(ctx, &pbconversation.GetConversationsNeedDestructMsgsReq{}) + conversations, err := conversationClient.GetConversationsNeedClearMsg(ctx, &pbconversation.GetConversationsNeedClearMsgReq{}) if err != nil { log.ZError(ctx, "Get conversation need Destruct msgs failed.", err) return - } else { - _, err := msgClient.DestructMsgs(ctx, &msg.DestructMsgsReq{Conversations: conversations.Conversations}) - if err != nil { - log.ZError(ctx, "Destruct Msgs failed.", err) - return - } } - log.ZDebug(ctx, "msg destruct cron task completed", "cont", time.Since(now)) + + _, err = msgClient.ClearMsg(ctx, &msg.ClearMsgReq{Conversations: conversations.Conversations}) + if err != nil { + log.ZError(ctx, "Clear Msg failed.", err) + return + } + + log.ZDebug(ctx, "clear msg cron task completed", "cont", time.Since(now)) } - if _, err := crontab.AddFunc(config.CronTask.CronExecuteTime, msgDestructFunc); err != nil { + if _, err := crontab.AddFunc(config.CronTask.CronExecuteTime, clearMsgFunc); err != nil { return errs.Wrap(err) } diff --git a/pkg/common/storage/controller/conversation.go b/pkg/common/storage/controller/conversation.go index f0b7d70db..bf41cce95 100644 --- a/pkg/common/storage/controller/conversation.go +++ b/pkg/common/storage/controller/conversation.go @@ -179,7 +179,7 @@ func (c *conversationDatabase) CreateConversation(ctx context.Context, conversat if conversation.RecvMsgOpt == constant.ReceiveNotNotifyMessage { notNotifyUserIDs = append(notNotifyUserIDs, conversation.OwnerUserID) } - if conversation.IsPinned == true { + if conversation.IsPinned { pinnedUserIDs = append(pinnedUserIDs, conversation.OwnerUserID) } } diff --git a/pkg/common/storage/controller/msg.go b/pkg/common/storage/controller/msg.go index 3cd706458..85b797c40 100644 --- a/pkg/common/storage/controller/msg.go +++ b/pkg/common/storage/controller/msg.go @@ -57,8 +57,8 @@ type CommonMsgDatabase interface { // DeleteConversationMsgsAndSetMinSeq deletes conversation messages and resets the minimum sequence number. If `remainTime` is 0, all messages are deleted (this method does not delete Redis // cache). DeleteConversationMsgsAndSetMinSeq(ctx context.Context, conversationID string, remainTime int64) error - // UserMsgsDestruct marks messages for deletion based on destruct time and returns a list of sequence numbers for marked messages. - UserMsgsDestruct(ctx context.Context, userID string, conversationID string, destructTime int64, lastMsgDestructTime time.Time) (seqs []int64, err error) + // ClearUserMsgs marks messages for deletion based on clear time and returns a list of sequence numbers for marked messages. + ClearUserMsgs(ctx context.Context, userID string, conversationID string, clearTime int64, lastMsgClearTime time.Time) (seqs []int64, err error) // DeleteUserMsgsBySeqs allows a user to delete messages based on sequence numbers. DeleteUserMsgsBySeqs(ctx context.Context, userID string, conversationID string, seqs []int64) error // DeleteMsgsPhysicalBySeqs physically deletes messages by emptying them based on sequence numbers. @@ -92,7 +92,7 @@ type CommonMsgDatabase interface { RangeGroupSendCount(ctx context.Context, start time.Time, end time.Time, ase bool, pageNumber int32, showNumber int32) (msgCount int64, userCount int64, groups []*model.GroupCount, dateCount map[string]int64, err error) ConvertMsgsDocLen(ctx context.Context, conversationIDs []string) - // clear msg + // get Msg when destruct msg before GetBeforeMsg(ctx context.Context, ts int64, docIds []string, limit int) ([]*model.MsgDocModel, error) DeleteDocMsgBefore(ctx context.Context, ts int64, doc *model.MsgDocModel) ([]int, error) @@ -528,10 +528,10 @@ func (db *commonMsgDatabase) DeleteConversationMsgsAndSetMinSeq(ctx context.Cont return db.seqConversation.SetMinSeq(ctx, conversationID, minSeq) } -func (db *commonMsgDatabase) UserMsgsDestruct(ctx context.Context, userID string, conversationID string, destructTime int64, lastMsgDestructTime time.Time) (seqs []int64, err error) { +func (db *commonMsgDatabase) ClearUserMsgs(ctx context.Context, userID string, conversationID string, clearTime int64, lastMsgClearTime time.Time) (seqs []int64, err error) { var index int64 for { - // from oldest 2 newest + // from oldest 2 newest, ASC msgDocModel, err := db.msgDocDatabase.GetMsgDocModelByIndex(ctx, conversationID, index, 1) if err != nil || msgDocModel.DocID == "" { if err != nil { @@ -544,15 +544,19 @@ func (db *commonMsgDatabase) UserMsgsDestruct(ctx context.Context, userID string // If an error is reported, or the error cannot be obtained, it is physically deleted and seq delMongoMsgsPhysical(delStruct.delDocIDList) is returned to end the recursion break } + index++ - // && msgDocModel.Msg[0].Msg.SendTime > lastMsgDestructTime.UnixMilli() + + // && msgDocModel.Msg[0].Msg.SendTime > lastMsgClearTime.UnixMilli() if len(msgDocModel.Msg) > 0 { i := 0 var over bool for _, msg := range msgDocModel.Msg { i++ - if msg != nil && msg.Msg != nil && msg.Msg.SendTime+destructTime*1000 <= time.Now().UnixMilli() { - if msg.Msg.SendTime+destructTime*1000 > lastMsgDestructTime.UnixMilli() && !datautil.Contain(userID, msg.DelList...) { + // over clear time, need to clear + if msg != nil && msg.Msg != nil && msg.Msg.SendTime+clearTime*1000 <= time.Now().UnixMilli() { + // if msg is not in del list, add to del list + if msg.Msg.SendTime+clearTime*1000 > lastMsgClearTime.UnixMilli() && !datautil.Contain(userID, msg.DelList...) { seqs = append(seqs, msg.Msg.Seq) } } else { @@ -567,13 +571,18 @@ func (db *commonMsgDatabase) UserMsgsDestruct(ctx context.Context, userID string } } - log.ZDebug(ctx, "UserMsgsDestruct", "conversationID", conversationID, "userID", userID, "seqs", seqs) + log.ZDebug(ctx, "ClearUserMsgs", "conversationID", conversationID, "userID", userID, "seqs", seqs) + + // have msg need to destruct if len(seqs) > 0 { - userMinSeq := seqs[len(seqs)-1] + 1 - currentUserMinSeq, err := db.seqUser.GetUserMinSeq(ctx, conversationID, userID) + // update min seq to clear after + userMinSeq := seqs[len(seqs)-1] + 1 // user min seq when clear after + currentUserMinSeq, err := db.seqUser.GetUserMinSeq(ctx, conversationID, userID) // user min seq when clear before if err != nil { return nil, err } + + // if before < after, update min seq if currentUserMinSeq < userMinSeq { if err := db.seqUser.SetUserMinSeq(ctx, conversationID, userID, userMinSeq); err != nil { return nil, err diff --git a/pkg/common/storage/database/conversation.go b/pkg/common/storage/database/conversation.go index 5a9b19035..30ca01ee7 100644 --- a/pkg/common/storage/database/conversation.go +++ b/pkg/common/storage/database/conversation.go @@ -16,6 +16,7 @@ package database import ( "context" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" "github.com/openimsdk/tools/db/pagination" ) diff --git a/pkg/common/storage/database/mgo/conversation.go b/pkg/common/storage/database/mgo/conversation.go index f7ced1c2c..10e223c89 100644 --- a/pkg/common/storage/database/mgo/conversation.go +++ b/pkg/common/storage/database/mgo/conversation.go @@ -16,9 +16,10 @@ package mgo import ( "context" + "time" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" - "time" "github.com/openimsdk/protocol/constant" "github.com/openimsdk/tools/db/mongoutil" diff --git a/pkg/common/storage/database/mgo/group_member.go b/pkg/common/storage/database/mgo/group_member.go index 2fdf2003b..510a049d0 100644 --- a/pkg/common/storage/database/mgo/group_member.go +++ b/pkg/common/storage/database/mgo/group_member.go @@ -60,7 +60,7 @@ type GroupMemberMgo struct { } func (g *GroupMemberMgo) memberSort() any { - return bson.D{{"role_level", -1}, {"create_time", 1}} + return bson.D{{Key: "role_level", Value: -1}, {Key: "create_time", Value: 1}} } func (g *GroupMemberMgo) Create(ctx context.Context, groupMembers []*model.GroupMember) (err error) { diff --git a/pkg/rpcclient/conversation.go b/pkg/rpcclient/conversation.go index ccca85619..c69d355d6 100644 --- a/pkg/rpcclient/conversation.go +++ b/pkg/rpcclient/conversation.go @@ -152,8 +152,8 @@ func (c *ConversationRpcClient) GetConversationNotReceiveMessageUserIDs(ctx cont return resp.UserIDs, nil } -func (c *ConversationRpcClient) GetConversationsNeedDestructMsgs(ctx context.Context) ([]*pbconversation.Conversation, error) { - resp, err := c.Client.GetConversationsNeedDestructMsgs(ctx, &pbconversation.GetConversationsNeedDestructMsgsReq{}) +func (c *ConversationRpcClient) GetConversationsNeedClearMsg(ctx context.Context) ([]*pbconversation.Conversation, error) { + resp, err := c.Client.GetConversationsNeedClearMsg(ctx, &pbconversation.GetConversationsNeedClearMsgReq{}) if err != nil { return nil, err } diff --git a/pkg/rpcclient/msg.go b/pkg/rpcclient/msg.go index 9b26a7abd..8313937cd 100644 --- a/pkg/rpcclient/msg.go +++ b/pkg/rpcclient/msg.go @@ -244,8 +244,8 @@ func (m *MessageRpcClient) GetConversationMaxSeq(ctx context.Context, conversati return resp.MaxSeq, nil } -func (m *MessageRpcClient) ClearMsg(ctx context.Context, ts int64) error { - _, err := m.Client.ClearMsg(ctx, &msg.ClearMsgReq{Timestamp: ts}) +func (m *MessageRpcClient) DestructMsgs(ctx context.Context, ts int64) error { + _, err := m.Client.DestructMsgs(ctx, &msg.DestructMsgsReq{Timestamp: ts}) return err } From a69d174e86817c9f702e98e20d49c1b370208839 Mon Sep 17 00:00:00 2001 From: Morya Date: Wed, 4 Dec 2024 11:18:39 +0800 Subject: [PATCH 060/199] fix https://github.com/openimsdk/open-im-server/issues/2895 (#2896) * fix: minor log typo * fix: #2895 no need to specify listen port in config file, just use system random * drop useless code --- config/openim-msggateway.yml | 4 +- config/openim-push.yml | 2 - config/openim-rpc-auth.yml | 3 +- config/openim-rpc-conversation.yml | 2 - config/openim-rpc-friend.yml | 2 - config/openim-rpc-group.yml | 2 - config/openim-rpc-msg.yml | 2 - config/openim-rpc-third.yml | 2 - config/openim-rpc-user.yml | 2 - internal/msggateway/hub_server.go | 8 +- internal/msggateway/init.go | 9 +- internal/tools/addr/addr.go | 165 +++++++++++++++++++++++++++++ pkg/common/cmd/auth.go | 1 - pkg/common/cmd/conversation.go | 1 - pkg/common/cmd/friend.go | 1 - pkg/common/cmd/group.go | 1 - pkg/common/cmd/msg.go | 1 - pkg/common/cmd/push.go | 1 - pkg/common/cmd/third.go | 1 - pkg/common/cmd/user.go | 1 - pkg/common/config/config.go | 1 - pkg/common/startrpc/start.go | 53 +++++---- 22 files changed, 198 insertions(+), 67 deletions(-) create mode 100644 internal/tools/addr/addr.go diff --git a/config/openim-msggateway.yml b/config/openim-msggateway.yml index 6c46b52a8..84e232ccb 100644 --- a/config/openim-msggateway.yml +++ b/config/openim-msggateway.yml @@ -1,8 +1,6 @@ rpc: # The IP address where this RPC service registers itself; if left blank, it defaults to the internal network IP - registerIP: - # List of ports that the RPC service listens on; configuring multiple ports will launch multiple instances. These must match the number of configured prometheus ports - ports: [ 10140, 10141, 10142, 10143, 10144, 10145, 10146, 10147, 10148, 10149, 10150, 10151, 10152, 10153, 10154, 10155 ] + registerIP: prometheus: # Enable or disable Prometheus monitoring diff --git a/config/openim-push.yml b/config/openim-push.yml index 92f716ba2..2db09b37e 100644 --- a/config/openim-push.yml +++ b/config/openim-push.yml @@ -3,8 +3,6 @@ rpc: registerIP: # IP address that the RPC service listens on; setting to 0.0.0.0 listens on both internal and external IPs. If left blank, it automatically uses the internal network IP listenIP: 0.0.0.0 - # List of ports that the RPC service listens on; configuring multiple ports will launch multiple instances. These must match the number of configured prometheus ports - ports: [ 10170, 10171, 10172, 10173, 10174, 10175, 10176, 10177, 10178, 10179, 10180, 10181, 10182, 10183, 10184, 10185 ] prometheus: # Enable or disable Prometheus monitoring diff --git a/config/openim-rpc-auth.yml b/config/openim-rpc-auth.yml index 496803e43..9ebd78070 100644 --- a/config/openim-rpc-auth.yml +++ b/config/openim-rpc-auth.yml @@ -3,8 +3,7 @@ rpc: registerIP: # IP address that the RPC service listens on; setting to 0.0.0.0 listens on both internal and external IPs. If left blank, it automatically uses the internal network IP listenIP: 0.0.0.0 - # List of ports that the RPC service listens on; configuring multiple ports will launch multiple instances. These must match the number of configured prometheus ports - ports: [ 10200 ] + prometheus: # Enable or disable Prometheus monitoring diff --git a/config/openim-rpc-conversation.yml b/config/openim-rpc-conversation.yml index 3581d7e19..1cb56ac60 100644 --- a/config/openim-rpc-conversation.yml +++ b/config/openim-rpc-conversation.yml @@ -3,8 +3,6 @@ rpc: registerIP: # IP address that the RPC service listens on; setting to 0.0.0.0 listens on both internal and external IPs. If left blank, it automatically uses the internal network IP listenIP: 0.0.0.0 - # List of ports that the RPC service listens on; configuring multiple ports will launch multiple instances. These must match the number of configured prometheus ports - ports: [ 10220 ] prometheus: # Enable or disable Prometheus monitoring diff --git a/config/openim-rpc-friend.yml b/config/openim-rpc-friend.yml index 3022c09f3..11b805299 100644 --- a/config/openim-rpc-friend.yml +++ b/config/openim-rpc-friend.yml @@ -3,8 +3,6 @@ rpc: registerIP: # IP address that the RPC service listens on; setting to 0.0.0.0 listens on both internal and external IPs. If left blank, it automatically uses the internal network IP listenIP: 0.0.0.0 - # List of ports that the RPC service listens on; configuring multiple ports will launch multiple instances. These must match the number of configured prometheus ports - ports: [ 10240 ] prometheus: # Enable or disable Prometheus monitoring diff --git a/config/openim-rpc-group.yml b/config/openim-rpc-group.yml index 9a634d12f..d16ae7b02 100644 --- a/config/openim-rpc-group.yml +++ b/config/openim-rpc-group.yml @@ -3,8 +3,6 @@ rpc: registerIP: # IP address that the RPC service listens on; setting to 0.0.0.0 listens on both internal and external IPs. If left blank, it automatically uses the internal network IP listenIP: 0.0.0.0 - # List of ports that the RPC service listens on; configuring multiple ports will launch multiple instances. These must match the number of configured prometheus ports - ports: [ 10260 ] prometheus: # Enable or disable Prometheus monitoring diff --git a/config/openim-rpc-msg.yml b/config/openim-rpc-msg.yml index 82d6e2f53..5917ff5ae 100644 --- a/config/openim-rpc-msg.yml +++ b/config/openim-rpc-msg.yml @@ -3,8 +3,6 @@ rpc: registerIP: # IP address that the RPC service listens on; setting to 0.0.0.0 listens on both internal and external IPs. If left blank, it automatically uses the internal network IP listenIP: 0.0.0.0 - # List of ports that the RPC service listens on; configuring multiple ports will launch multiple instances. These must match the number of configured prometheus ports - ports: [ 10280 ] prometheus: # Enable or disable Prometheus monitoring diff --git a/config/openim-rpc-third.yml b/config/openim-rpc-third.yml index d8f2d427f..54123d8ea 100644 --- a/config/openim-rpc-third.yml +++ b/config/openim-rpc-third.yml @@ -3,8 +3,6 @@ rpc: registerIP: # IP address that the RPC service listens on; setting to 0.0.0.0 listens on both internal and external IPs. If left blank, it automatically uses the internal network IP listenIP: 0.0.0.0 - # List of ports that the RPC service listens on; configuring multiple ports will launch multiple instances. These must match the number of configured prometheus ports - ports: [ 10300 ] prometheus: # Enable or disable Prometheus monitoring diff --git a/config/openim-rpc-user.yml b/config/openim-rpc-user.yml index 798105472..4bd6444a7 100644 --- a/config/openim-rpc-user.yml +++ b/config/openim-rpc-user.yml @@ -3,8 +3,6 @@ rpc: registerIP: # Listening IP; 0.0.0.0 means both internal and external IPs are listened to, if blank, the internal network IP is automatically obtained by default listenIP: 0.0.0.0 - # Listening ports; if multiple are configured, multiple instances will be launched, and must be consistent with the number of prometheus.ports - ports: [ 10320 ] prometheus: # Whether to enable prometheus diff --git a/internal/msggateway/hub_server.go b/internal/msggateway/hub_server.go index 8eff32899..04376b27f 100644 --- a/internal/msggateway/hub_server.go +++ b/internal/msggateway/hub_server.go @@ -46,8 +46,7 @@ func (s *Server) InitServer(ctx context.Context, config *Config, disCov discover func (s *Server) Start(ctx context.Context, index int, conf *Config) error { return startrpc.Start(ctx, &conf.Discovery, &conf.MsgGateway.Prometheus, conf.MsgGateway.ListenIP, - conf.MsgGateway.RPC.RegisterIP, - conf.MsgGateway.RPC.Ports, index, + index, conf.Share.RpcRegisterName.MessageGateway, &conf.Share, conf, @@ -57,7 +56,7 @@ func (s *Server) Start(ctx context.Context, index int, conf *Config) error { type Server struct { msggateway.UnimplementedMsgGatewayServer - rpcPort int + LongConnServer LongConnServer config *Config pushTerminal map[int]struct{} @@ -70,9 +69,8 @@ func (s *Server) SetLongConnServer(LongConnServer LongConnServer) { s.LongConnServer = LongConnServer } -func NewServer(rpcPort int, longConnServer LongConnServer, conf *Config, ready func(srv *Server) error) *Server { +func NewServer(longConnServer LongConnServer, conf *Config, ready func(srv *Server) error) *Server { s := &Server{ - rpcPort: rpcPort, LongConnServer: longConnServer, pushTerminal: make(map[int]struct{}), config: conf, diff --git a/internal/msggateway/init.go b/internal/msggateway/init.go index 50da06097..77bccc88c 100644 --- a/internal/msggateway/init.go +++ b/internal/msggateway/init.go @@ -35,16 +35,13 @@ type Config struct { // Start run ws server. func Start(ctx context.Context, index int, conf *Config) error { - log.CInfo(ctx, "MSG-GATEWAY server is initializing", "rpcPorts", conf.MsgGateway.RPC.Ports, + log.CInfo(ctx, "MSG-GATEWAY server is initializing", "wsPort", conf.MsgGateway.LongConnSvr.Ports, "prometheusPorts", conf.MsgGateway.Prometheus.Ports) wsPort, err := datautil.GetElemByIndex(conf.MsgGateway.LongConnSvr.Ports, index) if err != nil { return err } - rpcPort, err := datautil.GetElemByIndex(conf.MsgGateway.RPC.Ports, index) - if err != nil { - return err - } + rdb, err := redisutil.NewRedisClient(ctx, conf.RedisConfig.Build()) if err != nil { return err @@ -57,7 +54,7 @@ func Start(ctx context.Context, index int, conf *Config) error { WithMessageMaxMsgLength(conf.MsgGateway.LongConnSvr.WebsocketMaxMsgLen), ) - hubServer := NewServer(rpcPort, longServer, conf, func(srv *Server) error { + hubServer := NewServer(longServer, conf, func(srv *Server) error { longServer.online, _ = rpccache.NewOnlineCache(srv.userRcp, nil, rdb, false, longServer.subscriberUserOnlineStatusChanges) return nil }) diff --git a/internal/tools/addr/addr.go b/internal/tools/addr/addr.go new file mode 100644 index 000000000..b7cf8a2db --- /dev/null +++ b/internal/tools/addr/addr.go @@ -0,0 +1,165 @@ +// addr provides functions to retrieve local IP addresses from device interfaces. +package addr + +import ( + "net" + + "github.com/pkg/errors" +) + +var ( + // ErrIPNotFound no IP address found, and explicit IP not provided. + ErrIPNotFound = errors.New("no IP address found, and explicit IP not provided") +) + +// IsLocal checks whether an IP belongs to one of the device's interfaces. +func IsLocal(addr string) bool { + // Extract the host + host, _, err := net.SplitHostPort(addr) + if err == nil { + addr = host + } + + if addr == "localhost" { + return true + } + + // Check against all local ips + for _, ip := range IPs() { + if addr == ip { + return true + } + } + + return false +} + +// Extract returns a valid IP address. If the address provided is a valid +// address, it will be returned directly. Otherwise, the available interfaces +// will be iterated over to find an IP address, preferably private. +func Extract(addr string) (string, error) { + // if addr is already specified then it's directly returned + if len(addr) > 0 && (addr != "0.0.0.0" && addr != "[::]" && addr != "::") { + return addr, nil + } + + var ( + addrs []net.Addr + loAddrs []net.Addr + ) + + ifaces, err := net.Interfaces() + if err != nil { + return "", errors.Wrap(err, "failed to get interfaces") + } + + for _, iface := range ifaces { + ifaceAddrs, err := iface.Addrs() + if err != nil { + // ignore error, interface can disappear from system + continue + } + + if iface.Flags&net.FlagLoopback != 0 { + loAddrs = append(loAddrs, ifaceAddrs...) + continue + } + + addrs = append(addrs, ifaceAddrs...) + } + + // Add loopback addresses to the end of the list + addrs = append(addrs, loAddrs...) + + // Try to find private IP in list, public IP otherwise + ip, err := findIP(addrs) + if err != nil { + return "", err + } + + return ip.String(), nil +} + +// IPs returns all available interface IP addresses. +func IPs() []string { + ifaces, err := net.Interfaces() + if err != nil { + return nil + } + + var ipAddrs []string + + for _, i := range ifaces { + addrs, err := i.Addrs() + if err != nil { + continue + } + + for _, addr := range addrs { + var ip net.IP + switch v := addr.(type) { + case *net.IPNet: + ip = v.IP + case *net.IPAddr: + ip = v.IP + } + + if ip == nil { + continue + } + + ipAddrs = append(ipAddrs, ip.String()) + } + } + + return ipAddrs +} + +// findIP will return the first private IP available in the list. +// If no private IP is available it will return the first public IP, if present. +// If no public IP is available, it will return the first loopback IP, if present. +func findIP(addresses []net.Addr) (net.IP, error) { + var publicIP net.IP + var localIP net.IP + + for _, rawAddr := range addresses { + var ip net.IP + switch addr := rawAddr.(type) { + case *net.IPAddr: + ip = addr.IP + case *net.IPNet: + ip = addr.IP + default: + continue + } + + if ip.IsLoopback() { + if localIP == nil { + localIP = ip + } + continue + } + + if !ip.IsPrivate() { + if publicIP == nil { + publicIP = ip + } + continue + } + + // Return private IP if available + return ip, nil + } + + // Return public or virtual IP + if len(publicIP) > 0 { + return publicIP, nil + } + + // Return local IP + if len(localIP) > 0 { + return localIP, nil + } + + return nil, ErrIPNotFound +} diff --git a/pkg/common/cmd/auth.go b/pkg/common/cmd/auth.go index b35a95f39..f8232bc26 100644 --- a/pkg/common/cmd/auth.go +++ b/pkg/common/cmd/auth.go @@ -55,6 +55,5 @@ func (a *AuthRpcCmd) Exec() error { func (a *AuthRpcCmd) runE() error { return startrpc.Start(a.ctx, &a.authConfig.Discovery, &a.authConfig.RpcConfig.Prometheus, a.authConfig.RpcConfig.RPC.ListenIP, - a.authConfig.RpcConfig.RPC.RegisterIP, a.authConfig.RpcConfig.RPC.Ports, a.Index(), a.authConfig.Share.RpcRegisterName.Auth, &a.authConfig.Share, a.authConfig, auth.Start) } diff --git a/pkg/common/cmd/conversation.go b/pkg/common/cmd/conversation.go index bdb4447f4..6a0da11e5 100644 --- a/pkg/common/cmd/conversation.go +++ b/pkg/common/cmd/conversation.go @@ -57,6 +57,5 @@ func (a *ConversationRpcCmd) Exec() error { func (a *ConversationRpcCmd) runE() error { return startrpc.Start(a.ctx, &a.conversationConfig.Discovery, &a.conversationConfig.RpcConfig.Prometheus, a.conversationConfig.RpcConfig.RPC.ListenIP, - a.conversationConfig.RpcConfig.RPC.RegisterIP, a.conversationConfig.RpcConfig.RPC.Ports, a.Index(), a.conversationConfig.Share.RpcRegisterName.Conversation, &a.conversationConfig.Share, a.conversationConfig, conversation.Start) } diff --git a/pkg/common/cmd/friend.go b/pkg/common/cmd/friend.go index a564facd0..2ff25090e 100644 --- a/pkg/common/cmd/friend.go +++ b/pkg/common/cmd/friend.go @@ -58,6 +58,5 @@ func (a *FriendRpcCmd) Exec() error { func (a *FriendRpcCmd) runE() error { return startrpc.Start(a.ctx, &a.relationConfig.Discovery, &a.relationConfig.RpcConfig.Prometheus, a.relationConfig.RpcConfig.RPC.ListenIP, - a.relationConfig.RpcConfig.RPC.RegisterIP, a.relationConfig.RpcConfig.RPC.Ports, a.Index(), a.relationConfig.Share.RpcRegisterName.Friend, &a.relationConfig.Share, a.relationConfig, relation.Start) } diff --git a/pkg/common/cmd/group.go b/pkg/common/cmd/group.go index 9b0fbf8de..f1757c7fa 100644 --- a/pkg/common/cmd/group.go +++ b/pkg/common/cmd/group.go @@ -59,6 +59,5 @@ func (a *GroupRpcCmd) Exec() error { func (a *GroupRpcCmd) runE() error { return startrpc.Start(a.ctx, &a.groupConfig.Discovery, &a.groupConfig.RpcConfig.Prometheus, a.groupConfig.RpcConfig.RPC.ListenIP, - a.groupConfig.RpcConfig.RPC.RegisterIP, a.groupConfig.RpcConfig.RPC.Ports, a.Index(), a.groupConfig.Share.RpcRegisterName.Group, &a.groupConfig.Share, a.groupConfig, group.Start, versionctx.EnableVersionCtx()) } diff --git a/pkg/common/cmd/msg.go b/pkg/common/cmd/msg.go index bfd29398e..ae718a11d 100644 --- a/pkg/common/cmd/msg.go +++ b/pkg/common/cmd/msg.go @@ -59,6 +59,5 @@ func (a *MsgRpcCmd) Exec() error { func (a *MsgRpcCmd) runE() error { return startrpc.Start(a.ctx, &a.msgConfig.Discovery, &a.msgConfig.RpcConfig.Prometheus, a.msgConfig.RpcConfig.RPC.ListenIP, - a.msgConfig.RpcConfig.RPC.RegisterIP, a.msgConfig.RpcConfig.RPC.Ports, a.Index(), a.msgConfig.Share.RpcRegisterName.Msg, &a.msgConfig.Share, a.msgConfig, msg.Start) } diff --git a/pkg/common/cmd/push.go b/pkg/common/cmd/push.go index ca22a697d..2f9e248f1 100644 --- a/pkg/common/cmd/push.go +++ b/pkg/common/cmd/push.go @@ -59,6 +59,5 @@ func (a *PushRpcCmd) Exec() error { func (a *PushRpcCmd) runE() error { return startrpc.Start(a.ctx, &a.pushConfig.Discovery, &a.pushConfig.RpcConfig.Prometheus, a.pushConfig.RpcConfig.RPC.ListenIP, - a.pushConfig.RpcConfig.RPC.RegisterIP, a.pushConfig.RpcConfig.RPC.Ports, a.Index(), a.pushConfig.Share.RpcRegisterName.Push, &a.pushConfig.Share, a.pushConfig, push.Start) } diff --git a/pkg/common/cmd/third.go b/pkg/common/cmd/third.go index a301b738f..fa1d5d42d 100644 --- a/pkg/common/cmd/third.go +++ b/pkg/common/cmd/third.go @@ -58,6 +58,5 @@ func (a *ThirdRpcCmd) Exec() error { func (a *ThirdRpcCmd) runE() error { return startrpc.Start(a.ctx, &a.thirdConfig.Discovery, &a.thirdConfig.RpcConfig.Prometheus, a.thirdConfig.RpcConfig.RPC.ListenIP, - a.thirdConfig.RpcConfig.RPC.RegisterIP, a.thirdConfig.RpcConfig.RPC.Ports, a.Index(), a.thirdConfig.Share.RpcRegisterName.Third, &a.thirdConfig.Share, a.thirdConfig, third.Start) } diff --git a/pkg/common/cmd/user.go b/pkg/common/cmd/user.go index 9a614afca..8a22bfce9 100644 --- a/pkg/common/cmd/user.go +++ b/pkg/common/cmd/user.go @@ -59,6 +59,5 @@ func (a *UserRpcCmd) Exec() error { func (a *UserRpcCmd) runE() error { return startrpc.Start(a.ctx, &a.userConfig.Discovery, &a.userConfig.RpcConfig.Prometheus, a.userConfig.RpcConfig.RPC.ListenIP, - a.userConfig.RpcConfig.RPC.RegisterIP, a.userConfig.RpcConfig.RPC.Ports, a.Index(), a.userConfig.Share.RpcRegisterName.User, &a.userConfig.Share, a.userConfig, user.Start) } diff --git a/pkg/common/config/config.go b/pkg/common/config/config.go index fa571e9f2..5fb12b399 100644 --- a/pkg/common/config/config.go +++ b/pkg/common/config/config.go @@ -197,7 +197,6 @@ type Push struct { RPC struct { RegisterIP string `mapstructure:"registerIP"` ListenIP string `mapstructure:"listenIP"` - Ports []int `mapstructure:"ports"` } `mapstructure:"rpc"` Prometheus Prometheus `mapstructure:"prometheus"` MaxConcurrentWorkers int `mapstructure:"maxConcurrentWorkers"` diff --git a/pkg/common/startrpc/start.go b/pkg/common/startrpc/start.go index fb8782d30..5c42586bf 100644 --- a/pkg/common/startrpc/start.go +++ b/pkg/common/startrpc/start.go @@ -24,10 +24,10 @@ import ( "net/http" "os" "os/signal" - "strconv" "syscall" "time" + "github.com/openimsdk/open-im-server/v3/internal/tools/addr" kdisc "github.com/openimsdk/open-im-server/v3/pkg/common/discoveryregister" "github.com/openimsdk/open-im-server/v3/pkg/common/prommetrics" "github.com/openimsdk/tools/discovery" @@ -37,22 +37,15 @@ import ( "github.com/openimsdk/tools/utils/network" "google.golang.org/grpc" "google.golang.org/grpc/credentials/insecure" + "strconv" ) // Start rpc server. -func Start[T any](ctx context.Context, discovery *config.Discovery, prometheusConfig *config.Prometheus, listenIP, - registerIP string, rpcPorts []int, index int, rpcRegisterName string, share *config.Share, config T, rpcFn func(ctx context.Context, - config T, client discovery.SvcDiscoveryRegistry, server *grpc.Server) error, options ...grpc.ServerOption) error { - - rpcPort, err := datautil.GetElemByIndex(rpcPorts, index) - if err != nil { - return err - } - - log.CInfo(ctx, "RPC server is initializing", "rpcRegisterName", rpcRegisterName, "rpcPort", rpcPort, - "prometheusPorts", prometheusConfig.Ports) - rpcTcpAddr := net.JoinHostPort(network.GetListenIP(listenIP), strconv.Itoa(rpcPort)) +func Start[T any](ctx context.Context, discovery *config.Discovery, prometheusConfig *config.Prometheus, listenIP string, + index int, rpcRegisterName string, share *config.Share, config T, rpcFn func(ctx context.Context, + config T, client discovery.SvcDiscoveryRegistry, server *grpc.Server) error, options ...grpc.ServerOption) error { + rpcTcpAddr := net.JoinHostPort(network.GetListenIP(listenIP), "0") listener, err := net.Listen( "tcp", rpcTcpAddr, @@ -60,6 +53,14 @@ func Start[T any](ctx context.Context, discovery *config.Discovery, prometheusCo if err != nil { return errs.WrapMsg(err, "listen err", "rpcTcpAddr", rpcTcpAddr) } + + h, portStr, _ := net.SplitHostPort(listener.Addr().String()) + host, _ := addr.Extract(h) + port, _ := strconv.Atoi(portStr) + + log.CInfo(ctx, "RPC server is initializing", "rpcRegisterName", rpcRegisterName, "rpcPort", portStr, + "prometheusPorts", prometheusConfig.Ports) + defer listener.Close() client, err := kdisc.NewDiscoveryRegister(discovery, share) if err != nil { @@ -68,17 +69,13 @@ func Start[T any](ctx context.Context, discovery *config.Discovery, prometheusCo defer client.Close() client.AddOption(mw.GrpcClient(), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithDefaultServiceConfig(fmt.Sprintf(`{"LoadBalancingPolicy": "%s"}`, "round_robin"))) - registerIP, err = network.GetRpcRegisterIP(registerIP) - if err != nil { - return err - } - //var reg *prometheus.Registry - //var metric *grpcprometheus.ServerMetrics + // var reg *prometheus.Registry + // var metric *grpcprometheus.ServerMetrics if prometheusConfig.Enable { - //cusMetrics := prommetrics.GetGrpcCusMetrics(rpcRegisterName, share) - //reg, metric, _ = prommetrics.NewGrpcPromObj(cusMetrics) - //options = append(options, mw.GrpcServer(), grpc.StreamInterceptor(metric.StreamServerInterceptor()), + // cusMetrics := prommetrics.GetGrpcCusMetrics(rpcRegisterName, share) + // reg, metric, _ = prommetrics.NewGrpcPromObj(cusMetrics) + // options = append(options, mw.GrpcServer(), grpc.StreamInterceptor(metric.StreamServerInterceptor()), // grpc.UnaryInterceptor(metric.UnaryServerInterceptor())) options = append( options, mw.GrpcServer(), @@ -98,8 +95,8 @@ func Start[T any](ctx context.Context, discovery *config.Discovery, prometheusCo err = client.Register( rpcRegisterName, - registerIP, - rpcPort, + host, + port, grpc.WithTransportCredentials(insecure.NewCredentials()), ) if err != nil { @@ -123,13 +120,13 @@ func Start[T any](ctx context.Context, discovery *config.Discovery, prometheusCo netErr = errs.WrapMsg(err, fmt.Sprintf("rpc %s prometheus start err: %d", rpcRegisterName, prometheusPort)) netDone <- struct{}{} } - //metric.InitializeMetrics(srv) + // metric.InitializeMetrics(srv) // Create a HTTP server for prometheus. - //httpServer = &http.Server{Handler: promhttp.HandlerFor(reg, promhttp.HandlerOpts{}), Addr: fmt.Sprintf("0.0.0.0:%d", prometheusPort)} - //if err := httpServer.ListenAndServe(); err != nil && err != http.ErrServerClosed { + // httpServer = &http.Server{Handler: promhttp.HandlerFor(reg, promhttp.HandlerOpts{}), Addr: fmt.Sprintf("0.0.0.0:%d", prometheusPort)} + // if err := httpServer.ListenAndServe(); err != nil && err != http.ErrServerClosed { // netErr = errs.WrapMsg(err, "prometheus start err", httpServer.Addr) // netDone <- struct{}{} - //} + // } }() } From 17d5df88d1fa7333254e235a9efc486c106c02ae Mon Sep 17 00:00:00 2001 From: chao <48119764+withchao@users.noreply.github.com> Date: Wed, 4 Dec 2024 16:41:54 +0800 Subject: [PATCH 061/199] feat: seq user and conversation seq synchronization (#2924) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * update gomake version * update gomake version * fix: seq conversion bug * fix: redis pipe exec * fix: ImportFriends * fix: A large number of logs keysAndValues ​​length is not even * feat: mark read aggregate write * feat: online status supports redis cluster * feat: online status supports redis cluster * feat: online status supports redis cluster * merge * merge * read seq is written to mongo * read seq is written to mongo * fix: invitation to join group notification * fix: friend op_user_id * feat: optimizing asynchronous context * feat: optimizing memamq size * feat: add GetSeqMessage * feat: implement no gob encoder. * update unitTest content. * Update hub_server.go * feat: GroupApplicationAgreeMemberEnterNotification * fix: encoder replace to json encoder. * feat: GroupApplicationAgreeMemberEnterNotification * feat: go.mod * feat: go.mod * feat: join group notification and get seq * feat: join group notification and get seq * feat: avoid pulling messages from sessions with a large number of max seq values of 0 * feat: API supports gzip * go.mod * fix: nil pointer error on close * fix: listen error * fix: listen error * update go.mod * feat: add log * fix: token parse token value * fix: GetMsgBySeqs boundary issues * merge: merge main code into js branch. (#2648) * feat: update group notification when set to null. (#2590) * refactor: refactor workflows contents. * add tool workflows. * update field. * fix: remove chat error. * Fix err. * fix error. * remove cn comment. * update workflows files. * update infra config. * move workflows. * feat: update bot. * fix: solve uncorrect outdated msg get. * update get docIDs logic. * update * update skip logic. * fix * update. * fix: delay deleteObject func. * remove unused content. * feat: update group notification when set to null. * update log standard. * feat: add long time push msg in prometheus (#2584) * feat: add long time push msg in prometheus * fix: log print * fix: go mod * fix: log msg * fix: log init * feat: push msg * feat: go mod ,remove cgo package * feat: remove error log * feat: test dummy push * feat:redis pool config * feat: push to kafka log * feat: supports getting messages based on session ID and seq (#2582) * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * update gomake version * update gomake version * fix: seq conversion bug * fix: redis pipe exec * fix: ImportFriends * fix: A large number of logs keysAndValues ​​length is not even * feat: mark read aggregate write * feat: online status supports redis cluster * feat: online status supports redis cluster * feat: online status supports redis cluster * merge * merge * read seq is written to mongo * read seq is written to mongo * fix: invitation to join group notification * fix: friend op_user_id * feat: optimizing asynchronous context * feat: optimizing memamq size * feat: add GetSeqMessage * feat: GroupApplicationAgreeMemberEnterNotification * feat: GroupApplicationAgreeMemberEnterNotification * feat: go.mod * feat: go.mod * feat: join group notification and get seq --------- Co-authored-by: withchao * feat: implement request batch count limit. (#2591) * refactor: refactor workflows contents. * add tool workflows. * update field. * fix: remove chat error. * Fix err. * fix error. * remove cn comment. * update workflows files. * update infra config. * move workflows. * feat: update bot. * fix: solve uncorrect outdated msg get. * update get docIDs logic. * update * update skip logic. * fix * update. * fix: delay deleteObject func. * remove unused content. * update log type. * feat: implement request batch count limit. * update * update * fix: getting messages based on session ID and seq (#2595) * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * update gomake version * update gomake version * fix: seq conversion bug * fix: redis pipe exec * fix: ImportFriends * fix: A large number of logs keysAndValues ​​length is not even * feat: mark read aggregate write * feat: online status supports redis cluster * feat: online status supports redis cluster * feat: online status supports redis cluster * merge * merge * read seq is written to mongo * read seq is written to mongo * fix: invitation to join group notification * fix: friend op_user_id * feat: optimizing asynchronous context * feat: optimizing memamq size * feat: add GetSeqMessage * feat: GroupApplicationAgreeMemberEnterNotification * feat: GroupApplicationAgreeMemberEnterNotification * feat: go.mod * feat: go.mod * feat: join group notification and get seq * feat: join group notification and get seq --------- Co-authored-by: withchao * feat: avoid pulling messages from sessions with a large number of max seq values of 0 (#2602) * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * update gomake version * update gomake version * fix: seq conversion bug * fix: redis pipe exec * fix: ImportFriends * fix: A large number of logs keysAndValues ​​length is not even * feat: mark read aggregate write * feat: online status supports redis cluster * feat: online status supports redis cluster * feat: online status supports redis cluster * merge * merge * read seq is written to mongo * read seq is written to mongo * fix: invitation to join group notification * fix: friend op_user_id * feat: optimizing asynchronous context * feat: optimizing memamq size * feat: add GetSeqMessage * feat: GroupApplicationAgreeMemberEnterNotification * feat: GroupApplicationAgreeMemberEnterNotification * feat: go.mod * feat: go.mod * feat: join group notification and get seq * feat: join group notification and get seq * feat: avoid pulling messages from sessions with a large number of max seq values of 0 --------- Co-authored-by: withchao * refactor: improve db structure in `storage/controller` (#2604) * refactor: refactor workflows contents. * add tool workflows. * update field. * fix: remove chat error. * Fix err. * fix error. * remove cn comment. * update workflows files. * update infra config. * move workflows. * feat: update bot. * fix: solve uncorrect outdated msg get. * update get docIDs logic. * update * update skip logic. * fix * update. * fix: delay deleteObject func. * remove unused content. * update log type. * feat: implement request batch count limit. * update * update * refactor: improve db structure in `storage/controller` * feat: implement offline push using kafka (#2600) * refactor: refactor workflows contents. * add tool workflows. * update field. * fix: remove chat error. * Fix err. * fix error. * remove cn comment. * update workflows files. * update infra config. * move workflows. * feat: update bot. * fix: solve uncorrect outdated msg get. * update get docIDs logic. * update * update skip logic. * fix * update. * fix: delay deleteObject func. * remove unused content. * update log type. * feat: implement request batch count limit. * update * update * feat: implement offline push. * feat: implement batch Push spilt * update go mod * feat: implement kafka producer and consumer. * update format, * add PushMQ log. * feat: update Handler logic. * update MQ logic. * update * update * fix: update OfflinePushConsumerHandler. * feat: API supports gzip (#2609) * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * update gomake version * update gomake version * fix: seq conversion bug * fix: redis pipe exec * fix: ImportFriends * fix: A large number of logs keysAndValues ​​length is not even * feat: mark read aggregate write * feat: online status supports redis cluster * feat: online status supports redis cluster * feat: online status supports redis cluster * merge * merge * read seq is written to mongo * read seq is written to mongo * fix: invitation to join group notification * fix: friend op_user_id * feat: optimizing asynchronous context * feat: optimizing memamq size * feat: add GetSeqMessage * feat: GroupApplicationAgreeMemberEnterNotification * feat: GroupApplicationAgreeMemberEnterNotification * feat: go.mod * feat: go.mod * feat: join group notification and get seq * feat: join group notification and get seq * feat: avoid pulling messages from sessions with a large number of max seq values of 0 * feat: API supports gzip --------- Co-authored-by: withchao * Fix err (#2608) * refactor: refactor workflows contents. * add tool workflows. * update field. * fix: remove chat error. * Fix err. * fix error. * remove cn comment. * update workflows files. * update infra config. * move workflows. * feat: update bot. * fix: solve uncorrect outdated msg get. * update get docIDs logic. * update * update skip logic. * fix * update. * fix: delay deleteObject func. * remove unused content. * update log type. * feat: implement request batch count limit. * update * update * feat: add rocksTimeout * feat: wrap logs * feat: add logs * feat: listen config * feat: enable listen TIME_WAIT port * feat: add logs * feat: cache batch * chore: enable fullUserCache * feat: push rpc num * feat: push err * feat: with operationID * feat: sleep * feat: change 1s * feat: change log * feat: implement Getbatch in rpcCache. * feat: print getOnline cost * feat: change log * feat: change kafka and push config * feat: del interface * feat: fix err * feat: change config * feat: go mod * feat: change config * feat: change config * feat: add sleep in push * feat: warn logs * feat: logs * feat: logs * feat: change port * feat: start config * feat: remove port reuse * feat: prometheus config * feat: prometheus config * feat: prometheus config * feat: add long time send msg to grafana * feat: init * feat: init * feat: implement offline push. * feat: batch get user online * feat: implement batch Push spilt * update go mod * Revert "feat: change port" This reverts commit 06d5e944 * feat: change port * feat: change config * feat: implement kafka producer and consumer. * update format, * add PushMQ log. * feat: get all online users and init push * feat: lock in online cache * feat: config * fix: init online status * fix: add logs * fix: userIDs * fix: add logs * feat: update Handler logic. * update MQ logic. * update * update * fix: method name * fix: update OfflinePushConsumerHandler. * fix: prommetrics * fix: add logs * fix: ctx * fix: log * fix: config * feat: change port * fix: atomic online cache status --------- Co-authored-by: Monet Lee * feature: add GetConversationsHasReadAndMaxSeq interface to the WebSocket API. (#2611) * fix: lru lock (#2613) * fix: lru lock * fix: lru lock * fix: lru lock * fix: nil pointer error on close (#2618) * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * update gomake version * update gomake version * fix: seq conversion bug * fix: redis pipe exec * fix: ImportFriends * fix: A large number of logs keysAndValues ​​length is not even * feat: mark read aggregate write * feat: online status supports redis cluster * feat: online status supports redis cluster * feat: online status supports redis cluster * merge * merge * read seq is written to mongo * read seq is written to mongo * fix: invitation to join group notification * fix: friend op_user_id * feat: optimizing asynchronous context * feat: optimizing memamq size * feat: add GetSeqMessage * feat: GroupApplicationAgreeMemberEnterNotification * feat: GroupApplicationAgreeMemberEnterNotification * feat: go.mod * feat: go.mod * feat: join group notification and get seq * feat: join group notification and get seq * feat: avoid pulling messages from sessions with a large number of max seq values of 0 * feat: API supports gzip * go.mod * fix: nil pointer error on close --------- Co-authored-by: withchao * feat: create group can push notification (#2617) * fix: blockage caused by listen error (#2620) * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * update gomake version * update gomake version * fix: seq conversion bug * fix: redis pipe exec * fix: ImportFriends * fix: A large number of logs keysAndValues ​​length is not even * feat: mark read aggregate write * feat: online status supports redis cluster * feat: online status supports redis cluster * feat: online status supports redis cluster * merge * merge * read seq is written to mongo * read seq is written to mongo * fix: invitation to join group notification * fix: friend op_user_id * feat: optimizing asynchronous context * feat: optimizing memamq size * feat: add GetSeqMessage * feat: GroupApplicationAgreeMemberEnterNotification * feat: GroupApplicationAgreeMemberEnterNotification * feat: go.mod * feat: go.mod * feat: join group notification and get seq * feat: join group notification and get seq * feat: avoid pulling messages from sessions with a large number of max seq values of 0 * feat: API supports gzip * go.mod * fix: nil pointer error on close * fix: listen error --------- Co-authored-by: withchao * fix: go.mod (#2621) * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * update gomake version * update gomake version * fix: seq conversion bug * fix: redis pipe exec * fix: ImportFriends * fix: A large number of logs keysAndValues ​​length is not even * feat: mark read aggregate write * feat: online status supports redis cluster * feat: online status supports redis cluster * feat: online status supports redis cluster * merge * merge * read seq is written to mongo * read seq is written to mongo * fix: invitation to join group notification * fix: friend op_user_id * feat: optimizing asynchronous context * feat: optimizing memamq size * feat: add GetSeqMessage * feat: GroupApplicationAgreeMemberEnterNotification * feat: GroupApplicationAgreeMemberEnterNotification * feat: go.mod * feat: go.mod * feat: join group notification and get seq * feat: join group notification and get seq * feat: avoid pulling messages from sessions with a large number of max seq values of 0 * feat: API supports gzip * go.mod * fix: nil pointer error on close * fix: listen error * fix: listen error * update go.mod --------- Co-authored-by: withchao * feat: improve searchMsg implement. (#2614) * refactor: refactor workflows contents. * add tool workflows. * update field. * fix: remove chat error. * Fix err. * fix error. * remove cn comment. * update workflows files. * update infra config. * move workflows. * feat: update bot. * fix: solve uncorrect outdated msg get. * update get docIDs logic. * update * update skip logic. * fix * update. * fix: delay deleteObject func. * remove unused content. * update log type. * feat: implement request batch count limit. * update * update * remove unused script. * feat: improve searchMsg implement. * update mongo config. * Fix lock (#2622) * fix:log * fix: lock * fix: update setGroupInfoEX field name. (#2625) * refactor: refactor workflows contents. * add tool workflows. * update field. * fix: remove chat error. * Fix err. * fix error. * remove cn comment. * update workflows files. * update infra config. * move workflows. * feat: update bot. * fix: solve uncorrect outdated msg get. * update get docIDs logic. * update * update skip logic. * fix * update. * fix: delay deleteObject func. * remove unused content. * update log type. * feat: implement request batch count limit. * update * update * fix: update setGroupInfoEX field name. * fix: update setGroupInfoEX field name (#2626) * refactor: refactor workflows contents. * add tool workflows. * update field. * fix: remove chat error. * Fix err. * fix error. * remove cn comment. * update workflows files. * update infra config. * move workflows. * feat: update bot. * fix: solve uncorrect outdated msg get. * update get docIDs logic. * update * update skip logic. * fix * update. * fix: delay deleteObject func. * remove unused content. * update log type. * feat: implement request batch count limit. * update * update * fix: update setGroupInfoEX field name. * fix: update setGroupInfoEX field name * feat: msg gateway add log (#2631) * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * update gomake version * update gomake version * fix: seq conversion bug * fix: redis pipe exec * fix: ImportFriends * fix: A large number of logs keysAndValues ​​length is not even * feat: mark read aggregate write * feat: online status supports redis cluster * feat: online status supports redis cluster * feat: online status supports redis cluster * merge * merge * read seq is written to mongo * read seq is written to mongo * fix: invitation to join group notification * fix: friend op_user_id * feat: optimizing asynchronous context * feat: optimizing memamq size * feat: add GetSeqMessage * feat: GroupApplicationAgreeMemberEnterNotification * feat: GroupApplicationAgreeMemberEnterNotification * feat: go.mod * feat: go.mod * feat: join group notification and get seq * feat: join group notification and get seq * feat: avoid pulling messages from sessions with a large number of max seq values of 0 * feat: API supports gzip * go.mod * fix: nil pointer error on close * fix: listen error * fix: listen error * update go.mod * feat: add log --------- Co-authored-by: withchao * fix: update setGroupInfoEx func name and field. (#2634) * refactor: refactor workflows contents. * add tool workflows. * update field. * fix: remove chat error. * Fix err. * fix error. * remove cn comment. * update workflows files. * update infra config. * move workflows. * feat: update bot. * fix: solve uncorrect outdated msg get. * update get docIDs logic. * update * update skip logic. * fix * update. * fix: delay deleteObject func. * remove unused content. * update log type. * feat: implement request batch count limit. * update * update * fix: update setGroupInfoEx func name and field. * refactor: update groupinfoEx field. * refactor: update database name in mongodb.yml * add groupName Condition * fix: fix setConversations req fill. (#2645) * refactor: refactor workflows contents. * add tool workflows. * update field. * fix: remove chat error. * Fix err. * fix error. * remove cn comment. * update workflows files. * update infra config. * move workflows. * feat: update bot. * fix: solve uncorrect outdated msg get. * update get docIDs logic. * update * update skip logic. * fix * update. * fix: delay deleteObject func. * remove unused content. * update log type. * feat: implement request batch count limit. * update * update * fix: fix setConversations req fill. * fix: GetMsgBySeqs boundary issues (#2647) * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * update gomake version * update gomake version * fix: seq conversion bug * fix: redis pipe exec * fix: ImportFriends * fix: A large number of logs keysAndValues ​​length is not even * feat: mark read aggregate write * feat: online status supports redis cluster * feat: online status supports redis cluster * feat: online status supports redis cluster * merge * merge * read seq is written to mongo * read seq is written to mongo * fix: invitation to join group notification * fix: friend op_user_id * feat: optimizing asynchronous context * feat: optimizing memamq size * feat: add GetSeqMessage * feat: GroupApplicationAgreeMemberEnterNotification * feat: GroupApplicationAgreeMemberEnterNotification * feat: go.mod * feat: go.mod * feat: join group notification and get seq * feat: join group notification and get seq * feat: avoid pulling messages from sessions with a large number of max seq values of 0 * feat: API supports gzip * go.mod * fix: nil pointer error on close * fix: listen error * fix: listen error * update go.mod * feat: add log * fix: token parse token value * fix: GetMsgBySeqs boundary issues --------- Co-authored-by: withchao * fix: the attribute version is obsolete, remove it (#2644) * refactor: update Userregister request field. (#2650) --------- Co-authored-by: Monet Lee Co-authored-by: icey-yu <119291641+icey-yu@users.noreply.github.com> Co-authored-by: chao <48119764+withchao@users.noreply.github.com> Co-authored-by: withchao Co-authored-by: 蔡相跃 * update go mod * fix: sn_ not sort * fix: sn_ not sort * fix: sn_ not sort * fix: jssdk add * fix: jssdk support * fix: jssdk support * fix: jssdk support * fix: the message I sent is not set to read seq in mongodb * fix: cannot modify group member avatars * merge: update code from main to v3.8-js-sdk-only. (#2720) * fix: fix update groupName invalid. (#2673) * refactor: change platform to platformID (#2670) * feat: don`t return nil data (#2675) Co-authored-by: Monet Lee * refactor: update fields type in userStatus and check registered. (#2676) * fix: usertoken auth. (#2677) * refactor: update fields type in userStatus and check registered. * fix: usertoken auth. * update contents. * update content. * update * fix * update pb file. * feat: add friend agree after callback (#2680) * fix: sn not sort (#2682) * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * update gomake version * update gomake version * fix: seq conversion bug * fix: redis pipe exec * fix: ImportFriends * fix: A large number of logs keysAndValues ​​length is not even * feat: mark read aggregate write * feat: online status supports redis cluster * feat: online status supports redis cluster * feat: online status supports redis cluster * merge * merge * read seq is written to mongo * read seq is written to mongo * fix: invitation to join group notification * fix: friend op_user_id * feat: optimizing asynchronous context * feat: optimizing memamq size * feat: add GetSeqMessage * feat: GroupApplicationAgreeMemberEnterNotification * feat: GroupApplicationAgreeMemberEnterNotification * feat: go.mod * feat: go.mod * feat: join group notification and get seq * feat: join group notification and get seq * feat: avoid pulling messages from sessions with a large number of max seq values of 0 * feat: API supports gzip * go.mod * fix: nil pointer error on close * fix: listen error * fix: listen error * update go.mod * feat: add log * fix: token parse token value * fix: GetMsgBySeqs boundary issues * fix: sn_ not sort * fix: sn_ not sort * fix: sn_ not sort --------- Co-authored-by: withchao * refactor: add GetAdminToken interface. (#2684) * refactor: add GetAdminToken interface. * update config. * fix: admin token (#2686) * fix: update workflows logic. (#2688) * refactor: add GetAdminToken interface. * update config. * update workflows logic. * fix: admin token (#2687) * update the front image (#2692) * update the front image * update version * feat: improve publish docker image workflows (#2697) * refactor: add GetAdminToken interface. * update config. * update workflows logic. * feat: improve publish docker image workflows * update condition logic. * fix: update load file logic. (#2700) * refactor: add GetAdminToken interface. * update config. * update workflows logic. * feat: improve publish docker image workflows * update condition logic. * fix: update load file logic. * feat: Msg filter (#2703) * feat: msg filter * feat: msg filter * feat: msg filter * feat: provide the interface required by js sdk (#2712) * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * update gomake version * update gomake version * fix: seq conversion bug * fix: redis pipe exec * fix: ImportFriends * fix: A large number of logs keysAndValues ​​length is not even * feat: mark read aggregate write * feat: online status supports redis cluster * feat: online status supports redis cluster * feat: online status supports redis cluster * merge * merge * read seq is written to mongo * read seq is written to mongo * fix: invitation to join group notification * fix: friend op_user_id * feat: optimizing asynchronous context * feat: optimizing memamq size * feat: add GetSeqMessage * feat: GroupApplicationAgreeMemberEnterNotification * feat: GroupApplicationAgreeMemberEnterNotification * feat: go.mod * feat: go.mod * feat: join group notification and get seq * feat: join group notification and get seq * feat: avoid pulling messages from sessions with a large number of max seq values of 0 * feat: API supports gzip * go.mod * fix: nil pointer error on close * fix: listen error * fix: listen error * update go.mod * feat: add log * fix: token parse token value * fix: GetMsgBySeqs boundary issues * fix: sn_ not sort * fix: sn_ not sort * fix: sn_ not sort * fix: jssdk add * fix: jssdk support * fix: jssdk support * fix: jssdk support --------- Co-authored-by: withchao * Line webhook (#2716) * feat: online and offline webhook * feat: online and offline webhook * feat: remove zk * fix: the message I sent is not set to read seq in mongodb (#2718) * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * update gomake version * update gomake version * fix: seq conversion bug * fix: redis pipe exec * fix: ImportFriends * fix: A large number of logs keysAndValues ​​length is not even * feat: mark read aggregate write * feat: online status supports redis cluster * feat: online status supports redis cluster * feat: online status supports redis cluster * merge * merge * read seq is written to mongo * read seq is written to mongo * fix: invitation to join group notification * fix: friend op_user_id * feat: optimizing asynchronous context * feat: optimizing memamq size * feat: add GetSeqMessage * feat: GroupApplicationAgreeMemberEnterNotification * feat: GroupApplicationAgreeMemberEnterNotification * feat: go.mod * feat: go.mod * feat: join group notification and get seq * feat: join group notification and get seq * feat: avoid pulling messages from sessions with a large number of max seq values of 0 * feat: API supports gzip * go.mod * fix: nil pointer error on close * fix: listen error * fix: listen error * update go.mod * feat: add log * fix: token parse token value * fix: GetMsgBySeqs boundary issues * fix: sn_ not sort * fix: sn_ not sort * fix: sn_ not sort * fix: jssdk add * fix: jssdk support * fix: jssdk support * fix: jssdk support * fix: the message I sent is not set to read seq in mongodb --------- Co-authored-by: withchao * fix: cannot modify group member avatars (#2719) * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * update gomake version * update gomake version * fix: seq conversion bug * fix: redis pipe exec * fix: ImportFriends * fix: A large number of logs keysAndValues ​​length is not even * feat: mark read aggregate write * feat: online status supports redis cluster * feat: online status supports redis cluster * feat: online status supports redis cluster * merge * merge * read seq is written to mongo * read seq is written to mongo * fix: invitation to join group notification * fix: friend op_user_id * feat: optimizing asynchronous context * feat: optimizing memamq size * feat: add GetSeqMessage * feat: GroupApplicationAgreeMemberEnterNotification * feat: GroupApplicationAgreeMemberEnterNotification * feat: go.mod * feat: go.mod * feat: join group notification and get seq * feat: join group notification and get seq * feat: avoid pulling messages from sessions with a large number of max seq values of 0 * feat: API supports gzip * go.mod * fix: nil pointer error on close * fix: listen error * fix: listen error * update go.mod * feat: add log * fix: token parse token value * fix: GetMsgBySeqs boundary issues * fix: sn_ not sort * fix: sn_ not sort * fix: sn_ not sort * fix: jssdk add * fix: jssdk support * fix: jssdk support * fix: jssdk support * fix: the message I sent is not set to read seq in mongodb * fix: cannot modify group member avatars --------- Co-authored-by: withchao --------- Co-authored-by: Monet Lee Co-authored-by: icey-yu <119291641+icey-yu@users.noreply.github.com> Co-authored-by: chao <48119764+withchao@users.noreply.github.com> Co-authored-by: withchao Co-authored-by: skiffer-git <72860476+skiffer-git@users.noreply.github.com> * fix: MemberEnterNotification * fix: MemberEnterNotification * fix: MsgData status * merge * merge: update code from main to v3.8-js-sdk-only. (#2818) * feat: implement merge milestone PR to target-branch. (#2796) * build: improve workflows logic. (#2801) * fix: improve time condition check mehtod. (#2804) * fix: improve time condition check mehtod. * fix * fix: webhook before online push (#2805) * fix: set own read seq in MongoDB when sender send a message. (#2808) * fix: solve err Notification when setGroupInfo. (#2806) * fix: solve err Notification when setGroupInfo. * build: update checkout version. * fix: update notification contents. * Introducing OpenIM Guru on Gurubase.io (#2788) * feat: support app update service (#2811) * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * update gomake version * update gomake version * fix: seq conversion bug * fix: redis pipe exec * fix: ImportFriends * fix: A large number of logs keysAndValues ​​length is not even * feat: mark read aggregate write * feat: online status supports redis cluster * feat: online status supports redis cluster * feat: online status supports redis cluster * merge * merge * read seq is written to mongo * read seq is written to mongo * fix: invitation to join group notification * fix: friend op_user_id * feat: optimizing asynchronous context * feat: optimizing memamq size * feat: add GetSeqMessage * feat: GroupApplicationAgreeMemberEnterNotification * feat: GroupApplicationAgreeMemberEnterNotification * feat: go.mod * feat: go.mod * feat: join group notification and get seq * feat: join group notification and get seq * feat: avoid pulling messages from sessions with a large number of max seq values of 0 * feat: API supports gzip * go.mod * fix: nil pointer error on close * fix: listen error * fix: listen error * update go.mod * feat: add log * fix: token parse token value * fix: GetMsgBySeqs boundary issues * fix: sn_ not sort * fix: sn_ not sort * fix: sn_ not sort * fix: jssdk add * fix: jssdk support * fix: jssdk support * fix: jssdk support * fix: the message I sent is not set to read seq in mongodb * fix: cannot modify group member avatars * fix: MemberEnterNotification * fix: MemberEnterNotification * fix: MsgData status * feat: add ApplicationVersion * feat: add ApplicationVersion * feat: add ApplicationVersion --------- Co-authored-by: withchao * feat: ApplicationVersion move chat (#2813) * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * update gomake version * update gomake version * fix: seq conversion bug * fix: redis pipe exec * fix: ImportFriends * fix: A large number of logs keysAndValues ​​length is not even * feat: mark read aggregate write * feat: online status supports redis cluster * feat: online status supports redis cluster * feat: online status supports redis cluster * merge * merge * read seq is written to mongo * read seq is written to mongo * fix: invitation to join group notification * fix: friend op_user_id * feat: optimizing asynchronous context * feat: optimizing memamq size * feat: add GetSeqMessage * feat: GroupApplicationAgreeMemberEnterNotification * feat: GroupApplicationAgreeMemberEnterNotification * feat: go.mod * feat: go.mod * feat: join group notification and get seq * feat: join group notification and get seq * feat: avoid pulling messages from sessions with a large number of max seq values of 0 * feat: API supports gzip * go.mod * fix: nil pointer error on close * fix: listen error * fix: listen error * update go.mod * feat: add log * fix: token parse token value * fix: GetMsgBySeqs boundary issues * fix: sn_ not sort * fix: sn_ not sort * fix: sn_ not sort * fix: jssdk add * fix: jssdk support * fix: jssdk support * fix: jssdk support * fix: the message I sent is not set to read seq in mongodb * fix: cannot modify group member avatars * fix: MemberEnterNotification * fix: MemberEnterNotification * fix: MsgData status * feat: add ApplicationVersion * feat: ApplicationVersion move chat --------- Co-authored-by: withchao * fix: improve condition check. (#2815) --------- Co-authored-by: Monet Lee Co-authored-by: icey-yu <119291641+icey-yu@users.noreply.github.com> Co-authored-by: Kürşat Aktaş Co-authored-by: chao <48119764+withchao@users.noreply.github.com> Co-authored-by: withchao * feat: support text ping pong * feat: support text ping pong * feat: gob json encoder * feat: gob json encoder * feat: gob json encoder * feat: gob json encoder * feat: gob json encoder * feat: gob json encoder * fix: concurrent write to websocket connection * fix: concurrent write to websocket connection * fix: concurrent write to websocket connection * fix: group member update face_url * feat: seq user hook set conversation * feat: seq user hook set conversation * feat: seq user hook set conversation * feat: seq user hook set conversation --------- Co-authored-by: withchao Co-authored-by: Monet Lee Co-authored-by: OpenIM-Gordon <46924906+FGadvancer@users.noreply.github.com> Co-authored-by: icey-yu <119291641+icey-yu@users.noreply.github.com> Co-authored-by: 蔡相跃 Co-authored-by: skiffer-git <72860476+skiffer-git@users.noreply.github.com> Co-authored-by: Kürşat Aktaş --- go.mod | 2 +- go.sum | 4 ++-- internal/rpc/conversation/conversation.go | 19 ++++++++++++++++++- internal/rpc/group/group.go | 1 - internal/rpc/msg/seq.go | 18 ++++++++++++++++++ internal/rpc/msg/server.go | 1 - internal/rpc/third/log.go | 2 +- pkg/common/storage/controller/msg.go | 11 +++++++++++ 8 files changed, 51 insertions(+), 7 deletions(-) diff --git a/go.mod b/go.mod index 8eae1edb2..840016646 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( github.com/gorilla/websocket v1.5.1 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/mitchellh/mapstructure v1.5.0 - github.com/openimsdk/protocol v0.0.72-alpha.59 + github.com/openimsdk/protocol v0.0.72-alpha.61 github.com/openimsdk/tools v0.0.50-alpha.38 github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 diff --git a/go.sum b/go.sum index e572f0b47..1ca89efd4 100644 --- a/go.sum +++ b/go.sum @@ -319,8 +319,8 @@ github.com/onsi/gomega v1.25.0 h1:Vw7br2PCDYijJHSfBOWhov+8cAnUf8MfMaIOV323l6Y= github.com/onsi/gomega v1.25.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdMPGhM= github.com/openimsdk/gomake v0.0.14-alpha.5 h1:VY9c5x515lTfmdhhPjMvR3BBRrRquAUCFsz7t7vbv7Y= github.com/openimsdk/gomake v0.0.14-alpha.5/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= -github.com/openimsdk/protocol v0.0.72-alpha.59 h1:+ycb2+68mLKPIo7VrxF0id/GXP6OqZ2/nBM1YZQr7qY= -github.com/openimsdk/protocol v0.0.72-alpha.59/go.mod h1:Iet+piS/jaS+kWWyj6EEr36mk4ISzIRYjoMSVA4dq2M= +github.com/openimsdk/protocol v0.0.72-alpha.61 h1:RuZR9/Sg3p6Bpb2CKPjPoA2AUmTvHITmhZ3PT/RbWMs= +github.com/openimsdk/protocol v0.0.72-alpha.61/go.mod h1:Iet+piS/jaS+kWWyj6EEr36mk4ISzIRYjoMSVA4dq2M= github.com/openimsdk/tools v0.0.50-alpha.38 h1:AU6/cvDfN4ciIOwAj8IWEwze3DeEp2cHYPgW3y0OlbU= github.com/openimsdk/tools v0.0.50-alpha.38/go.mod h1:/Em/fQH46CuWf60+hcmvZyboGCQpSDEb2MdQ4nmQRAk= github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= diff --git a/internal/rpc/conversation/conversation.go b/internal/rpc/conversation/conversation.go index 0b6b656a4..6a8a6d800 100644 --- a/internal/rpc/conversation/conversation.go +++ b/internal/rpc/conversation/conversation.go @@ -16,6 +16,8 @@ package conversation import ( "context" + "github.com/openimsdk/open-im-server/v3/pkg/msgprocessor" + pbmsg "github.com/openimsdk/protocol/msg" "sort" "time" @@ -433,23 +435,38 @@ func (c *conversationServer) CreateGroupChatConversations(ctx context.Context, r if err != nil { return nil, err } + conversationID := msgprocessor.GetConversationIDBySessionType(constant.ReadGroupChatType, req.GroupID) + if _, err := c.msgRpcClient.Client.SetUserConversationMaxSeq(ctx, &pbmsg.SetUserConversationMaxSeqReq{ConversationID: conversationID, OwnerUserID: req.UserIDs, MaxSeq: 0}); err != nil { + return nil, err + } return &pbconversation.CreateGroupChatConversationsResp{}, nil } func (c *conversationServer) SetConversationMaxSeq(ctx context.Context, req *pbconversation.SetConversationMaxSeqReq) (*pbconversation.SetConversationMaxSeqResp, error) { + if _, err := c.msgRpcClient.Client.SetUserConversationMaxSeq(ctx, &pbmsg.SetUserConversationMaxSeqReq{ConversationID: req.ConversationID, OwnerUserID: req.OwnerUserID, MaxSeq: req.MaxSeq}); err != nil { + return nil, err + } if err := c.conversationDatabase.UpdateUsersConversationField(ctx, req.OwnerUserID, req.ConversationID, map[string]any{"max_seq": req.MaxSeq}); err != nil { return nil, err } - + for _, userID := range req.OwnerUserID { + c.conversationNotificationSender.ConversationChangeNotification(ctx, userID, []string{req.ConversationID}) + } return &pbconversation.SetConversationMaxSeqResp{}, nil } func (c *conversationServer) SetConversationMinSeq(ctx context.Context, req *pbconversation.SetConversationMinSeqReq) (*pbconversation.SetConversationMinSeqResp, error) { + if _, err := c.msgRpcClient.Client.SetUserConversationMinSeq(ctx, &pbmsg.SetUserConversationMinSeqReq{ConversationID: req.ConversationID, OwnerUserID: req.OwnerUserID, MinSeq: req.MinSeq}); err != nil { + return nil, err + } if err := c.conversationDatabase.UpdateUsersConversationField(ctx, req.OwnerUserID, req.ConversationID, map[string]any{"min_seq": req.MinSeq}); err != nil { return nil, err } + for _, userID := range req.OwnerUserID { + c.conversationNotificationSender.ConversationChangeNotification(ctx, userID, []string{req.ConversationID}) + } return &pbconversation.SetConversationMinSeqResp{}, nil } diff --git a/internal/rpc/group/group.go b/internal/rpc/group/group.go index ba0c8a42d..9e610df0f 100644 --- a/internal/rpc/group/group.go +++ b/internal/rpc/group/group.go @@ -964,7 +964,6 @@ func (g *groupServer) deleteMemberAndSetConversationSeq(ctx context.Context, gro if err != nil { return err } - return g.conversationRpcClient.SetConversationMaxSeq(ctx, userIDs, conevrsationID, maxSeq) } diff --git a/internal/rpc/msg/seq.go b/internal/rpc/msg/seq.go index 7f5fa1adb..bd68138fb 100644 --- a/internal/rpc/msg/seq.go +++ b/internal/rpc/msg/seq.go @@ -84,3 +84,21 @@ func (m *msgServer) GetActiveConversation(ctx context.Context, req *pbmsg.GetAct } return &pbmsg.GetActiveConversationResp{Conversations: conversations}, nil } + +func (m *msgServer) SetUserConversationMaxSeq(ctx context.Context, req *pbmsg.SetUserConversationMaxSeqReq) (*pbmsg.SetUserConversationMaxSeqResp, error) { + for _, userID := range req.OwnerUserID { + if err := m.MsgDatabase.SetUserConversationsMaxSeq(ctx, req.ConversationID, userID, req.MaxSeq); err != nil { + return nil, err + } + } + return &pbmsg.SetUserConversationMaxSeqResp{}, nil +} + +func (m *msgServer) SetUserConversationMinSeq(ctx context.Context, req *pbmsg.SetUserConversationMinSeqReq) (*pbmsg.SetUserConversationMinSeqResp, error) { + for _, userID := range req.OwnerUserID { + if err := m.MsgDatabase.SetUserConversationsMinSeq(ctx, req.ConversationID, userID, req.MinSeq); err != nil { + return nil, err + } + } + return &pbmsg.SetUserConversationMinSeqResp{}, nil +} diff --git a/internal/rpc/msg/server.go b/internal/rpc/msg/server.go index 6d5922ce3..758a7cf19 100644 --- a/internal/rpc/msg/server.go +++ b/internal/rpc/msg/server.go @@ -16,7 +16,6 @@ package msg import ( "context" - "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/redis" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database/mgo" diff --git a/internal/rpc/third/log.go b/internal/rpc/third/log.go index 657ea1689..cb4678b34 100644 --- a/internal/rpc/third/log.go +++ b/internal/rpc/third/log.go @@ -55,7 +55,7 @@ func (t *thirdServer) UploadLogs(ctx context.Context, req *third.UploadLogsReq) CreateTime: time.Now(), Url: fileURL.URL, FileName: fileURL.Filename, - SystemType: req.SystemType, + SystemType: req.AppFramework, Version: req.Version, Ex: req.Ex, } diff --git a/pkg/common/storage/controller/msg.go b/pkg/common/storage/controller/msg.go index 85b797c40..464ad7604 100644 --- a/pkg/common/storage/controller/msg.go +++ b/pkg/common/storage/controller/msg.go @@ -97,6 +97,9 @@ type CommonMsgDatabase interface { DeleteDocMsgBefore(ctx context.Context, ts int64, doc *model.MsgDocModel) ([]int, error) GetDocIDs(ctx context.Context) ([]string, error) + + SetUserConversationsMaxSeq(ctx context.Context, conversationID string, userID string, seq int64) error + SetUserConversationsMinSeq(ctx context.Context, conversationID string, userID string, seq int64) error } func NewCommonMsgDatabase(msgDocModel database.Msg, msg cache.MsgCache, seqUser cache.SeqUser, seqConversation cache.SeqConversationCache, kafkaConf *config.Kafka) (CommonMsgDatabase, error) { @@ -702,6 +705,14 @@ func (db *commonMsgDatabase) SetUserConversationsMinSeqs(ctx context.Context, us return db.seqUser.SetUserMinSeqs(ctx, userID, seqs) } +func (db *commonMsgDatabase) SetUserConversationsMaxSeq(ctx context.Context, conversationID string, userID string, seq int64) error { + return db.seqUser.SetUserMaxSeq(ctx, conversationID, userID, seq) +} + +func (db *commonMsgDatabase) SetUserConversationsMinSeq(ctx context.Context, conversationID string, userID string, seq int64) error { + return db.seqUser.SetUserMinSeq(ctx, conversationID, userID, seq) +} + func (db *commonMsgDatabase) UserSetHasReadSeqs(ctx context.Context, userID string, hasReadSeqs map[string]int64) error { return db.seqUser.SetUserReadSeqs(ctx, userID, hasReadSeqs) } From fceaaa199b461c77c0526ece9ab3087bf04a4ffd Mon Sep 17 00:00:00 2001 From: icey-yu <119291641+icey-yu@users.noreply.github.com> Date: Fri, 6 Dec 2024 17:25:42 +0800 Subject: [PATCH 062/199] feat: compatible autoSetPorts (#2929) --- config/openim-msggateway.yml | 5 + config/openim-push.yml | 6 ++ config/openim-rpc-auth.yml | 5 + config/openim-rpc-conversation.yml | 5 + config/openim-rpc-friend.yml | 5 + config/openim-rpc-group.yml | 5 + config/openim-rpc-msg.yml | 5 + config/openim-rpc-third.yml | 5 + config/openim-rpc-user.yml | 5 + internal/msggateway/hub_server.go | 3 +- internal/msggateway/init.go | 3 +- internal/tools/addr/addr.go | 165 ----------------------------- pkg/common/cmd/auth.go | 1 + pkg/common/cmd/conversation.go | 1 + pkg/common/cmd/friend.go | 1 + pkg/common/cmd/group.go | 1 + pkg/common/cmd/msg.go | 1 + pkg/common/cmd/push.go | 1 + pkg/common/cmd/third.go | 1 + pkg/common/cmd/user.go | 1 + pkg/common/config/config.go | 60 ++++++----- pkg/common/startrpc/start.go | 108 ++++++++++--------- 22 files changed, 150 insertions(+), 243 deletions(-) delete mode 100644 internal/tools/addr/addr.go diff --git a/config/openim-msggateway.yml b/config/openim-msggateway.yml index 84e232ccb..74eab35d5 100644 --- a/config/openim-msggateway.yml +++ b/config/openim-msggateway.yml @@ -1,6 +1,11 @@ rpc: # The IP address where this RPC service registers itself; if left blank, it defaults to the internal network IP registerIP: + # autoSetPorts indicates whether to automatically set the ports + autoSetPorts: true + # List of ports that the RPC service listens on; configuring multiple ports will launch multiple instances. These must match the number of configured prometheus ports + # It will only take effect when autoSetPorts is set to false. + ports: [ 10140, 10141, 10142, 10143, 10144, 10145, 10146, 10147, 10148, 10149, 10150, 10151, 10152, 10153, 10154, 10155 ] prometheus: # Enable or disable Prometheus monitoring diff --git a/config/openim-push.yml b/config/openim-push.yml index 2db09b37e..53c88ed41 100644 --- a/config/openim-push.yml +++ b/config/openim-push.yml @@ -3,6 +3,12 @@ rpc: registerIP: # IP address that the RPC service listens on; setting to 0.0.0.0 listens on both internal and external IPs. If left blank, it automatically uses the internal network IP listenIP: 0.0.0.0 + # autoSetPorts indicates whether to automatically set the ports + autoSetPorts: true + # List of ports that the RPC service listens on; configuring multiple ports will launch multiple instances. These must match the number of configured prometheus ports + # It will only take effect when autoSetPorts is set to false. + ports: [ 10170, 10171, 10172, 10173, 10174, 10175, 10176, 10177, 10178, 10179, 10180, 10181, 10182, 10183, 10184, 10185 ] + prometheus: # Enable or disable Prometheus monitoring diff --git a/config/openim-rpc-auth.yml b/config/openim-rpc-auth.yml index 9ebd78070..230aa8720 100644 --- a/config/openim-rpc-auth.yml +++ b/config/openim-rpc-auth.yml @@ -3,6 +3,11 @@ rpc: registerIP: # IP address that the RPC service listens on; setting to 0.0.0.0 listens on both internal and external IPs. If left blank, it automatically uses the internal network IP listenIP: 0.0.0.0 + # autoSetPorts indicates whether to automatically set the ports + autoSetPorts: true + # List of ports that the RPC service listens on; configuring multiple ports will launch multiple instances. These must match the number of configured prometheus ports + # It will only take effect when autoSetPorts is set to false. + ports: [ 10200 ] prometheus: diff --git a/config/openim-rpc-conversation.yml b/config/openim-rpc-conversation.yml index 1cb56ac60..ed61f7010 100644 --- a/config/openim-rpc-conversation.yml +++ b/config/openim-rpc-conversation.yml @@ -3,6 +3,11 @@ rpc: registerIP: # IP address that the RPC service listens on; setting to 0.0.0.0 listens on both internal and external IPs. If left blank, it automatically uses the internal network IP listenIP: 0.0.0.0 + # autoSetPorts indicates whether to automatically set the ports + autoSetPorts: true + # List of ports that the RPC service listens on; configuring multiple ports will launch multiple instances. These must match the number of configured prometheus ports + # It will only take effect when autoSetPorts is set to false. + ports: [ 10220 ] prometheus: # Enable or disable Prometheus monitoring diff --git a/config/openim-rpc-friend.yml b/config/openim-rpc-friend.yml index 11b805299..e10ef496a 100644 --- a/config/openim-rpc-friend.yml +++ b/config/openim-rpc-friend.yml @@ -3,6 +3,11 @@ rpc: registerIP: # IP address that the RPC service listens on; setting to 0.0.0.0 listens on both internal and external IPs. If left blank, it automatically uses the internal network IP listenIP: 0.0.0.0 + # autoSetPorts indicates whether to automatically set the ports + autoSetPorts: true + # List of ports that the RPC service listens on; configuring multiple ports will launch multiple instances. These must match the number of configured prometheus ports + # It will only take effect when autoSetPorts is set to false. + ports: [ 10240 ] prometheus: # Enable or disable Prometheus monitoring diff --git a/config/openim-rpc-group.yml b/config/openim-rpc-group.yml index d16ae7b02..9f5ceabab 100644 --- a/config/openim-rpc-group.yml +++ b/config/openim-rpc-group.yml @@ -3,6 +3,11 @@ rpc: registerIP: # IP address that the RPC service listens on; setting to 0.0.0.0 listens on both internal and external IPs. If left blank, it automatically uses the internal network IP listenIP: 0.0.0.0 + # autoSetPorts indicates whether to automatically set the ports + autoSetPorts: true + # List of ports that the RPC service listens on; configuring multiple ports will launch multiple instances. These must match the number of configured prometheus ports + # It will only take effect when autoSetPorts is set to false. + ports: [ 10260 ] prometheus: # Enable or disable Prometheus monitoring diff --git a/config/openim-rpc-msg.yml b/config/openim-rpc-msg.yml index 5917ff5ae..72e055a5d 100644 --- a/config/openim-rpc-msg.yml +++ b/config/openim-rpc-msg.yml @@ -3,6 +3,11 @@ rpc: registerIP: # IP address that the RPC service listens on; setting to 0.0.0.0 listens on both internal and external IPs. If left blank, it automatically uses the internal network IP listenIP: 0.0.0.0 + # autoSetPorts indicates whether to automatically set the ports + autoSetPorts: true + # List of ports that the RPC service listens on; configuring multiple ports will launch multiple instances. These must match the number of configured prometheus ports + # It will only take effect when autoSetPorts is set to false. + ports: [ 10280 ] prometheus: # Enable or disable Prometheus monitoring diff --git a/config/openim-rpc-third.yml b/config/openim-rpc-third.yml index 54123d8ea..9578e82c6 100644 --- a/config/openim-rpc-third.yml +++ b/config/openim-rpc-third.yml @@ -3,6 +3,11 @@ rpc: registerIP: # IP address that the RPC service listens on; setting to 0.0.0.0 listens on both internal and external IPs. If left blank, it automatically uses the internal network IP listenIP: 0.0.0.0 + # autoSetPorts indicates whether to automatically set the ports + autoSetPorts: true + # List of ports that the RPC service listens on; configuring multiple ports will launch multiple instances. These must match the number of configured prometheus ports + # It will only take effect when autoSetPorts is set to false. + ports: [ 10300 ] prometheus: # Enable or disable Prometheus monitoring diff --git a/config/openim-rpc-user.yml b/config/openim-rpc-user.yml index 4bd6444a7..6f7b9648b 100644 --- a/config/openim-rpc-user.yml +++ b/config/openim-rpc-user.yml @@ -3,6 +3,11 @@ rpc: registerIP: # Listening IP; 0.0.0.0 means both internal and external IPs are listened to, if blank, the internal network IP is automatically obtained by default listenIP: 0.0.0.0 + # autoSetPorts indicates whether to automatically set the ports + autoSetPorts: true + # List of ports that the RPC service listens on; configuring multiple ports will launch multiple instances. These must match the number of configured prometheus ports + # It will only take effect when autoSetPorts is set to false. + ports: [ 10320 ] prometheus: # Whether to enable prometheus diff --git a/internal/msggateway/hub_server.go b/internal/msggateway/hub_server.go index 04376b27f..b6760ed0f 100644 --- a/internal/msggateway/hub_server.go +++ b/internal/msggateway/hub_server.go @@ -46,7 +46,8 @@ func (s *Server) InitServer(ctx context.Context, config *Config, disCov discover func (s *Server) Start(ctx context.Context, index int, conf *Config) error { return startrpc.Start(ctx, &conf.Discovery, &conf.MsgGateway.Prometheus, conf.MsgGateway.ListenIP, - index, + conf.MsgGateway.RPC.RegisterIP, + conf.MsgGateway.RPC.AutoSetPorts, conf.MsgGateway.RPC.Ports, index, conf.Share.RpcRegisterName.MessageGateway, &conf.Share, conf, diff --git a/internal/msggateway/init.go b/internal/msggateway/init.go index 77bccc88c..1796a5482 100644 --- a/internal/msggateway/init.go +++ b/internal/msggateway/init.go @@ -35,7 +35,8 @@ type Config struct { // Start run ws server. func Start(ctx context.Context, index int, conf *Config) error { - log.CInfo(ctx, "MSG-GATEWAY server is initializing", + log.CInfo(ctx, "MSG-GATEWAY server is initializing", "autoSetPorts", conf.MsgGateway.RPC.AutoSetPorts, + "rpcPorts", conf.MsgGateway.RPC.Ports, "wsPort", conf.MsgGateway.LongConnSvr.Ports, "prometheusPorts", conf.MsgGateway.Prometheus.Ports) wsPort, err := datautil.GetElemByIndex(conf.MsgGateway.LongConnSvr.Ports, index) if err != nil { diff --git a/internal/tools/addr/addr.go b/internal/tools/addr/addr.go deleted file mode 100644 index b7cf8a2db..000000000 --- a/internal/tools/addr/addr.go +++ /dev/null @@ -1,165 +0,0 @@ -// addr provides functions to retrieve local IP addresses from device interfaces. -package addr - -import ( - "net" - - "github.com/pkg/errors" -) - -var ( - // ErrIPNotFound no IP address found, and explicit IP not provided. - ErrIPNotFound = errors.New("no IP address found, and explicit IP not provided") -) - -// IsLocal checks whether an IP belongs to one of the device's interfaces. -func IsLocal(addr string) bool { - // Extract the host - host, _, err := net.SplitHostPort(addr) - if err == nil { - addr = host - } - - if addr == "localhost" { - return true - } - - // Check against all local ips - for _, ip := range IPs() { - if addr == ip { - return true - } - } - - return false -} - -// Extract returns a valid IP address. If the address provided is a valid -// address, it will be returned directly. Otherwise, the available interfaces -// will be iterated over to find an IP address, preferably private. -func Extract(addr string) (string, error) { - // if addr is already specified then it's directly returned - if len(addr) > 0 && (addr != "0.0.0.0" && addr != "[::]" && addr != "::") { - return addr, nil - } - - var ( - addrs []net.Addr - loAddrs []net.Addr - ) - - ifaces, err := net.Interfaces() - if err != nil { - return "", errors.Wrap(err, "failed to get interfaces") - } - - for _, iface := range ifaces { - ifaceAddrs, err := iface.Addrs() - if err != nil { - // ignore error, interface can disappear from system - continue - } - - if iface.Flags&net.FlagLoopback != 0 { - loAddrs = append(loAddrs, ifaceAddrs...) - continue - } - - addrs = append(addrs, ifaceAddrs...) - } - - // Add loopback addresses to the end of the list - addrs = append(addrs, loAddrs...) - - // Try to find private IP in list, public IP otherwise - ip, err := findIP(addrs) - if err != nil { - return "", err - } - - return ip.String(), nil -} - -// IPs returns all available interface IP addresses. -func IPs() []string { - ifaces, err := net.Interfaces() - if err != nil { - return nil - } - - var ipAddrs []string - - for _, i := range ifaces { - addrs, err := i.Addrs() - if err != nil { - continue - } - - for _, addr := range addrs { - var ip net.IP - switch v := addr.(type) { - case *net.IPNet: - ip = v.IP - case *net.IPAddr: - ip = v.IP - } - - if ip == nil { - continue - } - - ipAddrs = append(ipAddrs, ip.String()) - } - } - - return ipAddrs -} - -// findIP will return the first private IP available in the list. -// If no private IP is available it will return the first public IP, if present. -// If no public IP is available, it will return the first loopback IP, if present. -func findIP(addresses []net.Addr) (net.IP, error) { - var publicIP net.IP - var localIP net.IP - - for _, rawAddr := range addresses { - var ip net.IP - switch addr := rawAddr.(type) { - case *net.IPAddr: - ip = addr.IP - case *net.IPNet: - ip = addr.IP - default: - continue - } - - if ip.IsLoopback() { - if localIP == nil { - localIP = ip - } - continue - } - - if !ip.IsPrivate() { - if publicIP == nil { - publicIP = ip - } - continue - } - - // Return private IP if available - return ip, nil - } - - // Return public or virtual IP - if len(publicIP) > 0 { - return publicIP, nil - } - - // Return local IP - if len(localIP) > 0 { - return localIP, nil - } - - return nil, ErrIPNotFound -} diff --git a/pkg/common/cmd/auth.go b/pkg/common/cmd/auth.go index f8232bc26..34f450fb1 100644 --- a/pkg/common/cmd/auth.go +++ b/pkg/common/cmd/auth.go @@ -55,5 +55,6 @@ func (a *AuthRpcCmd) Exec() error { func (a *AuthRpcCmd) runE() error { return startrpc.Start(a.ctx, &a.authConfig.Discovery, &a.authConfig.RpcConfig.Prometheus, a.authConfig.RpcConfig.RPC.ListenIP, + a.authConfig.RpcConfig.RPC.RegisterIP, a.authConfig.RpcConfig.RPC.AutoSetPorts, a.authConfig.RpcConfig.RPC.Ports, a.Index(), a.authConfig.Share.RpcRegisterName.Auth, &a.authConfig.Share, a.authConfig, auth.Start) } diff --git a/pkg/common/cmd/conversation.go b/pkg/common/cmd/conversation.go index 6a0da11e5..f8ec570a7 100644 --- a/pkg/common/cmd/conversation.go +++ b/pkg/common/cmd/conversation.go @@ -57,5 +57,6 @@ func (a *ConversationRpcCmd) Exec() error { func (a *ConversationRpcCmd) runE() error { return startrpc.Start(a.ctx, &a.conversationConfig.Discovery, &a.conversationConfig.RpcConfig.Prometheus, a.conversationConfig.RpcConfig.RPC.ListenIP, + a.conversationConfig.RpcConfig.RPC.RegisterIP, a.conversationConfig.RpcConfig.RPC.AutoSetPorts, a.conversationConfig.RpcConfig.RPC.Ports, a.Index(), a.conversationConfig.Share.RpcRegisterName.Conversation, &a.conversationConfig.Share, a.conversationConfig, conversation.Start) } diff --git a/pkg/common/cmd/friend.go b/pkg/common/cmd/friend.go index 2ff25090e..786be0d8e 100644 --- a/pkg/common/cmd/friend.go +++ b/pkg/common/cmd/friend.go @@ -58,5 +58,6 @@ func (a *FriendRpcCmd) Exec() error { func (a *FriendRpcCmd) runE() error { return startrpc.Start(a.ctx, &a.relationConfig.Discovery, &a.relationConfig.RpcConfig.Prometheus, a.relationConfig.RpcConfig.RPC.ListenIP, + a.relationConfig.RpcConfig.RPC.RegisterIP, a.relationConfig.RpcConfig.RPC.AutoSetPorts, a.relationConfig.RpcConfig.RPC.Ports, a.Index(), a.relationConfig.Share.RpcRegisterName.Friend, &a.relationConfig.Share, a.relationConfig, relation.Start) } diff --git a/pkg/common/cmd/group.go b/pkg/common/cmd/group.go index f1757c7fa..e405c9135 100644 --- a/pkg/common/cmd/group.go +++ b/pkg/common/cmd/group.go @@ -59,5 +59,6 @@ func (a *GroupRpcCmd) Exec() error { func (a *GroupRpcCmd) runE() error { return startrpc.Start(a.ctx, &a.groupConfig.Discovery, &a.groupConfig.RpcConfig.Prometheus, a.groupConfig.RpcConfig.RPC.ListenIP, + a.groupConfig.RpcConfig.RPC.RegisterIP, a.groupConfig.RpcConfig.RPC.AutoSetPorts, a.groupConfig.RpcConfig.RPC.Ports, a.Index(), a.groupConfig.Share.RpcRegisterName.Group, &a.groupConfig.Share, a.groupConfig, group.Start, versionctx.EnableVersionCtx()) } diff --git a/pkg/common/cmd/msg.go b/pkg/common/cmd/msg.go index ae718a11d..3f1a2fa31 100644 --- a/pkg/common/cmd/msg.go +++ b/pkg/common/cmd/msg.go @@ -59,5 +59,6 @@ func (a *MsgRpcCmd) Exec() error { func (a *MsgRpcCmd) runE() error { return startrpc.Start(a.ctx, &a.msgConfig.Discovery, &a.msgConfig.RpcConfig.Prometheus, a.msgConfig.RpcConfig.RPC.ListenIP, + a.msgConfig.RpcConfig.RPC.RegisterIP, a.msgConfig.RpcConfig.RPC.AutoSetPorts, a.msgConfig.RpcConfig.RPC.Ports, a.Index(), a.msgConfig.Share.RpcRegisterName.Msg, &a.msgConfig.Share, a.msgConfig, msg.Start) } diff --git a/pkg/common/cmd/push.go b/pkg/common/cmd/push.go index 2f9e248f1..e0f6b39f6 100644 --- a/pkg/common/cmd/push.go +++ b/pkg/common/cmd/push.go @@ -59,5 +59,6 @@ func (a *PushRpcCmd) Exec() error { func (a *PushRpcCmd) runE() error { return startrpc.Start(a.ctx, &a.pushConfig.Discovery, &a.pushConfig.RpcConfig.Prometheus, a.pushConfig.RpcConfig.RPC.ListenIP, + a.pushConfig.RpcConfig.RPC.RegisterIP, a.pushConfig.RpcConfig.RPC.AutoSetPorts, a.pushConfig.RpcConfig.RPC.Ports, a.Index(), a.pushConfig.Share.RpcRegisterName.Push, &a.pushConfig.Share, a.pushConfig, push.Start) } diff --git a/pkg/common/cmd/third.go b/pkg/common/cmd/third.go index fa1d5d42d..c5a21f91b 100644 --- a/pkg/common/cmd/third.go +++ b/pkg/common/cmd/third.go @@ -58,5 +58,6 @@ func (a *ThirdRpcCmd) Exec() error { func (a *ThirdRpcCmd) runE() error { return startrpc.Start(a.ctx, &a.thirdConfig.Discovery, &a.thirdConfig.RpcConfig.Prometheus, a.thirdConfig.RpcConfig.RPC.ListenIP, + a.thirdConfig.RpcConfig.RPC.RegisterIP, a.thirdConfig.RpcConfig.RPC.AutoSetPorts, a.thirdConfig.RpcConfig.RPC.Ports, a.Index(), a.thirdConfig.Share.RpcRegisterName.Third, &a.thirdConfig.Share, a.thirdConfig, third.Start) } diff --git a/pkg/common/cmd/user.go b/pkg/common/cmd/user.go index 8a22bfce9..1d916ea37 100644 --- a/pkg/common/cmd/user.go +++ b/pkg/common/cmd/user.go @@ -59,5 +59,6 @@ func (a *UserRpcCmd) Exec() error { func (a *UserRpcCmd) runE() error { return startrpc.Start(a.ctx, &a.userConfig.Discovery, &a.userConfig.RpcConfig.Prometheus, a.userConfig.RpcConfig.RPC.ListenIP, + a.userConfig.RpcConfig.RPC.RegisterIP, a.userConfig.RpcConfig.RPC.AutoSetPorts, a.userConfig.RpcConfig.RPC.Ports, a.Index(), a.userConfig.Share.RpcRegisterName.User, &a.userConfig.Share, a.userConfig, user.Start) } diff --git a/pkg/common/config/config.go b/pkg/common/config/config.go index 5fb12b399..ed803359c 100644 --- a/pkg/common/config/config.go +++ b/pkg/common/config/config.go @@ -176,8 +176,9 @@ type Prometheus struct { type MsgGateway struct { RPC struct { - RegisterIP string `mapstructure:"registerIP"` - Ports []int `mapstructure:"ports"` + RegisterIP string `mapstructure:"registerIP"` + AutoSetPorts bool `mapstructure:"autoSetPorts"` + Ports []int `mapstructure:"ports"` } `mapstructure:"rpc"` Prometheus Prometheus `mapstructure:"prometheus"` ListenIP string `mapstructure:"listenIP"` @@ -195,8 +196,10 @@ type MsgTransfer struct { type Push struct { RPC struct { - RegisterIP string `mapstructure:"registerIP"` - ListenIP string `mapstructure:"listenIP"` + RegisterIP string `mapstructure:"registerIP"` + ListenIP string `mapstructure:"listenIP"` + AutoSetPorts bool `mapstructure:"autoSetPorts"` + Ports []int `mapstructure:"ports"` } `mapstructure:"rpc"` Prometheus Prometheus `mapstructure:"prometheus"` MaxConcurrentWorkers int `mapstructure:"maxConcurrentWorkers"` @@ -229,9 +232,10 @@ type Push struct { type Auth struct { RPC struct { - RegisterIP string `mapstructure:"registerIP"` - ListenIP string `mapstructure:"listenIP"` - Ports []int `mapstructure:"ports"` + RegisterIP string `mapstructure:"registerIP"` + ListenIP string `mapstructure:"listenIP"` + AutoSetPorts bool `mapstructure:"autoSetPorts"` + Ports []int `mapstructure:"ports"` } `mapstructure:"rpc"` Prometheus Prometheus `mapstructure:"prometheus"` TokenPolicy struct { @@ -241,27 +245,30 @@ type Auth struct { type Conversation struct { RPC struct { - RegisterIP string `mapstructure:"registerIP"` - ListenIP string `mapstructure:"listenIP"` - Ports []int `mapstructure:"ports"` + RegisterIP string `mapstructure:"registerIP"` + ListenIP string `mapstructure:"listenIP"` + AutoSetPorts bool `mapstructure:"autoSetPorts"` + Ports []int `mapstructure:"ports"` } `mapstructure:"rpc"` Prometheus Prometheus `mapstructure:"prometheus"` } type Friend struct { RPC struct { - RegisterIP string `mapstructure:"registerIP"` - ListenIP string `mapstructure:"listenIP"` - Ports []int `mapstructure:"ports"` + RegisterIP string `mapstructure:"registerIP"` + ListenIP string `mapstructure:"listenIP"` + AutoSetPorts bool `mapstructure:"autoSetPorts"` + Ports []int `mapstructure:"ports"` } `mapstructure:"rpc"` Prometheus Prometheus `mapstructure:"prometheus"` } type Group struct { RPC struct { - RegisterIP string `mapstructure:"registerIP"` - ListenIP string `mapstructure:"listenIP"` - Ports []int `mapstructure:"ports"` + RegisterIP string `mapstructure:"registerIP"` + ListenIP string `mapstructure:"listenIP"` + AutoSetPorts bool `mapstructure:"autoSetPorts"` + Ports []int `mapstructure:"ports"` } `mapstructure:"rpc"` Prometheus Prometheus `mapstructure:"prometheus"` EnableHistoryForNewMembers bool `mapstructure:"enableHistoryForNewMembers"` @@ -269,9 +276,10 @@ type Group struct { type Msg struct { RPC struct { - RegisterIP string `mapstructure:"registerIP"` - ListenIP string `mapstructure:"listenIP"` - Ports []int `mapstructure:"ports"` + RegisterIP string `mapstructure:"registerIP"` + ListenIP string `mapstructure:"listenIP"` + AutoSetPorts bool `mapstructure:"autoSetPorts"` + Ports []int `mapstructure:"ports"` } `mapstructure:"rpc"` Prometheus Prometheus `mapstructure:"prometheus"` FriendVerify bool `mapstructure:"friendVerify"` @@ -279,9 +287,10 @@ type Msg struct { type Third struct { RPC struct { - RegisterIP string `mapstructure:"registerIP"` - ListenIP string `mapstructure:"listenIP"` - Ports []int `mapstructure:"ports"` + RegisterIP string `mapstructure:"registerIP"` + ListenIP string `mapstructure:"listenIP"` + AutoSetPorts bool `mapstructure:"autoSetPorts"` + Ports []int `mapstructure:"ports"` } `mapstructure:"rpc"` Prometheus Prometheus `mapstructure:"prometheus"` Object struct { @@ -328,9 +337,10 @@ type Kodo struct { type User struct { RPC struct { - RegisterIP string `mapstructure:"registerIP"` - ListenIP string `mapstructure:"listenIP"` - Ports []int `mapstructure:"ports"` + RegisterIP string `mapstructure:"registerIP"` + ListenIP string `mapstructure:"listenIP"` + AutoSetPorts bool `mapstructure:"autoSetPorts"` + Ports []int `mapstructure:"ports"` } `mapstructure:"rpc"` Prometheus Prometheus `mapstructure:"prometheus"` } diff --git a/pkg/common/startrpc/start.go b/pkg/common/startrpc/start.go index 5c42586bf..26fbb0ffa 100644 --- a/pkg/common/startrpc/start.go +++ b/pkg/common/startrpc/start.go @@ -16,6 +16,7 @@ package startrpc import ( "context" + "errors" "fmt" "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/tools/utils/datautil" @@ -27,7 +28,6 @@ import ( "syscall" "time" - "github.com/openimsdk/open-im-server/v3/internal/tools/addr" kdisc "github.com/openimsdk/open-im-server/v3/pkg/common/discoveryregister" "github.com/openimsdk/open-im-server/v3/pkg/common/prommetrics" "github.com/openimsdk/tools/discovery" @@ -41,11 +41,60 @@ import ( ) // Start rpc server. -func Start[T any](ctx context.Context, discovery *config.Discovery, prometheusConfig *config.Prometheus, listenIP string, - index int, rpcRegisterName string, share *config.Share, config T, rpcFn func(ctx context.Context, - config T, client discovery.SvcDiscoveryRegistry, server *grpc.Server) error, options ...grpc.ServerOption) error { +func Start[T any](ctx context.Context, discovery *config.Discovery, prometheusConfig *config.Prometheus, listenIP, + registerIP string, autoSetPorts bool, rpcPorts []int, index int, rpcRegisterName string, share *config.Share, config T, rpcFn func(ctx context.Context, + config T, client discovery.SvcDiscoveryRegistry, server *grpc.Server) error, options ...grpc.ServerOption) error { + + var ( + rpcTcpAddr string + netDone = make(chan struct{}, 2) + netErr error + ) + + if !autoSetPorts { + rpcPort, err := datautil.GetElemByIndex(rpcPorts, index) + if err != nil { + return err + } + rpcTcpAddr = net.JoinHostPort(network.GetListenIP(listenIP), strconv.Itoa(rpcPort)) + } else { + rpcTcpAddr = net.JoinHostPort(network.GetListenIP(listenIP), "0") + } + + // var reg *prometheus.Registry + // var metric *grpcprometheus.ServerMetrics + if prometheusConfig.Enable { + // cusMetrics := prommetrics.GetGrpcCusMetrics(rpcRegisterName, share) + // reg, metric, _ = prommetrics.NewGrpcPromObj(cusMetrics) + // options = append(options, mw.GrpcServer(), grpc.StreamInterceptor(metric.StreamServerInterceptor()), + // grpc.UnaryInterceptor(metric.UnaryServerInterceptor())) + options = append( + options, mw.GrpcServer(), + prommetricsUnaryInterceptor(rpcRegisterName), + prommetricsStreamInterceptor(rpcRegisterName), + ) + prometheusPort, err := datautil.GetElemByIndex(prometheusConfig.Ports, index) + if err != nil { + return err + } + cs := prommetrics.GetGrpcCusMetrics(rpcRegisterName, share) + go func() { + if err := prommetrics.RpcInit(cs, prometheusPort); err != nil && !errors.Is(err, http.ErrServerClosed) { + netErr = errs.WrapMsg(err, fmt.Sprintf("rpc %s prometheus start err: %d", rpcRegisterName, prometheusPort)) + netDone <- struct{}{} + } + // metric.InitializeMetrics(srv) + // Create a HTTP server for prometheus. + // httpServer = &http.Server{Handler: promhttp.HandlerFor(reg, promhttp.HandlerOpts{}), Addr: fmt.Sprintf("0.0.0.0:%d", prometheusPort)} + // if err := httpServer.ListenAndServe(); err != nil && err != http.ErrServerClosed { + // netErr = errs.WrapMsg(err, "prometheus start err", httpServer.Addr) + // netDone <- struct{}{} + // } + }() + } else { + options = append(options, mw.GrpcServer()) + } - rpcTcpAddr := net.JoinHostPort(network.GetListenIP(listenIP), "0") listener, err := net.Listen( "tcp", rpcTcpAddr, @@ -54,8 +103,8 @@ func Start[T any](ctx context.Context, discovery *config.Discovery, prometheusCo return errs.WrapMsg(err, "listen err", "rpcTcpAddr", rpcTcpAddr) } - h, portStr, _ := net.SplitHostPort(listener.Addr().String()) - host, _ := addr.Extract(h) + _, portStr, _ := net.SplitHostPort(listener.Addr().String()) + registerIP = network.GetListenIP(registerIP) port, _ := strconv.Atoi(portStr) log.CInfo(ctx, "RPC server is initializing", "rpcRegisterName", rpcRegisterName, "rpcPort", portStr, @@ -70,22 +119,6 @@ func Start[T any](ctx context.Context, discovery *config.Discovery, prometheusCo defer client.Close() client.AddOption(mw.GrpcClient(), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithDefaultServiceConfig(fmt.Sprintf(`{"LoadBalancingPolicy": "%s"}`, "round_robin"))) - // var reg *prometheus.Registry - // var metric *grpcprometheus.ServerMetrics - if prometheusConfig.Enable { - // cusMetrics := prommetrics.GetGrpcCusMetrics(rpcRegisterName, share) - // reg, metric, _ = prommetrics.NewGrpcPromObj(cusMetrics) - // options = append(options, mw.GrpcServer(), grpc.StreamInterceptor(metric.StreamServerInterceptor()), - // grpc.UnaryInterceptor(metric.UnaryServerInterceptor())) - options = append( - options, mw.GrpcServer(), - prommetricsUnaryInterceptor(rpcRegisterName), - prommetricsStreamInterceptor(rpcRegisterName), - ) - } else { - options = append(options, mw.GrpcServer()) - } - srv := grpc.NewServer(options...) err = rpcFn(ctx, config, client, srv) @@ -95,7 +128,7 @@ func Start[T any](ctx context.Context, discovery *config.Discovery, prometheusCo err = client.Register( rpcRegisterName, - host, + registerIP, port, grpc.WithTransportCredentials(insecure.NewCredentials()), ) @@ -103,33 +136,6 @@ func Start[T any](ctx context.Context, discovery *config.Discovery, prometheusCo return err } - var ( - netDone = make(chan struct{}, 2) - netErr error - ) - if prometheusConfig.Enable { - go func() { - prometheusPort, err := datautil.GetElemByIndex(prometheusConfig.Ports, index) - if err != nil { - netErr = err - netDone <- struct{}{} - return - } - cs := prommetrics.GetGrpcCusMetrics(rpcRegisterName, share) - if err := prommetrics.RpcInit(cs, prometheusPort); err != nil && err != http.ErrServerClosed { - netErr = errs.WrapMsg(err, fmt.Sprintf("rpc %s prometheus start err: %d", rpcRegisterName, prometheusPort)) - netDone <- struct{}{} - } - // metric.InitializeMetrics(srv) - // Create a HTTP server for prometheus. - // httpServer = &http.Server{Handler: promhttp.HandlerFor(reg, promhttp.HandlerOpts{}), Addr: fmt.Sprintf("0.0.0.0:%d", prometheusPort)} - // if err := httpServer.ListenAndServe(); err != nil && err != http.ErrServerClosed { - // netErr = errs.WrapMsg(err, "prometheus start err", httpServer.Addr) - // netDone <- struct{}{} - // } - }() - } - go func() { err := srv.Serve(listener) if err != nil { From bbbf17cbe4330ba0fcac45d28f129f33a4fc4b7e Mon Sep 17 00:00:00 2001 From: icey-yu <119291641+icey-yu@users.noreply.github.com> Date: Sat, 7 Dec 2024 12:11:12 +0800 Subject: [PATCH 063/199] feat: config (#2934) --- config/discovery.yml | 11 ++++++++++ internal/api/init.go | 2 +- internal/api/router.go | 14 ++++++------ internal/msggateway/hub_server.go | 4 ++-- internal/msggateway/message_handler.go | 2 +- internal/msggateway/ws_server.go | 10 ++++----- internal/msgtransfer/init.go | 6 ++--- internal/push/onlinepusher.go | 2 +- internal/push/push_handler.go | 10 ++++----- internal/rpc/auth/auth.go | 4 ++-- internal/rpc/conversation/conversation.go | 6 ++--- internal/rpc/group/group.go | 6 ++--- internal/rpc/msg/server.go | 8 +++---- internal/rpc/relation/friend.go | 6 ++--- internal/rpc/third/third.go | 2 +- internal/rpc/user/user.go | 6 ++--- internal/tools/cron_task.go | 22 ++++--------------- pkg/common/cmd/auth.go | 2 +- pkg/common/cmd/conversation.go | 2 +- pkg/common/cmd/friend.go | 2 +- pkg/common/cmd/group.go | 2 +- pkg/common/cmd/msg.go | 2 +- pkg/common/cmd/push.go | 2 +- pkg/common/cmd/third.go | 2 +- pkg/common/cmd/user.go | 2 +- pkg/common/config/config.go | 17 +++++++------- .../discoveryregister/discoveryregister.go | 14 ++---------- pkg/common/prommetrics/rpc.go | 12 +++++----- pkg/common/startrpc/start.go | 4 ++-- tools/check-component/main.go | 4 ---- 30 files changed, 85 insertions(+), 103 deletions(-) diff --git a/config/discovery.yml b/config/discovery.yml index 6e68cbff4..fad49e3ed 100644 --- a/config/discovery.yml +++ b/config/discovery.yml @@ -5,4 +5,15 @@ etcd: username: '' password: '' +rpcService: + user: user-rpc-service + friend: friend-rpc-service + msg: msg-rpc-service + push: push-rpc-service + messageGateway: messageGateway-rpc-service + group: group-rpc-service + auth: auth-rpc-service + conversation: conversation-rpc-service + third: third-rpc-service + diff --git a/internal/api/init.go b/internal/api/init.go index e83dfc2ea..6c25609f6 100644 --- a/internal/api/init.go +++ b/internal/api/init.go @@ -51,7 +51,7 @@ func Start(ctx context.Context, index int, config *Config) error { var client discovery.SvcDiscoveryRegistry // Determine whether zk is passed according to whether it is a clustered deployment - client, err = kdisc.NewDiscoveryRegister(&config.Discovery, &config.Share) + client, err = kdisc.NewDiscoveryRegister(&config.Discovery) if err != nil { return errs.WrapMsg(err, "failed to register discovery service") } diff --git a/internal/api/router.go b/internal/api/router.go index 8e4d17ef1..52d26bdc5 100644 --- a/internal/api/router.go +++ b/internal/api/router.go @@ -57,14 +57,14 @@ func newGinRouter(disCov discovery.SvcDiscoveryRegistry, config *Config) *gin.En _ = v.RegisterValidation("required_if", RequiredIf) } // init rpc client here - userRpc := rpcclient.NewUser(disCov, config.Share.RpcRegisterName.User, config.Share.RpcRegisterName.MessageGateway, + userRpc := rpcclient.NewUser(disCov, config.Discovery.RpcService.User, config.Discovery.RpcService.MessageGateway, config.Share.IMAdminUserID) - groupRpc := rpcclient.NewGroup(disCov, config.Share.RpcRegisterName.Group) - friendRpc := rpcclient.NewFriend(disCov, config.Share.RpcRegisterName.Friend) - messageRpc := rpcclient.NewMessage(disCov, config.Share.RpcRegisterName.Msg) - conversationRpc := rpcclient.NewConversation(disCov, config.Share.RpcRegisterName.Conversation) - authRpc := rpcclient.NewAuth(disCov, config.Share.RpcRegisterName.Auth) - thirdRpc := rpcclient.NewThird(disCov, config.Share.RpcRegisterName.Third, config.API.Prometheus.GrafanaURL) + groupRpc := rpcclient.NewGroup(disCov, config.Discovery.RpcService.Group) + friendRpc := rpcclient.NewFriend(disCov, config.Discovery.RpcService.Friend) + messageRpc := rpcclient.NewMessage(disCov, config.Discovery.RpcService.Msg) + conversationRpc := rpcclient.NewConversation(disCov, config.Discovery.RpcService.Conversation) + authRpc := rpcclient.NewAuth(disCov, config.Discovery.RpcService.Auth) + thirdRpc := rpcclient.NewThird(disCov, config.Discovery.RpcService.Third, config.API.Prometheus.GrafanaURL) switch config.API.Api.CompressionLevel { case NoCompression: case DefaultCompression: diff --git a/internal/msggateway/hub_server.go b/internal/msggateway/hub_server.go index b6760ed0f..8e1edbec7 100644 --- a/internal/msggateway/hub_server.go +++ b/internal/msggateway/hub_server.go @@ -37,7 +37,7 @@ import ( func (s *Server) InitServer(ctx context.Context, config *Config, disCov discovery.SvcDiscoveryRegistry, server *grpc.Server) error { s.LongConnServer.SetDiscoveryRegistry(disCov, config) msggateway.RegisterMsgGatewayServer(server, s) - s.userRcp = rpcclient.NewUserRpcClient(disCov, config.Share.RpcRegisterName.User, config.Share.IMAdminUserID) + s.userRcp = rpcclient.NewUserRpcClient(disCov, config.Discovery.RpcService.User, config.Share.IMAdminUserID) if s.ready != nil { return s.ready(s) } @@ -48,7 +48,7 @@ func (s *Server) Start(ctx context.Context, index int, conf *Config) error { return startrpc.Start(ctx, &conf.Discovery, &conf.MsgGateway.Prometheus, conf.MsgGateway.ListenIP, conf.MsgGateway.RPC.RegisterIP, conf.MsgGateway.RPC.AutoSetPorts, conf.MsgGateway.RPC.Ports, index, - conf.Share.RpcRegisterName.MessageGateway, + conf.Discovery.RpcService.MessageGateway, &conf.Share, conf, s.InitServer, diff --git a/internal/msggateway/message_handler.go b/internal/msggateway/message_handler.go index 5407ba90c..f2b0ce9da 100644 --- a/internal/msggateway/message_handler.go +++ b/internal/msggateway/message_handler.go @@ -120,7 +120,7 @@ type GrpcHandler struct { validate *validator.Validate } -func NewGrpcHandler(validate *validator.Validate, client discovery.SvcDiscoveryRegistry, rpcRegisterName *config.RpcRegisterName) *GrpcHandler { +func NewGrpcHandler(validate *validator.Validate, client discovery.SvcDiscoveryRegistry, rpcRegisterName *config.RpcService) *GrpcHandler { msgRpcClient := rpcclient.NewMessageRpcClient(client, rpcRegisterName.Msg) pushRpcClient := rpcclient.NewPushRpcClient(client, rpcRegisterName.Push) return &GrpcHandler{ diff --git a/internal/msggateway/ws_server.go b/internal/msggateway/ws_server.go index e6b4f3fa4..335556ae9 100644 --- a/internal/msggateway/ws_server.go +++ b/internal/msggateway/ws_server.go @@ -72,9 +72,9 @@ type kickHandler struct { } func (ws *WsServer) SetDiscoveryRegistry(disCov discovery.SvcDiscoveryRegistry, config *Config) { - ws.MessageHandler = NewGrpcHandler(ws.validate, disCov, &config.Share.RpcRegisterName) - u := rpcclient.NewUserRpcClient(disCov, config.Share.RpcRegisterName.User, config.Share.IMAdminUserID) - ws.authClient = rpcclient.NewAuth(disCov, config.Share.RpcRegisterName.Auth) + ws.MessageHandler = NewGrpcHandler(ws.validate, disCov, &config.Discovery.RpcService) + u := rpcclient.NewUserRpcClient(disCov, config.Discovery.RpcService.User, config.Share.IMAdminUserID) + ws.authClient = rpcclient.NewAuth(disCov, config.Discovery.RpcService.Auth) ws.userClient = &u ws.disCov = disCov } @@ -113,7 +113,7 @@ func NewWsServer(msgGatewayConfig *Config, opts ...Option) *WsServer { for _, o := range opts { o(&config) } - //userRpcClient := rpcclient.NewUserRpcClient(client, config.Share.RpcRegisterName.User, config.Share.IMAdminUserID) + //userRpcClient := rpcclient.NewUserRpcClient(client, config.Discovery.RpcService.User, config.Share.IMAdminUserID) v := validator.New() return &WsServer{ @@ -192,7 +192,7 @@ func (ws *WsServer) Run(done chan error) error { var concurrentRequest = 3 func (ws *WsServer) sendUserOnlineInfoToOtherNode(ctx context.Context, client *Client) error { - conns, err := ws.disCov.GetConns(ctx, ws.msgGatewayConfig.Share.RpcRegisterName.MessageGateway) + conns, err := ws.disCov.GetConns(ctx, ws.msgGatewayConfig.Discovery.RpcService.MessageGateway) if err != nil { return err } diff --git a/internal/msgtransfer/init.go b/internal/msgtransfer/init.go index 92053931c..b65532a8e 100644 --- a/internal/msgtransfer/init.go +++ b/internal/msgtransfer/init.go @@ -75,7 +75,7 @@ func Start(ctx context.Context, index int, config *Config) error { if err != nil { return err } - client, err := discRegister.NewDiscoveryRegister(&config.Discovery, &config.Share) + client, err := discRegister.NewDiscoveryRegister(&config.Discovery) if err != nil { return err } @@ -101,8 +101,8 @@ func Start(ctx context.Context, index int, config *Config) error { if err != nil { return err } - conversationRpcClient := rpcclient.NewConversationRpcClient(client, config.Share.RpcRegisterName.Conversation) - groupRpcClient := rpcclient.NewGroupRpcClient(client, config.Share.RpcRegisterName.Group) + conversationRpcClient := rpcclient.NewConversationRpcClient(client, config.Discovery.RpcService.Conversation) + groupRpcClient := rpcclient.NewGroupRpcClient(client, config.Discovery.RpcService.Group) historyCH, err := NewOnlineHistoryRedisConsumerHandler(&config.KafkaConfig, msgTransferDatabase, &conversationRpcClient, &groupRpcClient) if err != nil { return err diff --git a/internal/push/onlinepusher.go b/internal/push/onlinepusher.go index 9521a84a0..f1ff7fed5 100644 --- a/internal/push/onlinepusher.go +++ b/internal/push/onlinepusher.go @@ -60,7 +60,7 @@ func NewDefaultAllNode(disCov discovery.SvcDiscoveryRegistry, config *Config) *D func (d *DefaultAllNode) GetConnsAndOnlinePush(ctx context.Context, msg *sdkws.MsgData, pushToUserIDs []string) (wsResults []*msggateway.SingleMsgToUserResults, err error) { - conns, err := d.disCov.GetConns(ctx, d.config.Share.RpcRegisterName.MessageGateway) + conns, err := d.disCov.GetConns(ctx, d.config.Discovery.RpcService.MessageGateway) if len(conns) == 0 { log.ZWarn(ctx, "get gateway conn 0 ", nil) } else { diff --git a/internal/push/push_handler.go b/internal/push/push_handler.go index c1d1ac2f9..ee3dc5b84 100644 --- a/internal/push/push_handler.go +++ b/internal/push/push_handler.go @@ -58,14 +58,14 @@ func NewConsumerHandler(config *Config, database controller.PushDatabase, offlin return nil, err } - userRpcClient := rpcclient.NewUserRpcClient(client, config.Share.RpcRegisterName.User, config.Share.IMAdminUserID) + userRpcClient := rpcclient.NewUserRpcClient(client, config.Discovery.RpcService.User, config.Share.IMAdminUserID) consumerHandler.offlinePusher = offlinePusher consumerHandler.onlinePusher = NewOnlinePusher(client, config) - consumerHandler.groupRpcClient = rpcclient.NewGroupRpcClient(client, config.Share.RpcRegisterName.Group) + consumerHandler.groupRpcClient = rpcclient.NewGroupRpcClient(client, config.Discovery.RpcService.Group) consumerHandler.groupLocalCache = rpccache.NewGroupLocalCache(consumerHandler.groupRpcClient, &config.LocalCacheConfig, rdb) - consumerHandler.msgRpcClient = rpcclient.NewMessageRpcClient(client, config.Share.RpcRegisterName.Msg) - consumerHandler.conversationRpcClient = rpcclient.NewConversationRpcClient(client, config.Share.RpcRegisterName.Conversation) + consumerHandler.msgRpcClient = rpcclient.NewMessageRpcClient(client, config.Discovery.RpcService.Msg) + consumerHandler.conversationRpcClient = rpcclient.NewConversationRpcClient(client, config.Discovery.RpcService.Conversation) consumerHandler.conversationLocalCache = rpccache.NewConversationLocalCache(consumerHandler.conversationRpcClient, &config.LocalCacheConfig, rdb) consumerHandler.webhookClient = webhook.NewWebhookClient(config.WebhooksConfig.URL) consumerHandler.config = config @@ -414,7 +414,7 @@ func (c *ConsumerHandler) DeleteMemberAndSetConversationSeq(ctx context.Context, if err != nil { return err } - + return c.conversationRpcClient.SetConversationMaxSeq(ctx, userIDs, conversationID, maxSeq) } diff --git a/internal/rpc/auth/auth.go b/internal/rpc/auth/auth.go index f0f4a022d..c9cf8d345 100644 --- a/internal/rpc/auth/auth.go +++ b/internal/rpc/auth/auth.go @@ -59,7 +59,7 @@ func Start(ctx context.Context, config *Config, client discovery.SvcDiscoveryReg if err != nil { return err } - userRpcClient := rpcclient.NewUserRpcClient(client, config.Share.RpcRegisterName.User, config.Share.IMAdminUserID) + userRpcClient := rpcclient.NewUserRpcClient(client, config.Discovery.RpcService.User, config.Share.IMAdminUserID) pbauth.RegisterAuthServer(server, &authServer{ userRpcClient: &userRpcClient, RegisterCenter: client, @@ -182,7 +182,7 @@ func (s *authServer) ForceLogout(ctx context.Context, req *pbauth.ForceLogoutReq } func (s *authServer) forceKickOff(ctx context.Context, userID string, platformID int32) error { - conns, err := s.RegisterCenter.GetConns(ctx, s.config.Share.RpcRegisterName.MessageGateway) + conns, err := s.RegisterCenter.GetConns(ctx, s.config.Discovery.RpcService.MessageGateway) if err != nil { return err } diff --git a/internal/rpc/conversation/conversation.go b/internal/rpc/conversation/conversation.go index 6a8a6d800..7345c965d 100644 --- a/internal/rpc/conversation/conversation.go +++ b/internal/rpc/conversation/conversation.go @@ -78,9 +78,9 @@ func Start(ctx context.Context, config *Config, client discovery.SvcDiscoveryReg if err != nil { return err } - groupRpcClient := rpcclient.NewGroupRpcClient(client, config.Share.RpcRegisterName.Group) - msgRpcClient := rpcclient.NewMessageRpcClient(client, config.Share.RpcRegisterName.Msg) - userRpcClient := rpcclient.NewUserRpcClient(client, config.Share.RpcRegisterName.User, config.Share.IMAdminUserID) + groupRpcClient := rpcclient.NewGroupRpcClient(client, config.Discovery.RpcService.Group) + msgRpcClient := rpcclient.NewMessageRpcClient(client, config.Discovery.RpcService.Msg) + userRpcClient := rpcclient.NewUserRpcClient(client, config.Discovery.RpcService.User, config.Share.IMAdminUserID) localcache.InitLocalCache(&config.LocalCacheConfig) pbconversation.RegisterConversationServer(server, &conversationServer{ msgRpcClient: &msgRpcClient, diff --git a/internal/rpc/group/group.go b/internal/rpc/group/group.go index 9e610df0f..62020f980 100644 --- a/internal/rpc/group/group.go +++ b/internal/rpc/group/group.go @@ -99,9 +99,9 @@ func Start(ctx context.Context, config *Config, client discovery.SvcDiscoveryReg if err != nil { return err } - userRpcClient := rpcclient.NewUserRpcClient(client, config.Share.RpcRegisterName.User, config.Share.IMAdminUserID) - msgRpcClient := rpcclient.NewMessageRpcClient(client, config.Share.RpcRegisterName.Msg) - conversationRpcClient := rpcclient.NewConversationRpcClient(client, config.Share.RpcRegisterName.Conversation) + userRpcClient := rpcclient.NewUserRpcClient(client, config.Discovery.RpcService.User, config.Share.IMAdminUserID) + msgRpcClient := rpcclient.NewMessageRpcClient(client, config.Discovery.RpcService.Msg) + conversationRpcClient := rpcclient.NewConversationRpcClient(client, config.Discovery.RpcService.Conversation) var gs groupServer database := controller.NewGroupDatabase(rdb, &config.LocalCacheConfig, groupDB, groupMemberDB, groupRequestDB, mgocli.GetTx(), grouphash.NewGroupHashFromGroupServer(&gs)) gs.db = database diff --git a/internal/rpc/msg/server.go b/internal/rpc/msg/server.go index 758a7cf19..b0cd771a4 100644 --- a/internal/rpc/msg/server.go +++ b/internal/rpc/msg/server.go @@ -89,10 +89,10 @@ func Start(ctx context.Context, config *Config, client discovery.SvcDiscoveryReg return err } msgModel := redis.NewMsgCache(rdb) - conversationClient := rpcclient.NewConversationRpcClient(client, config.Share.RpcRegisterName.Conversation) - userRpcClient := rpcclient.NewUserRpcClient(client, config.Share.RpcRegisterName.User, config.Share.IMAdminUserID) - groupRpcClient := rpcclient.NewGroupRpcClient(client, config.Share.RpcRegisterName.Group) - friendRpcClient := rpcclient.NewFriendRpcClient(client, config.Share.RpcRegisterName.Friend) + conversationClient := rpcclient.NewConversationRpcClient(client, config.Discovery.RpcService.Conversation) + userRpcClient := rpcclient.NewUserRpcClient(client, config.Discovery.RpcService.User, config.Share.IMAdminUserID) + groupRpcClient := rpcclient.NewGroupRpcClient(client, config.Discovery.RpcService.Group) + friendRpcClient := rpcclient.NewFriendRpcClient(client, config.Discovery.RpcService.Friend) seqConversation, err := mgo.NewSeqConversationMongo(mgocli.GetDB()) if err != nil { return err diff --git a/internal/rpc/relation/friend.go b/internal/rpc/relation/friend.go index 036c7aff5..617e31348 100644 --- a/internal/rpc/relation/friend.go +++ b/internal/rpc/relation/friend.go @@ -93,8 +93,8 @@ func Start(ctx context.Context, config *Config, client discovery.SvcDiscoveryReg } // Initialize RPC clients - userRpcClient := rpcclient.NewUserRpcClient(client, config.Share.RpcRegisterName.User, config.Share.IMAdminUserID) - msgRpcClient := rpcclient.NewMessageRpcClient(client, config.Share.RpcRegisterName.Msg) + userRpcClient := rpcclient.NewUserRpcClient(client, config.Discovery.RpcService.User, config.Share.IMAdminUserID) + msgRpcClient := rpcclient.NewMessageRpcClient(client, config.Discovery.RpcService.Msg) // Initialize notification sender notificationSender := NewFriendNotificationSender( @@ -119,7 +119,7 @@ func Start(ctx context.Context, config *Config, client discovery.SvcDiscoveryReg userRpcClient: &userRpcClient, notificationSender: notificationSender, RegisterCenter: client, - conversationRpcClient: rpcclient.NewConversationRpcClient(client, config.Share.RpcRegisterName.Conversation), + conversationRpcClient: rpcclient.NewConversationRpcClient(client, config.Discovery.RpcService.Conversation), config: config, webhookClient: webhook.NewWebhookClient(config.WebhooksConfig.URL), queue: memamq.NewMemoryQueue(16, 1024*1024), diff --git a/internal/rpc/third/third.go b/internal/rpc/third/third.go index 4206a2d6f..75076ab74 100644 --- a/internal/rpc/third/third.go +++ b/internal/rpc/third/third.go @@ -101,7 +101,7 @@ func Start(ctx context.Context, config *Config, client discovery.SvcDiscoveryReg localcache.InitLocalCache(&config.LocalCacheConfig) third.RegisterThirdServer(server, &thirdServer{ thirdDatabase: controller.NewThirdDatabase(redis.NewThirdCache(rdb), logdb), - userRpcClient: rpcclient.NewUserRpcClient(client, config.Share.RpcRegisterName.User, config.Share.IMAdminUserID), + userRpcClient: rpcclient.NewUserRpcClient(client, config.Discovery.RpcService.User, config.Share.IMAdminUserID), s3dataBase: controller.NewS3Database(rdb, o, s3db), defaultExpire: time.Hour * 24 * 7, config: config, diff --git a/internal/rpc/user/user.go b/internal/rpc/user/user.go index 2dfbb01df..ae02b997f 100644 --- a/internal/rpc/user/user.go +++ b/internal/rpc/user/user.go @@ -96,9 +96,9 @@ func Start(ctx context.Context, config *Config, client registry.SvcDiscoveryRegi } userCache := redis.NewUserCacheRedis(rdb, &config.LocalCacheConfig, userDB, redis.GetRocksCacheOptions()) database := controller.NewUserDatabase(userDB, userCache, mgocli.GetTx()) - friendRpcClient := rpcclient.NewFriendRpcClient(client, config.Share.RpcRegisterName.Friend) - groupRpcClient := rpcclient.NewGroupRpcClient(client, config.Share.RpcRegisterName.Group) - msgRpcClient := rpcclient.NewMessageRpcClient(client, config.Share.RpcRegisterName.Msg) + friendRpcClient := rpcclient.NewFriendRpcClient(client, config.Discovery.RpcService.Friend) + groupRpcClient := rpcclient.NewGroupRpcClient(client, config.Discovery.RpcService.Group) + msgRpcClient := rpcclient.NewMessageRpcClient(client, config.Discovery.RpcService.Msg) localcache.InitLocalCache(&config.LocalCacheConfig) u := &userServer{ online: redis.NewUserOnline(rdb), diff --git a/internal/tools/cron_task.go b/internal/tools/cron_task.go index 4f87036c7..3864c43dc 100644 --- a/internal/tools/cron_task.go +++ b/internal/tools/cron_task.go @@ -1,17 +1,3 @@ -// Copyright © 2023 OpenIM. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - package tools import ( @@ -47,24 +33,24 @@ func Start(ctx context.Context, config *CronTaskConfig) error { if config.CronTask.RetainChatRecords < 1 { return errs.New("msg destruct time must be greater than 1").Wrap() } - client, err := kdisc.NewDiscoveryRegister(&config.Discovery, &config.Share) + client, err := kdisc.NewDiscoveryRegister(&config.Discovery) if err != nil { return errs.WrapMsg(err, "failed to register discovery service") } client.AddOption(mw.GrpcClient(), grpc.WithTransportCredentials(insecure.NewCredentials())) ctx = mcontext.SetOpUserID(ctx, config.Share.IMAdminUserID[0]) - msgConn, err := client.GetConn(ctx, config.Share.RpcRegisterName.Msg) + msgConn, err := client.GetConn(ctx, config.Discovery.RpcService.Msg) if err != nil { return err } - thirdConn, err := client.GetConn(ctx, config.Share.RpcRegisterName.Third) + thirdConn, err := client.GetConn(ctx, config.Discovery.RpcService.Third) if err != nil { return err } - conversationConn, err := client.GetConn(ctx, config.Share.RpcRegisterName.Conversation) + conversationConn, err := client.GetConn(ctx, config.Discovery.RpcService.Conversation) if err != nil { return err } diff --git a/pkg/common/cmd/auth.go b/pkg/common/cmd/auth.go index 34f450fb1..80a675ace 100644 --- a/pkg/common/cmd/auth.go +++ b/pkg/common/cmd/auth.go @@ -56,5 +56,5 @@ func (a *AuthRpcCmd) Exec() error { func (a *AuthRpcCmd) runE() error { return startrpc.Start(a.ctx, &a.authConfig.Discovery, &a.authConfig.RpcConfig.Prometheus, a.authConfig.RpcConfig.RPC.ListenIP, a.authConfig.RpcConfig.RPC.RegisterIP, a.authConfig.RpcConfig.RPC.AutoSetPorts, a.authConfig.RpcConfig.RPC.Ports, - a.Index(), a.authConfig.Share.RpcRegisterName.Auth, &a.authConfig.Share, a.authConfig, auth.Start) + a.Index(), a.authConfig.Discovery.RpcService.Auth, &a.authConfig.Share, a.authConfig, auth.Start) } diff --git a/pkg/common/cmd/conversation.go b/pkg/common/cmd/conversation.go index f8ec570a7..72fa694ed 100644 --- a/pkg/common/cmd/conversation.go +++ b/pkg/common/cmd/conversation.go @@ -58,5 +58,5 @@ func (a *ConversationRpcCmd) Exec() error { func (a *ConversationRpcCmd) runE() error { return startrpc.Start(a.ctx, &a.conversationConfig.Discovery, &a.conversationConfig.RpcConfig.Prometheus, a.conversationConfig.RpcConfig.RPC.ListenIP, a.conversationConfig.RpcConfig.RPC.RegisterIP, a.conversationConfig.RpcConfig.RPC.AutoSetPorts, a.conversationConfig.RpcConfig.RPC.Ports, - a.Index(), a.conversationConfig.Share.RpcRegisterName.Conversation, &a.conversationConfig.Share, a.conversationConfig, conversation.Start) + a.Index(), a.conversationConfig.Discovery.RpcService.Conversation, &a.conversationConfig.Share, a.conversationConfig, conversation.Start) } diff --git a/pkg/common/cmd/friend.go b/pkg/common/cmd/friend.go index 786be0d8e..b40449b2b 100644 --- a/pkg/common/cmd/friend.go +++ b/pkg/common/cmd/friend.go @@ -59,5 +59,5 @@ func (a *FriendRpcCmd) Exec() error { func (a *FriendRpcCmd) runE() error { return startrpc.Start(a.ctx, &a.relationConfig.Discovery, &a.relationConfig.RpcConfig.Prometheus, a.relationConfig.RpcConfig.RPC.ListenIP, a.relationConfig.RpcConfig.RPC.RegisterIP, a.relationConfig.RpcConfig.RPC.AutoSetPorts, a.relationConfig.RpcConfig.RPC.Ports, - a.Index(), a.relationConfig.Share.RpcRegisterName.Friend, &a.relationConfig.Share, a.relationConfig, relation.Start) + a.Index(), a.relationConfig.Discovery.RpcService.Friend, &a.relationConfig.Share, a.relationConfig, relation.Start) } diff --git a/pkg/common/cmd/group.go b/pkg/common/cmd/group.go index e405c9135..faed56233 100644 --- a/pkg/common/cmd/group.go +++ b/pkg/common/cmd/group.go @@ -60,5 +60,5 @@ func (a *GroupRpcCmd) Exec() error { func (a *GroupRpcCmd) runE() error { return startrpc.Start(a.ctx, &a.groupConfig.Discovery, &a.groupConfig.RpcConfig.Prometheus, a.groupConfig.RpcConfig.RPC.ListenIP, a.groupConfig.RpcConfig.RPC.RegisterIP, a.groupConfig.RpcConfig.RPC.AutoSetPorts, a.groupConfig.RpcConfig.RPC.Ports, - a.Index(), a.groupConfig.Share.RpcRegisterName.Group, &a.groupConfig.Share, a.groupConfig, group.Start, versionctx.EnableVersionCtx()) + a.Index(), a.groupConfig.Discovery.RpcService.Group, &a.groupConfig.Share, a.groupConfig, group.Start, versionctx.EnableVersionCtx()) } diff --git a/pkg/common/cmd/msg.go b/pkg/common/cmd/msg.go index 3f1a2fa31..962d1468a 100644 --- a/pkg/common/cmd/msg.go +++ b/pkg/common/cmd/msg.go @@ -60,5 +60,5 @@ func (a *MsgRpcCmd) Exec() error { func (a *MsgRpcCmd) runE() error { return startrpc.Start(a.ctx, &a.msgConfig.Discovery, &a.msgConfig.RpcConfig.Prometheus, a.msgConfig.RpcConfig.RPC.ListenIP, a.msgConfig.RpcConfig.RPC.RegisterIP, a.msgConfig.RpcConfig.RPC.AutoSetPorts, a.msgConfig.RpcConfig.RPC.Ports, - a.Index(), a.msgConfig.Share.RpcRegisterName.Msg, &a.msgConfig.Share, a.msgConfig, msg.Start) + a.Index(), a.msgConfig.Discovery.RpcService.Msg, &a.msgConfig.Share, a.msgConfig, msg.Start) } diff --git a/pkg/common/cmd/push.go b/pkg/common/cmd/push.go index e0f6b39f6..09883fc34 100644 --- a/pkg/common/cmd/push.go +++ b/pkg/common/cmd/push.go @@ -60,5 +60,5 @@ func (a *PushRpcCmd) Exec() error { func (a *PushRpcCmd) runE() error { return startrpc.Start(a.ctx, &a.pushConfig.Discovery, &a.pushConfig.RpcConfig.Prometheus, a.pushConfig.RpcConfig.RPC.ListenIP, a.pushConfig.RpcConfig.RPC.RegisterIP, a.pushConfig.RpcConfig.RPC.AutoSetPorts, a.pushConfig.RpcConfig.RPC.Ports, - a.Index(), a.pushConfig.Share.RpcRegisterName.Push, &a.pushConfig.Share, a.pushConfig, push.Start) + a.Index(), a.pushConfig.Discovery.RpcService.Push, &a.pushConfig.Share, a.pushConfig, push.Start) } diff --git a/pkg/common/cmd/third.go b/pkg/common/cmd/third.go index c5a21f91b..8e9ffb9bd 100644 --- a/pkg/common/cmd/third.go +++ b/pkg/common/cmd/third.go @@ -59,5 +59,5 @@ func (a *ThirdRpcCmd) Exec() error { func (a *ThirdRpcCmd) runE() error { return startrpc.Start(a.ctx, &a.thirdConfig.Discovery, &a.thirdConfig.RpcConfig.Prometheus, a.thirdConfig.RpcConfig.RPC.ListenIP, a.thirdConfig.RpcConfig.RPC.RegisterIP, a.thirdConfig.RpcConfig.RPC.AutoSetPorts, a.thirdConfig.RpcConfig.RPC.Ports, - a.Index(), a.thirdConfig.Share.RpcRegisterName.Third, &a.thirdConfig.Share, a.thirdConfig, third.Start) + a.Index(), a.thirdConfig.Discovery.RpcService.Third, &a.thirdConfig.Share, a.thirdConfig, third.Start) } diff --git a/pkg/common/cmd/user.go b/pkg/common/cmd/user.go index 1d916ea37..40f372770 100644 --- a/pkg/common/cmd/user.go +++ b/pkg/common/cmd/user.go @@ -60,5 +60,5 @@ func (a *UserRpcCmd) Exec() error { func (a *UserRpcCmd) runE() error { return startrpc.Start(a.ctx, &a.userConfig.Discovery, &a.userConfig.RpcConfig.Prometheus, a.userConfig.RpcConfig.RPC.ListenIP, a.userConfig.RpcConfig.RPC.RegisterIP, a.userConfig.RpcConfig.RPC.AutoSetPorts, a.userConfig.RpcConfig.RPC.Ports, - a.Index(), a.userConfig.Share.RpcRegisterName.User, &a.userConfig.Share, a.userConfig, user.Start) + a.Index(), a.userConfig.Discovery.RpcService.User, &a.userConfig.Share, a.userConfig, user.Start) } diff --git a/pkg/common/config/config.go b/pkg/common/config/config.go index ed803359c..6082b5f23 100644 --- a/pkg/common/config/config.go +++ b/pkg/common/config/config.go @@ -372,10 +372,9 @@ type AfterConfig struct { } type Share struct { - Secret string `mapstructure:"secret"` - RpcRegisterName RpcRegisterName `mapstructure:"rpcRegisterName"` - IMAdminUserID []string `mapstructure:"imAdminUserID"` - MultiLogin MultiLogin `mapstructure:"multiLogin"` + Secret string `mapstructure:"secret"` + IMAdminUserID []string `mapstructure:"imAdminUserID"` + MultiLogin MultiLogin `mapstructure:"multiLogin"` } type MultiLogin struct { @@ -383,7 +382,7 @@ type MultiLogin struct { MaxNumOneEnd int `mapstructure:"maxNumOneEnd"` } -type RpcRegisterName struct { +type RpcService struct { User string `mapstructure:"user"` Friend string `mapstructure:"friend"` Msg string `mapstructure:"msg"` @@ -395,7 +394,7 @@ type RpcRegisterName struct { Third string `mapstructure:"third"` } -func (r *RpcRegisterName) GetServiceNames() []string { +func (r *RpcService) GetServiceNames() []string { return []string{ r.User, r.Friend, @@ -470,9 +469,9 @@ type ZooKeeper struct { } type Discovery struct { - Enable string `mapstructure:"enable"` - Etcd Etcd `mapstructure:"etcd"` - ZooKeeper ZooKeeper `mapstructure:"zooKeeper"` + Enable string `mapstructure:"enable"` + Etcd Etcd `mapstructure:"etcd"` + RpcService RpcService `mapstructure:"rpcService"` } type Etcd struct { diff --git a/pkg/common/discoveryregister/discoveryregister.go b/pkg/common/discoveryregister/discoveryregister.go index 559c937c1..be82aa526 100644 --- a/pkg/common/discoveryregister/discoveryregister.go +++ b/pkg/common/discoveryregister/discoveryregister.go @@ -19,25 +19,15 @@ import ( "github.com/openimsdk/open-im-server/v3/pkg/common/discoveryregister/kubernetes" "github.com/openimsdk/tools/discovery" "github.com/openimsdk/tools/discovery/etcd" - "github.com/openimsdk/tools/discovery/zookeeper" "github.com/openimsdk/tools/errs" "time" ) // NewDiscoveryRegister creates a new service discovery and registry client based on the provided environment type. -func NewDiscoveryRegister(discovery *config.Discovery, share *config.Share) (discovery.SvcDiscoveryRegistry, error) { +func NewDiscoveryRegister(discovery *config.Discovery) (discovery.SvcDiscoveryRegistry, error) { switch discovery.Enable { - case "zookeeper": - return zookeeper.NewZkClient( - discovery.ZooKeeper.Address, - discovery.ZooKeeper.Schema, - zookeeper.WithFreq(time.Hour), - zookeeper.WithUserNameAndPassword(discovery.ZooKeeper.Username, discovery.ZooKeeper.Password), - zookeeper.WithRoundRobin(), - zookeeper.WithTimeout(10), - ) case "k8s": - return kubernetes.NewK8sDiscoveryRegister(share.RpcRegisterName.MessageGateway) + return kubernetes.NewK8sDiscoveryRegister(discovery.RpcService.MessageGateway) case "etcd": return etcd.NewSvcDiscoveryRegistry( discovery.Etcd.RootDirectory, diff --git a/pkg/common/prommetrics/rpc.go b/pkg/common/prommetrics/rpc.go index 7162fa7e8..809d509b2 100644 --- a/pkg/common/prommetrics/rpc.go +++ b/pkg/common/prommetrics/rpc.go @@ -42,25 +42,25 @@ func GetGrpcServerMetrics() *gp.ServerMetrics { return grpcMetrics } -func GetGrpcCusMetrics(registerName string, share *config.Share) []prometheus.Collector { +func GetGrpcCusMetrics(registerName string, discovery *config.Discovery) []prometheus.Collector { switch registerName { - case share.RpcRegisterName.MessageGateway: + case discovery.RpcService.MessageGateway: return []prometheus.Collector{OnlineUserGauge} - case share.RpcRegisterName.Msg: + case discovery.RpcService.Msg: return []prometheus.Collector{ SingleChatMsgProcessSuccessCounter, SingleChatMsgProcessFailedCounter, GroupChatMsgProcessSuccessCounter, GroupChatMsgProcessFailedCounter, } - case share.RpcRegisterName.Push: + case discovery.RpcService.Push: return []prometheus.Collector{ MsgOfflinePushFailedCounter, MsgLoneTimePushCounter, } - case share.RpcRegisterName.Auth: + case discovery.RpcService.Auth: return []prometheus.Collector{UserLoginCounter} - case share.RpcRegisterName.User: + case discovery.RpcService.User: return []prometheus.Collector{UserRegisterCounter} default: return nil diff --git a/pkg/common/startrpc/start.go b/pkg/common/startrpc/start.go index 26fbb0ffa..7671b1736 100644 --- a/pkg/common/startrpc/start.go +++ b/pkg/common/startrpc/start.go @@ -77,7 +77,7 @@ func Start[T any](ctx context.Context, discovery *config.Discovery, prometheusCo if err != nil { return err } - cs := prommetrics.GetGrpcCusMetrics(rpcRegisterName, share) + cs := prommetrics.GetGrpcCusMetrics(rpcRegisterName, discovery) go func() { if err := prommetrics.RpcInit(cs, prometheusPort); err != nil && !errors.Is(err, http.ErrServerClosed) { netErr = errs.WrapMsg(err, fmt.Sprintf("rpc %s prometheus start err: %d", rpcRegisterName, prometheusPort)) @@ -111,7 +111,7 @@ func Start[T any](ctx context.Context, discovery *config.Discovery, prometheusCo "prometheusPorts", prometheusConfig.Ports) defer listener.Close() - client, err := kdisc.NewDiscoveryRegister(discovery, share) + client, err := kdisc.NewDiscoveryRegister(discovery) if err != nil { return err } diff --git a/tools/check-component/main.go b/tools/check-component/main.go index 94dbd613c..a84b7bd87 100644 --- a/tools/check-component/main.go +++ b/tools/check-component/main.go @@ -160,10 +160,6 @@ func performChecks(ctx context.Context, mongoConfig *config.Mongo, redisConfig * checks["Etcd"] = func(ctx context.Context) error { return CheckEtcd(ctx, &discovery.Etcd) } - } else if discovery.Enable == "zookeeper" { - checks["Zookeeper"] = func(ctx context.Context) error { - return CheckZookeeper(ctx, &discovery.ZooKeeper) - } } for i := 0; i < maxRetry; i++ { From 59fb9886e5c182f4cde5fd1acdc87d1b268739fa Mon Sep 17 00:00:00 2001 From: chao <48119764+withchao@users.noreply.github.com> Date: Sat, 7 Dec 2024 14:21:14 +0800 Subject: [PATCH 064/199] feat: support aws (#2938) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * update gomake version * update gomake version * fix: seq conversion bug * fix: redis pipe exec * fix: ImportFriends * fix: A large number of logs keysAndValues ​​length is not even * feat: mark read aggregate write * feat: online status supports redis cluster * feat: online status supports redis cluster * feat: online status supports redis cluster * merge * merge * read seq is written to mongo * read seq is written to mongo * fix: invitation to join group notification * fix: friend op_user_id * feat: optimizing asynchronous context * feat: optimizing memamq size * feat: add GetSeqMessage * feat: implement no gob encoder. * update unitTest content. * Update hub_server.go * feat: GroupApplicationAgreeMemberEnterNotification * fix: encoder replace to json encoder. * feat: GroupApplicationAgreeMemberEnterNotification * feat: go.mod * feat: go.mod * feat: join group notification and get seq * feat: join group notification and get seq * feat: avoid pulling messages from sessions with a large number of max seq values of 0 * feat: API supports gzip * go.mod * fix: nil pointer error on close * fix: listen error * fix: listen error * update go.mod * feat: add log * fix: token parse token value * fix: GetMsgBySeqs boundary issues * merge: merge main code into js branch. (#2648) * feat: update group notification when set to null. (#2590) * refactor: refactor workflows contents. * add tool workflows. * update field. * fix: remove chat error. * Fix err. * fix error. * remove cn comment. * update workflows files. * update infra config. * move workflows. * feat: update bot. * fix: solve uncorrect outdated msg get. * update get docIDs logic. * update * update skip logic. * fix * update. * fix: delay deleteObject func. * remove unused content. * feat: update group notification when set to null. * update log standard. * feat: add long time push msg in prometheus (#2584) * feat: add long time push msg in prometheus * fix: log print * fix: go mod * fix: log msg * fix: log init * feat: push msg * feat: go mod ,remove cgo package * feat: remove error log * feat: test dummy push * feat:redis pool config * feat: push to kafka log * feat: supports getting messages based on session ID and seq (#2582) * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * update gomake version * update gomake version * fix: seq conversion bug * fix: redis pipe exec * fix: ImportFriends * fix: A large number of logs keysAndValues ​​length is not even * feat: mark read aggregate write * feat: online status supports redis cluster * feat: online status supports redis cluster * feat: online status supports redis cluster * merge * merge * read seq is written to mongo * read seq is written to mongo * fix: invitation to join group notification * fix: friend op_user_id * feat: optimizing asynchronous context * feat: optimizing memamq size * feat: add GetSeqMessage * feat: GroupApplicationAgreeMemberEnterNotification * feat: GroupApplicationAgreeMemberEnterNotification * feat: go.mod * feat: go.mod * feat: join group notification and get seq --------- Co-authored-by: withchao * feat: implement request batch count limit. (#2591) * refactor: refactor workflows contents. * add tool workflows. * update field. * fix: remove chat error. * Fix err. * fix error. * remove cn comment. * update workflows files. * update infra config. * move workflows. * feat: update bot. * fix: solve uncorrect outdated msg get. * update get docIDs logic. * update * update skip logic. * fix * update. * fix: delay deleteObject func. * remove unused content. * update log type. * feat: implement request batch count limit. * update * update * fix: getting messages based on session ID and seq (#2595) * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * update gomake version * update gomake version * fix: seq conversion bug * fix: redis pipe exec * fix: ImportFriends * fix: A large number of logs keysAndValues ​​length is not even * feat: mark read aggregate write * feat: online status supports redis cluster * feat: online status supports redis cluster * feat: online status supports redis cluster * merge * merge * read seq is written to mongo * read seq is written to mongo * fix: invitation to join group notification * fix: friend op_user_id * feat: optimizing asynchronous context * feat: optimizing memamq size * feat: add GetSeqMessage * feat: GroupApplicationAgreeMemberEnterNotification * feat: GroupApplicationAgreeMemberEnterNotification * feat: go.mod * feat: go.mod * feat: join group notification and get seq * feat: join group notification and get seq --------- Co-authored-by: withchao * feat: avoid pulling messages from sessions with a large number of max seq values of 0 (#2602) * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * update gomake version * update gomake version * fix: seq conversion bug * fix: redis pipe exec * fix: ImportFriends * fix: A large number of logs keysAndValues ​​length is not even * feat: mark read aggregate write * feat: online status supports redis cluster * feat: online status supports redis cluster * feat: online status supports redis cluster * merge * merge * read seq is written to mongo * read seq is written to mongo * fix: invitation to join group notification * fix: friend op_user_id * feat: optimizing asynchronous context * feat: optimizing memamq size * feat: add GetSeqMessage * feat: GroupApplicationAgreeMemberEnterNotification * feat: GroupApplicationAgreeMemberEnterNotification * feat: go.mod * feat: go.mod * feat: join group notification and get seq * feat: join group notification and get seq * feat: avoid pulling messages from sessions with a large number of max seq values of 0 --------- Co-authored-by: withchao * refactor: improve db structure in `storage/controller` (#2604) * refactor: refactor workflows contents. * add tool workflows. * update field. * fix: remove chat error. * Fix err. * fix error. * remove cn comment. * update workflows files. * update infra config. * move workflows. * feat: update bot. * fix: solve uncorrect outdated msg get. * update get docIDs logic. * update * update skip logic. * fix * update. * fix: delay deleteObject func. * remove unused content. * update log type. * feat: implement request batch count limit. * update * update * refactor: improve db structure in `storage/controller` * feat: implement offline push using kafka (#2600) * refactor: refactor workflows contents. * add tool workflows. * update field. * fix: remove chat error. * Fix err. * fix error. * remove cn comment. * update workflows files. * update infra config. * move workflows. * feat: update bot. * fix: solve uncorrect outdated msg get. * update get docIDs logic. * update * update skip logic. * fix * update. * fix: delay deleteObject func. * remove unused content. * update log type. * feat: implement request batch count limit. * update * update * feat: implement offline push. * feat: implement batch Push spilt * update go mod * feat: implement kafka producer and consumer. * update format, * add PushMQ log. * feat: update Handler logic. * update MQ logic. * update * update * fix: update OfflinePushConsumerHandler. * feat: API supports gzip (#2609) * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * update gomake version * update gomake version * fix: seq conversion bug * fix: redis pipe exec * fix: ImportFriends * fix: A large number of logs keysAndValues ​​length is not even * feat: mark read aggregate write * feat: online status supports redis cluster * feat: online status supports redis cluster * feat: online status supports redis cluster * merge * merge * read seq is written to mongo * read seq is written to mongo * fix: invitation to join group notification * fix: friend op_user_id * feat: optimizing asynchronous context * feat: optimizing memamq size * feat: add GetSeqMessage * feat: GroupApplicationAgreeMemberEnterNotification * feat: GroupApplicationAgreeMemberEnterNotification * feat: go.mod * feat: go.mod * feat: join group notification and get seq * feat: join group notification and get seq * feat: avoid pulling messages from sessions with a large number of max seq values of 0 * feat: API supports gzip --------- Co-authored-by: withchao * Fix err (#2608) * refactor: refactor workflows contents. * add tool workflows. * update field. * fix: remove chat error. * Fix err. * fix error. * remove cn comment. * update workflows files. * update infra config. * move workflows. * feat: update bot. * fix: solve uncorrect outdated msg get. * update get docIDs logic. * update * update skip logic. * fix * update. * fix: delay deleteObject func. * remove unused content. * update log type. * feat: implement request batch count limit. * update * update * feat: add rocksTimeout * feat: wrap logs * feat: add logs * feat: listen config * feat: enable listen TIME_WAIT port * feat: add logs * feat: cache batch * chore: enable fullUserCache * feat: push rpc num * feat: push err * feat: with operationID * feat: sleep * feat: change 1s * feat: change log * feat: implement Getbatch in rpcCache. * feat: print getOnline cost * feat: change log * feat: change kafka and push config * feat: del interface * feat: fix err * feat: change config * feat: go mod * feat: change config * feat: change config * feat: add sleep in push * feat: warn logs * feat: logs * feat: logs * feat: change port * feat: start config * feat: remove port reuse * feat: prometheus config * feat: prometheus config * feat: prometheus config * feat: add long time send msg to grafana * feat: init * feat: init * feat: implement offline push. * feat: batch get user online * feat: implement batch Push spilt * update go mod * Revert "feat: change port" This reverts commit 06d5e944 * feat: change port * feat: change config * feat: implement kafka producer and consumer. * update format, * add PushMQ log. * feat: get all online users and init push * feat: lock in online cache * feat: config * fix: init online status * fix: add logs * fix: userIDs * fix: add logs * feat: update Handler logic. * update MQ logic. * update * update * fix: method name * fix: update OfflinePushConsumerHandler. * fix: prommetrics * fix: add logs * fix: ctx * fix: log * fix: config * feat: change port * fix: atomic online cache status --------- Co-authored-by: Monet Lee * feature: add GetConversationsHasReadAndMaxSeq interface to the WebSocket API. (#2611) * fix: lru lock (#2613) * fix: lru lock * fix: lru lock * fix: lru lock * fix: nil pointer error on close (#2618) * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * update gomake version * update gomake version * fix: seq conversion bug * fix: redis pipe exec * fix: ImportFriends * fix: A large number of logs keysAndValues ​​length is not even * feat: mark read aggregate write * feat: online status supports redis cluster * feat: online status supports redis cluster * feat: online status supports redis cluster * merge * merge * read seq is written to mongo * read seq is written to mongo * fix: invitation to join group notification * fix: friend op_user_id * feat: optimizing asynchronous context * feat: optimizing memamq size * feat: add GetSeqMessage * feat: GroupApplicationAgreeMemberEnterNotification * feat: GroupApplicationAgreeMemberEnterNotification * feat: go.mod * feat: go.mod * feat: join group notification and get seq * feat: join group notification and get seq * feat: avoid pulling messages from sessions with a large number of max seq values of 0 * feat: API supports gzip * go.mod * fix: nil pointer error on close --------- Co-authored-by: withchao * feat: create group can push notification (#2617) * fix: blockage caused by listen error (#2620) * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * update gomake version * update gomake version * fix: seq conversion bug * fix: redis pipe exec * fix: ImportFriends * fix: A large number of logs keysAndValues ​​length is not even * feat: mark read aggregate write * feat: online status supports redis cluster * feat: online status supports redis cluster * feat: online status supports redis cluster * merge * merge * read seq is written to mongo * read seq is written to mongo * fix: invitation to join group notification * fix: friend op_user_id * feat: optimizing asynchronous context * feat: optimizing memamq size * feat: add GetSeqMessage * feat: GroupApplicationAgreeMemberEnterNotification * feat: GroupApplicationAgreeMemberEnterNotification * feat: go.mod * feat: go.mod * feat: join group notification and get seq * feat: join group notification and get seq * feat: avoid pulling messages from sessions with a large number of max seq values of 0 * feat: API supports gzip * go.mod * fix: nil pointer error on close * fix: listen error --------- Co-authored-by: withchao * fix: go.mod (#2621) * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * update gomake version * update gomake version * fix: seq conversion bug * fix: redis pipe exec * fix: ImportFriends * fix: A large number of logs keysAndValues ​​length is not even * feat: mark read aggregate write * feat: online status supports redis cluster * feat: online status supports redis cluster * feat: online status supports redis cluster * merge * merge * read seq is written to mongo * read seq is written to mongo * fix: invitation to join group notification * fix: friend op_user_id * feat: optimizing asynchronous context * feat: optimizing memamq size * feat: add GetSeqMessage * feat: GroupApplicationAgreeMemberEnterNotification * feat: GroupApplicationAgreeMemberEnterNotification * feat: go.mod * feat: go.mod * feat: join group notification and get seq * feat: join group notification and get seq * feat: avoid pulling messages from sessions with a large number of max seq values of 0 * feat: API supports gzip * go.mod * fix: nil pointer error on close * fix: listen error * fix: listen error * update go.mod --------- Co-authored-by: withchao * feat: improve searchMsg implement. (#2614) * refactor: refactor workflows contents. * add tool workflows. * update field. * fix: remove chat error. * Fix err. * fix error. * remove cn comment. * update workflows files. * update infra config. * move workflows. * feat: update bot. * fix: solve uncorrect outdated msg get. * update get docIDs logic. * update * update skip logic. * fix * update. * fix: delay deleteObject func. * remove unused content. * update log type. * feat: implement request batch count limit. * update * update * remove unused script. * feat: improve searchMsg implement. * update mongo config. * Fix lock (#2622) * fix:log * fix: lock * fix: update setGroupInfoEX field name. (#2625) * refactor: refactor workflows contents. * add tool workflows. * update field. * fix: remove chat error. * Fix err. * fix error. * remove cn comment. * update workflows files. * update infra config. * move workflows. * feat: update bot. * fix: solve uncorrect outdated msg get. * update get docIDs logic. * update * update skip logic. * fix * update. * fix: delay deleteObject func. * remove unused content. * update log type. * feat: implement request batch count limit. * update * update * fix: update setGroupInfoEX field name. * fix: update setGroupInfoEX field name (#2626) * refactor: refactor workflows contents. * add tool workflows. * update field. * fix: remove chat error. * Fix err. * fix error. * remove cn comment. * update workflows files. * update infra config. * move workflows. * feat: update bot. * fix: solve uncorrect outdated msg get. * update get docIDs logic. * update * update skip logic. * fix * update. * fix: delay deleteObject func. * remove unused content. * update log type. * feat: implement request batch count limit. * update * update * fix: update setGroupInfoEX field name. * fix: update setGroupInfoEX field name * feat: msg gateway add log (#2631) * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * update gomake version * update gomake version * fix: seq conversion bug * fix: redis pipe exec * fix: ImportFriends * fix: A large number of logs keysAndValues ​​length is not even * feat: mark read aggregate write * feat: online status supports redis cluster * feat: online status supports redis cluster * feat: online status supports redis cluster * merge * merge * read seq is written to mongo * read seq is written to mongo * fix: invitation to join group notification * fix: friend op_user_id * feat: optimizing asynchronous context * feat: optimizing memamq size * feat: add GetSeqMessage * feat: GroupApplicationAgreeMemberEnterNotification * feat: GroupApplicationAgreeMemberEnterNotification * feat: go.mod * feat: go.mod * feat: join group notification and get seq * feat: join group notification and get seq * feat: avoid pulling messages from sessions with a large number of max seq values of 0 * feat: API supports gzip * go.mod * fix: nil pointer error on close * fix: listen error * fix: listen error * update go.mod * feat: add log --------- Co-authored-by: withchao * fix: update setGroupInfoEx func name and field. (#2634) * refactor: refactor workflows contents. * add tool workflows. * update field. * fix: remove chat error. * Fix err. * fix error. * remove cn comment. * update workflows files. * update infra config. * move workflows. * feat: update bot. * fix: solve uncorrect outdated msg get. * update get docIDs logic. * update * update skip logic. * fix * update. * fix: delay deleteObject func. * remove unused content. * update log type. * feat: implement request batch count limit. * update * update * fix: update setGroupInfoEx func name and field. * refactor: update groupinfoEx field. * refactor: update database name in mongodb.yml * add groupName Condition * fix: fix setConversations req fill. (#2645) * refactor: refactor workflows contents. * add tool workflows. * update field. * fix: remove chat error. * Fix err. * fix error. * remove cn comment. * update workflows files. * update infra config. * move workflows. * feat: update bot. * fix: solve uncorrect outdated msg get. * update get docIDs logic. * update * update skip logic. * fix * update. * fix: delay deleteObject func. * remove unused content. * update log type. * feat: implement request batch count limit. * update * update * fix: fix setConversations req fill. * fix: GetMsgBySeqs boundary issues (#2647) * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * update gomake version * update gomake version * fix: seq conversion bug * fix: redis pipe exec * fix: ImportFriends * fix: A large number of logs keysAndValues ​​length is not even * feat: mark read aggregate write * feat: online status supports redis cluster * feat: online status supports redis cluster * feat: online status supports redis cluster * merge * merge * read seq is written to mongo * read seq is written to mongo * fix: invitation to join group notification * fix: friend op_user_id * feat: optimizing asynchronous context * feat: optimizing memamq size * feat: add GetSeqMessage * feat: GroupApplicationAgreeMemberEnterNotification * feat: GroupApplicationAgreeMemberEnterNotification * feat: go.mod * feat: go.mod * feat: join group notification and get seq * feat: join group notification and get seq * feat: avoid pulling messages from sessions with a large number of max seq values of 0 * feat: API supports gzip * go.mod * fix: nil pointer error on close * fix: listen error * fix: listen error * update go.mod * feat: add log * fix: token parse token value * fix: GetMsgBySeqs boundary issues --------- Co-authored-by: withchao * fix: the attribute version is obsolete, remove it (#2644) * refactor: update Userregister request field. (#2650) --------- Co-authored-by: Monet Lee Co-authored-by: icey-yu <119291641+icey-yu@users.noreply.github.com> Co-authored-by: chao <48119764+withchao@users.noreply.github.com> Co-authored-by: withchao Co-authored-by: 蔡相跃 * update go mod * fix: sn_ not sort * fix: sn_ not sort * fix: sn_ not sort * fix: jssdk add * fix: jssdk support * fix: jssdk support * fix: jssdk support * fix: the message I sent is not set to read seq in mongodb * fix: cannot modify group member avatars * merge: update code from main to v3.8-js-sdk-only. (#2720) * fix: fix update groupName invalid. (#2673) * refactor: change platform to platformID (#2670) * feat: don`t return nil data (#2675) Co-authored-by: Monet Lee * refactor: update fields type in userStatus and check registered. (#2676) * fix: usertoken auth. (#2677) * refactor: update fields type in userStatus and check registered. * fix: usertoken auth. * update contents. * update content. * update * fix * update pb file. * feat: add friend agree after callback (#2680) * fix: sn not sort (#2682) * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * update gomake version * update gomake version * fix: seq conversion bug * fix: redis pipe exec * fix: ImportFriends * fix: A large number of logs keysAndValues ​​length is not even * feat: mark read aggregate write * feat: online status supports redis cluster * feat: online status supports redis cluster * feat: online status supports redis cluster * merge * merge * read seq is written to mongo * read seq is written to mongo * fix: invitation to join group notification * fix: friend op_user_id * feat: optimizing asynchronous context * feat: optimizing memamq size * feat: add GetSeqMessage * feat: GroupApplicationAgreeMemberEnterNotification * feat: GroupApplicationAgreeMemberEnterNotification * feat: go.mod * feat: go.mod * feat: join group notification and get seq * feat: join group notification and get seq * feat: avoid pulling messages from sessions with a large number of max seq values of 0 * feat: API supports gzip * go.mod * fix: nil pointer error on close * fix: listen error * fix: listen error * update go.mod * feat: add log * fix: token parse token value * fix: GetMsgBySeqs boundary issues * fix: sn_ not sort * fix: sn_ not sort * fix: sn_ not sort --------- Co-authored-by: withchao * refactor: add GetAdminToken interface. (#2684) * refactor: add GetAdminToken interface. * update config. * fix: admin token (#2686) * fix: update workflows logic. (#2688) * refactor: add GetAdminToken interface. * update config. * update workflows logic. * fix: admin token (#2687) * update the front image (#2692) * update the front image * update version * feat: improve publish docker image workflows (#2697) * refactor: add GetAdminToken interface. * update config. * update workflows logic. * feat: improve publish docker image workflows * update condition logic. * fix: update load file logic. (#2700) * refactor: add GetAdminToken interface. * update config. * update workflows logic. * feat: improve publish docker image workflows * update condition logic. * fix: update load file logic. * feat: Msg filter (#2703) * feat: msg filter * feat: msg filter * feat: msg filter * feat: provide the interface required by js sdk (#2712) * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * update gomake version * update gomake version * fix: seq conversion bug * fix: redis pipe exec * fix: ImportFriends * fix: A large number of logs keysAndValues ​​length is not even * feat: mark read aggregate write * feat: online status supports redis cluster * feat: online status supports redis cluster * feat: online status supports redis cluster * merge * merge * read seq is written to mongo * read seq is written to mongo * fix: invitation to join group notification * fix: friend op_user_id * feat: optimizing asynchronous context * feat: optimizing memamq size * feat: add GetSeqMessage * feat: GroupApplicationAgreeMemberEnterNotification * feat: GroupApplicationAgreeMemberEnterNotification * feat: go.mod * feat: go.mod * feat: join group notification and get seq * feat: join group notification and get seq * feat: avoid pulling messages from sessions with a large number of max seq values of 0 * feat: API supports gzip * go.mod * fix: nil pointer error on close * fix: listen error * fix: listen error * update go.mod * feat: add log * fix: token parse token value * fix: GetMsgBySeqs boundary issues * fix: sn_ not sort * fix: sn_ not sort * fix: sn_ not sort * fix: jssdk add * fix: jssdk support * fix: jssdk support * fix: jssdk support --------- Co-authored-by: withchao * Line webhook (#2716) * feat: online and offline webhook * feat: online and offline webhook * feat: remove zk * fix: the message I sent is not set to read seq in mongodb (#2718) * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * update gomake version * update gomake version * fix: seq conversion bug * fix: redis pipe exec * fix: ImportFriends * fix: A large number of logs keysAndValues ​​length is not even * feat: mark read aggregate write * feat: online status supports redis cluster * feat: online status supports redis cluster * feat: online status supports redis cluster * merge * merge * read seq is written to mongo * read seq is written to mongo * fix: invitation to join group notification * fix: friend op_user_id * feat: optimizing asynchronous context * feat: optimizing memamq size * feat: add GetSeqMessage * feat: GroupApplicationAgreeMemberEnterNotification * feat: GroupApplicationAgreeMemberEnterNotification * feat: go.mod * feat: go.mod * feat: join group notification and get seq * feat: join group notification and get seq * feat: avoid pulling messages from sessions with a large number of max seq values of 0 * feat: API supports gzip * go.mod * fix: nil pointer error on close * fix: listen error * fix: listen error * update go.mod * feat: add log * fix: token parse token value * fix: GetMsgBySeqs boundary issues * fix: sn_ not sort * fix: sn_ not sort * fix: sn_ not sort * fix: jssdk add * fix: jssdk support * fix: jssdk support * fix: jssdk support * fix: the message I sent is not set to read seq in mongodb --------- Co-authored-by: withchao * fix: cannot modify group member avatars (#2719) * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * update gomake version * update gomake version * fix: seq conversion bug * fix: redis pipe exec * fix: ImportFriends * fix: A large number of logs keysAndValues ​​length is not even * feat: mark read aggregate write * feat: online status supports redis cluster * feat: online status supports redis cluster * feat: online status supports redis cluster * merge * merge * read seq is written to mongo * read seq is written to mongo * fix: invitation to join group notification * fix: friend op_user_id * feat: optimizing asynchronous context * feat: optimizing memamq size * feat: add GetSeqMessage * feat: GroupApplicationAgreeMemberEnterNotification * feat: GroupApplicationAgreeMemberEnterNotification * feat: go.mod * feat: go.mod * feat: join group notification and get seq * feat: join group notification and get seq * feat: avoid pulling messages from sessions with a large number of max seq values of 0 * feat: API supports gzip * go.mod * fix: nil pointer error on close * fix: listen error * fix: listen error * update go.mod * feat: add log * fix: token parse token value * fix: GetMsgBySeqs boundary issues * fix: sn_ not sort * fix: sn_ not sort * fix: sn_ not sort * fix: jssdk add * fix: jssdk support * fix: jssdk support * fix: jssdk support * fix: the message I sent is not set to read seq in mongodb * fix: cannot modify group member avatars --------- Co-authored-by: withchao --------- Co-authored-by: Monet Lee Co-authored-by: icey-yu <119291641+icey-yu@users.noreply.github.com> Co-authored-by: chao <48119764+withchao@users.noreply.github.com> Co-authored-by: withchao Co-authored-by: skiffer-git <72860476+skiffer-git@users.noreply.github.com> * fix: MemberEnterNotification * fix: MemberEnterNotification * fix: MsgData status * merge * merge: update code from main to v3.8-js-sdk-only. (#2818) * feat: implement merge milestone PR to target-branch. (#2796) * build: improve workflows logic. (#2801) * fix: improve time condition check mehtod. (#2804) * fix: improve time condition check mehtod. * fix * fix: webhook before online push (#2805) * fix: set own read seq in MongoDB when sender send a message. (#2808) * fix: solve err Notification when setGroupInfo. (#2806) * fix: solve err Notification when setGroupInfo. * build: update checkout version. * fix: update notification contents. * Introducing OpenIM Guru on Gurubase.io (#2788) * feat: support app update service (#2811) * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * update gomake version * update gomake version * fix: seq conversion bug * fix: redis pipe exec * fix: ImportFriends * fix: A large number of logs keysAndValues ​​length is not even * feat: mark read aggregate write * feat: online status supports redis cluster * feat: online status supports redis cluster * feat: online status supports redis cluster * merge * merge * read seq is written to mongo * read seq is written to mongo * fix: invitation to join group notification * fix: friend op_user_id * feat: optimizing asynchronous context * feat: optimizing memamq size * feat: add GetSeqMessage * feat: GroupApplicationAgreeMemberEnterNotification * feat: GroupApplicationAgreeMemberEnterNotification * feat: go.mod * feat: go.mod * feat: join group notification and get seq * feat: join group notification and get seq * feat: avoid pulling messages from sessions with a large number of max seq values of 0 * feat: API supports gzip * go.mod * fix: nil pointer error on close * fix: listen error * fix: listen error * update go.mod * feat: add log * fix: token parse token value * fix: GetMsgBySeqs boundary issues * fix: sn_ not sort * fix: sn_ not sort * fix: sn_ not sort * fix: jssdk add * fix: jssdk support * fix: jssdk support * fix: jssdk support * fix: the message I sent is not set to read seq in mongodb * fix: cannot modify group member avatars * fix: MemberEnterNotification * fix: MemberEnterNotification * fix: MsgData status * feat: add ApplicationVersion * feat: add ApplicationVersion * feat: add ApplicationVersion --------- Co-authored-by: withchao * feat: ApplicationVersion move chat (#2813) * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * update gomake version * update gomake version * fix: seq conversion bug * fix: redis pipe exec * fix: ImportFriends * fix: A large number of logs keysAndValues ​​length is not even * feat: mark read aggregate write * feat: online status supports redis cluster * feat: online status supports redis cluster * feat: online status supports redis cluster * merge * merge * read seq is written to mongo * read seq is written to mongo * fix: invitation to join group notification * fix: friend op_user_id * feat: optimizing asynchronous context * feat: optimizing memamq size * feat: add GetSeqMessage * feat: GroupApplicationAgreeMemberEnterNotification * feat: GroupApplicationAgreeMemberEnterNotification * feat: go.mod * feat: go.mod * feat: join group notification and get seq * feat: join group notification and get seq * feat: avoid pulling messages from sessions with a large number of max seq values of 0 * feat: API supports gzip * go.mod * fix: nil pointer error on close * fix: listen error * fix: listen error * update go.mod * feat: add log * fix: token parse token value * fix: GetMsgBySeqs boundary issues * fix: sn_ not sort * fix: sn_ not sort * fix: sn_ not sort * fix: jssdk add * fix: jssdk support * fix: jssdk support * fix: jssdk support * fix: the message I sent is not set to read seq in mongodb * fix: cannot modify group member avatars * fix: MemberEnterNotification * fix: MemberEnterNotification * fix: MsgData status * feat: add ApplicationVersion * feat: ApplicationVersion move chat --------- Co-authored-by: withchao * fix: improve condition check. (#2815) --------- Co-authored-by: Monet Lee Co-authored-by: icey-yu <119291641+icey-yu@users.noreply.github.com> Co-authored-by: Kürşat Aktaş Co-authored-by: chao <48119764+withchao@users.noreply.github.com> Co-authored-by: withchao * feat: support text ping pong * feat: support text ping pong * feat: gob json encoder * feat: gob json encoder * feat: gob json encoder * feat: gob json encoder * feat: gob json encoder * feat: gob json encoder * fix: concurrent write to websocket connection * fix: concurrent write to websocket connection * fix: concurrent write to websocket connection * fix: group member update face_url * feat: support aws * feat: support aws * feat: support aws --------- Co-authored-by: withchao Co-authored-by: Monet Lee Co-authored-by: OpenIM-Gordon <46924906+FGadvancer@users.noreply.github.com> Co-authored-by: icey-yu <119291641+icey-yu@users.noreply.github.com> Co-authored-by: 蔡相跃 Co-authored-by: skiffer-git <72860476+skiffer-git@users.noreply.github.com> Co-authored-by: Kürşat Aktaş --- config/openim-rpc-third.yml | 17 +++++++++---- go.mod | 15 ++++++++--- go.sum | 50 ++++++++++++++++++++++++++----------- internal/rpc/third/third.go | 5 +++- pkg/common/config/config.go | 30 ++++++++++++++++------ 5 files changed, 86 insertions(+), 31 deletions(-) diff --git a/config/openim-rpc-third.yml b/config/openim-rpc-third.yml index 9578e82c6..719c65e21 100644 --- a/config/openim-rpc-third.yml +++ b/config/openim-rpc-third.yml @@ -34,10 +34,17 @@ object: sessionToken: publicRead: false kodo: - endpoint: http://s3.cn-south-1.qiniucs.com - bucket: kodo-bucket-test - bucketURL: http://kodo-bucket-test-oetobfb.qiniudns.com - accessKeyID: - accessKeySecret: + endpoint: https://s3.cn-south-1.qiniucs.com + bucket: testdemo12313 + bucketURL: http://so2at6d05.hn-bkt.clouddn.com + accessKeyID: + accessKeySecret: sessionToken: publicRead: false + aws: + region: ap-southeast-2 + bucket: testdemo832234 + accessKeyID: + secretAccessKey: + sessionToken: + publicRead: false \ No newline at end of file diff --git a/go.mod b/go.mod index 840016646..e7f340894 100644 --- a/go.mod +++ b/go.mod @@ -15,7 +15,7 @@ require ( github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/mitchellh/mapstructure v1.5.0 github.com/openimsdk/protocol v0.0.72-alpha.61 - github.com/openimsdk/tools v0.0.50-alpha.38 + github.com/openimsdk/tools v0.0.50-alpha.43 github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 github.com/stretchr/testify v1.9.0 @@ -55,9 +55,11 @@ require ( cloud.google.com/go/iam v1.1.7 // indirect cloud.google.com/go/longrunning v0.5.5 // indirect cloud.google.com/go/storage v1.40.0 // indirect + github.com/BurntSushi/toml v1.3.2 // indirect github.com/MicahParks/keyfunc v1.9.0 // indirect + github.com/alex-ant/gomath v0.0.0-20160516115720-89013a210a82 // indirect github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible // indirect - github.com/aws/aws-sdk-go-v2 v1.32.5 // indirect + github.com/aws/aws-sdk-go-v2 v1.32.6 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.1 // indirect github.com/aws/aws-sdk-go-v2/config v1.28.5 // indirect github.com/aws/aws-sdk-go-v2/credentials v1.17.46 // indirect @@ -90,15 +92,19 @@ require ( github.com/eapache/go-resiliency v1.6.0 // indirect github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 // indirect github.com/eapache/queue v1.1.0 // indirect + github.com/elastic/go-sysinfo v1.0.2 // indirect + github.com/elastic/go-windows v1.0.0 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/gabriel-vasile/mimetype v1.4.3 // indirect + github.com/gammazero/toposort v0.1.1 // indirect github.com/gin-contrib/sse v0.1.0 // indirect github.com/go-logr/logr v1.4.2 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-ole/go-ole v1.2.6 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect github.com/go-zookeeper/zk v1.0.3 // indirect + github.com/gofrs/flock v0.8.1 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/golang/snappy v0.0.4 // indirect @@ -119,6 +125,7 @@ require ( github.com/jinzhu/copier v0.4.0 // indirect github.com/jinzhu/inflection v1.0.0 // indirect github.com/jinzhu/now v1.1.5 // indirect + github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/kelindar/simd v1.1.2 // indirect github.com/klauspost/compress v1.17.7 // indirect @@ -143,7 +150,7 @@ require ( github.com/prometheus/client_model v0.5.0 // indirect github.com/prometheus/common v0.45.0 // indirect github.com/prometheus/procfs v0.12.0 // indirect - github.com/qiniu/go-sdk/v7 v7.18.2 // indirect + github.com/qiniu/go-sdk/v7 v7.25.0 // indirect github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect github.com/rs/xid v1.5.0 // indirect github.com/sagikazarmark/locafero v0.4.0 // indirect @@ -186,6 +193,8 @@ require ( google.golang.org/genproto/googleapis/api v0.0.0-20240903143218-8af14fe29dc1 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 // indirect gorm.io/gorm v1.25.8 // indirect + howett.net/plist v0.0.0-20181124034731-591f970eefbb // indirect + modernc.org/fileutil v1.0.0 // indirect stathat.com/c/consistent v1.0.0 // indirect ) diff --git a/go.sum b/go.sum index 1ca89efd4..afbd01f09 100644 --- a/go.sum +++ b/go.sum @@ -14,15 +14,19 @@ cloud.google.com/go/storage v1.40.0/go.mod h1:Rrj7/hKlG87BLqDJYtwR0fbPld8uJPbQ2u firebase.google.com/go/v4 v4.14.1 h1:4qiUETaFRWoFGE1XP5VbcEdtPX93Qs+8B/7KvP2825g= firebase.google.com/go/v4 v4.14.1/go.mod h1:fgk2XshgNDEKaioKco+AouiegSI9oTWVqRaBdTTGBoM= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/toml v1.3.2 h1:o7IhLm0Msx3BaB+n3Ag7L8EVlByGnpq14C4YWiu/gL8= +github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/IBM/sarama v1.43.0 h1:YFFDn8mMI2QL0wOrG0J2sFoVIAFl7hS9JQi2YZsXtJc= github.com/IBM/sarama v1.43.0/go.mod h1:zlE6HEbC/SMQ9mhEYaF7nNLYOUyrs0obySKCckWP9BM= github.com/MicahParks/keyfunc v1.9.0 h1:lhKd5xrFHLNOWrDc4Tyb/Q1AJ4LCzQ48GVJyVIID3+o= github.com/MicahParks/keyfunc v1.9.0/go.mod h1:IdnCilugA0O/99dW+/MkvlyrsX8+L8+x95xuVNtM5jw= github.com/QcloudApi/qcloud_sign_golang v0.0.0-20141224014652-e4130a326409/go.mod h1:1pk82RBxDY/JZnPQrtqHlUFfCctgdorsd9M06fMynOM= +github.com/alex-ant/gomath v0.0.0-20160516115720-89013a210a82 h1:7dONQ3WNZ1zy960TmkxJPuwoolZwL7xKtpcM04MBnt4= +github.com/alex-ant/gomath v0.0.0-20160516115720-89013a210a82/go.mod h1:nLnM0KdK1CmygvjpDUO6m1TjSsiQtL61juhNsvV/JVI= github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible h1:8psS8a+wKfiLt1iVDX79F7Y6wUM49Lcha2FMXt4UM8g= github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8= -github.com/aws/aws-sdk-go-v2 v1.32.5 h1:U8vdWJuY7ruAkzaOdD7guwJjD06YSKmnKCJs7s3IkIo= -github.com/aws/aws-sdk-go-v2 v1.32.5/go.mod h1:P5WJBrYqqbWVaOxgH0X/FYYD47/nooaPOZPlQdmiN2U= +github.com/aws/aws-sdk-go-v2 v1.32.6 h1:7BokKRgRPuGmKkFMhEg/jSul+tB9VvXhcViILtfG8b4= +github.com/aws/aws-sdk-go-v2 v1.32.6/go.mod h1:P5WJBrYqqbWVaOxgH0X/FYYD47/nooaPOZPlQdmiN2U= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.1 h1:ZY3108YtBNq96jNZTICHxN1gSBSbnvIdYwwqnvCV4Mc= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.1/go.mod h1:t8PYl/6LzdAqsU4/9tz28V/kU+asFePvpOMkdul0gEQ= github.com/aws/aws-sdk-go-v2/config v1.28.5 h1:Za41twdCXbuyyWv9LndXxZZv3QhTG1DinqlFsSuvtI0= @@ -86,6 +90,7 @@ github.com/coreos/go-systemd/v22 v22.3.2 h1:D9/bQk5vlXQFZ6Kwuu6zaiXJ9oTPe68++AzA github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/dave/jennifer v1.6.1/go.mod h1:nXbxhEmQfOZhWml3D1cDK5M1FLnMSozpbFN/m3RmGZc= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= @@ -103,6 +108,10 @@ github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 h1:Oy0F4A github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3/go.mod h1:YvSRo5mw33fLEx1+DlK6L2VV43tJt5Eyel9n9XBcR+0= github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc= github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= +github.com/elastic/go-sysinfo v1.0.2 h1:Wq1bOgnSz7Obl7DbMjbn0tzx1bE5G8Cfy3MVFa6C1Cc= +github.com/elastic/go-sysinfo v1.0.2/go.mod h1:O/D5m1VpYLwGjCYzEt63g3Z1uO3jXfwyzzjiW90t8cY= +github.com/elastic/go-windows v1.0.0 h1:qLURgZFkkrYyTTkvYpsZIgf83AUsdIHfvlJaqaZ7aSY= +github.com/elastic/go-windows v1.0.0/go.mod h1:TsU0Nrp7/y3+VwE82FoZF8gC/XFg/Elz6CcloAxnPgU= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= @@ -119,6 +128,8 @@ github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nos github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0= github.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk= +github.com/gammazero/toposort v0.1.1 h1:OivGxsWxF3U3+U80VoLJ+f50HcPU1MIqE1JlKzoJ2Eg= +github.com/gammazero/toposort v0.1.1/go.mod h1:H2cozTnNpMw0hg2VHAYsAxmkHXBYroNangj2NTBQDvw= github.com/gin-contrib/gzip v1.0.1 h1:HQ8ENHODeLY7a4g1Au/46Z92bdGFl74OhxcZble9WJE= github.com/gin-contrib/gzip v1.0.1/go.mod h1:njt428fdUNRvjuJf16tZMYZ2Yl+WQB53X5wmhDwXvC4= github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= @@ -143,7 +154,7 @@ github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+ github.com/go-playground/universal-translator v0.18.0/go.mod h1:UvRDBj+xPUEGrFYl+lu/H90nyDXpg0fqeB/AQUGNTVA= github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= -github.com/go-playground/validator/v10 v10.8.0/go.mod h1:9JhgTzTaE31GZDpH/HSvHiRJrJ3iKAgqqH0Bl/Ocjdk= +github.com/go-playground/validator/v10 v10.7.0/go.mod h1:xm76BBt941f7yWdGnI2DVPFFg1UK3YY04qifoXU3lOk= github.com/go-playground/validator/v10 v10.20.0 h1:K9ISHbSaI0lyB2eWMPJo+kOS/FBExVwjEviJTixqxL8= github.com/go-playground/validator/v10 v10.20.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM= github.com/go-redis/redis v6.15.9+incompatible h1:K0pv1D7EQUjfyoMql+r/jZqCLizCGKFlFgcHWWmHQjg= @@ -155,6 +166,8 @@ github.com/go-zookeeper/zk v1.0.3/go.mod h1:nOB03cncLtlp4t+UAkGSV+9beXP/akpekBwL github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/gofrs/flock v0.8.1 h1:+gYjHKf32LDeiEEFhQaotPbLuUXjY5ZqxKgXy7n59aw= +github.com/gofrs/flock v0.8.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang-jwt/jwt/v4 v4.4.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= @@ -204,6 +217,8 @@ github.com/googleapis/enterprise-certificate-proxy v0.3.2 h1:Vie5ybvEvT75RniqhfF github.com/googleapis/enterprise-certificate-proxy v0.3.2/go.mod h1:VLSiSSBs/ksPL8kq3OBOQ6WRI2QnaFynd1DCjZ62+V0= github.com/googleapis/gax-go/v2 v2.12.3 h1:5/zPPDvw8Q1SuXjrqrZslrqT7dL/uJT2CQii/cLCKqA= github.com/googleapis/gax-go/v2 v2.12.3/go.mod h1:AKloxT6GtNbaLm8QTNSidHUVsHYcBHwWRvkNFJUQcS4= +github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY= +github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ= github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4= github.com/gorilla/sessions v1.2.1/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM= github.com/gorilla/websocket v1.5.1 h1:gmztn0JnHVt9JZquRuzLw3g4wouNVzKL15iLr/zn/QY= @@ -222,6 +237,7 @@ github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/iancoleman/strcase v0.3.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/jcmturner/aescts/v2 v2.0.0 h1:9YKLH6ey7H4eDBXW8khjYslgyqG2xZikXP0EQFKrle8= @@ -236,12 +252,15 @@ github.com/jcmturner/gokrb5/v8 v8.4.4 h1:x1Sv4HaTpepFkXbt2IkL29DXRf8sOfZXo8eRKh6 github.com/jcmturner/gokrb5/v8 v8.4.4/go.mod h1:1btQEpgT6k+unzCwX1KdWMEwPPkkgBtP+F6aCACiMrs= github.com/jcmturner/rpc/v2 v2.0.3 h1:7FXXj8Ti1IaVFpSAziCZWNzbNuZmnvw/i6CqLNdWfZY= github.com/jcmturner/rpc/v2 v2.0.3/go.mod h1:VUJYCIDm3PVOEHw8sgt091/20OJjskO/YJki3ELg/Hc= +github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jinzhu/copier v0.4.0 h1:w3ciUoD19shMCRargcpm0cm91ytaBhDvuRpz1ODO/U8= github.com/jinzhu/copier v0.4.0/go.mod h1:DfbEm0FYsaqBcKcFuvmOZb218JkPGtvSHsKg8S8hyyg= github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= +github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901 h1:rp+c0RAYOWj8l6qbCUTSiRLG/iKnW3K3/QfPPuSsBt4= +github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901/go.mod h1:Z86h9688Y0wesXCyonoVr47MasHilkuLMqGhRZ4Hpak= github.com/jonboulle/clockwork v0.4.0 h1:p4Cf1aMWXnXAUh8lVfewRBx1zaTSYKrKMF2g3ST4RZ4= github.com/jonboulle/clockwork v0.4.0/go.mod h1:xgRqUGwRcjKCO1vbZUEtSLrqKoPSsUpK7fnezOII0kc= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= @@ -268,6 +287,7 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY= github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= @@ -321,13 +341,14 @@ github.com/openimsdk/gomake v0.0.14-alpha.5 h1:VY9c5x515lTfmdhhPjMvR3BBRrRquAUCF github.com/openimsdk/gomake v0.0.14-alpha.5/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= github.com/openimsdk/protocol v0.0.72-alpha.61 h1:RuZR9/Sg3p6Bpb2CKPjPoA2AUmTvHITmhZ3PT/RbWMs= github.com/openimsdk/protocol v0.0.72-alpha.61/go.mod h1:Iet+piS/jaS+kWWyj6EEr36mk4ISzIRYjoMSVA4dq2M= -github.com/openimsdk/tools v0.0.50-alpha.38 h1:AU6/cvDfN4ciIOwAj8IWEwze3DeEp2cHYPgW3y0OlbU= -github.com/openimsdk/tools v0.0.50-alpha.38/go.mod h1:/Em/fQH46CuWf60+hcmvZyboGCQpSDEb2MdQ4nmQRAk= +github.com/openimsdk/tools v0.0.50-alpha.43 h1:IKa8++fT46YBAVlfy/DcNPOJvJRlWX38pWIfwFIQ2AE= +github.com/openimsdk/tools v0.0.50-alpha.43/go.mod h1:YOtTcSQ7Ik6NYkjXCTXZT3YDM7smGp9D2xvti3KvNoM= github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= @@ -342,11 +363,12 @@ github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cY github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI= github.com/prometheus/common v0.45.0 h1:2BGz0eBc2hdMDLnO/8n0jeB3oPrt2D08CekT0lneoxM= github.com/prometheus/common v0.45.0/go.mod h1:YJmSTw9BoKxJplESWWxlbyttQR4uaEcGyv9MZjVOJsY= +github.com/prometheus/procfs v0.0.0-20190425082905-87a4384529e0/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= github.com/qiniu/dyn v1.3.0/go.mod h1:E8oERcm8TtwJiZvkQPbcAh0RL8jO1G0VXJMW3FAWdkk= -github.com/qiniu/go-sdk/v7 v7.18.2 h1:vk9eo5OO7aqgAOPF0Ytik/gt7CMKuNgzC/IPkhda6rk= -github.com/qiniu/go-sdk/v7 v7.18.2/go.mod h1:nqoYCNo53ZlGA521RvRethvxUDvXKt4gtYXOwye868w= +github.com/qiniu/go-sdk/v7 v7.25.0 h1:Roi4XMxRly9K4wb87DhQOKaQylyiphEXC7/l8uqJZaQ= +github.com/qiniu/go-sdk/v7 v7.25.0/go.mod h1:uZE85Pi0ftIHT/UNLShosdzwsovqpdas0LwAGO7cPao= github.com/qiniu/x v1.10.5/go.mod h1:03Ni9tj+N2h2aKnAz+6N0Xfl8FwMEDRC2PAlxekASDs= github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:N/ElC8H3+5XpJzTSTfLsJV/mx9Q9g7kxmchpfZyxgzM= github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= @@ -461,9 +483,7 @@ golang.org/x/arch v0.7.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= golang.org/x/crypto v0.27.0 h1:GXm2NjJrPaiv/h1tb2UH8QfgC/hOf/+z0p6PT8o1w7A= golang.org/x/crypto v0.27.0/go.mod h1:1Xngt8kV6Dvbssa53Ziq6Eqn0HqbZi5Z6R0ZpwQzt70= @@ -491,7 +511,6 @@ golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwY golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20220708220712-1185a9018129/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.29.0 h1:5ORfpBpCs4HzDYoodCDBbwHzdR5UrLBZ3sOnUJmFoHo= @@ -501,6 +520,7 @@ golang.org/x/oauth2 v0.23.0 h1:PbgcYx2W7i4LvjJWEbf0ngHV6qJYr86PkAV3bXdLEbs= golang.org/x/oauth2 v0.23.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -510,22 +530,20 @@ golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190425145619-16072639606e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34= golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= @@ -533,7 +551,6 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= -golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224= golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= @@ -596,6 +613,7 @@ gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= @@ -607,6 +625,10 @@ gorm.io/gorm v1.25.8 h1:WAGEZ/aEcznN4D03laj8DKnehe1e9gYQAjW8xyPRdeo= gorm.io/gorm v1.25.8/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +howett.net/plist v0.0.0-20181124034731-591f970eefbb h1:jhnBjNi9UFpfpl8YZhA9CrOqpnJdvzuiHsl/dnxl11M= +howett.net/plist v0.0.0-20181124034731-591f970eefbb/go.mod h1:vMygbs4qMhSZSc4lCUl2OEE+rDiIIJAIdR4m7MiMcm0= +modernc.org/fileutil v1.0.0 h1:Z1AFLZwl6BO8A5NldQg/xTSjGLetp+1Ubvl4alfGx8w= +modernc.org/fileutil v1.0.0/go.mod h1:JHsWpkrk/CnVV1H/eGlFf85BEpfkrp56ro8nojIq9Q8= nullprogram.com/x/optparse v1.0.0/go.mod h1:KdyPE+Igbe0jQUrVfMqDMeJQIJZEuyV7pjYmp6pbG50= rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= stathat.com/c/consistent v1.0.0 h1:ezyc51EGcRPJUxfHGSgJjWzJdj3NiMU9pNfLNGiXV0c= diff --git a/internal/rpc/third/third.go b/internal/rpc/third/third.go index 75076ab74..52e48855c 100644 --- a/internal/rpc/third/third.go +++ b/internal/rpc/third/third.go @@ -21,6 +21,8 @@ import ( "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/redis" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database/mgo" "github.com/openimsdk/open-im-server/v3/pkg/localcache" + "github.com/openimsdk/tools/s3/aws" + "github.com/openimsdk/tools/s3/kodo" "time" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/controller" @@ -31,7 +33,6 @@ import ( "github.com/openimsdk/tools/discovery" "github.com/openimsdk/tools/s3" "github.com/openimsdk/tools/s3/cos" - "github.com/openimsdk/tools/s3/kodo" "github.com/openimsdk/tools/s3/minio" "github.com/openimsdk/tools/s3/oss" "google.golang.org/grpc" @@ -92,6 +93,8 @@ func Start(ctx context.Context, config *Config, client discovery.SvcDiscoveryReg o, err = oss.NewOSS(*config.RpcConfig.Object.Oss.Build()) case "kodo": o, err = kodo.NewKodo(*config.RpcConfig.Object.Kodo.Build()) + case "aws": + o, err = aws.NewAws(*config.RpcConfig.Object.Aws.Build()) default: err = fmt.Errorf("invalid object enable: %s", enable) } diff --git a/pkg/common/config/config.go b/pkg/common/config/config.go index 6082b5f23..b13da1197 100644 --- a/pkg/common/config/config.go +++ b/pkg/common/config/config.go @@ -15,6 +15,7 @@ package config import ( + "github.com/openimsdk/tools/s3/aws" "strings" "time" @@ -298,14 +299,7 @@ type Third struct { Cos Cos `mapstructure:"cos"` Oss Oss `mapstructure:"oss"` Kodo Kodo `mapstructure:"kodo"` - Aws struct { - Endpoint string `mapstructure:"endpoint"` - Region string `mapstructure:"region"` - Bucket string `mapstructure:"bucket"` - AccessKeyID string `mapstructure:"accessKeyID"` - AccessKeySecret string `mapstructure:"accessKeySecret"` - PublicRead bool `mapstructure:"publicRead"` - } `mapstructure:"aws"` + Aws Aws `mapstructure:"aws"` } `mapstructure:"object"` } type Cos struct { @@ -335,6 +329,15 @@ type Kodo struct { PublicRead bool `mapstructure:"publicRead"` } +type Aws struct { + Region string `mapstructure:"region"` + Bucket string `mapstructure:"bucket"` + AccessKeyID string `mapstructure:"accessKeyID"` + SecretAccessKey string `mapstructure:"secretAccessKey"` + SessionToken string `mapstructure:"sessionToken"` + PublicRead bool `mapstructure:"publicRead"` +} + type User struct { RPC struct { RegisterIP string `mapstructure:"registerIP"` @@ -541,6 +544,7 @@ func (m *Minio) Build() *minio.Config { SignEndpoint: formatEndpoint(m.ExternalAddress), } } + func (c *Cos) Build() *cos.Config { return &cos.Config{ BucketURL: c.BucketURL, @@ -575,6 +579,16 @@ func (o *Kodo) Build() *kodo.Config { } } +func (o *Aws) Build() *aws.Config { + return &aws.Config{ + Region: o.Region, + Bucket: o.Bucket, + AccessKeyID: o.AccessKeyID, + SecretAccessKey: o.SecretAccessKey, + SessionToken: o.SessionToken, + } +} + func (l *CacheConfig) Failed() time.Duration { return time.Second * time.Duration(l.FailedExpire) } From 24ab94087560595f3b95f8b2da6b84049b0847d4 Mon Sep 17 00:00:00 2001 From: Monet Lee Date: Sat, 7 Dec 2024 17:35:42 +0800 Subject: [PATCH 065/199] build: implement services image build and CI release. (#2920) * build: implement services image build. * remove unused tools * update test. * update images. * update dockerfile and go mod. * update go mod. * Add comments. * update go pkg. * update loadConfig and discovery logic in kubernetes. * update go pkg and discovery field. * update Load method args. --- .../build-and-release-services-images.yml | 91 +++++ Dockerfile | 2 +- build/images/Dockerfile | 38 +- build/images/openim-api/Dockerfile | 56 ++- build/images/openim-cmdutils/Dockerfile | 44 --- build/images/openim-crontask/Dockerfile | 55 ++- build/images/openim-msggateway/Dockerfile | 55 ++- build/images/openim-msgtransfer/Dockerfile | 55 ++- build/images/openim-push/Dockerfile | 55 ++- build/images/openim-rpc-auth/Dockerfile | 55 ++- .../images/openim-rpc-conversation/Dockerfile | 55 ++- build/images/openim-rpc-friend/Dockerfile | 55 ++- build/images/openim-rpc-group/Dockerfile | 55 ++- build/images/openim-rpc-msg/Dockerfile | 55 ++- build/images/openim-rpc-third/Dockerfile | 55 ++- build/images/openim-rpc-user/Dockerfile | 55 ++- .../images/openim-tools/component/Dockerfile | 124 ++++-- config/discovery.yml | 7 +- config/openim-msggateway.yml | 1 + config/openim-push.yml | 1 + config/openim-rpc-auth.yml | 1 + config/openim-rpc-conversation.yml | 1 + config/openim-rpc-friend.yml | 1 + config/openim-rpc-group.yml | 1 + config/openim-rpc-msg.yml | 1 + config/openim-rpc-third.yml | 1 + config/openim-rpc-user.yml | 1 + go.mod | 30 +- go.sum | 75 +++- internal/api/init.go | 16 +- internal/msggateway/init.go | 10 +- internal/msgtransfer/init.go | 10 +- internal/tools/cron_task.go | 9 +- magefile.go | 13 +- pkg/common/cmd/root.go | 11 +- pkg/common/config/config.go | 8 +- pkg/common/config/constant.go | 6 + pkg/common/config/load_config.go | 20 +- pkg/common/config/load_config_test.go | 40 +- .../discoveryregister/discoveryregister.go | 17 +- .../kubernetes/kubernetes.go | 365 +++++++++++------- pkg/common/startrpc/start.go | 17 +- tools/check-component/main.go | 15 +- 43 files changed, 980 insertions(+), 658 deletions(-) create mode 100644 .github/workflows/build-and-release-services-images.yml delete mode 100644 build/images/openim-cmdutils/Dockerfile diff --git a/.github/workflows/build-and-release-services-images.yml b/.github/workflows/build-and-release-services-images.yml new file mode 100644 index 000000000..d53fd2142 --- /dev/null +++ b/.github/workflows/build-and-release-services-images.yml @@ -0,0 +1,91 @@ +name: Build and release services Images + +on: + push: + branches: + - release-* + release: + types: [published] + workflow_dispatch: + inputs: + tag: + description: "Tag version to be used for Docker image" + required: true + default: "v3.8.3" + +jobs: + build-and-push: + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Log in to Docker Hub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + + - name: Log in to GitHub Container Registry + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Log in to Aliyun Container Registry + uses: docker/login-action@v2 + with: + registry: registry.cn-hangzhou.aliyuncs.com + username: ${{ secrets.ALIREGISTRY_USERNAME }} + password: ${{ secrets.ALIREGISTRY_TOKEN }} + + - name: Extract metadata for Docker (tags, labels) + id: meta + uses: docker/metadata-action@v5 + with: + tags: | + type=ref,event=tag + type=schedule + type=ref,event=branch + type=semver,pattern={{version}} + type=semver,pattern=v{{version}} + # type=semver,pattern={{major}}.{{minor}} + type=semver,pattern=release-{{raw}} + type=sha + type=raw,value=${{ github.event.inputs.tag }} + + - name: Build and push Docker images + run: | + ROOT_DIR="build/images" + for dir in "$ROOT_DIR"/*/; do + # Find Dockerfile or *.dockerfile in a case-insensitive manner + dockerfile=$(find "$dir" -maxdepth 1 -type f \( -iname 'dockerfile' -o -iname '*.dockerfile' \) | head -n 1) + + if [ -n "$dockerfile" ] && [ -f "$dockerfile" ]; then + IMAGE_NAME=$(basename "$dir") + echo "Building Docker image for $IMAGE_NAME with tags:" + + # Initialize tag arguments + tag_args=() + + # Read each tag and append --tag arguments + while IFS= read -r tag; do + tag_args+=(--tag "${{ secrets.DOCKER_USERNAME }}/$IMAGE_NAME:$tag") + tag_args+=(--tag "ghcr.io/${{ github.repository_owner }}/$IMAGE_NAME:$tag") + tag_args+=(--tag "registry.cn-hangzhou.aliyuncs.com/openimsdk/$IMAGE_NAME:$tag") + done <<< "${{ steps.meta.outputs.tags }}" + + # Build and push the Docker image with all tags + docker buildx build --platform linux/amd64,linux/arm64 \ + --file "$dockerfile" \ + "${tag_args[@]}" \ + --push "$dir" + else + echo "No valid Dockerfile found in $dir" + fi + done \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index f6a2ee9fe..8a95b6851 100644 --- a/Dockerfile +++ b/Dockerfile @@ -43,7 +43,7 @@ COPY --from=builder $SERVER_DIR/start-config.yml $SERVER_DIR/ COPY --from=builder $SERVER_DIR/go.mod $SERVER_DIR/ COPY --from=builder $SERVER_DIR/go.sum $SERVER_DIR/ -RUN go get github.com/openimsdk/gomake@v0.0.14-alpha.5 +RUN go get github.com/openimsdk/gomake@v0.0.15-alpha.1 # Set the command to run when the container starts ENTRYPOINT ["sh", "-c", "mage start && tail -f /dev/null"] diff --git a/build/images/Dockerfile b/build/images/Dockerfile index 51fe94e1c..020d50786 100644 --- a/build/images/Dockerfile +++ b/build/images/Dockerfile @@ -1,24 +1,24 @@ -# Copyright © 2023 OpenIM. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# # Copyright © 2023 OpenIM. All rights reserved. +# # +# # Licensed under the Apache License, Version 2.0 (the "License"); +# # you may not use this file except in compliance with the License. +# # You may obtain a copy of the License at +# # +# # http://www.apache.org/licenses/LICENSE-2.0 +# # +# # Unless required by applicable law or agreed to in writing, software +# # distributed under the License is distributed on an "AS IS" BASIS, +# # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# # See the License for the specific language governing permissions and +# # limitations under the License. -FROM BASE_IMAGE +# FROM BASE_IMAGE -WORKDIR ${SERVER_WORKDIR} +# WORKDIR ${SERVER_WORKDIR} -# Set HTTP proxy -ARG BINARY_NAME +# # Set HTTP proxy +# ARG BINARY_NAME -COPY BINARY_NAME ./bin/BINARY_NAME +# COPY BINARY_NAME ./bin/BINARY_NAME -ENTRYPOINT ["./bin/BINARY_NAME"] \ No newline at end of file +# ENTRYPOINT ["./bin/BINARY_NAME"] \ No newline at end of file diff --git a/build/images/openim-api/Dockerfile b/build/images/openim-api/Dockerfile index 662223956..eb9971c9b 100644 --- a/build/images/openim-api/Dockerfile +++ b/build/images/openim-api/Dockerfile @@ -1,44 +1,36 @@ -# Copyright © 2023 OpenIM. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# Use Go 1.22 Alpine as the base image for building the application +FROM golang:1.22-alpine AS builder -# OpenIM base image: https://github.com/openim-sigs/openim-base-image +# Define the base directory for the application as an environment variable +ENV SERVER_DIR=/openim-server -# Set go mod installation source and proxy +# Set the working directory inside the container based on the environment variable +WORKDIR $SERVER_DIR -FROM golang:1.20 AS builder +# Set the Go proxy to improve dependency resolution speed +#ENV GOPROXY=https://goproxy.io,direct -ARG GO111MODULE=on - -WORKDIR /openim/openim-server - -ENV GO111MODULE=$GO111MODULE -ENV GOPROXY=$GOPROXY +# Copy all files from the current directory into the container +COPY . . -COPY go.mod go.sum ./ -RUN go mod download +RUN go mod tidy -COPY . . +RUN go build -o _output/openim-api ./cmd/openim-api -RUN make build BINS=openim-api +# Using Alpine Linux for the final image +FROM alpine:latest -RUN cp /openim/openim-server/_output/bin/platforms/$(go env GOOS)/$(go env GOARCH)/openim-api /usr/bin/openim-api +# Install necessary packages, such as bash +RUN apk add --no-cache bash -# FROM ghcr.io/openim-sigs/openim-bash-image:latest -FROM ghcr.io/openim-sigs/openim-bash-image:latest +# Set the environment and work directory +ENV SERVER_DIR=/openim-server +WORKDIR $SERVER_DIR -WORKDIR /openim/openim-server -COPY --from=builder /usr/bin/openim-api ./bin/openim-api +# Copy the compiled binaries and mage from the builder image to the final image +COPY --from=builder $SERVER_DIR/_output $SERVER_DIR/_output +COPY --from=builder $SERVER_DIR/config $SERVER_DIR/config -ENTRYPOINT ["./bin/openim-api"] +# Set the command to run when the container starts +ENTRYPOINT ["sh", "-c", "_output/openim-api"] diff --git a/build/images/openim-cmdutils/Dockerfile b/build/images/openim-cmdutils/Dockerfile deleted file mode 100644 index 34bcd41f5..000000000 --- a/build/images/openim-cmdutils/Dockerfile +++ /dev/null @@ -1,44 +0,0 @@ -# Copyright © 2023 OpenIM. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# OpenIM base image: https://github.com/openim-sigs/openim-base-image - -# Set go mod installation source and proxy - -FROM golang:1.20 AS builder - -ARG GO111MODULE=on - -WORKDIR /openim/openim-server - -ENV GO111MODULE=$GO111MODULE -ENV GOPROXY=$GOPROXY - -COPY go.mod go.sum ./ -RUN go mod download - -COPY . . - -RUN make build BINS=openim-cmdutils -RUN cp /openim/openim-server/_output/bin/platforms/$(go env GOOS)/$(go env GOARCH)/openim-cmdutils /usr/bin/openim-cmdutils - -FROM ghcr.io/openim-sigs/openim-bash-image:latest - -WORKDIR /openim/openim-server - -COPY --from=builder /usr/bin/openim-cmdutils ./bin/openim-cmdutils - -ENTRYPOINT ["./bin/openim-cmdutils"] - -CMD ["--help"] diff --git a/build/images/openim-crontask/Dockerfile b/build/images/openim-crontask/Dockerfile index 90a562926..9ac0d8eca 100644 --- a/build/images/openim-crontask/Dockerfile +++ b/build/images/openim-crontask/Dockerfile @@ -1,44 +1,39 @@ -# Copyright © 2023 OpenIM. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# Use Go 1.22 Alpine as the base image for building the application +FROM golang:1.22-alpine AS builder +# Define the base directory for the application as an environment variable +ENV SERVER_DIR=/openim-server -# OpenIM base image: https://github.com/openim-sigs/openim-base-image +# Set the working directory inside the container based on the environment variable +WORKDIR $SERVER_DIR -# Set go mod installation source and proxy +# Set the Go proxy to improve dependency resolution speed -FROM golang:1.20 AS builder +#ENV GOPROXY=https://goproxy.io,direct -ARG GO111MODULE=on +# Copy all files from the current directory into the container +COPY . . -WORKDIR /openim/openim-server +RUN go mod tidy -ENV GO111MODULE=$GO111MODULE -ENV GOPROXY=$GOPROXY -COPY go.mod go.sum ./ -RUN go mod download -COPY . . +RUN go build -o _output/openim-crontask ./cmd/openim-crontask + -RUN make build BINS=openim-crontask +# Using Alpine Linux for the final image +FROM alpine:latest -RUN cp /openim/openim-server/_output/bin/platforms/$(go env GOOS)/$(go env GOARCH)/openim-crontask /usr/bin/openim-crontask +# Install necessary packages, such as bash +RUN apk add --no-cache bash -# FROM ghcr.io/openim-sigs/openim-bash-image:latest -FROM ghcr.io/openim-sigs/openim-bash-image:latest +# Set the environment and work directory +ENV SERVER_DIR=/openim-server +WORKDIR $SERVER_DIR -WORKDIR /openim/openim-server -COPY --from=builder /usr/bin/openim-crontask ./bin/openim-crontask +# Copy the compiled binaries and mage from the builder image to the final image +COPY --from=builder $SERVER_DIR/_output $SERVER_DIR/_output +# COPY --from=builder $SERVER_DIR/config $SERVER_DIR/config -ENTRYPOINT ["./bin/openim-crontask"] +# Set the command to run when the container starts +ENTRYPOINT ["sh", "-c", "_output/openim-crontask"] \ No newline at end of file diff --git a/build/images/openim-msggateway/Dockerfile b/build/images/openim-msggateway/Dockerfile index d3a8694ed..90b9c1fa6 100644 --- a/build/images/openim-msggateway/Dockerfile +++ b/build/images/openim-msggateway/Dockerfile @@ -1,44 +1,39 @@ -# Copyright © 2023 OpenIM. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# Use Go 1.22 Alpine as the base image for building the application +FROM golang:1.22-alpine AS builder +# Define the base directory for the application as an environment variable +ENV SERVER_DIR=/openim-server -# OpenIM base image: https://github.com/openim-sigs/openim-base-image +# Set the working directory inside the container based on the environment variable +WORKDIR $SERVER_DIR -# Set go mod installation source and proxy +# Set the Go proxy to improve dependency resolution speed -FROM golang:1.20 AS builder +#ENV GOPROXY=https://goproxy.io,direct -ARG GO111MODULE=on +# Copy all files from the current directory into the container +COPY . . -WORKDIR /openim/openim-server +RUN go mod tidy -ENV GO111MODULE=$GO111MODULE -ENV GOPROXY=$GOPROXY -COPY go.mod go.sum ./ -RUN go mod download -COPY . . +RUN go build -o _output/openim-msggateway ./cmd/openim-msggateway + -RUN make build BINS=openim-msggateway +# Using Alpine Linux for the final image +FROM alpine:latest -RUN cp /openim/openim-server/_output/bin/platforms/$(go env GOOS)/$(go env GOARCH)/openim-msggateway /usr/bin/openim-msggateway +# Install necessary packages, such as bash +RUN apk add --no-cache bash -# FROM ghcr.io/openim-sigs/openim-bash-image:latest -FROM ghcr.io/openim-sigs/openim-bash-image:latest +# Set the environment and work directory +ENV SERVER_DIR=/openim-server +WORKDIR $SERVER_DIR -WORKDIR /openim/openim-server -COPY --from=builder /usr/bin/openim-msggateway ./bin/openim-msggateway +# Copy the compiled binaries and mage from the builder image to the final image +COPY --from=builder $SERVER_DIR/_output $SERVER_DIR/_output +# COPY --from=builder $SERVER_DIR/config $SERVER_DIR/config -ENTRYPOINT ["./bin/openim-msggateway"] +# Set the command to run when the container starts +ENTRYPOINT ["sh", "-c", "_output/openim-msggateway"] \ No newline at end of file diff --git a/build/images/openim-msgtransfer/Dockerfile b/build/images/openim-msgtransfer/Dockerfile index f94978648..4c679863b 100644 --- a/build/images/openim-msgtransfer/Dockerfile +++ b/build/images/openim-msgtransfer/Dockerfile @@ -1,44 +1,39 @@ -# Copyright © 2023 OpenIM. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# Use Go 1.22 Alpine as the base image for building the application +FROM golang:1.22-alpine AS builder +# Define the base directory for the application as an environment variable +ENV SERVER_DIR=/openim-server -# OpenIM base image: https://github.com/openim-sigs/openim-base-image +# Set the working directory inside the container based on the environment variable +WORKDIR $SERVER_DIR -# Set go mod installation source and proxy +# Set the Go proxy to improve dependency resolution speed -FROM golang:1.20 AS builder +#ENV GOPROXY=https://goproxy.io,direct -ARG GO111MODULE=on +# Copy all files from the current directory into the container +COPY . . -WORKDIR /openim/openim-server +RUN go mod tidy -ENV GO111MODULE=$GO111MODULE -ENV GOPROXY=$GOPROXY -COPY go.mod go.sum ./ -RUN go mod download -COPY . . +RUN go build -o _output/openim-msgtransfer ./cmd/openim-msgtransfer + -RUN make build BINS=openim-msgtransfer +# Using Alpine Linux for the final image +FROM alpine:latest -RUN cp /openim/openim-server/_output/bin/platforms/$(go env GOOS)/$(go env GOARCH)/openim-msgtransfer /usr/bin/openim-msgtransfer +# Install necessary packages, such as bash +RUN apk add --no-cache bash -# FROM ghcr.io/openim-sigs/openim-bash-image:latest -FROM ghcr.io/openim-sigs/openim-bash-image:latest +# Set the environment and work directory +ENV SERVER_DIR=/openim-server +WORKDIR $SERVER_DIR -WORKDIR /openim/openim-server -COPY --from=builder /usr/bin/openim-msgtransfer ./bin/openim-msgtransfer +# Copy the compiled binaries and mage from the builder image to the final image +COPY --from=builder $SERVER_DIR/_output $SERVER_DIR/_output +# COPY --from=builder $SERVER_DIR/config $SERVER_DIR/config -ENTRYPOINT ["./bin/openim-msgtransfer"] +# Set the command to run when the container starts +ENTRYPOINT ["sh", "-c", "_output/openim-msgtransfer"] \ No newline at end of file diff --git a/build/images/openim-push/Dockerfile b/build/images/openim-push/Dockerfile index faebbe9c0..68b711025 100644 --- a/build/images/openim-push/Dockerfile +++ b/build/images/openim-push/Dockerfile @@ -1,44 +1,39 @@ -# Copyright © 2023 OpenIM. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# Use Go 1.22 Alpine as the base image for building the application +FROM golang:1.22-alpine AS builder +# Define the base directory for the application as an environment variable +ENV SERVER_DIR=/openim-server -# OpenIM base image: https://github.com/openim-sigs/openim-base-image +# Set the working directory inside the container based on the environment variable +WORKDIR $SERVER_DIR -# Set go mod installation source and proxy +# Set the Go proxy to improve dependency resolution speed -FROM golang:1.20 AS builder +#ENV GOPROXY=https://goproxy.io,direct -ARG GO111MODULE=on +# Copy all files from the current directory into the container +COPY . . -WORKDIR /openim/openim-server +RUN go mod tidy -ENV GO111MODULE=$GO111MODULE -ENV GOPROXY=$GOPROXY -COPY go.mod go.sum ./ -RUN go mod download -COPY . . +RUN go build -o _output/openim-push ./cmd/openim-push + -RUN make build BINS=openim-push +# Using Alpine Linux for the final image +FROM alpine:latest -RUN cp /openim/openim-server/_output/bin/platforms/$(go env GOOS)/$(go env GOARCH)/openim-push /usr/bin/openim-push +# Install necessary packages, such as bash +RUN apk add --no-cache bash -# FROM ghcr.io/openim-sigs/openim-bash-image:latest -FROM ghcr.io/openim-sigs/openim-bash-image:latest +# Set the environment and work directory +ENV SERVER_DIR=/openim-server +WORKDIR $SERVER_DIR -WORKDIR /openim/openim-server -COPY --from=builder /usr/bin/openim-push ./bin/openim-push +# Copy the compiled binaries and mage from the builder image to the final image +COPY --from=builder $SERVER_DIR/_output $SERVER_DIR/_output +# COPY --from=builder $SERVER_DIR/config $SERVER_DIR/config -ENTRYPOINT ["./bin/openim-push"] +# Set the command to run when the container starts +ENTRYPOINT ["sh", "-c", "_output/openim-push"] \ No newline at end of file diff --git a/build/images/openim-rpc-auth/Dockerfile b/build/images/openim-rpc-auth/Dockerfile index 1e905d4b2..0f9383805 100644 --- a/build/images/openim-rpc-auth/Dockerfile +++ b/build/images/openim-rpc-auth/Dockerfile @@ -1,44 +1,39 @@ -# Copyright © 2023 OpenIM. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# Use Go 1.22 Alpine as the base image for building the application +FROM golang:1.22-alpine AS builder +# Define the base directory for the application as an environment variable +ENV SERVER_DIR=/openim-server -# OpenIM base image: https://github.com/openim-sigs/openim-base-image +# Set the working directory inside the container based on the environment variable +WORKDIR $SERVER_DIR -# Set go mod installation source and proxy +# Set the Go proxy to improve dependency resolution speed -FROM golang:1.20 AS builder +#ENV GOPROXY=https://goproxy.io,direct -ARG GO111MODULE=on +# Copy all files from the current directory into the container +COPY . . -WORKDIR /openim/openim-server +RUN go mod tidy -ENV GO111MODULE=$GO111MODULE -ENV GOPROXY=$GOPROXY -COPY go.mod go.sum ./ -RUN go mod download -COPY . . +RUN go build -o _output/openim-rpc-auth ./cmd/openim-rpc/openim-rpc-auth + -RUN make build BINS=openim-rpc-auth +# Using Alpine Linux for the final image +FROM alpine:latest -RUN cp /openim/openim-server/_output/bin/platforms/$(go env GOOS)/$(go env GOARCH)/openim-rpc-auth /usr/bin/openim-rpc-auth +# Install necessary packages, such as bash +RUN apk add --no-cache bash -# FROM ghcr.io/openim-sigs/openim-bash-image:latest -FROM ghcr.io/openim-sigs/openim-bash-image:latest +# Set the environment and work directory +ENV SERVER_DIR=/openim-server +WORKDIR $SERVER_DIR -WORKDIR /openim/openim-server -COPY --from=builder /usr/bin/openim-rpc-auth ./bin/openim-rpc-auth +# Copy the compiled binaries and mage from the builder image to the final image +COPY --from=builder $SERVER_DIR/_output $SERVER_DIR/_output +# COPY --from=builder $SERVER_DIR/config $SERVER_DIR/config -ENTRYPOINT ["./bin/openim-rpc-auth"] +# Set the command to run when the container starts +ENTRYPOINT ["sh", "-c", "_output/openim-rpc-auth"] \ No newline at end of file diff --git a/build/images/openim-rpc-conversation/Dockerfile b/build/images/openim-rpc-conversation/Dockerfile index 5a69aa89f..2c3578c0c 100644 --- a/build/images/openim-rpc-conversation/Dockerfile +++ b/build/images/openim-rpc-conversation/Dockerfile @@ -1,44 +1,39 @@ -# Copyright © 2023 OpenIM. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# Use Go 1.22 Alpine as the base image for building the application +FROM golang:1.22-alpine AS builder +# Define the base directory for the application as an environment variable +ENV SERVER_DIR=/openim-server -# OpenIM base image: https://github.com/openim-sigs/openim-base-image +# Set the working directory inside the container based on the environment variable +WORKDIR $SERVER_DIR -# Set go mod installation source and proxy +# Set the Go proxy to improve dependency resolution speed -FROM golang:1.20 AS builder +#ENV GOPROXY=https://goproxy.io,direct -ARG GO111MODULE=on +# Copy all files from the current directory into the container +COPY . . -WORKDIR /openim/openim-server +RUN go mod tidy -ENV GO111MODULE=$GO111MODULE -ENV GOPROXY=$GOPROXY -COPY go.mod go.sum ./ -RUN go mod download -COPY . . +RUN go build -o _output/openim-rpc-conversation ./cmd/openim-rpc/openim-rpc-conversation + -RUN make build BINS=openim-rpc-conversation +# Using Alpine Linux for the final image +FROM alpine:latest -RUN cp /openim/openim-server/_output/bin/platforms/$(go env GOOS)/$(go env GOARCH)/openim-rpc-conversation /usr/bin/openim-rpc-conversation +# Install necessary packages, such as bash +RUN apk add --no-cache bash -# FROM ghcr.io/openim-sigs/openim-bash-image:latest -FROM ghcr.io/openim-sigs/openim-bash-image:latest +# Set the environment and work directory +ENV SERVER_DIR=/openim-server +WORKDIR $SERVER_DIR -WORKDIR /openim/openim-server -COPY --from=builder /usr/bin/openim-rpc-conversation ./bin/openim-rpc-conversation +# Copy the compiled binaries and mage from the builder image to the final image +COPY --from=builder $SERVER_DIR/_output $SERVER_DIR/_output +# COPY --from=builder $SERVER_DIR/config $SERVER_DIR/config -ENTRYPOINT ["./bin/openim-rpc-conversation"] +# Set the command to run when the container starts +ENTRYPOINT ["sh", "-c", "_output/openim-rpc-conversation"] \ No newline at end of file diff --git a/build/images/openim-rpc-friend/Dockerfile b/build/images/openim-rpc-friend/Dockerfile index fad21a880..7aa4b6a54 100644 --- a/build/images/openim-rpc-friend/Dockerfile +++ b/build/images/openim-rpc-friend/Dockerfile @@ -1,44 +1,39 @@ -# Copyright © 2023 OpenIM. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# Use Go 1.22 Alpine as the base image for building the application +FROM golang:1.22-alpine AS builder +# Define the base directory for the application as an environment variable +ENV SERVER_DIR=/openim-server -# OpenIM base image: https://github.com/openim-sigs/openim-base-image +# Set the working directory inside the container based on the environment variable +WORKDIR $SERVER_DIR -# Set go mod installation source and proxy +# Set the Go proxy to improve dependency resolution speed -FROM golang:1.20 AS builder +#ENV GOPROXY=https://goproxy.io,direct -ARG GO111MODULE=on +# Copy all files from the current directory into the container +COPY . . -WORKDIR /openim/openim-server +RUN go mod tidy -ENV GO111MODULE=$GO111MODULE -ENV GOPROXY=$GOPROXY -COPY go.mod go.sum ./ -RUN go mod download -COPY . . +RUN go build -o _output/openim-rpc-friend ./cmd/openim-rpc/openim-rpc-friend + -RUN make build BINS=openim-rpc-friend +# Using Alpine Linux for the final image +FROM alpine:latest -RUN cp /openim/openim-server/_output/bin/platforms/$(go env GOOS)/$(go env GOARCH)/openim-rpc-friend /usr/bin/openim-rpc-friend +# Install necessary packages, such as bash +RUN apk add --no-cache bash -# FROM ghcr.io/openim-sigs/openim-bash-image:latest -FROM ghcr.io/openim-sigs/openim-bash-image:latest +# Set the environment and work directory +ENV SERVER_DIR=/openim-server +WORKDIR $SERVER_DIR -WORKDIR /openim/openim-server -COPY --from=builder /usr/bin/openim-rpc-friend ./bin/openim-rpc-friend +# Copy the compiled binaries and mage from the builder image to the final image +COPY --from=builder $SERVER_DIR/_output $SERVER_DIR/_output +# COPY --from=builder $SERVER_DIR/config $SERVER_DIR/config -ENTRYPOINT ["./bin/openim-rpc-friend"] +# Set the command to run when the container starts +ENTRYPOINT ["sh", "-c", "_output/openim-rpc-friend"] \ No newline at end of file diff --git a/build/images/openim-rpc-group/Dockerfile b/build/images/openim-rpc-group/Dockerfile index d7dede6b9..d6a4e64e0 100644 --- a/build/images/openim-rpc-group/Dockerfile +++ b/build/images/openim-rpc-group/Dockerfile @@ -1,44 +1,39 @@ -# Copyright © 2023 OpenIM. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# Use Go 1.22 Alpine as the base image for building the application +FROM golang:1.22-alpine AS builder +# Define the base directory for the application as an environment variable +ENV SERVER_DIR=/openim-server -# OpenIM base image: https://github.com/openim-sigs/openim-base-image +# Set the working directory inside the container based on the environment variable +WORKDIR $SERVER_DIR -# Set go mod installation source and proxy +# Set the Go proxy to improve dependency resolution speed -FROM golang:1.20 AS builder +#ENV GOPROXY=https://goproxy.io,direct -ARG GO111MODULE=on +# Copy all files from the current directory into the container +COPY . . -WORKDIR /openim/openim-server +RUN go mod tidy -ENV GO111MODULE=$GO111MODULE -ENV GOPROXY=$GOPROXY -COPY go.mod go.sum ./ -RUN go mod download -COPY . . +RUN go build -o _output/openim-rpc-group ./cmd/openim-rpc/openim-rpc-group + -RUN make build BINS=openim-rpc-group +# Using Alpine Linux for the final image +FROM alpine:latest -RUN cp /openim/openim-server/_output/bin/platforms/$(go env GOOS)/$(go env GOARCH)/openim-rpc-group /usr/bin/openim-rpc-group +# Install necessary packages, such as bash +RUN apk add --no-cache bash -# FROM ghcr.io/openim-sigs/openim-bash-image:latest -FROM ghcr.io/openim-sigs/openim-bash-image:latest +# Set the environment and work directory +ENV SERVER_DIR=/openim-server +WORKDIR $SERVER_DIR -WORKDIR /openim/openim-server -COPY --from=builder /usr/bin/openim-rpc-group ./bin/openim-rpc-group +# Copy the compiled binaries and mage from the builder image to the final image +COPY --from=builder $SERVER_DIR/_output $SERVER_DIR/_output +# COPY --from=builder $SERVER_DIR/config $SERVER_DIR/config -ENTRYPOINT ["./bin/openim-rpc-group"] +# Set the command to run when the container starts +ENTRYPOINT ["sh", "-c", "_output/openim-rpc-group"] \ No newline at end of file diff --git a/build/images/openim-rpc-msg/Dockerfile b/build/images/openim-rpc-msg/Dockerfile index 71ad85f37..9dac2620a 100644 --- a/build/images/openim-rpc-msg/Dockerfile +++ b/build/images/openim-rpc-msg/Dockerfile @@ -1,44 +1,39 @@ -# Copyright © 2023 OpenIM. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# Use Go 1.22 Alpine as the base image for building the application +FROM golang:1.22-alpine AS builder +# Define the base directory for the application as an environment variable +ENV SERVER_DIR=/openim-server -# OpenIM base image: https://github.com/openim-sigs/openim-base-image +# Set the working directory inside the container based on the environment variable +WORKDIR $SERVER_DIR -# Set go mod installation source and proxy +# Set the Go proxy to improve dependency resolution speed -FROM golang:1.20 AS builder +#ENV GOPROXY=https://goproxy.io,direct -ARG GO111MODULE=on +# Copy all files from the current directory into the container +COPY . . -WORKDIR /openim/openim-server +RUN go mod tidy -ENV GO111MODULE=$GO111MODULE -ENV GOPROXY=$GOPROXY -COPY go.mod go.sum ./ -RUN go mod download -COPY . . +RUN go build -o _output/openim-rpc-msg ./cmd/openim-rpc/openim-rpc-msg + -RUN make build BINS=openim-rpc-msg +# Using Alpine Linux for the final image +FROM alpine:latest -RUN cp /openim/openim-server/_output/bin/platforms/$(go env GOOS)/$(go env GOARCH)/openim-rpc-msg /usr/bin/openim-rpc-msg +# Install necessary packages, such as bash +RUN apk add --no-cache bash -# FROM ghcr.io/openim-sigs/openim-bash-image:latest -FROM ghcr.io/openim-sigs/openim-bash-image:latest +# Set the environment and work directory +ENV SERVER_DIR=/openim-server +WORKDIR $SERVER_DIR -WORKDIR /openim/openim-server -COPY --from=builder /usr/bin/openim-rpc-msg ./bin/openim-rpc-msg +# Copy the compiled binaries and mage from the builder image to the final image +COPY --from=builder $SERVER_DIR/_output $SERVER_DIR/_output +# COPY --from=builder $SERVER_DIR/config $SERVER_DIR/config -ENTRYPOINT ["./bin/openim-rpc-msg"] +# Set the command to run when the container starts +ENTRYPOINT ["sh", "-c", "_output/openim-rpc-msg"] \ No newline at end of file diff --git a/build/images/openim-rpc-third/Dockerfile b/build/images/openim-rpc-third/Dockerfile index 3560ad0d7..e68f9fe98 100644 --- a/build/images/openim-rpc-third/Dockerfile +++ b/build/images/openim-rpc-third/Dockerfile @@ -1,44 +1,39 @@ -# Copyright © 2023 OpenIM. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# Use Go 1.22 Alpine as the base image for building the application +FROM golang:1.22-alpine AS builder +# Define the base directory for the application as an environment variable +ENV SERVER_DIR=/openim-server -# OpenIM base image: https://github.com/openim-sigs/openim-base-image +# Set the working directory inside the container based on the environment variable +WORKDIR $SERVER_DIR -# Set go mod installation source and proxy +# Set the Go proxy to improve dependency resolution speed -FROM golang:1.20 AS builder +#ENV GOPROXY=https://goproxy.io,direct -ARG GO111MODULE=on +# Copy all files from the current directory into the container +COPY . . -WORKDIR /openim/openim-server +RUN go mod tidy -ENV GO111MODULE=$GO111MODULE -ENV GOPROXY=$GOPROXY -COPY go.mod go.sum ./ -RUN go mod download -COPY . . +RUN go build -o _output/openim-rpc-third ./cmd/openim-rpc/openim-rpc-third + -RUN make build BINS=openim-rpc-third +# Using Alpine Linux for the final image +FROM alpine:latest -RUN cp /openim/openim-server/_output/bin/platforms/$(go env GOOS)/$(go env GOARCH)/openim-rpc-third /usr/bin/openim-rpc-third +# Install necessary packages, such as bash +RUN apk add --no-cache bash -# FROM ghcr.io/openim-sigs/openim-bash-image:latest -FROM ghcr.io/openim-sigs/openim-bash-image:latest +# Set the environment and work directory +ENV SERVER_DIR=/openim-server +WORKDIR $SERVER_DIR -WORKDIR /openim/openim-server -COPY --from=builder /usr/bin/openim-rpc-third ./bin/openim-rpc-third +# Copy the compiled binaries and mage from the builder image to the final image +COPY --from=builder $SERVER_DIR/_output $SERVER_DIR/_output +# COPY --from=builder $SERVER_DIR/config $SERVER_DIR/config -ENTRYPOINT ["./bin/openim-rpc-third"] +# Set the command to run when the container starts +ENTRYPOINT ["sh", "-c", "_output/openim-rpc-third"] \ No newline at end of file diff --git a/build/images/openim-rpc-user/Dockerfile b/build/images/openim-rpc-user/Dockerfile index a9891ece8..42f9ceec6 100644 --- a/build/images/openim-rpc-user/Dockerfile +++ b/build/images/openim-rpc-user/Dockerfile @@ -1,44 +1,37 @@ -# Copyright © 2023 OpenIM. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# Use Go 1.22 Alpine as the base image for building the application +FROM golang:1.22-alpine AS builder +# Define the base directory for the application as an environment variable +ENV SERVER_DIR=/openim-server -# OpenIM base image: https://github.com/openim-sigs/openim-base-image +# Set the working directory inside the container based on the environment variable +WORKDIR $SERVER_DIR -# Set go mod installation source and proxy +# Set the Go proxy to improve dependency resolution speed -FROM golang:1.20 AS builder +#ENV GOPROXY=https://goproxy.io,direct -ARG GO111MODULE=on - -WORKDIR /openim/openim-server +# Copy all files from the current directory into the container +COPY . . -ENV GO111MODULE=$GO111MODULE -ENV GOPROXY=$GOPROXY +RUN go mod tidy -COPY go.mod go.sum ./ -RUN go mod download +RUN go build -o _output/openim-rpc-user ./cmd/openim-rpc/openim-rpc-user -COPY . . -RUN make build BINS=openim-rpc-user +# Using Alpine Linux for the final image +FROM alpine:latest -RUN cp /openim/openim-server/_output/bin/platforms/$(go env GOOS)/$(go env GOARCH)/openim-rpc-user /usr/bin/openim-rpc-user +# Install necessary packages, such as bash +RUN apk add --no-cache bash -# FROM ghcr.io/openim-sigs/openim-bash-image:latest -FROM ghcr.io/openim-sigs/openim-bash-image:latest +# Set the environment and work directory +ENV SERVER_DIR=/openim-server +WORKDIR $SERVER_DIR -WORKDIR /openim/openim-server -COPY --from=builder /usr/bin/openim-rpc-user ./bin/openim-rpc-user +# Copy the compiled binaries and mage from the builder image to the final image +COPY --from=builder $SERVER_DIR/_output $SERVER_DIR/_output +# COPY --from=builder $SERVER_DIR/config $SERVER_DIR/config -ENTRYPOINT ["./bin/openim-rpc-user"] +# Set the command to run when the container starts +ENTRYPOINT ["sh", "-c", "_output/openim-rpc-user"] \ No newline at end of file diff --git a/build/images/openim-tools/component/Dockerfile b/build/images/openim-tools/component/Dockerfile index 6bdfa6942..ae8de800f 100644 --- a/build/images/openim-tools/component/Dockerfile +++ b/build/images/openim-tools/component/Dockerfile @@ -1,48 +1,108 @@ -# Copyright © 2023 OpenIM. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# # Copyright © 2023 OpenIM. All rights reserved. +# # +# # Licensed under the Apache License, Version 2.0 (the "License"); +# # you may not use this file except in compliance with the License. +# # You may obtain a copy of the License at +# # +# # http://www.apache.org/licenses/LICENSE-2.0 +# # +# # Unless required by applicable law or agreed to in writing, software +# # distributed under the License is distributed on an "AS IS" BASIS, +# # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# # See the License for the specific language governing permissions and +# # limitations under the License. -# OpenIM base image: https://github.com/openim-sigs/openim-base-image +# # OpenIM base image: https://github.com/openim-sigs/openim-base-image -# Set go mod installation source and proxy +# # Set go mod installation source and proxy -FROM golang:1.20 AS builder +# FROM golang:1.20 AS builder -ARG GO111MODULE=on +# -WORKDIR /openim/openim-server +# WORKDIR /openim/openim-server -ENV GO111MODULE=$GO111MODULE -ENV GOPROXY=$GOPROXY +# +# ENV GOPROXY=$GOPROXY -COPY go.mod go.sum ./ -RUN go mod download +# COPY go.mod go.sum ./ +# RUN go mod download -COPY . . +# COPY . . -RUN make clean -RUN make build BINS=component +# RUN make clean +# RUN make build BINS=component +# # FROM ghcr.io/openim-sigs/openim-bash-image:latest # FROM ghcr.io/openim-sigs/openim-bash-image:latest -FROM ghcr.io/openim-sigs/openim-bash-image:latest -WORKDIR /openim/openim-server +# WORKDIR /openim/openim-server + +# COPY --from=builder /openim/openim-server/_output/bin/tools /openim/openim-server/_output/bin/tools/ +# COPY --from=builder /openim/openim-server/config /openim/openim-server/config + +# ENV OPENIM_SERVER_CONFIG_NAME=/openim/openim-server/config + +# RUN mv ${OPENIM_SERVER_BINDIR}/platforms/$(get_os)/$(get_arch)/component /usr/bin/component + +# ENTRYPOINT ["bash", "-c", "component -c $OPENIM_SERVER_CONFIG_NAME"] + + +# Use Go 1.22 Alpine as the base image for building the application +FROM golang:1.22-alpine AS builder +# Define the base directory for the application as an environment variable +ENV SERVER_DIR=/openim-server + +# Set the working directory inside the container based on the environment variable +WORKDIR $SERVER_DIR + +# Set the Go proxy to improve dependency resolution speed + +#ENV GOPROXY=https://goproxy.io,direct + +# Copy all files from the current directory into the container +COPY . . + +RUN go mod download + +# Install Mage to use for building the application +RUN go install github.com/magefile/mage@v1.15.0 + +# ENV BINS=openim-rpc-user + +# Optionally build your application if needed +# RUN mage build ${BINS} check-free-memory seq || true +RUN mage build check-free-memory seq || true + +# Using Alpine Linux with Go environment for the final image +FROM golang:1.22-alpine + +# Install necessary packages, such as bash +RUN apk add bash + +# Set the environment and work directory +ENV SERVER_DIR=/openim-server +WORKDIR $SERVER_DIR + + +# Copy the compiled binaries and mage from the builder image to the final image +COPY --from=builder $SERVER_DIR/_output $SERVER_DIR/_output +COPY --from=builder $SERVER_DIR/config $SERVER_DIR/config +COPY --from=builder /go/bin/mage /usr/local/bin/mage +COPY --from=builder $SERVER_DIR/magefile_windows.go $SERVER_DIR/ +COPY --from=builder $SERVER_DIR/magefile_unix.go $SERVER_DIR/ +COPY --from=builder $SERVER_DIR/magefile.go $SERVER_DIR/ +# COPY --from=builder $SERVER_DIR/start-config.yml $SERVER_DIR/ +COPY --from=builder $SERVER_DIR/go.mod $SERVER_DIR/ +COPY --from=builder $SERVER_DIR/go.sum $SERVER_DIR/ -COPY --from=builder /openim/openim-server/_output/bin/tools /openim/openim-server/_output/bin/tools/ -COPY --from=builder /openim/openim-server/config /openim/openim-server/config -ENV OPENIM_SERVER_CONFIG_NAME=/openim/openim-server/config +RUN echo -e "serviceBinaries:\n \n" \ + > $SERVER_DIR/start-config.yml && \ + echo -e "toolBinaries:\n - check-free-memory\n - seq\n" >> $SERVER_DIR/start-config.yml && \ + echo "maxFileDescriptors: 10000" >> $SERVER_DIR/start-config.yml -RUN mv ${OPENIM_SERVER_BINDIR}/platforms/$(get_os)/$(get_arch)/component /usr/bin/component +RUN go get github.com/openimsdk/gomake@v0.0.15-alpha.1 -ENTRYPOINT ["bash", "-c", "component -c $OPENIM_SERVER_CONFIG_NAME"] +# Set the command to run when the container starts +ENTRYPOINT ["sh", "-c", "mage start && tail -f /dev/null"] diff --git a/config/discovery.yml b/config/discovery.yml index fad49e3ed..a04d7e40f 100644 --- a/config/discovery.yml +++ b/config/discovery.yml @@ -5,6 +5,9 @@ etcd: username: '' password: '' +kubernetes: + namespace: default + rpcService: user: user-rpc-service friend: friend-rpc-service @@ -14,6 +17,4 @@ rpcService: group: group-rpc-service auth: auth-rpc-service conversation: conversation-rpc-service - third: third-rpc-service - - + third: third-rpc-service \ No newline at end of file diff --git a/config/openim-msggateway.yml b/config/openim-msggateway.yml index 74eab35d5..696bc3d06 100644 --- a/config/openim-msggateway.yml +++ b/config/openim-msggateway.yml @@ -2,6 +2,7 @@ rpc: # The IP address where this RPC service registers itself; if left blank, it defaults to the internal network IP registerIP: # autoSetPorts indicates whether to automatically set the ports + # if you use in kubernetes, set it to false autoSetPorts: true # List of ports that the RPC service listens on; configuring multiple ports will launch multiple instances. These must match the number of configured prometheus ports # It will only take effect when autoSetPorts is set to false. diff --git a/config/openim-push.yml b/config/openim-push.yml index 53c88ed41..a3fe0c765 100644 --- a/config/openim-push.yml +++ b/config/openim-push.yml @@ -4,6 +4,7 @@ rpc: # IP address that the RPC service listens on; setting to 0.0.0.0 listens on both internal and external IPs. If left blank, it automatically uses the internal network IP listenIP: 0.0.0.0 # autoSetPorts indicates whether to automatically set the ports + # if you use in kubernetes, set it to false autoSetPorts: true # List of ports that the RPC service listens on; configuring multiple ports will launch multiple instances. These must match the number of configured prometheus ports # It will only take effect when autoSetPorts is set to false. diff --git a/config/openim-rpc-auth.yml b/config/openim-rpc-auth.yml index 230aa8720..93f365386 100644 --- a/config/openim-rpc-auth.yml +++ b/config/openim-rpc-auth.yml @@ -4,6 +4,7 @@ rpc: # IP address that the RPC service listens on; setting to 0.0.0.0 listens on both internal and external IPs. If left blank, it automatically uses the internal network IP listenIP: 0.0.0.0 # autoSetPorts indicates whether to automatically set the ports + # if you use in kubernetes, set it to false autoSetPorts: true # List of ports that the RPC service listens on; configuring multiple ports will launch multiple instances. These must match the number of configured prometheus ports # It will only take effect when autoSetPorts is set to false. diff --git a/config/openim-rpc-conversation.yml b/config/openim-rpc-conversation.yml index ed61f7010..c17b5709f 100644 --- a/config/openim-rpc-conversation.yml +++ b/config/openim-rpc-conversation.yml @@ -4,6 +4,7 @@ rpc: # IP address that the RPC service listens on; setting to 0.0.0.0 listens on both internal and external IPs. If left blank, it automatically uses the internal network IP listenIP: 0.0.0.0 # autoSetPorts indicates whether to automatically set the ports + # if you use in kubernetes, set it to false autoSetPorts: true # List of ports that the RPC service listens on; configuring multiple ports will launch multiple instances. These must match the number of configured prometheus ports # It will only take effect when autoSetPorts is set to false. diff --git a/config/openim-rpc-friend.yml b/config/openim-rpc-friend.yml index e10ef496a..5b51981d2 100644 --- a/config/openim-rpc-friend.yml +++ b/config/openim-rpc-friend.yml @@ -4,6 +4,7 @@ rpc: # IP address that the RPC service listens on; setting to 0.0.0.0 listens on both internal and external IPs. If left blank, it automatically uses the internal network IP listenIP: 0.0.0.0 # autoSetPorts indicates whether to automatically set the ports + # if you use in kubernetes, set it to false autoSetPorts: true # List of ports that the RPC service listens on; configuring multiple ports will launch multiple instances. These must match the number of configured prometheus ports # It will only take effect when autoSetPorts is set to false. diff --git a/config/openim-rpc-group.yml b/config/openim-rpc-group.yml index 9f5ceabab..06fb489d2 100644 --- a/config/openim-rpc-group.yml +++ b/config/openim-rpc-group.yml @@ -4,6 +4,7 @@ rpc: # IP address that the RPC service listens on; setting to 0.0.0.0 listens on both internal and external IPs. If left blank, it automatically uses the internal network IP listenIP: 0.0.0.0 # autoSetPorts indicates whether to automatically set the ports + # if you use in kubernetes, set it to false autoSetPorts: true # List of ports that the RPC service listens on; configuring multiple ports will launch multiple instances. These must match the number of configured prometheus ports # It will only take effect when autoSetPorts is set to false. diff --git a/config/openim-rpc-msg.yml b/config/openim-rpc-msg.yml index 72e055a5d..5a4beae5c 100644 --- a/config/openim-rpc-msg.yml +++ b/config/openim-rpc-msg.yml @@ -4,6 +4,7 @@ rpc: # IP address that the RPC service listens on; setting to 0.0.0.0 listens on both internal and external IPs. If left blank, it automatically uses the internal network IP listenIP: 0.0.0.0 # autoSetPorts indicates whether to automatically set the ports + # if you use in kubernetes, set it to false autoSetPorts: true # List of ports that the RPC service listens on; configuring multiple ports will launch multiple instances. These must match the number of configured prometheus ports # It will only take effect when autoSetPorts is set to false. diff --git a/config/openim-rpc-third.yml b/config/openim-rpc-third.yml index 719c65e21..e24f93883 100644 --- a/config/openim-rpc-third.yml +++ b/config/openim-rpc-third.yml @@ -4,6 +4,7 @@ rpc: # IP address that the RPC service listens on; setting to 0.0.0.0 listens on both internal and external IPs. If left blank, it automatically uses the internal network IP listenIP: 0.0.0.0 # autoSetPorts indicates whether to automatically set the ports + # if you use in kubernetes, set it to false autoSetPorts: true # List of ports that the RPC service listens on; configuring multiple ports will launch multiple instances. These must match the number of configured prometheus ports # It will only take effect when autoSetPorts is set to false. diff --git a/config/openim-rpc-user.yml b/config/openim-rpc-user.yml index 6f7b9648b..02b6d45b1 100644 --- a/config/openim-rpc-user.yml +++ b/config/openim-rpc-user.yml @@ -4,6 +4,7 @@ rpc: # Listening IP; 0.0.0.0 means both internal and external IPs are listened to, if blank, the internal network IP is automatically obtained by default listenIP: 0.0.0.0 # autoSetPorts indicates whether to automatically set the ports + # if you use in kubernetes, set it to false autoSetPorts: true # List of ports that the RPC service listens on; configuring multiple ports will launch multiple instances. These must match the number of configured prometheus ports # It will only take effect when autoSetPorts is set to false. diff --git a/go.mod b/go.mod index e7f340894..b278f2a9b 100644 --- a/go.mod +++ b/go.mod @@ -15,7 +15,7 @@ require ( github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/mitchellh/mapstructure v1.5.0 github.com/openimsdk/protocol v0.0.72-alpha.61 - github.com/openimsdk/tools v0.0.50-alpha.43 + github.com/openimsdk/tools v0.0.50-alpha.45 github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 github.com/stretchr/testify v1.9.0 @@ -37,15 +37,17 @@ require ( github.com/hashicorp/golang-lru/v2 v2.0.7 github.com/kelindar/bitmap v1.5.2 github.com/likexian/gokit v0.25.13 - github.com/openimsdk/gomake v0.0.14-alpha.5 + github.com/openimsdk/gomake v0.0.15-alpha.1 github.com/redis/go-redis/v9 v9.4.0 github.com/robfig/cron/v3 v3.0.1 github.com/shirou/gopsutil v3.21.11+incompatible github.com/spf13/viper v1.18.2 - github.com/stathat/consistent v1.0.0 go.uber.org/automaxprocs v1.5.3 golang.org/x/exp v0.0.0-20230905200255-921286631fa9 golang.org/x/sync v0.8.0 + k8s.io/api v0.31.2 + k8s.io/apimachinery v0.31.2 + k8s.io/client-go v0.31.2 ) require ( @@ -94,21 +96,29 @@ require ( github.com/eapache/queue v1.1.0 // indirect github.com/elastic/go-sysinfo v1.0.2 // indirect github.com/elastic/go-windows v1.0.0 // indirect + github.com/emicklei/go-restful/v3 v3.11.0 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect + github.com/fxamacker/cbor/v2 v2.7.0 // indirect github.com/gabriel-vasile/mimetype v1.4.3 // indirect github.com/gammazero/toposort v0.1.1 // indirect github.com/gin-contrib/sse v0.1.0 // indirect github.com/go-logr/logr v1.4.2 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-ole/go-ole v1.2.6 // indirect + github.com/go-openapi/jsonpointer v0.19.6 // indirect + github.com/go-openapi/jsonreference v0.20.2 // indirect + github.com/go-openapi/swag v0.22.4 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect github.com/go-zookeeper/zk v1.0.3 // indirect github.com/gofrs/flock v0.8.1 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/golang/snappy v0.0.4 // indirect + github.com/google/gnostic-models v0.6.8 // indirect + github.com/google/go-cmp v0.6.0 // indirect github.com/google/go-querystring v1.1.0 // indirect + github.com/google/gofuzz v1.2.0 // indirect github.com/google/s2a-go v0.1.7 // indirect github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect github.com/googleapis/gax-go/v2 v2.12.3 // indirect @@ -126,6 +136,7 @@ require ( github.com/jinzhu/inflection v1.0.0 // indirect github.com/jinzhu/now v1.1.5 // indirect github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901 // indirect + github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/kelindar/simd v1.1.2 // indirect github.com/klauspost/compress v1.17.7 // indirect @@ -135,6 +146,7 @@ require ( github.com/lithammer/shortuuid v3.0.0+incompatible // indirect github.com/magefile/mage v1.15.0 // indirect github.com/magiconair/properties v1.8.7 // indirect + github.com/mailru/easyjson v0.7.7 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect github.com/minio/md5-simd v1.1.2 // indirect @@ -144,6 +156,7 @@ require ( github.com/modern-go/reflect2 v1.0.2 // indirect github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe // indirect github.com/mozillazg/go-httpheader v0.4.0 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/pelletier/go-toml/v2 v2.2.2 // indirect github.com/pierrec/lz4/v4 v4.1.21 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect @@ -165,6 +178,7 @@ require ( github.com/tklauser/go-sysconf v0.3.13 // indirect github.com/tklauser/numcpus v0.7.0 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect + github.com/x448/float16 v0.8.4 // indirect github.com/xdg-go/pbkdf2 v1.0.0 // indirect github.com/xdg-go/scram v1.1.2 // indirect github.com/xdg-go/stringprep v1.0.4 // indirect @@ -186,16 +200,24 @@ require ( golang.org/x/net v0.29.0 // indirect golang.org/x/oauth2 v0.23.0 // indirect golang.org/x/sys v0.25.0 // indirect + golang.org/x/term v0.24.0 // indirect golang.org/x/text v0.18.0 // indirect golang.org/x/time v0.5.0 // indirect google.golang.org/appengine/v2 v2.0.2 // indirect google.golang.org/genproto v0.0.0-20240213162025-012b6fc9bca9 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20240903143218-8af14fe29dc1 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 // indirect + gopkg.in/inf.v0 v0.9.1 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect gorm.io/gorm v1.25.8 // indirect howett.net/plist v0.0.0-20181124034731-591f970eefbb // indirect + k8s.io/klog/v2 v2.130.1 // indirect + k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 // indirect + k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 // indirect modernc.org/fileutil v1.0.0 // indirect - stathat.com/c/consistent v1.0.0 // indirect + sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect + sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect + sigs.k8s.io/yaml v1.4.0 // indirect ) require ( diff --git a/go.sum b/go.sum index afbd01f09..4657f0eca 100644 --- a/go.sum +++ b/go.sum @@ -112,6 +112,8 @@ github.com/elastic/go-sysinfo v1.0.2 h1:Wq1bOgnSz7Obl7DbMjbn0tzx1bE5G8Cfy3MVFa6C github.com/elastic/go-sysinfo v1.0.2/go.mod h1:O/D5m1VpYLwGjCYzEt63g3Z1uO3jXfwyzzjiW90t8cY= github.com/elastic/go-windows v1.0.0 h1:qLURgZFkkrYyTTkvYpsZIgf83AUsdIHfvlJaqaZ7aSY= github.com/elastic/go-windows v1.0.0/go.mod h1:TsU0Nrp7/y3+VwE82FoZF8gC/XFg/Elz6CcloAxnPgU= +github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g= +github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= @@ -126,6 +128,8 @@ github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHk github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= +github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E= +github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ= github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0= github.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk= github.com/gammazero/toposort v0.1.1 h1:OivGxsWxF3U3+U80VoLJ+f50HcPU1MIqE1JlKzoJ2Eg= @@ -143,6 +147,13 @@ github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= +github.com/go-openapi/jsonpointer v0.19.6 h1:eCs3fxoIi3Wh6vtgmLTOjdhSpiqphQ+DaPn38N2ZdrE= +github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= +github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE= +github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= +github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= +github.com/go-openapi/swag v0.22.4 h1:QLMzNJnMGPRNDCbySlcj1x01tzU8/9LTTL9hZZZogBU= +github.com/go-openapi/swag v0.22.4/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= @@ -161,6 +172,8 @@ github.com/go-redis/redis v6.15.9+incompatible h1:K0pv1D7EQUjfyoMql+r/jZqCLizCGK github.com/go-redis/redis v6.15.9+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA= github.com/go-redis/redismock/v9 v9.2.0 h1:ZrMYQeKPECZPjOj5u9eyOjg8Nnb0BS9lkVIZ6IpsKLw= github.com/go-redis/redismock/v9 v9.2.0/go.mod h1:18KHfGDK4Y6c2R0H38EUGWAdc7ZQS9gfYxc94k7rWT0= +github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= +github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= github.com/go-zookeeper/zk v1.0.3 h1:7M2kwOsc//9VeeFiPtf+uSJlVpU66x9Ba5+8XK7/TDg= github.com/go-zookeeper/zk v1.0.3/go.mod h1:nOB03cncLtlp4t+UAkGSV+9beXP/akpekBwL+UX1Qcw= github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= @@ -192,6 +205,8 @@ github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= +github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -199,14 +214,19 @@ github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= +github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian/v3 v3.3.2 h1:IqNFLAmvJOgVlpdEBiQbDc2EwKW77amAycfTuWKdfvw= github.com/google/martian/v3 v3.3.2/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= +github.com/google/pprof v0.0.0-20240525223248-4bfdf5a9a2af h1:kmjWCqn2qkEml422C2Rrd27c3VGxi6a/6HNq8QmHRKM= +github.com/google/pprof v0.0.0-20240525223248-4bfdf5a9a2af/go.mod h1:K1liHPHnj73Fdn/EKuT8nrFqBihUSKXoLYU0BuatOYo= github.com/google/s2a-go v0.1.7 h1:60BLSyTrOV4/haCDW4zb1guZItoSq8foHCXrAnjBo/o= github.com/google/s2a-go v0.1.7/go.mod h1:50CgR4k1jNlWBu4UfS4AcfhVe1r6pdZPygJ3R8F0Qdw= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -263,6 +283,8 @@ github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901 h1:rp+c0RAYOWj8 github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901/go.mod h1:Z86h9688Y0wesXCyonoVr47MasHilkuLMqGhRZ4Hpak= github.com/jonboulle/clockwork v0.4.0 h1:p4Cf1aMWXnXAUh8lVfewRBx1zaTSYKrKMF2g3ST4RZ4= github.com/jonboulle/clockwork v0.4.0/go.mod h1:xgRqUGwRcjKCO1vbZUEtSLrqKoPSsUpK7fnezOII0kc= +github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/kelindar/bitmap v1.5.2 h1:XwX7CTvJtetQZ64zrOkApoZZHBJRkjE23NfqUALA/HE= @@ -305,6 +327,8 @@ github.com/magefile/mage v1.15.0 h1:BvGheCMAsG3bWUDbZ8AyXXpCNwU9u5CB6sM+HNb9HYg= github.com/magefile/mage v1.15.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A= github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= +github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= @@ -331,18 +355,22 @@ github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJ github.com/mozillazg/go-httpheader v0.2.1/go.mod h1:jJ8xECTlalr6ValeXYdOF8fFUISeBAdw6E61aqQma60= github.com/mozillazg/go-httpheader v0.4.0 h1:aBn6aRXtFzyDLZ4VIRLsZbbJloagQfMnCiYgOq6hK4w= github.com/mozillazg/go-httpheader v0.4.0/go.mod h1:PuT8h0pw6efvp8ZeUec1Rs7dwjK08bt6gKSReGMqtdA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= +github.com/onsi/ginkgo/v2 v2.19.0 h1:9Cnnf7UHo57Hy3k6/m5k3dRfGTMXGvxhHFvkDTCTpvA= +github.com/onsi/ginkgo/v2 v2.19.0/go.mod h1:rlwLi9PilAFJ8jCg9UE1QP6VBpd6/xj3SRC0d6TU0To= github.com/onsi/gomega v1.25.0 h1:Vw7br2PCDYijJHSfBOWhov+8cAnUf8MfMaIOV323l6Y= github.com/onsi/gomega v1.25.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdMPGhM= -github.com/openimsdk/gomake v0.0.14-alpha.5 h1:VY9c5x515lTfmdhhPjMvR3BBRrRquAUCFsz7t7vbv7Y= -github.com/openimsdk/gomake v0.0.14-alpha.5/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= +github.com/openimsdk/gomake v0.0.15-alpha.1 h1:a3U0gayzEJJucOoIOt9Jju+HQTG2BtKcuLG4AVjeVF4= +github.com/openimsdk/gomake v0.0.15-alpha.1/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= github.com/openimsdk/protocol v0.0.72-alpha.61 h1:RuZR9/Sg3p6Bpb2CKPjPoA2AUmTvHITmhZ3PT/RbWMs= github.com/openimsdk/protocol v0.0.72-alpha.61/go.mod h1:Iet+piS/jaS+kWWyj6EEr36mk4ISzIRYjoMSVA4dq2M= -github.com/openimsdk/tools v0.0.50-alpha.43 h1:IKa8++fT46YBAVlfy/DcNPOJvJRlWX38pWIfwFIQ2AE= -github.com/openimsdk/tools v0.0.50-alpha.43/go.mod h1:YOtTcSQ7Ik6NYkjXCTXZT3YDM7smGp9D2xvti3KvNoM= +github.com/openimsdk/tools v0.0.50-alpha.45 h1:bu3HCkdzLNQAPUR1VhF40XA9OhaNGHGxZqcC4rIYJyY= +github.com/openimsdk/tools v0.0.50-alpha.45/go.mod h1:YOtTcSQ7Ik6NYkjXCTXZT3YDM7smGp9D2xvti3KvNoM= github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= @@ -378,8 +406,8 @@ github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE= -github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= -github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= +github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= +github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc= github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= @@ -401,8 +429,6 @@ github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.18.2 h1:LUXCnvUvSM6FXAsj6nnfc8Q2tp1dIgUfY9Kc8GsSOiQ= github.com/spf13/viper v1.18.2/go.mod h1:EKmWIqdnk5lOcmR72yw6hS+8OPYcwD0jteitLMVB+yk= -github.com/stathat/consistent v1.0.0 h1:ZFJ1QTRn8npNBKW065raSZ8xfOqhpb8vLOkfp4CcL/U= -github.com/stathat/consistent v1.0.0/go.mod h1:uajTPbgSygZBJ+V+0mY7meZ8i0XAcZs7AQ6V121XSxw= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= @@ -432,6 +458,8 @@ github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= github.com/ugorji/go/codec v1.2.12 h1:9LC83zGrHhuUA9l16C9AHXAqEV/2wBQ4nkvumAE65EE= github.com/ugorji/go/codec v1.2.12/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= +github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= +github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c= github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY= @@ -471,8 +499,8 @@ go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/automaxprocs v1.5.3 h1:kWazyxZUrS3Gs4qUpbwo5kEIMGe/DAvi5Z4tl2NW4j8= go.uber.org/automaxprocs v1.5.3/go.mod h1:eRbA25aqJrxAbsLO0xy5jVwPt7FQnRgjW+efnwa1WM0= -go.uber.org/goleak v1.1.11 h1:wy28qYRKZgnJTxGxvye5/wgWr1EKjmUDGYox5mGlRlI= -go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60= @@ -545,6 +573,8 @@ golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.24.0 h1:Mh5cbb+Zk2hqqXNO7S1iTjEphVL+jb8ZWaqh/g+JWkM= +golang.org/x/term v0.24.0/go.mod h1:lOBK/LVxemqiMij05LGJ0tzNr8xlmwBRJ81PX6wVLH8= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -565,6 +595,8 @@ golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -609,12 +641,15 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= @@ -627,9 +662,25 @@ honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= howett.net/plist v0.0.0-20181124034731-591f970eefbb h1:jhnBjNi9UFpfpl8YZhA9CrOqpnJdvzuiHsl/dnxl11M= howett.net/plist v0.0.0-20181124034731-591f970eefbb/go.mod h1:vMygbs4qMhSZSc4lCUl2OEE+rDiIIJAIdR4m7MiMcm0= +k8s.io/api v0.31.2 h1:3wLBbL5Uom/8Zy98GRPXpJ254nEFpl+hwndmk9RwmL0= +k8s.io/api v0.31.2/go.mod h1:bWmGvrGPssSK1ljmLzd3pwCQ9MgoTsRCuK35u6SygUk= +k8s.io/apimachinery v0.31.2 h1:i4vUt2hPK56W6mlT7Ry+AO8eEsyxMD1U44NR22CLTYw= +k8s.io/apimachinery v0.31.2/go.mod h1:rsPdaZJfTfLsNJSQzNHQvYoTmxhoOEofxtOsF3rtsMo= +k8s.io/client-go v0.31.2 h1:Y2F4dxU5d3AQj+ybwSMqQnpZH9F30//1ObxOKlTI9yc= +k8s.io/client-go v0.31.2/go.mod h1:NPa74jSVR/+eez2dFsEIHNa+3o09vtNaWwWwb1qSxSs= +k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= +k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= +k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 h1:BZqlfIlq5YbRMFko6/PM7FjZpUb45WallggurYhKGag= +k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340/go.mod h1:yD4MZYeKMBwQKVht279WycxKyM84kkAx2DPrTXaeb98= +k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 h1:pUdcCO1Lk/tbT5ztQWOBi5HBgbBP1J8+AsQnQCKsi8A= +k8s.io/utils v0.0.0-20240711033017-18e509b52bc8/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= modernc.org/fileutil v1.0.0 h1:Z1AFLZwl6BO8A5NldQg/xTSjGLetp+1Ubvl4alfGx8w= modernc.org/fileutil v1.0.0/go.mod h1:JHsWpkrk/CnVV1H/eGlFf85BEpfkrp56ro8nojIq9Q8= nullprogram.com/x/optparse v1.0.0/go.mod h1:KdyPE+Igbe0jQUrVfMqDMeJQIJZEuyV7pjYmp6pbG50= rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= -stathat.com/c/consistent v1.0.0 h1:ezyc51EGcRPJUxfHGSgJjWzJdj3NiMU9pNfLNGiXV0c= -stathat.com/c/consistent v1.0.0/go.mod h1:QkzMWzcbB+yQBL2AttO6sgsQS/JSTapcDISJalmCDS0= +sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= +sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= +sigs.k8s.io/structured-merge-diff/v4 v4.4.1 h1:150L+0vs/8DA78h1u02ooW1/fFq/Lwr+sGiqlzvrtq4= +sigs.k8s.io/structured-merge-diff/v4 v4.4.1/go.mod h1:N8hJocpFajUSSeSJ9bOZ77VzejKZaXsTtZo4/u7Io08= +sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= +sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY= diff --git a/internal/api/init.go b/internal/api/init.go index 6c25609f6..ea7b6133f 100644 --- a/internal/api/init.go +++ b/internal/api/init.go @@ -17,9 +17,6 @@ package api import ( "context" "fmt" - "github.com/openimsdk/open-im-server/v3/pkg/common/config" - "github.com/openimsdk/tools/utils/datautil" - "github.com/openimsdk/tools/utils/network" "net" "net/http" "os" @@ -28,6 +25,11 @@ import ( "syscall" "time" + "github.com/openimsdk/open-im-server/v3/pkg/common/config" + "github.com/openimsdk/tools/utils/datautil" + "github.com/openimsdk/tools/utils/network" + "github.com/openimsdk/tools/utils/runtimeenv" + kdisc "github.com/openimsdk/open-im-server/v3/pkg/common/discoveryregister" "github.com/openimsdk/open-im-server/v3/pkg/common/prommetrics" "github.com/openimsdk/tools/discovery" @@ -40,6 +42,8 @@ type Config struct { API config.API Share config.Share Discovery config.Discovery + + RuntimeEnv string } func Start(ctx context.Context, index int, config *Config) error { @@ -48,10 +52,12 @@ func Start(ctx context.Context, index int, config *Config) error { return err } + config.RuntimeEnv = runtimeenv.PrintRuntimeEnvironment() + var client discovery.SvcDiscoveryRegistry // Determine whether zk is passed according to whether it is a clustered deployment - client, err = kdisc.NewDiscoveryRegister(&config.Discovery) + client, err = kdisc.NewDiscoveryRegister(&config.Discovery, config.RuntimeEnv) if err != nil { return errs.WrapMsg(err, "failed to register discovery service") } @@ -81,7 +87,7 @@ func Start(ctx context.Context, index int, config *Config) error { address := net.JoinHostPort(network.GetListenIP(config.API.Api.ListenIP), strconv.Itoa(apiPort)) server := http.Server{Addr: address, Handler: router} - log.CInfo(ctx, "API server is initializing", "address", address, "apiPort", apiPort, "prometheusPort", prometheusPort) + log.CInfo(ctx, "API server is initializing", "runtimeEnv", config.RuntimeEnv, "address", address, "apiPort", apiPort, "prometheusPort", prometheusPort) go func() { err = server.ListenAndServe() if err != nil && err != http.ErrServerClosed { diff --git a/internal/msggateway/init.go b/internal/msggateway/init.go index 1796a5482..7603147de 100644 --- a/internal/msggateway/init.go +++ b/internal/msggateway/init.go @@ -16,11 +16,13 @@ package msggateway import ( "context" + "time" + "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/open-im-server/v3/pkg/rpccache" "github.com/openimsdk/tools/db/redisutil" "github.com/openimsdk/tools/utils/datautil" - "time" + "github.com/openimsdk/tools/utils/runtimeenv" "github.com/openimsdk/tools/log" ) @@ -31,11 +33,15 @@ type Config struct { RedisConfig config.Redis WebhooksConfig config.Webhooks Discovery config.Discovery + + RuntimeEnv string } // Start run ws server. func Start(ctx context.Context, index int, conf *Config) error { - log.CInfo(ctx, "MSG-GATEWAY server is initializing", "autoSetPorts", conf.MsgGateway.RPC.AutoSetPorts, + conf.RuntimeEnv = runtimeenv.PrintRuntimeEnvironment() + + log.CInfo(ctx, "MSG-GATEWAY server is initializing", "runtimeEnv", conf.RuntimeEnv, "autoSetPorts", conf.MsgGateway.RPC.AutoSetPorts, "rpcPorts", conf.MsgGateway.RPC.Ports, "wsPort", conf.MsgGateway.LongConnSvr.Ports, "prometheusPorts", conf.MsgGateway.Prometheus.Ports) wsPort, err := datautil.GetElemByIndex(conf.MsgGateway.LongConnSvr.Ports, index) diff --git a/internal/msgtransfer/init.go b/internal/msgtransfer/init.go index b65532a8e..6131d8c77 100644 --- a/internal/msgtransfer/init.go +++ b/internal/msgtransfer/init.go @@ -29,6 +29,7 @@ import ( "github.com/openimsdk/tools/db/mongoutil" "github.com/openimsdk/tools/db/redisutil" "github.com/openimsdk/tools/utils/datautil" + "github.com/openimsdk/tools/utils/runtimeenv" "github.com/openimsdk/open-im-server/v3/pkg/common/config" discRegister "github.com/openimsdk/open-im-server/v3/pkg/common/discoveryregister" @@ -51,6 +52,8 @@ type MsgTransfer struct { historyMongoCH *OnlineHistoryMongoConsumerHandler ctx context.Context cancel context.CancelFunc + + runTimeEnv string } type Config struct { @@ -64,7 +67,9 @@ type Config struct { } func Start(ctx context.Context, index int, config *Config) error { - log.CInfo(ctx, "MSG-TRANSFER server is initializing", "prometheusPorts", + runTimeEnv := runtimeenv.PrintRuntimeEnvironment() + + log.CInfo(ctx, "MSG-TRANSFER server is initializing", "runTimeEnv", runTimeEnv, "prometheusPorts", config.MsgTransfer.Prometheus.Ports, "index", index) mgocli, err := mongoutil.NewMongoDB(ctx, config.MongodbConfig.Build()) @@ -75,7 +80,7 @@ func Start(ctx context.Context, index int, config *Config) error { if err != nil { return err } - client, err := discRegister.NewDiscoveryRegister(&config.Discovery) + client, err := discRegister.NewDiscoveryRegister(&config.Discovery, runTimeEnv) if err != nil { return err } @@ -115,6 +120,7 @@ func Start(ctx context.Context, index int, config *Config) error { msgTransfer := &MsgTransfer{ historyCH: historyCH, historyMongoCH: historyMongoCH, + runTimeEnv: runTimeEnv, } return msgTransfer.Start(index, config) } diff --git a/internal/tools/cron_task.go b/internal/tools/cron_task.go index 3864c43dc..2fe7d0e39 100644 --- a/internal/tools/cron_task.go +++ b/internal/tools/cron_task.go @@ -14,6 +14,7 @@ import ( "github.com/openimsdk/tools/mcontext" "github.com/openimsdk/tools/mw" + "github.com/openimsdk/tools/utils/runtimeenv" "google.golang.org/grpc" "google.golang.org/grpc/credentials/insecure" @@ -26,14 +27,18 @@ type CronTaskConfig struct { CronTask config.CronTask Share config.Share Discovery config.Discovery + + runTimeEnv string } func Start(ctx context.Context, config *CronTaskConfig) error { - log.CInfo(ctx, "CRON-TASK server is initializing", "chatRecordsClearTime", config.CronTask.CronExecuteTime, "msgDestructTime", config.CronTask.RetainChatRecords) + config.runTimeEnv = runtimeenv.PrintRuntimeEnvironment() + + log.CInfo(ctx, "CRON-TASK server is initializing", "runTimeEnv", config.runTimeEnv, "chatRecordsClearTime", config.CronTask.CronExecuteTime, "msgDestructTime", config.CronTask.RetainChatRecords) if config.CronTask.RetainChatRecords < 1 { return errs.New("msg destruct time must be greater than 1").Wrap() } - client, err := kdisc.NewDiscoveryRegister(&config.Discovery) + client, err := kdisc.NewDiscoveryRegister(&config.Discovery, config.runTimeEnv) if err != nil { return errs.WrapMsg(err, "failed to register discovery service") } diff --git a/magefile.go b/magefile.go index a8a1c4040..d0e77f6fc 100644 --- a/magefile.go +++ b/magefile.go @@ -4,14 +4,23 @@ package main import ( - "github.com/openimsdk/gomake/mageutil" + "flag" "os" + + "github.com/openimsdk/gomake/mageutil" ) var Default = Build func Build() { - mageutil.Build() + flag.Parse() + + bin := flag.Args() + if len(bin) != 0 { + bin = bin[1:] + } + + mageutil.Build(bin) } func Start() { diff --git a/pkg/common/cmd/root.go b/pkg/common/cmd/root.go index 5edea4377..87252c133 100644 --- a/pkg/common/cmd/root.go +++ b/pkg/common/cmd/root.go @@ -16,12 +16,12 @@ package cmd import ( "fmt" - "path/filepath" "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/open-im-server/v3/version" "github.com/openimsdk/tools/errs" "github.com/openimsdk/tools/log" + "github.com/openimsdk/tools/utils/runtimeenv" "github.com/spf13/cobra" ) @@ -105,18 +105,19 @@ func (r *RootCmd) initializeConfiguration(cmd *cobra.Command, opts *CmdOpts) err if err != nil { return err } + + runtimeEnv := runtimeenv.PrintRuntimeEnvironment() + // Load common configuration file //opts.configMap[ShareFileName] = StructEnvPrefix{EnvPrefix: shareEnvPrefix, ConfigStruct: &r.share} for configFileName, configStruct := range opts.configMap { - err := config.LoadConfig(filepath.Join(configDirectory, configFileName), - ConfigEnvPrefixMap[configFileName], configStruct) + err := config.Load(configDirectory, configFileName, ConfigEnvPrefixMap[configFileName], runtimeEnv, configStruct) if err != nil { return err } } // Load common log configuration file - return config.LoadConfig(filepath.Join(configDirectory, LogConfigFileName), - ConfigEnvPrefixMap[LogConfigFileName], &r.log) + return config.Load(configDirectory, LogConfigFileName, ConfigEnvPrefixMap[LogConfigFileName], runtimeEnv, &r.log) } func (r *RootCmd) applyOptions(opts ...func(*CmdOpts)) *CmdOpts { diff --git a/pkg/common/config/config.go b/pkg/common/config/config.go index b13da1197..b8a60c3df 100644 --- a/pkg/common/config/config.go +++ b/pkg/common/config/config.go @@ -15,10 +15,11 @@ package config import ( - "github.com/openimsdk/tools/s3/aws" "strings" "time" + "github.com/openimsdk/tools/s3/aws" + "github.com/openimsdk/tools/db/mongoutil" "github.com/openimsdk/tools/db/redisutil" "github.com/openimsdk/tools/mq/kafka" @@ -474,9 +475,14 @@ type ZooKeeper struct { type Discovery struct { Enable string `mapstructure:"enable"` Etcd Etcd `mapstructure:"etcd"` + Kubernetes Kubernetes `mapstructure:"kubernetes"` RpcService RpcService `mapstructure:"rpcService"` } +type Kubernetes struct { + Namespace string `mapstructure:"namespace"` +} + type Etcd struct { RootDirectory string `mapstructure:"rootDirectory"` Address []string `mapstructure:"address"` diff --git a/pkg/common/config/constant.go b/pkg/common/config/constant.go index f425a624c..9aeaedca9 100644 --- a/pkg/common/config/constant.go +++ b/pkg/common/config/constant.go @@ -16,6 +16,12 @@ package config const ConfKey = "conf" +const ( + MountConfigFilePath = "CONFIG_PATH" + DeploymentType = "DEPLOYMENT_TYPE" + KUBERNETES = "kubernetes" +) + const ( // DefaultDirPerm is used for creating general directories, allowing the owner to read, write, and execute, // while the group and others can only read and execute. diff --git a/pkg/common/config/load_config.go b/pkg/common/config/load_config.go index 9272896b4..aa87211f9 100644 --- a/pkg/common/config/load_config.go +++ b/pkg/common/config/load_config.go @@ -1,13 +1,29 @@ package config import ( + "os" + "path/filepath" + "strings" + "github.com/mitchellh/mapstructure" "github.com/openimsdk/tools/errs" "github.com/spf13/viper" - "strings" ) -func LoadConfig(path string, envPrefix string, config any) error { +func Load(configDirectory string, configFileName string, envPrefix string, runtimeEnv string, config any) error { + if runtimeEnv == KUBERNETES { + mountPath := os.Getenv(MountConfigFilePath) + if mountPath == "" { + return errs.ErrArgs.WrapMsg(MountConfigFilePath + " env is empty") + } + + return loadConfig(filepath.Join(mountPath, configFileName), envPrefix, config) + } + + return loadConfig(filepath.Join(configDirectory, configFileName), envPrefix, config) +} + +func loadConfig(path string, envPrefix string, config any) error { v := viper.New() v.SetConfigFile(path) v.SetEnvPrefix(envPrefix) diff --git a/pkg/common/config/load_config_test.go b/pkg/common/config/load_config_test.go index a0345fc7a..ea11b0574 100644 --- a/pkg/common/config/load_config_test.go +++ b/pkg/common/config/load_config_test.go @@ -1,27 +1,51 @@ package config import ( - "github.com/stretchr/testify/assert" + "os" "testing" + + "github.com/stretchr/testify/assert" ) func TestLoadLogConfig(t *testing.T) { var log Log - err := LoadConfig("../../../config/log.yml", "IMENV_LOG", &log) + os.Setenv("IMENV_LOG_REMAINLOGLEVEL", "5") + err := Load("../../../config/", "log.yml", "IMENV_LOG", "source", &log) + assert.Nil(t, err) + t.Log(log.RemainLogLevel) + // assert.Equal(t, "../../../../logs/", log.StorageLocation) +} + +func TestLoadMongoConfig(t *testing.T) { + var mongo Mongo + // os.Setenv("DEPLOYMENT_TYPE", "kubernetes") + os.Setenv("IMENV_MONGODB_PASSWORD", "openIM1231231") + // os.Setenv("IMENV_MONGODB_URI", "openIM123") + // os.Setenv("IMENV_MONGODB_USERNAME", "openIM123") + err := Load("../../../config/", "mongodb.yml", "IMENV_MONGODB", "source", &mongo) + // err := LoadConfig("../../../config/mongodb.yml", "IMENV_MONGODB", &mongo) + assert.Nil(t, err) - assert.Equal(t, "../../../../logs/", log.StorageLocation) + t.Log(mongo.Password) + // assert.Equal(t, "openIM123", mongo.Password) + t.Log(os.Getenv("IMENV_MONGODB_PASSWORD")) + t.Log(mongo) + // //export IMENV_OPENIM_RPC_USER_RPC_LISTENIP="0.0.0.0" + // assert.Equal(t, "0.0.0.0", user.RPC.ListenIP) + // //export IMENV_OPENIM_RPC_USER_RPC_PORTS="10110,10111,10112" + // assert.Equal(t, []int{10110, 10111, 10112}, user.RPC.Ports) } func TestLoadMinioConfig(t *testing.T) { var storageConfig Minio - err := LoadConfig("../../../config/minio.yml", "IMENV_MINIO", &storageConfig) + err := Load("../../../config/minio.yml", "IMENV_MINIO", "", "source", &storageConfig) assert.Nil(t, err) assert.Equal(t, "openim", storageConfig.Bucket) } func TestLoadWebhooksConfig(t *testing.T) { var webhooks Webhooks - err := LoadConfig("../../../config/webhooks.yml", "IMENV_WEBHOOKS", &webhooks) + err := Load("../../../config/webhooks.yml", "IMENV_WEBHOOKS", "", "source", &webhooks) assert.Nil(t, err) assert.Equal(t, 5, webhooks.BeforeAddBlack.Timeout) @@ -29,7 +53,7 @@ func TestLoadWebhooksConfig(t *testing.T) { func TestLoadOpenIMRpcUserConfig(t *testing.T) { var user User - err := LoadConfig("../../../config/openim-rpc-user.yml", "IMENV_OPENIM_RPC_USER", &user) + err := Load("../../../config/openim-rpc-user.yml", "IMENV_OPENIM_RPC_USER", "", "source", &user) assert.Nil(t, err) //export IMENV_OPENIM_RPC_USER_RPC_LISTENIP="0.0.0.0" assert.Equal(t, "0.0.0.0", user.RPC.ListenIP) @@ -39,14 +63,14 @@ func TestLoadOpenIMRpcUserConfig(t *testing.T) { func TestLoadNotificationConfig(t *testing.T) { var noti Notification - err := LoadConfig("../../../config/notification.yml", "IMENV_NOTIFICATION", ¬i) + err := Load("../../../config/notification.yml", "IMENV_NOTIFICATION", "", "source", ¬i) assert.Nil(t, err) assert.Equal(t, "Your friend's profile has been changed", noti.FriendRemarkSet.OfflinePush.Title) } func TestLoadOpenIMThirdConfig(t *testing.T) { var third Third - err := LoadConfig("../../../config/openim-rpc-third.yml", "IMENV_OPENIM_RPC_THIRD", &third) + err := Load("../../../config/openim-rpc-third.yml", "IMENV_OPENIM_RPC_THIRD", "", "source", &third) assert.Nil(t, err) assert.Equal(t, "enabled", third.Object.Enable) assert.Equal(t, "https://oss-cn-chengdu.aliyuncs.com", third.Object.Oss.Endpoint) diff --git a/pkg/common/discoveryregister/discoveryregister.go b/pkg/common/discoveryregister/discoveryregister.go index be82aa526..419293a91 100644 --- a/pkg/common/discoveryregister/discoveryregister.go +++ b/pkg/common/discoveryregister/discoveryregister.go @@ -15,19 +15,26 @@ package discoveryregister import ( + "time" + "github.com/openimsdk/open-im-server/v3/pkg/common/config" - "github.com/openimsdk/open-im-server/v3/pkg/common/discoveryregister/kubernetes" "github.com/openimsdk/tools/discovery" + + "github.com/openimsdk/tools/discovery/kubernetes" + "github.com/openimsdk/tools/discovery/etcd" "github.com/openimsdk/tools/errs" - "time" ) // NewDiscoveryRegister creates a new service discovery and registry client based on the provided environment type. -func NewDiscoveryRegister(discovery *config.Discovery) (discovery.SvcDiscoveryRegistry, error) { +func NewDiscoveryRegister(discovery *config.Discovery, runtimeEnv string) (discovery.SvcDiscoveryRegistry, error) { + if runtimeEnv == "kubernetes" { + discovery.Enable = "kubernetes" + } + switch discovery.Enable { - case "k8s": - return kubernetes.NewK8sDiscoveryRegister(discovery.RpcService.MessageGateway) + case "kubernetes": + return kubernetes.NewKubernetesConnManager(discovery.Kubernetes.Namespace) case "etcd": return etcd.NewSvcDiscoveryRegistry( discovery.Etcd.RootDirectory, diff --git a/pkg/common/discoveryregister/kubernetes/kubernetes.go b/pkg/common/discoveryregister/kubernetes/kubernetes.go index f1ce0bbdc..cf6a904be 100644 --- a/pkg/common/discoveryregister/kubernetes/kubernetes.go +++ b/pkg/common/discoveryregister/kubernetes/kubernetes.go @@ -16,184 +16,281 @@ package kubernetes import ( "context" - "errors" "fmt" - "os" - "strconv" - "strings" + "sync" + "time" - "github.com/openimsdk/tools/discovery" - "github.com/openimsdk/tools/log" - "github.com/stathat/consistent" "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/informers" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/cache" ) -// K8sDR represents the Kubernetes service discovery and registration client. -type K8sDR struct { - options []grpc.DialOption - rpcRegisterAddr string - gatewayHostConsistent *consistent.Consistent - gatewayName string +type KubernetesConnManager struct { + clientset *kubernetes.Clientset + namespace string + dialOptions []grpc.DialOption + + selfTarget string + + mu sync.RWMutex + connMap map[string][]*grpc.ClientConn } -func NewK8sDiscoveryRegister(gatewayName string) (discovery.SvcDiscoveryRegistry, error) { - gatewayConsistent := consistent.New() - gatewayHosts := getMsgGatewayHost(context.Background(), gatewayName) - for _, v := range gatewayHosts { - gatewayConsistent.Add(v) +// NewKubernetesConnManager creates a new connection manager that uses Kubernetes services for service discovery. +func NewKubernetesConnManager(namespace string, options ...grpc.DialOption) (*KubernetesConnManager, error) { + config, err := rest.InClusterConfig() + if err != nil { + return nil, fmt.Errorf("failed to create in-cluster config: %v", err) } - return &K8sDR{gatewayHostConsistent: gatewayConsistent}, nil -} -func (cli *K8sDR) Register(serviceName, host string, port int, opts ...grpc.DialOption) error { - if serviceName != cli.gatewayName { - cli.rpcRegisterAddr = serviceName - } else { - cli.rpcRegisterAddr = getSelfHost(context.Background(), cli.gatewayName) + clientset, err := kubernetes.NewForConfig(config) + if err != nil { + return nil, fmt.Errorf("failed to create clientset: %v", err) } - return nil -} + k := &KubernetesConnManager{ + clientset: clientset, + namespace: namespace, + dialOptions: options, + connMap: make(map[string][]*grpc.ClientConn), + } -func (cli *K8sDR) UnRegister() error { + go k.watchEndpoints() - return nil + return k, nil } -func (cli *K8sDR) CreateRpcRootNodes(serviceNames []string) error { +func (k *KubernetesConnManager) initializeConns(serviceName string) error { + port, err := k.getServicePort(serviceName) + if err != nil { + return err + } - return nil -} + endpoints, err := k.clientset.CoreV1().Endpoints(k.namespace).Get(context.Background(), serviceName, metav1.GetOptions{}) + if err != nil { + return fmt.Errorf("failed to get endpoints for service %s: %v", serviceName, err) + } + + var conns []*grpc.ClientConn + for _, subset := range endpoints.Subsets { + for _, address := range subset.Addresses { + target := fmt.Sprintf("%s:%d", address.IP, port) + conn, err := grpc.Dial(target, grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + return fmt.Errorf("failed to dial endpoint %s: %v", target, err) + } + conns = append(conns, conn) + } + } + + k.mu.Lock() + defer k.mu.Unlock() + k.connMap[serviceName] = conns -func (cli *K8sDR) RegisterConf2Registry(key string, conf []byte) error { + // go k.watchEndpoints(serviceName) return nil } -func (cli *K8sDR) GetConfFromRegistry(key string) ([]byte, error) { +// GetConns returns gRPC client connections for a given Kubernetes service name. +func (k *KubernetesConnManager) GetConns(ctx context.Context, serviceName string, opts ...grpc.DialOption) ([]*grpc.ClientConn, error) { + k.mu.RLock() + conns, exists := k.connMap[serviceName] + defer k.mu.RUnlock() + + if exists { + return conns, nil + } + + k.mu.Lock() + defer k.mu.Unlock() + + // Check if another goroutine has already initialized the connections when we released the read lock + conns, exists = k.connMap[serviceName] + if exists { + return conns, nil + } + + if err := k.initializeConns(serviceName); err != nil { + return nil, fmt.Errorf("failed to initialize connections for service %s: %v", serviceName, err) + } - return nil, nil + return k.connMap[serviceName], nil } -func (cli *K8sDR) GetUserIdHashGatewayHost(ctx context.Context, userId string) (string, error) { - host, err := cli.gatewayHostConsistent.Get(userId) +// GetConn returns a single gRPC client connection for a given Kubernetes service name. +func (k *KubernetesConnManager) GetConn(ctx context.Context, serviceName string, opts ...grpc.DialOption) (*grpc.ClientConn, error) { + port, err := k.getServicePort(serviceName) if err != nil { - log.ZError(ctx, "GetUserIdHashGatewayHost error", err) + return nil, err } - return host, err + + fmt.Println("SVC port:", port) + + target := fmt.Sprintf("%s.%s.svc.cluster.local:%d", serviceName, k.namespace, port) + + fmt.Println("SVC target:", target) + + return grpc.DialContext( + ctx, + target, + append([]grpc.DialOption{grpc.WithTransportCredentials(insecure.NewCredentials())}, k.dialOptions...)..., + ) } -func getSelfHost(ctx context.Context, gatewayName string) string { - port := 88 - instance := "openimserver" - selfPodName := os.Getenv("MY_POD_NAME") - ns := os.Getenv("MY_POD_NAMESPACE") - statefuleIndex := 0 - gatewayEnds := strings.Split(gatewayName, ":") - if len(gatewayEnds) != 2 { - log.ZError(ctx, "msggateway RpcRegisterName is error:config.RpcRegisterName.OpenImMessageGatewayName", errors.New("config error")) - } else { - port, _ = strconv.Atoi(gatewayEnds[1]) - } - podInfo := strings.Split(selfPodName, "-") - instance = podInfo[0] - count := len(podInfo) - statefuleIndex, _ = strconv.Atoi(podInfo[count-1]) - host := fmt.Sprintf("%s-openim-msggateway-%d.%s-openim-msggateway-headless.%s.svc.cluster.local:%d", instance, statefuleIndex, instance, ns, port) - return host +// GetSelfConnTarget returns the connection target for the current service. +func (k *KubernetesConnManager) GetSelfConnTarget() string { + return k.selfTarget } -// like openimserver-openim-msggateway-0.openimserver-openim-msggateway-headless.openim-lin.svc.cluster.local:88. -// Replica set in kubernetes environment -func getMsgGatewayHost(ctx context.Context, gatewayName string) []string { - port := 88 - instance := "openimserver" - selfPodName := os.Getenv("MY_POD_NAME") - replicas := os.Getenv("MY_MSGGATEWAY_REPLICACOUNT") - ns := os.Getenv("MY_POD_NAMESPACE") - gatewayEnds := strings.Split(gatewayName, ":") - if len(gatewayEnds) != 2 { - log.ZError(ctx, "msggateway RpcRegisterName is error:config.RpcRegisterName.OpenImMessageGatewayName", errors.New("config error")) - } else { - port, _ = strconv.Atoi(gatewayEnds[1]) - } - nReplicas, _ := strconv.Atoi(replicas) - podInfo := strings.Split(selfPodName, "-") - instance = podInfo[0] - var ret []string - for i := 0; i < nReplicas; i++ { - host := fmt.Sprintf("%s-openim-msggateway-%d.%s-openim-msggateway-headless.%s.svc.cluster.local:%d", instance, i, instance, ns, port) - ret = append(ret, host) - } - log.ZDebug(ctx, "getMsgGatewayHost", "instance", instance, "selfPodName", selfPodName, "replicas", replicas, "ns", ns, "ret", ret) - return ret +// AddOption appends gRPC dial options to the existing options. +func (k *KubernetesConnManager) AddOption(opts ...grpc.DialOption) { + k.mu.Lock() + defer k.mu.Unlock() + k.dialOptions = append(k.dialOptions, opts...) } -// GetConns returns the gRPC client connections to the specified service. -func (cli *K8sDR) GetConns(ctx context.Context, serviceName string, opts ...grpc.DialOption) ([]*grpc.ClientConn, error) { - - // This conditional checks if the serviceName is not the OpenImMessageGatewayName. - // It seems to handle a special case for the OpenImMessageGateway. - if serviceName != cli.gatewayName { - // DialContext creates a client connection to the given target (serviceName) using the specified context. - // 'cli.options' are likely default or common options for all connections in this struct. - // 'opts...' allows for additional gRPC dial options to be passed and used. - conn, err := grpc.DialContext(ctx, serviceName, append(cli.options, opts...)...) - - // The function returns a slice of client connections with the new connection, or an error if occurred. - return []*grpc.ClientConn{conn}, err - } else { - // This block is executed if the serviceName is OpenImMessageGatewayName. - // 'ret' will accumulate the connections to return. - var ret []*grpc.ClientConn - - // getMsgGatewayHost presumably retrieves hosts for the message gateway service. - // The context is passed, likely for cancellation and timeout control. - gatewayHosts := getMsgGatewayHost(ctx, cli.gatewayName) - - // Iterating over the retrieved gateway hosts. - for _, host := range gatewayHosts { - // Establishes a connection to each host. - // Again, appending cli.options with any additional opts provided. - conn, err := grpc.DialContext(ctx, host, append(cli.options, opts...)...) - - // If there's an error while dialing any host, the function returns immediately with the error. - if err != nil { - return nil, err - } else { - // If the connection is successful, it is added to the 'ret' slice. - ret = append(ret, conn) - } +// CloseConn closes a given gRPC client connection. +func (k *KubernetesConnManager) CloseConn(conn *grpc.ClientConn) { + conn.Close() +} + +// Close closes all gRPC connections managed by KubernetesConnManager. +func (k *KubernetesConnManager) Close() { + k.mu.Lock() + defer k.mu.Unlock() + for _, conns := range k.connMap { + for _, conn := range conns { + _ = conn.Close() } - // After all hosts are processed, the slice of connections is returned. - return ret, nil } + k.connMap = make(map[string][]*grpc.ClientConn) } -func (cli *K8sDR) GetConn(ctx context.Context, serviceName string, opts ...grpc.DialOption) (*grpc.ClientConn, error) { +func (k *KubernetesConnManager) Register(serviceName, host string, port int, opts ...grpc.DialOption) error { + return nil +} +func (k *KubernetesConnManager) UnRegister() error { + return nil +} - return grpc.DialContext(ctx, serviceName, append(cli.options, opts...)...) +func (k *KubernetesConnManager) GetUserIdHashGatewayHost(ctx context.Context, userId string) (string, error) { + return "", nil } -func (cli *K8sDR) GetSelfConnTarget() string { +func (k *KubernetesConnManager) getServicePort(serviceName string) (int32, error) { + svc, err := k.clientset.CoreV1().Services(k.namespace).Get(context.Background(), serviceName, metav1.GetOptions{}) + if err != nil { + fmt.Print("namespace:", k.namespace) + return 0, fmt.Errorf("failed to get service %s: %v", serviceName, err) + } - return cli.rpcRegisterAddr -} + if len(svc.Spec.Ports) == 0 { + return 0, fmt.Errorf("service %s has no ports defined", serviceName) + } -func (cli *K8sDR) AddOption(opts ...grpc.DialOption) { - cli.options = append(cli.options, opts...) + return svc.Spec.Ports[0].Port, nil } -func (cli *K8sDR) CloseConn(conn *grpc.ClientConn) { - conn.Close() +// watchEndpoints listens for changes in Pod resources. +func (k *KubernetesConnManager) watchEndpoints() { + informerFactory := informers.NewSharedInformerFactory(k.clientset, time.Minute*10) + informer := informerFactory.Core().V1().Pods().Informer() + + // Watch for Pod changes (add, update, delete) + informer.AddEventHandler(cache.ResourceEventHandlerFuncs{ + AddFunc: func(obj interface{}) { + k.handleEndpointChange(obj) + }, + UpdateFunc: func(oldObj, newObj interface{}) { + k.handleEndpointChange(newObj) + }, + DeleteFunc: func(obj interface{}) { + k.handleEndpointChange(obj) + }, + }) + + informerFactory.Start(context.Background().Done()) + <-context.Background().Done() // Block forever } -// do not use this method for call rpc. -func (cli *K8sDR) GetClientLocalConns() map[string][]*grpc.ClientConn { - log.ZError(context.Background(), "should not call this function!", nil) - return nil +func (k *KubernetesConnManager) handleEndpointChange(obj interface{}) { + endpoint, ok := obj.(*v1.Endpoints) + if !ok { + return + } + serviceName := endpoint.Name + if err := k.initializeConns(serviceName); err != nil { + fmt.Printf("Error initializing connections for %s: %v\n", serviceName, err) + } } -func (cli *K8sDR) Close() { - -} +// ================= + +// initEndpoints initializes connections by fetching all available endpoints in the specified namespace. + +// func (k *KubernetesConnManager) initEndpoints() error { +// k.mu.Lock() +// defer k.mu.Unlock() + +// pods, err := k.clientset.CoreV1().Pods(k.namespace).List(context.TODO(), metav1.ListOptions{}) +// if err != nil { +// return fmt.Errorf("failed to list pods: %v", err) +// } + +// for _, pod := range pods.Items { +// if pod.Status.Phase == v1.PodRunning { +// target := fmt.Sprintf("%s:%d", address.IP, port) +// conn, err := grpc.Dial(target, grpc.WithTransportCredentials(insecure.NewCredentials())) +// conn, err := k.createGRPCConnection(pod) +// if err != nil { +// return fmt.Errorf("failed to create GRPC connection for pod %s: %v", pod.Name, err) +// } +// k.connMap[pod.Name] = append(k.connMap[pod.Name], conn) +// } +// } + +// return nil +// } + +// ----- + +// func (k *KubernetesConnManager) watchEndpoints1(serviceName string) { +// // watch for changes to the service's endpoints +// informerFactory := informers.NewSharedInformerFactory(k.clientset, time.Minute) +// endpointsInformer := informerFactory.Core().V1().Endpoints().Informer() + +// endpointsInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{ +// AddFunc: func(obj interface{}) { +// eps := obj.(*v1.Endpoints) +// if eps.Name == serviceName { +// k.initializeConns(serviceName) +// } +// }, +// UpdateFunc: func(oldObj, newObj interface{}) { +// eps := newObj.(*v1.Endpoints) +// if eps.Name == serviceName { +// k.initializeConns(serviceName) +// } +// }, +// DeleteFunc: func(obj interface{}) { +// eps := obj.(*v1.Endpoints) +// if eps.Name == serviceName { +// k.mu.Lock() +// defer k.mu.Unlock() +// for _, conn := range k.connMap[serviceName] { +// _ = conn.Close() +// } +// delete(k.connMap, serviceName) +// } +// }, +// }) + +// informerFactory.Start(wait.NeverStop) +// informerFactory.WaitForCacheSync(wait.NeverStop) +// } diff --git a/pkg/common/startrpc/start.go b/pkg/common/startrpc/start.go index 7671b1736..ae8344923 100644 --- a/pkg/common/startrpc/start.go +++ b/pkg/common/startrpc/start.go @@ -18,9 +18,6 @@ import ( "context" "errors" "fmt" - "github.com/openimsdk/open-im-server/v3/pkg/common/config" - "github.com/openimsdk/tools/utils/datautil" - "google.golang.org/grpc/status" "net" "net/http" "os" @@ -28,6 +25,13 @@ import ( "syscall" "time" + "github.com/openimsdk/open-im-server/v3/pkg/common/config" + "github.com/openimsdk/tools/utils/datautil" + "github.com/openimsdk/tools/utils/runtimeenv" + "google.golang.org/grpc/status" + + "strconv" + kdisc "github.com/openimsdk/open-im-server/v3/pkg/common/discoveryregister" "github.com/openimsdk/open-im-server/v3/pkg/common/prommetrics" "github.com/openimsdk/tools/discovery" @@ -37,7 +41,6 @@ import ( "github.com/openimsdk/tools/utils/network" "google.golang.org/grpc" "google.golang.org/grpc/credentials/insecure" - "strconv" ) // Start rpc server. @@ -51,6 +54,8 @@ func Start[T any](ctx context.Context, discovery *config.Discovery, prometheusCo netErr error ) + runTimeEnv := runtimeenv.PrintRuntimeEnvironment() + if !autoSetPorts { rpcPort, err := datautil.GetElemByIndex(rpcPorts, index) if err != nil { @@ -107,11 +112,11 @@ func Start[T any](ctx context.Context, discovery *config.Discovery, prometheusCo registerIP = network.GetListenIP(registerIP) port, _ := strconv.Atoi(portStr) - log.CInfo(ctx, "RPC server is initializing", "rpcRegisterName", rpcRegisterName, "rpcPort", portStr, + log.CInfo(ctx, "RPC server is initializing", "runTimeEnv", runTimeEnv, "rpcRegisterName", rpcRegisterName, "rpcPort", portStr, "prometheusPorts", prometheusConfig.Ports) defer listener.Close() - client, err := kdisc.NewDiscoveryRegister(discovery) + client, err := kdisc.NewDiscoveryRegister(discovery, runTimeEnv) if err != nil { return err } diff --git a/tools/check-component/main.go b/tools/check-component/main.go index a84b7bd87..130035456 100644 --- a/tools/check-component/main.go +++ b/tools/check-component/main.go @@ -33,6 +33,7 @@ import ( "github.com/openimsdk/tools/mq/kafka" "github.com/openimsdk/tools/s3/minio" "github.com/openimsdk/tools/system/program" + "github.com/openimsdk/tools/utils/runtimeenv" ) const maxRetry = 180 @@ -78,35 +79,37 @@ func initConfig(configDir string) (*config.Mongo, *config.Redis, *config.Kafka, discovery = &config.Discovery{} thirdConfig = &config.Third{} ) - err := config.LoadConfig(filepath.Join(configDir, cmd.MongodbConfigFileName), cmd.ConfigEnvPrefixMap[cmd.MongodbConfigFileName], mongoConfig) + runtimeEnv := runtimeenv.PrintRuntimeEnvironment() + + err := config.Load(configDir, cmd.MongodbConfigFileName, cmd.ConfigEnvPrefixMap[cmd.MongodbConfigFileName], runtimeEnv, mongoConfig) if err != nil { return nil, nil, nil, nil, nil, err } - err = config.LoadConfig(filepath.Join(configDir, cmd.RedisConfigFileName), cmd.ConfigEnvPrefixMap[cmd.RedisConfigFileName], redisConfig) + err = config.Load(configDir, cmd.RedisConfigFileName, cmd.ConfigEnvPrefixMap[cmd.RedisConfigFileName], runtimeEnv, redisConfig) if err != nil { return nil, nil, nil, nil, nil, err } - err = config.LoadConfig(filepath.Join(configDir, cmd.KafkaConfigFileName), cmd.ConfigEnvPrefixMap[cmd.KafkaConfigFileName], kafkaConfig) + err = config.Load(configDir, cmd.KafkaConfigFileName, cmd.ConfigEnvPrefixMap[cmd.KafkaConfigFileName], runtimeEnv, kafkaConfig) if err != nil { return nil, nil, nil, nil, nil, err } - err = config.LoadConfig(filepath.Join(configDir, cmd.OpenIMRPCThirdCfgFileName), cmd.ConfigEnvPrefixMap[cmd.OpenIMRPCThirdCfgFileName], thirdConfig) + err = config.Load(configDir, cmd.OpenIMRPCThirdCfgFileName, cmd.ConfigEnvPrefixMap[cmd.OpenIMRPCThirdCfgFileName], runtimeEnv, thirdConfig) if err != nil { return nil, nil, nil, nil, nil, err } if thirdConfig.Object.Enable == "minio" { - err = config.LoadConfig(filepath.Join(configDir, cmd.MinioConfigFileName), cmd.ConfigEnvPrefixMap[cmd.MinioConfigFileName], minioConfig) + err = config.Load(configDir, cmd.MinioConfigFileName, cmd.ConfigEnvPrefixMap[cmd.MinioConfigFileName], runtimeEnv, minioConfig) if err != nil { return nil, nil, nil, nil, nil, err } } else { minioConfig = nil } - err = config.LoadConfig(filepath.Join(configDir, cmd.DiscoveryConfigFilename), cmd.ConfigEnvPrefixMap[cmd.DiscoveryConfigFilename], discovery) + err = config.Load(configDir, cmd.DiscoveryConfigFilename, cmd.ConfigEnvPrefixMap[cmd.DiscoveryConfigFilename], runtimeEnv, discovery) if err != nil { return nil, nil, nil, nil, nil, err } From 54be83776555c57c600a8adeee6b81e54dea6975 Mon Sep 17 00:00:00 2001 From: chao <48119764+withchao@users.noreply.github.com> Date: Mon, 9 Dec 2024 17:57:20 +0800 Subject: [PATCH 066/199] fix: compilation failed under Windows (#2940) * fix: kodo sdk windows build error * fix: kodo sdk windows build error --- go.mod | 17 ++++------------- go.sum | 54 ++++++++++++++++-------------------------------------- 2 files changed, 20 insertions(+), 51 deletions(-) diff --git a/go.mod b/go.mod index b278f2a9b..53392f739 100644 --- a/go.mod +++ b/go.mod @@ -15,7 +15,7 @@ require ( github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/mitchellh/mapstructure v1.5.0 github.com/openimsdk/protocol v0.0.72-alpha.61 - github.com/openimsdk/tools v0.0.50-alpha.45 + github.com/openimsdk/tools v0.0.50-alpha.46 github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 github.com/stretchr/testify v1.9.0 @@ -37,7 +37,7 @@ require ( github.com/hashicorp/golang-lru/v2 v2.0.7 github.com/kelindar/bitmap v1.5.2 github.com/likexian/gokit v0.25.13 - github.com/openimsdk/gomake v0.0.15-alpha.1 + github.com/openimsdk/gomake v0.0.15-alpha.2 github.com/redis/go-redis/v9 v9.4.0 github.com/robfig/cron/v3 v3.0.1 github.com/shirou/gopsutil v3.21.11+incompatible @@ -57,11 +57,9 @@ require ( cloud.google.com/go/iam v1.1.7 // indirect cloud.google.com/go/longrunning v0.5.5 // indirect cloud.google.com/go/storage v1.40.0 // indirect - github.com/BurntSushi/toml v1.3.2 // indirect github.com/MicahParks/keyfunc v1.9.0 // indirect - github.com/alex-ant/gomath v0.0.0-20160516115720-89013a210a82 // indirect github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible // indirect - github.com/aws/aws-sdk-go-v2 v1.32.6 // indirect + github.com/aws/aws-sdk-go-v2 v1.32.5 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.1 // indirect github.com/aws/aws-sdk-go-v2/config v1.28.5 // indirect github.com/aws/aws-sdk-go-v2/credentials v1.17.46 // indirect @@ -94,14 +92,11 @@ require ( github.com/eapache/go-resiliency v1.6.0 // indirect github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 // indirect github.com/eapache/queue v1.1.0 // indirect - github.com/elastic/go-sysinfo v1.0.2 // indirect - github.com/elastic/go-windows v1.0.0 // indirect github.com/emicklei/go-restful/v3 v3.11.0 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/fxamacker/cbor/v2 v2.7.0 // indirect github.com/gabriel-vasile/mimetype v1.4.3 // indirect - github.com/gammazero/toposort v0.1.1 // indirect github.com/gin-contrib/sse v0.1.0 // indirect github.com/go-logr/logr v1.4.2 // indirect github.com/go-logr/stdr v1.2.2 // indirect @@ -111,7 +106,6 @@ require ( github.com/go-openapi/swag v0.22.4 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect github.com/go-zookeeper/zk v1.0.3 // indirect - github.com/gofrs/flock v0.8.1 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/golang/snappy v0.0.4 // indirect @@ -135,7 +129,6 @@ require ( github.com/jinzhu/copier v0.4.0 // indirect github.com/jinzhu/inflection v1.0.0 // indirect github.com/jinzhu/now v1.1.5 // indirect - github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/kelindar/simd v1.1.2 // indirect @@ -163,7 +156,7 @@ require ( github.com/prometheus/client_model v0.5.0 // indirect github.com/prometheus/common v0.45.0 // indirect github.com/prometheus/procfs v0.12.0 // indirect - github.com/qiniu/go-sdk/v7 v7.25.0 // indirect + github.com/qiniu/go-sdk/v7 v7.18.2 // indirect github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect github.com/rs/xid v1.5.0 // indirect github.com/sagikazarmark/locafero v0.4.0 // indirect @@ -210,11 +203,9 @@ require ( gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gorm.io/gorm v1.25.8 // indirect - howett.net/plist v0.0.0-20181124034731-591f970eefbb // indirect k8s.io/klog/v2 v2.130.1 // indirect k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 // indirect k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 // indirect - modernc.org/fileutil v1.0.0 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect sigs.k8s.io/yaml v1.4.0 // indirect diff --git a/go.sum b/go.sum index 4657f0eca..9bfbb38fb 100644 --- a/go.sum +++ b/go.sum @@ -14,19 +14,15 @@ cloud.google.com/go/storage v1.40.0/go.mod h1:Rrj7/hKlG87BLqDJYtwR0fbPld8uJPbQ2u firebase.google.com/go/v4 v4.14.1 h1:4qiUETaFRWoFGE1XP5VbcEdtPX93Qs+8B/7KvP2825g= firebase.google.com/go/v4 v4.14.1/go.mod h1:fgk2XshgNDEKaioKco+AouiegSI9oTWVqRaBdTTGBoM= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/BurntSushi/toml v1.3.2 h1:o7IhLm0Msx3BaB+n3Ag7L8EVlByGnpq14C4YWiu/gL8= -github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/IBM/sarama v1.43.0 h1:YFFDn8mMI2QL0wOrG0J2sFoVIAFl7hS9JQi2YZsXtJc= github.com/IBM/sarama v1.43.0/go.mod h1:zlE6HEbC/SMQ9mhEYaF7nNLYOUyrs0obySKCckWP9BM= github.com/MicahParks/keyfunc v1.9.0 h1:lhKd5xrFHLNOWrDc4Tyb/Q1AJ4LCzQ48GVJyVIID3+o= github.com/MicahParks/keyfunc v1.9.0/go.mod h1:IdnCilugA0O/99dW+/MkvlyrsX8+L8+x95xuVNtM5jw= github.com/QcloudApi/qcloud_sign_golang v0.0.0-20141224014652-e4130a326409/go.mod h1:1pk82RBxDY/JZnPQrtqHlUFfCctgdorsd9M06fMynOM= -github.com/alex-ant/gomath v0.0.0-20160516115720-89013a210a82 h1:7dONQ3WNZ1zy960TmkxJPuwoolZwL7xKtpcM04MBnt4= -github.com/alex-ant/gomath v0.0.0-20160516115720-89013a210a82/go.mod h1:nLnM0KdK1CmygvjpDUO6m1TjSsiQtL61juhNsvV/JVI= github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible h1:8psS8a+wKfiLt1iVDX79F7Y6wUM49Lcha2FMXt4UM8g= github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8= -github.com/aws/aws-sdk-go-v2 v1.32.6 h1:7BokKRgRPuGmKkFMhEg/jSul+tB9VvXhcViILtfG8b4= -github.com/aws/aws-sdk-go-v2 v1.32.6/go.mod h1:P5WJBrYqqbWVaOxgH0X/FYYD47/nooaPOZPlQdmiN2U= +github.com/aws/aws-sdk-go-v2 v1.32.5 h1:U8vdWJuY7ruAkzaOdD7guwJjD06YSKmnKCJs7s3IkIo= +github.com/aws/aws-sdk-go-v2 v1.32.5/go.mod h1:P5WJBrYqqbWVaOxgH0X/FYYD47/nooaPOZPlQdmiN2U= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.1 h1:ZY3108YtBNq96jNZTICHxN1gSBSbnvIdYwwqnvCV4Mc= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.1/go.mod h1:t8PYl/6LzdAqsU4/9tz28V/kU+asFePvpOMkdul0gEQ= github.com/aws/aws-sdk-go-v2/config v1.28.5 h1:Za41twdCXbuyyWv9LndXxZZv3QhTG1DinqlFsSuvtI0= @@ -90,7 +86,6 @@ github.com/coreos/go-systemd/v22 v22.3.2 h1:D9/bQk5vlXQFZ6Kwuu6zaiXJ9oTPe68++AzA github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/dave/jennifer v1.6.1/go.mod h1:nXbxhEmQfOZhWml3D1cDK5M1FLnMSozpbFN/m3RmGZc= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= @@ -108,10 +103,6 @@ github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 h1:Oy0F4A github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3/go.mod h1:YvSRo5mw33fLEx1+DlK6L2VV43tJt5Eyel9n9XBcR+0= github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc= github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= -github.com/elastic/go-sysinfo v1.0.2 h1:Wq1bOgnSz7Obl7DbMjbn0tzx1bE5G8Cfy3MVFa6C1Cc= -github.com/elastic/go-sysinfo v1.0.2/go.mod h1:O/D5m1VpYLwGjCYzEt63g3Z1uO3jXfwyzzjiW90t8cY= -github.com/elastic/go-windows v1.0.0 h1:qLURgZFkkrYyTTkvYpsZIgf83AUsdIHfvlJaqaZ7aSY= -github.com/elastic/go-windows v1.0.0/go.mod h1:TsU0Nrp7/y3+VwE82FoZF8gC/XFg/Elz6CcloAxnPgU= github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g= github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= @@ -132,8 +123,6 @@ github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ= github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0= github.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk= -github.com/gammazero/toposort v0.1.1 h1:OivGxsWxF3U3+U80VoLJ+f50HcPU1MIqE1JlKzoJ2Eg= -github.com/gammazero/toposort v0.1.1/go.mod h1:H2cozTnNpMw0hg2VHAYsAxmkHXBYroNangj2NTBQDvw= github.com/gin-contrib/gzip v1.0.1 h1:HQ8ENHODeLY7a4g1Au/46Z92bdGFl74OhxcZble9WJE= github.com/gin-contrib/gzip v1.0.1/go.mod h1:njt428fdUNRvjuJf16tZMYZ2Yl+WQB53X5wmhDwXvC4= github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= @@ -165,7 +154,7 @@ github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+ github.com/go-playground/universal-translator v0.18.0/go.mod h1:UvRDBj+xPUEGrFYl+lu/H90nyDXpg0fqeB/AQUGNTVA= github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= -github.com/go-playground/validator/v10 v10.7.0/go.mod h1:xm76BBt941f7yWdGnI2DVPFFg1UK3YY04qifoXU3lOk= +github.com/go-playground/validator/v10 v10.8.0/go.mod h1:9JhgTzTaE31GZDpH/HSvHiRJrJ3iKAgqqH0Bl/Ocjdk= github.com/go-playground/validator/v10 v10.20.0 h1:K9ISHbSaI0lyB2eWMPJo+kOS/FBExVwjEviJTixqxL8= github.com/go-playground/validator/v10 v10.20.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM= github.com/go-redis/redis v6.15.9+incompatible h1:K0pv1D7EQUjfyoMql+r/jZqCLizCGKFlFgcHWWmHQjg= @@ -179,8 +168,6 @@ github.com/go-zookeeper/zk v1.0.3/go.mod h1:nOB03cncLtlp4t+UAkGSV+9beXP/akpekBwL github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= -github.com/gofrs/flock v0.8.1 h1:+gYjHKf32LDeiEEFhQaotPbLuUXjY5ZqxKgXy7n59aw= -github.com/gofrs/flock v0.8.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang-jwt/jwt/v4 v4.4.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= @@ -237,8 +224,6 @@ github.com/googleapis/enterprise-certificate-proxy v0.3.2 h1:Vie5ybvEvT75RniqhfF github.com/googleapis/enterprise-certificate-proxy v0.3.2/go.mod h1:VLSiSSBs/ksPL8kq3OBOQ6WRI2QnaFynd1DCjZ62+V0= github.com/googleapis/gax-go/v2 v2.12.3 h1:5/zPPDvw8Q1SuXjrqrZslrqT7dL/uJT2CQii/cLCKqA= github.com/googleapis/gax-go/v2 v2.12.3/go.mod h1:AKloxT6GtNbaLm8QTNSidHUVsHYcBHwWRvkNFJUQcS4= -github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY= -github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ= github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4= github.com/gorilla/sessions v1.2.1/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM= github.com/gorilla/websocket v1.5.1 h1:gmztn0JnHVt9JZquRuzLw3g4wouNVzKL15iLr/zn/QY= @@ -257,7 +242,6 @@ github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= -github.com/iancoleman/strcase v0.3.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/jcmturner/aescts/v2 v2.0.0 h1:9YKLH6ey7H4eDBXW8khjYslgyqG2xZikXP0EQFKrle8= @@ -272,15 +256,12 @@ github.com/jcmturner/gokrb5/v8 v8.4.4 h1:x1Sv4HaTpepFkXbt2IkL29DXRf8sOfZXo8eRKh6 github.com/jcmturner/gokrb5/v8 v8.4.4/go.mod h1:1btQEpgT6k+unzCwX1KdWMEwPPkkgBtP+F6aCACiMrs= github.com/jcmturner/rpc/v2 v2.0.3 h1:7FXXj8Ti1IaVFpSAziCZWNzbNuZmnvw/i6CqLNdWfZY= github.com/jcmturner/rpc/v2 v2.0.3/go.mod h1:VUJYCIDm3PVOEHw8sgt091/20OJjskO/YJki3ELg/Hc= -github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jinzhu/copier v0.4.0 h1:w3ciUoD19shMCRargcpm0cm91ytaBhDvuRpz1ODO/U8= github.com/jinzhu/copier v0.4.0/go.mod h1:DfbEm0FYsaqBcKcFuvmOZb218JkPGtvSHsKg8S8hyyg= github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= -github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901 h1:rp+c0RAYOWj8l6qbCUTSiRLG/iKnW3K3/QfPPuSsBt4= -github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901/go.mod h1:Z86h9688Y0wesXCyonoVr47MasHilkuLMqGhRZ4Hpak= github.com/jonboulle/clockwork v0.4.0 h1:p4Cf1aMWXnXAUh8lVfewRBx1zaTSYKrKMF2g3ST4RZ4= github.com/jonboulle/clockwork v0.4.0/go.mod h1:xgRqUGwRcjKCO1vbZUEtSLrqKoPSsUpK7fnezOII0kc= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= @@ -309,7 +290,6 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY= github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= @@ -365,18 +345,17 @@ github.com/onsi/ginkgo/v2 v2.19.0 h1:9Cnnf7UHo57Hy3k6/m5k3dRfGTMXGvxhHFvkDTCTpvA github.com/onsi/ginkgo/v2 v2.19.0/go.mod h1:rlwLi9PilAFJ8jCg9UE1QP6VBpd6/xj3SRC0d6TU0To= github.com/onsi/gomega v1.25.0 h1:Vw7br2PCDYijJHSfBOWhov+8cAnUf8MfMaIOV323l6Y= github.com/onsi/gomega v1.25.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdMPGhM= -github.com/openimsdk/gomake v0.0.15-alpha.1 h1:a3U0gayzEJJucOoIOt9Jju+HQTG2BtKcuLG4AVjeVF4= -github.com/openimsdk/gomake v0.0.15-alpha.1/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= +github.com/openimsdk/gomake v0.0.15-alpha.2 h1:5Q8yl8ezy2yx+q8/ucU/t4kJnDfCzNOrkXcDACCqtyM= +github.com/openimsdk/gomake v0.0.15-alpha.2/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= github.com/openimsdk/protocol v0.0.72-alpha.61 h1:RuZR9/Sg3p6Bpb2CKPjPoA2AUmTvHITmhZ3PT/RbWMs= github.com/openimsdk/protocol v0.0.72-alpha.61/go.mod h1:Iet+piS/jaS+kWWyj6EEr36mk4ISzIRYjoMSVA4dq2M= -github.com/openimsdk/tools v0.0.50-alpha.45 h1:bu3HCkdzLNQAPUR1VhF40XA9OhaNGHGxZqcC4rIYJyY= -github.com/openimsdk/tools v0.0.50-alpha.45/go.mod h1:YOtTcSQ7Ik6NYkjXCTXZT3YDM7smGp9D2xvti3KvNoM= +github.com/openimsdk/tools v0.0.50-alpha.46 h1:j3HxPxhDptVHwr7eChL2rCH8mKfpUEcr4nHi5k4yDME= +github.com/openimsdk/tools v0.0.50-alpha.46/go.mod h1:muCtxguNJv8lFwLei27UASu2Nvg4ERSeN0R4K5tivk0= github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= -github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= @@ -391,12 +370,11 @@ github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cY github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI= github.com/prometheus/common v0.45.0 h1:2BGz0eBc2hdMDLnO/8n0jeB3oPrt2D08CekT0lneoxM= github.com/prometheus/common v0.45.0/go.mod h1:YJmSTw9BoKxJplESWWxlbyttQR4uaEcGyv9MZjVOJsY= -github.com/prometheus/procfs v0.0.0-20190425082905-87a4384529e0/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= github.com/qiniu/dyn v1.3.0/go.mod h1:E8oERcm8TtwJiZvkQPbcAh0RL8jO1G0VXJMW3FAWdkk= -github.com/qiniu/go-sdk/v7 v7.25.0 h1:Roi4XMxRly9K4wb87DhQOKaQylyiphEXC7/l8uqJZaQ= -github.com/qiniu/go-sdk/v7 v7.25.0/go.mod h1:uZE85Pi0ftIHT/UNLShosdzwsovqpdas0LwAGO7cPao= +github.com/qiniu/go-sdk/v7 v7.18.2 h1:vk9eo5OO7aqgAOPF0Ytik/gt7CMKuNgzC/IPkhda6rk= +github.com/qiniu/go-sdk/v7 v7.18.2/go.mod h1:nqoYCNo53ZlGA521RvRethvxUDvXKt4gtYXOwye868w= github.com/qiniu/x v1.10.5/go.mod h1:03Ni9tj+N2h2aKnAz+6N0Xfl8FwMEDRC2PAlxekASDs= github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:N/ElC8H3+5XpJzTSTfLsJV/mx9Q9g7kxmchpfZyxgzM= github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= @@ -511,7 +489,9 @@ golang.org/x/arch v0.7.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= golang.org/x/crypto v0.27.0 h1:GXm2NjJrPaiv/h1tb2UH8QfgC/hOf/+z0p6PT8o1w7A= golang.org/x/crypto v0.27.0/go.mod h1:1Xngt8kV6Dvbssa53Ziq6Eqn0HqbZi5Z6R0ZpwQzt70= @@ -539,6 +519,7 @@ golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwY golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20220708220712-1185a9018129/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.29.0 h1:5ORfpBpCs4HzDYoodCDBbwHzdR5UrLBZ3sOnUJmFoHo= @@ -548,7 +529,6 @@ golang.org/x/oauth2 v0.23.0 h1:PbgcYx2W7i4LvjJWEbf0ngHV6qJYr86PkAV3bXdLEbs= golang.org/x/oauth2 v0.23.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -558,20 +538,22 @@ golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190425145619-16072639606e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34= golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.24.0 h1:Mh5cbb+Zk2hqqXNO7S1iTjEphVL+jb8ZWaqh/g+JWkM= golang.org/x/term v0.24.0/go.mod h1:lOBK/LVxemqiMij05LGJ0tzNr8xlmwBRJ81PX6wVLH8= @@ -581,6 +563,7 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= +golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224= golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= @@ -647,7 +630,6 @@ gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= -gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= @@ -660,8 +642,6 @@ gorm.io/gorm v1.25.8 h1:WAGEZ/aEcznN4D03laj8DKnehe1e9gYQAjW8xyPRdeo= gorm.io/gorm v1.25.8/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -howett.net/plist v0.0.0-20181124034731-591f970eefbb h1:jhnBjNi9UFpfpl8YZhA9CrOqpnJdvzuiHsl/dnxl11M= -howett.net/plist v0.0.0-20181124034731-591f970eefbb/go.mod h1:vMygbs4qMhSZSc4lCUl2OEE+rDiIIJAIdR4m7MiMcm0= k8s.io/api v0.31.2 h1:3wLBbL5Uom/8Zy98GRPXpJ254nEFpl+hwndmk9RwmL0= k8s.io/api v0.31.2/go.mod h1:bWmGvrGPssSK1ljmLzd3pwCQ9MgoTsRCuK35u6SygUk= k8s.io/apimachinery v0.31.2 h1:i4vUt2hPK56W6mlT7Ry+AO8eEsyxMD1U44NR22CLTYw= @@ -674,8 +654,6 @@ k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 h1:BZqlfIlq5YbRMFko6/PM7F k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340/go.mod h1:yD4MZYeKMBwQKVht279WycxKyM84kkAx2DPrTXaeb98= k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 h1:pUdcCO1Lk/tbT5ztQWOBi5HBgbBP1J8+AsQnQCKsi8A= k8s.io/utils v0.0.0-20240711033017-18e509b52bc8/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= -modernc.org/fileutil v1.0.0 h1:Z1AFLZwl6BO8A5NldQg/xTSjGLetp+1Ubvl4alfGx8w= -modernc.org/fileutil v1.0.0/go.mod h1:JHsWpkrk/CnVV1H/eGlFf85BEpfkrp56ro8nojIq9Q8= nullprogram.com/x/optparse v1.0.0/go.mod h1:KdyPE+Igbe0jQUrVfMqDMeJQIJZEuyV7pjYmp6pbG50= rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= From 6a7ae690ee00673cf0e5b9cd0254d2714a0969d8 Mon Sep 17 00:00:00 2001 From: icey-yu <119291641+icey-yu@users.noreply.github.com> Date: Tue, 10 Dec 2024 11:43:19 +0800 Subject: [PATCH 067/199] feat: Prometheus can auto set port (#2943) * feat: config * feat: prometheus auto set port --- config/openim-api.yml | 3 + config/openim-msggateway.yml | 1 + config/openim-msgtransfer.yml | 4 +- config/openim-push.yml | 1 + config/openim-rpc-auth.yml | 1 + config/openim-rpc-conversation.yml | 1 + config/openim-rpc-friend.yml | 1 + config/openim-rpc-group.yml | 1 + config/openim-rpc-msg.yml | 1 + config/openim-rpc-third.yml | 1 + config/openim-rpc-user.yml | 1 + config/prometheus.yml | 127 +++++++++++------- docker-compose.yml | 92 ++++++------- go.mod | 4 +- go.sum | 4 +- internal/api/init.go | 65 +++++++-- internal/api/prometheus_discovery.go | 113 ++++++++++++++++ internal/api/router.go | 16 ++- internal/msggateway/init.go | 2 +- internal/msgtransfer/init.go | 69 ++++++++-- pkg/common/config/config.go | 16 ++- pkg/common/config/load_config_test.go | 8 ++ .../discoveryregister/discoveryregister.go | 6 +- pkg/common/prommetrics/api.go | 5 +- pkg/common/prommetrics/discovery.go | 31 +++++ pkg/common/prommetrics/prommetrics.go | 6 +- pkg/common/prommetrics/rpc.go | 5 +- pkg/common/prommetrics/transfer.go | 5 +- pkg/common/startrpc/start.go | 95 +++++++++---- 29 files changed, 521 insertions(+), 164 deletions(-) create mode 100644 internal/api/prometheus_discovery.go create mode 100644 pkg/common/prommetrics/discovery.go diff --git a/config/openim-api.yml b/config/openim-api.yml index 4c38e1005..a23b5fb31 100644 --- a/config/openim-api.yml +++ b/config/openim-api.yml @@ -10,7 +10,10 @@ api: prometheus: # Whether to enable prometheus enable: true + # autoSetPorts indicates whether to automatically set the ports + autoSetPorts: true # Prometheus listening ports, must match the number of api.ports + # It will only take effect when autoSetPorts is set to false. ports: [ 12002 ] # This address can be accessed via a browser grafanaURL: http://127.0.0.1:13000/ diff --git a/config/openim-msggateway.yml b/config/openim-msggateway.yml index 696bc3d06..b7d6d9847 100644 --- a/config/openim-msggateway.yml +++ b/config/openim-msggateway.yml @@ -12,6 +12,7 @@ prometheus: # Enable or disable Prometheus monitoring enable: true # List of ports that Prometheus listens on; these must match the number of rpc.ports to ensure correct monitoring setup + # It will only take effect when autoSetPorts is set to false. ports: [ 12140, 12141, 12142, 12143, 12144, 12145, 12146, 12147, 12148, 12149, 12150, 12151, 12152, 12153, 12154, 12155 ] # IP address that the RPC/WebSocket service listens on; setting to 0.0.0.0 listens on both internal and external IPs. If left blank, it automatically uses the internal network IP diff --git a/config/openim-msgtransfer.yml b/config/openim-msgtransfer.yml index 94ed073d8..39b23b222 100644 --- a/config/openim-msgtransfer.yml +++ b/config/openim-msgtransfer.yml @@ -1,6 +1,8 @@ prometheus: # Enable or disable Prometheus monitoring enable: true + # autoSetPorts indicates whether to automatically set the ports + autoSetPorts: true # List of ports that Prometheus listens on; each port corresponds to an instance of monitoring. Ensure these are managed accordingly - # Because four instances have been launched, four ports need to be specified + # It will only take effect when autoSetPorts is set to false. ports: [ 12020, 12021, 12022, 12023, 12024, 12025, 12026, 12027, 12028, 12029, 12030, 12031, 12032, 12033, 12034, 12035 ] diff --git a/config/openim-push.yml b/config/openim-push.yml index a3fe0c765..dcc8f1aec 100644 --- a/config/openim-push.yml +++ b/config/openim-push.yml @@ -15,6 +15,7 @@ prometheus: # Enable or disable Prometheus monitoring enable: true # List of ports that Prometheus listens on; these must match the number of rpc.ports to ensure correct monitoring setup + # It will only take effect when autoSetPorts is set to false. ports: [ 12170, 12171, 12172, 12173, 12174, 12175, 12176, 12177, 12178, 12179, 12180, 12182, 12183, 12184, 12185, 12186 ] maxConcurrentWorkers: 3 diff --git a/config/openim-rpc-auth.yml b/config/openim-rpc-auth.yml index 93f365386..961244861 100644 --- a/config/openim-rpc-auth.yml +++ b/config/openim-rpc-auth.yml @@ -15,6 +15,7 @@ prometheus: # Enable or disable Prometheus monitoring enable: true # List of ports that Prometheus listens on; these must match the number of rpc.ports to ensure correct monitoring setup + # It will only take effect when autoSetPorts is set to false. ports: [ 12200 ] tokenPolicy: diff --git a/config/openim-rpc-conversation.yml b/config/openim-rpc-conversation.yml index c17b5709f..eaedfe21f 100644 --- a/config/openim-rpc-conversation.yml +++ b/config/openim-rpc-conversation.yml @@ -14,4 +14,5 @@ prometheus: # Enable or disable Prometheus monitoring enable: true # List of ports that Prometheus listens on; these must match the number of rpc.ports to ensure correct monitoring setup + # It will only take effect when autoSetPorts is set to false. ports: [ 12220 ] diff --git a/config/openim-rpc-friend.yml b/config/openim-rpc-friend.yml index 5b51981d2..920c4860b 100644 --- a/config/openim-rpc-friend.yml +++ b/config/openim-rpc-friend.yml @@ -14,4 +14,5 @@ prometheus: # Enable or disable Prometheus monitoring enable: true # List of ports that Prometheus listens on; these must match the number of rpc.ports to ensure correct monitoring setup + # It will only take effect when autoSetPorts is set to false. ports: [ 12240 ] diff --git a/config/openim-rpc-group.yml b/config/openim-rpc-group.yml index 06fb489d2..c48065cca 100644 --- a/config/openim-rpc-group.yml +++ b/config/openim-rpc-group.yml @@ -14,6 +14,7 @@ prometheus: # Enable or disable Prometheus monitoring enable: true # List of ports that Prometheus listens on; these must match the number of rpc.ports to ensure correct monitoring setup + # It will only take effect when autoSetPorts is set to false. ports: [ 12260 ] diff --git a/config/openim-rpc-msg.yml b/config/openim-rpc-msg.yml index 5a4beae5c..d07854622 100644 --- a/config/openim-rpc-msg.yml +++ b/config/openim-rpc-msg.yml @@ -14,6 +14,7 @@ prometheus: # Enable or disable Prometheus monitoring enable: true # List of ports that Prometheus listens on; these must match the number of rpc.ports to ensure correct monitoring setup + # It will only take effect when autoSetPorts is set to false. ports: [ 12280 ] diff --git a/config/openim-rpc-third.yml b/config/openim-rpc-third.yml index e24f93883..95a50fac4 100644 --- a/config/openim-rpc-third.yml +++ b/config/openim-rpc-third.yml @@ -14,6 +14,7 @@ prometheus: # Enable or disable Prometheus monitoring enable: true # List of ports that Prometheus listens on; these must match the number of rpc.ports to ensure correct monitoring setup + # It will only take effect when autoSetPorts is set to false. ports: [ 12300 ] diff --git a/config/openim-rpc-user.yml b/config/openim-rpc-user.yml index 02b6d45b1..3a1335895 100644 --- a/config/openim-rpc-user.yml +++ b/config/openim-rpc-user.yml @@ -14,4 +14,5 @@ prometheus: # Whether to enable prometheus enable: true # Prometheus listening ports, must be consistent with the number of rpc.ports + # It will only take effect when autoSetPorts is set to false. ports: [ 12320 ] diff --git a/config/prometheus.yml b/config/prometheus.yml index ab427ee82..6fb112824 100644 --- a/config/prometheus.yml +++ b/config/prometheus.yml @@ -26,61 +26,94 @@ scrape_configs: - job_name: node_exporter static_configs: - targets: [ internal_ip:20500 ] + - job_name: openimserver-openim-api - static_configs: - - targets: [ internal_ip:12002 ] - labels: - namespace: default + http_sd_configs: + - url: "http://internal_ip:10002/prometheus_discovery/api" +# static_configs: +# - targets: [ internal_ip:12002 ] +# labels: +# namespace: default + - job_name: openimserver-openim-msggateway - static_configs: - - targets: [ internal_ip:12140 ] -# - targets: [ internal_ip:12140, internal_ip:12141, internal_ip:12142, internal_ip:12143, internal_ip:12144, internal_ip:12145, internal_ip:12146, internal_ip:12147, internal_ip:12148, internal_ip:12149, internal_ip:12150, internal_ip:12151, internal_ip:12152, internal_ip:12153, internal_ip:12154, internal_ip:12155 ] - labels: - namespace: default + http_sd_configs: + - url: "http://internal_ip:10002/prometheus_discovery/msg_gateway" +# static_configs: +# - targets: [ internal_ip:12140 ] +# # - targets: [ internal_ip:12140, internal_ip:12141, internal_ip:12142, internal_ip:12143, internal_ip:12144, internal_ip:12145, internal_ip:12146, internal_ip:12147, internal_ip:12148, internal_ip:12149, internal_ip:12150, internal_ip:12151, internal_ip:12152, internal_ip:12153, internal_ip:12154, internal_ip:12155 ] +# labels: +# namespace: default + - job_name: openimserver-openim-msgtransfer - static_configs: - - targets: [ internal_ip:12020, internal_ip:12021, internal_ip:12022, internal_ip:12023, internal_ip:12024, internal_ip:12025, internal_ip:12026, internal_ip:12027 ] -# - targets: [ internal_ip:12020, internal_ip:12021, internal_ip:12022, internal_ip:12023, internal_ip:12024, internal_ip:12025, internal_ip:12026, internal_ip:12027, internal_ip:12028, internal_ip:12029, internal_ip:12030, internal_ip:12031, internal_ip:12032, internal_ip:12033, internal_ip:12034, internal_ip:12035 ] - labels: - namespace: default + http_sd_configs: + - url: "http://internal_ip:10002/prometheus_discovery/msg_transfer" +# static_configs: +# - targets: [ internal_ip:12020, internal_ip:12021, internal_ip:12022, internal_ip:12023, internal_ip:12024, internal_ip:12025, internal_ip:12026, internal_ip:12027 ] +# # - targets: [ internal_ip:12020, internal_ip:12021, internal_ip:12022, internal_ip:12023, internal_ip:12024, internal_ip:12025, internal_ip:12026, internal_ip:12027, internal_ip:12028, internal_ip:12029, internal_ip:12030, internal_ip:12031, internal_ip:12032, internal_ip:12033, internal_ip:12034, internal_ip:12035 ] +# labels: +# namespace: default + - job_name: openimserver-openim-push - static_configs: - - targets: [ internal_ip:12170, internal_ip:12171, internal_ip:12172, internal_ip:12173, internal_ip:12174, internal_ip:12175, internal_ip:12176, internal_ip:12177 ] -# - targets: [ internal_ip:12170, internal_ip:12171, internal_ip:12172, internal_ip:12173, internal_ip:12174, internal_ip:12175, internal_ip:12176, internal_ip:12177, internal_ip:12178, internal_ip:12179, internal_ip:12180, internal_ip:12182, internal_ip:12183, internal_ip:12184, internal_ip:12185, internal_ip:12186 ] - labels: - namespace: default + http_sd_configs: + - url: "http://internal_ip:10002/prometheus_discovery/push" +# static_configs: +# - targets: [ internal_ip:12170, internal_ip:12171, internal_ip:12172, internal_ip:12173, internal_ip:12174, internal_ip:12175, internal_ip:12176, internal_ip:12177 ] +## - targets: [ internal_ip:12170, internal_ip:12171, internal_ip:12172, internal_ip:12173, internal_ip:12174, internal_ip:12175, internal_ip:12176, internal_ip:12177, internal_ip:12178, internal_ip:12179, internal_ip:12180, internal_ip:12182, internal_ip:12183, internal_ip:12184, internal_ip:12185, internal_ip:12186 ] +# labels: +# namespace: default + - job_name: openimserver-openim-rpc-auth - static_configs: - - targets: [ internal_ip:12200 ] - labels: - namespace: default + http_sd_configs: + - url: "http://internal_ip:10002/prometheus_discovery/auth" +# static_configs: +# - targets: [ internal_ip:12200 ] +# labels: +# namespace: default + - job_name: openimserver-openim-rpc-conversation - static_configs: - - targets: [ internal_ip:12220 ] - labels: - namespace: default + http_sd_configs: + - url: "http://internal_ip:10002/prometheus_discovery/conversation" +# static_configs: +# - targets: [ internal_ip:12220 ] +# labels: +# namespace: default + - job_name: openimserver-openim-rpc-friend - static_configs: - - targets: [ internal_ip:12240 ] - labels: - namespace: default + http_sd_configs: + - url: "http://internal_ip:10002/prometheus_discovery/friend" +# static_configs: +# - targets: [ internal_ip:12240 ] +# labels: +# namespace: default + - job_name: openimserver-openim-rpc-group - static_configs: - - targets: [ internal_ip:12260 ] - labels: - namespace: default + http_sd_configs: + - url: "http://internal_ip:10002/prometheus_discovery/group" +# static_configs: +# - targets: [ internal_ip:12260 ] +# labels: +# namespace: default. + - job_name: openimserver-openim-rpc-msg - static_configs: - - targets: [ internal_ip:12280 ] - labels: - namespace: default + http_sd_configs: + - url: "http://internal_ip:10002/prometheus_discovery/msg" +# static_configs: +# - targets: [ internal_ip:12280 ] +# labels: +# namespace: default + - job_name: openimserver-openim-rpc-third - static_configs: - - targets: [ internal_ip:12300 ] - labels: - namespace: default + http_sd_configs: + - url: "http://internal_ip:10002/prometheus_discovery/third" +# static_configs: +# - targets: [ internal_ip:12300 ] +# labels: +# namespace: default + - job_name: openimserver-openim-rpc-user - static_configs: - - targets: [ internal_ip:12320 ] - labels: - namespace: default \ No newline at end of file + http_sd_configs: + - url: "http://internal_ip:10002/prometheus_discovery/user" +# static_configs: +# - targets: [ internal_ip:12320 ] +# labels: +# namespace: default \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 8d25383bc..57e654208 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -146,49 +146,49 @@ services: networks: - openim -# prometheus: -# image: ${PROMETHEUS_IMAGE} -# container_name: prometheus -# restart: always -# user: root -# volumes: -# - ./config/prometheus.yml:/etc/prometheus/prometheus.yml -# - ./config/instance-down-rules.yml:/etc/prometheus/instance-down-rules.yml -# - ${DATA_DIR}/components/prometheus/data:/prometheus -# command: -# - '--config.file=/etc/prometheus/prometheus.yml' -# - '--storage.tsdb.path=/prometheus' -# ports: -# - "19091:9090" -# networks: -# - openim -# -# alertmanager: -# image: ${ALERTMANAGER_IMAGE} -# container_name: alertmanager -# restart: always -# volumes: -# - ./config/alertmanager.yml:/etc/alertmanager/alertmanager.yml -# - ./config/email.tmpl:/etc/alertmanager/email.tmpl -# ports: -# - "19093:9093" -# networks: -# - openim -# -# grafana: -# image: ${GRAFANA_IMAGE} -# container_name: grafana -# user: root -# restart: always -# environment: -# - GF_SECURITY_ALLOW_EMBEDDING=true -# - GF_SESSION_COOKIE_SAMESITE=none -# - GF_SESSION_COOKIE_SECURE=true -# - GF_AUTH_ANONYMOUS_ENABLED=true -# - GF_AUTH_ANONYMOUS_ORG_ROLE=Admin -# ports: -# - "13000:3000" -# volumes: -# - ${DATA_DIR:-./}/components/grafana:/var/lib/grafana -# networks: -# - openim + prometheus: + image: ${PROMETHEUS_IMAGE} + container_name: prometheus + restart: always + user: root + volumes: + - ./config/prometheus.yml:/etc/prometheus/prometheus.yml + - ./config/instance-down-rules.yml:/etc/prometheus/instance-down-rules.yml + - ${DATA_DIR}/components/prometheus/data:/prometheus + command: + - '--config.file=/etc/prometheus/prometheus.yml' + - '--storage.tsdb.path=/prometheus' + ports: + - "19091:9090" + networks: + - openim + + alertmanager: + image: ${ALERTMANAGER_IMAGE} + container_name: alertmanager + restart: always + volumes: + - ./config/alertmanager.yml:/etc/alertmanager/alertmanager.yml + - ./config/email.tmpl:/etc/alertmanager/email.tmpl + ports: + - "19093:9093" + networks: + - openim + + grafana: + image: ${GRAFANA_IMAGE} + container_name: grafana + user: root + restart: always + environment: + - GF_SECURITY_ALLOW_EMBEDDING=true + - GF_SESSION_COOKIE_SAMESITE=none + - GF_SESSION_COOKIE_SECURE=true + - GF_AUTH_ANONYMOUS_ENABLED=true + - GF_AUTH_ANONYMOUS_ORG_ROLE=Admin + ports: + - "13000:3000" + volumes: + - ${DATA_DIR:-./}/components/grafana:/var/lib/grafana + networks: + - openim diff --git a/go.mod b/go.mod index 53392f739..5d81e67d3 100644 --- a/go.mod +++ b/go.mod @@ -15,7 +15,7 @@ require ( github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/mitchellh/mapstructure v1.5.0 github.com/openimsdk/protocol v0.0.72-alpha.61 - github.com/openimsdk/tools v0.0.50-alpha.46 + github.com/openimsdk/tools v0.0.50-alpha.47 github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 github.com/stretchr/testify v1.9.0 @@ -42,6 +42,7 @@ require ( github.com/robfig/cron/v3 v3.0.1 github.com/shirou/gopsutil v3.21.11+incompatible github.com/spf13/viper v1.18.2 + go.etcd.io/etcd/client/v3 v3.5.13 go.uber.org/automaxprocs v1.5.3 golang.org/x/exp v0.0.0-20230905200255-921286631fa9 golang.org/x/sync v0.8.0 @@ -179,7 +180,6 @@ require ( github.com/yusufpapurcu/wmi v1.2.4 // indirect go.etcd.io/etcd/api/v3 v3.5.13 // indirect go.etcd.io/etcd/client/pkg/v3 v3.5.13 // indirect - go.etcd.io/etcd/client/v3 v3.5.13 // indirect go.opencensus.io v0.24.0 // indirect go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect diff --git a/go.sum b/go.sum index 9bfbb38fb..99f569a38 100644 --- a/go.sum +++ b/go.sum @@ -349,8 +349,8 @@ github.com/openimsdk/gomake v0.0.15-alpha.2 h1:5Q8yl8ezy2yx+q8/ucU/t4kJnDfCzNOrk github.com/openimsdk/gomake v0.0.15-alpha.2/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= github.com/openimsdk/protocol v0.0.72-alpha.61 h1:RuZR9/Sg3p6Bpb2CKPjPoA2AUmTvHITmhZ3PT/RbWMs= github.com/openimsdk/protocol v0.0.72-alpha.61/go.mod h1:Iet+piS/jaS+kWWyj6EEr36mk4ISzIRYjoMSVA4dq2M= -github.com/openimsdk/tools v0.0.50-alpha.46 h1:j3HxPxhDptVHwr7eChL2rCH8mKfpUEcr4nHi5k4yDME= -github.com/openimsdk/tools v0.0.50-alpha.46/go.mod h1:muCtxguNJv8lFwLei27UASu2Nvg4ERSeN0R4K5tivk0= +github.com/openimsdk/tools v0.0.50-alpha.47 h1:Cfe2va/g6WhLjOoQqZkjrdlEDq1dUsfcQsdUB5oADVA= +github.com/openimsdk/tools v0.0.50-alpha.47/go.mod h1:muCtxguNJv8lFwLei27UASu2Nvg4ERSeN0R4K5tivk0= github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= diff --git a/internal/api/init.go b/internal/api/init.go index ea7b6133f..8fb6baff5 100644 --- a/internal/api/init.go +++ b/internal/api/init.go @@ -16,6 +16,7 @@ package api import ( "context" + "errors" "fmt" "net" "net/http" @@ -26,16 +27,17 @@ import ( "time" "github.com/openimsdk/open-im-server/v3/pkg/common/config" - "github.com/openimsdk/tools/utils/datautil" - "github.com/openimsdk/tools/utils/network" - "github.com/openimsdk/tools/utils/runtimeenv" - kdisc "github.com/openimsdk/open-im-server/v3/pkg/common/discoveryregister" "github.com/openimsdk/open-im-server/v3/pkg/common/prommetrics" "github.com/openimsdk/tools/discovery" + "github.com/openimsdk/tools/discovery/etcd" "github.com/openimsdk/tools/errs" "github.com/openimsdk/tools/log" "github.com/openimsdk/tools/system/program" + "github.com/openimsdk/tools/utils/datautil" + "github.com/openimsdk/tools/utils/jsonutil" + "github.com/openimsdk/tools/utils/network" + "github.com/openimsdk/tools/utils/runtimeenv" ) type Config struct { @@ -68,16 +70,57 @@ func Start(ctx context.Context, index int, config *Config) error { prometheusPort int ) - router := newGinRouter(client, config) + registerIP, err := network.GetRpcRegisterIP("") + if err != nil { + return err + } + + getAutoPort := func() (net.Listener, int, error) { + registerAddr := net.JoinHostPort(registerIP, "0") + listener, err := net.Listen("tcp", registerAddr) + if err != nil { + return nil, 0, errs.WrapMsg(err, "listen err", "registerAddr", registerAddr) + } + _, portStr, _ := net.SplitHostPort(listener.Addr().String()) + port, _ := strconv.Atoi(portStr) + return listener, port, nil + } + + if config.API.Prometheus.AutoSetPorts && config.Discovery.Enable != kdisc.Etcd { + return errs.New("only etcd support autoSetPorts", "RegisterName", "api").Wrap() + } + + router := newGinRouter(client, config, client) if config.API.Prometheus.Enable { - go func() { + var ( + listener net.Listener + ) + + if config.API.Prometheus.AutoSetPorts { + listener, prometheusPort, err = getAutoPort() + if err != nil { + return err + } + + etcdClient := client.(*etcd.SvcDiscoveryRegistryImpl).GetClient() + + _, err = etcdClient.Put(ctx, prommetrics.BuildDiscoveryKey(prommetrics.APIKeyName), jsonutil.StructToJsonString(prommetrics.BuildDefaultTarget(registerIP, prometheusPort))) + if err != nil { + return errs.WrapMsg(err, "etcd put err") + } + } else { prometheusPort, err = datautil.GetElemByIndex(config.API.Prometheus.Ports, index) if err != nil { - netErr = err - netDone <- struct{}{} - return + return err + } + listener, err = net.Listen("tcp", fmt.Sprintf(":%d", prometheusPort)) + if err != nil { + return errs.WrapMsg(err, "listen err", "addr", fmt.Sprintf(":%d", prometheusPort)) } - if err := prommetrics.ApiInit(prometheusPort); err != nil && err != http.ErrServerClosed { + } + + go func() { + if err := prommetrics.ApiInit(listener); err != nil && !errors.Is(err, http.ErrServerClosed) { netErr = errs.WrapMsg(err, fmt.Sprintf("api prometheus start err: %d", prometheusPort)) netDone <- struct{}{} } @@ -90,7 +133,7 @@ func Start(ctx context.Context, index int, config *Config) error { log.CInfo(ctx, "API server is initializing", "runtimeEnv", config.RuntimeEnv, "address", address, "apiPort", apiPort, "prometheusPort", prometheusPort) go func() { err = server.ListenAndServe() - if err != nil && err != http.ErrServerClosed { + if err != nil && !errors.Is(err, http.ErrServerClosed) { netErr = errs.WrapMsg(err, fmt.Sprintf("api start err: %s", server.Addr)) netDone <- struct{}{} diff --git a/internal/api/prometheus_discovery.go b/internal/api/prometheus_discovery.go new file mode 100644 index 000000000..6e33274be --- /dev/null +++ b/internal/api/prometheus_discovery.go @@ -0,0 +1,113 @@ +package api + +import ( + "encoding/json" + "github.com/gin-gonic/gin" + "github.com/openimsdk/open-im-server/v3/pkg/common/discoveryregister" + "github.com/openimsdk/open-im-server/v3/pkg/common/prommetrics" + "github.com/openimsdk/tools/apiresp" + "github.com/openimsdk/tools/discovery" + "github.com/openimsdk/tools/discovery/etcd" + "github.com/openimsdk/tools/errs" + "github.com/openimsdk/tools/log" + clientv3 "go.etcd.io/etcd/client/v3" + "net/http" +) + +type PrometheusDiscoveryApi struct { + config *Config + client *clientv3.Client +} + +func NewPrometheusDiscoveryApi(config *Config, client discovery.SvcDiscoveryRegistry) *PrometheusDiscoveryApi { + api := &PrometheusDiscoveryApi{ + config: config, + } + if config.Discovery.Enable == discoveryregister.Etcd { + api.client = client.(*etcd.SvcDiscoveryRegistryImpl).GetClient() + } + return api +} + +func (p *PrometheusDiscoveryApi) Enable(c *gin.Context) { + if p.config.Discovery.Enable != discoveryregister.Etcd { + c.JSON(http.StatusOK, []struct{}{}) + c.Abort() + } +} + +func (p *PrometheusDiscoveryApi) discovery(c *gin.Context, key string) { + eResp, err := p.client.Get(c, prommetrics.BuildDiscoveryKey(key)) + if err != nil { + // Log and respond with an error if preparation fails. + apiresp.GinError(c, errs.WrapMsg(err, "etcd get err")) + return + } + if len(eResp.Kvs) == 0 { + c.JSON(http.StatusOK, []*prommetrics.Target{}) + } + + var ( + resp = &prommetrics.RespTarget{ + Targets: make([]string, 0, len(eResp.Kvs)), + } + ) + + for i := range eResp.Kvs { + var target prommetrics.Target + err = json.Unmarshal(eResp.Kvs[i].Value, &target) + if err != nil { + log.ZError(c, "prometheus unmarshal err", errs.Wrap(err)) + } + resp.Targets = append(resp.Targets, target.Target) + if resp.Labels == nil { + resp.Labels = target.Labels + } + } + + c.JSON(200, []*prommetrics.RespTarget{resp}) +} + +func (p *PrometheusDiscoveryApi) Api(c *gin.Context) { + p.discovery(c, prommetrics.APIKeyName) +} + +func (p *PrometheusDiscoveryApi) User(c *gin.Context) { + p.discovery(c, p.config.Discovery.RpcService.User) +} + +func (p *PrometheusDiscoveryApi) Group(c *gin.Context) { + p.discovery(c, p.config.Discovery.RpcService.Group) +} + +func (p *PrometheusDiscoveryApi) Msg(c *gin.Context) { + p.discovery(c, p.config.Discovery.RpcService.Msg) +} + +func (p *PrometheusDiscoveryApi) Friend(c *gin.Context) { + p.discovery(c, p.config.Discovery.RpcService.Friend) +} + +func (p *PrometheusDiscoveryApi) Conversation(c *gin.Context) { + p.discovery(c, p.config.Discovery.RpcService.Conversation) +} + +func (p *PrometheusDiscoveryApi) Third(c *gin.Context) { + p.discovery(c, p.config.Discovery.RpcService.Third) +} + +func (p *PrometheusDiscoveryApi) Auth(c *gin.Context) { + p.discovery(c, p.config.Discovery.RpcService.Auth) +} + +func (p *PrometheusDiscoveryApi) Push(c *gin.Context) { + p.discovery(c, p.config.Discovery.RpcService.Push) +} + +func (p *PrometheusDiscoveryApi) MessageGateway(c *gin.Context) { + p.discovery(c, p.config.Discovery.RpcService.MessageGateway) +} + +func (p *PrometheusDiscoveryApi) MessageTransfer(c *gin.Context) { + p.discovery(c, prommetrics.MessageTransferKeyName) +} diff --git a/internal/api/router.go b/internal/api/router.go index 52d26bdc5..6714d645c 100644 --- a/internal/api/router.go +++ b/internal/api/router.go @@ -48,7 +48,7 @@ func prommetricsGin() gin.HandlerFunc { } } -func newGinRouter(disCov discovery.SvcDiscoveryRegistry, config *Config) *gin.Engine { +func newGinRouter(disCov discovery.SvcDiscoveryRegistry, config *Config, client discovery.SvcDiscoveryRegistry) *gin.Engine { disCov.AddOption(mw.GrpcClient(), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithDefaultServiceConfig(fmt.Sprintf(`{"LoadBalancingPolicy": "%s"}`, "round_robin"))) gin.SetMode(gin.ReleaseMode) @@ -78,6 +78,7 @@ func newGinRouter(disCov discovery.SvcDiscoveryRegistry, config *Config) *gin.En u := NewUserApi(*userRpc) m := NewMessageApi(messageRpc, userRpc, config.Share.IMAdminUserID) j := jssdk.NewJSSdkApi(userRpc.Client, friendRpc.Client, groupRpc.Client, messageRpc.Client, conversationRpc.Client) + pd := NewPrometheusDiscoveryApi(config, client) userRouterGroup := r.Group("/user") { userRouterGroup.POST("/user_register", u.UserRegister) @@ -254,6 +255,19 @@ func newGinRouter(disCov discovery.SvcDiscoveryRegistry, config *Config) *gin.En jssdk.POST("/get_conversations", j.GetConversations) jssdk.POST("/get_active_conversations", j.GetActiveConversations) + proDiscoveryGroup := r.Group("/prometheus_discovery", pd.Enable) + proDiscoveryGroup.GET("/api", pd.Api) + proDiscoveryGroup.GET("/user", pd.User) + proDiscoveryGroup.GET("/group", pd.Group) + proDiscoveryGroup.GET("/msg", pd.Msg) + proDiscoveryGroup.GET("/friend", pd.Friend) + proDiscoveryGroup.GET("/conversation", pd.Conversation) + proDiscoveryGroup.GET("/third", pd.Third) + proDiscoveryGroup.GET("/auth", pd.Auth) + proDiscoveryGroup.GET("/push", pd.Push) + proDiscoveryGroup.GET("/msg_gateway", pd.MessageGateway) + proDiscoveryGroup.GET("/msg_transfer", pd.MessageTransfer) + return r } diff --git a/internal/msggateway/init.go b/internal/msggateway/init.go index 7603147de..6654e6598 100644 --- a/internal/msggateway/init.go +++ b/internal/msggateway/init.go @@ -41,7 +41,7 @@ type Config struct { func Start(ctx context.Context, index int, conf *Config) error { conf.RuntimeEnv = runtimeenv.PrintRuntimeEnvironment() - log.CInfo(ctx, "MSG-GATEWAY server is initializing", "runtimeEnv", conf.RuntimeEnv, "autoSetPorts", conf.MsgGateway.RPC.AutoSetPorts, + log.CInfo(ctx, "MSG-GATEWAY server is initializing", "runtimeEnv", conf.RuntimeEnv, "rpcPorts", conf.MsgGateway.RPC.Ports, "wsPort", conf.MsgGateway.LongConnSvr.Ports, "prometheusPorts", conf.MsgGateway.Prometheus.Ports) wsPort, err := datautil.GetElemByIndex(conf.MsgGateway.LongConnSvr.Ports, index) diff --git a/internal/msgtransfer/init.go b/internal/msgtransfer/init.go index 6131d8c77..9da2e7974 100644 --- a/internal/msgtransfer/init.go +++ b/internal/msgtransfer/init.go @@ -18,11 +18,17 @@ import ( "context" "errors" "fmt" + "net" "net/http" "os" "os/signal" + "strconv" "syscall" + "github.com/openimsdk/tools/discovery/etcd" + "github.com/openimsdk/tools/utils/jsonutil" + "github.com/openimsdk/tools/utils/network" + "github.com/openimsdk/open-im-server/v3/pkg/common/prommetrics" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/redis" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database/mgo" @@ -33,6 +39,7 @@ import ( "github.com/openimsdk/open-im-server/v3/pkg/common/config" discRegister "github.com/openimsdk/open-im-server/v3/pkg/common/discoveryregister" + kdisc "github.com/openimsdk/open-im-server/v3/pkg/common/discoveryregister" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/controller" "github.com/openimsdk/open-im-server/v3/pkg/rpcclient" "github.com/openimsdk/tools/errs" @@ -140,21 +147,67 @@ func (m *MsgTransfer) Start(index int, config *Config) error { return err } + client, err := kdisc.NewDiscoveryRegister(&config.Discovery, m.runTimeEnv) + if err != nil { + return errs.WrapMsg(err, "failed to register discovery service") + } + + registerIP, err := network.GetRpcRegisterIP("") + if err != nil { + return err + } + + getAutoPort := func() (net.Listener, int, error) { + registerAddr := net.JoinHostPort(registerIP, "0") + listener, err := net.Listen("tcp", registerAddr) + if err != nil { + return nil, 0, errs.WrapMsg(err, "listen err", "registerAddr", registerAddr) + } + _, portStr, _ := net.SplitHostPort(listener.Addr().String()) + port, _ := strconv.Atoi(portStr) + return listener, port, nil + } + + if config.MsgTransfer.Prometheus.AutoSetPorts && config.Discovery.Enable != kdisc.Etcd { + return errs.New("only etcd support autoSetPorts", "RegisterName", "api").Wrap() + } + if config.MsgTransfer.Prometheus.Enable { + var ( + listener net.Listener + prometheusPort int + ) + + if config.MsgTransfer.Prometheus.AutoSetPorts { + listener, prometheusPort, err = getAutoPort() + if err != nil { + return err + } + + etcdClient := client.(*etcd.SvcDiscoveryRegistryImpl).GetClient() + + _, err = etcdClient.Put(context.TODO(), prommetrics.BuildDiscoveryKey(prommetrics.MessageTransferKeyName), jsonutil.StructToJsonString(prommetrics.BuildDefaultTarget(registerIP, prometheusPort))) + if err != nil { + return errs.WrapMsg(err, "etcd put err") + } + } else { + prometheusPort, err = datautil.GetElemByIndex(config.MsgTransfer.Prometheus.Ports, index) + if err != nil { + return err + } + listener, err = net.Listen("tcp", fmt.Sprintf(":%d", prometheusPort)) + if err != nil { + return errs.WrapMsg(err, "listen err", "addr", fmt.Sprintf(":%d", prometheusPort)) + } + } + go func() { defer func() { if r := recover(); r != nil { mw.PanicStackToLog(m.ctx, r) } }() - prometheusPort, err := datautil.GetElemByIndex(config.MsgTransfer.Prometheus.Ports, index) - if err != nil { - netErr = err - netDone <- struct{}{} - return - } - - if err := prommetrics.TransferInit(prometheusPort); err != nil && !errors.Is(err, http.ErrServerClosed) { + if err := prommetrics.TransferInit(listener); err != nil && !errors.Is(err, http.ErrServerClosed) { netErr = errs.WrapMsg(err, "prometheus start error", "prometheusPort", prometheusPort) netDone <- struct{}{} } diff --git a/pkg/common/config/config.go b/pkg/common/config/config.go index b8a60c3df..25748a618 100644 --- a/pkg/common/config/config.go +++ b/pkg/common/config/config.go @@ -18,11 +18,10 @@ import ( "strings" "time" - "github.com/openimsdk/tools/s3/aws" - "github.com/openimsdk/tools/db/mongoutil" "github.com/openimsdk/tools/db/redisutil" "github.com/openimsdk/tools/mq/kafka" + "github.com/openimsdk/tools/s3/aws" "github.com/openimsdk/tools/s3/cos" "github.com/openimsdk/tools/s3/kodo" "github.com/openimsdk/tools/s3/minio" @@ -108,9 +107,10 @@ type API struct { CompressionLevel int `mapstructure:"compressionLevel"` } `mapstructure:"api"` Prometheus struct { - Enable bool `mapstructure:"enable"` - Ports []int `mapstructure:"ports"` - GrafanaURL string `mapstructure:"grafanaURL"` + Enable bool `mapstructure:"enable"` + AutoSetPorts bool `mapstructure:"autoSetPorts"` + Ports []int `mapstructure:"ports"` + GrafanaURL string `mapstructure:"grafanaURL"` } `mapstructure:"prometheus"` } @@ -193,7 +193,11 @@ type MsgGateway struct { } type MsgTransfer struct { - Prometheus Prometheus `mapstructure:"prometheus"` + Prometheus struct { + Enable bool `mapstructure:"enable"` + AutoSetPorts bool `mapstructure:"autoSetPorts"` + Ports []int `mapstructure:"ports"` + } `mapstructure:"prometheus"` } type Push struct { diff --git a/pkg/common/config/load_config_test.go b/pkg/common/config/load_config_test.go index ea11b0574..1552c77d2 100644 --- a/pkg/common/config/load_config_test.go +++ b/pkg/common/config/load_config_test.go @@ -83,3 +83,11 @@ func TestLoadOpenIMThirdConfig(t *testing.T) { // Environment: IMENV_OPENIM_RPC_THIRD_OBJECT_ENABLE=enabled;IMENV_OPENIM_RPC_THIRD_OBJECT_OSS_ENDPOINT=https://oss-cn-chengdu.aliyuncs.com;IMENV_OPENIM_RPC_THIRD_OBJECT_OSS_BUCKET=my_bucket_name;IMENV_OPENIM_RPC_THIRD_OBJECT_OSS_BUCKETURL=https://my_bucket_name.oss-cn-chengdu.aliyuncs.com;IMENV_OPENIM_RPC_THIRD_OBJECT_OSS_ACCESSKEYID=AKID1234567890;IMENV_OPENIM_RPC_THIRD_OBJECT_OSS_ACCESSKEYSECRET=abc123xyz789;IMENV_OPENIM_RPC_THIRD_OBJECT_OSS_SESSIONTOKEN=session_token_value;IMENV_OPENIM_RPC_THIRD_OBJECT_OSS_PUBLICREAD=true } + +func TestTransferConfig(t *testing.T) { + var tran MsgTransfer + err := LoadConfig("../../../config/openim-msgtransfer.yml", "IMENV_OPENIM-MSGTRANSFER", &tran) + assert.Nil(t, err) + assert.Equal(t, true, tran.Prometheus.Enable) + assert.Equal(t, true, tran.Prometheus.AutoSetPorts) +} diff --git a/pkg/common/discoveryregister/discoveryregister.go b/pkg/common/discoveryregister/discoveryregister.go index 419293a91..dde629308 100644 --- a/pkg/common/discoveryregister/discoveryregister.go +++ b/pkg/common/discoveryregister/discoveryregister.go @@ -26,6 +26,10 @@ import ( "github.com/openimsdk/tools/errs" ) +const ( + Etcd = "etcd" +) + // NewDiscoveryRegister creates a new service discovery and registry client based on the provided environment type. func NewDiscoveryRegister(discovery *config.Discovery, runtimeEnv string) (discovery.SvcDiscoveryRegistry, error) { if runtimeEnv == "kubernetes" { @@ -35,7 +39,7 @@ func NewDiscoveryRegister(discovery *config.Discovery, runtimeEnv string) (disco switch discovery.Enable { case "kubernetes": return kubernetes.NewKubernetesConnManager(discovery.Kubernetes.Namespace) - case "etcd": + case Etcd: return etcd.NewSvcDiscoveryRegistry( discovery.Etcd.RootDirectory, discovery.Etcd.Address, diff --git a/pkg/common/prommetrics/api.go b/pkg/common/prommetrics/api.go index 95b5c06b6..2dc5cb65d 100644 --- a/pkg/common/prommetrics/api.go +++ b/pkg/common/prommetrics/api.go @@ -3,6 +3,7 @@ package prommetrics import ( "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" + "net" "strconv" ) @@ -23,14 +24,14 @@ var ( ) ) -func ApiInit(prometheusPort int) error { +func ApiInit(listener net.Listener) error { apiRegistry := prometheus.NewRegistry() cs := append( baseCollector, apiCounter, httpCounter, ) - return Init(apiRegistry, prometheusPort, commonPath, promhttp.HandlerFor(apiRegistry, promhttp.HandlerOpts{}), cs...) + return Init(apiRegistry, listener, commonPath, promhttp.HandlerFor(apiRegistry, promhttp.HandlerOpts{}), cs...) } func APICall(path string, method string, apiCode int) { diff --git a/pkg/common/prommetrics/discovery.go b/pkg/common/prommetrics/discovery.go new file mode 100644 index 000000000..8f03bc2ae --- /dev/null +++ b/pkg/common/prommetrics/discovery.go @@ -0,0 +1,31 @@ +package prommetrics + +import "fmt" + +const ( + APIKeyName = "api" + MessageTransferKeyName = "message-transfer" +) + +type Target struct { + Target string `json:"target"` + Labels map[string]string `json:"labels"` +} + +type RespTarget struct { + Targets []string `json:"targets"` + Labels map[string]string `json:"labels"` +} + +func BuildDiscoveryKey(name string) string { + return fmt.Sprintf("%s/%s/%s", "openim", "prometheus_discovery", name) +} + +func BuildDefaultTarget(host string, ip int) Target { + return Target{ + Target: fmt.Sprintf("%s:%d", host, ip), + Labels: map[string]string{ + "namespace": "default", + }, + } +} diff --git a/pkg/common/prommetrics/prommetrics.go b/pkg/common/prommetrics/prommetrics.go index 02e408d63..2fc5d76b4 100644 --- a/pkg/common/prommetrics/prommetrics.go +++ b/pkg/common/prommetrics/prommetrics.go @@ -15,9 +15,9 @@ package prommetrics import ( - "fmt" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/collectors" + "net" "net/http" ) @@ -30,9 +30,9 @@ var ( } ) -func Init(registry *prometheus.Registry, prometheusPort int, path string, handler http.Handler, cs ...prometheus.Collector) error { +func Init(registry *prometheus.Registry, listener net.Listener, path string, handler http.Handler, cs ...prometheus.Collector) error { registry.MustRegister(cs...) srv := http.NewServeMux() srv.Handle(path, handler) - return http.ListenAndServe(fmt.Sprintf(":%d", prometheusPort), srv) + return http.Serve(listener, srv) } diff --git a/pkg/common/prommetrics/rpc.go b/pkg/common/prommetrics/rpc.go index 809d509b2..3f115d30b 100644 --- a/pkg/common/prommetrics/rpc.go +++ b/pkg/common/prommetrics/rpc.go @@ -5,6 +5,7 @@ import ( "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" + "net" "strconv" ) @@ -21,13 +22,13 @@ var ( ) ) -func RpcInit(cs []prometheus.Collector, prometheusPort int) error { +func RpcInit(cs []prometheus.Collector, listener net.Listener) error { reg := prometheus.NewRegistry() cs = append(append( baseCollector, rpcCounter, ), cs...) - return Init(reg, prometheusPort, rpcPath, promhttp.HandlerFor(reg, promhttp.HandlerOpts{Registry: reg}), cs...) + return Init(reg, listener, rpcPath, promhttp.HandlerFor(reg, promhttp.HandlerOpts{Registry: reg}), cs...) } func RPCCall(name string, path string, code int) { diff --git a/pkg/common/prommetrics/transfer.go b/pkg/common/prommetrics/transfer.go index f0abb8285..36fe1d568 100644 --- a/pkg/common/prommetrics/transfer.go +++ b/pkg/common/prommetrics/transfer.go @@ -17,6 +17,7 @@ package prommetrics import ( "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" + "net" ) var ( @@ -42,7 +43,7 @@ var ( }) ) -func TransferInit(prometheusPort int) error { +func TransferInit(listener net.Listener) error { reg := prometheus.NewRegistry() cs := append( baseCollector, @@ -52,5 +53,5 @@ func TransferInit(prometheusPort int) error { MsgInsertMongoFailedCounter, SeqSetFailedCounter, ) - return Init(reg, prometheusPort, commonPath, promhttp.HandlerFor(reg, promhttp.HandlerOpts{Registry: reg}), cs...) + return Init(reg, listener, commonPath, promhttp.HandlerFor(reg, promhttp.HandlerOpts{Registry: reg}), cs...) } diff --git a/pkg/common/startrpc/start.go b/pkg/common/startrpc/start.go index ae8344923..f75a50541 100644 --- a/pkg/common/startrpc/start.go +++ b/pkg/common/startrpc/start.go @@ -22,15 +22,17 @@ import ( "net/http" "os" "os/signal" + "strconv" "syscall" "time" "github.com/openimsdk/open-im-server/v3/pkg/common/config" + "github.com/openimsdk/tools/discovery/etcd" "github.com/openimsdk/tools/utils/datautil" - "github.com/openimsdk/tools/utils/runtimeenv" + "github.com/openimsdk/tools/utils/jsonutil" "google.golang.org/grpc/status" - "strconv" + "github.com/openimsdk/tools/utils/runtimeenv" kdisc "github.com/openimsdk/open-im-server/v3/pkg/common/discoveryregister" "github.com/openimsdk/open-im-server/v3/pkg/common/prommetrics" @@ -49,11 +51,17 @@ func Start[T any](ctx context.Context, discovery *config.Discovery, prometheusCo config T, client discovery.SvcDiscoveryRegistry, server *grpc.Server) error, options ...grpc.ServerOption) error { var ( - rpcTcpAddr string - netDone = make(chan struct{}, 2) - netErr error + rpcTcpAddr string + netDone = make(chan struct{}, 2) + netErr error + prometheusPort int ) + registerIP, err := network.GetRpcRegisterIP(registerIP) + if err != nil { + return err + } + runTimeEnv := runtimeenv.PrintRuntimeEnvironment() if !autoSetPorts { @@ -66,6 +74,27 @@ func Start[T any](ctx context.Context, discovery *config.Discovery, prometheusCo rpcTcpAddr = net.JoinHostPort(network.GetListenIP(listenIP), "0") } + getAutoPort := func() (net.Listener, int, error) { + listener, err := net.Listen("tcp", rpcTcpAddr) + if err != nil { + return nil, 0, errs.WrapMsg(err, "listen err", "rpcTcpAddr", rpcTcpAddr) + } + _, portStr, _ := net.SplitHostPort(listener.Addr().String()) + port, _ := strconv.Atoi(portStr) + return listener, port, nil + } + + if autoSetPorts && discovery.Enable != kdisc.Etcd { + return errs.New("only etcd support autoSetPorts", "rpcRegisterName", rpcRegisterName).Wrap() + } + client, err := kdisc.NewDiscoveryRegister(discovery, runTimeEnv) + if err != nil { + return err + } + + defer client.Close() + client.AddOption(mw.GrpcClient(), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithDefaultServiceConfig(fmt.Sprintf(`{"LoadBalancingPolicy": "%s"}`, "round_robin"))) + // var reg *prometheus.Registry // var metric *grpcprometheus.ServerMetrics if prometheusConfig.Enable { @@ -78,17 +107,40 @@ func Start[T any](ctx context.Context, discovery *config.Discovery, prometheusCo prommetricsUnaryInterceptor(rpcRegisterName), prommetricsStreamInterceptor(rpcRegisterName), ) - prometheusPort, err := datautil.GetElemByIndex(prometheusConfig.Ports, index) - if err != nil { - return err + + var ( + listener net.Listener + ) + + if autoSetPorts { + listener, prometheusPort, err = getAutoPort() + if err != nil { + return err + } + + etcdClient := client.(*etcd.SvcDiscoveryRegistryImpl).GetClient() + + _, err = etcdClient.Put(ctx, prommetrics.BuildDiscoveryKey(rpcRegisterName), jsonutil.StructToJsonString(prommetrics.BuildDefaultTarget(registerIP, prometheusPort))) + if err != nil { + return errs.WrapMsg(err, "etcd put err") + } + } else { + prometheusPort, err = datautil.GetElemByIndex(prometheusConfig.Ports, index) + if err != nil { + return err + } + listener, err = net.Listen("tcp", fmt.Sprintf(":%d", prometheusPort)) + if err != nil { + return errs.WrapMsg(err, "listen err", "rpcTcpAddr", rpcTcpAddr) + } } cs := prommetrics.GetGrpcCusMetrics(rpcRegisterName, discovery) go func() { - if err := prommetrics.RpcInit(cs, prometheusPort); err != nil && !errors.Is(err, http.ErrServerClosed) { + if err := prommetrics.RpcInit(cs, listener); err != nil && !errors.Is(err, http.ErrServerClosed) { netErr = errs.WrapMsg(err, fmt.Sprintf("rpc %s prometheus start err: %d", rpcRegisterName, prometheusPort)) netDone <- struct{}{} } - // metric.InitializeMetrics(srv) + //metric.InitializeMetrics(srv) // Create a HTTP server for prometheus. // httpServer = &http.Server{Handler: promhttp.HandlerFor(reg, promhttp.HandlerOpts{}), Addr: fmt.Sprintf("0.0.0.0:%d", prometheusPort)} // if err := httpServer.ListenAndServe(); err != nil && err != http.ErrServerClosed { @@ -100,30 +152,15 @@ func Start[T any](ctx context.Context, discovery *config.Discovery, prometheusCo options = append(options, mw.GrpcServer()) } - listener, err := net.Listen( - "tcp", - rpcTcpAddr, - ) - if err != nil { - return errs.WrapMsg(err, "listen err", "rpcTcpAddr", rpcTcpAddr) - } - - _, portStr, _ := net.SplitHostPort(listener.Addr().String()) - registerIP = network.GetListenIP(registerIP) - port, _ := strconv.Atoi(portStr) - - log.CInfo(ctx, "RPC server is initializing", "runTimeEnv", runTimeEnv, "rpcRegisterName", rpcRegisterName, "rpcPort", portStr, - "prometheusPorts", prometheusConfig.Ports) - - defer listener.Close() - client, err := kdisc.NewDiscoveryRegister(discovery, runTimeEnv) + listener, port, err := getAutoPort() if err != nil { return err } - defer client.Close() - client.AddOption(mw.GrpcClient(), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithDefaultServiceConfig(fmt.Sprintf(`{"LoadBalancingPolicy": "%s"}`, "round_robin"))) + log.CInfo(ctx, "RPC server is initializing", "rpcRegisterName", rpcRegisterName, "rpcPort", port, + "prometheusPort", prometheusPort) + defer listener.Close() srv := grpc.NewServer(options...) err = rpcFn(ctx, config, client, srv) From 42136c6b93170f90e63fc7fa4034a3c98b123a21 Mon Sep 17 00:00:00 2001 From: OpenIM-Gordon <1432970085@qq.com> Date: Thu, 12 Dec 2024 10:50:40 +0800 Subject: [PATCH 068/199] fix: server can return isEnd to control fetch messages when sdk pull messages end normally. (#2949) --- go.mod | 2 +- go.sum | 4 +- internal/rpc/msg/sync_msg.go | 4 +- pkg/common/storage/controller/msg.go | 81 +++++++++++++++++++++++++++- 4 files changed, 85 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 5d81e67d3..6c5bf79e3 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( github.com/gorilla/websocket v1.5.1 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/mitchellh/mapstructure v1.5.0 - github.com/openimsdk/protocol v0.0.72-alpha.61 + github.com/openimsdk/protocol v0.0.72-alpha.63 github.com/openimsdk/tools v0.0.50-alpha.47 github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 diff --git a/go.sum b/go.sum index 99f569a38..a7cf42eca 100644 --- a/go.sum +++ b/go.sum @@ -347,8 +347,8 @@ github.com/onsi/gomega v1.25.0 h1:Vw7br2PCDYijJHSfBOWhov+8cAnUf8MfMaIOV323l6Y= github.com/onsi/gomega v1.25.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdMPGhM= github.com/openimsdk/gomake v0.0.15-alpha.2 h1:5Q8yl8ezy2yx+q8/ucU/t4kJnDfCzNOrkXcDACCqtyM= github.com/openimsdk/gomake v0.0.15-alpha.2/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= -github.com/openimsdk/protocol v0.0.72-alpha.61 h1:RuZR9/Sg3p6Bpb2CKPjPoA2AUmTvHITmhZ3PT/RbWMs= -github.com/openimsdk/protocol v0.0.72-alpha.61/go.mod h1:Iet+piS/jaS+kWWyj6EEr36mk4ISzIRYjoMSVA4dq2M= +github.com/openimsdk/protocol v0.0.72-alpha.63 h1:IyPBibEvwBtTmD8DSrlqcekfEXe74k4+KeeHsgdhGh0= +github.com/openimsdk/protocol v0.0.72-alpha.63/go.mod h1:Iet+piS/jaS+kWWyj6EEr36mk4ISzIRYjoMSVA4dq2M= github.com/openimsdk/tools v0.0.50-alpha.47 h1:Cfe2va/g6WhLjOoQqZkjrdlEDq1dUsfcQsdUB5oADVA= github.com/openimsdk/tools v0.0.50-alpha.47/go.mod h1:muCtxguNJv8lFwLei27UASu2Nvg4ERSeN0R4K5tivk0= github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= diff --git a/internal/rpc/msg/sync_msg.go b/internal/rpc/msg/sync_msg.go index 2f7788167..13e3cfd33 100644 --- a/internal/rpc/msg/sync_msg.go +++ b/internal/rpc/msg/sync_msg.go @@ -92,11 +92,13 @@ func (m *msgServer) GetSeqMessage(ctx context.Context, req *msg.GetSeqMessageReq NotificationMsgs: make(map[string]*sdkws.PullMsgs), } for _, conv := range req.Conversations { - _, _, msgs, err := m.MsgDatabase.GetMsgBySeqs(ctx, req.UserID, conv.ConversationID, conv.Seqs) + isEnd, endSeq, msgs, err := m.MsgDatabase.GetMessagesBySeqWithBounds(ctx, req.UserID, conv.ConversationID, conv.Seqs, req.GetOrder()) if err != nil { return nil, err } var pullMsgs *sdkws.PullMsgs + pullMsgs.IsEnd = isEnd + pullMsgs.EndSeq = endSeq if ok := false; conversationutil.IsNotificationConversationID(conv.ConversationID) { pullMsgs, ok = resp.NotificationMsgs[conv.ConversationID] if !ok { diff --git a/pkg/common/storage/controller/msg.go b/pkg/common/storage/controller/msg.go index 464ad7604..8d82d8543 100644 --- a/pkg/common/storage/controller/msg.go +++ b/pkg/common/storage/controller/msg.go @@ -24,6 +24,9 @@ import ( "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" + "github.com/redis/go-redis/v9" + "go.mongodb.org/mongo-driver/mongo" + "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/open-im-server/v3/pkg/common/convert" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache" @@ -35,8 +38,6 @@ import ( "github.com/openimsdk/tools/mq/kafka" "github.com/openimsdk/tools/utils/datautil" "github.com/openimsdk/tools/utils/timeutil" - "github.com/redis/go-redis/v9" - "go.mongodb.org/mongo-driver/mongo" ) const ( @@ -56,6 +57,7 @@ type CommonMsgDatabase interface { GetMsgBySeqs(ctx context.Context, userID string, conversationID string, seqs []int64) (minSeq int64, maxSeq int64, seqMsg []*sdkws.MsgData, err error) // DeleteConversationMsgsAndSetMinSeq deletes conversation messages and resets the minimum sequence number. If `remainTime` is 0, all messages are deleted (this method does not delete Redis // cache). + GetMessagesBySeqWithBounds(ctx context.Context, userID string, conversationID string, seqs []int64, pullOrder sdkws.PullOrder) (bool, int64, []*sdkws.MsgData, error) DeleteConversationMsgsAndSetMinSeq(ctx context.Context, conversationID string, remainTime int64) error // ClearUserMsgs marks messages for deletion based on clear time and returns a list of sequence numbers for marked messages. ClearUserMsgs(ctx context.Context, userID string, conversationID string, clearTime int64, lastMsgClearTime time.Time) (seqs []int64, err error) @@ -517,6 +519,81 @@ func (db *commonMsgDatabase) GetMsgBySeqs(ctx context.Context, userID string, co return minSeq, maxSeq, successMsgs, nil } +func (db *commonMsgDatabase) GetMessagesBySeqWithBounds(ctx context.Context, userID string, conversationID string, seqs []int64, pullOrder sdkws.PullOrder) (bool, int64, []*sdkws.MsgData, error) { + var endSeq int64 + var isEnd bool + userMinSeq, err := db.seqUser.GetUserMinSeq(ctx, conversationID, userID) + if err != nil { + return false, 0, nil, err + } + minSeq, err := db.seqConversation.GetMinSeq(ctx, conversationID) + if err != nil { + return false, 0, nil, err + } + maxSeq, err := db.seqConversation.GetMaxSeq(ctx, conversationID) + if err != nil { + return false, 0, nil, err + } + userMaxSeq, err := db.seqUser.GetUserMaxSeq(ctx, conversationID, userID) + if err != nil { + return false, 0, nil, err + } + if userMinSeq > minSeq { + minSeq = userMinSeq + } + if userMaxSeq > 0 && userMaxSeq < maxSeq { + maxSeq = userMaxSeq + } + newSeqs := make([]int64, 0, len(seqs)) + for _, seq := range seqs { + if seq <= 0 { + continue + } + // The normal range and can fetch messages + if seq >= minSeq && seq <= maxSeq { + newSeqs = append(newSeqs, seq) + continue + } + // If the requested seq is smaller than the minimum seq and the pull order is descending (pulling older messages) + if seq < minSeq && pullOrder == sdkws.PullOrder_PullOrderDesc { + isEnd = true + endSeq = minSeq + } + // If the requested seq is larger than the maximum seq and the pull order is ascending (pulling newer messages) + if seq > maxSeq && pullOrder == sdkws.PullOrder_PullOrderAsc { + isEnd = true + endSeq = maxSeq + } + } + if len(newSeqs) == 0 { + return isEnd, endSeq, nil, nil + } + successMsgs, failedSeqs, err := db.msg.GetMessagesBySeq(ctx, conversationID, newSeqs) + if err != nil { + if !errors.Is(err, redis.Nil) { + log.ZWarn(ctx, "get message from redis exception", err, "failedSeqs", failedSeqs, "conversationID", conversationID) + } + } + log.ZDebug(ctx, "db.seq.GetMessagesBySeq", "userID", userID, "conversationID", conversationID, "seqs", + seqs, "len(successMsgs)", len(successMsgs), "failedSeqs", failedSeqs) + + if len(failedSeqs) > 0 { + mongoMsgs, err := db.getMsgBySeqs(ctx, userID, conversationID, failedSeqs) + if err != nil { + + return false, 0, nil, err + } + + successMsgs = append(successMsgs, mongoMsgs...) + + //_, err = db.msg.SetMessagesToCache(ctx, conversationID, mongoMsgs) + //if err != nil { + // return 0, 0, nil, err + //} + } + return isEnd, endSeq, successMsgs, nil +} + func (db *commonMsgDatabase) DeleteConversationMsgsAndSetMinSeq(ctx context.Context, conversationID string, remainTime int64) error { var delStruct delMsgRecursionStruct var skip int64 From 120afdf6ae386542d74bddec20f2dafeda2cfc3d Mon Sep 17 00:00:00 2001 From: icey-yu <119291641+icey-yu@users.noreply.github.com> Date: Thu, 12 Dec 2024 11:47:52 +0800 Subject: [PATCH 069/199] feat: Change upload logs systemType to AppFramework. (#2927) * feat: change upload logs systemType to AppFramework * feat: change upload logs systemType to AppFramework * Merge: main --- internal/rpc/third/log.go | 16 ++++++++-------- pkg/common/storage/model/log.go | 19 ++++++++++--------- 2 files changed, 18 insertions(+), 17 deletions(-) diff --git a/internal/rpc/third/log.go b/internal/rpc/third/log.go index cb4678b34..4eeb5d558 100644 --- a/internal/rpc/third/log.go +++ b/internal/rpc/third/log.go @@ -50,14 +50,14 @@ func (t *thirdServer) UploadLogs(ctx context.Context, req *third.UploadLogsReq) platform := constant.PlatformID2Name[int(req.Platform)] for _, fileURL := range req.FileURLs { log := relationtb.Log{ - Platform: platform, - UserID: userID, - CreateTime: time.Now(), - Url: fileURL.URL, - FileName: fileURL.Filename, - SystemType: req.AppFramework, - Version: req.Version, - Ex: req.Ex, + Platform: platform, + UserID: userID, + CreateTime: time.Now(), + Url: fileURL.URL, + FileName: fileURL.Filename, + AppFramework: req.AppFramework, + Version: req.Version, + Ex: req.Ex, } for i := 0; i < 20; i++ { id := genLogID() diff --git a/pkg/common/storage/model/log.go b/pkg/common/storage/model/log.go index 9db72c695..9dc392179 100644 --- a/pkg/common/storage/model/log.go +++ b/pkg/common/storage/model/log.go @@ -19,13 +19,14 @@ import ( ) type Log struct { - LogID string `bson:"log_id"` - Platform string `bson:"platform"` - UserID string `bson:"user_id"` - CreateTime time.Time `bson:"create_time"` - Url string `bson:"url"` - FileName string `bson:"file_name"` - SystemType string `bson:"system_type"` - Version string `bson:"version"` - Ex string `bson:"ex"` + LogID string `bson:"log_id"` + Platform string `bson:"platform"` + UserID string `bson:"user_id"` + CreateTime time.Time `bson:"create_time"` + Url string `bson:"url"` + FileName string `bson:"file_name"` + SystemType string `bson:"system_type"` + AppFramework string `bson:"app_framework"` + Version string `bson:"version"` + Ex string `bson:"ex"` } From 92ea52febfa30c1d5597baeb4b13e7ae0b9d304f Mon Sep 17 00:00:00 2001 From: icey-yu <119291641+icey-yu@users.noreply.github.com> Date: Thu, 12 Dec 2024 11:59:15 +0800 Subject: [PATCH 070/199] fix:Only print panic function frame && feat: log.ZPanic (#2947) * fix: log print panic * Merge: main --- go.mod | 2 +- go.sum | 4 ++-- internal/msggateway/client.go | 3 +-- internal/msgtransfer/init.go | 2 +- internal/msgtransfer/online_history_msg_handler.go | 6 +++--- internal/rpc/msg/send.go | 3 +-- pkg/rpccache/subscriber.go | 3 +-- 7 files changed, 10 insertions(+), 13 deletions(-) diff --git a/go.mod b/go.mod index 6c5bf79e3..cd32a118a 100644 --- a/go.mod +++ b/go.mod @@ -15,7 +15,7 @@ require ( github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/mitchellh/mapstructure v1.5.0 github.com/openimsdk/protocol v0.0.72-alpha.63 - github.com/openimsdk/tools v0.0.50-alpha.47 + github.com/openimsdk/tools v0.0.50-alpha.50 github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 github.com/stretchr/testify v1.9.0 diff --git a/go.sum b/go.sum index a7cf42eca..26ed22efb 100644 --- a/go.sum +++ b/go.sum @@ -349,8 +349,8 @@ github.com/openimsdk/gomake v0.0.15-alpha.2 h1:5Q8yl8ezy2yx+q8/ucU/t4kJnDfCzNOrk github.com/openimsdk/gomake v0.0.15-alpha.2/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= github.com/openimsdk/protocol v0.0.72-alpha.63 h1:IyPBibEvwBtTmD8DSrlqcekfEXe74k4+KeeHsgdhGh0= github.com/openimsdk/protocol v0.0.72-alpha.63/go.mod h1:Iet+piS/jaS+kWWyj6EEr36mk4ISzIRYjoMSVA4dq2M= -github.com/openimsdk/tools v0.0.50-alpha.47 h1:Cfe2va/g6WhLjOoQqZkjrdlEDq1dUsfcQsdUB5oADVA= -github.com/openimsdk/tools v0.0.50-alpha.47/go.mod h1:muCtxguNJv8lFwLei27UASu2Nvg4ERSeN0R4K5tivk0= +github.com/openimsdk/tools v0.0.50-alpha.50 h1:+naDlvHcqJDj2NsCGnQd1LLQOET5IRPbrtmWbM/o7JQ= +github.com/openimsdk/tools v0.0.50-alpha.50/go.mod h1:muCtxguNJv8lFwLei27UASu2Nvg4ERSeN0R4K5tivk0= github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= diff --git a/internal/msggateway/client.go b/internal/msggateway/client.go index 0da7d7220..7e1ba6405 100644 --- a/internal/msggateway/client.go +++ b/internal/msggateway/client.go @@ -18,7 +18,6 @@ import ( "context" "encoding/json" "fmt" - "github.com/openimsdk/tools/mw" "runtime/debug" "sync" "sync/atomic" @@ -378,7 +377,7 @@ func (c *Client) activeHeartbeat(ctx context.Context) { go func() { defer func() { if r := recover(); r != nil { - mw.PanicStackToLog(ctx, r) + log.ZPanic(ctx, "activeHeartbeat Panic", r) } }() log.ZDebug(ctx, "server initiative send heartbeat start.") diff --git a/internal/msgtransfer/init.go b/internal/msgtransfer/init.go index 9da2e7974..5be5e107d 100644 --- a/internal/msgtransfer/init.go +++ b/internal/msgtransfer/init.go @@ -204,7 +204,7 @@ func (m *MsgTransfer) Start(index int, config *Config) error { go func() { defer func() { if r := recover(); r != nil { - mw.PanicStackToLog(m.ctx, r) + log.ZPanic(m.ctx, "MsgTransfer Start Panic", r) } }() if err := prommetrics.TransferInit(listener); err != nil && !errors.Is(err, http.ErrServerClosed) { diff --git a/internal/msgtransfer/online_history_msg_handler.go b/internal/msgtransfer/online_history_msg_handler.go index 4a5d5ba89..0104f6633 100644 --- a/internal/msgtransfer/online_history_msg_handler.go +++ b/internal/msgtransfer/online_history_msg_handler.go @@ -18,13 +18,13 @@ import ( "context" "encoding/json" "errors" - "github.com/openimsdk/open-im-server/v3/pkg/common/prommetrics" - "github.com/openimsdk/tools/mw" "strconv" "strings" "sync" "time" + "github.com/openimsdk/open-im-server/v3/pkg/common/prommetrics" + "github.com/IBM/sarama" "github.com/go-redis/redis" "github.com/openimsdk/open-im-server/v3/pkg/common/config" @@ -349,7 +349,7 @@ func (och *OnlineHistoryRedisConsumerHandler) handleNotification(ctx context.Con func (och *OnlineHistoryRedisConsumerHandler) HandleUserHasReadSeqMessages(ctx context.Context) { defer func() { if r := recover(); r != nil { - mw.PanicStackToLog(ctx, r) + log.ZPanic(ctx, "HandleUserHasReadSeqMessages Panic", r) } }() diff --git a/internal/rpc/msg/send.go b/internal/rpc/msg/send.go index 2cbbcd1fc..034d549ec 100644 --- a/internal/rpc/msg/send.go +++ b/internal/rpc/msg/send.go @@ -16,7 +16,6 @@ package msg import ( "context" - "github.com/openimsdk/tools/mw" "github.com/openimsdk/open-im-server/v3/pkg/common/prommetrics" "github.com/openimsdk/open-im-server/v3/pkg/msgprocessor" @@ -89,7 +88,7 @@ func (m *msgServer) setConversationAtInfo(nctx context.Context, msg *sdkws.MsgDa defer func() { if r := recover(); r != nil { - mw.PanicStackToLog(nctx, r) + log.ZPanic(nctx, "setConversationAtInfo Panic", r) } }() diff --git a/pkg/rpccache/subscriber.go b/pkg/rpccache/subscriber.go index 3c73ef449..44e1f5885 100644 --- a/pkg/rpccache/subscriber.go +++ b/pkg/rpccache/subscriber.go @@ -17,7 +17,6 @@ package rpccache import ( "context" "encoding/json" - "github.com/openimsdk/tools/mw" "github.com/openimsdk/tools/log" "github.com/redis/go-redis/v9" @@ -26,7 +25,7 @@ import ( func subscriberRedisDeleteCache(ctx context.Context, client redis.UniversalClient, channel string, del func(ctx context.Context, key ...string)) { defer func() { if r := recover(); r != nil { - mw.PanicStackToLog(ctx, r) + log.ZPanic(ctx, "subscriberRedisDeleteCache Panic", r) } }() for message := range client.Subscribe(ctx, channel).Channel() { From 0435915df1702611864caf224139a577ce4df9d0 Mon Sep 17 00:00:00 2001 From: Monet Lee Date: Thu, 12 Dec 2024 12:13:04 +0800 Subject: [PATCH 071/199] feat: support quote ContentType in SendMsg. (#2819) --- internal/api/msg.go | 2 ++ pkg/apistruct/msg.go | 43 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+) diff --git a/internal/api/msg.go b/internal/api/msg.go index ce94b5f4f..9f79067a8 100644 --- a/internal/api/msg.go +++ b/internal/api/msg.go @@ -173,6 +173,8 @@ func (m *MessageApi) getSendMsgReq(c *gin.Context, req apistruct.SendMsg) (sendM data = apistruct.AtElem{} case constant.Custom: data = apistruct.CustomElem{} + case constant.Quote: + data = apistruct.QuoteElem{} case constant.Stream: data = apistruct.StreamMsgElem{} case constant.OANotification: diff --git a/pkg/apistruct/msg.go b/pkg/apistruct/msg.go index dda3ff317..44f157b6d 100644 --- a/pkg/apistruct/msg.go +++ b/pkg/apistruct/msg.go @@ -14,6 +14,8 @@ package apistruct +import "github.com/openimsdk/protocol/sdkws" + type PictureBaseInfo struct { UUID string `mapstructure:"uuid"` Type string `mapstructure:"type" validate:"required"` @@ -90,6 +92,11 @@ type RevokeElem struct { RevokeMsgClientID string `mapstructure:"revokeMsgClientID" validate:"required"` } +type QuoteElem struct { + Text string `json:"text,omitempty"` + QuoteMessage *MsgStruct `json:"quoteMessage,omitempty"` +} + type OANotificationElem struct { NotificationName string `mapstructure:"notificationName" json:"notificationName" validate:"required"` NotificationFaceURL string `mapstructure:"notificationFaceURL" json:"notificationFaceURL"` @@ -103,6 +110,7 @@ type OANotificationElem struct { FileElem *FileElem `mapstructure:"fileElem" json:"fileElem"` Ex string `mapstructure:"ex" json:"ex"` } + type MessageRevoked struct { RevokerID string `mapstructure:"revokerID" json:"revokerID" validate:"required"` RevokerRole int32 `mapstructure:"revokerRole" json:"revokerRole" validate:"required"` @@ -111,3 +119,38 @@ type MessageRevoked struct { SessionType int32 `mapstructure:"sessionType" json:"sessionType" validate:"required"` Seq uint32 `mapstructure:"seq" json:"seq" validate:"required"` } + +type MsgStruct struct { + ClientMsgID string `json:"clientMsgID,omitempty"` + ServerMsgID string `json:"serverMsgID,omitempty"` + CreateTime int64 `json:"createTime"` + SendTime int64 `json:"sendTime"` + SessionType int32 `json:"sessionType"` + SendID string `json:"sendID,omitempty"` + RecvID string `json:"recvID,omitempty"` + MsgFrom int32 `json:"msgFrom"` + ContentType int32 `json:"contentType"` + SenderPlatformID int32 `json:"senderPlatformID"` + SenderNickname string `json:"senderNickname,omitempty"` + SenderFaceURL string `json:"senderFaceUrl,omitempty"` + GroupID string `json:"groupID,omitempty"` + Content string `json:"content,omitempty"` + Seq int64 `json:"seq"` + IsRead bool `json:"isRead"` + Status int32 `json:"status"` + IsReact bool `json:"isReact,omitempty"` + IsExternalExtensions bool `json:"isExternalExtensions,omitempty"` + OfflinePush *sdkws.OfflinePushInfo `json:"offlinePush,omitempty"` + AttachedInfo string `json:"attachedInfo,omitempty"` + Ex string `json:"ex,omitempty"` + LocalEx string `json:"localEx,omitempty"` + TextElem *TextElem `json:"textElem,omitempty"` + PictureElem *PictureElem `json:"pictureElem,omitempty"` + SoundElem *SoundElem `json:"soundElem,omitempty"` + VideoElem *VideoElem `json:"videoElem,omitempty"` + FileElem *FileElem `json:"fileElem,omitempty"` + AtTextElem *AtElem `json:"atTextElem,omitempty"` + LocationElem *LocationElem `json:"locationElem,omitempty"` + CustomElem *CustomElem `json:"customElem,omitempty"` + QuoteElem *QuoteElem `json:"quoteElem,omitempty"` +} From 97f506c36678db1c455ec7f911242cebe7ff05db Mon Sep 17 00:00:00 2001 From: chao <48119764+withchao@users.noreply.github.com> Date: Thu, 12 Dec 2024 17:32:03 +0800 Subject: [PATCH 072/199] pb (#2958) --- internal/rpc/msg/delete.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/internal/rpc/msg/delete.go b/internal/rpc/msg/delete.go index e19bba867..371c50e2e 100644 --- a/internal/rpc/msg/delete.go +++ b/internal/rpc/msg/delete.go @@ -138,9 +138,16 @@ func (m *msgServer) clearConversation(ctx context.Context, conversationIDs []str } isSyncSelf, isSyncOther := m.validateDeleteSyncOpt(deleteSyncOpt) if !isSyncOther { - if err := m.MsgDatabase.SetUserConversationsMinSeqs(ctx, userID, m.getMinSeqs(maxSeqs)); err != nil { + setSeqs := m.getMinSeqs(maxSeqs) + if err := m.MsgDatabase.SetUserConversationsMinSeqs(ctx, userID, setSeqs); err != nil { return err } + ownerUserIDs := []string{userID} + for conversationID, seq := range setSeqs { + if err := m.Conversation.SetConversationMinSeq(ctx, ownerUserIDs, conversationID, seq); err != nil { + return err + } + } // notification 2 self if isSyncSelf { tips := &sdkws.ClearConversationTips{UserID: userID, ConversationIDs: existConversationIDs} From 1eaae5f9804b406f70deb8977c9ae9ff59890f89 Mon Sep 17 00:00:00 2001 From: OpenIM-Gordon <1432970085@qq.com> Date: Thu, 12 Dec 2024 18:13:35 +0800 Subject: [PATCH 073/199] fix: fetch message return isEnd and endSeq panic. (#2959) * fix: server can return isEnd to control fetch messages when sdk pull messages end normally. * fix: server can return isEnd to control fetch messages when sdk pull messages end normally. * fix: server can return isEnd to control fetch messages when sdk pull messages end normally. --- internal/rpc/msg/sync_msg.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/rpc/msg/sync_msg.go b/internal/rpc/msg/sync_msg.go index 13e3cfd33..7d4ffa3e6 100644 --- a/internal/rpc/msg/sync_msg.go +++ b/internal/rpc/msg/sync_msg.go @@ -97,8 +97,6 @@ func (m *msgServer) GetSeqMessage(ctx context.Context, req *msg.GetSeqMessageReq return nil, err } var pullMsgs *sdkws.PullMsgs - pullMsgs.IsEnd = isEnd - pullMsgs.EndSeq = endSeq if ok := false; conversationutil.IsNotificationConversationID(conv.ConversationID) { pullMsgs, ok = resp.NotificationMsgs[conv.ConversationID] if !ok { @@ -113,6 +111,8 @@ func (m *msgServer) GetSeqMessage(ctx context.Context, req *msg.GetSeqMessageReq } } pullMsgs.Msgs = append(pullMsgs.Msgs, msgs...) + pullMsgs.IsEnd = isEnd + pullMsgs.EndSeq = endSeq } return resp, nil } From 7c7a99f801b67a5d6f70908782229c6619073734 Mon Sep 17 00:00:00 2001 From: Monet Lee Date: Fri, 13 Dec 2024 11:02:45 +0800 Subject: [PATCH 074/199] build: update kubernetes deployment Run. (#2919) * build: k8s improve. * refactor: update docker image contents. * rename seq file. * build: update k8s origin deploys. * update check logic. * update magefile * update image name. * update readme * update Kubernetes Discovery. * revert pkg. * update create in k8s * update service image release CI. * update deployment image source. * update mage contents. * update pkg source. * update go get pkg. * fix test file. * update discovery register. * update * update deploy yaml. * update replica. * update deployment. * remove notfication config. * remove notification and zookeeper. * update discovery in kubernetes. * build: improve kubernetes deployment. * update config field in discovery. * update ReadMe in deployments. * update go mod. * update const quote. * fix test fields. * remove unused method. * remove unused contents. --- .../workflows/docker-build-and-release.yml | 91 ++ build/images/openim-crontask/Dockerfile | 2 +- build/images/openim-msggateway/Dockerfile | 2 +- build/images/openim-msgtransfer/Dockerfile | 2 +- build/images/openim-push/Dockerfile | 2 +- build/images/openim-rpc-auth/Dockerfile | 2 +- .../images/openim-rpc-conversation/Dockerfile | 2 +- build/images/openim-rpc-friend/Dockerfile | 2 +- build/images/openim-rpc-group/Dockerfile | 2 +- build/images/openim-rpc-msg/Dockerfile | 2 +- build/images/openim-rpc-third/Dockerfile | 2 +- build/images/openim-rpc-user/Dockerfile | 2 +- config/discovery.yml | 2 +- config/openim-rpc-auth.yml | 1 - deployments/Readme.md | 174 +-- deployments/charts/helmfile.yaml | 34 - deployments/charts/openim-api/.helmignore | 23 - deployments/charts/openim-api/Chart.yaml | 53 - deployments/charts/openim-api/LICENSE | 201 ---- .../charts/openim-api/templates/NOTES.txt | 22 - .../charts/openim-api/templates/_helpers.tpl | 62 - .../charts/openim-api/templates/app-cm.yaml | 27 - .../openim-api/templates/deployment.yaml | 86 -- .../charts/openim-api/templates/hpa.yaml | 42 - .../charts/openim-api/templates/ingress.yaml | 75 -- .../charts/openim-api/templates/service.yaml | 29 - .../openim-api/templates/serviceaccount.yaml | 26 - deployments/charts/openim-api/values.yaml | 100 -- .../charts/openim-msggateway/.helmignore | 23 - .../charts/openim-msggateway/Chart.yaml | 53 - .../openim-msggateway/templates/NOTES.txt | 22 - .../openim-msggateway/templates/_helpers.tpl | 62 - .../templates/deployment.yaml | 89 -- .../openim-msggateway/templates/hpa.yaml | 42 - .../openim-msggateway/templates/ingress.yaml | 75 -- .../openim-msggateway/templates/service.yaml | 33 - .../templates/serviceaccount.yaml | 26 - .../charts/openim-msggateway/values.yaml | 96 -- .../charts/openim-msgtransfer/.helmignore | 23 - .../charts/openim-msgtransfer/Chart.yaml | 53 - .../openim-msgtransfer/templates/NOTES.txt | 22 - .../openim-msgtransfer/templates/_helpers.tpl | 62 - .../templates/deployment.yaml | 86 -- .../openim-msgtransfer/templates/hpa.yaml | 42 - .../openim-msgtransfer/templates/ingress.yaml | 75 -- .../openim-msgtransfer/templates/service.yaml | 29 - .../templates/serviceaccount.yaml | 26 - .../charts/openim-msgtransfer/values.yaml | 96 -- deployments/charts/openim-push/.helmignore | 23 - deployments/charts/openim-push/Chart.yaml | 53 - .../charts/openim-push/templates/NOTES.txt | 22 - .../charts/openim-push/templates/_helpers.tpl | 62 - .../openim-push/templates/deployment.yaml | 86 -- .../charts/openim-push/templates/hpa.yaml | 42 - .../charts/openim-push/templates/ingress.yaml | 75 -- .../charts/openim-push/templates/service.yaml | 29 - .../openim-push/templates/serviceaccount.yaml | 26 - deployments/charts/openim-push/values.yaml | 96 -- .../charts/openim-rpc-auth/.helmignore | 23 - deployments/charts/openim-rpc-auth/Chart.yaml | 53 - .../openim-rpc-auth/templates/NOTES.txt | 22 - .../openim-rpc-auth/templates/_helpers.tpl | 62 - .../openim-rpc-auth/templates/deployment.yaml | 86 -- .../charts/openim-rpc-auth/templates/hpa.yaml | 42 - .../openim-rpc-auth/templates/ingress.yaml | 75 -- .../openim-rpc-auth/templates/service.yaml | 29 - .../templates/serviceaccount.yaml | 26 - .../charts/openim-rpc-auth/values.yaml | 96 -- .../openim-rpc-conversation/.helmignore | 23 - .../charts/openim-rpc-conversation/Chart.yaml | 53 - .../templates/NOTES.txt | 22 - .../templates/_helpers.tpl | 62 - .../templates/deployment.yaml | 86 -- .../templates/hpa.yaml | 42 - .../templates/ingress.yaml | 75 -- .../templates/service.yaml | 29 - .../templates/serviceaccount.yaml | 26 - .../openim-rpc-conversation/values.yaml | 96 -- .../charts/openim-rpc-friend/.helmignore | 23 - .../charts/openim-rpc-friend/Chart.yaml | 53 - .../openim-rpc-friend/templates/NOTES.txt | 22 - .../openim-rpc-friend/templates/_helpers.tpl | 62 - .../templates/deployment.yaml | 86 -- .../openim-rpc-friend/templates/hpa.yaml | 42 - .../openim-rpc-friend/templates/ingress.yaml | 75 -- .../openim-rpc-friend/templates/service.yaml | 29 - .../templates/serviceaccount.yaml | 26 - .../charts/openim-rpc-friend/values.yaml | 96 -- .../charts/openim-rpc-group/.helmignore | 23 - .../charts/openim-rpc-group/Chart.yaml | 53 - .../openim-rpc-group/templates/NOTES.txt | 22 - .../openim-rpc-group/templates/_helpers.tpl | 62 - .../templates/deployment.yaml | 86 -- .../openim-rpc-group/templates/hpa.yaml | 42 - .../openim-rpc-group/templates/ingress.yaml | 75 -- .../openim-rpc-group/templates/service.yaml | 29 - .../templates/serviceaccount.yaml | 26 - .../charts/openim-rpc-group/values.yaml | 96 -- deployments/charts/openim-rpc-msg/.helmignore | 23 - deployments/charts/openim-rpc-msg/Chart.yaml | 53 - .../charts/openim-rpc-msg/templates/NOTES.txt | 22 - .../openim-rpc-msg/templates/_helpers.tpl | 62 - .../openim-rpc-msg/templates/deployment.yaml | 86 -- .../charts/openim-rpc-msg/templates/hpa.yaml | 42 - .../openim-rpc-msg/templates/ingress.yaml | 75 -- .../openim-rpc-msg/templates/service.yaml | 29 - .../templates/serviceaccount.yaml | 26 - deployments/charts/openim-rpc-msg/values.yaml | 96 -- .../charts/openim-rpc-third/.helmignore | 23 - .../charts/openim-rpc-third/Chart.yaml | 53 - .../openim-rpc-third/templates/NOTES.txt | 22 - .../openim-rpc-third/templates/_helpers.tpl | 62 - .../templates/deployment.yaml | 86 -- .../openim-rpc-third/templates/hpa.yaml | 42 - .../openim-rpc-third/templates/ingress.yaml | 75 -- .../openim-rpc-third/templates/service.yaml | 29 - .../templates/serviceaccount.yaml | 26 - .../charts/openim-rpc-third/values.yaml | 96 -- .../charts/openim-rpc-user/.helmignore | 23 - deployments/charts/openim-rpc-user/Chart.yaml | 53 - .../openim-rpc-user/templates/NOTES.txt | 22 - .../openim-rpc-user/templates/_helpers.tpl | 62 - .../openim-rpc-user/templates/deployment.yaml | 86 -- .../charts/openim-rpc-user/templates/hpa.yaml | 42 - .../openim-rpc-user/templates/ingress.yaml | 75 -- .../openim-rpc-user/templates/service.yaml | 29 - .../templates/serviceaccount.yaml | 26 - .../charts/openim-rpc-user/values.yaml | 96 -- deployments/deploy/README.md | 85 ++ deployments/deploy/clusterRole.yml | 24 + deployments/deploy/ingress.yml | 25 + deployments/deploy/kafka-service.yml | 20 + deployments/deploy/kafka-statefulset.yml | 71 ++ deployments/deploy/minio-service.yml | 18 + deployments/deploy/minio-statefulset.yml | 87 ++ deployments/deploy/mongo-service.yml | 13 + deployments/deploy/mongo-statefulset.yml | 95 ++ deployments/deploy/openim-api-deployment.yml | 49 + deployments/deploy/openim-api-service.yml | 17 + deployments/deploy/openim-config.yml | 1070 +++++++++++++++++ .../deploy/openim-crontask-deployment.yml | 28 + .../deploy/openim-msggateway-deployment.yml | 36 + .../deploy/openim-msggateway-service.yml | 21 + .../deploy/openim-msgtransfer-deployment.yml | 40 + .../deploy/openim-msgtransfer-service.yml | 13 + deployments/deploy/openim-push-deployment.yml | 36 + deployments/deploy/openim-push-service.yml | 17 + .../deploy/openim-rpc-auth-deployment.yml | 37 + .../deploy/openim-rpc-auth-service.yml | 17 + .../openim-rpc-conversation-deployment.yml | 41 + .../openim-rpc-conversation-service.yml | 17 + .../deploy/openim-rpc-friend-deployment.yml | 41 + .../deploy/openim-rpc-friend-service.yml | 17 + .../deploy/openim-rpc-group-deployment.yml | 42 + .../deploy/openim-rpc-group-service.yml | 17 + .../deploy/openim-rpc-msg-deployment.yml | 41 + deployments/deploy/openim-rpc-msg-service.yml | 17 + .../deploy/openim-rpc-third-deployment.yml | 51 + .../deploy/openim-rpc-third-service.yml | 17 + .../deploy/openim-rpc-user-deployment.yml | 41 + .../deploy/openim-rpc-user-service.yml | 17 + deployments/deploy/redis-service.yml | 15 + deployments/deploy/redis-statefulset.yml | 66 + deployments/templates/alertmanager.yml | 47 - deployments/templates/charts-value.yaml | 110 -- deployments/templates/config.yaml | 573 --------- deployments/templates/email.tmpl | 16 - deployments/templates/env-template.yaml | 237 ---- deployments/templates/helm-image.yaml | 103 -- deployments/templates/instance-down-rules.yml | 36 - deployments/templates/notification.yaml | 354 ------ deployments/templates/openim-api.yaml | 17 - deployments/templates/openim.service | 13 - deployments/templates/openim.target | 7 - deployments/templates/prometheus.yml | 99 -- go.mod | 2 +- go.sum | 4 +- internal/api/init.go | 10 +- internal/api/prometheus_discovery.go | 9 +- internal/msggateway/ws_server.go | 9 +- internal/msgtransfer/init.go | 18 +- internal/push/onlinepusher.go | 20 +- internal/push/push.go | 5 + magefile_unix.go | 3 +- pkg/common/config/constant.go | 1 + pkg/common/config/load_config_test.go | 2 +- .../discoveryregister/discoveryregister.go | 17 +- .../kubernetes/kubernetes.go | 166 ++- pkg/common/discoveryregister/zookeeper/doc.go | 15 - pkg/common/startrpc/start.go | 8 +- tools/check-component/main.go | 10 +- tools/seq/internal/{main.go => seq.go} | 24 +- 192 files changed, 2517 insertions(+), 7894 deletions(-) create mode 100644 .github/workflows/docker-build-and-release.yml delete mode 100644 deployments/charts/helmfile.yaml delete mode 100644 deployments/charts/openim-api/.helmignore delete mode 100644 deployments/charts/openim-api/Chart.yaml delete mode 100644 deployments/charts/openim-api/LICENSE delete mode 100644 deployments/charts/openim-api/templates/NOTES.txt delete mode 100644 deployments/charts/openim-api/templates/_helpers.tpl delete mode 100644 deployments/charts/openim-api/templates/app-cm.yaml delete mode 100644 deployments/charts/openim-api/templates/deployment.yaml delete mode 100644 deployments/charts/openim-api/templates/hpa.yaml delete mode 100644 deployments/charts/openim-api/templates/ingress.yaml delete mode 100644 deployments/charts/openim-api/templates/service.yaml delete mode 100644 deployments/charts/openim-api/templates/serviceaccount.yaml delete mode 100644 deployments/charts/openim-api/values.yaml delete mode 100644 deployments/charts/openim-msggateway/.helmignore delete mode 100644 deployments/charts/openim-msggateway/Chart.yaml delete mode 100644 deployments/charts/openim-msggateway/templates/NOTES.txt delete mode 100644 deployments/charts/openim-msggateway/templates/_helpers.tpl delete mode 100644 deployments/charts/openim-msggateway/templates/deployment.yaml delete mode 100644 deployments/charts/openim-msggateway/templates/hpa.yaml delete mode 100644 deployments/charts/openim-msggateway/templates/ingress.yaml delete mode 100644 deployments/charts/openim-msggateway/templates/service.yaml delete mode 100644 deployments/charts/openim-msggateway/templates/serviceaccount.yaml delete mode 100644 deployments/charts/openim-msggateway/values.yaml delete mode 100644 deployments/charts/openim-msgtransfer/.helmignore delete mode 100644 deployments/charts/openim-msgtransfer/Chart.yaml delete mode 100644 deployments/charts/openim-msgtransfer/templates/NOTES.txt delete mode 100644 deployments/charts/openim-msgtransfer/templates/_helpers.tpl delete mode 100644 deployments/charts/openim-msgtransfer/templates/deployment.yaml delete mode 100644 deployments/charts/openim-msgtransfer/templates/hpa.yaml delete mode 100644 deployments/charts/openim-msgtransfer/templates/ingress.yaml delete mode 100644 deployments/charts/openim-msgtransfer/templates/service.yaml delete mode 100644 deployments/charts/openim-msgtransfer/templates/serviceaccount.yaml delete mode 100644 deployments/charts/openim-msgtransfer/values.yaml delete mode 100644 deployments/charts/openim-push/.helmignore delete mode 100644 deployments/charts/openim-push/Chart.yaml delete mode 100644 deployments/charts/openim-push/templates/NOTES.txt delete mode 100644 deployments/charts/openim-push/templates/_helpers.tpl delete mode 100644 deployments/charts/openim-push/templates/deployment.yaml delete mode 100644 deployments/charts/openim-push/templates/hpa.yaml delete mode 100644 deployments/charts/openim-push/templates/ingress.yaml delete mode 100644 deployments/charts/openim-push/templates/service.yaml delete mode 100644 deployments/charts/openim-push/templates/serviceaccount.yaml delete mode 100644 deployments/charts/openim-push/values.yaml delete mode 100644 deployments/charts/openim-rpc-auth/.helmignore delete mode 100644 deployments/charts/openim-rpc-auth/Chart.yaml delete mode 100644 deployments/charts/openim-rpc-auth/templates/NOTES.txt delete mode 100644 deployments/charts/openim-rpc-auth/templates/_helpers.tpl delete mode 100644 deployments/charts/openim-rpc-auth/templates/deployment.yaml delete mode 100644 deployments/charts/openim-rpc-auth/templates/hpa.yaml delete mode 100644 deployments/charts/openim-rpc-auth/templates/ingress.yaml delete mode 100644 deployments/charts/openim-rpc-auth/templates/service.yaml delete mode 100644 deployments/charts/openim-rpc-auth/templates/serviceaccount.yaml delete mode 100644 deployments/charts/openim-rpc-auth/values.yaml delete mode 100644 deployments/charts/openim-rpc-conversation/.helmignore delete mode 100644 deployments/charts/openim-rpc-conversation/Chart.yaml delete mode 100644 deployments/charts/openim-rpc-conversation/templates/NOTES.txt delete mode 100644 deployments/charts/openim-rpc-conversation/templates/_helpers.tpl delete mode 100644 deployments/charts/openim-rpc-conversation/templates/deployment.yaml delete mode 100644 deployments/charts/openim-rpc-conversation/templates/hpa.yaml delete mode 100644 deployments/charts/openim-rpc-conversation/templates/ingress.yaml delete mode 100644 deployments/charts/openim-rpc-conversation/templates/service.yaml delete mode 100644 deployments/charts/openim-rpc-conversation/templates/serviceaccount.yaml delete mode 100644 deployments/charts/openim-rpc-conversation/values.yaml delete mode 100644 deployments/charts/openim-rpc-friend/.helmignore delete mode 100644 deployments/charts/openim-rpc-friend/Chart.yaml delete mode 100644 deployments/charts/openim-rpc-friend/templates/NOTES.txt delete mode 100644 deployments/charts/openim-rpc-friend/templates/_helpers.tpl delete mode 100644 deployments/charts/openim-rpc-friend/templates/deployment.yaml delete mode 100644 deployments/charts/openim-rpc-friend/templates/hpa.yaml delete mode 100644 deployments/charts/openim-rpc-friend/templates/ingress.yaml delete mode 100644 deployments/charts/openim-rpc-friend/templates/service.yaml delete mode 100644 deployments/charts/openim-rpc-friend/templates/serviceaccount.yaml delete mode 100644 deployments/charts/openim-rpc-friend/values.yaml delete mode 100644 deployments/charts/openim-rpc-group/.helmignore delete mode 100644 deployments/charts/openim-rpc-group/Chart.yaml delete mode 100644 deployments/charts/openim-rpc-group/templates/NOTES.txt delete mode 100644 deployments/charts/openim-rpc-group/templates/_helpers.tpl delete mode 100644 deployments/charts/openim-rpc-group/templates/deployment.yaml delete mode 100644 deployments/charts/openim-rpc-group/templates/hpa.yaml delete mode 100644 deployments/charts/openim-rpc-group/templates/ingress.yaml delete mode 100644 deployments/charts/openim-rpc-group/templates/service.yaml delete mode 100644 deployments/charts/openim-rpc-group/templates/serviceaccount.yaml delete mode 100644 deployments/charts/openim-rpc-group/values.yaml delete mode 100644 deployments/charts/openim-rpc-msg/.helmignore delete mode 100644 deployments/charts/openim-rpc-msg/Chart.yaml delete mode 100644 deployments/charts/openim-rpc-msg/templates/NOTES.txt delete mode 100644 deployments/charts/openim-rpc-msg/templates/_helpers.tpl delete mode 100644 deployments/charts/openim-rpc-msg/templates/deployment.yaml delete mode 100644 deployments/charts/openim-rpc-msg/templates/hpa.yaml delete mode 100644 deployments/charts/openim-rpc-msg/templates/ingress.yaml delete mode 100644 deployments/charts/openim-rpc-msg/templates/service.yaml delete mode 100644 deployments/charts/openim-rpc-msg/templates/serviceaccount.yaml delete mode 100644 deployments/charts/openim-rpc-msg/values.yaml delete mode 100644 deployments/charts/openim-rpc-third/.helmignore delete mode 100644 deployments/charts/openim-rpc-third/Chart.yaml delete mode 100644 deployments/charts/openim-rpc-third/templates/NOTES.txt delete mode 100644 deployments/charts/openim-rpc-third/templates/_helpers.tpl delete mode 100644 deployments/charts/openim-rpc-third/templates/deployment.yaml delete mode 100644 deployments/charts/openim-rpc-third/templates/hpa.yaml delete mode 100644 deployments/charts/openim-rpc-third/templates/ingress.yaml delete mode 100644 deployments/charts/openim-rpc-third/templates/service.yaml delete mode 100644 deployments/charts/openim-rpc-third/templates/serviceaccount.yaml delete mode 100644 deployments/charts/openim-rpc-third/values.yaml delete mode 100644 deployments/charts/openim-rpc-user/.helmignore delete mode 100644 deployments/charts/openim-rpc-user/Chart.yaml delete mode 100644 deployments/charts/openim-rpc-user/templates/NOTES.txt delete mode 100644 deployments/charts/openim-rpc-user/templates/_helpers.tpl delete mode 100644 deployments/charts/openim-rpc-user/templates/deployment.yaml delete mode 100644 deployments/charts/openim-rpc-user/templates/hpa.yaml delete mode 100644 deployments/charts/openim-rpc-user/templates/ingress.yaml delete mode 100644 deployments/charts/openim-rpc-user/templates/service.yaml delete mode 100644 deployments/charts/openim-rpc-user/templates/serviceaccount.yaml delete mode 100644 deployments/charts/openim-rpc-user/values.yaml create mode 100644 deployments/deploy/README.md create mode 100644 deployments/deploy/clusterRole.yml create mode 100644 deployments/deploy/ingress.yml create mode 100644 deployments/deploy/kafka-service.yml create mode 100644 deployments/deploy/kafka-statefulset.yml create mode 100644 deployments/deploy/minio-service.yml create mode 100644 deployments/deploy/minio-statefulset.yml create mode 100644 deployments/deploy/mongo-service.yml create mode 100644 deployments/deploy/mongo-statefulset.yml create mode 100644 deployments/deploy/openim-api-deployment.yml create mode 100644 deployments/deploy/openim-api-service.yml create mode 100644 deployments/deploy/openim-config.yml create mode 100644 deployments/deploy/openim-crontask-deployment.yml create mode 100644 deployments/deploy/openim-msggateway-deployment.yml create mode 100644 deployments/deploy/openim-msggateway-service.yml create mode 100644 deployments/deploy/openim-msgtransfer-deployment.yml create mode 100644 deployments/deploy/openim-msgtransfer-service.yml create mode 100644 deployments/deploy/openim-push-deployment.yml create mode 100644 deployments/deploy/openim-push-service.yml create mode 100644 deployments/deploy/openim-rpc-auth-deployment.yml create mode 100644 deployments/deploy/openim-rpc-auth-service.yml create mode 100644 deployments/deploy/openim-rpc-conversation-deployment.yml create mode 100644 deployments/deploy/openim-rpc-conversation-service.yml create mode 100644 deployments/deploy/openim-rpc-friend-deployment.yml create mode 100644 deployments/deploy/openim-rpc-friend-service.yml create mode 100644 deployments/deploy/openim-rpc-group-deployment.yml create mode 100644 deployments/deploy/openim-rpc-group-service.yml create mode 100644 deployments/deploy/openim-rpc-msg-deployment.yml create mode 100644 deployments/deploy/openim-rpc-msg-service.yml create mode 100644 deployments/deploy/openim-rpc-third-deployment.yml create mode 100644 deployments/deploy/openim-rpc-third-service.yml create mode 100644 deployments/deploy/openim-rpc-user-deployment.yml create mode 100644 deployments/deploy/openim-rpc-user-service.yml create mode 100644 deployments/deploy/redis-service.yml create mode 100644 deployments/deploy/redis-statefulset.yml delete mode 100644 deployments/templates/alertmanager.yml delete mode 100644 deployments/templates/charts-value.yaml delete mode 100644 deployments/templates/config.yaml delete mode 100644 deployments/templates/email.tmpl delete mode 100644 deployments/templates/env-template.yaml delete mode 100644 deployments/templates/helm-image.yaml delete mode 100644 deployments/templates/instance-down-rules.yml delete mode 100644 deployments/templates/notification.yaml delete mode 100644 deployments/templates/openim-api.yaml delete mode 100644 deployments/templates/openim.service delete mode 100644 deployments/templates/openim.target delete mode 100644 deployments/templates/prometheus.yml delete mode 100644 pkg/common/discoveryregister/zookeeper/doc.go rename tools/seq/internal/{main.go => seq.go} (98%) diff --git a/.github/workflows/docker-build-and-release.yml b/.github/workflows/docker-build-and-release.yml new file mode 100644 index 000000000..76602c4d1 --- /dev/null +++ b/.github/workflows/docker-build-and-release.yml @@ -0,0 +1,91 @@ +name: Build and release services Docker Images + +on: + push: + branches: + - release-* + release: + types: [published] + workflow_dispatch: + inputs: + tag: + description: "Tag version to be used for Docker image" + required: true + default: "v3.8.3" + +jobs: + build-and-push: + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Log in to Docker Hub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + + - name: Log in to GitHub Container Registry + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Log in to Aliyun Container Registry + uses: docker/login-action@v2 + with: + registry: registry.cn-hangzhou.aliyuncs.com + username: ${{ secrets.ALIREGISTRY_USERNAME }} + password: ${{ secrets.ALIREGISTRY_TOKEN }} + + - name: Extract metadata for Docker (tags, labels) + id: meta + uses: docker/metadata-action@v5 + with: + tags: | + type=ref,event=tag + type=schedule + type=ref,event=branch + type=semver,pattern={{version}} + type=semver,pattern=v{{version}} + # type=semver,pattern={{major}}.{{minor}} + type=semver,pattern=release-{{raw}} + type=sha + type=raw,value=${{ github.event.inputs.tag }} + + - name: Build and push Docker images + run: | + ROOT_DIR="build/images" + for dir in "$ROOT_DIR"/*/; do + # Find Dockerfile or *.dockerfile in a case-insensitive manner + dockerfile=$(find "$dir" -maxdepth 1 -type f \( -iname 'dockerfile' -o -iname '*.dockerfile' \) | head -n 1) + + if [ -n "$dockerfile" ] && [ -f "$dockerfile" ]; then + IMAGE_NAME=$(basename "$dir") + echo "Building Docker image for $IMAGE_NAME with tags:" + + # Initialize tag arguments + tag_args=() + + # Read each tag and append --tag arguments + while IFS= read -r tag; do + tag_args+=(--tag "${{ secrets.DOCKER_USERNAME }}/$IMAGE_NAME:$tag") + tag_args+=(--tag "ghcr.io/${{ github.repository_owner }}/$IMAGE_NAME:$tag") + tag_args+=(--tag "registry.cn-hangzhou.aliyuncs.com/openimsdk/$IMAGE_NAME:$tag") + done <<< "${{ steps.meta.outputs.tags }}" + + # Build and push the Docker image with all tags + docker buildx build --platform linux/amd64,linux/arm64 \ + --file "$dockerfile" \ + "${tag_args[@]}" \ + --push "$dir" + else + echo "No valid Dockerfile found in $dir" + fi + done \ No newline at end of file diff --git a/build/images/openim-crontask/Dockerfile b/build/images/openim-crontask/Dockerfile index 9ac0d8eca..863ef4e65 100644 --- a/build/images/openim-crontask/Dockerfile +++ b/build/images/openim-crontask/Dockerfile @@ -36,4 +36,4 @@ COPY --from=builder $SERVER_DIR/_output $SERVER_DIR/_output # COPY --from=builder $SERVER_DIR/config $SERVER_DIR/config # Set the command to run when the container starts -ENTRYPOINT ["sh", "-c", "_output/openim-crontask"] \ No newline at end of file +ENTRYPOINT ["sh", "-c", "_output/openim-crontask"] diff --git a/build/images/openim-msggateway/Dockerfile b/build/images/openim-msggateway/Dockerfile index 90b9c1fa6..981c8215b 100644 --- a/build/images/openim-msggateway/Dockerfile +++ b/build/images/openim-msggateway/Dockerfile @@ -36,4 +36,4 @@ COPY --from=builder $SERVER_DIR/_output $SERVER_DIR/_output # COPY --from=builder $SERVER_DIR/config $SERVER_DIR/config # Set the command to run when the container starts -ENTRYPOINT ["sh", "-c", "_output/openim-msggateway"] \ No newline at end of file +ENTRYPOINT ["sh", "-c", "_output/openim-msggateway"] diff --git a/build/images/openim-msgtransfer/Dockerfile b/build/images/openim-msgtransfer/Dockerfile index 4c679863b..b765b01ed 100644 --- a/build/images/openim-msgtransfer/Dockerfile +++ b/build/images/openim-msgtransfer/Dockerfile @@ -36,4 +36,4 @@ COPY --from=builder $SERVER_DIR/_output $SERVER_DIR/_output # COPY --from=builder $SERVER_DIR/config $SERVER_DIR/config # Set the command to run when the container starts -ENTRYPOINT ["sh", "-c", "_output/openim-msgtransfer"] \ No newline at end of file +ENTRYPOINT ["sh", "-c", "_output/openim-msgtransfer"] diff --git a/build/images/openim-push/Dockerfile b/build/images/openim-push/Dockerfile index 68b711025..ee5ede79a 100644 --- a/build/images/openim-push/Dockerfile +++ b/build/images/openim-push/Dockerfile @@ -36,4 +36,4 @@ COPY --from=builder $SERVER_DIR/_output $SERVER_DIR/_output # COPY --from=builder $SERVER_DIR/config $SERVER_DIR/config # Set the command to run when the container starts -ENTRYPOINT ["sh", "-c", "_output/openim-push"] \ No newline at end of file +ENTRYPOINT ["sh", "-c", "_output/openim-push"] diff --git a/build/images/openim-rpc-auth/Dockerfile b/build/images/openim-rpc-auth/Dockerfile index 0f9383805..b09258eee 100644 --- a/build/images/openim-rpc-auth/Dockerfile +++ b/build/images/openim-rpc-auth/Dockerfile @@ -36,4 +36,4 @@ COPY --from=builder $SERVER_DIR/_output $SERVER_DIR/_output # COPY --from=builder $SERVER_DIR/config $SERVER_DIR/config # Set the command to run when the container starts -ENTRYPOINT ["sh", "-c", "_output/openim-rpc-auth"] \ No newline at end of file +ENTRYPOINT ["sh", "-c", "_output/openim-rpc-auth"] diff --git a/build/images/openim-rpc-conversation/Dockerfile b/build/images/openim-rpc-conversation/Dockerfile index 2c3578c0c..ff6b315b2 100644 --- a/build/images/openim-rpc-conversation/Dockerfile +++ b/build/images/openim-rpc-conversation/Dockerfile @@ -36,4 +36,4 @@ COPY --from=builder $SERVER_DIR/_output $SERVER_DIR/_output # COPY --from=builder $SERVER_DIR/config $SERVER_DIR/config # Set the command to run when the container starts -ENTRYPOINT ["sh", "-c", "_output/openim-rpc-conversation"] \ No newline at end of file +ENTRYPOINT ["sh", "-c", "_output/openim-rpc-conversation"] diff --git a/build/images/openim-rpc-friend/Dockerfile b/build/images/openim-rpc-friend/Dockerfile index 7aa4b6a54..540678310 100644 --- a/build/images/openim-rpc-friend/Dockerfile +++ b/build/images/openim-rpc-friend/Dockerfile @@ -36,4 +36,4 @@ COPY --from=builder $SERVER_DIR/_output $SERVER_DIR/_output # COPY --from=builder $SERVER_DIR/config $SERVER_DIR/config # Set the command to run when the container starts -ENTRYPOINT ["sh", "-c", "_output/openim-rpc-friend"] \ No newline at end of file +ENTRYPOINT ["sh", "-c", "_output/openim-rpc-friend"] diff --git a/build/images/openim-rpc-group/Dockerfile b/build/images/openim-rpc-group/Dockerfile index d6a4e64e0..d1aab9ad9 100644 --- a/build/images/openim-rpc-group/Dockerfile +++ b/build/images/openim-rpc-group/Dockerfile @@ -36,4 +36,4 @@ COPY --from=builder $SERVER_DIR/_output $SERVER_DIR/_output # COPY --from=builder $SERVER_DIR/config $SERVER_DIR/config # Set the command to run when the container starts -ENTRYPOINT ["sh", "-c", "_output/openim-rpc-group"] \ No newline at end of file +ENTRYPOINT ["sh", "-c", "_output/openim-rpc-group"] diff --git a/build/images/openim-rpc-msg/Dockerfile b/build/images/openim-rpc-msg/Dockerfile index 9dac2620a..da40bb293 100644 --- a/build/images/openim-rpc-msg/Dockerfile +++ b/build/images/openim-rpc-msg/Dockerfile @@ -36,4 +36,4 @@ COPY --from=builder $SERVER_DIR/_output $SERVER_DIR/_output # COPY --from=builder $SERVER_DIR/config $SERVER_DIR/config # Set the command to run when the container starts -ENTRYPOINT ["sh", "-c", "_output/openim-rpc-msg"] \ No newline at end of file +ENTRYPOINT ["sh", "-c", "_output/openim-rpc-msg"] diff --git a/build/images/openim-rpc-third/Dockerfile b/build/images/openim-rpc-third/Dockerfile index e68f9fe98..52cdb0de8 100644 --- a/build/images/openim-rpc-third/Dockerfile +++ b/build/images/openim-rpc-third/Dockerfile @@ -36,4 +36,4 @@ COPY --from=builder $SERVER_DIR/_output $SERVER_DIR/_output # COPY --from=builder $SERVER_DIR/config $SERVER_DIR/config # Set the command to run when the container starts -ENTRYPOINT ["sh", "-c", "_output/openim-rpc-third"] \ No newline at end of file +ENTRYPOINT ["sh", "-c", "_output/openim-rpc-third"] diff --git a/build/images/openim-rpc-user/Dockerfile b/build/images/openim-rpc-user/Dockerfile index 42f9ceec6..1b3481c66 100644 --- a/build/images/openim-rpc-user/Dockerfile +++ b/build/images/openim-rpc-user/Dockerfile @@ -34,4 +34,4 @@ COPY --from=builder $SERVER_DIR/_output $SERVER_DIR/_output # COPY --from=builder $SERVER_DIR/config $SERVER_DIR/config # Set the command to run when the container starts -ENTRYPOINT ["sh", "-c", "_output/openim-rpc-user"] \ No newline at end of file +ENTRYPOINT ["sh", "-c", "_output/openim-rpc-user"] diff --git a/config/discovery.yml b/config/discovery.yml index a04d7e40f..e8d733e9f 100644 --- a/config/discovery.yml +++ b/config/discovery.yml @@ -13,7 +13,7 @@ rpcService: friend: friend-rpc-service msg: msg-rpc-service push: push-rpc-service - messageGateway: messageGateway-rpc-service + messageGateway: messagegateway-rpc-service group: group-rpc-service auth: auth-rpc-service conversation: conversation-rpc-service diff --git a/config/openim-rpc-auth.yml b/config/openim-rpc-auth.yml index 961244861..5d6d85b2f 100644 --- a/config/openim-rpc-auth.yml +++ b/config/openim-rpc-auth.yml @@ -10,7 +10,6 @@ rpc: # It will only take effect when autoSetPorts is set to false. ports: [ 10200 ] - prometheus: # Enable or disable Prometheus monitoring enable: true diff --git a/deployments/Readme.md b/deployments/Readme.md index a7b288130..775baaea0 100644 --- a/deployments/Readme.md +++ b/deployments/Readme.md @@ -1,175 +1,3 @@ # OpenIM Application Containerization Deployment Guide -OpenIM supports a variety of cluster deployment methods, including but not limited to `helm`, `sealos`, `kustomize` - -Various contributors, as well as previous official releases, have provided some referenceable solutions: - -+ [k8s-jenkins Repository](https://github.com/OpenIMSDK/k8s-jenkins) -+ [open-im-server-k8s-deploy Repository](https://github.com/openimsdk/open-im-server-k8s-deploy) -+ [openim-charts Repository](https://github.com/OpenIMSDK/openim-charts) -+ [deploy-openim Repository](https://github.com/showurl/deploy-openim) - -### Dependency Check - -```bash -Kubernetes: >= 1.16.0-0 -Helm: >= 3.0 -``` - -### Minimum Configuration - -The recommended minimum configuration for a production environment is as follows: - -```yaml -CPU: 4 -Memory: 8G -Disk: 100G -``` - -## Configuration File Generation - -We have automated all the files, making the generation of configuration files optional for OpenIM. However, if you desire custom configurations, you can follow the steps below: - -```bash -$ make init -# Alternatively, use script: -# ./scripts/init-config.sh -``` - -At this point, configuration files will be generated under `deployments/openim/config`, which you can modify as per your requirements. - -## Cluster Setup - -If you already have a `kubernetes` cluster, or if you wish to build a `kubernetes` cluster from scratch, you can skip this step. - -For a quick start, I used [sealos](https://github.com/labring/sealos) to rapidly set up the cluster, with sealos also being a wrapper for kubeadm at its core: - -```bash -$ SEALOS_VERSION=`curl -s https://api.github.com/repos/labring/sealos/releases/latest | grep -oE '"tag_name": "[^"]+"' | head -n1 | cut -d'"' -f4` && \ - curl -sfL https://raw.githubusercontent.com/labring/sealos/${SEALOS_VERSION}/scripts/install.sh | - sh -s ${SEALOS_VERSION} labring/sealos -``` - -**Supported Versions:** - -+ docker: `labring/kubernetes-docker`:(v1.24.0~v1.27.0) -+ containerd: `labring/kubernetes`:(v1.24.0~v1.27.0) - -#### Cluster Installation: - -Cluster details are as follows: - -| Hostname | IP Address | System Info | -| -------- | ---------- | ------------------------------------------------------------ | -| master01 | 10.0.0.9 | `Linux VM-0-9-ubuntu 5.15.0-76-generic #83-Ubuntu SMP Thu Jun 15 19:16:32 UTC 2023 x86_64 x86_64 x86_64 GNU/Linux` | -| node01 | 10.0.0.4 | Similar to master01 | -| node02 | 10.0.0.10 | Similar to master01 | - -```bash -$ export CLUSTER_USERNAME=ubuntu -$ export CLUSTER_PASSWORD=123456 -$ sudo sealos run labring/kubernetes:v1.25.0 labring/helm:v3.8.2 labring/calico:v3.24.1 \ - --masters 10.0.0.9 \ - --nodes 10.0.0.4,10.0.0.10 \ - -u "$CLUSTER_USERNAME" \ - -p "$CLUSTER_PASSWORD" -``` - -> **Node** Uninstallation method: using `kubeadm` for uninstallation does not remove `etcd` and `cni` related configurations. Manual clearance or using `sealos` for uninstallation is needed. -> -> ```bash -> $ sealos reset -> ``` - -If you are local, you can also use Kind and Minikube to test, for example, using Kind: - -```bash -$ GO111MODULE="on" go get sigs.k8s.io/kind@v0.11.1 -$ kind create cluster -``` - -### Installing helm - -Helm simplifies the deployment and management of Kubernetes applications to a large extent by offering version control and release management through packaging. - -**Using Script:** - -```bash -$ curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash -``` - -**Adding Repository:** - -```bash -$ helm repo add brigade https://openimsdk.github.io/openim-charts -``` - -### OpenIM Image Strategy - -Automated offerings include aliyun, ghcr, docker hub: [Image Documentation](https://github.com/openimsdk/open-im-server/blob/main/docs/contrib/images.md) - -**Local Test Build Method:** - -```bash -$ make image -``` - -> This command assists in quickly building the required images locally. For a detailed build strategy, refer to the [Build Documentation](https://github.com/openimsdk/open-im-server/blob/main/build/README.md). - -## Installation - -Explore our Helm-Charts repository and read through: [Helm-Charts Repository](https://github.com/openimsdk/helm-charts) - - -Using the helm charts repository, you can ignore the following configuration, but if you want to just use the server and scale on top of it, you can go ahead: - -**Use the Helm template to generate the deployment yaml file: `openim-charts.yaml`** - -**Gen Image:** - -```bash -../scripts/genconfig.sh ../scripts/install/environment.sh ./templates/helm-image.yaml > ./charts/generated-configs/helm-image.yaml -``` - -**Gen Charts:** - -```bash -for chart in ./charts/*/; do - if [[ "$chart" == *"generated-configs"* || "$chart" == *"helmfile.yaml"* ]]; then - continue - fi - - if [ -f "${chart}values.yaml" ]; then - helm template "$chart" -f "./charts/generated-configs/helm-image.yaml" -f "./charts/generated-configs/config.yaml" -f "./charts/generated-configs/notification.yaml" >> openim-charts.yaml - else - helm template "$chart" >> openim-charts.yaml - fi -done -``` - -**Use Helmfile:** - -```bash -GO111MODULE=on go get github.com/roboll/helmfile@latest -``` - -```bash -export MONGO_ADDRESS=im-mongo -export MONGO_PORT=27017 -export REDIS_ADDRESS=im-redis-master -export REDIS_PORT=6379 -export KAFKA_ADDRESS=im-kafka -export KAFKA_PORT=9092 -export OBJECT_APIURL="https://openim.server.com/api" -export MINIO_ENDPOINT="http://im-minio:9000" -export MINIO_SIGN_ENDPOINT="https://openim.server.com/im-minio-api" - -mkdir ./charts/generated-configs -../scripts/genconfig.sh ../scripts/install/environment.sh ./templates/config.yaml > ./charts/generated-configs/config.yaml -cp ../config/notification.yaml ./charts/generated-configs/notification.yaml -../scripts/genconfig.sh ../scripts/install/environment.sh ./templates/helm-image.yaml > ./charts/generated-configs/helm-image.yaml -``` - -```bash -helmfile apply -``` +view deploy [README](./deploy/README.md) \ No newline at end of file diff --git a/deployments/charts/helmfile.yaml b/deployments/charts/helmfile.yaml deleted file mode 100644 index eb064be9a..000000000 --- a/deployments/charts/helmfile.yaml +++ /dev/null @@ -1,34 +0,0 @@ -# Copyright © 2023 OpenIM. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -{{- define "defaultValues" -}} -- ./generated-configs/helm-image.yaml -- ./generated-configs/config.yaml -- ./generated-configs/notification.yaml -{{- end -}} - -{{- define "defaultRelease" -}} -namespace: openim -chart: ./{{ .name }} -values: - - ./{{ .name }}/values.yaml - {{- template "defaultValues" . }} -{{- end -}} - -releases: -{{- $apps := list "openim-api" "openim-msggateway" "openim-msgtransfer" "openim-push" "openim-rpc-auth" "openim-rpc-conversation" "openim-rpc-friend" "openim-rpc-group" "openim-rpc-msg" "openim-rpc-third" "openim-rpc-user" }} -{{- range $app := $apps }} - - name: {{ $app }} - {{- template "defaultRelease" dict "name" $app }} -{{- end }} diff --git a/deployments/charts/openim-api/.helmignore b/deployments/charts/openim-api/.helmignore deleted file mode 100644 index 0e8a0eb36..000000000 --- a/deployments/charts/openim-api/.helmignore +++ /dev/null @@ -1,23 +0,0 @@ -# Patterns to ignore when building packages. -# This supports shell glob matching, relative path matching, and -# negation (prefixed with !). Only one pattern per line. -.DS_Store -# Common VCS dirs -.git/ -.gitignore -.bzr/ -.bzrignore -.hg/ -.hgignore -.svn/ -# Common backup files -*.swp -*.bak -*.tmp -*.orig -*~ -# Various IDEs -.project -.idea/ -*.tmproj -.vscode/ diff --git a/deployments/charts/openim-api/Chart.yaml b/deployments/charts/openim-api/Chart.yaml deleted file mode 100644 index e79d2a2c2..000000000 --- a/deployments/charts/openim-api/Chart.yaml +++ /dev/null @@ -1,53 +0,0 @@ -# Copyright © 2023 OpenIM. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -apiVersion: v2 -name: openim-api -description: A OpenIM Api Helm chart for Kubernetes - -# A chart can be either an 'application' or a 'library' chart. -# -# Application charts are a collection of templates that can be packaged into versioned archives -# to be deployed. -# -# Library charts provide useful utilities or functions for the chart developer. They're included as -# a dependency of application charts to inject those utilities and functions into the rendering -# pipeline. Library charts do not define any templates and therefore cannot be deployed. -type: application - -# This is the chart version. This version number should be incremented each time you make changes -# to the chart and its templates, including the app version. -# Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.0 - -# This is the version number of the application being deployed. This version number should be -# incremented each time you make changes to the application. Versions are not expected to -# follow Semantic Versioning. They should reflect the version the application is using. -# It is recommended to use it with quotes. -appVersion: "1.16.0" - -icon: https://raw.githubusercontent.com/openimsdk/open-im-server/main/assets/openim-logo-gradient.svg - -maintainers: - - name: "OpenIM" - url: "https://github.com/openimsdk" - -keywords: - - openim - - im - - chat - -sources: - - "https://github.com/openimsdk/open-im-server" - - "https://github.com/openimsdk/helm-charts" \ No newline at end of file diff --git a/deployments/charts/openim-api/LICENSE b/deployments/charts/openim-api/LICENSE deleted file mode 100644 index 261eeb9e9..000000000 --- a/deployments/charts/openim-api/LICENSE +++ /dev/null @@ -1,201 +0,0 @@ - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/deployments/charts/openim-api/templates/NOTES.txt b/deployments/charts/openim-api/templates/NOTES.txt deleted file mode 100644 index e9ce81bdb..000000000 --- a/deployments/charts/openim-api/templates/NOTES.txt +++ /dev/null @@ -1,22 +0,0 @@ -1. Get the application URL by running these commands: -{{- if .Values.ingress.enabled }} -{{- range $host := .Values.ingress.hosts }} - {{- range .paths }} - http{{ if $.Values.ingress.tls }}s{{ end }}://{{ $host.host }}{{ .path }} - {{- end }} -{{- end }} -{{- else if contains "NodePort" .Values.service.type }} - export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "openim-api.fullname" . }}) - export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}") - echo http://$NODE_IP:$NODE_PORT -{{- else if contains "LoadBalancer" .Values.service.type }} - NOTE: It may take a few minutes for the LoadBalancer IP to be available. - You can watch the status of by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "openim-api.fullname" . }}' - export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "openim-api.fullname" . }} --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}") - echo http://$SERVICE_IP:{{ .Values.service.port }} -{{- else if contains "ClusterIP" .Values.service.type }} - export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "openim-api.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") - export CONTAINER_PORT=$(kubectl get pod --namespace {{ .Release.Namespace }} $POD_NAME -o jsonpath="{.spec.containers[0].ports[0].containerPort}") - echo "Visit http://127.0.0.1:8080 to use your application" - kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:$CONTAINER_PORT -{{- end }} diff --git a/deployments/charts/openim-api/templates/_helpers.tpl b/deployments/charts/openim-api/templates/_helpers.tpl deleted file mode 100644 index f137492f1..000000000 --- a/deployments/charts/openim-api/templates/_helpers.tpl +++ /dev/null @@ -1,62 +0,0 @@ -{{/* -Expand the name of the chart. -*/}} -{{- define "openim-api.name" -}} -{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} -{{- end }} - -{{/* -Create a default fully qualified app name. -We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). -If release name contains chart name it will be used as a full name. -*/}} -{{- define "openim-api.fullname" -}} -{{- if .Values.fullnameOverride }} -{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} -{{- else }} -{{- $name := default .Chart.Name .Values.nameOverride }} -{{- if contains $name .Release.Name }} -{{- .Release.Name | trunc 63 | trimSuffix "-" }} -{{- else }} -{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} -{{- end }} -{{- end }} -{{- end }} - -{{/* -Create chart name and version as used by the chart label. -*/}} -{{- define "openim-api.chart" -}} -{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} -{{- end }} - -{{/* -Common labels -*/}} -{{- define "openim-api.labels" -}} -helm.sh/chart: {{ include "openim-api.chart" . }} -{{ include "openim-api.selectorLabels" . }} -{{- if .Chart.AppVersion }} -app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} -{{- end }} -app.kubernetes.io/managed-by: {{ .Release.Service }} -{{- end }} - -{{/* -Selector labels -*/}} -{{- define "openim-api.selectorLabels" -}} -app.kubernetes.io/name: {{ include "openim-api.name" . }} -app.kubernetes.io/instance: {{ .Release.Name }} -{{- end }} - -{{/* -Create the name of the service account to use -*/}} -{{- define "openim-api.serviceAccountName" -}} -{{- if .Values.serviceAccount.create }} -{{- default (include "openim-api.fullname" .) .Values.serviceAccount.name }} -{{- else }} -{{- default "default" .Values.serviceAccount.name }} -{{- end }} -{{- end }} diff --git a/deployments/charts/openim-api/templates/app-cm.yaml b/deployments/charts/openim-api/templates/app-cm.yaml deleted file mode 100644 index 9d4e96e8b..000000000 --- a/deployments/charts/openim-api/templates/app-cm.yaml +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright © 2023 OpenIM. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -apiVersion: v1 -kind: ConfigMap -metadata: - name: openim-cm -data: - config.yaml: |+ - {{- with .Values.config }} - {{- toYaml . | nindent 4 }} - {{- end }} - notification.yaml: |+ - {{- with .Values.notification }} - {{- toYaml . | nindent 4 }} - {{- end }} diff --git a/deployments/charts/openim-api/templates/deployment.yaml b/deployments/charts/openim-api/templates/deployment.yaml deleted file mode 100644 index b0076393f..000000000 --- a/deployments/charts/openim-api/templates/deployment.yaml +++ /dev/null @@ -1,86 +0,0 @@ -# Copyright © 2023 OpenIM. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ include "openim-api.fullname" . }} - labels: - {{- include "openim-api.labels" . | nindent 4 }} -spec: - {{- if not .Values.autoscaling.enabled }} - replicas: {{ .Values.replicaCount }} - {{- end }} - selector: - matchLabels: - {{- include "openim-api.selectorLabels" . | nindent 6 }} - template: - metadata: - {{- with .Values.podAnnotations }} - annotations: - {{- toYaml . | nindent 8 }} - {{- end }} - labels: - {{- include "openim-api.selectorLabels" . | nindent 8 }} - spec: - {{- with .Values.imagePullSecrets }} - imagePullSecrets: - {{- toYaml . | nindent 8 }} - {{- end }} - serviceAccountName: {{ include "openim-api.serviceAccountName" . }} - securityContext: - {{- toYaml .Values.podSecurityContext | nindent 8 }} - containers: - - name: {{ .Chart.Name }} - securityContext: - {{- toYaml .Values.securityContext | nindent 12 }} - image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" - imagePullPolicy: {{ .Values.image.pullPolicy }} - ports: - - name: webhook - containerPort: 80 - protocol: TCP - #livenessProbe: - # httpGet: - # path: / - # port: webhook - #readinessProbe: - # httpGet: - # path: / - # port: webhook - resources: - {{- toYaml .Values.resources | nindent 12 }} - volumeMounts: - - mountPath: /openim/openim-server/config/config.yaml - name: config - subPath: config.yaml - - mountPath: /openim/openim-server/config/ - name: config - subPath: notification.yaml - volumes: - - name: config - configMap: - name: openim-cm - {{- with .Values.nodeSelector }} - nodeSelector: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.affinity }} - affinity: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.tolerations }} - tolerations: - {{- toYaml . | nindent 8 }} - {{- end }} diff --git a/deployments/charts/openim-api/templates/hpa.yaml b/deployments/charts/openim-api/templates/hpa.yaml deleted file mode 100644 index dc0742a25..000000000 --- a/deployments/charts/openim-api/templates/hpa.yaml +++ /dev/null @@ -1,42 +0,0 @@ -# Copyright © 2023 OpenIM. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -{{- if .Values.autoscaling.enabled }} -apiVersion: autoscaling/v2beta1 -kind: HorizontalPodAutoscaler -metadata: - name: {{ include "openim-api.fullname" . }} - labels: - {{- include "openim-api.labels" . | nindent 4 }} -spec: - scaleTargetRef: - apiVersion: apps/v1 - kind: Deployment - name: {{ include "openim-api.fullname" . }} - minReplicas: {{ .Values.autoscaling.minReplicas }} - maxReplicas: {{ .Values.autoscaling.maxReplicas }} - metrics: - {{- if .Values.autoscaling.targetCPUUtilizationPercentage }} - - type: Resource - resource: - name: cpu - targetAverageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} - {{- end }} - {{- if .Values.autoscaling.targetMemoryUtilizationPercentage }} - - type: Resource - resource: - name: memory - targetAverageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} - {{- end }} -{{- end }} diff --git a/deployments/charts/openim-api/templates/ingress.yaml b/deployments/charts/openim-api/templates/ingress.yaml deleted file mode 100644 index 55bc69710..000000000 --- a/deployments/charts/openim-api/templates/ingress.yaml +++ /dev/null @@ -1,75 +0,0 @@ -# Copyright © 2023 OpenIM. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -{{- if .Values.ingress.enabled -}} -{{- $fullName := include "openim-api.fullname" . -}} -{{- $svcPort := .Values.service.port -}} -{{- if and .Values.ingress.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }} - {{- if not (hasKey .Values.ingress.annotations "kubernetes.io/ingress.class") }} - {{- $_ := set .Values.ingress.annotations "kubernetes.io/ingress.class" .Values.ingress.className}} - {{- end }} -{{- end }} -{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}} -apiVersion: networking.k8s.io/v1 -{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}} -apiVersion: networking.k8s.io/v1beta1 -{{- else -}} -apiVersion: extensions/v1beta1 -{{- end }} -kind: Ingress -metadata: - name: {{ $fullName }} - labels: - {{- include "openim-api.labels" . | nindent 4 }} - {{- with .Values.ingress.annotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -spec: - {{- if and .Values.ingress.className (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }} - ingressClassName: {{ .Values.ingress.className }} - {{- end }} - {{- if .Values.ingress.tls }} - tls: - {{- range .Values.ingress.tls }} - - hosts: - {{- range .hosts }} - - {{ . | quote }} - {{- end }} - secretName: {{ .secretName }} - {{- end }} - {{- end }} - rules: - {{- range .Values.ingress.hosts }} - - host: {{ .host | quote }} - http: - paths: - {{- range .paths }} - - path: {{ .path }} - {{- if and .pathType (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }} - pathType: {{ .pathType }} - {{- end }} - backend: - {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }} - service: - name: {{ $fullName }} - port: - number: {{ $svcPort }} - {{- else }} - serviceName: {{ $fullName }} - servicePort: {{ $svcPort }} - {{- end }} - {{- end }} - {{- end }} -{{- end }} diff --git a/deployments/charts/openim-api/templates/service.yaml b/deployments/charts/openim-api/templates/service.yaml deleted file mode 100644 index 74f75a25e..000000000 --- a/deployments/charts/openim-api/templates/service.yaml +++ /dev/null @@ -1,29 +0,0 @@ -# Copyright © 2023 OpenIM. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -apiVersion: v1 -kind: Service -metadata: - name: {{ include "openim-api.fullname" . }} - labels: - {{- include "openim-api.labels" . | nindent 4 }} -spec: - type: {{ .Values.service.type }} - ports: - - port: {{ .Values.service.port }} - targetPort: webhook - protocol: TCP - name: webhook - selector: - {{- include "openim-api.selectorLabels" . | nindent 4 }} diff --git a/deployments/charts/openim-api/templates/serviceaccount.yaml b/deployments/charts/openim-api/templates/serviceaccount.yaml deleted file mode 100644 index 556a71e34..000000000 --- a/deployments/charts/openim-api/templates/serviceaccount.yaml +++ /dev/null @@ -1,26 +0,0 @@ -# Copyright © 2023 OpenIM. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -{{- if .Values.serviceAccount.create -}} -apiVersion: v1 -kind: ServiceAccount -metadata: - name: {{ include "openim-api.serviceAccountName" . }} - labels: - {{- include "openim-api.labels" . | nindent 4 }} - {{- with .Values.serviceAccount.annotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -{{- end }} diff --git a/deployments/charts/openim-api/values.yaml b/deployments/charts/openim-api/values.yaml deleted file mode 100644 index 4208335de..000000000 --- a/deployments/charts/openim-api/values.yaml +++ /dev/null @@ -1,100 +0,0 @@ -# Copyright © 2023 OpenIM. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Default values for openim-api. -# This is a YAML-formatted file. -# Declare variables to be passed into your templates. - -replicaCount: 1 - -image: - repository: ghcr.io/openimsdk/openim-api - pullPolicy: IfNotPresent - # Overrides the image tag whose default is the chart appVersion. - tag: "" - -imagePullSecrets: [] -nameOverride: "" -fullnameOverride: "" - -serviceAccount: - # Specifies whether a service account should be created - create: false - # Annotations to add to the service account - annotations: {} - # The name of the service account to use. - # If not set and create is true, a name is generated using the fullname template - name: "" - -podAnnotations: {} - -podSecurityContext: {} - # fsGroup: 2000 - -securityContext: {} - # capabilities: - # drop: - # - ALL - # readOnlyRootFilesystem: true - # runAsNonRoot: true - # runAsUser: 1000 - -service: - type: ClusterIP - port: 80 - -ingress: - enabled: false - className: "nginx" - annotations: - # kubernetes.io/ingress.class: nginx - # kubernetes.io/tls-acme: "true" - hosts: - - host: chart-example.local - paths: - - path: / - pathType: ImplementationSpecific - tls: [] - # - secretName: chart-example-tls - # hosts: - # - chart-example.local - -resources: {} - # We usually recommend not to specify default resources and to leave this as a conscious - # choice for the user. This also increases chances charts run on environments with little - # resources, such as Minikube. If you do want to specify resources, uncomment the following - # lines, adjust them as necessary, and remove the curly braces after 'resources:'. - # limits: - # cpu: 100m - # memory: 128Mi - # requests: - # cpu: 100m - # memory: 128Mi - -autoscaling: - enabled: false - minReplicas: 1 - maxReplicas: 100 - targetCPUUtilizationPercentage: 80 - # targetMemoryUtilizationPercentage: 80 - -nodeSelector: {} - -tolerations: [] - -affinity: {} - -notification: - -config: diff --git a/deployments/charts/openim-msggateway/.helmignore b/deployments/charts/openim-msggateway/.helmignore deleted file mode 100644 index 0e8a0eb36..000000000 --- a/deployments/charts/openim-msggateway/.helmignore +++ /dev/null @@ -1,23 +0,0 @@ -# Patterns to ignore when building packages. -# This supports shell glob matching, relative path matching, and -# negation (prefixed with !). Only one pattern per line. -.DS_Store -# Common VCS dirs -.git/ -.gitignore -.bzr/ -.bzrignore -.hg/ -.hgignore -.svn/ -# Common backup files -*.swp -*.bak -*.tmp -*.orig -*~ -# Various IDEs -.project -.idea/ -*.tmproj -.vscode/ diff --git a/deployments/charts/openim-msggateway/Chart.yaml b/deployments/charts/openim-msggateway/Chart.yaml deleted file mode 100644 index fb6d0d153..000000000 --- a/deployments/charts/openim-msggateway/Chart.yaml +++ /dev/null @@ -1,53 +0,0 @@ -# Copyright © 2023 OpenIM. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -apiVersion: v2 -name: openim-msggateway -description: A Helm chart for Kubernetes - -# A chart can be either an 'application' or a 'library' chart. -# -# Application charts are a collection of templates that can be packaged into versioned archives -# to be deployed. -# -# Library charts provide useful utilities or functions for the chart developer. They're included as -# a dependency of application charts to inject those utilities and functions into the rendering -# pipeline. Library charts do not define any templates and therefore cannot be deployed. -type: application - -# This is the chart version. This version number should be incremented each time you make changes -# to the chart and its templates, including the app version. -# Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.0 - -# This is the version number of the application being deployed. This version number should be -# incremented each time you make changes to the application. Versions are not expected to -# follow Semantic Versioning. They should reflect the version the application is using. -# It is recommended to use it with quotes. -appVersion: "1.16.0" - -icon: https://raw.githubusercontent.com/openimsdk/open-im-server/main/assets/openim-logo-gradient.svg - -maintainers: - - name: "OpenIM" - url: "https://github.com/openimsdk" - -keywords: - - openim - - im - - chat - -sources: - - "https://github.com/openimsdk/open-im-server" - - "https://github.com/openimsdk/helm-charts" \ No newline at end of file diff --git a/deployments/charts/openim-msggateway/templates/NOTES.txt b/deployments/charts/openim-msggateway/templates/NOTES.txt deleted file mode 100644 index c1f032586..000000000 --- a/deployments/charts/openim-msggateway/templates/NOTES.txt +++ /dev/null @@ -1,22 +0,0 @@ -1. Get the application URL by running these commands: -{{- if .Values.ingress.enabled }} -{{- range $host := .Values.ingress.hosts }} - {{- range .paths }} - http{{ if $.Values.ingress.tls }}s{{ end }}://{{ $host.host }}{{ .path }} - {{- end }} -{{- end }} -{{- else if contains "NodePort" .Values.service.type }} - export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "openim-msggateway.fullname" . }}) - export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}") - echo http://$NODE_IP:$NODE_PORT -{{- else if contains "LoadBalancer" .Values.service.type }} - NOTE: It may take a few minutes for the LoadBalancer IP to be available. - You can watch the status of by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "openim-msggateway.fullname" . }}' - export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "openim-msggateway.fullname" . }} --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}") - echo http://$SERVICE_IP:{{ .Values.service.port }} -{{- else if contains "ClusterIP" .Values.service.type }} - export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "openim-msggateway.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") - export CONTAINER_PORT=$(kubectl get pod --namespace {{ .Release.Namespace }} $POD_NAME -o jsonpath="{.spec.containers[0].ports[0].containerPort}") - echo "Visit http://127.0.0.1:8080 to use your application" - kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:$CONTAINER_PORT -{{- end }} diff --git a/deployments/charts/openim-msggateway/templates/_helpers.tpl b/deployments/charts/openim-msggateway/templates/_helpers.tpl deleted file mode 100644 index c902a1b30..000000000 --- a/deployments/charts/openim-msggateway/templates/_helpers.tpl +++ /dev/null @@ -1,62 +0,0 @@ -{{/* -Expand the name of the chart. -*/}} -{{- define "openim-msggateway.name" -}} -{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} -{{- end }} - -{{/* -Create a default fully qualified app name. -We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). -If release name contains chart name it will be used as a full name. -*/}} -{{- define "openim-msggateway.fullname" -}} -{{- if .Values.fullnameOverride }} -{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} -{{- else }} -{{- $name := default .Chart.Name .Values.nameOverride }} -{{- if contains $name .Release.Name }} -{{- .Release.Name | trunc 63 | trimSuffix "-" }} -{{- else }} -{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} -{{- end }} -{{- end }} -{{- end }} - -{{/* -Create chart name and version as used by the chart label. -*/}} -{{- define "openim-msggateway.chart" -}} -{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} -{{- end }} - -{{/* -Common labels -*/}} -{{- define "openim-msggateway.labels" -}} -helm.sh/chart: {{ include "openim-msggateway.chart" . }} -{{ include "openim-msggateway.selectorLabels" . }} -{{- if .Chart.AppVersion }} -app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} -{{- end }} -app.kubernetes.io/managed-by: {{ .Release.Service }} -{{- end }} - -{{/* -Selector labels -*/}} -{{- define "openim-msggateway.selectorLabels" -}} -app.kubernetes.io/name: {{ include "openim-msggateway.name" . }} -app.kubernetes.io/instance: {{ .Release.Name }} -{{- end }} - -{{/* -Create the name of the service account to use -*/}} -{{- define "openim-msggateway.serviceAccountName" -}} -{{- if .Values.serviceAccount.create }} -{{- default (include "openim-msggateway.fullname" .) .Values.serviceAccount.name }} -{{- else }} -{{- default "default" .Values.serviceAccount.name }} -{{- end }} -{{- end }} diff --git a/deployments/charts/openim-msggateway/templates/deployment.yaml b/deployments/charts/openim-msggateway/templates/deployment.yaml deleted file mode 100644 index e938fa9bf..000000000 --- a/deployments/charts/openim-msggateway/templates/deployment.yaml +++ /dev/null @@ -1,89 +0,0 @@ -# Copyright © 2023 OpenIM. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ include "openim-msggateway.fullname" . }} - labels: - {{- include "openim-msggateway.labels" . | nindent 4 }} -spec: - {{- if not .Values.autoscaling.enabled }} - replicas: {{ .Values.replicaCount }} - {{- end }} - selector: - matchLabels: - {{- include "openim-msggateway.selectorLabels" . | nindent 6 }} - template: - metadata: - {{- with .Values.podAnnotations }} - annotations: - {{- toYaml . | nindent 8 }} - {{- end }} - labels: - {{- include "openim-msggateway.selectorLabels" . | nindent 8 }} - spec: - {{- with .Values.imagePullSecrets }} - imagePullSecrets: - {{- toYaml . | nindent 8 }} - {{- end }} - serviceAccountName: {{ include "openim-msggateway.serviceAccountName" . }} - securityContext: - {{- toYaml .Values.podSecurityContext | nindent 8 }} - containers: - - name: {{ .Chart.Name }} - securityContext: - {{- toYaml .Values.securityContext | nindent 12 }} - image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" - imagePullPolicy: {{ .Values.image.pullPolicy }} - ports: - - name: webhook - containerPort: 80 - protocol: TCP - - name: rpc - containerPort: 88 - protocol: TCP - #livenessProbe: - # httpGet: - # path: / - # port: webhook - #readinessProbe: - # httpGet: - # path: / - # port: webhook - resources: - {{- toYaml .Values.resources | nindent 12 }} - volumeMounts: - - mountPath: /openim/openim-server/config/config.yaml - name: config - subPath: config.yaml - - mountPath: /openim/openim-server/config/ - name: config - subPath: notification.yaml - volumes: - - name: config - configMap: - name: openim-cm - {{- with .Values.nodeSelector }} - nodeSelector: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.affinity }} - affinity: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.tolerations }} - tolerations: - {{- toYaml . | nindent 8 }} - {{- end }} diff --git a/deployments/charts/openim-msggateway/templates/hpa.yaml b/deployments/charts/openim-msggateway/templates/hpa.yaml deleted file mode 100644 index 99121afba..000000000 --- a/deployments/charts/openim-msggateway/templates/hpa.yaml +++ /dev/null @@ -1,42 +0,0 @@ -# Copyright © 2023 OpenIM. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -{{- if .Values.autoscaling.enabled }} -apiVersion: autoscaling/v2beta1 -kind: HorizontalPodAutoscaler -metadata: - name: {{ include "openim-msggateway.fullname" . }} - labels: - {{- include "openim-msggateway.labels" . | nindent 4 }} -spec: - scaleTargetRef: - apiVersion: apps/v1 - kind: Deployment - name: {{ include "openim-msggateway.fullname" . }} - minReplicas: {{ .Values.autoscaling.minReplicas }} - maxReplicas: {{ .Values.autoscaling.maxReplicas }} - metrics: - {{- if .Values.autoscaling.targetCPUUtilizationPercentage }} - - type: Resource - resource: - name: cpu - targetAverageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} - {{- end }} - {{- if .Values.autoscaling.targetMemoryUtilizationPercentage }} - - type: Resource - resource: - name: memory - targetAverageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} - {{- end }} -{{- end }} diff --git a/deployments/charts/openim-msggateway/templates/ingress.yaml b/deployments/charts/openim-msggateway/templates/ingress.yaml deleted file mode 100644 index 0e22e1936..000000000 --- a/deployments/charts/openim-msggateway/templates/ingress.yaml +++ /dev/null @@ -1,75 +0,0 @@ -# Copyright © 2023 OpenIM. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -{{- if .Values.ingress.enabled -}} -{{- $fullName := include "openim-msggateway.fullname" . -}} -{{- $svcPort := .Values.service.port -}} -{{- if and .Values.ingress.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }} - {{- if not (hasKey .Values.ingress.annotations "kubernetes.io/ingress.class") }} - {{- $_ := set .Values.ingress.annotations "kubernetes.io/ingress.class" .Values.ingress.className}} - {{- end }} -{{- end }} -{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}} -apiVersion: networking.k8s.io/v1 -{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}} -apiVersion: networking.k8s.io/v1beta1 -{{- else -}} -apiVersion: extensions/v1beta1 -{{- end }} -kind: Ingress -metadata: - name: {{ $fullName }} - labels: - {{- include "openim-msggateway.labels" . | nindent 4 }} - {{- with .Values.ingress.annotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -spec: - {{- if and .Values.ingress.className (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }} - ingressClassName: {{ .Values.ingress.className }} - {{- end }} - {{- if .Values.ingress.tls }} - tls: - {{- range .Values.ingress.tls }} - - hosts: - {{- range .hosts }} - - {{ . | quote }} - {{- end }} - secretName: {{ .secretName }} - {{- end }} - {{- end }} - rules: - {{- range .Values.ingress.hosts }} - - host: {{ .host | quote }} - http: - paths: - {{- range .paths }} - - path: {{ .path }} - {{- if and .pathType (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }} - pathType: {{ .pathType }} - {{- end }} - backend: - {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }} - service: - name: {{ $fullName }} - port: - number: {{ $svcPort }} - {{- else }} - serviceName: {{ $fullName }} - servicePort: {{ $svcPort }} - {{- end }} - {{- end }} - {{- end }} -{{- end }} diff --git a/deployments/charts/openim-msggateway/templates/service.yaml b/deployments/charts/openim-msggateway/templates/service.yaml deleted file mode 100644 index e914ee1d4..000000000 --- a/deployments/charts/openim-msggateway/templates/service.yaml +++ /dev/null @@ -1,33 +0,0 @@ -# Copyright © 2023 OpenIM. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -apiVersion: v1 -kind: Service -metadata: - name: {{ include "openim-msggateway.fullname" . }} - labels: - {{- include "openim-msggateway.labels" . | nindent 4 }} -spec: - type: {{ .Values.service.type }} - ports: - - port: {{ .Values.service.port }} - targetPort: webhook - protocol: TCP - name: webhook - - port: 88 - targetPort: rpc - protocol: TCP - name: rpc - selector: - {{- include "openim-msggateway.selectorLabels" . | nindent 4 }} diff --git a/deployments/charts/openim-msggateway/templates/serviceaccount.yaml b/deployments/charts/openim-msggateway/templates/serviceaccount.yaml deleted file mode 100644 index 718880d71..000000000 --- a/deployments/charts/openim-msggateway/templates/serviceaccount.yaml +++ /dev/null @@ -1,26 +0,0 @@ -# Copyright © 2023 OpenIM. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -{{- if .Values.serviceAccount.create -}} -apiVersion: v1 -kind: ServiceAccount -metadata: - name: {{ include "openim-msggateway.serviceAccountName" . }} - labels: - {{- include "openim-msggateway.labels" . | nindent 4 }} - {{- with .Values.serviceAccount.annotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -{{- end }} diff --git a/deployments/charts/openim-msggateway/values.yaml b/deployments/charts/openim-msggateway/values.yaml deleted file mode 100644 index 059601807..000000000 --- a/deployments/charts/openim-msggateway/values.yaml +++ /dev/null @@ -1,96 +0,0 @@ -# Copyright © 2023 OpenIM. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Default values for openim-msggateway. -# This is a YAML-formatted file. -# Declare variables to be passed into your templates. - -replicaCount: 1 - -image: - repository: ghcr.io/openimsdk/openim-msggateway - pullPolicy: IfNotPresent - # Overrides the image tag whose default is the chart appVersion. - tag: "" - -imagePullSecrets: [] -nameOverride: "" -fullnameOverride: "" - -serviceAccount: - # Specifies whether a service account should be created - create: false - # Annotations to add to the service account - annotations: {} - # The name of the service account to use. - # If not set and create is true, a name is generated using the fullname template - name: "" - -podAnnotations: {} - -podSecurityContext: {} - # fsGroup: 2000 - -securityContext: {} - # capabilities: - # drop: - # - ALL - # readOnlyRootFilesystem: true - # runAsNonRoot: true - # runAsUser: 1000 - -service: - type: ClusterIP - port: 80 - -ingress: - enabled: false - className: "nginx" - annotations: - # kubernetes.io/ingress.class: nginx - # kubernetes.io/tls-acme: "true" - hosts: - - host: chart-example.local - paths: - - path: / - pathType: ImplementationSpecific - tls: [] - # - secretName: chart-example-tls - # hosts: - # - chart-example.local - -resources: {} - # We usually recommend not to specify default resources and to leave this as a conscious - # choice for the user. This also increases chances charts run on environments with little - # resources, such as Minikube. If you do want to specify resources, uncomment the following - # lines, adjust them as necessary, and remove the curly braces after 'resources:'. - # limits: - # cpu: 100m - # memory: 128Mi - # requests: - # cpu: 100m - # memory: 128Mi - -autoscaling: - enabled: false - minReplicas: 1 - maxReplicas: 100 - targetCPUUtilizationPercentage: 80 - # targetMemoryUtilizationPercentage: 80 - -nodeSelector: {} - -tolerations: [] - -affinity: {} diff --git a/deployments/charts/openim-msgtransfer/.helmignore b/deployments/charts/openim-msgtransfer/.helmignore deleted file mode 100644 index 0e8a0eb36..000000000 --- a/deployments/charts/openim-msgtransfer/.helmignore +++ /dev/null @@ -1,23 +0,0 @@ -# Patterns to ignore when building packages. -# This supports shell glob matching, relative path matching, and -# negation (prefixed with !). Only one pattern per line. -.DS_Store -# Common VCS dirs -.git/ -.gitignore -.bzr/ -.bzrignore -.hg/ -.hgignore -.svn/ -# Common backup files -*.swp -*.bak -*.tmp -*.orig -*~ -# Various IDEs -.project -.idea/ -*.tmproj -.vscode/ diff --git a/deployments/charts/openim-msgtransfer/Chart.yaml b/deployments/charts/openim-msgtransfer/Chart.yaml deleted file mode 100644 index 37d897728..000000000 --- a/deployments/charts/openim-msgtransfer/Chart.yaml +++ /dev/null @@ -1,53 +0,0 @@ -# Copyright © 2023 OpenIM. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -apiVersion: v2 -name: openim-msgtransfer -description: A Helm chart for Kubernetes - -# A chart can be either an 'application' or a 'library' chart. -# -# Application charts are a collection of templates that can be packaged into versioned archives -# to be deployed. -# -# Library charts provide useful utilities or functions for the chart developer. They're included as -# a dependency of application charts to inject those utilities and functions into the rendering -# pipeline. Library charts do not define any templates and therefore cannot be deployed. -type: application - -# This is the chart version. This version number should be incremented each time you make changes -# to the chart and its templates, including the app version. -# Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.0 - -# This is the version number of the application being deployed. This version number should be -# incremented each time you make changes to the application. Versions are not expected to -# follow Semantic Versioning. They should reflect the version the application is using. -# It is recommended to use it with quotes. -appVersion: "1.16.0" - -icon: https://raw.githubusercontent.com/openimsdk/open-im-server/main/assets/openim-logo-gradient.svg - -maintainers: - - name: "OpenIM" - url: "https://github.com/openimsdk" - -keywords: - - openim - - im - - chat - -sources: - - "https://github.com/openimsdk/open-im-server" - - "https://github.com/openimsdk/helm-charts" \ No newline at end of file diff --git a/deployments/charts/openim-msgtransfer/templates/NOTES.txt b/deployments/charts/openim-msgtransfer/templates/NOTES.txt deleted file mode 100644 index a9876065d..000000000 --- a/deployments/charts/openim-msgtransfer/templates/NOTES.txt +++ /dev/null @@ -1,22 +0,0 @@ -1. Get the application URL by running these commands: -{{- if .Values.ingress.enabled }} -{{- range $host := .Values.ingress.hosts }} - {{- range .paths }} - http{{ if $.Values.ingress.tls }}s{{ end }}://{{ $host.host }}{{ .path }} - {{- end }} -{{- end }} -{{- else if contains "NodePort" .Values.service.type }} - export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "openim-msgtransfer.fullname" . }}) - export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}") - echo http://$NODE_IP:$NODE_PORT -{{- else if contains "LoadBalancer" .Values.service.type }} - NOTE: It may take a few minutes for the LoadBalancer IP to be available. - You can watch the status of by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "openim-msgtransfer.fullname" . }}' - export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "openim-msgtransfer.fullname" . }} --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}") - echo http://$SERVICE_IP:{{ .Values.service.port }} -{{- else if contains "ClusterIP" .Values.service.type }} - export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "openim-msgtransfer.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") - export CONTAINER_PORT=$(kubectl get pod --namespace {{ .Release.Namespace }} $POD_NAME -o jsonpath="{.spec.containers[0].ports[0].containerPort}") - echo "Visit http://127.0.0.1:8080 to use your application" - kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:$CONTAINER_PORT -{{- end }} diff --git a/deployments/charts/openim-msgtransfer/templates/_helpers.tpl b/deployments/charts/openim-msgtransfer/templates/_helpers.tpl deleted file mode 100644 index 39f7a9acf..000000000 --- a/deployments/charts/openim-msgtransfer/templates/_helpers.tpl +++ /dev/null @@ -1,62 +0,0 @@ -{{/* -Expand the name of the chart. -*/}} -{{- define "openim-msgtransfer.name" -}} -{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} -{{- end }} - -{{/* -Create a default fully qualified app name. -We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). -If release name contains chart name it will be used as a full name. -*/}} -{{- define "openim-msgtransfer.fullname" -}} -{{- if .Values.fullnameOverride }} -{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} -{{- else }} -{{- $name := default .Chart.Name .Values.nameOverride }} -{{- if contains $name .Release.Name }} -{{- .Release.Name | trunc 63 | trimSuffix "-" }} -{{- else }} -{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} -{{- end }} -{{- end }} -{{- end }} - -{{/* -Create chart name and version as used by the chart label. -*/}} -{{- define "openim-msgtransfer.chart" -}} -{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} -{{- end }} - -{{/* -Common labels -*/}} -{{- define "openim-msgtransfer.labels" -}} -helm.sh/chart: {{ include "openim-msgtransfer.chart" . }} -{{ include "openim-msgtransfer.selectorLabels" . }} -{{- if .Chart.AppVersion }} -app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} -{{- end }} -app.kubernetes.io/managed-by: {{ .Release.Service }} -{{- end }} - -{{/* -Selector labels -*/}} -{{- define "openim-msgtransfer.selectorLabels" -}} -app.kubernetes.io/name: {{ include "openim-msgtransfer.name" . }} -app.kubernetes.io/instance: {{ .Release.Name }} -{{- end }} - -{{/* -Create the name of the service account to use -*/}} -{{- define "openim-msgtransfer.serviceAccountName" -}} -{{- if .Values.serviceAccount.create }} -{{- default (include "openim-msgtransfer.fullname" .) .Values.serviceAccount.name }} -{{- else }} -{{- default "default" .Values.serviceAccount.name }} -{{- end }} -{{- end }} diff --git a/deployments/charts/openim-msgtransfer/templates/deployment.yaml b/deployments/charts/openim-msgtransfer/templates/deployment.yaml deleted file mode 100644 index 019e307d5..000000000 --- a/deployments/charts/openim-msgtransfer/templates/deployment.yaml +++ /dev/null @@ -1,86 +0,0 @@ -# Copyright © 2023 OpenIM. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ include "openim-msgtransfer.fullname" . }} - labels: - {{- include "openim-msgtransfer.labels" . | nindent 4 }} -spec: - {{- if not .Values.autoscaling.enabled }} - replicas: {{ .Values.replicaCount }} - {{- end }} - selector: - matchLabels: - {{- include "openim-msgtransfer.selectorLabels" . | nindent 6 }} - template: - metadata: - {{- with .Values.podAnnotations }} - annotations: - {{- toYaml . | nindent 8 }} - {{- end }} - labels: - {{- include "openim-msgtransfer.selectorLabels" . | nindent 8 }} - spec: - {{- with .Values.imagePullSecrets }} - imagePullSecrets: - {{- toYaml . | nindent 8 }} - {{- end }} - serviceAccountName: {{ include "openim-msgtransfer.serviceAccountName" . }} - securityContext: - {{- toYaml .Values.podSecurityContext | nindent 8 }} - containers: - - name: {{ .Chart.Name }} - securityContext: - {{- toYaml .Values.securityContext | nindent 12 }} - image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" - imagePullPolicy: {{ .Values.image.pullPolicy }} - ports: - - name: webhook - containerPort: 80 - protocol: TCP - #livenessProbe: - # httpGet: - # path: / - # port: webhook - #readinessProbe: - # httpGet: - # path: / - # port: webhook - resources: - {{- toYaml .Values.resources | nindent 12 }} - volumeMounts: - - mountPath: /openim/openim-server/config/config.yaml - name: config - subPath: config.yaml - - mountPath: /openim/openim-server/config/ - name: config - subPath: notification.yaml - volumes: - - name: config - configMap: - name: openim-cm - {{- with .Values.nodeSelector }} - nodeSelector: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.affinity }} - affinity: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.tolerations }} - tolerations: - {{- toYaml . | nindent 8 }} - {{- end }} diff --git a/deployments/charts/openim-msgtransfer/templates/hpa.yaml b/deployments/charts/openim-msgtransfer/templates/hpa.yaml deleted file mode 100644 index 89921bf30..000000000 --- a/deployments/charts/openim-msgtransfer/templates/hpa.yaml +++ /dev/null @@ -1,42 +0,0 @@ -# Copyright © 2023 OpenIM. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -{{- if .Values.autoscaling.enabled }} -apiVersion: autoscaling/v2beta1 -kind: HorizontalPodAutoscaler -metadata: - name: {{ include "openim-msgtransfer.fullname" . }} - labels: - {{- include "openim-msgtransfer.labels" . | nindent 4 }} -spec: - scaleTargetRef: - apiVersion: apps/v1 - kind: Deployment - name: {{ include "openim-msgtransfer.fullname" . }} - minReplicas: {{ .Values.autoscaling.minReplicas }} - maxReplicas: {{ .Values.autoscaling.maxReplicas }} - metrics: - {{- if .Values.autoscaling.targetCPUUtilizationPercentage }} - - type: Resource - resource: - name: cpu - targetAverageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} - {{- end }} - {{- if .Values.autoscaling.targetMemoryUtilizationPercentage }} - - type: Resource - resource: - name: memory - targetAverageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} - {{- end }} -{{- end }} diff --git a/deployments/charts/openim-msgtransfer/templates/ingress.yaml b/deployments/charts/openim-msgtransfer/templates/ingress.yaml deleted file mode 100644 index ab28a9e71..000000000 --- a/deployments/charts/openim-msgtransfer/templates/ingress.yaml +++ /dev/null @@ -1,75 +0,0 @@ -# Copyright © 2023 OpenIM. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -{{- if .Values.ingress.enabled -}} -{{- $fullName := include "openim-msgtransfer.fullname" . -}} -{{- $svcPort := .Values.service.port -}} -{{- if and .Values.ingress.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }} - {{- if not (hasKey .Values.ingress.annotations "kubernetes.io/ingress.class") }} - {{- $_ := set .Values.ingress.annotations "kubernetes.io/ingress.class" .Values.ingress.className}} - {{- end }} -{{- end }} -{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}} -apiVersion: networking.k8s.io/v1 -{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}} -apiVersion: networking.k8s.io/v1beta1 -{{- else -}} -apiVersion: extensions/v1beta1 -{{- end }} -kind: Ingress -metadata: - name: {{ $fullName }} - labels: - {{- include "openim-msgtransfer.labels" . | nindent 4 }} - {{- with .Values.ingress.annotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -spec: - {{- if and .Values.ingress.className (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }} - ingressClassName: {{ .Values.ingress.className }} - {{- end }} - {{- if .Values.ingress.tls }} - tls: - {{- range .Values.ingress.tls }} - - hosts: - {{- range .hosts }} - - {{ . | quote }} - {{- end }} - secretName: {{ .secretName }} - {{- end }} - {{- end }} - rules: - {{- range .Values.ingress.hosts }} - - host: {{ .host | quote }} - http: - paths: - {{- range .paths }} - - path: {{ .path }} - {{- if and .pathType (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }} - pathType: {{ .pathType }} - {{- end }} - backend: - {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }} - service: - name: {{ $fullName }} - port: - number: {{ $svcPort }} - {{- else }} - serviceName: {{ $fullName }} - servicePort: {{ $svcPort }} - {{- end }} - {{- end }} - {{- end }} -{{- end }} diff --git a/deployments/charts/openim-msgtransfer/templates/service.yaml b/deployments/charts/openim-msgtransfer/templates/service.yaml deleted file mode 100644 index 467f7d13c..000000000 --- a/deployments/charts/openim-msgtransfer/templates/service.yaml +++ /dev/null @@ -1,29 +0,0 @@ -# Copyright © 2023 OpenIM. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -apiVersion: v1 -kind: Service -metadata: - name: {{ include "openim-msgtransfer.fullname" . }} - labels: - {{- include "openim-msgtransfer.labels" . | nindent 4 }} -spec: - type: {{ .Values.service.type }} - ports: - - port: {{ .Values.service.port }} - targetPort: webhook - protocol: TCP - name: webhook - selector: - {{- include "openim-msgtransfer.selectorLabels" . | nindent 4 }} diff --git a/deployments/charts/openim-msgtransfer/templates/serviceaccount.yaml b/deployments/charts/openim-msgtransfer/templates/serviceaccount.yaml deleted file mode 100644 index 78816bd80..000000000 --- a/deployments/charts/openim-msgtransfer/templates/serviceaccount.yaml +++ /dev/null @@ -1,26 +0,0 @@ -# Copyright © 2023 OpenIM. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -{{- if .Values.serviceAccount.create -}} -apiVersion: v1 -kind: ServiceAccount -metadata: - name: {{ include "openim-msgtransfer.serviceAccountName" . }} - labels: - {{- include "openim-msgtransfer.labels" . | nindent 4 }} - {{- with .Values.serviceAccount.annotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -{{- end }} diff --git a/deployments/charts/openim-msgtransfer/values.yaml b/deployments/charts/openim-msgtransfer/values.yaml deleted file mode 100644 index 6e26d72c8..000000000 --- a/deployments/charts/openim-msgtransfer/values.yaml +++ /dev/null @@ -1,96 +0,0 @@ -# Copyright © 2023 OpenIM. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Default values for openim-msgtransfer. -# This is a YAML-formatted file. -# Declare variables to be passed into your templates. - -replicaCount: 1 - -image: - repository: ghcr.io/openimsdk/openim-msgtransfer - pullPolicy: IfNotPresent - # Overrides the image tag whose default is the chart appVersion. - tag: "" - -imagePullSecrets: [] -nameOverride: "" -fullnameOverride: "" - -serviceAccount: - # Specifies whether a service account should be created - create: false - # Annotations to add to the service account - annotations: {} - # The name of the service account to use. - # If not set and create is true, a name is generated using the fullname template - name: "" - -podAnnotations: {} - -podSecurityContext: {} - # fsGroup: 2000 - -securityContext: {} - # capabilities: - # drop: - # - ALL - # readOnlyRootFilesystem: true - # runAsNonRoot: true - # runAsUser: 1000 - -service: - type: ClusterIP - port: 80 - -ingress: - enabled: false - className: "" - annotations: {} - # kubernetes.io/ingress.class: nginx - # kubernetes.io/tls-acme: "true" - hosts: - - host: chart-example.local - paths: - - path: / - pathType: ImplementationSpecific - tls: [] - # - secretName: chart-example-tls - # hosts: - # - chart-example.local - -resources: {} - # We usually recommend not to specify default resources and to leave this as a conscious - # choice for the user. This also increases chances charts run on environments with little - # resources, such as Minikube. If you do want to specify resources, uncomment the following - # lines, adjust them as necessary, and remove the curly braces after 'resources:'. - # limits: - # cpu: 100m - # memory: 128Mi - # requests: - # cpu: 100m - # memory: 128Mi - -autoscaling: - enabled: false - minReplicas: 1 - maxReplicas: 100 - targetCPUUtilizationPercentage: 80 - # targetMemoryUtilizationPercentage: 80 - -nodeSelector: {} - -tolerations: [] - -affinity: {} diff --git a/deployments/charts/openim-push/.helmignore b/deployments/charts/openim-push/.helmignore deleted file mode 100644 index 0e8a0eb36..000000000 --- a/deployments/charts/openim-push/.helmignore +++ /dev/null @@ -1,23 +0,0 @@ -# Patterns to ignore when building packages. -# This supports shell glob matching, relative path matching, and -# negation (prefixed with !). Only one pattern per line. -.DS_Store -# Common VCS dirs -.git/ -.gitignore -.bzr/ -.bzrignore -.hg/ -.hgignore -.svn/ -# Common backup files -*.swp -*.bak -*.tmp -*.orig -*~ -# Various IDEs -.project -.idea/ -*.tmproj -.vscode/ diff --git a/deployments/charts/openim-push/Chart.yaml b/deployments/charts/openim-push/Chart.yaml deleted file mode 100644 index 966769b00..000000000 --- a/deployments/charts/openim-push/Chart.yaml +++ /dev/null @@ -1,53 +0,0 @@ -# Copyright © 2023 OpenIM. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -apiVersion: v2 -name: openim-push -description: A Helm chart for Kubernetes - -# A chart can be either an 'application' or a 'library' chart. -# -# Application charts are a collection of templates that can be packaged into versioned archives -# to be deployed. -# -# Library charts provide useful utilities or functions for the chart developer. They're included as -# a dependency of application charts to inject those utilities and functions into the rendering -# pipeline. Library charts do not define any templates and therefore cannot be deployed. -type: application - -# This is the chart version. This version number should be incremented each time you make changes -# to the chart and its templates, including the app version. -# Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.0 - -# This is the version number of the application being deployed. This version number should be -# incremented each time you make changes to the application. Versions are not expected to -# follow Semantic Versioning. They should reflect the version the application is using. -# It is recommended to use it with quotes. -appVersion: "1.16.0" - -icon: https://raw.githubusercontent.com/openimsdk/open-im-server/main/assets/openim-logo-gradient.svg - -maintainers: - - name: "OpenIM" - url: "https://github.com/openimsdk" - -keywords: - - openim - - im - - chat - -sources: - - "https://github.com/openimsdk/open-im-server" - - "https://github.com/openimsdk/helm-charts" diff --git a/deployments/charts/openim-push/templates/NOTES.txt b/deployments/charts/openim-push/templates/NOTES.txt deleted file mode 100644 index 625c39050..000000000 --- a/deployments/charts/openim-push/templates/NOTES.txt +++ /dev/null @@ -1,22 +0,0 @@ -1. Get the application URL by running these commands: -{{- if .Values.ingress.enabled }} -{{- range $host := .Values.ingress.hosts }} - {{- range .paths }} - http{{ if $.Values.ingress.tls }}s{{ end }}://{{ $host.host }}{{ .path }} - {{- end }} -{{- end }} -{{- else if contains "NodePort" .Values.service.type }} - export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "openim-push.fullname" . }}) - export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}") - echo http://$NODE_IP:$NODE_PORT -{{- else if contains "LoadBalancer" .Values.service.type }} - NOTE: It may take a few minutes for the LoadBalancer IP to be available. - You can watch the status of by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "openim-push.fullname" . }}' - export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "openim-push.fullname" . }} --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}") - echo http://$SERVICE_IP:{{ .Values.service.port }} -{{- else if contains "ClusterIP" .Values.service.type }} - export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "openim-push.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") - export CONTAINER_PORT=$(kubectl get pod --namespace {{ .Release.Namespace }} $POD_NAME -o jsonpath="{.spec.containers[0].ports[0].containerPort}") - echo "Visit http://127.0.0.1:8080 to use your application" - kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:$CONTAINER_PORT -{{- end }} diff --git a/deployments/charts/openim-push/templates/_helpers.tpl b/deployments/charts/openim-push/templates/_helpers.tpl deleted file mode 100644 index 050428471..000000000 --- a/deployments/charts/openim-push/templates/_helpers.tpl +++ /dev/null @@ -1,62 +0,0 @@ -{{/* -Expand the name of the chart. -*/}} -{{- define "openim-push.name" -}} -{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} -{{- end }} - -{{/* -Create a default fully qualified app name. -We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). -If release name contains chart name it will be used as a full name. -*/}} -{{- define "openim-push.fullname" -}} -{{- if .Values.fullnameOverride }} -{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} -{{- else }} -{{- $name := default .Chart.Name .Values.nameOverride }} -{{- if contains $name .Release.Name }} -{{- .Release.Name | trunc 63 | trimSuffix "-" }} -{{- else }} -{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} -{{- end }} -{{- end }} -{{- end }} - -{{/* -Create chart name and version as used by the chart label. -*/}} -{{- define "openim-push.chart" -}} -{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} -{{- end }} - -{{/* -Common labels -*/}} -{{- define "openim-push.labels" -}} -helm.sh/chart: {{ include "openim-push.chart" . }} -{{ include "openim-push.selectorLabels" . }} -{{- if .Chart.AppVersion }} -app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} -{{- end }} -app.kubernetes.io/managed-by: {{ .Release.Service }} -{{- end }} - -{{/* -Selector labels -*/}} -{{- define "openim-push.selectorLabels" -}} -app.kubernetes.io/name: {{ include "openim-push.name" . }} -app.kubernetes.io/instance: {{ .Release.Name }} -{{- end }} - -{{/* -Create the name of the service account to use -*/}} -{{- define "openim-push.serviceAccountName" -}} -{{- if .Values.serviceAccount.create }} -{{- default (include "openim-push.fullname" .) .Values.serviceAccount.name }} -{{- else }} -{{- default "default" .Values.serviceAccount.name }} -{{- end }} -{{- end }} diff --git a/deployments/charts/openim-push/templates/deployment.yaml b/deployments/charts/openim-push/templates/deployment.yaml deleted file mode 100644 index 86c27d14c..000000000 --- a/deployments/charts/openim-push/templates/deployment.yaml +++ /dev/null @@ -1,86 +0,0 @@ -# Copyright © 2023 OpenIM. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ include "openim-push.fullname" . }} - labels: - {{- include "openim-push.labels" . | nindent 4 }} -spec: - {{- if not .Values.autoscaling.enabled }} - replicas: {{ .Values.replicaCount }} - {{- end }} - selector: - matchLabels: - {{- include "openim-push.selectorLabels" . | nindent 6 }} - template: - metadata: - {{- with .Values.podAnnotations }} - annotations: - {{- toYaml . | nindent 8 }} - {{- end }} - labels: - {{- include "openim-push.selectorLabels" . | nindent 8 }} - spec: - {{- with .Values.imagePullSecrets }} - imagePullSecrets: - {{- toYaml . | nindent 8 }} - {{- end }} - serviceAccountName: {{ include "openim-push.serviceAccountName" . }} - securityContext: - {{- toYaml .Values.podSecurityContext | nindent 8 }} - containers: - - name: {{ .Chart.Name }} - securityContext: - {{- toYaml .Values.securityContext | nindent 12 }} - image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" - imagePullPolicy: {{ .Values.image.pullPolicy }} - ports: - - name: webhook - containerPort: 80 - protocol: TCP - #livenessProbe: - # httpGet: - # path: / - # port: webhook - #readinessProbe: - # httpGet: - # path: / - # port: webhook - resources: - {{- toYaml .Values.resources | nindent 12 }} - volumeMounts: - - mountPath: /openim/openim-server/config/config.yaml - name: config - subPath: config.yaml - - mountPath: /openim/openim-server/config/ - name: config - subPath: notification.yaml - volumes: - - name: config - configMap: - name: openim-cm - {{- with .Values.nodeSelector }} - nodeSelector: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.affinity }} - affinity: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.tolerations }} - tolerations: - {{- toYaml . | nindent 8 }} - {{- end }} diff --git a/deployments/charts/openim-push/templates/hpa.yaml b/deployments/charts/openim-push/templates/hpa.yaml deleted file mode 100644 index 40c77b048..000000000 --- a/deployments/charts/openim-push/templates/hpa.yaml +++ /dev/null @@ -1,42 +0,0 @@ -# Copyright © 2023 OpenIM. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -{{- if .Values.autoscaling.enabled }} -apiVersion: autoscaling/v2beta1 -kind: HorizontalPodAutoscaler -metadata: - name: {{ include "openim-push.fullname" . }} - labels: - {{- include "openim-push.labels" . | nindent 4 }} -spec: - scaleTargetRef: - apiVersion: apps/v1 - kind: Deployment - name: {{ include "openim-push.fullname" . }} - minReplicas: {{ .Values.autoscaling.minReplicas }} - maxReplicas: {{ .Values.autoscaling.maxReplicas }} - metrics: - {{- if .Values.autoscaling.targetCPUUtilizationPercentage }} - - type: Resource - resource: - name: cpu - targetAverageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} - {{- end }} - {{- if .Values.autoscaling.targetMemoryUtilizationPercentage }} - - type: Resource - resource: - name: memory - targetAverageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} - {{- end }} -{{- end }} diff --git a/deployments/charts/openim-push/templates/ingress.yaml b/deployments/charts/openim-push/templates/ingress.yaml deleted file mode 100644 index 6638f256d..000000000 --- a/deployments/charts/openim-push/templates/ingress.yaml +++ /dev/null @@ -1,75 +0,0 @@ -# Copyright © 2023 OpenIM. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -{{- if .Values.ingress.enabled -}} -{{- $fullName := include "openim-push.fullname" . -}} -{{- $svcPort := .Values.service.port -}} -{{- if and .Values.ingress.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }} - {{- if not (hasKey .Values.ingress.annotations "kubernetes.io/ingress.class") }} - {{- $_ := set .Values.ingress.annotations "kubernetes.io/ingress.class" .Values.ingress.className}} - {{- end }} -{{- end }} -{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}} -apiVersion: networking.k8s.io/v1 -{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}} -apiVersion: networking.k8s.io/v1beta1 -{{- else -}} -apiVersion: extensions/v1beta1 -{{- end }} -kind: Ingress -metadata: - name: {{ $fullName }} - labels: - {{- include "openim-push.labels" . | nindent 4 }} - {{- with .Values.ingress.annotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -spec: - {{- if and .Values.ingress.className (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }} - ingressClassName: {{ .Values.ingress.className }} - {{- end }} - {{- if .Values.ingress.tls }} - tls: - {{- range .Values.ingress.tls }} - - hosts: - {{- range .hosts }} - - {{ . | quote }} - {{- end }} - secretName: {{ .secretName }} - {{- end }} - {{- end }} - rules: - {{- range .Values.ingress.hosts }} - - host: {{ .host | quote }} - http: - paths: - {{- range .paths }} - - path: {{ .path }} - {{- if and .pathType (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }} - pathType: {{ .pathType }} - {{- end }} - backend: - {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }} - service: - name: {{ $fullName }} - port: - number: {{ $svcPort }} - {{- else }} - serviceName: {{ $fullName }} - servicePort: {{ $svcPort }} - {{- end }} - {{- end }} - {{- end }} -{{- end }} diff --git a/deployments/charts/openim-push/templates/service.yaml b/deployments/charts/openim-push/templates/service.yaml deleted file mode 100644 index c2ef8db35..000000000 --- a/deployments/charts/openim-push/templates/service.yaml +++ /dev/null @@ -1,29 +0,0 @@ -# Copyright © 2023 OpenIM. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -apiVersion: v1 -kind: Service -metadata: - name: {{ include "openim-push.fullname" . }} - labels: - {{- include "openim-push.labels" . | nindent 4 }} -spec: - type: {{ .Values.service.type }} - ports: - - port: {{ .Values.service.port }} - targetPort: webhook - protocol: TCP - name: webhook - selector: - {{- include "openim-push.selectorLabels" . | nindent 4 }} diff --git a/deployments/charts/openim-push/templates/serviceaccount.yaml b/deployments/charts/openim-push/templates/serviceaccount.yaml deleted file mode 100644 index 66dfedfdf..000000000 --- a/deployments/charts/openim-push/templates/serviceaccount.yaml +++ /dev/null @@ -1,26 +0,0 @@ -# Copyright © 2023 OpenIM. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -{{- if .Values.serviceAccount.create -}} -apiVersion: v1 -kind: ServiceAccount -metadata: - name: {{ include "openim-push.serviceAccountName" . }} - labels: - {{- include "openim-push.labels" . | nindent 4 }} - {{- with .Values.serviceAccount.annotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -{{- end }} diff --git a/deployments/charts/openim-push/values.yaml b/deployments/charts/openim-push/values.yaml deleted file mode 100644 index 3348fe79d..000000000 --- a/deployments/charts/openim-push/values.yaml +++ /dev/null @@ -1,96 +0,0 @@ -# Copyright © 2023 OpenIM. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Default values for openim-push. -# This is a YAML-formatted file. -# Declare variables to be passed into your templates. - -replicaCount: 1 - -image: - repository: ghcr.io/openimsdk/openim-push - pullPolicy: IfNotPresent - # Overrides the image tag whose default is the chart appVersion. - tag: "" - -imagePullSecrets: [] -nameOverride: "" -fullnameOverride: "" - -serviceAccount: - # Specifies whether a service account should be created - create: false - # Annotations to add to the service account - annotations: {} - # The name of the service account to use. - # If not set and create is true, a name is generated using the fullname template - name: "" - -podAnnotations: {} - -podSecurityContext: {} - # fsGroup: 2000 - -securityContext: {} - # capabilities: - # drop: - # - ALL - # readOnlyRootFilesystem: true - # runAsNonRoot: true - # runAsUser: 1000 - -service: - type: ClusterIP - port: 80 - -ingress: - enabled: false - className: "" - annotations: {} - # kubernetes.io/ingress.class: nginx - # kubernetes.io/tls-acme: "true" - hosts: - - host: chart-example.local - paths: - - path: / - pathType: ImplementationSpecific - tls: [] - # - secretName: chart-example-tls - # hosts: - # - chart-example.local - -resources: {} - # We usually recommend not to specify default resources and to leave this as a conscious - # choice for the user. This also increases chances charts run on environments with little - # resources, such as Minikube. If you do want to specify resources, uncomment the following - # lines, adjust them as necessary, and remove the curly braces after 'resources:'. - # limits: - # cpu: 100m - # memory: 128Mi - # requests: - # cpu: 100m - # memory: 128Mi - -autoscaling: - enabled: false - minReplicas: 1 - maxReplicas: 100 - targetCPUUtilizationPercentage: 80 - # targetMemoryUtilizationPercentage: 80 - -nodeSelector: {} - -tolerations: [] - -affinity: {} diff --git a/deployments/charts/openim-rpc-auth/.helmignore b/deployments/charts/openim-rpc-auth/.helmignore deleted file mode 100644 index 0e8a0eb36..000000000 --- a/deployments/charts/openim-rpc-auth/.helmignore +++ /dev/null @@ -1,23 +0,0 @@ -# Patterns to ignore when building packages. -# This supports shell glob matching, relative path matching, and -# negation (prefixed with !). Only one pattern per line. -.DS_Store -# Common VCS dirs -.git/ -.gitignore -.bzr/ -.bzrignore -.hg/ -.hgignore -.svn/ -# Common backup files -*.swp -*.bak -*.tmp -*.orig -*~ -# Various IDEs -.project -.idea/ -*.tmproj -.vscode/ diff --git a/deployments/charts/openim-rpc-auth/Chart.yaml b/deployments/charts/openim-rpc-auth/Chart.yaml deleted file mode 100644 index 8e8fb03b4..000000000 --- a/deployments/charts/openim-rpc-auth/Chart.yaml +++ /dev/null @@ -1,53 +0,0 @@ -# Copyright © 2023 OpenIM. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -apiVersion: v2 -name: openim-rpc-auth -description: A Helm chart for Kubernetes - -# A chart can be either an 'application' or a 'library' chart. -# -# Application charts are a collection of templates that can be packaged into versioned archives -# to be deployed. -# -# Library charts provide useful utilities or functions for the chart developer. They're included as -# a dependency of application charts to inject those utilities and functions into the rendering -# pipeline. Library charts do not define any templates and therefore cannot be deployed. -type: application - -# This is the chart version. This version number should be incremented each time you make changes -# to the chart and its templates, including the app version. -# Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.0 - -# This is the version number of the application being deployed. This version number should be -# incremented each time you make changes to the application. Versions are not expected to -# follow Semantic Versioning. They should reflect the version the application is using. -# It is recommended to use it with quotes. -appVersion: "1.16.0" - -icon: https://raw.githubusercontent.com/openimsdk/open-im-server/main/assets/openim-logo-gradient.svg - -maintainers: - - name: "OpenIM" - url: "https://github.com/openimsdk" - -keywords: - - openim - - im - - chat - -sources: - - "https://github.com/openimsdk/open-im-server" - - "https://github.com/openimsdk/helm-charts" \ No newline at end of file diff --git a/deployments/charts/openim-rpc-auth/templates/NOTES.txt b/deployments/charts/openim-rpc-auth/templates/NOTES.txt deleted file mode 100644 index 220602077..000000000 --- a/deployments/charts/openim-rpc-auth/templates/NOTES.txt +++ /dev/null @@ -1,22 +0,0 @@ -1. Get the application URL by running these commands: -{{- if .Values.ingress.enabled }} -{{- range $host := .Values.ingress.hosts }} - {{- range .paths }} - http{{ if $.Values.ingress.tls }}s{{ end }}://{{ $host.host }}{{ .path }} - {{- end }} -{{- end }} -{{- else if contains "NodePort" .Values.service.type }} - export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "openim-rpc-auth.fullname" . }}) - export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}") - echo http://$NODE_IP:$NODE_PORT -{{- else if contains "LoadBalancer" .Values.service.type }} - NOTE: It may take a few minutes for the LoadBalancer IP to be available. - You can watch the status of by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "openim-rpc-auth.fullname" . }}' - export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "openim-rpc-auth.fullname" . }} --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}") - echo http://$SERVICE_IP:{{ .Values.service.port }} -{{- else if contains "ClusterIP" .Values.service.type }} - export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "openim-rpc-auth.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") - export CONTAINER_PORT=$(kubectl get pod --namespace {{ .Release.Namespace }} $POD_NAME -o jsonpath="{.spec.containers[0].ports[0].containerPort}") - echo "Visit http://127.0.0.1:8080 to use your application" - kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:$CONTAINER_PORT -{{- end }} diff --git a/deployments/charts/openim-rpc-auth/templates/_helpers.tpl b/deployments/charts/openim-rpc-auth/templates/_helpers.tpl deleted file mode 100644 index cb6575872..000000000 --- a/deployments/charts/openim-rpc-auth/templates/_helpers.tpl +++ /dev/null @@ -1,62 +0,0 @@ -{{/* -Expand the name of the chart. -*/}} -{{- define "openim-rpc-auth.name" -}} -{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} -{{- end }} - -{{/* -Create a default fully qualified app name. -We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). -If release name contains chart name it will be used as a full name. -*/}} -{{- define "openim-rpc-auth.fullname" -}} -{{- if .Values.fullnameOverride }} -{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} -{{- else }} -{{- $name := default .Chart.Name .Values.nameOverride }} -{{- if contains $name .Release.Name }} -{{- .Release.Name | trunc 63 | trimSuffix "-" }} -{{- else }} -{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} -{{- end }} -{{- end }} -{{- end }} - -{{/* -Create chart name and version as used by the chart label. -*/}} -{{- define "openim-rpc-auth.chart" -}} -{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} -{{- end }} - -{{/* -Common labels -*/}} -{{- define "openim-rpc-auth.labels" -}} -helm.sh/chart: {{ include "openim-rpc-auth.chart" . }} -{{ include "openim-rpc-auth.selectorLabels" . }} -{{- if .Chart.AppVersion }} -app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} -{{- end }} -app.kubernetes.io/managed-by: {{ .Release.Service }} -{{- end }} - -{{/* -Selector labels -*/}} -{{- define "openim-rpc-auth.selectorLabels" -}} -app.kubernetes.io/name: {{ include "openim-rpc-auth.name" . }} -app.kubernetes.io/instance: {{ .Release.Name }} -{{- end }} - -{{/* -Create the name of the service account to use -*/}} -{{- define "openim-rpc-auth.serviceAccountName" -}} -{{- if .Values.serviceAccount.create }} -{{- default (include "openim-rpc-auth.fullname" .) .Values.serviceAccount.name }} -{{- else }} -{{- default "default" .Values.serviceAccount.name }} -{{- end }} -{{- end }} diff --git a/deployments/charts/openim-rpc-auth/templates/deployment.yaml b/deployments/charts/openim-rpc-auth/templates/deployment.yaml deleted file mode 100644 index 98c43ecb7..000000000 --- a/deployments/charts/openim-rpc-auth/templates/deployment.yaml +++ /dev/null @@ -1,86 +0,0 @@ -# Copyright © 2023 OpenIM. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ include "openim-rpc-auth.fullname" . }} - labels: - {{- include "openim-rpc-auth.labels" . | nindent 4 }} -spec: - {{- if not .Values.autoscaling.enabled }} - replicas: {{ .Values.replicaCount }} - {{- end }} - selector: - matchLabels: - {{- include "openim-rpc-auth.selectorLabels" . | nindent 6 }} - template: - metadata: - {{- with .Values.podAnnotations }} - annotations: - {{- toYaml . | nindent 8 }} - {{- end }} - labels: - {{- include "openim-rpc-auth.selectorLabels" . | nindent 8 }} - spec: - {{- with .Values.imagePullSecrets }} - imagePullSecrets: - {{- toYaml . | nindent 8 }} - {{- end }} - serviceAccountName: {{ include "openim-rpc-auth.serviceAccountName" . }} - securityContext: - {{- toYaml .Values.podSecurityContext | nindent 8 }} - containers: - - name: {{ .Chart.Name }} - securityContext: - {{- toYaml .Values.securityContext | nindent 12 }} - image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" - imagePullPolicy: {{ .Values.image.pullPolicy }} - ports: - - name: webhook - containerPort: 80 - protocol: TCP - #livenessProbe: - # httpGet: - # path: / - # port: webhook - #readinessProbe: - # httpGet: - # path: / - # port: webhook - resources: - {{- toYaml .Values.resources | nindent 12 }} - volumeMounts: - - mountPath: /openim/openim-server/config/config.yaml - name: config - subPath: config.yaml - - mountPath: /openim/openim-server/config/ - name: config - subPath: notification.yaml - volumes: - - name: config - configMap: - name: openim-cm - {{- with .Values.nodeSelector }} - nodeSelector: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.affinity }} - affinity: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.tolerations }} - tolerations: - {{- toYaml . | nindent 8 }} - {{- end }} diff --git a/deployments/charts/openim-rpc-auth/templates/hpa.yaml b/deployments/charts/openim-rpc-auth/templates/hpa.yaml deleted file mode 100644 index e99536cf1..000000000 --- a/deployments/charts/openim-rpc-auth/templates/hpa.yaml +++ /dev/null @@ -1,42 +0,0 @@ -# Copyright © 2023 OpenIM. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -{{- if .Values.autoscaling.enabled }} -apiVersion: autoscaling/v2beta1 -kind: HorizontalPodAutoscaler -metadata: - name: {{ include "openim-rpc-auth.fullname" . }} - labels: - {{- include "openim-rpc-auth.labels" . | nindent 4 }} -spec: - scaleTargetRef: - apiVersion: apps/v1 - kind: Deployment - name: {{ include "openim-rpc-auth.fullname" . }} - minReplicas: {{ .Values.autoscaling.minReplicas }} - maxReplicas: {{ .Values.autoscaling.maxReplicas }} - metrics: - {{- if .Values.autoscaling.targetCPUUtilizationPercentage }} - - type: Resource - resource: - name: cpu - targetAverageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} - {{- end }} - {{- if .Values.autoscaling.targetMemoryUtilizationPercentage }} - - type: Resource - resource: - name: memory - targetAverageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} - {{- end }} -{{- end }} diff --git a/deployments/charts/openim-rpc-auth/templates/ingress.yaml b/deployments/charts/openim-rpc-auth/templates/ingress.yaml deleted file mode 100644 index ece395de7..000000000 --- a/deployments/charts/openim-rpc-auth/templates/ingress.yaml +++ /dev/null @@ -1,75 +0,0 @@ -# Copyright © 2023 OpenIM. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -{{- if .Values.ingress.enabled -}} -{{- $fullName := include "openim-rpc-auth.fullname" . -}} -{{- $svcPort := .Values.service.port -}} -{{- if and .Values.ingress.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }} - {{- if not (hasKey .Values.ingress.annotations "kubernetes.io/ingress.class") }} - {{- $_ := set .Values.ingress.annotations "kubernetes.io/ingress.class" .Values.ingress.className}} - {{- end }} -{{- end }} -{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}} -apiVersion: networking.k8s.io/v1 -{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}} -apiVersion: networking.k8s.io/v1beta1 -{{- else -}} -apiVersion: extensions/v1beta1 -{{- end }} -kind: Ingress -metadata: - name: {{ $fullName }} - labels: - {{- include "openim-rpc-auth.labels" . | nindent 4 }} - {{- with .Values.ingress.annotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -spec: - {{- if and .Values.ingress.className (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }} - ingressClassName: {{ .Values.ingress.className }} - {{- end }} - {{- if .Values.ingress.tls }} - tls: - {{- range .Values.ingress.tls }} - - hosts: - {{- range .hosts }} - - {{ . | quote }} - {{- end }} - secretName: {{ .secretName }} - {{- end }} - {{- end }} - rules: - {{- range .Values.ingress.hosts }} - - host: {{ .host | quote }} - http: - paths: - {{- range .paths }} - - path: {{ .path }} - {{- if and .pathType (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }} - pathType: {{ .pathType }} - {{- end }} - backend: - {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }} - service: - name: {{ $fullName }} - port: - number: {{ $svcPort }} - {{- else }} - serviceName: {{ $fullName }} - servicePort: {{ $svcPort }} - {{- end }} - {{- end }} - {{- end }} -{{- end }} diff --git a/deployments/charts/openim-rpc-auth/templates/service.yaml b/deployments/charts/openim-rpc-auth/templates/service.yaml deleted file mode 100644 index 785512347..000000000 --- a/deployments/charts/openim-rpc-auth/templates/service.yaml +++ /dev/null @@ -1,29 +0,0 @@ -# Copyright © 2023 OpenIM. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -apiVersion: v1 -kind: Service -metadata: - name: {{ include "openim-rpc-auth.fullname" . }} - labels: - {{- include "openim-rpc-auth.labels" . | nindent 4 }} -spec: - type: {{ .Values.service.type }} - ports: - - port: {{ .Values.service.port }} - targetPort: webhook - protocol: TCP - name: webhook - selector: - {{- include "openim-rpc-auth.selectorLabels" . | nindent 4 }} diff --git a/deployments/charts/openim-rpc-auth/templates/serviceaccount.yaml b/deployments/charts/openim-rpc-auth/templates/serviceaccount.yaml deleted file mode 100644 index 555d4f6ff..000000000 --- a/deployments/charts/openim-rpc-auth/templates/serviceaccount.yaml +++ /dev/null @@ -1,26 +0,0 @@ -# Copyright © 2023 OpenIM. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -{{- if .Values.serviceAccount.create -}} -apiVersion: v1 -kind: ServiceAccount -metadata: - name: {{ include "openim-rpc-auth.serviceAccountName" . }} - labels: - {{- include "openim-rpc-auth.labels" . | nindent 4 }} - {{- with .Values.serviceAccount.annotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -{{- end }} diff --git a/deployments/charts/openim-rpc-auth/values.yaml b/deployments/charts/openim-rpc-auth/values.yaml deleted file mode 100644 index 93fa68b87..000000000 --- a/deployments/charts/openim-rpc-auth/values.yaml +++ /dev/null @@ -1,96 +0,0 @@ -# Copyright © 2023 OpenIM. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Default values for openim-rpc-auth. -# This is a YAML-formatted file. -# Declare variables to be passed into your templates. - -replicaCount: 1 - -image: - repository: ghcr.io/openimsdk/openim-rpc-auth - pullPolicy: IfNotPresent - # Overrides the image tag whose default is the chart appVersion. - tag: "" - -imagePullSecrets: [] -nameOverride: "" -fullnameOverride: "" - -serviceAccount: - # Specifies whether a service account should be created - create: false - # Annotations to add to the service account - annotations: {} - # The name of the service account to use. - # If not set and create is true, a name is generated using the fullname template - name: "" - -podAnnotations: {} - -podSecurityContext: {} - # fsGroup: 2000 - -securityContext: {} - # capabilities: - # drop: - # - ALL - # readOnlyRootFilesystem: true - # runAsNonRoot: true - # runAsUser: 1000 - -service: - type: ClusterIP - port: 80 - -ingress: - enabled: false - className: "" - annotations: {} - # kubernetes.io/ingress.class: nginx - # kubernetes.io/tls-acme: "true" - hosts: - - host: chart-example.local - paths: - - path: / - pathType: ImplementationSpecific - tls: [] - # - secretName: chart-example-tls - # hosts: - # - chart-example.local - -resources: {} - # We usually recommend not to specify default resources and to leave this as a conscious - # choice for the user. This also increases chances charts run on environments with little - # resources, such as Minikube. If you do want to specify resources, uncomment the following - # lines, adjust them as necessary, and remove the curly braces after 'resources:'. - # limits: - # cpu: 100m - # memory: 128Mi - # requests: - # cpu: 100m - # memory: 128Mi - -autoscaling: - enabled: false - minReplicas: 1 - maxReplicas: 100 - targetCPUUtilizationPercentage: 80 - # targetMemoryUtilizationPercentage: 80 - -nodeSelector: {} - -tolerations: [] - -affinity: {} diff --git a/deployments/charts/openim-rpc-conversation/.helmignore b/deployments/charts/openim-rpc-conversation/.helmignore deleted file mode 100644 index 0e8a0eb36..000000000 --- a/deployments/charts/openim-rpc-conversation/.helmignore +++ /dev/null @@ -1,23 +0,0 @@ -# Patterns to ignore when building packages. -# This supports shell glob matching, relative path matching, and -# negation (prefixed with !). Only one pattern per line. -.DS_Store -# Common VCS dirs -.git/ -.gitignore -.bzr/ -.bzrignore -.hg/ -.hgignore -.svn/ -# Common backup files -*.swp -*.bak -*.tmp -*.orig -*~ -# Various IDEs -.project -.idea/ -*.tmproj -.vscode/ diff --git a/deployments/charts/openim-rpc-conversation/Chart.yaml b/deployments/charts/openim-rpc-conversation/Chart.yaml deleted file mode 100644 index 0ca8f5513..000000000 --- a/deployments/charts/openim-rpc-conversation/Chart.yaml +++ /dev/null @@ -1,53 +0,0 @@ -# Copyright © 2023 OpenIM. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -apiVersion: v2 -name: openim-rpc-conversation -description: A Helm chart for Kubernetes - -# A chart can be either an 'application' or a 'library' chart. -# -# Application charts are a collection of templates that can be packaged into versioned archives -# to be deployed. -# -# Library charts provide useful utilities or functions for the chart developer. They're included as -# a dependency of application charts to inject those utilities and functions into the rendering -# pipeline. Library charts do not define any templates and therefore cannot be deployed. -type: application - -# This is the chart version. This version number should be incremented each time you make changes -# to the chart and its templates, including the app version. -# Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.0 - -# This is the version number of the application being deployed. This version number should be -# incremented each time you make changes to the application. Versions are not expected to -# follow Semantic Versioning. They should reflect the version the application is using. -# It is recommended to use it with quotes. -appVersion: "1.16.0" - -icon: https://raw.githubusercontent.com/openimsdk/open-im-server/main/assets/openim-logo-gradient.svg - -maintainers: - - name: "OpenIM" - url: "https://github.com/openimsdk" - -keywords: - - openim - - im - - chat - -sources: - - "https://github.com/openimsdk/open-im-server" - - "https://github.com/openimsdk/helm-charts" diff --git a/deployments/charts/openim-rpc-conversation/templates/NOTES.txt b/deployments/charts/openim-rpc-conversation/templates/NOTES.txt deleted file mode 100644 index 287f2fd64..000000000 --- a/deployments/charts/openim-rpc-conversation/templates/NOTES.txt +++ /dev/null @@ -1,22 +0,0 @@ -1. Get the application URL by running these commands: -{{- if .Values.ingress.enabled }} -{{- range $host := .Values.ingress.hosts }} - {{- range .paths }} - http{{ if $.Values.ingress.tls }}s{{ end }}://{{ $host.host }}{{ .path }} - {{- end }} -{{- end }} -{{- else if contains "NodePort" .Values.service.type }} - export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "openim-rpc-conversation.fullname" . }}) - export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}") - echo http://$NODE_IP:$NODE_PORT -{{- else if contains "LoadBalancer" .Values.service.type }} - NOTE: It may take a few minutes for the LoadBalancer IP to be available. - You can watch the status of by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "openim-rpc-conversation.fullname" . }}' - export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "openim-rpc-conversation.fullname" . }} --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}") - echo http://$SERVICE_IP:{{ .Values.service.port }} -{{- else if contains "ClusterIP" .Values.service.type }} - export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "openim-rpc-conversation.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") - export CONTAINER_PORT=$(kubectl get pod --namespace {{ .Release.Namespace }} $POD_NAME -o jsonpath="{.spec.containers[0].ports[0].containerPort}") - echo "Visit http://127.0.0.1:8080 to use your application" - kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:$CONTAINER_PORT -{{- end }} diff --git a/deployments/charts/openim-rpc-conversation/templates/_helpers.tpl b/deployments/charts/openim-rpc-conversation/templates/_helpers.tpl deleted file mode 100644 index 184271057..000000000 --- a/deployments/charts/openim-rpc-conversation/templates/_helpers.tpl +++ /dev/null @@ -1,62 +0,0 @@ -{{/* -Expand the name of the chart. -*/}} -{{- define "openim-rpc-conversation.name" -}} -{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} -{{- end }} - -{{/* -Create a default fully qualified app name. -We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). -If release name contains chart name it will be used as a full name. -*/}} -{{- define "openim-rpc-conversation.fullname" -}} -{{- if .Values.fullnameOverride }} -{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} -{{- else }} -{{- $name := default .Chart.Name .Values.nameOverride }} -{{- if contains $name .Release.Name }} -{{- .Release.Name | trunc 63 | trimSuffix "-" }} -{{- else }} -{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} -{{- end }} -{{- end }} -{{- end }} - -{{/* -Create chart name and version as used by the chart label. -*/}} -{{- define "openim-rpc-conversation.chart" -}} -{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} -{{- end }} - -{{/* -Common labels -*/}} -{{- define "openim-rpc-conversation.labels" -}} -helm.sh/chart: {{ include "openim-rpc-conversation.chart" . }} -{{ include "openim-rpc-conversation.selectorLabels" . }} -{{- if .Chart.AppVersion }} -app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} -{{- end }} -app.kubernetes.io/managed-by: {{ .Release.Service }} -{{- end }} - -{{/* -Selector labels -*/}} -{{- define "openim-rpc-conversation.selectorLabels" -}} -app.kubernetes.io/name: {{ include "openim-rpc-conversation.name" . }} -app.kubernetes.io/instance: {{ .Release.Name }} -{{- end }} - -{{/* -Create the name of the service account to use -*/}} -{{- define "openim-rpc-conversation.serviceAccountName" -}} -{{- if .Values.serviceAccount.create }} -{{- default (include "openim-rpc-conversation.fullname" .) .Values.serviceAccount.name }} -{{- else }} -{{- default "default" .Values.serviceAccount.name }} -{{- end }} -{{- end }} diff --git a/deployments/charts/openim-rpc-conversation/templates/deployment.yaml b/deployments/charts/openim-rpc-conversation/templates/deployment.yaml deleted file mode 100644 index 6dcb001f4..000000000 --- a/deployments/charts/openim-rpc-conversation/templates/deployment.yaml +++ /dev/null @@ -1,86 +0,0 @@ -# Copyright © 2023 OpenIM. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ include "openim-rpc-conversation.fullname" . }} - labels: - {{- include "openim-rpc-conversation.labels" . | nindent 4 }} -spec: - {{- if not .Values.autoscaling.enabled }} - replicas: {{ .Values.replicaCount }} - {{- end }} - selector: - matchLabels: - {{- include "openim-rpc-conversation.selectorLabels" . | nindent 6 }} - template: - metadata: - {{- with .Values.podAnnotations }} - annotations: - {{- toYaml . | nindent 8 }} - {{- end }} - labels: - {{- include "openim-rpc-conversation.selectorLabels" . | nindent 8 }} - spec: - {{- with .Values.imagePullSecrets }} - imagePullSecrets: - {{- toYaml . | nindent 8 }} - {{- end }} - serviceAccountName: {{ include "openim-rpc-conversation.serviceAccountName" . }} - securityContext: - {{- toYaml .Values.podSecurityContext | nindent 8 }} - containers: - - name: {{ .Chart.Name }} - securityContext: - {{- toYaml .Values.securityContext | nindent 12 }} - image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" - imagePullPolicy: {{ .Values.image.pullPolicy }} - ports: - - name: webhook - containerPort: 80 - protocol: TCP - #livenessProbe: - # httpGet: - # path: / - # port: webhook - #readinessProbe: - # httpGet: - # path: / - # port: webhook - resources: - {{- toYaml .Values.resources | nindent 12 }} - volumeMounts: - - mountPath: /openim/openim-server/config/config.yaml - name: config - subPath: config.yaml - - mountPath: /openim/openim-server/config/ - name: config - subPath: notification.yaml - volumes: - - name: config - configMap: - name: openim-cm - {{- with .Values.nodeSelector }} - nodeSelector: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.affinity }} - affinity: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.tolerations }} - tolerations: - {{- toYaml . | nindent 8 }} - {{- end }} diff --git a/deployments/charts/openim-rpc-conversation/templates/hpa.yaml b/deployments/charts/openim-rpc-conversation/templates/hpa.yaml deleted file mode 100644 index 524245562..000000000 --- a/deployments/charts/openim-rpc-conversation/templates/hpa.yaml +++ /dev/null @@ -1,42 +0,0 @@ -# Copyright © 2023 OpenIM. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -{{- if .Values.autoscaling.enabled }} -apiVersion: autoscaling/v2beta1 -kind: HorizontalPodAutoscaler -metadata: - name: {{ include "openim-rpc-conversation.fullname" . }} - labels: - {{- include "openim-rpc-conversation.labels" . | nindent 4 }} -spec: - scaleTargetRef: - apiVersion: apps/v1 - kind: Deployment - name: {{ include "openim-rpc-conversation.fullname" . }} - minReplicas: {{ .Values.autoscaling.minReplicas }} - maxReplicas: {{ .Values.autoscaling.maxReplicas }} - metrics: - {{- if .Values.autoscaling.targetCPUUtilizationPercentage }} - - type: Resource - resource: - name: cpu - targetAverageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} - {{- end }} - {{- if .Values.autoscaling.targetMemoryUtilizationPercentage }} - - type: Resource - resource: - name: memory - targetAverageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} - {{- end }} -{{- end }} diff --git a/deployments/charts/openim-rpc-conversation/templates/ingress.yaml b/deployments/charts/openim-rpc-conversation/templates/ingress.yaml deleted file mode 100644 index 078c472b8..000000000 --- a/deployments/charts/openim-rpc-conversation/templates/ingress.yaml +++ /dev/null @@ -1,75 +0,0 @@ -# Copyright © 2023 OpenIM. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -{{- if .Values.ingress.enabled -}} -{{- $fullName := include "openim-rpc-conversation.fullname" . -}} -{{- $svcPort := .Values.service.port -}} -{{- if and .Values.ingress.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }} - {{- if not (hasKey .Values.ingress.annotations "kubernetes.io/ingress.class") }} - {{- $_ := set .Values.ingress.annotations "kubernetes.io/ingress.class" .Values.ingress.className}} - {{- end }} -{{- end }} -{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}} -apiVersion: networking.k8s.io/v1 -{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}} -apiVersion: networking.k8s.io/v1beta1 -{{- else -}} -apiVersion: extensions/v1beta1 -{{- end }} -kind: Ingress -metadata: - name: {{ $fullName }} - labels: - {{- include "openim-rpc-conversation.labels" . | nindent 4 }} - {{- with .Values.ingress.annotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -spec: - {{- if and .Values.ingress.className (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }} - ingressClassName: {{ .Values.ingress.className }} - {{- end }} - {{- if .Values.ingress.tls }} - tls: - {{- range .Values.ingress.tls }} - - hosts: - {{- range .hosts }} - - {{ . | quote }} - {{- end }} - secretName: {{ .secretName }} - {{- end }} - {{- end }} - rules: - {{- range .Values.ingress.hosts }} - - host: {{ .host | quote }} - http: - paths: - {{- range .paths }} - - path: {{ .path }} - {{- if and .pathType (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }} - pathType: {{ .pathType }} - {{- end }} - backend: - {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }} - service: - name: {{ $fullName }} - port: - number: {{ $svcPort }} - {{- else }} - serviceName: {{ $fullName }} - servicePort: {{ $svcPort }} - {{- end }} - {{- end }} - {{- end }} -{{- end }} diff --git a/deployments/charts/openim-rpc-conversation/templates/service.yaml b/deployments/charts/openim-rpc-conversation/templates/service.yaml deleted file mode 100644 index 8559c4d11..000000000 --- a/deployments/charts/openim-rpc-conversation/templates/service.yaml +++ /dev/null @@ -1,29 +0,0 @@ -# Copyright © 2023 OpenIM. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -apiVersion: v1 -kind: Service -metadata: - name: {{ include "openim-rpc-conversation.fullname" . }} - labels: - {{- include "openim-rpc-conversation.labels" . | nindent 4 }} -spec: - type: {{ .Values.service.type }} - ports: - - port: {{ .Values.service.port }} - targetPort: webhook - protocol: TCP - name: webhook - selector: - {{- include "openim-rpc-conversation.selectorLabels" . | nindent 4 }} diff --git a/deployments/charts/openim-rpc-conversation/templates/serviceaccount.yaml b/deployments/charts/openim-rpc-conversation/templates/serviceaccount.yaml deleted file mode 100644 index 14b1b2129..000000000 --- a/deployments/charts/openim-rpc-conversation/templates/serviceaccount.yaml +++ /dev/null @@ -1,26 +0,0 @@ -# Copyright © 2023 OpenIM. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -{{- if .Values.serviceAccount.create -}} -apiVersion: v1 -kind: ServiceAccount -metadata: - name: {{ include "openim-rpc-conversation.serviceAccountName" . }} - labels: - {{- include "openim-rpc-conversation.labels" . | nindent 4 }} - {{- with .Values.serviceAccount.annotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -{{- end }} diff --git a/deployments/charts/openim-rpc-conversation/values.yaml b/deployments/charts/openim-rpc-conversation/values.yaml deleted file mode 100644 index bb88ab613..000000000 --- a/deployments/charts/openim-rpc-conversation/values.yaml +++ /dev/null @@ -1,96 +0,0 @@ -# Copyright © 2023 OpenIM. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Default values for openim-rpc-conversation. -# This is a YAML-formatted file. -# Declare variables to be passed into your templates. - -replicaCount: 1 - -image: - repository: ghcr.io/openimsdk/openim-rpc-conversation - pullPolicy: IfNotPresent - # Overrides the image tag whose default is the chart appVersion. - tag: "" - -imagePullSecrets: [] -nameOverride: "" -fullnameOverride: "" - -serviceAccount: - # Specifies whether a service account should be created - create: false - # Annotations to add to the service account - annotations: {} - # The name of the service account to use. - # If not set and create is true, a name is generated using the fullname template - name: "" - -podAnnotations: {} - -podSecurityContext: {} - # fsGroup: 2000 - -securityContext: {} - # capabilities: - # drop: - # - ALL - # readOnlyRootFilesystem: true - # runAsNonRoot: true - # runAsUser: 1000 - -service: - type: ClusterIP - port: 80 - -ingress: - enabled: false - className: "" - annotations: {} - # kubernetes.io/ingress.class: nginx - # kubernetes.io/tls-acme: "true" - hosts: - - host: chart-example.local - paths: - - path: / - pathType: ImplementationSpecific - tls: [] - # - secretName: chart-example-tls - # hosts: - # - chart-example.local - -resources: {} - # We usually recommend not to specify default resources and to leave this as a conscious - # choice for the user. This also increases chances charts run on environments with little - # resources, such as Minikube. If you do want to specify resources, uncomment the following - # lines, adjust them as necessary, and remove the curly braces after 'resources:'. - # limits: - # cpu: 100m - # memory: 128Mi - # requests: - # cpu: 100m - # memory: 128Mi - -autoscaling: - enabled: false - minReplicas: 1 - maxReplicas: 100 - targetCPUUtilizationPercentage: 80 - # targetMemoryUtilizationPercentage: 80 - -nodeSelector: {} - -tolerations: [] - -affinity: {} diff --git a/deployments/charts/openim-rpc-friend/.helmignore b/deployments/charts/openim-rpc-friend/.helmignore deleted file mode 100644 index 0e8a0eb36..000000000 --- a/deployments/charts/openim-rpc-friend/.helmignore +++ /dev/null @@ -1,23 +0,0 @@ -# Patterns to ignore when building packages. -# This supports shell glob matching, relative path matching, and -# negation (prefixed with !). Only one pattern per line. -.DS_Store -# Common VCS dirs -.git/ -.gitignore -.bzr/ -.bzrignore -.hg/ -.hgignore -.svn/ -# Common backup files -*.swp -*.bak -*.tmp -*.orig -*~ -# Various IDEs -.project -.idea/ -*.tmproj -.vscode/ diff --git a/deployments/charts/openim-rpc-friend/Chart.yaml b/deployments/charts/openim-rpc-friend/Chart.yaml deleted file mode 100644 index 62e92db9b..000000000 --- a/deployments/charts/openim-rpc-friend/Chart.yaml +++ /dev/null @@ -1,53 +0,0 @@ -# Copyright © 2023 OpenIM. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -apiVersion: v2 -name: openim-rpc-friend -description: A Helm chart for Kubernetes - -# A chart can be either an 'application' or a 'library' chart. -# -# Application charts are a collection of templates that can be packaged into versioned archives -# to be deployed. -# -# Library charts provide useful utilities or functions for the chart developer. They're included as -# a dependency of application charts to inject those utilities and functions into the rendering -# pipeline. Library charts do not define any templates and therefore cannot be deployed. -type: application - -# This is the chart version. This version number should be incremented each time you make changes -# to the chart and its templates, including the app version. -# Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.0 - -# This is the version number of the application being deployed. This version number should be -# incremented each time you make changes to the application. Versions are not expected to -# follow Semantic Versioning. They should reflect the version the application is using. -# It is recommended to use it with quotes. -appVersion: "1.16.0" - -icon: https://raw.githubusercontent.com/openimsdk/open-im-server/main/assets/openim-logo-gradient.svg - -maintainers: - - name: "OpenIM" - url: "https://github.com/openimsdk" - -keywords: - - openim - - im - - chat - -sources: - - "https://github.com/openimsdk/open-im-server" - - "https://github.com/openimsdk/helm-charts" diff --git a/deployments/charts/openim-rpc-friend/templates/NOTES.txt b/deployments/charts/openim-rpc-friend/templates/NOTES.txt deleted file mode 100644 index 004ee5c5e..000000000 --- a/deployments/charts/openim-rpc-friend/templates/NOTES.txt +++ /dev/null @@ -1,22 +0,0 @@ -1. Get the application URL by running these commands: -{{- if .Values.ingress.enabled }} -{{- range $host := .Values.ingress.hosts }} - {{- range .paths }} - http{{ if $.Values.ingress.tls }}s{{ end }}://{{ $host.host }}{{ .path }} - {{- end }} -{{- end }} -{{- else if contains "NodePort" .Values.service.type }} - export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "openim-rpc-friend.fullname" . }}) - export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}") - echo http://$NODE_IP:$NODE_PORT -{{- else if contains "LoadBalancer" .Values.service.type }} - NOTE: It may take a few minutes for the LoadBalancer IP to be available. - You can watch the status of by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "openim-rpc-friend.fullname" . }}' - export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "openim-rpc-friend.fullname" . }} --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}") - echo http://$SERVICE_IP:{{ .Values.service.port }} -{{- else if contains "ClusterIP" .Values.service.type }} - export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "openim-rpc-friend.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") - export CONTAINER_PORT=$(kubectl get pod --namespace {{ .Release.Namespace }} $POD_NAME -o jsonpath="{.spec.containers[0].ports[0].containerPort}") - echo "Visit http://127.0.0.1:8080 to use your application" - kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:$CONTAINER_PORT -{{- end }} diff --git a/deployments/charts/openim-rpc-friend/templates/_helpers.tpl b/deployments/charts/openim-rpc-friend/templates/_helpers.tpl deleted file mode 100644 index 83719fb31..000000000 --- a/deployments/charts/openim-rpc-friend/templates/_helpers.tpl +++ /dev/null @@ -1,62 +0,0 @@ -{{/* -Expand the name of the chart. -*/}} -{{- define "openim-rpc-friend.name" -}} -{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} -{{- end }} - -{{/* -Create a default fully qualified app name. -We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). -If release name contains chart name it will be used as a full name. -*/}} -{{- define "openim-rpc-friend.fullname" -}} -{{- if .Values.fullnameOverride }} -{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} -{{- else }} -{{- $name := default .Chart.Name .Values.nameOverride }} -{{- if contains $name .Release.Name }} -{{- .Release.Name | trunc 63 | trimSuffix "-" }} -{{- else }} -{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} -{{- end }} -{{- end }} -{{- end }} - -{{/* -Create chart name and version as used by the chart label. -*/}} -{{- define "openim-rpc-friend.chart" -}} -{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} -{{- end }} - -{{/* -Common labels -*/}} -{{- define "openim-rpc-friend.labels" -}} -helm.sh/chart: {{ include "openim-rpc-friend.chart" . }} -{{ include "openim-rpc-friend.selectorLabels" . }} -{{- if .Chart.AppVersion }} -app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} -{{- end }} -app.kubernetes.io/managed-by: {{ .Release.Service }} -{{- end }} - -{{/* -Selector labels -*/}} -{{- define "openim-rpc-friend.selectorLabels" -}} -app.kubernetes.io/name: {{ include "openim-rpc-friend.name" . }} -app.kubernetes.io/instance: {{ .Release.Name }} -{{- end }} - -{{/* -Create the name of the service account to use -*/}} -{{- define "openim-rpc-friend.serviceAccountName" -}} -{{- if .Values.serviceAccount.create }} -{{- default (include "openim-rpc-friend.fullname" .) .Values.serviceAccount.name }} -{{- else }} -{{- default "default" .Values.serviceAccount.name }} -{{- end }} -{{- end }} diff --git a/deployments/charts/openim-rpc-friend/templates/deployment.yaml b/deployments/charts/openim-rpc-friend/templates/deployment.yaml deleted file mode 100644 index 01251cdfa..000000000 --- a/deployments/charts/openim-rpc-friend/templates/deployment.yaml +++ /dev/null @@ -1,86 +0,0 @@ -# Copyright © 2023 OpenIM. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ include "openim-rpc-friend.fullname" . }} - labels: - {{- include "openim-rpc-friend.labels" . | nindent 4 }} -spec: - {{- if not .Values.autoscaling.enabled }} - replicas: {{ .Values.replicaCount }} - {{- end }} - selector: - matchLabels: - {{- include "openim-rpc-friend.selectorLabels" . | nindent 6 }} - template: - metadata: - {{- with .Values.podAnnotations }} - annotations: - {{- toYaml . | nindent 8 }} - {{- end }} - labels: - {{- include "openim-rpc-friend.selectorLabels" . | nindent 8 }} - spec: - {{- with .Values.imagePullSecrets }} - imagePullSecrets: - {{- toYaml . | nindent 8 }} - {{- end }} - serviceAccountName: {{ include "openim-rpc-friend.serviceAccountName" . }} - securityContext: - {{- toYaml .Values.podSecurityContext | nindent 8 }} - containers: - - name: {{ .Chart.Name }} - securityContext: - {{- toYaml .Values.securityContext | nindent 12 }} - image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" - imagePullPolicy: {{ .Values.image.pullPolicy }} - ports: - - name: webhook - containerPort: 80 - protocol: TCP - #livenessProbe: - # httpGet: - # path: / - # port: webhook - #readinessProbe: - # httpGet: - # path: / - # port: webhook - resources: - {{- toYaml .Values.resources | nindent 12 }} - volumeMounts: - - mountPath: /openim/openim-server/config/config.yaml - name: config - subPath: config.yaml - - mountPath: /openim/openim-server/config/ - name: config - subPath: notification.yaml - volumes: - - name: config - configMap: - name: openim-cm - {{- with .Values.nodeSelector }} - nodeSelector: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.affinity }} - affinity: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.tolerations }} - tolerations: - {{- toYaml . | nindent 8 }} - {{- end }} diff --git a/deployments/charts/openim-rpc-friend/templates/hpa.yaml b/deployments/charts/openim-rpc-friend/templates/hpa.yaml deleted file mode 100644 index ae88e2621..000000000 --- a/deployments/charts/openim-rpc-friend/templates/hpa.yaml +++ /dev/null @@ -1,42 +0,0 @@ -# Copyright © 2023 OpenIM. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -{{- if .Values.autoscaling.enabled }} -apiVersion: autoscaling/v2beta1 -kind: HorizontalPodAutoscaler -metadata: - name: {{ include "openim-rpc-friend.fullname" . }} - labels: - {{- include "openim-rpc-friend.labels" . | nindent 4 }} -spec: - scaleTargetRef: - apiVersion: apps/v1 - kind: Deployment - name: {{ include "openim-rpc-friend.fullname" . }} - minReplicas: {{ .Values.autoscaling.minReplicas }} - maxReplicas: {{ .Values.autoscaling.maxReplicas }} - metrics: - {{- if .Values.autoscaling.targetCPUUtilizationPercentage }} - - type: Resource - resource: - name: cpu - targetAverageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} - {{- end }} - {{- if .Values.autoscaling.targetMemoryUtilizationPercentage }} - - type: Resource - resource: - name: memory - targetAverageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} - {{- end }} -{{- end }} diff --git a/deployments/charts/openim-rpc-friend/templates/ingress.yaml b/deployments/charts/openim-rpc-friend/templates/ingress.yaml deleted file mode 100644 index 0845aa783..000000000 --- a/deployments/charts/openim-rpc-friend/templates/ingress.yaml +++ /dev/null @@ -1,75 +0,0 @@ -# Copyright © 2023 OpenIM. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -{{- if .Values.ingress.enabled -}} -{{- $fullName := include "openim-rpc-friend.fullname" . -}} -{{- $svcPort := .Values.service.port -}} -{{- if and .Values.ingress.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }} - {{- if not (hasKey .Values.ingress.annotations "kubernetes.io/ingress.class") }} - {{- $_ := set .Values.ingress.annotations "kubernetes.io/ingress.class" .Values.ingress.className}} - {{- end }} -{{- end }} -{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}} -apiVersion: networking.k8s.io/v1 -{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}} -apiVersion: networking.k8s.io/v1beta1 -{{- else -}} -apiVersion: extensions/v1beta1 -{{- end }} -kind: Ingress -metadata: - name: {{ $fullName }} - labels: - {{- include "openim-rpc-friend.labels" . | nindent 4 }} - {{- with .Values.ingress.annotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -spec: - {{- if and .Values.ingress.className (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }} - ingressClassName: {{ .Values.ingress.className }} - {{- end }} - {{- if .Values.ingress.tls }} - tls: - {{- range .Values.ingress.tls }} - - hosts: - {{- range .hosts }} - - {{ . | quote }} - {{- end }} - secretName: {{ .secretName }} - {{- end }} - {{- end }} - rules: - {{- range .Values.ingress.hosts }} - - host: {{ .host | quote }} - http: - paths: - {{- range .paths }} - - path: {{ .path }} - {{- if and .pathType (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }} - pathType: {{ .pathType }} - {{- end }} - backend: - {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }} - service: - name: {{ $fullName }} - port: - number: {{ $svcPort }} - {{- else }} - serviceName: {{ $fullName }} - servicePort: {{ $svcPort }} - {{- end }} - {{- end }} - {{- end }} -{{- end }} diff --git a/deployments/charts/openim-rpc-friend/templates/service.yaml b/deployments/charts/openim-rpc-friend/templates/service.yaml deleted file mode 100644 index 892a007dd..000000000 --- a/deployments/charts/openim-rpc-friend/templates/service.yaml +++ /dev/null @@ -1,29 +0,0 @@ -# Copyright © 2023 OpenIM. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -apiVersion: v1 -kind: Service -metadata: - name: {{ include "openim-rpc-friend.fullname" . }} - labels: - {{- include "openim-rpc-friend.labels" . | nindent 4 }} -spec: - type: {{ .Values.service.type }} - ports: - - port: {{ .Values.service.port }} - targetPort: webhook - protocol: TCP - name: webhook - selector: - {{- include "openim-rpc-friend.selectorLabels" . | nindent 4 }} diff --git a/deployments/charts/openim-rpc-friend/templates/serviceaccount.yaml b/deployments/charts/openim-rpc-friend/templates/serviceaccount.yaml deleted file mode 100644 index 5146d6781..000000000 --- a/deployments/charts/openim-rpc-friend/templates/serviceaccount.yaml +++ /dev/null @@ -1,26 +0,0 @@ -# Copyright © 2023 OpenIM. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -{{- if .Values.serviceAccount.create -}} -apiVersion: v1 -kind: ServiceAccount -metadata: - name: {{ include "openim-rpc-friend.serviceAccountName" . }} - labels: - {{- include "openim-rpc-friend.labels" . | nindent 4 }} - {{- with .Values.serviceAccount.annotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -{{- end }} diff --git a/deployments/charts/openim-rpc-friend/values.yaml b/deployments/charts/openim-rpc-friend/values.yaml deleted file mode 100644 index 34dcda9da..000000000 --- a/deployments/charts/openim-rpc-friend/values.yaml +++ /dev/null @@ -1,96 +0,0 @@ -# Copyright © 2023 OpenIM. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Default values for openim-rpc-friend. -# This is a YAML-formatted file. -# Declare variables to be passed into your templates. - -replicaCount: 1 - -image: - repository: ghcr.io/openimsdk/openim-rpc-friend - pullPolicy: IfNotPresent - # Overrides the image tag whose default is the chart appVersion. - tag: "" - -imagePullSecrets: [] -nameOverride: "" -fullnameOverride: "" - -serviceAccount: - # Specifies whether a service account should be created - create: false - # Annotations to add to the service account - annotations: {} - # The name of the service account to use. - # If not set and create is true, a name is generated using the fullname template - name: "" - -podAnnotations: {} - -podSecurityContext: {} - # fsGroup: 2000 - -securityContext: {} - # capabilities: - # drop: - # - ALL - # readOnlyRootFilesystem: true - # runAsNonRoot: true - # runAsUser: 1000 - -service: - type: ClusterIP - port: 80 - -ingress: - enabled: false - className: "" - annotations: {} - # kubernetes.io/ingress.class: nginx - # kubernetes.io/tls-acme: "true" - hosts: - - host: chart-example.local - paths: - - path: / - pathType: ImplementationSpecific - tls: [] - # - secretName: chart-example-tls - # hosts: - # - chart-example.local - -resources: {} - # We usually recommend not to specify default resources and to leave this as a conscious - # choice for the user. This also increases chances charts run on environments with little - # resources, such as Minikube. If you do want to specify resources, uncomment the following - # lines, adjust them as necessary, and remove the curly braces after 'resources:'. - # limits: - # cpu: 100m - # memory: 128Mi - # requests: - # cpu: 100m - # memory: 128Mi - -autoscaling: - enabled: false - minReplicas: 1 - maxReplicas: 100 - targetCPUUtilizationPercentage: 80 - # targetMemoryUtilizationPercentage: 80 - -nodeSelector: {} - -tolerations: [] - -affinity: {} diff --git a/deployments/charts/openim-rpc-group/.helmignore b/deployments/charts/openim-rpc-group/.helmignore deleted file mode 100644 index 0e8a0eb36..000000000 --- a/deployments/charts/openim-rpc-group/.helmignore +++ /dev/null @@ -1,23 +0,0 @@ -# Patterns to ignore when building packages. -# This supports shell glob matching, relative path matching, and -# negation (prefixed with !). Only one pattern per line. -.DS_Store -# Common VCS dirs -.git/ -.gitignore -.bzr/ -.bzrignore -.hg/ -.hgignore -.svn/ -# Common backup files -*.swp -*.bak -*.tmp -*.orig -*~ -# Various IDEs -.project -.idea/ -*.tmproj -.vscode/ diff --git a/deployments/charts/openim-rpc-group/Chart.yaml b/deployments/charts/openim-rpc-group/Chart.yaml deleted file mode 100644 index c3f7e3bfa..000000000 --- a/deployments/charts/openim-rpc-group/Chart.yaml +++ /dev/null @@ -1,53 +0,0 @@ -# Copyright © 2023 OpenIM. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -apiVersion: v2 -name: openim-rpc-group -description: A Helm chart for Kubernetes - -# A chart can be either an 'application' or a 'library' chart. -# -# Application charts are a collection of templates that can be packaged into versioned archives -# to be deployed. -# -# Library charts provide useful utilities or functions for the chart developer. They're included as -# a dependency of application charts to inject those utilities and functions into the rendering -# pipeline. Library charts do not define any templates and therefore cannot be deployed. -type: application - -# This is the chart version. This version number should be incremented each time you make changes -# to the chart and its templates, including the app version. -# Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.0 - -# This is the version number of the application being deployed. This version number should be -# incremented each time you make changes to the application. Versions are not expected to -# follow Semantic Versioning. They should reflect the version the application is using. -# It is recommended to use it with quotes. -appVersion: "1.16.0" - -icon: https://raw.githubusercontent.com/openimsdk/open-im-server/main/assets/openim-logo-gradient.svg - -maintainers: - - name: "OpenIM" - url: "https://github.com/openimsdk" - -keywords: - - openim - - im - - chat - -sources: - - "https://github.com/openimsdk/open-im-server" - - "https://github.com/openimsdk/helm-charts" \ No newline at end of file diff --git a/deployments/charts/openim-rpc-group/templates/NOTES.txt b/deployments/charts/openim-rpc-group/templates/NOTES.txt deleted file mode 100644 index 0cc117a49..000000000 --- a/deployments/charts/openim-rpc-group/templates/NOTES.txt +++ /dev/null @@ -1,22 +0,0 @@ -1. Get the application URL by running these commands: -{{- if .Values.ingress.enabled }} -{{- range $host := .Values.ingress.hosts }} - {{- range .paths }} - http{{ if $.Values.ingress.tls }}s{{ end }}://{{ $host.host }}{{ .path }} - {{- end }} -{{- end }} -{{- else if contains "NodePort" .Values.service.type }} - export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "openim-rpc-group.fullname" . }}) - export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}") - echo http://$NODE_IP:$NODE_PORT -{{- else if contains "LoadBalancer" .Values.service.type }} - NOTE: It may take a few minutes for the LoadBalancer IP to be available. - You can watch the status of by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "openim-rpc-group.fullname" . }}' - export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "openim-rpc-group.fullname" . }} --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}") - echo http://$SERVICE_IP:{{ .Values.service.port }} -{{- else if contains "ClusterIP" .Values.service.type }} - export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "openim-rpc-group.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") - export CONTAINER_PORT=$(kubectl get pod --namespace {{ .Release.Namespace }} $POD_NAME -o jsonpath="{.spec.containers[0].ports[0].containerPort}") - echo "Visit http://127.0.0.1:8080 to use your application" - kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:$CONTAINER_PORT -{{- end }} diff --git a/deployments/charts/openim-rpc-group/templates/_helpers.tpl b/deployments/charts/openim-rpc-group/templates/_helpers.tpl deleted file mode 100644 index 7835b84e1..000000000 --- a/deployments/charts/openim-rpc-group/templates/_helpers.tpl +++ /dev/null @@ -1,62 +0,0 @@ -{{/* -Expand the name of the chart. -*/}} -{{- define "openim-rpc-group.name" -}} -{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} -{{- end }} - -{{/* -Create a default fully qualified app name. -We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). -If release name contains chart name it will be used as a full name. -*/}} -{{- define "openim-rpc-group.fullname" -}} -{{- if .Values.fullnameOverride }} -{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} -{{- else }} -{{- $name := default .Chart.Name .Values.nameOverride }} -{{- if contains $name .Release.Name }} -{{- .Release.Name | trunc 63 | trimSuffix "-" }} -{{- else }} -{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} -{{- end }} -{{- end }} -{{- end }} - -{{/* -Create chart name and version as used by the chart label. -*/}} -{{- define "openim-rpc-group.chart" -}} -{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} -{{- end }} - -{{/* -Common labels -*/}} -{{- define "openim-rpc-group.labels" -}} -helm.sh/chart: {{ include "openim-rpc-group.chart" . }} -{{ include "openim-rpc-group.selectorLabels" . }} -{{- if .Chart.AppVersion }} -app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} -{{- end }} -app.kubernetes.io/managed-by: {{ .Release.Service }} -{{- end }} - -{{/* -Selector labels -*/}} -{{- define "openim-rpc-group.selectorLabels" -}} -app.kubernetes.io/name: {{ include "openim-rpc-group.name" . }} -app.kubernetes.io/instance: {{ .Release.Name }} -{{- end }} - -{{/* -Create the name of the service account to use -*/}} -{{- define "openim-rpc-group.serviceAccountName" -}} -{{- if .Values.serviceAccount.create }} -{{- default (include "openim-rpc-group.fullname" .) .Values.serviceAccount.name }} -{{- else }} -{{- default "default" .Values.serviceAccount.name }} -{{- end }} -{{- end }} diff --git a/deployments/charts/openim-rpc-group/templates/deployment.yaml b/deployments/charts/openim-rpc-group/templates/deployment.yaml deleted file mode 100644 index e738f33be..000000000 --- a/deployments/charts/openim-rpc-group/templates/deployment.yaml +++ /dev/null @@ -1,86 +0,0 @@ -# Copyright © 2023 OpenIM. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ include "openim-rpc-group.fullname" . }} - labels: - {{- include "openim-rpc-group.labels" . | nindent 4 }} -spec: - {{- if not .Values.autoscaling.enabled }} - replicas: {{ .Values.replicaCount }} - {{- end }} - selector: - matchLabels: - {{- include "openim-rpc-group.selectorLabels" . | nindent 6 }} - template: - metadata: - {{- with .Values.podAnnotations }} - annotations: - {{- toYaml . | nindent 8 }} - {{- end }} - labels: - {{- include "openim-rpc-group.selectorLabels" . | nindent 8 }} - spec: - {{- with .Values.imagePullSecrets }} - imagePullSecrets: - {{- toYaml . | nindent 8 }} - {{- end }} - serviceAccountName: {{ include "openim-rpc-group.serviceAccountName" . }} - securityContext: - {{- toYaml .Values.podSecurityContext | nindent 8 }} - containers: - - name: {{ .Chart.Name }} - securityContext: - {{- toYaml .Values.securityContext | nindent 12 }} - image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" - imagePullPolicy: {{ .Values.image.pullPolicy }} - ports: - - name: webhook - containerPort: 80 - protocol: TCP - #livenessProbe: - # httpGet: - # path: / - # port: webhook - #readinessProbe: - # httpGet: - # path: / - # port: webhook - resources: - {{- toYaml .Values.resources | nindent 12 }} - volumeMounts: - - mountPath: /openim/openim-server/config/config.yaml - name: config - subPath: config.yaml - - mountPath: /openim/openim-server/config/ - name: config - subPath: notification.yaml - volumes: - - name: config - configMap: - name: openim-cm - {{- with .Values.nodeSelector }} - nodeSelector: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.affinity }} - affinity: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.tolerations }} - tolerations: - {{- toYaml . | nindent 8 }} - {{- end }} diff --git a/deployments/charts/openim-rpc-group/templates/hpa.yaml b/deployments/charts/openim-rpc-group/templates/hpa.yaml deleted file mode 100644 index 91f313abe..000000000 --- a/deployments/charts/openim-rpc-group/templates/hpa.yaml +++ /dev/null @@ -1,42 +0,0 @@ -# Copyright © 2023 OpenIM. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -{{- if .Values.autoscaling.enabled }} -apiVersion: autoscaling/v2beta1 -kind: HorizontalPodAutoscaler -metadata: - name: {{ include "openim-rpc-group.fullname" . }} - labels: - {{- include "openim-rpc-group.labels" . | nindent 4 }} -spec: - scaleTargetRef: - apiVersion: apps/v1 - kind: Deployment - name: {{ include "openim-rpc-group.fullname" . }} - minReplicas: {{ .Values.autoscaling.minReplicas }} - maxReplicas: {{ .Values.autoscaling.maxReplicas }} - metrics: - {{- if .Values.autoscaling.targetCPUUtilizationPercentage }} - - type: Resource - resource: - name: cpu - targetAverageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} - {{- end }} - {{- if .Values.autoscaling.targetMemoryUtilizationPercentage }} - - type: Resource - resource: - name: memory - targetAverageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} - {{- end }} -{{- end }} diff --git a/deployments/charts/openim-rpc-group/templates/ingress.yaml b/deployments/charts/openim-rpc-group/templates/ingress.yaml deleted file mode 100644 index e3cad781a..000000000 --- a/deployments/charts/openim-rpc-group/templates/ingress.yaml +++ /dev/null @@ -1,75 +0,0 @@ -# Copyright © 2023 OpenIM. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -{{- if .Values.ingress.enabled -}} -{{- $fullName := include "openim-rpc-group.fullname" . -}} -{{- $svcPort := .Values.service.port -}} -{{- if and .Values.ingress.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }} - {{- if not (hasKey .Values.ingress.annotations "kubernetes.io/ingress.class") }} - {{- $_ := set .Values.ingress.annotations "kubernetes.io/ingress.class" .Values.ingress.className}} - {{- end }} -{{- end }} -{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}} -apiVersion: networking.k8s.io/v1 -{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}} -apiVersion: networking.k8s.io/v1beta1 -{{- else -}} -apiVersion: extensions/v1beta1 -{{- end }} -kind: Ingress -metadata: - name: {{ $fullName }} - labels: - {{- include "openim-rpc-group.labels" . | nindent 4 }} - {{- with .Values.ingress.annotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -spec: - {{- if and .Values.ingress.className (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }} - ingressClassName: {{ .Values.ingress.className }} - {{- end }} - {{- if .Values.ingress.tls }} - tls: - {{- range .Values.ingress.tls }} - - hosts: - {{- range .hosts }} - - {{ . | quote }} - {{- end }} - secretName: {{ .secretName }} - {{- end }} - {{- end }} - rules: - {{- range .Values.ingress.hosts }} - - host: {{ .host | quote }} - http: - paths: - {{- range .paths }} - - path: {{ .path }} - {{- if and .pathType (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }} - pathType: {{ .pathType }} - {{- end }} - backend: - {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }} - service: - name: {{ $fullName }} - port: - number: {{ $svcPort }} - {{- else }} - serviceName: {{ $fullName }} - servicePort: {{ $svcPort }} - {{- end }} - {{- end }} - {{- end }} -{{- end }} diff --git a/deployments/charts/openim-rpc-group/templates/service.yaml b/deployments/charts/openim-rpc-group/templates/service.yaml deleted file mode 100644 index 42e1f78ca..000000000 --- a/deployments/charts/openim-rpc-group/templates/service.yaml +++ /dev/null @@ -1,29 +0,0 @@ -# Copyright © 2023 OpenIM. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -apiVersion: v1 -kind: Service -metadata: - name: {{ include "openim-rpc-group.fullname" . }} - labels: - {{- include "openim-rpc-group.labels" . | nindent 4 }} -spec: - type: {{ .Values.service.type }} - ports: - - port: {{ .Values.service.port }} - targetPort: webhook - protocol: TCP - name: webhook - selector: - {{- include "openim-rpc-group.selectorLabels" . | nindent 4 }} diff --git a/deployments/charts/openim-rpc-group/templates/serviceaccount.yaml b/deployments/charts/openim-rpc-group/templates/serviceaccount.yaml deleted file mode 100644 index 304bbda56..000000000 --- a/deployments/charts/openim-rpc-group/templates/serviceaccount.yaml +++ /dev/null @@ -1,26 +0,0 @@ -# Copyright © 2023 OpenIM. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -{{- if .Values.serviceAccount.create -}} -apiVersion: v1 -kind: ServiceAccount -metadata: - name: {{ include "openim-rpc-group.serviceAccountName" . }} - labels: - {{- include "openim-rpc-group.labels" . | nindent 4 }} - {{- with .Values.serviceAccount.annotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -{{- end }} diff --git a/deployments/charts/openim-rpc-group/values.yaml b/deployments/charts/openim-rpc-group/values.yaml deleted file mode 100644 index f04c64147..000000000 --- a/deployments/charts/openim-rpc-group/values.yaml +++ /dev/null @@ -1,96 +0,0 @@ -# Copyright © 2023 OpenIM. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Default values for openim-rpc-group. -# This is a YAML-formatted file. -# Declare variables to be passed into your templates. - -replicaCount: 1 - -image: - repository: ghcr.io/openimsdk/openim-rpc-group - pullPolicy: IfNotPresent - # Overrides the image tag whose default is the chart appVersion. - tag: "" - -imagePullSecrets: [] -nameOverride: "" -fullnameOverride: "" - -serviceAccount: - # Specifies whether a service account should be created - create: false - # Annotations to add to the service account - annotations: {} - # The name of the service account to use. - # If not set and create is true, a name is generated using the fullname template - name: "" - -podAnnotations: {} - -podSecurityContext: {} - # fsGroup: 2000 - -securityContext: {} - # capabilities: - # drop: - # - ALL - # readOnlyRootFilesystem: true - # runAsNonRoot: true - # runAsUser: 1000 - -service: - type: ClusterIP - port: 80 - -ingress: - enabled: false - className: "" - annotations: {} - # kubernetes.io/ingress.class: nginx - # kubernetes.io/tls-acme: "true" - hosts: - - host: chart-example.local - paths: - - path: / - pathType: ImplementationSpecific - tls: [] - # - secretName: chart-example-tls - # hosts: - # - chart-example.local - -resources: {} - # We usually recommend not to specify default resources and to leave this as a conscious - # choice for the user. This also increases chances charts run on environments with little - # resources, such as Minikube. If you do want to specify resources, uncomment the following - # lines, adjust them as necessary, and remove the curly braces after 'resources:'. - # limits: - # cpu: 100m - # memory: 128Mi - # requests: - # cpu: 100m - # memory: 128Mi - -autoscaling: - enabled: false - minReplicas: 1 - maxReplicas: 100 - targetCPUUtilizationPercentage: 80 - # targetMemoryUtilizationPercentage: 80 - -nodeSelector: {} - -tolerations: [] - -affinity: {} diff --git a/deployments/charts/openim-rpc-msg/.helmignore b/deployments/charts/openim-rpc-msg/.helmignore deleted file mode 100644 index 0e8a0eb36..000000000 --- a/deployments/charts/openim-rpc-msg/.helmignore +++ /dev/null @@ -1,23 +0,0 @@ -# Patterns to ignore when building packages. -# This supports shell glob matching, relative path matching, and -# negation (prefixed with !). Only one pattern per line. -.DS_Store -# Common VCS dirs -.git/ -.gitignore -.bzr/ -.bzrignore -.hg/ -.hgignore -.svn/ -# Common backup files -*.swp -*.bak -*.tmp -*.orig -*~ -# Various IDEs -.project -.idea/ -*.tmproj -.vscode/ diff --git a/deployments/charts/openim-rpc-msg/Chart.yaml b/deployments/charts/openim-rpc-msg/Chart.yaml deleted file mode 100644 index c32fe64c1..000000000 --- a/deployments/charts/openim-rpc-msg/Chart.yaml +++ /dev/null @@ -1,53 +0,0 @@ -# Copyright © 2023 OpenIM. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -apiVersion: v2 -name: openim-rpc-msg -description: A Helm chart for Kubernetes - -# A chart can be either an 'application' or a 'library' chart. -# -# Application charts are a collection of templates that can be packaged into versioned archives -# to be deployed. -# -# Library charts provide useful utilities or functions for the chart developer. They're included as -# a dependency of application charts to inject those utilities and functions into the rendering -# pipeline. Library charts do not define any templates and therefore cannot be deployed. -type: application - -# This is the chart version. This version number should be incremented each time you make changes -# to the chart and its templates, including the app version. -# Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.0 - -# This is the version number of the application being deployed. This version number should be -# incremented each time you make changes to the application. Versions are not expected to -# follow Semantic Versioning. They should reflect the version the application is using. -# It is recommended to use it with quotes. -appVersion: "1.16.0" - -icon: https://raw.githubusercontent.com/openimsdk/open-im-server/main/assets/openim-logo-gradient.svg - -maintainers: - - name: "OpenIM" - url: "https://github.com/openimsdk" - -keywords: - - openim - - im - - chat - -sources: - - "https://github.com/openimsdk/open-im-server" - - "https://github.com/openimsdk/helm-charts" \ No newline at end of file diff --git a/deployments/charts/openim-rpc-msg/templates/NOTES.txt b/deployments/charts/openim-rpc-msg/templates/NOTES.txt deleted file mode 100644 index 65033b107..000000000 --- a/deployments/charts/openim-rpc-msg/templates/NOTES.txt +++ /dev/null @@ -1,22 +0,0 @@ -1. Get the application URL by running these commands: -{{- if .Values.ingress.enabled }} -{{- range $host := .Values.ingress.hosts }} - {{- range .paths }} - http{{ if $.Values.ingress.tls }}s{{ end }}://{{ $host.host }}{{ .path }} - {{- end }} -{{- end }} -{{- else if contains "NodePort" .Values.service.type }} - export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "openim-rpc-msg.fullname" . }}) - export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}") - echo http://$NODE_IP:$NODE_PORT -{{- else if contains "LoadBalancer" .Values.service.type }} - NOTE: It may take a few minutes for the LoadBalancer IP to be available. - You can watch the status of by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "openim-rpc-msg.fullname" . }}' - export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "openim-rpc-msg.fullname" . }} --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}") - echo http://$SERVICE_IP:{{ .Values.service.port }} -{{- else if contains "ClusterIP" .Values.service.type }} - export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "openim-rpc-msg.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") - export CONTAINER_PORT=$(kubectl get pod --namespace {{ .Release.Namespace }} $POD_NAME -o jsonpath="{.spec.containers[0].ports[0].containerPort}") - echo "Visit http://127.0.0.1:8080 to use your application" - kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:$CONTAINER_PORT -{{- end }} diff --git a/deployments/charts/openim-rpc-msg/templates/_helpers.tpl b/deployments/charts/openim-rpc-msg/templates/_helpers.tpl deleted file mode 100644 index da6956822..000000000 --- a/deployments/charts/openim-rpc-msg/templates/_helpers.tpl +++ /dev/null @@ -1,62 +0,0 @@ -{{/* -Expand the name of the chart. -*/}} -{{- define "openim-rpc-msg.name" -}} -{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} -{{- end }} - -{{/* -Create a default fully qualified app name. -We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). -If release name contains chart name it will be used as a full name. -*/}} -{{- define "openim-rpc-msg.fullname" -}} -{{- if .Values.fullnameOverride }} -{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} -{{- else }} -{{- $name := default .Chart.Name .Values.nameOverride }} -{{- if contains $name .Release.Name }} -{{- .Release.Name | trunc 63 | trimSuffix "-" }} -{{- else }} -{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} -{{- end }} -{{- end }} -{{- end }} - -{{/* -Create chart name and version as used by the chart label. -*/}} -{{- define "openim-rpc-msg.chart" -}} -{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} -{{- end }} - -{{/* -Common labels -*/}} -{{- define "openim-rpc-msg.labels" -}} -helm.sh/chart: {{ include "openim-rpc-msg.chart" . }} -{{ include "openim-rpc-msg.selectorLabels" . }} -{{- if .Chart.AppVersion }} -app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} -{{- end }} -app.kubernetes.io/managed-by: {{ .Release.Service }} -{{- end }} - -{{/* -Selector labels -*/}} -{{- define "openim-rpc-msg.selectorLabels" -}} -app.kubernetes.io/name: {{ include "openim-rpc-msg.name" . }} -app.kubernetes.io/instance: {{ .Release.Name }} -{{- end }} - -{{/* -Create the name of the service account to use -*/}} -{{- define "openim-rpc-msg.serviceAccountName" -}} -{{- if .Values.serviceAccount.create }} -{{- default (include "openim-rpc-msg.fullname" .) .Values.serviceAccount.name }} -{{- else }} -{{- default "default" .Values.serviceAccount.name }} -{{- end }} -{{- end }} diff --git a/deployments/charts/openim-rpc-msg/templates/deployment.yaml b/deployments/charts/openim-rpc-msg/templates/deployment.yaml deleted file mode 100644 index f7267fabb..000000000 --- a/deployments/charts/openim-rpc-msg/templates/deployment.yaml +++ /dev/null @@ -1,86 +0,0 @@ -# Copyright © 2023 OpenIM. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ include "openim-rpc-msg.fullname" . }} - labels: - {{- include "openim-rpc-msg.labels" . | nindent 4 }} -spec: - {{- if not .Values.autoscaling.enabled }} - replicas: {{ .Values.replicaCount }} - {{- end }} - selector: - matchLabels: - {{- include "openim-rpc-msg.selectorLabels" . | nindent 6 }} - template: - metadata: - {{- with .Values.podAnnotations }} - annotations: - {{- toYaml . | nindent 8 }} - {{- end }} - labels: - {{- include "openim-rpc-msg.selectorLabels" . | nindent 8 }} - spec: - {{- with .Values.imagePullSecrets }} - imagePullSecrets: - {{- toYaml . | nindent 8 }} - {{- end }} - serviceAccountName: {{ include "openim-rpc-msg.serviceAccountName" . }} - securityContext: - {{- toYaml .Values.podSecurityContext | nindent 8 }} - containers: - - name: {{ .Chart.Name }} - securityContext: - {{- toYaml .Values.securityContext | nindent 12 }} - image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" - imagePullPolicy: {{ .Values.image.pullPolicy }} - ports: - - name: webhook - containerPort: 80 - protocol: TCP - #livenessProbe: - # httpGet: - # path: / - # port: webhook - #readinessProbe: - # httpGet: - # path: / - # port: webhook - resources: - {{- toYaml .Values.resources | nindent 12 }} - volumeMounts: - - mountPath: /openim/openim-server/config/config.yaml - name: config - subPath: config.yaml - - mountPath: /openim/openim-server/config/ - name: config - subPath: notification.yaml - volumes: - - name: config - configMap: - name: openim-cm - {{- with .Values.nodeSelector }} - nodeSelector: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.affinity }} - affinity: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.tolerations }} - tolerations: - {{- toYaml . | nindent 8 }} - {{- end }} diff --git a/deployments/charts/openim-rpc-msg/templates/hpa.yaml b/deployments/charts/openim-rpc-msg/templates/hpa.yaml deleted file mode 100644 index 0e6dde369..000000000 --- a/deployments/charts/openim-rpc-msg/templates/hpa.yaml +++ /dev/null @@ -1,42 +0,0 @@ -# Copyright © 2023 OpenIM. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -{{- if .Values.autoscaling.enabled }} -apiVersion: autoscaling/v2beta1 -kind: HorizontalPodAutoscaler -metadata: - name: {{ include "openim-rpc-msg.fullname" . }} - labels: - {{- include "openim-rpc-msg.labels" . | nindent 4 }} -spec: - scaleTargetRef: - apiVersion: apps/v1 - kind: Deployment - name: {{ include "openim-rpc-msg.fullname" . }} - minReplicas: {{ .Values.autoscaling.minReplicas }} - maxReplicas: {{ .Values.autoscaling.maxReplicas }} - metrics: - {{- if .Values.autoscaling.targetCPUUtilizationPercentage }} - - type: Resource - resource: - name: cpu - targetAverageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} - {{- end }} - {{- if .Values.autoscaling.targetMemoryUtilizationPercentage }} - - type: Resource - resource: - name: memory - targetAverageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} - {{- end }} -{{- end }} diff --git a/deployments/charts/openim-rpc-msg/templates/ingress.yaml b/deployments/charts/openim-rpc-msg/templates/ingress.yaml deleted file mode 100644 index 2ba587338..000000000 --- a/deployments/charts/openim-rpc-msg/templates/ingress.yaml +++ /dev/null @@ -1,75 +0,0 @@ -# Copyright © 2023 OpenIM. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -{{- if .Values.ingress.enabled -}} -{{- $fullName := include "openim-rpc-msg.fullname" . -}} -{{- $svcPort := .Values.service.port -}} -{{- if and .Values.ingress.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }} - {{- if not (hasKey .Values.ingress.annotations "kubernetes.io/ingress.class") }} - {{- $_ := set .Values.ingress.annotations "kubernetes.io/ingress.class" .Values.ingress.className}} - {{- end }} -{{- end }} -{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}} -apiVersion: networking.k8s.io/v1 -{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}} -apiVersion: networking.k8s.io/v1beta1 -{{- else -}} -apiVersion: extensions/v1beta1 -{{- end }} -kind: Ingress -metadata: - name: {{ $fullName }} - labels: - {{- include "openim-rpc-msg.labels" . | nindent 4 }} - {{- with .Values.ingress.annotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -spec: - {{- if and .Values.ingress.className (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }} - ingressClassName: {{ .Values.ingress.className }} - {{- end }} - {{- if .Values.ingress.tls }} - tls: - {{- range .Values.ingress.tls }} - - hosts: - {{- range .hosts }} - - {{ . | quote }} - {{- end }} - secretName: {{ .secretName }} - {{- end }} - {{- end }} - rules: - {{- range .Values.ingress.hosts }} - - host: {{ .host | quote }} - http: - paths: - {{- range .paths }} - - path: {{ .path }} - {{- if and .pathType (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }} - pathType: {{ .pathType }} - {{- end }} - backend: - {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }} - service: - name: {{ $fullName }} - port: - number: {{ $svcPort }} - {{- else }} - serviceName: {{ $fullName }} - servicePort: {{ $svcPort }} - {{- end }} - {{- end }} - {{- end }} -{{- end }} diff --git a/deployments/charts/openim-rpc-msg/templates/service.yaml b/deployments/charts/openim-rpc-msg/templates/service.yaml deleted file mode 100644 index ba403d5ab..000000000 --- a/deployments/charts/openim-rpc-msg/templates/service.yaml +++ /dev/null @@ -1,29 +0,0 @@ -# Copyright © 2023 OpenIM. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -apiVersion: v1 -kind: Service -metadata: - name: {{ include "openim-rpc-msg.fullname" . }} - labels: - {{- include "openim-rpc-msg.labels" . | nindent 4 }} -spec: - type: {{ .Values.service.type }} - ports: - - port: {{ .Values.service.port }} - targetPort: webhook - protocol: TCP - name: webhook - selector: - {{- include "openim-rpc-msg.selectorLabels" . | nindent 4 }} diff --git a/deployments/charts/openim-rpc-msg/templates/serviceaccount.yaml b/deployments/charts/openim-rpc-msg/templates/serviceaccount.yaml deleted file mode 100644 index 70fd82069..000000000 --- a/deployments/charts/openim-rpc-msg/templates/serviceaccount.yaml +++ /dev/null @@ -1,26 +0,0 @@ -# Copyright © 2023 OpenIM. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -{{- if .Values.serviceAccount.create -}} -apiVersion: v1 -kind: ServiceAccount -metadata: - name: {{ include "openim-rpc-msg.serviceAccountName" . }} - labels: - {{- include "openim-rpc-msg.labels" . | nindent 4 }} - {{- with .Values.serviceAccount.annotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -{{- end }} diff --git a/deployments/charts/openim-rpc-msg/values.yaml b/deployments/charts/openim-rpc-msg/values.yaml deleted file mode 100644 index cf2f4e84d..000000000 --- a/deployments/charts/openim-rpc-msg/values.yaml +++ /dev/null @@ -1,96 +0,0 @@ -# Copyright © 2023 OpenIM. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Default values for openim-rpc-msg. -# This is a YAML-formatted file. -# Declare variables to be passed into your templates. - -replicaCount: 1 - -image: - repository: ghcr.io/openimsdk/openim-rpc-msg - pullPolicy: IfNotPresent - # Overrides the image tag whose default is the chart appVersion. - tag: "" - -imagePullSecrets: [] -nameOverride: "" -fullnameOverride: "" - -serviceAccount: - # Specifies whether a service account should be created - create: false - # Annotations to add to the service account - annotations: {} - # The name of the service account to use. - # If not set and create is true, a name is generated using the fullname template - name: "" - -podAnnotations: {} - -podSecurityContext: {} - # fsGroup: 2000 - -securityContext: {} - # capabilities: - # drop: - # - ALL - # readOnlyRootFilesystem: true - # runAsNonRoot: true - # runAsUser: 1000 - -service: - type: ClusterIP - port: 80 - -ingress: - enabled: false - className: "" - annotations: {} - # kubernetes.io/ingress.class: nginx - # kubernetes.io/tls-acme: "true" - hosts: - - host: chart-example.local - paths: - - path: / - pathType: ImplementationSpecific - tls: [] - # - secretName: chart-example-tls - # hosts: - # - chart-example.local - -resources: {} - # We usually recommend not to specify default resources and to leave this as a conscious - # choice for the user. This also increases chances charts run on environments with little - # resources, such as Minikube. If you do want to specify resources, uncomment the following - # lines, adjust them as necessary, and remove the curly braces after 'resources:'. - # limits: - # cpu: 100m - # memory: 128Mi - # requests: - # cpu: 100m - # memory: 128Mi - -autoscaling: - enabled: false - minReplicas: 1 - maxReplicas: 100 - targetCPUUtilizationPercentage: 80 - # targetMemoryUtilizationPercentage: 80 - -nodeSelector: {} - -tolerations: [] - -affinity: {} diff --git a/deployments/charts/openim-rpc-third/.helmignore b/deployments/charts/openim-rpc-third/.helmignore deleted file mode 100644 index 0e8a0eb36..000000000 --- a/deployments/charts/openim-rpc-third/.helmignore +++ /dev/null @@ -1,23 +0,0 @@ -# Patterns to ignore when building packages. -# This supports shell glob matching, relative path matching, and -# negation (prefixed with !). Only one pattern per line. -.DS_Store -# Common VCS dirs -.git/ -.gitignore -.bzr/ -.bzrignore -.hg/ -.hgignore -.svn/ -# Common backup files -*.swp -*.bak -*.tmp -*.orig -*~ -# Various IDEs -.project -.idea/ -*.tmproj -.vscode/ diff --git a/deployments/charts/openim-rpc-third/Chart.yaml b/deployments/charts/openim-rpc-third/Chart.yaml deleted file mode 100644 index ff624aa34..000000000 --- a/deployments/charts/openim-rpc-third/Chart.yaml +++ /dev/null @@ -1,53 +0,0 @@ -# Copyright © 2023 OpenIM. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -apiVersion: v2 -name: openim-rpc-third -description: A Helm chart for Kubernetes - -# A chart can be either an 'application' or a 'library' chart. -# -# Application charts are a collection of templates that can be packaged into versioned archives -# to be deployed. -# -# Library charts provide useful utilities or functions for the chart developer. They're included as -# a dependency of application charts to inject those utilities and functions into the rendering -# pipeline. Library charts do not define any templates and therefore cannot be deployed. -type: application - -# This is the chart version. This version number should be incremented each time you make changes -# to the chart and its templates, including the app version. -# Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.0 - -# This is the version number of the application being deployed. This version number should be -# incremented each time you make changes to the application. Versions are not expected to -# follow Semantic Versioning. They should reflect the version the application is using. -# It is recommended to use it with quotes. -appVersion: "1.16.0" - -icon: https://raw.githubusercontent.com/openimsdk/open-im-server/main/assets/openim-logo-gradient.svg - -maintainers: - - name: "OpenIM" - url: "https://github.com/openimsdk" - -keywords: - - openim - - im - - chat - -sources: - - "https://github.com/openimsdk/open-im-server" - - "https://github.com/openimsdk/helm-charts" \ No newline at end of file diff --git a/deployments/charts/openim-rpc-third/templates/NOTES.txt b/deployments/charts/openim-rpc-third/templates/NOTES.txt deleted file mode 100644 index a2228c84d..000000000 --- a/deployments/charts/openim-rpc-third/templates/NOTES.txt +++ /dev/null @@ -1,22 +0,0 @@ -1. Get the application URL by running these commands: -{{- if .Values.ingress.enabled }} -{{- range $host := .Values.ingress.hosts }} - {{- range .paths }} - http{{ if $.Values.ingress.tls }}s{{ end }}://{{ $host.host }}{{ .path }} - {{- end }} -{{- end }} -{{- else if contains "NodePort" .Values.service.type }} - export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "openim-rpc-third.fullname" . }}) - export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}") - echo http://$NODE_IP:$NODE_PORT -{{- else if contains "LoadBalancer" .Values.service.type }} - NOTE: It may take a few minutes for the LoadBalancer IP to be available. - You can watch the status of by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "openim-rpc-third.fullname" . }}' - export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "openim-rpc-third.fullname" . }} --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}") - echo http://$SERVICE_IP:{{ .Values.service.port }} -{{- else if contains "ClusterIP" .Values.service.type }} - export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "openim-rpc-third.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") - export CONTAINER_PORT=$(kubectl get pod --namespace {{ .Release.Namespace }} $POD_NAME -o jsonpath="{.spec.containers[0].ports[0].containerPort}") - echo "Visit http://127.0.0.1:8080 to use your application" - kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:$CONTAINER_PORT -{{- end }} diff --git a/deployments/charts/openim-rpc-third/templates/_helpers.tpl b/deployments/charts/openim-rpc-third/templates/_helpers.tpl deleted file mode 100644 index 46039ce2c..000000000 --- a/deployments/charts/openim-rpc-third/templates/_helpers.tpl +++ /dev/null @@ -1,62 +0,0 @@ -{{/* -Expand the name of the chart. -*/}} -{{- define "openim-rpc-third.name" -}} -{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} -{{- end }} - -{{/* -Create a default fully qualified app name. -We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). -If release name contains chart name it will be used as a full name. -*/}} -{{- define "openim-rpc-third.fullname" -}} -{{- if .Values.fullnameOverride }} -{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} -{{- else }} -{{- $name := default .Chart.Name .Values.nameOverride }} -{{- if contains $name .Release.Name }} -{{- .Release.Name | trunc 63 | trimSuffix "-" }} -{{- else }} -{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} -{{- end }} -{{- end }} -{{- end }} - -{{/* -Create chart name and version as used by the chart label. -*/}} -{{- define "openim-rpc-third.chart" -}} -{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} -{{- end }} - -{{/* -Common labels -*/}} -{{- define "openim-rpc-third.labels" -}} -helm.sh/chart: {{ include "openim-rpc-third.chart" . }} -{{ include "openim-rpc-third.selectorLabels" . }} -{{- if .Chart.AppVersion }} -app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} -{{- end }} -app.kubernetes.io/managed-by: {{ .Release.Service }} -{{- end }} - -{{/* -Selector labels -*/}} -{{- define "openim-rpc-third.selectorLabels" -}} -app.kubernetes.io/name: {{ include "openim-rpc-third.name" . }} -app.kubernetes.io/instance: {{ .Release.Name }} -{{- end }} - -{{/* -Create the name of the service account to use -*/}} -{{- define "openim-rpc-third.serviceAccountName" -}} -{{- if .Values.serviceAccount.create }} -{{- default (include "openim-rpc-third.fullname" .) .Values.serviceAccount.name }} -{{- else }} -{{- default "default" .Values.serviceAccount.name }} -{{- end }} -{{- end }} diff --git a/deployments/charts/openim-rpc-third/templates/deployment.yaml b/deployments/charts/openim-rpc-third/templates/deployment.yaml deleted file mode 100644 index 779415535..000000000 --- a/deployments/charts/openim-rpc-third/templates/deployment.yaml +++ /dev/null @@ -1,86 +0,0 @@ -# Copyright © 2023 OpenIM. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ include "openim-rpc-third.fullname" . }} - labels: - {{- include "openim-rpc-third.labels" . | nindent 4 }} -spec: - {{- if not .Values.autoscaling.enabled }} - replicas: {{ .Values.replicaCount }} - {{- end }} - selector: - matchLabels: - {{- include "openim-rpc-third.selectorLabels" . | nindent 6 }} - template: - metadata: - {{- with .Values.podAnnotations }} - annotations: - {{- toYaml . | nindent 8 }} - {{- end }} - labels: - {{- include "openim-rpc-third.selectorLabels" . | nindent 8 }} - spec: - {{- with .Values.imagePullSecrets }} - imagePullSecrets: - {{- toYaml . | nindent 8 }} - {{- end }} - serviceAccountName: {{ include "openim-rpc-third.serviceAccountName" . }} - securityContext: - {{- toYaml .Values.podSecurityContext | nindent 8 }} - containers: - - name: {{ .Chart.Name }} - securityContext: - {{- toYaml .Values.securityContext | nindent 12 }} - image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" - imagePullPolicy: {{ .Values.image.pullPolicy }} - ports: - - name: webhook - containerPort: 80 - protocol: TCP - #livenessProbe: - # httpGet: - # path: / - # port: webhook - #readinessProbe: - # httpGet: - # path: / - # port: webhook - resources: - {{- toYaml .Values.resources | nindent 12 }} - volumeMounts: - - mountPath: /openim/openim-server/config/config.yaml - name: config - subPath: config.yaml - - mountPath: /openim/openim-server/config/ - name: config - subPath: notification.yaml - volumes: - - name: config - configMap: - name: openim-cm - {{- with .Values.nodeSelector }} - nodeSelector: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.affinity }} - affinity: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.tolerations }} - tolerations: - {{- toYaml . | nindent 8 }} - {{- end }} diff --git a/deployments/charts/openim-rpc-third/templates/hpa.yaml b/deployments/charts/openim-rpc-third/templates/hpa.yaml deleted file mode 100644 index f20ecb973..000000000 --- a/deployments/charts/openim-rpc-third/templates/hpa.yaml +++ /dev/null @@ -1,42 +0,0 @@ -# Copyright © 2023 OpenIM. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -{{- if .Values.autoscaling.enabled }} -apiVersion: autoscaling/v2beta1 -kind: HorizontalPodAutoscaler -metadata: - name: {{ include "openim-rpc-third.fullname" . }} - labels: - {{- include "openim-rpc-third.labels" . | nindent 4 }} -spec: - scaleTargetRef: - apiVersion: apps/v1 - kind: Deployment - name: {{ include "openim-rpc-third.fullname" . }} - minReplicas: {{ .Values.autoscaling.minReplicas }} - maxReplicas: {{ .Values.autoscaling.maxReplicas }} - metrics: - {{- if .Values.autoscaling.targetCPUUtilizationPercentage }} - - type: Resource - resource: - name: cpu - targetAverageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} - {{- end }} - {{- if .Values.autoscaling.targetMemoryUtilizationPercentage }} - - type: Resource - resource: - name: memory - targetAverageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} - {{- end }} -{{- end }} diff --git a/deployments/charts/openim-rpc-third/templates/ingress.yaml b/deployments/charts/openim-rpc-third/templates/ingress.yaml deleted file mode 100644 index 1c64ac545..000000000 --- a/deployments/charts/openim-rpc-third/templates/ingress.yaml +++ /dev/null @@ -1,75 +0,0 @@ -# Copyright © 2023 OpenIM. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -{{- if .Values.ingress.enabled -}} -{{- $fullName := include "openim-rpc-third.fullname" . -}} -{{- $svcPort := .Values.service.port -}} -{{- if and .Values.ingress.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }} - {{- if not (hasKey .Values.ingress.annotations "kubernetes.io/ingress.class") }} - {{- $_ := set .Values.ingress.annotations "kubernetes.io/ingress.class" .Values.ingress.className}} - {{- end }} -{{- end }} -{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}} -apiVersion: networking.k8s.io/v1 -{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}} -apiVersion: networking.k8s.io/v1beta1 -{{- else -}} -apiVersion: extensions/v1beta1 -{{- end }} -kind: Ingress -metadata: - name: {{ $fullName }} - labels: - {{- include "openim-rpc-third.labels" . | nindent 4 }} - {{- with .Values.ingress.annotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -spec: - {{- if and .Values.ingress.className (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }} - ingressClassName: {{ .Values.ingress.className }} - {{- end }} - {{- if .Values.ingress.tls }} - tls: - {{- range .Values.ingress.tls }} - - hosts: - {{- range .hosts }} - - {{ . | quote }} - {{- end }} - secretName: {{ .secretName }} - {{- end }} - {{- end }} - rules: - {{- range .Values.ingress.hosts }} - - host: {{ .host | quote }} - http: - paths: - {{- range .paths }} - - path: {{ .path }} - {{- if and .pathType (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }} - pathType: {{ .pathType }} - {{- end }} - backend: - {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }} - service: - name: {{ $fullName }} - port: - number: {{ $svcPort }} - {{- else }} - serviceName: {{ $fullName }} - servicePort: {{ $svcPort }} - {{- end }} - {{- end }} - {{- end }} -{{- end }} diff --git a/deployments/charts/openim-rpc-third/templates/service.yaml b/deployments/charts/openim-rpc-third/templates/service.yaml deleted file mode 100644 index af112794e..000000000 --- a/deployments/charts/openim-rpc-third/templates/service.yaml +++ /dev/null @@ -1,29 +0,0 @@ -# Copyright © 2023 OpenIM. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -apiVersion: v1 -kind: Service -metadata: - name: {{ include "openim-rpc-third.fullname" . }} - labels: - {{- include "openim-rpc-third.labels" . | nindent 4 }} -spec: - type: {{ .Values.service.type }} - ports: - - port: {{ .Values.service.port }} - targetPort: webhook - protocol: TCP - name: webhook - selector: - {{- include "openim-rpc-third.selectorLabels" . | nindent 4 }} diff --git a/deployments/charts/openim-rpc-third/templates/serviceaccount.yaml b/deployments/charts/openim-rpc-third/templates/serviceaccount.yaml deleted file mode 100644 index 27f2bf067..000000000 --- a/deployments/charts/openim-rpc-third/templates/serviceaccount.yaml +++ /dev/null @@ -1,26 +0,0 @@ -# Copyright © 2023 OpenIM. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -{{- if .Values.serviceAccount.create -}} -apiVersion: v1 -kind: ServiceAccount -metadata: - name: {{ include "openim-rpc-third.serviceAccountName" . }} - labels: - {{- include "openim-rpc-third.labels" . | nindent 4 }} - {{- with .Values.serviceAccount.annotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -{{- end }} diff --git a/deployments/charts/openim-rpc-third/values.yaml b/deployments/charts/openim-rpc-third/values.yaml deleted file mode 100644 index 71d9a3ede..000000000 --- a/deployments/charts/openim-rpc-third/values.yaml +++ /dev/null @@ -1,96 +0,0 @@ -# Copyright © 2023 OpenIM. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Default values for openim-rpc-third. -# This is a YAML-formatted file. -# Declare variables to be passed into your templates. - -replicaCount: 1 - -image: - repository: ghcr.io/openimsdk/openim-rpc-third - pullPolicy: IfNotPresent - # Overrides the image tag whose default is the chart appVersion. - tag: "" - -imagePullSecrets: [] -nameOverride: "" -fullnameOverride: "" - -serviceAccount: - # Specifies whether a service account should be created - create: false - # Annotations to add to the service account - annotations: {} - # The name of the service account to use. - # If not set and create is true, a name is generated using the fullname template - name: "" - -podAnnotations: {} - -podSecurityContext: {} - # fsGroup: 2000 - -securityContext: {} - # capabilities: - # drop: - # - ALL - # readOnlyRootFilesystem: true - # runAsNonRoot: true - # runAsUser: 1000 - -service: - type: ClusterIP - port: 80 - -ingress: - enabled: false - className: "" - annotations: {} - # kubernetes.io/ingress.class: nginx - # kubernetes.io/tls-acme: "true" - hosts: - - host: chart-example.local - paths: - - path: / - pathType: ImplementationSpecific - tls: [] - # - secretName: chart-example-tls - # hosts: - # - chart-example.local - -resources: {} - # We usually recommend not to specify default resources and to leave this as a conscious - # choice for the user. This also increases chances charts run on environments with little - # resources, such as Minikube. If you do want to specify resources, uncomment the following - # lines, adjust them as necessary, and remove the curly braces after 'resources:'. - # limits: - # cpu: 100m - # memory: 128Mi - # requests: - # cpu: 100m - # memory: 128Mi - -autoscaling: - enabled: false - minReplicas: 1 - maxReplicas: 100 - targetCPUUtilizationPercentage: 80 - # targetMemoryUtilizationPercentage: 80 - -nodeSelector: {} - -tolerations: [] - -affinity: {} diff --git a/deployments/charts/openim-rpc-user/.helmignore b/deployments/charts/openim-rpc-user/.helmignore deleted file mode 100644 index 0e8a0eb36..000000000 --- a/deployments/charts/openim-rpc-user/.helmignore +++ /dev/null @@ -1,23 +0,0 @@ -# Patterns to ignore when building packages. -# This supports shell glob matching, relative path matching, and -# negation (prefixed with !). Only one pattern per line. -.DS_Store -# Common VCS dirs -.git/ -.gitignore -.bzr/ -.bzrignore -.hg/ -.hgignore -.svn/ -# Common backup files -*.swp -*.bak -*.tmp -*.orig -*~ -# Various IDEs -.project -.idea/ -*.tmproj -.vscode/ diff --git a/deployments/charts/openim-rpc-user/Chart.yaml b/deployments/charts/openim-rpc-user/Chart.yaml deleted file mode 100644 index feb5a3195..000000000 --- a/deployments/charts/openim-rpc-user/Chart.yaml +++ /dev/null @@ -1,53 +0,0 @@ -# Copyright © 2023 OpenIM. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -apiVersion: v2 -name: openim-rpc-user -description: A Helm chart for Kubernetes - -# A chart can be either an 'application' or a 'library' chart. -# -# Application charts are a collection of templates that can be packaged into versioned archives -# to be deployed. -# -# Library charts provide useful utilities or functions for the chart developer. They're included as -# a dependency of application charts to inject those utilities and functions into the rendering -# pipeline. Library charts do not define any templates and therefore cannot be deployed. -type: application - -# This is the chart version. This version number should be incremented each time you make changes -# to the chart and its templates, including the app version. -# Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.0 - -# This is the version number of the application being deployed. This version number should be -# incremented each time you make changes to the application. Versions are not expected to -# follow Semantic Versioning. They should reflect the version the application is using. -# It is recommended to use it with quotes. -appVersion: "1.16.0" - -icon: https://raw.githubusercontent.com/openimsdk/open-im-server/main/assets/openim-logo-gradient.svg - -maintainers: - - name: "OpenIM" - url: "https://github.com/openimsdk" - -keywords: - - openim - - im - - chat - -sources: - - "https://github.com/openimsdk/open-im-server" - - "https://github.com/openimsdk/helm-charts" \ No newline at end of file diff --git a/deployments/charts/openim-rpc-user/templates/NOTES.txt b/deployments/charts/openim-rpc-user/templates/NOTES.txt deleted file mode 100644 index 9928bd162..000000000 --- a/deployments/charts/openim-rpc-user/templates/NOTES.txt +++ /dev/null @@ -1,22 +0,0 @@ -1. Get the application URL by running these commands: -{{- if .Values.ingress.enabled }} -{{- range $host := .Values.ingress.hosts }} - {{- range .paths }} - http{{ if $.Values.ingress.tls }}s{{ end }}://{{ $host.host }}{{ .path }} - {{- end }} -{{- end }} -{{- else if contains "NodePort" .Values.service.type }} - export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "openim-rpc-user.fullname" . }}) - export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}") - echo http://$NODE_IP:$NODE_PORT -{{- else if contains "LoadBalancer" .Values.service.type }} - NOTE: It may take a few minutes for the LoadBalancer IP to be available. - You can watch the status of by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "openim-rpc-user.fullname" . }}' - export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "openim-rpc-user.fullname" . }} --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}") - echo http://$SERVICE_IP:{{ .Values.service.port }} -{{- else if contains "ClusterIP" .Values.service.type }} - export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "openim-rpc-user.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") - export CONTAINER_PORT=$(kubectl get pod --namespace {{ .Release.Namespace }} $POD_NAME -o jsonpath="{.spec.containers[0].ports[0].containerPort}") - echo "Visit http://127.0.0.1:8080 to use your application" - kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:$CONTAINER_PORT -{{- end }} diff --git a/deployments/charts/openim-rpc-user/templates/_helpers.tpl b/deployments/charts/openim-rpc-user/templates/_helpers.tpl deleted file mode 100644 index 473f66556..000000000 --- a/deployments/charts/openim-rpc-user/templates/_helpers.tpl +++ /dev/null @@ -1,62 +0,0 @@ -{{/* -Expand the name of the chart. -*/}} -{{- define "openim-rpc-user.name" -}} -{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} -{{- end }} - -{{/* -Create a default fully qualified app name. -We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). -If release name contains chart name it will be used as a full name. -*/}} -{{- define "openim-rpc-user.fullname" -}} -{{- if .Values.fullnameOverride }} -{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} -{{- else }} -{{- $name := default .Chart.Name .Values.nameOverride }} -{{- if contains $name .Release.Name }} -{{- .Release.Name | trunc 63 | trimSuffix "-" }} -{{- else }} -{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} -{{- end }} -{{- end }} -{{- end }} - -{{/* -Create chart name and version as used by the chart label. -*/}} -{{- define "openim-rpc-user.chart" -}} -{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} -{{- end }} - -{{/* -Common labels -*/}} -{{- define "openim-rpc-user.labels" -}} -helm.sh/chart: {{ include "openim-rpc-user.chart" . }} -{{ include "openim-rpc-user.selectorLabels" . }} -{{- if .Chart.AppVersion }} -app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} -{{- end }} -app.kubernetes.io/managed-by: {{ .Release.Service }} -{{- end }} - -{{/* -Selector labels -*/}} -{{- define "openim-rpc-user.selectorLabels" -}} -app.kubernetes.io/name: {{ include "openim-rpc-user.name" . }} -app.kubernetes.io/instance: {{ .Release.Name }} -{{- end }} - -{{/* -Create the name of the service account to use -*/}} -{{- define "openim-rpc-user.serviceAccountName" -}} -{{- if .Values.serviceAccount.create }} -{{- default (include "openim-rpc-user.fullname" .) .Values.serviceAccount.name }} -{{- else }} -{{- default "default" .Values.serviceAccount.name }} -{{- end }} -{{- end }} diff --git a/deployments/charts/openim-rpc-user/templates/deployment.yaml b/deployments/charts/openim-rpc-user/templates/deployment.yaml deleted file mode 100644 index 26497d837..000000000 --- a/deployments/charts/openim-rpc-user/templates/deployment.yaml +++ /dev/null @@ -1,86 +0,0 @@ -# Copyright © 2023 OpenIM. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ include "openim-rpc-user.fullname" . }} - labels: - {{- include "openim-rpc-user.labels" . | nindent 4 }} -spec: - {{- if not .Values.autoscaling.enabled }} - replicas: {{ .Values.replicaCount }} - {{- end }} - selector: - matchLabels: - {{- include "openim-rpc-user.selectorLabels" . | nindent 6 }} - template: - metadata: - {{- with .Values.podAnnotations }} - annotations: - {{- toYaml . | nindent 8 }} - {{- end }} - labels: - {{- include "openim-rpc-user.selectorLabels" . | nindent 8 }} - spec: - {{- with .Values.imagePullSecrets }} - imagePullSecrets: - {{- toYaml . | nindent 8 }} - {{- end }} - serviceAccountName: {{ include "openim-rpc-user.serviceAccountName" . }} - securityContext: - {{- toYaml .Values.podSecurityContext | nindent 8 }} - containers: - - name: {{ .Chart.Name }} - securityContext: - {{- toYaml .Values.securityContext | nindent 12 }} - image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" - imagePullPolicy: {{ .Values.image.pullPolicy }} - ports: - - name: webhook - containerPort: 80 - protocol: TCP - #livenessProbe: - # httpGet: - # path: / - # port: webhook - #readinessProbe: - # httpGet: - # path: / - # port: webhook - resources: - {{- toYaml .Values.resources | nindent 12 }} - volumeMounts: - - mountPath: /openim/openim-server/config/config.yaml - name: config - subPath: config.yaml - - mountPath: /openim/openim-server/config/ - name: config - subPath: notification.yaml - volumes: - - name: config - configMap: - name: openim-cm - {{- with .Values.nodeSelector }} - nodeSelector: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.affinity }} - affinity: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.tolerations }} - tolerations: - {{- toYaml . | nindent 8 }} - {{- end }} diff --git a/deployments/charts/openim-rpc-user/templates/hpa.yaml b/deployments/charts/openim-rpc-user/templates/hpa.yaml deleted file mode 100644 index 012b15ae3..000000000 --- a/deployments/charts/openim-rpc-user/templates/hpa.yaml +++ /dev/null @@ -1,42 +0,0 @@ -# Copyright © 2023 OpenIM. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -{{- if .Values.autoscaling.enabled }} -apiVersion: autoscaling/v2beta1 -kind: HorizontalPodAutoscaler -metadata: - name: {{ include "openim-rpc-user.fullname" . }} - labels: - {{- include "openim-rpc-user.labels" . | nindent 4 }} -spec: - scaleTargetRef: - apiVersion: apps/v1 - kind: Deployment - name: {{ include "openim-rpc-user.fullname" . }} - minReplicas: {{ .Values.autoscaling.minReplicas }} - maxReplicas: {{ .Values.autoscaling.maxReplicas }} - metrics: - {{- if .Values.autoscaling.targetCPUUtilizationPercentage }} - - type: Resource - resource: - name: cpu - targetAverageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} - {{- end }} - {{- if .Values.autoscaling.targetMemoryUtilizationPercentage }} - - type: Resource - resource: - name: memory - targetAverageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} - {{- end }} -{{- end }} diff --git a/deployments/charts/openim-rpc-user/templates/ingress.yaml b/deployments/charts/openim-rpc-user/templates/ingress.yaml deleted file mode 100644 index 38aa6fdff..000000000 --- a/deployments/charts/openim-rpc-user/templates/ingress.yaml +++ /dev/null @@ -1,75 +0,0 @@ -# Copyright © 2023 OpenIM. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -{{- if .Values.ingress.enabled -}} -{{- $fullName := include "openim-rpc-user.fullname" . -}} -{{- $svcPort := .Values.service.port -}} -{{- if and .Values.ingress.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }} - {{- if not (hasKey .Values.ingress.annotations "kubernetes.io/ingress.class") }} - {{- $_ := set .Values.ingress.annotations "kubernetes.io/ingress.class" .Values.ingress.className}} - {{- end }} -{{- end }} -{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}} -apiVersion: networking.k8s.io/v1 -{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}} -apiVersion: networking.k8s.io/v1beta1 -{{- else -}} -apiVersion: extensions/v1beta1 -{{- end }} -kind: Ingress -metadata: - name: {{ $fullName }} - labels: - {{- include "openim-rpc-user.labels" . | nindent 4 }} - {{- with .Values.ingress.annotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -spec: - {{- if and .Values.ingress.className (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }} - ingressClassName: {{ .Values.ingress.className }} - {{- end }} - {{- if .Values.ingress.tls }} - tls: - {{- range .Values.ingress.tls }} - - hosts: - {{- range .hosts }} - - {{ . | quote }} - {{- end }} - secretName: {{ .secretName }} - {{- end }} - {{- end }} - rules: - {{- range .Values.ingress.hosts }} - - host: {{ .host | quote }} - http: - paths: - {{- range .paths }} - - path: {{ .path }} - {{- if and .pathType (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }} - pathType: {{ .pathType }} - {{- end }} - backend: - {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }} - service: - name: {{ $fullName }} - port: - number: {{ $svcPort }} - {{- else }} - serviceName: {{ $fullName }} - servicePort: {{ $svcPort }} - {{- end }} - {{- end }} - {{- end }} -{{- end }} diff --git a/deployments/charts/openim-rpc-user/templates/service.yaml b/deployments/charts/openim-rpc-user/templates/service.yaml deleted file mode 100644 index af8a53e19..000000000 --- a/deployments/charts/openim-rpc-user/templates/service.yaml +++ /dev/null @@ -1,29 +0,0 @@ -# Copyright © 2023 OpenIM. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -apiVersion: v1 -kind: Service -metadata: - name: {{ include "openim-rpc-user.fullname" . }} - labels: - {{- include "openim-rpc-user.labels" . | nindent 4 }} -spec: - type: {{ .Values.service.type }} - ports: - - port: {{ .Values.service.port }} - targetPort: webhook - protocol: TCP - name: webhook - selector: - {{- include "openim-rpc-user.selectorLabels" . | nindent 4 }} diff --git a/deployments/charts/openim-rpc-user/templates/serviceaccount.yaml b/deployments/charts/openim-rpc-user/templates/serviceaccount.yaml deleted file mode 100644 index ebe3e3d68..000000000 --- a/deployments/charts/openim-rpc-user/templates/serviceaccount.yaml +++ /dev/null @@ -1,26 +0,0 @@ -# Copyright © 2023 OpenIM. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -{{- if .Values.serviceAccount.create -}} -apiVersion: v1 -kind: ServiceAccount -metadata: - name: {{ include "openim-rpc-user.serviceAccountName" . }} - labels: - {{- include "openim-rpc-user.labels" . | nindent 4 }} - {{- with .Values.serviceAccount.annotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -{{- end }} diff --git a/deployments/charts/openim-rpc-user/values.yaml b/deployments/charts/openim-rpc-user/values.yaml deleted file mode 100644 index 4f1ad7ede..000000000 --- a/deployments/charts/openim-rpc-user/values.yaml +++ /dev/null @@ -1,96 +0,0 @@ -# Copyright © 2023 OpenIM. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Default values for openim-rpc-user. -# This is a YAML-formatted file. -# Declare variables to be passed into your templates. - -replicaCount: 1 - -image: - repository: ghcr.io/openimsdk/openim-rpc-user - pullPolicy: IfNotPresent - # Overrides the image tag whose default is the chart appVersion. - tag: "" - -imagePullSecrets: [] -nameOverride: "" -fullnameOverride: "" - -serviceAccount: - # Specifies whether a service account should be created - create: false - # Annotations to add to the service account - annotations: {} - # The name of the service account to use. - # If not set and create is true, a name is generated using the fullname template - name: "" - -podAnnotations: {} - -podSecurityContext: {} - # fsGroup: 2000 - -securityContext: {} - # capabilities: - # drop: - # - ALL - # readOnlyRootFilesystem: true - # runAsNonRoot: true - # runAsUser: 1000 - -service: - type: ClusterIP - port: 80 - -ingress: - enabled: false - className: "" - annotations: {} - # kubernetes.io/ingress.class: nginx - # kubernetes.io/tls-acme: "true" - hosts: - - host: chart-example.local - paths: - - path: / - pathType: ImplementationSpecific - tls: [] - # - secretName: chart-example-tls - # hosts: - # - chart-example.local - -resources: {} - # We usually recommend not to specify default resources and to leave this as a conscious - # choice for the user. This also increases chances charts run on environments with little - # resources, such as Minikube. If you do want to specify resources, uncomment the following - # lines, adjust them as necessary, and remove the curly braces after 'resources:'. - # limits: - # cpu: 100m - # memory: 128Mi - # requests: - # cpu: 100m - # memory: 128Mi - -autoscaling: - enabled: false - minReplicas: 1 - maxReplicas: 100 - targetCPUUtilizationPercentage: 80 - # targetMemoryUtilizationPercentage: 80 - -nodeSelector: {} - -tolerations: [] - -affinity: {} diff --git a/deployments/deploy/README.md b/deployments/deploy/README.md new file mode 100644 index 000000000..d6b083bc5 --- /dev/null +++ b/deployments/deploy/README.md @@ -0,0 +1,85 @@ +# Kubernetes Deployment + +## Resource Requests + +- CPU: 2 cores +- Memory: 4 GiB +- Disk usage: 20 GiB (on Node) + +## Origin Deploy + +1. Enter the target dir + `cd ./deployments/deploy/` + +2. Deploy configs and dependencies + Upate your `openim-config.yml` + +Apply all config and dependencies +`kubectl apply -f ./openim-config.yml` + +> Attation: If you use `default` namespace, you can excute `clusterRile.yml` to create a cluster role binding for default service account. +> +> Namespace is modify to `discovery.yml` in `openim-config.yml`, you can change `kubernetes.namespace` to your namespace. + +Excute `clusterRole.yml` +`kubectl apply -f ./clusterRole.yml` + +Run infrasturcture components. + +`kubectl apply -f minio-service.yml -f minio-statefulset.yml -f mongo-service.yml -f mongo-statefulset.yml -f redis-service.yml -f redis-statefulset.yml -f kafka-service.yml -f kafka-statefulset.yml` + +> Note: Ensure that infrastructure services like MinIO, Redis, and Kafka are running before deploying the main applications. + +3. run all deployments and services + +```bash +kubectl apply \ + -f openim-api-deployment.yml \ + -f openim-api-service.yml \ + -f openim-crontask-deployment.yml \ + -f openim-rpc-user-deployment.yml \ + -f openim-rpc-user-service.yml \ + -f openim-msggateway-deployment.yml \ + -f openim-msggateway-service.yml \ + -f openim-push-deployment.yml \ + -f openim-push-service.yml \ + -f openim-msgtransfer-service.yml \ + -f openim-msgtransfer-deployment.yml \ + -f openim-rpc-conversation-deployment.yml \ + -f openim-rpc-conversation-service.yml \ + -f openim-rpc-auth-deployment.yml \ + -f openim-rpc-auth-service.yml \ + -f openim-rpc-group-deployment.yml \ + -f openim-rpc-group-service.yml \ + -f openim-rpc-friend-deployment.yml \ + -f openim-rpc-friend-service.yml \ + -f openim-rpc-msg-deployment.yml \ + -f openim-rpc-msg-service.yml \ + -f openim-rpc-third-deployment.yml \ + -f openim-rpc-third-service.yml +``` + +4. Verification + After deploying the services, verify that everything is running smoothly: + +```bash +# Check the status of all pods +kubectl get pods + +# Check the status of services +kubectl get svc + +# Check the status of deployments +kubectl get deployments + +# View all resources +kubectl get all +``` + +5. clean all + +`kubectl delete -f ./` + +### Notes: + +- If you use a specific namespace for your deployment, be sure to append the -n flag to your kubectl commands. diff --git a/deployments/deploy/clusterRole.yml b/deployments/deploy/clusterRole.yml new file mode 100644 index 000000000..190c0b268 --- /dev/null +++ b/deployments/deploy/clusterRole.yml @@ -0,0 +1,24 @@ +# ClusterRole.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: service-reader +rules: + - apiGroups: [""] + resources: ["services", "endpoints"] + verbs: ["get", "list", "watch"] + +--- +# ClusterRoleBinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: default-service-reader-binding +subjects: + - kind: ServiceAccount + name: default + namespace: default +roleRef: + kind: ClusterRole + name: service-reader + apiGroup: rbac.authorization.k8s.io diff --git a/deployments/deploy/ingress.yml b/deployments/deploy/ingress.yml new file mode 100644 index 000000000..8a4fbaa02 --- /dev/null +++ b/deployments/deploy/ingress.yml @@ -0,0 +1,25 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: openim-ingress + annotations: + nginx.ingress.kubernetes.io/rewrite-target: / +spec: + ingressClassName: openim-nginx + rules: + - http: + paths: + - path: /openim-api + pathType: Prefix + backend: + service: + name: openim-api-service + port: + number: 10002 + - path: /openim-msggateway + pathType: Prefix + backend: + service: + name: openim-msggateway-service + port: + number: 10001 diff --git a/deployments/deploy/kafka-service.yml b/deployments/deploy/kafka-service.yml new file mode 100644 index 000000000..675600b98 --- /dev/null +++ b/deployments/deploy/kafka-service.yml @@ -0,0 +1,20 @@ +apiVersion: v1 +kind: Service +metadata: + name: kafka-service + labels: + app: kafka +spec: + ports: + - name: plaintext + port: 9092 + targetPort: 9092 + - name: controller + port: 9093 + targetPort: 9093 + - name: external + port: 19094 + targetPort: 9094 + selector: + app: kafka + type: ClusterIP diff --git a/deployments/deploy/kafka-statefulset.yml b/deployments/deploy/kafka-statefulset.yml new file mode 100644 index 000000000..0e3c78b69 --- /dev/null +++ b/deployments/deploy/kafka-statefulset.yml @@ -0,0 +1,71 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: kafka-statefulset + labels: + app: kafka +spec: + replicas: 2 + selector: + matchLabels: + app: kafka + serviceName: "kafka-service" + template: + metadata: + labels: + app: kafka + spec: + containers: + - name: kafka + image: bitnami/kafka:3.5.1 + imagePullPolicy: IfNotPresent + resources: + limits: + memory: "2Gi" + cpu: "1000m" + requests: + memory: "1Gi" + cpu: "500m" + ports: + - containerPort: 9092 # PLAINTEXT + - containerPort: 9093 # CONTROLLER + - containerPort: 9094 # EXTERNAL + env: + - name: TZ + value: "Asia/Shanghai" + - name: KAFKA_CFG_NODE_ID + value: "0" + - name: KAFKA_CFG_PROCESS_ROLES + value: "controller,broker" + - name: KAFKA_CFG_CONTROLLER_QUORUM_VOTERS + value: "0@kafka-service:9093" + - name: KAFKA_CFG_LISTENERS + value: "PLAINTEXT://:9092,CONTROLLER://:9093,EXTERNAL://:9094" + - name: KAFKA_CFG_ADVERTISED_LISTENERS + value: "PLAINTEXT://kafka-service:9092,EXTERNAL://kafka-service:19094" + - name: KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP + value: "CONTROLLER:PLAINTEXT,EXTERNAL:PLAINTEXT,PLAINTEXT:PLAINTEXT" + - name: KAFKA_CFG_CONTROLLER_LISTENER_NAMES + value: "CONTROLLER" + - name: KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE + value: "true" + volumeMounts: + - name: kafka-data + mountPath: /bitnami/kafka + + volumes: + - name: kafka-data + persistentVolumeClaim: + claimName: kafka-pvc + +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: kafka-pvc +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi diff --git a/deployments/deploy/minio-service.yml b/deployments/deploy/minio-service.yml new file mode 100644 index 000000000..1aeeb5f6c --- /dev/null +++ b/deployments/deploy/minio-service.yml @@ -0,0 +1,18 @@ +--- +apiVersion: v1 +kind: Service +metadata: + name: minio-service +spec: + selector: + app: minio + ports: + - name: minio + protocol: TCP + port: 10005 # External port for accessing MinIO service + targetPort: 9000 # Container port for MinIO service + - name: minio-console + protocol: TCP + port: 19090 # External port for accessing MinIO console + targetPort: 9090 # Container port for MinIO console + type: NodePort diff --git a/deployments/deploy/minio-statefulset.yml b/deployments/deploy/minio-statefulset.yml new file mode 100644 index 000000000..c8806ff12 --- /dev/null +++ b/deployments/deploy/minio-statefulset.yml @@ -0,0 +1,87 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: minio + labels: + app: minio +spec: + replicas: 2 + selector: + matchLabels: + app: minio + template: + metadata: + labels: + app: minio + spec: + containers: + - name: minio + image: minio/minio:RELEASE.2024-01-11T07-46-16Z + ports: + - containerPort: 9000 # MinIO service port + - containerPort: 9090 # MinIO console port + volumeMounts: + - name: minio-data + mountPath: /data + - name: minio-config + mountPath: /root/.minio + env: + - name: TZ + value: "Asia/Shanghai" + - name: MINIO_ROOT_USER + valueFrom: + secretKeyRef: + name: minio-secret + key: minio-root-user + - name: MINIO_ROOT_PASSWORD + valueFrom: + secretKeyRef: + name: minio-secret + key: minio-root-password + command: + - "/bin/sh" + - "-c" + - | + mkdir -p /data && \ + minio server /data --console-address ":9090" + volumes: + - name: minio-data + persistentVolumeClaim: + claimName: minio-pvc + - name: minio-config + persistentVolumeClaim: + claimName: minio-config-pvc + +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: minio-pvc +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi + +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: minio-config-pvc +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 2Gi + +--- +apiVersion: v1 +kind: Secret +metadata: + name: minio-secret +type: Opaque +data: + minio-root-user: cm9vdA== # Base64 encoded "root" + minio-root-password: b3BlbklNMTIz # Base64 encoded "openIM123" diff --git a/deployments/deploy/mongo-service.yml b/deployments/deploy/mongo-service.yml new file mode 100644 index 000000000..c3b3a1027 --- /dev/null +++ b/deployments/deploy/mongo-service.yml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: Service +metadata: + name: mongo-service +spec: + selector: + app: mongo + ports: + - name: mongodb-port + protocol: TCP + port: 37017 + targetPort: 27017 + type: NodePort diff --git a/deployments/deploy/mongo-statefulset.yml b/deployments/deploy/mongo-statefulset.yml new file mode 100644 index 000000000..e8510fdf7 --- /dev/null +++ b/deployments/deploy/mongo-statefulset.yml @@ -0,0 +1,95 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: mongo-statefulset +spec: + serviceName: "mongo" + replicas: 2 + selector: + matchLabels: + app: mongo + template: + metadata: + labels: + app: mongo + spec: + containers: + - name: mongo + image: mongo:7.0 + command: ["/bin/bash", "-c"] + args: + - > + docker-entrypoint.sh mongod --wiredTigerCacheSizeGB ${wiredTigerCacheSizeGB} --auth & + until mongosh -u ${MONGO_INITDB_ROOT_USERNAME} -p ${MONGO_INITDB_ROOT_PASSWORD} --authenticationDatabase admin --eval "db.runCommand({ ping: 1 })" &>/dev/null; do + echo "Waiting for MongoDB to start..."; + sleep 1; + done && + mongosh -u ${MONGO_INITDB_ROOT_USERNAME} -p ${MONGO_INITDB_ROOT_PASSWORD} --authenticationDatabase admin --eval " + db = db.getSiblingDB(\"${MONGO_INITDB_DATABASE}\"); + if (!db.getUser(\"${MONGO_OPENIM_USERNAME}\")) { + db.createUser({ + user: \"${MONGO_OPENIM_USERNAME}\", + pwd: \"${MONGO_OPENIM_PASSWORD}\", + roles: [{role: \"readWrite\", db: \"${MONGO_INITDB_DATABASE}\"}] + }); + print(\"User created successfully: \"); + print(\"Username: ${MONGO_OPENIM_USERNAME}\"); + print(\"Password: ${MONGO_OPENIM_PASSWORD}\"); + print(\"Database: ${MONGO_INITDB_DATABASE}\"); + } else { + print(\"User already exists in database: ${MONGO_INITDB_DATABASE}, Username: ${MONGO_OPENIM_USERNAME}\"); + } + " && + tail -f /dev/null + ports: + - containerPort: 27017 + env: + - name: MONGO_INITDB_ROOT_USERNAME + valueFrom: + secretKeyRef: + name: mongo-secret + key: mongo_initdb_root_username + - name: MONGO_INITDB_ROOT_PASSWORD + valueFrom: + secretKeyRef: + name: mongo-secret + key: mongo_initdb_root_password + - name: MONGO_INITDB_DATABASE + valueFrom: + secretKeyRef: + name: mongo-secret + key: mongo_initdb_database + - name: MONGO_OPENIM_USERNAME + valueFrom: + secretKeyRef: + name: mongo-secret + key: mongo_openim_username + - name: MONGO_OPENIM_PASSWORD + valueFrom: + secretKeyRef: + name: mongo-secret + key: mongo_openim_password + - name: TZ + value: "Asia/Shanghai" + - name: wiredTigerCacheSizeGB + value: "1" + volumeMounts: + - name: mongo-storage + mountPath: /data/db + + volumes: + - name: mongo-storage + persistentVolumeClaim: + claimName: mongo-pvc + +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: mongo-pvc +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 5Gi diff --git a/deployments/deploy/openim-api-deployment.yml b/deployments/deploy/openim-api-deployment.yml new file mode 100644 index 000000000..cb1a68075 --- /dev/null +++ b/deployments/deploy/openim-api-deployment.yml @@ -0,0 +1,49 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: openim-api +spec: + replicas: 2 + selector: + matchLabels: + app: openim-api + template: + metadata: + labels: + app: openim-api + spec: + containers: + - name: openim-api-container + image: openim/openim-api:v3.8.3 + env: + - name: CONFIG_PATH + value: "/config" + - name: IMENV_REDIS_PASSWORD + valueFrom: + secretKeyRef: + name: redis-secret + key: redis-password + + - name: IMENV_MONGODB_PASSWORD + valueFrom: + secretKeyRef: + name: mongo-secret + key: mongo_openim_password + + - name: IMENV_MONGODB_USERNAME + valueFrom: + secretKeyRef: + name: mongo-secret + key: mongo_openim_username + + volumeMounts: + - name: openim-config + mountPath: "/config" + readOnly: true + ports: + - containerPort: 10002 + - containerPort: 12002 + volumes: + - name: openim-config + configMap: + name: openim-config \ No newline at end of file diff --git a/deployments/deploy/openim-api-service.yml b/deployments/deploy/openim-api-service.yml new file mode 100644 index 000000000..a75bcd34e --- /dev/null +++ b/deployments/deploy/openim-api-service.yml @@ -0,0 +1,17 @@ +apiVersion: v1 +kind: Service +metadata: + name: openim-api-service +spec: + selector: + app: openim-api + ports: + - name: http-10002 + protocol: TCP + port: 10002 + targetPort: 10002 + - name: prometheus-12002 + protocol: TCP + port: 12002 + targetPort: 12002 + type: NodePort diff --git a/deployments/deploy/openim-config.yml b/deployments/deploy/openim-config.yml new file mode 100644 index 000000000..d0651bdea --- /dev/null +++ b/deployments/deploy/openim-config.yml @@ -0,0 +1,1070 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: openim-config +data: + discovery.yml: | + enable: "kubernetes" + kubernetes: + namespace: default + etcd: + rootDirectory: openim + address: [ localhost:12379 ] + username: '' + password: '' + + rpcService: + user: user-rpc-service + friend: friend-rpc-service + msg: msg-rpc-service + push: push-rpc-service + messageGateway: messagegateway-rpc-service + group: group-rpc-service + auth: auth-rpc-service + conversation: conversation-rpc-service + third: third-rpc-service + + log.yml: | + # Log storage path, default is acceptable, change to a full path if modification is needed + # storageLocation: ../../../../logs/ + storageLocation: ./logs/ + # Log rotation period (in hours), default is acceptable + rotationTime: 24 + # Number of log files to retain, default is acceptable + remainRotationCount: 2 + # Log level settings: 3 for production environment; 6 for more verbose logging in debugging environments + remainLogLevel: 6 + # Whether to output to standard output, default is acceptable + isStdout: true + # Whether to log in JSON format, default is acceptable + isJson: false + # output simplify log when KeyAndValues's value len is bigger than 50 in rpc method log + isSimplify: true + + mongodb.yml: | + # URI for database connection, leave empty if using address and credential settings directly + uri: '' + # List of MongoDB server addresses + address: [ mongo-service:37017 ] + # Name of the database + database: openim_v3 + # Username for database authentication + username: openIM + # Password for database authentication + password: openIM123 + # Authentication source for database authentication, if use root user, set it to admin + authSource: openim_v3 + # Maximum number of connections in the connection pool + maxPoolSize: 100 + # Maximum number of retry attempts for a failed database connection + maxRetry: 10 + + local-cache.yml: | + user: + topic: DELETE_CACHE_USER + slotNum: 100 + slotSize: 2000 + successExpire: 300 + failedExpire: 5 + group: + topic: DELETE_CACHE_GROUP + slotNum: 100 + slotSize: 2000 + successExpire: 300 + failedExpire: 5 + friend: + topic: DELETE_CACHE_FRIEND + slotNum: 100 + slotSize: 2000 + successExpire: 300 + failedExpire: 5 + conversation: + topic: DELETE_CACHE_CONVERSATION + slotNum: 100 + slotSize: 2000 + successExpire: 300 + failedExpire: 5 + + openim-api.yml: | + api: + # Listening IP; 0.0.0.0 means both internal and external IPs are listened to, default is recommended + listenIP: 0.0.0.0 + # Listening ports; if multiple are configured, multiple instances will be launched, must be consistent with the number of prometheus.ports + ports: [ 10002 ] + # API compression level; 0: default compression, 1: best compression, 2: best speed, -1: no compression + compressionLevel: 0 + + prometheus: + # Whether to enable prometheus + enable: true + # Prometheus listening ports, must match the number of api.ports + ports: [ 12002 ] + # This address can be accessed via a browser + grafanaURL: http://127.0.0.1:13000/ + + openim-rpc-user.yml: | + rpc: + # API or other RPCs can access this RPC through this IP; if left blank, the internal network IP is obtained by default + registerIP: + # Listening IP; 0.0.0.0 means both internal and external IPs are listened to, if blank, the internal network IP is automatically obtained by default + listenIP: 0.0.0.0 + # autoSetPorts indicates whether to automatically set the ports + # if you use in kubernetes, set it to false + autoSetPorts: false + # List of ports that the RPC service listens on; configuring multiple ports will launch multiple instances. These must match the number of configured prometheus ports + # It will only take effect when autoSetPorts is set to false. + ports: [ 10320 ] + prometheus: + # Whether to enable prometheus + enable: true + # Prometheus listening ports, must be consistent with the number of rpc.ports + ports: [ 12320 ] + + openim-crontask.yml: | + cronExecuteTime: 0 2 * * * + retainChatRecords: 365 + fileExpireTime: 180 + deleteObjectType: ["msg-picture","msg-file", "msg-voice","msg-video","msg-video-snapshot","sdklog"] + + openim-msggateway.yml: | + rpc: + # The IP address where this RPC service registers itself; if left blank, it defaults to the internal network IP + registerIP: + # autoSetPorts indicates whether to automatically set the ports + # if you use in kubernetes, set it to false + autoSetPorts: false + # List of ports that the RPC service listens on; configuring multiple ports will launch multiple instances. These must match the number of configured prometheus ports + # It will only take effect when autoSetPorts is set to false. + ports: [ 10140 ] + + prometheus: + # Enable or disable Prometheus monitoring + enable: true + # List of ports that Prometheus listens on; these must match the number of rpc.ports to ensure correct monitoring setup + ports: [ 12140 ] + + # IP address that the RPC/WebSocket service listens on; setting to 0.0.0.0 listens on both internal and external IPs. If left blank, it automatically uses the internal network IP + listenIP: 0.0.0.0 + + longConnSvr: + # WebSocket listening ports, must match the number of rpc.ports + ports: [ 10001 ] + # Maximum number of WebSocket connections + websocketMaxConnNum: 100000 + # Maximum length of the entire WebSocket message packet + websocketMaxMsgLen: 4096 + # WebSocket connection handshake timeout in seconds + websocketTimeout: 10 + + openim-msgtransfer.yml: | + prometheus: + # Enable or disable Prometheus monitoring + enable: true + # List of ports that Prometheus listens on; each port corresponds to an instance of monitoring. Ensure these are managed accordingly + # Because four instances have been launched, four ports need to be specified + ports: [ 12020 ] + + openim-push.yml: | + rpc: + # The IP address where this RPC service registers itself; if left blank, it defaults to the internal network IP + registerIP: + # IP address that the RPC service listens on; setting to 0.0.0.0 listens on both internal and external IPs. If left blank, it automatically uses the internal network IP + listenIP: 0.0.0.0 + # autoSetPorts indicates whether to automatically set the ports + # if you use in kubernetes, set it to false + autoSetPorts: false + # List of ports that the RPC service listens on; configuring multiple ports will launch multiple instances. These must match the number of configured prometheus ports + # It will only take effect when autoSetPorts is set to false. + ports: [ 10170 ] + + + prometheus: + # Enable or disable Prometheus monitoring + enable: true + # List of ports that Prometheus listens on; these must match the number of rpc.ports to ensure correct monitoring setup + ports: [ 12170 ] + + maxConcurrentWorkers: 3 + #Use geTui for offline push notifications, or choose fcm or jpns; corresponding configuration settings must be specified. + enable: + geTui: + pushUrl: https://restapi.getui.com/v2/$appId + masterSecret: + appKey: + intent: + channelID: + channelName: + fcm: + # Prioritize using file paths. If the file path is empty, use URL + filePath: # File path is concatenated with the parameters passed in through - c(`mage` default pass in `config/`) and filePath. + authURL: # Must start with https or http. + jpush: + appKey: + masterSecret: + pushURL: + pushIntent: + + # iOS system push sound and badge count + iosPush: + pushSound: xxx + badgeCount: true + production: false + + fullUserCache: true + + openim-rpc-auth.yml: | + rpc: + # The IP address where this RPC service registers itself; if left blank, it defaults to the internal network IP + registerIP: + # IP address that the RPC service listens on; setting to 0.0.0.0 listens on both internal and external IPs. If left blank, it automatically uses the internal network IP + listenIP: 0.0.0.0 + # autoSetPorts indicates whether to automatically set the ports + # if you use in kubernetes, set it to false + autoSetPorts: false + # List of ports that the RPC service listens on; configuring multiple ports will launch multiple instances. These must match the number of configured prometheus ports + # It will only take effect when autoSetPorts is set to false. + ports: [ 10200 ] + + prometheus: + # Enable or disable Prometheus monitoring + enable: true + # List of ports that Prometheus listens on; these must match the number of rpc.ports to ensure correct monitoring setup + ports: [12200] + + tokenPolicy: + # Token validity period, in days + expire: 90 + + openim-rpc-conversation.yml: | + rpc: + # The IP address where this RPC service registers itself; if left blank, it defaults to the internal network IP + registerIP: + # IP address that the RPC service listens on; setting to 0.0.0.0 listens on both internal and external IPs. If left blank, it automatically uses the internal network IP + listenIP: 0.0.0.0 + # autoSetPorts indicates whether to automatically set the ports + # if you use in kubernetes, set it to false + autoSetPorts: false + # List of ports that the RPC service listens on; configuring multiple ports will launch multiple instances. These must match the number of configured prometheus ports + # It will only take effect when autoSetPorts is set to false. + ports: [ 10220 ] + + prometheus: + # Enable or disable Prometheus monitoring + enable: true + # List of ports that Prometheus listens on; these must match the number of rpc.ports to ensure correct monitoring setup + ports: [ 12200 ] + + tokenPolicy: + # Token validity period, in days + expire: 90 + + openim-rpc-friend.yml: | + rpc: + # The IP address where this RPC service registers itself; if left blank, it defaults to the internal network IP + registerIP: + # IP address that the RPC service listens on; setting to 0.0.0.0 listens on both internal and external IPs. If left blank, it automatically uses the internal network IP + listenIP: 0.0.0.0 + # autoSetPorts indicates whether to automatically set the ports + # if you use in kubernetes, set it to false + autoSetPorts: false + # List of ports that the RPC service listens on; configuring multiple ports will launch multiple instances. These must match the number of configured prometheus ports + # It will only take effect when autoSetPorts is set to false. + ports: [ 10240 ] + + prometheus: + # Enable or disable Prometheus monitoring + enable: true + # List of ports that Prometheus listens on; these must match the number of rpc.ports to ensure correct monitoring setup + ports: [ 12240 ] + + openim-rpc-group.yml: | + rpc: + # The IP address where this RPC service registers itself; if left blank, it defaults to the internal network IP + registerIP: + # IP address that the RPC service listens on; setting to 0.0.0.0 listens on both internal and external IPs. If left blank, it automatically uses the internal network IP + listenIP: 0.0.0.0 + # autoSetPorts indicates whether to automatically set the ports + # if you use in kubernetes, set it to false + autoSetPorts: false + # List of ports that the RPC service listens on; configuring multiple ports will launch multiple instances. These must match the number of configured prometheus ports + # It will only take effect when autoSetPorts is set to false. + ports: [ 10260 ] + + prometheus: + # Enable or disable Prometheus monitoring + enable: true + # List of ports that Prometheus listens on; these must match the number of rpc.ports to ensure correct monitoring setup + ports: [ 12260 ] + + enableHistoryForNewMembers: true + + openim-rpc-msg.yml: | + rpc: + # The IP address where this RPC service registers itself; if left blank, it defaults to the internal network IP + registerIP: + # IP address that the RPC service listens on; setting to 0.0.0.0 listens on both internal and external IPs. If left blank, it automatically uses the internal network IP + listenIP: 0.0.0.0 + # autoSetPorts indicates whether to automatically set the ports + # if you use in kubernetes, set it to false + autoSetPorts: false + # List of ports that the RPC service listens on; configuring multiple ports will launch multiple instances. These must match the number of configured prometheus ports + ports: [ 10280 ] + + prometheus: + # Enable or disable Prometheus monitoring + enable: true + # List of ports that Prometheus listens on; these must match the number of rpc.ports to ensure correct monitoring setup + ports: [ 12280 ] + + + # Does sending messages require friend verification + friendVerify: false + + openim-rpc-third.yml: | + rpc: + # The IP address where this RPC service registers itself; if left blank, it defaults to the internal network IP + registerIP: + # IP address that the RPC service listens on; setting to 0.0.0.0 listens on both internal and external IPs. If left blank, it automatically uses the internal network IP + listenIP: 0.0.0.0 + # autoSetPorts indicates whether to automatically set the ports + # if you use in kubernetes, set it to false + autoSetPorts: false + # List of ports that the RPC service listens on; configuring multiple ports will launch multiple instances. These must match the number of configured prometheus ports + # It will only take effect when autoSetPorts is set to false. + ports: [ 10300 ] + + prometheus: + # Enable or disable Prometheus monitoring + enable: true + # List of ports that Prometheus listens on; these must match the number of rpc.ports to ensure correct monitoring setup + ports: [ 12300 ] + + + object: + # Use MinIO as object storage, or set to "cos", "oss", "kodo", "aws", while also configuring the corresponding settings + enable: minio + cos: + bucketURL: https://temp-1252357374.cos.ap-chengdu.myqcloud.com + secretID: + secretKey: + sessionToken: + publicRead: false + oss: + endpoint: https://oss-cn-chengdu.aliyuncs.com + bucket: demo-9999999 + bucketURL: https://demo-9999999.oss-cn-chengdu.aliyuncs.com + accessKeyID: + accessKeySecret: + sessionToken: + publicRead: false + kodo: + endpoint: http://s3.cn-south-1.qiniucs.com + bucket: kodo-bucket-test + bucketURL: http://kodo-bucket-test-oetobfb.qiniudns.com + accessKeyID: + accessKeySecret: + sessionToken: + publicRead: false + aws: + region: ap-southeast-2 + bucket: testdemo832234 + accessKeyID: + secretAccessKey: + sessionToken: + publicRead: false + + share.yml: | + secret: openIM123 + + imAdminUserID: ["imAdmin"] + + # 1: For Android, iOS, Windows, Mac, and web platforms, only one instance can be online at a time + multiLogin: + policy: 1 + maxNumOneEnd: 30 + + kafka.yml: | + # Username for authentication + username: '' + # Password for authentication + password: '' + # Producer acknowledgment settings + producerAck: + # Compression type to use (e.g., none, gzip, snappy) + compressType: none + # List of Kafka broker addresses + address: [ "kafka-service:19094" ] + # Kafka topic for Redis integration + toRedisTopic: toRedis + # Kafka topic for MongoDB integration + toMongoTopic: toMongo + # Kafka topic for push notifications + toPushTopic: toPush + # Kafka topic for offline push notifications + toOfflinePushTopic: toOfflinePush + # Consumer group ID for Redis topic + toRedisGroupID: redis + # Consumer group ID for MongoDB topic + toMongoGroupID: mongo + # Consumer group ID for push notifications topic + toPushGroupID: push + # Consumer group ID for offline push notifications topic + toOfflinePushGroupID: offlinePush + # TLS (Transport Layer Security) configuration + tls: + # Enable or disable TLS + enableTLS: false + # CA certificate file path + caCrt: + # Client certificate file path + clientCrt: + # Client key file path + clientKey: + # Client key password + clientKeyPwd: + # Whether to skip TLS verification (not recommended for production) + insecureSkipVerify: false + + redis.yml: | + address: [ "redis-service:16379" ] + username: + password: # openIM123 + clusterMode: false + db: 0 + maxRetry: 10 + poolSize: 100 + + minio.yml: | + # Name of the bucket in MinIO + bucket: openim + # Access key ID for MinIO authentication + accessKeyID: root + # Secret access key for MinIO authentication + secretAccessKey: # openIM123 + # Session token for MinIO authentication (optional) + sessionToken: + # Internal address of the MinIO server + internalAddress: minio-service:10005 + # External address of the MinIO server, accessible from outside. Supports both HTTP and HTTPS using a domain name + externalAddress: http://minio-service:10005 + # Flag to enable or disable public read access to the bucket + publicRead: "false" + + notification.yml: | + groupCreated: + isSendMsg: true + # Reliability level of the message sending. + # Set to 1 to send only when online, 2 for guaranteed delivery. + reliabilityLevel: 1 + # This setting is effective only when 'isSendMsg' is true. + # It controls whether to count unread messages. + unreadCount: false + # Configuration for offline push notifications. + offlinePush: + # Enables or disables offline push notifications. + enable: false + # Title for the notification when a group is created. + title: create group title + # Description for the notification. + desc: create group desc + # Additional information for the notification. + ext: create group ext + + groupInfoSet: + isSendMsg: false + reliabilityLevel: 1 + unreadCount: false + offlinePush: + enable: false + title: groupInfoSet title + desc: groupInfoSet desc + ext: groupInfoSet ext + + joinGroupApplication: + isSendMsg: false + reliabilityLevel: 1 + unreadCount: false + offlinePush: + enable: false + title: joinGroupApplication title + desc: joinGroupApplication desc + ext: joinGroupApplication ext + + memberQuit: + isSendMsg: true + reliabilityLevel: 1 + unreadCount: false + offlinePush: + enable: false + title: memberQuit title + desc: memberQuit desc + ext: memberQuit ext + + groupApplicationAccepted: + isSendMsg: false + reliabilityLevel: 1 + unreadCount: false + offlinePush: + enable: false + title: groupApplicationAccepted title + desc: groupApplicationAccepted desc + ext: groupApplicationAccepted ext + + groupApplicationRejected: + isSendMsg: false + reliabilityLevel: 1 + unreadCount: false + offlinePush: + enable: false + title: groupApplicationRejected title + desc: groupApplicationRejected desc + ext: groupApplicationRejected ext + + groupOwnerTransferred: + isSendMsg: true + reliabilityLevel: 1 + unreadCount: false + offlinePush: + enable: false + title: groupOwnerTransferred title + desc: groupOwnerTransferred desc + ext: groupOwnerTransferred ext + + memberKicked: + isSendMsg: true + reliabilityLevel: 1 + unreadCount: false + offlinePush: + enable: false + title: memberKicked title + desc: memberKicked desc + ext: memberKicked ext + + memberInvited: + isSendMsg: true + reliabilityLevel: 1 + unreadCount: false + offlinePush: + enable: false + title: memberInvited title + desc: memberInvited desc + ext: memberInvited ext + + memberEnter: + isSendMsg: true + reliabilityLevel: 1 + unreadCount: false + offlinePush: + enable: false + title: memberEnter title + desc: memberEnter desc + ext: memberEnter ext + + groupDismissed: + isSendMsg: true + reliabilityLevel: 1 + unreadCount: false + offlinePush: + enable: false + title: groupDismissed title + desc: groupDismissed desc + ext: groupDismissed ext + + groupMuted: + isSendMsg: true + reliabilityLevel: 1 + unreadCount: false + offlinePush: + enable: false + title: groupMuted title + desc: groupMuted desc + ext: groupMuted ext + + groupCancelMuted: + isSendMsg: true + reliabilityLevel: 1 + unreadCount: false + offlinePush: + enable: false + title: groupCancelMuted title + desc: groupCancelMuted desc + ext: groupCancelMuted ext + defaultTips: + tips: group Cancel Muted + + groupMemberMuted: + isSendMsg: true + reliabilityLevel: 1 + unreadCount: false + offlinePush: + enable: false + title: groupMemberMuted title + desc: groupMemberMuted desc + ext: groupMemberMuted ext + + groupMemberCancelMuted: + isSendMsg: true + reliabilityLevel: 1 + unreadCount: false + offlinePush: + enable: false + title: groupMemberCancelMuted title + desc: groupMemberCancelMuted desc + ext: groupMemberCancelMuted ext + + groupMemberInfoSet: + isSendMsg: false + reliabilityLevel: 1 + unreadCount: false + offlinePush: + enable: false + title: groupMemberInfoSet title + desc: groupMemberInfoSet desc + ext: groupMemberInfoSet ext + + groupInfoSetAnnouncement: + isSendMsg: true + reliabilityLevel: 1 + unreadCount: false + offlinePush: + enable: false + title: groupInfoSetAnnouncement title + desc: groupInfoSetAnnouncement desc + ext: groupInfoSetAnnouncement ext + + groupInfoSetName: + isSendMsg: true + reliabilityLevel: 1 + unreadCount: false + offlinePush: + enable: false + title: groupInfoSetName title + desc: groupInfoSetName desc + ext: groupInfoSetName ext + + #############################friend################################# + friendApplicationAdded: + isSendMsg: false + reliabilityLevel: 1 + unreadCount: false + offlinePush: + enable: false + title: Somebody applies to add you as a friend + desc: Somebody applies to add you as a friend + ext: Somebody applies to add you as a friend + + friendApplicationApproved: + isSendMsg: true + reliabilityLevel: 1 + unreadCount: false + offlinePush: + enable: true + title: Someone applies to add your friend application + desc: Someone applies to add your friend application + ext: Someone applies to add your friend application + + friendApplicationRejected: + isSendMsg: false + reliabilityLevel: 1 + unreadCount: false + offlinePush: + enable: true + title: Someone rejected your friend application + desc: Someone rejected your friend application + ext: Someone rejected your friend application + + friendAdded: + isSendMsg: false + reliabilityLevel: 1 + unreadCount: false + offlinePush: + enable: true + title: We have become friends + desc: We have become friends + ext: We have become friends + + friendDeleted: + isSendMsg: false + reliabilityLevel: 1 + unreadCount: false + offlinePush: + enable: true + title: deleted a friend + desc: deleted a friend + ext: deleted a friend + + friendRemarkSet: + isSendMsg: false + reliabilityLevel: 1 + unreadCount: false + offlinePush: + enable: true + title: Your friend's profile has been changed + desc: Your friend's profile has been changed + ext: Your friend's profile has been changed + + blackAdded: + isSendMsg: false + reliabilityLevel: 1 + unreadCount: false + offlinePush: + enable: true + title: blocked a user + desc: blocked a user + ext: blocked a user + + blackDeleted: + isSendMsg: false + reliabilityLevel: 1 + unreadCount: false + offlinePush: + enable: true + title: Remove a blocked user + desc: Remove a blocked user + ext: Remove a blocked user + + friendInfoUpdated: + isSendMsg: false + reliabilityLevel: 1 + unreadCount: false + offlinePush: + enable: true + title: friend info updated + desc: friend info updated + ext: friend info updated + + #####################user######################### + userInfoUpdated: + isSendMsg: false + reliabilityLevel: 1 + unreadCount: false + offlinePush: + enable: true + title: userInfo updated + desc: userInfo updated + ext: userInfo updated + + userStatusChanged: + isSendMsg: false + reliabilityLevel: 1 + unreadCount: false + offlinePush: + enable: false + title: user status changed + desc: user status changed + ext: user status changed + + #####################conversation######################### + conversationChanged: + isSendMsg: false + reliabilityLevel: 1 + unreadCount: false + offlinePush: + enable: true + title: conversation changed + desc: conversation changed + ext: conversation changed + + conversationSetPrivate: + isSendMsg: true + reliabilityLevel: 1 + unreadCount: false + offlinePush: + enable: true + title: burn after reading + desc: burn after reading + ext: burn after reading + + webhooks.yml: | + url: http://127.0.0.1:10006/callbackExample + beforeSendSingleMsg: + enable: false + timeout: 5 + failedContinue: true + # Only the contentType in allowedTypes will send the callback. + # Supports two formats: a single type or a range. The range is defined by the lower and upper bounds connected with a hyphen ("-"). + # e.g. allowedTypes: [1, 100, 200-500, 600-700] means that only contentType within the range + # {1, 100} ∪ [200, 500] ∪ [600, 700] will be allowed through the filter. + # If not set, all contentType messages will through this filter. + allowedTypes: [] + # Only the contentType not in deniedTypes will send the callback. + # Supports two formats, same as allowedTypes. + # If not set, all contentType messages will through this filter. + deniedTypes: [] + beforeUpdateUserInfoEx: + enable: false + timeout: 5 + failedContinue: true + afterUpdateUserInfoEx: + enable: false + timeout: 5 + afterSendSingleMsg: + enable: false + timeout: 5 + # Only the senID/recvID specified in attentionIds will send the callback + # if not set, all user messages will be callback + attentionIds: [] + # See beforeSendSingleMsg comment. + allowedTypes: [] + deniedTypes: [] + beforeSendGroupMsg: + enable: false + timeout: 5 + failedContinue: true + # See beforeSendSingleMsg comment. + allowedTypes: [] + deniedTypes: [] + beforeMsgModify: + enable: false + timeout: 5 + failedContinue: true + # See beforeSendSingleMsg comment. + allowedTypes: [] + deniedTypes: [] + afterSendGroupMsg: + enable: false + timeout: 5 + # See beforeSendSingleMsg comment. + allowedTypes: [] + deniedTypes: [] + afterUserOnline: + enable: false + timeout: 5 + afterUserOffline: + enable: false + timeout: 5 + afterUserKickOff: + enable: false + timeout: 5 + beforeOfflinePush: + enable: false + timeout: 5 + failedContinue: true + beforeOnlinePush: + enable: false + timeout: 5 + failedContinue: true + beforeGroupOnlinePush: + enable: false + timeout: 5 + failedContinue: true + beforeAddFriend: + enable: false + timeout: 5 + failedContinue: true + beforeUpdateUserInfo: + enable: false + timeout: 5 + failedContinue: true + afterUpdateUserInfo: + enable: false + timeout: 5 + beforeCreateGroup: + enable: false + timeout: 5 + failedContinue: true + afterCreateGroup: + enable: false + timeout: 5 + beforeMemberJoinGroup: + enable: false + timeout: 5 + failedContinue: true + beforeSetGroupMemberInfo: + enable: false + timeout: 5 + failedContinue: true + afterSetGroupMemberInfo: + enable: false + timeout: 5 + afterQuitGroup: + enable: false + timeout: 5 + afterKickGroupMember: + enable: false + timeout: 5 + afterDismissGroup: + enable: false + timeout: 5 + beforeApplyJoinGroup: + enable: false + timeout: 5 + failedContinue: true + afterGroupMsgRead: + enable: false + timeout: 5 + afterSingleMsgRead: + enable: false + timeout: 5 + beforeUserRegister: + enable: false + timeout: 5 + failedContinue: true + afterUserRegister: + enable: false + timeout: 5 + afterTransferGroupOwner: + enable: false + timeout: 5 + beforeSetFriendRemark: + enable: false + timeout: 5 + failedContinue: true + afterSetFriendRemark: + enable: false + timeout: 5 + afterGroupMsgRevoke: + enable: false + timeout: 5 + afterJoinGroup: + enable: false + timeout: 5 + beforeInviteUserToGroup: + enable: false + timeout: 5 + failedContinue: true + afterSetGroupInfo: + enable: false + timeout: 5 + beforeSetGroupInfo: + enable: false + timeout: 5 + failedContinue: true + afterSetGroupInfoEx: + enable: false + timeout: 5 + beforeSetGroupInfoEx: + enable: false + timeout: 5 + failedContinue: true + afterRevokeMsg: + enable: false + timeout: 5 + beforeAddBlack: + enable: false + timeout: 5 + failedContinue: + afterAddFriend: + enable: false + timeout: 5 + beforeAddFriendAgree: + enable: false + timeout: 5 + failedContinue: true + afterAddFriendAgree: + enable: false + timeout: 5 + afterDeleteFriend: + enable: false + timeout: 5 + beforeImportFriends: + enable: false + timeout: 5 + failedContinue: true + afterImportFriends: + enable: false + timeout: 5 + afterRemoveBlack: + enable: false + timeout: 5 + + prometheus.yml: | + # my global config + global: + scrape_interval: 15s # Set the scrape interval to every 15 seconds. Default is every 1 minute. + evaluation_interval: 15s # Evaluate rules every 15 seconds. The default is every 1 minute. + # scrape_timeout is set to the global default (10s). + + # Alertmanager configuration + alerting: + alertmanagers: + - static_configs: + - targets: [internal_ip:19093] + + # Load rules once and periodically evaluate them according to the global evaluation_interval. + rule_files: + - instance-down-rules.yml + # - first_rules.yml + # - second_rules.yml + + # A scrape configuration containing exactly one endpoint to scrape: + # Here it's Prometheus itself. + scrape_configs: + # The job name is added as a label "job=job_name" to any timeseries scraped from this config. + # Monitored information captured by prometheus + + # prometheus fetches application services + - job_name: node_exporter + static_configs: + - targets: [ internal_ip:20500 ] + - job_name: openimserver-openim-api + static_configs: + - targets: [ internal_ip:12002 ] + labels: + namespace: default + - job_name: openimserver-openim-msggateway + static_configs: + - targets: [ internal_ip:12140 ] + # - targets: [ internal_ip:12140, internal_ip:12141, internal_ip:12142, internal_ip:12143, internal_ip:12144, internal_ip:12145, internal_ip:12146, internal_ip:12147, internal_ip:12148, internal_ip:12149, internal_ip:12150, internal_ip:12151, internal_ip:12152, internal_ip:12153, internal_ip:12154, internal_ip:12155 ] + labels: + namespace: default + - job_name: openimserver-openim-msgtransfer + static_configs: + - targets: [ internal_ip:12020, internal_ip:12021, internal_ip:12022, internal_ip:12023, internal_ip:12024, internal_ip:12025, internal_ip:12026, internal_ip:12027 ] + # - targets: [ internal_ip:12020, internal_ip:12021, internal_ip:12022, internal_ip:12023, internal_ip:12024, internal_ip:12025, internal_ip:12026, internal_ip:12027, internal_ip:12028, internal_ip:12029, internal_ip:12030, internal_ip:12031, internal_ip:12032, internal_ip:12033, internal_ip:12034, internal_ip:12035 ] + labels: + namespace: default + - job_name: openimserver-openim-push + static_configs: + - targets: [ internal_ip:12170, internal_ip:12171, internal_ip:12172, internal_ip:12173, internal_ip:12174, internal_ip:12175, internal_ip:12176, internal_ip:12177 ] + # - targets: [ internal_ip:12170, internal_ip:12171, internal_ip:12172, internal_ip:12173, internal_ip:12174, internal_ip:12175, internal_ip:12176, internal_ip:12177, internal_ip:12178, internal_ip:12179, internal_ip:12180, internal_ip:12182, internal_ip:12183, internal_ip:12184, internal_ip:12185, internal_ip:12186 ] + labels: + namespace: default + - job_name: openimserver-openim-rpc-auth + static_configs: + - targets: [ internal_ip:12200 ] + labels: + namespace: default + - job_name: openimserver-openim-rpc-conversation + static_configs: + - targets: [ internal_ip:12220 ] + labels: + namespace: default + - job_name: openimserver-openim-rpc-friend + static_configs: + - targets: [ internal_ip:12240 ] + labels: + namespace: default + - job_name: openimserver-openim-rpc-group + static_configs: + - targets: [ internal_ip:12260 ] + labels: + namespace: default + - job_name: openimserver-openim-rpc-msg + static_configs: + - targets: [ internal_ip:12280 ] + labels: + namespace: default + - job_name: openimserver-openim-rpc-third + static_configs: + - targets: [ internal_ip:12300 ] + labels: + namespace: default + - job_name: openimserver-openim-rpc-user + static_configs: + - targets: [ internal_ip:12320 ] + labels: + namespace: default + +--- +apiVersion: v1 +kind: Secret +metadata: + name: mongo-secret +type: Opaque +data: + mongo_initdb_root_username: cm9vdA== # base64 for "root" + mongo_initdb_root_password: b3BlbklNMTIz # base64 for "openIM123" + mongo_initdb_database: b3BlbmltX3Yz # base64 for "openim_v3" + mongo_openim_username: b3BlbklN # base64 for "openIM" + mongo_openim_password: b3BlbklNMTIz # base64 for "openIM123" \ No newline at end of file diff --git a/deployments/deploy/openim-crontask-deployment.yml b/deployments/deploy/openim-crontask-deployment.yml new file mode 100644 index 000000000..ca62714cd --- /dev/null +++ b/deployments/deploy/openim-crontask-deployment.yml @@ -0,0 +1,28 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: openim-crontask +spec: + replicas: 2 + selector: + matchLabels: + app: crontask + template: + metadata: + labels: + app: crontask + spec: + containers: + - name: crontask-container + image: openim/openim-crontask:v3.8.3 + env: + - name: CONFIG_PATH + value: "/config" + volumeMounts: + - name: openim-config + mountPath: "/config" + readOnly: true + volumes: + - name: openim-config + configMap: + name: openim-config diff --git a/deployments/deploy/openim-msggateway-deployment.yml b/deployments/deploy/openim-msggateway-deployment.yml new file mode 100644 index 000000000..ba2b1b84e --- /dev/null +++ b/deployments/deploy/openim-msggateway-deployment.yml @@ -0,0 +1,36 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: messagegateway-rpc-server +spec: + replicas: 2 + selector: + matchLabels: + app: messagegateway-rpc-server + template: + metadata: + labels: + app: messagegateway-rpc-server + spec: + containers: + - name: openim-msggateway-container + image: openim/openim-msggateway:v3.8.3 + env: + - name: CONFIG_PATH + value: "/config" + - name: IMENV_REDIS_PASSWORD + valueFrom: + secretKeyRef: + name: redis-secret + key: redis-password + volumeMounts: + - name: openim-config + mountPath: "/config" + readOnly: true + ports: + - containerPort: 10140 + - containerPort: 12001 + volumes: + - name: openim-config + configMap: + name: openim-config diff --git a/deployments/deploy/openim-msggateway-service.yml b/deployments/deploy/openim-msggateway-service.yml new file mode 100644 index 000000000..8b8ea2f16 --- /dev/null +++ b/deployments/deploy/openim-msggateway-service.yml @@ -0,0 +1,21 @@ +apiVersion: v1 +kind: Service +metadata: + name: messagegateway-rpc-service +spec: + selector: + app: messagegateway-rpc-server + ports: + - name: longConnServer-10001 + protocol: TCP + port: 10001 + targetPort: 10001 + - name: grpc-10140 + protocol: TCP + port: 10140 + targetPort: 10140 + - name: prometheus-12001 + protocol: TCP + port: 12001 + targetPort: 12001 + type: NodePort diff --git a/deployments/deploy/openim-msgtransfer-deployment.yml b/deployments/deploy/openim-msgtransfer-deployment.yml new file mode 100644 index 000000000..be608845c --- /dev/null +++ b/deployments/deploy/openim-msgtransfer-deployment.yml @@ -0,0 +1,40 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: openim-msgtransfer-server +spec: + replicas: 2 + selector: + matchLabels: + app: openim-msgtransfer-server + template: + metadata: + labels: + app: openim-msgtransfer-server + spec: + containers: + - name: openim-msgtransfer-container + image: openim/openim-msgtransfer:v3.8.3 + env: + - name: CONFIG_PATH + value: "/config" + - name: IMENV_REDIS_PASSWORD + valueFrom: + secretKeyRef: + name: redis-secret + key: redis-password + - name: IMENV_MONGODB_PASSWORD + valueFrom: + secretKeyRef: + name: mongo-secret + key: mongo_openim_password + volumeMounts: + - name: openim-config + mountPath: "/config" + readOnly: true + ports: + - containerPort: 12020 + volumes: + - name: openim-config + configMap: + name: openim-config diff --git a/deployments/deploy/openim-msgtransfer-service.yml b/deployments/deploy/openim-msgtransfer-service.yml new file mode 100644 index 000000000..387208bb8 --- /dev/null +++ b/deployments/deploy/openim-msgtransfer-service.yml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: Service +metadata: + name: openim-msgtransfer-service +spec: + selector: + app: openim-msgtransfer-server + ports: + - name: prometheus-12020 + protocol: TCP + port: 12020 + targetPort: 12020 + type: ClusterIP diff --git a/deployments/deploy/openim-push-deployment.yml b/deployments/deploy/openim-push-deployment.yml new file mode 100644 index 000000000..2092b343c --- /dev/null +++ b/deployments/deploy/openim-push-deployment.yml @@ -0,0 +1,36 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: push-rpc-server +spec: + replicas: 2 + selector: + matchLabels: + app: push-rpc-server + template: + metadata: + labels: + app: push-rpc-server + spec: + containers: + - name: push-rpc-server-container + image: openim/openim-push:v3.8.3 + env: + - name: CONFIG_PATH + value: "/config" + - name: IMENV_REDIS_PASSWORD + valueFrom: + secretKeyRef: + name: redis-secret + key: redis-password + volumeMounts: + - name: openim-config + mountPath: "/config" + readOnly: true + ports: + - containerPort: 10170 + - containerPort: 12170 + volumes: + - name: openim-config + configMap: + name: openim-config diff --git a/deployments/deploy/openim-push-service.yml b/deployments/deploy/openim-push-service.yml new file mode 100644 index 000000000..33f39c2d6 --- /dev/null +++ b/deployments/deploy/openim-push-service.yml @@ -0,0 +1,17 @@ +apiVersion: v1 +kind: Service +metadata: + name: push-rpc-service +spec: + selector: + app: push-rpc-server + ports: + - name: http-10170 + protocol: TCP + port: 10170 + targetPort: 10170 + - name: prometheus-12170 + protocol: TCP + port: 12170 + targetPort: 12170 + type: ClusterIP diff --git a/deployments/deploy/openim-rpc-auth-deployment.yml b/deployments/deploy/openim-rpc-auth-deployment.yml new file mode 100644 index 000000000..b785ea92f --- /dev/null +++ b/deployments/deploy/openim-rpc-auth-deployment.yml @@ -0,0 +1,37 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: auth-rpc-server +spec: + replicas: 2 + selector: + matchLabels: + app: auth-rpc-server + template: + metadata: + labels: + app: auth-rpc-server + spec: + containers: + - name: auth-rpc-server-container + image: openim/openim-rpc-auth:v3.8.3 + imagePullPolicy: Never + env: + - name: CONFIG_PATH + value: "/config" + - name: IMENV_REDIS_PASSWORD + valueFrom: + secretKeyRef: + name: redis-secret + key: redis-password + volumeMounts: + - name: openim-config + mountPath: "/config" + readOnly: true + ports: + - containerPort: 10200 + - containerPort: 12200 + volumes: + - name: openim-config + configMap: + name: openim-config diff --git a/deployments/deploy/openim-rpc-auth-service.yml b/deployments/deploy/openim-rpc-auth-service.yml new file mode 100644 index 000000000..7d798383f --- /dev/null +++ b/deployments/deploy/openim-rpc-auth-service.yml @@ -0,0 +1,17 @@ +apiVersion: v1 +kind: Service +metadata: + name: auth-rpc-service +spec: + selector: + app: auth-rpc-server + ports: + - name: http-10200 + protocol: TCP + port: 10200 + targetPort: 10200 + - name: prometheus-12200 + protocol: TCP + port: 12200 + targetPort: 12200 + type: ClusterIP diff --git a/deployments/deploy/openim-rpc-conversation-deployment.yml b/deployments/deploy/openim-rpc-conversation-deployment.yml new file mode 100644 index 000000000..4d7a32497 --- /dev/null +++ b/deployments/deploy/openim-rpc-conversation-deployment.yml @@ -0,0 +1,41 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: conversation-rpc-server +spec: + replicas: 2 + selector: + matchLabels: + app: conversation-rpc-server + template: + metadata: + labels: + app: conversation-rpc-server + spec: + containers: + - name: conversation-rpc-server-container + image: openim/openim-rpc-conversation:v3.8.3 + env: + - name: CONFIG_PATH + value: "/config" + - name: IMENV_REDIS_PASSWORD + valueFrom: + secretKeyRef: + name: redis-secret + key: redis-password + - name: IMENV_MONGODB_PASSWORD + valueFrom: + secretKeyRef: + name: mongo-secret + key: mongo_openim_password + volumeMounts: + - name: openim-config + mountPath: "/config" + readOnly: true + ports: + - containerPort: 10220 + - containerPort: 12220 + volumes: + - name: openim-config + configMap: + name: openim-config diff --git a/deployments/deploy/openim-rpc-conversation-service.yml b/deployments/deploy/openim-rpc-conversation-service.yml new file mode 100644 index 000000000..f9be231ad --- /dev/null +++ b/deployments/deploy/openim-rpc-conversation-service.yml @@ -0,0 +1,17 @@ +apiVersion: v1 +kind: Service +metadata: + name: conversation-rpc-service +spec: + selector: + app: conversation-rpc-server + ports: + - name: http-10220 + protocol: TCP + port: 10220 + targetPort: 10220 + - name: prometheus-12220 + protocol: TCP + port: 12220 + targetPort: 12220 + type: ClusterIP diff --git a/deployments/deploy/openim-rpc-friend-deployment.yml b/deployments/deploy/openim-rpc-friend-deployment.yml new file mode 100644 index 000000000..5fdd3bf62 --- /dev/null +++ b/deployments/deploy/openim-rpc-friend-deployment.yml @@ -0,0 +1,41 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: friend-rpc-server +spec: + replicas: 2 + selector: + matchLabels: + app: friend-rpc-server + template: + metadata: + labels: + app: friend-rpc-server + spec: + containers: + - name: friend-rpc-server-container + image: openim/openim-rpc-friend:v3.8.3 + env: + - name: CONFIG_PATH + value: "/config" + - name: IMENV_REDIS_PASSWORD + valueFrom: + secretKeyRef: + name: redis-secret + key: redis-password + - name: IMENV_MONGODB_PASSWORD + valueFrom: + secretKeyRef: + name: mongo-secret + key: mongo_openim_password + volumeMounts: + - name: openim-config + mountPath: "/config" + readOnly: true + ports: + - containerPort: 10240 + - containerPort: 12240 + volumes: + - name: openim-config + configMap: + name: openim-config \ No newline at end of file diff --git a/deployments/deploy/openim-rpc-friend-service.yml b/deployments/deploy/openim-rpc-friend-service.yml new file mode 100644 index 000000000..b6b512baa --- /dev/null +++ b/deployments/deploy/openim-rpc-friend-service.yml @@ -0,0 +1,17 @@ +apiVersion: v1 +kind: Service +metadata: + name: friend-rpc-service +spec: + selector: + app: friend-rpc-server + ports: + - name: http-10240 + protocol: TCP + port: 10240 + targetPort: 10240 + - name: prometheus-12240 + protocol: TCP + port: 12240 + targetPort: 12240 + type: ClusterIP diff --git a/deployments/deploy/openim-rpc-group-deployment.yml b/deployments/deploy/openim-rpc-group-deployment.yml new file mode 100644 index 000000000..313fec897 --- /dev/null +++ b/deployments/deploy/openim-rpc-group-deployment.yml @@ -0,0 +1,42 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: group-rpc-server +spec: + replicas: 2 + selector: + matchLabels: + app: group-rpc-server + template: + metadata: + labels: + app: group-rpc-server + spec: + containers: + - name: group-rpc-server-container + image: openim/openim-rpc-group:v3.8.3 + + env: + - name: CONFIG_PATH + value: "/config" + - name: IMENV_REDIS_PASSWORD + valueFrom: + secretKeyRef: + name: redis-secret + key: redis-password + - name: IMENV_MONGODB_PASSWORD + valueFrom: + secretKeyRef: + name: mongo-secret + key: mongo_openim_password + volumeMounts: + - name: openim-config + mountPath: "/config" + readOnly: true + ports: + - containerPort: 10260 + - containerPort: 12260 + volumes: + - name: openim-config + configMap: + name: openim-config \ No newline at end of file diff --git a/deployments/deploy/openim-rpc-group-service.yml b/deployments/deploy/openim-rpc-group-service.yml new file mode 100644 index 000000000..bccc0803f --- /dev/null +++ b/deployments/deploy/openim-rpc-group-service.yml @@ -0,0 +1,17 @@ +apiVersion: v1 +kind: Service +metadata: + name: group-rpc-service +spec: + selector: + app: group-rpc-server + ports: + - name: http-10260 + protocol: TCP + port: 10260 + targetPort: 10260 + - name: prometheus-12260 + protocol: TCP + port: 12260 + targetPort: 12260 + type: ClusterIP diff --git a/deployments/deploy/openim-rpc-msg-deployment.yml b/deployments/deploy/openim-rpc-msg-deployment.yml new file mode 100644 index 000000000..e883f5849 --- /dev/null +++ b/deployments/deploy/openim-rpc-msg-deployment.yml @@ -0,0 +1,41 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: msg-rpc-server +spec: + replicas: 2 + selector: + matchLabels: + app: msg-rpc-server + template: + metadata: + labels: + app: msg-rpc-server + spec: + containers: + - name: msg-rpc-server-container + image: openim/openim-rpc-msg:v3.8.3 + env: + - name: CONFIG_PATH + value: "/config" + - name: IMENV_REDIS_PASSWORD + valueFrom: + secretKeyRef: + name: redis-secret + key: redis-password + - name: IMENV_MONGODB_PASSWORD + valueFrom: + secretKeyRef: + name: mongo-secret + key: mongo_openim_password + volumeMounts: + - name: openim-config + mountPath: "/config" + readOnly: true + ports: + - containerPort: 10280 + - containerPort: 12280 + volumes: + - name: openim-config + configMap: + name: openim-config \ No newline at end of file diff --git a/deployments/deploy/openim-rpc-msg-service.yml b/deployments/deploy/openim-rpc-msg-service.yml new file mode 100644 index 000000000..db7610e8b --- /dev/null +++ b/deployments/deploy/openim-rpc-msg-service.yml @@ -0,0 +1,17 @@ +apiVersion: v1 +kind: Service +metadata: + name: msg-rpc-service +spec: + selector: + app: msg-rpc-server + ports: + - name: http-10280 + protocol: TCP + port: 10280 + targetPort: 10280 + - name: prometheus-12280 + protocol: TCP + port: 12280 + targetPort: 12280 + type: ClusterIP diff --git a/deployments/deploy/openim-rpc-third-deployment.yml b/deployments/deploy/openim-rpc-third-deployment.yml new file mode 100644 index 000000000..326aaee03 --- /dev/null +++ b/deployments/deploy/openim-rpc-third-deployment.yml @@ -0,0 +1,51 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: third-rpc-server +spec: + replicas: 2 + selector: + matchLabels: + app: third-rpc-server + template: + metadata: + labels: + app: third-rpc-server + spec: + containers: + - name: third-rpc-server-container + image: openim/openim-rpc-third:v3.8.3 + env: + - name: CONFIG_PATH + value: "/config" + - name: IMENV_MINIO_ACCESSKEYID + valueFrom: + secretKeyRef: + name: minio-secret + key: minio-root-user + - name: IMENV_MINIO_SECRETACCESSKEY + valueFrom: + secretKeyRef: + name: minio-secret + key: minio-root-password + - name: IMENV_REDIS_PASSWORD + valueFrom: + secretKeyRef: + name: redis-secret + key: redis-password + - name: IMENV_MONGODB_PASSWORD + valueFrom: + secretKeyRef: + name: mongo-secret + key: mongo_openim_password + volumeMounts: + - name: openim-config + mountPath: "/config" + readOnly: true + ports: + - containerPort: 10300 + - containerPort: 12300 + volumes: + - name: openim-config + configMap: + name: openim-config diff --git a/deployments/deploy/openim-rpc-third-service.yml b/deployments/deploy/openim-rpc-third-service.yml new file mode 100644 index 000000000..8cd34c285 --- /dev/null +++ b/deployments/deploy/openim-rpc-third-service.yml @@ -0,0 +1,17 @@ +apiVersion: v1 +kind: Service +metadata: + name: third-rpc-service +spec: + selector: + app: third-rpc-server + ports: + - name: http-10300 + protocol: TCP + port: 10300 + targetPort: 10300 + - name: prometheus-12300 + protocol: TCP + port: 12300 + targetPort: 12300 + type: ClusterIP diff --git a/deployments/deploy/openim-rpc-user-deployment.yml b/deployments/deploy/openim-rpc-user-deployment.yml new file mode 100644 index 000000000..c6a47e125 --- /dev/null +++ b/deployments/deploy/openim-rpc-user-deployment.yml @@ -0,0 +1,41 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: user-rpc-server +spec: + replicas: 2 + selector: + matchLabels: + app: user-rpc-server + template: + metadata: + labels: + app: user-rpc-server + spec: + containers: + - name: user-rpc-server-container + image: openim/openim-rpc-user:v3.8.3 + env: + - name: CONFIG_PATH + value: "/config" + - name: IMENV_REDIS_PASSWORD + valueFrom: + secretKeyRef: + name: redis-secret + key: redis-password + - name: IMENV_MONGODB_PASSWORD + valueFrom: + secretKeyRef: + name: mongo-secret + key: mongo_openim_password + volumeMounts: + - name: openim-config + mountPath: "/config" + readOnly: true + ports: + - containerPort: 10320 + - containerPort: 12320 + volumes: + - name: openim-config + configMap: + name: openim-config diff --git a/deployments/deploy/openim-rpc-user-service.yml b/deployments/deploy/openim-rpc-user-service.yml new file mode 100644 index 000000000..50cef3c88 --- /dev/null +++ b/deployments/deploy/openim-rpc-user-service.yml @@ -0,0 +1,17 @@ +apiVersion: v1 +kind: Service +metadata: + name: user-rpc-service +spec: + selector: + app: user-rpc-server + ports: + - name: http-10320 + protocol: TCP + port: 10320 + targetPort: 10320 + - name: prometheus-12320 + protocol: TCP + port: 12320 + targetPort: 12320 + type: ClusterIP diff --git a/deployments/deploy/redis-service.yml b/deployments/deploy/redis-service.yml new file mode 100644 index 000000000..d076fd119 --- /dev/null +++ b/deployments/deploy/redis-service.yml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Service +metadata: + name: redis-service + labels: + app: redis +spec: + type: ClusterIP + selector: + app: redis + ports: + - name: redis-port + protocol: TCP + port: 16379 + targetPort: 6379 diff --git a/deployments/deploy/redis-statefulset.yml b/deployments/deploy/redis-statefulset.yml new file mode 100644 index 000000000..3f6dc41bc --- /dev/null +++ b/deployments/deploy/redis-statefulset.yml @@ -0,0 +1,66 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: redis-statefulset +spec: + serviceName: "redis" + replicas: 2 + selector: + matchLabels: + app: redis + template: + metadata: + labels: + app: redis + spec: + containers: + - name: redis + image: redis:7.0.0 + ports: + - containerPort: 6379 + env: + - name: TZ + value: "Asia/Shanghai" + - name: REDIS_PASSWORD + valueFrom: + secretKeyRef: + name: redis-secret + key: redis-password + volumeMounts: + - name: redis-data + mountPath: /data + # - name: redis-config-volume + # mountPath: /usr/local/redis/config/redis.conf + # subPath: redis.conf + command: + [ + "/bin/sh", + "-c", + 'redis-server --requirepass "$REDIS_PASSWORD" --appendonly yes', + ] + volumes: + - name: redis-config-volume + configMap: + name: openim-config + - name: redis-data + persistentVolumeClaim: + claimName: redis-pvc +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: redis-pvc +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 5Gi +--- +apiVersion: v1 +kind: Secret +metadata: + name: redis-secret +type: Opaque +data: + redis-password: b3BlbklNMTIz # "openIM123" in base64 diff --git a/deployments/templates/alertmanager.yml b/deployments/templates/alertmanager.yml deleted file mode 100644 index 4e390b2bb..000000000 --- a/deployments/templates/alertmanager.yml +++ /dev/null @@ -1,47 +0,0 @@ -# Copyright © 2023 OpenIM. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -###################### AlertManager Configuration ###################### -# AlertManager configuration using environment variables -# -# Resolve timeout -# SMTP configuration for sending alerts -# Templates for email notifications -# Routing configurations for alerts -# Receiver configurations -global: - resolve_timeout: ${ALERTMANAGER_RESOLVE_TIMEOUT} - smtp_from: ${ALERTMANAGER_SMTP_FROM} - smtp_smarthost: ${ALERTMANAGER_SMTP_SMARTHOST} - smtp_auth_username: ${ALERTMANAGER_SMTP_AUTH_USERNAME} - smtp_auth_password: ${ALERTMANAGER_SMTP_AUTH_PASSWORD} - smtp_require_tls: ${ALERTMANAGER_SMTP_REQUIRE_TLS} - smtp_hello: ${ALERTMANAGER_SMTP_HELLO} - -templates: - - /etc/alertmanager/email.tmpl - -route: - group_by: ['alertname'] - group_wait: 5s - group_interval: 5s - repeat_interval: 5m - receiver: email -receivers: - - name: email - email_configs: - - to: '${ALERTMANAGER_EMAIL_TO}' - html: '{{ template "email.to.html" . }}' - headers: { Subject: "[OPENIM-SERVER]Alarm" } - send_resolved: true diff --git a/deployments/templates/charts-value.yaml b/deployments/templates/charts-value.yaml deleted file mode 100644 index 26bbe2926..000000000 --- a/deployments/templates/charts-value.yaml +++ /dev/null @@ -1,110 +0,0 @@ -# Copyright © 2023 OpenIM. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -#This configuration file is used to override the use of the value.yaml variable. -#Currently, only the configuration with ingressName as nginx is provided. -#If it is another gateway such as istio or treafik, please modify the corresponding gateway requirements - -global: - commonRepository: ghcr.io/openimsdk - commonTag: "latest" - pullPolicy: Always - -openim-api: - image: - repository: ghcr.io/openimsdk/openim-api - pullPolicy: Always - tag: "latest" - ingress: - enabled: true - className: "nginx" - annotations: - nginx.ingress.kubernetes.io/use-regex: "true" - nginx.ingress.kubernetes.io/rewrite-target: /$2 - hosts: - - host: openim.server.com # your hostname - paths: - - path: /api(/|$)(.*) - pathType: ImplementationSpecific - tls: - - secretName: webapitls #your hostname tls - hosts: - - openim.server.com - -openim-msggateway: - image: - repository: ghcr.io/openimsdk/openim-msggateway - tag: "latest" - pullPolicy: Always - ingress: - enabled: true - className: "nginx" - annotations: - nginx.ingress.kubernetes.io/use-regex: "true" - nginx.ingress.kubernetes.io/rewrite-target: /$2 - hosts: - - host: openim.server.com # your hostname - paths: - - path: /msg_gateway(/|$)(.*) - pathType: ImplementationSpecific - tls: - - secretName: webapitls #your hostname tls - hosts: - - openim.server.com - -openim-msgtransfer: - image: - repository: ghcr.io/openimsdk/openim-msgtransfer - tag: "latest" - pullPolicy: Always -openim-push: - image: - repository: ghcr.io/openimsdk/openim-push - tag: "latest" - pullPolicy: Always -openim-rpc-auth: - image: - repository: ghcr.io/openimsdk/openim-rpc-auth - tag: "latest" - pullPolicy: Always -openim-rpc-conversation: - image: - repository: ghcr.io/openimsdk/openim-rpc-conversation - tag: "latest" - pullPolicy: Always -openim-rpc-friend: - image: - repository: ghcr.io/openimsdk/openim-rpc-friend - tag: "latest" - pullPolicy: Always -openim-rpc-group: - image: - repository: ghcr.io/openimsdk/openim-rpc-group - tag: "latest" - pullPolicy: Always -openim-rpc-msg: - image: - repository: ghcr.io/openimsdk/openim-rpc-msg - tag: "latest" - pullPolicy: Always -openim-rpc-third: - image: - repository: ghcr.io/openimsdk/openim-rpc-third - tag: "latest" - pullPolicy: Always -openim-rpc-user: - image: - repository: ghcr.io/openimsdk/openim-rpc-user - tag: "latest" - pullPolicy: Always diff --git a/deployments/templates/config.yaml b/deployments/templates/config.yaml deleted file mode 100644 index c108de5e8..000000000 --- a/deployments/templates/config.yaml +++ /dev/null @@ -1,573 +0,0 @@ -# Copyright © 2023 OpenIM. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the License); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# ----------------------------------------------------------------- -# TODO: This config file is the template file -# --| source: deployments/templates/config.yaml -# --| env: scripts/install/environment -# --| target: config/config.yaml -# ----------------------------------------------------------------- - -envs: - discovery: ${ENVS_DISCOVERY} - -###################### Zookeeper ###################### -# Zookeeper configuration -# It's not recommended to modify the schema -# -# Zookeeper address -# Zookeeper username -# Zookeeper password -zookeeper: - schema: ${ZOOKEEPER_SCHEMA} - address: [ ${ZOOKEEPER_ADDRESS}:${ZOOKEEPER_PORT} ] - username: ${ZOOKEEPER_USERNAME} - password: ${ZOOKEEPER_PASSWORD} - -###################### Mongo ###################### -# MongoDB configuration - -# If uri is not empty, it will be used directly for the MongoDB connection. -# This is a complete MongoDB URI string. -# Example: mongodb://user:password@host1:port1,host2:port2/dbname?options -mongo: - uri: ${MONGO_URI} - -# List of MongoDB server addresses. -# Used for constructing the MongoDB URI if 'uri' above is empty. -# For a standalone setup, specify the address of the single server. -# For a sharded cluster, specify the addresses of the Mongos servers. -# Example: [ '172.28.0.1:37017', '172.28.0.2:37017' ] -# Default MongoDB database name -# Maximum connection pool size - address: [ ${MONGO_ADDRESS}:${MONGO_PORT} ] - database: ${MONGO_DATABASE} - username: ${MONGO_OPENIM_USERNAME} - password: ${MONGO_OPENIM_PASSWORD} - maxPoolSize: ${MONGO_MAX_POOL_SIZE} - -###################### Redis configuration information ###################### -# Redis configuration -# -# Username is required only for Redis version 6.0+ -redis: - address: [ ${REDIS_ADDRESS}:${REDIS_PORT} ] - username: ${REDIS_USERNAME} - password: ${REDIS_PASSWORD} - -###################### Kafka configuration information ###################### -# Kafka configuration -# -# Kafka username -# Kafka password -# It's not recommended to modify this topic name -# Consumer group ID, it's not recommended to modify -kafka: - username: ${KAFKA_USERNAME} - password: ${KAFKA_PASSWORD} - addr: [ ${KAFKA_ADDRESS}:${KAFKA_PORT} ] - latestMsgToRedis: - topic: "${KAFKA_LATESTMSG_REDIS_TOPIC}" - offlineMsgToMongo: - topic: "${KAFKA_OFFLINEMSG_MONGO_TOPIC}" - msgToPush: - topic: "${KAFKA_MSG_PUSH_TOPIC}" - consumerGroupID: - msgToRedis: ${KAFKA_CONSUMERGROUPID_REDIS} - msgToMongo: ${KAFKA_CONSUMERGROUPID_MONGO} - msgToMySql: ${KAFKA_CONSUMERGROUPID_MYSQL} - msgToPush: ${KAFKA_CONSUMERGROUPID_PUSH} - -###################### RPC configuration information ###################### -# RPC configuration -# -# IP address to register with zookeeper when starting RPC, the IP and corresponding rpcPort should be accessible by api/gateway -# Default listen IP is 0.0.0.0 -rpc: - registerIP: ${RPC_REGISTER_IP} - listenIP: ${RPC_LISTEN_IP} - -###################### API configuration information ###################### -# API configuration -# -# API service port -# Default listen IP is 0.0.0.0 -api: - openImApiPort: [ ${API_OPENIM_PORT} ] - listenIP: ${API_LISTEN_IP} - -###################### Object configuration information ###################### -# Object storage configuration -# -# Use minio for object storage -# API URL should be accessible by the app -# It's not recommended to modify the bucket name -# Endpoint should be accessible by the app -# Session token -# Configuration for Tencent COS -# Configuration for Aliyun OSS -# apiURL is the address of the api, the access address of the app, use s3 must be configured -# minio.endpoint can be configured as an intranet address, -# minio.signEndpoint is minio public network address -object: - enable: "${OBJECT_ENABLE}" - apiURL: "${OBJECT_APIURL}" - minio: - bucket: "${MINIO_BUCKET}" - endpoint: "${MINIO_ENDPOINT}" - accessKeyID: "${MINIO_ACCESS_KEY}" - secretAccessKey: "${MINIO_SECRET_KEY}" - sessionToken: ${MINIO_SESSION_TOKEN} - signEndpoint: "${MINIO_SIGN_ENDPOINT}" - publicRead: ${MINIO_PUBLIC_READ} - cos: - bucketURL: ${COS_BUCKET_URL} - secretID: ${COS_SECRET_ID} - secretKey: ${COS_SECRET_KEY} - sessionToken: ${COS_SESSION_TOKEN} - publicRead: ${COS_PUBLIC_READ} - oss: - endpoint: "${OSS_ENDPOINT}" - bucket: "${OSS_BUCKET}" - bucketURL: "${OSS_BUCKET_URL}" - accessKeyID: ${OSS_ACCESS_KEY_ID} - accessKeySecret: ${OSS_ACCESS_KEY_SECRET} - sessionToken: ${OSS_SESSION_TOKEN} - publicRead: ${OSS_PUBLIC_READ} - kodo: - endpoint: "${KODO_ENDPOINT}" - bucket: "${KODO_BUCKET}" - bucketURL: "${KODO_BUCKET_URL}" - accessKeyID: ${KODO_ACCESS_KEY_ID} - accessKeySecret: ${KODO_ACCESS_KEY_SECRET} - sessionToken: ${KODO_SESSION_TOKEN} - publicRead: ${KODO_PUBLIC_READ} - aws: - endpoint: "${AWS_ENDPOINT}" # This might not be necessary unless you're using a custom endpoint - region: "${AWS_REGION}" - bucket: "${AWS_BUCKET}" - accessKeyID: ${AWS_ACCESS_KEY_ID} - accessKeySecret: ${AWS_SECRET_ACCESS_KEY} - publicRead: ${AWS_PUBLIC_READ} - -###################### RPC Port Configuration ###################### -# RPC service ports -# These ports are passed into the program by the script and are not recommended to modify -# For launching multiple programs, just fill in multiple ports separated by commas -# For example, [10110, 10111] -rpcPort: - openImUserPort: [ ${OPENIM_USER_PORT} ] - openImFriendPort: [ ${OPENIM_FRIEND_PORT} ] - openImMessagePort: [ ${OPENIM_MESSAGE_PORT} ] - openImGroupPort: [ ${OPENIM_GROUP_PORT} ] - openImAuthPort: [ ${OPENIM_AUTH_PORT} ] - openImPushPort: [ ${OPENIM_PUSH_PORT} ] - openImConversationPort: [ ${OPENIM_CONVERSATION_PORT} ] - openImThirdPort: [ ${OPENIM_THIRD_PORT} ] - -###################### RPC Register Name Configuration ###################### -# RPC service names for registration, it's not recommended to modify these -rpcRegisterName: - openImUserName: ${OPENIM_USER_NAME} - openImFriendName: ${OPENIM_FRIEND_NAME} - openImMsgName: ${OPENIM_MSG_NAME} - openImPushName: ${OPENIM_PUSH_NAME} - openImMessageGatewayName: ${OPENIM_MESSAGE_GATEWAY_NAME} - openImGroupName: ${OPENIM_GROUP_NAME} - openImAuthName: ${OPENIM_AUTH_NAME} - openImConversationName: ${OPENIM_CONVERSATION_NAME} - openImThirdName: ${OPENIM_THIRD_NAME} - -###################### Log Configuration ###################### -# Log configuration -# -# Storage directory -# Log rotation time -# Maximum number of logs to retain -# Log level, 6 means all levels -# Whether to output to stdout -# Whether to output in json format -# Whether to include stack trace in logs -log: - storageLocation: ${LOG_STORAGE_LOCATION} - rotationTime: ${LOG_ROTATION_TIME} - remainRotationCount: ${LOG_REMAIN_ROTATION_COUNT} - remainLogLevel: ${LOG_REMAIN_LOG_LEVEL} - isStdout: ${LOG_IS_STDOUT} - isJson: ${LOG_IS_JSON} - withStack: ${LOG_WITH_STACK} - -###################### Variables definition ###################### -# Long connection server configuration -# -# Websocket port for msg_gateway -# Maximum number of websocket connections -# Maximum length of websocket request package -# Websocket connection handshake timeout -longConnSvr: - openImWsPort: [ ${OPENIM_WS_PORT} ] - websocketMaxConnNum: ${WEBSOCKET_MAX_CONN_NUM} - openImMessageGatewayPort: [ ${OPENIM_MESSAGE_GATEWAY_PORT} ] - websocketMaxMsgLen: ${WEBSOCKET_MAX_MSG_LEN} - websocketTimeout: ${WEBSOCKET_TIMEOUT} - -# Push notification service configuration -# -# Use GeTui for push notifications -# GeTui offline push configuration -# FCM offline push configuration -# Account file, place it in the config directory -# JPush configuration, modify these after applying in JPush backend -push: - enable: ${PUSH_ENABLE} - geTui: - pushUrl: "${GETUI_PUSH_URL}" - masterSecret: ${GETUI_MASTER_SECRET} - appKey: ${GETUI_APP_KEY} - intent: ${GETUI_INTENT} - channelID: ${GETUI_CHANNEL_ID} - channelName: ${GETUI_CHANNEL_NAME} - fcm: - serviceAccount: "${FCM_SERVICE_ACCOUNT}" - jpush: - appKey: ${JPUSH_APP_KEY} - masterSecret: ${JPUSH_MASTER_SECRET} - pushUrl: ${JPUSH_PUSH_URL} - pushIntent: ${JPUSH_PUSH_INTENT} - -# App manager configuration -# -# Built-in app manager user IDs -# Built-in app manager nicknames -# Attention, this configure is discarded. If you have used him before, configure your own -manager: - userID: - nickname: - -# chatAdmin, use for send notification -# -# Built-in app system notification account ID -# Built-in app system notification account nickname -im-admin: - userID: [ "${IM_ADMIN_USERID}" ] - nickname: [ "${IM_ADMIN_NAME}" ] - -# Multi-platform login policy -# For each platform(Android, iOS, Windows, Mac, web), only one can be online at a time -multiLoginPolicy: ${MULTILOGIN_POLICY} - -# Whether to store messages in MySQL, messages in MySQL are only used for management background -chatPersistenceMysql: ${CHAT_PERSISTENCE_MYSQL} - -# Message cache timeout in seconds, it's not recommended to modify -msgCacheTimeout: ${MSG_CACHE_TIMEOUT} - -# Whether to enable read receipts for group chat -groupMessageHasReadReceiptEnable: ${GROUP_MSG_READ_RECEIPT} - -# Whether to enable read receipts for single chat -singleMessageHasReadReceiptEnable: ${SINGLE_MSG_READ_RECEIPT} - -# MongoDB offline message retention period in days -retainChatRecords: ${RETAIN_CHAT_RECORDS} - -# Schedule to clear expired messages(older than retainChatRecords days) in MongoDB every Wednesday at 2am -# This deletion is just for cleaning up disk usage according to previous configuration retainChatRecords, no notification will be sent -chatRecordsClearTime: "${CHAT_RECORDS_CLEAR_TIME}" - -# Schedule to auto delete messages every day at 2am -# This deletion is for messages that have been retained for more than msg_destruct_time (seconds) in the conversation field -msgDestructTime: "${MSG_DESTRUCT_TIME}" - -# Secret key -secret: ${SECRET} - -# Token policy -# -# Token expiration period in days -tokenPolicy: - expire: ${TOKEN_EXPIRE} - -# Message verification policy -# -# Whether to verify friendship when sending messages -messageVerify: - friendVerify: false - -# iOS push notification configuration -# -# iOS push notification sound -# Whether to count badge -# Whether it's production environment -iosPush: - pushSound: "xxx" - badgeCount: true - production: false - -###################### Third-party service configuration ###################### -# Callback configuration -# -# Callback URL -# Whether to enable this callback event -# Timeout in seconds -# Whether to continue execution if callback fails -callback: - url: "webhook://127.0.0.1:10008/callbackExample" - beforeSendSingleMsg: - enable: ${CALLBACK_ENABLE} - timeout: ${CALLBACK_TIMEOUT} - failedContinue: ${CALLBACK_FAILED_CONTINUE} - beforeUpdateUserInfoEx: - enable: ${CALLBACK_ENABLE} - timeout: ${CALLBACK_TIMEOUT} - failedContinue: ${CALLBACK_FAILED_CONTINUE} - afterUpdateUserInfoEx: - enable: ${CALLBACK_ENABLE} - timeout: ${CALLBACK_TIMEOUT} - failedContinue: ${CALLBACK_FAILED_CONTINUE} - afterSendSingleMsg: - enable: ${CALLBACK_ENABLE} - timeout: ${CALLBACK_TIMEOUT} - failedContinue: ${CALLBACK_FAILED_CONTINUE} - beforeSendGroupMsg: - enable: ${CALLBACK_ENABLE} - timeout: ${CALLBACK_TIMEOUT} - failedContinue: ${CALLBACK_FAILED_CONTINUE} - afterSendGroupMsg: - enable: ${CALLBACK_ENABLE} - timeout: ${CALLBACK_TIMEOUT} - failedContinue: ${CALLBACK_FAILED_CONTINUE} - msgModify: - enable: ${CALLBACK_ENABLE} - timeout: ${CALLBACK_TIMEOUT} - failedContinue: ${CALLBACK_FAILED_CONTINUE} - userOnline: - enable: ${CALLBACK_ENABLE} - timeout: ${CALLBACK_TIMEOUT} - failedContinue: ${CALLBACK_FAILED_CONTINUE} - userOffline: - enable: ${CALLBACK_ENABLE} - timeout: ${CALLBACK_TIMEOUT} - failedContinue: ${CALLBACK_FAILED_CONTINUE} - userKickOff: - enable: ${CALLBACK_ENABLE} - timeout: ${CALLBACK_TIMEOUT} - failedContinue: ${CALLBACK_FAILED_CONTINUE} - offlinePush: - enable: ${CALLBACK_ENABLE} - timeout: ${CALLBACK_TIMEOUT} - failedContinue: ${CALLBACK_FAILED_CONTINUE} - onlinePush: - enable: ${CALLBACK_ENABLE} - timeout: ${CALLBACK_TIMEOUT} - failedContinue: ${CALLBACK_FAILED_CONTINUE} - superGroupOnlinePush: - enable: ${CALLBACK_ENABLE} - timeout: ${CALLBACK_TIMEOUT} - failedContinue: ${CALLBACK_FAILED_CONTINUE} - beforeAddFriend: - enable: ${CALLBACK_ENABLE} - timeout: ${CALLBACK_TIMEOUT} - failedContinue: ${CALLBACK_FAILED_CONTINUE} - beforeUpdateUserInfo: - enable: ${CALLBACK_ENABLE} - timeout: ${CALLBACK_TIMEOUT} - failedContinue: ${CALLBACK_FAILED_CONTINUE} - beforeCreateGroup: - enable: ${CALLBACK_ENABLE} - timeout: ${CALLBACK_TIMEOUT} - failedContinue: ${CALLBACK_FAILED_CONTINUE} - afterCreateGroup: - enable: ${CALLBACK_ENABLE} - timeout: ${CALLBACK_TIMEOUT} - failedContinue: ${CALLBACK_FAILED_CONTINUE} - beforeMemberJoinGroup: - enable: ${CALLBACK_ENABLE} - timeout: ${CALLBACK_TIMEOUT} - failedContinue: ${CALLBACK_FAILED_CONTINUE} - beforeSetGroupMemberInfo: - enable: ${CALLBACK_ENABLE} - timeout: ${CALLBACK_TIMEOUT} - failedContinue: ${CALLBACK_FAILED_CONTINUE} - afterSetGroupMemberInfo: - enable: ${CALLBACK_ENABLE} - timeout: ${CALLBACK_TIMEOUT} - failedContinue: ${CALLBACK_FAILED_CONTINUE} - setMessageReactionExtensions: - enable: ${CALLBACK_ENABLE} - timeout: ${CALLBACK_TIMEOUT} - failedContinue: ${CALLBACK_FAILED_CONTINUE} - quitGroup: - enable: ${CALLBACK_ENABLE} - timeout: ${CALLBACK_TIMEOUT} - failedContinue: ${CALLBACK_FAILED_CONTINUE} - killGroupMember: - enable: ${CALLBACK_ENABLE} - timeout: ${CALLBACK_TIMEOUT} - failedContinue: ${CALLBACK_FAILED_CONTINUE} - dismissGroup: - enable: ${CALLBACK_ENABLE} - timeout: ${CALLBACK_TIMEOUT} - failedContinue: ${CALLBACK_FAILED_CONTINUE} - joinGroup: - enable: ${CALLBACK_ENABLE} - timeout: ${CALLBACK_TIMEOUT} - failedContinue: ${CALLBACK_FAILED_CONTINUE} - groupMsgRead: - enable: ${CALLBACK_ENABLE} - timeout: ${CALLBACK_TIMEOUT} - failedContinue: ${CALLBACK_FAILED_CONTINUE} - singleMsgRead: - enable: ${CALLBACK_ENABLE} - timeout: ${CALLBACK_TIMEOUT} - failedContinue: ${CALLBACK_FAILED_CONTINUE} - updateUserInfo: - enable: ${CALLBACK_ENABLE} - timeout: ${CALLBACK_TIMEOUT} - failedContinue: ${CALLBACK_FAILED_CONTINUE} - beforeUserRegister: - enable: ${CALLBACK_ENABLE} - timeout: ${CALLBACK_TIMEOUT} - failedContinue: ${CALLBACK_FAILED_CONTINUE} - afterUserRegister: - enable: ${CALLBACK_ENABLE} - timeout: ${CALLBACK_TIMEOUT} - failedContinue: ${CALLBACK_FAILED_CONTINUE} - transferGroupOwner: - enable: ${CALLBACK_ENABLE} - timeout: ${CALLBACK_TIMEOUT} - failedContinue: ${CALLBACK_FAILED_CONTINUE} - beforeSetFriendRemark: - enable: ${CALLBACK_ENABLE} - timeout: ${CALLBACK_TIMEOUT} - failedContinue: ${CALLBACK_FAILED_CONTINUE} - afterSetFriendRemark: - enable: ${CALLBACK_ENABLE} - timeout: ${CALLBACK_TIMEOUT} - failedContinue: ${CALLBACK_FAILED_CONTINUE} - afterGroupMsgRead: - enable: ${CALLBACK_ENABLE} - timeout: ${CALLBACK_TIMEOUT} - failedContinue: ${CALLBACK_FAILED_CONTINUE} - afterGroupMsgRevoke: - enable: ${CALLBACK_ENABLE} - timeout: ${CALLBACK_TIMEOUT} - failedContinue: ${CALLBACK_FAILED_CONTINUE} - afterJoinGroup: - enable: ${CALLBACK_ENABLE} - timeout: ${CALLBACK_TIMEOUT} - failedContinue: ${CALLBACK_FAILED_CONTINUE} - beforeInviteUserToGroup: - enable: ${CALLBACK_ENABLE} - timeout: ${CALLBACK_TIMEOUT} - failedContinue: ${CALLBACK_FAILED_CONTINUE} - joinGroupAfter: - enable: ${CALLBACK_ENABLE} - timeout: ${CALLBACK_TIMEOUT} - failedContinue: ${CALLBACK_FAILED_CONTINUE} - setGroupInfoAfter: - enable: ${CALLBACK_ENABLE} - timeout: ${CALLBACK_TIMEOUT} - failedContinue: ${CALLBACK_FAILED_CONTINUE} - setGroupInfoBefore: - enable: ${CALLBACK_ENABLE} - timeout: ${CALLBACK_TIMEOUT} - failedContinue: ${CALLBACK_FAILED_CONTINUE} - revokeMsgAfter: - enable: ${CALLBACK_ENABLE} - timeout: ${CALLBACK_TIMEOUT} - failedContinue: ${CALLBACK_FAILED_CONTINUE} - addBlackBefore: - enable: ${CALLBACK_ENABLE} - timeout: ${CALLBACK_TIMEOUT} - failedContinue: ${CALLBACK_FAILED_CONTINUE} - addFriendAfter: - enable: ${CALLBACK_ENABLE} - timeout: ${CALLBACK_TIMEOUT} - failedContinue: ${CALLBACK_FAILED_CONTINUE} - addFriendAgreeBefore: - enable: ${CALLBACK_ENABLE} - timeout: ${CALLBACK_TIMEOUT} - failedContinue: ${CALLBACK_FAILED_CONTINUE} - deleteFriendAfter: - enable: ${CALLBACK_ENABLE} - timeout: ${CALLBACK_TIMEOUT} - failedContinue: ${CALLBACK_FAILED_CONTINUE} - importFriendsBefore: - enable: ${CALLBACK_ENABLE} - timeout: ${CALLBACK_TIMEOUT} - failedContinue: ${CALLBACK_FAILED_CONTINUE} - importFriendsAfter: - enable: ${CALLBACK_ENABLE} - timeout: ${CALLBACK_TIMEOUT} - failedContinue: ${CALLBACK_FAILED_CONTINUE} - removeBlackAfter: - enable: ${CALLBACK_ENABLE} - timeout: ${CALLBACK_TIMEOUT} - failedContinue: ${CALLBACK_FAILED_CONTINUE} -###################### Prometheus ###################### -# Prometheus configuration for various services -# The number of Prometheus ports per service needs to correspond to rpcPort -# The number of ports needs to be consistent with msg_transfer_service_num in script/path_info.sh -prometheus: - enable: ${PROMETHEUS_ENABLE} - grafanaUrl: ${GRAFANA_URL} - apiPrometheusPort: [${API_PROM_PORT}] - userPrometheusPort: [ ${USER_PROM_PORT} ] - friendPrometheusPort: [ ${FRIEND_PROM_PORT} ] - messagePrometheusPort: [ ${MESSAGE_PROM_PORT} ] - messageGatewayPrometheusPort: [ ${MSG_GATEWAY_PROM_PORT} ] - groupPrometheusPort: [ ${GROUP_PROM_PORT} ] - authPrometheusPort: [ ${AUTH_PROM_PORT} ] - pushPrometheusPort: [ ${PUSH_PROM_PORT} ] - conversationPrometheusPort: [ ${CONVERSATION_PROM_PORT} ] - rtcPrometheusPort: [ ${RTC_PROM_PORT} ] - thirdPrometheusPort: [ ${THIRD_PROM_PORT} ] - messageTransferPrometheusPort: [ ${MSG_TRANSFER_PROM_PORT} ] # List of ports - -###################### LocalCache configuration information ###################### -# topic: redis subscriber channel -# slotNum: number of slots, multiple slots can prevent too many keys from competing for a lock -# slotSize: number of slots, the number of cached keys per slot, the overall cache quantity is slotNum * slotSize -# successExpire: successful cache time seconds -# failedExpire: failed cache time seconds -# disable local caching and annotate topic, slotNum, and slotSize -localCache: - user: - topic: DELETE_CACHE_USER - slotNum: 100 - slotSize: 2000 - successExpire: 300 - failedExpire: 5 - - group: - topic: DELETE_CACHE_GROUP - slotNum: 100 - slotSize: 2000 - successExpire: 300 - failedExpire: 5 - - friend: - topic: DELETE_CACHE_FRIEND - slotNum: 100 - slotSize: 2000 - successExpire: 300 - failedExpire: 5 - - conversation: - topic: DELETE_CACHE_CONVERSATION - slotNum: 100 - slotSize: 2000 - successExpire: 300 - failedExpire: 5 \ No newline at end of file diff --git a/deployments/templates/email.tmpl b/deployments/templates/email.tmpl deleted file mode 100644 index 0385601d0..000000000 --- a/deployments/templates/email.tmpl +++ /dev/null @@ -1,16 +0,0 @@ -{{ define "email.to.html" }} -{{ range .Alerts }} - -

-

OpenIM Alert

-

Alert Program: Prometheus Alert

-

Severity Level: {{ .Labels.severity }}

-

Alert Type: {{ .Labels.alertname }}

-

Affected Host: {{ .Labels.instance }}

-

Affected Service: {{ .Labels.job }}

-

Alert Subject: {{ .Annotations.summary }}

-

Trigger Time: {{ .StartsAt.Format "2006-01-02 15:04:05" }}

-
- -{{ end }} -{{ end }} diff --git a/deployments/templates/env-template.yaml b/deployments/templates/env-template.yaml deleted file mode 100644 index b019f97e5..000000000 --- a/deployments/templates/env-template.yaml +++ /dev/null @@ -1,237 +0,0 @@ -# Copyright © 2024 OpenIM. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# ----------------------------------------------------------------------------- -# General Configuration -# This section contains general configuration options for the entire environment. -# These options can be set via environment variables. If both environment variables -# and settings in this .env file exist, the environment variables take precedence. -# ----------------------------------------------------------------------------- -# ========================== -# General Configuration -# ========================== -# These settings apply to the overall environment. - -# Data storage directory for persistent data. -# Example: DATA_DIR=/path/to/data -DATA_DIR=${DATA_DIR} - -# Docker image registry. Uncomment the preferred one. -# Options: ghcr.io/openimsdk, openim, registry.cn-hangzhou.aliyuncs.com/openimsdk -# IMAGE_REGISTRY="ghcr.io/openimsdk" -# IMAGE_REGISTRY="openim" -# IMAGE_REGISTRY="registry.cn-hangzhou.aliyuncs.com/openimsdk" -IMAGE_REGISTRY=${IMAGE_REGISTRY} - -# ====================================== -# ========= Network Configuration ====== -# ====================================== - -# Subnet for the Docker network. -# Default: DOCKER_BRIDGE_SUBNET=172.28.0.0/16 -DOCKER_BRIDGE_SUBNET=${DOCKER_BRIDGE_SUBNET} - -# Set and specify the IP addresses of some containers. Generally speaking, -# you do not need to modify these configurations to facilitate debugging -DOCKER_BRIDGE_GATEWAY=${DOCKER_BRIDGE_GATEWAY} -MONGO_NETWORK_ADDRESS=${MONGO_NETWORK_ADDRESS} -REDIS_NETWORK_ADDRESS=${REDIS_NETWORK_ADDRESS} -KAFKA_NETWORK_ADDRESS=${KAFKA_NETWORK_ADDRESS} -ZOOKEEPER_NETWORK_ADDRESS=${ZOOKEEPER_NETWORK_ADDRESS} -MINIO_NETWORK_ADDRESS=${MINIO_NETWORK_ADDRESS} -OPENIM_WEB_NETWORK_ADDRESS=${OPENIM_WEB_NETWORK_ADDRESS} -OPENIM_SERVER_NETWORK_ADDRESS=${OPENIM_SERVER_NETWORK_ADDRESS} -OPENIM_CHAT_NETWORK_ADDRESS=${OPENIM_CHAT_NETWORK_ADDRESS} -PROMETHEUS_NETWORK_ADDRESS=${PROMETHEUS_NETWORK_ADDRESS} -GRAFANA_NETWORK_ADDRESS=${GRAFANA_NETWORK_ADDRESS} -NODE_EXPORTER_NETWORK_ADDRESS=${NODE_EXPORTER_NETWORK_ADDRESS} -OPENIM_ADMIN_FRONT_NETWORK_ADDRESS=${OPENIM_ADMIN_FRONT_NETWORK_ADDRESS} -ALERT_MANAGER_NETWORK_ADDRESS=${ALERT_MANAGER_NETWORK_ADDRESS} - -# ============================================================================== -# Configuration Update Instructions -# ============================================================================== -# This header outlines the methods to update common variables in config.yaml and .env files. -# These instructions are vital for maintaining the OpenIM environment's configuration. -# -# METHOD 1: Regenerate All Configurations -# ---------------------------------------- -# Use this method to regenerate all configurations. -# Steps: -# 1. Delete existing config files: -# - openim-server/config/config.yaml -# - openim-chat/config/config.yaml -# 2. Modify the .env file as required. -# 3. Run 'docker compose up -d'. This will regenerate: -# - config/config.yaml -# -# METHOD 2: Modify Individual Configuration Files -# ----------------------------------------------- -# Use this method to update specific configuration files. -# Steps: -# 1. Modify the .env file as necessary. -# 2. Update the corresponding entries in: -# - config/config.yaml -# 3. Restart the services with 'docker compose up -d'. -# 4. Special Note: If you modify OPENIM_IP, API_OPENIM_PORT, or MINIO_PORT in .env, -# ensure to update the corresponding services and configurations accordingly. -# -# It is essential to follow these methods to ensure consistent and correct application behavior. -# ============================================================================== -# Local IP address of the service. Modify if necessary. -# Example: OPENIM_IP=172.28.0.1， -OPENIM_IP=${OPENIM_IP} - -# ----- ZooKeeper Configuration ----- -# Port for ZooKeeper service. -# Default: ZOOKEEPER_PORT=12181 -ZOOKEEPER_PORT=${ZOOKEEPER_PORT} - -# MongoDB service port configuration. -# Default: MONGO_PORT=37017 -MONGO_PORT=${MONGO_PORT} - -# Password for MongoDB admin user. Used for service authentication. -# Default: MONGO_PASSWORD=openIM123 -MONGO_PASSWORD=${MONGO_PASSWORD} - -# Username for a regular OpenIM user in MongoDB. -# Default: MONGO_OPENIM_USERNAME=openIM -MONGO_OPENIM_USERNAME=${MONGO_OPENIM_USERNAME} - -# Password for a regular OpenIM user in MongoDB. -# Default: MONGO_OPENIM_PASSWORD=openIM123456 -MONGO_OPENIM_PASSWORD=${MONGO_OPENIM_PASSWORD} - -# Specifies the database name to be used within MongoDB. -# Default: MONGO_DATABASE=openim_v3 -MONGO_DATABASE=${MONGO_DATABASE} - -MONGO_MAX_POOL_SIZE=${MONGO_MAX_POOL_SIZE} -# ----- Redis Configuration ----- - -# Port on which Redis in-memory data structure store is running. -# Default: REDIS_PORT=16379 -REDIS_PORT=${REDIS_PORT} - -# Password to authenticate with the Redis service. -# Default: REDIS_PASSWORD=openIM123 -REDIS_PASSWORD=${REDIS_PASSWORD} - -# Kafka username to authenticate with the Kafka service. -# KAFKA_USERNAME=${KAFKA_USERNAME} - -# Port on which Kafka distributed streaming platform is running. -# Default: KAFKA_PORT=19092 -KAFKA_PORT=${KAFKA_PORT} - -# Topic in Kafka for storing the latest messages in Redis. -# Default: KAFKA_LATESTMSG_REDIS_TOPIC=latestMsgToRedis -KAFKA_LATESTMSG_REDIS_TOPIC=${KAFKA_LATESTMSG_REDIS_TOPIC} - -# MINIO_PORT -# ---------- -# MINIO_PORT sets the port for the MinIO object storage service. -# Upon changing this port, the MinIO endpoint URLs in the config/config.yaml file must be updated -# to reflect this change. The endpoints include both the 'endpoint' and 'signEndpoint' -# under the MinIO configuration. -# -# Default: MINIO_PORT=10005 -MINIO_PORT=${MINIO_PORT} - -# Access key to authenticate with the MinIO service. -# Default: MINIO_ACCESS_KEY=root -# MINIO_ACCESS_KEY=${MINIO_ACCESS_KEY} - -# Secret key corresponding to the access key for MinIO authentication. -# Default: MINIO_SECRET_KEY=openIM123 -MINIO_SECRET_KEY=${MINIO_SECRET_KEY} - -# ----- Prometheus Configuration ----- -# Port on which Prometheus service is running. -# Default: PROMETHEUS_PORT=19090 -PROMETHEUS_PORT=${PROMETHEUS_PORT} - -# ----- Grafana Configuration ----- -# Port on which Grafana service is running. -# Default: GRAFANA_PORT=13000 -GRAFANA_PORT=${GRAFANA_PORT} - -# ====================================== -# ============ OpenIM Web =============== -# ====================================== - -# Port on which OpenIM web service is running. -# Default: OPENIM_WEB_PORT=11001 -OPENIM_WEB_PORT=${OPENIM_WEB_PORT} - -# ====================================== -# ========= OpenIM Server ============== -# ====================================== -# Port for the OpenIM WebSockets. -# Default: OPENIM_WS_PORT=10001 -OPENIM_WS_PORT=${OPENIM_WS_PORT} - -# API_OPENIM_PORT -# --------------- -# This variable defines the port on which the OpenIM API service will listen. -# When changing this port, it's essential to update the apiURL in the config.yaml file -# to ensure the API service is accessible at the new port. -# -# Default: API_OPENIM_PORT=10002 -API_OPENIM_PORT=${API_OPENIM_PORT} - -# ====================================== -# ========== OpenIM Chat =============== -# ====================================== - -# Branch name for OpenIM chat. -# Default: CHAT_IMAGE_VERSION=main -CHAT_IMAGE_VERSION=${CHAT_IMAGE_VERSION} - -# Port for the OpenIM chat API. -# Default: OPENIM_CHAT_API_PORT=10008 -OPENIM_CHAT_API_PORT=${OPENIM_CHAT_API_PORT} - -# Port for the OpenIM admin API. -# Default: OPENIM_ADMIN_API_PORT=10009 -OPENIM_ADMIN_API_PORT=${OPENIM_ADMIN_API_PORT} - -# ====================================== -# ========== OpenIM Admin ============== -# ====================================== - -# Branch name for OpenIM server. -# Default: SERVER_IMAGE_VERSION=main -SERVER_IMAGE_VERSION=${SERVER_IMAGE_VERSION} - -# Port for the node exporter. -# Default: NODE_EXPORTER_PORT=19100 -NODE_EXPORTER_PORT=${NODE_EXPORTER_PORT} - -# Port for the prometheus. -# Default: PROMETHEUS_PORT=19090 -PROMETHEUS_PORT=${PROMETHEUS_PORT} - -# Port for the grafana. -# Default: GRAFANA_PORT=13000 -GRAFANA_PORT=${GRAFANA_PORT} - -# Port for the admin front. -# Default: OPENIM_ADMIN_FRONT_PORT=11002 -OPENIM_ADMIN_FRONT_PORT=${OPENIM_ADMIN_FRONT_PORT} - -# Port for the alertmanager. -# Default: ALERT_MANAGER_PORT=19093 -ALERT_MANAGER_PORT=${ALERT_MANAGER_PORT} diff --git a/deployments/templates/helm-image.yaml b/deployments/templates/helm-image.yaml deleted file mode 100644 index ef64abf91..000000000 --- a/deployments/templates/helm-image.yaml +++ /dev/null @@ -1,103 +0,0 @@ -# Copyright © 2023 OpenIM. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -#This configuration file is used to override the use of the value.yaml variable. -#Currently, only the configuration with ingressName as nginx is provided. -#If it is another gateway such as istio or treafik, please modify the corresponding gateway requirements -image: - repository: ${IMAGE_REGISTRY}/openim-api - pullPolicy: Always - tag: "${SERVER_IMAGE_TAG}" -ingress: - enabled: true - className: "nginx" - annotations: - nginx.ingress.kubernetes.io/use-regex: "true" - nginx.ingress.kubernetes.io/rewrite-target: /$2 - hosts: - - host: openim.server.com # your hostname - paths: - - path: /api(/|$)(.*) - pathType: ImplementationSpecific - tls: - - secretName: webapitls #your hostname tls - hosts: - - openim.server.com - -openim-msggateway: - image: - repository: ${IMAGE_REGISTRY}/openim-msggateway - tag: "${SERVER_IMAGE_TAG}" - pullPolicy: Always - ingress: - enabled: true - className: "nginx" - annotations: - nginx.ingress.kubernetes.io/use-regex: "true" - nginx.ingress.kubernetes.io/rewrite-target: /$2 - hosts: - - host: openim.server.com # your hostname - paths: - - path: /msg_gateway(/|$)(.*) - pathType: ImplementationSpecific - tls: - - secretName: webapitls #your hostname tls - hosts: - - openim.server.com - -openim-msgtransfer: - image: - repository: ${IMAGE_REGISTRY}/openim-msgtransfer - tag: "${SERVER_IMAGE_TAG}" - pullPolicy: Always -openim-push: - image: - repository: ${IMAGE_REGISTRY}/openim-push - tag: "${SERVER_IMAGE_TAG}" - pullPolicy: Always -openim-rpc-auth: - image: - repository: ${IMAGE_REGISTRY}/openim-rpc-auth - tag: "${SERVER_IMAGE_TAG}" - pullPolicy: Always -openim-rpc-conversation: - image: - repository: ${IMAGE_REGISTRY}/openim-rpc-conversation - tag: "${SERVER_IMAGE_TAG}" - pullPolicy: Always -openim-rpc-friend: - image: - repository: ${IMAGE_REGISTRY}/openim-rpc-friend - tag: "${SERVER_IMAGE_TAG}" - pullPolicy: Always -openim-rpc-group: - image: - repository: ${IMAGE_REGISTRY}/openim-rpc-group - tag: "${SERVER_IMAGE_TAG}" - pullPolicy: Always -openim-rpc-msg: - image: - repository: ${IMAGE_REGISTRY}/openim-rpc-msg - tag: "${SERVER_IMAGE_TAG}" - pullPolicy: Always -openim-rpc-third: - image: - repository: ${IMAGE_REGISTRY}/openim-rpc-third - tag: "${SERVER_IMAGE_TAG}" - pullPolicy: Always -openim-rpc-user: - image: - repository: ${IMAGE_REGISTRY}/openim-rpc-user - tag: "${SERVER_IMAGE_TAG}" - pullPolicy: Always \ No newline at end of file diff --git a/deployments/templates/instance-down-rules.yml b/deployments/templates/instance-down-rules.yml deleted file mode 100644 index 7a6e9fda9..000000000 --- a/deployments/templates/instance-down-rules.yml +++ /dev/null @@ -1,36 +0,0 @@ -# Copyright © 2023 OpenIM. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -groups: - - name: instance_down - rules: - - alert: InstanceDown - expr: up == 0 - for: 1m - labels: - severity: critical - annotations: - summary: "Instance {{ $labels.instance }} down" - description: "{{ $labels.instance }} of job {{ $labels.job }} has been down for more than 1 minutes." - - - name: database_insert_failure_alerts - rules: - - alert: DatabaseInsertFailed - expr: (increase(msg_insert_redis_failed_total[5m]) > 0) or (increase(msg_insert_mongo_failed_total[5m]) > 0) - for: 1m - labels: - severity: critical - annotations: - summary: "Increase in MsgInsertRedisFailedCounter or MsgInsertMongoFailedCounter detected" - description: "Either MsgInsertRedisFailedCounter or MsgInsertMongoFailedCounter has increased in the last 5 minutes, indicating failures in message insert operations to Redis or MongoDB,maybe the redis or mongodb is crash." diff --git a/deployments/templates/notification.yaml b/deployments/templates/notification.yaml deleted file mode 100644 index 665c21261..000000000 --- a/deployments/templates/notification.yaml +++ /dev/null @@ -1,354 +0,0 @@ -# Copyright © 2023 OpenIM. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Determines if a message should be sent. If set to false, it triggers a silent sync without a message. If true, it requires triggering a conversation. -# For rpc notification, send twice: once as a message and once as a notification. -# The options field 'isNotification' indicates if it's a notification. -groupCreated: - isSendMsg: true - -# Reliability level of the message sending. -# Set to 1 to send only when online, 2 for guaranteed delivery. - reliabilityLevel: 1 - -# This setting is effective only when 'isSendMsg' is true. -# It controls whether to count unread messages. - unreadCount: false - -# Configuration for offline push notifications. - offlinePush: - # Enables or disables offline push notifications. - enable: false - - # Title for the notification when a group is created. - title: "create group title" - - # Description for the notification. - desc: "create group desc" - - # Additional information for the notification. - ext: "create group ext" - -# Content type is not added here. -# Content should use a JSON structure conforming to the protobuf format. - -groupInfoSet: - isSendMsg: false - reliabilityLevel: 1 - unreadCount: false - offlinePush: - enable: false - title: "groupInfoSet title" - desc: "groupInfoSet desc" - ext: "groupInfoSet ext" - - -joinGroupApplication: - isSendMsg: false - reliabilityLevel: 1 - unreadCount: false - offlinePush: - enable: false - title: "joinGroupApplication title" - desc: "joinGroupApplication desc" - ext: "joinGroupApplication ext" - -memberQuit: - isSendMsg: true - reliabilityLevel: 1 - unreadCount: false - offlinePush: - enable: false - title: "memberQuit title" - desc: "memberQuit desc" - ext: "memberQuit ext" - -groupApplicationAccepted: - isSendMsg: false - reliabilityLevel: 1 - unreadCount: false - offlinePush: - enable: false - title: "groupApplicationAccepted title" - desc: "groupApplicationAccepted desc" - ext: "groupApplicationAccepted ext" - -groupApplicationRejected: - isSendMsg: false - reliabilityLevel: 1 - unreadCount: false - offlinePush: - enable: false - title: " title" - desc: " desc" - ext: " ext" - - -groupOwnerTransferred: - isSendMsg: true - reliabilityLevel: 1 - unreadCount: false - offlinePush: - enable: false - title: "groupOwnerTransferred title" - desc: "groupOwnerTransferred desc" - ext: "groupOwnerTransferred ext" - -memberKicked: - isSendMsg: true - reliabilityLevel: 1 - unreadCount: false - offlinePush: - enable: false - title: "memberKicked title" - desc: "memberKicked desc" - ext: "memberKicked ext" - -memberInvited: - isSendMsg: true - reliabilityLevel: 1 - unreadCount: false - offlinePush: - enable: false - title: "memberInvited title" - desc: "memberInvited desc" - ext: "memberInvited ext" - -memberEnter: - isSendMsg: true - reliabilityLevel: 1 - unreadCount: false - offlinePush: - enable: false - title: "memberEnter title" - desc: "memberEnter desc" - ext: "memberEnter ext" - -groupDismissed: - isSendMsg: true - reliabilityLevel: 1 - unreadCount: false - offlinePush: - enable: false - title: "groupDismissed title" - desc: "groupDismissed desc" - ext: "groupDismissed ext" - -groupMuted: - isSendMsg: true - reliabilityLevel: 1 - unreadCount: false - offlinePush: - enable: false - title: "groupMuted title" - desc: "groupMuted desc" - ext: "groupMuted ext" - -groupCancelMuted: - isSendMsg: true - reliabilityLevel: 1 - unreadCount: false - offlinePush: - enable: false - title: "groupCancelMuted title" - desc: "groupCancelMuted desc" - ext: "groupCancelMuted ext" - defaultTips: - tips: "group Cancel Muted" - - -groupMemberMuted: - isSendMsg: true - reliabilityLevel: 1 - unreadCount: false - offlinePush: - enable: false - title: "groupMemberMuted title" - desc: "groupMemberMuted desc" - ext: "groupMemberMuted ext" - -groupMemberCancelMuted: - isSendMsg: true - reliabilityLevel: 1 - unreadCount: false - offlinePush: - enable: false - title: "groupMemberCancelMuted title" - desc: "groupMemberCancelMuted desc" - ext: "groupMemberCancelMuted ext" - -groupMemberInfoSet: - isSendMsg: false - reliabilityLevel: 1 - unreadCount: false - offlinePush: - enable: false - title: "groupMemberInfoSet title" - desc: "groupMemberInfoSet desc" - ext: "groupMemberInfoSet ext" - -groupInfoSetAnnouncement: - isSendMsg: true - reliabilityLevel: 1 - unreadCount: false - offlinePush: - enable: false - title: "groupInfoSetAnnouncement title" - desc: "groupInfoSetAnnouncement desc" - ext: "groupInfoSetAnnouncement ext" - - -groupInfoSetName: - isSendMsg: true - reliabilityLevel: 1 - unreadCount: false - offlinePush: - enable: false - title: "groupInfoSetName title" - desc: "groupInfoSetName desc" - ext: "groupInfoSetName ext" - - -#############################friend################################# -friendApplicationAdded: - isSendMsg: false - reliabilityLevel: 1 - unreadCount: false - offlinePush: - enable: false - title: "Somebody applies to add you as a friend" - desc: "Somebody applies to add you as a friend" - ext: "Somebody applies to add you as a friend" - -friendApplicationApproved: - isSendMsg: true - reliabilityLevel: 1 - unreadCount: false - offlinePush: - enable: true - title: "Someone applies to add your friend application" - desc: "Someone applies to add your friend application" - ext: "Someone applies to add your friend application" - -friendApplicationRejected: - isSendMsg: false - reliabilityLevel: 1 - unreadCount: false - offlinePush: - enable: true - title: "Someone rejected your friend application" - desc: "Someone rejected your friend application" - ext: "Someone rejected your friend application" - -friendAdded: - isSendMsg: false - reliabilityLevel: 1 - unreadCount: false - offlinePush: - enable: true - title: "We have become friends" - desc: "We have become friends" - ext: "We have become friends" - -friendDeleted: - isSendMsg: false - reliabilityLevel: 1 - unreadCount: false - offlinePush: - enable: true - title: "deleted a friend" - desc: "deleted a friend" - ext: "deleted a friend" - -friendRemarkSet: - isSendMsg: false - reliabilityLevel: 1 - unreadCount: false - offlinePush: - enable: true - title: "Your friend's profile has been changed" - desc: "Your friend's profile has been changed" - ext: "Your friend's profile has been changed" - -blackAdded: - isSendMsg: false - reliabilityLevel: 1 - unreadCount: false - offlinePush: - enable: true - title: "blocked a user" - desc: "blocked a user" - ext: "blocked a user" - -blackDeleted: - isSendMsg: false - reliabilityLevel: 1 - unreadCount: false - offlinePush: - enable: true - title: "Remove a blocked user" - desc: "Remove a blocked user" - ext: "Remove a blocked user" - -friendInfoUpdated: - isSendMsg: false - reliabilityLevel: 1 - unreadCount: false - offlinePush: - enable: true - title: "friend info updated" - desc: "friend info updated" - ext: "friend info updated" - -#####################user######################### -userInfoUpdated: - isSendMsg: false - reliabilityLevel: 1 - unreadCount: false - offlinePush: - enable: true - title: "Remove a blocked user" - desc: "Remove a blocked user" - ext: "Remove a blocked user" - -userStatusChanged: - isSendMsg: false - reliabilityLevel: 1 - unreadCount: false - offlinePush: - enable: false - title: "user status changed" - desc: "user status changed" - ext: "user status changed" - -#####################conversation######################### -conversationChanged: - isSendMsg: false - reliabilityLevel: 1 - unreadCount: false - offlinePush: - enable: true - title: "conversation changed" - desc: "conversation changed" - ext: "conversation changed" - -conversationSetPrivate: - isSendMsg: true - reliabilityLevel: 1 - unreadCount: false - offlinePush: - enable: true - title: "burn after reading" - desc: "burn after reading" - ext: "burn after reading" diff --git a/deployments/templates/openim-api.yaml b/deployments/templates/openim-api.yaml deleted file mode 100644 index 15fae592c..000000000 --- a/deployments/templates/openim-api.yaml +++ /dev/null @@ -1,17 +0,0 @@ -# Copyright © 2023 OpenIM. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -name: openim-api -description: "This is a description for openim-api" -type: "service" diff --git a/deployments/templates/openim.service b/deployments/templates/openim.service deleted file mode 100644 index 030e44c32..000000000 --- a/deployments/templates/openim.service +++ /dev/null @@ -1,13 +0,0 @@ -[Unit] -Description=${SERVER_NAME} for OpenIM -Documentation=https://github.com/openimsdk/open-im-server/blob/main/deployment/README.md - -[Service] -WorkingDirectory=${OPENIM_DATA_DIR} -ExecStart=${OPENIM_INSTALL_DIR}/${SERVER_NAME} -c ${OPENIM_CONFIG_DIR} -Restart=always -RestartSec=5 -StartLimitInterval=0 - -[Install] -WantedBy=multi-user.target diff --git a/deployments/templates/openim.target b/deployments/templates/openim.target deleted file mode 100644 index 1d481ce31..000000000 --- a/deployments/templates/openim.target +++ /dev/null @@ -1,7 +0,0 @@ -[Unit] -Description=Unified target for OpenIM servers -Requires=openim-api.service openim-push.service openim-rpc-group.service openim-crontask.service openim-rpc-auth.service openim-rpc-msg.service openim-msggateway.service openim-rpc-conversation.service openim-rpc-third.service openim-msgtransfer.service openim-rpc-friend.service openim-rpc-user.service -After=openim-api.service openim-push.service openim-rpc-group.service openim-crontask.service openim-rpc-auth.service openim-rpc-msg.service openim-msggateway.service openim-rpc-conversation.service openim-rpc-third.service openim-msgtransfer.service openim-rpc-friend.service openim-rpc-user.service - -[Install] -WantedBy=multi-user.target diff --git a/deployments/templates/prometheus.yml b/deployments/templates/prometheus.yml deleted file mode 100644 index f1a551a62..000000000 --- a/deployments/templates/prometheus.yml +++ /dev/null @@ -1,99 +0,0 @@ -# Copyright © 2023 OpenIM. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# my global config -global: - scrape_interval: 15s # Set the scrape interval to every 15 seconds. Default is every 1 minute. - evaluation_interval: 15s # Evaluate rules every 15 seconds. The default is every 1 minute. - # scrape_timeout is set to the global default (10s). - -# Alertmanager configuration -alerting: - alertmanagers: - - static_configs: - - targets: ['${ALERT_MANAGER_ADDRESS}:${ALERT_MANAGER_PORT}'] - -# Load rules once and periodically evaluate them according to the global 'evaluation_interval'. -rule_files: - - "instance-down-rules.yml" -# - "first_rules.yml" -# - "second_rules.yml" - -# A scrape configuration containing exactly one endpoint to scrape: -# Here it's Prometheus itself. -scrape_configs: - # The job name is added as a label "job='job_name'"" to any timeseries scraped from this config. - # Monitored information captured by prometheus - - job_name: 'node-exporter' - static_configs: - - targets: [ '${NODE_EXPORTER_ADDRESS}:${NODE_EXPORTER_PORT}' ] - labels: - namespace: 'default' - - # prometheus fetches application services - - job_name: 'openimserver-openim-api' - static_configs: - - targets: [ '${DOCKER_BRIDGE_GATEWAY}:${API_PROM_PORT}' ] - labels: - namespace: 'default' - - job_name: 'openimserver-openim-msggateway' - static_configs: - - targets: [ '${DOCKER_BRIDGE_GATEWAY}:${MSG_GATEWAY_PROM_PORT}' ] - labels: - namespace: 'default' - - job_name: 'openimserver-openim-msgtransfer' - static_configs: - - targets: [ ${MSG_TRANSFER_PROM_ADDRESS_PORT} ] - labels: - namespace: 'default' - - job_name: 'openimserver-openim-push' - static_configs: - - targets: [ '${DOCKER_BRIDGE_GATEWAY}:${PUSH_PROM_PORT}' ] - labels: - namespace: 'default' - - job_name: 'openimserver-openim-rpc-auth' - static_configs: - - targets: [ '${DOCKER_BRIDGE_GATEWAY}:${AUTH_PROM_PORT}' ] - labels: - namespace: 'default' - - job_name: 'openimserver-openim-rpc-conversation' - static_configs: - - targets: [ '${DOCKER_BRIDGE_GATEWAY}:${CONVERSATION_PROM_PORT}' ] - labels: - namespace: 'default' - - job_name: 'openimserver-openim-rpc-friend' - static_configs: - - targets: [ '${DOCKER_BRIDGE_GATEWAY}:${FRIEND_PROM_PORT}' ] - labels: - namespace: 'default' - - job_name: 'openimserver-openim-rpc-group' - static_configs: - - targets: [ '${DOCKER_BRIDGE_GATEWAY}:${GROUP_PROM_PORT}' ] - labels: - namespace: 'default' - - job_name: 'openimserver-openim-rpc-msg' - static_configs: - - targets: [ '${DOCKER_BRIDGE_GATEWAY}:${MESSAGE_PROM_PORT}' ] - labels: - namespace: 'default' - - job_name: 'openimserver-openim-rpc-third' - static_configs: - - targets: [ '${DOCKER_BRIDGE_GATEWAY}:${THIRD_PROM_PORT}' ] - labels: - namespace: 'default' - - job_name: 'openimserver-openim-rpc-user' - static_configs: - - targets: [ '${DOCKER_BRIDGE_GATEWAY}:${USER_PROM_PORT}' ] - labels: - namespace: 'default' diff --git a/go.mod b/go.mod index cd32a118a..123e1ff91 100644 --- a/go.mod +++ b/go.mod @@ -15,7 +15,7 @@ require ( github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/mitchellh/mapstructure v1.5.0 github.com/openimsdk/protocol v0.0.72-alpha.63 - github.com/openimsdk/tools v0.0.50-alpha.50 + github.com/openimsdk/tools v0.0.50-alpha.51 github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 github.com/stretchr/testify v1.9.0 diff --git a/go.sum b/go.sum index 26ed22efb..3d25082ea 100644 --- a/go.sum +++ b/go.sum @@ -349,8 +349,8 @@ github.com/openimsdk/gomake v0.0.15-alpha.2 h1:5Q8yl8ezy2yx+q8/ucU/t4kJnDfCzNOrk github.com/openimsdk/gomake v0.0.15-alpha.2/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= github.com/openimsdk/protocol v0.0.72-alpha.63 h1:IyPBibEvwBtTmD8DSrlqcekfEXe74k4+KeeHsgdhGh0= github.com/openimsdk/protocol v0.0.72-alpha.63/go.mod h1:Iet+piS/jaS+kWWyj6EEr36mk4ISzIRYjoMSVA4dq2M= -github.com/openimsdk/tools v0.0.50-alpha.50 h1:+naDlvHcqJDj2NsCGnQd1LLQOET5IRPbrtmWbM/o7JQ= -github.com/openimsdk/tools v0.0.50-alpha.50/go.mod h1:muCtxguNJv8lFwLei27UASu2Nvg4ERSeN0R4K5tivk0= +github.com/openimsdk/tools v0.0.50-alpha.51 h1:M3dMUoHjggx5Ry6XSkK0FTSJmRQjjkSBpuzXiFzKtC4= +github.com/openimsdk/tools v0.0.50-alpha.51/go.mod h1:muCtxguNJv8lFwLei27UASu2Nvg4ERSeN0R4K5tivk0= github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= diff --git a/internal/api/init.go b/internal/api/init.go index 8fb6baff5..59b19ee89 100644 --- a/internal/api/init.go +++ b/internal/api/init.go @@ -26,7 +26,7 @@ import ( "syscall" "time" - "github.com/openimsdk/open-im-server/v3/pkg/common/config" + conf "github.com/openimsdk/open-im-server/v3/pkg/common/config" kdisc "github.com/openimsdk/open-im-server/v3/pkg/common/discoveryregister" "github.com/openimsdk/open-im-server/v3/pkg/common/prommetrics" "github.com/openimsdk/tools/discovery" @@ -41,9 +41,9 @@ import ( ) type Config struct { - API config.API - Share config.Share - Discovery config.Discovery + API conf.API + Share conf.Share + Discovery conf.Discovery RuntimeEnv string } @@ -86,7 +86,7 @@ func Start(ctx context.Context, index int, config *Config) error { return listener, port, nil } - if config.API.Prometheus.AutoSetPorts && config.Discovery.Enable != kdisc.Etcd { + if config.API.Prometheus.AutoSetPorts && config.Discovery.Enable != conf.ETCD { return errs.New("only etcd support autoSetPorts", "RegisterName", "api").Wrap() } diff --git a/internal/api/prometheus_discovery.go b/internal/api/prometheus_discovery.go index 6e33274be..5e1a9cae2 100644 --- a/internal/api/prometheus_discovery.go +++ b/internal/api/prometheus_discovery.go @@ -2,8 +2,10 @@ package api import ( "encoding/json" + "net/http" + "github.com/gin-gonic/gin" - "github.com/openimsdk/open-im-server/v3/pkg/common/discoveryregister" + conf "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/open-im-server/v3/pkg/common/prommetrics" "github.com/openimsdk/tools/apiresp" "github.com/openimsdk/tools/discovery" @@ -11,7 +13,6 @@ import ( "github.com/openimsdk/tools/errs" "github.com/openimsdk/tools/log" clientv3 "go.etcd.io/etcd/client/v3" - "net/http" ) type PrometheusDiscoveryApi struct { @@ -23,14 +24,14 @@ func NewPrometheusDiscoveryApi(config *Config, client discovery.SvcDiscoveryRegi api := &PrometheusDiscoveryApi{ config: config, } - if config.Discovery.Enable == discoveryregister.Etcd { + if config.Discovery.Enable == conf.ETCD { api.client = client.(*etcd.SvcDiscoveryRegistryImpl).GetClient() } return api } func (p *PrometheusDiscoveryApi) Enable(c *gin.Context) { - if p.config.Discovery.Enable != discoveryregister.Etcd { + if p.config.Discovery.Enable != conf.ETCD { c.JSON(http.StatusOK, []struct{}{}) c.Abort() } diff --git a/internal/msggateway/ws_server.go b/internal/msggateway/ws_server.go index 335556ae9..2e23262b1 100644 --- a/internal/msggateway/ws_server.go +++ b/internal/msggateway/ws_server.go @@ -3,15 +3,16 @@ package msggateway import ( "context" "fmt" - "github.com/openimsdk/open-im-server/v3/pkg/common/webhook" - "github.com/openimsdk/open-im-server/v3/pkg/rpccache" - pbAuth "github.com/openimsdk/protocol/auth" - "github.com/openimsdk/tools/mcontext" "net/http" "sync" "sync/atomic" "time" + "github.com/openimsdk/open-im-server/v3/pkg/common/webhook" + "github.com/openimsdk/open-im-server/v3/pkg/rpccache" + pbAuth "github.com/openimsdk/protocol/auth" + "github.com/openimsdk/tools/mcontext" + "github.com/go-playground/validator/v10" "github.com/openimsdk/open-im-server/v3/pkg/common/prommetrics" "github.com/openimsdk/open-im-server/v3/pkg/common/servererrs" diff --git a/internal/msgtransfer/init.go b/internal/msgtransfer/init.go index 5be5e107d..4c09bc4e2 100644 --- a/internal/msgtransfer/init.go +++ b/internal/msgtransfer/init.go @@ -37,7 +37,7 @@ import ( "github.com/openimsdk/tools/utils/datautil" "github.com/openimsdk/tools/utils/runtimeenv" - "github.com/openimsdk/open-im-server/v3/pkg/common/config" + conf "github.com/openimsdk/open-im-server/v3/pkg/common/config" discRegister "github.com/openimsdk/open-im-server/v3/pkg/common/discoveryregister" kdisc "github.com/openimsdk/open-im-server/v3/pkg/common/discoveryregister" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/controller" @@ -64,13 +64,13 @@ type MsgTransfer struct { } type Config struct { - MsgTransfer config.MsgTransfer - RedisConfig config.Redis - MongodbConfig config.Mongo - KafkaConfig config.Kafka - Share config.Share - WebhooksConfig config.Webhooks - Discovery config.Discovery + MsgTransfer conf.MsgTransfer + RedisConfig conf.Redis + MongodbConfig conf.Mongo + KafkaConfig conf.Kafka + Share conf.Share + WebhooksConfig conf.Webhooks + Discovery conf.Discovery } func Start(ctx context.Context, index int, config *Config) error { @@ -168,7 +168,7 @@ func (m *MsgTransfer) Start(index int, config *Config) error { return listener, port, nil } - if config.MsgTransfer.Prometheus.AutoSetPorts && config.Discovery.Enable != kdisc.Etcd { + if config.MsgTransfer.Prometheus.AutoSetPorts && config.Discovery.Enable != conf.ETCD { return errs.New("only etcd support autoSetPorts", "RegisterName", "api").Wrap() } diff --git a/internal/push/onlinepusher.go b/internal/push/onlinepusher.go index f1ff7fed5..23e68339c 100644 --- a/internal/push/onlinepusher.go +++ b/internal/push/onlinepusher.go @@ -2,14 +2,19 @@ package push import ( "context" + "errors" + "sync" + "github.com/openimsdk/protocol/msggateway" "github.com/openimsdk/protocol/sdkws" "github.com/openimsdk/tools/discovery" + "github.com/openimsdk/tools/errs" "github.com/openimsdk/tools/log" "github.com/openimsdk/tools/utils/datautil" "golang.org/x/sync/errgroup" "google.golang.org/grpc" - "sync" + + conf "github.com/openimsdk/open-im-server/v3/pkg/common/config" ) type OnlinePusher interface { @@ -37,15 +42,16 @@ func (u emptyOnlinePusher) GetOnlinePushFailedUserIDs(ctx context.Context, msg * } func NewOnlinePusher(disCov discovery.SvcDiscoveryRegistry, config *Config) OnlinePusher { - switch config.Discovery.Enable { - case "k8s": - return NewK8sStaticConsistentHash(disCov, config) - case "zookeeper": + + if config.runTimeEnv == conf.KUBERNETES { return NewDefaultAllNode(disCov, config) - case "etcd": + } + switch config.Discovery.Enable { + case conf.ETCD: return NewDefaultAllNode(disCov, config) default: - return newEmptyOnlinePusher() + log.ZError(context.Background(), "NewOnlinePusher is error", errs.Wrap(errors.New("unsupported discovery type")), "type", config.Discovery.Enable) + return nil } } diff --git a/internal/push/push.go b/internal/push/push.go index 7f14bced7..74a4a0a4a 100644 --- a/internal/push/push.go +++ b/internal/push/push.go @@ -10,6 +10,7 @@ import ( pbpush "github.com/openimsdk/protocol/push" "github.com/openimsdk/tools/db/redisutil" "github.com/openimsdk/tools/discovery" + "github.com/openimsdk/tools/utils/runtimeenv" "google.golang.org/grpc" ) @@ -32,6 +33,8 @@ type Config struct { LocalCacheConfig config.LocalCache Discovery config.Discovery FcmConfigPath string + + runTimeEnv string } func (p pushServer) PushMsg(ctx context.Context, req *pbpush.PushMsgReq) (*pbpush.PushMsgResp, error) { @@ -48,6 +51,8 @@ func (p pushServer) DelUserPushToken(ctx context.Context, } func Start(ctx context.Context, config *Config, client discovery.SvcDiscoveryRegistry, server *grpc.Server) error { + config.runTimeEnv = runtimeenv.PrintRuntimeEnvironment() + rdb, err := redisutil.NewRedisClient(ctx, config.RedisConfig.Build()) if err != nil { return err diff --git a/magefile_unix.go b/magefile_unix.go index ff6b6de4e..4bb0cc1a9 100644 --- a/magefile_unix.go +++ b/magefile_unix.go @@ -4,8 +4,9 @@ package main import ( - "github.com/openimsdk/gomake/mageutil" "syscall" + + "github.com/openimsdk/gomake/mageutil" ) func setMaxOpenFiles() error { diff --git a/pkg/common/config/constant.go b/pkg/common/config/constant.go index 9aeaedca9..f3fcc67ef 100644 --- a/pkg/common/config/constant.go +++ b/pkg/common/config/constant.go @@ -20,6 +20,7 @@ const ( MountConfigFilePath = "CONFIG_PATH" DeploymentType = "DEPLOYMENT_TYPE" KUBERNETES = "kubernetes" + ETCD = "etcd" ) const ( diff --git a/pkg/common/config/load_config_test.go b/pkg/common/config/load_config_test.go index 1552c77d2..e575b27a9 100644 --- a/pkg/common/config/load_config_test.go +++ b/pkg/common/config/load_config_test.go @@ -86,7 +86,7 @@ func TestLoadOpenIMThirdConfig(t *testing.T) { func TestTransferConfig(t *testing.T) { var tran MsgTransfer - err := LoadConfig("../../../config/openim-msgtransfer.yml", "IMENV_OPENIM-MSGTRANSFER", &tran) + err := Load("../../../config/openim-msgtransfer.yml", "IMENV_OPENIM-MSGTRANSFER", "", "source", &tran) assert.Nil(t, err) assert.Equal(t, true, tran.Prometheus.Enable) assert.Equal(t, true, tran.Prometheus.AutoSetPorts) diff --git a/pkg/common/discoveryregister/discoveryregister.go b/pkg/common/discoveryregister/discoveryregister.go index dde629308..ae4229e1b 100644 --- a/pkg/common/discoveryregister/discoveryregister.go +++ b/pkg/common/discoveryregister/discoveryregister.go @@ -19,6 +19,7 @@ import ( "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/tools/discovery" + "google.golang.org/grpc" "github.com/openimsdk/tools/discovery/kubernetes" @@ -26,20 +27,18 @@ import ( "github.com/openimsdk/tools/errs" ) -const ( - Etcd = "etcd" -) - // NewDiscoveryRegister creates a new service discovery and registry client based on the provided environment type. func NewDiscoveryRegister(discovery *config.Discovery, runtimeEnv string) (discovery.SvcDiscoveryRegistry, error) { - if runtimeEnv == "kubernetes" { - discovery.Enable = "kubernetes" + if runtimeEnv == config.KUBERNETES { + return kubernetes.NewKubernetesConnManager(discovery.Kubernetes.Namespace, + grpc.WithDefaultCallOptions( + grpc.MaxCallSendMsgSize(1024*1024*20), + ), + ) } switch discovery.Enable { - case "kubernetes": - return kubernetes.NewKubernetesConnManager(discovery.Kubernetes.Namespace) - case Etcd: + case config.ETCD: return etcd.NewSvcDiscoveryRegistry( discovery.Etcd.RootDirectory, discovery.Etcd.Address, diff --git a/pkg/common/discoveryregister/kubernetes/kubernetes.go b/pkg/common/discoveryregister/kubernetes/kubernetes.go index cf6a904be..459f20e0b 100644 --- a/pkg/common/discoveryregister/kubernetes/kubernetes.go +++ b/pkg/common/discoveryregister/kubernetes/kubernetes.go @@ -1,22 +1,10 @@ -// Copyright © 2023 OpenIM. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - package kubernetes import ( "context" "fmt" + "log" + "os" "sync" "time" @@ -35,6 +23,7 @@ type KubernetesConnManager struct { namespace string dialOptions []grpc.DialOption + rpcTargets map[string]string selfTarget string mu sync.RWMutex @@ -76,11 +65,14 @@ func (k *KubernetesConnManager) initializeConns(serviceName string) error { return fmt.Errorf("failed to get endpoints for service %s: %v", serviceName, err) } + // fmt.Println("Endpoints:", endpoints, "endpoints.Subsets:", endpoints.Subsets) + var conns []*grpc.ClientConn for _, subset := range endpoints.Subsets { for _, address := range subset.Addresses { target := fmt.Sprintf("%s:%d", address.IP, port) - conn, err := grpc.Dial(target, grpc.WithTransportCredentials(insecure.NewCredentials())) + // fmt.Println("IP target:", target) + conn, err := grpc.Dial(target, append(k.dialOptions, grpc.WithTransportCredentials(insecure.NewCredentials()))...) if err != nil { return fmt.Errorf("failed to dial endpoint %s: %v", target, err) } @@ -89,10 +81,8 @@ func (k *KubernetesConnManager) initializeConns(serviceName string) error { } k.mu.Lock() - defer k.mu.Unlock() k.connMap[serviceName] = conns - - // go k.watchEndpoints(serviceName) + k.mu.Unlock() return nil } @@ -100,23 +90,23 @@ func (k *KubernetesConnManager) initializeConns(serviceName string) error { // GetConns returns gRPC client connections for a given Kubernetes service name. func (k *KubernetesConnManager) GetConns(ctx context.Context, serviceName string, opts ...grpc.DialOption) ([]*grpc.ClientConn, error) { k.mu.RLock() - conns, exists := k.connMap[serviceName] - defer k.mu.RUnlock() + conns, exists := k.connMap[serviceName] + k.mu.RUnlock() if exists { return conns, nil } k.mu.Lock() - defer k.mu.Unlock() - // Check if another goroutine has already initialized the connections when we released the read lock conns, exists = k.connMap[serviceName] if exists { return conns, nil } + k.mu.Unlock() if err := k.initializeConns(serviceName); err != nil { + fmt.Println("Failed to initialize connections:", err) return nil, fmt.Errorf("failed to initialize connections for service %s: %v", serviceName, err) } @@ -125,26 +115,64 @@ func (k *KubernetesConnManager) GetConns(ctx context.Context, serviceName string // GetConn returns a single gRPC client connection for a given Kubernetes service name. func (k *KubernetesConnManager) GetConn(ctx context.Context, serviceName string, opts ...grpc.DialOption) (*grpc.ClientConn, error) { - port, err := k.getServicePort(serviceName) - if err != nil { - return nil, err - } + var target string - fmt.Println("SVC port:", port) + if k.rpcTargets[serviceName] == "" { + var err error + + svcPort, err := k.getServicePort(serviceName) + if err != nil { + return nil, err + } - target := fmt.Sprintf("%s.%s.svc.cluster.local:%d", serviceName, k.namespace, port) + target = fmt.Sprintf("%s.%s.svc.cluster.local:%d", serviceName, k.namespace, svcPort) - fmt.Println("SVC target:", target) + // fmt.Println("SVC target:", target) + } else { + target = k.rpcTargets[serviceName] + } return grpc.DialContext( ctx, target, - append([]grpc.DialOption{grpc.WithTransportCredentials(insecure.NewCredentials())}, k.dialOptions...)..., + append([]grpc.DialOption{ + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(1024*1024*10), grpc.MaxCallSendMsgSize(1024*1024*20)), + }, k.dialOptions...)..., ) } // GetSelfConnTarget returns the connection target for the current service. func (k *KubernetesConnManager) GetSelfConnTarget() string { + if k.selfTarget == "" { + hostName := os.Getenv("HOSTNAME") + + pod, err := k.clientset.CoreV1().Pods(k.namespace).Get(context.Background(), hostName, metav1.GetOptions{}) + if err != nil { + log.Printf("failed to get pod %s: %v \n", hostName, err) + } + + for pod.Status.PodIP == "" { + pod, err = k.clientset.CoreV1().Pods(k.namespace).Get(context.TODO(), hostName, metav1.GetOptions{}) + if err != nil { + log.Printf("Error getting pod: %v \n", err) + } + + time.Sleep(3 * time.Second) + } + + var selfPort int32 + + for _, port := range pod.Spec.Containers[0].Ports { + if port.ContainerPort != 10001 { + selfPort = port.ContainerPort + break + } + } + + k.selfTarget = fmt.Sprintf("%s:%d", pod.Status.PodIP, selfPort) + } + return k.selfTarget } @@ -175,6 +203,7 @@ func (k *KubernetesConnManager) Close() { func (k *KubernetesConnManager) Register(serviceName, host string, port int, opts ...grpc.DialOption) error { return nil } + func (k *KubernetesConnManager) UnRegister() error { return nil } @@ -184,6 +213,8 @@ func (k *KubernetesConnManager) GetUserIdHashGatewayHost(ctx context.Context, us } func (k *KubernetesConnManager) getServicePort(serviceName string) (int32, error) { + var svcPort int32 + svc, err := k.clientset.CoreV1().Services(k.namespace).Get(context.Background(), serviceName, metav1.GetOptions{}) if err != nil { fmt.Print("namespace:", k.namespace) @@ -194,7 +225,15 @@ func (k *KubernetesConnManager) getServicePort(serviceName string) (int32, error return 0, fmt.Errorf("service %s has no ports defined", serviceName) } - return svc.Spec.Ports[0].Port, nil + for _, port := range svc.Spec.Ports { + // fmt.Println(serviceName, " Now Get Port:", port.Port) + if port.Port != 10001 { + svcPort = port.Port + break + } + } + + return svcPort, nil } // watchEndpoints listens for changes in Pod resources. @@ -229,68 +268,3 @@ func (k *KubernetesConnManager) handleEndpointChange(obj interface{}) { fmt.Printf("Error initializing connections for %s: %v\n", serviceName, err) } } - -// ================= - -// initEndpoints initializes connections by fetching all available endpoints in the specified namespace. - -// func (k *KubernetesConnManager) initEndpoints() error { -// k.mu.Lock() -// defer k.mu.Unlock() - -// pods, err := k.clientset.CoreV1().Pods(k.namespace).List(context.TODO(), metav1.ListOptions{}) -// if err != nil { -// return fmt.Errorf("failed to list pods: %v", err) -// } - -// for _, pod := range pods.Items { -// if pod.Status.Phase == v1.PodRunning { -// target := fmt.Sprintf("%s:%d", address.IP, port) -// conn, err := grpc.Dial(target, grpc.WithTransportCredentials(insecure.NewCredentials())) -// conn, err := k.createGRPCConnection(pod) -// if err != nil { -// return fmt.Errorf("failed to create GRPC connection for pod %s: %v", pod.Name, err) -// } -// k.connMap[pod.Name] = append(k.connMap[pod.Name], conn) -// } -// } - -// return nil -// } - -// ----- - -// func (k *KubernetesConnManager) watchEndpoints1(serviceName string) { -// // watch for changes to the service's endpoints -// informerFactory := informers.NewSharedInformerFactory(k.clientset, time.Minute) -// endpointsInformer := informerFactory.Core().V1().Endpoints().Informer() - -// endpointsInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{ -// AddFunc: func(obj interface{}) { -// eps := obj.(*v1.Endpoints) -// if eps.Name == serviceName { -// k.initializeConns(serviceName) -// } -// }, -// UpdateFunc: func(oldObj, newObj interface{}) { -// eps := newObj.(*v1.Endpoints) -// if eps.Name == serviceName { -// k.initializeConns(serviceName) -// } -// }, -// DeleteFunc: func(obj interface{}) { -// eps := obj.(*v1.Endpoints) -// if eps.Name == serviceName { -// k.mu.Lock() -// defer k.mu.Unlock() -// for _, conn := range k.connMap[serviceName] { -// _ = conn.Close() -// } -// delete(k.connMap, serviceName) -// } -// }, -// }) - -// informerFactory.Start(wait.NeverStop) -// informerFactory.WaitForCacheSync(wait.NeverStop) -// } diff --git a/pkg/common/discoveryregister/zookeeper/doc.go b/pkg/common/discoveryregister/zookeeper/doc.go deleted file mode 100644 index 1c24d77ac..000000000 --- a/pkg/common/discoveryregister/zookeeper/doc.go +++ /dev/null @@ -1,15 +0,0 @@ -// Copyright © 2024 OpenIM. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package zookeeper // import "github.com/openimsdk/open-im-server/v3/pkg/common/discoveryregister/zookeeper" diff --git a/pkg/common/startrpc/start.go b/pkg/common/startrpc/start.go index f75a50541..259f4caed 100644 --- a/pkg/common/startrpc/start.go +++ b/pkg/common/startrpc/start.go @@ -26,7 +26,7 @@ import ( "syscall" "time" - "github.com/openimsdk/open-im-server/v3/pkg/common/config" + conf "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/tools/discovery/etcd" "github.com/openimsdk/tools/utils/datautil" "github.com/openimsdk/tools/utils/jsonutil" @@ -46,8 +46,8 @@ import ( ) // Start rpc server. -func Start[T any](ctx context.Context, discovery *config.Discovery, prometheusConfig *config.Prometheus, listenIP, - registerIP string, autoSetPorts bool, rpcPorts []int, index int, rpcRegisterName string, share *config.Share, config T, rpcFn func(ctx context.Context, +func Start[T any](ctx context.Context, discovery *conf.Discovery, prometheusConfig *conf.Prometheus, listenIP, + registerIP string, autoSetPorts bool, rpcPorts []int, index int, rpcRegisterName string, share *conf.Share, config T, rpcFn func(ctx context.Context, config T, client discovery.SvcDiscoveryRegistry, server *grpc.Server) error, options ...grpc.ServerOption) error { var ( @@ -84,7 +84,7 @@ func Start[T any](ctx context.Context, discovery *config.Discovery, prometheusCo return listener, port, nil } - if autoSetPorts && discovery.Enable != kdisc.Etcd { + if autoSetPorts && discovery.Enable != conf.ETCD { return errs.New("only etcd support autoSetPorts", "rpcRegisterName", rpcRegisterName).Wrap() } client, err := kdisc.NewDiscoveryRegister(discovery, runTimeEnv) diff --git a/tools/check-component/main.go b/tools/check-component/main.go index 130035456..15d8640c7 100644 --- a/tools/check-component/main.go +++ b/tools/check-component/main.go @@ -18,7 +18,7 @@ import ( "context" "flag" "fmt" - "io/ioutil" + "io" "log" "os" "path/filepath" @@ -38,10 +38,16 @@ import ( const maxRetry = 180 +const ( + MountConfigFilePath = "CONFIG_PATH" + DeploymentType = "DEPLOYMENT_TYPE" + KUBERNETES = "kubernetes" +) + func CheckZookeeper(ctx context.Context, config *config.ZooKeeper) error { // Temporary disable logging originalLogger := log.Default().Writer() - log.SetOutput(ioutil.Discard) + log.SetOutput(io.Discard) defer log.SetOutput(originalLogger) // Ensure logging is restored return zookeeper.Check(ctx, config.Address, config.Schema, zookeeper.WithUserNameAndPassword(config.Username, config.Password)) } diff --git a/tools/seq/internal/main.go b/tools/seq/internal/seq.go similarity index 98% rename from tools/seq/internal/main.go rename to tools/seq/internal/seq.go index 2bec5a8f1..62466670e 100644 --- a/tools/seq/internal/main.go +++ b/tools/seq/internal/seq.go @@ -5,25 +5,27 @@ import ( "context" "errors" "fmt" + "os" + "os/signal" + "path/filepath" + "strconv" + "strings" + "sync" + "sync/atomic" + "syscall" + "time" + "github.com/openimsdk/open-im-server/v3/pkg/common/cmd" "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database/mgo" "github.com/openimsdk/tools/db/mongoutil" "github.com/openimsdk/tools/db/redisutil" + "github.com/openimsdk/tools/utils/runtimeenv" "github.com/redis/go-redis/v9" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/mongo" "go.mongodb.org/mongo-driver/mongo/options" "gopkg.in/yaml.v3" - "os" - "os/signal" - "path/filepath" - "strconv" - "strings" - "sync" - "sync/atomic" - "syscall" - "time" ) const ( @@ -41,6 +43,10 @@ const ( ) func readConfig[T any](dir string, name string) (*T, error) { + if runtimeenv.PrintRuntimeEnvironment() == config.KUBERNETES { + dir = os.Getenv(config.MountConfigFilePath) + } + data, err := os.ReadFile(filepath.Join(dir, name)) if err != nil { return nil, err From b8024379aa9fbefeeb72f348c77b67189ec53df1 Mon Sep 17 00:00:00 2001 From: icey-yu <119291641+icey-yu@users.noreply.github.com> Date: Fri, 13 Dec 2024 14:49:18 +0800 Subject: [PATCH 075/199] fix: rpc panic recover (#2957) --- go.mod | 2 +- go.sum | 4 ++-- internal/push/push.go | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index 123e1ff91..dfb34e485 100644 --- a/go.mod +++ b/go.mod @@ -15,7 +15,7 @@ require ( github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/mitchellh/mapstructure v1.5.0 github.com/openimsdk/protocol v0.0.72-alpha.63 - github.com/openimsdk/tools v0.0.50-alpha.51 + github.com/openimsdk/tools v0.0.50-alpha.52 github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 github.com/stretchr/testify v1.9.0 diff --git a/go.sum b/go.sum index 3d25082ea..16423b772 100644 --- a/go.sum +++ b/go.sum @@ -349,8 +349,8 @@ github.com/openimsdk/gomake v0.0.15-alpha.2 h1:5Q8yl8ezy2yx+q8/ucU/t4kJnDfCzNOrk github.com/openimsdk/gomake v0.0.15-alpha.2/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= github.com/openimsdk/protocol v0.0.72-alpha.63 h1:IyPBibEvwBtTmD8DSrlqcekfEXe74k4+KeeHsgdhGh0= github.com/openimsdk/protocol v0.0.72-alpha.63/go.mod h1:Iet+piS/jaS+kWWyj6EEr36mk4ISzIRYjoMSVA4dq2M= -github.com/openimsdk/tools v0.0.50-alpha.51 h1:M3dMUoHjggx5Ry6XSkK0FTSJmRQjjkSBpuzXiFzKtC4= -github.com/openimsdk/tools v0.0.50-alpha.51/go.mod h1:muCtxguNJv8lFwLei27UASu2Nvg4ERSeN0R4K5tivk0= +github.com/openimsdk/tools v0.0.50-alpha.52 h1:SAxnn6xgHPcEBHTebNLFgvUqmxd4d2XpBBh9jHpUEvs= +github.com/openimsdk/tools v0.0.50-alpha.52/go.mod h1:muCtxguNJv8lFwLei27UASu2Nvg4ERSeN0R4K5tivk0= github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= diff --git a/internal/push/push.go b/internal/push/push.go index 74a4a0a4a..3247426e9 100644 --- a/internal/push/push.go +++ b/internal/push/push.go @@ -33,8 +33,8 @@ type Config struct { LocalCacheConfig config.LocalCache Discovery config.Discovery FcmConfigPath string - - runTimeEnv string + + runTimeEnv string } func (p pushServer) PushMsg(ctx context.Context, req *pbpush.PushMsgReq) (*pbpush.PushMsgResp, error) { From 7563affc64c6270454e2174c2032523095993956 Mon Sep 17 00:00:00 2001 From: icey-yu <119291641+icey-yu@users.noreply.github.com> Date: Fri, 13 Dec 2024 18:28:57 +0800 Subject: [PATCH 076/199] fix: Add a lead time for the token's issuance time. (#2914) --- internal/rpc/auth/auth.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/rpc/auth/auth.go b/internal/rpc/auth/auth.go index c9cf8d345..987ccd4ac 100644 --- a/internal/rpc/auth/auth.go +++ b/internal/rpc/auth/auth.go @@ -130,7 +130,7 @@ func (s *authServer) GetUserToken(ctx context.Context, req *pbauth.GetUserTokenR func (s *authServer) parseToken(ctx context.Context, tokensString string) (claims *tokenverify.Claims, err error) { claims, err = tokenverify.GetClaimFromToken(tokensString, authverify.Secret(s.config.Share.Secret)) if err != nil { - return nil, errs.Wrap(err) + return nil, err } isAdmin := authverify.IsManagerUserID(claims.UserID, s.config.Share.IMAdminUserID) if isAdmin { From 5a8f82d45947c719a5aa1ded18ac7d10726e6a80 Mon Sep 17 00:00:00 2001 From: chao <48119764+withchao@users.noreply.github.com> Date: Tue, 17 Dec 2024 14:22:59 +0800 Subject: [PATCH 077/199] fix: modifying other fields while setting IsPrivateChat does not take effect (#2972) * pb * fix: Modifying other fields while setting IsPrivateChat does not take effect --- internal/rpc/conversation/conversation.go | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/internal/rpc/conversation/conversation.go b/internal/rpc/conversation/conversation.go index 7345c965d..8ed1323d5 100644 --- a/internal/rpc/conversation/conversation.go +++ b/internal/rpc/conversation/conversation.go @@ -354,7 +354,15 @@ func (c *conversationServer) SetConversations(ctx context.Context, req *pbconver needUpdateUsersList = append(needUpdateUsersList, userID) } } + if len(m) != 0 && len(needUpdateUsersList) != 0 { + if err := c.conversationDatabase.SetUsersConversationFieldTx(ctx, needUpdateUsersList, &conversation, m); err != nil { + return nil, err + } + for _, v := range needUpdateUsersList { + c.conversationNotificationSender.ConversationChangeNotification(ctx, v, []string{req.Conversation.ConversationID}) + } + } if req.Conversation.IsPrivateChat != nil && req.Conversation.ConversationType != constant.ReadGroupChatType { var conversations []*dbModel.Conversation for _, ownerUserID := range req.UserIDs { @@ -372,16 +380,6 @@ func (c *conversationServer) SetConversations(ctx context.Context, req *pbconver c.conversationNotificationSender.ConversationSetPrivateNotification(ctx, userID, req.Conversation.UserID, req.Conversation.IsPrivateChat.Value, req.Conversation.ConversationID) } - } else { - if len(m) != 0 && len(needUpdateUsersList) != 0 { - if err := c.conversationDatabase.SetUsersConversationFieldTx(ctx, needUpdateUsersList, &conversation, m); err != nil { - return nil, err - } - - for _, v := range needUpdateUsersList { - c.conversationNotificationSender.ConversationChangeNotification(ctx, v, []string{req.Conversation.ConversationID}) - } - } } return &pbconversation.SetConversationsResp{}, nil From 1c35db760103a2435dda16e1daa82dbda4911dd0 Mon Sep 17 00:00:00 2001 From: icey-yu <119291641+icey-yu@users.noreply.github.com> Date: Tue, 17 Dec 2024 18:48:08 +0800 Subject: [PATCH 078/199] refactor: Refactor rpc call && auto gen rpc_call code (#2969) * refactor: rpcclient * chore: err * fix: err * fix: err * fix: err * feat: change api --- go.mod | 4 +- go.sum | 8 +- internal/api/auth.go | 15 +- internal/api/conversation.go | 29 ++- internal/api/friend.go | 49 ++--- internal/api/group.go | 71 +++--- internal/api/init.go | 16 +- internal/api/jssdk/jssdk.go | 44 ++-- internal/api/msg.go | 66 +++--- internal/api/router.go | 42 ++-- internal/api/statistics.go | 9 +- internal/api/third.go | 40 ++-- internal/api/user.go | 54 +++-- internal/msggateway/client.go | 3 +- internal/msggateway/hub_server.go | 3 - internal/msggateway/init.go | 2 +- internal/msggateway/message_handler.go | 38 ++-- internal/msggateway/online.go | 13 +- internal/msggateway/ws_server.go | 19 +- internal/msgtransfer/init.go | 20 +- .../msgtransfer/online_history_msg_handler.go | 34 +-- internal/push/push_handler.go | 38 ++-- internal/rpc/auth/auth.go | 7 +- internal/rpc/conversation/conversation.go | 56 +++-- internal/rpc/conversation/notification.go | 4 +- internal/rpc/group/cache.go | 8 +- internal/rpc/group/callback.go | 64 +++--- internal/rpc/group/convert.go | 8 +- internal/rpc/group/fill.go | 5 +- internal/rpc/group/group.go | 84 +++---- internal/rpc/group/notification.go | 39 ++-- internal/rpc/group/statistics.go | 8 +- internal/rpc/group/sync.go | 54 ++--- internal/rpc/msg/clear.go | 21 +- internal/rpc/msg/delete.go | 16 +- internal/rpc/msg/send.go | 19 +- internal/rpc/msg/server.go | 15 +- internal/rpc/msg/stream_msg.go | 10 +- internal/rpc/relation/black.go | 10 +- internal/rpc/relation/friend.go | 52 ++--- internal/rpc/relation/notification.go | 4 +- internal/rpc/third/log.go | 3 +- internal/rpc/third/third.go | 6 +- internal/rpc/user/notification.go | 5 +- internal/rpc/user/user.go | 16 +- pkg/common/startrpc/start.go | 5 + pkg/rpccache/conversation.go | 38 ++-- pkg/rpccache/friend.go | 12 +- pkg/rpccache/group.go | 24 +- pkg/rpccache/online.go | 34 ++- pkg/rpccache/user.go | 10 +- pkg/rpcclient/auth.go | 74 ------- pkg/rpcclient/conversation.go | 161 -------------- pkg/rpcclient/friend.go | 87 -------- pkg/rpcclient/group.go | 205 ------------------ pkg/rpcclient/init.go | 60 +++++ pkg/rpcclient/msg.go | 133 +----------- pkg/rpcclient/push.go | 52 ----- pkg/rpcclient/third.go | 47 ---- pkg/rpcclient/user.go | 146 +------------ 60 files changed, 719 insertions(+), 1500 deletions(-) delete mode 100644 pkg/rpcclient/auth.go delete mode 100644 pkg/rpcclient/conversation.go delete mode 100644 pkg/rpcclient/friend.go delete mode 100644 pkg/rpcclient/group.go create mode 100644 pkg/rpcclient/init.go delete mode 100644 pkg/rpcclient/push.go delete mode 100644 pkg/rpcclient/third.go diff --git a/go.mod b/go.mod index dfb34e485..4eaa18ccc 100644 --- a/go.mod +++ b/go.mod @@ -14,8 +14,8 @@ require ( github.com/gorilla/websocket v1.5.1 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/mitchellh/mapstructure v1.5.0 - github.com/openimsdk/protocol v0.0.72-alpha.63 - github.com/openimsdk/tools v0.0.50-alpha.52 + github.com/openimsdk/protocol v0.0.72-alpha.66 + github.com/openimsdk/tools v0.0.50-alpha.57 github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 github.com/stretchr/testify v1.9.0 diff --git a/go.sum b/go.sum index 16423b772..6e283eb66 100644 --- a/go.sum +++ b/go.sum @@ -347,10 +347,10 @@ github.com/onsi/gomega v1.25.0 h1:Vw7br2PCDYijJHSfBOWhov+8cAnUf8MfMaIOV323l6Y= github.com/onsi/gomega v1.25.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdMPGhM= github.com/openimsdk/gomake v0.0.15-alpha.2 h1:5Q8yl8ezy2yx+q8/ucU/t4kJnDfCzNOrkXcDACCqtyM= github.com/openimsdk/gomake v0.0.15-alpha.2/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= -github.com/openimsdk/protocol v0.0.72-alpha.63 h1:IyPBibEvwBtTmD8DSrlqcekfEXe74k4+KeeHsgdhGh0= -github.com/openimsdk/protocol v0.0.72-alpha.63/go.mod h1:Iet+piS/jaS+kWWyj6EEr36mk4ISzIRYjoMSVA4dq2M= -github.com/openimsdk/tools v0.0.50-alpha.52 h1:SAxnn6xgHPcEBHTebNLFgvUqmxd4d2XpBBh9jHpUEvs= -github.com/openimsdk/tools v0.0.50-alpha.52/go.mod h1:muCtxguNJv8lFwLei27UASu2Nvg4ERSeN0R4K5tivk0= +github.com/openimsdk/protocol v0.0.72-alpha.66 h1:5KoDY6M4T+pXg449ScF6hqeQ+WenBwNyUJn/t8W0oBQ= +github.com/openimsdk/protocol v0.0.72-alpha.66/go.mod h1:Iet+piS/jaS+kWWyj6EEr36mk4ISzIRYjoMSVA4dq2M= +github.com/openimsdk/tools v0.0.50-alpha.57 h1:oIKV6vYhqp7TRmZ6Pe+r9RNl1D5s7aB/kE9yQVEWcSY= +github.com/openimsdk/tools v0.0.50-alpha.57/go.mod h1:muCtxguNJv8lFwLei27UASu2Nvg4ERSeN0R4K5tivk0= github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= diff --git a/internal/api/auth.go b/internal/api/auth.go index f41b530bf..c7e7ff704 100644 --- a/internal/api/auth.go +++ b/internal/api/auth.go @@ -16,29 +16,28 @@ package api import ( "github.com/gin-gonic/gin" - "github.com/openimsdk/open-im-server/v3/pkg/rpcclient" "github.com/openimsdk/protocol/auth" "github.com/openimsdk/tools/a2r" ) -type AuthApi rpcclient.Auth +type AuthApi struct{} -func NewAuthApi(client rpcclient.Auth) AuthApi { - return AuthApi(client) +func NewAuthApi() AuthApi { + return AuthApi{} } func (o *AuthApi) GetAdminToken(c *gin.Context) { - a2r.Call(auth.AuthClient.GetAdminToken, o.Client, c) + a2r.CallV2(c, auth.GetAdminTokenCaller.Invoke) } func (o *AuthApi) GetUserToken(c *gin.Context) { - a2r.Call(auth.AuthClient.GetUserToken, o.Client, c) + a2r.CallV2(c, auth.GetUserTokenCaller.Invoke) } func (o *AuthApi) ParseToken(c *gin.Context) { - a2r.Call(auth.AuthClient.ParseToken, o.Client, c) + a2r.CallV2(c, auth.ParseTokenCaller.Invoke) } func (o *AuthApi) ForceLogout(c *gin.Context) { - a2r.Call(auth.AuthClient.ForceLogout, o.Client, c) + a2r.CallV2(c, auth.ForceLogoutCaller.Invoke) } diff --git a/internal/api/conversation.go b/internal/api/conversation.go index 8e3a3ca82..4004944e5 100644 --- a/internal/api/conversation.go +++ b/internal/api/conversation.go @@ -16,57 +16,56 @@ package api import ( "github.com/gin-gonic/gin" - "github.com/openimsdk/open-im-server/v3/pkg/rpcclient" "github.com/openimsdk/protocol/conversation" "github.com/openimsdk/tools/a2r" ) -type ConversationApi rpcclient.Conversation +type ConversationApi struct{} -func NewConversationApi(client rpcclient.Conversation) ConversationApi { - return ConversationApi(client) +func NewConversationApi() ConversationApi { + return ConversationApi{} } func (o *ConversationApi) GetAllConversations(c *gin.Context) { - a2r.Call(conversation.ConversationClient.GetAllConversations, o.Client, c) + a2r.CallV2(c, conversation.GetAllConversationsCaller.Invoke) } func (o *ConversationApi) GetSortedConversationList(c *gin.Context) { - a2r.Call(conversation.ConversationClient.GetSortedConversationList, o.Client, c) + a2r.CallV2(c, conversation.GetSortedConversationListCaller.Invoke) } func (o *ConversationApi) GetConversation(c *gin.Context) { - a2r.Call(conversation.ConversationClient.GetConversation, o.Client, c) + a2r.CallV2(c, conversation.GetConversationCaller.Invoke) } func (o *ConversationApi) GetConversations(c *gin.Context) { - a2r.Call(conversation.ConversationClient.GetConversations, o.Client, c) + a2r.CallV2(c, conversation.GetConversationsCaller.Invoke) } func (o *ConversationApi) SetConversations(c *gin.Context) { - a2r.Call(conversation.ConversationClient.SetConversations, o.Client, c) + a2r.CallV2(c, conversation.SetConversationsCaller.Invoke) } func (o *ConversationApi) GetConversationOfflinePushUserIDs(c *gin.Context) { - a2r.Call(conversation.ConversationClient.GetConversationOfflinePushUserIDs, o.Client, c) + a2r.CallV2(c, conversation.GetConversationOfflinePushUserIDsCaller.Invoke) } func (o *ConversationApi) GetFullOwnerConversationIDs(c *gin.Context) { - a2r.Call(conversation.ConversationClient.GetFullOwnerConversationIDs, o.Client, c) + a2r.CallV2(c, conversation.GetFullOwnerConversationIDsCaller.Invoke) } func (o *ConversationApi) GetIncrementalConversation(c *gin.Context) { - a2r.Call(conversation.ConversationClient.GetIncrementalConversation, o.Client, c) + a2r.CallV2(c, conversation.GetIncrementalConversationCaller.Invoke) } func (o *ConversationApi) GetOwnerConversation(c *gin.Context) { - a2r.Call(conversation.ConversationClient.GetOwnerConversation, o.Client, c) + a2r.CallV2(c, conversation.GetOwnerConversationCaller.Invoke) } func (o *ConversationApi) GetNotNotifyConversationIDs(c *gin.Context) { - a2r.Call(conversation.ConversationClient.GetNotNotifyConversationIDs, o.Client, c) + a2r.CallV2(c, conversation.GetNotNotifyConversationIDsCaller.Invoke) } func (o *ConversationApi) GetPinnedConversationIDs(c *gin.Context) { - a2r.Call(conversation.ConversationClient.GetPinnedConversationIDs, o.Client, c) + a2r.CallV2(c, conversation.GetPinnedConversationIDsCaller.Invoke) } diff --git a/internal/api/friend.go b/internal/api/friend.go index d000cccdd..4e08474fb 100644 --- a/internal/api/friend.go +++ b/internal/api/friend.go @@ -17,99 +17,98 @@ package api import ( "github.com/gin-gonic/gin" - "github.com/openimsdk/open-im-server/v3/pkg/rpcclient" "github.com/openimsdk/protocol/relation" "github.com/openimsdk/tools/a2r" ) -type FriendApi rpcclient.Friend +type FriendApi struct{} -func NewFriendApi(client rpcclient.Friend) FriendApi { - return FriendApi(client) +func NewFriendApi() FriendApi { + return FriendApi{} } func (o *FriendApi) ApplyToAddFriend(c *gin.Context) { - a2r.Call(relation.FriendClient.ApplyToAddFriend, o.Client, c) + a2r.CallV2(c, relation.ApplyToAddFriendCaller.Invoke) } func (o *FriendApi) RespondFriendApply(c *gin.Context) { - a2r.Call(relation.FriendClient.RespondFriendApply, o.Client, c) + a2r.CallV2(c, relation.RespondFriendApplyCaller.Invoke) } func (o *FriendApi) DeleteFriend(c *gin.Context) { - a2r.Call(relation.FriendClient.DeleteFriend, o.Client, c) + a2r.CallV2(c, relation.DeleteFriendCaller.Invoke) } func (o *FriendApi) GetFriendApplyList(c *gin.Context) { - a2r.Call(relation.FriendClient.GetPaginationFriendsApplyTo, o.Client, c) + a2r.CallV2(c, relation.GetPaginationFriendsApplyToCaller.Invoke) } func (o *FriendApi) GetDesignatedFriendsApply(c *gin.Context) { - a2r.Call(relation.FriendClient.GetDesignatedFriendsApply, o.Client, c) + a2r.CallV2(c, relation.GetDesignatedFriendsApplyCaller.Invoke) } func (o *FriendApi) GetSelfApplyList(c *gin.Context) { - a2r.Call(relation.FriendClient.GetPaginationFriendsApplyFrom, o.Client, c) + a2r.CallV2(c, relation.GetPaginationFriendsApplyFromCaller.Invoke) } func (o *FriendApi) GetFriendList(c *gin.Context) { - a2r.Call(relation.FriendClient.GetPaginationFriends, o.Client, c) + a2r.CallV2(c, relation.GetPaginationFriendsCaller.Invoke) } func (o *FriendApi) GetDesignatedFriends(c *gin.Context) { - a2r.Call(relation.FriendClient.GetDesignatedFriends, o.Client, c) + a2r.CallV2(c, relation.GetDesignatedFriendsCaller.Invoke) } func (o *FriendApi) SetFriendRemark(c *gin.Context) { - a2r.Call(relation.FriendClient.SetFriendRemark, o.Client, c) + a2r.CallV2(c, relation.SetFriendRemarkCaller.Invoke) } func (o *FriendApi) AddBlack(c *gin.Context) { - a2r.Call(relation.FriendClient.AddBlack, o.Client, c) + a2r.CallV2(c, relation.AddBlackCaller.Invoke) } func (o *FriendApi) GetPaginationBlacks(c *gin.Context) { - a2r.Call(relation.FriendClient.GetPaginationBlacks, o.Client, c) + a2r.CallV2(c, relation.GetPaginationBlacksCaller.Invoke) } func (o *FriendApi) GetSpecifiedBlacks(c *gin.Context) { - a2r.Call(relation.FriendClient.GetSpecifiedBlacks, o.Client, c) + a2r.CallV2(c, relation.GetSpecifiedBlacksCaller.Invoke) } func (o *FriendApi) RemoveBlack(c *gin.Context) { - a2r.Call(relation.FriendClient.RemoveBlack, o.Client, c) + a2r.CallV2(c, relation.RemoveBlackCaller.Invoke) } func (o *FriendApi) ImportFriends(c *gin.Context) { - a2r.Call(relation.FriendClient.ImportFriends, o.Client, c) + a2r.CallV2(c, relation.ImportFriendsCaller.Invoke) } func (o *FriendApi) IsFriend(c *gin.Context) { - a2r.Call(relation.FriendClient.IsFriend, o.Client, c) + a2r.CallV2(c, relation.IsFriendCaller.Invoke) } func (o *FriendApi) GetFriendIDs(c *gin.Context) { - a2r.Call(relation.FriendClient.GetFriendIDs, o.Client, c) + a2r.CallV2(c, relation.GetFriendIDsCaller.Invoke) } func (o *FriendApi) GetSpecifiedFriendsInfo(c *gin.Context) { - a2r.Call(relation.FriendClient.GetSpecifiedFriendsInfo, o.Client, c) + a2r.CallV2(c, relation.GetSpecifiedFriendsInfoCaller.Invoke) } func (o *FriendApi) UpdateFriends(c *gin.Context) { - a2r.Call(relation.FriendClient.UpdateFriends, o.Client, c) + a2r.CallV2(c, relation.UpdateFriendsCaller.Invoke) } func (o *FriendApi) GetIncrementalFriends(c *gin.Context) { - a2r.Call(relation.FriendClient.GetIncrementalFriends, o.Client, c) + a2r.CallV2(c, relation.GetIncrementalFriendsCaller.Invoke) } // GetIncrementalBlacks is temporarily unused. // Deprecated: This function is currently unused and may be removed in future versions. func (o *FriendApi) GetIncrementalBlacks(c *gin.Context) { - a2r.Call(relation.FriendClient.GetIncrementalBlacks, o.Client, c) + a2r.CallV2(c, relation.GetIncrementalBlacksCaller.Invoke) } func (o *FriendApi) GetFullFriendUserIDs(c *gin.Context) { - a2r.Call(relation.FriendClient.GetFullFriendUserIDs, o.Client, c) + a2r.CallV2(c, relation.GetFullFriendUserIDsCaller.Invoke) } diff --git a/internal/api/group.go b/internal/api/group.go index 9c35da708..784197fd8 100644 --- a/internal/api/group.go +++ b/internal/api/group.go @@ -16,113 +16,112 @@ package api import ( "github.com/gin-gonic/gin" - "github.com/openimsdk/open-im-server/v3/pkg/rpcclient" "github.com/openimsdk/protocol/group" "github.com/openimsdk/tools/a2r" ) -type GroupApi rpcclient.Group +type GroupApi struct{} -func NewGroupApi(client rpcclient.Group) GroupApi { - return GroupApi(client) +func NewGroupApi() GroupApi { + return GroupApi{} } func (o *GroupApi) CreateGroup(c *gin.Context) { - a2r.Call(group.GroupClient.CreateGroup, o.Client, c) + a2r.CallV2(c, group.CreateGroupCaller.Invoke) } func (o *GroupApi) SetGroupInfo(c *gin.Context) { - a2r.Call(group.GroupClient.SetGroupInfo, o.Client, c) + a2r.CallV2(c, group.SetGroupInfoCaller.Invoke) } func (o *GroupApi) SetGroupInfoEx(c *gin.Context) { - a2r.Call(group.GroupClient.SetGroupInfoEx, o.Client, c) + a2r.CallV2(c, group.SetGroupInfoExCaller.Invoke) } func (o *GroupApi) JoinGroup(c *gin.Context) { - a2r.Call(group.GroupClient.JoinGroup, o.Client, c) + a2r.CallV2(c, group.JoinGroupCaller.Invoke) } func (o *GroupApi) QuitGroup(c *gin.Context) { - a2r.Call(group.GroupClient.QuitGroup, o.Client, c) + a2r.CallV2(c, group.QuitGroupCaller.Invoke) } func (o *GroupApi) ApplicationGroupResponse(c *gin.Context) { - a2r.Call(group.GroupClient.GroupApplicationResponse, o.Client, c) + a2r.CallV2(c, group.GroupApplicationResponseCaller.Invoke) } func (o *GroupApi) TransferGroupOwner(c *gin.Context) { - a2r.Call(group.GroupClient.TransferGroupOwner, o.Client, c) + a2r.CallV2(c, group.TransferGroupOwnerCaller.Invoke) } func (o *GroupApi) GetRecvGroupApplicationList(c *gin.Context) { - a2r.Call(group.GroupClient.GetGroupApplicationList, o.Client, c) + a2r.CallV2(c, group.GetGroupApplicationListCaller.Invoke) } func (o *GroupApi) GetUserReqGroupApplicationList(c *gin.Context) { - a2r.Call(group.GroupClient.GetUserReqApplicationList, o.Client, c) + a2r.CallV2(c, group.GetUserReqApplicationListCaller.Invoke) } func (o *GroupApi) GetGroupUsersReqApplicationList(c *gin.Context) { - a2r.Call(group.GroupClient.GetGroupUsersReqApplicationList, o.Client, c) + a2r.CallV2(c, group.GetGroupUsersReqApplicationListCaller.Invoke) } func (o *GroupApi) GetSpecifiedUserGroupRequestInfo(c *gin.Context) { - a2r.Call(group.GroupClient.GetSpecifiedUserGroupRequestInfo, o.Client, c) + a2r.CallV2(c, group.GetSpecifiedUserGroupRequestInfoCaller.Invoke) } func (o *GroupApi) GetGroupsInfo(c *gin.Context) { - a2r.Call(group.GroupClient.GetGroupsInfo, o.Client, c) + a2r.CallV2(c, group.GetGroupsInfoCaller.Invoke) //a2r.Call(group.GroupClient.GetGroupsInfo, o.Client, c, a2r.NewNilReplaceOption(group.GroupClient.GetGroupsInfo)) } func (o *GroupApi) KickGroupMember(c *gin.Context) { - a2r.Call(group.GroupClient.KickGroupMember, o.Client, c) + a2r.CallV2(c, group.KickGroupMemberCaller.Invoke) } func (o *GroupApi) GetGroupMembersInfo(c *gin.Context) { - a2r.Call(group.GroupClient.GetGroupMembersInfo, o.Client, c) + a2r.CallV2(c, group.GetGroupMembersInfoCaller.Invoke) //a2r.Call(group.GroupClient.GetGroupMembersInfo, o.Client, c, a2r.NewNilReplaceOption(group.GroupClient.GetGroupMembersInfo)) } func (o *GroupApi) GetGroupMemberList(c *gin.Context) { - a2r.Call(group.GroupClient.GetGroupMemberList, o.Client, c) + a2r.CallV2(c, group.GetGroupMemberListCaller.Invoke) } func (o *GroupApi) InviteUserToGroup(c *gin.Context) { - a2r.Call(group.GroupClient.InviteUserToGroup, o.Client, c) + a2r.CallV2(c, group.InviteUserToGroupCaller.Invoke) } func (o *GroupApi) GetJoinedGroupList(c *gin.Context) { - a2r.Call(group.GroupClient.GetJoinedGroupList, o.Client, c) + a2r.CallV2(c, group.GetJoinedGroupListCaller.Invoke) } func (o *GroupApi) DismissGroup(c *gin.Context) { - a2r.Call(group.GroupClient.DismissGroup, o.Client, c) + a2r.CallV2(c, group.DismissGroupCaller.Invoke) } func (o *GroupApi) MuteGroupMember(c *gin.Context) { - a2r.Call(group.GroupClient.MuteGroupMember, o.Client, c) + a2r.CallV2(c, group.MuteGroupMemberCaller.Invoke) } func (o *GroupApi) CancelMuteGroupMember(c *gin.Context) { - a2r.Call(group.GroupClient.CancelMuteGroupMember, o.Client, c) + a2r.CallV2(c, group.CancelMuteGroupMemberCaller.Invoke) } func (o *GroupApi) MuteGroup(c *gin.Context) { - a2r.Call(group.GroupClient.MuteGroup, o.Client, c) + a2r.CallV2(c, group.MuteGroupCaller.Invoke) } func (o *GroupApi) CancelMuteGroup(c *gin.Context) { - a2r.Call(group.GroupClient.CancelMuteGroup, o.Client, c) + a2r.CallV2(c, group.CancelMuteGroupCaller.Invoke) } func (o *GroupApi) SetGroupMemberInfo(c *gin.Context) { - a2r.Call(group.GroupClient.SetGroupMemberInfo, o.Client, c) + a2r.CallV2(c, group.SetGroupMemberInfoCaller.Invoke) } func (o *GroupApi) GetGroupAbstractInfo(c *gin.Context) { - a2r.Call(group.GroupClient.GetGroupAbstractInfo, o.Client, c) + a2r.CallV2(c, group.GetGroupAbstractInfoCaller.Invoke) } // func (g *Group) SetGroupMemberNickname(c *gin.Context) { @@ -134,33 +133,33 @@ func (o *GroupApi) GetGroupAbstractInfo(c *gin.Context) { //} func (o *GroupApi) GroupCreateCount(c *gin.Context) { - a2r.Call(group.GroupClient.GroupCreateCount, o.Client, c) + a2r.CallV2(c, group.GroupCreateCountCaller.Invoke) } func (o *GroupApi) GetGroups(c *gin.Context) { - a2r.Call(group.GroupClient.GetGroups, o.Client, c) + a2r.CallV2(c, group.GetGroupsCaller.Invoke) } func (o *GroupApi) GetGroupMemberUserIDs(c *gin.Context) { - a2r.Call(group.GroupClient.GetGroupMemberUserIDs, o.Client, c) + a2r.CallV2(c, group.GetGroupMemberUserIDsCaller.Invoke) } func (o *GroupApi) GetIncrementalJoinGroup(c *gin.Context) { - a2r.Call(group.GroupClient.GetIncrementalJoinGroup, o.Client, c) + a2r.CallV2(c, group.GetIncrementalJoinGroupCaller.Invoke) } func (o *GroupApi) GetIncrementalGroupMember(c *gin.Context) { - a2r.Call(group.GroupClient.GetIncrementalGroupMember, o.Client, c) + a2r.CallV2(c, group.GetIncrementalGroupMemberCaller.Invoke) } func (o *GroupApi) GetIncrementalGroupMemberBatch(c *gin.Context) { - a2r.Call(group.GroupClient.BatchGetIncrementalGroupMember, o.Client, c) + a2r.CallV2(c, group.BatchGetIncrementalGroupMemberCaller.Invoke) } func (o *GroupApi) GetFullGroupMemberUserIDs(c *gin.Context) { - a2r.Call(group.GroupClient.GetFullGroupMemberUserIDs, o.Client, c) + a2r.CallV2(c, group.GetFullGroupMemberUserIDsCaller.Invoke) } func (o *GroupApi) GetFullJoinGroupIDs(c *gin.Context) { - a2r.Call(group.GroupClient.GetFullJoinGroupIDs, o.Client, c) + a2r.CallV2(c, group.GetFullJoinGroupIDsCaller.Invoke) } diff --git a/internal/api/init.go b/internal/api/init.go index 59b19ee89..fcc69dc68 100644 --- a/internal/api/init.go +++ b/internal/api/init.go @@ -29,15 +29,18 @@ import ( conf "github.com/openimsdk/open-im-server/v3/pkg/common/config" kdisc "github.com/openimsdk/open-im-server/v3/pkg/common/discoveryregister" "github.com/openimsdk/open-im-server/v3/pkg/common/prommetrics" - "github.com/openimsdk/tools/discovery" + "github.com/openimsdk/open-im-server/v3/pkg/rpcclient" "github.com/openimsdk/tools/discovery/etcd" "github.com/openimsdk/tools/errs" "github.com/openimsdk/tools/log" + "github.com/openimsdk/tools/mw" "github.com/openimsdk/tools/system/program" "github.com/openimsdk/tools/utils/datautil" "github.com/openimsdk/tools/utils/jsonutil" "github.com/openimsdk/tools/utils/network" "github.com/openimsdk/tools/utils/runtimeenv" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" ) type Config struct { @@ -56,13 +59,14 @@ func Start(ctx context.Context, index int, config *Config) error { config.RuntimeEnv = runtimeenv.PrintRuntimeEnvironment() - var client discovery.SvcDiscoveryRegistry - - // Determine whether zk is passed according to whether it is a clustered deployment - client, err = kdisc.NewDiscoveryRegister(&config.Discovery, config.RuntimeEnv) + client, err := kdisc.NewDiscoveryRegister(&config.Discovery, config.RuntimeEnv) if err != nil { return errs.WrapMsg(err, "failed to register discovery service") } + client.AddOption(mw.GrpcClient(), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithDefaultServiceConfig(fmt.Sprintf(`{"LoadBalancingPolicy": "%s"}`, "round_robin"))) + if err = rpcclient.InitRpcCaller(client, config.Discovery.RpcService); err != nil { + return err + } var ( netDone = make(chan struct{}, 1) @@ -90,7 +94,7 @@ func Start(ctx context.Context, index int, config *Config) error { return errs.New("only etcd support autoSetPorts", "RegisterName", "api").Wrap() } - router := newGinRouter(client, config, client) + router := newGinRouter(client, config) if config.API.Prometheus.Enable { var ( listener net.Listener diff --git a/internal/api/jssdk/jssdk.go b/internal/api/jssdk/jssdk.go index 2fcbf5ec6..409fcbf79 100644 --- a/internal/api/jssdk/jssdk.go +++ b/internal/api/jssdk/jssdk.go @@ -2,6 +2,8 @@ package jssdk import ( "context" + "sort" + "github.com/gin-gonic/gin" "github.com/openimsdk/protocol/conversation" "github.com/openimsdk/protocol/group" @@ -12,7 +14,6 @@ import ( "github.com/openimsdk/protocol/user" "github.com/openimsdk/tools/mcontext" "github.com/openimsdk/tools/utils/datautil" - "sort" ) const ( @@ -20,22 +21,11 @@ const ( defaultGetActiveConversation = 100 ) -func NewJSSdkApi(user user.UserClient, friend relation.FriendClient, group group.GroupClient, msg msg.MsgClient, conv conversation.ConversationClient) *JSSdk { - return &JSSdk{ - user: user, - friend: friend, - group: group, - msg: msg, - conv: conv, - } +func NewJSSdkApi() *JSSdk { + return &JSSdk{} } type JSSdk struct { - user user.UserClient - friend relation.FriendClient - group group.GroupClient - msg msg.MsgClient - conv conversation.ConversationClient } func (x *JSSdk) GetActiveConversations(c *gin.Context) { @@ -67,11 +57,11 @@ func (x *JSSdk) fillConversations(ctx context.Context, conversations []*jssdk.Co groupMap map[string]*sdkws.GroupInfo ) if len(userIDs) > 0 { - users, err := field(ctx, x.user.GetDesignateUsers, &user.GetDesignateUsersReq{UserIDs: userIDs}, (*user.GetDesignateUsersResp).GetUsersInfo) + users, err := field(ctx, user.GetDesignateUsersCaller.Invoke, &user.GetDesignateUsersReq{UserIDs: userIDs}, (*user.GetDesignateUsersResp).GetUsersInfo) if err != nil { return err } - friends, err := field(ctx, x.friend.GetFriendInfo, &relation.GetFriendInfoReq{OwnerUserID: conversations[0].Conversation.OwnerUserID, FriendUserIDs: userIDs}, (*relation.GetFriendInfoResp).GetFriendInfos) + friends, err := field(ctx, relation.GetFriendInfoCaller.Invoke, &relation.GetFriendInfoReq{OwnerUserID: conversations[0].Conversation.OwnerUserID, FriendUserIDs: userIDs}, (*relation.GetFriendInfoResp).GetFriendInfos) if err != nil { return err } @@ -79,7 +69,7 @@ func (x *JSSdk) fillConversations(ctx context.Context, conversations []*jssdk.Co friendMap = datautil.SliceToMap(friends, (*relation.FriendInfoOnly).GetFriendUserID) } if len(groupIDs) > 0 { - resp, err := x.group.GetGroupsInfo(ctx, &group.GetGroupsInfoReq{GroupIDs: groupIDs}) + resp, err := group.GetGroupsInfoCaller.Invoke(ctx, &group.GetGroupsInfoReq{GroupIDs: groupIDs}) if err != nil { return err } @@ -101,7 +91,7 @@ func (x *JSSdk) getActiveConversations(ctx context.Context, req *jssdk.GetActive req.Count = defaultGetActiveConversation } req.OwnerUserID = mcontext.GetOpUserID(ctx) - conversationIDs, err := field(ctx, x.conv.GetConversationIDs, + conversationIDs, err := field(ctx, conversation.GetConversationIDsCaller.Invoke, &conversation.GetConversationIDsReq{UserID: req.OwnerUserID}, (*conversation.GetConversationIDsResp).GetConversationIDs) if err != nil { return nil, err @@ -109,12 +99,12 @@ func (x *JSSdk) getActiveConversations(ctx context.Context, req *jssdk.GetActive if len(conversationIDs) == 0 { return &jssdk.GetActiveConversationsResp{}, nil } - readSeq, err := field(ctx, x.msg.GetHasReadSeqs, + readSeq, err := field(ctx, msg.GetHasReadSeqsCaller.Invoke, &msg.GetHasReadSeqsReq{UserID: req.OwnerUserID, ConversationIDs: conversationIDs}, (*msg.SeqsInfoResp).GetMaxSeqs) if err != nil { return nil, err } - activeConversation, err := field(ctx, x.msg.GetActiveConversation, + activeConversation, err := field(ctx, msg.GetActiveConversationCaller.Invoke, &msg.GetActiveConversationReq{ConversationIDs: conversationIDs}, (*msg.GetActiveConversationResp).GetConversations) if err != nil { return nil, err @@ -126,7 +116,7 @@ func (x *JSSdk) getActiveConversations(ctx context.Context, req *jssdk.GetActive Conversation: activeConversation, } if len(activeConversation) > 1 { - pinnedConversationIDs, err := field(ctx, x.conv.GetPinnedConversationIDs, + pinnedConversationIDs, err := field(ctx, conversation.GetPinnedConversationIDsCaller.Invoke, &conversation.GetPinnedConversationIDsReq{UserID: req.OwnerUserID}, (*conversation.GetPinnedConversationIDsResp).GetConversationIDs) if err != nil { return nil, err @@ -135,7 +125,7 @@ func (x *JSSdk) getActiveConversations(ctx context.Context, req *jssdk.GetActive } sort.Sort(&sortConversations) sortList := sortConversations.Top(int(req.Count)) - conversations, err := field(ctx, x.conv.GetConversations, + conversations, err := field(ctx, conversation.GetConversationsCaller.Invoke, &conversation.GetConversationsReq{ OwnerUserID: req.OwnerUserID, ConversationIDs: datautil.Slice(sortList, func(c *msg.ActiveConversation) string { @@ -144,7 +134,7 @@ func (x *JSSdk) getActiveConversations(ctx context.Context, req *jssdk.GetActive if err != nil { return nil, err } - msgs, err := field(ctx, x.msg.GetSeqMessage, + msgs, err := field(ctx, msg.GetSeqMessageCaller.Invoke, &msg.GetSeqMessageReq{ UserID: req.OwnerUserID, Conversations: datautil.Slice(sortList, func(c *msg.ActiveConversation) *msg.ConversationSeqs { @@ -195,7 +185,7 @@ func (x *JSSdk) getActiveConversations(ctx context.Context, req *jssdk.GetActive func (x *JSSdk) getConversations(ctx context.Context, req *jssdk.GetConversationsReq) (*jssdk.GetConversationsResp, error) { req.OwnerUserID = mcontext.GetOpUserID(ctx) - conversations, err := field(ctx, x.conv.GetConversations, &conversation.GetConversationsReq{OwnerUserID: req.OwnerUserID, ConversationIDs: req.ConversationIDs}, (*conversation.GetConversationsResp).GetConversations) + conversations, err := field(ctx, conversation.GetConversationsCaller.Invoke, &conversation.GetConversationsReq{OwnerUserID: req.OwnerUserID, ConversationIDs: req.ConversationIDs}, (*conversation.GetConversationsResp).GetConversations) if err != nil { return nil, err } @@ -205,12 +195,12 @@ func (x *JSSdk) getConversations(ctx context.Context, req *jssdk.GetConversation req.ConversationIDs = datautil.Slice(conversations, func(c *conversation.Conversation) string { return c.ConversationID }) - maxSeqs, err := field(ctx, x.msg.GetMaxSeqs, + maxSeqs, err := field(ctx, msg.GetMaxSeqsCaller.Invoke, &msg.GetMaxSeqsReq{ConversationIDs: req.ConversationIDs}, (*msg.SeqsInfoResp).GetMaxSeqs) if err != nil { return nil, err } - readSeqs, err := field(ctx, x.msg.GetHasReadSeqs, + readSeqs, err := field(ctx, msg.GetHasReadSeqsCaller.Invoke, &msg.GetHasReadSeqsReq{UserID: req.OwnerUserID, ConversationIDs: req.ConversationIDs}, (*msg.SeqsInfoResp).GetMaxSeqs) if err != nil { return nil, err @@ -226,7 +216,7 @@ func (x *JSSdk) getConversations(ctx context.Context, req *jssdk.GetConversation } var msgs map[string]*sdkws.PullMsgs if len(conversationSeqs) > 0 { - msgs, err = field(ctx, x.msg.GetSeqMessage, + msgs, err = field(ctx, msg.GetSeqMessageCaller.Invoke, &msg.GetSeqMessageReq{UserID: req.OwnerUserID, Conversations: conversationSeqs}, (*msg.GetSeqMessageResp).GetMsgs) if err != nil { return nil, err diff --git a/internal/api/msg.go b/internal/api/msg.go index 9f79067a8..8efd34894 100644 --- a/internal/api/msg.go +++ b/internal/api/msg.go @@ -21,10 +21,11 @@ import ( "github.com/openimsdk/open-im-server/v3/pkg/apistruct" "github.com/openimsdk/open-im-server/v3/pkg/authverify" "github.com/openimsdk/open-im-server/v3/pkg/common/config" - "github.com/openimsdk/open-im-server/v3/pkg/rpcclient" "github.com/openimsdk/protocol/constant" "github.com/openimsdk/protocol/msg" + "github.com/openimsdk/protocol/rpccall" "github.com/openimsdk/protocol/sdkws" + "github.com/openimsdk/protocol/user" "github.com/openimsdk/tools/a2r" "github.com/openimsdk/tools/apiresp" "github.com/openimsdk/tools/errs" @@ -37,16 +38,12 @@ import ( ) type MessageApi struct { - *rpcclient.Message validate *validator.Validate - userRpcClient *rpcclient.UserRpcClient imAdminUserID []string } -func NewMessageApi(msgRpcClient *rpcclient.Message, userRpcClient *rpcclient.User, - imAdminUserID []string) MessageApi { - return MessageApi{Message: msgRpcClient, validate: validator.New(), - userRpcClient: rpcclient.NewUserRpcClientByUser(userRpcClient), imAdminUserID: imAdminUserID} +func NewMessageApi(imAdminUserID []string) MessageApi { + return MessageApi{validate: validator.New(), imAdminUserID: imAdminUserID} } func (*MessageApi) SetOptions(options map[string]bool, value bool) { @@ -108,51 +105,51 @@ func (m *MessageApi) newUserSendMsgReq(_ *gin.Context, params *apistruct.SendMsg } func (m *MessageApi) GetSeq(c *gin.Context) { - a2r.Call(msg.MsgClient.GetMaxSeq, m.Client, c) + a2r.CallV2(c, msg.GetMaxSeqCaller.Invoke) } func (m *MessageApi) PullMsgBySeqs(c *gin.Context) { - a2r.Call(msg.MsgClient.PullMessageBySeqs, m.Client, c) + a2r.CallV2(c, msg.PullMessageBySeqsCaller.Invoke) } func (m *MessageApi) RevokeMsg(c *gin.Context) { - a2r.Call(msg.MsgClient.RevokeMsg, m.Client, c) + a2r.CallV2(c, msg.RevokeMsgCaller.Invoke) } func (m *MessageApi) MarkMsgsAsRead(c *gin.Context) { - a2r.Call(msg.MsgClient.MarkMsgsAsRead, m.Client, c) + a2r.CallV2(c, msg.MarkMsgsAsReadCaller.Invoke) } func (m *MessageApi) MarkConversationAsRead(c *gin.Context) { - a2r.Call(msg.MsgClient.MarkConversationAsRead, m.Client, c) + a2r.CallV2(c, msg.MarkConversationAsReadCaller.Invoke) } func (m *MessageApi) GetConversationsHasReadAndMaxSeq(c *gin.Context) { - a2r.Call(msg.MsgClient.GetConversationsHasReadAndMaxSeq, m.Client, c) + a2r.CallV2(c, msg.GetConversationsHasReadAndMaxSeqCaller.Invoke) } func (m *MessageApi) SetConversationHasReadSeq(c *gin.Context) { - a2r.Call(msg.MsgClient.SetConversationHasReadSeq, m.Client, c) + a2r.CallV2(c, msg.SetConversationHasReadSeqCaller.Invoke) } func (m *MessageApi) ClearConversationsMsg(c *gin.Context) { - a2r.Call(msg.MsgClient.ClearConversationsMsg, m.Client, c) + a2r.CallV2(c, msg.ClearConversationsMsgCaller.Invoke) } func (m *MessageApi) UserClearAllMsg(c *gin.Context) { - a2r.Call(msg.MsgClient.UserClearAllMsg, m.Client, c) + a2r.CallV2(c, msg.UserClearAllMsgCaller.Invoke) } func (m *MessageApi) DeleteMsgs(c *gin.Context) { - a2r.Call(msg.MsgClient.DeleteMsgs, m.Client, c) + a2r.CallV2(c, msg.DeleteMsgsCaller.Invoke) } func (m *MessageApi) DeleteMsgPhysicalBySeq(c *gin.Context) { - a2r.Call(msg.MsgClient.DeleteMsgPhysicalBySeq, m.Client, c) + a2r.CallV2(c, msg.DeleteMsgPhysicalBySeqCaller.Invoke) } func (m *MessageApi) DeleteMsgPhysical(c *gin.Context) { - a2r.Call(msg.MsgClient.DeleteMsgPhysical, m.Client, c) + a2r.CallV2(c, msg.DeleteMsgPhysicalCaller.Invoke) } func (m *MessageApi) getSendMsgReq(c *gin.Context, req apistruct.SendMsg) (sendMsgReq *msg.SendMsgReq, err error) { @@ -180,7 +177,7 @@ func (m *MessageApi) getSendMsgReq(c *gin.Context, req apistruct.SendMsg) (sendM case constant.OANotification: data = apistruct.OANotificationElem{} req.SessionType = constant.NotificationChatType - if err = m.userRpcClient.GetNotificationByID(c, req.SendID); err != nil { + if err = user.GetNotificationAccountCaller.Execute(c, &user.GetNotificationAccountReq{UserID: req.SendID}); err != nil { return nil, err } default: @@ -227,7 +224,7 @@ func (m *MessageApi) SendMessage(c *gin.Context) { sendMsgReq.MsgData.RecvID = req.RecvID // Attempt to send the message using the client. - respPb, err := m.Client.SendMsg(c, sendMsgReq) + respPb, err := msg.SendMsgCaller.Invoke(c, sendMsgReq) if err != nil { // Set the status to failed and respond with an error if sending fails. apiresp.GinError(c, err) @@ -238,7 +235,7 @@ func (m *MessageApi) SendMessage(c *gin.Context) { var status = constant.MsgSendSuccessed // Attempt to update the message sending status in the system. - _, err = m.Client.SetSendMsgStatus(c, &msg.SetSendMsgStatusReq{ + err = msg.SetSendMsgStatusCaller.Execute(c, &msg.SetSendMsgStatusReq{ Status: int32(status), }) @@ -290,7 +287,7 @@ func (m *MessageApi) SendBusinessNotification(c *gin.Context) { }), }, } - respPb, err := m.Client.SendMsg(c, &sendMsgReq) + respPb, err := msg.SendMsgCaller.Invoke(c, &sendMsgReq) if err != nil { apiresp.GinError(c, err) return @@ -317,7 +314,10 @@ func (m *MessageApi) BatchSendMsg(c *gin.Context) { pageNumber := 1 showNumber := 500 for { - recvIDsPart, err := m.userRpcClient.GetAllUserIDs(c, int32(pageNumber), int32(showNumber)) + recvIDsPart, err := rpccall.ExtractField(c, user.GetAllUserIDCaller.Invoke, &user.GetAllUserIDReq{Pagination: &sdkws.RequestPagination{ + PageNumber: int32(pageNumber), + ShowNumber: int32(showNumber), + }}, (*user.GetAllUserIDResp).GetUserIDs) if err != nil { apiresp.GinError(c, err) return @@ -339,7 +339,7 @@ func (m *MessageApi) BatchSendMsg(c *gin.Context) { } for _, recvID := range recvIDs { sendMsgReq.MsgData.RecvID = recvID - rpcResp, err := m.Client.SendMsg(c, sendMsgReq) + rpcResp, err := msg.SendMsgCaller.Invoke(c, sendMsgReq) if err != nil { resp.FailedIDs = append(resp.FailedIDs, recvID) continue @@ -355,33 +355,33 @@ func (m *MessageApi) BatchSendMsg(c *gin.Context) { } func (m *MessageApi) CheckMsgIsSendSuccess(c *gin.Context) { - a2r.Call(msg.MsgClient.GetSendMsgStatus, m.Client, c) + a2r.CallV2(c, msg.GetSendMsgStatusCaller.Invoke) } func (m *MessageApi) GetUsersOnlineStatus(c *gin.Context) { - a2r.Call(msg.MsgClient.GetSendMsgStatus, m.Client, c) + a2r.CallV2(c, msg.GetSendMsgStatusCaller.Invoke) } func (m *MessageApi) GetActiveUser(c *gin.Context) { - a2r.Call(msg.MsgClient.GetActiveUser, m.Client, c) + a2r.CallV2(c, msg.GetActiveUserCaller.Invoke) } func (m *MessageApi) GetActiveGroup(c *gin.Context) { - a2r.Call(msg.MsgClient.GetActiveGroup, m.Client, c) + a2r.CallV2(c, msg.GetActiveGroupCaller.Invoke) } func (m *MessageApi) SearchMsg(c *gin.Context) { - a2r.Call(msg.MsgClient.SearchMessage, m.Client, c) + a2r.CallV2(c, msg.SearchMessageCaller.Invoke) } func (m *MessageApi) GetServerTime(c *gin.Context) { - a2r.Call(msg.MsgClient.GetServerTime, m.Client, c) + a2r.CallV2(c, msg.GetServerTimeCaller.Invoke) } func (m *MessageApi) GetStreamMsg(c *gin.Context) { - a2r.Call(msg.MsgClient.GetStreamMsg, m.Client, c) + a2r.CallV2(c, msg.GetStreamMsgCaller.Invoke) } func (m *MessageApi) AppendStreamMsg(c *gin.Context) { - a2r.Call(msg.MsgClient.AppendStreamMsg, m.Client, c) + a2r.CallV2(c, msg.AppendStreamMsgCaller.Invoke) } diff --git a/internal/api/router.go b/internal/api/router.go index 6714d645c..d6bf4e130 100644 --- a/internal/api/router.go +++ b/internal/api/router.go @@ -2,8 +2,11 @@ package api import ( "fmt" + "net/http" + "strings" "github.com/openimsdk/open-im-server/v3/internal/api/jssdk" + pbAuth "github.com/openimsdk/protocol/auth" "github.com/gin-contrib/gzip" @@ -13,12 +16,8 @@ import ( "google.golang.org/grpc" "google.golang.org/grpc/credentials/insecure" - "net/http" - "strings" - "github.com/openimsdk/open-im-server/v3/pkg/common/prommetrics" "github.com/openimsdk/open-im-server/v3/pkg/common/servererrs" - "github.com/openimsdk/open-im-server/v3/pkg/rpcclient" "github.com/openimsdk/protocol/constant" "github.com/openimsdk/tools/apiresp" "github.com/openimsdk/tools/discovery" @@ -48,7 +47,7 @@ func prommetricsGin() gin.HandlerFunc { } } -func newGinRouter(disCov discovery.SvcDiscoveryRegistry, config *Config, client discovery.SvcDiscoveryRegistry) *gin.Engine { +func newGinRouter(disCov discovery.SvcDiscoveryRegistry, config *Config) *gin.Engine { disCov.AddOption(mw.GrpcClient(), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithDefaultServiceConfig(fmt.Sprintf(`{"LoadBalancingPolicy": "%s"}`, "round_robin"))) gin.SetMode(gin.ReleaseMode) @@ -56,15 +55,6 @@ func newGinRouter(disCov discovery.SvcDiscoveryRegistry, config *Config, client if v, ok := binding.Validator.Engine().(*validator.Validate); ok { _ = v.RegisterValidation("required_if", RequiredIf) } - // init rpc client here - userRpc := rpcclient.NewUser(disCov, config.Discovery.RpcService.User, config.Discovery.RpcService.MessageGateway, - config.Share.IMAdminUserID) - groupRpc := rpcclient.NewGroup(disCov, config.Discovery.RpcService.Group) - friendRpc := rpcclient.NewFriend(disCov, config.Discovery.RpcService.Friend) - messageRpc := rpcclient.NewMessage(disCov, config.Discovery.RpcService.Msg) - conversationRpc := rpcclient.NewConversation(disCov, config.Discovery.RpcService.Conversation) - authRpc := rpcclient.NewAuth(disCov, config.Discovery.RpcService.Auth) - thirdRpc := rpcclient.NewThird(disCov, config.Discovery.RpcService.Third, config.API.Prometheus.GrafanaURL) switch config.API.Api.CompressionLevel { case NoCompression: case DefaultCompression: @@ -74,11 +64,11 @@ func newGinRouter(disCov discovery.SvcDiscoveryRegistry, config *Config, client case BestSpeed: r.Use(gzip.Gzip(gzip.BestSpeed)) } - r.Use(prommetricsGin(), gin.RecoveryWithWriter(gin.DefaultErrorWriter, mw.GinPanicErr), mw.CorsHandler(), mw.GinParseOperationID(), GinParseToken(authRpc)) - u := NewUserApi(*userRpc) - m := NewMessageApi(messageRpc, userRpc, config.Share.IMAdminUserID) - j := jssdk.NewJSSdkApi(userRpc.Client, friendRpc.Client, groupRpc.Client, messageRpc.Client, conversationRpc.Client) - pd := NewPrometheusDiscoveryApi(config, client) + r.Use(prommetricsGin(), gin.RecoveryWithWriter(gin.DefaultErrorWriter, mw.GinPanicErr), mw.CorsHandler(), mw.GinParseOperationID(), GinParseToken()) + u := NewUserApi(disCov, config.Discovery.RpcService.MessageGateway) + m := NewMessageApi(config.Share.IMAdminUserID) + j := jssdk.NewJSSdkApi() + pd := NewPrometheusDiscoveryApi(config, disCov) userRouterGroup := r.Group("/user") { userRouterGroup.POST("/user_register", u.UserRegister) @@ -108,7 +98,7 @@ func newGinRouter(disCov discovery.SvcDiscoveryRegistry, config *Config, client // friend routing group friendRouterGroup := r.Group("/friend") { - f := NewFriendApi(*friendRpc) + f := NewFriendApi() friendRouterGroup.POST("/delete_friend", f.DeleteFriend) friendRouterGroup.POST("/get_friend_apply_list", f.GetFriendApplyList) friendRouterGroup.POST("/get_designated_friend_apply", f.GetDesignatedFriendsApply) @@ -131,7 +121,7 @@ func newGinRouter(disCov discovery.SvcDiscoveryRegistry, config *Config, client friendRouterGroup.POST("/get_incremental_friends", f.GetIncrementalFriends) friendRouterGroup.POST("/get_full_friend_user_ids", f.GetFullFriendUserIDs) } - g := NewGroupApi(*groupRpc) + g := NewGroupApi() groupRouterGroup := r.Group("/group") { groupRouterGroup.POST("/create_group", g.CreateGroup) @@ -169,7 +159,7 @@ func newGinRouter(disCov discovery.SvcDiscoveryRegistry, config *Config, client // certificate authRouterGroup := r.Group("/auth") { - a := NewAuthApi(*authRpc) + a := NewAuthApi() authRouterGroup.POST("/get_admin_token", a.GetAdminToken) authRouterGroup.POST("/get_user_token", a.GetUserToken) authRouterGroup.POST("/parse_token", a.ParseToken) @@ -178,7 +168,7 @@ func newGinRouter(disCov discovery.SvcDiscoveryRegistry, config *Config, client // Third service thirdGroup := r.Group("/third") { - t := NewThirdApi(*thirdRpc) + t := NewThirdApi(config.API.Prometheus.GrafanaURL) thirdGroup.GET("/prometheus", t.GetPrometheus) thirdGroup.POST("/fcm_update_token", t.FcmUpdateToken) thirdGroup.POST("/set_app_badge", t.SetAppBadge) @@ -229,7 +219,7 @@ func newGinRouter(disCov discovery.SvcDiscoveryRegistry, config *Config, client // Conversation conversationGroup := r.Group("/conversation") { - c := NewConversationApi(*conversationRpc) + c := NewConversationApi() conversationGroup.POST("/get_sorted_conversation_list", c.GetSortedConversationList) conversationGroup.POST("/get_all_conversations", c.GetAllConversations) conversationGroup.POST("/get_conversation", c.GetConversation) @@ -271,7 +261,7 @@ func newGinRouter(disCov discovery.SvcDiscoveryRegistry, config *Config, client return r } -func GinParseToken(authRPC *rpcclient.Auth) gin.HandlerFunc { +func GinParseToken() gin.HandlerFunc { return func(c *gin.Context) { switch c.Request.Method { case http.MethodPost: @@ -289,7 +279,7 @@ func GinParseToken(authRPC *rpcclient.Auth) gin.HandlerFunc { c.Abort() return } - resp, err := authRPC.ParseToken(c, token) + resp, err := pbAuth.ParseTokenCaller.Invoke(c, &pbAuth.ParseTokenReq{Token: token}) if err != nil { apiresp.GinError(c, err) c.Abort() diff --git a/internal/api/statistics.go b/internal/api/statistics.go index f5ee99f73..f60bddb2e 100644 --- a/internal/api/statistics.go +++ b/internal/api/statistics.go @@ -16,17 +16,16 @@ package api import ( "github.com/gin-gonic/gin" - "github.com/openimsdk/open-im-server/v3/pkg/rpcclient" "github.com/openimsdk/protocol/user" "github.com/openimsdk/tools/a2r" ) -type StatisticsApi rpcclient.User +type StatisticsApi struct{} -func NewStatisticsApi(client rpcclient.User) StatisticsApi { - return StatisticsApi(client) +func NewStatisticsApi() StatisticsApi { + return StatisticsApi{} } func (s *StatisticsApi) UserRegister(c *gin.Context) { - a2r.Call(user.UserClient.UserRegisterCount, s.Client, c) + a2r.CallV2(c, user.UserRegisterCountCaller.Invoke) } diff --git a/internal/api/third.go b/internal/api/third.go index 6baa70ee5..ac02e3734 100644 --- a/internal/api/third.go +++ b/internal/api/third.go @@ -16,33 +16,35 @@ package api import ( "context" - "google.golang.org/grpc" "math/rand" "net/http" "net/url" "strconv" "strings" + "google.golang.org/grpc" + "github.com/gin-gonic/gin" - "github.com/openimsdk/open-im-server/v3/pkg/rpcclient" "github.com/openimsdk/protocol/third" "github.com/openimsdk/tools/a2r" "github.com/openimsdk/tools/errs" "github.com/openimsdk/tools/mcontext" ) -type ThirdApi rpcclient.Third +type ThirdApi struct { + GrafanaUrl string +} -func NewThirdApi(client rpcclient.Third) ThirdApi { - return ThirdApi(client) +func NewThirdApi(grafanaUrl string) ThirdApi { + return ThirdApi{GrafanaUrl: grafanaUrl} } func (o *ThirdApi) FcmUpdateToken(c *gin.Context) { - a2r.Call(third.ThirdClient.FcmUpdateToken, o.Client, c) + a2r.CallV2(c, third.FcmUpdateTokenCaller.Invoke) } func (o *ThirdApi) SetAppBadge(c *gin.Context) { - a2r.Call(third.ThirdClient.SetAppBadge, o.Client, c) + a2r.CallV2(c, third.SetAppBadgeCaller.Invoke) } // #################### s3 #################### @@ -77,44 +79,44 @@ func setURLPrefix(c *gin.Context, urlPrefix *string) error { } func (o *ThirdApi) PartLimit(c *gin.Context) { - a2r.Call(third.ThirdClient.PartLimit, o.Client, c) + a2r.CallV2(c, third.PartLimitCaller.Invoke) } func (o *ThirdApi) PartSize(c *gin.Context) { - a2r.Call(third.ThirdClient.PartSize, o.Client, c) + a2r.CallV2(c, third.PartSizeCaller.Invoke) } func (o *ThirdApi) InitiateMultipartUpload(c *gin.Context) { opt := setURLPrefixOption(third.ThirdClient.InitiateMultipartUpload, func(req *third.InitiateMultipartUploadReq) error { return setURLPrefix(c, &req.UrlPrefix) }) - a2r.Call(third.ThirdClient.InitiateMultipartUpload, o.Client, c, opt) + a2r.CallV2(c, third.InitiateMultipartUploadCaller.Invoke, opt) } func (o *ThirdApi) AuthSign(c *gin.Context) { - a2r.Call(third.ThirdClient.AuthSign, o.Client, c) + a2r.CallV2(c, third.AuthSignCaller.Invoke) } func (o *ThirdApi) CompleteMultipartUpload(c *gin.Context) { opt := setURLPrefixOption(third.ThirdClient.CompleteMultipartUpload, func(req *third.CompleteMultipartUploadReq) error { return setURLPrefix(c, &req.UrlPrefix) }) - a2r.Call(third.ThirdClient.CompleteMultipartUpload, o.Client, c, opt) + a2r.CallV2(c, third.CompleteMultipartUploadCaller.Invoke, opt) } func (o *ThirdApi) AccessURL(c *gin.Context) { - a2r.Call(third.ThirdClient.AccessURL, o.Client, c) + a2r.CallV2(c, third.AccessURLCaller.Invoke) } func (o *ThirdApi) InitiateFormData(c *gin.Context) { - a2r.Call(third.ThirdClient.InitiateFormData, o.Client, c) + a2r.CallV2(c, third.InitiateFormDataCaller.Invoke) } func (o *ThirdApi) CompleteFormData(c *gin.Context) { opt := setURLPrefixOption(third.ThirdClient.CompleteFormData, func(req *third.CompleteFormDataReq) error { return setURLPrefix(c, &req.UrlPrefix) }) - a2r.Call(third.ThirdClient.CompleteFormData, o.Client, c, opt) + a2r.CallV2(c, third.CompleteFormDataCaller.Invoke, opt) } func (o *ThirdApi) ObjectRedirect(c *gin.Context) { @@ -138,7 +140,7 @@ func (o *ThirdApi) ObjectRedirect(c *gin.Context) { } query[key] = values[0] } - resp, err := o.Client.AccessURL(ctx, &third.AccessURLReq{Name: name, Query: query}) + resp, err := third.AccessURLCaller.Invoke(ctx, &third.AccessURLReq{Name: name, Query: query}) if err != nil { if errs.ErrArgs.Is(err) { c.String(http.StatusBadRequest, err.Error()) @@ -156,15 +158,15 @@ func (o *ThirdApi) ObjectRedirect(c *gin.Context) { // #################### logs ####################. func (o *ThirdApi) UploadLogs(c *gin.Context) { - a2r.Call(third.ThirdClient.UploadLogs, o.Client, c) + a2r.CallV2(c, third.UploadLogsCaller.Invoke) } func (o *ThirdApi) DeleteLogs(c *gin.Context) { - a2r.Call(third.ThirdClient.DeleteLogs, o.Client, c) + a2r.CallV2(c, third.DeleteLogsCaller.Invoke) } func (o *ThirdApi) SearchLogs(c *gin.Context) { - a2r.Call(third.ThirdClient.SearchLogs, o.Client, c) + a2r.CallV2(c, third.SearchLogsCaller.Invoke) } func (o *ThirdApi) GetPrometheus(c *gin.Context) { diff --git a/internal/api/user.go b/internal/api/user.go index b499f71dc..e1024e50b 100644 --- a/internal/api/user.go +++ b/internal/api/user.go @@ -16,52 +16,58 @@ package api import ( "github.com/gin-gonic/gin" - "github.com/openimsdk/open-im-server/v3/pkg/rpcclient" "github.com/openimsdk/protocol/constant" "github.com/openimsdk/protocol/msggateway" "github.com/openimsdk/protocol/user" "github.com/openimsdk/tools/a2r" "github.com/openimsdk/tools/apiresp" + "github.com/openimsdk/tools/discovery" "github.com/openimsdk/tools/errs" "github.com/openimsdk/tools/log" ) -type UserApi rpcclient.User +type UserApi struct { + Discov discovery.SvcDiscoveryRegistry + MessageGateWayRpcName string +} -func NewUserApi(client rpcclient.User) UserApi { - return UserApi(client) +func NewUserApi(discov discovery.SvcDiscoveryRegistry, messageGateWayRpcName string) UserApi { + return UserApi{ + Discov: discov, + MessageGateWayRpcName: messageGateWayRpcName, + } } func (u *UserApi) UserRegister(c *gin.Context) { - a2r.Call(user.UserClient.UserRegister, u.Client, c) + a2r.CallV2(c, user.UserRegisterCaller.Invoke) } // UpdateUserInfo is deprecated. Use UpdateUserInfoEx func (u *UserApi) UpdateUserInfo(c *gin.Context) { - a2r.Call(user.UserClient.UpdateUserInfo, u.Client, c) + a2r.CallV2(c, user.UpdateUserInfoCaller.Invoke) } func (u *UserApi) UpdateUserInfoEx(c *gin.Context) { - a2r.Call(user.UserClient.UpdateUserInfoEx, u.Client, c) + a2r.CallV2(c, user.UpdateUserInfoExCaller.Invoke) } func (u *UserApi) SetGlobalRecvMessageOpt(c *gin.Context) { - a2r.Call(user.UserClient.SetGlobalRecvMessageOpt, u.Client, c) + a2r.CallV2(c, user.SetGlobalRecvMessageOptCaller.Invoke) } func (u *UserApi) GetUsersPublicInfo(c *gin.Context) { - a2r.Call(user.UserClient.GetDesignateUsers, u.Client, c) + a2r.CallV2(c, user.GetDesignateUsersCaller.Invoke) } func (u *UserApi) GetAllUsersID(c *gin.Context) { - a2r.Call(user.UserClient.GetAllUserID, u.Client, c) + a2r.CallV2(c, user.GetAllUserIDCaller.Invoke) } func (u *UserApi) AccountCheck(c *gin.Context) { - a2r.Call(user.UserClient.AccountCheck, u.Client, c) + a2r.CallV2(c, user.AccountCheckCaller.Invoke) } func (u *UserApi) GetUsers(c *gin.Context) { - a2r.Call(user.UserClient.GetPaginationUsers, u.Client, c) + a2r.CallV2(c, user.GetPaginationUsersCaller.Invoke) } // GetUsersOnlineStatus Get user online status. @@ -122,7 +128,7 @@ func (u *UserApi) GetUsersOnlineStatus(c *gin.Context) { } func (u *UserApi) UserRegisterCount(c *gin.Context) { - a2r.Call(user.UserClient.UserRegisterCount, u.Client, c) + a2r.CallV2(c, user.UserRegisterCountCaller.Invoke) } // GetUsersOnlineTokenDetail Get user online token details. @@ -188,52 +194,52 @@ func (u *UserApi) GetUsersOnlineTokenDetail(c *gin.Context) { // SubscriberStatus Presence status of subscribed users. func (u *UserApi) SubscriberStatus(c *gin.Context) { - a2r.Call(user.UserClient.SubscribeOrCancelUsersStatus, u.Client, c) + a2r.CallV2(c, user.SubscribeOrCancelUsersStatusCaller.Invoke) } // GetUserStatus Get the online status of the user. func (u *UserApi) GetUserStatus(c *gin.Context) { - a2r.Call(user.UserClient.GetUserStatus, u.Client, c) + a2r.CallV2(c, user.GetUserStatusCaller.Invoke) } // GetSubscribeUsersStatus Get the online status of subscribers. func (u *UserApi) GetSubscribeUsersStatus(c *gin.Context) { - a2r.Call(user.UserClient.GetSubscribeUsersStatus, u.Client, c) + a2r.CallV2(c, user.GetSubscribeUsersStatusCaller.Invoke) } // ProcessUserCommandAdd user general function add. func (u *UserApi) ProcessUserCommandAdd(c *gin.Context) { - a2r.Call(user.UserClient.ProcessUserCommandAdd, u.Client, c) + a2r.CallV2(c, user.ProcessUserCommandAddCaller.Invoke) } // ProcessUserCommandDelete user general function delete. func (u *UserApi) ProcessUserCommandDelete(c *gin.Context) { - a2r.Call(user.UserClient.ProcessUserCommandDelete, u.Client, c) + a2r.CallV2(c, user.ProcessUserCommandDeleteCaller.Invoke) } // ProcessUserCommandUpdate user general function update. func (u *UserApi) ProcessUserCommandUpdate(c *gin.Context) { - a2r.Call(user.UserClient.ProcessUserCommandUpdate, u.Client, c) + a2r.CallV2(c, user.ProcessUserCommandUpdateCaller.Invoke) } // ProcessUserCommandGet user general function get. func (u *UserApi) ProcessUserCommandGet(c *gin.Context) { - a2r.Call(user.UserClient.ProcessUserCommandGet, u.Client, c) + a2r.CallV2(c, user.ProcessUserCommandGetCaller.Invoke) } // ProcessUserCommandGet user general function get all. func (u *UserApi) ProcessUserCommandGetAll(c *gin.Context) { - a2r.Call(user.UserClient.ProcessUserCommandGetAll, u.Client, c) + a2r.CallV2(c, user.ProcessUserCommandGetAllCaller.Invoke) } func (u *UserApi) AddNotificationAccount(c *gin.Context) { - a2r.Call(user.UserClient.AddNotificationAccount, u.Client, c) + a2r.CallV2(c, user.AddNotificationAccountCaller.Invoke) } func (u *UserApi) UpdateNotificationAccountInfo(c *gin.Context) { - a2r.Call(user.UserClient.UpdateNotificationAccountInfo, u.Client, c) + a2r.CallV2(c, user.UpdateNotificationAccountInfoCaller.Invoke) } func (u *UserApi) SearchNotificationAccount(c *gin.Context) { - a2r.Call(user.UserClient.SearchNotificationAccount, u.Client, c) + a2r.CallV2(c, user.SearchNotificationAccountCaller.Invoke) } diff --git a/internal/msggateway/client.go b/internal/msggateway/client.go index 7e1ba6405..161300d6d 100644 --- a/internal/msggateway/client.go +++ b/internal/msggateway/client.go @@ -18,7 +18,6 @@ import ( "context" "encoding/json" "fmt" - "runtime/debug" "sync" "sync/atomic" "time" @@ -132,7 +131,7 @@ func (c *Client) readMessage() { defer func() { if r := recover(); r != nil { c.closedErr = ErrPanic - fmt.Println("socket have panic err:", r, string(debug.Stack())) + log.ZPanic(c.ctx, "socket have panic err:", r) } c.close() }() diff --git a/internal/msggateway/hub_server.go b/internal/msggateway/hub_server.go index 8e1edbec7..5c9237ed1 100644 --- a/internal/msggateway/hub_server.go +++ b/internal/msggateway/hub_server.go @@ -21,7 +21,6 @@ import ( "github.com/openimsdk/open-im-server/v3/pkg/authverify" "github.com/openimsdk/open-im-server/v3/pkg/common/servererrs" "github.com/openimsdk/open-im-server/v3/pkg/common/startrpc" - "github.com/openimsdk/open-im-server/v3/pkg/rpcclient" "github.com/openimsdk/protocol/constant" "github.com/openimsdk/protocol/msggateway" "github.com/openimsdk/protocol/sdkws" @@ -37,7 +36,6 @@ import ( func (s *Server) InitServer(ctx context.Context, config *Config, disCov discovery.SvcDiscoveryRegistry, server *grpc.Server) error { s.LongConnServer.SetDiscoveryRegistry(disCov, config) msggateway.RegisterMsgGatewayServer(server, s) - s.userRcp = rpcclient.NewUserRpcClient(disCov, config.Discovery.RpcService.User, config.Share.IMAdminUserID) if s.ready != nil { return s.ready(s) } @@ -62,7 +60,6 @@ type Server struct { config *Config pushTerminal map[int]struct{} ready func(srv *Server) error - userRcp rpcclient.UserRpcClient queue *memamq.MemoryQueue } diff --git a/internal/msggateway/init.go b/internal/msggateway/init.go index 6654e6598..156d32b4d 100644 --- a/internal/msggateway/init.go +++ b/internal/msggateway/init.go @@ -62,7 +62,7 @@ func Start(ctx context.Context, index int, conf *Config) error { ) hubServer := NewServer(longServer, conf, func(srv *Server) error { - longServer.online, _ = rpccache.NewOnlineCache(srv.userRcp, nil, rdb, false, longServer.subscriberUserOnlineStatusChanges) + longServer.online, _ = rpccache.NewOnlineCache(conf.Share.IMAdminUserID, nil, rdb, false, longServer.subscriberUserOnlineStatusChanges) return nil }) diff --git a/internal/msggateway/message_handler.go b/internal/msggateway/message_handler.go index f2b0ce9da..d88d2fbfd 100644 --- a/internal/msggateway/message_handler.go +++ b/internal/msggateway/message_handler.go @@ -22,12 +22,9 @@ import ( "github.com/go-playground/validator/v10" "google.golang.org/protobuf/proto" - "github.com/openimsdk/open-im-server/v3/pkg/common/config" - "github.com/openimsdk/open-im-server/v3/pkg/rpcclient" "github.com/openimsdk/protocol/msg" "github.com/openimsdk/protocol/push" "github.com/openimsdk/protocol/sdkws" - "github.com/openimsdk/tools/discovery" "github.com/openimsdk/tools/errs" "github.com/openimsdk/tools/utils/jsonutil" ) @@ -115,18 +112,11 @@ type MessageHandler interface { var _ MessageHandler = (*GrpcHandler)(nil) type GrpcHandler struct { - msgRpcClient *rpcclient.MessageRpcClient - pushClient *rpcclient.PushRpcClient - validate *validator.Validate + validate *validator.Validate } -func NewGrpcHandler(validate *validator.Validate, client discovery.SvcDiscoveryRegistry, rpcRegisterName *config.RpcService) *GrpcHandler { - msgRpcClient := rpcclient.NewMessageRpcClient(client, rpcRegisterName.Msg) - pushRpcClient := rpcclient.NewPushRpcClient(client, rpcRegisterName.Push) - return &GrpcHandler{ - msgRpcClient: &msgRpcClient, - pushClient: &pushRpcClient, validate: validate, - } +func NewGrpcHandler(validate *validator.Validate) *GrpcHandler { + return &GrpcHandler{validate: validate} } func (g GrpcHandler) GetSeq(ctx context.Context, data *Req) ([]byte, error) { @@ -137,7 +127,7 @@ func (g GrpcHandler) GetSeq(ctx context.Context, data *Req) ([]byte, error) { if err := g.validate.Struct(&req); err != nil { return nil, errs.WrapMsg(err, "GetSeq: validation failed", "action", "validate", "dataType", "GetMaxSeqReq") } - resp, err := g.msgRpcClient.GetMaxSeq(ctx, &req) + resp, err := msg.GetMaxSeqCaller.Invoke(ctx, &req) if err != nil { return nil, err } @@ -161,7 +151,7 @@ func (g GrpcHandler) SendMessage(ctx context.Context, data *Req) ([]byte, error) } req := msg.SendMsgReq{MsgData: &msgData} - resp, err := g.msgRpcClient.SendMsg(ctx, &req) + resp, err := msg.SendMsgCaller.Invoke(ctx, &req) if err != nil { return nil, err } @@ -175,7 +165,7 @@ func (g GrpcHandler) SendMessage(ctx context.Context, data *Req) ([]byte, error) } func (g GrpcHandler) SendSignalMessage(context context.Context, data *Req) ([]byte, error) { - resp, err := g.msgRpcClient.SendMsg(context, nil) + resp, err := msg.SendMsgCaller.Invoke(context, nil) if err != nil { return nil, err } @@ -186,7 +176,7 @@ func (g GrpcHandler) SendSignalMessage(context context.Context, data *Req) ([]by return c, nil } -func (g GrpcHandler) PullMessageBySeqList(context context.Context, data *Req) ([]byte, error) { +func (g GrpcHandler) PullMessageBySeqList(ctx context.Context, data *Req) ([]byte, error) { req := sdkws.PullMessageBySeqsReq{} if err := proto.Unmarshal(data.Data, &req); err != nil { return nil, errs.WrapMsg(err, "err proto unmarshal", "action", "unmarshal", "dataType", "PullMessageBySeqsReq") @@ -194,7 +184,7 @@ func (g GrpcHandler) PullMessageBySeqList(context context.Context, data *Req) ([ if err := g.validate.Struct(data); err != nil { return nil, errs.WrapMsg(err, "validation failed", "action", "validate", "dataType", "PullMessageBySeqsReq") } - resp, err := g.msgRpcClient.PullMessageBySeqList(context, &req) + resp, err := msg.PullMessageBySeqsCaller.Invoke(ctx, &req) if err != nil { return nil, err } @@ -205,7 +195,7 @@ func (g GrpcHandler) PullMessageBySeqList(context context.Context, data *Req) ([ return c, nil } -func (g GrpcHandler) GetConversationsHasReadAndMaxSeq(context context.Context, data *Req) ([]byte, error) { +func (g GrpcHandler) GetConversationsHasReadAndMaxSeq(ctx context.Context, data *Req) ([]byte, error) { req := msg.GetConversationsHasReadAndMaxSeqReq{} if err := proto.Unmarshal(data.Data, &req); err != nil { return nil, errs.WrapMsg(err, "err proto unmarshal", "action", "unmarshal", "dataType", "GetConversationsHasReadAndMaxSeq") @@ -213,7 +203,7 @@ func (g GrpcHandler) GetConversationsHasReadAndMaxSeq(context context.Context, d if err := g.validate.Struct(data); err != nil { return nil, errs.WrapMsg(err, "validation failed", "action", "validate", "dataType", "GetConversationsHasReadAndMaxSeq") } - resp, err := g.msgRpcClient.GetConversationsHasReadAndMaxSeq(context, &req) + resp, err := msg.GetConversationsHasReadAndMaxSeqCaller.Invoke(ctx, &req) if err != nil { return nil, err } @@ -224,7 +214,7 @@ func (g GrpcHandler) GetConversationsHasReadAndMaxSeq(context context.Context, d return c, nil } -func (g GrpcHandler) GetSeqMessage(context context.Context, data *Req) ([]byte, error) { +func (g GrpcHandler) GetSeqMessage(ctx context.Context, data *Req) ([]byte, error) { req := msg.GetSeqMessageReq{} if err := proto.Unmarshal(data.Data, &req); err != nil { return nil, errs.WrapMsg(err, "error unmarshaling request", "action", "unmarshal", "dataType", "GetSeqMessage") @@ -232,7 +222,7 @@ func (g GrpcHandler) GetSeqMessage(context context.Context, data *Req) ([]byte, if err := g.validate.Struct(data); err != nil { return nil, errs.WrapMsg(err, "validation failed", "action", "validate", "dataType", "GetSeqMessage") } - resp, err := g.msgRpcClient.GetSeqMessage(context, &req) + resp, err := msg.GetSeqMessageCaller.Invoke(ctx, &req) if err != nil { return nil, err } @@ -243,12 +233,12 @@ func (g GrpcHandler) GetSeqMessage(context context.Context, data *Req) ([]byte, return c, nil } -func (g GrpcHandler) UserLogout(context context.Context, data *Req) ([]byte, error) { +func (g GrpcHandler) UserLogout(ctx context.Context, data *Req) ([]byte, error) { req := push.DelUserPushTokenReq{} if err := proto.Unmarshal(data.Data, &req); err != nil { return nil, errs.WrapMsg(err, "error unmarshaling request", "action", "unmarshal", "dataType", "DelUserPushTokenReq") } - resp, err := g.pushClient.DelUserPushToken(context, &req) + resp, err := push.DelUserPushTokenCaller.Invoke(ctx, &req) if err != nil { return nil, err } diff --git a/internal/msggateway/online.go b/internal/msggateway/online.go index f29869b6e..bff263997 100644 --- a/internal/msggateway/online.go +++ b/internal/msggateway/online.go @@ -5,16 +5,17 @@ import ( "crypto/md5" "encoding/binary" "fmt" - "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/cachekey" - pbuser "github.com/openimsdk/protocol/user" - "github.com/openimsdk/tools/log" - "github.com/openimsdk/tools/mcontext" - "github.com/openimsdk/tools/utils/datautil" "math/rand" "os" "strconv" "sync/atomic" "time" + + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/cachekey" + pbuser "github.com/openimsdk/protocol/user" + "github.com/openimsdk/tools/log" + "github.com/openimsdk/tools/mcontext" + "github.com/openimsdk/tools/utils/datautil" ) func (ws *WsServer) ChangeOnlineStatus(concurrent int) { @@ -87,7 +88,7 @@ func (ws *WsServer) ChangeOnlineStatus(concurrent int) { opIdCtx := mcontext.SetOperationID(context.Background(), operationIDPrefix+strconv.FormatInt(count.Add(1), 10)) ctx, cancel := context.WithTimeout(opIdCtx, time.Second*5) defer cancel() - if _, err := ws.userClient.Client.SetUserOnlineStatus(ctx, req); err != nil { + if err := pbuser.SetUserOnlineStatusCaller.Execute(ctx, req); err != nil { log.ZError(ctx, "update user online status", err) } for _, ss := range req.Status { diff --git a/internal/msggateway/ws_server.go b/internal/msggateway/ws_server.go index 2e23262b1..7271c3727 100644 --- a/internal/msggateway/ws_server.go +++ b/internal/msggateway/ws_server.go @@ -16,7 +16,6 @@ import ( "github.com/go-playground/validator/v10" "github.com/openimsdk/open-im-server/v3/pkg/common/prommetrics" "github.com/openimsdk/open-im-server/v3/pkg/common/servererrs" - "github.com/openimsdk/open-im-server/v3/pkg/rpcclient" "github.com/openimsdk/protocol/constant" "github.com/openimsdk/protocol/msggateway" "github.com/openimsdk/tools/discovery" @@ -57,8 +56,6 @@ type WsServer struct { handshakeTimeout time.Duration writeBufferSize int validate *validator.Validate - userClient *rpcclient.UserRpcClient - authClient *rpcclient.Auth disCov discovery.SvcDiscoveryRegistry Compressor //Encoder @@ -73,10 +70,7 @@ type kickHandler struct { } func (ws *WsServer) SetDiscoveryRegistry(disCov discovery.SvcDiscoveryRegistry, config *Config) { - ws.MessageHandler = NewGrpcHandler(ws.validate, disCov, &config.Discovery.RpcService) - u := rpcclient.NewUserRpcClient(disCov, config.Discovery.RpcService.User, config.Share.IMAdminUserID) - ws.authClient = rpcclient.NewAuth(disCov, config.Discovery.RpcService.Auth) - ws.userClient = &u + ws.MessageHandler = NewGrpcHandler(ws.validate) ws.disCov = disCov } @@ -312,7 +306,8 @@ func (ws *WsServer) multiTerminalLoginChecker(clientOK bool, oldClients []*Clien []string{newClient.ctx.GetOperationID(), newClient.ctx.GetUserID(), constant.PlatformIDToName(newClient.PlatformID), newClient.ctx.GetConnID()}, ) - if _, err := ws.authClient.KickTokens(ctx, kickTokens); err != nil { + + if err := pbAuth.KickTokensCaller.Execute(ctx, &pbAuth.KickTokensReq{Tokens: kickTokens}); err != nil { log.ZWarn(newClient.ctx, "kickTokens err", err) } } @@ -339,7 +334,11 @@ func (ws *WsServer) multiTerminalLoginChecker(clientOK bool, oldClients []*Clien []string{newClient.ctx.GetOperationID(), newClient.ctx.GetUserID(), constant.PlatformIDToName(newClient.PlatformID), newClient.ctx.GetConnID()}, ) - if _, err := ws.authClient.InvalidateToken(ctx, newClient.token, newClient.UserID, newClient.PlatformID); err != nil { + if err := pbAuth.InvalidateTokenCaller.Execute(ctx, &pbAuth.InvalidateTokenReq{ + PreservedToken: newClient.token, + UserID: newClient.UserID, + PlatformID: int32(newClient.PlatformID), + }); err != nil { log.ZWarn(newClient.ctx, "InvalidateToken err", err, "userID", newClient.UserID, "platformID", newClient.PlatformID) } @@ -410,7 +409,7 @@ func (ws *WsServer) wsHandler(w http.ResponseWriter, r *http.Request) { } // Call the authentication client to parse the Token obtained from the context - resp, err := ws.authClient.ParseToken(connContext, connContext.GetToken()) + resp, err := pbAuth.ParseTokenCaller.Invoke(connContext, &pbAuth.ParseTokenReq{Token: connContext.GetToken()}) if err != nil { // If there's an error parsing the Token, decide whether to send the error message via WebSocket based on the context flag shouldSendError := connContext.ShouldSendResp() diff --git a/internal/msgtransfer/init.go b/internal/msgtransfer/init.go index 4c09bc4e2..19a53ebd5 100644 --- a/internal/msgtransfer/init.go +++ b/internal/msgtransfer/init.go @@ -25,6 +25,8 @@ import ( "strconv" "syscall" + "github.com/openimsdk/open-im-server/v3/pkg/rpcclient" + "github.com/openimsdk/tools/discovery" "github.com/openimsdk/tools/discovery/etcd" "github.com/openimsdk/tools/utils/jsonutil" "github.com/openimsdk/tools/utils/network" @@ -39,9 +41,7 @@ import ( conf "github.com/openimsdk/open-im-server/v3/pkg/common/config" discRegister "github.com/openimsdk/open-im-server/v3/pkg/common/discoveryregister" - kdisc "github.com/openimsdk/open-im-server/v3/pkg/common/discoveryregister" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/controller" - "github.com/openimsdk/open-im-server/v3/pkg/rpcclient" "github.com/openimsdk/tools/errs" "github.com/openimsdk/tools/log" "github.com/openimsdk/tools/mw" @@ -93,6 +93,9 @@ func Start(ctx context.Context, index int, config *Config) error { } client.AddOption(mw.GrpcClient(), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithDefaultServiceConfig(fmt.Sprintf(`{"LoadBalancingPolicy": "%s"}`, "round_robin"))) + if err = rpcclient.InitRpcCaller(client, config.Discovery.RpcService); err != nil { + return err + } msgModel := redis.NewMsgCache(rdb) msgDocModel, err := mgo.NewMsgMongo(mgocli.GetDB()) @@ -113,9 +116,7 @@ func Start(ctx context.Context, index int, config *Config) error { if err != nil { return err } - conversationRpcClient := rpcclient.NewConversationRpcClient(client, config.Discovery.RpcService.Conversation) - groupRpcClient := rpcclient.NewGroupRpcClient(client, config.Discovery.RpcService.Group) - historyCH, err := NewOnlineHistoryRedisConsumerHandler(&config.KafkaConfig, msgTransferDatabase, &conversationRpcClient, &groupRpcClient) + historyCH, err := NewOnlineHistoryRedisConsumerHandler(&config.KafkaConfig, msgTransferDatabase) if err != nil { return err } @@ -129,10 +130,10 @@ func Start(ctx context.Context, index int, config *Config) error { historyMongoCH: historyMongoCH, runTimeEnv: runTimeEnv, } - return msgTransfer.Start(index, config) + return msgTransfer.Start(index, config, client) } -func (m *MsgTransfer) Start(index int, config *Config) error { +func (m *MsgTransfer) Start(index int, config *Config, client discovery.SvcDiscoveryRegistry) error { m.ctx, m.cancel = context.WithCancel(context.Background()) var ( netDone = make(chan struct{}, 1) @@ -147,11 +148,6 @@ func (m *MsgTransfer) Start(index int, config *Config) error { return err } - client, err := kdisc.NewDiscoveryRegister(&config.Discovery, m.runTimeEnv) - if err != nil { - return errs.WrapMsg(err, "failed to register discovery service") - } - registerIP, err := network.GetRpcRegisterIP("") if err != nil { return err diff --git a/internal/msgtransfer/online_history_msg_handler.go b/internal/msgtransfer/online_history_msg_handler.go index 0104f6633..9287d6b61 100644 --- a/internal/msgtransfer/online_history_msg_handler.go +++ b/internal/msgtransfer/online_history_msg_handler.go @@ -23,16 +23,17 @@ import ( "sync" "time" - "github.com/openimsdk/open-im-server/v3/pkg/common/prommetrics" - "github.com/IBM/sarama" "github.com/go-redis/redis" "github.com/openimsdk/open-im-server/v3/pkg/common/config" + "github.com/openimsdk/open-im-server/v3/pkg/common/prommetrics" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/controller" "github.com/openimsdk/open-im-server/v3/pkg/msgprocessor" - "github.com/openimsdk/open-im-server/v3/pkg/rpcclient" "github.com/openimsdk/open-im-server/v3/pkg/tools/batcher" "github.com/openimsdk/protocol/constant" + pbconv "github.com/openimsdk/protocol/conversation" + "github.com/openimsdk/protocol/group" + "github.com/openimsdk/protocol/rpccall" "github.com/openimsdk/protocol/sdkws" "github.com/openimsdk/tools/errs" "github.com/openimsdk/tools/log" @@ -69,14 +70,11 @@ type OnlineHistoryRedisConsumerHandler struct { redisMessageBatches *batcher.Batcher[sarama.ConsumerMessage] msgTransferDatabase controller.MsgTransferDatabase - conversationRpcClient *rpcclient.ConversationRpcClient - groupRpcClient *rpcclient.GroupRpcClient conversationUserHasReadChan chan *userHasReadSeq wg sync.WaitGroup } -func NewOnlineHistoryRedisConsumerHandler(kafkaConf *config.Kafka, database controller.MsgTransferDatabase, - conversationRpcClient *rpcclient.ConversationRpcClient, groupRpcClient *rpcclient.GroupRpcClient) (*OnlineHistoryRedisConsumerHandler, error) { +func NewOnlineHistoryRedisConsumerHandler(kafkaConf *config.Kafka, database controller.MsgTransferDatabase) (*OnlineHistoryRedisConsumerHandler, error) { historyConsumerGroup, err := kafka.NewMConsumerGroup(kafkaConf.Build(), kafkaConf.ToRedisGroupID, []string{kafkaConf.ToRedisTopic}, false) if err != nil { return nil, err @@ -103,8 +101,6 @@ func NewOnlineHistoryRedisConsumerHandler(kafkaConf *config.Kafka, database cont } b.Do = och.do och.redisMessageBatches = b - och.conversationRpcClient = conversationRpcClient - och.groupRpcClient = groupRpcClient och.historyConsumerGroup = historyConsumerGroup return &och, err @@ -285,22 +281,32 @@ func (och *OnlineHistoryRedisConsumerHandler) handleMsg(ctx context.Context, key case constant.ReadGroupChatType: log.ZDebug(ctx, "group chat first create conversation", "conversationID", conversationID) - userIDs, err := och.groupRpcClient.GetGroupMemberIDs(ctx, msg.GroupID) + + userIDs, err := rpccall.ExtractField(ctx, group.GetGroupMemberUserIDsCaller.Invoke, + &group.GetGroupMemberUserIDsReq{ + GroupID: msg.GroupID, + }, (*group.GetGroupMemberUserIDsResp).GetUserIDs) if err != nil { log.ZWarn(ctx, "get group member ids error", err, "conversationID", conversationID) } else { log.ZInfo(ctx, "GetGroupMemberIDs end") - if err := och.conversationRpcClient.GroupChatFirstCreateConversation(ctx, - msg.GroupID, userIDs); err != nil { + if err := pbconv.CreateGroupChatConversationsCaller.Execute(ctx, &pbconv.CreateGroupChatConversationsReq{ + UserIDs: userIDs, + GroupID: msg.GroupID, + }); err != nil { log.ZWarn(ctx, "single chat first create conversation error", err, "conversationID", conversationID) } } case constant.SingleChatType, constant.NotificationChatType: - if err := och.conversationRpcClient.SingleChatFirstCreateConversation(ctx, msg.RecvID, - msg.SendID, conversationID, msg.SessionType); err != nil { + if err := pbconv.CreateSingleChatConversationsCaller.Execute(ctx, &pbconv.CreateSingleChatConversationsReq{ + RecvID: msg.RecvID, + SendID: msg.SendID, + ConversationID: conversationID, + ConversationType: msg.SessionType, + }); err != nil { log.ZWarn(ctx, "single chat or notification first create conversation error", err, "conversationID", conversationID, "sessionType", msg.SessionType) } diff --git a/internal/push/push_handler.go b/internal/push/push_handler.go index ee3dc5b84..ee855e122 100644 --- a/internal/push/push_handler.go +++ b/internal/push/push_handler.go @@ -3,7 +3,6 @@ package push import ( "context" "encoding/json" - "math/rand" "strconv" "time" @@ -16,11 +15,14 @@ import ( "github.com/openimsdk/open-im-server/v3/pkg/common/webhook" "github.com/openimsdk/open-im-server/v3/pkg/msgprocessor" "github.com/openimsdk/open-im-server/v3/pkg/rpccache" - "github.com/openimsdk/open-im-server/v3/pkg/rpcclient" "github.com/openimsdk/open-im-server/v3/pkg/util/conversationutil" "github.com/openimsdk/protocol/constant" + pbconv "github.com/openimsdk/protocol/conversation" + "github.com/openimsdk/protocol/group" + "github.com/openimsdk/protocol/msg" "github.com/openimsdk/protocol/msggateway" pbpush "github.com/openimsdk/protocol/push" + "github.com/openimsdk/protocol/rpccall" "github.com/openimsdk/protocol/sdkws" "github.com/openimsdk/tools/discovery" "github.com/openimsdk/tools/log" @@ -41,9 +43,6 @@ type ConsumerHandler struct { onlineCache *rpccache.OnlineCache groupLocalCache *rpccache.GroupLocalCache conversationLocalCache *rpccache.ConversationLocalCache - msgRpcClient rpcclient.MessageRpcClient - conversationRpcClient rpcclient.ConversationRpcClient - groupRpcClient rpcclient.GroupRpcClient webhookClient *webhook.Client config *Config } @@ -58,19 +57,14 @@ func NewConsumerHandler(config *Config, database controller.PushDatabase, offlin return nil, err } - userRpcClient := rpcclient.NewUserRpcClient(client, config.Discovery.RpcService.User, config.Share.IMAdminUserID) - consumerHandler.offlinePusher = offlinePusher consumerHandler.onlinePusher = NewOnlinePusher(client, config) - consumerHandler.groupRpcClient = rpcclient.NewGroupRpcClient(client, config.Discovery.RpcService.Group) - consumerHandler.groupLocalCache = rpccache.NewGroupLocalCache(consumerHandler.groupRpcClient, &config.LocalCacheConfig, rdb) - consumerHandler.msgRpcClient = rpcclient.NewMessageRpcClient(client, config.Discovery.RpcService.Msg) - consumerHandler.conversationRpcClient = rpcclient.NewConversationRpcClient(client, config.Discovery.RpcService.Conversation) - consumerHandler.conversationLocalCache = rpccache.NewConversationLocalCache(consumerHandler.conversationRpcClient, &config.LocalCacheConfig, rdb) + consumerHandler.groupLocalCache = rpccache.NewGroupLocalCache(&config.LocalCacheConfig, rdb) + consumerHandler.conversationLocalCache = rpccache.NewConversationLocalCache(&config.LocalCacheConfig, rdb) consumerHandler.webhookClient = webhook.NewWebhookClient(config.WebhooksConfig.URL) consumerHandler.config = config consumerHandler.pushDatabase = database - consumerHandler.onlineCache, err = rpccache.NewOnlineCache(userRpcClient, consumerHandler.groupLocalCache, rdb, config.RpcConfig.FullUserCache, nil) + consumerHandler.onlineCache, err = rpccache.NewOnlineCache(config.Share.IMAdminUserID, consumerHandler.groupLocalCache, rdb, config.RpcConfig.FullUserCache, nil) if err != nil { return nil, err } @@ -327,7 +321,7 @@ func (c *ConsumerHandler) groupMessagesHandler(ctx context.Context, groupID stri ctx = mcontext.WithOpUserIDContext(ctx, c.config.Share.IMAdminUserID[0]) } defer func(groupID string) { - if err = c.groupRpcClient.DismissGroup(ctx, groupID); err != nil { + if err = group.DismissGroupCaller.Execute(ctx, &group.DismissGroupReq{GroupID: groupID}); err != nil { log.ZError(ctx, "DismissGroup Notification clear members", err, "groupID", groupID) } }(groupID) @@ -355,8 +349,10 @@ func (c *ConsumerHandler) filterGroupMessageOfflinePush(ctx context.Context, gro offlinePushUserIDs []string) (userIDs []string, err error) { //todo local cache Obtain the difference set through local comparison. - needOfflinePushUserIDs, err := c.conversationRpcClient.GetConversationOfflinePushUserIDs( - ctx, conversationutil.GenGroupConversationID(groupID), offlinePushUserIDs) + needOfflinePushUserIDs, err := rpccall.ExtractField(ctx, pbconv.GetConversationOfflinePushUserIDsCaller.Invoke, &pbconv.GetConversationOfflinePushUserIDsReq{ + ConversationID: conversationutil.GenGroupConversationID(groupID), + UserIDs: offlinePushUserIDs, + }, (*pbconv.GetConversationOfflinePushUserIDsResp).GetUserIDs) if err != nil { return nil, err } @@ -410,12 +406,18 @@ func (c *ConsumerHandler) getOfflinePushInfos(msg *sdkws.MsgData) (title, conten func (c *ConsumerHandler) DeleteMemberAndSetConversationSeq(ctx context.Context, groupID string, userIDs []string) error { conversationID := msgprocessor.GetConversationIDBySessionType(constant.ReadGroupChatType, groupID) - maxSeq, err := c.msgRpcClient.GetConversationMaxSeq(ctx, conversationID) + maxSeq, err := rpccall.ExtractField(ctx, msg.GetConversationMaxSeqCaller.Invoke, + &msg.GetConversationMaxSeqReq{ConversationID: conversationID}, + (*msg.GetConversationMaxSeqResp).GetMaxSeq) if err != nil { return err } - return c.conversationRpcClient.SetConversationMaxSeq(ctx, userIDs, conversationID, maxSeq) + return pbconv.SetConversationMaxSeqCaller.Execute(ctx, &pbconv.SetConversationMaxSeqReq{ + ConversationID: conversationID, + OwnerUserID: userIDs, + MaxSeq: maxSeq, + }) } func unmarshalNotificationElem(bytes []byte, t any) error { diff --git a/internal/rpc/auth/auth.go b/internal/rpc/auth/auth.go index 987ccd4ac..c220863c6 100644 --- a/internal/rpc/auth/auth.go +++ b/internal/rpc/auth/auth.go @@ -42,7 +42,6 @@ import ( type authServer struct { pbauth.UnimplementedAuthServer authDatabase controller.AuthDatabase - userRpcClient *rpcclient.UserRpcClient RegisterCenter discovery.SvcDiscoveryRegistry config *Config } @@ -59,9 +58,7 @@ func Start(ctx context.Context, config *Config, client discovery.SvcDiscoveryReg if err != nil { return err } - userRpcClient := rpcclient.NewUserRpcClient(client, config.Discovery.RpcService.User, config.Share.IMAdminUserID) pbauth.RegisterAuthServer(server, &authServer{ - userRpcClient: &userRpcClient, RegisterCenter: client, authDatabase: controller.NewAuthDatabase( redis2.NewTokenCacheModel(rdb, config.RpcConfig.TokenPolicy.Expire), @@ -86,7 +83,7 @@ func (s *authServer) GetAdminToken(ctx context.Context, req *pbauth.GetAdminToke } - if _, err := s.userRpcClient.GetUserInfo(ctx, req.UserID); err != nil { + if _, err := rpcclient.GetUserInfo(ctx, req.UserID); err != nil { return nil, err } @@ -115,7 +112,7 @@ func (s *authServer) GetUserToken(ctx context.Context, req *pbauth.GetUserTokenR if authverify.IsManagerUserID(req.UserID, s.config.Share.IMAdminUserID) { return nil, errs.ErrNoPermission.WrapMsg("don't get Admin token") } - if _, err := s.userRpcClient.GetUserInfo(ctx, req.UserID); err != nil { + if _, err := rpcclient.GetUserInfo(ctx, req.UserID); err != nil { return nil, err } token, err := s.authDatabase.CreateToken(ctx, req.UserID, int(req.PlatformID)) diff --git a/internal/rpc/conversation/conversation.go b/internal/rpc/conversation/conversation.go index 8ed1323d5..f534ca6de 100644 --- a/internal/rpc/conversation/conversation.go +++ b/internal/rpc/conversation/conversation.go @@ -16,8 +16,6 @@ package conversation import ( "context" - "github.com/openimsdk/open-im-server/v3/pkg/msgprocessor" - pbmsg "github.com/openimsdk/protocol/msg" "sort" "time" @@ -27,6 +25,10 @@ import ( "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" dbModel "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" "github.com/openimsdk/open-im-server/v3/pkg/localcache" + "github.com/openimsdk/open-im-server/v3/pkg/msgprocessor" + pbgroup "github.com/openimsdk/protocol/group" + pbmsg "github.com/openimsdk/protocol/msg" + "github.com/openimsdk/protocol/rpccall" "github.com/openimsdk/tools/db/redisutil" "github.com/openimsdk/open-im-server/v3/pkg/common/convert" @@ -46,9 +48,6 @@ import ( type conversationServer struct { pbconversation.UnimplementedConversationServer - msgRpcClient *rpcclient.MessageRpcClient - user *rpcclient.UserRpcClient - groupRpcClient *rpcclient.GroupRpcClient conversationDatabase controller.ConversationDatabase conversationNotificationSender *ConversationNotificationSender @@ -78,15 +77,9 @@ func Start(ctx context.Context, config *Config, client discovery.SvcDiscoveryReg if err != nil { return err } - groupRpcClient := rpcclient.NewGroupRpcClient(client, config.Discovery.RpcService.Group) - msgRpcClient := rpcclient.NewMessageRpcClient(client, config.Discovery.RpcService.Msg) - userRpcClient := rpcclient.NewUserRpcClient(client, config.Discovery.RpcService.User, config.Share.IMAdminUserID) localcache.InitLocalCache(&config.LocalCacheConfig) pbconversation.RegisterConversationServer(server, &conversationServer{ - msgRpcClient: &msgRpcClient, - user: &userRpcClient, - conversationNotificationSender: NewConversationNotificationSender(&config.NotificationConfig, &msgRpcClient), - groupRpcClient: &groupRpcClient, + conversationNotificationSender: NewConversationNotificationSender(&config.NotificationConfig), conversationDatabase: controller.NewConversationDatabase(conversationDB, redis.NewConversationRedis(rdb, &config.LocalCacheConfig, redis.GetRocksCacheOptions(), conversationDB), mgocli.GetTx()), }) @@ -126,12 +119,18 @@ func (c *conversationServer) GetSortedConversationList(ctx context.Context, req return nil, errs.ErrRecordNotFound.Wrap() } - maxSeqs, err := c.msgRpcClient.GetMaxSeqs(ctx, conversationIDs) + maxSeqs, err := rpccall.ExtractField(ctx, pbmsg.GetMaxSeqsCaller.Invoke, + &pbmsg.GetMaxSeqsReq{ConversationIDs: conversationIDs}, + (*pbmsg.SeqsInfoResp).GetMaxSeqs) if err != nil { return nil, err } - chatLogs, err := c.msgRpcClient.GetMsgByConversationIDs(ctx, conversationIDs, maxSeqs) + chatLogs, err := rpccall.ExtractField(ctx, pbmsg.GetMsgByConversationIDsCaller.Invoke, + &pbmsg.GetMsgByConversationIDsReq{ + ConversationIDs: conversationIDs, + MaxSeqs: maxSeqs, + }, (*pbmsg.GetMsgByConversationIDsResp).GetMsgDatas) if err != nil { return nil, err } @@ -141,7 +140,9 @@ func (c *conversationServer) GetSortedConversationList(ctx context.Context, req return nil, err } - hasReadSeqs, err := c.msgRpcClient.GetHasReadSeqs(ctx, req.UserID, conversationIDs) + hasReadSeqs, err := rpccall.ExtractField(ctx, pbmsg.GetHasReadSeqsCaller.Invoke, + &pbmsg.GetHasReadSeqsReq{ConversationIDs: conversationIDs}, + (*pbmsg.SeqsInfoResp).GetMaxSeqs) if err != nil { return nil, err } @@ -228,12 +229,21 @@ func (c *conversationServer) SetConversations(ctx context.Context, req *pbconver if req.Conversation == nil { return nil, errs.ErrArgs.WrapMsg("conversation must not be nil") } - if req.Conversation.ConversationType == constant.WriteGroupChatType { - groupInfo, err := c.groupRpcClient.GetGroupInfo(ctx, req.Conversation.GroupID) + groupInfo, err := rpccall.ExtractField(ctx, pbgroup.GetGroupsInfoCaller.Invoke, + &pbgroup.GetGroupsInfoReq{GroupIDs: []string{req.Conversation.GroupID}}, + func(r *pbgroup.GetGroupsInfoResp) *sdkws.GroupInfo { + if len(r.GroupInfos) > 0 { + return r.GroupInfos[0] + } + return nil + }) if err != nil { return nil, err } + if groupInfo == nil { + return nil, servererrs.ErrGroupIDNotFound.WrapMsg(req.Conversation.GroupID) + } if groupInfo.Status == constant.GroupStatusDismissed { return nil, servererrs.ErrDismissedAlready.WrapMsg("group dismissed") } @@ -434,14 +444,14 @@ func (c *conversationServer) CreateGroupChatConversations(ctx context.Context, r return nil, err } conversationID := msgprocessor.GetConversationIDBySessionType(constant.ReadGroupChatType, req.GroupID) - if _, err := c.msgRpcClient.Client.SetUserConversationMaxSeq(ctx, &pbmsg.SetUserConversationMaxSeqReq{ConversationID: conversationID, OwnerUserID: req.UserIDs, MaxSeq: 0}); err != nil { + if _, err := pbmsg.SetUserConversationMaxSeqCaller.Invoke(ctx, &pbmsg.SetUserConversationMaxSeqReq{ConversationID: conversationID, OwnerUserID: req.UserIDs, MaxSeq: 0}); err != nil { return nil, err } return &pbconversation.CreateGroupChatConversationsResp{}, nil } func (c *conversationServer) SetConversationMaxSeq(ctx context.Context, req *pbconversation.SetConversationMaxSeqReq) (*pbconversation.SetConversationMaxSeqResp, error) { - if _, err := c.msgRpcClient.Client.SetUserConversationMaxSeq(ctx, &pbmsg.SetUserConversationMaxSeqReq{ConversationID: req.ConversationID, OwnerUserID: req.OwnerUserID, MaxSeq: req.MaxSeq}); err != nil { + if _, err := pbmsg.SetUserConversationMaxSeqCaller.Invoke(ctx, &pbmsg.SetUserConversationMaxSeqReq{ConversationID: req.ConversationID, OwnerUserID: req.OwnerUserID, MaxSeq: req.MaxSeq}); err != nil { return nil, err } if err := c.conversationDatabase.UpdateUsersConversationField(ctx, req.OwnerUserID, req.ConversationID, @@ -455,7 +465,7 @@ func (c *conversationServer) SetConversationMaxSeq(ctx context.Context, req *pbc } func (c *conversationServer) SetConversationMinSeq(ctx context.Context, req *pbconversation.SetConversationMinSeqReq) (*pbconversation.SetConversationMinSeqResp, error) { - if _, err := c.msgRpcClient.Client.SetUserConversationMinSeq(ctx, &pbmsg.SetUserConversationMinSeqReq{ConversationID: req.ConversationID, OwnerUserID: req.OwnerUserID, MinSeq: req.MinSeq}); err != nil { + if _, err := pbmsg.SetUserConversationMinSeqCaller.Invoke(ctx, &pbmsg.SetUserConversationMinSeqReq{ConversationID: req.ConversationID, OwnerUserID: req.OwnerUserID, MinSeq: req.MinSeq}); err != nil { return nil, err } if err := c.conversationDatabase.UpdateUsersConversationField(ctx, req.OwnerUserID, req.ConversationID, @@ -565,7 +575,7 @@ func (c *conversationServer) getConversationInfo( } } if len(sendIDs) != 0 { - sendInfos, err := c.user.GetUsersInfo(ctx, sendIDs) + sendInfos, err := rpcclient.GetUsersInfo(ctx, sendIDs) if err != nil { return nil, err } @@ -574,7 +584,9 @@ func (c *conversationServer) getConversationInfo( } } if len(groupIDs) != 0 { - groupInfos, err := c.groupRpcClient.GetGroupInfos(ctx, groupIDs, false) + groupInfos, err := rpccall.ExtractField(ctx, pbgroup.GetGroupsInfoCaller.Invoke, + &pbgroup.GetGroupsInfoReq{GroupIDs: groupIDs}, + (*pbgroup.GetGroupsInfoResp).GetGroupInfos) if err != nil { return nil, err } diff --git a/internal/rpc/conversation/notification.go b/internal/rpc/conversation/notification.go index 994e1d57a..f94c0cd07 100644 --- a/internal/rpc/conversation/notification.go +++ b/internal/rpc/conversation/notification.go @@ -27,8 +27,8 @@ type ConversationNotificationSender struct { *rpcclient.NotificationSender } -func NewConversationNotificationSender(conf *config.Notification, msgRpcClient *rpcclient.MessageRpcClient) *ConversationNotificationSender { - return &ConversationNotificationSender{rpcclient.NewNotificationSender(conf, rpcclient.WithRpcClient(msgRpcClient))} +func NewConversationNotificationSender(conf *config.Notification) *ConversationNotificationSender { + return &ConversationNotificationSender{rpcclient.NewNotificationSender(conf, rpcclient.WithRpcClient())} } // SetPrivate invote. diff --git a/internal/rpc/group/cache.go b/internal/rpc/group/cache.go index 022a0f4ef..bcd246267 100644 --- a/internal/rpc/group/cache.go +++ b/internal/rpc/group/cache.go @@ -22,8 +22,8 @@ import ( ) // GetGroupInfoCache get group info from cache. -func (s *groupServer) GetGroupInfoCache(ctx context.Context, req *pbgroup.GetGroupInfoCacheReq) (*pbgroup.GetGroupInfoCacheResp, error) { - group, err := s.db.TakeGroup(ctx, req.GroupID) +func (g *groupServer) GetGroupInfoCache(ctx context.Context, req *pbgroup.GetGroupInfoCacheReq) (*pbgroup.GetGroupInfoCacheResp, error) { + group, err := g.db.TakeGroup(ctx, req.GroupID) if err != nil { return nil, err } @@ -32,8 +32,8 @@ func (s *groupServer) GetGroupInfoCache(ctx context.Context, req *pbgroup.GetGro }, nil } -func (s *groupServer) GetGroupMemberCache(ctx context.Context, req *pbgroup.GetGroupMemberCacheReq) (*pbgroup.GetGroupMemberCacheResp, error) { - members, err := s.db.TakeGroupMember(ctx, req.GroupID, req.GroupMemberID) +func (g *groupServer) GetGroupMemberCache(ctx context.Context, req *pbgroup.GetGroupMemberCacheReq) (*pbgroup.GetGroupMemberCacheResp, error) { + members, err := g.db.TakeGroupMember(ctx, req.GroupID, req.GroupMemberID) if err != nil { return nil, err } diff --git a/internal/rpc/group/callback.go b/internal/rpc/group/callback.go index 559d64ff4..8ed4afe5a 100644 --- a/internal/rpc/group/callback.go +++ b/internal/rpc/group/callback.go @@ -32,7 +32,7 @@ import ( ) // CallbackBeforeCreateGroup callback before create group. -func (s *groupServer) webhookBeforeCreateGroup(ctx context.Context, before *config.BeforeConfig, req *group.CreateGroupReq) error { +func (g *groupServer) webhookBeforeCreateGroup(ctx context.Context, before *config.BeforeConfig, req *group.CreateGroupReq) error { return webhook.WithCondition(ctx, before, func(ctx context.Context) error { cbReq := &callbackstruct.CallbackBeforeCreateGroupReq{ CallbackCommand: callbackstruct.CallbackBeforeCreateGroupCommand, @@ -57,7 +57,7 @@ func (s *groupServer) webhookBeforeCreateGroup(ctx context.Context, before *conf } resp := &callbackstruct.CallbackBeforeCreateGroupResp{} - if err := s.webhookClient.SyncPost(ctx, cbReq.GetCallbackCommand(), cbReq, resp, before); err != nil { + if err := g.webhookClient.SyncPost(ctx, cbReq.GetCallbackCommand(), cbReq, resp, before); err != nil { return err } @@ -77,7 +77,7 @@ func (s *groupServer) webhookBeforeCreateGroup(ctx context.Context, before *conf }) } -func (s *groupServer) webhookAfterCreateGroup(ctx context.Context, after *config.AfterConfig, req *group.CreateGroupReq) { +func (g *groupServer) webhookAfterCreateGroup(ctx context.Context, after *config.AfterConfig, req *group.CreateGroupReq) { cbReq := &callbackstruct.CallbackAfterCreateGroupReq{ CallbackCommand: callbackstruct.CallbackAfterCreateGroupCommand, GroupInfo: req.GroupInfo, @@ -98,10 +98,10 @@ func (s *groupServer) webhookAfterCreateGroup(ctx context.Context, after *config RoleLevel: constant.GroupOrdinaryUsers, }) } - s.webhookClient.AsyncPost(ctx, cbReq.GetCallbackCommand(), cbReq, &callbackstruct.CallbackAfterCreateGroupResp{}, after) + g.webhookClient.AsyncPost(ctx, cbReq.GetCallbackCommand(), cbReq, &callbackstruct.CallbackAfterCreateGroupResp{}, after) } -func (s *groupServer) webhookBeforeMembersJoinGroup(ctx context.Context, before *config.BeforeConfig, groupMembers []*model.GroupMember, groupID string, groupEx string) error { +func (g *groupServer) webhookBeforeMembersJoinGroup(ctx context.Context, before *config.BeforeConfig, groupMembers []*model.GroupMember, groupID string, groupEx string) error { return webhook.WithCondition(ctx, before, func(ctx context.Context) error { groupMembersMap := datautil.SliceToMap(groupMembers, func(e *model.GroupMember) string { return e.UserID @@ -123,7 +123,7 @@ func (s *groupServer) webhookBeforeMembersJoinGroup(ctx context.Context, before } resp := &callbackstruct.CallbackBeforeMembersJoinGroupResp{} - if err := s.webhookClient.SyncPost(ctx, cbReq.GetCallbackCommand(), cbReq, resp, before); err != nil { + if err := g.webhookClient.SyncPost(ctx, cbReq.GetCallbackCommand(), cbReq, resp, before); err != nil { return err } @@ -144,7 +144,7 @@ func (s *groupServer) webhookBeforeMembersJoinGroup(ctx context.Context, before }) } -func (s *groupServer) webhookBeforeSetGroupMemberInfo(ctx context.Context, before *config.BeforeConfig, req *group.SetGroupMemberInfo) error { +func (g *groupServer) webhookBeforeSetGroupMemberInfo(ctx context.Context, before *config.BeforeConfig, req *group.SetGroupMemberInfo) error { return webhook.WithCondition(ctx, before, func(ctx context.Context) error { cbReq := callbackstruct.CallbackBeforeSetGroupMemberInfoReq{ CallbackCommand: callbackstruct.CallbackBeforeSetGroupMemberInfoCommand, @@ -164,7 +164,7 @@ func (s *groupServer) webhookBeforeSetGroupMemberInfo(ctx context.Context, befor cbReq.Ex = &req.Ex.Value } resp := &callbackstruct.CallbackBeforeSetGroupMemberInfoResp{} - if err := s.webhookClient.SyncPost(ctx, cbReq.GetCallbackCommand(), cbReq, resp, before); err != nil { + if err := g.webhookClient.SyncPost(ctx, cbReq.GetCallbackCommand(), cbReq, resp, before); err != nil { return err } if resp.FaceURL != nil { @@ -183,7 +183,7 @@ func (s *groupServer) webhookBeforeSetGroupMemberInfo(ctx context.Context, befor }) } -func (s *groupServer) webhookAfterSetGroupMemberInfo(ctx context.Context, after *config.AfterConfig, req *group.SetGroupMemberInfo) { +func (g *groupServer) webhookAfterSetGroupMemberInfo(ctx context.Context, after *config.AfterConfig, req *group.SetGroupMemberInfo) { cbReq := callbackstruct.CallbackAfterSetGroupMemberInfoReq{ CallbackCommand: callbackstruct.CallbackAfterSetGroupMemberInfoCommand, GroupID: req.GroupID, @@ -201,55 +201,55 @@ func (s *groupServer) webhookAfterSetGroupMemberInfo(ctx context.Context, after if req.Ex != nil { cbReq.Ex = &req.Ex.Value } - s.webhookClient.AsyncPost(ctx, cbReq.GetCallbackCommand(), cbReq, &callbackstruct.CallbackAfterSetGroupMemberInfoResp{}, after) + g.webhookClient.AsyncPost(ctx, cbReq.GetCallbackCommand(), cbReq, &callbackstruct.CallbackAfterSetGroupMemberInfoResp{}, after) } -func (s *groupServer) webhookAfterQuitGroup(ctx context.Context, after *config.AfterConfig, req *group.QuitGroupReq) { +func (g *groupServer) webhookAfterQuitGroup(ctx context.Context, after *config.AfterConfig, req *group.QuitGroupReq) { cbReq := &callbackstruct.CallbackQuitGroupReq{ CallbackCommand: callbackstruct.CallbackAfterQuitGroupCommand, GroupID: req.GroupID, UserID: req.UserID, } - s.webhookClient.AsyncPost(ctx, cbReq.GetCallbackCommand(), cbReq, &callbackstruct.CallbackQuitGroupResp{}, after) + g.webhookClient.AsyncPost(ctx, cbReq.GetCallbackCommand(), cbReq, &callbackstruct.CallbackQuitGroupResp{}, after) } -func (s *groupServer) webhookAfterKickGroupMember(ctx context.Context, after *config.AfterConfig, req *group.KickGroupMemberReq) { +func (g *groupServer) webhookAfterKickGroupMember(ctx context.Context, after *config.AfterConfig, req *group.KickGroupMemberReq) { cbReq := &callbackstruct.CallbackKillGroupMemberReq{ CallbackCommand: callbackstruct.CallbackAfterKickGroupCommand, GroupID: req.GroupID, KickedUserIDs: req.KickedUserIDs, Reason: req.Reason, } - s.webhookClient.AsyncPost(ctx, cbReq.GetCallbackCommand(), cbReq, &callbackstruct.CallbackKillGroupMemberResp{}, after) + g.webhookClient.AsyncPost(ctx, cbReq.GetCallbackCommand(), cbReq, &callbackstruct.CallbackKillGroupMemberResp{}, after) } -func (s *groupServer) webhookAfterDismissGroup(ctx context.Context, after *config.AfterConfig, req *callbackstruct.CallbackDisMissGroupReq) { +func (g *groupServer) webhookAfterDismissGroup(ctx context.Context, after *config.AfterConfig, req *callbackstruct.CallbackDisMissGroupReq) { req.CallbackCommand = callbackstruct.CallbackAfterDisMissGroupCommand - s.webhookClient.AsyncPost(ctx, req.GetCallbackCommand(), req, &callbackstruct.CallbackDisMissGroupResp{}, after) + g.webhookClient.AsyncPost(ctx, req.GetCallbackCommand(), req, &callbackstruct.CallbackDisMissGroupResp{}, after) } -func (s *groupServer) webhookBeforeApplyJoinGroup(ctx context.Context, before *config.BeforeConfig, req *callbackstruct.CallbackJoinGroupReq) (err error) { +func (g *groupServer) webhookBeforeApplyJoinGroup(ctx context.Context, before *config.BeforeConfig, req *callbackstruct.CallbackJoinGroupReq) (err error) { return webhook.WithCondition(ctx, before, func(ctx context.Context) error { req.CallbackCommand = callbackstruct.CallbackBeforeJoinGroupCommand resp := &callbackstruct.CallbackJoinGroupResp{} - if err := s.webhookClient.SyncPost(ctx, req.GetCallbackCommand(), req, resp, before); err != nil { + if err := g.webhookClient.SyncPost(ctx, req.GetCallbackCommand(), req, resp, before); err != nil { return err } return nil }) } -func (s *groupServer) webhookAfterTransferGroupOwner(ctx context.Context, after *config.AfterConfig, req *group.TransferGroupOwnerReq) { +func (g *groupServer) webhookAfterTransferGroupOwner(ctx context.Context, after *config.AfterConfig, req *group.TransferGroupOwnerReq) { cbReq := &callbackstruct.CallbackTransferGroupOwnerReq{ CallbackCommand: callbackstruct.CallbackAfterTransferGroupOwnerCommand, GroupID: req.GroupID, OldOwnerUserID: req.OldOwnerUserID, NewOwnerUserID: req.NewOwnerUserID, } - s.webhookClient.AsyncPost(ctx, cbReq.GetCallbackCommand(), cbReq, &callbackstruct.CallbackTransferGroupOwnerResp{}, after) + g.webhookClient.AsyncPost(ctx, cbReq.GetCallbackCommand(), cbReq, &callbackstruct.CallbackTransferGroupOwnerResp{}, after) } -func (s *groupServer) webhookBeforeInviteUserToGroup(ctx context.Context, before *config.BeforeConfig, req *group.InviteUserToGroupReq) (err error) { +func (g *groupServer) webhookBeforeInviteUserToGroup(ctx context.Context, before *config.BeforeConfig, req *group.InviteUserToGroupReq) (err error) { return webhook.WithCondition(ctx, before, func(ctx context.Context) error { cbReq := &callbackstruct.CallbackBeforeInviteUserToGroupReq{ CallbackCommand: callbackstruct.CallbackBeforeInviteJoinGroupCommand, @@ -260,7 +260,7 @@ func (s *groupServer) webhookBeforeInviteUserToGroup(ctx context.Context, before } resp := &callbackstruct.CallbackBeforeInviteUserToGroupResp{} - if err := s.webhookClient.SyncPost(ctx, cbReq.GetCallbackCommand(), cbReq, resp, before); err != nil { + if err := g.webhookClient.SyncPost(ctx, cbReq.GetCallbackCommand(), cbReq, resp, before); err != nil { return err } @@ -275,7 +275,7 @@ func (s *groupServer) webhookBeforeInviteUserToGroup(ctx context.Context, before }) } -func (s *groupServer) webhookAfterJoinGroup(ctx context.Context, after *config.AfterConfig, req *group.JoinGroupReq) { +func (g *groupServer) webhookAfterJoinGroup(ctx context.Context, after *config.AfterConfig, req *group.JoinGroupReq) { cbReq := &callbackstruct.CallbackAfterJoinGroupReq{ CallbackCommand: callbackstruct.CallbackAfterJoinGroupCommand, OperationID: mcontext.GetOperationID(ctx), @@ -284,10 +284,10 @@ func (s *groupServer) webhookAfterJoinGroup(ctx context.Context, after *config.A JoinSource: req.JoinSource, InviterUserID: req.InviterUserID, } - s.webhookClient.AsyncPost(ctx, cbReq.GetCallbackCommand(), cbReq, &callbackstruct.CallbackAfterJoinGroupResp{}, after) + g.webhookClient.AsyncPost(ctx, cbReq.GetCallbackCommand(), cbReq, &callbackstruct.CallbackAfterJoinGroupResp{}, after) } -func (s *groupServer) webhookBeforeSetGroupInfo(ctx context.Context, before *config.BeforeConfig, req *group.SetGroupInfoReq) error { +func (g *groupServer) webhookBeforeSetGroupInfo(ctx context.Context, before *config.BeforeConfig, req *group.SetGroupInfoReq) error { return webhook.WithCondition(ctx, before, func(ctx context.Context) error { cbReq := &callbackstruct.CallbackBeforeSetGroupInfoReq{ CallbackCommand: callbackstruct.CallbackBeforeSetGroupInfoCommand, @@ -312,7 +312,7 @@ func (s *groupServer) webhookBeforeSetGroupInfo(ctx context.Context, before *con } resp := &callbackstruct.CallbackBeforeSetGroupInfoResp{} - if err := s.webhookClient.SyncPost(ctx, cbReq.GetCallbackCommand(), cbReq, resp, before); err != nil { + if err := g.webhookClient.SyncPost(ctx, cbReq.GetCallbackCommand(), cbReq, resp, before); err != nil { return err } @@ -336,7 +336,7 @@ func (s *groupServer) webhookBeforeSetGroupInfo(ctx context.Context, before *con }) } -func (s *groupServer) webhookAfterSetGroupInfo(ctx context.Context, after *config.AfterConfig, req *group.SetGroupInfoReq) { +func (g *groupServer) webhookAfterSetGroupInfo(ctx context.Context, after *config.AfterConfig, req *group.SetGroupInfoReq) { cbReq := &callbackstruct.CallbackAfterSetGroupInfoReq{ CallbackCommand: callbackstruct.CallbackAfterSetGroupInfoCommand, GroupID: req.GroupInfoForSet.GroupID, @@ -357,10 +357,10 @@ func (s *groupServer) webhookAfterSetGroupInfo(ctx context.Context, after *confi if req.GroupInfoForSet.ApplyMemberFriend != nil { cbReq.ApplyMemberFriend = &req.GroupInfoForSet.ApplyMemberFriend.Value } - s.webhookClient.AsyncPost(ctx, cbReq.GetCallbackCommand(), cbReq, &callbackstruct.CallbackAfterSetGroupInfoResp{}, after) + g.webhookClient.AsyncPost(ctx, cbReq.GetCallbackCommand(), cbReq, &callbackstruct.CallbackAfterSetGroupInfoResp{}, after) } -func (s *groupServer) webhookBeforeSetGroupInfoEx(ctx context.Context, before *config.BeforeConfig, req *group.SetGroupInfoExReq) error { +func (g *groupServer) webhookBeforeSetGroupInfoEx(ctx context.Context, before *config.BeforeConfig, req *group.SetGroupInfoExReq) error { return webhook.WithCondition(ctx, before, func(ctx context.Context) error { cbReq := &callbackstruct.CallbackBeforeSetGroupInfoExReq{ CallbackCommand: callbackstruct.CallbackBeforeSetGroupInfoExCommand, @@ -388,7 +388,7 @@ func (s *groupServer) webhookBeforeSetGroupInfoEx(ctx context.Context, before *c resp := &callbackstruct.CallbackBeforeSetGroupInfoExResp{} - if err := s.webhookClient.SyncPost(ctx, cbReq.GetCallbackCommand(), cbReq, resp, before); err != nil { + if err := g.webhookClient.SyncPost(ctx, cbReq.GetCallbackCommand(), cbReq, resp, before); err != nil { return err } @@ -405,7 +405,7 @@ func (s *groupServer) webhookBeforeSetGroupInfoEx(ctx context.Context, before *c }) } -func (s *groupServer) webhookAfterSetGroupInfoEx(ctx context.Context, after *config.AfterConfig, req *group.SetGroupInfoExReq) { +func (g *groupServer) webhookAfterSetGroupInfoEx(ctx context.Context, after *config.AfterConfig, req *group.SetGroupInfoExReq) { cbReq := &callbackstruct.CallbackAfterSetGroupInfoExReq{ CallbackCommand: callbackstruct.CallbackAfterSetGroupInfoExCommand, GroupID: req.GroupID, @@ -428,5 +428,5 @@ func (s *groupServer) webhookAfterSetGroupInfoEx(ctx context.Context, after *con cbReq.ApplyMemberFriend = req.ApplyMemberFriend } - s.webhookClient.AsyncPost(ctx, cbReq.GetCallbackCommand(), cbReq, &callbackstruct.CallbackAfterSetGroupInfoExResp{}, after) + g.webhookClient.AsyncPost(ctx, cbReq.GetCallbackCommand(), cbReq, &callbackstruct.CallbackAfterSetGroupInfoExResp{}, after) } diff --git a/internal/rpc/group/convert.go b/internal/rpc/group/convert.go index 8026430c3..26277a56d 100644 --- a/internal/rpc/group/convert.go +++ b/internal/rpc/group/convert.go @@ -19,7 +19,7 @@ import ( "github.com/openimsdk/protocol/sdkws" ) -func (s *groupServer) groupDB2PB(group *model.Group, ownerUserID string, memberCount uint32) *sdkws.GroupInfo { +func (g *groupServer) groupDB2PB(group *model.Group, ownerUserID string, memberCount uint32) *sdkws.GroupInfo { return &sdkws.GroupInfo{ GroupID: group.GroupID, GroupName: group.GroupName, @@ -41,7 +41,7 @@ func (s *groupServer) groupDB2PB(group *model.Group, ownerUserID string, memberC } } -func (s *groupServer) groupMemberDB2PB(member *model.GroupMember, appMangerLevel int32) *sdkws.GroupMemberFullInfo { +func (g *groupServer) groupMemberDB2PB(member *model.GroupMember, appMangerLevel int32) *sdkws.GroupMemberFullInfo { return &sdkws.GroupMemberFullInfo{ GroupID: member.GroupID, UserID: member.UserID, @@ -58,6 +58,6 @@ func (s *groupServer) groupMemberDB2PB(member *model.GroupMember, appMangerLevel } } -func (s *groupServer) groupMemberDB2PB2(member *model.GroupMember) *sdkws.GroupMemberFullInfo { - return s.groupMemberDB2PB(member, 0) +func (g *groupServer) groupMemberDB2PB2(member *model.GroupMember) *sdkws.GroupMemberFullInfo { + return g.groupMemberDB2PB(member, 0) } diff --git a/internal/rpc/group/fill.go b/internal/rpc/group/fill.go index 1c86481df..186506a55 100644 --- a/internal/rpc/group/fill.go +++ b/internal/rpc/group/fill.go @@ -16,9 +16,10 @@ package group import ( "context" + relationtb "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" ) -func (s *groupServer) PopulateGroupMember(ctx context.Context, members ...*relationtb.GroupMember) error { - return s.notification.PopulateGroupMember(ctx, members...) +func (g *groupServer) PopulateGroupMember(ctx context.Context, members ...*relationtb.GroupMember) error { + return g.notification.PopulateGroupMember(ctx, members...) } diff --git a/internal/rpc/group/group.go b/internal/rpc/group/group.go index 62020f980..8af09b4c5 100644 --- a/internal/rpc/group/group.go +++ b/internal/rpc/group/group.go @@ -23,25 +23,26 @@ import ( "strings" "time" + "github.com/openimsdk/open-im-server/v3/pkg/authverify" + "github.com/openimsdk/open-im-server/v3/pkg/callbackstruct" "github.com/openimsdk/open-im-server/v3/pkg/common/config" + "github.com/openimsdk/open-im-server/v3/pkg/common/convert" + "github.com/openimsdk/open-im-server/v3/pkg/common/servererrs" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/common" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/controller" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database/mgo" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" "github.com/openimsdk/open-im-server/v3/pkg/common/webhook" "github.com/openimsdk/open-im-server/v3/pkg/localcache" - - "github.com/openimsdk/open-im-server/v3/pkg/authverify" - "github.com/openimsdk/open-im-server/v3/pkg/callbackstruct" - "github.com/openimsdk/open-im-server/v3/pkg/common/convert" - "github.com/openimsdk/open-im-server/v3/pkg/common/servererrs" - "github.com/openimsdk/open-im-server/v3/pkg/common/storage/controller" "github.com/openimsdk/open-im-server/v3/pkg/msgprocessor" "github.com/openimsdk/open-im-server/v3/pkg/rpcclient" "github.com/openimsdk/open-im-server/v3/pkg/rpcclient/grouphash" "github.com/openimsdk/open-im-server/v3/pkg/rpcclient/notification" "github.com/openimsdk/protocol/constant" - pbconversation "github.com/openimsdk/protocol/conversation" + pbconv "github.com/openimsdk/protocol/conversation" pbgroup "github.com/openimsdk/protocol/group" + "github.com/openimsdk/protocol/msg" + "github.com/openimsdk/protocol/rpccall" "github.com/openimsdk/protocol/sdkws" "github.com/openimsdk/protocol/wrapperspb" "github.com/openimsdk/tools/db/mongoutil" @@ -58,13 +59,10 @@ import ( type groupServer struct { pbgroup.UnimplementedGroupServer - db controller.GroupDatabase - user rpcclient.UserRpcClient - notification *GroupNotificationSender - conversationRpcClient rpcclient.ConversationRpcClient - msgRpcClient rpcclient.MessageRpcClient - config *Config - webhookClient *webhook.Client + db controller.GroupDatabase + notification *GroupNotificationSender + config *Config + webhookClient *webhook.Client } type Config struct { @@ -99,30 +97,22 @@ func Start(ctx context.Context, config *Config, client discovery.SvcDiscoveryReg if err != nil { return err } - userRpcClient := rpcclient.NewUserRpcClient(client, config.Discovery.RpcService.User, config.Share.IMAdminUserID) - msgRpcClient := rpcclient.NewMessageRpcClient(client, config.Discovery.RpcService.Msg) - conversationRpcClient := rpcclient.NewConversationRpcClient(client, config.Discovery.RpcService.Conversation) var gs groupServer database := controller.NewGroupDatabase(rdb, &config.LocalCacheConfig, groupDB, groupMemberDB, groupRequestDB, mgocli.GetTx(), grouphash.NewGroupHashFromGroupServer(&gs)) gs.db = database - gs.user = userRpcClient gs.notification = NewGroupNotificationSender( database, - &msgRpcClient, - &userRpcClient, - &conversationRpcClient, config, func(ctx context.Context, userIDs []string) ([]notification.CommonUser, error) { - users, err := userRpcClient.GetUsersInfo(ctx, userIDs) + users, err := rpcclient.GetUsersInfo(ctx, userIDs) if err != nil { return nil, err } + return datautil.Slice(users, func(e *sdkws.UserInfo) notification.CommonUser { return e }), nil }, ) localcache.InitLocalCache(&config.LocalCacheConfig) - gs.conversationRpcClient = conversationRpcClient - gs.msgRpcClient = msgRpcClient gs.config = config gs.webhookClient = webhook.NewWebhookClient(config.WebhooksConfig.URL) pbgroup.RegisterGroupServer(server, &gs) @@ -172,7 +162,7 @@ func (g *groupServer) GetPublicUserInfoMap(ctx context.Context, userIDs []string if len(userIDs) == 0 { return map[string]*sdkws.PublicUserInfo{}, nil } - users, err := g.user.GetPublicUserInfos(ctx, userIDs) + users, err := rpcclient.GetPublicUserInfos(ctx, userIDs) if err != nil { return nil, err } @@ -235,7 +225,7 @@ func (g *groupServer) CreateGroup(ctx context.Context, req *pbgroup.CreateGroupR return nil, errs.ErrArgs.WrapMsg("group member repeated") } - userMap, err := g.user.GetUsersInfoMap(ctx, userIDs) + userMap, err := rpcclient.GetUsersInfoMap(ctx, userIDs) if err != nil { return nil, err } @@ -386,7 +376,7 @@ func (g *groupServer) InviteUserToGroup(ctx context.Context, req *pbgroup.Invite return nil, servererrs.ErrDismissedAlready.WrapMsg("group dismissed checking group status found it dismissed") } - userMap, err := g.user.GetUsersInfoMap(ctx, req.InvitedUserIDs) + userMap, err := rpcclient.GetUsersInfoMap(ctx, req.InvitedUserIDs) if err != nil { return nil, err } @@ -697,7 +687,7 @@ func (g *groupServer) GetGroupApplicationList(ctx context.Context, req *pbgroup. userIDs = append(userIDs, gr.UserID) } userIDs = datautil.Distinct(userIDs) - userMap, err := g.user.GetPublicUserInfoMap(ctx, userIDs) + userMap, err := rpcclient.GetPublicUserInfoMap(ctx, userIDs) if err != nil { return nil, err } @@ -809,7 +799,7 @@ func (g *groupServer) GroupApplicationResponse(ctx context.Context, req *pbgroup } else if !g.IsNotFound(err) { return nil, err } - if _, err := g.user.GetPublicUserInfo(ctx, req.FromUserID); err != nil { + if _, err := rpcclient.GetPublicUserInfo(ctx, req.FromUserID); err != nil { return nil, err } var member *model.GroupMember @@ -853,7 +843,7 @@ func (g *groupServer) GroupApplicationResponse(ctx context.Context, req *pbgroup } func (g *groupServer) JoinGroup(ctx context.Context, req *pbgroup.JoinGroupReq) (*pbgroup.JoinGroupResp, error) { - user, err := g.user.GetUserInfo(ctx, req.InviterUserID) + user, err := rpcclient.GetUserInfo(ctx, req.InviterUserID) if err != nil { return nil, err } @@ -959,12 +949,19 @@ func (g *groupServer) QuitGroup(ctx context.Context, req *pbgroup.QuitGroupReq) } func (g *groupServer) deleteMemberAndSetConversationSeq(ctx context.Context, groupID string, userIDs []string) error { - conevrsationID := msgprocessor.GetConversationIDBySessionType(constant.ReadGroupChatType, groupID) - maxSeq, err := g.msgRpcClient.GetConversationMaxSeq(ctx, conevrsationID) + conversationID := msgprocessor.GetConversationIDBySessionType(constant.ReadGroupChatType, groupID) + maxSeq, err := rpccall.ExtractField(ctx, msg.GetConversationMaxSeqCaller.Invoke, + &msg.GetConversationMaxSeqReq{ConversationID: conversationID}, + (*msg.GetConversationMaxSeqResp).GetMaxSeq) if err != nil { return err } - return g.conversationRpcClient.SetConversationMaxSeq(ctx, userIDs, conevrsationID, maxSeq) + + return pbconv.SetConversationMaxSeqCaller.Execute(ctx, &pbconv.SetConversationMaxSeqReq{ + ConversationID: conversationID, + OwnerUserID: userIDs, + MaxSeq: maxSeq, + }) } func (g *groupServer) SetGroupInfo(ctx context.Context, req *pbgroup.SetGroupInfoReq) (*pbgroup.SetGroupInfoResp, error) { @@ -1029,7 +1026,7 @@ func (g *groupServer) SetGroupInfo(ctx context.Context, req *pbgroup.SetGroupInf if req.GroupInfoForSet.Notification != "" { num -= 3 func() { - conversation := &pbconversation.ConversationReq{ + conversation := &pbconv.ConversationReq{ ConversationID: msgprocessor.GetConversationIDBySessionType(constant.ReadGroupChatType, req.GroupInfoForSet.GroupID), ConversationType: constant.ReadGroupChatType, GroupID: req.GroupInfoForSet.GroupID, @@ -1040,7 +1037,11 @@ func (g *groupServer) SetGroupInfo(ctx context.Context, req *pbgroup.SetGroupInf return } conversation.GroupAtType = &wrapperspb.Int32Value{Value: constant.GroupNotification} - if err := g.conversationRpcClient.SetConversations(ctx, resp.UserIDs, conversation); err != nil { + + if err := pbconv.SetConversationsCaller.Execute(ctx, &pbconv.SetConversationsReq{ + UserIDs: resp.UserIDs, + Conversation: conversation, + }); err != nil { log.ZWarn(ctx, "SetConversations", err, "UserIDs", resp.UserIDs, "conversation", conversation) } }() @@ -1140,7 +1141,7 @@ func (g *groupServer) SetGroupInfoEx(ctx context.Context, req *pbgroup.SetGroupI if req.Notification.Value != "" { func() { - conversation := &pbconversation.ConversationReq{ + conversation := &pbconv.ConversationReq{ ConversationID: msgprocessor.GetConversationIDBySessionType(constant.ReadGroupChatType, req.GroupID), ConversationType: constant.ReadGroupChatType, GroupID: req.GroupID, @@ -1154,7 +1155,10 @@ func (g *groupServer) SetGroupInfoEx(ctx context.Context, req *pbgroup.SetGroupI conversation.GroupAtType = &wrapperspb.Int32Value{Value: constant.GroupNotification} - if err := g.conversationRpcClient.SetConversations(ctx, resp.UserIDs, conversation); err != nil { + if err := pbconv.SetConversationsCaller.Execute(ctx, &pbconv.SetConversationsReq{ + UserIDs: resp.UserIDs, + Conversation: conversation, + }); err != nil { log.ZWarn(ctx, "SetConversations", err, "UserIDs", resp.UserIDs, "conversation", conversation) } }() @@ -1306,7 +1310,7 @@ func (g *groupServer) GetGroupMembersCMS(ctx context.Context, req *pbgroup.GetGr } func (g *groupServer) GetUserReqApplicationList(ctx context.Context, req *pbgroup.GetUserReqApplicationListReq) (*pbgroup.GetUserReqApplicationListResp, error) { - user, err := g.user.GetPublicUserInfo(ctx, req.UserID) + user, err := rpcclient.GetPublicUserInfo(ctx, req.UserID) if err != nil { return nil, err } @@ -1762,7 +1766,7 @@ func (g *groupServer) GetGroupUsersReqApplicationList(ctx context.Context, req * return nil, servererrs.ErrGroupIDNotFound.WrapMsg(strings.Join(ids, ",")) } - userMap, err := g.user.GetPublicUserInfoMap(ctx, req.UserIDs) + userMap, err := rpcclient.GetPublicUserInfoMap(ctx, req.UserIDs) if err != nil { return nil, err } @@ -1839,7 +1843,7 @@ func (g *groupServer) GetSpecifiedUserGroupRequestInfo(ctx context.Context, req return nil, err } - userInfos, err := g.user.GetPublicUserInfos(ctx, []string{req.UserID}) + userInfos, err := rpcclient.GetPublicUserInfos(ctx, []string{req.UserID}) if err != nil { return nil, err } diff --git a/internal/rpc/group/notification.go b/internal/rpc/group/notification.go index 54a6146f5..784ec8943 100644 --- a/internal/rpc/group/notification.go +++ b/internal/rpc/group/notification.go @@ -18,6 +18,8 @@ import ( "context" "errors" "fmt" + "time" + "github.com/openimsdk/open-im-server/v3/pkg/authverify" "github.com/openimsdk/open-im-server/v3/pkg/common/convert" "github.com/openimsdk/open-im-server/v3/pkg/common/servererrs" @@ -29,8 +31,10 @@ import ( "github.com/openimsdk/open-im-server/v3/pkg/rpcclient" "github.com/openimsdk/open-im-server/v3/pkg/rpcclient/notification" "github.com/openimsdk/protocol/constant" + pbconv "github.com/openimsdk/protocol/conversation" pbgroup "github.com/openimsdk/protocol/group" "github.com/openimsdk/protocol/msg" + "github.com/openimsdk/protocol/rpccall" "github.com/openimsdk/protocol/sdkws" "github.com/openimsdk/tools/errs" "github.com/openimsdk/tools/log" @@ -38,7 +42,6 @@ import ( "github.com/openimsdk/tools/utils/datautil" "github.com/openimsdk/tools/utils/stringutil" "go.mongodb.org/mongo-driver/mongo" - "time" ) // GroupApplicationReceiver @@ -49,20 +52,14 @@ const ( func NewGroupNotificationSender( db controller.GroupDatabase, - msgRpcClient *rpcclient.MessageRpcClient, - userRpcClient *rpcclient.UserRpcClient, - conversationRpcClient *rpcclient.ConversationRpcClient, config *Config, fn func(ctx context.Context, userIDs []string) ([]notification.CommonUser, error), ) *GroupNotificationSender { return &GroupNotificationSender{ - NotificationSender: rpcclient.NewNotificationSender(&config.NotificationConfig, rpcclient.WithRpcClient(msgRpcClient), rpcclient.WithUserRpcClient(userRpcClient)), + NotificationSender: rpcclient.NewNotificationSender(&config.NotificationConfig, rpcclient.WithRpcClient(), rpcclient.WithUserRpcClient()), getUsersInfo: fn, db: db, config: config, - - conversationRpcClient: conversationRpcClient, - msgRpcClient: msgRpcClient, } } @@ -71,9 +68,6 @@ type GroupNotificationSender struct { getUsersInfo func(ctx context.Context, userIDs []string) ([]notification.CommonUser, error) db controller.GroupDatabase config *Config - - conversationRpcClient *rpcclient.ConversationRpcClient - msgRpcClient *rpcclient.MessageRpcClient } func (g *GroupNotificationSender) PopulateGroupMember(ctx context.Context, members ...*model.GroupMember) error { @@ -524,11 +518,14 @@ func (g *GroupNotificationSender) GroupApplicationAgreeMemberEnterNotification(c if !g.config.RpcConfig.EnableHistoryForNewMembers { conversationID := msgprocessor.GetConversationIDBySessionType(constant.ReadGroupChatType, groupID) - maxSeq, err := g.msgRpcClient.GetConversationMaxSeq(ctx, conversationID) + maxSeq, err := rpccall.ExtractField(ctx, msg.GetConversationMaxSeqCaller.Invoke, + &msg.GetConversationMaxSeqReq{ConversationID: conversationID}, + (*msg.GetConversationMaxSeqResp).GetMaxSeq) if err != nil { return err } - if _, err = g.msgRpcClient.SetUserConversationsMinSeq(ctx, &msg.SetUserConversationsMinSeqReq{ + + if err := msg.SetUserConversationsMinSeqCaller.Execute(ctx, &msg.SetUserConversationsMinSeqReq{ UserIDs: entrantUserID, ConversationID: conversationID, Seq: maxSeq, @@ -537,7 +534,10 @@ func (g *GroupNotificationSender) GroupApplicationAgreeMemberEnterNotification(c } } - if err := g.conversationRpcClient.GroupChatFirstCreateConversation(ctx, groupID, entrantUserID); err != nil { + if err := pbconv.CreateGroupChatConversationsCaller.Execute(ctx, &pbconv.CreateGroupChatConversationsReq{ + UserIDs: entrantUserID, + GroupID: groupID, + }); err != nil { return err } @@ -583,11 +583,13 @@ func (g *GroupNotificationSender) MemberEnterNotification(ctx context.Context, g if !g.config.RpcConfig.EnableHistoryForNewMembers { conversationID := msgprocessor.GetConversationIDBySessionType(constant.ReadGroupChatType, groupID) - maxSeq, err := g.msgRpcClient.GetConversationMaxSeq(ctx, conversationID) + maxSeq, err := rpccall.ExtractField(ctx, msg.GetConversationMaxSeqCaller.Invoke, + &msg.GetConversationMaxSeqReq{ConversationID: conversationID}, + (*msg.GetConversationMaxSeqResp).GetMaxSeq) if err != nil { return err } - if _, err = g.msgRpcClient.SetUserConversationsMinSeq(ctx, &msg.SetUserConversationsMinSeqReq{ + if err := msg.SetUserConversationsMinSeqCaller.Execute(ctx, &msg.SetUserConversationsMinSeqReq{ UserIDs: []string{entrantUserID}, ConversationID: conversationID, Seq: maxSeq, @@ -596,7 +598,10 @@ func (g *GroupNotificationSender) MemberEnterNotification(ctx context.Context, g } } - if err := g.conversationRpcClient.GroupChatFirstCreateConversation(ctx, groupID, []string{entrantUserID}); err != nil { + if err := pbconv.CreateGroupChatConversationsCaller.Execute(ctx, &pbconv.CreateGroupChatConversationsReq{ + UserIDs: []string{entrantUserID}, + GroupID: groupID, + }); err != nil { return err } diff --git a/internal/rpc/group/statistics.go b/internal/rpc/group/statistics.go index 6adb1261a..1c582fda1 100644 --- a/internal/rpc/group/statistics.go +++ b/internal/rpc/group/statistics.go @@ -22,20 +22,20 @@ import ( "github.com/openimsdk/tools/errs" ) -func (s *groupServer) GroupCreateCount(ctx context.Context, req *group.GroupCreateCountReq) (*group.GroupCreateCountResp, error) { +func (g *groupServer) GroupCreateCount(ctx context.Context, req *group.GroupCreateCountReq) (*group.GroupCreateCountResp, error) { if req.Start > req.End { return nil, errs.ErrArgs.WrapMsg("start > end: %d > %d", req.Start, req.End) } - total, err := s.db.CountTotal(ctx, nil) + total, err := g.db.CountTotal(ctx, nil) if err != nil { return nil, err } start := time.UnixMilli(req.Start) - before, err := s.db.CountTotal(ctx, &start) + before, err := g.db.CountTotal(ctx, &start) if err != nil { return nil, err } - count, err := s.db.CountRangeEverydayTotal(ctx, start, time.UnixMilli(req.End)) + count, err := g.db.CountRangeEverydayTotal(ctx, start, time.UnixMilli(req.End)) if err != nil { return nil, err } diff --git a/internal/rpc/group/sync.go b/internal/rpc/group/sync.go index 0592aa811..50ac8252f 100644 --- a/internal/rpc/group/sync.go +++ b/internal/rpc/group/sync.go @@ -15,12 +15,12 @@ import ( "github.com/openimsdk/tools/log" ) -func (s *groupServer) GetFullGroupMemberUserIDs(ctx context.Context, req *pbgroup.GetFullGroupMemberUserIDsReq) (*pbgroup.GetFullGroupMemberUserIDsResp, error) { - vl, err := s.db.FindMaxGroupMemberVersionCache(ctx, req.GroupID) +func (g *groupServer) GetFullGroupMemberUserIDs(ctx context.Context, req *pbgroup.GetFullGroupMemberUserIDsReq) (*pbgroup.GetFullGroupMemberUserIDsResp, error) { + vl, err := g.db.FindMaxGroupMemberVersionCache(ctx, req.GroupID) if err != nil { return nil, err } - userIDs, err := s.db.FindGroupMemberUserID(ctx, req.GroupID) + userIDs, err := g.db.FindGroupMemberUserID(ctx, req.GroupID) if err != nil { return nil, err } @@ -36,12 +36,12 @@ func (s *groupServer) GetFullGroupMemberUserIDs(ctx context.Context, req *pbgrou }, nil } -func (s *groupServer) GetFullJoinGroupIDs(ctx context.Context, req *pbgroup.GetFullJoinGroupIDsReq) (*pbgroup.GetFullJoinGroupIDsResp, error) { - vl, err := s.db.FindMaxJoinGroupVersionCache(ctx, req.UserID) +func (g *groupServer) GetFullJoinGroupIDs(ctx context.Context, req *pbgroup.GetFullJoinGroupIDsReq) (*pbgroup.GetFullJoinGroupIDsResp, error) { + vl, err := g.db.FindMaxJoinGroupVersionCache(ctx, req.UserID) if err != nil { return nil, err } - groupIDs, err := s.db.FindJoinGroupID(ctx, req.UserID) + groupIDs, err := g.db.FindJoinGroupID(ctx, req.UserID) if err != nil { return nil, err } @@ -57,8 +57,8 @@ func (s *groupServer) GetFullJoinGroupIDs(ctx context.Context, req *pbgroup.GetF }, nil } -func (s *groupServer) GetIncrementalGroupMember(ctx context.Context, req *pbgroup.GetIncrementalGroupMemberReq) (*pbgroup.GetIncrementalGroupMemberResp, error) { - group, err := s.db.TakeGroup(ctx, req.GroupID) +func (g *groupServer) GetIncrementalGroupMember(ctx context.Context, req *pbgroup.GetIncrementalGroupMemberReq) (*pbgroup.GetIncrementalGroupMemberResp, error) { + group, err := g.db.TakeGroup(ctx, req.GroupID) if err != nil { return nil, err } @@ -75,7 +75,7 @@ func (s *groupServer) GetIncrementalGroupMember(ctx context.Context, req *pbgrou VersionID: req.VersionID, VersionNumber: req.Version, Version: func(ctx context.Context, groupID string, version uint, limit int) (*model.VersionLog, error) { - vl, err := s.db.FindMemberIncrVersion(ctx, groupID, version, limit) + vl, err := g.db.FindMemberIncrVersion(ctx, groupID, version, limit) if err != nil { return nil, err } @@ -98,9 +98,9 @@ func (s *groupServer) GetIncrementalGroupMember(ctx context.Context, req *pbgrou } return vl, nil }, - CacheMaxVersion: s.db.FindMaxGroupMemberVersionCache, + CacheMaxVersion: g.db.FindMaxGroupMemberVersionCache, Find: func(ctx context.Context, ids []string) ([]*sdkws.GroupMemberFullInfo, error) { - return s.getGroupMembersInfo(ctx, req.GroupID, ids) + return g.getGroupMembersInfo(ctx, req.GroupID, ids) }, Resp: func(version *model.VersionLog, delIDs []string, insertList, updateList []*sdkws.GroupMemberFullInfo, full bool) *pbgroup.GetIncrementalGroupMemberResp { return &pbgroup.GetIncrementalGroupMemberResp{ @@ -119,20 +119,20 @@ func (s *groupServer) GetIncrementalGroupMember(ctx context.Context, req *pbgrou return nil, err } if resp.Full || hasGroupUpdate { - count, err := s.db.FindGroupMemberNum(ctx, group.GroupID) + count, err := g.db.FindGroupMemberNum(ctx, group.GroupID) if err != nil { return nil, err } - owner, err := s.db.TakeGroupOwner(ctx, group.GroupID) + owner, err := g.db.TakeGroupOwner(ctx, group.GroupID) if err != nil { return nil, err } - resp.Group = s.groupDB2PB(group, owner.UserID, count) + resp.Group = g.groupDB2PB(group, owner.UserID, count) } return resp, nil } -func (s *groupServer) BatchGetIncrementalGroupMember(ctx context.Context, req *pbgroup.BatchGetIncrementalGroupMemberReq) (resp *pbgroup.BatchGetIncrementalGroupMemberResp, err error) { +func (g *groupServer) BatchGetIncrementalGroupMember(ctx context.Context, req *pbgroup.BatchGetIncrementalGroupMemberReq) (resp *pbgroup.BatchGetIncrementalGroupMemberResp, err error) { type VersionInfo struct { GroupID string VersionID string @@ -161,7 +161,7 @@ func (s *groupServer) BatchGetIncrementalGroupMember(ctx context.Context, req *p groupIDs = append(groupIDs, group.GroupID) } - groups, err := s.db.FindGroup(ctx, groupIDs) + groups, err := g.db.FindGroup(ctx, groupIDs) if err != nil { return nil, errs.Wrap(err) } @@ -189,7 +189,7 @@ func (s *groupServer) BatchGetIncrementalGroupMember(ctx context.Context, req *p VersionIDs: versionIDs, VersionNumbers: versionNumbers, Versions: func(ctx context.Context, groupIDs []string, versions []uint64, limits []int) (map[string]*model.VersionLog, error) { - vLogs, err := s.db.BatchFindMemberIncrVersion(ctx, groupIDs, versions, limits) + vLogs, err := g.db.BatchFindMemberIncrVersion(ctx, groupIDs, versions, limits) if err != nil { return nil, errs.Wrap(err) } @@ -216,9 +216,9 @@ func (s *groupServer) BatchGetIncrementalGroupMember(ctx context.Context, req *p return vLogs, nil }, - CacheMaxVersions: s.db.BatchFindMaxGroupMemberVersionCache, + CacheMaxVersions: g.db.BatchFindMaxGroupMemberVersionCache, Find: func(ctx context.Context, groupID string, ids []string) ([]*sdkws.GroupMemberFullInfo, error) { - memberInfo, err := s.getGroupMembersInfo(ctx, groupID, ids) + memberInfo, err := g.getGroupMembersInfo(ctx, groupID, ids) if err != nil { return nil, err } @@ -258,17 +258,17 @@ func (s *groupServer) BatchGetIncrementalGroupMember(ctx context.Context, req *p for groupID, val := range resp.RespList { if val.Full || hasGroupUpdateMap[groupID] { - count, err := s.db.FindGroupMemberNum(ctx, groupID) + count, err := g.db.FindGroupMemberNum(ctx, groupID) if err != nil { return nil, err } - owner, err := s.db.TakeGroupOwner(ctx, groupID) + owner, err := g.db.TakeGroupOwner(ctx, groupID) if err != nil { return nil, err } - resp.RespList[groupID].Group = s.groupDB2PB(groupsMap[groupID], owner.UserID, count) + resp.RespList[groupID].Group = g.groupDB2PB(groupsMap[groupID], owner.UserID, count) } } @@ -276,8 +276,8 @@ func (s *groupServer) BatchGetIncrementalGroupMember(ctx context.Context, req *p } -func (s *groupServer) GetIncrementalJoinGroup(ctx context.Context, req *pbgroup.GetIncrementalJoinGroupReq) (*pbgroup.GetIncrementalJoinGroupResp, error) { - if err := authverify.CheckAccessV3(ctx, req.UserID, s.config.Share.IMAdminUserID); err != nil { +func (g *groupServer) GetIncrementalJoinGroup(ctx context.Context, req *pbgroup.GetIncrementalJoinGroupReq) (*pbgroup.GetIncrementalJoinGroupResp, error) { + if err := authverify.CheckAccessV3(ctx, req.UserID, g.config.Share.IMAdminUserID); err != nil { return nil, err } opt := incrversion.Option[*sdkws.GroupInfo, pbgroup.GetIncrementalJoinGroupResp]{ @@ -285,9 +285,9 @@ func (s *groupServer) GetIncrementalJoinGroup(ctx context.Context, req *pbgroup. VersionKey: req.UserID, VersionID: req.VersionID, VersionNumber: req.Version, - Version: s.db.FindJoinIncrVersion, - CacheMaxVersion: s.db.FindMaxJoinGroupVersionCache, - Find: s.getGroupsInfo, + Version: g.db.FindJoinIncrVersion, + CacheMaxVersion: g.db.FindMaxJoinGroupVersionCache, + Find: g.getGroupsInfo, Resp: func(version *model.VersionLog, delIDs []string, insertList, updateList []*sdkws.GroupInfo, full bool) *pbgroup.GetIncrementalJoinGroupResp { return &pbgroup.GetIncrementalJoinGroupResp{ VersionID: version.ID.Hex(), diff --git a/internal/rpc/msg/clear.go b/internal/rpc/msg/clear.go index ff732136a..7d62e7c8f 100644 --- a/internal/rpc/msg/clear.go +++ b/internal/rpc/msg/clear.go @@ -6,7 +6,7 @@ import ( "github.com/openimsdk/open-im-server/v3/pkg/authverify" "github.com/openimsdk/open-im-server/v3/pkg/common/convert" - pbconversation "github.com/openimsdk/protocol/conversation" + pbconv "github.com/openimsdk/protocol/conversation" "github.com/openimsdk/protocol/msg" "github.com/openimsdk/protocol/wrapperspb" "github.com/openimsdk/tools/errs" @@ -105,18 +105,21 @@ func (m *msgServer) ClearMsg(ctx context.Context, req *msg.ClearMsgReq) (_ *msg. minseq := datautil.Max(seqs...) // update - if err := m.Conversation.UpdateConversation(handleCtx, - &pbconversation.UpdateConversationReq{ - UserIDs: []string{conversation.OwnerUserID}, - ConversationID: conversation.ConversationID, - LatestMsgDestructTime: wrapperspb.Int64(time.Now().UnixMilli()), - MinSeq: wrapperspb.Int64(minseq), - }); err != nil { + if err := pbconv.UpdateConversationCaller.Execute(ctx, &pbconv.UpdateConversationReq{ + ConversationID: conversation.ConversationID, + UserIDs: []string{conversation.OwnerUserID}, + MinSeq: wrapperspb.Int64(minseq), + LatestMsgDestructTime: wrapperspb.Int64(time.Now().UnixMilli()), + }); err != nil { log.ZError(handleCtx, "updateUsersConversationField failed", err, "conversationID", conversation.ConversationID, "ownerUserID", conversation.OwnerUserID) continue } - if err := m.Conversation.SetConversationMinSeq(handleCtx, []string{conversation.OwnerUserID}, conversation.ConversationID, minseq); err != nil { + if err := pbconv.SetConversationMinSeqCaller.Execute(ctx, &pbconv.SetConversationMinSeqReq{ + ConversationID: conversation.ConversationID, + OwnerUserID: []string{conversation.OwnerUserID}, + MinSeq: minseq, + }); err != nil { return err } diff --git a/internal/rpc/msg/delete.go b/internal/rpc/msg/delete.go index 371c50e2e..e0c3a89ed 100644 --- a/internal/rpc/msg/delete.go +++ b/internal/rpc/msg/delete.go @@ -21,6 +21,7 @@ import ( "github.com/openimsdk/protocol/constant" "github.com/openimsdk/protocol/conversation" "github.com/openimsdk/protocol/msg" + "github.com/openimsdk/protocol/rpccall" "github.com/openimsdk/protocol/sdkws" "github.com/openimsdk/tools/log" "github.com/openimsdk/tools/utils/timeutil" @@ -74,7 +75,10 @@ func (m *msgServer) DeleteMsgs(ctx context.Context, req *msg.DeleteMsgsReq) (*ms if err := m.MsgDatabase.DeleteMsgsPhysicalBySeqs(ctx, req.ConversationID, req.Seqs); err != nil { return nil, err } - conversations, err := m.Conversation.GetConversationsByConversationID(ctx, []string{req.ConversationID}) + + conversations, err := rpccall.ExtractField(ctx, conversation.GetConversationsByConversationIDCaller.Invoke, &conversation.GetConversationsByConversationIDReq{ + ConversationIDs: []string{req.ConversationID}, + }, (*conversation.GetConversationsByConversationIDResp).GetConversations) if err != nil { return nil, err } @@ -121,7 +125,9 @@ func (m *msgServer) DeleteMsgPhysical(ctx context.Context, req *msg.DeleteMsgPhy } func (m *msgServer) clearConversation(ctx context.Context, conversationIDs []string, userID string, deleteSyncOpt *msg.DeleteSyncOpt) error { - conversations, err := m.Conversation.GetConversationsByConversationID(ctx, conversationIDs) + conversations, err := rpccall.ExtractField(ctx, conversation.GetConversationsByConversationIDCaller.Invoke, &conversation.GetConversationsByConversationIDReq{ + ConversationIDs: conversationIDs, + }, (*conversation.GetConversationsByConversationIDResp).GetConversations) if err != nil { return err } @@ -144,7 +150,11 @@ func (m *msgServer) clearConversation(ctx context.Context, conversationIDs []str } ownerUserIDs := []string{userID} for conversationID, seq := range setSeqs { - if err := m.Conversation.SetConversationMinSeq(ctx, ownerUserIDs, conversationID, seq); err != nil { + if err := conversation.SetConversationMinSeqCaller.Execute(ctx, &conversation.SetConversationMinSeqReq{ + ConversationID: conversationID, + OwnerUserID: ownerUserIDs, + MinSeq: seq, + }); err != nil { return err } } diff --git a/internal/rpc/msg/send.go b/internal/rpc/msg/send.go index 034d549ec..bb20206f7 100644 --- a/internal/rpc/msg/send.go +++ b/internal/rpc/msg/send.go @@ -21,7 +21,7 @@ import ( "github.com/openimsdk/open-im-server/v3/pkg/msgprocessor" "github.com/openimsdk/open-im-server/v3/pkg/util/conversationutil" "github.com/openimsdk/protocol/constant" - pbconversation "github.com/openimsdk/protocol/conversation" + pbconv "github.com/openimsdk/protocol/conversation" pbmsg "github.com/openimsdk/protocol/msg" "github.com/openimsdk/protocol/sdkws" "github.com/openimsdk/protocol/wrapperspb" @@ -96,7 +96,7 @@ func (m *msgServer) setConversationAtInfo(nctx context.Context, msg *sdkws.MsgDa var atUserID []string - conversation := &pbconversation.ConversationReq{ + conversation := &pbconv.ConversationReq{ ConversationID: msgprocessor.GetConversationIDByMsg(msg), ConversationType: msg.SessionType, GroupID: msg.GroupID, @@ -119,7 +119,10 @@ func (m *msgServer) setConversationAtInfo(nctx context.Context, msg *sdkws.MsgDa } else { // @Everyone and @other people conversation.GroupAtType = &wrapperspb.Int32Value{Value: constant.AtAllAtMe} - err = m.Conversation.SetConversations(ctx, atUserID, conversation) + err = pbconv.SetConversationsCaller.Execute(ctx, &pbconv.SetConversationsReq{ + UserIDs: atUserID, + Conversation: conversation, + }) if err != nil { log.ZWarn(ctx, "SetConversations", err, "userID", atUserID, "conversation", conversation) } @@ -129,7 +132,10 @@ func (m *msgServer) setConversationAtInfo(nctx context.Context, msg *sdkws.MsgDa conversation.GroupAtType = &wrapperspb.Int32Value{Value: constant.AtAll} - err = m.Conversation.SetConversations(ctx, memberUserIDList, conversation) + err = pbconv.SetConversationsCaller.Execute(ctx, &pbconv.SetConversationsReq{ + UserIDs: memberUserIDList, + Conversation: conversation, + }) if err != nil { log.ZWarn(ctx, "SetConversations", err, "userID", memberUserIDList, "conversation", conversation) } @@ -138,7 +144,10 @@ func (m *msgServer) setConversationAtInfo(nctx context.Context, msg *sdkws.MsgDa } conversation.GroupAtType = &wrapperspb.Int32Value{Value: constant.AtMe} - err := m.Conversation.SetConversations(ctx, msg.AtUserIDList, conversation) + err := pbconv.SetConversationsCaller.Execute(ctx, &pbconv.SetConversationsReq{ + UserIDs: msg.AtUserIDList, + Conversation: conversation, + }) if err != nil { log.ZWarn(ctx, "SetConversations", err, msg.AtUserIDList, conversation) } diff --git a/internal/rpc/msg/server.go b/internal/rpc/msg/server.go index b0cd771a4..9bd2a6946 100644 --- a/internal/rpc/msg/server.go +++ b/internal/rpc/msg/server.go @@ -16,6 +16,7 @@ package msg import ( "context" + "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/redis" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database/mgo" @@ -44,7 +45,6 @@ type ( RegisterCenter discovery.SvcDiscoveryRegistry // Service discovery registry for service registration. MsgDatabase controller.CommonMsgDatabase // Interface for message database operations. StreamMsgDatabase controller.StreamMsgDatabase - Conversation *rpcclient.ConversationRpcClient // RPC client for conversation service. UserLocalCache *rpccache.UserLocalCache // Local cache for user data. FriendLocalCache *rpccache.FriendLocalCache // Local cache for friend data. GroupLocalCache *rpccache.GroupLocalCache // Local cache for group data. @@ -89,10 +89,6 @@ func Start(ctx context.Context, config *Config, client discovery.SvcDiscoveryReg return err } msgModel := redis.NewMsgCache(rdb) - conversationClient := rpcclient.NewConversationRpcClient(client, config.Discovery.RpcService.Conversation) - userRpcClient := rpcclient.NewUserRpcClient(client, config.Discovery.RpcService.User, config.Share.IMAdminUserID) - groupRpcClient := rpcclient.NewGroupRpcClient(client, config.Discovery.RpcService.Group) - friendRpcClient := rpcclient.NewFriendRpcClient(client, config.Discovery.RpcService.Friend) seqConversation, err := mgo.NewSeqConversationMongo(mgocli.GetDB()) if err != nil { return err @@ -112,14 +108,13 @@ func Start(ctx context.Context, config *Config, client discovery.SvcDiscoveryReg return err } s := &msgServer{ - Conversation: &conversationClient, MsgDatabase: msgDatabase, StreamMsgDatabase: controller.NewStreamMsgDatabase(streamMsg), RegisterCenter: client, - UserLocalCache: rpccache.NewUserLocalCache(userRpcClient, &config.LocalCacheConfig, rdb), - GroupLocalCache: rpccache.NewGroupLocalCache(groupRpcClient, &config.LocalCacheConfig, rdb), - ConversationLocalCache: rpccache.NewConversationLocalCache(conversationClient, &config.LocalCacheConfig, rdb), - FriendLocalCache: rpccache.NewFriendLocalCache(friendRpcClient, &config.LocalCacheConfig, rdb), + UserLocalCache: rpccache.NewUserLocalCache(&config.LocalCacheConfig, rdb), + GroupLocalCache: rpccache.NewGroupLocalCache(&config.LocalCacheConfig, rdb), + ConversationLocalCache: rpccache.NewConversationLocalCache(&config.LocalCacheConfig, rdb), + FriendLocalCache: rpccache.NewFriendLocalCache(&config.LocalCacheConfig, rdb), config: config, webhookClient: webhook.NewWebhookClient(config.WebhooksConfig.URL), } diff --git a/internal/rpc/msg/stream_msg.go b/internal/rpc/msg/stream_msg.go index 5db2aad48..e216b1087 100644 --- a/internal/rpc/msg/stream_msg.go +++ b/internal/rpc/msg/stream_msg.go @@ -3,13 +3,16 @@ package msg import ( "context" "fmt" + "time" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" "github.com/openimsdk/open-im-server/v3/pkg/msgprocessor" "github.com/openimsdk/protocol/constant" + pbconv "github.com/openimsdk/protocol/conversation" "github.com/openimsdk/protocol/msg" + "github.com/openimsdk/protocol/rpccall" "github.com/openimsdk/protocol/sdkws" "github.com/openimsdk/tools/errs" - "time" ) const StreamDeadlineTime = time.Second * 60 * 10 @@ -71,7 +74,10 @@ func (m *msgServer) AppendStreamMsg(ctx context.Context, req *msg.AppendStreamMs if err := m.StreamMsgDatabase.AppendStreamMsg(ctx, req.ClientMsgID, int(req.StartIndex), req.Packets, req.End, deadlineTime); err != nil { return nil, err } - conversation, err := m.Conversation.GetConversation(ctx, res.UserID, res.ConversationID) + conversation, err := rpccall.ExtractField(ctx, pbconv.GetConversationCaller.Invoke, &pbconv.GetConversationReq{ + ConversationID: res.ConversationID, + OwnerUserID: res.UserID, + }, (*pbconv.GetConversationResp).GetConversation) if err != nil { return nil, err } diff --git a/internal/rpc/relation/black.go b/internal/rpc/relation/black.go index d8d457dac..0fd9b8766 100644 --- a/internal/rpc/relation/black.go +++ b/internal/rpc/relation/black.go @@ -18,10 +18,10 @@ import ( "context" "time" - "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" - "github.com/openimsdk/open-im-server/v3/pkg/authverify" "github.com/openimsdk/open-im-server/v3/pkg/common/convert" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" + "github.com/openimsdk/open-im-server/v3/pkg/rpcclient" "github.com/openimsdk/protocol/relation" "github.com/openimsdk/protocol/sdkws" "github.com/openimsdk/tools/errs" @@ -39,7 +39,7 @@ func (s *friendServer) GetPaginationBlacks(ctx context.Context, req *relation.Ge return nil, err } resp = &relation.GetPaginationBlacksResp{} - resp.Blacks, err = convert.BlackDB2Pb(ctx, blacks, s.userRpcClient.GetUsersInfoMap) + resp.Blacks, err = convert.BlackDB2Pb(ctx, blacks, rpcclient.GetUsersInfoMap) if err != nil { return nil, err } @@ -82,7 +82,7 @@ func (s *friendServer) AddBlack(ctx context.Context, req *relation.AddBlackReq) return nil, err } - _, err := s.userRpcClient.GetUsersInfo(ctx, []string{req.OwnerUserID, req.BlackUserID}) + _, err := rpcclient.GetUsersInfo(ctx, []string{req.OwnerUserID, req.BlackUserID}) if err != nil { return nil, err } @@ -114,7 +114,7 @@ func (s *friendServer) GetSpecifiedBlacks(ctx context.Context, req *relation.Get return nil, errs.ErrArgs.WrapMsg("userIDList repeated") } - userMap, err := s.userRpcClient.GetPublicUserInfoMap(ctx, req.UserIDList) + userMap, err := rpcclient.GetPublicUserInfoMap(ctx, req.UserIDList) if err != nil { return nil, err } diff --git a/internal/rpc/relation/friend.go b/internal/rpc/relation/friend.go index 617e31348..b97a9d01f 100644 --- a/internal/rpc/relation/friend.go +++ b/internal/rpc/relation/friend.go @@ -44,15 +44,13 @@ import ( type friendServer struct { relation.UnimplementedFriendServer - db controller.FriendDatabase - blackDatabase controller.BlackDatabase - userRpcClient *rpcclient.UserRpcClient - notificationSender *FriendNotificationSender - conversationRpcClient rpcclient.ConversationRpcClient - RegisterCenter discovery.SvcDiscoveryRegistry - config *Config - webhookClient *webhook.Client - queue *memamq.MemoryQueue + db controller.FriendDatabase + blackDatabase controller.BlackDatabase + notificationSender *FriendNotificationSender + RegisterCenter discovery.SvcDiscoveryRegistry + config *Config + webhookClient *webhook.Client + queue *memamq.MemoryQueue } type Config struct { @@ -92,15 +90,10 @@ func Start(ctx context.Context, config *Config, client discovery.SvcDiscoveryReg return err } - // Initialize RPC clients - userRpcClient := rpcclient.NewUserRpcClient(client, config.Discovery.RpcService.User, config.Share.IMAdminUserID) - msgRpcClient := rpcclient.NewMessageRpcClient(client, config.Discovery.RpcService.Msg) - // Initialize notification sender notificationSender := NewFriendNotificationSender( &config.NotificationConfig, - &msgRpcClient, - WithRpcFunc(userRpcClient.GetUsersInfo), + WithRpcFunc(rpcclient.GetUsersInfo), ) localcache.InitLocalCache(&config.LocalCacheConfig) @@ -116,13 +109,11 @@ func Start(ctx context.Context, config *Config, client discovery.SvcDiscoveryReg blackMongoDB, redis.NewBlackCacheRedis(rdb, &config.LocalCacheConfig, blackMongoDB, redis.GetRocksCacheOptions()), ), - userRpcClient: &userRpcClient, - notificationSender: notificationSender, - RegisterCenter: client, - conversationRpcClient: rpcclient.NewConversationRpcClient(client, config.Discovery.RpcService.Conversation), - config: config, - webhookClient: webhook.NewWebhookClient(config.WebhooksConfig.URL), - queue: memamq.NewMemoryQueue(16, 1024*1024), + notificationSender: notificationSender, + RegisterCenter: client, + config: config, + webhookClient: webhook.NewWebhookClient(config.WebhooksConfig.URL), + queue: memamq.NewMemoryQueue(16, 1024*1024), }) return nil } @@ -139,7 +130,7 @@ func (s *friendServer) ApplyToAddFriend(ctx context.Context, req *relation.Apply if err = s.webhookBeforeAddFriend(ctx, &s.config.WebhooksConfig.BeforeAddFriend, req); err != nil && err != servererrs.ErrCallbackContinue { return nil, err } - if _, err := s.userRpcClient.GetUsersInfoMap(ctx, []string{req.ToUserID, req.FromUserID}); err != nil { + if _, err := rpcclient.GetUsersInfoMap(ctx, []string{req.ToUserID, req.FromUserID}); err != nil { return nil, err } @@ -163,7 +154,8 @@ func (s *friendServer) ImportFriends(ctx context.Context, req *relation.ImportFr if err := authverify.CheckAdmin(ctx, s.config.Share.IMAdminUserID); err != nil { return nil, err } - if _, err := s.userRpcClient.GetUsersInfo(ctx, append([]string{req.OwnerUserID}, req.FriendUserIDs...)); err != nil { + + if _, err := rpcclient.GetUsersInfo(ctx, append([]string{req.OwnerUserID}, req.FriendUserIDs...)); err != nil { return nil, err } if datautil.Contain(req.OwnerUserID, req.FriendUserIDs...) { @@ -304,7 +296,7 @@ func (s *friendServer) getFriend(ctx context.Context, ownerUserID string, friend if err != nil { return nil, err } - return convert.FriendsDB2Pb(ctx, friends, s.userRpcClient.GetUsersInfoMap) + return convert.FriendsDB2Pb(ctx, friends, rpcclient.GetUsersInfoMap) } // Get the list of friend requests sent out proactively. @@ -316,7 +308,7 @@ func (s *friendServer) GetDesignatedFriendsApply(ctx context.Context, return nil, err } resp = &relation.GetDesignatedFriendsApplyResp{} - resp.FriendRequests, err = convert.FriendRequestDB2Pb(ctx, friendRequests, s.userRpcClient.GetUsersInfoMap) + resp.FriendRequests, err = convert.FriendRequestDB2Pb(ctx, friendRequests, rpcclient.GetUsersInfoMap) if err != nil { return nil, err } @@ -335,7 +327,7 @@ func (s *friendServer) GetPaginationFriendsApplyTo(ctx context.Context, req *rel } resp = &relation.GetPaginationFriendsApplyToResp{} - resp.FriendRequests, err = convert.FriendRequestDB2Pb(ctx, friendRequests, s.userRpcClient.GetUsersInfoMap) + resp.FriendRequests, err = convert.FriendRequestDB2Pb(ctx, friendRequests, rpcclient.GetUsersInfoMap) if err != nil { return nil, err } @@ -357,7 +349,7 @@ func (s *friendServer) GetPaginationFriendsApplyFrom(ctx context.Context, req *r return nil, err } - resp.FriendRequests, err = convert.FriendRequestDB2Pb(ctx, friendRequests, s.userRpcClient.GetUsersInfoMap) + resp.FriendRequests, err = convert.FriendRequestDB2Pb(ctx, friendRequests, rpcclient.GetUsersInfoMap) if err != nil { return nil, err } @@ -388,7 +380,7 @@ func (s *friendServer) GetPaginationFriends(ctx context.Context, req *relation.G } resp = &relation.GetPaginationFriendsResp{} - resp.FriendsInfo, err = convert.FriendsDB2Pb(ctx, friends, s.userRpcClient.GetUsersInfoMap) + resp.FriendsInfo, err = convert.FriendsDB2Pb(ctx, friends, rpcclient.GetUsersInfoMap) if err != nil { return nil, err } @@ -421,7 +413,7 @@ func (s *friendServer) GetSpecifiedFriendsInfo(ctx context.Context, req *relatio return nil, errs.ErrArgs.WrapMsg("userIDList repeated") } - userMap, err := s.userRpcClient.GetUsersInfoMap(ctx, req.UserIDList) + userMap, err := rpcclient.GetUsersInfoMap(ctx, req.UserIDList) if err != nil { return nil, err } diff --git a/internal/rpc/relation/notification.go b/internal/rpc/relation/notification.go index 83c5d2ca9..ba4365200 100644 --- a/internal/rpc/relation/notification.go +++ b/internal/rpc/relation/notification.go @@ -16,6 +16,7 @@ package relation import ( "context" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/versionctx" @@ -86,11 +87,10 @@ func WithRpcFunc( func NewFriendNotificationSender( conf *config.Notification, - msgRpcClient *rpcclient.MessageRpcClient, opts ...friendNotificationSenderOptions, ) *FriendNotificationSender { f := &FriendNotificationSender{ - NotificationSender: rpcclient.NewNotificationSender(conf, rpcclient.WithRpcClient(msgRpcClient)), + NotificationSender: rpcclient.NewNotificationSender(conf, rpcclient.WithRpcClient()), } for _, opt := range opts { opt(f) diff --git a/internal/rpc/third/log.go b/internal/rpc/third/log.go index 4eeb5d558..222cfad1d 100644 --- a/internal/rpc/third/log.go +++ b/internal/rpc/third/log.go @@ -20,6 +20,7 @@ import ( "time" relationtb "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" + "github.com/openimsdk/open-im-server/v3/pkg/rpcclient" "github.com/openimsdk/open-im-server/v3/pkg/authverify" "github.com/openimsdk/open-im-server/v3/pkg/common/servererrs" @@ -149,7 +150,7 @@ func (t *thirdServer) SearchLogs(ctx context.Context, req *third.SearchLogsReq) for _, log := range logs { userIDs = append(userIDs, log.UserID) } - userMap, err := t.userRpcClient.GetUsersInfoMap(ctx, userIDs) + userMap, err := rpcclient.GetUsersInfoMap(ctx, userIDs) if err != nil { return nil, err } diff --git a/internal/rpc/third/third.go b/internal/rpc/third/third.go index 52e48855c..77e6d459f 100644 --- a/internal/rpc/third/third.go +++ b/internal/rpc/third/third.go @@ -17,16 +17,16 @@ package third import ( "context" "fmt" + "time" + "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/redis" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database/mgo" "github.com/openimsdk/open-im-server/v3/pkg/localcache" "github.com/openimsdk/tools/s3/aws" "github.com/openimsdk/tools/s3/kodo" - "time" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/controller" - "github.com/openimsdk/open-im-server/v3/pkg/rpcclient" "github.com/openimsdk/protocol/third" "github.com/openimsdk/tools/db/mongoutil" "github.com/openimsdk/tools/db/redisutil" @@ -42,7 +42,6 @@ type thirdServer struct { third.UnimplementedThirdServer thirdDatabase controller.ThirdDatabase s3dataBase controller.S3Database - userRpcClient rpcclient.UserRpcClient defaultExpire time.Duration config *Config minio *minio.Minio @@ -104,7 +103,6 @@ func Start(ctx context.Context, config *Config, client discovery.SvcDiscoveryReg localcache.InitLocalCache(&config.LocalCacheConfig) third.RegisterThirdServer(server, &thirdServer{ thirdDatabase: controller.NewThirdDatabase(redis.NewThirdCache(rdb), logdb), - userRpcClient: rpcclient.NewUserRpcClient(client, config.Discovery.RpcService.User, config.Share.IMAdminUserID), s3dataBase: controller.NewS3Database(rdb, o, s3db), defaultExpire: time.Hour * 24 * 7, config: config, diff --git a/internal/rpc/user/notification.go b/internal/rpc/user/notification.go index b992c9d12..54e5b27d7 100644 --- a/internal/rpc/user/notification.go +++ b/internal/rpc/user/notification.go @@ -16,6 +16,7 @@ package user import ( "context" + relationtb "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" "github.com/openimsdk/open-im-server/v3/pkg/rpcclient/notification" @@ -58,9 +59,9 @@ func WithUserFunc( } } -func NewUserNotificationSender(config *Config, msgRpcClient *rpcclient.MessageRpcClient, opts ...userNotificationSenderOptions) *UserNotificationSender { +func NewUserNotificationSender(config *Config, opts ...userNotificationSenderOptions) *UserNotificationSender { f := &UserNotificationSender{ - NotificationSender: rpcclient.NewNotificationSender(&config.NotificationConfig, rpcclient.WithRpcClient(msgRpcClient)), + NotificationSender: rpcclient.NewNotificationSender(&config.NotificationConfig, rpcclient.WithRpcClient()), } for _, opt := range opts { opt(f) diff --git a/internal/rpc/user/user.go b/internal/rpc/user/user.go index ae02b997f..c2bcabaa1 100644 --- a/internal/rpc/user/user.go +++ b/internal/rpc/user/user.go @@ -39,7 +39,6 @@ import ( "github.com/openimsdk/open-im-server/v3/pkg/common/convert" "github.com/openimsdk/open-im-server/v3/pkg/common/servererrs" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/controller" - "github.com/openimsdk/open-im-server/v3/pkg/rpcclient" "github.com/openimsdk/protocol/constant" "github.com/openimsdk/protocol/sdkws" pbuser "github.com/openimsdk/protocol/user" @@ -57,8 +56,6 @@ type userServer struct { db controller.UserDatabase friendNotificationSender *relation.FriendNotificationSender userNotificationSender *UserNotificationSender - friendRpcClient *rpcclient.FriendRpcClient - groupRpcClient *rpcclient.GroupRpcClient RegisterCenter registry.SvcDiscoveryRegistry config *Config webhookClient *webhook.Client @@ -96,18 +93,13 @@ func Start(ctx context.Context, config *Config, client registry.SvcDiscoveryRegi } userCache := redis.NewUserCacheRedis(rdb, &config.LocalCacheConfig, userDB, redis.GetRocksCacheOptions()) database := controller.NewUserDatabase(userDB, userCache, mgocli.GetTx()) - friendRpcClient := rpcclient.NewFriendRpcClient(client, config.Discovery.RpcService.Friend) - groupRpcClient := rpcclient.NewGroupRpcClient(client, config.Discovery.RpcService.Group) - msgRpcClient := rpcclient.NewMessageRpcClient(client, config.Discovery.RpcService.Msg) localcache.InitLocalCache(&config.LocalCacheConfig) u := &userServer{ online: redis.NewUserOnline(rdb), db: database, RegisterCenter: client, - friendRpcClient: &friendRpcClient, - groupRpcClient: &groupRpcClient, - friendNotificationSender: relation.NewFriendNotificationSender(&config.NotificationConfig, &msgRpcClient, relation.WithDBFunc(database.FindWithError)), - userNotificationSender: NewUserNotificationSender(config, &msgRpcClient, WithUserFunc(database.FindWithError)), + friendNotificationSender: relation.NewFriendNotificationSender(&config.NotificationConfig, relation.WithDBFunc(database.FindWithError)), + userNotificationSender: NewUserNotificationSender(config, WithUserFunc(database.FindWithError)), config: config, webhookClient: webhook.NewWebhookClient(config.WebhooksConfig.URL), } @@ -641,7 +633,7 @@ func (s *userServer) NotificationUserInfoUpdate(ctx context.Context, userID stri wg.Add(len(es)) go func() { defer wg.Done() - _, es[0] = s.groupRpcClient.Client.NotificationUserInfoUpdate(ctx, &group.NotificationUserInfoUpdateReq{ + _, es[0] = group.NotificationUserInfoUpdateCaller.Invoke(ctx, &group.NotificationUserInfoUpdateReq{ UserID: userID, OldUserInfo: oldUserInfo, NewUserInfo: newUserInfo, @@ -650,7 +642,7 @@ func (s *userServer) NotificationUserInfoUpdate(ctx context.Context, userID stri go func() { defer wg.Done() - _, es[1] = s.friendRpcClient.Client.NotificationUserInfoUpdate(ctx, &friendpb.NotificationUserInfoUpdateReq{ + _, es[1] = friendpb.NotificationUserInfoUpdateCaller.Invoke(ctx, &friendpb.NotificationUserInfoUpdateReq{ UserID: userID, OldUserInfo: oldUserInfo, NewUserInfo: newUserInfo, diff --git a/pkg/common/startrpc/start.go b/pkg/common/startrpc/start.go index 259f4caed..c38ee27a1 100644 --- a/pkg/common/startrpc/start.go +++ b/pkg/common/startrpc/start.go @@ -27,6 +27,7 @@ import ( "time" conf "github.com/openimsdk/open-im-server/v3/pkg/common/config" + "github.com/openimsdk/open-im-server/v3/pkg/rpcclient" "github.com/openimsdk/tools/discovery/etcd" "github.com/openimsdk/tools/utils/datautil" "github.com/openimsdk/tools/utils/jsonutil" @@ -95,6 +96,10 @@ func Start[T any](ctx context.Context, discovery *conf.Discovery, prometheusConf defer client.Close() client.AddOption(mw.GrpcClient(), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithDefaultServiceConfig(fmt.Sprintf(`{"LoadBalancingPolicy": "%s"}`, "round_robin"))) + if err = rpcclient.InitRpcCaller(client, discovery.RpcService); err != nil { + return err + } + // var reg *prometheus.Registry // var metric *grpcprometheus.ServerMetrics if prometheusConfig.Enable { diff --git a/pkg/rpccache/conversation.go b/pkg/rpccache/conversation.go index 925d2a37c..ba3690f44 100644 --- a/pkg/rpccache/conversation.go +++ b/pkg/rpccache/conversation.go @@ -16,11 +16,12 @@ package rpccache import ( "context" + "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/cachekey" "github.com/openimsdk/open-im-server/v3/pkg/localcache" - "github.com/openimsdk/open-im-server/v3/pkg/rpcclient" - pbconversation "github.com/openimsdk/protocol/conversation" + pbconv "github.com/openimsdk/protocol/conversation" + "github.com/openimsdk/protocol/rpccall" "github.com/openimsdk/tools/errs" "github.com/openimsdk/tools/log" "github.com/openimsdk/tools/utils/datautil" @@ -32,11 +33,10 @@ const ( conversationWorkerCount = 20 ) -func NewConversationLocalCache(client rpcclient.ConversationRpcClient, localCache *config.LocalCache, cli redis.UniversalClient) *ConversationLocalCache { +func NewConversationLocalCache(localCache *config.LocalCache, cli redis.UniversalClient) *ConversationLocalCache { lc := localCache.Conversation log.ZDebug(context.Background(), "ConversationLocalCache", "topic", lc.Topic, "slotNum", lc.SlotNum, "slotSize", lc.SlotSize, "enable", lc.Enable()) x := &ConversationLocalCache{ - client: client, local: localcache.New[[]byte]( localcache.WithLocalSlotNum(lc.SlotNum), localcache.WithLocalSlotSize(lc.SlotSize), @@ -52,8 +52,7 @@ func NewConversationLocalCache(client rpcclient.ConversationRpcClient, localCach } type ConversationLocalCache struct { - client rpcclient.ConversationRpcClient - local localcache.Cache[[]byte] + local localcache.Cache[[]byte] } func (c *ConversationLocalCache) GetConversationIDs(ctx context.Context, ownerUserID string) (val []string, err error) { @@ -64,7 +63,7 @@ func (c *ConversationLocalCache) GetConversationIDs(ctx context.Context, ownerUs return resp.ConversationIDs, nil } -func (c *ConversationLocalCache) getConversationIDs(ctx context.Context, ownerUserID string) (val *pbconversation.GetConversationIDsResp, err error) { +func (c *ConversationLocalCache) getConversationIDs(ctx context.Context, ownerUserID string) (val *pbconv.GetConversationIDsResp, err error) { log.ZDebug(ctx, "ConversationLocalCache getConversationIDs req", "ownerUserID", ownerUserID) defer func() { if err == nil { @@ -73,14 +72,14 @@ func (c *ConversationLocalCache) getConversationIDs(ctx context.Context, ownerUs log.ZError(ctx, "ConversationLocalCache getConversationIDs return", err, "ownerUserID", ownerUserID) } }() - var cache cacheProto[pbconversation.GetConversationIDsResp] + var cache cacheProto[pbconv.GetConversationIDsResp] return cache.Unmarshal(c.local.Get(ctx, cachekey.GetConversationIDsKey(ownerUserID), func(ctx context.Context) ([]byte, error) { log.ZDebug(ctx, "ConversationLocalCache getConversationIDs rpc", "ownerUserID", ownerUserID) - return cache.Marshal(c.client.Client.GetConversationIDs(ctx, &pbconversation.GetConversationIDsReq{UserID: ownerUserID})) + return cache.Marshal(pbconv.GetConversationIDsCaller.Invoke(ctx, &pbconv.GetConversationIDsReq{UserID: ownerUserID})) })) } -func (c *ConversationLocalCache) GetConversation(ctx context.Context, userID, conversationID string) (val *pbconversation.Conversation, err error) { +func (c *ConversationLocalCache) GetConversation(ctx context.Context, userID, conversationID string) (val *pbconv.Conversation, err error) { log.ZDebug(ctx, "ConversationLocalCache GetConversation req", "userID", userID, "conversationID", conversationID) defer func() { if err == nil { @@ -89,10 +88,13 @@ func (c *ConversationLocalCache) GetConversation(ctx context.Context, userID, co log.ZWarn(ctx, "ConversationLocalCache GetConversation return", err, "userID", userID, "conversationID", conversationID) } }() - var cache cacheProto[pbconversation.Conversation] + var cache cacheProto[pbconv.Conversation] return cache.Unmarshal(c.local.Get(ctx, cachekey.GetConversationKey(userID, conversationID), func(ctx context.Context) ([]byte, error) { log.ZDebug(ctx, "ConversationLocalCache GetConversation rpc", "userID", userID, "conversationID", conversationID) - return cache.Marshal(c.client.GetConversation(ctx, userID, conversationID)) + return cache.Marshal(rpccall.ExtractField(ctx, pbconv.GetConversationCaller.Invoke, &pbconv.GetConversationReq{ + ConversationID: conversationID, + OwnerUserID: userID, + }, (*pbconv.GetConversationResp).GetConversation)) })) } @@ -104,10 +106,10 @@ func (c *ConversationLocalCache) GetSingleConversationRecvMsgOpt(ctx context.Con return conv.RecvMsgOpt, nil } -func (c *ConversationLocalCache) GetConversations(ctx context.Context, ownerUserID string, conversationIDs []string) ([]*pbconversation.Conversation, error) { +func (c *ConversationLocalCache) GetConversations(ctx context.Context, ownerUserID string, conversationIDs []string) ([]*pbconv.Conversation, error) { var ( - conversations = make([]*pbconversation.Conversation, 0, len(conversationIDs)) - conversationsChan = make(chan *pbconversation.Conversation, len(conversationIDs)) + conversations = make([]*pbconv.Conversation, 0, len(conversationIDs)) + conversationsChan = make(chan *pbconv.Conversation, len(conversationIDs)) ) g, ctx := errgroup.WithContext(ctx) @@ -137,7 +139,7 @@ func (c *ConversationLocalCache) GetConversations(ctx context.Context, ownerUser return conversations, nil } -func (c *ConversationLocalCache) getConversationNotReceiveMessageUserIDs(ctx context.Context, conversationID string) (val *pbconversation.GetConversationNotReceiveMessageUserIDsResp, err error) { +func (c *ConversationLocalCache) getConversationNotReceiveMessageUserIDs(ctx context.Context, conversationID string) (val *pbconv.GetConversationNotReceiveMessageUserIDsResp, err error) { log.ZDebug(ctx, "ConversationLocalCache getConversationNotReceiveMessageUserIDs req", "conversationID", conversationID) defer func() { if err == nil { @@ -146,10 +148,10 @@ func (c *ConversationLocalCache) getConversationNotReceiveMessageUserIDs(ctx con log.ZError(ctx, "ConversationLocalCache getConversationNotReceiveMessageUserIDs return", err, "conversationID", conversationID) } }() - var cache cacheProto[pbconversation.GetConversationNotReceiveMessageUserIDsResp] + var cache cacheProto[pbconv.GetConversationNotReceiveMessageUserIDsResp] return cache.Unmarshal(c.local.Get(ctx, cachekey.GetConversationNotReceiveMessageUserIDsKey(conversationID), func(ctx context.Context) ([]byte, error) { log.ZDebug(ctx, "ConversationLocalCache getConversationNotReceiveMessageUserIDs rpc", "conversationID", conversationID) - return cache.Marshal(c.client.Client.GetConversationNotReceiveMessageUserIDs(ctx, &pbconversation.GetConversationNotReceiveMessageUserIDsReq{ConversationID: conversationID})) + return cache.Marshal(pbconv.GetConversationNotReceiveMessageUserIDsCaller.Invoke(ctx, &pbconv.GetConversationNotReceiveMessageUserIDsReq{ConversationID: conversationID})) })) } diff --git a/pkg/rpccache/friend.go b/pkg/rpccache/friend.go index dca3b4c97..865cac7b5 100644 --- a/pkg/rpccache/friend.go +++ b/pkg/rpccache/friend.go @@ -16,21 +16,20 @@ package rpccache import ( "context" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/cachekey" "github.com/openimsdk/protocol/relation" "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/open-im-server/v3/pkg/localcache" - "github.com/openimsdk/open-im-server/v3/pkg/rpcclient" "github.com/openimsdk/tools/log" "github.com/redis/go-redis/v9" ) -func NewFriendLocalCache(client rpcclient.FriendRpcClient, localCache *config.LocalCache, cli redis.UniversalClient) *FriendLocalCache { +func NewFriendLocalCache(localCache *config.LocalCache, cli redis.UniversalClient) *FriendLocalCache { lc := localCache.Friend log.ZDebug(context.Background(), "FriendLocalCache", "topic", lc.Topic, "slotNum", lc.SlotNum, "slotSize", lc.SlotSize, "enable", lc.Enable()) x := &FriendLocalCache{ - client: client, local: localcache.New[[]byte]( localcache.WithLocalSlotNum(lc.SlotNum), localcache.WithLocalSlotSize(lc.SlotSize), @@ -46,8 +45,7 @@ func NewFriendLocalCache(client rpcclient.FriendRpcClient, localCache *config.Lo } type FriendLocalCache struct { - client rpcclient.FriendRpcClient - local localcache.Cache[[]byte] + local localcache.Cache[[]byte] } func (f *FriendLocalCache) IsFriend(ctx context.Context, possibleFriendUserID, userID string) (val bool, err error) { @@ -70,7 +68,7 @@ func (f *FriendLocalCache) isFriend(ctx context.Context, possibleFriendUserID, u var cache cacheProto[relation.IsFriendResp] return cache.Unmarshal(f.local.GetLink(ctx, cachekey.GetIsFriendKey(possibleFriendUserID, userID), func(ctx context.Context) ([]byte, error) { log.ZDebug(ctx, "FriendLocalCache isFriend rpc", "possibleFriendUserID", possibleFriendUserID, "userID", userID) - return cache.Marshal(f.client.Client.IsFriend(ctx, &relation.IsFriendReq{UserID1: userID, UserID2: possibleFriendUserID})) + return cache.Marshal(relation.IsFriendCaller.Invoke(ctx, &relation.IsFriendReq{UserID1: userID, UserID2: possibleFriendUserID})) }, cachekey.GetFriendIDsKey(possibleFriendUserID))) } @@ -96,6 +94,6 @@ func (f *FriendLocalCache) isBlack(ctx context.Context, possibleBlackUserID, use var cache cacheProto[relation.IsBlackResp] return cache.Unmarshal(f.local.GetLink(ctx, cachekey.GetIsBlackIDsKey(possibleBlackUserID, userID), func(ctx context.Context) ([]byte, error) { log.ZDebug(ctx, "FriendLocalCache IsBlack rpc", "possibleBlackUserID", possibleBlackUserID, "userID", userID) - return cache.Marshal(f.client.Client.IsBlack(ctx, &relation.IsBlackReq{UserID1: possibleBlackUserID, UserID2: userID})) + return cache.Marshal(relation.IsBlackCaller.Invoke(ctx, &relation.IsBlackReq{UserID1: possibleBlackUserID, UserID2: userID})) }, cachekey.GetBlackIDsKey(userID))) } diff --git a/pkg/rpccache/group.go b/pkg/rpccache/group.go index b2d852fc5..111813103 100644 --- a/pkg/rpccache/group.go +++ b/pkg/rpccache/group.go @@ -16,24 +16,24 @@ package rpccache import ( "context" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/cachekey" "github.com/openimsdk/protocol/group" + "github.com/openimsdk/protocol/rpccall" "github.com/openimsdk/tools/utils/datautil" "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/open-im-server/v3/pkg/localcache" - "github.com/openimsdk/open-im-server/v3/pkg/rpcclient" "github.com/openimsdk/protocol/sdkws" "github.com/openimsdk/tools/errs" "github.com/openimsdk/tools/log" "github.com/redis/go-redis/v9" ) -func NewGroupLocalCache(client rpcclient.GroupRpcClient, localCache *config.LocalCache, cli redis.UniversalClient) *GroupLocalCache { +func NewGroupLocalCache(localCache *config.LocalCache, cli redis.UniversalClient) *GroupLocalCache { lc := localCache.Group log.ZDebug(context.Background(), "GroupLocalCache", "topic", lc.Topic, "slotNum", lc.SlotNum, "slotSize", lc.SlotSize, "enable", lc.Enable()) x := &GroupLocalCache{ - client: client, local: localcache.New[[]byte]( localcache.WithLocalSlotNum(lc.SlotNum), localcache.WithLocalSlotSize(lc.SlotSize), @@ -49,8 +49,7 @@ func NewGroupLocalCache(client rpcclient.GroupRpcClient, localCache *config.Loca } type GroupLocalCache struct { - client rpcclient.GroupRpcClient - local localcache.Cache[[]byte] + local localcache.Cache[[]byte] } func (g *GroupLocalCache) getGroupMemberIDs(ctx context.Context, groupID string) (val *group.GetGroupMemberUserIDsResp, err error) { @@ -65,7 +64,7 @@ func (g *GroupLocalCache) getGroupMemberIDs(ctx context.Context, groupID string) var cache cacheProto[group.GetGroupMemberUserIDsResp] return cache.Unmarshal(g.local.Get(ctx, cachekey.GetGroupMemberIDsKey(groupID), func(ctx context.Context) ([]byte, error) { log.ZDebug(ctx, "GroupLocalCache getGroupMemberIDs rpc", "groupID", groupID) - return cache.Marshal(g.client.Client.GetGroupMemberUserIDs(ctx, &group.GetGroupMemberUserIDsReq{GroupID: groupID})) + return cache.Marshal(group.GetGroupMemberUserIDsCaller.Invoke(ctx, &group.GetGroupMemberUserIDsReq{GroupID: groupID})) })) } @@ -81,7 +80,13 @@ func (g *GroupLocalCache) GetGroupMember(ctx context.Context, groupID, userID st var cache cacheProto[sdkws.GroupMemberFullInfo] return cache.Unmarshal(g.local.Get(ctx, cachekey.GetGroupMemberInfoKey(groupID, userID), func(ctx context.Context) ([]byte, error) { log.ZDebug(ctx, "GroupLocalCache GetGroupInfo rpc", "groupID", groupID, "userID", userID) - return cache.Marshal(g.client.GetGroupMemberCache(ctx, groupID, userID)) + return cache.Marshal(rpccall.ExtractField(ctx, group.GetGroupMemberCacheCaller.Invoke, + &group.GetGroupMemberCacheReq{ + GroupID: groupID, + GroupMemberID: userID, + }, + (*group.GetGroupMemberCacheResp).GetMember, + )) })) } @@ -97,7 +102,10 @@ func (g *GroupLocalCache) GetGroupInfo(ctx context.Context, groupID string) (val var cache cacheProto[sdkws.GroupInfo] return cache.Unmarshal(g.local.Get(ctx, cachekey.GetGroupInfoKey(groupID), func(ctx context.Context) ([]byte, error) { log.ZDebug(ctx, "GroupLocalCache GetGroupInfo rpc", "groupID", groupID) - return cache.Marshal(g.client.GetGroupInfoCache(ctx, groupID)) + return cache.Marshal(rpccall.ExtractField(ctx, group.GetGroupInfoCacheCaller.Invoke, + &group.GetGroupInfoCacheReq{ + GroupID: groupID, + }, (*group.GetGroupInfoCacheResp).GetGroupInfo)) })) } diff --git a/pkg/rpccache/online.go b/pkg/rpccache/online.go index a02a0662d..25362b529 100644 --- a/pkg/rpccache/online.go +++ b/pkg/rpccache/online.go @@ -3,18 +3,19 @@ package rpccache import ( "context" "fmt" - "github.com/openimsdk/protocol/constant" - "github.com/openimsdk/protocol/user" "math/rand" "strconv" "sync" "sync/atomic" "time" + "github.com/openimsdk/protocol/constant" + "github.com/openimsdk/protocol/rpccall" + "github.com/openimsdk/protocol/user" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/cachekey" "github.com/openimsdk/open-im-server/v3/pkg/localcache" "github.com/openimsdk/open-im-server/v3/pkg/localcache/lru" - "github.com/openimsdk/open-im-server/v3/pkg/rpcclient" "github.com/openimsdk/open-im-server/v3/pkg/util/useronline" "github.com/openimsdk/tools/db/cacheutil" "github.com/openimsdk/tools/log" @@ -22,10 +23,10 @@ import ( "github.com/redis/go-redis/v9" ) -func NewOnlineCache(user rpcclient.UserRpcClient, group *GroupLocalCache, rdb redis.UniversalClient, fullUserCache bool, fn func(ctx context.Context, userID string, platformIDs []int32)) (*OnlineCache, error) { +func NewOnlineCache(adminUserID []string, group *GroupLocalCache, rdb redis.UniversalClient, fullUserCache bool, fn func(ctx context.Context, userID string, platformIDs []int32)) (*OnlineCache, error) { l := &sync.Mutex{} x := &OnlineCache{ - user: user, + adminUserID: adminUserID, group: group, fullUserCache: fullUserCache, Lock: l, @@ -65,8 +66,8 @@ const ( ) type OnlineCache struct { - user rpcclient.UserRpcClient - group *GroupLocalCache + adminUserID []string + group *GroupLocalCache // fullUserCache if enabled, caches the online status of all users using mapCache; // otherwise, only a portion of users' online statuses (regardless of whether they are online) will be cached using lruCache. @@ -112,7 +113,7 @@ func (o *OnlineCache) initUsersOnlineStatus(ctx context.Context) (err error) { cursor := uint64(0) for resp == nil || resp.NextCursor != 0 { if err = retryOperation(func() error { - resp, err = o.user.GetAllOnlineUsers(ctx, cursor) + resp, err = user.GetAllOnlineUsersCaller.Invoke(ctx, &user.GetAllOnlineUsersReq{Cursor: cursor}) if err != nil { return err } @@ -186,7 +187,17 @@ func (o *OnlineCache) doSubscribe(ctx context.Context, rdb redis.UniversalClient func (o *OnlineCache) getUserOnlinePlatform(ctx context.Context, userID string) ([]int32, error) { platformIDs, err := o.lruCache.Get(userID, func() ([]int32, error) { - return o.user.GetUserOnlinePlatform(ctx, userID) + resp, err := rpccall.ExtractField(ctx, user.GetUserStatusCaller.Invoke, &user.GetUserStatusReq{ + UserID: o.adminUserID[0], + UserIDs: []string{userID}, + }, (*user.GetUserStatusResp).GetStatusList) + if err != nil { + return nil, err + } + if len(resp) == 0 { + return nil, nil + } + return resp[0].PlatformIDs, nil }) if err != nil { log.ZError(ctx, "OnlineCache GetUserOnlinePlatform", err, "userID", userID) @@ -228,7 +239,10 @@ func (o *OnlineCache) getUserOnlinePlatformBatch(ctx context.Context, userIDs [] platformIDsMap, err := o.lruCache.GetBatch(userIDs, func(missingUsers []string) (map[string][]int32, error) { platformIDsMap := make(map[string][]int32) - usersStatus, err := o.user.GetUsersOnlinePlatform(ctx, missingUsers) + usersStatus, err := rpccall.ExtractField(ctx, user.GetUserStatusCaller.Invoke, &user.GetUserStatusReq{ + UserID: o.adminUserID[0], + UserIDs: missingUsers, + }, (*user.GetUserStatusResp).GetStatusList) if err != nil { return nil, err } diff --git a/pkg/rpccache/user.go b/pkg/rpccache/user.go index 79a768597..fce2c911a 100644 --- a/pkg/rpccache/user.go +++ b/pkg/rpccache/user.go @@ -28,11 +28,10 @@ import ( "github.com/redis/go-redis/v9" ) -func NewUserLocalCache(client rpcclient.UserRpcClient, localCache *config.LocalCache, cli redis.UniversalClient) *UserLocalCache { +func NewUserLocalCache(localCache *config.LocalCache, cli redis.UniversalClient) *UserLocalCache { lc := localCache.User log.ZDebug(context.Background(), "UserLocalCache", "topic", lc.Topic, "slotNum", lc.SlotNum, "slotSize", lc.SlotSize, "enable", lc.Enable()) x := &UserLocalCache{ - client: client, local: localcache.New[[]byte]( localcache.WithLocalSlotNum(lc.SlotNum), localcache.WithLocalSlotSize(lc.SlotSize), @@ -48,8 +47,7 @@ func NewUserLocalCache(client rpcclient.UserRpcClient, localCache *config.LocalC } type UserLocalCache struct { - client rpcclient.UserRpcClient - local localcache.Cache[[]byte] + local localcache.Cache[[]byte] } func (u *UserLocalCache) GetUserInfo(ctx context.Context, userID string) (val *sdkws.UserInfo, err error) { @@ -64,7 +62,7 @@ func (u *UserLocalCache) GetUserInfo(ctx context.Context, userID string) (val *s var cache cacheProto[sdkws.UserInfo] return cache.Unmarshal(u.local.Get(ctx, cachekey.GetUserInfoKey(userID), func(ctx context.Context) ([]byte, error) { log.ZDebug(ctx, "UserLocalCache GetUserInfo rpc", "userID", userID) - return cache.Marshal(u.client.GetUserInfo(ctx, userID)) + return cache.Marshal(rpcclient.GetUserInfo(ctx, userID)) })) } @@ -88,7 +86,7 @@ func (u *UserLocalCache) getUserGlobalMsgRecvOpt(ctx context.Context, userID str var cache cacheProto[user.GetGlobalRecvMessageOptResp] return cache.Unmarshal(u.local.Get(ctx, cachekey.GetUserGlobalRecvMsgOptKey(userID), func(ctx context.Context) ([]byte, error) { log.ZDebug(ctx, "UserLocalCache GetUserGlobalMsgRecvOpt rpc", "userID", userID) - return cache.Marshal(u.client.Client.GetGlobalRecvMessageOpt(ctx, &user.GetGlobalRecvMessageOptReq{UserID: userID})) + return cache.Marshal(user.GetGlobalRecvMessageOptCaller.Invoke(ctx, &user.GetGlobalRecvMessageOptReq{UserID: userID})) })) } diff --git a/pkg/rpcclient/auth.go b/pkg/rpcclient/auth.go deleted file mode 100644 index 05fec35a0..000000000 --- a/pkg/rpcclient/auth.go +++ /dev/null @@ -1,74 +0,0 @@ -// Copyright © 2023 OpenIM. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package rpcclient - -import ( - "context" - - "github.com/openimsdk/protocol/auth" - "github.com/openimsdk/tools/discovery" - "github.com/openimsdk/tools/system/program" - "google.golang.org/grpc" -) - -func NewAuth(discov discovery.SvcDiscoveryRegistry, rpcRegisterName string) *Auth { - conn, err := discov.GetConn(context.Background(), rpcRegisterName) - if err != nil { - program.ExitWithError(err) - } - client := auth.NewAuthClient(conn) - return &Auth{discov: discov, conn: conn, Client: client} -} - -type Auth struct { - conn grpc.ClientConnInterface - Client auth.AuthClient - discov discovery.SvcDiscoveryRegistry -} - -func (a *Auth) ParseToken(ctx context.Context, token string) (*auth.ParseTokenResp, error) { - req := auth.ParseTokenReq{ - Token: token, - } - resp, err := a.Client.ParseToken(ctx, &req) - if err != nil { - return nil, err - } - return resp, err -} - -func (a *Auth) InvalidateToken(ctx context.Context, preservedToken, userID string, platformID int) (*auth.InvalidateTokenResp, error) { - req := auth.InvalidateTokenReq{ - PreservedToken: preservedToken, - UserID: userID, - PlatformID: int32(platformID), - } - resp, err := a.Client.InvalidateToken(ctx, &req) - if err != nil { - return nil, err - } - return resp, err -} - -func (a *Auth) KickTokens(ctx context.Context, tokens []string) (*auth.KickTokensResp, error) { - req := auth.KickTokensReq{ - Tokens: tokens, - } - resp, err := a.Client.KickTokens(ctx, &req) - if err != nil { - return nil, err - } - return resp, err -} diff --git a/pkg/rpcclient/conversation.go b/pkg/rpcclient/conversation.go deleted file mode 100644 index c69d355d6..000000000 --- a/pkg/rpcclient/conversation.go +++ /dev/null @@ -1,161 +0,0 @@ -// Copyright © 2023 OpenIM. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package rpcclient - -import ( - "context" - "fmt" - - pbconversation "github.com/openimsdk/protocol/conversation" - "github.com/openimsdk/tools/discovery" - "github.com/openimsdk/tools/errs" - "github.com/openimsdk/tools/system/program" - "google.golang.org/grpc" -) - -type Conversation struct { - Client pbconversation.ConversationClient - conn grpc.ClientConnInterface - discov discovery.SvcDiscoveryRegistry -} - -func NewConversation(discov discovery.SvcDiscoveryRegistry, rpcRegisterName string) *Conversation { - conn, err := discov.GetConn(context.Background(), rpcRegisterName) - if err != nil { - program.ExitWithError(err) - } - client := pbconversation.NewConversationClient(conn) - return &Conversation{discov: discov, conn: conn, Client: client} -} - -type ConversationRpcClient Conversation - -func NewConversationRpcClient(discov discovery.SvcDiscoveryRegistry, rpcRegisterName string) ConversationRpcClient { - return ConversationRpcClient(*NewConversation(discov, rpcRegisterName)) -} - -func (c *ConversationRpcClient) GetSingleConversationRecvMsgOpt(ctx context.Context, userID, conversationID string) (int32, error) { - var req pbconversation.GetConversationReq - req.OwnerUserID = userID - req.ConversationID = conversationID - conversation, err := c.Client.GetConversation(ctx, &req) - if err != nil { - return 0, err - } - return conversation.GetConversation().RecvMsgOpt, err -} - -func (c *ConversationRpcClient) SingleChatFirstCreateConversation(ctx context.Context, recvID, sendID, - conversationID string, conversationType int32) error { - _, err := c.Client.CreateSingleChatConversations(ctx, - &pbconversation.CreateSingleChatConversationsReq{ - RecvID: recvID, SendID: sendID, ConversationID: conversationID, - ConversationType: conversationType, - }) - return err -} - -func (c *ConversationRpcClient) GroupChatFirstCreateConversation(ctx context.Context, groupID string, userIDs []string) error { - _, err := c.Client.CreateGroupChatConversations(ctx, &pbconversation.CreateGroupChatConversationsReq{UserIDs: userIDs, GroupID: groupID}) - return err -} - -func (c *ConversationRpcClient) SetConversationMaxSeq(ctx context.Context, ownerUserIDs []string, conversationID string, maxSeq int64) error { - _, err := c.Client.SetConversationMaxSeq(ctx, &pbconversation.SetConversationMaxSeqReq{OwnerUserID: ownerUserIDs, ConversationID: conversationID, MaxSeq: maxSeq}) - return err -} - -func (c *ConversationRpcClient) SetConversationMinSeq(ctx context.Context, ownerUserIDs []string, conversationID string, minSeq int64) error { - _, err := c.Client.SetConversationMinSeq(ctx, &pbconversation.SetConversationMinSeqReq{OwnerUserID: ownerUserIDs, ConversationID: conversationID, MinSeq: minSeq}) - return err -} - -func (c *ConversationRpcClient) SetConversations(ctx context.Context, userIDs []string, conversation *pbconversation.ConversationReq) error { - _, err := c.Client.SetConversations(ctx, &pbconversation.SetConversationsReq{UserIDs: userIDs, Conversation: conversation}) - return err -} - -func (c *ConversationRpcClient) UpdateConversation(ctx context.Context, conversation *pbconversation.UpdateConversationReq) error { - _, err := c.Client.UpdateConversation(ctx, conversation) - return err -} - -func (c *ConversationRpcClient) GetConversationIDs(ctx context.Context, ownerUserID string) ([]string, error) { - resp, err := c.Client.GetConversationIDs(ctx, &pbconversation.GetConversationIDsReq{UserID: ownerUserID}) - if err != nil { - return nil, err - } - return resp.ConversationIDs, nil -} - -func (c *ConversationRpcClient) GetConversation(ctx context.Context, ownerUserID, conversationID string) (*pbconversation.Conversation, error) { - resp, err := c.Client.GetConversation(ctx, &pbconversation.GetConversationReq{OwnerUserID: ownerUserID, ConversationID: conversationID}) - if err != nil { - return nil, err - } - return resp.Conversation, nil -} - -func (c *ConversationRpcClient) GetConversationsByConversationID(ctx context.Context, conversationIDs []string) ([]*pbconversation.Conversation, error) { - if len(conversationIDs) == 0 { - return nil, nil - } - resp, err := c.Client.GetConversationsByConversationID(ctx, &pbconversation.GetConversationsByConversationIDReq{ConversationIDs: conversationIDs}) - if err != nil { - return nil, err - } - if len(resp.Conversations) == 0 { - return nil, errs.ErrRecordNotFound.WrapMsg(fmt.Sprintf("conversationIDs: %v not found", conversationIDs)) - } - return resp.Conversations, nil -} - -func (c *ConversationRpcClient) GetConversationOfflinePushUserIDs(ctx context.Context, conversationID string, userIDs []string) ([]string, error) { - resp, err := c.Client.GetConversationOfflinePushUserIDs(ctx, &pbconversation.GetConversationOfflinePushUserIDsReq{ConversationID: conversationID, UserIDs: userIDs}) - if err != nil { - return nil, err - } - return resp.UserIDs, nil -} - -func (c *ConversationRpcClient) GetConversations(ctx context.Context, ownerUserID string, conversationIDs []string) ([]*pbconversation.Conversation, error) { - if len(conversationIDs) == 0 { - return nil, nil - } - resp, err := c.Client.GetConversations( - ctx, - &pbconversation.GetConversationsReq{OwnerUserID: ownerUserID, ConversationIDs: conversationIDs}, - ) - if err != nil { - return nil, err - } - return resp.Conversations, nil -} - -func (c *ConversationRpcClient) GetConversationNotReceiveMessageUserIDs(ctx context.Context, conversationID string) ([]string, error) { - resp, err := c.Client.GetConversationNotReceiveMessageUserIDs(ctx, &pbconversation.GetConversationNotReceiveMessageUserIDsReq{ConversationID: conversationID}) - if err != nil { - return nil, err - } - return resp.UserIDs, nil -} - -func (c *ConversationRpcClient) GetConversationsNeedClearMsg(ctx context.Context) ([]*pbconversation.Conversation, error) { - resp, err := c.Client.GetConversationsNeedClearMsg(ctx, &pbconversation.GetConversationsNeedClearMsgReq{}) - if err != nil { - return nil, err - } - return resp.Conversations, nil -} diff --git a/pkg/rpcclient/friend.go b/pkg/rpcclient/friend.go deleted file mode 100644 index 359ed3a8b..000000000 --- a/pkg/rpcclient/friend.go +++ /dev/null @@ -1,87 +0,0 @@ -// Copyright © 2023 OpenIM. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package rpcclient - -import ( - "context" - - "github.com/openimsdk/protocol/relation" - sdkws "github.com/openimsdk/protocol/sdkws" - "github.com/openimsdk/tools/discovery" - "github.com/openimsdk/tools/system/program" - "google.golang.org/grpc" -) - -type Friend struct { - conn grpc.ClientConnInterface - Client relation.FriendClient - discov discovery.SvcDiscoveryRegistry -} - -func NewFriend(discov discovery.SvcDiscoveryRegistry, rpcRegisterName string) *Friend { - conn, err := discov.GetConn(context.Background(), rpcRegisterName) - if err != nil { - program.ExitWithError(err) - } - client := relation.NewFriendClient(conn) - return &Friend{discov: discov, conn: conn, Client: client} -} - -type FriendRpcClient Friend - -func NewFriendRpcClient(discov discovery.SvcDiscoveryRegistry, rpcRegisterName string) FriendRpcClient { - return FriendRpcClient(*NewFriend(discov, rpcRegisterName)) -} - -func (f *FriendRpcClient) GetFriendsInfo( - ctx context.Context, - ownerUserID, relationUserID string, -) (resp *sdkws.FriendInfo, err error) { - r, err := f.Client.GetDesignatedFriends( - ctx, - &relation.GetDesignatedFriendsReq{OwnerUserID: ownerUserID, FriendUserIDs: []string{relationUserID}}, - ) - if err != nil { - return nil, err - } - resp = r.FriendsInfo[0] - return -} - -// possibleFriendUserID Is PossibleFriendUserId's relations. -func (f *FriendRpcClient) IsFriend(ctx context.Context, possibleFriendUserID, userID string) (bool, error) { - resp, err := f.Client.IsFriend(ctx, &relation.IsFriendReq{UserID1: userID, UserID2: possibleFriendUserID}) - if err != nil { - return false, err - } - return resp.InUser1Friends, nil -} - -func (f *FriendRpcClient) GetFriendIDs(ctx context.Context, ownerUserID string) (relationIDs []string, err error) { - req := relation.GetFriendIDsReq{UserID: ownerUserID} - resp, err := f.Client.GetFriendIDs(ctx, &req) - if err != nil { - return nil, err - } - return resp.FriendIDs, nil -} - -func (f *FriendRpcClient) IsBlack(ctx context.Context, possibleBlackUserID, userID string) (bool, error) { - r, err := f.Client.IsBlack(ctx, &relation.IsBlackReq{UserID1: possibleBlackUserID, UserID2: userID}) - if err != nil { - return false, err - } - return r.InUser2Blacks, nil -} diff --git a/pkg/rpcclient/group.go b/pkg/rpcclient/group.go deleted file mode 100644 index 30d0b3288..000000000 --- a/pkg/rpcclient/group.go +++ /dev/null @@ -1,205 +0,0 @@ -// Copyright © 2023 OpenIM. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package rpcclient - -import ( - "context" - "strings" - - "github.com/openimsdk/open-im-server/v3/pkg/common/servererrs" - "github.com/openimsdk/protocol/constant" - "github.com/openimsdk/protocol/group" - "github.com/openimsdk/protocol/sdkws" - "github.com/openimsdk/tools/discovery" - "github.com/openimsdk/tools/system/program" - "github.com/openimsdk/tools/utils/datautil" -) - -type Group struct { - Client group.GroupClient - discov discovery.SvcDiscoveryRegistry -} - -func NewGroup(discov discovery.SvcDiscoveryRegistry, rpcRegisterName string) *Group { - conn, err := discov.GetConn(context.Background(), rpcRegisterName) - if err != nil { - program.ExitWithError(err) - } - client := group.NewGroupClient(conn) - return &Group{discov: discov, Client: client} -} - -type GroupRpcClient Group - -func NewGroupRpcClient(discov discovery.SvcDiscoveryRegistry, rpcRegisterName string) GroupRpcClient { - return GroupRpcClient(*NewGroup(discov, rpcRegisterName)) -} - -func (g *GroupRpcClient) GetGroupInfos(ctx context.Context, groupIDs []string, complete bool) ([]*sdkws.GroupInfo, error) { - resp, err := g.Client.GetGroupsInfo(ctx, &group.GetGroupsInfoReq{ - GroupIDs: groupIDs, - }) - if err != nil { - return nil, err - } - if complete { - if ids := datautil.Single(groupIDs, datautil.Slice(resp.GroupInfos, func(e *sdkws.GroupInfo) string { - return e.GroupID - })); len(ids) > 0 { - return nil, servererrs.ErrGroupIDNotFound.WrapMsg(strings.Join(ids, ",")) - } - } - return resp.GroupInfos, nil -} - -func (g *GroupRpcClient) GetGroupInfo(ctx context.Context, groupID string) (*sdkws.GroupInfo, error) { - groups, err := g.GetGroupInfos(ctx, []string{groupID}, true) - if err != nil { - return nil, err - } - return groups[0], nil -} - -func (g *GroupRpcClient) GetGroupInfoMap( - ctx context.Context, - groupIDs []string, - complete bool, -) (map[string]*sdkws.GroupInfo, error) { - groups, err := g.GetGroupInfos(ctx, groupIDs, complete) - if err != nil { - return nil, err - } - return datautil.SliceToMap(groups, func(e *sdkws.GroupInfo) string { - return e.GroupID - }), nil -} - -func (g *GroupRpcClient) GetGroupMemberInfos( - ctx context.Context, - groupID string, - userIDs []string, - complete bool, -) ([]*sdkws.GroupMemberFullInfo, error) { - resp, err := g.Client.GetGroupMembersInfo(ctx, &group.GetGroupMembersInfoReq{ - GroupID: groupID, - UserIDs: userIDs, - }) - if err != nil { - return nil, err - } - if complete { - if ids := datautil.Single(userIDs, datautil.Slice(resp.Members, func(e *sdkws.GroupMemberFullInfo) string { - return e.UserID - })); len(ids) > 0 { - return nil, servererrs.ErrNotInGroupYet.WrapMsg(strings.Join(ids, ",")) - } - } - return resp.Members, nil -} - -func (g *GroupRpcClient) GetGroupMemberInfo( - ctx context.Context, - groupID string, - userID string, -) (*sdkws.GroupMemberFullInfo, error) { - members, err := g.GetGroupMemberInfos(ctx, groupID, []string{userID}, true) - if err != nil { - return nil, err - } - return members[0], nil -} - -func (g *GroupRpcClient) GetGroupMemberInfoMap( - ctx context.Context, - groupID string, - userIDs []string, - complete bool, -) (map[string]*sdkws.GroupMemberFullInfo, error) { - members, err := g.GetGroupMemberInfos(ctx, groupID, userIDs, true) - if err != nil { - return nil, err - } - return datautil.SliceToMap(members, func(e *sdkws.GroupMemberFullInfo) string { - return e.UserID - }), nil -} - -func (g *GroupRpcClient) GetOwnerAndAdminInfos( - ctx context.Context, - groupID string, -) ([]*sdkws.GroupMemberFullInfo, error) { - resp, err := g.Client.GetGroupMemberRoleLevel(ctx, &group.GetGroupMemberRoleLevelReq{ - GroupID: groupID, - RoleLevels: []int32{constant.GroupOwner, constant.GroupAdmin}, - }) - if err != nil { - return nil, err - } - return resp.Members, nil -} - -func (g *GroupRpcClient) GetOwnerInfo(ctx context.Context, groupID string) (*sdkws.GroupMemberFullInfo, error) { - resp, err := g.Client.GetGroupMemberRoleLevel(ctx, &group.GetGroupMemberRoleLevelReq{ - GroupID: groupID, - RoleLevels: []int32{constant.GroupOwner}, - }) - return resp.Members[0], err -} - -func (g *GroupRpcClient) GetGroupMemberIDs(ctx context.Context, groupID string) ([]string, error) { - resp, err := g.Client.GetGroupMemberUserIDs(ctx, &group.GetGroupMemberUserIDsReq{ - GroupID: groupID, - }) - if err != nil { - return nil, err - } - return resp.UserIDs, nil -} - -func (g *GroupRpcClient) GetGroupInfoCache(ctx context.Context, groupID string) (*sdkws.GroupInfo, error) { - resp, err := g.Client.GetGroupInfoCache(ctx, &group.GetGroupInfoCacheReq{ - GroupID: groupID, - }) - if err != nil { - return nil, err - } - return resp.GroupInfo, nil -} - -func (g *GroupRpcClient) GetGroupMemberCache(ctx context.Context, groupID string, groupMemberID string) (*sdkws.GroupMemberFullInfo, error) { - resp, err := g.Client.GetGroupMemberCache(ctx, &group.GetGroupMemberCacheReq{ - GroupID: groupID, - GroupMemberID: groupMemberID, - }) - if err != nil { - return nil, err - } - return resp.Member, nil -} - -func (g *GroupRpcClient) DismissGroup(ctx context.Context, groupID string) error { - _, err := g.Client.DismissGroup(ctx, &group.DismissGroupReq{ - GroupID: groupID, - DeleteMember: true, - }) - return err -} - -func (g *GroupRpcClient) NotificationUserInfoUpdate(ctx context.Context, userID string) error { - _, err := g.Client.NotificationUserInfoUpdate(ctx, &group.NotificationUserInfoUpdateReq{ - UserID: userID, - }) - return err -} diff --git a/pkg/rpcclient/init.go b/pkg/rpcclient/init.go new file mode 100644 index 000000000..3d3f68aef --- /dev/null +++ b/pkg/rpcclient/init.go @@ -0,0 +1,60 @@ +package rpcclient + +import ( + "context" + + "github.com/openimsdk/open-im-server/v3/pkg/common/config" + pbauth "github.com/openimsdk/protocol/auth" + pbconversation "github.com/openimsdk/protocol/conversation" + pbgroup "github.com/openimsdk/protocol/group" + pbmsg "github.com/openimsdk/protocol/msg" + pbmsggateway "github.com/openimsdk/protocol/msggateway" + pbpush "github.com/openimsdk/protocol/push" + pbrelation "github.com/openimsdk/protocol/relation" + pbthird "github.com/openimsdk/protocol/third" + pbuser "github.com/openimsdk/protocol/user" + "github.com/openimsdk/tools/discovery" + "github.com/openimsdk/tools/system/program" + "google.golang.org/grpc" +) + +func InitRpcCaller(discov discovery.SvcDiscoveryRegistry, service config.RpcService) error { + initConn := func(discov discovery.SvcDiscoveryRegistry, name string, initFunc func(conn *grpc.ClientConn)) error { + conn, err := discov.GetConn(context.Background(), name) + if err != nil { + program.ExitWithError(err) + return err + } + initFunc(conn) + return nil + } + if err := initConn(discov, service.Auth, pbauth.InitAuth); err != nil { + return err + } + if err := initConn(discov, service.Conversation, pbconversation.InitConversation); err != nil { + return err + } + if err := initConn(discov, service.Group, pbgroup.InitGroup); err != nil { + return err + } + if err := initConn(discov, service.Msg, pbmsg.InitMsg); err != nil { + return err + } + if err := initConn(discov, service.MessageGateway, pbmsggateway.InitMsgGateway); err != nil { + return err + } + if err := initConn(discov, service.Push, pbpush.InitPushMsgService); err != nil { + return err + } + if err := initConn(discov, service.Friend, pbrelation.InitFriend); err != nil { + return err + } + if err := initConn(discov, service.Third, pbthird.InitThird); err != nil { + return err + } + if err := initConn(discov, service.User, pbuser.InitUser); err != nil { + return err + } + + return nil +} diff --git a/pkg/rpcclient/msg.go b/pkg/rpcclient/msg.go index 8313937cd..ca9b8fc68 100644 --- a/pkg/rpcclient/msg.go +++ b/pkg/rpcclient/msg.go @@ -19,17 +19,14 @@ import ( "encoding/json" "time" - "google.golang.org/grpc" "google.golang.org/protobuf/proto" "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/protocol/constant" "github.com/openimsdk/protocol/msg" "github.com/openimsdk/protocol/sdkws" - "github.com/openimsdk/tools/discovery" "github.com/openimsdk/tools/log" "github.com/openimsdk/tools/mq/memamq" - "github.com/openimsdk/tools/system/program" "github.com/openimsdk/tools/utils/idutil" "github.com/openimsdk/tools/utils/jsonutil" "github.com/openimsdk/tools/utils/timeutil" @@ -129,126 +126,6 @@ func newSessionTypeConf() map[int32]int32 { } } -type Message struct { - conn grpc.ClientConnInterface - Client msg.MsgClient - discov discovery.SvcDiscoveryRegistry -} - -func NewMessage(discov discovery.SvcDiscoveryRegistry, rpcRegisterName string) *Message { - conn, err := discov.GetConn(context.Background(), rpcRegisterName) - if err != nil { - program.ExitWithError(err) - } - client := msg.NewMsgClient(conn) - return &Message{discov: discov, conn: conn, Client: client} -} - -type MessageRpcClient Message - -func NewMessageRpcClient(discov discovery.SvcDiscoveryRegistry, rpcRegisterName string) MessageRpcClient { - return MessageRpcClient(*NewMessage(discov, rpcRegisterName)) -} - -// SendMsg sends a message through the gRPC client and returns the response. -// It wraps any encountered error for better error handling and context understanding. -func (m *MessageRpcClient) SendMsg(ctx context.Context, req *msg.SendMsgReq) (*msg.SendMsgResp, error) { - resp, err := m.Client.SendMsg(ctx, req) - if err != nil { - return nil, err - } - return resp, nil -} - -// SetUserConversationsMinSeq set min seq -func (m *MessageRpcClient) SetUserConversationsMinSeq(ctx context.Context, req *msg.SetUserConversationsMinSeqReq) (*msg.SetUserConversationsMinSeqResp, error) { - resp, err := m.Client.SetUserConversationsMinSeq(ctx, req) - if err != nil { - return nil, err - } - return resp, nil -} - -// GetMaxSeq retrieves the maximum sequence number from the gRPC client. -// Errors during the gRPC call are wrapped to provide additional context. -func (m *MessageRpcClient) GetMaxSeq(ctx context.Context, req *sdkws.GetMaxSeqReq) (*sdkws.GetMaxSeqResp, error) { - resp, err := m.Client.GetMaxSeq(ctx, req) - if err != nil { - return nil, err - } - return resp, nil -} - -func (m *MessageRpcClient) GetMaxSeqs(ctx context.Context, conversationIDs []string) (map[string]int64, error) { - log.ZDebug(ctx, "GetMaxSeqs", "conversationIDs", conversationIDs) - resp, err := m.Client.GetMaxSeqs(ctx, &msg.GetMaxSeqsReq{ - ConversationIDs: conversationIDs, - }) - if err != nil { - return nil, err - } - return resp.MaxSeqs, err -} - -func (m *MessageRpcClient) GetHasReadSeqs(ctx context.Context, userID string, conversationIDs []string) (map[string]int64, error) { - resp, err := m.Client.GetHasReadSeqs(ctx, &msg.GetHasReadSeqsReq{ - UserID: userID, - ConversationIDs: conversationIDs, - }) - if err != nil { - return nil, err - } - return resp.MaxSeqs, err -} - -func (m *MessageRpcClient) GetMsgByConversationIDs(ctx context.Context, docIDs []string, seqs map[string]int64) (map[string]*sdkws.MsgData, error) { - resp, err := m.Client.GetMsgByConversationIDs(ctx, &msg.GetMsgByConversationIDsReq{ - ConversationIDs: docIDs, - MaxSeqs: seqs, - }) - if err != nil { - return nil, err - } - return resp.MsgDatas, err -} - -// PullMessageBySeqList retrieves messages by their sequence numbers using the gRPC client. -// It directly forwards the request to the gRPC client and returns the response along with any error encountered. -func (m *MessageRpcClient) PullMessageBySeqList(ctx context.Context, req *sdkws.PullMessageBySeqsReq) (*sdkws.PullMessageBySeqsResp, error) { - resp, err := m.Client.PullMessageBySeqs(ctx, req) - if err != nil { - // Wrap the error to provide more context if the gRPC call fails. - return nil, err - } - return resp, nil -} - -func (m *MessageRpcClient) GetConversationsHasReadAndMaxSeq(ctx context.Context, req *msg.GetConversationsHasReadAndMaxSeqReq) (*msg.GetConversationsHasReadAndMaxSeqResp, error) { - resp, err := m.Client.GetConversationsHasReadAndMaxSeq(ctx, req) - if err != nil { - // Wrap the error to provide more context if the gRPC call fails. - return nil, err - } - return resp, nil -} - -func (m *MessageRpcClient) GetSeqMessage(ctx context.Context, req *msg.GetSeqMessageReq) (*msg.GetSeqMessageResp, error) { - return m.Client.GetSeqMessage(ctx, req) -} - -func (m *MessageRpcClient) GetConversationMaxSeq(ctx context.Context, conversationID string) (int64, error) { - resp, err := m.Client.GetConversationMaxSeq(ctx, &msg.GetConversationMaxSeqReq{ConversationID: conversationID}) - if err != nil { - return 0, err - } - return resp.MaxSeq, nil -} - -func (m *MessageRpcClient) DestructMsgs(ctx context.Context, ts int64) error { - _, err := m.Client.DestructMsgs(ctx, &msg.DestructMsgsReq{Timestamp: ts}) - return err -} - type NotificationSender struct { contentTypeConf map[int32]config.NotificationConfig sessionTypeConf map[int32]int32 @@ -271,15 +148,17 @@ func WithLocalSendMsg(sendMsg func(ctx context.Context, req *msg.SendMsgReq) (*m } } -func WithRpcClient(msgRpcClient *MessageRpcClient) NotificationSenderOptions { +func WithRpcClient() NotificationSenderOptions { return func(s *NotificationSender) { - s.sendMsg = msgRpcClient.SendMsg + s.sendMsg = func(ctx context.Context, req *msg.SendMsgReq) (*msg.SendMsgResp, error) { + return msg.SendMsgCaller.Invoke(ctx, req) + } } } -func WithUserRpcClient(userRpcClient *UserRpcClient) NotificationSenderOptions { +func WithUserRpcClient() NotificationSenderOptions { return func(s *NotificationSender) { - s.getUserInfo = userRpcClient.GetUserInfo + s.getUserInfo = GetUserInfo } } diff --git a/pkg/rpcclient/push.go b/pkg/rpcclient/push.go deleted file mode 100644 index c549e454a..000000000 --- a/pkg/rpcclient/push.go +++ /dev/null @@ -1,52 +0,0 @@ -// Copyright © 2023 OpenIM. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package rpcclient - -import ( - "context" - - "github.com/openimsdk/protocol/push" - "github.com/openimsdk/tools/discovery" - "github.com/openimsdk/tools/system/program" - "google.golang.org/grpc" -) - -type Push struct { - conn grpc.ClientConnInterface - Client push.PushMsgServiceClient - discov discovery.SvcDiscoveryRegistry -} - -func NewPush(discov discovery.SvcDiscoveryRegistry, rpcRegisterName string) *Push { - conn, err := discov.GetConn(context.Background(), rpcRegisterName) - if err != nil { - program.ExitWithError(err) - } - return &Push{ - discov: discov, - conn: conn, - Client: push.NewPushMsgServiceClient(conn), - } -} - -type PushRpcClient Push - -func NewPushRpcClient(discov discovery.SvcDiscoveryRegistry, rpcRegisterName string) PushRpcClient { - return PushRpcClient(*NewPush(discov, rpcRegisterName)) -} - -func (p *PushRpcClient) DelUserPushToken(ctx context.Context, req *push.DelUserPushTokenReq) (*push.DelUserPushTokenResp, error) { - return p.Client.DelUserPushToken(ctx, req) -} diff --git a/pkg/rpcclient/third.go b/pkg/rpcclient/third.go deleted file mode 100644 index 7cdc60d52..000000000 --- a/pkg/rpcclient/third.go +++ /dev/null @@ -1,47 +0,0 @@ -// Copyright © 2023 OpenIM. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package rpcclient - -import ( - "context" - - "github.com/openimsdk/protocol/third" - "github.com/openimsdk/tools/discovery" - "github.com/openimsdk/tools/system/program" - "google.golang.org/grpc" -) - -type Third struct { - conn grpc.ClientConnInterface - Client third.ThirdClient - discov discovery.SvcDiscoveryRegistry - GrafanaUrl string -} - -func NewThird(discov discovery.SvcDiscoveryRegistry, rpcRegisterName, grafanaUrl string) *Third { - conn, err := discov.GetConn(context.Background(), rpcRegisterName) - if err != nil { - program.ExitWithError(err) - } - client := third.NewThirdClient(conn) - if err != nil { - program.ExitWithError(err) - } - return &Third{discov: discov, Client: client, conn: conn, GrafanaUrl: grafanaUrl} -} -func (t *Third) DeleteOutdatedData(ctx context.Context, expires int64) error { - _, err := t.Client.DeleteOutdatedData(ctx, &third.DeleteOutdatedDataReq{ExpireTime: expires}) - return err -} diff --git a/pkg/rpcclient/user.go b/pkg/rpcclient/user.go index bdc1a2e01..463dd9a39 100644 --- a/pkg/rpcclient/user.go +++ b/pkg/rpcclient/user.go @@ -18,60 +18,18 @@ import ( "context" "strings" - "github.com/openimsdk/open-im-server/v3/pkg/authverify" "github.com/openimsdk/open-im-server/v3/pkg/common/servererrs" "github.com/openimsdk/protocol/sdkws" "github.com/openimsdk/protocol/user" - "github.com/openimsdk/tools/discovery" - "github.com/openimsdk/tools/system/program" "github.com/openimsdk/tools/utils/datautil" - "google.golang.org/grpc" ) -// User represents a structure holding connection details for the User RPC client. -type User struct { - conn grpc.ClientConnInterface - Client user.UserClient - Discov discovery.SvcDiscoveryRegistry - MessageGateWayRpcName string - imAdminUserID []string -} - -// NewUser initializes and returns a User instance based on the provided service discovery registry. -func NewUser(discov discovery.SvcDiscoveryRegistry, rpcRegisterName, messageGateWayRpcName string, - imAdminUserID []string) *User { - conn, err := discov.GetConn(context.Background(), rpcRegisterName) - if err != nil { - program.ExitWithError(err) - } - client := user.NewUserClient(conn) - return &User{Discov: discov, Client: client, - conn: conn, - MessageGateWayRpcName: messageGateWayRpcName, - imAdminUserID: imAdminUserID} -} - -// UserRpcClient represents the structure for a User RPC client. -type UserRpcClient User - -// NewUserRpcClientByUser initializes a UserRpcClient based on the provided User instance. -func NewUserRpcClientByUser(user *User) *UserRpcClient { - rpc := UserRpcClient(*user) - return &rpc -} - -// NewUserRpcClient initializes a UserRpcClient based on the provided service discovery registry. -func NewUserRpcClient(client discovery.SvcDiscoveryRegistry, rpcRegisterName string, - imAdminUserID []string) UserRpcClient { - return UserRpcClient(*NewUser(client, rpcRegisterName, "", imAdminUserID)) -} - // GetUsersInfo retrieves information for multiple users based on their user IDs. -func (u *UserRpcClient) GetUsersInfo(ctx context.Context, userIDs []string) ([]*sdkws.UserInfo, error) { +func GetUsersInfo(ctx context.Context, userIDs []string) ([]*sdkws.UserInfo, error) { if len(userIDs) == 0 { return []*sdkws.UserInfo{}, nil } - resp, err := u.Client.GetDesignateUsers(ctx, &user.GetDesignateUsersReq{ + resp, err := user.GetDesignateUsersCaller.Invoke(ctx, &user.GetDesignateUsersReq{ UserIDs: userIDs, }) if err != nil { @@ -86,8 +44,8 @@ func (u *UserRpcClient) GetUsersInfo(ctx context.Context, userIDs []string) ([]* } // GetUserInfo retrieves information for a single user based on the provided user ID. -func (u *UserRpcClient) GetUserInfo(ctx context.Context, userID string) (*sdkws.UserInfo, error) { - users, err := u.GetUsersInfo(ctx, []string{userID}) +func GetUserInfo(ctx context.Context, userID string) (*sdkws.UserInfo, error) { + users, err := GetUsersInfo(ctx, []string{userID}) if err != nil { return nil, err } @@ -95,8 +53,8 @@ func (u *UserRpcClient) GetUserInfo(ctx context.Context, userID string) (*sdkws. } // GetUsersInfoMap retrieves a map of user information indexed by their user IDs. -func (u *UserRpcClient) GetUsersInfoMap(ctx context.Context, userIDs []string) (map[string]*sdkws.UserInfo, error) { - users, err := u.GetUsersInfo(ctx, userIDs) +func GetUsersInfoMap(ctx context.Context, userIDs []string) (map[string]*sdkws.UserInfo, error) { + users, err := GetUsersInfo(ctx, userIDs) if err != nil { return nil, err } @@ -106,11 +64,11 @@ func (u *UserRpcClient) GetUsersInfoMap(ctx context.Context, userIDs []string) ( } // GetPublicUserInfos retrieves public information for multiple users based on their user IDs. -func (u *UserRpcClient) GetPublicUserInfos( +func GetPublicUserInfos( ctx context.Context, userIDs []string, ) ([]*sdkws.PublicUserInfo, error) { - users, err := u.GetUsersInfo(ctx, userIDs) + users, err := GetUsersInfo(ctx, userIDs) if err != nil { return nil, err } @@ -126,8 +84,8 @@ func (u *UserRpcClient) GetPublicUserInfos( } // GetPublicUserInfo retrieves public information for a single user based on the provided user ID. -func (u *UserRpcClient) GetPublicUserInfo(ctx context.Context, userID string) (*sdkws.PublicUserInfo, error) { - users, err := u.GetPublicUserInfos(ctx, []string{userID}) +func GetPublicUserInfo(ctx context.Context, userID string) (*sdkws.PublicUserInfo, error) { + users, err := GetPublicUserInfos(ctx, []string{userID}) if err != nil { return nil, err } @@ -136,11 +94,11 @@ func (u *UserRpcClient) GetPublicUserInfo(ctx context.Context, userID string) (* } // GetPublicUserInfoMap retrieves a map of public user information indexed by their user IDs. -func (u *UserRpcClient) GetPublicUserInfoMap( +func GetPublicUserInfoMap( ctx context.Context, userIDs []string, ) (map[string]*sdkws.PublicUserInfo, error) { - users, err := u.GetPublicUserInfos(ctx, userIDs) + users, err := GetPublicUserInfos(ctx, userIDs) if err != nil { return nil, err } @@ -149,83 +107,3 @@ func (u *UserRpcClient) GetPublicUserInfoMap( return e.UserID }), nil } - -// GetUserGlobalMsgRecvOpt retrieves the global message receive option for a user based on the provided user ID. -func (u *UserRpcClient) GetUserGlobalMsgRecvOpt(ctx context.Context, userID string) (int32, error) { - resp, err := u.Client.GetGlobalRecvMessageOpt(ctx, &user.GetGlobalRecvMessageOptReq{ - UserID: userID, - }) - if err != nil { - return 0, err - } - return resp.GlobalRecvMsgOpt, nil -} - -// Access verifies the access rights for the provided user ID. -func (u *UserRpcClient) Access(ctx context.Context, ownerUserID string) error { - _, err := u.GetUserInfo(ctx, ownerUserID) - if err != nil { - return err - } - return authverify.CheckAccessV3(ctx, ownerUserID, u.imAdminUserID) -} - -// GetAllUserID retrieves all user IDs with pagination options. -func (u *UserRpcClient) GetAllUserID(ctx context.Context, pageNumber, showNumber int32) (*user.GetAllUserIDResp, error) { - resp, err := u.Client.GetAllUserID(ctx, &user.GetAllUserIDReq{Pagination: &sdkws.RequestPagination{PageNumber: pageNumber, ShowNumber: showNumber}}) - if err != nil { - return nil, err - } - return resp, nil -} - -// GetAllUserIDs retrieves all user IDs with pagination options. -func (u *UserRpcClient) GetAllUserIDs(ctx context.Context, pageNumber, showNumber int32) ([]string, error) { - resp, err := u.Client.GetAllUserID(ctx, &user.GetAllUserIDReq{Pagination: &sdkws.RequestPagination{PageNumber: pageNumber, ShowNumber: showNumber}}) - if err != nil { - return nil, err - } - return resp.UserIDs, nil -} - -// SetUserStatus sets the status for a user based on the provided user ID, status, and platform ID. -func (u *UserRpcClient) SetUserStatus(ctx context.Context, userID string, status int32, platformID int) error { - _, err := u.Client.SetUserStatus(ctx, &user.SetUserStatusReq{ - UserID: userID, - Status: status, PlatformID: int32(platformID), - }) - return err -} - -func (u *UserRpcClient) GetNotificationByID(ctx context.Context, userID string) error { - _, err := u.Client.GetNotificationAccount(ctx, &user.GetNotificationAccountReq{ - UserID: userID, - }) - return err -} - -func (u *UserRpcClient) GetUsersOnlinePlatform(ctx context.Context, userIDs []string) ([]*user.OnlineStatus, error) { - if len(userIDs) == 0 { - return nil, nil - } - resp, err := u.Client.GetUserStatus(ctx, &user.GetUserStatusReq{UserIDs: userIDs, UserID: u.imAdminUserID[0]}) - if err != nil { - return nil, err - } - return resp.StatusList, nil -} - -func (u *UserRpcClient) GetUserOnlinePlatform(ctx context.Context, userID string) ([]int32, error) { - resp, err := u.GetUsersOnlinePlatform(ctx, []string{userID}) - if err != nil { - return nil, err - } - if len(resp) == 0 { - return nil, nil - } - return resp[0].PlatformIDs, nil -} - -func (u *UserRpcClient) GetAllOnlineUsers(ctx context.Context, cursor uint64) (*user.GetAllOnlineUsersResp, error) { - return u.Client.GetAllOnlineUsers(ctx, &user.GetAllOnlineUsersReq{Cursor: cursor}) -} From fdc97c6c4829cb606da5f962b8747d8b44bb1e68 Mon Sep 17 00:00:00 2001 From: chao <48119764+withchao@users.noreply.github.com> Date: Tue, 17 Dec 2024 18:48:52 +0800 Subject: [PATCH 079/199] fix: when fetching a referenced message, it indicates that the original message has been deleted. (#2977) * pb * fix: Modifying other fields while setting IsPrivateChat does not take effect * fix: quote message error revoke --- pkg/common/storage/controller/msg.go | 52 +++++++++++++++++++++++----- 1 file changed, 44 insertions(+), 8 deletions(-) diff --git a/pkg/common/storage/controller/msg.go b/pkg/common/storage/controller/msg.go index 8d82d8543..3c3cd9671 100644 --- a/pkg/common/storage/controller/msg.go +++ b/pkg/common/storage/controller/msg.go @@ -269,7 +269,6 @@ func (db *commonMsgDatabase) getMsgBySeqs(ctx context.Context, userID, conversat } return totalMsgs, nil } - func (db *commonMsgDatabase) handlerDBMsg(ctx context.Context, cache map[int64][]*model.MsgInfoModel, userID, conversationID string, msg *model.MsgInfoModel) { if msg.IsRead { msg.Msg.IsRead = true @@ -280,16 +279,53 @@ func (db *commonMsgDatabase) handlerDBMsg(ctx context.Context, cache map[int64][ if msg.Msg.Content == "" { return } + type MsgData struct { + SendID string `json:"sendID"` + RecvID string `json:"recvID"` + GroupID string `json:"groupID"` + ClientMsgID string `json:"clientMsgID"` + ServerMsgID string `json:"serverMsgID"` + SenderPlatformID int32 `json:"senderPlatformID"` + SenderNickname string `json:"senderNickname"` + SenderFaceURL string `json:"senderFaceURL"` + SessionType int32 `json:"sessionType"` + MsgFrom int32 `json:"msgFrom"` + ContentType int32 `json:"contentType"` + Content string `json:"content"` + Seq int64 `json:"seq"` + SendTime int64 `json:"sendTime"` + CreateTime int64 `json:"createTime"` + Status int32 `json:"status"` + IsRead bool `json:"isRead"` + Options map[string]bool `json:"options,omitempty"` + OfflinePushInfo *sdkws.OfflinePushInfo `json:"offlinePushInfo"` + AtUserIDList []string `json:"atUserIDList"` + AttachedInfo string `json:"attachedInfo"` + Ex string `json:"ex"` + KeyVersion int32 `json:"keyVersion"` + DstUserIDs []string `json:"dstUserIDs"` + } var quoteMsg struct { Text string `json:"text,omitempty"` - QuoteMessage *sdkws.MsgData `json:"quoteMessage,omitempty"` + QuoteMessage *MsgData `json:"quoteMessage,omitempty"` MessageEntityList json.RawMessage `json:"messageEntityList,omitempty"` } if err := json.Unmarshal([]byte(msg.Msg.Content), "eMsg); err != nil { log.ZError(ctx, "json.Unmarshal", err) return } - if quoteMsg.QuoteMessage == nil || quoteMsg.QuoteMessage.ContentType == constant.MsgRevokeNotification { + if quoteMsg.QuoteMessage == nil || quoteMsg.QuoteMessage.Content == "" { + return + } + if quoteMsg.QuoteMessage.Content == "e30=" { + quoteMsg.QuoteMessage.Content = "{}" + data, err := json.Marshal("eMsg) + if err != nil { + return + } + msg.Msg.Content = string(data) + } + if quoteMsg.QuoteMessage.Seq <= 0 && quoteMsg.QuoteMessage.ContentType == constant.MsgRevokeNotification { return } var msgs []*model.MsgInfoModel @@ -311,9 +347,9 @@ func (db *commonMsgDatabase) handlerDBMsg(ctx context.Context, cache map[int64][ } quoteMsg.QuoteMessage.ContentType = constant.MsgRevokeNotification if len(msgs) > 0 { - quoteMsg.QuoteMessage.Content = []byte(msgs[0].Msg.Content) + quoteMsg.QuoteMessage.Content = msgs[0].Msg.Content } else { - quoteMsg.QuoteMessage.Content = []byte("{}") + quoteMsg.QuoteMessage.Content = "{}" } data, err := json.Marshal("eMsg) if err != nil { @@ -321,9 +357,9 @@ func (db *commonMsgDatabase) handlerDBMsg(ctx context.Context, cache map[int64][ return } msg.Msg.Content = string(data) - if _, err := db.msgDocDatabase.UpdateMsg(ctx, db.msgTable.GetDocID(conversationID, msg.Msg.Seq), db.msgTable.GetMsgIndex(msg.Msg.Seq), "msg", msg.Msg); err != nil { - log.ZError(ctx, "UpdateMsgContent", err) - } + //if _, err := db.msgDocDatabase.UpdateMsg(ctx, db.msgTable.GetDocID(conversationID, msg.Msg.Seq), db.msgTable.GetMsgIndex(msg.Msg.Seq), "msg", msg.Msg); err != nil { + // log.ZError(ctx, "UpdateMsgContent", err) + //} } func (db *commonMsgDatabase) findMsgInfoBySeq(ctx context.Context, userID, docID string, conversationID string, seqs []int64) (totalMsgs []*model.MsgInfoModel, err error) { From 9929bd3ce5b93c6669114e3468d602a61d3d21d0 Mon Sep 17 00:00:00 2001 From: icey-yu <119291641+icey-yu@users.noreply.github.com> Date: Wed, 18 Dec 2024 10:45:45 +0800 Subject: [PATCH 080/199] feat: Group Monitoring Components, Enable Host Mode && Deprecate reliabilityLevel and unreadCount in notification.yml (#2975) * chore: docker config * chore: docker config * feat: Group Monitoring Components, Enable Host Mode && Deprecate reliabilityLevel and unreadCount in notification.yml * feat: Group Monitoring Components, Enable Host Mode && Deprecate reliabilityLevel and unreadCount in notification.yml * feat: Group Monitoring Components, Enable Host Mode && Deprecate reliabilityLevel and unreadCount in notification.yml * feat: cicd --- .env | 4 ++ .github/workflows/go-build-test.yml | 4 +- config/notification.yml | 6 +-- config/openim-api.yml | 2 +- config/prometheus.yml | 54 +++++++++++----------- config/share.yml | 10 ----- docker-compose.yml | 26 ++++++----- internal/msggateway/hub_server.go | 2 +- pkg/common/cmd/auth.go | 2 +- pkg/common/cmd/conversation.go | 2 +- pkg/common/cmd/friend.go | 2 +- pkg/common/cmd/group.go | 2 +- pkg/common/cmd/msg.go | 2 +- pkg/common/cmd/push.go | 2 +- pkg/common/cmd/third.go | 2 +- pkg/common/cmd/user.go | 2 +- pkg/common/config/config.go | 69 +++++++++++++++++++++++++++++ pkg/common/config/parse.go | 2 +- pkg/common/startrpc/start.go | 9 +++- 19 files changed, 136 insertions(+), 68 deletions(-) diff --git a/.env b/.env index 87f4badba..4a39255b8 100644 --- a/.env +++ b/.env @@ -15,3 +15,7 @@ OPENIM_ADMIN_FRONT_IMAGE=openim/openim-admin-front:release-v1.8.3 #OPENIM_ADMIN_FRONT_IMAGE=registry.cn-hangzhou.aliyuncs.com/openimsdk/openim-admin-front:release-v1.8.3 DATA_DIR=./ + +PROMETHEUS_PORT=19091 +ALERTMANAGER_PORT=19093 +GRAFANA_PORT=13000 \ No newline at end of file diff --git a/.github/workflows/go-build-test.yml b/.github/workflows/go-build-test.yml index 2ca960cc9..10a4154d6 100644 --- a/.github/workflows/go-build-test.yml +++ b/.github/workflows/go-build-test.yml @@ -126,8 +126,8 @@ jobs: - name: Modify Server Configuration run: | - yq e '.groupCreated.unreadCount = true' -i ${{ env.CONFIG_PATH }} - yq e '.friendApplicationApproved.unreadCount = true' -i ${{ env.CONFIG_PATH }} + yq e '.groupCreated.isSendMsg = true' -i ${{ env.CONFIG_PATH }} + yq e '.friendApplicationApproved.isSendMsg = true' -i ${{ env.CONFIG_PATH }} - name: Start Server Services run: | diff --git a/config/notification.yml b/config/notification.yml index ba5ca1c21..c3ae01539 100644 --- a/config/notification.yml +++ b/config/notification.yml @@ -1,10 +1,8 @@ groupCreated: isSendMsg: true -# Reliability level of the message sending. -# Set to 1 to send only when online, 2 for guaranteed delivery. +# Deprecated. Fixed as 1. reliabilityLevel: 1 -# This setting is effective only when 'isSendMsg' is true. -# It controls whether to count unread messages. +# Deprecated. Fixed as false. unreadCount: false # Configuration for offline push notifications. offlinePush: diff --git a/config/openim-api.yml b/config/openim-api.yml index a23b5fb31..f063005ec 100644 --- a/config/openim-api.yml +++ b/config/openim-api.yml @@ -16,4 +16,4 @@ prometheus: # It will only take effect when autoSetPorts is set to false. ports: [ 12002 ] # This address can be accessed via a browser - grafanaURL: http://127.0.0.1:13000/ + grafanaURL: diff --git a/config/prometheus.yml b/config/prometheus.yml index 6fb112824..d176bfcbd 100644 --- a/config/prometheus.yml +++ b/config/prometheus.yml @@ -8,7 +8,7 @@ global: alerting: alertmanagers: - static_configs: - - targets: [internal_ip:19093] + - targets: [127.0.0.1:19093] # Load rules once and periodically evaluate them according to the global evaluation_interval. rule_files: @@ -25,95 +25,95 @@ scrape_configs: # prometheus fetches application services - job_name: node_exporter static_configs: - - targets: [ internal_ip:20500 ] + - targets: [ 127.0.0.1:20500 ] - job_name: openimserver-openim-api http_sd_configs: - - url: "http://internal_ip:10002/prometheus_discovery/api" + - url: "http://127.0.0.1:10002/prometheus_discovery/api" # static_configs: -# - targets: [ internal_ip:12002 ] +# - targets: [ 127.0.0.1:12002 ] # labels: # namespace: default - job_name: openimserver-openim-msggateway http_sd_configs: - - url: "http://internal_ip:10002/prometheus_discovery/msg_gateway" + - url: "http://127.0.0.1:10002/prometheus_discovery/msg_gateway" # static_configs: -# - targets: [ internal_ip:12140 ] -# # - targets: [ internal_ip:12140, internal_ip:12141, internal_ip:12142, internal_ip:12143, internal_ip:12144, internal_ip:12145, internal_ip:12146, internal_ip:12147, internal_ip:12148, internal_ip:12149, internal_ip:12150, internal_ip:12151, internal_ip:12152, internal_ip:12153, internal_ip:12154, internal_ip:12155 ] +# - targets: [ 127.0.0.1:12140 ] +# # - targets: [ 127.0.0.1:12140, 127.0.0.1:12141, 127.0.0.1:12142, 127.0.0.1:12143, 127.0.0.1:12144, 127.0.0.1:12145, 127.0.0.1:12146, 127.0.0.1:12147, 127.0.0.1:12148, 127.0.0.1:12149, 127.0.0.1:12150, 127.0.0.1:12151, 127.0.0.1:12152, 127.0.0.1:12153, 127.0.0.1:12154, 127.0.0.1:12155 ] # labels: # namespace: default - job_name: openimserver-openim-msgtransfer http_sd_configs: - - url: "http://internal_ip:10002/prometheus_discovery/msg_transfer" + - url: "http://127.0.0.1:10002/prometheus_discovery/msg_transfer" # static_configs: -# - targets: [ internal_ip:12020, internal_ip:12021, internal_ip:12022, internal_ip:12023, internal_ip:12024, internal_ip:12025, internal_ip:12026, internal_ip:12027 ] -# # - targets: [ internal_ip:12020, internal_ip:12021, internal_ip:12022, internal_ip:12023, internal_ip:12024, internal_ip:12025, internal_ip:12026, internal_ip:12027, internal_ip:12028, internal_ip:12029, internal_ip:12030, internal_ip:12031, internal_ip:12032, internal_ip:12033, internal_ip:12034, internal_ip:12035 ] +# - targets: [ 127.0.0.1:12020, 127.0.0.1:12021, 127.0.0.1:12022, 127.0.0.1:12023, 127.0.0.1:12024, 127.0.0.1:12025, 127.0.0.1:12026, 127.0.0.1:12027 ] +# # - targets: [ 127.0.0.1:12020, 127.0.0.1:12021, 127.0.0.1:12022, 127.0.0.1:12023, 127.0.0.1:12024, 127.0.0.1:12025, 127.0.0.1:12026, 127.0.0.1:12027, 127.0.0.1:12028, 127.0.0.1:12029, 127.0.0.1:12030, 127.0.0.1:12031, 127.0.0.1:12032, 127.0.0.1:12033, 127.0.0.1:12034, 127.0.0.1:12035 ] # labels: # namespace: default - job_name: openimserver-openim-push http_sd_configs: - - url: "http://internal_ip:10002/prometheus_discovery/push" + - url: "http://127.0.0.1:10002/prometheus_discovery/push" # static_configs: -# - targets: [ internal_ip:12170, internal_ip:12171, internal_ip:12172, internal_ip:12173, internal_ip:12174, internal_ip:12175, internal_ip:12176, internal_ip:12177 ] -## - targets: [ internal_ip:12170, internal_ip:12171, internal_ip:12172, internal_ip:12173, internal_ip:12174, internal_ip:12175, internal_ip:12176, internal_ip:12177, internal_ip:12178, internal_ip:12179, internal_ip:12180, internal_ip:12182, internal_ip:12183, internal_ip:12184, internal_ip:12185, internal_ip:12186 ] +# - targets: [ 127.0.0.1:12170, 127.0.0.1:12171, 127.0.0.1:12172, 127.0.0.1:12173, 127.0.0.1:12174, 127.0.0.1:12175, 127.0.0.1:12176, 127.0.0.1:12177 ] +## - targets: [ 127.0.0.1:12170, 127.0.0.1:12171, 127.0.0.1:12172, 127.0.0.1:12173, 127.0.0.1:12174, 127.0.0.1:12175, 127.0.0.1:12176, 127.0.0.1:12177, 127.0.0.1:12178, 127.0.0.1:12179, 127.0.0.1:12180, 127.0.0.1:12182, 127.0.0.1:12183, 127.0.0.1:12184, 127.0.0.1:12185, 127.0.0.1:12186 ] # labels: # namespace: default - job_name: openimserver-openim-rpc-auth http_sd_configs: - - url: "http://internal_ip:10002/prometheus_discovery/auth" + - url: "http://127.0.0.1:10002/prometheus_discovery/auth" # static_configs: -# - targets: [ internal_ip:12200 ] +# - targets: [ 127.0.0.1:12200 ] # labels: # namespace: default - job_name: openimserver-openim-rpc-conversation http_sd_configs: - - url: "http://internal_ip:10002/prometheus_discovery/conversation" + - url: "http://127.0.0.1:10002/prometheus_discovery/conversation" # static_configs: -# - targets: [ internal_ip:12220 ] +# - targets: [ 127.0.0.1:12220 ] # labels: # namespace: default - job_name: openimserver-openim-rpc-friend http_sd_configs: - - url: "http://internal_ip:10002/prometheus_discovery/friend" + - url: "http://127.0.0.1:10002/prometheus_discovery/friend" # static_configs: -# - targets: [ internal_ip:12240 ] +# - targets: [ 127.0.0.1:12240 ] # labels: # namespace: default - job_name: openimserver-openim-rpc-group http_sd_configs: - - url: "http://internal_ip:10002/prometheus_discovery/group" + - url: "http://127.0.0.1:10002/prometheus_discovery/group" # static_configs: -# - targets: [ internal_ip:12260 ] +# - targets: [ 127.0.0.1:12260 ] # labels: # namespace: default. - job_name: openimserver-openim-rpc-msg http_sd_configs: - - url: "http://internal_ip:10002/prometheus_discovery/msg" + - url: "http://127.0.0.1:10002/prometheus_discovery/msg" # static_configs: -# - targets: [ internal_ip:12280 ] +# - targets: [ 127.0.0.1:12280 ] # labels: # namespace: default - job_name: openimserver-openim-rpc-third http_sd_configs: - - url: "http://internal_ip:10002/prometheus_discovery/third" + - url: "http://127.0.0.1:10002/prometheus_discovery/third" # static_configs: -# - targets: [ internal_ip:12300 ] +# - targets: [ 127.0.0.1:12300 ] # labels: # namespace: default - job_name: openimserver-openim-rpc-user http_sd_configs: - - url: "http://internal_ip:10002/prometheus_discovery/user" + - url: "http://127.0.0.1:10002/prometheus_discovery/user" # static_configs: -# - targets: [ internal_ip:12320 ] +# - targets: [ 127.0.0.1:12320 ] # labels: # namespace: default \ No newline at end of file diff --git a/config/share.yml b/config/share.yml index 1726af2dc..0ed59f2cd 100644 --- a/config/share.yml +++ b/config/share.yml @@ -1,14 +1,4 @@ secret: openIM123 -rpcRegisterName: - user: user - friend: friend - msg: msg - push: push - messageGateway: messageGateway - group: group - auth: auth - conversation: conversation - third: third imAdminUserID: [ imAdmin ] diff --git a/docker-compose.yml b/docker-compose.yml index 57e654208..4334790e1 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -151,6 +151,8 @@ services: container_name: prometheus restart: always user: root + profiles: + - m volumes: - ./config/prometheus.yml:/etc/prometheus/prometheus.yml - ./config/instance-down-rules.yml:/etc/prometheus/instance-down-rules.yml @@ -158,37 +160,37 @@ services: command: - '--config.file=/etc/prometheus/prometheus.yml' - '--storage.tsdb.path=/prometheus' - ports: - - "19091:9090" - networks: - - openim + - '--web.listen-address=:${PROMETHEUS_PORT}' + network_mode: host alertmanager: image: ${ALERTMANAGER_IMAGE} container_name: alertmanager restart: always + profiles: + - m volumes: - ./config/alertmanager.yml:/etc/alertmanager/alertmanager.yml - ./config/email.tmpl:/etc/alertmanager/email.tmpl - ports: - - "19093:9093" - networks: - - openim + command: + - '--config.file=/etc/alertmanager/alertmanager.yml' + - '--web.listen-address=:${ALERTMANAGER_PORT}' + network_mode: host grafana: image: ${GRAFANA_IMAGE} container_name: grafana user: root restart: always + profiles: + - m environment: - GF_SECURITY_ALLOW_EMBEDDING=true - GF_SESSION_COOKIE_SAMESITE=none - GF_SESSION_COOKIE_SECURE=true - GF_AUTH_ANONYMOUS_ENABLED=true - GF_AUTH_ANONYMOUS_ORG_ROLE=Admin - ports: - - "13000:3000" + - GF_SERVER_HTTP_PORT=${GRAFANA_PORT} volumes: - ${DATA_DIR:-./}/components/grafana:/var/lib/grafana - networks: - - openim + network_mode: host diff --git a/internal/msggateway/hub_server.go b/internal/msggateway/hub_server.go index 5c9237ed1..f6c12350c 100644 --- a/internal/msggateway/hub_server.go +++ b/internal/msggateway/hub_server.go @@ -47,7 +47,7 @@ func (s *Server) Start(ctx context.Context, index int, conf *Config) error { conf.MsgGateway.RPC.RegisterIP, conf.MsgGateway.RPC.AutoSetPorts, conf.MsgGateway.RPC.Ports, index, conf.Discovery.RpcService.MessageGateway, - &conf.Share, + nil, conf, s.InitServer, ) diff --git a/pkg/common/cmd/auth.go b/pkg/common/cmd/auth.go index 80a675ace..bf51a2cf3 100644 --- a/pkg/common/cmd/auth.go +++ b/pkg/common/cmd/auth.go @@ -56,5 +56,5 @@ func (a *AuthRpcCmd) Exec() error { func (a *AuthRpcCmd) runE() error { return startrpc.Start(a.ctx, &a.authConfig.Discovery, &a.authConfig.RpcConfig.Prometheus, a.authConfig.RpcConfig.RPC.ListenIP, a.authConfig.RpcConfig.RPC.RegisterIP, a.authConfig.RpcConfig.RPC.AutoSetPorts, a.authConfig.RpcConfig.RPC.Ports, - a.Index(), a.authConfig.Discovery.RpcService.Auth, &a.authConfig.Share, a.authConfig, auth.Start) + a.Index(), a.authConfig.Discovery.RpcService.Auth, nil, a.authConfig, auth.Start) } diff --git a/pkg/common/cmd/conversation.go b/pkg/common/cmd/conversation.go index 72fa694ed..4d38f7fd4 100644 --- a/pkg/common/cmd/conversation.go +++ b/pkg/common/cmd/conversation.go @@ -58,5 +58,5 @@ func (a *ConversationRpcCmd) Exec() error { func (a *ConversationRpcCmd) runE() error { return startrpc.Start(a.ctx, &a.conversationConfig.Discovery, &a.conversationConfig.RpcConfig.Prometheus, a.conversationConfig.RpcConfig.RPC.ListenIP, a.conversationConfig.RpcConfig.RPC.RegisterIP, a.conversationConfig.RpcConfig.RPC.AutoSetPorts, a.conversationConfig.RpcConfig.RPC.Ports, - a.Index(), a.conversationConfig.Discovery.RpcService.Conversation, &a.conversationConfig.Share, a.conversationConfig, conversation.Start) + a.Index(), a.conversationConfig.Discovery.RpcService.Conversation, &a.conversationConfig.NotificationConfig, a.conversationConfig, conversation.Start) } diff --git a/pkg/common/cmd/friend.go b/pkg/common/cmd/friend.go index b40449b2b..800116531 100644 --- a/pkg/common/cmd/friend.go +++ b/pkg/common/cmd/friend.go @@ -59,5 +59,5 @@ func (a *FriendRpcCmd) Exec() error { func (a *FriendRpcCmd) runE() error { return startrpc.Start(a.ctx, &a.relationConfig.Discovery, &a.relationConfig.RpcConfig.Prometheus, a.relationConfig.RpcConfig.RPC.ListenIP, a.relationConfig.RpcConfig.RPC.RegisterIP, a.relationConfig.RpcConfig.RPC.AutoSetPorts, a.relationConfig.RpcConfig.RPC.Ports, - a.Index(), a.relationConfig.Discovery.RpcService.Friend, &a.relationConfig.Share, a.relationConfig, relation.Start) + a.Index(), a.relationConfig.Discovery.RpcService.Friend, &a.relationConfig.NotificationConfig, a.relationConfig, relation.Start) } diff --git a/pkg/common/cmd/group.go b/pkg/common/cmd/group.go index faed56233..4f8d17516 100644 --- a/pkg/common/cmd/group.go +++ b/pkg/common/cmd/group.go @@ -60,5 +60,5 @@ func (a *GroupRpcCmd) Exec() error { func (a *GroupRpcCmd) runE() error { return startrpc.Start(a.ctx, &a.groupConfig.Discovery, &a.groupConfig.RpcConfig.Prometheus, a.groupConfig.RpcConfig.RPC.ListenIP, a.groupConfig.RpcConfig.RPC.RegisterIP, a.groupConfig.RpcConfig.RPC.AutoSetPorts, a.groupConfig.RpcConfig.RPC.Ports, - a.Index(), a.groupConfig.Discovery.RpcService.Group, &a.groupConfig.Share, a.groupConfig, group.Start, versionctx.EnableVersionCtx()) + a.Index(), a.groupConfig.Discovery.RpcService.Group, &a.groupConfig.NotificationConfig, a.groupConfig, group.Start, versionctx.EnableVersionCtx()) } diff --git a/pkg/common/cmd/msg.go b/pkg/common/cmd/msg.go index 962d1468a..b6647f9c1 100644 --- a/pkg/common/cmd/msg.go +++ b/pkg/common/cmd/msg.go @@ -60,5 +60,5 @@ func (a *MsgRpcCmd) Exec() error { func (a *MsgRpcCmd) runE() error { return startrpc.Start(a.ctx, &a.msgConfig.Discovery, &a.msgConfig.RpcConfig.Prometheus, a.msgConfig.RpcConfig.RPC.ListenIP, a.msgConfig.RpcConfig.RPC.RegisterIP, a.msgConfig.RpcConfig.RPC.AutoSetPorts, a.msgConfig.RpcConfig.RPC.Ports, - a.Index(), a.msgConfig.Discovery.RpcService.Msg, &a.msgConfig.Share, a.msgConfig, msg.Start) + a.Index(), a.msgConfig.Discovery.RpcService.Msg, &a.msgConfig.NotificationConfig, a.msgConfig, msg.Start) } diff --git a/pkg/common/cmd/push.go b/pkg/common/cmd/push.go index 09883fc34..93b502cd8 100644 --- a/pkg/common/cmd/push.go +++ b/pkg/common/cmd/push.go @@ -60,5 +60,5 @@ func (a *PushRpcCmd) Exec() error { func (a *PushRpcCmd) runE() error { return startrpc.Start(a.ctx, &a.pushConfig.Discovery, &a.pushConfig.RpcConfig.Prometheus, a.pushConfig.RpcConfig.RPC.ListenIP, a.pushConfig.RpcConfig.RPC.RegisterIP, a.pushConfig.RpcConfig.RPC.AutoSetPorts, a.pushConfig.RpcConfig.RPC.Ports, - a.Index(), a.pushConfig.Discovery.RpcService.Push, &a.pushConfig.Share, a.pushConfig, push.Start) + a.Index(), a.pushConfig.Discovery.RpcService.Push, &a.pushConfig.NotificationConfig, a.pushConfig, push.Start) } diff --git a/pkg/common/cmd/third.go b/pkg/common/cmd/third.go index 8e9ffb9bd..e43f61732 100644 --- a/pkg/common/cmd/third.go +++ b/pkg/common/cmd/third.go @@ -59,5 +59,5 @@ func (a *ThirdRpcCmd) Exec() error { func (a *ThirdRpcCmd) runE() error { return startrpc.Start(a.ctx, &a.thirdConfig.Discovery, &a.thirdConfig.RpcConfig.Prometheus, a.thirdConfig.RpcConfig.RPC.ListenIP, a.thirdConfig.RpcConfig.RPC.RegisterIP, a.thirdConfig.RpcConfig.RPC.AutoSetPorts, a.thirdConfig.RpcConfig.RPC.Ports, - a.Index(), a.thirdConfig.Discovery.RpcService.Third, &a.thirdConfig.Share, a.thirdConfig, third.Start) + a.Index(), a.thirdConfig.Discovery.RpcService.Third, &a.thirdConfig.NotificationConfig, a.thirdConfig, third.Start) } diff --git a/pkg/common/cmd/user.go b/pkg/common/cmd/user.go index 40f372770..dc848a775 100644 --- a/pkg/common/cmd/user.go +++ b/pkg/common/cmd/user.go @@ -60,5 +60,5 @@ func (a *UserRpcCmd) Exec() error { func (a *UserRpcCmd) runE() error { return startrpc.Start(a.ctx, &a.userConfig.Discovery, &a.userConfig.RpcConfig.Prometheus, a.userConfig.RpcConfig.RPC.ListenIP, a.userConfig.RpcConfig.RPC.RegisterIP, a.userConfig.RpcConfig.RPC.AutoSetPorts, a.userConfig.RpcConfig.RPC.Ports, - a.Index(), a.userConfig.Discovery.RpcService.User, &a.userConfig.Share, a.userConfig, user.Start) + a.Index(), a.userConfig.Discovery.RpcService.User, &a.userConfig.NotificationConfig, a.userConfig, user.Start) } diff --git a/pkg/common/config/config.go b/pkg/common/config/config.go index 25748a618..da11a20e7 100644 --- a/pkg/common/config/config.go +++ b/pkg/common/config/config.go @@ -610,3 +610,72 @@ func (l *CacheConfig) Success() time.Duration { func (l *CacheConfig) Enable() bool { return l.Topic != "" && l.SlotNum > 0 && l.SlotSize > 0 } + +func InitNotification(notification *Notification) { + notification.GroupCreated.UnreadCount = false + notification.GroupCreated.ReliabilityLevel = 1 + notification.GroupInfoSet.UnreadCount = false + notification.GroupInfoSet.ReliabilityLevel = 1 + notification.JoinGroupApplication.UnreadCount = false + notification.JoinGroupApplication.ReliabilityLevel = 1 + notification.MemberQuit.UnreadCount = false + notification.MemberQuit.ReliabilityLevel = 1 + notification.GroupApplicationAccepted.UnreadCount = false + notification.GroupApplicationAccepted.ReliabilityLevel = 1 + notification.GroupApplicationRejected.UnreadCount = false + notification.GroupApplicationRejected.ReliabilityLevel = 1 + notification.GroupOwnerTransferred.UnreadCount = false + notification.GroupOwnerTransferred.ReliabilityLevel = 1 + notification.MemberKicked.UnreadCount = false + notification.MemberKicked.ReliabilityLevel = 1 + notification.MemberInvited.UnreadCount = false + notification.MemberInvited.ReliabilityLevel = 1 + notification.MemberEnter.UnreadCount = false + notification.MemberEnter.ReliabilityLevel = 1 + notification.GroupDismissed.UnreadCount = false + notification.GroupDismissed.ReliabilityLevel = 1 + notification.GroupMuted.UnreadCount = false + notification.GroupMuted.ReliabilityLevel = 1 + notification.GroupCancelMuted.UnreadCount = false + notification.GroupCancelMuted.ReliabilityLevel = 1 + notification.GroupMemberMuted.UnreadCount = false + notification.GroupMemberMuted.ReliabilityLevel = 1 + notification.GroupMemberCancelMuted.UnreadCount = false + notification.GroupMemberCancelMuted.ReliabilityLevel = 1 + notification.GroupMemberInfoSet.UnreadCount = false + notification.GroupMemberInfoSet.ReliabilityLevel = 1 + notification.GroupMemberSetToAdmin.UnreadCount = false + notification.GroupMemberSetToAdmin.ReliabilityLevel = 1 + notification.GroupMemberSetToOrdinary.UnreadCount = false + notification.GroupMemberSetToOrdinary.ReliabilityLevel = 1 + notification.GroupInfoSetAnnouncement.UnreadCount = false + notification.GroupInfoSetAnnouncement.ReliabilityLevel = 1 + notification.GroupInfoSetName.UnreadCount = false + notification.GroupInfoSetName.ReliabilityLevel = 1 + notification.FriendApplicationAdded.UnreadCount = false + notification.FriendApplicationAdded.ReliabilityLevel = 1 + notification.FriendApplicationApproved.UnreadCount = false + notification.FriendApplicationApproved.ReliabilityLevel = 1 + notification.FriendApplicationRejected.UnreadCount = false + notification.FriendApplicationRejected.ReliabilityLevel = 1 + notification.FriendAdded.UnreadCount = false + notification.FriendAdded.ReliabilityLevel = 1 + notification.FriendDeleted.UnreadCount = false + notification.FriendDeleted.ReliabilityLevel = 1 + notification.FriendRemarkSet.UnreadCount = false + notification.FriendRemarkSet.ReliabilityLevel = 1 + notification.BlackAdded.UnreadCount = false + notification.BlackAdded.ReliabilityLevel = 1 + notification.BlackDeleted.UnreadCount = false + notification.BlackDeleted.ReliabilityLevel = 1 + notification.FriendInfoUpdated.UnreadCount = false + notification.FriendInfoUpdated.ReliabilityLevel = 1 + notification.UserInfoUpdated.UnreadCount = false + notification.UserInfoUpdated.ReliabilityLevel = 1 + notification.UserStatusChanged.UnreadCount = false + notification.UserStatusChanged.ReliabilityLevel = 1 + notification.ConversationChanged.UnreadCount = false + notification.ConversationChanged.ReliabilityLevel = 1 + notification.ConversationSetPrivate.UnreadCount = false + notification.ConversationSetPrivate.ReliabilityLevel = 1 +} diff --git a/pkg/common/config/parse.go b/pkg/common/config/parse.go index 08f82ac7d..ea62c1de0 100644 --- a/pkg/common/config/parse.go +++ b/pkg/common/config/parse.go @@ -61,7 +61,7 @@ func GetProjectRoot() (string, error) { func GetOptionsByNotification(cfg NotificationConfig) msgprocessor.Options { opts := msgprocessor.NewOptions() - if cfg.UnreadCount { + if cfg.IsSendMsg { opts = msgprocessor.WithOptions(opts, msgprocessor.WithUnreadCount(true)) } if cfg.OfflinePush.Enable { diff --git a/pkg/common/startrpc/start.go b/pkg/common/startrpc/start.go index c38ee27a1..089bc3d97 100644 --- a/pkg/common/startrpc/start.go +++ b/pkg/common/startrpc/start.go @@ -48,8 +48,9 @@ import ( // Start rpc server. func Start[T any](ctx context.Context, discovery *conf.Discovery, prometheusConfig *conf.Prometheus, listenIP, - registerIP string, autoSetPorts bool, rpcPorts []int, index int, rpcRegisterName string, share *conf.Share, config T, rpcFn func(ctx context.Context, - config T, client discovery.SvcDiscoveryRegistry, server *grpc.Server) error, options ...grpc.ServerOption) error { + registerIP string, autoSetPorts bool, rpcPorts []int, index int, rpcRegisterName string, notification *conf.Notification, config T, + rpcFn func(ctx context.Context, config T, client discovery.SvcDiscoveryRegistry, server *grpc.Server) error, + options ...grpc.ServerOption) error { var ( rpcTcpAddr string @@ -58,6 +59,10 @@ func Start[T any](ctx context.Context, discovery *conf.Discovery, prometheusConf prometheusPort int ) + if notification != nil { + conf.InitNotification(notification) + } + registerIP, err := network.GetRpcRegisterIP(registerIP) if err != nil { return err From 04f3c99697c2262469a5c393acaae0e05720a23b Mon Sep 17 00:00:00 2001 From: icey-yu <119291641+icey-yu@users.noreply.github.com> Date: Wed, 18 Dec 2024 18:10:25 +0800 Subject: [PATCH 081/199] chore: node exporter (#2979) --- .env | 1 + config/openim-api.yml | 2 +- config/openim-msggateway.yml | 5 ++--- config/openim-msgtransfer.yml | 2 +- config/openim-push.yml | 4 ++-- config/openim-rpc-auth.yml | 4 ++-- config/openim-rpc-conversation.yml | 4 ++-- config/openim-rpc-friend.yml | 4 ++-- config/openim-rpc-group.yml | 4 ++-- config/openim-rpc-msg.yml | 4 ++-- config/openim-rpc-third.yml | 4 ++-- config/openim-rpc-user.yml | 4 ++-- config/prometheus.yml | 2 +- config/share.yml | 1 + docker-compose.yml | 18 ++++++++++++++++++ 15 files changed, 41 insertions(+), 22 deletions(-) diff --git a/.env b/.env index 4a39255b8..a615e5096 100644 --- a/.env +++ b/.env @@ -6,6 +6,7 @@ ETCD_IMAGE=quay.io/coreos/etcd:v3.5.13 PROMETHEUS_IMAGE=prom/prometheus:v2.45.6 ALERTMANAGER_IMAGE=prom/alertmanager:v0.27.0 GRAFANA_IMAGE=grafana/grafana:11.0.1 +NODE_EXPORTER_IMAGE=prom/node-exporter:v1.7.0 OPENIM_WEB_FRONT_IMAGE=openim/openim-web-front:release-v3.8.1 OPENIM_ADMIN_FRONT_IMAGE=openim/openim-admin-front:release-v1.8.3 diff --git a/config/openim-api.yml b/config/openim-api.yml index f063005ec..103c36f95 100644 --- a/config/openim-api.yml +++ b/config/openim-api.yml @@ -14,6 +14,6 @@ prometheus: autoSetPorts: true # Prometheus listening ports, must match the number of api.ports # It will only take effect when autoSetPorts is set to false. - ports: [ 12002 ] + ports: # This address can be accessed via a browser grafanaURL: diff --git a/config/openim-msggateway.yml b/config/openim-msggateway.yml index b7d6d9847..812df90f7 100644 --- a/config/openim-msggateway.yml +++ b/config/openim-msggateway.yml @@ -6,15 +6,14 @@ rpc: autoSetPorts: true # List of ports that the RPC service listens on; configuring multiple ports will launch multiple instances. These must match the number of configured prometheus ports # It will only take effect when autoSetPorts is set to false. - ports: [ 10140, 10141, 10142, 10143, 10144, 10145, 10146, 10147, 10148, 10149, 10150, 10151, 10152, 10153, 10154, 10155 ] + ports: prometheus: # Enable or disable Prometheus monitoring enable: true # List of ports that Prometheus listens on; these must match the number of rpc.ports to ensure correct monitoring setup # It will only take effect when autoSetPorts is set to false. - ports: [ 12140, 12141, 12142, 12143, 12144, 12145, 12146, 12147, 12148, 12149, 12150, 12151, 12152, 12153, 12154, 12155 ] - + ports: # IP address that the RPC/WebSocket service listens on; setting to 0.0.0.0 listens on both internal and external IPs. If left blank, it automatically uses the internal network IP listenIP: 0.0.0.0 diff --git a/config/openim-msgtransfer.yml b/config/openim-msgtransfer.yml index 39b23b222..52d6a805e 100644 --- a/config/openim-msgtransfer.yml +++ b/config/openim-msgtransfer.yml @@ -5,4 +5,4 @@ prometheus: autoSetPorts: true # List of ports that Prometheus listens on; each port corresponds to an instance of monitoring. Ensure these are managed accordingly # It will only take effect when autoSetPorts is set to false. - ports: [ 12020, 12021, 12022, 12023, 12024, 12025, 12026, 12027, 12028, 12029, 12030, 12031, 12032, 12033, 12034, 12035 ] + ports: diff --git a/config/openim-push.yml b/config/openim-push.yml index dcc8f1aec..e98324620 100644 --- a/config/openim-push.yml +++ b/config/openim-push.yml @@ -8,7 +8,7 @@ rpc: autoSetPorts: true # List of ports that the RPC service listens on; configuring multiple ports will launch multiple instances. These must match the number of configured prometheus ports # It will only take effect when autoSetPorts is set to false. - ports: [ 10170, 10171, 10172, 10173, 10174, 10175, 10176, 10177, 10178, 10179, 10180, 10181, 10182, 10183, 10184, 10185 ] + ports: prometheus: @@ -16,7 +16,7 @@ prometheus: enable: true # List of ports that Prometheus listens on; these must match the number of rpc.ports to ensure correct monitoring setup # It will only take effect when autoSetPorts is set to false. - ports: [ 12170, 12171, 12172, 12173, 12174, 12175, 12176, 12177, 12178, 12179, 12180, 12182, 12183, 12184, 12185, 12186 ] + ports: maxConcurrentWorkers: 3 #Use geTui for offline push notifications, or choose fcm or jpns; corresponding configuration settings must be specified. diff --git a/config/openim-rpc-auth.yml b/config/openim-rpc-auth.yml index 5d6d85b2f..c42e556c4 100644 --- a/config/openim-rpc-auth.yml +++ b/config/openim-rpc-auth.yml @@ -8,14 +8,14 @@ rpc: autoSetPorts: true # List of ports that the RPC service listens on; configuring multiple ports will launch multiple instances. These must match the number of configured prometheus ports # It will only take effect when autoSetPorts is set to false. - ports: [ 10200 ] + ports: prometheus: # Enable or disable Prometheus monitoring enable: true # List of ports that Prometheus listens on; these must match the number of rpc.ports to ensure correct monitoring setup # It will only take effect when autoSetPorts is set to false. - ports: [ 12200 ] + ports: tokenPolicy: # Token validity period, in days diff --git a/config/openim-rpc-conversation.yml b/config/openim-rpc-conversation.yml index eaedfe21f..e722ac2b0 100644 --- a/config/openim-rpc-conversation.yml +++ b/config/openim-rpc-conversation.yml @@ -8,11 +8,11 @@ rpc: autoSetPorts: true # List of ports that the RPC service listens on; configuring multiple ports will launch multiple instances. These must match the number of configured prometheus ports # It will only take effect when autoSetPorts is set to false. - ports: [ 10220 ] + ports: prometheus: # Enable or disable Prometheus monitoring enable: true # List of ports that Prometheus listens on; these must match the number of rpc.ports to ensure correct monitoring setup # It will only take effect when autoSetPorts is set to false. - ports: [ 12220 ] + ports: diff --git a/config/openim-rpc-friend.yml b/config/openim-rpc-friend.yml index 920c4860b..e722ac2b0 100644 --- a/config/openim-rpc-friend.yml +++ b/config/openim-rpc-friend.yml @@ -8,11 +8,11 @@ rpc: autoSetPorts: true # List of ports that the RPC service listens on; configuring multiple ports will launch multiple instances. These must match the number of configured prometheus ports # It will only take effect when autoSetPorts is set to false. - ports: [ 10240 ] + ports: prometheus: # Enable or disable Prometheus monitoring enable: true # List of ports that Prometheus listens on; these must match the number of rpc.ports to ensure correct monitoring setup # It will only take effect when autoSetPorts is set to false. - ports: [ 12240 ] + ports: diff --git a/config/openim-rpc-group.yml b/config/openim-rpc-group.yml index c48065cca..252f64c28 100644 --- a/config/openim-rpc-group.yml +++ b/config/openim-rpc-group.yml @@ -8,14 +8,14 @@ rpc: autoSetPorts: true # List of ports that the RPC service listens on; configuring multiple ports will launch multiple instances. These must match the number of configured prometheus ports # It will only take effect when autoSetPorts is set to false. - ports: [ 10260 ] + ports: prometheus: # Enable or disable Prometheus monitoring enable: true # List of ports that Prometheus listens on; these must match the number of rpc.ports to ensure correct monitoring setup # It will only take effect when autoSetPorts is set to false. - ports: [ 12260 ] + ports: enableHistoryForNewMembers: true diff --git a/config/openim-rpc-msg.yml b/config/openim-rpc-msg.yml index d07854622..17fd3b8f4 100644 --- a/config/openim-rpc-msg.yml +++ b/config/openim-rpc-msg.yml @@ -8,14 +8,14 @@ rpc: autoSetPorts: true # List of ports that the RPC service listens on; configuring multiple ports will launch multiple instances. These must match the number of configured prometheus ports # It will only take effect when autoSetPorts is set to false. - ports: [ 10280 ] + ports: prometheus: # Enable or disable Prometheus monitoring enable: true # List of ports that Prometheus listens on; these must match the number of rpc.ports to ensure correct monitoring setup # It will only take effect when autoSetPorts is set to false. - ports: [ 12280 ] + ports: # Does sending messages require friend verification diff --git a/config/openim-rpc-third.yml b/config/openim-rpc-third.yml index 95a50fac4..7169e6c61 100644 --- a/config/openim-rpc-third.yml +++ b/config/openim-rpc-third.yml @@ -8,14 +8,14 @@ rpc: autoSetPorts: true # List of ports that the RPC service listens on; configuring multiple ports will launch multiple instances. These must match the number of configured prometheus ports # It will only take effect when autoSetPorts is set to false. - ports: [ 10300 ] + ports: prometheus: # Enable or disable Prometheus monitoring enable: true # List of ports that Prometheus listens on; these must match the number of rpc.ports to ensure correct monitoring setup # It will only take effect when autoSetPorts is set to false. - ports: [ 12300 ] + ports: object: diff --git a/config/openim-rpc-user.yml b/config/openim-rpc-user.yml index 3a1335895..337cacd35 100644 --- a/config/openim-rpc-user.yml +++ b/config/openim-rpc-user.yml @@ -8,11 +8,11 @@ rpc: autoSetPorts: true # List of ports that the RPC service listens on; configuring multiple ports will launch multiple instances. These must match the number of configured prometheus ports # It will only take effect when autoSetPorts is set to false. - ports: [ 10320 ] + ports: prometheus: # Whether to enable prometheus enable: true # Prometheus listening ports, must be consistent with the number of rpc.ports # It will only take effect when autoSetPorts is set to false. - ports: [ 12320 ] + ports: diff --git a/config/prometheus.yml b/config/prometheus.yml index d176bfcbd..0b13326d1 100644 --- a/config/prometheus.yml +++ b/config/prometheus.yml @@ -25,7 +25,7 @@ scrape_configs: # prometheus fetches application services - job_name: node_exporter static_configs: - - targets: [ 127.0.0.1:20500 ] + - targets: [ 127.0.0.1:19100 ] - job_name: openimserver-openim-api http_sd_configs: diff --git a/config/share.yml b/config/share.yml index 0ed59f2cd..a5fbeac75 100644 --- a/config/share.yml +++ b/config/share.yml @@ -5,4 +5,5 @@ imAdminUserID: [ imAdmin ] # 1: For Android, iOS, Windows, Mac, and web platforms, only one instance can be online at a time multiLogin: policy: 1 + # max num of tokens in one end maxNumOneEnd: 30 \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 4334790e1..51dd4d04f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -194,3 +194,21 @@ services: volumes: - ${DATA_DIR:-./}/components/grafana:/var/lib/grafana network_mode: host + + node-exporter: + image: ${NODE_EXPORTER_IMAGE} + container_name: node-exporter + restart: always + profiles: + - m + volumes: + - /proc:/host/proc:ro + - /sys:/host/sys:ro + - /:/rootfs:ro + command: + - '--path.procfs=/host/proc' + - '--path.sysfs=/host/sys' + - '--path.rootfs=/rootfs' + - '--web.listen-address=:19100' + network_mode: host + From e9895062cb1fd7c3fcf788621591fbb87e991143 Mon Sep 17 00:00:00 2001 From: chao <48119764+withchao@users.noreply.github.com> Date: Fri, 20 Dec 2024 15:45:37 +0800 Subject: [PATCH 082/199] feat: Optimize Scheduled Task (#2985) * pb * fix: Modifying other fields while setting IsPrivateChat does not take effect * fix: quote message error revoke * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks --- go.mod | 4 +- go.sum | 8 +- internal/api/router.go | 1 + internal/rpc/conversation/conversation.go | 50 +++++++ internal/rpc/msg/clear.go | 85 ++++++------ internal/rpc/third/s3.go | 99 +++----------- internal/rpc/third/third.go | 14 +- internal/tools/cron_task.go | 125 +++++++----------- internal/tools/cron_test.go | 63 +++++++++ internal/tools/msg.go | 36 +++++ internal/tools/s3.go | 79 +++++++++++ internal/tools/user_msg.go | 34 +++++ pkg/common/storage/controller/conversation.go | 6 + pkg/common/storage/controller/msg.go | 110 ++++++++------- pkg/common/storage/controller/s3.go | 24 ++-- pkg/common/storage/database/conversation.go | 1 + .../storage/database/mgo/conversation.go | 32 +++++ pkg/common/storage/database/mgo/msg.go | 119 +++++++++-------- pkg/common/storage/database/mgo/msg_test.go | 101 +++++++++----- pkg/common/storage/database/mgo/object.go | 29 ++-- pkg/common/storage/database/msg.go | 6 +- pkg/common/storage/database/object.go | 7 +- 22 files changed, 640 insertions(+), 393 deletions(-) create mode 100644 internal/tools/cron_test.go create mode 100644 internal/tools/msg.go create mode 100644 internal/tools/s3.go create mode 100644 internal/tools/user_msg.go diff --git a/go.mod b/go.mod index 4eaa18ccc..6c1d421c8 100644 --- a/go.mod +++ b/go.mod @@ -14,8 +14,8 @@ require ( github.com/gorilla/websocket v1.5.1 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/mitchellh/mapstructure v1.5.0 - github.com/openimsdk/protocol v0.0.72-alpha.66 - github.com/openimsdk/tools v0.0.50-alpha.57 + github.com/openimsdk/protocol v0.0.72-alpha.67 + github.com/openimsdk/tools v0.0.50-alpha.58 github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 github.com/stretchr/testify v1.9.0 diff --git a/go.sum b/go.sum index 6e283eb66..86ed9ed6e 100644 --- a/go.sum +++ b/go.sum @@ -347,10 +347,10 @@ github.com/onsi/gomega v1.25.0 h1:Vw7br2PCDYijJHSfBOWhov+8cAnUf8MfMaIOV323l6Y= github.com/onsi/gomega v1.25.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdMPGhM= github.com/openimsdk/gomake v0.0.15-alpha.2 h1:5Q8yl8ezy2yx+q8/ucU/t4kJnDfCzNOrkXcDACCqtyM= github.com/openimsdk/gomake v0.0.15-alpha.2/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= -github.com/openimsdk/protocol v0.0.72-alpha.66 h1:5KoDY6M4T+pXg449ScF6hqeQ+WenBwNyUJn/t8W0oBQ= -github.com/openimsdk/protocol v0.0.72-alpha.66/go.mod h1:Iet+piS/jaS+kWWyj6EEr36mk4ISzIRYjoMSVA4dq2M= -github.com/openimsdk/tools v0.0.50-alpha.57 h1:oIKV6vYhqp7TRmZ6Pe+r9RNl1D5s7aB/kE9yQVEWcSY= -github.com/openimsdk/tools v0.0.50-alpha.57/go.mod h1:muCtxguNJv8lFwLei27UASu2Nvg4ERSeN0R4K5tivk0= +github.com/openimsdk/protocol v0.0.72-alpha.67 h1:zlLbVkoT0OYsjO2RCutQuDFllcfNvZfdYchvlR6UIe0= +github.com/openimsdk/protocol v0.0.72-alpha.67/go.mod h1:Iet+piS/jaS+kWWyj6EEr36mk4ISzIRYjoMSVA4dq2M= +github.com/openimsdk/tools v0.0.50-alpha.58 h1:hkFL02Bzzp/l5x+tb7kJ9zes7hilh65EQ4qEIthsQX4= +github.com/openimsdk/tools v0.0.50-alpha.58/go.mod h1:muCtxguNJv8lFwLei27UASu2Nvg4ERSeN0R4K5tivk0= github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= diff --git a/internal/api/router.go b/internal/api/router.go index d6bf4e130..5e8038068 100644 --- a/internal/api/router.go +++ b/internal/api/router.go @@ -164,6 +164,7 @@ func newGinRouter(disCov discovery.SvcDiscoveryRegistry, config *Config) *gin.En authRouterGroup.POST("/get_user_token", a.GetUserToken) authRouterGroup.POST("/parse_token", a.ParseToken) authRouterGroup.POST("/force_logout", a.ForceLogout) + } // Third service thirdGroup := r.Group("/third") diff --git a/internal/rpc/conversation/conversation.go b/internal/rpc/conversation/conversation.go index f534ca6de..696ada152 100644 --- a/internal/rpc/conversation/conversation.go +++ b/internal/rpc/conversation/conversation.go @@ -763,3 +763,53 @@ func (c *conversationServer) GetPinnedConversationIDs(ctx context.Context, req * } return &pbconversation.GetPinnedConversationIDsResp{ConversationIDs: conversationIDs}, nil } + +func (c *conversationServer) ClearUserConversationMsg(ctx context.Context, req *pbconversation.ClearUserConversationMsgReq) (*pbconversation.ClearUserConversationMsgResp, error) { + conversations, err := c.conversationDatabase.FindRandConversation(ctx, req.Timestamp, int(req.Limit)) + if err != nil { + return nil, err + } + latestMsgDestructTime := time.UnixMilli(req.Timestamp) + for i, conversation := range conversations { + if conversation.IsMsgDestruct == false || conversation.MsgDestructTime == 0 { + continue + } + rcpReq := &pbmsg.GetLastMessageSeqByTimeReq{ConversationID: conversation.ConversationID, Time: req.Timestamp - conversation.MsgDestructTime} + resp, err := pbmsg.GetLastMessageSeqByTime.Invoke(ctx, rcpReq) + if err != nil { + return nil, err + } + if resp.Seq <= 0 { + log.ZDebug(ctx, "ClearUserConversationMsg GetLastMessageSeqByTime seq <= 0", "index", i, "conversationID", conversation.ConversationID, "ownerUserID", conversation.OwnerUserID, "msgDestructTime", conversation.MsgDestructTime, "seq", resp.Seq) + if err := c.setConversationMinSeqAndLatestMsgDestructTime(ctx, conversation.ConversationID, conversation.OwnerUserID, -1, latestMsgDestructTime); err != nil { + return nil, err + } + continue + } + resp.Seq++ + if err := c.setConversationMinSeqAndLatestMsgDestructTime(ctx, conversation.ConversationID, conversation.OwnerUserID, resp.Seq, latestMsgDestructTime); err != nil { + return nil, err + } + log.ZDebug(ctx, "ClearUserConversationMsg set min seq", "index", i, "conversationID", conversation.ConversationID, "ownerUserID", conversation.OwnerUserID, "seq", resp.Seq, "msgDestructTime", conversation.MsgDestructTime) + } + return &pbconversation.ClearUserConversationMsgResp{Count: int32(len(conversations))}, nil +} + +func (c *conversationServer) setConversationMinSeqAndLatestMsgDestructTime(ctx context.Context, conversationID string, ownerUserID string, minSeq int64, latestMsgDestructTime time.Time) error { + update := map[string]any{ + "latest_msg_destruct_time": latestMsgDestructTime, + } + if minSeq >= 0 { + req := &pbmsg.SetUserConversationMinSeqReq{ConversationID: conversationID, OwnerUserID: []string{ownerUserID}, MinSeq: minSeq} + if _, err := pbmsg.SetUserConversationMinSeqCaller.Invoke(ctx, req); err != nil { + return err + } + update["min_seq"] = minSeq + } + + if err := c.conversationDatabase.UpdateUsersConversationField(ctx, []string{ownerUserID}, conversationID, update); err != nil { + return err + } + c.conversationNotificationSender.ConversationChangeNotification(ctx, ownerUserID, []string{conversationID}) + return nil +} diff --git a/internal/rpc/msg/clear.go b/internal/rpc/msg/clear.go index 7d62e7c8f..7a2d36300 100644 --- a/internal/rpc/msg/clear.go +++ b/internal/rpc/msg/clear.go @@ -2,6 +2,7 @@ package msg import ( "context" + "strings" "time" "github.com/openimsdk/open-im-server/v3/pkg/authverify" @@ -9,7 +10,6 @@ import ( pbconv "github.com/openimsdk/protocol/conversation" "github.com/openimsdk/protocol/msg" "github.com/openimsdk/protocol/wrapperspb" - "github.com/openimsdk/tools/errs" "github.com/openimsdk/tools/log" "github.com/openimsdk/tools/mcontext" "github.com/openimsdk/tools/utils/datautil" @@ -19,63 +19,50 @@ import ( ) // hard delete in Database. -func (m *msgServer) DestructMsgs(ctx context.Context, req *msg.DestructMsgsReq) (_ *msg.DestructMsgsResp, err error) { +func (m *msgServer) DestructMsgs(ctx context.Context, req *msg.DestructMsgsReq) (*msg.DestructMsgsResp, error) { if err := authverify.CheckAdmin(ctx, m.config.Share.IMAdminUserID); err != nil { return nil, err } - if req.Timestamp > time.Now().UnixMilli() { - return nil, errs.ErrArgs.WrapMsg("request millisecond timestamp error") + docs, err := m.MsgDatabase.GetRandBeforeMsg(ctx, req.Timestamp, int(req.Limit)) + if err != nil { + return nil, err } - var ( - docNum int - msgNum int - start = time.Now() - getLimit = 5000 - ) - - destructMsg := func(ctx context.Context) (bool, error) { - docIDs, err := m.MsgDatabase.GetDocIDs(ctx) - if err != nil { - return false, err - } - - msgs, err := m.MsgDatabase.GetBeforeMsg(ctx, req.Timestamp, docIDs, getLimit) - if err != nil { - return false, err + for i, doc := range docs { + if err := m.MsgDatabase.DeleteDoc(ctx, doc.DocID); err != nil { + return nil, err } - if len(msgs) == 0 { - return false, nil + log.ZDebug(ctx, "DestructMsgs delete doc", "index", i, "docID", doc.DocID) + index := strings.LastIndex(doc.DocID, ":") + if index < 0 { + continue } - - for _, msg := range msgs { - index, err := m.MsgDatabase.DeleteDocMsgBefore(ctx, req.Timestamp, msg) - if err != nil { - return false, err + var minSeq int64 + for _, model := range doc.Msg { + if model.Msg == nil { + continue } - if len(index) == 0 { - return false, errs.ErrInternalServer.WrapMsg("delete doc msg failed") + if model.Msg.Seq > minSeq { + minSeq = model.Msg.Seq } - - docNum++ - msgNum += len(index) } - - return true, nil - } - - _, err = destructMsg(ctx) - if err != nil { - log.ZError(ctx, "clear msg failed", err, "docNum", docNum, "msgNum", msgNum, "cost", time.Since(start)) - return nil, err + if minSeq <= 0 { + continue + } + conversationID := doc.DocID[:index] + if conversationID == "" { + continue + } + minSeq++ + if err := m.MsgDatabase.SetMinSeq(ctx, conversationID, minSeq); err != nil { + return nil, err + } + log.ZDebug(ctx, "DestructMsgs delete doc set min seq", "index", i, "docID", doc.DocID, "conversationID", conversationID, "setMinSeq", minSeq) } - - log.ZDebug(ctx, "clearing message", "docNum", docNum, "msgNum", msgNum, "cost", time.Since(start)) - - return &msg.DestructMsgsResp{}, nil + return &msg.DestructMsgsResp{Count: int32(len(docs))}, nil } // soft delete for user self -func (m *msgServer) ClearMsg(ctx context.Context, req *msg.ClearMsgReq) (_ *msg.ClearMsgResp, err error) { +func (m *msgServer) ClearMsg(ctx context.Context, req *msg.ClearMsgReq) (*msg.ClearMsgResp, error) { temp := convert.ConversationsPb2DB(req.Conversations) batchNum := 100 @@ -137,3 +124,11 @@ func (m *msgServer) ClearMsg(ctx context.Context, req *msg.ClearMsgReq) (_ *msg. return nil, nil } + +func (m *msgServer) GetLastMessageSeqByTime(ctx context.Context, req *msg.GetLastMessageSeqByTimeReq) (*msg.GetLastMessageSeqByTimeResp, error) { + seq, err := m.MsgDatabase.GetLastMessageSeqByTime(ctx, req.ConversationID, req.Time) + if err != nil { + return nil, err + } + return &msg.GetLastMessageSeqByTimeResp{Seq: seq}, nil +} diff --git a/internal/rpc/third/s3.go b/internal/rpc/third/s3.go index b4b064d7f..8796fe824 100644 --- a/internal/rpc/third/s3.go +++ b/internal/rpc/third/s3.go @@ -19,17 +19,14 @@ import ( "encoding/base64" "encoding/hex" "encoding/json" + "github.com/openimsdk/open-im-server/v3/pkg/authverify" "path" "strconv" "time" - "github.com/openimsdk/open-im-server/v3/pkg/common/config" - "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" - "go.mongodb.org/mongo-driver/mongo" - "github.com/google/uuid" "github.com/openimsdk/open-im-server/v3/pkg/common/servererrs" - "github.com/openimsdk/protocol/sdkws" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" "github.com/openimsdk/protocol/third" "github.com/openimsdk/tools/errs" "github.com/openimsdk/tools/log" @@ -288,87 +285,35 @@ func (t *thirdServer) apiAddress(prefix, name string) string { } func (t *thirdServer) DeleteOutdatedData(ctx context.Context, req *third.DeleteOutdatedDataReq) (*third.DeleteOutdatedDataResp, error) { - var conf config.Third - expireTime := time.UnixMilli(req.ExpireTime) - - findPagination := &sdkws.RequestPagination{ - PageNumber: 1, - ShowNumber: 500, + if err := authverify.CheckAdmin(ctx, t.config.Share.IMAdminUserID); err != nil { + return nil, err } - + engine := t.config.RpcConfig.Object.Enable + expireTime := time.UnixMilli(req.ExpireTime) // Find all expired data in S3 database - total, models, err := t.s3dataBase.FindNeedDeleteObjectByDB(ctx, expireTime, req.ObjectGroup, findPagination) - if err != nil && errs.Unwrap(err) != mongo.ErrNoDocuments { - return nil, errs.Wrap(err) - } - - if total == 0 { - log.ZDebug(ctx, "Not have OutdatedData", "delete Total", total) - return &third.DeleteOutdatedDataResp{Count: int32(total)}, nil - } - - needDelObjectKeys := make([]string, len(models)) - for _, model := range models { - needDelObjectKeys = append(needDelObjectKeys, model.Key) + models, err := t.s3dataBase.FindExpirationObject(ctx, engine, expireTime, req.ObjectGroup, int64(req.Limit)) + if err != nil { + return nil, err } - - // Remove duplicate keys, have the same key use in different models - needDelObjectKeys = datautil.Distinct(needDelObjectKeys) - - for _, key := range needDelObjectKeys { - // Find all models by key - keyModels, err := t.s3dataBase.FindModelsByKey(ctx, key) - if err != nil && errs.Unwrap(err) != mongo.ErrNoDocuments { + for i, obj := range models { + if err := t.s3dataBase.DeleteSpecifiedData(ctx, engine, []string{obj.Name}); err != nil { return nil, errs.Wrap(err) } - - // check keyModels, if all keyModels. - needDelKey := true // Default can delete - for _, keymodel := range keyModels { - // If group is empty or CreateTime is after expireTime, can't delete this key - if keymodel.Group == "" || keymodel.CreateTime.After(expireTime) { - needDelKey = false - break - } - } - - // If this object is not referenced by not expire data, delete it - if needDelKey && t.minio != nil { - // If have a thumbnail, delete it - thumbnailKey, _ := t.getMinioImageThumbnailKey(ctx, key) - if thumbnailKey != "" { - err := t.s3dataBase.DeleteObject(ctx, thumbnailKey) - if err != nil { - log.ZWarn(ctx, "Delete thumbnail object is error:", errs.Wrap(err), "thumbnailKey", thumbnailKey) - } - } - - // Delete object - err = t.s3dataBase.DeleteObject(ctx, key) - if err != nil { - log.ZWarn(ctx, "Delete object is error", errs.Wrap(err), "object key", key) - } - - // Delete cache key - err = t.s3dataBase.DelS3Key(ctx, conf.Object.Enable, key) - if err != nil { - log.ZWarn(ctx, "Delete cache key is error:", errs.Wrap(err), "cache S3 key:", key) - } + if err := t.s3dataBase.DelS3Key(ctx, engine, obj.Name); err != nil { + return nil, err } - } - - // handle delete data in S3 database - for _, model := range models { - // Delete all expired data row in S3 database - err := t.s3dataBase.DeleteSpecifiedData(ctx, model.Engine, model.Name) + count, err := t.s3dataBase.GetKeyCount(ctx, engine, obj.Key) if err != nil { - return nil, errs.Wrap(err) + return nil, err + } + log.ZDebug(ctx, "delete s3 object record", "index", i, "s3", obj, "count", count) + if count == 0 { + if err := t.s3.DeleteObject(ctx, obj.Key); err != nil { + return nil, err + } } } - - log.ZDebug(ctx, "DeleteOutdatedData", "delete Total", total) - - return &third.DeleteOutdatedDataResp{Count: int32(total)}, nil + return &third.DeleteOutdatedDataResp{Count: int32(len(models))}, nil } type FormDataMate struct { diff --git a/internal/rpc/third/third.go b/internal/rpc/third/third.go index 77e6d459f..dc964fdb1 100644 --- a/internal/rpc/third/third.go +++ b/internal/rpc/third/third.go @@ -44,7 +44,7 @@ type thirdServer struct { s3dataBase controller.S3Database defaultExpire time.Duration config *Config - minio *minio.Minio + s3 s3.Interface } type Config struct { @@ -79,13 +79,11 @@ func Start(ctx context.Context, config *Config, client discovery.SvcDiscoveryReg // Select the oss method according to the profile policy enable := config.RpcConfig.Object.Enable var ( - o s3.Interface - minioCli *minio.Minio + o s3.Interface ) switch enable { case "minio": - minioCli, err = minio.NewMinio(ctx, redis.NewMinioCache(rdb), *config.MinioConfig.Build()) - o = minioCli + o, err = minio.NewMinio(ctx, redis.NewMinioCache(rdb), *config.MinioConfig.Build()) case "cos": o, err = cos.NewCos(*config.RpcConfig.Object.Cos.Build()) case "oss": @@ -106,15 +104,11 @@ func Start(ctx context.Context, config *Config, client discovery.SvcDiscoveryReg s3dataBase: controller.NewS3Database(rdb, o, s3db), defaultExpire: time.Hour * 24 * 7, config: config, - minio: minioCli, + s3: o, }) return nil } -func (t *thirdServer) getMinioImageThumbnailKey(ctx context.Context, name string) (string, error) { - return t.minio.GetImageThumbnailKey(ctx, name) -} - func (t *thirdServer) FcmUpdateToken(ctx context.Context, req *third.FcmUpdateTokenReq) (resp *third.FcmUpdateTokenResp, err error) { err = t.thirdDatabase.FcmUpdateToken(ctx, req.Account, int(req.PlatformID), req.FcmToken, req.ExpireTime) if err != nil { diff --git a/internal/tools/cron_task.go b/internal/tools/cron_task.go index 2fe7d0e39..049a6199c 100644 --- a/internal/tools/cron_task.go +++ b/internal/tools/cron_task.go @@ -2,10 +2,6 @@ package tools import ( "context" - "fmt" - "os" - "time" - "github.com/openimsdk/open-im-server/v3/pkg/common/config" kdisc "github.com/openimsdk/open-im-server/v3/pkg/common/discoveryregister" pbconversation "github.com/openimsdk/protocol/conversation" @@ -60,87 +56,58 @@ func Start(ctx context.Context, config *CronTaskConfig) error { return err } - msgClient := msg.NewMsgClient(msgConn) - conversationClient := pbconversation.NewConversationClient(conversationConn) - thirdClient := third.NewThirdClient(thirdConn) - - crontab := cron.New() - - // scheduled hard delete outdated Msgs in specific time. - destructMsgsFunc := func() { - now := time.Now() - deltime := now.Add(-time.Hour * 24 * time.Duration(config.CronTask.RetainChatRecords)) - ctx := mcontext.SetOperationID(ctx, fmt.Sprintf("cron_%d_%d", os.Getpid(), deltime.UnixMilli())) - log.ZDebug(ctx, "Destruct chat records", "deltime", deltime, "timestamp", deltime.UnixMilli()) - - if _, err := msgClient.DestructMsgs(ctx, &msg.DestructMsgsReq{Timestamp: deltime.UnixMilli()}); err != nil { - log.ZError(ctx, "cron destruct chat records failed", err, "deltime", deltime, "cont", time.Since(now)) - return - } - log.ZDebug(ctx, "cron destruct chat records success", "deltime", deltime, "cont", time.Since(now)) - } - if _, err := crontab.AddFunc(config.CronTask.CronExecuteTime, destructMsgsFunc); err != nil { - return errs.Wrap(err) + srv := &cronServer{ + ctx: ctx, + config: config, + cron: cron.New(), + msgClient: msg.NewMsgClient(msgConn), + conversationClient: pbconversation.NewConversationClient(conversationConn), + thirdClient: third.NewThirdClient(thirdConn), } - // scheduled soft delete outdated Msgs in specific time when user set `is_msg_destruct` feature. - clearMsgFunc := func() { - now := time.Now() - ctx := mcontext.SetOperationID(ctx, fmt.Sprintf("cron_%d_%d", os.Getpid(), now.UnixMilli())) - log.ZDebug(ctx, "clear msg cron start", "now", now) - - conversations, err := conversationClient.GetConversationsNeedClearMsg(ctx, &pbconversation.GetConversationsNeedClearMsgReq{}) - if err != nil { - log.ZError(ctx, "Get conversation need Destruct msgs failed.", err) - return - } - - _, err = msgClient.ClearMsg(ctx, &msg.ClearMsgReq{Conversations: conversations.Conversations}) - if err != nil { - log.ZError(ctx, "Clear Msg failed.", err) - return - } - - log.ZDebug(ctx, "clear msg cron task completed", "cont", time.Since(now)) - } - if _, err := crontab.AddFunc(config.CronTask.CronExecuteTime, clearMsgFunc); err != nil { - return errs.Wrap(err) + if err := srv.registerClearS3(); err != nil { + return err } - - // scheduled delete outdated file Objects and their datas in specific time. - deleteObjectFunc := func() { - now := time.Now() - executeNum := 5 - // number of pagination. if need modify, need update value in third.DeleteOutdatedData - pageShowNumber := 500 - deleteTime := now.Add(-time.Hour * 24 * time.Duration(config.CronTask.FileExpireTime)) - ctx := mcontext.SetOperationID(ctx, fmt.Sprintf("cron_%d_%d", os.Getpid(), deleteTime.UnixMilli())) - log.ZDebug(ctx, "deleteoutDatedData", "deletetime", deleteTime, "timestamp", deleteTime.UnixMilli()) - - if len(config.CronTask.DeleteObjectType) == 0 { - log.ZDebug(ctx, "cron deleteoutDatedData not type need delete", "deletetime", deleteTime, "DeleteObjectType", config.CronTask.DeleteObjectType, "cont", time.Since(now)) - return - } - - for i := 0; i < executeNum; i++ { - resp, err := thirdClient.DeleteOutdatedData(ctx, &third.DeleteOutdatedDataReq{ExpireTime: deleteTime.UnixMilli(), ObjectGroup: config.CronTask.DeleteObjectType}) - if err != nil { - log.ZError(ctx, "cron deleteoutDatedData failed", err, "deleteTime", deleteTime, "cont", time.Since(now)) - return - } - if resp.Count == 0 || resp.Count < int32(pageShowNumber) { - break - } - } - - log.ZDebug(ctx, "cron deleteoutDatedData success", "deltime", deleteTime, "cont", time.Since(now)) + if err := srv.registerDeleteMsg(); err != nil { + return err } - if _, err := crontab.AddFunc(config.CronTask.CronExecuteTime, deleteObjectFunc); err != nil { - return errs.Wrap(err) + if err := srv.registerClearUserMsg(); err != nil { + return err } - log.ZDebug(ctx, "start cron task", "CronExecuteTime", config.CronTask.CronExecuteTime) - crontab.Start() + srv.cron.Start() <-ctx.Done() return nil } + +type cronServer struct { + ctx context.Context + config *CronTaskConfig + cron *cron.Cron + msgClient msg.MsgClient + conversationClient pbconversation.ConversationClient + thirdClient third.ThirdClient +} + +func (c *cronServer) registerClearS3() error { + if c.config.CronTask.FileExpireTime <= 0 || len(c.config.CronTask.DeleteObjectType) == 0 { + log.ZInfo(c.ctx, "disable scheduled cleanup of s3", "fileExpireTime", c.config.CronTask.FileExpireTime, "deleteObjectType", c.config.CronTask.DeleteObjectType) + return nil + } + _, err := c.cron.AddFunc(c.config.CronTask.CronExecuteTime, c.clearS3) + return errs.WrapMsg(err, "failed to register clear s3 cron task") +} + +func (c *cronServer) registerDeleteMsg() error { + if c.config.CronTask.RetainChatRecords <= 0 { + log.ZInfo(c.ctx, "disable scheduled cleanup of chat records", "retainChatRecords", c.config.CronTask.RetainChatRecords) + return nil + } + _, err := c.cron.AddFunc(c.config.CronTask.CronExecuteTime, c.deleteMsg) + return errs.WrapMsg(err, "failed to register delete msg cron task") +} + +func (c *cronServer) registerClearUserMsg() error { + _, err := c.cron.AddFunc(c.config.CronTask.CronExecuteTime, c.clearUserMsg) + return errs.WrapMsg(err, "failed to register clear user msg cron task") +} diff --git a/internal/tools/cron_test.go b/internal/tools/cron_test.go new file mode 100644 index 000000000..890349069 --- /dev/null +++ b/internal/tools/cron_test.go @@ -0,0 +1,63 @@ +package tools + +import ( + "context" + "github.com/openimsdk/open-im-server/v3/pkg/common/config" + kdisc "github.com/openimsdk/open-im-server/v3/pkg/common/discoveryregister" + pbconversation "github.com/openimsdk/protocol/conversation" + "github.com/openimsdk/protocol/msg" + "github.com/openimsdk/protocol/third" + "github.com/openimsdk/tools/mcontext" + "github.com/openimsdk/tools/mw" + "github.com/robfig/cron/v3" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + "testing" +) + +func TestName(t *testing.T) { + conf := &config.Discovery{ + Enable: config.ETCD, + Etcd: config.Etcd{ + RootDirectory: "openim", + Address: []string{"localhost:12379"}, + }, + } + client, err := kdisc.NewDiscoveryRegister(conf, "source") + if err != nil { + panic(err) + } + client.AddOption(mw.GrpcClient(), grpc.WithTransportCredentials(insecure.NewCredentials())) + ctx := mcontext.SetOpUserID(context.Background(), "imAdmin") + msgConn, err := client.GetConn(ctx, "msg-rpc-service") + if err != nil { + panic(err) + } + thirdConn, err := client.GetConn(ctx, "third-rpc-service") + if err != nil { + panic(err) + } + + conversationConn, err := client.GetConn(ctx, "conversation-rpc-service") + if err != nil { + panic(err) + } + + srv := &cronServer{ + ctx: ctx, + config: &CronTaskConfig{ + CronTask: config.CronTask{ + RetainChatRecords: 1, + FileExpireTime: 1, + DeleteObjectType: []string{"msg-picture", "msg-file", "msg-voice", "msg-video", "msg-video-snapshot", "sdklog", ""}, + }, + }, + cron: cron.New(), + msgClient: msg.NewMsgClient(msgConn), + conversationClient: pbconversation.NewConversationClient(conversationConn), + thirdClient: third.NewThirdClient(thirdConn), + } + srv.deleteMsg() + //srv.clearS3() + //srv.clearUserMsg() +} diff --git a/internal/tools/msg.go b/internal/tools/msg.go new file mode 100644 index 000000000..cc00cc5b8 --- /dev/null +++ b/internal/tools/msg.go @@ -0,0 +1,36 @@ +package tools + +import ( + "fmt" + "github.com/openimsdk/protocol/msg" + "github.com/openimsdk/tools/log" + "github.com/openimsdk/tools/mcontext" + "os" + "time" +) + +func (c *cronServer) deleteMsg() { + now := time.Now() + deltime := now.Add(-time.Hour * 24 * time.Duration(c.config.CronTask.RetainChatRecords)) + operationID := fmt.Sprintf("cron_msg_%d_%d", os.Getpid(), deltime.UnixMilli()) + ctx := mcontext.SetOperationID(c.ctx, operationID) + log.ZDebug(ctx, "Destruct chat records", "deltime", deltime, "timestamp", deltime.UnixMilli()) + const ( + deleteCount = 10000 + deleteLimit = 50 + ) + var count int + for i := 1; i <= deleteCount; i++ { + ctx := mcontext.SetOperationID(c.ctx, fmt.Sprintf("%s_%d", operationID, i)) + resp, err := c.msgClient.DestructMsgs(ctx, &msg.DestructMsgsReq{Timestamp: deltime.UnixMilli(), Limit: deleteLimit}) + if err != nil { + log.ZError(ctx, "cron destruct chat records failed", err) + break + } + count += int(resp.Count) + if resp.Count < deleteLimit { + break + } + } + log.ZDebug(ctx, "cron destruct chat records end", "deltime", deltime, "cont", time.Since(now), "count", count) +} diff --git a/internal/tools/s3.go b/internal/tools/s3.go new file mode 100644 index 000000000..9b6b9c408 --- /dev/null +++ b/internal/tools/s3.go @@ -0,0 +1,79 @@ +package tools + +import ( + "fmt" + "github.com/openimsdk/protocol/third" + "github.com/openimsdk/tools/log" + "github.com/openimsdk/tools/mcontext" + "os" + "time" +) + +func (c *cronServer) clearS3() { + start := time.Now() + deleteTime := start.Add(-time.Hour * 24 * time.Duration(c.config.CronTask.FileExpireTime)) + operationID := fmt.Sprintf("cron_s3_%d_%d", os.Getpid(), deleteTime.UnixMilli()) + ctx := mcontext.SetOperationID(c.ctx, operationID) + log.ZDebug(ctx, "deleteoutDatedData", "deletetime", deleteTime, "timestamp", deleteTime.UnixMilli()) + const ( + deleteCount = 10000 + deleteLimit = 100 + ) + + var count int + for i := 1; i <= deleteCount; i++ { + resp, err := c.thirdClient.DeleteOutdatedData(ctx, &third.DeleteOutdatedDataReq{ExpireTime: deleteTime.UnixMilli(), ObjectGroup: c.config.CronTask.DeleteObjectType, Limit: deleteLimit}) + if err != nil { + log.ZError(ctx, "cron deleteoutDatedData failed", err) + return + } + count += int(resp.Count) + if resp.Count < deleteLimit { + break + } + } + log.ZDebug(ctx, "cron deleteoutDatedData success", "deltime", deleteTime, "cont", time.Since(start), "count", count) +} + +// var req *third.DeleteOutdatedDataReq +// count1, err := ExtractField(ctx, c.thirdClient.DeleteOutdatedData, req, (*third.DeleteOutdatedDataResp).GetCount) +// +// c.thirdClient.DeleteOutdatedData(ctx, &third.DeleteOutdatedDataReq{}) +// msggateway.GetUsersOnlineStatusCaller.Invoke(ctx, &msggateway.GetUsersOnlineStatusReq{}) +// +// var cli ThirdClient +// +// c111, err := cli.DeleteOutdatedData(ctx, 100) +// +// cli.ThirdClient.DeleteOutdatedData(ctx, &third.DeleteOutdatedDataReq{}) +// +// cli.AuthSign(ctx, &third.AuthSignReq{}) +// +// cli.SetAppBadge() +// +//} +// +//func extractField[A, B, C any](ctx context.Context, fn func(ctx context.Context, req *A, opts ...grpc.CallOption) (*B, error), req *A, get func(*B) C) (C, error) { +// resp, err := fn(ctx, req) +// if err != nil { +// var c C +// return c, err +// } +// return get(resp), nil +//} +// +//func ignore(_ any, err error) error { +// return err +//} +// +//type ThirdClient struct { +// third.ThirdClient +//} +// +//func (c *ThirdClient) DeleteOutdatedData(ctx context.Context, expireTime int64) (int32, error) { +// return extractField(ctx, c.ThirdClient.DeleteOutdatedData, &third.DeleteOutdatedDataReq{ExpireTime: expireTime}, (*third.DeleteOutdatedDataResp).GetCount) +//} +// +//func (c *ThirdClient) DeleteOutdatedData1(ctx context.Context, expireTime int64) error { +// return ignore(c.ThirdClient.DeleteOutdatedData(ctx, &third.DeleteOutdatedDataReq{ExpireTime: expireTime})) +//} diff --git a/internal/tools/user_msg.go b/internal/tools/user_msg.go new file mode 100644 index 000000000..a4afa769e --- /dev/null +++ b/internal/tools/user_msg.go @@ -0,0 +1,34 @@ +package tools + +import ( + "fmt" + pbconversation "github.com/openimsdk/protocol/conversation" + "github.com/openimsdk/tools/log" + "github.com/openimsdk/tools/mcontext" + "os" + "time" +) + +func (c *cronServer) clearUserMsg() { + now := time.Now() + operationID := fmt.Sprintf("cron_user_msg_%d_%d", os.Getpid(), now.UnixMilli()) + ctx := mcontext.SetOperationID(c.ctx, operationID) + log.ZDebug(ctx, "clear user msg cron start") + const ( + deleteCount = 10000 + deleteLimit = 100 + ) + var count int + for i := 1; i <= deleteCount; i++ { + resp, err := c.conversationClient.ClearUserConversationMsg(ctx, &pbconversation.ClearUserConversationMsgReq{Timestamp: now.UnixMilli(), Limit: deleteLimit}) + if err != nil { + log.ZError(ctx, "ClearUserConversationMsg failed.", err) + return + } + count += int(resp.Count) + if resp.Count < deleteLimit { + break + } + } + log.ZDebug(ctx, "clear user msg cron task completed", "cont", time.Since(now), "count", count) +} diff --git a/pkg/common/storage/controller/conversation.go b/pkg/common/storage/controller/conversation.go index bf41cce95..d4088e0c0 100644 --- a/pkg/common/storage/controller/conversation.go +++ b/pkg/common/storage/controller/conversation.go @@ -74,6 +74,8 @@ type ConversationDatabase interface { GetNotNotifyConversationIDs(ctx context.Context, userID string) ([]string, error) // GetPinnedConversationIDs gets pinned conversationIDs by userID GetPinnedConversationIDs(ctx context.Context, userID string) ([]string, error) + // FindRandConversation finds random conversations based on the specified timestamp and limit. + FindRandConversation(ctx context.Context, ts int64, limit int) ([]*relationtb.Conversation, error) } func NewConversationDatabase(conversation database.Conversation, cache cache.ConversationCache, tx tx.Tx) ConversationDatabase { @@ -401,3 +403,7 @@ func (c *conversationDatabase) GetPinnedConversationIDs(ctx context.Context, use } return conversationIDs, nil } + +func (c *conversationDatabase) FindRandConversation(ctx context.Context, ts int64, limit int) ([]*relationtb.Conversation, error) { + return c.conversationDB.FindRandConversation(ctx, ts, limit) +} diff --git a/pkg/common/storage/controller/msg.go b/pkg/common/storage/controller/msg.go index 3c3cd9671..c29544c33 100644 --- a/pkg/common/storage/controller/msg.go +++ b/pkg/common/storage/controller/msg.go @@ -18,7 +18,6 @@ import ( "context" "encoding/json" "errors" - "strings" "time" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" @@ -69,6 +68,7 @@ type CommonMsgDatabase interface { GetMaxSeqs(ctx context.Context, conversationIDs []string) (map[string]int64, error) GetMaxSeq(ctx context.Context, conversationID string) (int64, error) SetMinSeqs(ctx context.Context, seqs map[string]int64) error + SetMinSeq(ctx context.Context, conversationID string, seq int64) error SetUserConversationsMinSeqs(ctx context.Context, userID string, seqs map[string]int64) (err error) SetHasReadSeq(ctx context.Context, userID string, conversationID string, hasReadSeq int64) error @@ -95,13 +95,16 @@ type CommonMsgDatabase interface { ConvertMsgsDocLen(ctx context.Context, conversationIDs []string) // get Msg when destruct msg before - GetBeforeMsg(ctx context.Context, ts int64, docIds []string, limit int) ([]*model.MsgDocModel, error) - DeleteDocMsgBefore(ctx context.Context, ts int64, doc *model.MsgDocModel) ([]int, error) + //DeleteDocMsgBefore(ctx context.Context, ts int64, doc *model.MsgDocModel) ([]int, error) - GetDocIDs(ctx context.Context) ([]string, error) + GetRandBeforeMsg(ctx context.Context, ts int64, limit int) ([]*model.MsgDocModel, error) SetUserConversationsMaxSeq(ctx context.Context, conversationID string, userID string, seq int64) error SetUserConversationsMinSeq(ctx context.Context, conversationID string, userID string, seq int64) error + + DeleteDoc(ctx context.Context, docID string) error + + GetLastMessageSeqByTime(ctx context.Context, conversationID string, time int64) (int64, error) } func NewCommonMsgDatabase(msgDocModel database.Msg, msg cache.MsgCache, seqUser cache.SeqUser, seqConversation cache.SeqConversationCache, kafkaConf *config.Kafka) (CommonMsgDatabase, error) { @@ -806,9 +809,10 @@ func (db *commonMsgDatabase) GetMaxSeq(ctx context.Context, conversationID strin return db.seqConversation.GetMaxSeq(ctx, conversationID) } -func (db *commonMsgDatabase) SetMinSeq(ctx context.Context, conversationID string, minSeq int64) error { - return db.seqConversation.SetMinSeq(ctx, conversationID, minSeq) -} +// +//func (db *commonMsgDatabase) SetMinSeq(ctx context.Context, conversationID string, minSeq int64) error { +// return db.seqConversation.SetMinSeq(ctx, conversationID, minSeq) +//} func (db *commonMsgDatabase) SetMinSeqs(ctx context.Context, seqs map[string]int64) error { return db.seqConversation.SetMinSeqs(ctx, seqs) @@ -947,56 +951,40 @@ func (db *commonMsgDatabase) ConvertMsgsDocLen(ctx context.Context, conversation db.msgDocDatabase.ConvertMsgsDocLen(ctx, conversationIDs) } -func (db *commonMsgDatabase) GetBeforeMsg(ctx context.Context, ts int64, docIDs []string, limit int) ([]*model.MsgDocModel, error) { - var msgs []*model.MsgDocModel - for i := 0; i < len(docIDs); i += 1000 { - end := i + 1000 - if end > len(docIDs) { - end = len(docIDs) - } - - res, err := db.msgDocDatabase.GetBeforeMsg(ctx, ts, docIDs[i:end], limit) - if err != nil { - return nil, err - } - msgs = append(msgs, res...) - - if len(msgs) >= limit { - return msgs[:limit], nil - } - } - return msgs, nil +func (db *commonMsgDatabase) GetRandBeforeMsg(ctx context.Context, ts int64, limit int) ([]*model.MsgDocModel, error) { + return db.msgDocDatabase.GetRandBeforeMsg(ctx, ts, limit) } -func (db *commonMsgDatabase) DeleteDocMsgBefore(ctx context.Context, ts int64, doc *model.MsgDocModel) ([]int, error) { - var notNull int - index := make([]int, 0, len(doc.Msg)) - for i, message := range doc.Msg { - if message.Msg != nil { - notNull++ - if message.Msg.SendTime < ts { - index = append(index, i) - } - } - } - if len(index) == 0 { - return index, nil - } - maxSeq := doc.Msg[index[len(index)-1]].Msg.Seq - conversationID := doc.DocID[:strings.LastIndex(doc.DocID, ":")] - if err := db.setMinSeq(ctx, conversationID, maxSeq+1); err != nil { - return index, err - } - if len(index) == notNull { - log.ZDebug(ctx, "Delete db in Doc", "DocID", doc.DocID, "index", index, "maxSeq", maxSeq) - return index, db.msgDocDatabase.DeleteDoc(ctx, doc.DocID) - } else { - log.ZDebug(ctx, "delete db in index", "DocID", doc.DocID, "index", index, "maxSeq", maxSeq) - return index, db.msgDocDatabase.DeleteMsgByIndex(ctx, doc.DocID, index) - } -} - -func (db *commonMsgDatabase) setMinSeq(ctx context.Context, conversationID string, seq int64) error { +// +//func (db *commonMsgDatabase) DeleteDocMsgBefore(ctx context.Context, ts int64, doc *model.MsgDocModel) ([]int, error) { +// var notNull int +// index := make([]int, 0, len(doc.Msg)) +// for i, message := range doc.Msg { +// if message.Msg != nil { +// notNull++ +// if message.Msg.SendTime < ts { +// index = append(index, i) +// } +// } +// } +// if len(index) == 0 { +// return index, nil +// } +// maxSeq := doc.Msg[index[len(index)-1]].Msg.Seq +// conversationID := doc.DocID[:strings.LastIndex(doc.DocID, ":")] +// if err := db.SetMinSeq(ctx, conversationID, maxSeq+1); err != nil { +// return index, err +// } +// if len(index) == notNull { +// log.ZDebug(ctx, "Delete db in Doc", "DocID", doc.DocID, "index", index, "maxSeq", maxSeq) +// return index, db.msgDocDatabase.DeleteDoc(ctx, doc.DocID) +// } else { +// log.ZDebug(ctx, "delete db in index", "DocID", doc.DocID, "index", index, "maxSeq", maxSeq) +// return index, db.msgDocDatabase.DeleteMsgByIndex(ctx, doc.DocID, index) +// } +//} + +func (db *commonMsgDatabase) SetMinSeq(ctx context.Context, conversationID string, seq int64) error { dbSeq, err := db.seqConversation.GetMinSeq(ctx, conversationID) if err != nil { if errors.Is(errs.Unwrap(err), redis.Nil) { @@ -1010,8 +998,8 @@ func (db *commonMsgDatabase) setMinSeq(ctx context.Context, conversationID strin return db.seqConversation.SetMinSeq(ctx, conversationID, seq) } -func (db *commonMsgDatabase) GetDocIDs(ctx context.Context) ([]string, error) { - return db.msgDocDatabase.GetDocIDs(ctx) +func (db *commonMsgDatabase) GetRandDocIDs(ctx context.Context, limit int) ([]string, error) { + return db.msgDocDatabase.GetRandDocIDs(ctx, limit) } func (db *commonMsgDatabase) GetCacheMaxSeqWithTime(ctx context.Context, conversationIDs []string) (map[string]database.SeqTime, error) { @@ -1026,3 +1014,11 @@ func (db *commonMsgDatabase) GetMaxSeqsWithTime(ctx context.Context, conversatio // todo: only the time in the redis cache will be taken, not the message time return db.seqConversation.GetMaxSeqsWithTime(ctx, conversationIDs) } + +func (db *commonMsgDatabase) DeleteDoc(ctx context.Context, docID string) error { + return db.msgDocDatabase.DeleteDoc(ctx, docID) +} + +func (db *commonMsgDatabase) GetLastMessageSeqByTime(ctx context.Context, conversationID string, time int64) (int64, error) { + return db.msgDocDatabase.GetLastMessageSeqByTime(ctx, conversationID, time) +} diff --git a/pkg/common/storage/controller/s3.go b/pkg/common/storage/controller/s3.go index 4e5ad18b6..6693d2dde 100644 --- a/pkg/common/storage/controller/s3.go +++ b/pkg/common/storage/controller/s3.go @@ -24,7 +24,6 @@ import ( "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache" - "github.com/openimsdk/tools/db/pagination" "github.com/openimsdk/tools/s3" "github.com/openimsdk/tools/s3/cont" "github.com/redis/go-redis/v9" @@ -40,11 +39,10 @@ type S3Database interface { SetObject(ctx context.Context, info *model.Object) error StatObject(ctx context.Context, name string) (*s3.ObjectInfo, error) FormData(ctx context.Context, name string, size int64, contentType string, duration time.Duration) (*s3.FormData, error) - FindNeedDeleteObjectByDB(ctx context.Context, duration time.Time, needDelType []string, pagination pagination.Pagination) (total int64, objects []*model.Object, err error) - DeleteObject(ctx context.Context, name string) error - DeleteSpecifiedData(ctx context.Context, engine string, name string) error - FindModelsByKey(ctx context.Context, key string) (objects []*model.Object, err error) + FindExpirationObject(ctx context.Context, engine string, expiration time.Time, needDelType []string, count int64) ([]*model.Object, error) + DeleteSpecifiedData(ctx context.Context, engine string, name []string) error DelS3Key(ctx context.Context, engine string, keys ...string) error + GetKeyCount(ctx context.Context, engine string, key string) (int64, error) } func NewS3Database(rdb redis.UniversalClient, s3 s3.Interface, obj database.ObjectInfo) S3Database { @@ -120,19 +118,17 @@ func (s *s3Database) StatObject(ctx context.Context, name string) (*s3.ObjectInf func (s *s3Database) FormData(ctx context.Context, name string, size int64, contentType string, duration time.Duration) (*s3.FormData, error) { return s.s3.FormData(ctx, name, size, contentType, duration) } -func (s *s3Database) FindNeedDeleteObjectByDB(ctx context.Context, duration time.Time, needDelType []string, pagination pagination.Pagination) (total int64, objects []*model.Object, err error) { - return s.db.FindNeedDeleteObjectByDB(ctx, duration, needDelType, pagination) -} -func (s *s3Database) DeleteObject(ctx context.Context, name string) error { - return s.s3.DeleteObject(ctx, name) +func (s *s3Database) FindExpirationObject(ctx context.Context, engine string, expiration time.Time, needDelType []string, count int64) ([]*model.Object, error) { + return s.db.FindExpirationObject(ctx, engine, expiration, needDelType, count) } -func (s *s3Database) DeleteSpecifiedData(ctx context.Context, engine string, name string) error { - return s.db.Delete(ctx, engine, name) + +func (s *s3Database) GetKeyCount(ctx context.Context, engine string, key string) (int64, error) { + return s.db.GetKeyCount(ctx, engine, key) } -func (s *s3Database) FindModelsByKey(ctx context.Context, key string) (objects []*model.Object, err error) { - return s.db.FindModelsByKey(ctx, key) +func (s *s3Database) DeleteSpecifiedData(ctx context.Context, engine string, name []string) error { + return s.db.Delete(ctx, engine, name) } func (s *s3Database) DelS3Key(ctx context.Context, engine string, keys ...string) error { diff --git a/pkg/common/storage/database/conversation.go b/pkg/common/storage/database/conversation.go index 30ca01ee7..1fb53cfed 100644 --- a/pkg/common/storage/database/conversation.go +++ b/pkg/common/storage/database/conversation.go @@ -42,4 +42,5 @@ type Conversation interface { GetConversationIDsNeedDestruct(ctx context.Context) ([]*model.Conversation, error) GetConversationNotReceiveMessageUserIDs(ctx context.Context, conversationID string) ([]string, error) FindConversationUserVersion(ctx context.Context, userID string, version uint, limit int) (*model.VersionLog, error) + FindRandConversation(ctx context.Context, ts int64, limit int) ([]*model.Conversation, error) } diff --git a/pkg/common/storage/database/mgo/conversation.go b/pkg/common/storage/database/mgo/conversation.go index 10e223c89..851ec99c4 100644 --- a/pkg/common/storage/database/mgo/conversation.go +++ b/pkg/common/storage/database/mgo/conversation.go @@ -228,3 +228,35 @@ func (c *ConversationMgo) GetConversationNotReceiveMessageUserIDs(ctx context.Co func (c *ConversationMgo) FindConversationUserVersion(ctx context.Context, userID string, version uint, limit int) (*model.VersionLog, error) { return c.version.FindChangeLog(ctx, userID, version, limit) } + +func (c *ConversationMgo) FindRandConversation(ctx context.Context, ts int64, limit int) ([]*model.Conversation, error) { + pipeline := []bson.M{ + { + "$match": bson.M{ + "is_msg_destruct": true, + "msg_destruct_time": bson.M{"$ne": 0}, + }, + }, + { + "$addFields": bson.M{ + "next_msg_destruct_timestamp": bson.M{ + "$add": []any{ + bson.M{ + "$toLong": "$latest_msg_destruct_time", + }, "$msg_destruct_time"}, + }, + }, + }, + { + "$match": bson.M{ + "next_msg_destruct_timestamp": bson.M{"$lt": ts}, + }, + }, + { + "$sample": bson.M{ + "size": limit, + }, + }, + } + return mongoutil.Aggregate[*model.Conversation](ctx, c.coll, pipeline) +} diff --git a/pkg/common/storage/database/mgo/msg.go b/pkg/common/storage/database/mgo/msg.go index fc1fe47ea..f37176695 100644 --- a/pkg/common/storage/database/mgo/msg.go +++ b/pkg/common/storage/database/mgo/msg.go @@ -1227,8 +1227,7 @@ func (m *MsgMgo) ConvertMsgsDocLen(ctx context.Context, conversationIDs []string } } -func (m *MsgMgo) GetDocIDs(ctx context.Context) ([]string, error) { - limit := 5000 +func (m *MsgMgo) GetRandDocIDs(ctx context.Context, limit int) ([]string, error) { var skip int var docIDs []string var offset int @@ -1267,15 +1266,18 @@ func (m *MsgMgo) GetDocIDs(ctx context.Context) ([]string, error) { return docIDs, errs.Wrap(err) } -func (m *MsgMgo) GetBeforeMsg(ctx context.Context, ts int64, docIDs []string, limit int) ([]*model.MsgDocModel, error) { +func (m *MsgMgo) GetRandBeforeMsg(ctx context.Context, ts int64, limit int) ([]*model.MsgDocModel, error) { return mongoutil.Aggregate[*model.MsgDocModel](ctx, m.coll, []bson.M{ { "$match": bson.M{ - "doc_id": bson.M{ - "$in": docIDs, - }, - "msgs.msg.send_time": bson.M{ - "$lt": ts, + "msgs": bson.M{ + "$not": bson.M{ + "$elemMatch": bson.M{ + "msg.send_time": bson.M{ + "$gt": ts, + }, + }, + }, }, }, }, @@ -1288,7 +1290,9 @@ func (m *MsgMgo) GetBeforeMsg(ctx context.Context, ts int64, docIDs []string, li }, }, { - "$limit": limit, + "$sample": bson.M{ + "size": limit, + }, }, }) } @@ -1305,53 +1309,58 @@ func (m *MsgMgo) DeleteMsgByIndex(ctx context.Context, docID string, index []int return mongoutil.UpdateOne(ctx, m.coll, bson.M{"doc_id": docID}, bson.M{"$set": set}, true) } -//func (m *MsgMgo) ClearMsg(ctx context.Context, t time.Time) (int64, error) { -// ts := t.UnixMilli() -// var count int64 -// for { -// msgs, err := m.GetBeforeMsg(ctx, ts, 100) -// if err != nil { -// return count, err -// } -// if len(msgs) == 0 { -// return count, nil -// } -// for _, msg := range msgs { -// num, err := m.deleteOneMsg(ctx, ts, msg) -// count += num -// if err != nil { -// return count, err -// } -// } -// } -//} - func (m *MsgMgo) DeleteDoc(ctx context.Context, docID string) error { return mongoutil.DeleteOne(ctx, m.coll, bson.M{"doc_id": docID}) } -//func (m *MsgMgo) DeleteDocMsg(ctx context.Context, ts int64, doc *relation.MsgDocModel) (int64, error) { -// var notNull int -// index := make([]int, 0, len(doc.Msg)) -// for i, message := range doc.Msg { -// if message.Msg != nil { -// notNull++ -// if message.Msg.SendTime < ts { -// index = append(index, i) -// } -// } -// } -// if len(index) == 0 { -// return 0, errs.New("no msg to delete").WrapMsg("deleteOneMsg", "docID", doc.DocID) -// } -// if len(index) == notNull { -// if err := m.DeleteDoc(ctx, doc.DocID); err != nil { -// return 0, err -// } -// } else { -// if err := m.setNullMsg(ctx, doc.DocID, index); err != nil { -// return 0, err -// } -// } -// return int64(len(index)), nil -//} +func (m *MsgMgo) GetLastMessageSeqByTime(ctx context.Context, conversationID string, time int64) (int64, error) { + pipeline := []bson.M{ + { + "$match": bson.M{ + "doc_id": bson.M{ + "$regex": fmt.Sprintf("^%s", conversationID), + }, + }, + }, + { + "$match": bson.M{ + "msgs.msg.send_time": bson.M{ + "$lte": time, + }, + }, + }, + { + "$sort": bson.M{ + "_id": -1, + }, + }, + { + "$limit": 1, + }, + { + "$project": bson.M{ + "_id": 0, + "doc_id": 1, + "msgs.msg.send_time": 1, + "msgs.msg.seq": 1, + }, + }, + } + res, err := mongoutil.Aggregate[*model.MsgDocModel](ctx, m.coll, pipeline) + if err != nil { + return 0, err + } + if len(res) == 0 { + return 0, nil + } + var seq int64 + for _, v := range res[0].Msg { + if v.Msg == nil { + continue + } + if v.Msg.SendTime <= time { + seq = v.Msg.Seq + } + } + return seq, nil +} diff --git a/pkg/common/storage/database/mgo/msg_test.go b/pkg/common/storage/database/mgo/msg_test.go index 5aed4dc51..992090552 100644 --- a/pkg/common/storage/database/mgo/msg_test.go +++ b/pkg/common/storage/database/mgo/msg_test.go @@ -3,12 +3,11 @@ package mgo import ( "context" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" - "github.com/openimsdk/protocol/msg" - "github.com/openimsdk/protocol/sdkws" "github.com/openimsdk/tools/db/mongoutil" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/mongo" "go.mongodb.org/mongo-driver/mongo/options" + "math" "math/rand" "strconv" "testing" @@ -16,35 +15,45 @@ import ( ) func TestName1(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*300) - defer cancel() - cli := Result(mongo.Connect(ctx, options.Client().ApplyURI("mongodb://openIM:openIM123@172.16.8.48:37017/openim_v3?maxPoolSize=100").SetConnectTimeout(5*time.Second))) - - v := &MsgMgo{ - coll: cli.Database("openim_v3").Collection("msg3"), - } - - req := &msg.SearchMessageReq{ - //RecvID: "3187706596", - //SendID: "7009965934", - ContentType: 101, - //SendTime: "2024-05-06", - //SessionType: 3, - Pagination: &sdkws.RequestPagination{ - PageNumber: 1, - ShowNumber: 10, - }, - } - total, res, err := v.SearchMessage(ctx, req) - if err != nil { - panic(err) - } - - for i, re := range res { - t.Logf("%d => %d | %+v", i+1, re.Msg.Seq, re.Msg.Content) - } - - t.Log(total) + //ctx, cancel := context.WithTimeout(context.Background(), time.Second*300) + //defer cancel() + //cli := Result(mongo.Connect(ctx, options.Client().ApplyURI("mongodb://openIM:openIM123@172.16.8.66:37017/openim_v3?maxPoolSize=100").SetConnectTimeout(5*time.Second))) + // + //v := &MsgMgo{ + // coll: cli.Database("openim_v3").Collection("msg3"), + //} + // + //req := &msg.SearchMessageReq{ + // //RecvID: "3187706596", + // //SendID: "7009965934", + // ContentType: 101, + // //SendTime: "2024-05-06", + // //SessionType: 3, + // Pagination: &sdkws.RequestPagination{ + // PageNumber: 1, + // ShowNumber: 10, + // }, + //} + //total, res, err := v.SearchMessage(ctx, req) + //if err != nil { + // panic(err) + //} + // + //for i, re := range res { + // t.Logf("%d => %d | %+v", i+1, re.Msg.Seq, re.Msg.Content) + //} + // + //t.Log(total) + // + //msg, err := NewMsgMongo(cli.Database("openim_v3")) + //if err != nil { + // panic(err) + //} + //res, err := msg.GetBeforeMsg(ctx, time.Now().UnixMilli(), []string{"1:0"}, 1000) + //if err != nil { + // panic(err) + //} + //t.Log(len(res)) } func TestName10(t *testing.T) { @@ -73,3 +82,33 @@ func TestName10(t *testing.T) { } } + +func TestName3(t *testing.T) { + t.Log(uint64(math.MaxUint64)) + t.Log(int64(math.MaxInt64)) + + t.Log(int64(math.MinInt64)) +} + +func TestName4(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*300) + defer cancel() + cli := Result(mongo.Connect(ctx, options.Client().ApplyURI("mongodb://openIM:openIM123@172.16.8.66:37017/openim_v3?maxPoolSize=100").SetConnectTimeout(5*time.Second))) + + msg, err := NewMsgMongo(cli.Database("openim_v3")) + if err != nil { + panic(err) + } + ts := time.Now().Add(-time.Hour * 24 * 5).UnixMilli() + t.Log(ts) + res, err := msg.GetLastMessageSeqByTime(ctx, "sg_1523453548", ts) + if err != nil { + panic(err) + } + t.Log(res) +} + +func TestName5(t *testing.T) { + var v time.Time + t.Log(v.UnixMilli()) +} diff --git a/pkg/common/storage/database/mgo/object.go b/pkg/common/storage/database/mgo/object.go index 5bc329d33..624eb84a0 100644 --- a/pkg/common/storage/database/mgo/object.go +++ b/pkg/common/storage/database/mgo/object.go @@ -22,7 +22,6 @@ import ( "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" "github.com/openimsdk/tools/db/mongoutil" - "github.com/openimsdk/tools/db/pagination" "github.com/openimsdk/tools/errs" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/mongo" @@ -91,21 +90,25 @@ func (o *S3Mongo) Take(ctx context.Context, engine string, name string) (*model. return mongoutil.FindOne[*model.Object](ctx, o.coll, bson.M{"name": name, "engine": engine}) } -func (o *S3Mongo) Delete(ctx context.Context, engine string, name string) error { - return mongoutil.DeleteOne(ctx, o.coll, bson.M{"name": name, "engine": engine}) +func (o *S3Mongo) Delete(ctx context.Context, engine string, name []string) error { + if len(name) == 0 { + return nil + } + return mongoutil.DeleteOne(ctx, o.coll, bson.M{"engine": engine, "name": bson.M{"$in": name}}) } -// Find Expires object -func (o *S3Mongo) FindNeedDeleteObjectByDB(ctx context.Context, duration time.Time, needDelType []string, pagination pagination.Pagination) (total int64, objects []*model.Object, err error) { - return mongoutil.FindPage[*model.Object](ctx, o.coll, bson.M{ - "create_time": bson.M{"$lt": duration}, +func (o *S3Mongo) FindExpirationObject(ctx context.Context, engine string, expiration time.Time, needDelType []string, count int64) ([]*model.Object, error) { + opt := options.Find() + if count > 0 { + opt.SetLimit(count) + } + return mongoutil.Find[*model.Object](ctx, o.coll, bson.M{ + "engine": engine, + "create_time": bson.M{"$lt": expiration}, "group": bson.M{"$in": needDelType}, - }, pagination) + }, opt) } -// Find object by key -func (o *S3Mongo) FindModelsByKey(ctx context.Context, key string) (objects []*model.Object, err error) { - return mongoutil.Find[*model.Object](ctx, o.coll, bson.M{ - "key": key, - }) +func (o *S3Mongo) GetKeyCount(ctx context.Context, engine string, key string) (int64, error) { + return mongoutil.Count(ctx, o.coll, bson.M{"engine": engine, "key": key}) } diff --git a/pkg/common/storage/database/msg.go b/pkg/common/storage/database/msg.go index 23a99f5b9..abb2a44c2 100644 --- a/pkg/common/storage/database/msg.go +++ b/pkg/common/storage/database/msg.go @@ -45,7 +45,9 @@ type Msg interface { DeleteDoc(ctx context.Context, docID string) error DeleteMsgByIndex(ctx context.Context, docID string, index []int) error - GetBeforeMsg(ctx context.Context, ts int64, docIDs []string, limit int) ([]*model.MsgDocModel, error) + GetRandBeforeMsg(ctx context.Context, ts int64, limit int) ([]*model.MsgDocModel, error) - GetDocIDs(ctx context.Context) ([]string, error) + GetRandDocIDs(ctx context.Context, limit int) ([]string, error) + + GetLastMessageSeqByTime(ctx context.Context, conversationID string, time int64) (int64, error) } diff --git a/pkg/common/storage/database/object.go b/pkg/common/storage/database/object.go index c741e39a6..5541a159b 100644 --- a/pkg/common/storage/database/object.go +++ b/pkg/common/storage/database/object.go @@ -19,13 +19,12 @@ import ( "time" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" - "github.com/openimsdk/tools/db/pagination" ) type ObjectInfo interface { SetObject(ctx context.Context, obj *model.Object) error Take(ctx context.Context, engine string, name string) (*model.Object, error) - Delete(ctx context.Context, engine string, name string) error - FindNeedDeleteObjectByDB(ctx context.Context, duration time.Time, needDelType []string, pagination pagination.Pagination) (total int64, objects []*model.Object, err error) - FindModelsByKey(ctx context.Context, key string) (objects []*model.Object, err error) + Delete(ctx context.Context, engine string, name []string) error + FindExpirationObject(ctx context.Context, engine string, expiration time.Time, needDelType []string, count int64) ([]*model.Object, error) + GetKeyCount(ctx context.Context, engine string, key string) (int64, error) } From e3f609b3ae5194f8b1b3d309be5ec3b13c7664f7 Mon Sep 17 00:00:00 2001 From: chao <48119764+withchao@users.noreply.github.com> Date: Tue, 24 Dec 2024 10:51:38 +0800 Subject: [PATCH 083/199] feat: Optimizing RPC call (#2993) * pb * fix: Modifying other fields while setting IsPrivateChat does not take effect * fix: quote message error revoke * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * rpc client * rpc client * rpc client * rpc client * rpc client * rpc client * rpc client * rpc client --- go.mod | 6 +- go.sum | 8 +- internal/api/auth.go | 16 +- internal/api/conversation.go | 30 ++-- internal/api/friend.go | 50 +++--- internal/api/group.go | 80 +++++----- internal/api/init.go | 9 +- internal/api/jssdk/jssdk.go | 64 ++++---- internal/api/msg.go | 74 +++++---- internal/api/router.go | 117 +++++++++----- internal/api/statistics.go | 31 ---- internal/api/third.go | 36 ++--- internal/api/user.go | 57 ++++--- internal/msggateway/hub_server.go | 11 +- internal/msggateway/init.go | 5 +- internal/msggateway/message_handler.go | 59 +++---- internal/msggateway/online.go | 2 +- internal/msggateway/ws_server.go | 38 ++++- internal/msgtransfer/init.go | 7 +- .../msgtransfer/online_history_msg_handler.go | 42 ++--- internal/push/push.go | 7 +- internal/push/push_handler.go | 58 ++++--- internal/rpc/auth/auth.go | 19 ++- internal/rpc/conversation/conversation.go | 79 +++++----- internal/rpc/conversation/notification.go | 10 +- internal/rpc/group/group.go | 114 ++++++-------- internal/rpc/group/notification.go | 144 ++++++++---------- internal/rpc/msg/clear.go | 78 +--------- internal/rpc/msg/delete.go | 26 +--- internal/rpc/msg/notification.go | 2 +- internal/rpc/msg/send.go | 21 +-- internal/rpc/msg/server.go | 91 ++++++----- internal/rpc/msg/stream_msg.go | 7 +- internal/rpc/relation/black.go | 24 ++- internal/rpc/relation/friend.go | 48 +++--- internal/rpc/relation/notification.go | 21 +-- internal/rpc/third/log.go | 6 +- internal/rpc/third/third.go | 7 + internal/rpc/user/notification.go | 16 +- internal/rpc/user/user.go | 27 +++- pkg/common/convert/group.go | 13 +- pkg/common/startrpc/start.go | 5 - .../common_user}/common.go | 2 +- .../grouphash/grouphash.go | 0 pkg/{rpcclient => notification}/msg.go | 8 +- pkg/rpccache/conversation.go | 38 +++-- pkg/rpccache/friend.go | 12 +- pkg/rpccache/group.go | 24 +-- pkg/rpccache/online.go | 35 ++--- pkg/rpccache/subscriber.go | 1 - pkg/rpccache/user.go | 12 +- pkg/rpcclient/doc.go | 15 -- pkg/rpcclient/grouphash/doc.go | 15 -- pkg/rpcclient/init.go | 60 -------- pkg/rpcclient/notification/doc.go | 15 -- pkg/rpcclient/user.go | 109 ------------- pkg/rpcli/auth.go | 30 ++++ pkg/rpcli/conversation.go | 94 ++++++++++++ pkg/rpcli/group.go | 48 ++++++ pkg/rpcli/msg.go | 90 +++++++++++ pkg/rpcli/msggateway.go | 14 ++ pkg/rpcli/push.go | 14 ++ pkg/rpcli/relation.go | 23 +++ pkg/rpcli/rtc.go | 14 ++ pkg/rpcli/third.go | 14 ++ pkg/rpcli/tool.go | 32 ++++ pkg/rpcli/user.go | 95 ++++++++++++ 67 files changed, 1287 insertions(+), 1092 deletions(-) delete mode 100644 internal/api/statistics.go rename pkg/{rpcclient/notification => notification/common_user}/common.go (97%) rename pkg/{rpcclient => notification}/grouphash/grouphash.go (100%) rename pkg/{rpcclient => notification}/msg.go (97%) delete mode 100644 pkg/rpcclient/doc.go delete mode 100644 pkg/rpcclient/grouphash/doc.go delete mode 100644 pkg/rpcclient/init.go delete mode 100644 pkg/rpcclient/notification/doc.go delete mode 100644 pkg/rpcclient/user.go create mode 100644 pkg/rpcli/auth.go create mode 100644 pkg/rpcli/conversation.go create mode 100644 pkg/rpcli/group.go create mode 100644 pkg/rpcli/msg.go create mode 100644 pkg/rpcli/msggateway.go create mode 100644 pkg/rpcli/push.go create mode 100644 pkg/rpcli/relation.go create mode 100644 pkg/rpcli/rtc.go create mode 100644 pkg/rpcli/third.go create mode 100644 pkg/rpcli/tool.go create mode 100644 pkg/rpcli/user.go diff --git a/go.mod b/go.mod index 6c1d421c8..26f167a19 100644 --- a/go.mod +++ b/go.mod @@ -2,8 +2,6 @@ module github.com/openimsdk/open-im-server/v3 go 1.22.7 -toolchain go1.23.2 - require ( firebase.google.com/go/v4 v4.14.1 github.com/dtm-labs/rockscache v0.1.1 @@ -14,8 +12,8 @@ require ( github.com/gorilla/websocket v1.5.1 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/mitchellh/mapstructure v1.5.0 - github.com/openimsdk/protocol v0.0.72-alpha.67 - github.com/openimsdk/tools v0.0.50-alpha.58 + github.com/openimsdk/protocol v0.0.72-alpha.68 + github.com/openimsdk/tools v0.0.50-alpha.60 github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 github.com/stretchr/testify v1.9.0 diff --git a/go.sum b/go.sum index 86ed9ed6e..c0385481c 100644 --- a/go.sum +++ b/go.sum @@ -347,10 +347,10 @@ github.com/onsi/gomega v1.25.0 h1:Vw7br2PCDYijJHSfBOWhov+8cAnUf8MfMaIOV323l6Y= github.com/onsi/gomega v1.25.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdMPGhM= github.com/openimsdk/gomake v0.0.15-alpha.2 h1:5Q8yl8ezy2yx+q8/ucU/t4kJnDfCzNOrkXcDACCqtyM= github.com/openimsdk/gomake v0.0.15-alpha.2/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= -github.com/openimsdk/protocol v0.0.72-alpha.67 h1:zlLbVkoT0OYsjO2RCutQuDFllcfNvZfdYchvlR6UIe0= -github.com/openimsdk/protocol v0.0.72-alpha.67/go.mod h1:Iet+piS/jaS+kWWyj6EEr36mk4ISzIRYjoMSVA4dq2M= -github.com/openimsdk/tools v0.0.50-alpha.58 h1:hkFL02Bzzp/l5x+tb7kJ9zes7hilh65EQ4qEIthsQX4= -github.com/openimsdk/tools v0.0.50-alpha.58/go.mod h1:muCtxguNJv8lFwLei27UASu2Nvg4ERSeN0R4K5tivk0= +github.com/openimsdk/protocol v0.0.72-alpha.68 h1:Ekn6S9Ftt12Xs/p9kJ39RDr2gSwIczz+MmSHQE4lAek= +github.com/openimsdk/protocol v0.0.72-alpha.68/go.mod h1:Iet+piS/jaS+kWWyj6EEr36mk4ISzIRYjoMSVA4dq2M= +github.com/openimsdk/tools v0.0.50-alpha.60 h1:dYqYpSdSN5o6CxlEjua2USfwfUiG0tUWFBpqghTjbWE= +github.com/openimsdk/tools v0.0.50-alpha.60/go.mod h1:muCtxguNJv8lFwLei27UASu2Nvg4ERSeN0R4K5tivk0= github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= diff --git a/internal/api/auth.go b/internal/api/auth.go index c7e7ff704..92d911b71 100644 --- a/internal/api/auth.go +++ b/internal/api/auth.go @@ -20,24 +20,26 @@ import ( "github.com/openimsdk/tools/a2r" ) -type AuthApi struct{} +type AuthApi struct { + Client auth.AuthClient +} -func NewAuthApi() AuthApi { - return AuthApi{} +func NewAuthApi(client auth.AuthClient) AuthApi { + return AuthApi{client} } func (o *AuthApi) GetAdminToken(c *gin.Context) { - a2r.CallV2(c, auth.GetAdminTokenCaller.Invoke) + a2r.Call(c, auth.AuthClient.GetAdminToken, o.Client) } func (o *AuthApi) GetUserToken(c *gin.Context) { - a2r.CallV2(c, auth.GetUserTokenCaller.Invoke) + a2r.Call(c, auth.AuthClient.GetUserToken, o.Client) } func (o *AuthApi) ParseToken(c *gin.Context) { - a2r.CallV2(c, auth.ParseTokenCaller.Invoke) + a2r.Call(c, auth.AuthClient.ParseToken, o.Client) } func (o *AuthApi) ForceLogout(c *gin.Context) { - a2r.CallV2(c, auth.ForceLogoutCaller.Invoke) + a2r.Call(c, auth.AuthClient.ForceLogout, o.Client) } diff --git a/internal/api/conversation.go b/internal/api/conversation.go index 4004944e5..f7dbc133c 100644 --- a/internal/api/conversation.go +++ b/internal/api/conversation.go @@ -20,52 +20,54 @@ import ( "github.com/openimsdk/tools/a2r" ) -type ConversationApi struct{} +type ConversationApi struct { + Client conversation.ConversationClient +} -func NewConversationApi() ConversationApi { - return ConversationApi{} +func NewConversationApi(client conversation.ConversationClient) ConversationApi { + return ConversationApi{client} } func (o *ConversationApi) GetAllConversations(c *gin.Context) { - a2r.CallV2(c, conversation.GetAllConversationsCaller.Invoke) + a2r.Call(c, conversation.ConversationClient.GetAllConversations, o.Client) } func (o *ConversationApi) GetSortedConversationList(c *gin.Context) { - a2r.CallV2(c, conversation.GetSortedConversationListCaller.Invoke) + a2r.Call(c, conversation.ConversationClient.GetSortedConversationList, o.Client) } func (o *ConversationApi) GetConversation(c *gin.Context) { - a2r.CallV2(c, conversation.GetConversationCaller.Invoke) + a2r.Call(c, conversation.ConversationClient.GetConversation, o.Client) } func (o *ConversationApi) GetConversations(c *gin.Context) { - a2r.CallV2(c, conversation.GetConversationsCaller.Invoke) + a2r.Call(c, conversation.ConversationClient.GetConversations, o.Client) } func (o *ConversationApi) SetConversations(c *gin.Context) { - a2r.CallV2(c, conversation.SetConversationsCaller.Invoke) + a2r.Call(c, conversation.ConversationClient.SetConversations, o.Client) } func (o *ConversationApi) GetConversationOfflinePushUserIDs(c *gin.Context) { - a2r.CallV2(c, conversation.GetConversationOfflinePushUserIDsCaller.Invoke) + a2r.Call(c, conversation.ConversationClient.GetConversationOfflinePushUserIDs, o.Client) } func (o *ConversationApi) GetFullOwnerConversationIDs(c *gin.Context) { - a2r.CallV2(c, conversation.GetFullOwnerConversationIDsCaller.Invoke) + a2r.Call(c, conversation.ConversationClient.GetFullOwnerConversationIDs, o.Client) } func (o *ConversationApi) GetIncrementalConversation(c *gin.Context) { - a2r.CallV2(c, conversation.GetIncrementalConversationCaller.Invoke) + a2r.Call(c, conversation.ConversationClient.GetIncrementalConversation, o.Client) } func (o *ConversationApi) GetOwnerConversation(c *gin.Context) { - a2r.CallV2(c, conversation.GetOwnerConversationCaller.Invoke) + a2r.Call(c, conversation.ConversationClient.GetOwnerConversation, o.Client) } func (o *ConversationApi) GetNotNotifyConversationIDs(c *gin.Context) { - a2r.CallV2(c, conversation.GetNotNotifyConversationIDsCaller.Invoke) + a2r.Call(c, conversation.ConversationClient.GetNotNotifyConversationIDs, o.Client) } func (o *ConversationApi) GetPinnedConversationIDs(c *gin.Context) { - a2r.CallV2(c, conversation.GetPinnedConversationIDsCaller.Invoke) + a2r.Call(c, conversation.ConversationClient.GetPinnedConversationIDs, o.Client) } diff --git a/internal/api/friend.go b/internal/api/friend.go index 4e08474fb..7d84ff0dc 100644 --- a/internal/api/friend.go +++ b/internal/api/friend.go @@ -21,94 +21,96 @@ import ( "github.com/openimsdk/tools/a2r" ) -type FriendApi struct{} +type FriendApi struct { + Client relation.FriendClient +} -func NewFriendApi() FriendApi { - return FriendApi{} +func NewFriendApi(client relation.FriendClient) FriendApi { + return FriendApi{client} } func (o *FriendApi) ApplyToAddFriend(c *gin.Context) { - a2r.CallV2(c, relation.ApplyToAddFriendCaller.Invoke) + a2r.Call(c, relation.FriendClient.ApplyToAddFriend, o.Client) } func (o *FriendApi) RespondFriendApply(c *gin.Context) { - a2r.CallV2(c, relation.RespondFriendApplyCaller.Invoke) + a2r.Call(c, relation.FriendClient.RespondFriendApply, o.Client) } func (o *FriendApi) DeleteFriend(c *gin.Context) { - a2r.CallV2(c, relation.DeleteFriendCaller.Invoke) + a2r.Call(c, relation.FriendClient.DeleteFriend, o.Client) } func (o *FriendApi) GetFriendApplyList(c *gin.Context) { - a2r.CallV2(c, relation.GetPaginationFriendsApplyToCaller.Invoke) + a2r.Call(c, relation.FriendClient.GetPaginationFriendsApplyTo, o.Client) } func (o *FriendApi) GetDesignatedFriendsApply(c *gin.Context) { - a2r.CallV2(c, relation.GetDesignatedFriendsApplyCaller.Invoke) + a2r.Call(c, relation.FriendClient.GetDesignatedFriendsApply, o.Client) } func (o *FriendApi) GetSelfApplyList(c *gin.Context) { - a2r.CallV2(c, relation.GetPaginationFriendsApplyFromCaller.Invoke) + a2r.Call(c, relation.FriendClient.GetPaginationFriendsApplyFrom, o.Client) } func (o *FriendApi) GetFriendList(c *gin.Context) { - a2r.CallV2(c, relation.GetPaginationFriendsCaller.Invoke) + a2r.Call(c, relation.FriendClient.GetPaginationFriends, o.Client) } func (o *FriendApi) GetDesignatedFriends(c *gin.Context) { - a2r.CallV2(c, relation.GetDesignatedFriendsCaller.Invoke) + a2r.Call(c, relation.FriendClient.GetDesignatedFriends, o.Client) } func (o *FriendApi) SetFriendRemark(c *gin.Context) { - a2r.CallV2(c, relation.SetFriendRemarkCaller.Invoke) + a2r.Call(c, relation.FriendClient.SetFriendRemark, o.Client) } func (o *FriendApi) AddBlack(c *gin.Context) { - a2r.CallV2(c, relation.AddBlackCaller.Invoke) + a2r.Call(c, relation.FriendClient.AddBlack, o.Client) } func (o *FriendApi) GetPaginationBlacks(c *gin.Context) { - a2r.CallV2(c, relation.GetPaginationBlacksCaller.Invoke) + a2r.Call(c, relation.FriendClient.GetPaginationBlacks, o.Client) } func (o *FriendApi) GetSpecifiedBlacks(c *gin.Context) { - a2r.CallV2(c, relation.GetSpecifiedBlacksCaller.Invoke) + a2r.Call(c, relation.FriendClient.GetSpecifiedBlacks, o.Client) } func (o *FriendApi) RemoveBlack(c *gin.Context) { - a2r.CallV2(c, relation.RemoveBlackCaller.Invoke) + a2r.Call(c, relation.FriendClient.RemoveBlack, o.Client) } func (o *FriendApi) ImportFriends(c *gin.Context) { - a2r.CallV2(c, relation.ImportFriendsCaller.Invoke) + a2r.Call(c, relation.FriendClient.ImportFriends, o.Client) } func (o *FriendApi) IsFriend(c *gin.Context) { - a2r.CallV2(c, relation.IsFriendCaller.Invoke) + a2r.Call(c, relation.FriendClient.IsFriend, o.Client) } func (o *FriendApi) GetFriendIDs(c *gin.Context) { - a2r.CallV2(c, relation.GetFriendIDsCaller.Invoke) + a2r.Call(c, relation.FriendClient.GetFriendIDs, o.Client) } func (o *FriendApi) GetSpecifiedFriendsInfo(c *gin.Context) { - a2r.CallV2(c, relation.GetSpecifiedFriendsInfoCaller.Invoke) + a2r.Call(c, relation.FriendClient.GetSpecifiedFriendsInfo, o.Client) } func (o *FriendApi) UpdateFriends(c *gin.Context) { - a2r.CallV2(c, relation.UpdateFriendsCaller.Invoke) + a2r.Call(c, relation.FriendClient.UpdateFriends, o.Client) } func (o *FriendApi) GetIncrementalFriends(c *gin.Context) { - a2r.CallV2(c, relation.GetIncrementalFriendsCaller.Invoke) + a2r.Call(c, relation.FriendClient.GetIncrementalFriends, o.Client) } // GetIncrementalBlacks is temporarily unused. // Deprecated: This function is currently unused and may be removed in future versions. func (o *FriendApi) GetIncrementalBlacks(c *gin.Context) { - a2r.CallV2(c, relation.GetIncrementalBlacksCaller.Invoke) + a2r.Call(c, relation.FriendClient.GetIncrementalBlacks, o.Client) } func (o *FriendApi) GetFullFriendUserIDs(c *gin.Context) { - a2r.CallV2(c, relation.GetFullFriendUserIDsCaller.Invoke) + a2r.Call(c, relation.FriendClient.GetFullFriendUserIDs, o.Client) } diff --git a/internal/api/group.go b/internal/api/group.go index 784197fd8..97b6b73f0 100644 --- a/internal/api/group.go +++ b/internal/api/group.go @@ -20,146 +20,148 @@ import ( "github.com/openimsdk/tools/a2r" ) -type GroupApi struct{} +type GroupApi struct { + Client group.GroupClient +} -func NewGroupApi() GroupApi { - return GroupApi{} +func NewGroupApi(client group.GroupClient) GroupApi { + return GroupApi{client} } func (o *GroupApi) CreateGroup(c *gin.Context) { - a2r.CallV2(c, group.CreateGroupCaller.Invoke) + a2r.Call(c, group.GroupClient.CreateGroup, o.Client) } func (o *GroupApi) SetGroupInfo(c *gin.Context) { - a2r.CallV2(c, group.SetGroupInfoCaller.Invoke) + a2r.Call(c, group.GroupClient.SetGroupInfo, o.Client) } func (o *GroupApi) SetGroupInfoEx(c *gin.Context) { - a2r.CallV2(c, group.SetGroupInfoExCaller.Invoke) + a2r.Call(c, group.GroupClient.SetGroupInfoEx, o.Client) } func (o *GroupApi) JoinGroup(c *gin.Context) { - a2r.CallV2(c, group.JoinGroupCaller.Invoke) + a2r.Call(c, group.GroupClient.JoinGroup, o.Client) } func (o *GroupApi) QuitGroup(c *gin.Context) { - a2r.CallV2(c, group.QuitGroupCaller.Invoke) + a2r.Call(c, group.GroupClient.QuitGroup, o.Client) } func (o *GroupApi) ApplicationGroupResponse(c *gin.Context) { - a2r.CallV2(c, group.GroupApplicationResponseCaller.Invoke) + a2r.Call(c, group.GroupClient.GroupApplicationResponse, o.Client) } func (o *GroupApi) TransferGroupOwner(c *gin.Context) { - a2r.CallV2(c, group.TransferGroupOwnerCaller.Invoke) + a2r.Call(c, group.GroupClient.TransferGroupOwner, o.Client) } func (o *GroupApi) GetRecvGroupApplicationList(c *gin.Context) { - a2r.CallV2(c, group.GetGroupApplicationListCaller.Invoke) + a2r.Call(c, group.GroupClient.GetGroupApplicationList, o.Client) } func (o *GroupApi) GetUserReqGroupApplicationList(c *gin.Context) { - a2r.CallV2(c, group.GetUserReqApplicationListCaller.Invoke) + a2r.Call(c, group.GroupClient.GetUserReqApplicationList, o.Client) } func (o *GroupApi) GetGroupUsersReqApplicationList(c *gin.Context) { - a2r.CallV2(c, group.GetGroupUsersReqApplicationListCaller.Invoke) + a2r.Call(c, group.GroupClient.GetGroupUsersReqApplicationList, o.Client) } func (o *GroupApi) GetSpecifiedUserGroupRequestInfo(c *gin.Context) { - a2r.CallV2(c, group.GetSpecifiedUserGroupRequestInfoCaller.Invoke) + a2r.Call(c, group.GroupClient.GetSpecifiedUserGroupRequestInfo, o.Client) } func (o *GroupApi) GetGroupsInfo(c *gin.Context) { - a2r.CallV2(c, group.GetGroupsInfoCaller.Invoke) - //a2r.Call(group.GroupClient.GetGroupsInfo, o.Client, c, a2r.NewNilReplaceOption(group.GroupClient.GetGroupsInfo)) + a2r.Call(c, group.GroupClient.GetGroupsInfo, o.Client) + //a2r.Call(c, group.GroupClient.GetGroupsInfo, o.Client, c, a2r.NewNilReplaceOption(group.GroupClient.GetGroupsInfo)) } func (o *GroupApi) KickGroupMember(c *gin.Context) { - a2r.CallV2(c, group.KickGroupMemberCaller.Invoke) + a2r.Call(c, group.GroupClient.KickGroupMember, o.Client) } func (o *GroupApi) GetGroupMembersInfo(c *gin.Context) { - a2r.CallV2(c, group.GetGroupMembersInfoCaller.Invoke) - //a2r.Call(group.GroupClient.GetGroupMembersInfo, o.Client, c, a2r.NewNilReplaceOption(group.GroupClient.GetGroupMembersInfo)) + a2r.Call(c, group.GroupClient.GetGroupMembersInfo, o.Client) + //a2r.Call(c, group.GroupClient.GetGroupMembersInfo, o.Client, c, a2r.NewNilReplaceOption(group.GroupClient.GetGroupMembersInfo)) } func (o *GroupApi) GetGroupMemberList(c *gin.Context) { - a2r.CallV2(c, group.GetGroupMemberListCaller.Invoke) + a2r.Call(c, group.GroupClient.GetGroupMemberList, o.Client) } func (o *GroupApi) InviteUserToGroup(c *gin.Context) { - a2r.CallV2(c, group.InviteUserToGroupCaller.Invoke) + a2r.Call(c, group.GroupClient.InviteUserToGroup, o.Client) } func (o *GroupApi) GetJoinedGroupList(c *gin.Context) { - a2r.CallV2(c, group.GetJoinedGroupListCaller.Invoke) + a2r.Call(c, group.GroupClient.GetJoinedGroupList, o.Client) } func (o *GroupApi) DismissGroup(c *gin.Context) { - a2r.CallV2(c, group.DismissGroupCaller.Invoke) + a2r.Call(c, group.GroupClient.DismissGroup, o.Client) } func (o *GroupApi) MuteGroupMember(c *gin.Context) { - a2r.CallV2(c, group.MuteGroupMemberCaller.Invoke) + a2r.Call(c, group.GroupClient.MuteGroupMember, o.Client) } func (o *GroupApi) CancelMuteGroupMember(c *gin.Context) { - a2r.CallV2(c, group.CancelMuteGroupMemberCaller.Invoke) + a2r.Call(c, group.GroupClient.CancelMuteGroupMember, o.Client) } func (o *GroupApi) MuteGroup(c *gin.Context) { - a2r.CallV2(c, group.MuteGroupCaller.Invoke) + a2r.Call(c, group.GroupClient.MuteGroup, o.Client) } func (o *GroupApi) CancelMuteGroup(c *gin.Context) { - a2r.CallV2(c, group.CancelMuteGroupCaller.Invoke) + a2r.Call(c, group.GroupClient.CancelMuteGroup, o.Client) } func (o *GroupApi) SetGroupMemberInfo(c *gin.Context) { - a2r.CallV2(c, group.SetGroupMemberInfoCaller.Invoke) + a2r.Call(c, group.GroupClient.SetGroupMemberInfo, o.Client) } func (o *GroupApi) GetGroupAbstractInfo(c *gin.Context) { - a2r.CallV2(c, group.GetGroupAbstractInfoCaller.Invoke) + a2r.Call(c, group.GroupClient.GetGroupAbstractInfo, o.Client) } // func (g *Group) SetGroupMemberNickname(c *gin.Context) { -// a2r.Call(group.GroupClient.SetGroupMemberNickname, g.userClient, c) +// a2r.Call(c, group.GroupClient.SetGroupMemberNickname, g.userClient) //} // // func (g *Group) GetGroupAllMemberList(c *gin.Context) { -// a2r.Call(group.GroupClient.GetGroupAllMember, g.userClient, c) +// a2r.Call(c, group.GroupClient.GetGroupAllMember, g.userClient) //} func (o *GroupApi) GroupCreateCount(c *gin.Context) { - a2r.CallV2(c, group.GroupCreateCountCaller.Invoke) + a2r.Call(c, group.GroupClient.GroupCreateCount, o.Client) } func (o *GroupApi) GetGroups(c *gin.Context) { - a2r.CallV2(c, group.GetGroupsCaller.Invoke) + a2r.Call(c, group.GroupClient.GetGroups, o.Client) } func (o *GroupApi) GetGroupMemberUserIDs(c *gin.Context) { - a2r.CallV2(c, group.GetGroupMemberUserIDsCaller.Invoke) + a2r.Call(c, group.GroupClient.GetGroupMemberUserIDs, o.Client) } func (o *GroupApi) GetIncrementalJoinGroup(c *gin.Context) { - a2r.CallV2(c, group.GetIncrementalJoinGroupCaller.Invoke) + a2r.Call(c, group.GroupClient.GetIncrementalJoinGroup, o.Client) } func (o *GroupApi) GetIncrementalGroupMember(c *gin.Context) { - a2r.CallV2(c, group.GetIncrementalGroupMemberCaller.Invoke) + a2r.Call(c, group.GroupClient.GetIncrementalGroupMember, o.Client) } func (o *GroupApi) GetIncrementalGroupMemberBatch(c *gin.Context) { - a2r.CallV2(c, group.BatchGetIncrementalGroupMemberCaller.Invoke) + a2r.Call(c, group.GroupClient.BatchGetIncrementalGroupMember, o.Client) } func (o *GroupApi) GetFullGroupMemberUserIDs(c *gin.Context) { - a2r.CallV2(c, group.GetFullGroupMemberUserIDsCaller.Invoke) + a2r.Call(c, group.GroupClient.GetFullGroupMemberUserIDs, o.Client) } func (o *GroupApi) GetFullJoinGroupIDs(c *gin.Context) { - a2r.CallV2(c, group.GetFullJoinGroupIDsCaller.Invoke) + a2r.Call(c, group.GroupClient.GetFullJoinGroupIDs, o.Client) } diff --git a/internal/api/init.go b/internal/api/init.go index fcc69dc68..4a558c8b6 100644 --- a/internal/api/init.go +++ b/internal/api/init.go @@ -29,7 +29,6 @@ import ( conf "github.com/openimsdk/open-im-server/v3/pkg/common/config" kdisc "github.com/openimsdk/open-im-server/v3/pkg/common/discoveryregister" "github.com/openimsdk/open-im-server/v3/pkg/common/prommetrics" - "github.com/openimsdk/open-im-server/v3/pkg/rpcclient" "github.com/openimsdk/tools/discovery/etcd" "github.com/openimsdk/tools/errs" "github.com/openimsdk/tools/log" @@ -64,9 +63,6 @@ func Start(ctx context.Context, index int, config *Config) error { return errs.WrapMsg(err, "failed to register discovery service") } client.AddOption(mw.GrpcClient(), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithDefaultServiceConfig(fmt.Sprintf(`{"LoadBalancingPolicy": "%s"}`, "round_robin"))) - if err = rpcclient.InitRpcCaller(client, config.Discovery.RpcService); err != nil { - return err - } var ( netDone = make(chan struct{}, 1) @@ -94,7 +90,10 @@ func Start(ctx context.Context, index int, config *Config) error { return errs.New("only etcd support autoSetPorts", "RegisterName", "api").Wrap() } - router := newGinRouter(client, config) + router, err := newGinRouter(ctx, client, config) + if err != nil { + return err + } if config.API.Prometheus.Enable { var ( listener net.Listener diff --git a/internal/api/jssdk/jssdk.go b/internal/api/jssdk/jssdk.go index 409fcbf79..036cb027a 100644 --- a/internal/api/jssdk/jssdk.go +++ b/internal/api/jssdk/jssdk.go @@ -2,16 +2,15 @@ package jssdk import ( "context" + "github.com/openimsdk/open-im-server/v3/pkg/rpcli" "sort" "github.com/gin-gonic/gin" "github.com/openimsdk/protocol/conversation" - "github.com/openimsdk/protocol/group" "github.com/openimsdk/protocol/jssdk" "github.com/openimsdk/protocol/msg" "github.com/openimsdk/protocol/relation" "github.com/openimsdk/protocol/sdkws" - "github.com/openimsdk/protocol/user" "github.com/openimsdk/tools/mcontext" "github.com/openimsdk/tools/utils/datautil" ) @@ -26,6 +25,11 @@ func NewJSSdkApi() *JSSdk { } type JSSdk struct { + userClient rpcli.UserClient + relationClient rpcli.RelationClient + groupClient rpcli.GroupClient + conversationClient rpcli.ConversationClient + msgClient rpcli.MsgClient } func (x *JSSdk) GetActiveConversations(c *gin.Context) { @@ -57,11 +61,11 @@ func (x *JSSdk) fillConversations(ctx context.Context, conversations []*jssdk.Co groupMap map[string]*sdkws.GroupInfo ) if len(userIDs) > 0 { - users, err := field(ctx, user.GetDesignateUsersCaller.Invoke, &user.GetDesignateUsersReq{UserIDs: userIDs}, (*user.GetDesignateUsersResp).GetUsersInfo) + users, err := x.userClient.GetUsersInfo(ctx, userIDs) if err != nil { return err } - friends, err := field(ctx, relation.GetFriendInfoCaller.Invoke, &relation.GetFriendInfoReq{OwnerUserID: conversations[0].Conversation.OwnerUserID, FriendUserIDs: userIDs}, (*relation.GetFriendInfoResp).GetFriendInfos) + friends, err := x.relationClient.GetFriendsInfo(ctx, conversations[0].Conversation.OwnerUserID, userIDs) if err != nil { return err } @@ -69,11 +73,11 @@ func (x *JSSdk) fillConversations(ctx context.Context, conversations []*jssdk.Co friendMap = datautil.SliceToMap(friends, (*relation.FriendInfoOnly).GetFriendUserID) } if len(groupIDs) > 0 { - resp, err := group.GetGroupsInfoCaller.Invoke(ctx, &group.GetGroupsInfoReq{GroupIDs: groupIDs}) + groups, err := x.groupClient.GetGroupsInfo(ctx, groupIDs) if err != nil { return err } - groupMap = datautil.SliceToMap(resp.GroupInfos, (*sdkws.GroupInfo).GetGroupID) + groupMap = datautil.SliceToMap(groups, (*sdkws.GroupInfo).GetGroupID) } for _, c := range conversations { if c.Conversation.GroupID == "" { @@ -91,21 +95,18 @@ func (x *JSSdk) getActiveConversations(ctx context.Context, req *jssdk.GetActive req.Count = defaultGetActiveConversation } req.OwnerUserID = mcontext.GetOpUserID(ctx) - conversationIDs, err := field(ctx, conversation.GetConversationIDsCaller.Invoke, - &conversation.GetConversationIDsReq{UserID: req.OwnerUserID}, (*conversation.GetConversationIDsResp).GetConversationIDs) + conversationIDs, err := x.conversationClient.GetConversationIDs(ctx, req.OwnerUserID) if err != nil { return nil, err } if len(conversationIDs) == 0 { return &jssdk.GetActiveConversationsResp{}, nil } - readSeq, err := field(ctx, msg.GetHasReadSeqsCaller.Invoke, - &msg.GetHasReadSeqsReq{UserID: req.OwnerUserID, ConversationIDs: conversationIDs}, (*msg.SeqsInfoResp).GetMaxSeqs) + readSeq, err := x.msgClient.GetHasReadSeqs(ctx, conversationIDs, req.OwnerUserID) if err != nil { return nil, err } - activeConversation, err := field(ctx, msg.GetActiveConversationCaller.Invoke, - &msg.GetActiveConversationReq{ConversationIDs: conversationIDs}, (*msg.GetActiveConversationResp).GetConversations) + activeConversation, err := x.msgClient.GetActiveConversation(ctx, conversationIDs) if err != nil { return nil, err } @@ -116,8 +117,7 @@ func (x *JSSdk) getActiveConversations(ctx context.Context, req *jssdk.GetActive Conversation: activeConversation, } if len(activeConversation) > 1 { - pinnedConversationIDs, err := field(ctx, conversation.GetPinnedConversationIDsCaller.Invoke, - &conversation.GetPinnedConversationIDsReq{UserID: req.OwnerUserID}, (*conversation.GetPinnedConversationIDsResp).GetConversationIDs) + pinnedConversationIDs, err := x.conversationClient.GetPinnedConversationIDs(ctx, req.OwnerUserID) if err != nil { return nil, err } @@ -125,25 +125,18 @@ func (x *JSSdk) getActiveConversations(ctx context.Context, req *jssdk.GetActive } sort.Sort(&sortConversations) sortList := sortConversations.Top(int(req.Count)) - conversations, err := field(ctx, conversation.GetConversationsCaller.Invoke, - &conversation.GetConversationsReq{ - OwnerUserID: req.OwnerUserID, - ConversationIDs: datautil.Slice(sortList, func(c *msg.ActiveConversation) string { - return c.ConversationID - })}, (*conversation.GetConversationsResp).GetConversations) + conversations, err := x.conversationClient.GetConversations(ctx, datautil.Slice(sortList, func(c *msg.ActiveConversation) string { + return c.ConversationID + }), req.OwnerUserID) if err != nil { return nil, err } - msgs, err := field(ctx, msg.GetSeqMessageCaller.Invoke, - &msg.GetSeqMessageReq{ - UserID: req.OwnerUserID, - Conversations: datautil.Slice(sortList, func(c *msg.ActiveConversation) *msg.ConversationSeqs { - return &msg.ConversationSeqs{ - ConversationID: c.ConversationID, - Seqs: []int64{c.MaxSeq}, - } - }), - }, (*msg.GetSeqMessageResp).GetMsgs) + msgs, err := x.msgClient.GetSeqMessage(ctx, req.OwnerUserID, datautil.Slice(sortList, func(c *msg.ActiveConversation) *msg.ConversationSeqs { + return &msg.ConversationSeqs{ + ConversationID: c.ConversationID, + Seqs: []int64{c.MaxSeq}, + } + })) if err != nil { return nil, err } @@ -185,7 +178,7 @@ func (x *JSSdk) getActiveConversations(ctx context.Context, req *jssdk.GetActive func (x *JSSdk) getConversations(ctx context.Context, req *jssdk.GetConversationsReq) (*jssdk.GetConversationsResp, error) { req.OwnerUserID = mcontext.GetOpUserID(ctx) - conversations, err := field(ctx, conversation.GetConversationsCaller.Invoke, &conversation.GetConversationsReq{OwnerUserID: req.OwnerUserID, ConversationIDs: req.ConversationIDs}, (*conversation.GetConversationsResp).GetConversations) + conversations, err := x.conversationClient.GetConversations(ctx, req.ConversationIDs, req.OwnerUserID) if err != nil { return nil, err } @@ -195,13 +188,11 @@ func (x *JSSdk) getConversations(ctx context.Context, req *jssdk.GetConversation req.ConversationIDs = datautil.Slice(conversations, func(c *conversation.Conversation) string { return c.ConversationID }) - maxSeqs, err := field(ctx, msg.GetMaxSeqsCaller.Invoke, - &msg.GetMaxSeqsReq{ConversationIDs: req.ConversationIDs}, (*msg.SeqsInfoResp).GetMaxSeqs) + maxSeqs, err := x.msgClient.GetMaxSeqs(ctx, req.ConversationIDs) if err != nil { return nil, err } - readSeqs, err := field(ctx, msg.GetHasReadSeqsCaller.Invoke, - &msg.GetHasReadSeqsReq{UserID: req.OwnerUserID, ConversationIDs: req.ConversationIDs}, (*msg.SeqsInfoResp).GetMaxSeqs) + readSeqs, err := x.msgClient.GetHasReadSeqs(ctx, req.ConversationIDs, req.OwnerUserID) if err != nil { return nil, err } @@ -216,8 +207,7 @@ func (x *JSSdk) getConversations(ctx context.Context, req *jssdk.GetConversation } var msgs map[string]*sdkws.PullMsgs if len(conversationSeqs) > 0 { - msgs, err = field(ctx, msg.GetSeqMessageCaller.Invoke, - &msg.GetSeqMessageReq{UserID: req.OwnerUserID, Conversations: conversationSeqs}, (*msg.GetSeqMessageResp).GetMsgs) + msgs, err = x.msgClient.GetSeqMessage(ctx, req.OwnerUserID, conversationSeqs) if err != nil { return nil, err } diff --git a/internal/api/msg.go b/internal/api/msg.go index 8efd34894..fc235354c 100644 --- a/internal/api/msg.go +++ b/internal/api/msg.go @@ -21,11 +21,10 @@ import ( "github.com/openimsdk/open-im-server/v3/pkg/apistruct" "github.com/openimsdk/open-im-server/v3/pkg/authverify" "github.com/openimsdk/open-im-server/v3/pkg/common/config" + "github.com/openimsdk/open-im-server/v3/pkg/rpcli" "github.com/openimsdk/protocol/constant" "github.com/openimsdk/protocol/msg" - "github.com/openimsdk/protocol/rpccall" "github.com/openimsdk/protocol/sdkws" - "github.com/openimsdk/protocol/user" "github.com/openimsdk/tools/a2r" "github.com/openimsdk/tools/apiresp" "github.com/openimsdk/tools/errs" @@ -38,12 +37,14 @@ import ( ) type MessageApi struct { - validate *validator.Validate + Client msg.MsgClient + userClient *rpcli.UserClient imAdminUserID []string + validate *validator.Validate } -func NewMessageApi(imAdminUserID []string) MessageApi { - return MessageApi{validate: validator.New(), imAdminUserID: imAdminUserID} +func NewMessageApi(client msg.MsgClient, userClient *rpcli.UserClient, imAdminUserID []string) MessageApi { + return MessageApi{Client: client, userClient: userClient, imAdminUserID: imAdminUserID, validate: validator.New()} } func (*MessageApi) SetOptions(options map[string]bool, value bool) { @@ -105,51 +106,51 @@ func (m *MessageApi) newUserSendMsgReq(_ *gin.Context, params *apistruct.SendMsg } func (m *MessageApi) GetSeq(c *gin.Context) { - a2r.CallV2(c, msg.GetMaxSeqCaller.Invoke) + a2r.Call(c, msg.MsgClient.GetMaxSeq, m.Client) } func (m *MessageApi) PullMsgBySeqs(c *gin.Context) { - a2r.CallV2(c, msg.PullMessageBySeqsCaller.Invoke) + a2r.Call(c, msg.MsgClient.PullMessageBySeqs, m.Client) } func (m *MessageApi) RevokeMsg(c *gin.Context) { - a2r.CallV2(c, msg.RevokeMsgCaller.Invoke) + a2r.Call(c, msg.MsgClient.RevokeMsg, m.Client) } func (m *MessageApi) MarkMsgsAsRead(c *gin.Context) { - a2r.CallV2(c, msg.MarkMsgsAsReadCaller.Invoke) + a2r.Call(c, msg.MsgClient.MarkMsgsAsRead, m.Client) } func (m *MessageApi) MarkConversationAsRead(c *gin.Context) { - a2r.CallV2(c, msg.MarkConversationAsReadCaller.Invoke) + a2r.Call(c, msg.MsgClient.MarkConversationAsRead, m.Client) } func (m *MessageApi) GetConversationsHasReadAndMaxSeq(c *gin.Context) { - a2r.CallV2(c, msg.GetConversationsHasReadAndMaxSeqCaller.Invoke) + a2r.Call(c, msg.MsgClient.GetConversationsHasReadAndMaxSeq, m.Client) } func (m *MessageApi) SetConversationHasReadSeq(c *gin.Context) { - a2r.CallV2(c, msg.SetConversationHasReadSeqCaller.Invoke) + a2r.Call(c, msg.MsgClient.SetConversationHasReadSeq, m.Client) } func (m *MessageApi) ClearConversationsMsg(c *gin.Context) { - a2r.CallV2(c, msg.ClearConversationsMsgCaller.Invoke) + a2r.Call(c, msg.MsgClient.ClearConversationsMsg, m.Client) } func (m *MessageApi) UserClearAllMsg(c *gin.Context) { - a2r.CallV2(c, msg.UserClearAllMsgCaller.Invoke) + a2r.Call(c, msg.MsgClient.UserClearAllMsg, m.Client) } func (m *MessageApi) DeleteMsgs(c *gin.Context) { - a2r.CallV2(c, msg.DeleteMsgsCaller.Invoke) + a2r.Call(c, msg.MsgClient.DeleteMsgs, m.Client) } func (m *MessageApi) DeleteMsgPhysicalBySeq(c *gin.Context) { - a2r.CallV2(c, msg.DeleteMsgPhysicalBySeqCaller.Invoke) + a2r.Call(c, msg.MsgClient.DeleteMsgPhysicalBySeq, m.Client) } func (m *MessageApi) DeleteMsgPhysical(c *gin.Context) { - a2r.CallV2(c, msg.DeleteMsgPhysicalCaller.Invoke) + a2r.Call(c, msg.MsgClient.DeleteMsgPhysical, m.Client) } func (m *MessageApi) getSendMsgReq(c *gin.Context, req apistruct.SendMsg) (sendMsgReq *msg.SendMsgReq, err error) { @@ -170,14 +171,10 @@ func (m *MessageApi) getSendMsgReq(c *gin.Context, req apistruct.SendMsg) (sendM data = apistruct.AtElem{} case constant.Custom: data = apistruct.CustomElem{} - case constant.Quote: - data = apistruct.QuoteElem{} - case constant.Stream: - data = apistruct.StreamMsgElem{} case constant.OANotification: data = apistruct.OANotificationElem{} req.SessionType = constant.NotificationChatType - if err = user.GetNotificationAccountCaller.Execute(c, &user.GetNotificationAccountReq{UserID: req.SendID}); err != nil { + if err = m.userClient.GetNotificationByID(c, req.SendID); err != nil { return nil, err } default: @@ -224,7 +221,7 @@ func (m *MessageApi) SendMessage(c *gin.Context) { sendMsgReq.MsgData.RecvID = req.RecvID // Attempt to send the message using the client. - respPb, err := msg.SendMsgCaller.Invoke(c, sendMsgReq) + respPb, err := m.Client.SendMsg(c, sendMsgReq) if err != nil { // Set the status to failed and respond with an error if sending fails. apiresp.GinError(c, err) @@ -235,7 +232,7 @@ func (m *MessageApi) SendMessage(c *gin.Context) { var status = constant.MsgSendSuccessed // Attempt to update the message sending status in the system. - err = msg.SetSendMsgStatusCaller.Execute(c, &msg.SetSendMsgStatusReq{ + _, err = m.Client.SetSendMsgStatus(c, &msg.SetSendMsgStatusReq{ Status: int32(status), }) @@ -287,7 +284,7 @@ func (m *MessageApi) SendBusinessNotification(c *gin.Context) { }), }, } - respPb, err := msg.SendMsgCaller.Invoke(c, &sendMsgReq) + respPb, err := m.Client.SendMsg(c, &sendMsgReq) if err != nil { apiresp.GinError(c, err) return @@ -311,13 +308,10 @@ func (m *MessageApi) BatchSendMsg(c *gin.Context) { var recvIDs []string if req.IsSendAll { - pageNumber := 1 - showNumber := 500 + var pageNumber int32 = 1 + const showNumber = 500 for { - recvIDsPart, err := rpccall.ExtractField(c, user.GetAllUserIDCaller.Invoke, &user.GetAllUserIDReq{Pagination: &sdkws.RequestPagination{ - PageNumber: int32(pageNumber), - ShowNumber: int32(showNumber), - }}, (*user.GetAllUserIDResp).GetUserIDs) + recvIDsPart, err := m.userClient.GetAllUserIDs(c, pageNumber, showNumber) if err != nil { apiresp.GinError(c, err) return @@ -339,7 +333,7 @@ func (m *MessageApi) BatchSendMsg(c *gin.Context) { } for _, recvID := range recvIDs { sendMsgReq.MsgData.RecvID = recvID - rpcResp, err := msg.SendMsgCaller.Invoke(c, sendMsgReq) + rpcResp, err := m.Client.SendMsg(c, sendMsgReq) if err != nil { resp.FailedIDs = append(resp.FailedIDs, recvID) continue @@ -355,33 +349,33 @@ func (m *MessageApi) BatchSendMsg(c *gin.Context) { } func (m *MessageApi) CheckMsgIsSendSuccess(c *gin.Context) { - a2r.CallV2(c, msg.GetSendMsgStatusCaller.Invoke) + a2r.Call(c, msg.MsgClient.GetSendMsgStatus, m.Client) } func (m *MessageApi) GetUsersOnlineStatus(c *gin.Context) { - a2r.CallV2(c, msg.GetSendMsgStatusCaller.Invoke) + a2r.Call(c, msg.MsgClient.GetSendMsgStatus, m.Client) } func (m *MessageApi) GetActiveUser(c *gin.Context) { - a2r.CallV2(c, msg.GetActiveUserCaller.Invoke) + a2r.Call(c, msg.MsgClient.GetActiveUser, m.Client) } func (m *MessageApi) GetActiveGroup(c *gin.Context) { - a2r.CallV2(c, msg.GetActiveGroupCaller.Invoke) + a2r.Call(c, msg.MsgClient.GetActiveGroup, m.Client) } func (m *MessageApi) SearchMsg(c *gin.Context) { - a2r.CallV2(c, msg.SearchMessageCaller.Invoke) + a2r.Call(c, msg.MsgClient.SearchMessage, m.Client) } func (m *MessageApi) GetServerTime(c *gin.Context) { - a2r.CallV2(c, msg.GetServerTimeCaller.Invoke) + a2r.Call(c, msg.MsgClient.GetServerTime, m.Client) } func (m *MessageApi) GetStreamMsg(c *gin.Context) { - a2r.CallV2(c, msg.GetStreamMsgCaller.Invoke) + a2r.Call(c, msg.MsgClient.GetServerTime, m.Client) } func (m *MessageApi) AppendStreamMsg(c *gin.Context) { - a2r.CallV2(c, msg.AppendStreamMsgCaller.Invoke) + a2r.Call(c, msg.MsgClient.GetServerTime, m.Client) } diff --git a/internal/api/router.go b/internal/api/router.go index 5e8038068..62b8079e6 100644 --- a/internal/api/router.go +++ b/internal/api/router.go @@ -1,7 +1,15 @@ package api import ( + "context" "fmt" + "github.com/openimsdk/open-im-server/v3/pkg/rpcli" + "github.com/openimsdk/protocol/conversation" + "github.com/openimsdk/protocol/group" + "github.com/openimsdk/protocol/msg" + "github.com/openimsdk/protocol/relation" + "github.com/openimsdk/protocol/third" + "github.com/openimsdk/protocol/user" "net/http" "strings" @@ -47,9 +55,37 @@ func prommetricsGin() gin.HandlerFunc { } } -func newGinRouter(disCov discovery.SvcDiscoveryRegistry, config *Config) *gin.Engine { - disCov.AddOption(mw.GrpcClient(), grpc.WithTransportCredentials(insecure.NewCredentials()), +func newGinRouter(ctx context.Context, client discovery.SvcDiscoveryRegistry, config *Config) (*gin.Engine, error) { + client.AddOption(mw.GrpcClient(), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithDefaultServiceConfig(fmt.Sprintf(`{"LoadBalancingPolicy": "%s"}`, "round_robin"))) + authConn, err := client.GetConn(ctx, config.Discovery.RpcService.Auth) + if err != nil { + return nil, err + } + userConn, err := client.GetConn(ctx, config.Discovery.RpcService.User) + if err != nil { + return nil, err + } + groupConn, err := client.GetConn(ctx, config.Discovery.RpcService.Group) + if err != nil { + return nil, err + } + friendConn, err := client.GetConn(ctx, config.Discovery.RpcService.Friend) + if err != nil { + return nil, err + } + conversationConn, err := client.GetConn(ctx, config.Discovery.RpcService.Conversation) + if err != nil { + return nil, err + } + thirdConn, err := client.GetConn(ctx, config.Discovery.RpcService.Third) + if err != nil { + return nil, err + } + msgConn, err := client.GetConn(ctx, config.Discovery.RpcService.Msg) + if err != nil { + return nil, err + } gin.SetMode(gin.ReleaseMode) r := gin.New() if v, ok := binding.Validator.Engine().(*validator.Validate); ok { @@ -64,13 +100,12 @@ func newGinRouter(disCov discovery.SvcDiscoveryRegistry, config *Config) *gin.En case BestSpeed: r.Use(gzip.Gzip(gzip.BestSpeed)) } - r.Use(prommetricsGin(), gin.RecoveryWithWriter(gin.DefaultErrorWriter, mw.GinPanicErr), mw.CorsHandler(), mw.GinParseOperationID(), GinParseToken()) - u := NewUserApi(disCov, config.Discovery.RpcService.MessageGateway) - m := NewMessageApi(config.Share.IMAdminUserID) + r.Use(prommetricsGin(), gin.RecoveryWithWriter(gin.DefaultErrorWriter, mw.GinPanicErr), mw.CorsHandler(), mw.GinParseOperationID(), GinParseToken(rpcli.NewAuthClient(authConn))) j := jssdk.NewJSSdkApi() - pd := NewPrometheusDiscoveryApi(config, disCov) - userRouterGroup := r.Group("/user") + + u := NewUserApi(user.NewUserClient(userConn), client, config.Discovery.RpcService) { + userRouterGroup := r.Group("/user") userRouterGroup.POST("/user_register", u.UserRegister) userRouterGroup.POST("/update_user_info", u.UpdateUserInfo) userRouterGroup.POST("/update_user_info_ex", u.UpdateUserInfoEx) @@ -96,9 +131,9 @@ func newGinRouter(disCov discovery.SvcDiscoveryRegistry, config *Config) *gin.En userRouterGroup.POST("/search_notification_account", u.SearchNotificationAccount) } // friend routing group - friendRouterGroup := r.Group("/friend") { - f := NewFriendApi() + f := NewFriendApi(relation.NewFriendClient(friendConn)) + friendRouterGroup := r.Group("/friend") friendRouterGroup.POST("/delete_friend", f.DeleteFriend) friendRouterGroup.POST("/get_friend_apply_list", f.GetFriendApplyList) friendRouterGroup.POST("/get_designated_friend_apply", f.GetDesignatedFriendsApply) @@ -121,9 +156,10 @@ func newGinRouter(disCov discovery.SvcDiscoveryRegistry, config *Config) *gin.En friendRouterGroup.POST("/get_incremental_friends", f.GetIncrementalFriends) friendRouterGroup.POST("/get_full_friend_user_ids", f.GetFullFriendUserIDs) } - g := NewGroupApi() - groupRouterGroup := r.Group("/group") + + g := NewGroupApi(group.NewGroupClient(groupConn)) { + groupRouterGroup := r.Group("/group") groupRouterGroup.POST("/create_group", g.CreateGroup) groupRouterGroup.POST("/set_group_info", g.SetGroupInfo) groupRouterGroup.POST("/set_group_info_ex", g.SetGroupInfoEx) @@ -157,9 +193,9 @@ func newGinRouter(disCov discovery.SvcDiscoveryRegistry, config *Config) *gin.En groupRouterGroup.POST("/get_full_join_group_ids", g.GetFullJoinGroupIDs) } // certificate - authRouterGroup := r.Group("/auth") { - a := NewAuthApi() + a := NewAuthApi(pbAuth.NewAuthClient(authConn)) + authRouterGroup := r.Group("/auth") authRouterGroup.POST("/get_admin_token", a.GetAdminToken) authRouterGroup.POST("/get_user_token", a.GetUserToken) authRouterGroup.POST("/parse_token", a.ParseToken) @@ -167,9 +203,9 @@ func newGinRouter(disCov discovery.SvcDiscoveryRegistry, config *Config) *gin.En } // Third service - thirdGroup := r.Group("/third") { - t := NewThirdApi(config.API.Prometheus.GrafanaURL) + t := NewThirdApi(third.NewThirdClient(thirdConn), config.API.Prometheus.GrafanaURL) + thirdGroup := r.Group("/third") thirdGroup.GET("/prometheus", t.GetPrometheus) thirdGroup.POST("/fcm_update_token", t.FcmUpdateToken) thirdGroup.POST("/set_app_badge", t.SetAppBadge) @@ -192,8 +228,9 @@ func newGinRouter(disCov discovery.SvcDiscoveryRegistry, config *Config) *gin.En objectGroup.GET("/*name", t.ObjectRedirect) } // Message - msgGroup := r.Group("/msg") + m := NewMessageApi(msg.NewMsgClient(msgConn), rpcli.NewUserClient(userConn), config.Share.IMAdminUserID) { + msgGroup := r.Group("/msg") msgGroup.POST("/newest_seq", m.GetSeq) msgGroup.POST("/search_msg", m.SearchMsg) msgGroup.POST("/send_msg", m.SendMessage) @@ -218,9 +255,9 @@ func newGinRouter(disCov discovery.SvcDiscoveryRegistry, config *Config) *gin.En msgGroup.POST("/append_stream_msg", m.AppendStreamMsg) } // Conversation - conversationGroup := r.Group("/conversation") { - c := NewConversationApi() + c := NewConversationApi(conversation.NewConversationClient(conversationConn)) + conversationGroup := r.Group("/conversation") conversationGroup.POST("/get_sorted_conversation_list", c.GetSortedConversationList) conversationGroup.POST("/get_all_conversations", c.GetAllConversations) conversationGroup.POST("/get_conversation", c.GetConversation) @@ -234,35 +271,39 @@ func newGinRouter(disCov discovery.SvcDiscoveryRegistry, config *Config) *gin.En conversationGroup.POST("/get_pinned_conversation_ids", c.GetPinnedConversationIDs) } - statisticsGroup := r.Group("/statistics") { + statisticsGroup := r.Group("/statistics") statisticsGroup.POST("/user/register", u.UserRegisterCount) statisticsGroup.POST("/user/active", m.GetActiveUser) statisticsGroup.POST("/group/create", g.GroupCreateCount) statisticsGroup.POST("/group/active", m.GetActiveGroup) } - jssdk := r.Group("/jssdk") - jssdk.POST("/get_conversations", j.GetConversations) - jssdk.POST("/get_active_conversations", j.GetActiveConversations) - - proDiscoveryGroup := r.Group("/prometheus_discovery", pd.Enable) - proDiscoveryGroup.GET("/api", pd.Api) - proDiscoveryGroup.GET("/user", pd.User) - proDiscoveryGroup.GET("/group", pd.Group) - proDiscoveryGroup.GET("/msg", pd.Msg) - proDiscoveryGroup.GET("/friend", pd.Friend) - proDiscoveryGroup.GET("/conversation", pd.Conversation) - proDiscoveryGroup.GET("/third", pd.Third) - proDiscoveryGroup.GET("/auth", pd.Auth) - proDiscoveryGroup.GET("/push", pd.Push) - proDiscoveryGroup.GET("/msg_gateway", pd.MessageGateway) - proDiscoveryGroup.GET("/msg_transfer", pd.MessageTransfer) + { + jssdk := r.Group("/jssdk") + jssdk.POST("/get_conversations", j.GetConversations) + jssdk.POST("/get_active_conversations", j.GetActiveConversations) + } + { + pd := NewPrometheusDiscoveryApi(config, client) + proDiscoveryGroup := r.Group("/prometheus_discovery", pd.Enable) + proDiscoveryGroup.GET("/api", pd.Api) + proDiscoveryGroup.GET("/user", pd.User) + proDiscoveryGroup.GET("/group", pd.Group) + proDiscoveryGroup.GET("/msg", pd.Msg) + proDiscoveryGroup.GET("/friend", pd.Friend) + proDiscoveryGroup.GET("/conversation", pd.Conversation) + proDiscoveryGroup.GET("/third", pd.Third) + proDiscoveryGroup.GET("/auth", pd.Auth) + proDiscoveryGroup.GET("/push", pd.Push) + proDiscoveryGroup.GET("/msg_gateway", pd.MessageGateway) + proDiscoveryGroup.GET("/msg_transfer", pd.MessageTransfer) + } - return r + return r, nil } -func GinParseToken() gin.HandlerFunc { +func GinParseToken(authClient *rpcli.AuthClient) gin.HandlerFunc { return func(c *gin.Context) { switch c.Request.Method { case http.MethodPost: @@ -280,7 +321,7 @@ func GinParseToken() gin.HandlerFunc { c.Abort() return } - resp, err := pbAuth.ParseTokenCaller.Invoke(c, &pbAuth.ParseTokenReq{Token: token}) + resp, err := authClient.ParseToken(c, token) if err != nil { apiresp.GinError(c, err) c.Abort() diff --git a/internal/api/statistics.go b/internal/api/statistics.go deleted file mode 100644 index f60bddb2e..000000000 --- a/internal/api/statistics.go +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright © 2023 OpenIM. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package api - -import ( - "github.com/gin-gonic/gin" - "github.com/openimsdk/protocol/user" - "github.com/openimsdk/tools/a2r" -) - -type StatisticsApi struct{} - -func NewStatisticsApi() StatisticsApi { - return StatisticsApi{} -} - -func (s *StatisticsApi) UserRegister(c *gin.Context) { - a2r.CallV2(c, user.UserRegisterCountCaller.Invoke) -} diff --git a/internal/api/third.go b/internal/api/third.go index ac02e3734..7ecb32911 100644 --- a/internal/api/third.go +++ b/internal/api/third.go @@ -16,14 +16,13 @@ package api import ( "context" + "google.golang.org/grpc" "math/rand" "net/http" "net/url" "strconv" "strings" - "google.golang.org/grpc" - "github.com/gin-gonic/gin" "github.com/openimsdk/protocol/third" "github.com/openimsdk/tools/a2r" @@ -33,18 +32,19 @@ import ( type ThirdApi struct { GrafanaUrl string + Client third.ThirdClient } -func NewThirdApi(grafanaUrl string) ThirdApi { - return ThirdApi{GrafanaUrl: grafanaUrl} +func NewThirdApi(client third.ThirdClient, grafanaUrl string) ThirdApi { + return ThirdApi{Client: client, GrafanaUrl: grafanaUrl} } func (o *ThirdApi) FcmUpdateToken(c *gin.Context) { - a2r.CallV2(c, third.FcmUpdateTokenCaller.Invoke) + a2r.Call(c, third.ThirdClient.FcmUpdateToken, o.Client) } func (o *ThirdApi) SetAppBadge(c *gin.Context) { - a2r.CallV2(c, third.SetAppBadgeCaller.Invoke) + a2r.Call(c, third.ThirdClient.SetAppBadge, o.Client) } // #################### s3 #################### @@ -79,44 +79,44 @@ func setURLPrefix(c *gin.Context, urlPrefix *string) error { } func (o *ThirdApi) PartLimit(c *gin.Context) { - a2r.CallV2(c, third.PartLimitCaller.Invoke) + a2r.Call(c, third.ThirdClient.PartLimit, o.Client) } func (o *ThirdApi) PartSize(c *gin.Context) { - a2r.CallV2(c, third.PartSizeCaller.Invoke) + a2r.Call(c, third.ThirdClient.PartSize, o.Client) } func (o *ThirdApi) InitiateMultipartUpload(c *gin.Context) { opt := setURLPrefixOption(third.ThirdClient.InitiateMultipartUpload, func(req *third.InitiateMultipartUploadReq) error { return setURLPrefix(c, &req.UrlPrefix) }) - a2r.CallV2(c, third.InitiateMultipartUploadCaller.Invoke, opt) + a2r.Call(c, third.ThirdClient.InitiateMultipartUpload, o.Client, opt) } func (o *ThirdApi) AuthSign(c *gin.Context) { - a2r.CallV2(c, third.AuthSignCaller.Invoke) + a2r.Call(c, third.ThirdClient.AuthSign, o.Client) } func (o *ThirdApi) CompleteMultipartUpload(c *gin.Context) { opt := setURLPrefixOption(third.ThirdClient.CompleteMultipartUpload, func(req *third.CompleteMultipartUploadReq) error { return setURLPrefix(c, &req.UrlPrefix) }) - a2r.CallV2(c, third.CompleteMultipartUploadCaller.Invoke, opt) + a2r.Call(c, third.ThirdClient.CompleteMultipartUpload, o.Client, opt) } func (o *ThirdApi) AccessURL(c *gin.Context) { - a2r.CallV2(c, third.AccessURLCaller.Invoke) + a2r.Call(c, third.ThirdClient.AccessURL, o.Client) } func (o *ThirdApi) InitiateFormData(c *gin.Context) { - a2r.CallV2(c, third.InitiateFormDataCaller.Invoke) + a2r.Call(c, third.ThirdClient.InitiateFormData, o.Client) } func (o *ThirdApi) CompleteFormData(c *gin.Context) { opt := setURLPrefixOption(third.ThirdClient.CompleteFormData, func(req *third.CompleteFormDataReq) error { return setURLPrefix(c, &req.UrlPrefix) }) - a2r.CallV2(c, third.CompleteFormDataCaller.Invoke, opt) + a2r.Call(c, third.ThirdClient.CompleteFormData, o.Client, opt) } func (o *ThirdApi) ObjectRedirect(c *gin.Context) { @@ -140,7 +140,7 @@ func (o *ThirdApi) ObjectRedirect(c *gin.Context) { } query[key] = values[0] } - resp, err := third.AccessURLCaller.Invoke(ctx, &third.AccessURLReq{Name: name, Query: query}) + resp, err := o.Client.AccessURL(ctx, &third.AccessURLReq{Name: name, Query: query}) if err != nil { if errs.ErrArgs.Is(err) { c.String(http.StatusBadRequest, err.Error()) @@ -158,15 +158,15 @@ func (o *ThirdApi) ObjectRedirect(c *gin.Context) { // #################### logs ####################. func (o *ThirdApi) UploadLogs(c *gin.Context) { - a2r.CallV2(c, third.UploadLogsCaller.Invoke) + a2r.Call(c, third.ThirdClient.UploadLogs, o.Client) } func (o *ThirdApi) DeleteLogs(c *gin.Context) { - a2r.CallV2(c, third.DeleteLogsCaller.Invoke) + a2r.Call(c, third.ThirdClient.DeleteLogs, o.Client) } func (o *ThirdApi) SearchLogs(c *gin.Context) { - a2r.CallV2(c, third.SearchLogsCaller.Invoke) + a2r.Call(c, third.ThirdClient.SearchLogs, o.Client) } func (o *ThirdApi) GetPrometheus(c *gin.Context) { diff --git a/internal/api/user.go b/internal/api/user.go index e1024e50b..a88f8f65a 100644 --- a/internal/api/user.go +++ b/internal/api/user.go @@ -16,6 +16,7 @@ package api import ( "github.com/gin-gonic/gin" + "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/protocol/constant" "github.com/openimsdk/protocol/msggateway" "github.com/openimsdk/protocol/user" @@ -27,47 +28,45 @@ import ( ) type UserApi struct { - Discov discovery.SvcDiscoveryRegistry - MessageGateWayRpcName string + Client user.UserClient + discov discovery.SvcDiscoveryRegistry + config config.RpcService } -func NewUserApi(discov discovery.SvcDiscoveryRegistry, messageGateWayRpcName string) UserApi { - return UserApi{ - Discov: discov, - MessageGateWayRpcName: messageGateWayRpcName, - } +func NewUserApi(client user.UserClient, discov discovery.SvcDiscoveryRegistry, config config.RpcService) UserApi { + return UserApi{Client: client, discov: discov, config: config} } func (u *UserApi) UserRegister(c *gin.Context) { - a2r.CallV2(c, user.UserRegisterCaller.Invoke) + a2r.Call(c, user.UserClient.UserRegister, u.Client) } // UpdateUserInfo is deprecated. Use UpdateUserInfoEx func (u *UserApi) UpdateUserInfo(c *gin.Context) { - a2r.CallV2(c, user.UpdateUserInfoCaller.Invoke) + a2r.Call(c, user.UserClient.UpdateUserInfo, u.Client) } func (u *UserApi) UpdateUserInfoEx(c *gin.Context) { - a2r.CallV2(c, user.UpdateUserInfoExCaller.Invoke) + a2r.Call(c, user.UserClient.UpdateUserInfoEx, u.Client) } func (u *UserApi) SetGlobalRecvMessageOpt(c *gin.Context) { - a2r.CallV2(c, user.SetGlobalRecvMessageOptCaller.Invoke) + a2r.Call(c, user.UserClient.SetGlobalRecvMessageOpt, u.Client) } func (u *UserApi) GetUsersPublicInfo(c *gin.Context) { - a2r.CallV2(c, user.GetDesignateUsersCaller.Invoke) + a2r.Call(c, user.UserClient.GetDesignateUsers, u.Client) } func (u *UserApi) GetAllUsersID(c *gin.Context) { - a2r.CallV2(c, user.GetAllUserIDCaller.Invoke) + a2r.Call(c, user.UserClient.GetAllUserID, u.Client) } func (u *UserApi) AccountCheck(c *gin.Context) { - a2r.CallV2(c, user.AccountCheckCaller.Invoke) + a2r.Call(c, user.UserClient.AccountCheck, u.Client) } func (u *UserApi) GetUsers(c *gin.Context) { - a2r.CallV2(c, user.GetPaginationUsersCaller.Invoke) + a2r.Call(c, user.UserClient.GetPaginationUsers, u.Client) } // GetUsersOnlineStatus Get user online status. @@ -77,7 +76,7 @@ func (u *UserApi) GetUsersOnlineStatus(c *gin.Context) { apiresp.GinError(c, err) return } - conns, err := u.Discov.GetConns(c, u.MessageGateWayRpcName) + conns, err := u.discov.GetConns(c, u.config.MessageGateway) if err != nil { apiresp.GinError(c, err) return @@ -128,7 +127,7 @@ func (u *UserApi) GetUsersOnlineStatus(c *gin.Context) { } func (u *UserApi) UserRegisterCount(c *gin.Context) { - a2r.CallV2(c, user.UserRegisterCountCaller.Invoke) + a2r.Call(c, user.UserClient.UserRegisterCount, u.Client) } // GetUsersOnlineTokenDetail Get user online token details. @@ -141,7 +140,7 @@ func (u *UserApi) GetUsersOnlineTokenDetail(c *gin.Context) { apiresp.GinError(c, errs.ErrArgs.WithDetail(err.Error()).Wrap()) return } - conns, err := u.Discov.GetConns(c, u.MessageGateWayRpcName) + conns, err := u.discov.GetConns(c, u.config.MessageGateway) if err != nil { apiresp.GinError(c, err) return @@ -194,52 +193,52 @@ func (u *UserApi) GetUsersOnlineTokenDetail(c *gin.Context) { // SubscriberStatus Presence status of subscribed users. func (u *UserApi) SubscriberStatus(c *gin.Context) { - a2r.CallV2(c, user.SubscribeOrCancelUsersStatusCaller.Invoke) + a2r.Call(c, user.UserClient.SubscribeOrCancelUsersStatus, u.Client) } // GetUserStatus Get the online status of the user. func (u *UserApi) GetUserStatus(c *gin.Context) { - a2r.CallV2(c, user.GetUserStatusCaller.Invoke) + a2r.Call(c, user.UserClient.GetUserStatus, u.Client) } // GetSubscribeUsersStatus Get the online status of subscribers. func (u *UserApi) GetSubscribeUsersStatus(c *gin.Context) { - a2r.CallV2(c, user.GetSubscribeUsersStatusCaller.Invoke) + a2r.Call(c, user.UserClient.GetSubscribeUsersStatus, u.Client) } // ProcessUserCommandAdd user general function add. func (u *UserApi) ProcessUserCommandAdd(c *gin.Context) { - a2r.CallV2(c, user.ProcessUserCommandAddCaller.Invoke) + a2r.Call(c, user.UserClient.ProcessUserCommandAdd, u.Client) } // ProcessUserCommandDelete user general function delete. func (u *UserApi) ProcessUserCommandDelete(c *gin.Context) { - a2r.CallV2(c, user.ProcessUserCommandDeleteCaller.Invoke) + a2r.Call(c, user.UserClient.ProcessUserCommandDelete, u.Client) } // ProcessUserCommandUpdate user general function update. func (u *UserApi) ProcessUserCommandUpdate(c *gin.Context) { - a2r.CallV2(c, user.ProcessUserCommandUpdateCaller.Invoke) + a2r.Call(c, user.UserClient.ProcessUserCommandUpdate, u.Client) } // ProcessUserCommandGet user general function get. func (u *UserApi) ProcessUserCommandGet(c *gin.Context) { - a2r.CallV2(c, user.ProcessUserCommandGetCaller.Invoke) + a2r.Call(c, user.UserClient.ProcessUserCommandGet, u.Client) } // ProcessUserCommandGet user general function get all. func (u *UserApi) ProcessUserCommandGetAll(c *gin.Context) { - a2r.CallV2(c, user.ProcessUserCommandGetAllCaller.Invoke) + a2r.Call(c, user.UserClient.ProcessUserCommandGetAll, u.Client) } func (u *UserApi) AddNotificationAccount(c *gin.Context) { - a2r.CallV2(c, user.AddNotificationAccountCaller.Invoke) + a2r.Call(c, user.UserClient.AddNotificationAccount, u.Client) } func (u *UserApi) UpdateNotificationAccountInfo(c *gin.Context) { - a2r.CallV2(c, user.UpdateNotificationAccountInfoCaller.Invoke) + a2r.Call(c, user.UserClient.UpdateNotificationAccountInfo, u.Client) } func (u *UserApi) SearchNotificationAccount(c *gin.Context) { - a2r.CallV2(c, user.SearchNotificationAccountCaller.Invoke) + a2r.Call(c, user.UserClient.SearchNotificationAccount, u.Client) } diff --git a/internal/msggateway/hub_server.go b/internal/msggateway/hub_server.go index f6c12350c..753326726 100644 --- a/internal/msggateway/hub_server.go +++ b/internal/msggateway/hub_server.go @@ -16,6 +16,7 @@ package msggateway import ( "context" + "github.com/openimsdk/open-im-server/v3/pkg/rpcli" "sync/atomic" "github.com/openimsdk/open-im-server/v3/pkg/authverify" @@ -34,7 +35,14 @@ import ( ) func (s *Server) InitServer(ctx context.Context, config *Config, disCov discovery.SvcDiscoveryRegistry, server *grpc.Server) error { - s.LongConnServer.SetDiscoveryRegistry(disCov, config) + userConn, err := disCov.GetConn(ctx, config.Discovery.RpcService.User) + if err != nil { + return err + } + s.userClient = rpcli.NewUserClient(userConn) + if err := s.LongConnServer.SetDiscoveryRegistry(ctx, disCov, config); err != nil { + return err + } msggateway.RegisterMsgGatewayServer(server, s) if s.ready != nil { return s.ready(s) @@ -61,6 +69,7 @@ type Server struct { pushTerminal map[int]struct{} ready func(srv *Server) error queue *memamq.MemoryQueue + userClient *rpcli.UserClient } func (s *Server) SetLongConnServer(LongConnServer LongConnServer) { diff --git a/internal/msggateway/init.go b/internal/msggateway/init.go index 156d32b4d..6614b96bd 100644 --- a/internal/msggateway/init.go +++ b/internal/msggateway/init.go @@ -62,8 +62,9 @@ func Start(ctx context.Context, index int, conf *Config) error { ) hubServer := NewServer(longServer, conf, func(srv *Server) error { - longServer.online, _ = rpccache.NewOnlineCache(conf.Share.IMAdminUserID, nil, rdb, false, longServer.subscriberUserOnlineStatusChanges) - return nil + var err error + longServer.online, err = rpccache.NewOnlineCache(srv.userClient, nil, rdb, false, longServer.subscriberUserOnlineStatusChanges) + return err }) go longServer.ChangeOnlineStatus(4) diff --git a/internal/msggateway/message_handler.go b/internal/msggateway/message_handler.go index d88d2fbfd..9b59867d6 100644 --- a/internal/msggateway/message_handler.go +++ b/internal/msggateway/message_handler.go @@ -17,6 +17,7 @@ package msggateway import ( "context" "encoding/json" + "github.com/openimsdk/open-im-server/v3/pkg/rpcli" "sync" "github.com/go-playground/validator/v10" @@ -99,27 +100,33 @@ func (r *Resp) String() string { } type MessageHandler interface { - GetSeq(context context.Context, data *Req) ([]byte, error) - SendMessage(context context.Context, data *Req) ([]byte, error) - SendSignalMessage(context context.Context, data *Req) ([]byte, error) - PullMessageBySeqList(context context.Context, data *Req) ([]byte, error) - GetConversationsHasReadAndMaxSeq(context context.Context, data *Req) ([]byte, error) - GetSeqMessage(context context.Context, data *Req) ([]byte, error) - UserLogout(context context.Context, data *Req) ([]byte, error) - SetUserDeviceBackground(context context.Context, data *Req) ([]byte, bool, error) + GetSeq(ctx context.Context, data *Req) ([]byte, error) + SendMessage(ctx context.Context, data *Req) ([]byte, error) + SendSignalMessage(ctx context.Context, data *Req) ([]byte, error) + PullMessageBySeqList(ctx context.Context, data *Req) ([]byte, error) + GetConversationsHasReadAndMaxSeq(ctx context.Context, data *Req) ([]byte, error) + GetSeqMessage(ctx context.Context, data *Req) ([]byte, error) + UserLogout(ctx context.Context, data *Req) ([]byte, error) + SetUserDeviceBackground(ctx context.Context, data *Req) ([]byte, bool, error) } var _ MessageHandler = (*GrpcHandler)(nil) type GrpcHandler struct { - validate *validator.Validate + validate *validator.Validate + msgClient *rpcli.MsgClient + pushClient *rpcli.PushMsgServiceClient } -func NewGrpcHandler(validate *validator.Validate) *GrpcHandler { - return &GrpcHandler{validate: validate} +func NewGrpcHandler(validate *validator.Validate, msgClient *rpcli.MsgClient, pushClient *rpcli.PushMsgServiceClient) *GrpcHandler { + return &GrpcHandler{ + validate: validate, + msgClient: msgClient, + pushClient: pushClient, + } } -func (g GrpcHandler) GetSeq(ctx context.Context, data *Req) ([]byte, error) { +func (g *GrpcHandler) GetSeq(ctx context.Context, data *Req) ([]byte, error) { req := sdkws.GetMaxSeqReq{} if err := proto.Unmarshal(data.Data, &req); err != nil { return nil, errs.WrapMsg(err, "GetSeq: error unmarshaling request", "action", "unmarshal", "dataType", "GetMaxSeqReq") @@ -127,7 +134,7 @@ func (g GrpcHandler) GetSeq(ctx context.Context, data *Req) ([]byte, error) { if err := g.validate.Struct(&req); err != nil { return nil, errs.WrapMsg(err, "GetSeq: validation failed", "action", "validate", "dataType", "GetMaxSeqReq") } - resp, err := msg.GetMaxSeqCaller.Invoke(ctx, &req) + resp, err := g.msgClient.MsgClient.GetMaxSeq(ctx, &req) if err != nil { return nil, err } @@ -140,7 +147,7 @@ func (g GrpcHandler) GetSeq(ctx context.Context, data *Req) ([]byte, error) { // SendMessage handles the sending of messages through gRPC. It unmarshals the request data, // validates the message, and then sends it using the message RPC client. -func (g GrpcHandler) SendMessage(ctx context.Context, data *Req) ([]byte, error) { +func (g *GrpcHandler) SendMessage(ctx context.Context, data *Req) ([]byte, error) { var msgData sdkws.MsgData if err := proto.Unmarshal(data.Data, &msgData); err != nil { return nil, errs.WrapMsg(err, "SendMessage: error unmarshaling message data", "action", "unmarshal", "dataType", "MsgData") @@ -151,7 +158,7 @@ func (g GrpcHandler) SendMessage(ctx context.Context, data *Req) ([]byte, error) } req := msg.SendMsgReq{MsgData: &msgData} - resp, err := msg.SendMsgCaller.Invoke(ctx, &req) + resp, err := g.msgClient.MsgClient.SendMsg(ctx, &req) if err != nil { return nil, err } @@ -164,8 +171,8 @@ func (g GrpcHandler) SendMessage(ctx context.Context, data *Req) ([]byte, error) return c, nil } -func (g GrpcHandler) SendSignalMessage(context context.Context, data *Req) ([]byte, error) { - resp, err := msg.SendMsgCaller.Invoke(context, nil) +func (g *GrpcHandler) SendSignalMessage(ctx context.Context, data *Req) ([]byte, error) { + resp, err := g.msgClient.MsgClient.SendMsg(ctx, nil) if err != nil { return nil, err } @@ -176,7 +183,7 @@ func (g GrpcHandler) SendSignalMessage(context context.Context, data *Req) ([]by return c, nil } -func (g GrpcHandler) PullMessageBySeqList(ctx context.Context, data *Req) ([]byte, error) { +func (g *GrpcHandler) PullMessageBySeqList(ctx context.Context, data *Req) ([]byte, error) { req := sdkws.PullMessageBySeqsReq{} if err := proto.Unmarshal(data.Data, &req); err != nil { return nil, errs.WrapMsg(err, "err proto unmarshal", "action", "unmarshal", "dataType", "PullMessageBySeqsReq") @@ -184,7 +191,7 @@ func (g GrpcHandler) PullMessageBySeqList(ctx context.Context, data *Req) ([]byt if err := g.validate.Struct(data); err != nil { return nil, errs.WrapMsg(err, "validation failed", "action", "validate", "dataType", "PullMessageBySeqsReq") } - resp, err := msg.PullMessageBySeqsCaller.Invoke(ctx, &req) + resp, err := g.msgClient.MsgClient.PullMessageBySeqs(ctx, &req) if err != nil { return nil, err } @@ -195,7 +202,7 @@ func (g GrpcHandler) PullMessageBySeqList(ctx context.Context, data *Req) ([]byt return c, nil } -func (g GrpcHandler) GetConversationsHasReadAndMaxSeq(ctx context.Context, data *Req) ([]byte, error) { +func (g *GrpcHandler) GetConversationsHasReadAndMaxSeq(ctx context.Context, data *Req) ([]byte, error) { req := msg.GetConversationsHasReadAndMaxSeqReq{} if err := proto.Unmarshal(data.Data, &req); err != nil { return nil, errs.WrapMsg(err, "err proto unmarshal", "action", "unmarshal", "dataType", "GetConversationsHasReadAndMaxSeq") @@ -203,7 +210,7 @@ func (g GrpcHandler) GetConversationsHasReadAndMaxSeq(ctx context.Context, data if err := g.validate.Struct(data); err != nil { return nil, errs.WrapMsg(err, "validation failed", "action", "validate", "dataType", "GetConversationsHasReadAndMaxSeq") } - resp, err := msg.GetConversationsHasReadAndMaxSeqCaller.Invoke(ctx, &req) + resp, err := g.msgClient.MsgClient.GetConversationsHasReadAndMaxSeq(ctx, &req) if err != nil { return nil, err } @@ -214,7 +221,7 @@ func (g GrpcHandler) GetConversationsHasReadAndMaxSeq(ctx context.Context, data return c, nil } -func (g GrpcHandler) GetSeqMessage(ctx context.Context, data *Req) ([]byte, error) { +func (g *GrpcHandler) GetSeqMessage(ctx context.Context, data *Req) ([]byte, error) { req := msg.GetSeqMessageReq{} if err := proto.Unmarshal(data.Data, &req); err != nil { return nil, errs.WrapMsg(err, "error unmarshaling request", "action", "unmarshal", "dataType", "GetSeqMessage") @@ -222,7 +229,7 @@ func (g GrpcHandler) GetSeqMessage(ctx context.Context, data *Req) ([]byte, erro if err := g.validate.Struct(data); err != nil { return nil, errs.WrapMsg(err, "validation failed", "action", "validate", "dataType", "GetSeqMessage") } - resp, err := msg.GetSeqMessageCaller.Invoke(ctx, &req) + resp, err := g.msgClient.MsgClient.GetSeqMessage(ctx, &req) if err != nil { return nil, err } @@ -233,12 +240,12 @@ func (g GrpcHandler) GetSeqMessage(ctx context.Context, data *Req) ([]byte, erro return c, nil } -func (g GrpcHandler) UserLogout(ctx context.Context, data *Req) ([]byte, error) { +func (g *GrpcHandler) UserLogout(ctx context.Context, data *Req) ([]byte, error) { req := push.DelUserPushTokenReq{} if err := proto.Unmarshal(data.Data, &req); err != nil { return nil, errs.WrapMsg(err, "error unmarshaling request", "action", "unmarshal", "dataType", "DelUserPushTokenReq") } - resp, err := push.DelUserPushTokenCaller.Invoke(ctx, &req) + resp, err := g.pushClient.PushMsgServiceClient.DelUserPushToken(ctx, &req) if err != nil { return nil, err } @@ -249,7 +256,7 @@ func (g GrpcHandler) UserLogout(ctx context.Context, data *Req) ([]byte, error) return c, nil } -func (g GrpcHandler) SetUserDeviceBackground(_ context.Context, data *Req) ([]byte, bool, error) { +func (g *GrpcHandler) SetUserDeviceBackground(ctx context.Context, data *Req) ([]byte, bool, error) { req := sdkws.SetAppBackgroundStatusReq{} if err := proto.Unmarshal(data.Data, &req); err != nil { return nil, false, errs.WrapMsg(err, "error unmarshaling request", "action", "unmarshal", "dataType", "SetAppBackgroundStatusReq") diff --git a/internal/msggateway/online.go b/internal/msggateway/online.go index bff263997..52b6c5d05 100644 --- a/internal/msggateway/online.go +++ b/internal/msggateway/online.go @@ -88,7 +88,7 @@ func (ws *WsServer) ChangeOnlineStatus(concurrent int) { opIdCtx := mcontext.SetOperationID(context.Background(), operationIDPrefix+strconv.FormatInt(count.Add(1), 10)) ctx, cancel := context.WithTimeout(opIdCtx, time.Second*5) defer cancel() - if err := pbuser.SetUserOnlineStatusCaller.Execute(ctx, req); err != nil { + if err := ws.userClient.SetUserOnlineStatus(ctx, req); err != nil { log.ZError(ctx, "update user online status", err) } for _, ss := range req.Status { diff --git a/internal/msggateway/ws_server.go b/internal/msggateway/ws_server.go index 7271c3727..44b6ddb89 100644 --- a/internal/msggateway/ws_server.go +++ b/internal/msggateway/ws_server.go @@ -3,6 +3,7 @@ package msggateway import ( "context" "fmt" + "github.com/openimsdk/open-im-server/v3/pkg/rpcli" "net/http" "sync" "sync/atomic" @@ -31,7 +32,7 @@ type LongConnServer interface { GetUserAllCons(userID string) ([]*Client, bool) GetUserPlatformCons(userID string, platform int) ([]*Client, bool, bool) Validate(s any) error - SetDiscoveryRegistry(client discovery.SvcDiscoveryRegistry, config *Config) + SetDiscoveryRegistry(ctx context.Context, client discovery.SvcDiscoveryRegistry, config *Config) error KickUserConn(client *Client) error UnRegister(c *Client) SetKickHandlerInfo(i *kickHandler) @@ -61,6 +62,8 @@ type WsServer struct { //Encoder MessageHandler webhookClient *webhook.Client + userClient *rpcli.UserClient + authClient *rpcli.AuthClient } type kickHandler struct { @@ -69,9 +72,28 @@ type kickHandler struct { newClient *Client } -func (ws *WsServer) SetDiscoveryRegistry(disCov discovery.SvcDiscoveryRegistry, config *Config) { - ws.MessageHandler = NewGrpcHandler(ws.validate) +func (ws *WsServer) SetDiscoveryRegistry(ctx context.Context, disCov discovery.SvcDiscoveryRegistry, config *Config) error { + userConn, err := disCov.GetConn(ctx, config.Discovery.RpcService.User) + if err != nil { + return err + } + pushConn, err := disCov.GetConn(ctx, config.Discovery.RpcService.Push) + if err != nil { + return err + } + authConn, err := disCov.GetConn(ctx, config.Discovery.RpcService.Auth) + if err != nil { + return err + } + msgConn, err := disCov.GetConn(ctx, config.Discovery.RpcService.Msg) + if err != nil { + return err + } + ws.userClient = rpcli.NewUserClient(userConn) + ws.authClient = rpcli.NewAuthClient(authConn) + ws.MessageHandler = NewGrpcHandler(ws.validate, rpcli.NewMsgClient(msgConn), rpcli.NewPushMsgServiceClient(pushConn)) ws.disCov = disCov + return nil } //func (ws *WsServer) SetUserOnlineStatus(ctx context.Context, client *Client, status int32) { @@ -306,8 +328,7 @@ func (ws *WsServer) multiTerminalLoginChecker(clientOK bool, oldClients []*Clien []string{newClient.ctx.GetOperationID(), newClient.ctx.GetUserID(), constant.PlatformIDToName(newClient.PlatformID), newClient.ctx.GetConnID()}, ) - - if err := pbAuth.KickTokensCaller.Execute(ctx, &pbAuth.KickTokensReq{Tokens: kickTokens}); err != nil { + if err := ws.authClient.KickTokens(ctx, kickTokens); err != nil { log.ZWarn(newClient.ctx, "kickTokens err", err) } } @@ -334,11 +355,12 @@ func (ws *WsServer) multiTerminalLoginChecker(clientOK bool, oldClients []*Clien []string{newClient.ctx.GetOperationID(), newClient.ctx.GetUserID(), constant.PlatformIDToName(newClient.PlatformID), newClient.ctx.GetConnID()}, ) - if err := pbAuth.InvalidateTokenCaller.Execute(ctx, &pbAuth.InvalidateTokenReq{ + req := &pbAuth.InvalidateTokenReq{ PreservedToken: newClient.token, UserID: newClient.UserID, PlatformID: int32(newClient.PlatformID), - }); err != nil { + } + if err := ws.authClient.InvalidateToken(ctx, req); err != nil { log.ZWarn(newClient.ctx, "InvalidateToken err", err, "userID", newClient.UserID, "platformID", newClient.PlatformID) } @@ -409,7 +431,7 @@ func (ws *WsServer) wsHandler(w http.ResponseWriter, r *http.Request) { } // Call the authentication client to parse the Token obtained from the context - resp, err := pbAuth.ParseTokenCaller.Invoke(connContext, &pbAuth.ParseTokenReq{Token: connContext.GetToken()}) + resp, err := ws.authClient.ParseToken(connContext, connContext.GetToken()) if err != nil { // If there's an error parsing the Token, decide whether to send the error message via WebSocket based on the context flag shouldSendError := connContext.ShouldSendResp() diff --git a/internal/msgtransfer/init.go b/internal/msgtransfer/init.go index 19a53ebd5..5cb613123 100644 --- a/internal/msgtransfer/init.go +++ b/internal/msgtransfer/init.go @@ -25,7 +25,6 @@ import ( "strconv" "syscall" - "github.com/openimsdk/open-im-server/v3/pkg/rpcclient" "github.com/openimsdk/tools/discovery" "github.com/openimsdk/tools/discovery/etcd" "github.com/openimsdk/tools/utils/jsonutil" @@ -93,10 +92,6 @@ func Start(ctx context.Context, index int, config *Config) error { } client.AddOption(mw.GrpcClient(), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithDefaultServiceConfig(fmt.Sprintf(`{"LoadBalancingPolicy": "%s"}`, "round_robin"))) - if err = rpcclient.InitRpcCaller(client, config.Discovery.RpcService); err != nil { - return err - } - msgModel := redis.NewMsgCache(rdb) msgDocModel, err := mgo.NewMsgMongo(mgocli.GetDB()) if err != nil { @@ -116,7 +111,7 @@ func Start(ctx context.Context, index int, config *Config) error { if err != nil { return err } - historyCH, err := NewOnlineHistoryRedisConsumerHandler(&config.KafkaConfig, msgTransferDatabase) + historyCH, err := NewOnlineHistoryRedisConsumerHandler(ctx, client, config, msgTransferDatabase) if err != nil { return err } diff --git a/internal/msgtransfer/online_history_msg_handler.go b/internal/msgtransfer/online_history_msg_handler.go index 9287d6b61..83b075061 100644 --- a/internal/msgtransfer/online_history_msg_handler.go +++ b/internal/msgtransfer/online_history_msg_handler.go @@ -18,6 +18,8 @@ import ( "context" "encoding/json" "errors" + "github.com/openimsdk/open-im-server/v3/pkg/rpcli" + "github.com/openimsdk/tools/discovery" "strconv" "strings" "sync" @@ -25,15 +27,12 @@ import ( "github.com/IBM/sarama" "github.com/go-redis/redis" - "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/open-im-server/v3/pkg/common/prommetrics" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/controller" "github.com/openimsdk/open-im-server/v3/pkg/msgprocessor" "github.com/openimsdk/open-im-server/v3/pkg/tools/batcher" "github.com/openimsdk/protocol/constant" pbconv "github.com/openimsdk/protocol/conversation" - "github.com/openimsdk/protocol/group" - "github.com/openimsdk/protocol/rpccall" "github.com/openimsdk/protocol/sdkws" "github.com/openimsdk/tools/errs" "github.com/openimsdk/tools/log" @@ -72,16 +71,30 @@ type OnlineHistoryRedisConsumerHandler struct { msgTransferDatabase controller.MsgTransferDatabase conversationUserHasReadChan chan *userHasReadSeq wg sync.WaitGroup + + groupClient *rpcli.GroupClient + conversationClient *rpcli.ConversationClient } -func NewOnlineHistoryRedisConsumerHandler(kafkaConf *config.Kafka, database controller.MsgTransferDatabase) (*OnlineHistoryRedisConsumerHandler, error) { +func NewOnlineHistoryRedisConsumerHandler(ctx context.Context, client discovery.SvcDiscoveryRegistry, config *Config, database controller.MsgTransferDatabase) (*OnlineHistoryRedisConsumerHandler, error) { + kafkaConf := config.KafkaConfig historyConsumerGroup, err := kafka.NewMConsumerGroup(kafkaConf.Build(), kafkaConf.ToRedisGroupID, []string{kafkaConf.ToRedisTopic}, false) if err != nil { return nil, err } + groupConn, err := client.GetConn(ctx, config.Discovery.RpcService.Group) + if err != nil { + return nil, err + } + conversationConn, err := client.GetConn(ctx, config.Discovery.RpcService.Conversation) + if err != nil { + return nil, err + } var och OnlineHistoryRedisConsumerHandler och.msgTransferDatabase = database och.conversationUserHasReadChan = make(chan *userHasReadSeq, hasReadChanBuffer) + och.groupClient = rpcli.NewGroupClient(groupConn) + och.conversationClient = rpcli.NewConversationClient(conversationConn) och.wg.Add(1) b := batcher.New[sarama.ConsumerMessage]( @@ -109,15 +122,13 @@ func (och *OnlineHistoryRedisConsumerHandler) do(ctx context.Context, channelID ctx = mcontext.WithTriggerIDContext(ctx, val.TriggerID()) ctxMessages := och.parseConsumerMessages(ctx, val.Val()) ctx = withAggregationCtx(ctx, ctxMessages) - log.ZInfo(ctx, "msg arrived channel", "channel id", channelID, "msgList length", len(ctxMessages), - "key", val.Key()) + log.ZInfo(ctx, "msg arrived channel", "channel id", channelID, "msgList length", len(ctxMessages), "key", val.Key()) och.doSetReadSeq(ctx, ctxMessages) storageMsgList, notStorageMsgList, storageNotificationList, notStorageNotificationList := och.categorizeMessageLists(ctxMessages) log.ZDebug(ctx, "number of categorized messages", "storageMsgList", len(storageMsgList), "notStorageMsgList", - len(notStorageMsgList), "storageNotificationList", len(storageNotificationList), "notStorageNotificationList", - len(notStorageNotificationList)) + len(notStorageMsgList), "storageNotificationList", len(storageNotificationList), "notStorageNotificationList", len(notStorageNotificationList)) conversationIDMsg := msgprocessor.GetChatConversationIDByMsg(ctxMessages[0].message) conversationIDNotification := msgprocessor.GetNotificationConversationIDByMsg(ctxMessages[0].message) @@ -282,31 +293,26 @@ func (och *OnlineHistoryRedisConsumerHandler) handleMsg(ctx context.Context, key log.ZDebug(ctx, "group chat first create conversation", "conversationID", conversationID) - userIDs, err := rpccall.ExtractField(ctx, group.GetGroupMemberUserIDsCaller.Invoke, - &group.GetGroupMemberUserIDsReq{ - GroupID: msg.GroupID, - }, (*group.GetGroupMemberUserIDsResp).GetUserIDs) + userIDs, err := och.groupClient.GetGroupMemberUserIDs(ctx, msg.GroupID) if err != nil { log.ZWarn(ctx, "get group member ids error", err, "conversationID", conversationID) } else { log.ZInfo(ctx, "GetGroupMemberIDs end") - if err := pbconv.CreateGroupChatConversationsCaller.Execute(ctx, &pbconv.CreateGroupChatConversationsReq{ - UserIDs: userIDs, - GroupID: msg.GroupID, - }); err != nil { + if err := och.conversationClient.CreateGroupChatConversations(ctx, msg.GroupID, userIDs); err != nil { log.ZWarn(ctx, "single chat first create conversation error", err, "conversationID", conversationID) } } case constant.SingleChatType, constant.NotificationChatType: - if err := pbconv.CreateSingleChatConversationsCaller.Execute(ctx, &pbconv.CreateSingleChatConversationsReq{ + req := &pbconv.CreateSingleChatConversationsReq{ RecvID: msg.RecvID, SendID: msg.SendID, ConversationID: conversationID, ConversationType: msg.SessionType, - }); err != nil { + } + if err := och.conversationClient.CreateSingleChatConversations(ctx, req); err != nil { log.ZWarn(ctx, "single chat or notification first create conversation error", err, "conversationID", conversationID, "sessionType", msg.SessionType) } diff --git a/internal/push/push.go b/internal/push/push.go index 3247426e9..b7c1ec427 100644 --- a/internal/push/push.go +++ b/internal/push/push.go @@ -37,11 +37,6 @@ type Config struct { runTimeEnv string } -func (p pushServer) PushMsg(ctx context.Context, req *pbpush.PushMsgReq) (*pbpush.PushMsgResp, error) { - //todo reserved Interface - return nil, nil -} - func (p pushServer) DelUserPushToken(ctx context.Context, req *pbpush.DelUserPushTokenReq) (resp *pbpush.DelUserPushTokenResp, err error) { if err = p.database.DelFcmToken(ctx, req.UserID, int(req.PlatformID)); err != nil { @@ -65,7 +60,7 @@ func Start(ctx context.Context, config *Config, client discovery.SvcDiscoveryReg database := controller.NewPushDatabase(cacheModel, &config.KafkaConfig) - consumer, err := NewConsumerHandler(config, database, offlinePusher, rdb, client) + consumer, err := NewConsumerHandler(ctx, config, database, offlinePusher, rdb, client) if err != nil { return err } diff --git a/internal/push/push_handler.go b/internal/push/push_handler.go index ee855e122..d5d457c0d 100644 --- a/internal/push/push_handler.go +++ b/internal/push/push_handler.go @@ -3,6 +3,7 @@ package push import ( "context" "encoding/json" + "github.com/openimsdk/open-im-server/v3/pkg/rpcli" "math/rand" "strconv" "time" @@ -17,12 +18,8 @@ import ( "github.com/openimsdk/open-im-server/v3/pkg/rpccache" "github.com/openimsdk/open-im-server/v3/pkg/util/conversationutil" "github.com/openimsdk/protocol/constant" - pbconv "github.com/openimsdk/protocol/conversation" - "github.com/openimsdk/protocol/group" - "github.com/openimsdk/protocol/msg" "github.com/openimsdk/protocol/msggateway" pbpush "github.com/openimsdk/protocol/push" - "github.com/openimsdk/protocol/rpccall" "github.com/openimsdk/protocol/sdkws" "github.com/openimsdk/tools/discovery" "github.com/openimsdk/tools/log" @@ -45,9 +42,13 @@ type ConsumerHandler struct { conversationLocalCache *rpccache.ConversationLocalCache webhookClient *webhook.Client config *Config + userClient *rpcli.UserClient + groupClient *rpcli.GroupClient + msgClient *rpcli.MsgClient + conversationClient *rpcli.ConversationClient } -func NewConsumerHandler(config *Config, database controller.PushDatabase, offlinePusher offlinepush.OfflinePusher, rdb redis.UniversalClient, +func NewConsumerHandler(ctx context.Context, config *Config, database controller.PushDatabase, offlinePusher offlinepush.OfflinePusher, rdb redis.UniversalClient, client discovery.SvcDiscoveryRegistry) (*ConsumerHandler, error) { var consumerHandler ConsumerHandler var err error @@ -56,15 +57,35 @@ func NewConsumerHandler(config *Config, database controller.PushDatabase, offlin if err != nil { return nil, err } + userConn, err := client.GetConn(ctx, config.Discovery.RpcService.User) + if err != nil { + return nil, err + } + groupConn, err := client.GetConn(ctx, config.Discovery.RpcService.Group) + if err != nil { + return nil, err + } + msgConn, err := client.GetConn(ctx, config.Discovery.RpcService.Msg) + if err != nil { + return nil, err + } + conversationConn, err := client.GetConn(ctx, config.Discovery.RpcService.Conversation) + if err != nil { + return nil, err + } + consumerHandler.userClient = rpcli.NewUserClient(userConn) + consumerHandler.groupClient = rpcli.NewGroupClient(groupConn) + consumerHandler.msgClient = rpcli.NewMsgClient(msgConn) + consumerHandler.conversationClient = rpcli.NewConversationClient(conversationConn) consumerHandler.offlinePusher = offlinePusher consumerHandler.onlinePusher = NewOnlinePusher(client, config) - consumerHandler.groupLocalCache = rpccache.NewGroupLocalCache(&config.LocalCacheConfig, rdb) - consumerHandler.conversationLocalCache = rpccache.NewConversationLocalCache(&config.LocalCacheConfig, rdb) + consumerHandler.groupLocalCache = rpccache.NewGroupLocalCache(consumerHandler.groupClient, &config.LocalCacheConfig, rdb) + consumerHandler.conversationLocalCache = rpccache.NewConversationLocalCache(consumerHandler.conversationClient, &config.LocalCacheConfig, rdb) consumerHandler.webhookClient = webhook.NewWebhookClient(config.WebhooksConfig.URL) consumerHandler.config = config consumerHandler.pushDatabase = database - consumerHandler.onlineCache, err = rpccache.NewOnlineCache(config.Share.IMAdminUserID, consumerHandler.groupLocalCache, rdb, config.RpcConfig.FullUserCache, nil) + consumerHandler.onlineCache, err = rpccache.NewOnlineCache(consumerHandler.userClient, consumerHandler.groupLocalCache, rdb, config.RpcConfig.FullUserCache, nil) if err != nil { return nil, err } @@ -321,7 +342,7 @@ func (c *ConsumerHandler) groupMessagesHandler(ctx context.Context, groupID stri ctx = mcontext.WithOpUserIDContext(ctx, c.config.Share.IMAdminUserID[0]) } defer func(groupID string) { - if err = group.DismissGroupCaller.Execute(ctx, &group.DismissGroupReq{GroupID: groupID}); err != nil { + if err := c.groupClient.DismissGroup(ctx, groupID, true); err != nil { log.ZError(ctx, "DismissGroup Notification clear members", err, "groupID", groupID) } }(groupID) @@ -347,12 +368,7 @@ func (c *ConsumerHandler) offlinePushMsg(ctx context.Context, msg *sdkws.MsgData func (c *ConsumerHandler) filterGroupMessageOfflinePush(ctx context.Context, groupID string, msg *sdkws.MsgData, offlinePushUserIDs []string) (userIDs []string, err error) { - - //todo local cache Obtain the difference set through local comparison. - needOfflinePushUserIDs, err := rpccall.ExtractField(ctx, pbconv.GetConversationOfflinePushUserIDsCaller.Invoke, &pbconv.GetConversationOfflinePushUserIDsReq{ - ConversationID: conversationutil.GenGroupConversationID(groupID), - UserIDs: offlinePushUserIDs, - }, (*pbconv.GetConversationOfflinePushUserIDsResp).GetUserIDs) + needOfflinePushUserIDs, err := c.conversationClient.GetConversationOfflinePushUserIDs(ctx, conversationutil.GenGroupConversationID(groupID), offlinePushUserIDs) if err != nil { return nil, err } @@ -406,18 +422,11 @@ func (c *ConsumerHandler) getOfflinePushInfos(msg *sdkws.MsgData) (title, conten func (c *ConsumerHandler) DeleteMemberAndSetConversationSeq(ctx context.Context, groupID string, userIDs []string) error { conversationID := msgprocessor.GetConversationIDBySessionType(constant.ReadGroupChatType, groupID) - maxSeq, err := rpccall.ExtractField(ctx, msg.GetConversationMaxSeqCaller.Invoke, - &msg.GetConversationMaxSeqReq{ConversationID: conversationID}, - (*msg.GetConversationMaxSeqResp).GetMaxSeq) + maxSeq, err := c.msgClient.GetConversationMaxSeq(ctx, conversationID) if err != nil { return err } - - return pbconv.SetConversationMaxSeqCaller.Execute(ctx, &pbconv.SetConversationMaxSeqReq{ - ConversationID: conversationID, - OwnerUserID: userIDs, - MaxSeq: maxSeq, - }) + return c.conversationClient.SetConversationMaxSeq(ctx, conversationID, userIDs, maxSeq) } func unmarshalNotificationElem(bytes []byte, t any) error { @@ -425,6 +434,5 @@ func unmarshalNotificationElem(bytes []byte, t any) error { if err := json.Unmarshal(bytes, ¬ification); err != nil { return err } - return json.Unmarshal([]byte(notification.Detail), t) } diff --git a/internal/rpc/auth/auth.go b/internal/rpc/auth/auth.go index c220863c6..a2b14f40d 100644 --- a/internal/rpc/auth/auth.go +++ b/internal/rpc/auth/auth.go @@ -17,6 +17,7 @@ package auth import ( "context" "errors" + "github.com/openimsdk/open-im-server/v3/pkg/rpcli" "github.com/openimsdk/open-im-server/v3/pkg/common/config" redis2 "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/redis" @@ -28,7 +29,6 @@ import ( "github.com/openimsdk/open-im-server/v3/pkg/common/prommetrics" "github.com/openimsdk/open-im-server/v3/pkg/common/servererrs" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/controller" - "github.com/openimsdk/open-im-server/v3/pkg/rpcclient" pbauth "github.com/openimsdk/protocol/auth" "github.com/openimsdk/protocol/constant" "github.com/openimsdk/protocol/msggateway" @@ -44,6 +44,7 @@ type authServer struct { authDatabase controller.AuthDatabase RegisterCenter discovery.SvcDiscoveryRegistry config *Config + userClient *rpcli.UserClient } type Config struct { @@ -58,6 +59,10 @@ func Start(ctx context.Context, config *Config, client discovery.SvcDiscoveryReg if err != nil { return err } + userConn, err := client.GetConn(ctx, config.Discovery.RpcService.User) + if err != nil { + return err + } pbauth.RegisterAuthServer(server, &authServer{ RegisterCenter: client, authDatabase: controller.NewAuthDatabase( @@ -67,7 +72,8 @@ func Start(ctx context.Context, config *Config, client discovery.SvcDiscoveryReg config.Share.MultiLogin, config.Share.IMAdminUserID, ), - config: config, + config: config, + userClient: rpcli.NewUserClient(userConn), }) return nil } @@ -83,7 +89,7 @@ func (s *authServer) GetAdminToken(ctx context.Context, req *pbauth.GetAdminToke } - if _, err := rpcclient.GetUserInfo(ctx, req.UserID); err != nil { + if err := s.userClient.CheckUser(ctx, []string{req.UserID}); err != nil { return nil, err } @@ -112,7 +118,7 @@ func (s *authServer) GetUserToken(ctx context.Context, req *pbauth.GetUserTokenR if authverify.IsManagerUserID(req.UserID, s.config.Share.IMAdminUserID) { return nil, errs.ErrNoPermission.WrapMsg("don't get Admin token") } - if _, err := rpcclient.GetUserInfo(ctx, req.UserID); err != nil { + if err := s.userClient.CheckUser(ctx, []string{req.UserID}); err != nil { return nil, err } token, err := s.authDatabase.CreateToken(ctx, req.UserID, int(req.PlatformID)) @@ -153,10 +159,7 @@ func (s *authServer) parseToken(ctx context.Context, tokensString string) (claim return nil, servererrs.ErrTokenNotExist.Wrap() } -func (s *authServer) ParseToken( - ctx context.Context, - req *pbauth.ParseTokenReq, -) (resp *pbauth.ParseTokenResp, err error) { +func (s *authServer) ParseToken(ctx context.Context, req *pbauth.ParseTokenReq) (resp *pbauth.ParseTokenResp, err error) { resp = &pbauth.ParseTokenResp{} claims, err := s.parseToken(ctx, req.Token) if err != nil { diff --git a/internal/rpc/conversation/conversation.go b/internal/rpc/conversation/conversation.go index 696ada152..53364ff86 100644 --- a/internal/rpc/conversation/conversation.go +++ b/internal/rpc/conversation/conversation.go @@ -16,6 +16,7 @@ package conversation import ( "context" + "github.com/openimsdk/open-im-server/v3/pkg/rpcli" "sort" "time" @@ -26,15 +27,11 @@ import ( dbModel "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" "github.com/openimsdk/open-im-server/v3/pkg/localcache" "github.com/openimsdk/open-im-server/v3/pkg/msgprocessor" - pbgroup "github.com/openimsdk/protocol/group" - pbmsg "github.com/openimsdk/protocol/msg" - "github.com/openimsdk/protocol/rpccall" "github.com/openimsdk/tools/db/redisutil" "github.com/openimsdk/open-im-server/v3/pkg/common/convert" "github.com/openimsdk/open-im-server/v3/pkg/common/servererrs" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/controller" - "github.com/openimsdk/open-im-server/v3/pkg/rpcclient" "github.com/openimsdk/protocol/constant" pbconversation "github.com/openimsdk/protocol/conversation" "github.com/openimsdk/protocol/sdkws" @@ -52,6 +49,10 @@ type conversationServer struct { conversationNotificationSender *ConversationNotificationSender config *Config + + userClient *rpcli.UserClient + msgClient *rpcli.MsgClient + groupClient *rpcli.GroupClient } type Config struct { @@ -77,11 +78,27 @@ func Start(ctx context.Context, config *Config, client discovery.SvcDiscoveryReg if err != nil { return err } + userConn, err := client.GetConn(ctx, config.Discovery.RpcService.User) + if err != nil { + return err + } + groupConn, err := client.GetConn(ctx, config.Discovery.RpcService.Group) + if err != nil { + return err + } + msgConn, err := client.GetConn(ctx, config.Discovery.RpcService.Msg) + if err != nil { + return err + } + msgClient := rpcli.NewMsgClient(msgConn) localcache.InitLocalCache(&config.LocalCacheConfig) pbconversation.RegisterConversationServer(server, &conversationServer{ - conversationNotificationSender: NewConversationNotificationSender(&config.NotificationConfig), + conversationNotificationSender: NewConversationNotificationSender(&config.NotificationConfig, msgClient), conversationDatabase: controller.NewConversationDatabase(conversationDB, redis.NewConversationRedis(rdb, &config.LocalCacheConfig, redis.GetRocksCacheOptions(), conversationDB), mgocli.GetTx()), + userClient: rpcli.NewUserClient(userConn), + groupClient: rpcli.NewGroupClient(groupConn), + msgClient: msgClient, }) return nil } @@ -118,19 +135,12 @@ func (c *conversationServer) GetSortedConversationList(ctx context.Context, req if len(conversations) == 0 { return nil, errs.ErrRecordNotFound.Wrap() } - - maxSeqs, err := rpccall.ExtractField(ctx, pbmsg.GetMaxSeqsCaller.Invoke, - &pbmsg.GetMaxSeqsReq{ConversationIDs: conversationIDs}, - (*pbmsg.SeqsInfoResp).GetMaxSeqs) + maxSeqs, err := c.msgClient.GetMaxSeqs(ctx, conversationIDs) if err != nil { return nil, err } - chatLogs, err := rpccall.ExtractField(ctx, pbmsg.GetMsgByConversationIDsCaller.Invoke, - &pbmsg.GetMsgByConversationIDsReq{ - ConversationIDs: conversationIDs, - MaxSeqs: maxSeqs, - }, (*pbmsg.GetMsgByConversationIDsResp).GetMsgDatas) + chatLogs, err := c.msgClient.GetMsgByConversationIDs(ctx, conversationIDs, maxSeqs) if err != nil { return nil, err } @@ -140,9 +150,7 @@ func (c *conversationServer) GetSortedConversationList(ctx context.Context, req return nil, err } - hasReadSeqs, err := rpccall.ExtractField(ctx, pbmsg.GetHasReadSeqsCaller.Invoke, - &pbmsg.GetHasReadSeqsReq{ConversationIDs: conversationIDs}, - (*pbmsg.SeqsInfoResp).GetMaxSeqs) + hasReadSeqs, err := c.msgClient.GetHasReadSeqs(ctx, conversationIDs, req.UserID) if err != nil { return nil, err } @@ -230,14 +238,7 @@ func (c *conversationServer) SetConversations(ctx context.Context, req *pbconver return nil, errs.ErrArgs.WrapMsg("conversation must not be nil") } if req.Conversation.ConversationType == constant.WriteGroupChatType { - groupInfo, err := rpccall.ExtractField(ctx, pbgroup.GetGroupsInfoCaller.Invoke, - &pbgroup.GetGroupsInfoReq{GroupIDs: []string{req.Conversation.GroupID}}, - func(r *pbgroup.GetGroupsInfoResp) *sdkws.GroupInfo { - if len(r.GroupInfos) > 0 { - return r.GroupInfos[0] - } - return nil - }) + groupInfo, err := c.groupClient.GetGroupInfo(ctx, req.Conversation.GroupID) if err != nil { return nil, err } @@ -444,14 +445,14 @@ func (c *conversationServer) CreateGroupChatConversations(ctx context.Context, r return nil, err } conversationID := msgprocessor.GetConversationIDBySessionType(constant.ReadGroupChatType, req.GroupID) - if _, err := pbmsg.SetUserConversationMaxSeqCaller.Invoke(ctx, &pbmsg.SetUserConversationMaxSeqReq{ConversationID: conversationID, OwnerUserID: req.UserIDs, MaxSeq: 0}); err != nil { + if err := c.msgClient.SetUserConversationMaxSeq(ctx, conversationID, req.UserIDs, 0); err != nil { return nil, err } return &pbconversation.CreateGroupChatConversationsResp{}, nil } func (c *conversationServer) SetConversationMaxSeq(ctx context.Context, req *pbconversation.SetConversationMaxSeqReq) (*pbconversation.SetConversationMaxSeqResp, error) { - if _, err := pbmsg.SetUserConversationMaxSeqCaller.Invoke(ctx, &pbmsg.SetUserConversationMaxSeqReq{ConversationID: req.ConversationID, OwnerUserID: req.OwnerUserID, MaxSeq: req.MaxSeq}); err != nil { + if err := c.msgClient.SetUserConversationMaxSeq(ctx, req.ConversationID, req.OwnerUserID, req.MaxSeq); err != nil { return nil, err } if err := c.conversationDatabase.UpdateUsersConversationField(ctx, req.OwnerUserID, req.ConversationID, @@ -465,7 +466,7 @@ func (c *conversationServer) SetConversationMaxSeq(ctx context.Context, req *pbc } func (c *conversationServer) SetConversationMinSeq(ctx context.Context, req *pbconversation.SetConversationMinSeqReq) (*pbconversation.SetConversationMinSeqResp, error) { - if _, err := pbmsg.SetUserConversationMinSeqCaller.Invoke(ctx, &pbmsg.SetUserConversationMinSeqReq{ConversationID: req.ConversationID, OwnerUserID: req.OwnerUserID, MinSeq: req.MinSeq}); err != nil { + if err := c.msgClient.SetUserConversationMin(ctx, req.ConversationID, req.OwnerUserID, req.MinSeq); err != nil { return nil, err } if err := c.conversationDatabase.UpdateUsersConversationField(ctx, req.OwnerUserID, req.ConversationID, @@ -575,7 +576,7 @@ func (c *conversationServer) getConversationInfo( } } if len(sendIDs) != 0 { - sendInfos, err := rpcclient.GetUsersInfo(ctx, sendIDs) + sendInfos, err := c.userClient.GetUsersInfo(ctx, sendIDs) if err != nil { return nil, err } @@ -584,9 +585,7 @@ func (c *conversationServer) getConversationInfo( } } if len(groupIDs) != 0 { - groupInfos, err := rpccall.ExtractField(ctx, pbgroup.GetGroupsInfoCaller.Invoke, - &pbgroup.GetGroupsInfoReq{GroupIDs: groupIDs}, - (*pbgroup.GetGroupsInfoResp).GetGroupInfos) + groupInfos, err := c.groupClient.GetGroupsInfo(ctx, groupIDs) if err != nil { return nil, err } @@ -774,23 +773,22 @@ func (c *conversationServer) ClearUserConversationMsg(ctx context.Context, req * if conversation.IsMsgDestruct == false || conversation.MsgDestructTime == 0 { continue } - rcpReq := &pbmsg.GetLastMessageSeqByTimeReq{ConversationID: conversation.ConversationID, Time: req.Timestamp - conversation.MsgDestructTime} - resp, err := pbmsg.GetLastMessageSeqByTime.Invoke(ctx, rcpReq) + seq, err := c.msgClient.GetLastMessageSeqByTime(ctx, conversation.ConversationID, req.Timestamp-conversation.MsgDestructTime) if err != nil { return nil, err } - if resp.Seq <= 0 { - log.ZDebug(ctx, "ClearUserConversationMsg GetLastMessageSeqByTime seq <= 0", "index", i, "conversationID", conversation.ConversationID, "ownerUserID", conversation.OwnerUserID, "msgDestructTime", conversation.MsgDestructTime, "seq", resp.Seq) + if seq <= 0 { + log.ZDebug(ctx, "ClearUserConversationMsg GetLastMessageSeqByTime seq <= 0", "index", i, "conversationID", conversation.ConversationID, "ownerUserID", conversation.OwnerUserID, "msgDestructTime", conversation.MsgDestructTime, "seq", seq) if err := c.setConversationMinSeqAndLatestMsgDestructTime(ctx, conversation.ConversationID, conversation.OwnerUserID, -1, latestMsgDestructTime); err != nil { return nil, err } continue } - resp.Seq++ - if err := c.setConversationMinSeqAndLatestMsgDestructTime(ctx, conversation.ConversationID, conversation.OwnerUserID, resp.Seq, latestMsgDestructTime); err != nil { + seq++ + if err := c.setConversationMinSeqAndLatestMsgDestructTime(ctx, conversation.ConversationID, conversation.OwnerUserID, seq, latestMsgDestructTime); err != nil { return nil, err } - log.ZDebug(ctx, "ClearUserConversationMsg set min seq", "index", i, "conversationID", conversation.ConversationID, "ownerUserID", conversation.OwnerUserID, "seq", resp.Seq, "msgDestructTime", conversation.MsgDestructTime) + log.ZDebug(ctx, "ClearUserConversationMsg set min seq", "index", i, "conversationID", conversation.ConversationID, "ownerUserID", conversation.OwnerUserID, "seq", seq, "msgDestructTime", conversation.MsgDestructTime) } return &pbconversation.ClearUserConversationMsgResp{Count: int32(len(conversations))}, nil } @@ -800,8 +798,7 @@ func (c *conversationServer) setConversationMinSeqAndLatestMsgDestructTime(ctx c "latest_msg_destruct_time": latestMsgDestructTime, } if minSeq >= 0 { - req := &pbmsg.SetUserConversationMinSeqReq{ConversationID: conversationID, OwnerUserID: []string{ownerUserID}, MinSeq: minSeq} - if _, err := pbmsg.SetUserConversationMinSeqCaller.Invoke(ctx, req); err != nil { + if err := c.msgClient.SetUserConversationMin(ctx, conversationID, []string{ownerUserID}, minSeq); err != nil { return err } update["min_seq"] = minSeq diff --git a/internal/rpc/conversation/notification.go b/internal/rpc/conversation/notification.go index f94c0cd07..c6368b916 100644 --- a/internal/rpc/conversation/notification.go +++ b/internal/rpc/conversation/notification.go @@ -16,9 +16,11 @@ package conversation import ( "context" + "github.com/openimsdk/open-im-server/v3/pkg/rpcli" + "github.com/openimsdk/protocol/msg" "github.com/openimsdk/open-im-server/v3/pkg/common/config" - "github.com/openimsdk/open-im-server/v3/pkg/rpcclient" + "github.com/openimsdk/open-im-server/v3/pkg/notification" "github.com/openimsdk/protocol/constant" "github.com/openimsdk/protocol/sdkws" ) @@ -27,8 +29,10 @@ type ConversationNotificationSender struct { *rpcclient.NotificationSender } -func NewConversationNotificationSender(conf *config.Notification) *ConversationNotificationSender { - return &ConversationNotificationSender{rpcclient.NewNotificationSender(conf, rpcclient.WithRpcClient())} +func NewConversationNotificationSender(conf *config.Notification, msgClient *rpcli.MsgClient) *ConversationNotificationSender { + return &ConversationNotificationSender{rpcclient.NewNotificationSender(conf, rpcclient.WithRpcClient(func(ctx context.Context, req *msg.SendMsgReq) (*msg.SendMsgResp, error) { + return msgClient.SendMsg(ctx, req) + }))} } // SetPrivate invote. diff --git a/internal/rpc/group/group.go b/internal/rpc/group/group.go index 8af09b4c5..bea0e1af4 100644 --- a/internal/rpc/group/group.go +++ b/internal/rpc/group/group.go @@ -17,6 +17,7 @@ package group import ( "context" "fmt" + "github.com/openimsdk/open-im-server/v3/pkg/rpcli" "math/big" "math/rand" "strconv" @@ -35,14 +36,10 @@ import ( "github.com/openimsdk/open-im-server/v3/pkg/common/webhook" "github.com/openimsdk/open-im-server/v3/pkg/localcache" "github.com/openimsdk/open-im-server/v3/pkg/msgprocessor" - "github.com/openimsdk/open-im-server/v3/pkg/rpcclient" - "github.com/openimsdk/open-im-server/v3/pkg/rpcclient/grouphash" - "github.com/openimsdk/open-im-server/v3/pkg/rpcclient/notification" + "github.com/openimsdk/open-im-server/v3/pkg/notification/grouphash" "github.com/openimsdk/protocol/constant" pbconv "github.com/openimsdk/protocol/conversation" pbgroup "github.com/openimsdk/protocol/group" - "github.com/openimsdk/protocol/msg" - "github.com/openimsdk/protocol/rpccall" "github.com/openimsdk/protocol/sdkws" "github.com/openimsdk/protocol/wrapperspb" "github.com/openimsdk/tools/db/mongoutil" @@ -59,10 +56,13 @@ import ( type groupServer struct { pbgroup.UnimplementedGroupServer - db controller.GroupDatabase - notification *GroupNotificationSender - config *Config - webhookClient *webhook.Client + db controller.GroupDatabase + notification *NotificationSender + config *Config + webhookClient *webhook.Client + userClient *rpcli.UserClient + msgClient *rpcli.MsgClient + conversationClient *rpcli.ConversationClient } type Config struct { @@ -97,24 +97,33 @@ func Start(ctx context.Context, config *Config, client discovery.SvcDiscoveryReg if err != nil { return err } - var gs groupServer - database := controller.NewGroupDatabase(rdb, &config.LocalCacheConfig, groupDB, groupMemberDB, groupRequestDB, mgocli.GetTx(), grouphash.NewGroupHashFromGroupServer(&gs)) - gs.db = database - gs.notification = NewGroupNotificationSender( - database, - config, - func(ctx context.Context, userIDs []string) ([]notification.CommonUser, error) { - users, err := rpcclient.GetUsersInfo(ctx, userIDs) - if err != nil { - return nil, err - } - return datautil.Slice(users, func(e *sdkws.UserInfo) notification.CommonUser { return e }), nil - }, - ) + //userRpcClient := rpcclient.NewUserRpcClient(client, config.Share.RpcRegisterName.User, config.Share.IMAdminUserID) + //msgRpcClient := rpcclient.NewMessageRpcClient(client, config.Share.RpcRegisterName.Msg) + //conversationRpcClient := rpcclient.NewConversationRpcClient(client, config.Share.RpcRegisterName.Conversation) + + userConn, err := client.GetConn(ctx, config.Discovery.RpcService.User) + if err != nil { + return err + } + msgConn, err := client.GetConn(ctx, config.Discovery.RpcService.Msg) + if err != nil { + return err + } + conversationConn, err := client.GetConn(ctx, config.Discovery.RpcService.Conversation) + if err != nil { + return err + } + gs := groupServer{ + config: config, + webhookClient: webhook.NewWebhookClient(config.WebhooksConfig.URL), + userClient: rpcli.NewUserClient(userConn), + msgClient: rpcli.NewMsgClient(msgConn), + conversationClient: rpcli.NewConversationClient(conversationConn), + } + gs.db = controller.NewGroupDatabase(rdb, &config.LocalCacheConfig, groupDB, groupMemberDB, groupRequestDB, mgocli.GetTx(), grouphash.NewGroupHashFromGroupServer(&gs)) + gs.notification = NewNotificationSender(gs.db, config, gs.userClient, gs.msgClient, gs.conversationClient) localcache.InitLocalCache(&config.LocalCacheConfig) - gs.config = config - gs.webhookClient = webhook.NewWebhookClient(config.WebhooksConfig.URL) pbgroup.RegisterGroupServer(server, &gs) return nil } @@ -158,19 +167,6 @@ func (g *groupServer) CheckGroupAdmin(ctx context.Context, groupID string) error return nil } -func (g *groupServer) GetPublicUserInfoMap(ctx context.Context, userIDs []string) (map[string]*sdkws.PublicUserInfo, error) { - if len(userIDs) == 0 { - return map[string]*sdkws.PublicUserInfo{}, nil - } - users, err := rpcclient.GetPublicUserInfos(ctx, userIDs) - if err != nil { - return nil, err - } - return datautil.SliceToMapAny(users, func(e *sdkws.PublicUserInfo) (string, *sdkws.PublicUserInfo) { - return e.UserID, e - }), nil -} - func (g *groupServer) IsNotFound(err error) bool { return errs.ErrRecordNotFound.Is(specialerror.ErrCode(errs.Unwrap(err))) } @@ -212,7 +208,6 @@ func (g *groupServer) CreateGroup(ctx context.Context, req *pbgroup.CreateGroupR return nil, errs.ErrArgs.WrapMsg("no group owner") } if err := authverify.CheckAccessV3(ctx, req.OwnerUserID, g.config.Share.IMAdminUserID); err != nil { - return nil, err } userIDs := append(append(req.MemberUserIDs, req.AdminUserIDs...), req.OwnerUserID) @@ -225,7 +220,7 @@ func (g *groupServer) CreateGroup(ctx context.Context, req *pbgroup.CreateGroupR return nil, errs.ErrArgs.WrapMsg("group member repeated") } - userMap, err := rpcclient.GetUsersInfoMap(ctx, userIDs) + userMap, err := g.userClient.GetUsersInfoMap(ctx, userIDs) if err != nil { return nil, err } @@ -376,7 +371,7 @@ func (g *groupServer) InviteUserToGroup(ctx context.Context, req *pbgroup.Invite return nil, servererrs.ErrDismissedAlready.WrapMsg("group dismissed checking group status found it dismissed") } - userMap, err := rpcclient.GetUsersInfoMap(ctx, req.InvitedUserIDs) + userMap, err := g.userClient.GetUsersInfoMap(ctx, req.InvitedUserIDs) if err != nil { return nil, err } @@ -687,7 +682,7 @@ func (g *groupServer) GetGroupApplicationList(ctx context.Context, req *pbgroup. userIDs = append(userIDs, gr.UserID) } userIDs = datautil.Distinct(userIDs) - userMap, err := rpcclient.GetPublicUserInfoMap(ctx, userIDs) + userMap, err := g.userClient.GetUsersInfoMap(ctx, userIDs) if err != nil { return nil, err } @@ -799,7 +794,7 @@ func (g *groupServer) GroupApplicationResponse(ctx context.Context, req *pbgroup } else if !g.IsNotFound(err) { return nil, err } - if _, err := rpcclient.GetPublicUserInfo(ctx, req.FromUserID); err != nil { + if err := g.userClient.CheckUser(ctx, []string{req.FromUserID}); err != nil { return nil, err } var member *model.GroupMember @@ -843,7 +838,7 @@ func (g *groupServer) GroupApplicationResponse(ctx context.Context, req *pbgroup } func (g *groupServer) JoinGroup(ctx context.Context, req *pbgroup.JoinGroupReq) (*pbgroup.JoinGroupResp, error) { - user, err := rpcclient.GetUserInfo(ctx, req.InviterUserID) + user, err := g.userClient.GetUserInfo(ctx, req.InviterUserID) if err != nil { return nil, err } @@ -950,18 +945,11 @@ func (g *groupServer) QuitGroup(ctx context.Context, req *pbgroup.QuitGroupReq) func (g *groupServer) deleteMemberAndSetConversationSeq(ctx context.Context, groupID string, userIDs []string) error { conversationID := msgprocessor.GetConversationIDBySessionType(constant.ReadGroupChatType, groupID) - maxSeq, err := rpccall.ExtractField(ctx, msg.GetConversationMaxSeqCaller.Invoke, - &msg.GetConversationMaxSeqReq{ConversationID: conversationID}, - (*msg.GetConversationMaxSeqResp).GetMaxSeq) + maxSeq, err := g.msgClient.GetConversationMaxSeq(ctx, conversationID) if err != nil { return err } - - return pbconv.SetConversationMaxSeqCaller.Execute(ctx, &pbconv.SetConversationMaxSeqReq{ - ConversationID: conversationID, - OwnerUserID: userIDs, - MaxSeq: maxSeq, - }) + return g.conversationClient.SetConversationMaxSeq(ctx, conversationID, userIDs, maxSeq) } func (g *groupServer) SetGroupInfo(ctx context.Context, req *pbgroup.SetGroupInfoReq) (*pbgroup.SetGroupInfoResp, error) { @@ -1037,11 +1025,7 @@ func (g *groupServer) SetGroupInfo(ctx context.Context, req *pbgroup.SetGroupInf return } conversation.GroupAtType = &wrapperspb.Int32Value{Value: constant.GroupNotification} - - if err := pbconv.SetConversationsCaller.Execute(ctx, &pbconv.SetConversationsReq{ - UserIDs: resp.UserIDs, - Conversation: conversation, - }); err != nil { + if err := g.conversationClient.SetConversations(ctx, resp.UserIDs, conversation); err != nil { log.ZWarn(ctx, "SetConversations", err, "UserIDs", resp.UserIDs, "conversation", conversation) } }() @@ -1154,11 +1138,7 @@ func (g *groupServer) SetGroupInfoEx(ctx context.Context, req *pbgroup.SetGroupI } conversation.GroupAtType = &wrapperspb.Int32Value{Value: constant.GroupNotification} - - if err := pbconv.SetConversationsCaller.Execute(ctx, &pbconv.SetConversationsReq{ - UserIDs: resp.UserIDs, - Conversation: conversation, - }); err != nil { + if err := g.conversationClient.SetConversations(ctx, resp.UserIDs, conversation); err != nil { log.ZWarn(ctx, "SetConversations", err, "UserIDs", resp.UserIDs, "conversation", conversation) } }() @@ -1310,7 +1290,7 @@ func (g *groupServer) GetGroupMembersCMS(ctx context.Context, req *pbgroup.GetGr } func (g *groupServer) GetUserReqApplicationList(ctx context.Context, req *pbgroup.GetUserReqApplicationListReq) (*pbgroup.GetUserReqApplicationListResp, error) { - user, err := rpcclient.GetPublicUserInfo(ctx, req.UserID) + user, err := g.userClient.GetUserInfo(ctx, req.UserID) if err != nil { return nil, err } @@ -1766,7 +1746,7 @@ func (g *groupServer) GetGroupUsersReqApplicationList(ctx context.Context, req * return nil, servererrs.ErrGroupIDNotFound.WrapMsg(strings.Join(ids, ",")) } - userMap, err := rpcclient.GetPublicUserInfoMap(ctx, req.UserIDs) + userMap, err := g.userClient.GetUsersInfoMap(ctx, req.UserIDs) if err != nil { return nil, err } @@ -1797,7 +1777,7 @@ func (g *groupServer) GetGroupUsersReqApplicationList(ctx context.Context, req * ownerUserID = owner.UserID } - var userInfo *sdkws.PublicUserInfo + var userInfo *sdkws.UserInfo if user, ok := userMap[e.UserID]; !ok { userInfo = user } @@ -1843,7 +1823,7 @@ func (g *groupServer) GetSpecifiedUserGroupRequestInfo(ctx context.Context, req return nil, err } - userInfos, err := rpcclient.GetPublicUserInfos(ctx, []string{req.UserID}) + userInfos, err := g.userClient.GetUsersInfo(ctx, []string{req.UserID}) if err != nil { return nil, err } diff --git a/internal/rpc/group/notification.go b/internal/rpc/group/notification.go index 784ec8943..48fadbd07 100644 --- a/internal/rpc/group/notification.go +++ b/internal/rpc/group/notification.go @@ -18,6 +18,7 @@ import ( "context" "errors" "fmt" + "github.com/openimsdk/open-im-server/v3/pkg/rpcli" "time" "github.com/openimsdk/open-im-server/v3/pkg/authverify" @@ -28,13 +29,11 @@ import ( "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/versionctx" "github.com/openimsdk/open-im-server/v3/pkg/msgprocessor" - "github.com/openimsdk/open-im-server/v3/pkg/rpcclient" - "github.com/openimsdk/open-im-server/v3/pkg/rpcclient/notification" + "github.com/openimsdk/open-im-server/v3/pkg/notification" + "github.com/openimsdk/open-im-server/v3/pkg/notification/common_user" "github.com/openimsdk/protocol/constant" - pbconv "github.com/openimsdk/protocol/conversation" pbgroup "github.com/openimsdk/protocol/group" "github.com/openimsdk/protocol/msg" - "github.com/openimsdk/protocol/rpccall" "github.com/openimsdk/protocol/sdkws" "github.com/openimsdk/tools/errs" "github.com/openimsdk/tools/log" @@ -50,27 +49,38 @@ const ( adminReceiver ) -func NewGroupNotificationSender( - db controller.GroupDatabase, - config *Config, - fn func(ctx context.Context, userIDs []string) ([]notification.CommonUser, error), -) *GroupNotificationSender { - return &GroupNotificationSender{ - NotificationSender: rpcclient.NewNotificationSender(&config.NotificationConfig, rpcclient.WithRpcClient(), rpcclient.WithUserRpcClient()), - getUsersInfo: fn, +func NewNotificationSender(db controller.GroupDatabase, config *Config, userClient *rpcli.UserClient, msgClient *rpcli.MsgClient, conversationClient *rpcli.ConversationClient) *NotificationSender { + return &NotificationSender{ + NotificationSender: rpcclient.NewNotificationSender(&config.NotificationConfig, + rpcclient.WithRpcClient(func(ctx context.Context, req *msg.SendMsgReq) (*msg.SendMsgResp, error) { + return msgClient.SendMsg(ctx, req) + }), + rpcclient.WithUserRpcClient(userClient.GetUserInfo), + ), + getUsersInfo: func(ctx context.Context, userIDs []string) ([]common_user.CommonUser, error) { + users, err := userClient.GetUsersInfo(ctx, userIDs) + if err != nil { + return nil, err + } + return datautil.Slice(users, func(e *sdkws.UserInfo) common_user.CommonUser { return e }), nil + }, db: db, config: config, + msgClient: msgClient, + conversationClient: conversationClient, } } -type GroupNotificationSender struct { +type NotificationSender struct { *rpcclient.NotificationSender - getUsersInfo func(ctx context.Context, userIDs []string) ([]notification.CommonUser, error) - db controller.GroupDatabase - config *Config + getUsersInfo func(ctx context.Context, userIDs []string) ([]common_user.CommonUser, error) + db controller.GroupDatabase + config *Config + msgClient *rpcli.MsgClient + conversationClient *rpcli.ConversationClient } -func (g *GroupNotificationSender) PopulateGroupMember(ctx context.Context, members ...*model.GroupMember) error { +func (g *NotificationSender) PopulateGroupMember(ctx context.Context, members ...*model.GroupMember) error { if len(members) == 0 { return nil } @@ -85,7 +95,7 @@ func (g *GroupNotificationSender) PopulateGroupMember(ctx context.Context, membe if err != nil { return err } - userMap := make(map[string]notification.CommonUser) + userMap := make(map[string]common_user.CommonUser) for i, user := range users { userMap[user.GetUserID()] = users[i] } @@ -105,7 +115,7 @@ func (g *GroupNotificationSender) PopulateGroupMember(ctx context.Context, membe return nil } -func (g *GroupNotificationSender) getUser(ctx context.Context, userID string) (*sdkws.PublicUserInfo, error) { +func (g *NotificationSender) getUser(ctx context.Context, userID string) (*sdkws.PublicUserInfo, error) { users, err := g.getUsersInfo(ctx, []string{userID}) if err != nil { return nil, err @@ -121,7 +131,7 @@ func (g *GroupNotificationSender) getUser(ctx context.Context, userID string) (* }, nil } -func (g *GroupNotificationSender) getGroupInfo(ctx context.Context, groupID string) (*sdkws.GroupInfo, error) { +func (g *NotificationSender) getGroupInfo(ctx context.Context, groupID string) (*sdkws.GroupInfo, error) { gm, err := g.db.TakeGroup(ctx, groupID) if err != nil { return nil, err @@ -142,7 +152,7 @@ func (g *GroupNotificationSender) getGroupInfo(ctx context.Context, groupID stri return convert.Db2PbGroupInfo(gm, ownerUserID, num), nil } -func (g *GroupNotificationSender) getGroupMembers(ctx context.Context, groupID string, userIDs []string) ([]*sdkws.GroupMemberFullInfo, error) { +func (g *NotificationSender) getGroupMembers(ctx context.Context, groupID string, userIDs []string) ([]*sdkws.GroupMemberFullInfo, error) { members, err := g.db.FindGroupMembers(ctx, groupID, userIDs) if err != nil { return nil, err @@ -158,7 +168,7 @@ func (g *GroupNotificationSender) getGroupMembers(ctx context.Context, groupID s return res, nil } -func (g *GroupNotificationSender) getGroupMemberMap(ctx context.Context, groupID string, userIDs []string) (map[string]*sdkws.GroupMemberFullInfo, error) { +func (g *NotificationSender) getGroupMemberMap(ctx context.Context, groupID string, userIDs []string) (map[string]*sdkws.GroupMemberFullInfo, error) { members, err := g.getGroupMembers(ctx, groupID, userIDs) if err != nil { return nil, err @@ -170,7 +180,7 @@ func (g *GroupNotificationSender) getGroupMemberMap(ctx context.Context, groupID return m, nil } -func (g *GroupNotificationSender) getGroupMember(ctx context.Context, groupID string, userID string) (*sdkws.GroupMemberFullInfo, error) { +func (g *NotificationSender) getGroupMember(ctx context.Context, groupID string, userID string) (*sdkws.GroupMemberFullInfo, error) { members, err := g.getGroupMembers(ctx, groupID, []string{userID}) if err != nil { return nil, err @@ -181,7 +191,7 @@ func (g *GroupNotificationSender) getGroupMember(ctx context.Context, groupID st return members[0], nil } -func (g *GroupNotificationSender) getGroupOwnerAndAdminUserID(ctx context.Context, groupID string) ([]string, error) { +func (g *NotificationSender) getGroupOwnerAndAdminUserID(ctx context.Context, groupID string) ([]string, error) { members, err := g.db.FindGroupMemberRoleLevels(ctx, groupID, []int32{constant.GroupOwner, constant.GroupAdmin}) if err != nil { return nil, err @@ -193,7 +203,7 @@ func (g *GroupNotificationSender) getGroupOwnerAndAdminUserID(ctx context.Contex return datautil.Slice(members, fn), nil } -func (g *GroupNotificationSender) groupMemberDB2PB(member *model.GroupMember, appMangerLevel int32) *sdkws.GroupMemberFullInfo { +func (g *NotificationSender) groupMemberDB2PB(member *model.GroupMember, appMangerLevel int32) *sdkws.GroupMemberFullInfo { return &sdkws.GroupMemberFullInfo{ GroupID: member.GroupID, UserID: member.UserID, @@ -210,7 +220,7 @@ func (g *GroupNotificationSender) groupMemberDB2PB(member *model.GroupMember, ap } } -/* func (g *GroupNotificationSender) getUsersInfoMap(ctx context.Context, userIDs []string) (map[string]*sdkws.UserInfo, error) { +/* func (g *NotificationSender) getUsersInfoMap(ctx context.Context, userIDs []string) (map[string]*sdkws.UserInfo, error) { users, err := g.getUsersInfo(ctx, userIDs) if err != nil { return nil, err @@ -222,11 +232,11 @@ func (g *GroupNotificationSender) groupMemberDB2PB(member *model.GroupMember, ap return result, nil } */ -func (g *GroupNotificationSender) fillOpUser(ctx context.Context, opUser **sdkws.GroupMemberFullInfo, groupID string) (err error) { +func (g *NotificationSender) fillOpUser(ctx context.Context, opUser **sdkws.GroupMemberFullInfo, groupID string) (err error) { return g.fillOpUserByUserID(ctx, mcontext.GetOpUserID(ctx), opUser, groupID) } -func (g *GroupNotificationSender) fillOpUserByUserID(ctx context.Context, userID string, opUser **sdkws.GroupMemberFullInfo, groupID string) error { +func (g *NotificationSender) fillOpUserByUserID(ctx context.Context, userID string, opUser **sdkws.GroupMemberFullInfo, groupID string) error { if opUser == nil { return errs.ErrInternalServer.WrapMsg("**sdkws.GroupMemberFullInfo is nil") } @@ -270,7 +280,7 @@ func (g *GroupNotificationSender) fillOpUserByUserID(ctx context.Context, userID return nil } -func (g *GroupNotificationSender) setVersion(ctx context.Context, version *uint64, versionID *string, collName string, id string) { +func (g *NotificationSender) setVersion(ctx context.Context, version *uint64, versionID *string, collName string, id string) { versions := versionctx.GetVersionLog(ctx).Get() for _, coll := range versions { if coll.Name == collName && coll.Doc.DID == id { @@ -281,7 +291,7 @@ func (g *GroupNotificationSender) setVersion(ctx context.Context, version *uint6 } } -func (g *GroupNotificationSender) setSortVersion(ctx context.Context, version *uint64, versionID *string, collName string, id string, sortVersion *uint64) { +func (g *NotificationSender) setSortVersion(ctx context.Context, version *uint64, versionID *string, collName string, id string, sortVersion *uint64) { versions := versionctx.GetVersionLog(ctx).Get() for _, coll := range versions { if coll.Name == collName && coll.Doc.DID == id { @@ -296,7 +306,7 @@ func (g *GroupNotificationSender) setSortVersion(ctx context.Context, version *u } } -func (g *GroupNotificationSender) GroupCreatedNotification(ctx context.Context, tips *sdkws.GroupCreatedTips) { +func (g *NotificationSender) GroupCreatedNotification(ctx context.Context, tips *sdkws.GroupCreatedTips) { var err error defer func() { if err != nil { @@ -310,7 +320,7 @@ func (g *GroupNotificationSender) GroupCreatedNotification(ctx context.Context, g.Notification(ctx, mcontext.GetOpUserID(ctx), tips.Group.GroupID, constant.GroupCreatedNotification, tips) } -func (g *GroupNotificationSender) GroupInfoSetNotification(ctx context.Context, tips *sdkws.GroupInfoSetTips) { +func (g *NotificationSender) GroupInfoSetNotification(ctx context.Context, tips *sdkws.GroupInfoSetTips) { var err error defer func() { if err != nil { @@ -324,7 +334,7 @@ func (g *GroupNotificationSender) GroupInfoSetNotification(ctx context.Context, g.Notification(ctx, mcontext.GetOpUserID(ctx), tips.Group.GroupID, constant.GroupInfoSetNotification, tips, rpcclient.WithRpcGetUserName()) } -func (g *GroupNotificationSender) GroupInfoSetNameNotification(ctx context.Context, tips *sdkws.GroupInfoSetNameTips) { +func (g *NotificationSender) GroupInfoSetNameNotification(ctx context.Context, tips *sdkws.GroupInfoSetNameTips) { var err error defer func() { if err != nil { @@ -338,7 +348,7 @@ func (g *GroupNotificationSender) GroupInfoSetNameNotification(ctx context.Conte g.Notification(ctx, mcontext.GetOpUserID(ctx), tips.Group.GroupID, constant.GroupInfoSetNameNotification, tips) } -func (g *GroupNotificationSender) GroupInfoSetAnnouncementNotification(ctx context.Context, tips *sdkws.GroupInfoSetAnnouncementTips) { +func (g *NotificationSender) GroupInfoSetAnnouncementNotification(ctx context.Context, tips *sdkws.GroupInfoSetAnnouncementTips) { var err error defer func() { if err != nil { @@ -352,7 +362,7 @@ func (g *GroupNotificationSender) GroupInfoSetAnnouncementNotification(ctx conte g.Notification(ctx, mcontext.GetOpUserID(ctx), tips.Group.GroupID, constant.GroupInfoSetAnnouncementNotification, tips, rpcclient.WithRpcGetUserName()) } -func (g *GroupNotificationSender) JoinGroupApplicationNotification(ctx context.Context, req *pbgroup.JoinGroupReq) { +func (g *NotificationSender) JoinGroupApplicationNotification(ctx context.Context, req *pbgroup.JoinGroupReq) { var err error defer func() { if err != nil { @@ -380,7 +390,7 @@ func (g *GroupNotificationSender) JoinGroupApplicationNotification(ctx context.C } } -func (g *GroupNotificationSender) MemberQuitNotification(ctx context.Context, member *sdkws.GroupMemberFullInfo) { +func (g *NotificationSender) MemberQuitNotification(ctx context.Context, member *sdkws.GroupMemberFullInfo) { var err error defer func() { if err != nil { @@ -397,7 +407,7 @@ func (g *GroupNotificationSender) MemberQuitNotification(ctx context.Context, me g.Notification(ctx, mcontext.GetOpUserID(ctx), member.GroupID, constant.MemberQuitNotification, tips) } -func (g *GroupNotificationSender) GroupApplicationAcceptedNotification(ctx context.Context, req *pbgroup.GroupApplicationResponseReq) { +func (g *NotificationSender) GroupApplicationAcceptedNotification(ctx context.Context, req *pbgroup.GroupApplicationResponseReq) { var err error defer func() { if err != nil { @@ -430,7 +440,7 @@ func (g *GroupNotificationSender) GroupApplicationAcceptedNotification(ctx conte } } -func (g *GroupNotificationSender) GroupApplicationRejectedNotification(ctx context.Context, req *pbgroup.GroupApplicationResponseReq) { +func (g *NotificationSender) GroupApplicationRejectedNotification(ctx context.Context, req *pbgroup.GroupApplicationResponseReq) { var err error defer func() { if err != nil { @@ -463,7 +473,7 @@ func (g *GroupNotificationSender) GroupApplicationRejectedNotification(ctx conte } } -func (g *GroupNotificationSender) GroupOwnerTransferredNotification(ctx context.Context, req *pbgroup.TransferGroupOwnerReq) { +func (g *NotificationSender) GroupOwnerTransferredNotification(ctx context.Context, req *pbgroup.TransferGroupOwnerReq) { var err error defer func() { if err != nil { @@ -494,7 +504,7 @@ func (g *GroupNotificationSender) GroupOwnerTransferredNotification(ctx context. g.Notification(ctx, mcontext.GetOpUserID(ctx), group.GroupID, constant.GroupOwnerTransferredNotification, tips) } -func (g *GroupNotificationSender) MemberKickedNotification(ctx context.Context, tips *sdkws.MemberKickedTips) { +func (g *NotificationSender) MemberKickedNotification(ctx context.Context, tips *sdkws.MemberKickedTips) { var err error defer func() { if err != nil { @@ -508,7 +518,7 @@ func (g *GroupNotificationSender) MemberKickedNotification(ctx context.Context, g.Notification(ctx, mcontext.GetOpUserID(ctx), tips.Group.GroupID, constant.MemberKickedNotification, tips) } -func (g *GroupNotificationSender) GroupApplicationAgreeMemberEnterNotification(ctx context.Context, groupID string, invitedOpUserID string, entrantUserID ...string) error { +func (g *NotificationSender) GroupApplicationAgreeMemberEnterNotification(ctx context.Context, groupID string, invitedOpUserID string, entrantUserID ...string) error { var err error defer func() { if err != nil { @@ -518,26 +528,15 @@ func (g *GroupNotificationSender) GroupApplicationAgreeMemberEnterNotification(c if !g.config.RpcConfig.EnableHistoryForNewMembers { conversationID := msgprocessor.GetConversationIDBySessionType(constant.ReadGroupChatType, groupID) - maxSeq, err := rpccall.ExtractField(ctx, msg.GetConversationMaxSeqCaller.Invoke, - &msg.GetConversationMaxSeqReq{ConversationID: conversationID}, - (*msg.GetConversationMaxSeqResp).GetMaxSeq) + maxSeq, err := g.msgClient.GetConversationMaxSeq(ctx, conversationID) if err != nil { return err } - - if err := msg.SetUserConversationsMinSeqCaller.Execute(ctx, &msg.SetUserConversationsMinSeqReq{ - UserIDs: entrantUserID, - ConversationID: conversationID, - Seq: maxSeq, - }); err != nil { + if err := g.msgClient.SetUserConversationsMinSeq(ctx, conversationID, entrantUserID, maxSeq); err != nil { return err } } - - if err := pbconv.CreateGroupChatConversationsCaller.Execute(ctx, &pbconv.CreateGroupChatConversationsReq{ - UserIDs: entrantUserID, - GroupID: groupID, - }); err != nil { + if err := g.conversationClient.CreateGroupChatConversations(ctx, groupID, entrantUserID); err != nil { return err } @@ -573,7 +572,7 @@ func (g *GroupNotificationSender) GroupApplicationAgreeMemberEnterNotification(c return nil } -func (g *GroupNotificationSender) MemberEnterNotification(ctx context.Context, groupID string, entrantUserID string) error { +func (g *NotificationSender) MemberEnterNotification(ctx context.Context, groupID string, entrantUserID string) error { var err error defer func() { if err != nil { @@ -583,28 +582,17 @@ func (g *GroupNotificationSender) MemberEnterNotification(ctx context.Context, g if !g.config.RpcConfig.EnableHistoryForNewMembers { conversationID := msgprocessor.GetConversationIDBySessionType(constant.ReadGroupChatType, groupID) - maxSeq, err := rpccall.ExtractField(ctx, msg.GetConversationMaxSeqCaller.Invoke, - &msg.GetConversationMaxSeqReq{ConversationID: conversationID}, - (*msg.GetConversationMaxSeqResp).GetMaxSeq) + maxSeq, err := g.msgClient.GetConversationMaxSeq(ctx, conversationID) if err != nil { return err } - if err := msg.SetUserConversationsMinSeqCaller.Execute(ctx, &msg.SetUserConversationsMinSeqReq{ - UserIDs: []string{entrantUserID}, - ConversationID: conversationID, - Seq: maxSeq, - }); err != nil { + if err := g.msgClient.SetUserConversationsMinSeq(ctx, conversationID, []string{entrantUserID}, maxSeq); err != nil { return err } } - - if err := pbconv.CreateGroupChatConversationsCaller.Execute(ctx, &pbconv.CreateGroupChatConversationsReq{ - UserIDs: []string{entrantUserID}, - GroupID: groupID, - }); err != nil { + if err := g.conversationClient.CreateGroupChatConversations(ctx, groupID, []string{entrantUserID}); err != nil { return err } - var group *sdkws.GroupInfo group, err = g.getGroupInfo(ctx, groupID) if err != nil { @@ -625,7 +613,7 @@ func (g *GroupNotificationSender) MemberEnterNotification(ctx context.Context, g return nil } -func (g *GroupNotificationSender) GroupDismissedNotification(ctx context.Context, tips *sdkws.GroupDismissedTips) { +func (g *NotificationSender) GroupDismissedNotification(ctx context.Context, tips *sdkws.GroupDismissedTips) { var err error defer func() { if err != nil { @@ -638,7 +626,7 @@ func (g *GroupNotificationSender) GroupDismissedNotification(ctx context.Context g.Notification(ctx, mcontext.GetOpUserID(ctx), tips.Group.GroupID, constant.GroupDismissedNotification, tips) } -func (g *GroupNotificationSender) GroupMemberMutedNotification(ctx context.Context, groupID, groupMemberUserID string, mutedSeconds uint32) { +func (g *NotificationSender) GroupMemberMutedNotification(ctx context.Context, groupID, groupMemberUserID string, mutedSeconds uint32) { var err error defer func() { if err != nil { @@ -666,7 +654,7 @@ func (g *GroupNotificationSender) GroupMemberMutedNotification(ctx context.Conte g.Notification(ctx, mcontext.GetOpUserID(ctx), group.GroupID, constant.GroupMemberMutedNotification, tips) } -func (g *GroupNotificationSender) GroupMemberCancelMutedNotification(ctx context.Context, groupID, groupMemberUserID string) { +func (g *NotificationSender) GroupMemberCancelMutedNotification(ctx context.Context, groupID, groupMemberUserID string) { var err error defer func() { if err != nil { @@ -691,7 +679,7 @@ func (g *GroupNotificationSender) GroupMemberCancelMutedNotification(ctx context g.Notification(ctx, mcontext.GetOpUserID(ctx), group.GroupID, constant.GroupMemberCancelMutedNotification, tips) } -func (g *GroupNotificationSender) GroupMutedNotification(ctx context.Context, groupID string) { +func (g *NotificationSender) GroupMutedNotification(ctx context.Context, groupID string) { var err error defer func() { if err != nil { @@ -719,7 +707,7 @@ func (g *GroupNotificationSender) GroupMutedNotification(ctx context.Context, gr g.Notification(ctx, mcontext.GetOpUserID(ctx), group.GroupID, constant.GroupMutedNotification, tips) } -func (g *GroupNotificationSender) GroupCancelMutedNotification(ctx context.Context, groupID string) { +func (g *NotificationSender) GroupCancelMutedNotification(ctx context.Context, groupID string) { var err error defer func() { if err != nil { @@ -747,7 +735,7 @@ func (g *GroupNotificationSender) GroupCancelMutedNotification(ctx context.Conte g.Notification(ctx, mcontext.GetOpUserID(ctx), group.GroupID, constant.GroupCancelMutedNotification, tips) } -func (g *GroupNotificationSender) GroupMemberInfoSetNotification(ctx context.Context, groupID, groupMemberUserID string) { +func (g *NotificationSender) GroupMemberInfoSetNotification(ctx context.Context, groupID, groupMemberUserID string) { var err error defer func() { if err != nil { @@ -772,7 +760,7 @@ func (g *GroupNotificationSender) GroupMemberInfoSetNotification(ctx context.Con g.Notification(ctx, mcontext.GetOpUserID(ctx), group.GroupID, constant.GroupMemberInfoSetNotification, tips) } -func (g *GroupNotificationSender) GroupMemberSetToAdminNotification(ctx context.Context, groupID, groupMemberUserID string) { +func (g *NotificationSender) GroupMemberSetToAdminNotification(ctx context.Context, groupID, groupMemberUserID string) { var err error defer func() { if err != nil { @@ -796,7 +784,7 @@ func (g *GroupNotificationSender) GroupMemberSetToAdminNotification(ctx context. g.Notification(ctx, mcontext.GetOpUserID(ctx), group.GroupID, constant.GroupMemberSetToAdminNotification, tips) } -func (g *GroupNotificationSender) GroupMemberSetToOrdinaryUserNotification(ctx context.Context, groupID, groupMemberUserID string) { +func (g *NotificationSender) GroupMemberSetToOrdinaryUserNotification(ctx context.Context, groupID, groupMemberUserID string) { var err error defer func() { if err != nil { diff --git a/internal/rpc/msg/clear.go b/internal/rpc/msg/clear.go index 7a2d36300..8e14b281e 100644 --- a/internal/rpc/msg/clear.go +++ b/internal/rpc/msg/clear.go @@ -2,23 +2,13 @@ package msg import ( "context" - "strings" - "time" - "github.com/openimsdk/open-im-server/v3/pkg/authverify" - "github.com/openimsdk/open-im-server/v3/pkg/common/convert" - pbconv "github.com/openimsdk/protocol/conversation" "github.com/openimsdk/protocol/msg" - "github.com/openimsdk/protocol/wrapperspb" "github.com/openimsdk/tools/log" - "github.com/openimsdk/tools/mcontext" - "github.com/openimsdk/tools/utils/datautil" - "github.com/openimsdk/tools/utils/idutil" - "github.com/openimsdk/tools/utils/stringutil" - "golang.org/x/sync/errgroup" + "strings" ) -// hard delete in Database. +// DestructMsgs hard delete in Database. func (m *msgServer) DestructMsgs(ctx context.Context, req *msg.DestructMsgsReq) (*msg.DestructMsgsResp, error) { if err := authverify.CheckAdmin(ctx, m.config.Share.IMAdminUserID); err != nil { return nil, err @@ -61,70 +51,6 @@ func (m *msgServer) DestructMsgs(ctx context.Context, req *msg.DestructMsgsReq) return &msg.DestructMsgsResp{Count: int32(len(docs))}, nil } -// soft delete for user self -func (m *msgServer) ClearMsg(ctx context.Context, req *msg.ClearMsgReq) (*msg.ClearMsgResp, error) { - temp := convert.ConversationsPb2DB(req.Conversations) - - batchNum := 100 - - errg, _ := errgroup.WithContext(ctx) - errg.SetLimit(100) - - for i := 0; i < len(temp); i += batchNum { - batch := temp[i:min(i+batchNum, len(temp))] - - errg.Go(func() error { - for _, conversation := range batch { - handleCtx := mcontext.NewCtx(stringutil.GetSelfFuncName() + "-" + idutil.OperationIDGenerator() + "-" + conversation.ConversationID + "-" + conversation.OwnerUserID) - log.ZDebug(handleCtx, "User MsgsDestruct", - "conversationID", conversation.ConversationID, - "ownerUserID", conversation.OwnerUserID, - "msgDestructTime", conversation.MsgDestructTime, - "lastMsgDestructTime", conversation.LatestMsgDestructTime) - - seqs, err := m.MsgDatabase.ClearUserMsgs(handleCtx, conversation.OwnerUserID, conversation.ConversationID, conversation.MsgDestructTime, conversation.LatestMsgDestructTime) - if err != nil { - log.ZError(handleCtx, "user msg destruct failed", err, "conversationID", conversation.ConversationID, "ownerUserID", conversation.OwnerUserID) - continue - } - - if len(seqs) > 0 { - minseq := datautil.Max(seqs...) - - // update - if err := pbconv.UpdateConversationCaller.Execute(ctx, &pbconv.UpdateConversationReq{ - ConversationID: conversation.ConversationID, - UserIDs: []string{conversation.OwnerUserID}, - MinSeq: wrapperspb.Int64(minseq), - LatestMsgDestructTime: wrapperspb.Int64(time.Now().UnixMilli()), - }); err != nil { - log.ZError(handleCtx, "updateUsersConversationField failed", err, "conversationID", conversation.ConversationID, "ownerUserID", conversation.OwnerUserID) - continue - } - - if err := pbconv.SetConversationMinSeqCaller.Execute(ctx, &pbconv.SetConversationMinSeqReq{ - ConversationID: conversation.ConversationID, - OwnerUserID: []string{conversation.OwnerUserID}, - MinSeq: minseq, - }); err != nil { - return err - } - - // if you need Notify SDK client userseq is update. - // m.msgNotificationSender.UserDeleteMsgsNotification(handleCtx, conversation.OwnerUserID, conversation.ConversationID, seqs) - } - } - return nil - }) - } - - if err := errg.Wait(); err != nil { - return nil, err - } - - return nil, nil -} - func (m *msgServer) GetLastMessageSeqByTime(ctx context.Context, req *msg.GetLastMessageSeqByTimeReq) (*msg.GetLastMessageSeqByTimeResp, error) { seq, err := m.MsgDatabase.GetLastMessageSeqByTime(ctx, req.ConversationID, req.Time) if err != nil { diff --git a/internal/rpc/msg/delete.go b/internal/rpc/msg/delete.go index e0c3a89ed..f45713ff1 100644 --- a/internal/rpc/msg/delete.go +++ b/internal/rpc/msg/delete.go @@ -21,7 +21,6 @@ import ( "github.com/openimsdk/protocol/constant" "github.com/openimsdk/protocol/conversation" "github.com/openimsdk/protocol/msg" - "github.com/openimsdk/protocol/rpccall" "github.com/openimsdk/protocol/sdkws" "github.com/openimsdk/tools/log" "github.com/openimsdk/tools/utils/timeutil" @@ -75,22 +74,13 @@ func (m *msgServer) DeleteMsgs(ctx context.Context, req *msg.DeleteMsgsReq) (*ms if err := m.MsgDatabase.DeleteMsgsPhysicalBySeqs(ctx, req.ConversationID, req.Seqs); err != nil { return nil, err } - - conversations, err := rpccall.ExtractField(ctx, conversation.GetConversationsByConversationIDCaller.Invoke, &conversation.GetConversationsByConversationIDReq{ - ConversationIDs: []string{req.ConversationID}, - }, (*conversation.GetConversationsByConversationIDResp).GetConversations) + conv, err := m.conversationClient.GetConversationsByConversationID(ctx, req.ConversationID) if err != nil { return nil, err } tips := &sdkws.DeleteMsgsTips{UserID: req.UserID, ConversationID: req.ConversationID, Seqs: req.Seqs} - m.notificationSender.NotificationWithSessionType( - ctx, - req.UserID, - m.conversationAndGetRecvID(conversations[0], req.UserID), - constant.DeleteMsgsNotification, - conversations[0].ConversationType, - tips, - ) + m.notificationSender.NotificationWithSessionType(ctx, req.UserID, m.conversationAndGetRecvID(conv, req.UserID), + constant.DeleteMsgsNotification, conv.ConversationType, tips) } else { if err := m.MsgDatabase.DeleteUserMsgsBySeqs(ctx, req.UserID, req.ConversationID, req.Seqs); err != nil { return nil, err @@ -125,9 +115,7 @@ func (m *msgServer) DeleteMsgPhysical(ctx context.Context, req *msg.DeleteMsgPhy } func (m *msgServer) clearConversation(ctx context.Context, conversationIDs []string, userID string, deleteSyncOpt *msg.DeleteSyncOpt) error { - conversations, err := rpccall.ExtractField(ctx, conversation.GetConversationsByConversationIDCaller.Invoke, &conversation.GetConversationsByConversationIDReq{ - ConversationIDs: conversationIDs, - }, (*conversation.GetConversationsByConversationIDResp).GetConversations) + conversations, err := m.conversationClient.GetConversationsByConversationIDs(ctx, conversationIDs) if err != nil { return err } @@ -150,11 +138,7 @@ func (m *msgServer) clearConversation(ctx context.Context, conversationIDs []str } ownerUserIDs := []string{userID} for conversationID, seq := range setSeqs { - if err := conversation.SetConversationMinSeqCaller.Execute(ctx, &conversation.SetConversationMinSeqReq{ - ConversationID: conversationID, - OwnerUserID: ownerUserIDs, - MinSeq: seq, - }); err != nil { + if err := m.conversationClient.SetConversationMinSeq(ctx, conversationID, ownerUserIDs, seq); err != nil { return err } } diff --git a/internal/rpc/msg/notification.go b/internal/rpc/msg/notification.go index 26e3c7f46..14576f693 100644 --- a/internal/rpc/msg/notification.go +++ b/internal/rpc/msg/notification.go @@ -17,7 +17,7 @@ package msg import ( "context" - "github.com/openimsdk/open-im-server/v3/pkg/rpcclient" + "github.com/openimsdk/open-im-server/v3/pkg/notification" "github.com/openimsdk/protocol/constant" "github.com/openimsdk/protocol/sdkws" ) diff --git a/internal/rpc/msg/send.go b/internal/rpc/msg/send.go index bb20206f7..b9bbf615f 100644 --- a/internal/rpc/msg/send.go +++ b/internal/rpc/msg/send.go @@ -118,25 +118,14 @@ func (m *msgServer) setConversationAtInfo(nctx context.Context, msg *sdkws.MsgDa conversation.GroupAtType = &wrapperspb.Int32Value{Value: constant.AtAll} } else { // @Everyone and @other people conversation.GroupAtType = &wrapperspb.Int32Value{Value: constant.AtAllAtMe} - - err = pbconv.SetConversationsCaller.Execute(ctx, &pbconv.SetConversationsReq{ - UserIDs: atUserID, - Conversation: conversation, - }) - if err != nil { + if err := m.conversationClient.SetConversations(ctx, atUserID, conversation); err != nil { log.ZWarn(ctx, "SetConversations", err, "userID", atUserID, "conversation", conversation) } - memberUserIDList = datautil.Single(atUserID, memberUserIDList) } conversation.GroupAtType = &wrapperspb.Int32Value{Value: constant.AtAll} - - err = pbconv.SetConversationsCaller.Execute(ctx, &pbconv.SetConversationsReq{ - UserIDs: memberUserIDList, - Conversation: conversation, - }) - if err != nil { + if err := m.conversationClient.SetConversations(ctx, memberUserIDList, conversation); err != nil { log.ZWarn(ctx, "SetConversations", err, "userID", memberUserIDList, "conversation", conversation) } @@ -144,11 +133,7 @@ func (m *msgServer) setConversationAtInfo(nctx context.Context, msg *sdkws.MsgDa } conversation.GroupAtType = &wrapperspb.Int32Value{Value: constant.AtMe} - err := pbconv.SetConversationsCaller.Execute(ctx, &pbconv.SetConversationsReq{ - UserIDs: msg.AtUserIDList, - Conversation: conversation, - }) - if err != nil { + if err := m.conversationClient.SetConversations(ctx, msg.AtUserIDList, conversation); err != nil { log.ZWarn(ctx, "SetConversations", err, msg.AtUserIDList, conversation) } } diff --git a/internal/rpc/msg/server.go b/internal/rpc/msg/server.go index 9bd2a6946..991fb8704 100644 --- a/internal/rpc/msg/server.go +++ b/internal/rpc/msg/server.go @@ -16,6 +16,7 @@ package msg import ( "context" + "github.com/openimsdk/open-im-server/v3/pkg/rpcli" "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/redis" @@ -26,8 +27,8 @@ import ( "github.com/openimsdk/tools/db/redisutil" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/controller" + "github.com/openimsdk/open-im-server/v3/pkg/notification" "github.com/openimsdk/open-im-server/v3/pkg/rpccache" - "github.com/openimsdk/open-im-server/v3/pkg/rpcclient" "github.com/openimsdk/protocol/constant" "github.com/openimsdk/protocol/conversation" "github.com/openimsdk/protocol/msg" @@ -36,39 +37,39 @@ import ( ) type MessageInterceptorFunc func(ctx context.Context, globalConfig *Config, req *msg.SendMsgReq) (*sdkws.MsgData, error) -type ( - // MessageInterceptorChain defines a chain of message interceptor functions. - MessageInterceptorChain []MessageInterceptorFunc - // MsgServer encapsulates dependencies required for message handling. - msgServer struct { - RegisterCenter discovery.SvcDiscoveryRegistry // Service discovery registry for service registration. - MsgDatabase controller.CommonMsgDatabase // Interface for message database operations. - StreamMsgDatabase controller.StreamMsgDatabase - UserLocalCache *rpccache.UserLocalCache // Local cache for user data. - FriendLocalCache *rpccache.FriendLocalCache // Local cache for friend data. - GroupLocalCache *rpccache.GroupLocalCache // Local cache for group data. - ConversationLocalCache *rpccache.ConversationLocalCache // Local cache for conversation data. - Handlers MessageInterceptorChain // Chain of handlers for processing messages. - notificationSender *rpcclient.NotificationSender // RPC client for sending notifications. - msgNotificationSender *MsgNotificationSender // RPC client for sending msg notifications. - config *Config // Global configuration settings. - webhookClient *webhook.Client - msg.UnimplementedMsgServer - } +// MessageInterceptorChain defines a chain of message interceptor functions. +type MessageInterceptorChain []MessageInterceptorFunc - Config struct { - RpcConfig config.Msg - RedisConfig config.Redis - MongodbConfig config.Mongo - KafkaConfig config.Kafka - NotificationConfig config.Notification - Share config.Share - WebhooksConfig config.Webhooks - LocalCacheConfig config.LocalCache - Discovery config.Discovery - } -) +type Config struct { + RpcConfig config.Msg + RedisConfig config.Redis + MongodbConfig config.Mongo + KafkaConfig config.Kafka + NotificationConfig config.Notification + Share config.Share + WebhooksConfig config.Webhooks + LocalCacheConfig config.LocalCache + Discovery config.Discovery +} + +// MsgServer encapsulates dependencies required for message handling. +type msgServer struct { + msg.UnimplementedMsgServer + RegisterCenter discovery.SvcDiscoveryRegistry // Service discovery registry for service registration. + MsgDatabase controller.CommonMsgDatabase // Interface for message database operations. + StreamMsgDatabase controller.StreamMsgDatabase + UserLocalCache *rpccache.UserLocalCache // Local cache for user data. + FriendLocalCache *rpccache.FriendLocalCache // Local cache for friend data. + GroupLocalCache *rpccache.GroupLocalCache // Local cache for group data. + ConversationLocalCache *rpccache.ConversationLocalCache // Local cache for conversation data. + Handlers MessageInterceptorChain // Chain of handlers for processing messages. + notificationSender *rpcclient.NotificationSender // RPC client for sending notifications. + msgNotificationSender *MsgNotificationSender // RPC client for sending msg notifications. + config *Config // Global configuration settings. + webhookClient *webhook.Client + conversationClient *rpcli.ConversationClient +} func (m *msgServer) addInterceptorHandler(interceptorFunc ...MessageInterceptorFunc) { m.Handlers = append(m.Handlers, interceptorFunc...) @@ -107,16 +108,34 @@ func Start(ctx context.Context, config *Config, client discovery.SvcDiscoveryReg if err != nil { return err } + userConn, err := client.GetConn(ctx, config.Discovery.RpcService.User) + if err != nil { + return err + } + groupConn, err := client.GetConn(ctx, config.Discovery.RpcService.Group) + if err != nil { + return err + } + friendConn, err := client.GetConn(ctx, config.Discovery.RpcService.Friend) + if err != nil { + return err + } + conversationConn, err := client.GetConn(ctx, config.Discovery.RpcService.Conversation) + if err != nil { + return err + } + conversationClient := rpcli.NewConversationClient(conversationConn) s := &msgServer{ MsgDatabase: msgDatabase, StreamMsgDatabase: controller.NewStreamMsgDatabase(streamMsg), RegisterCenter: client, - UserLocalCache: rpccache.NewUserLocalCache(&config.LocalCacheConfig, rdb), - GroupLocalCache: rpccache.NewGroupLocalCache(&config.LocalCacheConfig, rdb), - ConversationLocalCache: rpccache.NewConversationLocalCache(&config.LocalCacheConfig, rdb), - FriendLocalCache: rpccache.NewFriendLocalCache(&config.LocalCacheConfig, rdb), + UserLocalCache: rpccache.NewUserLocalCache(rpcli.NewUserClient(userConn), &config.LocalCacheConfig, rdb), + GroupLocalCache: rpccache.NewGroupLocalCache(rpcli.NewGroupClient(groupConn), &config.LocalCacheConfig, rdb), + ConversationLocalCache: rpccache.NewConversationLocalCache(conversationClient, &config.LocalCacheConfig, rdb), + FriendLocalCache: rpccache.NewFriendLocalCache(rpcli.NewRelationClient(friendConn), &config.LocalCacheConfig, rdb), config: config, webhookClient: webhook.NewWebhookClient(config.WebhooksConfig.URL), + conversationClient: conversationClient, } s.notificationSender = rpcclient.NewNotificationSender(&config.NotificationConfig, rpcclient.WithLocalSendMsg(s.SendMsg)) diff --git a/internal/rpc/msg/stream_msg.go b/internal/rpc/msg/stream_msg.go index e216b1087..688d766c8 100644 --- a/internal/rpc/msg/stream_msg.go +++ b/internal/rpc/msg/stream_msg.go @@ -8,9 +8,7 @@ import ( "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" "github.com/openimsdk/open-im-server/v3/pkg/msgprocessor" "github.com/openimsdk/protocol/constant" - pbconv "github.com/openimsdk/protocol/conversation" "github.com/openimsdk/protocol/msg" - "github.com/openimsdk/protocol/rpccall" "github.com/openimsdk/protocol/sdkws" "github.com/openimsdk/tools/errs" ) @@ -74,10 +72,7 @@ func (m *msgServer) AppendStreamMsg(ctx context.Context, req *msg.AppendStreamMs if err := m.StreamMsgDatabase.AppendStreamMsg(ctx, req.ClientMsgID, int(req.StartIndex), req.Packets, req.End, deadlineTime); err != nil { return nil, err } - conversation, err := rpccall.ExtractField(ctx, pbconv.GetConversationCaller.Invoke, &pbconv.GetConversationReq{ - ConversationID: res.ConversationID, - OwnerUserID: res.UserID, - }, (*pbconv.GetConversationResp).GetConversation) + conversation, err := m.conversationClient.GetConversation(ctx, res.ConversationID, res.UserID) if err != nil { return nil, err } diff --git a/internal/rpc/relation/black.go b/internal/rpc/relation/black.go index 0fd9b8766..2108d7dc5 100644 --- a/internal/rpc/relation/black.go +++ b/internal/rpc/relation/black.go @@ -21,7 +21,6 @@ import ( "github.com/openimsdk/open-im-server/v3/pkg/authverify" "github.com/openimsdk/open-im-server/v3/pkg/common/convert" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" - "github.com/openimsdk/open-im-server/v3/pkg/rpcclient" "github.com/openimsdk/protocol/relation" "github.com/openimsdk/protocol/sdkws" "github.com/openimsdk/tools/errs" @@ -39,7 +38,7 @@ func (s *friendServer) GetPaginationBlacks(ctx context.Context, req *relation.Ge return nil, err } resp = &relation.GetPaginationBlacksResp{} - resp.Blacks, err = convert.BlackDB2Pb(ctx, blacks, rpcclient.GetUsersInfoMap) + resp.Blacks, err = convert.BlackDB2Pb(ctx, blacks, s.userClient.GetUsersInfoMap) if err != nil { return nil, err } @@ -81,9 +80,7 @@ func (s *friendServer) AddBlack(ctx context.Context, req *relation.AddBlackReq) if err := s.webhookBeforeAddBlack(ctx, &s.config.WebhooksConfig.BeforeAddBlack, req); err != nil { return nil, err } - - _, err := rpcclient.GetUsersInfo(ctx, []string{req.OwnerUserID, req.BlackUserID}) - if err != nil { + if err := s.userClient.CheckUser(ctx, []string{req.OwnerUserID, req.BlackUserID}); err != nil { return nil, err } black := model.Black{ @@ -114,7 +111,7 @@ func (s *friendServer) GetSpecifiedBlacks(ctx context.Context, req *relation.Get return nil, errs.ErrArgs.WrapMsg("userIDList repeated") } - userMap, err := rpcclient.GetPublicUserInfoMap(ctx, req.UserIDList) + userMap, err := s.userClient.GetUsersInfoMap(ctx, req.UserIDList) if err != nil { return nil, err } @@ -132,13 +129,26 @@ func (s *friendServer) GetSpecifiedBlacks(ctx context.Context, req *relation.Get Blacks: make([]*sdkws.BlackInfo, 0, len(req.UserIDList)), } + toPublcUser := func(userID string) *sdkws.PublicUserInfo { + v, ok := userMap[userID] + if !ok { + return nil + } + return &sdkws.PublicUserInfo{ + UserID: v.UserID, + Nickname: v.Nickname, + FaceURL: v.FaceURL, + Ex: v.Ex, + } + } + for _, userID := range req.UserIDList { if black := blackMap[userID]; black != nil { resp.Blacks = append(resp.Blacks, &sdkws.BlackInfo{ OwnerUserID: black.OwnerUserID, CreateTime: black.CreateTime.UnixMilli(), - BlackUserInfo: userMap[userID], + BlackUserInfo: toPublcUser(userID), AddSource: black.AddSource, OperatorUserID: black.OperatorUserID, Ex: black.Ex, diff --git a/internal/rpc/relation/friend.go b/internal/rpc/relation/friend.go index b97a9d01f..79db14970 100644 --- a/internal/rpc/relation/friend.go +++ b/internal/rpc/relation/friend.go @@ -16,6 +16,7 @@ package relation import ( "context" + "github.com/openimsdk/open-im-server/v3/pkg/rpcli" "github.com/openimsdk/tools/mq/memamq" @@ -31,7 +32,6 @@ import ( "github.com/openimsdk/open-im-server/v3/pkg/common/convert" "github.com/openimsdk/open-im-server/v3/pkg/common/servererrs" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/controller" - "github.com/openimsdk/open-im-server/v3/pkg/rpcclient" "github.com/openimsdk/protocol/constant" "github.com/openimsdk/protocol/relation" "github.com/openimsdk/protocol/sdkws" @@ -51,6 +51,7 @@ type friendServer struct { config *Config webhookClient *webhook.Client queue *memamq.MemoryQueue + userClient *rpcli.UserClient } type Config struct { @@ -90,10 +91,21 @@ func Start(ctx context.Context, config *Config, client discovery.SvcDiscoveryReg return err } + userConn, err := client.GetConn(ctx, config.Discovery.RpcService.User) + if err != nil { + return err + } + msgConn, err := client.GetConn(ctx, config.Discovery.RpcService.Msg) + if err != nil { + return err + } + userClient := rpcli.NewUserClient(userConn) + // Initialize notification sender notificationSender := NewFriendNotificationSender( &config.NotificationConfig, - WithRpcFunc(rpcclient.GetUsersInfo), + rpcli.NewMsgClient(msgConn), + WithRpcFunc(userClient.GetUsersInfo), ) localcache.InitLocalCache(&config.LocalCacheConfig) @@ -114,6 +126,7 @@ func Start(ctx context.Context, config *Config, client discovery.SvcDiscoveryReg config: config, webhookClient: webhook.NewWebhookClient(config.WebhooksConfig.URL), queue: memamq.NewMemoryQueue(16, 1024*1024), + userClient: userClient, }) return nil } @@ -130,7 +143,7 @@ func (s *friendServer) ApplyToAddFriend(ctx context.Context, req *relation.Apply if err = s.webhookBeforeAddFriend(ctx, &s.config.WebhooksConfig.BeforeAddFriend, req); err != nil && err != servererrs.ErrCallbackContinue { return nil, err } - if _, err := rpcclient.GetUsersInfoMap(ctx, []string{req.ToUserID, req.FromUserID}); err != nil { + if err := s.userClient.CheckUser(ctx, []string{req.ToUserID, req.FromUserID}); err != nil { return nil, err } @@ -155,7 +168,7 @@ func (s *friendServer) ImportFriends(ctx context.Context, req *relation.ImportFr return nil, err } - if _, err := rpcclient.GetUsersInfo(ctx, append([]string{req.OwnerUserID}, req.FriendUserIDs...)); err != nil { + if err := s.userClient.CheckUser(ctx, append([]string{req.OwnerUserID}, req.FriendUserIDs...)); err != nil { return nil, err } if datautil.Contain(req.OwnerUserID, req.FriendUserIDs...) { @@ -296,7 +309,7 @@ func (s *friendServer) getFriend(ctx context.Context, ownerUserID string, friend if err != nil { return nil, err } - return convert.FriendsDB2Pb(ctx, friends, rpcclient.GetUsersInfoMap) + return convert.FriendsDB2Pb(ctx, friends, s.userClient.GetUsersInfoMap) } // Get the list of friend requests sent out proactively. @@ -308,7 +321,7 @@ func (s *friendServer) GetDesignatedFriendsApply(ctx context.Context, return nil, err } resp = &relation.GetDesignatedFriendsApplyResp{} - resp.FriendRequests, err = convert.FriendRequestDB2Pb(ctx, friendRequests, rpcclient.GetUsersInfoMap) + resp.FriendRequests, err = convert.FriendRequestDB2Pb(ctx, friendRequests, s.userClient.GetUsersInfoMap) if err != nil { return nil, err } @@ -327,7 +340,7 @@ func (s *friendServer) GetPaginationFriendsApplyTo(ctx context.Context, req *rel } resp = &relation.GetPaginationFriendsApplyToResp{} - resp.FriendRequests, err = convert.FriendRequestDB2Pb(ctx, friendRequests, rpcclient.GetUsersInfoMap) + resp.FriendRequests, err = convert.FriendRequestDB2Pb(ctx, friendRequests, s.userClient.GetUsersInfoMap) if err != nil { return nil, err } @@ -349,7 +362,7 @@ func (s *friendServer) GetPaginationFriendsApplyFrom(ctx context.Context, req *r return nil, err } - resp.FriendRequests, err = convert.FriendRequestDB2Pb(ctx, friendRequests, rpcclient.GetUsersInfoMap) + resp.FriendRequests, err = convert.FriendRequestDB2Pb(ctx, friendRequests, s.userClient.GetUsersInfoMap) if err != nil { return nil, err } @@ -380,7 +393,7 @@ func (s *friendServer) GetPaginationFriends(ctx context.Context, req *relation.G } resp = &relation.GetPaginationFriendsResp{} - resp.FriendsInfo, err = convert.FriendsDB2Pb(ctx, friends, rpcclient.GetUsersInfoMap) + resp.FriendsInfo, err = convert.FriendsDB2Pb(ctx, friends, s.userClient.GetUsersInfoMap) if err != nil { return nil, err } @@ -413,7 +426,7 @@ func (s *friendServer) GetSpecifiedFriendsInfo(ctx context.Context, req *relatio return nil, errs.ErrArgs.WrapMsg("userIDList repeated") } - userMap, err := rpcclient.GetUsersInfoMap(ctx, req.UserIDList) + userMap, err := s.userClient.GetUsersInfoMap(ctx, req.UserIDList) if err != nil { return nil, err } @@ -517,18 +530,3 @@ func (s *friendServer) UpdateFriends( s.notificationSender.FriendsInfoUpdateNotification(ctx, req.OwnerUserID, req.FriendUserIDs) return resp, nil } - -func (s *friendServer) GetIncrementalFriendsApplyTo(ctx context.Context, req *relation.GetIncrementalFriendsApplyToReq) (*relation.GetIncrementalFriendsApplyToResp, error) { - // TODO implement me - return nil, nil -} - -func (s *friendServer) GetIncrementalFriendsApplyFrom(ctx context.Context, req *relation.GetIncrementalFriendsApplyFromReq) (*relation.GetIncrementalFriendsApplyFromResp, error) { - // TODO implement me - return nil, nil -} - -func (s *friendServer) GetIncrementalBlacks(ctx context.Context, req *relation.GetIncrementalBlacksReq) (*relation.GetIncrementalBlacksResp, error) { - // TODO implement me - return nil, nil -} diff --git a/internal/rpc/relation/notification.go b/internal/rpc/relation/notification.go index ba4365200..a34a4d322 100644 --- a/internal/rpc/relation/notification.go +++ b/internal/rpc/relation/notification.go @@ -16,6 +16,8 @@ package relation import ( "context" + "github.com/openimsdk/open-im-server/v3/pkg/rpcli" + "github.com/openimsdk/protocol/msg" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/versionctx" @@ -25,8 +27,8 @@ import ( "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/open-im-server/v3/pkg/common/convert" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/controller" - "github.com/openimsdk/open-im-server/v3/pkg/rpcclient" - "github.com/openimsdk/open-im-server/v3/pkg/rpcclient/notification" + "github.com/openimsdk/open-im-server/v3/pkg/notification" + "github.com/openimsdk/open-im-server/v3/pkg/notification/common_user" "github.com/openimsdk/protocol/constant" "github.com/openimsdk/protocol/relation" "github.com/openimsdk/protocol/sdkws" @@ -36,7 +38,7 @@ import ( type FriendNotificationSender struct { *rpcclient.NotificationSender // Target not found err - getUsersInfo func(ctx context.Context, userIDs []string) ([]notification.CommonUser, error) + getUsersInfo func(ctx context.Context, userIDs []string) ([]common_user.CommonUser, error) // db controller db controller.FriendDatabase } @@ -53,7 +55,7 @@ func WithDBFunc( fn func(ctx context.Context, userIDs []string) (users []*relationtb.User, err error), ) friendNotificationSenderOptions { return func(s *FriendNotificationSender) { - f := func(ctx context.Context, userIDs []string) (result []notification.CommonUser, err error) { + f := func(ctx context.Context, userIDs []string) (result []common_user.CommonUser, err error) { users, err := fn(ctx, userIDs) if err != nil { return nil, err @@ -71,7 +73,7 @@ func WithRpcFunc( fn func(ctx context.Context, userIDs []string) ([]*sdkws.UserInfo, error), ) friendNotificationSenderOptions { return func(s *FriendNotificationSender) { - f := func(ctx context.Context, userIDs []string) (result []notification.CommonUser, err error) { + f := func(ctx context.Context, userIDs []string) (result []common_user.CommonUser, err error) { users, err := fn(ctx, userIDs) if err != nil { return nil, err @@ -85,12 +87,11 @@ func WithRpcFunc( } } -func NewFriendNotificationSender( - conf *config.Notification, - opts ...friendNotificationSenderOptions, -) *FriendNotificationSender { +func NewFriendNotificationSender(conf *config.Notification, msgClient *rpcli.MsgClient, opts ...friendNotificationSenderOptions) *FriendNotificationSender { f := &FriendNotificationSender{ - NotificationSender: rpcclient.NewNotificationSender(conf, rpcclient.WithRpcClient()), + NotificationSender: rpcclient.NewNotificationSender(conf, rpcclient.WithRpcClient(func(ctx context.Context, req *msg.SendMsgReq) (*msg.SendMsgResp, error) { + return msgClient.SendMsg(ctx, req) + })), } for _, opt := range opts { opt(f) diff --git a/internal/rpc/third/log.go b/internal/rpc/third/log.go index 222cfad1d..4d8cbc0bb 100644 --- a/internal/rpc/third/log.go +++ b/internal/rpc/third/log.go @@ -19,11 +19,9 @@ import ( "crypto/rand" "time" - relationtb "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" - "github.com/openimsdk/open-im-server/v3/pkg/rpcclient" - "github.com/openimsdk/open-im-server/v3/pkg/authverify" "github.com/openimsdk/open-im-server/v3/pkg/common/servererrs" + relationtb "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" "github.com/openimsdk/protocol/constant" "github.com/openimsdk/protocol/third" "github.com/openimsdk/tools/errs" @@ -150,7 +148,7 @@ func (t *thirdServer) SearchLogs(ctx context.Context, req *third.SearchLogsReq) for _, log := range logs { userIDs = append(userIDs, log.UserID) } - userMap, err := rpcclient.GetUsersInfoMap(ctx, userIDs) + userMap, err := t.userClient.GetUsersInfoMap(ctx, userIDs) if err != nil { return nil, err } diff --git a/internal/rpc/third/third.go b/internal/rpc/third/third.go index dc964fdb1..0b8ca25a8 100644 --- a/internal/rpc/third/third.go +++ b/internal/rpc/third/third.go @@ -17,6 +17,7 @@ package third import ( "context" "fmt" + "github.com/openimsdk/open-im-server/v3/pkg/rpcli" "time" "github.com/openimsdk/open-im-server/v3/pkg/common/config" @@ -45,6 +46,7 @@ type thirdServer struct { defaultExpire time.Duration config *Config s3 s3.Interface + userClient *rpcli.UserClient } type Config struct { @@ -98,6 +100,10 @@ func Start(ctx context.Context, config *Config, client discovery.SvcDiscoveryReg if err != nil { return err } + userConn, err := client.GetConn(ctx, config.Discovery.RpcService.User) + if err != nil { + return err + } localcache.InitLocalCache(&config.LocalCacheConfig) third.RegisterThirdServer(server, &thirdServer{ thirdDatabase: controller.NewThirdDatabase(redis.NewThirdCache(rdb), logdb), @@ -105,6 +111,7 @@ func Start(ctx context.Context, config *Config, client discovery.SvcDiscoveryReg defaultExpire: time.Hour * 24 * 7, config: config, s3: o, + userClient: rpcli.NewUserClient(userConn), }) return nil } diff --git a/internal/rpc/user/notification.go b/internal/rpc/user/notification.go index 54e5b27d7..03fdf95bd 100644 --- a/internal/rpc/user/notification.go +++ b/internal/rpc/user/notification.go @@ -16,19 +16,21 @@ package user import ( "context" + "github.com/openimsdk/open-im-server/v3/pkg/rpcli" + "github.com/openimsdk/protocol/msg" relationtb "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" - "github.com/openimsdk/open-im-server/v3/pkg/rpcclient/notification" + "github.com/openimsdk/open-im-server/v3/pkg/notification/common_user" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/controller" - "github.com/openimsdk/open-im-server/v3/pkg/rpcclient" + "github.com/openimsdk/open-im-server/v3/pkg/notification" "github.com/openimsdk/protocol/constant" "github.com/openimsdk/protocol/sdkws" ) type UserNotificationSender struct { *rpcclient.NotificationSender - getUsersInfo func(ctx context.Context, userIDs []string) ([]notification.CommonUser, error) + getUsersInfo func(ctx context.Context, userIDs []string) ([]common_user.CommonUser, error) // db controller db controller.UserDatabase } @@ -45,7 +47,7 @@ func WithUserFunc( fn func(ctx context.Context, userIDs []string) (users []*relationtb.User, err error), ) userNotificationSenderOptions { return func(u *UserNotificationSender) { - f := func(ctx context.Context, userIDs []string) (result []notification.CommonUser, err error) { + f := func(ctx context.Context, userIDs []string) (result []common_user.CommonUser, err error) { users, err := fn(ctx, userIDs) if err != nil { return nil, err @@ -59,9 +61,11 @@ func WithUserFunc( } } -func NewUserNotificationSender(config *Config, opts ...userNotificationSenderOptions) *UserNotificationSender { +func NewUserNotificationSender(config *Config, msgClient *rpcli.MsgClient, opts ...userNotificationSenderOptions) *UserNotificationSender { f := &UserNotificationSender{ - NotificationSender: rpcclient.NewNotificationSender(&config.NotificationConfig, rpcclient.WithRpcClient()), + NotificationSender: rpcclient.NewNotificationSender(&config.NotificationConfig, rpcclient.WithRpcClient(func(ctx context.Context, req *msg.SendMsgReq) (*msg.SendMsgResp, error) { + return msgClient.SendMsg(ctx, req) + })), } for _, opt := range opts { opt(f) diff --git a/internal/rpc/user/user.go b/internal/rpc/user/user.go index c2bcabaa1..c5ee3be70 100644 --- a/internal/rpc/user/user.go +++ b/internal/rpc/user/user.go @@ -17,6 +17,7 @@ package user import ( "context" "errors" + "github.com/openimsdk/open-im-server/v3/pkg/rpcli" "math/rand" "strings" "sync" @@ -59,6 +60,8 @@ type userServer struct { RegisterCenter registry.SvcDiscoveryRegistry config *Config webhookClient *webhook.Client + groupClient *rpcli.GroupClient + relationClient *rpcli.RelationClient } type Config struct { @@ -91,6 +94,19 @@ func Start(ctx context.Context, config *Config, client registry.SvcDiscoveryRegi if err != nil { return err } + msgConn, err := client.GetConn(ctx, config.Discovery.RpcService.Msg) + if err != nil { + return err + } + groupConn, err := client.GetConn(ctx, config.Discovery.RpcService.Group) + if err != nil { + return err + } + friendConn, err := client.GetConn(ctx, config.Discovery.RpcService.Friend) + if err != nil { + return err + } + msgClient := rpcli.NewMsgClient(msgConn) userCache := redis.NewUserCacheRedis(rdb, &config.LocalCacheConfig, userDB, redis.GetRocksCacheOptions()) database := controller.NewUserDatabase(userDB, userCache, mgocli.GetTx()) localcache.InitLocalCache(&config.LocalCacheConfig) @@ -98,10 +114,13 @@ func Start(ctx context.Context, config *Config, client registry.SvcDiscoveryRegi online: redis.NewUserOnline(rdb), db: database, RegisterCenter: client, - friendNotificationSender: relation.NewFriendNotificationSender(&config.NotificationConfig, relation.WithDBFunc(database.FindWithError)), - userNotificationSender: NewUserNotificationSender(config, WithUserFunc(database.FindWithError)), + friendNotificationSender: relation.NewFriendNotificationSender(&config.NotificationConfig, msgClient, relation.WithDBFunc(database.FindWithError)), + userNotificationSender: NewUserNotificationSender(config, msgClient, WithUserFunc(database.FindWithError)), config: config, webhookClient: webhook.NewWebhookClient(config.WebhooksConfig.URL), + + groupClient: rpcli.NewGroupClient(groupConn), + relationClient: rpcli.NewRelationClient(friendConn), } pbuser.RegisterUserServer(server, u) return u.db.InitOnce(context.Background(), users) @@ -633,7 +652,7 @@ func (s *userServer) NotificationUserInfoUpdate(ctx context.Context, userID stri wg.Add(len(es)) go func() { defer wg.Done() - _, es[0] = group.NotificationUserInfoUpdateCaller.Invoke(ctx, &group.NotificationUserInfoUpdateReq{ + _, es[0] = s.groupClient.NotificationUserInfoUpdate(ctx, &group.NotificationUserInfoUpdateReq{ UserID: userID, OldUserInfo: oldUserInfo, NewUserInfo: newUserInfo, @@ -642,7 +661,7 @@ func (s *userServer) NotificationUserInfoUpdate(ctx context.Context, userID stri go func() { defer wg.Done() - _, es[1] = friendpb.NotificationUserInfoUpdateCaller.Invoke(ctx, &friendpb.NotificationUserInfoUpdateReq{ + _, es[1] = s.relationClient.NotificationUserInfoUpdate(ctx, &friendpb.NotificationUserInfoUpdateReq{ UserID: userID, OldUserInfo: oldUserInfo, NewUserInfo: newUserInfo, diff --git a/pkg/common/convert/group.go b/pkg/common/convert/group.go index bc2b2f998..5e41599c6 100644 --- a/pkg/common/convert/group.go +++ b/pkg/common/convert/group.go @@ -80,9 +80,18 @@ func Db2PbGroupMember(m *model.GroupMember) *sdkws.GroupMemberFullInfo { } } -func Db2PbGroupRequest(m *model.GroupRequest, user *sdkws.PublicUserInfo, group *sdkws.GroupInfo) *sdkws.GroupRequest { +func Db2PbGroupRequest(m *model.GroupRequest, user *sdkws.UserInfo, group *sdkws.GroupInfo) *sdkws.GroupRequest { + var pu *sdkws.PublicUserInfo + if user != nil { + pu = &sdkws.PublicUserInfo{ + UserID: user.UserID, + Nickname: user.Nickname, + FaceURL: user.FaceURL, + Ex: user.Ex, + } + } return &sdkws.GroupRequest{ - UserInfo: user, + UserInfo: pu, GroupInfo: group, HandleResult: m.HandleResult, ReqMsg: m.ReqMsg, diff --git a/pkg/common/startrpc/start.go b/pkg/common/startrpc/start.go index 089bc3d97..fd46fb45c 100644 --- a/pkg/common/startrpc/start.go +++ b/pkg/common/startrpc/start.go @@ -27,7 +27,6 @@ import ( "time" conf "github.com/openimsdk/open-im-server/v3/pkg/common/config" - "github.com/openimsdk/open-im-server/v3/pkg/rpcclient" "github.com/openimsdk/tools/discovery/etcd" "github.com/openimsdk/tools/utils/datautil" "github.com/openimsdk/tools/utils/jsonutil" @@ -101,10 +100,6 @@ func Start[T any](ctx context.Context, discovery *conf.Discovery, prometheusConf defer client.Close() client.AddOption(mw.GrpcClient(), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithDefaultServiceConfig(fmt.Sprintf(`{"LoadBalancingPolicy": "%s"}`, "round_robin"))) - if err = rpcclient.InitRpcCaller(client, discovery.RpcService); err != nil { - return err - } - // var reg *prometheus.Registry // var metric *grpcprometheus.ServerMetrics if prometheusConfig.Enable { diff --git a/pkg/rpcclient/notification/common.go b/pkg/notification/common_user/common.go similarity index 97% rename from pkg/rpcclient/notification/common.go rename to pkg/notification/common_user/common.go index 09d8b8798..470d9c7a2 100644 --- a/pkg/rpcclient/notification/common.go +++ b/pkg/notification/common_user/common.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package notification +package common_user type CommonUser interface { GetNickname() string diff --git a/pkg/rpcclient/grouphash/grouphash.go b/pkg/notification/grouphash/grouphash.go similarity index 100% rename from pkg/rpcclient/grouphash/grouphash.go rename to pkg/notification/grouphash/grouphash.go diff --git a/pkg/rpcclient/msg.go b/pkg/notification/msg.go similarity index 97% rename from pkg/rpcclient/msg.go rename to pkg/notification/msg.go index ca9b8fc68..0795982c8 100644 --- a/pkg/rpcclient/msg.go +++ b/pkg/notification/msg.go @@ -148,17 +148,17 @@ func WithLocalSendMsg(sendMsg func(ctx context.Context, req *msg.SendMsgReq) (*m } } -func WithRpcClient() NotificationSenderOptions { +func WithRpcClient(sendMsg func(ctx context.Context, req *msg.SendMsgReq) (*msg.SendMsgResp, error)) NotificationSenderOptions { return func(s *NotificationSender) { s.sendMsg = func(ctx context.Context, req *msg.SendMsgReq) (*msg.SendMsgResp, error) { - return msg.SendMsgCaller.Invoke(ctx, req) + return sendMsg(ctx, req) } } } -func WithUserRpcClient() NotificationSenderOptions { +func WithUserRpcClient(getUserInfo func(ctx context.Context, userID string) (*sdkws.UserInfo, error)) NotificationSenderOptions { return func(s *NotificationSender) { - s.getUserInfo = GetUserInfo + s.getUserInfo = getUserInfo } } diff --git a/pkg/rpccache/conversation.go b/pkg/rpccache/conversation.go index ba3690f44..70f5acfd1 100644 --- a/pkg/rpccache/conversation.go +++ b/pkg/rpccache/conversation.go @@ -16,12 +16,11 @@ package rpccache import ( "context" - "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/cachekey" "github.com/openimsdk/open-im-server/v3/pkg/localcache" - pbconv "github.com/openimsdk/protocol/conversation" - "github.com/openimsdk/protocol/rpccall" + "github.com/openimsdk/open-im-server/v3/pkg/rpcli" + pbconversation "github.com/openimsdk/protocol/conversation" "github.com/openimsdk/tools/errs" "github.com/openimsdk/tools/log" "github.com/openimsdk/tools/utils/datautil" @@ -33,10 +32,11 @@ const ( conversationWorkerCount = 20 ) -func NewConversationLocalCache(localCache *config.LocalCache, cli redis.UniversalClient) *ConversationLocalCache { +func NewConversationLocalCache(client *rpcli.ConversationClient, localCache *config.LocalCache, cli redis.UniversalClient) *ConversationLocalCache { lc := localCache.Conversation log.ZDebug(context.Background(), "ConversationLocalCache", "topic", lc.Topic, "slotNum", lc.SlotNum, "slotSize", lc.SlotSize, "enable", lc.Enable()) x := &ConversationLocalCache{ + client: client, local: localcache.New[[]byte]( localcache.WithLocalSlotNum(lc.SlotNum), localcache.WithLocalSlotSize(lc.SlotSize), @@ -52,7 +52,8 @@ func NewConversationLocalCache(localCache *config.LocalCache, cli redis.Universa } type ConversationLocalCache struct { - local localcache.Cache[[]byte] + client *rpcli.ConversationClient + local localcache.Cache[[]byte] } func (c *ConversationLocalCache) GetConversationIDs(ctx context.Context, ownerUserID string) (val []string, err error) { @@ -63,7 +64,7 @@ func (c *ConversationLocalCache) GetConversationIDs(ctx context.Context, ownerUs return resp.ConversationIDs, nil } -func (c *ConversationLocalCache) getConversationIDs(ctx context.Context, ownerUserID string) (val *pbconv.GetConversationIDsResp, err error) { +func (c *ConversationLocalCache) getConversationIDs(ctx context.Context, ownerUserID string) (val *pbconversation.GetConversationIDsResp, err error) { log.ZDebug(ctx, "ConversationLocalCache getConversationIDs req", "ownerUserID", ownerUserID) defer func() { if err == nil { @@ -72,14 +73,14 @@ func (c *ConversationLocalCache) getConversationIDs(ctx context.Context, ownerUs log.ZError(ctx, "ConversationLocalCache getConversationIDs return", err, "ownerUserID", ownerUserID) } }() - var cache cacheProto[pbconv.GetConversationIDsResp] + var cache cacheProto[pbconversation.GetConversationIDsResp] return cache.Unmarshal(c.local.Get(ctx, cachekey.GetConversationIDsKey(ownerUserID), func(ctx context.Context) ([]byte, error) { log.ZDebug(ctx, "ConversationLocalCache getConversationIDs rpc", "ownerUserID", ownerUserID) - return cache.Marshal(pbconv.GetConversationIDsCaller.Invoke(ctx, &pbconv.GetConversationIDsReq{UserID: ownerUserID})) + return cache.Marshal(c.client.ConversationClient.GetConversationIDs(ctx, &pbconversation.GetConversationIDsReq{UserID: ownerUserID})) })) } -func (c *ConversationLocalCache) GetConversation(ctx context.Context, userID, conversationID string) (val *pbconv.Conversation, err error) { +func (c *ConversationLocalCache) GetConversation(ctx context.Context, userID, conversationID string) (val *pbconversation.Conversation, err error) { log.ZDebug(ctx, "ConversationLocalCache GetConversation req", "userID", userID, "conversationID", conversationID) defer func() { if err == nil { @@ -88,13 +89,10 @@ func (c *ConversationLocalCache) GetConversation(ctx context.Context, userID, co log.ZWarn(ctx, "ConversationLocalCache GetConversation return", err, "userID", userID, "conversationID", conversationID) } }() - var cache cacheProto[pbconv.Conversation] + var cache cacheProto[pbconversation.Conversation] return cache.Unmarshal(c.local.Get(ctx, cachekey.GetConversationKey(userID, conversationID), func(ctx context.Context) ([]byte, error) { log.ZDebug(ctx, "ConversationLocalCache GetConversation rpc", "userID", userID, "conversationID", conversationID) - return cache.Marshal(rpccall.ExtractField(ctx, pbconv.GetConversationCaller.Invoke, &pbconv.GetConversationReq{ - ConversationID: conversationID, - OwnerUserID: userID, - }, (*pbconv.GetConversationResp).GetConversation)) + return cache.Marshal(c.client.GetConversation(ctx, conversationID, userID)) })) } @@ -106,10 +104,10 @@ func (c *ConversationLocalCache) GetSingleConversationRecvMsgOpt(ctx context.Con return conv.RecvMsgOpt, nil } -func (c *ConversationLocalCache) GetConversations(ctx context.Context, ownerUserID string, conversationIDs []string) ([]*pbconv.Conversation, error) { +func (c *ConversationLocalCache) GetConversations(ctx context.Context, ownerUserID string, conversationIDs []string) ([]*pbconversation.Conversation, error) { var ( - conversations = make([]*pbconv.Conversation, 0, len(conversationIDs)) - conversationsChan = make(chan *pbconv.Conversation, len(conversationIDs)) + conversations = make([]*pbconversation.Conversation, 0, len(conversationIDs)) + conversationsChan = make(chan *pbconversation.Conversation, len(conversationIDs)) ) g, ctx := errgroup.WithContext(ctx) @@ -139,7 +137,7 @@ func (c *ConversationLocalCache) GetConversations(ctx context.Context, ownerUser return conversations, nil } -func (c *ConversationLocalCache) getConversationNotReceiveMessageUserIDs(ctx context.Context, conversationID string) (val *pbconv.GetConversationNotReceiveMessageUserIDsResp, err error) { +func (c *ConversationLocalCache) getConversationNotReceiveMessageUserIDs(ctx context.Context, conversationID string) (val *pbconversation.GetConversationNotReceiveMessageUserIDsResp, err error) { log.ZDebug(ctx, "ConversationLocalCache getConversationNotReceiveMessageUserIDs req", "conversationID", conversationID) defer func() { if err == nil { @@ -148,10 +146,10 @@ func (c *ConversationLocalCache) getConversationNotReceiveMessageUserIDs(ctx con log.ZError(ctx, "ConversationLocalCache getConversationNotReceiveMessageUserIDs return", err, "conversationID", conversationID) } }() - var cache cacheProto[pbconv.GetConversationNotReceiveMessageUserIDsResp] + var cache cacheProto[pbconversation.GetConversationNotReceiveMessageUserIDsResp] return cache.Unmarshal(c.local.Get(ctx, cachekey.GetConversationNotReceiveMessageUserIDsKey(conversationID), func(ctx context.Context) ([]byte, error) { log.ZDebug(ctx, "ConversationLocalCache getConversationNotReceiveMessageUserIDs rpc", "conversationID", conversationID) - return cache.Marshal(pbconv.GetConversationNotReceiveMessageUserIDsCaller.Invoke(ctx, &pbconv.GetConversationNotReceiveMessageUserIDsReq{ConversationID: conversationID})) + return cache.Marshal(c.client.ConversationClient.GetConversationNotReceiveMessageUserIDs(ctx, &pbconversation.GetConversationNotReceiveMessageUserIDsReq{ConversationID: conversationID})) })) } diff --git a/pkg/rpccache/friend.go b/pkg/rpccache/friend.go index 865cac7b5..8ed6c1ae9 100644 --- a/pkg/rpccache/friend.go +++ b/pkg/rpccache/friend.go @@ -16,8 +16,8 @@ package rpccache import ( "context" - "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/cachekey" + "github.com/openimsdk/open-im-server/v3/pkg/rpcli" "github.com/openimsdk/protocol/relation" "github.com/openimsdk/open-im-server/v3/pkg/common/config" @@ -26,10 +26,11 @@ import ( "github.com/redis/go-redis/v9" ) -func NewFriendLocalCache(localCache *config.LocalCache, cli redis.UniversalClient) *FriendLocalCache { +func NewFriendLocalCache(client *rpcli.RelationClient, localCache *config.LocalCache, cli redis.UniversalClient) *FriendLocalCache { lc := localCache.Friend log.ZDebug(context.Background(), "FriendLocalCache", "topic", lc.Topic, "slotNum", lc.SlotNum, "slotSize", lc.SlotSize, "enable", lc.Enable()) x := &FriendLocalCache{ + client: client, local: localcache.New[[]byte]( localcache.WithLocalSlotNum(lc.SlotNum), localcache.WithLocalSlotSize(lc.SlotSize), @@ -45,7 +46,8 @@ func NewFriendLocalCache(localCache *config.LocalCache, cli redis.UniversalClien } type FriendLocalCache struct { - local localcache.Cache[[]byte] + client *rpcli.RelationClient + local localcache.Cache[[]byte] } func (f *FriendLocalCache) IsFriend(ctx context.Context, possibleFriendUserID, userID string) (val bool, err error) { @@ -68,7 +70,7 @@ func (f *FriendLocalCache) isFriend(ctx context.Context, possibleFriendUserID, u var cache cacheProto[relation.IsFriendResp] return cache.Unmarshal(f.local.GetLink(ctx, cachekey.GetIsFriendKey(possibleFriendUserID, userID), func(ctx context.Context) ([]byte, error) { log.ZDebug(ctx, "FriendLocalCache isFriend rpc", "possibleFriendUserID", possibleFriendUserID, "userID", userID) - return cache.Marshal(relation.IsFriendCaller.Invoke(ctx, &relation.IsFriendReq{UserID1: userID, UserID2: possibleFriendUserID})) + return cache.Marshal(f.client.FriendClient.IsFriend(ctx, &relation.IsFriendReq{UserID1: userID, UserID2: possibleFriendUserID})) }, cachekey.GetFriendIDsKey(possibleFriendUserID))) } @@ -94,6 +96,6 @@ func (f *FriendLocalCache) isBlack(ctx context.Context, possibleBlackUserID, use var cache cacheProto[relation.IsBlackResp] return cache.Unmarshal(f.local.GetLink(ctx, cachekey.GetIsBlackIDsKey(possibleBlackUserID, userID), func(ctx context.Context) ([]byte, error) { log.ZDebug(ctx, "FriendLocalCache IsBlack rpc", "possibleBlackUserID", possibleBlackUserID, "userID", userID) - return cache.Marshal(relation.IsBlackCaller.Invoke(ctx, &relation.IsBlackReq{UserID1: possibleBlackUserID, UserID2: userID})) + return cache.Marshal(f.client.FriendClient.IsBlack(ctx, &relation.IsBlackReq{UserID1: possibleBlackUserID, UserID2: userID})) }, cachekey.GetBlackIDsKey(userID))) } diff --git a/pkg/rpccache/group.go b/pkg/rpccache/group.go index 111813103..174ba7dc5 100644 --- a/pkg/rpccache/group.go +++ b/pkg/rpccache/group.go @@ -16,10 +16,9 @@ package rpccache import ( "context" - "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/cachekey" + "github.com/openimsdk/open-im-server/v3/pkg/rpcli" "github.com/openimsdk/protocol/group" - "github.com/openimsdk/protocol/rpccall" "github.com/openimsdk/tools/utils/datautil" "github.com/openimsdk/open-im-server/v3/pkg/common/config" @@ -30,10 +29,11 @@ import ( "github.com/redis/go-redis/v9" ) -func NewGroupLocalCache(localCache *config.LocalCache, cli redis.UniversalClient) *GroupLocalCache { +func NewGroupLocalCache(client *rpcli.GroupClient, localCache *config.LocalCache, cli redis.UniversalClient) *GroupLocalCache { lc := localCache.Group log.ZDebug(context.Background(), "GroupLocalCache", "topic", lc.Topic, "slotNum", lc.SlotNum, "slotSize", lc.SlotSize, "enable", lc.Enable()) x := &GroupLocalCache{ + client: client, local: localcache.New[[]byte]( localcache.WithLocalSlotNum(lc.SlotNum), localcache.WithLocalSlotSize(lc.SlotSize), @@ -49,7 +49,8 @@ func NewGroupLocalCache(localCache *config.LocalCache, cli redis.UniversalClient } type GroupLocalCache struct { - local localcache.Cache[[]byte] + client *rpcli.GroupClient + local localcache.Cache[[]byte] } func (g *GroupLocalCache) getGroupMemberIDs(ctx context.Context, groupID string) (val *group.GetGroupMemberUserIDsResp, err error) { @@ -64,7 +65,7 @@ func (g *GroupLocalCache) getGroupMemberIDs(ctx context.Context, groupID string) var cache cacheProto[group.GetGroupMemberUserIDsResp] return cache.Unmarshal(g.local.Get(ctx, cachekey.GetGroupMemberIDsKey(groupID), func(ctx context.Context) ([]byte, error) { log.ZDebug(ctx, "GroupLocalCache getGroupMemberIDs rpc", "groupID", groupID) - return cache.Marshal(group.GetGroupMemberUserIDsCaller.Invoke(ctx, &group.GetGroupMemberUserIDsReq{GroupID: groupID})) + return cache.Marshal(g.client.GroupClient.GetGroupMemberUserIDs(ctx, &group.GetGroupMemberUserIDsReq{GroupID: groupID})) })) } @@ -80,13 +81,7 @@ func (g *GroupLocalCache) GetGroupMember(ctx context.Context, groupID, userID st var cache cacheProto[sdkws.GroupMemberFullInfo] return cache.Unmarshal(g.local.Get(ctx, cachekey.GetGroupMemberInfoKey(groupID, userID), func(ctx context.Context) ([]byte, error) { log.ZDebug(ctx, "GroupLocalCache GetGroupInfo rpc", "groupID", groupID, "userID", userID) - return cache.Marshal(rpccall.ExtractField(ctx, group.GetGroupMemberCacheCaller.Invoke, - &group.GetGroupMemberCacheReq{ - GroupID: groupID, - GroupMemberID: userID, - }, - (*group.GetGroupMemberCacheResp).GetMember, - )) + return cache.Marshal(g.client.GetGroupMemberCache(ctx, groupID, userID)) })) } @@ -102,10 +97,7 @@ func (g *GroupLocalCache) GetGroupInfo(ctx context.Context, groupID string) (val var cache cacheProto[sdkws.GroupInfo] return cache.Unmarshal(g.local.Get(ctx, cachekey.GetGroupInfoKey(groupID), func(ctx context.Context) ([]byte, error) { log.ZDebug(ctx, "GroupLocalCache GetGroupInfo rpc", "groupID", groupID) - return cache.Marshal(rpccall.ExtractField(ctx, group.GetGroupInfoCacheCaller.Invoke, - &group.GetGroupInfoCacheReq{ - GroupID: groupID, - }, (*group.GetGroupInfoCacheResp).GetGroupInfo)) + return cache.Marshal(g.client.GetGroupInfoCache(ctx, groupID)) })) } diff --git a/pkg/rpccache/online.go b/pkg/rpccache/online.go index 25362b529..8f5323477 100644 --- a/pkg/rpccache/online.go +++ b/pkg/rpccache/online.go @@ -3,16 +3,15 @@ package rpccache import ( "context" "fmt" + "github.com/openimsdk/open-im-server/v3/pkg/rpcli" + "github.com/openimsdk/protocol/constant" + "github.com/openimsdk/protocol/user" "math/rand" "strconv" "sync" "sync/atomic" "time" - "github.com/openimsdk/protocol/constant" - "github.com/openimsdk/protocol/rpccall" - "github.com/openimsdk/protocol/user" - "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/cachekey" "github.com/openimsdk/open-im-server/v3/pkg/localcache" "github.com/openimsdk/open-im-server/v3/pkg/localcache/lru" @@ -23,10 +22,10 @@ import ( "github.com/redis/go-redis/v9" ) -func NewOnlineCache(adminUserID []string, group *GroupLocalCache, rdb redis.UniversalClient, fullUserCache bool, fn func(ctx context.Context, userID string, platformIDs []int32)) (*OnlineCache, error) { +func NewOnlineCache(client *rpcli.UserClient, group *GroupLocalCache, rdb redis.UniversalClient, fullUserCache bool, fn func(ctx context.Context, userID string, platformIDs []int32)) (*OnlineCache, error) { l := &sync.Mutex{} x := &OnlineCache{ - adminUserID: adminUserID, + client: client, group: group, fullUserCache: fullUserCache, Lock: l, @@ -66,8 +65,8 @@ const ( ) type OnlineCache struct { - adminUserID []string - group *GroupLocalCache + client *rpcli.UserClient + group *GroupLocalCache // fullUserCache if enabled, caches the online status of all users using mapCache; // otherwise, only a portion of users' online statuses (regardless of whether they are online) will be cached using lruCache. @@ -113,7 +112,7 @@ func (o *OnlineCache) initUsersOnlineStatus(ctx context.Context) (err error) { cursor := uint64(0) for resp == nil || resp.NextCursor != 0 { if err = retryOperation(func() error { - resp, err = user.GetAllOnlineUsersCaller.Invoke(ctx, &user.GetAllOnlineUsersReq{Cursor: cursor}) + resp, err = o.client.GetAllOnlineUsers(ctx, cursor) if err != nil { return err } @@ -187,17 +186,7 @@ func (o *OnlineCache) doSubscribe(ctx context.Context, rdb redis.UniversalClient func (o *OnlineCache) getUserOnlinePlatform(ctx context.Context, userID string) ([]int32, error) { platformIDs, err := o.lruCache.Get(userID, func() ([]int32, error) { - resp, err := rpccall.ExtractField(ctx, user.GetUserStatusCaller.Invoke, &user.GetUserStatusReq{ - UserID: o.adminUserID[0], - UserIDs: []string{userID}, - }, (*user.GetUserStatusResp).GetStatusList) - if err != nil { - return nil, err - } - if len(resp) == 0 { - return nil, nil - } - return resp[0].PlatformIDs, nil + return o.client.GetUserOnlinePlatform(ctx, userID) }) if err != nil { log.ZError(ctx, "OnlineCache GetUserOnlinePlatform", err, "userID", userID) @@ -238,11 +227,7 @@ func (o *OnlineCache) GetUserOnline(ctx context.Context, userID string) (bool, e func (o *OnlineCache) getUserOnlinePlatformBatch(ctx context.Context, userIDs []string) (map[string][]int32, error) { platformIDsMap, err := o.lruCache.GetBatch(userIDs, func(missingUsers []string) (map[string][]int32, error) { platformIDsMap := make(map[string][]int32) - - usersStatus, err := rpccall.ExtractField(ctx, user.GetUserStatusCaller.Invoke, &user.GetUserStatusReq{ - UserID: o.adminUserID[0], - UserIDs: missingUsers, - }, (*user.GetUserStatusResp).GetStatusList) + usersStatus, err := o.client.GetUsersOnlinePlatform(ctx, missingUsers) if err != nil { return nil, err } diff --git a/pkg/rpccache/subscriber.go b/pkg/rpccache/subscriber.go index 44e1f5885..d28d1aa29 100644 --- a/pkg/rpccache/subscriber.go +++ b/pkg/rpccache/subscriber.go @@ -17,7 +17,6 @@ package rpccache import ( "context" "encoding/json" - "github.com/openimsdk/tools/log" "github.com/redis/go-redis/v9" ) diff --git a/pkg/rpccache/user.go b/pkg/rpccache/user.go index fce2c911a..cb0704906 100644 --- a/pkg/rpccache/user.go +++ b/pkg/rpccache/user.go @@ -16,11 +16,11 @@ package rpccache import ( "context" + "github.com/openimsdk/open-im-server/v3/pkg/rpcli" "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/cachekey" "github.com/openimsdk/open-im-server/v3/pkg/localcache" - "github.com/openimsdk/open-im-server/v3/pkg/rpcclient" "github.com/openimsdk/protocol/sdkws" "github.com/openimsdk/protocol/user" "github.com/openimsdk/tools/errs" @@ -28,10 +28,11 @@ import ( "github.com/redis/go-redis/v9" ) -func NewUserLocalCache(localCache *config.LocalCache, cli redis.UniversalClient) *UserLocalCache { +func NewUserLocalCache(client *rpcli.UserClient, localCache *config.LocalCache, cli redis.UniversalClient) *UserLocalCache { lc := localCache.User log.ZDebug(context.Background(), "UserLocalCache", "topic", lc.Topic, "slotNum", lc.SlotNum, "slotSize", lc.SlotSize, "enable", lc.Enable()) x := &UserLocalCache{ + client: client, local: localcache.New[[]byte]( localcache.WithLocalSlotNum(lc.SlotNum), localcache.WithLocalSlotSize(lc.SlotSize), @@ -47,7 +48,8 @@ func NewUserLocalCache(localCache *config.LocalCache, cli redis.UniversalClient) } type UserLocalCache struct { - local localcache.Cache[[]byte] + client *rpcli.UserClient + local localcache.Cache[[]byte] } func (u *UserLocalCache) GetUserInfo(ctx context.Context, userID string) (val *sdkws.UserInfo, err error) { @@ -62,7 +64,7 @@ func (u *UserLocalCache) GetUserInfo(ctx context.Context, userID string) (val *s var cache cacheProto[sdkws.UserInfo] return cache.Unmarshal(u.local.Get(ctx, cachekey.GetUserInfoKey(userID), func(ctx context.Context) ([]byte, error) { log.ZDebug(ctx, "UserLocalCache GetUserInfo rpc", "userID", userID) - return cache.Marshal(rpcclient.GetUserInfo(ctx, userID)) + return cache.Marshal(u.client.GetUserInfo(ctx, userID)) })) } @@ -86,7 +88,7 @@ func (u *UserLocalCache) getUserGlobalMsgRecvOpt(ctx context.Context, userID str var cache cacheProto[user.GetGlobalRecvMessageOptResp] return cache.Unmarshal(u.local.Get(ctx, cachekey.GetUserGlobalRecvMsgOptKey(userID), func(ctx context.Context) ([]byte, error) { log.ZDebug(ctx, "UserLocalCache GetUserGlobalMsgRecvOpt rpc", "userID", userID) - return cache.Marshal(user.GetGlobalRecvMessageOptCaller.Invoke(ctx, &user.GetGlobalRecvMessageOptReq{UserID: userID})) + return cache.Marshal(u.client.UserClient.GetGlobalRecvMessageOpt(ctx, &user.GetGlobalRecvMessageOptReq{UserID: userID})) })) } diff --git a/pkg/rpcclient/doc.go b/pkg/rpcclient/doc.go deleted file mode 100644 index 66b0aee91..000000000 --- a/pkg/rpcclient/doc.go +++ /dev/null @@ -1,15 +0,0 @@ -// Copyright © 2024 OpenIM. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package rpcclient // import "github.com/openimsdk/open-im-server/v3/pkg/rpcclient" diff --git a/pkg/rpcclient/grouphash/doc.go b/pkg/rpcclient/grouphash/doc.go deleted file mode 100644 index c780701d9..000000000 --- a/pkg/rpcclient/grouphash/doc.go +++ /dev/null @@ -1,15 +0,0 @@ -// Copyright © 2024 OpenIM. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package grouphash // import "github.com/openimsdk/open-im-server/v3/pkg/rpcclient/grouphash" diff --git a/pkg/rpcclient/init.go b/pkg/rpcclient/init.go deleted file mode 100644 index 3d3f68aef..000000000 --- a/pkg/rpcclient/init.go +++ /dev/null @@ -1,60 +0,0 @@ -package rpcclient - -import ( - "context" - - "github.com/openimsdk/open-im-server/v3/pkg/common/config" - pbauth "github.com/openimsdk/protocol/auth" - pbconversation "github.com/openimsdk/protocol/conversation" - pbgroup "github.com/openimsdk/protocol/group" - pbmsg "github.com/openimsdk/protocol/msg" - pbmsggateway "github.com/openimsdk/protocol/msggateway" - pbpush "github.com/openimsdk/protocol/push" - pbrelation "github.com/openimsdk/protocol/relation" - pbthird "github.com/openimsdk/protocol/third" - pbuser "github.com/openimsdk/protocol/user" - "github.com/openimsdk/tools/discovery" - "github.com/openimsdk/tools/system/program" - "google.golang.org/grpc" -) - -func InitRpcCaller(discov discovery.SvcDiscoveryRegistry, service config.RpcService) error { - initConn := func(discov discovery.SvcDiscoveryRegistry, name string, initFunc func(conn *grpc.ClientConn)) error { - conn, err := discov.GetConn(context.Background(), name) - if err != nil { - program.ExitWithError(err) - return err - } - initFunc(conn) - return nil - } - if err := initConn(discov, service.Auth, pbauth.InitAuth); err != nil { - return err - } - if err := initConn(discov, service.Conversation, pbconversation.InitConversation); err != nil { - return err - } - if err := initConn(discov, service.Group, pbgroup.InitGroup); err != nil { - return err - } - if err := initConn(discov, service.Msg, pbmsg.InitMsg); err != nil { - return err - } - if err := initConn(discov, service.MessageGateway, pbmsggateway.InitMsgGateway); err != nil { - return err - } - if err := initConn(discov, service.Push, pbpush.InitPushMsgService); err != nil { - return err - } - if err := initConn(discov, service.Friend, pbrelation.InitFriend); err != nil { - return err - } - if err := initConn(discov, service.Third, pbthird.InitThird); err != nil { - return err - } - if err := initConn(discov, service.User, pbuser.InitUser); err != nil { - return err - } - - return nil -} diff --git a/pkg/rpcclient/notification/doc.go b/pkg/rpcclient/notification/doc.go deleted file mode 100644 index 8ce57ca4e..000000000 --- a/pkg/rpcclient/notification/doc.go +++ /dev/null @@ -1,15 +0,0 @@ -// Copyright © 2024 OpenIM. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package notification // import "github.com/openimsdk/open-im-server/v3/pkg/rpcclient/notification" diff --git a/pkg/rpcclient/user.go b/pkg/rpcclient/user.go deleted file mode 100644 index 463dd9a39..000000000 --- a/pkg/rpcclient/user.go +++ /dev/null @@ -1,109 +0,0 @@ -// Copyright © 2023 OpenIM. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package rpcclient - -import ( - "context" - "strings" - - "github.com/openimsdk/open-im-server/v3/pkg/common/servererrs" - "github.com/openimsdk/protocol/sdkws" - "github.com/openimsdk/protocol/user" - "github.com/openimsdk/tools/utils/datautil" -) - -// GetUsersInfo retrieves information for multiple users based on their user IDs. -func GetUsersInfo(ctx context.Context, userIDs []string) ([]*sdkws.UserInfo, error) { - if len(userIDs) == 0 { - return []*sdkws.UserInfo{}, nil - } - resp, err := user.GetDesignateUsersCaller.Invoke(ctx, &user.GetDesignateUsersReq{ - UserIDs: userIDs, - }) - if err != nil { - return nil, err - } - if ids := datautil.Single(userIDs, datautil.Slice(resp.UsersInfo, func(e *sdkws.UserInfo) string { - return e.UserID - })); len(ids) > 0 { - return nil, servererrs.ErrUserIDNotFound.WrapMsg(strings.Join(ids, ",")) - } - return resp.UsersInfo, nil -} - -// GetUserInfo retrieves information for a single user based on the provided user ID. -func GetUserInfo(ctx context.Context, userID string) (*sdkws.UserInfo, error) { - users, err := GetUsersInfo(ctx, []string{userID}) - if err != nil { - return nil, err - } - return users[0], nil -} - -// GetUsersInfoMap retrieves a map of user information indexed by their user IDs. -func GetUsersInfoMap(ctx context.Context, userIDs []string) (map[string]*sdkws.UserInfo, error) { - users, err := GetUsersInfo(ctx, userIDs) - if err != nil { - return nil, err - } - return datautil.SliceToMap(users, func(e *sdkws.UserInfo) string { - return e.UserID - }), nil -} - -// GetPublicUserInfos retrieves public information for multiple users based on their user IDs. -func GetPublicUserInfos( - ctx context.Context, - userIDs []string, -) ([]*sdkws.PublicUserInfo, error) { - users, err := GetUsersInfo(ctx, userIDs) - if err != nil { - return nil, err - } - - return datautil.Slice(users, func(e *sdkws.UserInfo) *sdkws.PublicUserInfo { - return &sdkws.PublicUserInfo{ - UserID: e.UserID, - Nickname: e.Nickname, - FaceURL: e.FaceURL, - Ex: e.Ex, - } - }), nil -} - -// GetPublicUserInfo retrieves public information for a single user based on the provided user ID. -func GetPublicUserInfo(ctx context.Context, userID string) (*sdkws.PublicUserInfo, error) { - users, err := GetPublicUserInfos(ctx, []string{userID}) - if err != nil { - return nil, err - } - - return users[0], nil -} - -// GetPublicUserInfoMap retrieves a map of public user information indexed by their user IDs. -func GetPublicUserInfoMap( - ctx context.Context, - userIDs []string, -) (map[string]*sdkws.PublicUserInfo, error) { - users, err := GetPublicUserInfos(ctx, userIDs) - if err != nil { - return nil, err - } - - return datautil.SliceToMap(users, func(e *sdkws.PublicUserInfo) string { - return e.UserID - }), nil -} diff --git a/pkg/rpcli/auth.go b/pkg/rpcli/auth.go new file mode 100644 index 000000000..17fc6ea28 --- /dev/null +++ b/pkg/rpcli/auth.go @@ -0,0 +1,30 @@ +package rpcli + +import ( + "context" + "github.com/openimsdk/protocol/auth" + "google.golang.org/grpc" +) + +func NewAuthClient(cc grpc.ClientConnInterface) *AuthClient { + return &AuthClient{auth.NewAuthClient(cc)} +} + +type AuthClient struct { + auth.AuthClient +} + +func (x *AuthClient) KickTokens(ctx context.Context, tokens []string) error { + if len(tokens) == 0 { + return nil + } + return ignoreResp(x.AuthClient.KickTokens(ctx, &auth.KickTokensReq{Tokens: tokens})) +} + +func (x *AuthClient) InvalidateToken(ctx context.Context, req *auth.InvalidateTokenReq) error { + return ignoreResp(x.AuthClient.InvalidateToken(ctx, req)) +} + +func (x *AuthClient) ParseToken(ctx context.Context, token string) (*auth.ParseTokenResp, error) { + return x.AuthClient.ParseToken(ctx, &auth.ParseTokenReq{Token: token}) +} diff --git a/pkg/rpcli/conversation.go b/pkg/rpcli/conversation.go new file mode 100644 index 000000000..ba5b90cb3 --- /dev/null +++ b/pkg/rpcli/conversation.go @@ -0,0 +1,94 @@ +package rpcli + +import ( + "context" + "github.com/openimsdk/protocol/conversation" + "google.golang.org/grpc" +) + +func NewConversationClient(cc grpc.ClientConnInterface) *ConversationClient { + return &ConversationClient{conversation.NewConversationClient(cc)} +} + +type ConversationClient struct { + conversation.ConversationClient +} + +func (x *ConversationClient) SetConversationMaxSeq(ctx context.Context, conversationID string, ownerUserIDs []string, maxSeq int64) error { + if len(ownerUserIDs) == 0 { + return nil + } + req := &conversation.SetConversationMaxSeqReq{ConversationID: conversationID, OwnerUserID: ownerUserIDs, MaxSeq: maxSeq} + return ignoreResp(x.ConversationClient.SetConversationMaxSeq(ctx, req)) +} + +func (x *ConversationClient) SetConversations(ctx context.Context, ownerUserIDs []string, info *conversation.ConversationReq) error { + if len(ownerUserIDs) == 0 { + return nil + } + req := &conversation.SetConversationsReq{UserIDs: ownerUserIDs, Conversation: info} + return ignoreResp(x.ConversationClient.SetConversations(ctx, req)) +} + +func (x *ConversationClient) GetConversationsByConversationIDs(ctx context.Context, conversationIDs []string) ([]*conversation.Conversation, error) { + if len(conversationIDs) == 0 { + return nil, nil + } + req := &conversation.GetConversationsByConversationIDReq{ConversationIDs: conversationIDs} + return extractField(ctx, x.ConversationClient.GetConversationsByConversationID, req, (*conversation.GetConversationsByConversationIDResp).GetConversations) +} + +func (x *ConversationClient) GetConversationsByConversationID(ctx context.Context, conversationID string) (*conversation.Conversation, error) { + return firstValue(x.GetConversationsByConversationIDs(ctx, []string{conversationID})) +} + +func (x *ConversationClient) SetConversationMinSeq(ctx context.Context, conversationID string, ownerUserIDs []string, minSeq int64) error { + if len(ownerUserIDs) == 0 { + return nil + } + req := &conversation.SetConversationMinSeqReq{ConversationID: conversationID, OwnerUserID: ownerUserIDs, MinSeq: minSeq} + return ignoreResp(x.ConversationClient.SetConversationMinSeq(ctx, req)) +} + +func (x *ConversationClient) GetConversation(ctx context.Context, conversationID string, ownerUserID string) (*conversation.Conversation, error) { + req := &conversation.GetConversationReq{ConversationID: conversationID, OwnerUserID: ownerUserID} + return extractField(ctx, x.ConversationClient.GetConversation, req, (*conversation.GetConversationResp).GetConversation) +} + +func (x *ConversationClient) GetConversations(ctx context.Context, conversationIDs []string, ownerUserID string) ([]*conversation.Conversation, error) { + if len(conversationIDs) == 0 { + return nil, nil + } + req := &conversation.GetConversationsReq{ConversationIDs: conversationIDs, OwnerUserID: ownerUserID} + return extractField(ctx, x.ConversationClient.GetConversations, req, (*conversation.GetConversationsResp).GetConversations) +} + +func (x *ConversationClient) GetConversationIDs(ctx context.Context, ownerUserID string) ([]string, error) { + req := &conversation.GetConversationIDsReq{UserID: ownerUserID} + return extractField(ctx, x.ConversationClient.GetConversationIDs, req, (*conversation.GetConversationIDsResp).GetConversationIDs) +} + +func (x *ConversationClient) GetPinnedConversationIDs(ctx context.Context, ownerUserID string) ([]string, error) { + req := &conversation.GetPinnedConversationIDsReq{UserID: ownerUserID} + return extractField(ctx, x.ConversationClient.GetPinnedConversationIDs, req, (*conversation.GetPinnedConversationIDsResp).GetConversationIDs) +} + +func (x *ConversationClient) CreateGroupChatConversations(ctx context.Context, groupID string, userIDs []string) error { + if len(userIDs) == 0 { + return nil + } + req := &conversation.CreateGroupChatConversationsReq{GroupID: groupID, UserIDs: userIDs} + return ignoreResp(x.ConversationClient.CreateGroupChatConversations(ctx, req)) +} + +func (x *ConversationClient) CreateSingleChatConversations(ctx context.Context, req *conversation.CreateSingleChatConversationsReq) error { + return ignoreResp(x.ConversationClient.CreateSingleChatConversations(ctx, req)) +} + +func (x *ConversationClient) GetConversationOfflinePushUserIDs(ctx context.Context, conversationID string, userIDs []string) ([]string, error) { + if len(userIDs) == 0 { + return nil, nil + } + req := &conversation.GetConversationOfflinePushUserIDsReq{ConversationID: conversationID, UserIDs: userIDs} + return extractField(ctx, x.ConversationClient.GetConversationOfflinePushUserIDs, req, (*conversation.GetConversationOfflinePushUserIDsResp).GetUserIDs) +} diff --git a/pkg/rpcli/group.go b/pkg/rpcli/group.go new file mode 100644 index 000000000..b51283c5d --- /dev/null +++ b/pkg/rpcli/group.go @@ -0,0 +1,48 @@ +package rpcli + +import ( + "context" + "github.com/openimsdk/protocol/group" + "github.com/openimsdk/protocol/sdkws" + "google.golang.org/grpc" +) + +func NewGroupClient(cc grpc.ClientConnInterface) *GroupClient { + return &GroupClient{group.NewGroupClient(cc)} +} + +type GroupClient struct { + group.GroupClient +} + +func (x *GroupClient) GetGroupsInfo(ctx context.Context, groupIDs []string) ([]*sdkws.GroupInfo, error) { + if len(groupIDs) == 0 { + return nil, nil + } + req := &group.GetGroupsInfoReq{GroupIDs: groupIDs} + return extractField(ctx, x.GroupClient.GetGroupsInfo, req, (*group.GetGroupsInfoResp).GetGroupInfos) +} + +func (x *GroupClient) GetGroupInfo(ctx context.Context, groupID string) (*sdkws.GroupInfo, error) { + return firstValue(x.GetGroupsInfo(ctx, []string{groupID})) +} + +func (x *GroupClient) GetGroupInfoCache(ctx context.Context, groupID string) (*sdkws.GroupInfo, error) { + req := &group.GetGroupInfoCacheReq{GroupID: groupID} + return extractField(ctx, x.GroupClient.GetGroupInfoCache, req, (*group.GetGroupInfoCacheResp).GetGroupInfo) +} + +func (x *GroupClient) GetGroupMemberCache(ctx context.Context, groupID string, userID string) (*sdkws.GroupMemberFullInfo, error) { + req := &group.GetGroupMemberCacheReq{GroupID: groupID, GroupMemberID: userID} + return extractField(ctx, x.GroupClient.GetGroupMemberCache, req, (*group.GetGroupMemberCacheResp).GetMember) +} + +func (x *GroupClient) DismissGroup(ctx context.Context, groupID string, deleteMember bool) error { + req := &group.DismissGroupReq{GroupID: groupID, DeleteMember: deleteMember} + return ignoreResp(x.GroupClient.DismissGroup(ctx, req)) +} + +func (x *GroupClient) GetGroupMemberUserIDs(ctx context.Context, groupID string) ([]string, error) { + req := &group.GetGroupMemberUserIDsReq{GroupID: groupID} + return extractField(ctx, x.GroupClient.GetGroupMemberUserIDs, req, (*group.GetGroupMemberUserIDsResp).GetUserIDs) +} diff --git a/pkg/rpcli/msg.go b/pkg/rpcli/msg.go new file mode 100644 index 000000000..0c44b7c8b --- /dev/null +++ b/pkg/rpcli/msg.go @@ -0,0 +1,90 @@ +package rpcli + +import ( + "context" + "github.com/openimsdk/protocol/msg" + "github.com/openimsdk/protocol/sdkws" + "google.golang.org/grpc" +) + +func NewMsgClient(cc grpc.ClientConnInterface) *MsgClient { + return &MsgClient{msg.NewMsgClient(cc)} +} + +type MsgClient struct { + msg.MsgClient +} + +func (x *MsgClient) GetMaxSeqs(ctx context.Context, conversationIDs []string) (map[string]int64, error) { + if len(conversationIDs) == 0 { + return nil, nil + } + req := &msg.GetMaxSeqsReq{ConversationIDs: conversationIDs} + return extractField(ctx, x.MsgClient.GetMaxSeqs, req, (*msg.SeqsInfoResp).GetMaxSeqs) +} + +func (x *MsgClient) GetMsgByConversationIDs(ctx context.Context, conversationIDs []string, maxSeqs map[string]int64) (map[string]*sdkws.MsgData, error) { + if len(conversationIDs) == 0 || len(maxSeqs) == 0 { + return nil, nil + } + req := &msg.GetMsgByConversationIDsReq{ConversationIDs: conversationIDs, MaxSeqs: maxSeqs} + return extractField(ctx, x.MsgClient.GetMsgByConversationIDs, req, (*msg.GetMsgByConversationIDsResp).GetMsgDatas) +} + +func (x *MsgClient) GetHasReadSeqs(ctx context.Context, conversationIDs []string, userID string) (map[string]int64, error) { + if len(conversationIDs) == 0 { + return nil, nil + } + req := &msg.GetHasReadSeqsReq{ConversationIDs: conversationIDs, UserID: userID} + return extractField(ctx, x.MsgClient.GetHasReadSeqs, req, (*msg.SeqsInfoResp).GetMaxSeqs) +} + +func (x *MsgClient) SetUserConversationMaxSeq(ctx context.Context, conversationID string, ownerUserIDs []string, maxSeq int64) error { + if len(ownerUserIDs) == 0 { + return nil + } + req := &msg.SetUserConversationMaxSeqReq{ConversationID: conversationID, OwnerUserID: ownerUserIDs, MaxSeq: maxSeq} + return ignoreResp(x.MsgClient.SetUserConversationMaxSeq(ctx, req)) +} + +func (x *MsgClient) SetUserConversationMin(ctx context.Context, conversationID string, ownerUserIDs []string, minSeq int64) error { + if len(ownerUserIDs) == 0 { + return nil + } + req := &msg.SetUserConversationsMinSeqReq{ConversationID: conversationID, UserIDs: ownerUserIDs, Seq: minSeq} + return ignoreResp(x.MsgClient.SetUserConversationsMinSeq(ctx, req)) +} + +func (x *MsgClient) GetLastMessageSeqByTime(ctx context.Context, conversationID string, lastTime int64) (int64, error) { + req := &msg.GetLastMessageSeqByTimeReq{ConversationID: conversationID, Time: lastTime} + return extractField(ctx, x.MsgClient.GetLastMessageSeqByTime, req, (*msg.GetLastMessageSeqByTimeResp).GetSeq) +} + +func (x *MsgClient) GetConversationMaxSeq(ctx context.Context, conversationID string) (int64, error) { + req := &msg.GetConversationMaxSeqReq{ConversationID: conversationID} + return extractField(ctx, x.MsgClient.GetConversationMaxSeq, req, (*msg.GetConversationMaxSeqResp).GetMaxSeq) +} + +func (x *MsgClient) GetActiveConversation(ctx context.Context, conversationIDs []string) ([]*msg.ActiveConversation, error) { + if len(conversationIDs) == 0 { + return nil, nil + } + req := &msg.GetActiveConversationReq{ConversationIDs: conversationIDs} + return extractField(ctx, x.MsgClient.GetActiveConversation, req, (*msg.GetActiveConversationResp).GetConversations) +} + +func (x *MsgClient) GetSeqMessage(ctx context.Context, userID string, conversations []*msg.ConversationSeqs) (map[string]*sdkws.PullMsgs, error) { + if len(conversations) == 0 { + return nil, nil + } + req := &msg.GetSeqMessageReq{UserID: userID, Conversations: conversations} + return extractField(ctx, x.MsgClient.GetSeqMessage, req, (*msg.GetSeqMessageResp).GetMsgs) +} + +func (x *MsgClient) SetUserConversationsMinSeq(ctx context.Context, conversationID string, userIDs []string, seq int64) error { + if len(userIDs) == 0 { + return nil + } + req := &msg.SetUserConversationsMinSeqReq{ConversationID: conversationID, UserIDs: userIDs, Seq: seq} + return ignoreResp(x.MsgClient.SetUserConversationsMinSeq(ctx, req)) +} diff --git a/pkg/rpcli/msggateway.go b/pkg/rpcli/msggateway.go new file mode 100644 index 000000000..3f19963d5 --- /dev/null +++ b/pkg/rpcli/msggateway.go @@ -0,0 +1,14 @@ +package rpcli + +import ( + "github.com/openimsdk/protocol/msggateway" + "google.golang.org/grpc" +) + +func NewMsgGatewayClient(cc grpc.ClientConnInterface) *MsgGatewayClient { + return &MsgGatewayClient{msggateway.NewMsgGatewayClient(cc)} +} + +type MsgGatewayClient struct { + msggateway.MsgGatewayClient +} diff --git a/pkg/rpcli/push.go b/pkg/rpcli/push.go new file mode 100644 index 000000000..bb33660e4 --- /dev/null +++ b/pkg/rpcli/push.go @@ -0,0 +1,14 @@ +package rpcli + +import ( + "github.com/openimsdk/protocol/push" + "google.golang.org/grpc" +) + +func NewPushMsgServiceClient(cc grpc.ClientConnInterface) *PushMsgServiceClient { + return &PushMsgServiceClient{push.NewPushMsgServiceClient(cc)} +} + +type PushMsgServiceClient struct { + push.PushMsgServiceClient +} diff --git a/pkg/rpcli/relation.go b/pkg/rpcli/relation.go new file mode 100644 index 000000000..dce0e7165 --- /dev/null +++ b/pkg/rpcli/relation.go @@ -0,0 +1,23 @@ +package rpcli + +import ( + "context" + "github.com/openimsdk/protocol/relation" + "google.golang.org/grpc" +) + +func NewRelationClient(cc grpc.ClientConnInterface) *RelationClient { + return &RelationClient{relation.NewFriendClient(cc)} +} + +type RelationClient struct { + relation.FriendClient +} + +func (x *RelationClient) GetFriendsInfo(ctx context.Context, ownerUserID string, friendUserIDs []string) ([]*relation.FriendInfoOnly, error) { + if len(friendUserIDs) == 0 { + return nil, nil + } + req := &relation.GetFriendInfoReq{OwnerUserID: ownerUserID, FriendUserIDs: friendUserIDs} + return extractField(ctx, x.FriendClient.GetFriendInfo, req, (*relation.GetFriendInfoResp).GetFriendInfos) +} diff --git a/pkg/rpcli/rtc.go b/pkg/rpcli/rtc.go new file mode 100644 index 000000000..18a79d6b4 --- /dev/null +++ b/pkg/rpcli/rtc.go @@ -0,0 +1,14 @@ +package rpcli + +import ( + "github.com/openimsdk/protocol/rtc" + "google.golang.org/grpc" +) + +func NewRtcServiceClient(cc grpc.ClientConnInterface) *RtcServiceClient { + return &RtcServiceClient{rtc.NewRtcServiceClient(cc)} +} + +type RtcServiceClient struct { + rtc.RtcServiceClient +} diff --git a/pkg/rpcli/third.go b/pkg/rpcli/third.go new file mode 100644 index 000000000..cbb28ff34 --- /dev/null +++ b/pkg/rpcli/third.go @@ -0,0 +1,14 @@ +package rpcli + +import ( + "github.com/openimsdk/protocol/third" + "google.golang.org/grpc" +) + +func NewThirdClient(cc grpc.ClientConnInterface) *ThirdClient { + return &ThirdClient{third.NewThirdClient(cc)} +} + +type ThirdClient struct { + third.ThirdClient +} diff --git a/pkg/rpcli/tool.go b/pkg/rpcli/tool.go new file mode 100644 index 000000000..2bd50bd00 --- /dev/null +++ b/pkg/rpcli/tool.go @@ -0,0 +1,32 @@ +package rpcli + +import ( + "context" + "github.com/openimsdk/tools/errs" + "google.golang.org/grpc" +) + +func extractField[A, B, C any](ctx context.Context, fn func(ctx context.Context, req *A, opts ...grpc.CallOption) (*B, error), req *A, get func(*B) C) (C, error) { + resp, err := fn(ctx, req) + if err != nil { + var c C + return c, err + } + return get(resp), nil +} + +func firstValue[A any](val []A, err error) (A, error) { + if err != nil { + var a A + return a, err + } + if len(val) == 0 { + var a A + return a, errs.ErrRecordNotFound.WrapMsg("record not found") + } + return val[0], nil +} + +func ignoreResp(_ any, err error) error { + return err +} diff --git a/pkg/rpcli/user.go b/pkg/rpcli/user.go new file mode 100644 index 000000000..357640345 --- /dev/null +++ b/pkg/rpcli/user.go @@ -0,0 +1,95 @@ +package rpcli + +import ( + "context" + "github.com/openimsdk/protocol/sdkws" + "github.com/openimsdk/protocol/user" + "github.com/openimsdk/tools/errs" + "github.com/openimsdk/tools/utils/datautil" + "google.golang.org/grpc" +) + +func NewUserClient(cc grpc.ClientConnInterface) *UserClient { + return &UserClient{user.NewUserClient(cc)} +} + +type UserClient struct { + user.UserClient +} + +func (x *UserClient) GetUsersInfo(ctx context.Context, userIDs []string) ([]*sdkws.UserInfo, error) { + if len(userIDs) == 0 { + return nil, nil + } + req := &user.GetDesignateUsersReq{UserIDs: userIDs} + return extractField(ctx, x.UserClient.GetDesignateUsers, req, (*user.GetDesignateUsersResp).GetUsersInfo) +} + +func (x *UserClient) GetUserInfo(ctx context.Context, userID string) (*sdkws.UserInfo, error) { + return firstValue(x.GetUsersInfo(ctx, []string{userID})) +} + +func (x *UserClient) CheckUser(ctx context.Context, userIDs []string) error { + if len(userIDs) == 0 { + return nil + } + users, err := x.GetUsersInfo(ctx, userIDs) + if err != nil { + return err + } + if len(users) != len(userIDs) { + return errs.ErrRecordNotFound.WrapMsg("user not found") + } + return nil +} + +func (x *UserClient) GetUsersInfoMap(ctx context.Context, userIDs []string) (map[string]*sdkws.UserInfo, error) { + users, err := x.GetUsersInfo(ctx, userIDs) + if err != nil { + return nil, err + } + return datautil.SliceToMap(users, func(e *sdkws.UserInfo) string { + return e.UserID + }), nil +} + +func (x *UserClient) GetAllOnlineUsers(ctx context.Context, cursor uint64) (*user.GetAllOnlineUsersResp, error) { + req := &user.GetAllOnlineUsersReq{Cursor: cursor} + return x.UserClient.GetAllOnlineUsers(ctx, req) +} + +func (x *UserClient) GetUsersOnlinePlatform(ctx context.Context, userIDs []string) ([]*user.OnlineStatus, error) { + if len(userIDs) == 0 { + return nil, nil + } + req := &user.GetUserStatusReq{UserIDs: userIDs} + return extractField(ctx, x.UserClient.GetUserStatus, req, (*user.GetUserStatusResp).GetStatusList) + +} + +func (x *UserClient) GetUserOnlinePlatform(ctx context.Context, userID string) ([]int32, error) { + status, err := x.GetUsersOnlinePlatform(ctx, []string{userID}) + if err != nil { + return nil, err + } + if len(status) == 0 { + return nil, nil + } + return status[0].PlatformIDs, nil +} + +func (x *UserClient) SetUserOnlineStatus(ctx context.Context, req *user.SetUserOnlineStatusReq) error { + if len(req.Status) == 0 { + return nil + } + return ignoreResp(x.UserClient.SetUserOnlineStatus(ctx, req)) +} + +func (x *UserClient) GetNotificationByID(ctx context.Context, userID string) error { + return ignoreResp(x.UserClient.GetNotificationAccount(ctx, &user.GetNotificationAccountReq{UserID: userID})) +} + +func (x *UserClient) GetAllUserIDs(ctx context.Context, pageNumber, showNumber int32) ([]string, error) { + req := &user.GetAllUserIDReq{Pagination: &sdkws.RequestPagination{PageNumber: pageNumber, ShowNumber: showNumber}} + return extractField(ctx, x.UserClient.GetAllUserID, req, (*user.GetAllUserIDResp).GetUserIDs) +} From 5503443f26037281fdb0cd6e2a860f93be966051 Mon Sep 17 00:00:00 2001 From: chao <48119764+withchao@users.noreply.github.com> Date: Tue, 24 Dec 2024 18:11:35 +0800 Subject: [PATCH 084/199] feat: optimize error stack information (#2995) * pb * fix: Modifying other fields while setting IsPrivateChat does not take effect * fix: quote message error revoke * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * upgrading pkg tools --- go.mod | 2 +- go.sum | 4 ++-- internal/msggateway/client.go | 4 ++-- internal/msgtransfer/init.go | 2 +- internal/msgtransfer/online_history_msg_handler.go | 2 +- internal/rpc/msg/send.go | 2 +- pkg/rpccache/subscriber.go | 3 ++- 7 files changed, 10 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index 26f167a19..05fe6db15 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/mitchellh/mapstructure v1.5.0 github.com/openimsdk/protocol v0.0.72-alpha.68 - github.com/openimsdk/tools v0.0.50-alpha.60 + github.com/openimsdk/tools v0.0.50-alpha.61 github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 github.com/stretchr/testify v1.9.0 diff --git a/go.sum b/go.sum index c0385481c..1fc3c33db 100644 --- a/go.sum +++ b/go.sum @@ -349,8 +349,8 @@ github.com/openimsdk/gomake v0.0.15-alpha.2 h1:5Q8yl8ezy2yx+q8/ucU/t4kJnDfCzNOrk github.com/openimsdk/gomake v0.0.15-alpha.2/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= github.com/openimsdk/protocol v0.0.72-alpha.68 h1:Ekn6S9Ftt12Xs/p9kJ39RDr2gSwIczz+MmSHQE4lAek= github.com/openimsdk/protocol v0.0.72-alpha.68/go.mod h1:Iet+piS/jaS+kWWyj6EEr36mk4ISzIRYjoMSVA4dq2M= -github.com/openimsdk/tools v0.0.50-alpha.60 h1:dYqYpSdSN5o6CxlEjua2USfwfUiG0tUWFBpqghTjbWE= -github.com/openimsdk/tools v0.0.50-alpha.60/go.mod h1:muCtxguNJv8lFwLei27UASu2Nvg4ERSeN0R4K5tivk0= +github.com/openimsdk/tools v0.0.50-alpha.61 h1:zKEZwrj+fUVuyC6KR3kZp9zFaCCIFgoSbHO0r0mZ6h4= +github.com/openimsdk/tools v0.0.50-alpha.61/go.mod h1:JowL2jYr8tu4vcQe+5hJh4v3BtSx1T0CIS3pgU/Mw+U= github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= diff --git a/internal/msggateway/client.go b/internal/msggateway/client.go index 161300d6d..1040f2be2 100644 --- a/internal/msggateway/client.go +++ b/internal/msggateway/client.go @@ -131,7 +131,7 @@ func (c *Client) readMessage() { defer func() { if r := recover(); r != nil { c.closedErr = ErrPanic - log.ZPanic(c.ctx, "socket have panic err:", r) + log.ZPanic(c.ctx, "socket have panic err:", errs.ErrPanic(r)) } c.close() }() @@ -376,7 +376,7 @@ func (c *Client) activeHeartbeat(ctx context.Context) { go func() { defer func() { if r := recover(); r != nil { - log.ZPanic(ctx, "activeHeartbeat Panic", r) + log.ZPanic(ctx, "activeHeartbeat Panic", errs.ErrPanic(r)) } }() log.ZDebug(ctx, "server initiative send heartbeat start.") diff --git a/internal/msgtransfer/init.go b/internal/msgtransfer/init.go index 5cb613123..ee9f06644 100644 --- a/internal/msgtransfer/init.go +++ b/internal/msgtransfer/init.go @@ -195,7 +195,7 @@ func (m *MsgTransfer) Start(index int, config *Config, client discovery.SvcDisco go func() { defer func() { if r := recover(); r != nil { - log.ZPanic(m.ctx, "MsgTransfer Start Panic", r) + log.ZPanic(m.ctx, "MsgTransfer Start Panic", errs.ErrPanic(r)) } }() if err := prommetrics.TransferInit(listener); err != nil && !errors.Is(err, http.ErrServerClosed) { diff --git a/internal/msgtransfer/online_history_msg_handler.go b/internal/msgtransfer/online_history_msg_handler.go index 83b075061..7b989070b 100644 --- a/internal/msgtransfer/online_history_msg_handler.go +++ b/internal/msgtransfer/online_history_msg_handler.go @@ -361,7 +361,7 @@ func (och *OnlineHistoryRedisConsumerHandler) handleNotification(ctx context.Con func (och *OnlineHistoryRedisConsumerHandler) HandleUserHasReadSeqMessages(ctx context.Context) { defer func() { if r := recover(); r != nil { - log.ZPanic(ctx, "HandleUserHasReadSeqMessages Panic", r) + log.ZPanic(ctx, "HandleUserHasReadSeqMessages Panic", errs.ErrPanic(r)) } }() diff --git a/internal/rpc/msg/send.go b/internal/rpc/msg/send.go index b9bbf615f..f01134f8f 100644 --- a/internal/rpc/msg/send.go +++ b/internal/rpc/msg/send.go @@ -88,7 +88,7 @@ func (m *msgServer) setConversationAtInfo(nctx context.Context, msg *sdkws.MsgDa defer func() { if r := recover(); r != nil { - log.ZPanic(nctx, "setConversationAtInfo Panic", r) + log.ZPanic(nctx, "setConversationAtInfo Panic", errs.ErrPanic(r)) } }() diff --git a/pkg/rpccache/subscriber.go b/pkg/rpccache/subscriber.go index d28d1aa29..0cb35bebf 100644 --- a/pkg/rpccache/subscriber.go +++ b/pkg/rpccache/subscriber.go @@ -17,6 +17,7 @@ package rpccache import ( "context" "encoding/json" + "github.com/openimsdk/tools/errs" "github.com/openimsdk/tools/log" "github.com/redis/go-redis/v9" ) @@ -24,7 +25,7 @@ import ( func subscriberRedisDeleteCache(ctx context.Context, client redis.UniversalClient, channel string, del func(ctx context.Context, key ...string)) { defer func() { if r := recover(); r != nil { - log.ZPanic(ctx, "subscriberRedisDeleteCache Panic", r) + log.ZPanic(ctx, "subscriberRedisDeleteCache Panic", errs.ErrPanic(r)) } }() for message := range client.Subscribe(ctx, channel).Channel() { From 870252927a6c1b0e9e8dfc8f80d18c9ce474b4b4 Mon Sep 17 00:00:00 2001 From: icey-yu <119291641+icey-yu@users.noreply.github.com> Date: Wed, 25 Dec 2024 18:07:12 +0800 Subject: [PATCH 085/199] fix: when unable EnableHistoryForNewMembers, new group member can read last one message. (#3001) --- internal/rpc/group/notification.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/internal/rpc/group/notification.go b/internal/rpc/group/notification.go index 48fadbd07..1aa5333b4 100644 --- a/internal/rpc/group/notification.go +++ b/internal/rpc/group/notification.go @@ -18,9 +18,10 @@ import ( "context" "errors" "fmt" - "github.com/openimsdk/open-im-server/v3/pkg/rpcli" "time" + "github.com/openimsdk/open-im-server/v3/pkg/rpcli" + "github.com/openimsdk/open-im-server/v3/pkg/authverify" "github.com/openimsdk/open-im-server/v3/pkg/common/convert" "github.com/openimsdk/open-im-server/v3/pkg/common/servererrs" @@ -532,7 +533,7 @@ func (g *NotificationSender) GroupApplicationAgreeMemberEnterNotification(ctx co if err != nil { return err } - if err := g.msgClient.SetUserConversationsMinSeq(ctx, conversationID, entrantUserID, maxSeq); err != nil { + if err := g.msgClient.SetUserConversationsMinSeq(ctx, conversationID, entrantUserID, maxSeq+1); err != nil { return err } } @@ -586,7 +587,7 @@ func (g *NotificationSender) MemberEnterNotification(ctx context.Context, groupI if err != nil { return err } - if err := g.msgClient.SetUserConversationsMinSeq(ctx, conversationID, []string{entrantUserID}, maxSeq); err != nil { + if err := g.msgClient.SetUserConversationsMinSeq(ctx, conversationID, []string{entrantUserID}, maxSeq+1); err != nil { return err } } From 66abd9e1b9d45e41876b2598aa5ca95985b1b0fa Mon Sep 17 00:00:00 2001 From: icey-yu <119291641+icey-yu@users.noreply.github.com> Date: Wed, 25 Dec 2024 18:07:25 +0800 Subject: [PATCH 086/199] fix: redis save error when KickTokens (#3002) --- pkg/common/storage/cache/redis/token.go | 9 +++++---- pkg/common/storage/cache/token.go | 2 +- pkg/common/storage/controller/auth.go | 5 +++-- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/pkg/common/storage/cache/redis/token.go b/pkg/common/storage/cache/redis/token.go index 998b4f1c9..510da43e3 100644 --- a/pkg/common/storage/cache/redis/token.go +++ b/pkg/common/storage/cache/redis/token.go @@ -2,13 +2,14 @@ package redis import ( "context" + "strconv" + "sync" + "time" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/cachekey" "github.com/openimsdk/tools/errs" "github.com/redis/go-redis/v9" - "strconv" - "sync" - "time" ) type tokenCache struct { @@ -99,7 +100,7 @@ func (c *tokenCache) SetTokenMapByUidPid(ctx context.Context, userID string, pla return errs.Wrap(c.rdb.HSet(ctx, cachekey.GetTokenKey(userID, platformID), mm).Err()) } -func (c *tokenCache) BatchSetTokenMapByUidPid(ctx context.Context, tokens map[string]map[string]int) error { +func (c *tokenCache) BatchSetTokenMapByUidPid(ctx context.Context, tokens map[string]map[string]any) error { pipe := c.rdb.Pipeline() for k, v := range tokens { pipe.HSet(ctx, k, v) diff --git a/pkg/common/storage/cache/token.go b/pkg/common/storage/cache/token.go index ee0004d7f..e5e0a9383 100644 --- a/pkg/common/storage/cache/token.go +++ b/pkg/common/storage/cache/token.go @@ -11,6 +11,6 @@ type TokenModel interface { GetTokensWithoutError(ctx context.Context, userID string, platformID int) (map[string]int, error) GetAllTokensWithoutError(ctx context.Context, userID string) (map[int]map[string]int, error) SetTokenMapByUidPid(ctx context.Context, userID string, platformID int, m map[string]int) error - BatchSetTokenMapByUidPid(ctx context.Context, tokens map[string]map[string]int) error + BatchSetTokenMapByUidPid(ctx context.Context, tokens map[string]map[string]any) error DeleteTokenByUidPid(ctx context.Context, userID string, platformID int, fields []string) error } diff --git a/pkg/common/storage/controller/auth.go b/pkg/common/storage/controller/auth.go index 0a7029662..2885b985a 100644 --- a/pkg/common/storage/controller/auth.go +++ b/pkg/common/storage/controller/auth.go @@ -2,6 +2,7 @@ package controller import ( "context" + "github.com/golang-jwt/jwt/v4" "github.com/openimsdk/open-im-server/v3/pkg/authverify" "github.com/openimsdk/open-im-server/v3/pkg/common/config" @@ -56,7 +57,7 @@ func (a *authDatabase) SetTokenMapByUidPid(ctx context.Context, userID string, p } func (a *authDatabase) BatchSetTokenMapByUidPid(ctx context.Context, tokens []string) error { - setMap := make(map[string]map[string]int) + setMap := make(map[string]map[string]any) for _, token := range tokens { claims, err := tokenverify.GetClaimFromToken(token, authverify.Secret(a.accessSecret)) key := cachekey.GetTokenKey(claims.UserID, claims.PlatformID) @@ -66,7 +67,7 @@ func (a *authDatabase) BatchSetTokenMapByUidPid(ctx context.Context, tokens []st if v, ok := setMap[key]; ok { v[token] = constant.KickedToken } else { - setMap[key] = map[string]int{ + setMap[key] = map[string]any{ token: constant.KickedToken, } } From 1110af98ef83f20a20f66ece61c89cc5879244e6 Mon Sep 17 00:00:00 2001 From: icey-yu <119291641+icey-yu@users.noreply.github.com> Date: Wed, 25 Dec 2024 18:08:08 +0800 Subject: [PATCH 087/199] feat: config center (#2997) * chore: config * chore: config * chore: config * chore: config * chore: config * feat: config * fix: config * fix: config * feat: config * feat: config * feat: config * feat: config * feat: config * feat: config * feat: config * feat: config * feat: config * feat: config * feat: config * feat: config * feat: config * feat: config * feat: config * feat: config * feat: config * feat: config * fix: config * fix: config * fix: config * fix: config * fix: config * fix: config * fix: config * fix: config * fix: config * fix: config * fix: config * fix: config * fix: config * fix: config * fix: config * fix: config * fix: config * fix: config --- config/openim-push.yml | 2 +- docker-compose.yml | 2 + internal/api/config_manager.go | 250 ++++++++++++++++++ internal/api/init.go | 31 ++- internal/api/router.go | 70 +++-- internal/msggateway/hub_server.go | 7 + internal/msggateway/ws_server.go | 24 +- internal/msgtransfer/init.go | 19 +- internal/tools/cron_task.go | 2 +- internal/tools/cron_test.go | 5 +- pkg/apistruct/config_manager.go | 16 ++ pkg/common/cmd/api.go | 29 +- pkg/common/cmd/auth.go | 18 +- pkg/common/cmd/constant.go | 96 ------- pkg/common/cmd/conversation.go | 27 +- pkg/common/cmd/cron_task.go | 7 +- pkg/common/cmd/friend.go | 30 ++- pkg/common/cmd/group.go | 30 ++- pkg/common/cmd/msg.go | 33 ++- pkg/common/cmd/msg_gateway.go | 11 +- pkg/common/cmd/msg_transfer.go | 15 +- pkg/common/cmd/msg_utils.go | 5 +- pkg/common/cmd/push.go | 30 ++- pkg/common/cmd/root.go | 89 +++++-- pkg/common/cmd/third.go | 30 ++- pkg/common/cmd/user.go | 33 ++- pkg/common/config/config.go | 217 +++++++++++++++ pkg/common/config/env.go | 30 +++ pkg/common/config/load_config_test.go | 2 +- pkg/common/config/parse.go | 4 +- .../direct/direct_resolver.go | 0 .../direct/directconn.go | 0 .../direct/doc.go | 2 +- .../discoveryregister.go | 2 +- .../discoveryregister_test.go | 2 +- .../{discoveryregister => discovery}/doc.go | 2 +- pkg/common/discovery/etcd/config_manager.go | 111 ++++++++ .../kubernetes => discovery/etcd}/doc.go | 2 +- .../etcd => discovery/kubernetes}/doc.go | 2 +- .../kubernetes/kubernetes.go | 0 pkg/common/startrpc/start.go | 12 +- tools/check-component/main.go | 13 +- tools/seq/internal/seq.go | 5 +- 43 files changed, 1039 insertions(+), 278 deletions(-) create mode 100644 internal/api/config_manager.go create mode 100644 pkg/apistruct/config_manager.go delete mode 100644 pkg/common/cmd/constant.go create mode 100644 pkg/common/config/env.go rename pkg/common/{discoveryregister => discovery}/direct/direct_resolver.go (100%) rename pkg/common/{discoveryregister => discovery}/direct/directconn.go (100%) rename pkg/common/{discoveryregister => discovery}/direct/doc.go (94%) rename pkg/common/{discoveryregister => discovery}/discoveryregister.go (98%) rename pkg/common/{discoveryregister => discovery}/discoveryregister_test.go (98%) rename pkg/common/{discoveryregister => discovery}/doc.go (85%) create mode 100644 pkg/common/discovery/etcd/config_manager.go rename pkg/common/{discoveryregister/kubernetes => discovery/etcd}/doc.go (84%) rename pkg/common/{discoveryregister/etcd => discovery/kubernetes}/doc.go (94%) rename pkg/common/{discoveryregister => discovery}/kubernetes/kubernetes.go (100%) diff --git a/config/openim-push.yml b/config/openim-push.yml index e98324620..5db5b541a 100644 --- a/config/openim-push.yml +++ b/config/openim-push.yml @@ -21,7 +21,7 @@ prometheus: maxConcurrentWorkers: 3 #Use geTui for offline push notifications, or choose fcm or jpns; corresponding configuration settings must be specified. enable: -geTui: +getui: pushUrl: https://restapi.getui.com/v2/$appId masterSecret: appKey: diff --git a/docker-compose.yml b/docker-compose.yml index 51dd4d04f..0cdeebe43 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -82,6 +82,8 @@ services: - ETCD_INITIAL_CLUSTER=s1=http://0.0.0.0:2380 - ETCD_INITIAL_CLUSTER_TOKEN=tkn - ETCD_INITIAL_CLUSTER_STATE=new + volumes: + - "${DATA_DIR}/components/etcd:/etcd-data" restart: always networks: - openim diff --git a/internal/api/config_manager.go b/internal/api/config_manager.go new file mode 100644 index 000000000..c330cad46 --- /dev/null +++ b/internal/api/config_manager.go @@ -0,0 +1,250 @@ +package api + +import ( + "encoding/json" + "reflect" + "strconv" + "time" + + "github.com/gin-gonic/gin" + "github.com/openimsdk/open-im-server/v3/pkg/apistruct" + "github.com/openimsdk/open-im-server/v3/pkg/authverify" + "github.com/openimsdk/open-im-server/v3/pkg/common/config" + "github.com/openimsdk/open-im-server/v3/pkg/common/discovery/etcd" + "github.com/openimsdk/open-im-server/v3/version" + "github.com/openimsdk/tools/apiresp" + "github.com/openimsdk/tools/errs" + "github.com/openimsdk/tools/log" + "github.com/openimsdk/tools/utils/runtimeenv" + clientv3 "go.etcd.io/etcd/client/v3" +) + +type ConfigManager struct { + imAdminUserID []string + config *config.AllConfig + client *clientv3.Client + configPath string + runtimeEnv string +} + +func NewConfigManager(IMAdminUserID []string, cfg *config.AllConfig, client *clientv3.Client, configPath string, runtimeEnv string) *ConfigManager { + return &ConfigManager{ + imAdminUserID: IMAdminUserID, + config: cfg, + client: client, + configPath: configPath, + runtimeEnv: runtimeEnv, + } +} + +func (cm *ConfigManager) CheckAdmin(c *gin.Context) { + if err := authverify.CheckAdmin(c, cm.imAdminUserID); err != nil { + apiresp.GinError(c, err) + c.Abort() + } +} + +func (cm *ConfigManager) GetConfig(c *gin.Context) { + var req apistruct.GetConfigReq + if err := c.BindJSON(&req); err != nil { + apiresp.GinError(c, errs.ErrArgs.WithDetail(err.Error()).Wrap()) + return + } + conf := cm.config.Name2Config(req.ConfigName) + if conf == nil { + apiresp.GinError(c, errs.ErrArgs.WithDetail("config name not found").Wrap()) + return + } + b, err := json.Marshal(conf) + if err != nil { + apiresp.GinError(c, err) + return + } + apiresp.GinSuccess(c, string(b)) +} + +func (cm *ConfigManager) GetConfigList(c *gin.Context) { + var resp apistruct.GetConfigListResp + resp.ConfigNames = cm.config.GetConfigNames() + resp.Environment = runtimeenv.PrintRuntimeEnvironment() + resp.Version = version.Version + + apiresp.GinSuccess(c, resp) +} + +func (cm *ConfigManager) SetConfig(c *gin.Context) { + if cm.config.Discovery.Enable != config.ETCD { + apiresp.GinError(c, errs.New("only etcd support set config").Wrap()) + return + } + var req apistruct.SetConfigReq + if err := c.BindJSON(&req); err != nil { + apiresp.GinError(c, errs.ErrArgs.WithDetail(err.Error()).Wrap()) + return + } + var err error + switch req.ConfigName { + case cm.config.Discovery.GetConfigFileName(): + err = compareAndSave[config.Discovery](c, cm.config.Name2Config(req.ConfigName), &req, cm.client) + case cm.config.Kafka.GetConfigFileName(): + err = compareAndSave[config.Kafka](c, cm.config.Name2Config(req.ConfigName), &req, cm.client) + case cm.config.LocalCache.GetConfigFileName(): + err = compareAndSave[config.LocalCache](c, cm.config.Name2Config(req.ConfigName), &req, cm.client) + case cm.config.Log.GetConfigFileName(): + err = compareAndSave[config.Log](c, cm.config.Name2Config(req.ConfigName), &req, cm.client) + case cm.config.Minio.GetConfigFileName(): + err = compareAndSave[config.Minio](c, cm.config.Name2Config(req.ConfigName), &req, cm.client) + case cm.config.Mongo.GetConfigFileName(): + err = compareAndSave[config.Mongo](c, cm.config.Name2Config(req.ConfigName), &req, cm.client) + case cm.config.Notification.GetConfigFileName(): + err = compareAndSave[config.Notification](c, cm.config.Name2Config(req.ConfigName), &req, cm.client) + case cm.config.API.GetConfigFileName(): + err = compareAndSave[config.API](c, cm.config.Name2Config(req.ConfigName), &req, cm.client) + case cm.config.CronTask.GetConfigFileName(): + err = compareAndSave[config.CronTask](c, cm.config.Name2Config(req.ConfigName), &req, cm.client) + case cm.config.MsgGateway.GetConfigFileName(): + err = compareAndSave[config.MsgGateway](c, cm.config.Name2Config(req.ConfigName), &req, cm.client) + case cm.config.MsgTransfer.GetConfigFileName(): + err = compareAndSave[config.MsgTransfer](c, cm.config.Name2Config(req.ConfigName), &req, cm.client) + case cm.config.Push.GetConfigFileName(): + err = compareAndSave[config.Push](c, cm.config.Name2Config(req.ConfigName), &req, cm.client) + case cm.config.Auth.GetConfigFileName(): + err = compareAndSave[config.Auth](c, cm.config.Name2Config(req.ConfigName), &req, cm.client) + case cm.config.Conversation.GetConfigFileName(): + err = compareAndSave[config.Conversation](c, cm.config.Name2Config(req.ConfigName), &req, cm.client) + case cm.config.Friend.GetConfigFileName(): + err = compareAndSave[config.Friend](c, cm.config.Name2Config(req.ConfigName), &req, cm.client) + case cm.config.Group.GetConfigFileName(): + err = compareAndSave[config.Group](c, cm.config.Name2Config(req.ConfigName), &req, cm.client) + case cm.config.Msg.GetConfigFileName(): + err = compareAndSave[config.Msg](c, cm.config.Name2Config(req.ConfigName), &req, cm.client) + case cm.config.Third.GetConfigFileName(): + err = compareAndSave[config.Third](c, cm.config.Name2Config(req.ConfigName), &req, cm.client) + case cm.config.User.GetConfigFileName(): + err = compareAndSave[config.User](c, cm.config.Name2Config(req.ConfigName), &req, cm.client) + case cm.config.Redis.GetConfigFileName(): + err = compareAndSave[config.Redis](c, cm.config.Name2Config(req.ConfigName), &req, cm.client) + case cm.config.Share.GetConfigFileName(): + err = compareAndSave[config.Share](c, cm.config.Name2Config(req.ConfigName), &req, cm.client) + case cm.config.Webhooks.GetConfigFileName(): + err = compareAndSave[config.Webhooks](c, cm.config.Name2Config(req.ConfigName), &req, cm.client) + default: + apiresp.GinError(c, errs.ErrArgs.Wrap()) + return + } + if err != nil { + apiresp.GinError(c, errs.ErrArgs.WithDetail(err.Error()).Wrap()) + return + } + apiresp.GinSuccess(c, nil) +} + +func compareAndSave[T any](c *gin.Context, old any, req *apistruct.SetConfigReq, client *clientv3.Client) error { + conf := new(T) + err := json.Unmarshal([]byte(req.Data), &conf) + if err != nil { + return errs.ErrArgs.WithDetail(err.Error()).Wrap() + } + eq := reflect.DeepEqual(old, conf) + if eq { + return nil + } + data, err := json.Marshal(conf) + if err != nil { + return errs.ErrArgs.WithDetail(err.Error()).Wrap() + } + _, err = client.Put(c, etcd.BuildKey(req.ConfigName), string(data)) + if err != nil { + return errs.WrapMsg(err, "save to etcd failed") + } + return nil +} + +func (cm *ConfigManager) ResetConfig(c *gin.Context) { + go cm.resetConfig(c) + apiresp.GinSuccess(c, nil) +} + +func (cm *ConfigManager) resetConfig(c *gin.Context) { + txn := cm.client.Txn(c) + type initConf struct { + old any + new any + isChanged bool + } + configMap := map[string]*initConf{ + cm.config.Discovery.GetConfigFileName(): {old: &cm.config.Discovery, new: new(config.Discovery)}, + cm.config.Kafka.GetConfigFileName(): {old: &cm.config.Kafka, new: new(config.Kafka)}, + cm.config.LocalCache.GetConfigFileName(): {old: &cm.config.LocalCache, new: new(config.LocalCache)}, + cm.config.Log.GetConfigFileName(): {old: &cm.config.Log, new: new(config.Log)}, + cm.config.Minio.GetConfigFileName(): {old: &cm.config.Minio, new: new(config.Minio)}, + cm.config.Mongo.GetConfigFileName(): {old: &cm.config.Mongo, new: new(config.Mongo)}, + cm.config.Notification.GetConfigFileName(): {old: &cm.config.Notification, new: new(config.Notification)}, + cm.config.API.GetConfigFileName(): {old: &cm.config.API, new: new(config.API)}, + cm.config.CronTask.GetConfigFileName(): {old: &cm.config.CronTask, new: new(config.CronTask)}, + cm.config.MsgGateway.GetConfigFileName(): {old: &cm.config.MsgGateway, new: new(config.MsgGateway)}, + cm.config.MsgTransfer.GetConfigFileName(): {old: &cm.config.MsgTransfer, new: new(config.MsgTransfer)}, + cm.config.Push.GetConfigFileName(): {old: &cm.config.Push, new: new(config.Push)}, + cm.config.Auth.GetConfigFileName(): {old: &cm.config.Auth, new: new(config.Auth)}, + cm.config.Conversation.GetConfigFileName(): {old: &cm.config.Conversation, new: new(config.Conversation)}, + cm.config.Friend.GetConfigFileName(): {old: &cm.config.Friend, new: new(config.Friend)}, + cm.config.Group.GetConfigFileName(): {old: &cm.config.Group, new: new(config.Group)}, + cm.config.Msg.GetConfigFileName(): {old: &cm.config.Msg, new: new(config.Msg)}, + cm.config.Third.GetConfigFileName(): {old: &cm.config.Third, new: new(config.Third)}, + cm.config.User.GetConfigFileName(): {old: &cm.config.User, new: new(config.User)}, + cm.config.Redis.GetConfigFileName(): {old: &cm.config.Redis, new: new(config.Redis)}, + cm.config.Share.GetConfigFileName(): {old: &cm.config.Share, new: new(config.Share)}, + cm.config.Webhooks.GetConfigFileName(): {old: &cm.config.Webhooks, new: new(config.Webhooks)}, + } + + changedKeys := make([]string, 0, len(configMap)) + for k, v := range configMap { + err := config.Load( + cm.configPath, + k, + config.EnvPrefixMap[k], + cm.runtimeEnv, + v.new, + ) + if err != nil { + log.ZError(c, "load config failed", err) + continue + } + v.isChanged = reflect.DeepEqual(v.old, v.new) + if !v.isChanged { + changedKeys = append(changedKeys, k) + } + } + + ops := make([]clientv3.Op, 0) + for _, k := range changedKeys { + data, err := json.Marshal(configMap[k].new) + if err != nil { + log.ZError(c, "marshal config failed", err) + continue + } + ops = append(ops, clientv3.OpPut(etcd.BuildKey(k), string(data))) + } + if len(ops) > 0 { + txn.Then(ops...) + _, err := txn.Commit() + if err != nil { + log.ZError(c, "commit etcd txn failed", err) + return + } + } +} + +func (cm *ConfigManager) Restart(c *gin.Context) { + go cm.restart(c) + apiresp.GinSuccess(c, nil) +} + +func (cm *ConfigManager) restart(c *gin.Context) { + time.Sleep(time.Millisecond * 200) // wait for Restart http call return + t := time.Now().Unix() + _, err := cm.client.Put(c, etcd.BuildKey(etcd.RestartKey), strconv.Itoa(int(t))) + if err != nil { + log.ZError(c, "restart etcd put key failed", err) + } +} diff --git a/internal/api/init.go b/internal/api/init.go index 4a558c8b6..780ecb913 100644 --- a/internal/api/init.go +++ b/internal/api/init.go @@ -27,7 +27,8 @@ import ( "time" conf "github.com/openimsdk/open-im-server/v3/pkg/common/config" - kdisc "github.com/openimsdk/open-im-server/v3/pkg/common/discoveryregister" + kdisc "github.com/openimsdk/open-im-server/v3/pkg/common/discovery" + disetcd "github.com/openimsdk/open-im-server/v3/pkg/common/discovery/etcd" "github.com/openimsdk/open-im-server/v3/pkg/common/prommetrics" "github.com/openimsdk/tools/discovery/etcd" "github.com/openimsdk/tools/errs" @@ -43,11 +44,10 @@ import ( ) type Config struct { - API conf.API - Share conf.Share - Discovery conf.Discovery + *conf.AllConfig RuntimeEnv string + ConfigPath string } func Start(ctx context.Context, index int, config *Config) error { @@ -139,22 +139,33 @@ func Start(ctx context.Context, index int, config *Config) error { if err != nil && !errors.Is(err, http.ErrServerClosed) { netErr = errs.WrapMsg(err, fmt.Sprintf("api start err: %s", server.Addr)) netDone <- struct{}{} - } }() + if config.Discovery.Enable == conf.ETCD { + cm := disetcd.NewConfigManager(client.(*etcd.SvcDiscoveryRegistryImpl).GetClient(), config.GetConfigNames()) + cm.Watch(ctx) + } + sigs := make(chan os.Signal, 1) signal.Notify(sigs, syscall.SIGTERM) - ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) - defer cancel() - select { - case <-sigs: - program.SIGTERMExit() + shutdown := func() error { + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + defer cancel() err := server.Shutdown(ctx) if err != nil { return errs.WrapMsg(err, "shutdown err") } + return nil + } + disetcd.RegisterShutDown(shutdown) + select { + case <-sigs: + program.SIGTERMExit() + if err := shutdown(); err != nil { + return err + } case <-netDone: close(netDone) return netErr diff --git a/internal/api/router.go b/internal/api/router.go index 62b8079e6..789b8205d 100644 --- a/internal/api/router.go +++ b/internal/api/router.go @@ -3,34 +3,34 @@ package api import ( "context" "fmt" - "github.com/openimsdk/open-im-server/v3/pkg/rpcli" - "github.com/openimsdk/protocol/conversation" - "github.com/openimsdk/protocol/group" - "github.com/openimsdk/protocol/msg" - "github.com/openimsdk/protocol/relation" - "github.com/openimsdk/protocol/third" - "github.com/openimsdk/protocol/user" "net/http" "strings" - "github.com/openimsdk/open-im-server/v3/internal/api/jssdk" - pbAuth "github.com/openimsdk/protocol/auth" - "github.com/gin-contrib/gzip" - "github.com/gin-gonic/gin" "github.com/gin-gonic/gin/binding" "github.com/go-playground/validator/v10" - "google.golang.org/grpc" - "google.golang.org/grpc/credentials/insecure" - + "github.com/openimsdk/open-im-server/v3/internal/api/jssdk" + "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/open-im-server/v3/pkg/common/prommetrics" "github.com/openimsdk/open-im-server/v3/pkg/common/servererrs" + "github.com/openimsdk/open-im-server/v3/pkg/rpcli" + pbAuth "github.com/openimsdk/protocol/auth" "github.com/openimsdk/protocol/constant" + "github.com/openimsdk/protocol/conversation" + "github.com/openimsdk/protocol/group" + "github.com/openimsdk/protocol/msg" + "github.com/openimsdk/protocol/relation" + "github.com/openimsdk/protocol/third" + "github.com/openimsdk/protocol/user" "github.com/openimsdk/tools/apiresp" "github.com/openimsdk/tools/discovery" + "github.com/openimsdk/tools/discovery/etcd" "github.com/openimsdk/tools/log" "github.com/openimsdk/tools/mw" + clientv3 "go.etcd.io/etcd/client/v3" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" ) const ( @@ -55,34 +55,34 @@ func prommetricsGin() gin.HandlerFunc { } } -func newGinRouter(ctx context.Context, client discovery.SvcDiscoveryRegistry, config *Config) (*gin.Engine, error) { +func newGinRouter(ctx context.Context, client discovery.SvcDiscoveryRegistry, cfg *Config) (*gin.Engine, error) { client.AddOption(mw.GrpcClient(), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithDefaultServiceConfig(fmt.Sprintf(`{"LoadBalancingPolicy": "%s"}`, "round_robin"))) - authConn, err := client.GetConn(ctx, config.Discovery.RpcService.Auth) + authConn, err := client.GetConn(ctx, cfg.Discovery.RpcService.Auth) if err != nil { return nil, err } - userConn, err := client.GetConn(ctx, config.Discovery.RpcService.User) + userConn, err := client.GetConn(ctx, cfg.Discovery.RpcService.User) if err != nil { return nil, err } - groupConn, err := client.GetConn(ctx, config.Discovery.RpcService.Group) + groupConn, err := client.GetConn(ctx, cfg.Discovery.RpcService.Group) if err != nil { return nil, err } - friendConn, err := client.GetConn(ctx, config.Discovery.RpcService.Friend) + friendConn, err := client.GetConn(ctx, cfg.Discovery.RpcService.Friend) if err != nil { return nil, err } - conversationConn, err := client.GetConn(ctx, config.Discovery.RpcService.Conversation) + conversationConn, err := client.GetConn(ctx, cfg.Discovery.RpcService.Conversation) if err != nil { return nil, err } - thirdConn, err := client.GetConn(ctx, config.Discovery.RpcService.Third) + thirdConn, err := client.GetConn(ctx, cfg.Discovery.RpcService.Third) if err != nil { return nil, err } - msgConn, err := client.GetConn(ctx, config.Discovery.RpcService.Msg) + msgConn, err := client.GetConn(ctx, cfg.Discovery.RpcService.Msg) if err != nil { return nil, err } @@ -91,7 +91,7 @@ func newGinRouter(ctx context.Context, client discovery.SvcDiscoveryRegistry, co if v, ok := binding.Validator.Engine().(*validator.Validate); ok { _ = v.RegisterValidation("required_if", RequiredIf) } - switch config.API.Api.CompressionLevel { + switch cfg.API.Api.CompressionLevel { case NoCompression: case DefaultCompression: r.Use(gzip.Gzip(gzip.DefaultCompression)) @@ -103,7 +103,7 @@ func newGinRouter(ctx context.Context, client discovery.SvcDiscoveryRegistry, co r.Use(prommetricsGin(), gin.RecoveryWithWriter(gin.DefaultErrorWriter, mw.GinPanicErr), mw.CorsHandler(), mw.GinParseOperationID(), GinParseToken(rpcli.NewAuthClient(authConn))) j := jssdk.NewJSSdkApi() - u := NewUserApi(user.NewUserClient(userConn), client, config.Discovery.RpcService) + u := NewUserApi(user.NewUserClient(userConn), client, cfg.Discovery.RpcService) { userRouterGroup := r.Group("/user") userRouterGroup.POST("/user_register", u.UserRegister) @@ -204,7 +204,7 @@ func newGinRouter(ctx context.Context, client discovery.SvcDiscoveryRegistry, co } // Third service { - t := NewThirdApi(third.NewThirdClient(thirdConn), config.API.Prometheus.GrafanaURL) + t := NewThirdApi(third.NewThirdClient(thirdConn), cfg.API.Prometheus.GrafanaURL) thirdGroup := r.Group("/third") thirdGroup.GET("/prometheus", t.GetPrometheus) thirdGroup.POST("/fcm_update_token", t.FcmUpdateToken) @@ -228,7 +228,7 @@ func newGinRouter(ctx context.Context, client discovery.SvcDiscoveryRegistry, co objectGroup.GET("/*name", t.ObjectRedirect) } // Message - m := NewMessageApi(msg.NewMsgClient(msgConn), rpcli.NewUserClient(userConn), config.Share.IMAdminUserID) + m := NewMessageApi(msg.NewMsgClient(msgConn), rpcli.NewUserClient(userConn), cfg.Share.IMAdminUserID) { msgGroup := r.Group("/msg") msgGroup.POST("/newest_seq", m.GetSeq) @@ -285,7 +285,7 @@ func newGinRouter(ctx context.Context, client discovery.SvcDiscoveryRegistry, co jssdk.POST("/get_active_conversations", j.GetActiveConversations) } { - pd := NewPrometheusDiscoveryApi(config, client) + pd := NewPrometheusDiscoveryApi(cfg, client) proDiscoveryGroup := r.Group("/prometheus_discovery", pd.Enable) proDiscoveryGroup.GET("/api", pd.Api) proDiscoveryGroup.GET("/user", pd.User) @@ -300,6 +300,22 @@ func newGinRouter(ctx context.Context, client discovery.SvcDiscoveryRegistry, co proDiscoveryGroup.GET("/msg_transfer", pd.MessageTransfer) } + var etcdClient *clientv3.Client + if cfg.Discovery.Enable == config.ETCD { + etcdClient = client.(*etcd.SvcDiscoveryRegistryImpl).GetClient() + } + cm := NewConfigManager(cfg.Share.IMAdminUserID, cfg.AllConfig, etcdClient, cfg.ConfigPath, cfg.RuntimeEnv) + { + + configGroup := r.Group("/config", cm.CheckAdmin) + configGroup.POST("/get_config_list", cm.GetConfigList) + configGroup.POST("/get_config", cm.GetConfig) + configGroup.POST("/set_config", cm.SetConfig) + configGroup.POST("/reset_config", cm.ResetConfig) + } + { + r.POST("/restart", cm.Restart) + } return r, nil } diff --git a/internal/msggateway/hub_server.go b/internal/msggateway/hub_server.go index 753326726..52afe495b 100644 --- a/internal/msggateway/hub_server.go +++ b/internal/msggateway/hub_server.go @@ -57,6 +57,13 @@ func (s *Server) Start(ctx context.Context, index int, conf *Config) error { conf.Discovery.RpcService.MessageGateway, nil, conf, + []string{ + conf.Share.GetConfigFileName(), + conf.Discovery.GetConfigFileName(), + conf.MsgGateway.GetConfigFileName(), + conf.WebhooksConfig.GetConfigFileName(), + conf.RedisConfig.GetConfigFileName(), + }, s.InitServer, ) } diff --git a/internal/msggateway/ws_server.go b/internal/msggateway/ws_server.go index 44b6ddb89..24dd823f6 100644 --- a/internal/msggateway/ws_server.go +++ b/internal/msggateway/ws_server.go @@ -2,13 +2,16 @@ package msggateway import ( "context" + "errors" "fmt" - "github.com/openimsdk/open-im-server/v3/pkg/rpcli" "net/http" "sync" "sync/atomic" "time" + "github.com/openimsdk/open-im-server/v3/pkg/rpcli" + + "github.com/openimsdk/open-im-server/v3/pkg/common/discovery/etcd" "github.com/openimsdk/open-im-server/v3/pkg/common/webhook" "github.com/openimsdk/open-im-server/v3/pkg/rpccache" pbAuth "github.com/openimsdk/protocol/auth" @@ -182,21 +185,28 @@ func (ws *WsServer) Run(done chan error) error { go func() { http.HandleFunc("/", ws.wsHandler) err := server.ListenAndServe() - defer close(netDone) - if err != nil && err != http.ErrServerClosed { + if err != nil && !errors.Is(err, http.ErrServerClosed) { netErr = errs.WrapMsg(err, "ws start err", server.Addr) + netDone <- struct{}{} } }() ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) - defer cancel() - var err error - select { - case err = <-done: + shutDown := func() error { sErr := server.Shutdown(ctx) if sErr != nil { return errs.WrapMsg(sErr, "shutdown err") } close(shutdownDone) + return nil + } + etcd.RegisterShutDown(shutDown) + defer cancel() + var err error + select { + case err = <-done: + if err := shutDown(); err != nil { + return err + } if err != nil { return err } diff --git a/internal/msgtransfer/init.go b/internal/msgtransfer/init.go index ee9f06644..c4a9a89c6 100644 --- a/internal/msgtransfer/init.go +++ b/internal/msgtransfer/init.go @@ -25,6 +25,7 @@ import ( "strconv" "syscall" + disetcd "github.com/openimsdk/open-im-server/v3/pkg/common/discovery/etcd" "github.com/openimsdk/tools/discovery" "github.com/openimsdk/tools/discovery/etcd" "github.com/openimsdk/tools/utils/jsonutil" @@ -39,7 +40,7 @@ import ( "github.com/openimsdk/tools/utils/runtimeenv" conf "github.com/openimsdk/open-im-server/v3/pkg/common/config" - discRegister "github.com/openimsdk/open-im-server/v3/pkg/common/discoveryregister" + discRegister "github.com/openimsdk/open-im-server/v3/pkg/common/discovery" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/controller" "github.com/openimsdk/tools/errs" "github.com/openimsdk/tools/log" @@ -92,6 +93,21 @@ func Start(ctx context.Context, index int, config *Config) error { } client.AddOption(mw.GrpcClient(), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithDefaultServiceConfig(fmt.Sprintf(`{"LoadBalancingPolicy": "%s"}`, "round_robin"))) + + if config.Discovery.Enable == conf.ETCD { + cm := disetcd.NewConfigManager(client.(*etcd.SvcDiscoveryRegistryImpl).GetClient(), []string{ + config.MsgTransfer.GetConfigFileName(), + config.RedisConfig.GetConfigFileName(), + config.MongodbConfig.GetConfigFileName(), + config.KafkaConfig.GetConfigFileName(), + config.Share.GetConfigFileName(), + config.WebhooksConfig.GetConfigFileName(), + config.Discovery.GetConfigFileName(), + conf.LogConfigFileName, + }) + cm.Watch(ctx) + } + msgModel := redis.NewMsgCache(rdb) msgDocModel, err := mgo.NewMsgMongo(mgocli.GetDB()) if err != nil { @@ -125,6 +141,7 @@ func Start(ctx context.Context, index int, config *Config) error { historyMongoCH: historyMongoCH, runTimeEnv: runTimeEnv, } + return msgTransfer.Start(index, config, client) } diff --git a/internal/tools/cron_task.go b/internal/tools/cron_task.go index 049a6199c..71fd886f6 100644 --- a/internal/tools/cron_task.go +++ b/internal/tools/cron_task.go @@ -3,7 +3,7 @@ package tools import ( "context" "github.com/openimsdk/open-im-server/v3/pkg/common/config" - kdisc "github.com/openimsdk/open-im-server/v3/pkg/common/discoveryregister" + kdisc "github.com/openimsdk/open-im-server/v3/pkg/common/discovery" pbconversation "github.com/openimsdk/protocol/conversation" "github.com/openimsdk/protocol/msg" "github.com/openimsdk/protocol/third" diff --git a/internal/tools/cron_test.go b/internal/tools/cron_test.go index 890349069..b4082a5a5 100644 --- a/internal/tools/cron_test.go +++ b/internal/tools/cron_test.go @@ -2,8 +2,10 @@ package tools import ( "context" + "testing" + "github.com/openimsdk/open-im-server/v3/pkg/common/config" - kdisc "github.com/openimsdk/open-im-server/v3/pkg/common/discoveryregister" + kdisc "github.com/openimsdk/open-im-server/v3/pkg/common/discovery" pbconversation "github.com/openimsdk/protocol/conversation" "github.com/openimsdk/protocol/msg" "github.com/openimsdk/protocol/third" @@ -12,7 +14,6 @@ import ( "github.com/robfig/cron/v3" "google.golang.org/grpc" "google.golang.org/grpc/credentials/insecure" - "testing" ) func TestName(t *testing.T) { diff --git a/pkg/apistruct/config_manager.go b/pkg/apistruct/config_manager.go new file mode 100644 index 000000000..84b8fb36b --- /dev/null +++ b/pkg/apistruct/config_manager.go @@ -0,0 +1,16 @@ +package apistruct + +type GetConfigReq struct { + ConfigName string `json:"configName"` +} + +type GetConfigListResp struct { + Environment string `json:"environment"` + Version string `json:"version"` + ConfigNames []string `json:"configNames"` +} + +type SetConfigReq struct { + ConfigName string `json:"configName"` + Data string `json:"data"` +} diff --git a/pkg/common/cmd/api.go b/pkg/common/cmd/api.go index 4088ecd09..050b313ff 100644 --- a/pkg/common/cmd/api.go +++ b/pkg/common/cmd/api.go @@ -18,6 +18,7 @@ import ( "context" "github.com/openimsdk/open-im-server/v3/internal/api" + "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/open-im-server/v3/version" "github.com/openimsdk/tools/system/program" "github.com/spf13/cobra" @@ -31,16 +32,36 @@ type ApiCmd struct { } func NewApiCmd() *ApiCmd { - var apiConfig api.Config + apiConfig := api.Config{AllConfig: &config.AllConfig{}} ret := &ApiCmd{apiConfig: &apiConfig} ret.configMap = map[string]any{ - OpenIMAPICfgFileName: &apiConfig.API, - ShareFileName: &apiConfig.Share, - DiscoveryConfigFilename: &apiConfig.Discovery, + config.DiscoveryConfigFilename: &apiConfig.Discovery, + config.KafkaConfigFileName: &apiConfig.Kafka, + config.LocalCacheConfigFileName: &apiConfig.LocalCache, + config.LogConfigFileName: &apiConfig.Log, + config.MinioConfigFileName: &apiConfig.Minio, + config.MongodbConfigFileName: &apiConfig.Mongo, + config.NotificationFileName: &apiConfig.Notification, + config.OpenIMAPICfgFileName: &apiConfig.API, + config.OpenIMCronTaskCfgFileName: &apiConfig.CronTask, + config.OpenIMMsgGatewayCfgFileName: &apiConfig.MsgGateway, + config.OpenIMMsgTransferCfgFileName: &apiConfig.MsgTransfer, + config.OpenIMPushCfgFileName: &apiConfig.Push, + config.OpenIMRPCAuthCfgFileName: &apiConfig.Auth, + config.OpenIMRPCConversationCfgFileName: &apiConfig.Conversation, + config.OpenIMRPCFriendCfgFileName: &apiConfig.Friend, + config.OpenIMRPCGroupCfgFileName: &apiConfig.Group, + config.OpenIMRPCMsgCfgFileName: &apiConfig.Msg, + config.OpenIMRPCThirdCfgFileName: &apiConfig.Third, + config.OpenIMRPCUserCfgFileName: &apiConfig.User, + config.RedisConfigFileName: &apiConfig.Redis, + config.ShareFileName: &apiConfig.Share, + config.WebhooksConfigFileName: &apiConfig.Webhooks, } ret.RootCmd = NewRootCmd(program.GetProcessName(), WithConfigMap(ret.configMap)) ret.ctx = context.WithValue(context.Background(), "version", version.Version) ret.Command.RunE = func(cmd *cobra.Command, args []string) error { + apiConfig.ConfigPath = ret.configPath return ret.runE() } return ret diff --git a/pkg/common/cmd/auth.go b/pkg/common/cmd/auth.go index bf51a2cf3..a5ab3fea7 100644 --- a/pkg/common/cmd/auth.go +++ b/pkg/common/cmd/auth.go @@ -18,6 +18,7 @@ import ( "context" "github.com/openimsdk/open-im-server/v3/internal/rpc/auth" + "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/open-im-server/v3/pkg/common/startrpc" "github.com/openimsdk/open-im-server/v3/version" "github.com/openimsdk/tools/system/program" @@ -35,10 +36,10 @@ func NewAuthRpcCmd() *AuthRpcCmd { var authConfig auth.Config ret := &AuthRpcCmd{authConfig: &authConfig} ret.configMap = map[string]any{ - OpenIMRPCAuthCfgFileName: &authConfig.RpcConfig, - RedisConfigFileName: &authConfig.RedisConfig, - ShareFileName: &authConfig.Share, - DiscoveryConfigFilename: &authConfig.Discovery, + config.OpenIMRPCAuthCfgFileName: &authConfig.RpcConfig, + config.RedisConfigFileName: &authConfig.RedisConfig, + config.ShareFileName: &authConfig.Share, + config.DiscoveryConfigFilename: &authConfig.Discovery, } ret.RootCmd = NewRootCmd(program.GetProcessName(), WithConfigMap(ret.configMap)) ret.ctx = context.WithValue(context.Background(), "version", version.Version) @@ -56,5 +57,12 @@ func (a *AuthRpcCmd) Exec() error { func (a *AuthRpcCmd) runE() error { return startrpc.Start(a.ctx, &a.authConfig.Discovery, &a.authConfig.RpcConfig.Prometheus, a.authConfig.RpcConfig.RPC.ListenIP, a.authConfig.RpcConfig.RPC.RegisterIP, a.authConfig.RpcConfig.RPC.AutoSetPorts, a.authConfig.RpcConfig.RPC.Ports, - a.Index(), a.authConfig.Discovery.RpcService.Auth, nil, a.authConfig, auth.Start) + a.Index(), a.authConfig.Discovery.RpcService.Auth, nil, a.authConfig, + []string{ + a.authConfig.RpcConfig.GetConfigFileName(), + a.authConfig.Share.GetConfigFileName(), + a.authConfig.RedisConfig.GetConfigFileName(), + a.authConfig.Discovery.GetConfigFileName(), + }, + auth.Start) } diff --git a/pkg/common/cmd/constant.go b/pkg/common/cmd/constant.go deleted file mode 100644 index 45dbcafda..000000000 --- a/pkg/common/cmd/constant.go +++ /dev/null @@ -1,96 +0,0 @@ -// Copyright © 2023 OpenIM. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package cmd - -import ( - "strings" -) - -var ( - FileName string - NotificationFileName string - ShareFileName string - WebhooksConfigFileName string - LocalCacheConfigFileName string - KafkaConfigFileName string - RedisConfigFileName string - MongodbConfigFileName string - MinioConfigFileName string - LogConfigFileName string - OpenIMAPICfgFileName string - OpenIMCronTaskCfgFileName string - OpenIMMsgGatewayCfgFileName string - OpenIMMsgTransferCfgFileName string - OpenIMPushCfgFileName string - OpenIMRPCAuthCfgFileName string - OpenIMRPCConversationCfgFileName string - OpenIMRPCFriendCfgFileName string - OpenIMRPCGroupCfgFileName string - OpenIMRPCMsgCfgFileName string - OpenIMRPCThirdCfgFileName string - OpenIMRPCUserCfgFileName string - DiscoveryConfigFilename string -) - -var ConfigEnvPrefixMap map[string]string - -func init() { - FileName = "config.yaml" - NotificationFileName = "notification.yml" - ShareFileName = "share.yml" - WebhooksConfigFileName = "webhooks.yml" - LocalCacheConfigFileName = "local-cache.yml" - KafkaConfigFileName = "kafka.yml" - RedisConfigFileName = "redis.yml" - MongodbConfigFileName = "mongodb.yml" - MinioConfigFileName = "minio.yml" - LogConfigFileName = "log.yml" - OpenIMAPICfgFileName = "openim-api.yml" - OpenIMCronTaskCfgFileName = "openim-crontask.yml" - OpenIMMsgGatewayCfgFileName = "openim-msggateway.yml" - OpenIMMsgTransferCfgFileName = "openim-msgtransfer.yml" - OpenIMPushCfgFileName = "openim-push.yml" - OpenIMRPCAuthCfgFileName = "openim-rpc-auth.yml" - OpenIMRPCConversationCfgFileName = "openim-rpc-conversation.yml" - OpenIMRPCFriendCfgFileName = "openim-rpc-friend.yml" - OpenIMRPCGroupCfgFileName = "openim-rpc-group.yml" - OpenIMRPCMsgCfgFileName = "openim-rpc-msg.yml" - OpenIMRPCThirdCfgFileName = "openim-rpc-third.yml" - OpenIMRPCUserCfgFileName = "openim-rpc-user.yml" - DiscoveryConfigFilename = "discovery.yml" - - ConfigEnvPrefixMap = make(map[string]string) - fileNames := []string{ - FileName, NotificationFileName, ShareFileName, WebhooksConfigFileName, - KafkaConfigFileName, RedisConfigFileName, - MongodbConfigFileName, MinioConfigFileName, LogConfigFileName, - OpenIMAPICfgFileName, OpenIMCronTaskCfgFileName, OpenIMMsgGatewayCfgFileName, - OpenIMMsgTransferCfgFileName, OpenIMPushCfgFileName, OpenIMRPCAuthCfgFileName, - OpenIMRPCConversationCfgFileName, OpenIMRPCFriendCfgFileName, OpenIMRPCGroupCfgFileName, - OpenIMRPCMsgCfgFileName, OpenIMRPCThirdCfgFileName, OpenIMRPCUserCfgFileName, DiscoveryConfigFilename, - } - - for _, fileName := range fileNames { - envKey := strings.TrimSuffix(strings.TrimSuffix(fileName, ".yml"), ".yaml") - envKey = "IMENV_" + envKey - envKey = strings.ToUpper(strings.ReplaceAll(envKey, "-", "_")) - ConfigEnvPrefixMap[fileName] = envKey - } -} - -const ( - FlagConf = "config_folder_path" - FlagTransferIndex = "index" -) diff --git a/pkg/common/cmd/conversation.go b/pkg/common/cmd/conversation.go index 4d38f7fd4..12c29a873 100644 --- a/pkg/common/cmd/conversation.go +++ b/pkg/common/cmd/conversation.go @@ -18,6 +18,7 @@ import ( "context" "github.com/openimsdk/open-im-server/v3/internal/rpc/conversation" + "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/open-im-server/v3/pkg/common/startrpc" "github.com/openimsdk/open-im-server/v3/version" "github.com/openimsdk/tools/system/program" @@ -35,13 +36,13 @@ func NewConversationRpcCmd() *ConversationRpcCmd { var conversationConfig conversation.Config ret := &ConversationRpcCmd{conversationConfig: &conversationConfig} ret.configMap = map[string]any{ - OpenIMRPCConversationCfgFileName: &conversationConfig.RpcConfig, - RedisConfigFileName: &conversationConfig.RedisConfig, - MongodbConfigFileName: &conversationConfig.MongodbConfig, - ShareFileName: &conversationConfig.Share, - NotificationFileName: &conversationConfig.NotificationConfig, - LocalCacheConfigFileName: &conversationConfig.LocalCacheConfig, - DiscoveryConfigFilename: &conversationConfig.Discovery, + config.OpenIMRPCConversationCfgFileName: &conversationConfig.RpcConfig, + config.RedisConfigFileName: &conversationConfig.RedisConfig, + config.MongodbConfigFileName: &conversationConfig.MongodbConfig, + config.ShareFileName: &conversationConfig.Share, + config.NotificationFileName: &conversationConfig.NotificationConfig, + config.LocalCacheConfigFileName: &conversationConfig.LocalCacheConfig, + config.DiscoveryConfigFilename: &conversationConfig.Discovery, } ret.RootCmd = NewRootCmd(program.GetProcessName(), WithConfigMap(ret.configMap)) ret.ctx = context.WithValue(context.Background(), "version", version.Version) @@ -58,5 +59,15 @@ func (a *ConversationRpcCmd) Exec() error { func (a *ConversationRpcCmd) runE() error { return startrpc.Start(a.ctx, &a.conversationConfig.Discovery, &a.conversationConfig.RpcConfig.Prometheus, a.conversationConfig.RpcConfig.RPC.ListenIP, a.conversationConfig.RpcConfig.RPC.RegisterIP, a.conversationConfig.RpcConfig.RPC.AutoSetPorts, a.conversationConfig.RpcConfig.RPC.Ports, - a.Index(), a.conversationConfig.Discovery.RpcService.Conversation, &a.conversationConfig.NotificationConfig, a.conversationConfig, conversation.Start) + a.Index(), a.conversationConfig.Discovery.RpcService.Conversation, &a.conversationConfig.NotificationConfig, a.conversationConfig, + []string{ + a.conversationConfig.RpcConfig.GetConfigFileName(), + a.conversationConfig.RedisConfig.GetConfigFileName(), + a.conversationConfig.MongodbConfig.GetConfigFileName(), + a.conversationConfig.NotificationConfig.GetConfigFileName(), + a.conversationConfig.Share.GetConfigFileName(), + a.conversationConfig.LocalCacheConfig.GetConfigFileName(), + a.conversationConfig.Discovery.GetConfigFileName(), + }, + conversation.Start) } diff --git a/pkg/common/cmd/cron_task.go b/pkg/common/cmd/cron_task.go index d6c5e472e..e7eb0ce18 100644 --- a/pkg/common/cmd/cron_task.go +++ b/pkg/common/cmd/cron_task.go @@ -18,6 +18,7 @@ import ( "context" "github.com/openimsdk/open-im-server/v3/internal/tools" + "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/open-im-server/v3/version" "github.com/openimsdk/tools/system/program" "github.com/spf13/cobra" @@ -34,9 +35,9 @@ func NewCronTaskCmd() *CronTaskCmd { var cronTaskConfig tools.CronTaskConfig ret := &CronTaskCmd{cronTaskConfig: &cronTaskConfig} ret.configMap = map[string]any{ - OpenIMCronTaskCfgFileName: &cronTaskConfig.CronTask, - ShareFileName: &cronTaskConfig.Share, - DiscoveryConfigFilename: &cronTaskConfig.Discovery, + config.OpenIMCronTaskCfgFileName: &cronTaskConfig.CronTask, + config.ShareFileName: &cronTaskConfig.Share, + config.DiscoveryConfigFilename: &cronTaskConfig.Discovery, } ret.RootCmd = NewRootCmd(program.GetProcessName(), WithConfigMap(ret.configMap)) ret.ctx = context.WithValue(context.Background(), "version", version.Version) diff --git a/pkg/common/cmd/friend.go b/pkg/common/cmd/friend.go index 800116531..209d481bb 100644 --- a/pkg/common/cmd/friend.go +++ b/pkg/common/cmd/friend.go @@ -18,6 +18,7 @@ import ( "context" "github.com/openimsdk/open-im-server/v3/internal/rpc/relation" + "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/open-im-server/v3/pkg/common/startrpc" "github.com/openimsdk/open-im-server/v3/version" "github.com/openimsdk/tools/system/program" @@ -35,14 +36,14 @@ func NewFriendRpcCmd() *FriendRpcCmd { var relationConfig relation.Config ret := &FriendRpcCmd{relationConfig: &relationConfig} ret.configMap = map[string]any{ - OpenIMRPCFriendCfgFileName: &relationConfig.RpcConfig, - RedisConfigFileName: &relationConfig.RedisConfig, - MongodbConfigFileName: &relationConfig.MongodbConfig, - ShareFileName: &relationConfig.Share, - NotificationFileName: &relationConfig.NotificationConfig, - WebhooksConfigFileName: &relationConfig.WebhooksConfig, - LocalCacheConfigFileName: &relationConfig.LocalCacheConfig, - DiscoveryConfigFilename: &relationConfig.Discovery, + config.OpenIMRPCFriendCfgFileName: &relationConfig.RpcConfig, + config.RedisConfigFileName: &relationConfig.RedisConfig, + config.MongodbConfigFileName: &relationConfig.MongodbConfig, + config.ShareFileName: &relationConfig.Share, + config.NotificationFileName: &relationConfig.NotificationConfig, + config.WebhooksConfigFileName: &relationConfig.WebhooksConfig, + config.LocalCacheConfigFileName: &relationConfig.LocalCacheConfig, + config.DiscoveryConfigFilename: &relationConfig.Discovery, } ret.RootCmd = NewRootCmd(program.GetProcessName(), WithConfigMap(ret.configMap)) ret.ctx = context.WithValue(context.Background(), "version", version.Version) @@ -59,5 +60,16 @@ func (a *FriendRpcCmd) Exec() error { func (a *FriendRpcCmd) runE() error { return startrpc.Start(a.ctx, &a.relationConfig.Discovery, &a.relationConfig.RpcConfig.Prometheus, a.relationConfig.RpcConfig.RPC.ListenIP, a.relationConfig.RpcConfig.RPC.RegisterIP, a.relationConfig.RpcConfig.RPC.AutoSetPorts, a.relationConfig.RpcConfig.RPC.Ports, - a.Index(), a.relationConfig.Discovery.RpcService.Friend, &a.relationConfig.NotificationConfig, a.relationConfig, relation.Start) + a.Index(), a.relationConfig.Discovery.RpcService.Friend, &a.relationConfig.NotificationConfig, a.relationConfig, + []string{ + a.relationConfig.RpcConfig.GetConfigFileName(), + a.relationConfig.RedisConfig.GetConfigFileName(), + a.relationConfig.MongodbConfig.GetConfigFileName(), + a.relationConfig.NotificationConfig.GetConfigFileName(), + a.relationConfig.Share.GetConfigFileName(), + a.relationConfig.WebhooksConfig.GetConfigFileName(), + a.relationConfig.LocalCacheConfig.GetConfigFileName(), + a.relationConfig.Discovery.GetConfigFileName(), + }, + relation.Start) } diff --git a/pkg/common/cmd/group.go b/pkg/common/cmd/group.go index 4f8d17516..23fd460f7 100644 --- a/pkg/common/cmd/group.go +++ b/pkg/common/cmd/group.go @@ -18,6 +18,7 @@ import ( "context" "github.com/openimsdk/open-im-server/v3/internal/rpc/group" + "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/open-im-server/v3/pkg/common/startrpc" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/versionctx" "github.com/openimsdk/open-im-server/v3/version" @@ -36,14 +37,14 @@ func NewGroupRpcCmd() *GroupRpcCmd { var groupConfig group.Config ret := &GroupRpcCmd{groupConfig: &groupConfig} ret.configMap = map[string]any{ - OpenIMRPCGroupCfgFileName: &groupConfig.RpcConfig, - RedisConfigFileName: &groupConfig.RedisConfig, - MongodbConfigFileName: &groupConfig.MongodbConfig, - ShareFileName: &groupConfig.Share, - NotificationFileName: &groupConfig.NotificationConfig, - WebhooksConfigFileName: &groupConfig.WebhooksConfig, - LocalCacheConfigFileName: &groupConfig.LocalCacheConfig, - DiscoveryConfigFilename: &groupConfig.Discovery, + config.OpenIMRPCGroupCfgFileName: &groupConfig.RpcConfig, + config.RedisConfigFileName: &groupConfig.RedisConfig, + config.MongodbConfigFileName: &groupConfig.MongodbConfig, + config.ShareFileName: &groupConfig.Share, + config.NotificationFileName: &groupConfig.NotificationConfig, + config.WebhooksConfigFileName: &groupConfig.WebhooksConfig, + config.LocalCacheConfigFileName: &groupConfig.LocalCacheConfig, + config.DiscoveryConfigFilename: &groupConfig.Discovery, } ret.RootCmd = NewRootCmd(program.GetProcessName(), WithConfigMap(ret.configMap)) ret.ctx = context.WithValue(context.Background(), "version", version.Version) @@ -60,5 +61,16 @@ func (a *GroupRpcCmd) Exec() error { func (a *GroupRpcCmd) runE() error { return startrpc.Start(a.ctx, &a.groupConfig.Discovery, &a.groupConfig.RpcConfig.Prometheus, a.groupConfig.RpcConfig.RPC.ListenIP, a.groupConfig.RpcConfig.RPC.RegisterIP, a.groupConfig.RpcConfig.RPC.AutoSetPorts, a.groupConfig.RpcConfig.RPC.Ports, - a.Index(), a.groupConfig.Discovery.RpcService.Group, &a.groupConfig.NotificationConfig, a.groupConfig, group.Start, versionctx.EnableVersionCtx()) + a.Index(), a.groupConfig.Discovery.RpcService.Group, &a.groupConfig.NotificationConfig, a.groupConfig, + []string{ + a.groupConfig.RpcConfig.GetConfigFileName(), + a.groupConfig.RedisConfig.GetConfigFileName(), + a.groupConfig.MongodbConfig.GetConfigFileName(), + a.groupConfig.NotificationConfig.GetConfigFileName(), + a.groupConfig.Share.GetConfigFileName(), + a.groupConfig.WebhooksConfig.GetConfigFileName(), + a.groupConfig.LocalCacheConfig.GetConfigFileName(), + a.groupConfig.Discovery.GetConfigFileName(), + }, + group.Start, versionctx.EnableVersionCtx()) } diff --git a/pkg/common/cmd/msg.go b/pkg/common/cmd/msg.go index b6647f9c1..b6f0b6131 100644 --- a/pkg/common/cmd/msg.go +++ b/pkg/common/cmd/msg.go @@ -18,6 +18,7 @@ import ( "context" "github.com/openimsdk/open-im-server/v3/internal/rpc/msg" + "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/open-im-server/v3/pkg/common/startrpc" "github.com/openimsdk/open-im-server/v3/version" "github.com/openimsdk/tools/system/program" @@ -35,15 +36,15 @@ func NewMsgRpcCmd() *MsgRpcCmd { var msgConfig msg.Config ret := &MsgRpcCmd{msgConfig: &msgConfig} ret.configMap = map[string]any{ - OpenIMRPCMsgCfgFileName: &msgConfig.RpcConfig, - RedisConfigFileName: &msgConfig.RedisConfig, - MongodbConfigFileName: &msgConfig.MongodbConfig, - KafkaConfigFileName: &msgConfig.KafkaConfig, - ShareFileName: &msgConfig.Share, - NotificationFileName: &msgConfig.NotificationConfig, - WebhooksConfigFileName: &msgConfig.WebhooksConfig, - LocalCacheConfigFileName: &msgConfig.LocalCacheConfig, - DiscoveryConfigFilename: &msgConfig.Discovery, + config.OpenIMRPCMsgCfgFileName: &msgConfig.RpcConfig, + config.RedisConfigFileName: &msgConfig.RedisConfig, + config.MongodbConfigFileName: &msgConfig.MongodbConfig, + config.KafkaConfigFileName: &msgConfig.KafkaConfig, + config.ShareFileName: &msgConfig.Share, + config.NotificationFileName: &msgConfig.NotificationConfig, + config.WebhooksConfigFileName: &msgConfig.WebhooksConfig, + config.LocalCacheConfigFileName: &msgConfig.LocalCacheConfig, + config.DiscoveryConfigFilename: &msgConfig.Discovery, } ret.RootCmd = NewRootCmd(program.GetProcessName(), WithConfigMap(ret.configMap)) ret.ctx = context.WithValue(context.Background(), "version", version.Version) @@ -60,5 +61,17 @@ func (a *MsgRpcCmd) Exec() error { func (a *MsgRpcCmd) runE() error { return startrpc.Start(a.ctx, &a.msgConfig.Discovery, &a.msgConfig.RpcConfig.Prometheus, a.msgConfig.RpcConfig.RPC.ListenIP, a.msgConfig.RpcConfig.RPC.RegisterIP, a.msgConfig.RpcConfig.RPC.AutoSetPorts, a.msgConfig.RpcConfig.RPC.Ports, - a.Index(), a.msgConfig.Discovery.RpcService.Msg, &a.msgConfig.NotificationConfig, a.msgConfig, msg.Start) + a.Index(), a.msgConfig.Discovery.RpcService.Msg, &a.msgConfig.NotificationConfig, a.msgConfig, + []string{ + a.msgConfig.RpcConfig.GetConfigFileName(), + a.msgConfig.RedisConfig.GetConfigFileName(), + a.msgConfig.MongodbConfig.GetConfigFileName(), + a.msgConfig.KafkaConfig.GetConfigFileName(), + a.msgConfig.NotificationConfig.GetConfigFileName(), + a.msgConfig.Share.GetConfigFileName(), + a.msgConfig.WebhooksConfig.GetConfigFileName(), + a.msgConfig.LocalCacheConfig.GetConfigFileName(), + a.msgConfig.Discovery.GetConfigFileName(), + }, + msg.Start) } diff --git a/pkg/common/cmd/msg_gateway.go b/pkg/common/cmd/msg_gateway.go index 6363bfbf9..3f66ea720 100644 --- a/pkg/common/cmd/msg_gateway.go +++ b/pkg/common/cmd/msg_gateway.go @@ -18,6 +18,7 @@ import ( "context" "github.com/openimsdk/open-im-server/v3/internal/msggateway" + "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/open-im-server/v3/version" "github.com/openimsdk/tools/system/program" @@ -35,11 +36,11 @@ func NewMsgGatewayCmd() *MsgGatewayCmd { var msgGatewayConfig msggateway.Config ret := &MsgGatewayCmd{msgGatewayConfig: &msgGatewayConfig} ret.configMap = map[string]any{ - OpenIMMsgGatewayCfgFileName: &msgGatewayConfig.MsgGateway, - ShareFileName: &msgGatewayConfig.Share, - RedisConfigFileName: &msgGatewayConfig.RedisConfig, - WebhooksConfigFileName: &msgGatewayConfig.WebhooksConfig, - DiscoveryConfigFilename: &msgGatewayConfig.Discovery, + config.OpenIMMsgGatewayCfgFileName: &msgGatewayConfig.MsgGateway, + config.ShareFileName: &msgGatewayConfig.Share, + config.RedisConfigFileName: &msgGatewayConfig.RedisConfig, + config.WebhooksConfigFileName: &msgGatewayConfig.WebhooksConfig, + config.DiscoveryConfigFilename: &msgGatewayConfig.Discovery, } ret.RootCmd = NewRootCmd(program.GetProcessName(), WithConfigMap(ret.configMap)) ret.ctx = context.WithValue(context.Background(), "version", version.Version) diff --git a/pkg/common/cmd/msg_transfer.go b/pkg/common/cmd/msg_transfer.go index 364393413..fbb83c65f 100644 --- a/pkg/common/cmd/msg_transfer.go +++ b/pkg/common/cmd/msg_transfer.go @@ -18,6 +18,7 @@ import ( "context" "github.com/openimsdk/open-im-server/v3/internal/msgtransfer" + "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/open-im-server/v3/version" "github.com/openimsdk/tools/system/program" "github.com/spf13/cobra" @@ -34,13 +35,13 @@ func NewMsgTransferCmd() *MsgTransferCmd { var msgTransferConfig msgtransfer.Config ret := &MsgTransferCmd{msgTransferConfig: &msgTransferConfig} ret.configMap = map[string]any{ - OpenIMMsgTransferCfgFileName: &msgTransferConfig.MsgTransfer, - RedisConfigFileName: &msgTransferConfig.RedisConfig, - MongodbConfigFileName: &msgTransferConfig.MongodbConfig, - KafkaConfigFileName: &msgTransferConfig.KafkaConfig, - ShareFileName: &msgTransferConfig.Share, - WebhooksConfigFileName: &msgTransferConfig.WebhooksConfig, - DiscoveryConfigFilename: &msgTransferConfig.Discovery, + config.OpenIMMsgTransferCfgFileName: &msgTransferConfig.MsgTransfer, + config.RedisConfigFileName: &msgTransferConfig.RedisConfig, + config.MongodbConfigFileName: &msgTransferConfig.MongodbConfig, + config.KafkaConfigFileName: &msgTransferConfig.KafkaConfig, + config.ShareFileName: &msgTransferConfig.Share, + config.WebhooksConfigFileName: &msgTransferConfig.WebhooksConfig, + config.DiscoveryConfigFilename: &msgTransferConfig.Discovery, } ret.RootCmd = NewRootCmd(program.GetProcessName(), WithConfigMap(ret.configMap)) ret.ctx = context.WithValue(context.Background(), "version", version.Version) diff --git a/pkg/common/cmd/msg_utils.go b/pkg/common/cmd/msg_utils.go index a0a9b0410..f0e590e2c 100644 --- a/pkg/common/cmd/msg_utils.go +++ b/pkg/common/cmd/msg_utils.go @@ -15,6 +15,7 @@ package cmd import ( + "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/spf13/cobra" ) @@ -26,11 +27,11 @@ func (m *MsgUtilsCmd) AddUserIDFlag() { m.Command.PersistentFlags().StringP("userID", "u", "", "openIM userID") } func (m *MsgUtilsCmd) AddIndexFlag() { - m.Command.PersistentFlags().IntP(FlagTransferIndex, "i", 0, "process startup sequence number") + m.Command.PersistentFlags().IntP(config.FlagTransferIndex, "i", 0, "process startup sequence number") } func (m *MsgUtilsCmd) AddConfigDirFlag() { - m.Command.PersistentFlags().StringP(FlagConf, "c", "", "path of config directory") + m.Command.PersistentFlags().StringP(config.FlagConf, "c", "", "path of config directory") } diff --git a/pkg/common/cmd/push.go b/pkg/common/cmd/push.go index 93b502cd8..41b9d56e6 100644 --- a/pkg/common/cmd/push.go +++ b/pkg/common/cmd/push.go @@ -18,6 +18,7 @@ import ( "context" "github.com/openimsdk/open-im-server/v3/internal/push" + "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/open-im-server/v3/pkg/common/startrpc" "github.com/openimsdk/open-im-server/v3/version" "github.com/openimsdk/tools/system/program" @@ -35,14 +36,14 @@ func NewPushRpcCmd() *PushRpcCmd { var pushConfig push.Config ret := &PushRpcCmd{pushConfig: &pushConfig} ret.configMap = map[string]any{ - OpenIMPushCfgFileName: &pushConfig.RpcConfig, - RedisConfigFileName: &pushConfig.RedisConfig, - KafkaConfigFileName: &pushConfig.KafkaConfig, - ShareFileName: &pushConfig.Share, - NotificationFileName: &pushConfig.NotificationConfig, - WebhooksConfigFileName: &pushConfig.WebhooksConfig, - LocalCacheConfigFileName: &pushConfig.LocalCacheConfig, - DiscoveryConfigFilename: &pushConfig.Discovery, + config.OpenIMPushCfgFileName: &pushConfig.RpcConfig, + config.RedisConfigFileName: &pushConfig.RedisConfig, + config.KafkaConfigFileName: &pushConfig.KafkaConfig, + config.ShareFileName: &pushConfig.Share, + config.NotificationFileName: &pushConfig.NotificationConfig, + config.WebhooksConfigFileName: &pushConfig.WebhooksConfig, + config.LocalCacheConfigFileName: &pushConfig.LocalCacheConfig, + config.DiscoveryConfigFilename: &pushConfig.Discovery, } ret.RootCmd = NewRootCmd(program.GetProcessName(), WithConfigMap(ret.configMap)) ret.ctx = context.WithValue(context.Background(), "version", version.Version) @@ -60,5 +61,16 @@ func (a *PushRpcCmd) Exec() error { func (a *PushRpcCmd) runE() error { return startrpc.Start(a.ctx, &a.pushConfig.Discovery, &a.pushConfig.RpcConfig.Prometheus, a.pushConfig.RpcConfig.RPC.ListenIP, a.pushConfig.RpcConfig.RPC.RegisterIP, a.pushConfig.RpcConfig.RPC.AutoSetPorts, a.pushConfig.RpcConfig.RPC.Ports, - a.Index(), a.pushConfig.Discovery.RpcService.Push, &a.pushConfig.NotificationConfig, a.pushConfig, push.Start) + a.Index(), a.pushConfig.Discovery.RpcService.Push, &a.pushConfig.NotificationConfig, a.pushConfig, + []string{ + a.pushConfig.RpcConfig.GetConfigFileName(), + a.pushConfig.RedisConfig.GetConfigFileName(), + a.pushConfig.KafkaConfig.GetConfigFileName(), + a.pushConfig.NotificationConfig.GetConfigFileName(), + a.pushConfig.Share.GetConfigFileName(), + a.pushConfig.WebhooksConfig.GetConfigFileName(), + a.pushConfig.LocalCacheConfig.GetConfigFileName(), + a.pushConfig.Discovery.GetConfigFileName(), + }, + push.Start) } diff --git a/pkg/common/cmd/root.go b/pkg/common/cmd/root.go index 87252c133..20db4126e 100644 --- a/pkg/common/cmd/root.go +++ b/pkg/common/cmd/root.go @@ -1,28 +1,20 @@ -// Copyright © 2023 OpenIM. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - package cmd import ( + "context" + "encoding/json" "fmt" "github.com/openimsdk/open-im-server/v3/pkg/common/config" + kdisc "github.com/openimsdk/open-im-server/v3/pkg/common/discovery" + disetcd "github.com/openimsdk/open-im-server/v3/pkg/common/discovery/etcd" "github.com/openimsdk/open-im-server/v3/version" + "github.com/openimsdk/tools/discovery/etcd" "github.com/openimsdk/tools/errs" "github.com/openimsdk/tools/log" "github.com/openimsdk/tools/utils/runtimeenv" "github.com/spf13/cobra" + clientv3 "go.etcd.io/etcd/client/v3" ) type RootCmd struct { @@ -33,6 +25,7 @@ type RootCmd struct { log config.Log index int configPath string + etcdClient *clientv3.Client } func (r *RootCmd) ConfigPath() string { @@ -80,19 +73,43 @@ func NewRootCmd(processName string, opts ...func(*CmdOpts)) *RootCmd { SilenceUsage: true, SilenceErrors: false, } - cmd.Flags().StringP(FlagConf, "c", "", "path of config directory") - cmd.Flags().IntP(FlagTransferIndex, "i", 0, "process startup sequence number") + cmd.Flags().StringP(config.FlagConf, "c", "", "path of config directory") + cmd.Flags().IntP(config.FlagTransferIndex, "i", 0, "process startup sequence number") rootCmd.Command = cmd return rootCmd } +func (r *RootCmd) initEtcd() error { + configDirectory, _, err := r.getFlag(&r.Command) + if err != nil { + return err + } + disConfig := config.Discovery{} + env := runtimeenv.PrintRuntimeEnvironment() + err = config.Load(configDirectory, config.DiscoveryConfigFilename, config.EnvPrefixMap[config.DiscoveryConfigFilename], + env, &disConfig) + if err != nil { + return err + } + if disConfig.Enable == config.ETCD { + discov, _ := kdisc.NewDiscoveryRegister(&disConfig, env) + r.etcdClient = discov.(*etcd.SvcDiscoveryRegistryImpl).GetClient() + } + return nil +} + func (r *RootCmd) persistentPreRun(cmd *cobra.Command, opts ...func(*CmdOpts)) error { + if err := r.initEtcd(); err != nil { + return err + } cmdOpts := r.applyOptions(opts...) if err := r.initializeConfiguration(cmd, cmdOpts); err != nil { return err } - + if err := r.updateConfigFromEtcd(cmdOpts); err != nil { + return err + } if err := r.initializeLogger(cmdOpts); err != nil { return errs.WrapMsg(err, "failed to initialize logger") } @@ -111,13 +128,43 @@ func (r *RootCmd) initializeConfiguration(cmd *cobra.Command, opts *CmdOpts) err // Load common configuration file //opts.configMap[ShareFileName] = StructEnvPrefix{EnvPrefix: shareEnvPrefix, ConfigStruct: &r.share} for configFileName, configStruct := range opts.configMap { - err := config.Load(configDirectory, configFileName, ConfigEnvPrefixMap[configFileName], runtimeEnv, configStruct) + err := config.Load(configDirectory, configFileName, config.EnvPrefixMap[configFileName], runtimeEnv, configStruct) if err != nil { return err } } // Load common log configuration file - return config.Load(configDirectory, LogConfigFileName, ConfigEnvPrefixMap[LogConfigFileName], runtimeEnv, &r.log) + return config.Load(configDirectory, config.LogConfigFileName, config.EnvPrefixMap[config.LogConfigFileName], runtimeEnv, &r.log) +} + +func (r *RootCmd) updateConfigFromEtcd(opts *CmdOpts) error { + if r.etcdClient == nil { + return nil + } + + update := func(configFileName string, configStruct any) error { + key := disetcd.BuildKey(configFileName) + etcdRes, err := r.etcdClient.Get(context.TODO(), key) + if err != nil || etcdRes.Count == 0 { + return nil + } + err = json.Unmarshal(etcdRes.Kvs[0].Value, configStruct) + if err != nil { + return errs.WrapMsg(err, "failed to unmarshal config from etcd") + } + return nil + } + for configFileName, configStruct := range opts.configMap { + if err := update(configFileName, configStruct); err != nil { + return err + } + } + if err := update(config.LogConfigFileName, &r.log); err != nil { + return err + } + // Load common log configuration file + return nil + } func (r *RootCmd) applyOptions(opts ...func(*CmdOpts)) *CmdOpts { @@ -158,12 +205,12 @@ func defaultCmdOpts() *CmdOpts { } func (r *RootCmd) getFlag(cmd *cobra.Command) (string, int, error) { - configDirectory, err := cmd.Flags().GetString(FlagConf) + configDirectory, err := cmd.Flags().GetString(config.FlagConf) if err != nil { return "", 0, errs.Wrap(err) } r.configPath = configDirectory - index, err := cmd.Flags().GetInt(FlagTransferIndex) + index, err := cmd.Flags().GetInt(config.FlagTransferIndex) if err != nil { return "", 0, errs.Wrap(err) } diff --git a/pkg/common/cmd/third.go b/pkg/common/cmd/third.go index e43f61732..5086116b5 100644 --- a/pkg/common/cmd/third.go +++ b/pkg/common/cmd/third.go @@ -18,6 +18,7 @@ import ( "context" "github.com/openimsdk/open-im-server/v3/internal/rpc/third" + "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/open-im-server/v3/pkg/common/startrpc" "github.com/openimsdk/open-im-server/v3/version" "github.com/openimsdk/tools/system/program" @@ -35,14 +36,14 @@ func NewThirdRpcCmd() *ThirdRpcCmd { var thirdConfig third.Config ret := &ThirdRpcCmd{thirdConfig: &thirdConfig} ret.configMap = map[string]any{ - OpenIMRPCThirdCfgFileName: &thirdConfig.RpcConfig, - RedisConfigFileName: &thirdConfig.RedisConfig, - MongodbConfigFileName: &thirdConfig.MongodbConfig, - ShareFileName: &thirdConfig.Share, - NotificationFileName: &thirdConfig.NotificationConfig, - MinioConfigFileName: &thirdConfig.MinioConfig, - LocalCacheConfigFileName: &thirdConfig.LocalCacheConfig, - DiscoveryConfigFilename: &thirdConfig.Discovery, + config.OpenIMRPCThirdCfgFileName: &thirdConfig.RpcConfig, + config.RedisConfigFileName: &thirdConfig.RedisConfig, + config.MongodbConfigFileName: &thirdConfig.MongodbConfig, + config.ShareFileName: &thirdConfig.Share, + config.NotificationFileName: &thirdConfig.NotificationConfig, + config.MinioConfigFileName: &thirdConfig.MinioConfig, + config.LocalCacheConfigFileName: &thirdConfig.LocalCacheConfig, + config.DiscoveryConfigFilename: &thirdConfig.Discovery, } ret.RootCmd = NewRootCmd(program.GetProcessName(), WithConfigMap(ret.configMap)) ret.ctx = context.WithValue(context.Background(), "version", version.Version) @@ -59,5 +60,16 @@ func (a *ThirdRpcCmd) Exec() error { func (a *ThirdRpcCmd) runE() error { return startrpc.Start(a.ctx, &a.thirdConfig.Discovery, &a.thirdConfig.RpcConfig.Prometheus, a.thirdConfig.RpcConfig.RPC.ListenIP, a.thirdConfig.RpcConfig.RPC.RegisterIP, a.thirdConfig.RpcConfig.RPC.AutoSetPorts, a.thirdConfig.RpcConfig.RPC.Ports, - a.Index(), a.thirdConfig.Discovery.RpcService.Third, &a.thirdConfig.NotificationConfig, a.thirdConfig, third.Start) + a.Index(), a.thirdConfig.Discovery.RpcService.Third, &a.thirdConfig.NotificationConfig, a.thirdConfig, + []string{ + a.thirdConfig.RpcConfig.GetConfigFileName(), + a.thirdConfig.RedisConfig.GetConfigFileName(), + a.thirdConfig.MongodbConfig.GetConfigFileName(), + a.thirdConfig.NotificationConfig.GetConfigFileName(), + a.thirdConfig.Share.GetConfigFileName(), + a.thirdConfig.MinioConfig.GetConfigFileName(), + a.thirdConfig.LocalCacheConfig.GetConfigFileName(), + a.thirdConfig.Discovery.GetConfigFileName(), + }, + third.Start) } diff --git a/pkg/common/cmd/user.go b/pkg/common/cmd/user.go index dc848a775..61125e0c3 100644 --- a/pkg/common/cmd/user.go +++ b/pkg/common/cmd/user.go @@ -18,6 +18,7 @@ import ( "context" "github.com/openimsdk/open-im-server/v3/internal/rpc/user" + "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/open-im-server/v3/pkg/common/startrpc" "github.com/openimsdk/open-im-server/v3/version" "github.com/openimsdk/tools/system/program" @@ -35,15 +36,15 @@ func NewUserRpcCmd() *UserRpcCmd { var userConfig user.Config ret := &UserRpcCmd{userConfig: &userConfig} ret.configMap = map[string]any{ - OpenIMRPCUserCfgFileName: &userConfig.RpcConfig, - RedisConfigFileName: &userConfig.RedisConfig, - MongodbConfigFileName: &userConfig.MongodbConfig, - KafkaConfigFileName: &userConfig.KafkaConfig, - ShareFileName: &userConfig.Share, - NotificationFileName: &userConfig.NotificationConfig, - WebhooksConfigFileName: &userConfig.WebhooksConfig, - LocalCacheConfigFileName: &userConfig.LocalCacheConfig, - DiscoveryConfigFilename: &userConfig.Discovery, + config.OpenIMRPCUserCfgFileName: &userConfig.RpcConfig, + config.RedisConfigFileName: &userConfig.RedisConfig, + config.MongodbConfigFileName: &userConfig.MongodbConfig, + config.KafkaConfigFileName: &userConfig.KafkaConfig, + config.ShareFileName: &userConfig.Share, + config.NotificationFileName: &userConfig.NotificationConfig, + config.WebhooksConfigFileName: &userConfig.WebhooksConfig, + config.LocalCacheConfigFileName: &userConfig.LocalCacheConfig, + config.DiscoveryConfigFilename: &userConfig.Discovery, } ret.RootCmd = NewRootCmd(program.GetProcessName(), WithConfigMap(ret.configMap)) ret.ctx = context.WithValue(context.Background(), "version", version.Version) @@ -60,5 +61,17 @@ func (a *UserRpcCmd) Exec() error { func (a *UserRpcCmd) runE() error { return startrpc.Start(a.ctx, &a.userConfig.Discovery, &a.userConfig.RpcConfig.Prometheus, a.userConfig.RpcConfig.RPC.ListenIP, a.userConfig.RpcConfig.RPC.RegisterIP, a.userConfig.RpcConfig.RPC.AutoSetPorts, a.userConfig.RpcConfig.RPC.Ports, - a.Index(), a.userConfig.Discovery.RpcService.User, &a.userConfig.NotificationConfig, a.userConfig, user.Start) + a.Index(), a.userConfig.Discovery.RpcService.User, &a.userConfig.NotificationConfig, a.userConfig, + []string{ + a.userConfig.RpcConfig.GetConfigFileName(), + a.userConfig.RedisConfig.GetConfigFileName(), + a.userConfig.MongodbConfig.GetConfigFileName(), + a.userConfig.KafkaConfig.GetConfigFileName(), + a.userConfig.NotificationConfig.GetConfigFileName(), + a.userConfig.Share.GetConfigFileName(), + a.userConfig.WebhooksConfig.GetConfigFileName(), + a.userConfig.LocalCacheConfig.GetConfigFileName(), + a.userConfig.Discovery.GetConfigFileName(), + }, + user.Start) } diff --git a/pkg/common/config/config.go b/pkg/common/config/config.go index da11a20e7..1b9121b7a 100644 --- a/pkg/common/config/config.go +++ b/pkg/common/config/config.go @@ -679,3 +679,220 @@ func InitNotification(notification *Notification) { notification.ConversationSetPrivate.UnreadCount = false notification.ConversationSetPrivate.ReliabilityLevel = 1 } + +type AllConfig struct { + Discovery Discovery + Kafka Kafka + LocalCache LocalCache + Log Log + Minio Minio + Mongo Mongo + Notification Notification + API API + CronTask CronTask + MsgGateway MsgGateway + MsgTransfer MsgTransfer + Push Push + Auth Auth + Conversation Conversation + Friend Friend + Group Group + Msg Msg + Third Third + User User + Redis Redis + Share Share + Webhooks Webhooks +} + +func (a *AllConfig) Name2Config(name string) any { + switch name { + case a.Discovery.GetConfigFileName(): + return a.Discovery + case a.Kafka.GetConfigFileName(): + return a.Kafka + case a.LocalCache.GetConfigFileName(): + return a.LocalCache + case a.Log.GetConfigFileName(): + return a.Log + case a.Minio.GetConfigFileName(): + return a.Minio + case a.Mongo.GetConfigFileName(): + return a.Mongo + case a.Notification.GetConfigFileName(): + return a.Notification + case a.API.GetConfigFileName(): + return a.API + case a.CronTask.GetConfigFileName(): + return a.CronTask + case a.MsgGateway.GetConfigFileName(): + return a.MsgGateway + case a.MsgTransfer.GetConfigFileName(): + return a.MsgTransfer + case a.Push.GetConfigFileName(): + return a.Push + case a.Auth.GetConfigFileName(): + return a.Auth + case a.Conversation.GetConfigFileName(): + return a.Conversation + case a.Friend.GetConfigFileName(): + return a.Friend + case a.Group.GetConfigFileName(): + return a.Group + case a.Msg.GetConfigFileName(): + return a.Msg + case a.Third.GetConfigFileName(): + return a.Third + case a.User.GetConfigFileName(): + return a.User + case a.Redis.GetConfigFileName(): + return a.Redis + case a.Share.GetConfigFileName(): + return a.Share + case a.Webhooks.GetConfigFileName(): + return a.Webhooks + default: + return nil + } +} + +func (a *AllConfig) GetConfigNames() []string { + return []string{ + a.Discovery.GetConfigFileName(), + a.Kafka.GetConfigFileName(), + a.LocalCache.GetConfigFileName(), + a.Log.GetConfigFileName(), + a.Minio.GetConfigFileName(), + a.Mongo.GetConfigFileName(), + a.Notification.GetConfigFileName(), + a.API.GetConfigFileName(), + a.CronTask.GetConfigFileName(), + a.MsgGateway.GetConfigFileName(), + a.MsgTransfer.GetConfigFileName(), + a.Push.GetConfigFileName(), + a.Auth.GetConfigFileName(), + a.Conversation.GetConfigFileName(), + a.Friend.GetConfigFileName(), + a.Group.GetConfigFileName(), + a.Msg.GetConfigFileName(), + a.Third.GetConfigFileName(), + a.User.GetConfigFileName(), + a.Redis.GetConfigFileName(), + a.Share.GetConfigFileName(), + a.Webhooks.GetConfigFileName(), + } +} + +var ( + FileName = "config.yaml" + DiscoveryConfigFilename = "discovery.yml" + KafkaConfigFileName = "kafka.yml" + LocalCacheConfigFileName = "local-cache.yml" + LogConfigFileName = "log.yml" + MinioConfigFileName = "minio.yml" + MongodbConfigFileName = "mongodb.yml" + NotificationFileName = "notification.yml" + OpenIMAPICfgFileName = "openim-api.yml" + OpenIMCronTaskCfgFileName = "openim-crontask.yml" + OpenIMMsgGatewayCfgFileName = "openim-msggateway.yml" + OpenIMMsgTransferCfgFileName = "openim-msgtransfer.yml" + OpenIMPushCfgFileName = "openim-push.yml" + OpenIMRPCAuthCfgFileName = "openim-rpc-auth.yml" + OpenIMRPCConversationCfgFileName = "openim-rpc-conversation.yml" + OpenIMRPCFriendCfgFileName = "openim-rpc-friend.yml" + OpenIMRPCGroupCfgFileName = "openim-rpc-group.yml" + OpenIMRPCMsgCfgFileName = "openim-rpc-msg.yml" + OpenIMRPCThirdCfgFileName = "openim-rpc-third.yml" + OpenIMRPCUserCfgFileName = "openim-rpc-user.yml" + RedisConfigFileName = "redis.yml" + ShareFileName = "share.yml" + WebhooksConfigFileName = "webhooks.yml" +) + +func (d *Discovery) GetConfigFileName() string { + return DiscoveryConfigFilename +} + +func (k *Kafka) GetConfigFileName() string { + return KafkaConfigFileName +} + +func (lc *LocalCache) GetConfigFileName() string { + return LocalCacheConfigFileName +} + +func (l *Log) GetConfigFileName() string { + return LogConfigFileName +} + +func (m *Minio) GetConfigFileName() string { + return MinioConfigFileName +} + +func (m *Mongo) GetConfigFileName() string { + return MongodbConfigFileName +} + +func (n *Notification) GetConfigFileName() string { + return NotificationFileName +} + +func (a *API) GetConfigFileName() string { + return OpenIMAPICfgFileName +} + +func (ct *CronTask) GetConfigFileName() string { + return OpenIMCronTaskCfgFileName +} + +func (mg *MsgGateway) GetConfigFileName() string { + return OpenIMMsgGatewayCfgFileName +} + +func (mt *MsgTransfer) GetConfigFileName() string { + return OpenIMMsgTransferCfgFileName +} + +func (p *Push) GetConfigFileName() string { + return OpenIMPushCfgFileName +} + +func (a *Auth) GetConfigFileName() string { + return OpenIMRPCAuthCfgFileName +} + +func (c *Conversation) GetConfigFileName() string { + return OpenIMRPCConversationCfgFileName +} + +func (f *Friend) GetConfigFileName() string { + return OpenIMRPCFriendCfgFileName +} + +func (g *Group) GetConfigFileName() string { + return OpenIMRPCGroupCfgFileName +} + +func (m *Msg) GetConfigFileName() string { + return OpenIMRPCMsgCfgFileName +} + +func (t *Third) GetConfigFileName() string { + return OpenIMRPCThirdCfgFileName +} + +func (u *User) GetConfigFileName() string { + return OpenIMRPCUserCfgFileName +} + +func (r *Redis) GetConfigFileName() string { + return RedisConfigFileName +} + +func (s *Share) GetConfigFileName() string { + return ShareFileName +} + +func (w *Webhooks) GetConfigFileName() string { + return WebhooksConfigFileName +} diff --git a/pkg/common/config/env.go b/pkg/common/config/env.go new file mode 100644 index 000000000..99ccb3ca0 --- /dev/null +++ b/pkg/common/config/env.go @@ -0,0 +1,30 @@ +package config + +import "strings" + +var EnvPrefixMap map[string]string + +func init() { + EnvPrefixMap = make(map[string]string) + fileNames := []string{ + FileName, NotificationFileName, ShareFileName, WebhooksConfigFileName, + KafkaConfigFileName, RedisConfigFileName, + MongodbConfigFileName, MinioConfigFileName, LogConfigFileName, + OpenIMAPICfgFileName, OpenIMCronTaskCfgFileName, OpenIMMsgGatewayCfgFileName, + OpenIMMsgTransferCfgFileName, OpenIMPushCfgFileName, OpenIMRPCAuthCfgFileName, + OpenIMRPCConversationCfgFileName, OpenIMRPCFriendCfgFileName, OpenIMRPCGroupCfgFileName, + OpenIMRPCMsgCfgFileName, OpenIMRPCThirdCfgFileName, OpenIMRPCUserCfgFileName, DiscoveryConfigFilename, + } + + for _, fileName := range fileNames { + envKey := strings.TrimSuffix(strings.TrimSuffix(fileName, ".yml"), ".yaml") + envKey = "IMENV_" + envKey + envKey = strings.ToUpper(strings.ReplaceAll(envKey, "-", "_")) + EnvPrefixMap[fileName] = envKey + } +} + +const ( + FlagConf = "config_folder_path" + FlagTransferIndex = "index" +) diff --git a/pkg/common/config/load_config_test.go b/pkg/common/config/load_config_test.go index e575b27a9..612d44335 100644 --- a/pkg/common/config/load_config_test.go +++ b/pkg/common/config/load_config_test.go @@ -23,7 +23,7 @@ func TestLoadMongoConfig(t *testing.T) { // os.Setenv("IMENV_MONGODB_URI", "openIM123") // os.Setenv("IMENV_MONGODB_USERNAME", "openIM123") err := Load("../../../config/", "mongodb.yml", "IMENV_MONGODB", "source", &mongo) - // err := LoadConfig("../../../config/mongodb.yml", "IMENV_MONGODB", &mongo) + // err := LoadApiConfig("../../../config/mongodb.yml", "IMENV_MONGODB", &mongo) assert.Nil(t, err) t.Log(mongo.Password) diff --git a/pkg/common/config/parse.go b/pkg/common/config/parse.go index ea62c1de0..8fa8812ef 100644 --- a/pkg/common/config/parse.go +++ b/pkg/common/config/parse.go @@ -26,9 +26,7 @@ import ( ) const ( - FileName = "config.yaml" - NotificationFileName = "notification.yaml" - DefaultFolderPath = "../config/" + DefaultFolderPath = "../config/" ) // return absolude path join ../config/, this is k8s container config path. diff --git a/pkg/common/discoveryregister/direct/direct_resolver.go b/pkg/common/discovery/direct/direct_resolver.go similarity index 100% rename from pkg/common/discoveryregister/direct/direct_resolver.go rename to pkg/common/discovery/direct/direct_resolver.go diff --git a/pkg/common/discoveryregister/direct/directconn.go b/pkg/common/discovery/direct/directconn.go similarity index 100% rename from pkg/common/discoveryregister/direct/directconn.go rename to pkg/common/discovery/direct/directconn.go diff --git a/pkg/common/discoveryregister/direct/doc.go b/pkg/common/discovery/direct/doc.go similarity index 94% rename from pkg/common/discoveryregister/direct/doc.go rename to pkg/common/discovery/direct/doc.go index b3cd0f804..0ba0d1437 100644 --- a/pkg/common/discoveryregister/direct/doc.go +++ b/pkg/common/discovery/direct/doc.go @@ -12,4 +12,4 @@ // See the License for the specific language governing permissions and // limitations under the License. -package direct // import "github.com/openimsdk/open-im-server/v3/pkg/common/discoveryregister/direct" +package direct // import "github.com/openimsdk/open-im-server/v3/pkg/common/discovery/direct" diff --git a/pkg/common/discoveryregister/discoveryregister.go b/pkg/common/discovery/discoveryregister.go similarity index 98% rename from pkg/common/discoveryregister/discoveryregister.go rename to pkg/common/discovery/discoveryregister.go index ae4229e1b..bc9fd0f5a 100644 --- a/pkg/common/discoveryregister/discoveryregister.go +++ b/pkg/common/discovery/discoveryregister.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package discoveryregister +package discovery import ( "time" diff --git a/pkg/common/discoveryregister/discoveryregister_test.go b/pkg/common/discovery/discoveryregister_test.go similarity index 98% rename from pkg/common/discoveryregister/discoveryregister_test.go rename to pkg/common/discovery/discoveryregister_test.go index 417226645..63f7e94cd 100644 --- a/pkg/common/discoveryregister/discoveryregister_test.go +++ b/pkg/common/discovery/discoveryregister_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package discoveryregister +package discovery import ( "os" diff --git a/pkg/common/discoveryregister/doc.go b/pkg/common/discovery/doc.go similarity index 85% rename from pkg/common/discoveryregister/doc.go rename to pkg/common/discovery/doc.go index 46bbe7001..7a5918cfa 100644 --- a/pkg/common/discoveryregister/doc.go +++ b/pkg/common/discovery/doc.go @@ -12,4 +12,4 @@ // See the License for the specific language governing permissions and // limitations under the License. -package discoveryregister // import "github.com/openimsdk/open-im-server/v3/pkg/common/discoveryregister" +package discovery // import "github.com/openimsdk/open-im-server/v3/pkg/common/discovery" diff --git a/pkg/common/discovery/etcd/config_manager.go b/pkg/common/discovery/etcd/config_manager.go new file mode 100644 index 000000000..013e2cce3 --- /dev/null +++ b/pkg/common/discovery/etcd/config_manager.go @@ -0,0 +1,111 @@ +package etcd + +import ( + "context" + "os" + "os/exec" + "runtime" + "sync" + "syscall" + + "github.com/openimsdk/tools/errs" + "github.com/openimsdk/tools/log" + "github.com/openimsdk/tools/utils/datautil" + clientv3 "go.etcd.io/etcd/client/v3" +) + +const ( + ConfigKeyPrefix = "/open-im/config/" + RestartKey = "restart" +) + +var ( + ShutDowns []func() error +) + +func RegisterShutDown(shutDown ...func() error) { + ShutDowns = append(ShutDowns, shutDown...) +} + +type ConfigManager struct { + client *clientv3.Client + watchConfigNames []string + lock sync.Mutex +} + +func BuildKey(s string) string { + return ConfigKeyPrefix + s +} + +func NewConfigManager(client *clientv3.Client, configNames []string) *ConfigManager { + return &ConfigManager{ + client: client, + watchConfigNames: datautil.Batch(func(s string) string { return BuildKey(s) }, append(configNames, RestartKey))} +} + +func (c *ConfigManager) Watch(ctx context.Context) { + chans := make([]clientv3.WatchChan, 0, len(c.watchConfigNames)) + for _, name := range c.watchConfigNames { + chans = append(chans, c.client.Watch(ctx, name, clientv3.WithPrefix())) + } + + doWatch := func(watchChan clientv3.WatchChan) { + for watchResp := range watchChan { + if watchResp.Err() != nil { + log.ZError(ctx, "watch err", errs.Wrap(watchResp.Err())) + continue + } + for _, event := range watchResp.Events { + if event.IsModify() { + if datautil.Contain(string(event.Kv.Key), c.watchConfigNames...) { + c.lock.Lock() + err := restartServer(ctx) + if err != nil { + log.ZError(ctx, "restart server err", err) + } + c.lock.Unlock() + } + } + } + } + } + for _, ch := range chans { + go doWatch(ch) + } +} + +func restartServer(ctx context.Context) error { + exePath, err := os.Executable() + if err != nil { + return errs.New("get executable path fail").Wrap() + } + + args := os.Args + env := os.Environ() + + cmd := exec.Command(exePath, args[1:]...) + cmd.Env = env + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + cmd.Stdin = os.Stdin + + if runtime.GOOS != "windows" { + cmd.SysProcAttr = &syscall.SysProcAttr{} + } + log.ZInfo(ctx, "shutdown server") + for _, f := range ShutDowns { + if err = f(); err != nil { + log.ZError(ctx, "shutdown fail", err) + } + } + + log.ZInfo(ctx, "restart server") + err = cmd.Start() + if err != nil { + return errs.New("restart server fail").Wrap() + } + log.ZInfo(ctx, "cmd start over") + + os.Exit(0) + return nil +} diff --git a/pkg/common/discoveryregister/kubernetes/doc.go b/pkg/common/discovery/etcd/doc.go similarity index 84% rename from pkg/common/discoveryregister/kubernetes/doc.go rename to pkg/common/discovery/etcd/doc.go index 8615caa6b..fedf5ad51 100644 --- a/pkg/common/discoveryregister/kubernetes/doc.go +++ b/pkg/common/discovery/etcd/doc.go @@ -12,4 +12,4 @@ // See the License for the specific language governing permissions and // limitations under the License. -package kubernetes // import "github.com/openimsdk/open-im-server/v3/pkg/common/discoveryregister/kubernetes" +package etcd // import "github.com/openimsdk/open-im-server/v3/pkg/common/discovery/etcd" diff --git a/pkg/common/discoveryregister/etcd/doc.go b/pkg/common/discovery/kubernetes/doc.go similarity index 94% rename from pkg/common/discoveryregister/etcd/doc.go rename to pkg/common/discovery/kubernetes/doc.go index 1da7508a1..dc23ac132 100644 --- a/pkg/common/discoveryregister/etcd/doc.go +++ b/pkg/common/discovery/kubernetes/doc.go @@ -12,4 +12,4 @@ // See the License for the specific language governing permissions and // limitations under the License. -package kubernetes // import "github.com/openimsdk/open-im-server/v3/pkg/common/discoveryregister/etcd" +package kubernetes // import "github.com/openimsdk/open-im-server/v3/pkg/common/discovery/kubernetes" diff --git a/pkg/common/discoveryregister/kubernetes/kubernetes.go b/pkg/common/discovery/kubernetes/kubernetes.go similarity index 100% rename from pkg/common/discoveryregister/kubernetes/kubernetes.go rename to pkg/common/discovery/kubernetes/kubernetes.go diff --git a/pkg/common/startrpc/start.go b/pkg/common/startrpc/start.go index fd46fb45c..5facc8f73 100644 --- a/pkg/common/startrpc/start.go +++ b/pkg/common/startrpc/start.go @@ -27,6 +27,7 @@ import ( "time" conf "github.com/openimsdk/open-im-server/v3/pkg/common/config" + disetcd "github.com/openimsdk/open-im-server/v3/pkg/common/discovery/etcd" "github.com/openimsdk/tools/discovery/etcd" "github.com/openimsdk/tools/utils/datautil" "github.com/openimsdk/tools/utils/jsonutil" @@ -34,7 +35,7 @@ import ( "github.com/openimsdk/tools/utils/runtimeenv" - kdisc "github.com/openimsdk/open-im-server/v3/pkg/common/discoveryregister" + kdisc "github.com/openimsdk/open-im-server/v3/pkg/common/discovery" "github.com/openimsdk/open-im-server/v3/pkg/common/prommetrics" "github.com/openimsdk/tools/discovery" "github.com/openimsdk/tools/errs" @@ -48,9 +49,11 @@ import ( // Start rpc server. func Start[T any](ctx context.Context, discovery *conf.Discovery, prometheusConfig *conf.Prometheus, listenIP, registerIP string, autoSetPorts bool, rpcPorts []int, index int, rpcRegisterName string, notification *conf.Notification, config T, + watchConfigNames []string, rpcFn func(ctx context.Context, config T, client discovery.SvcDiscoveryRegistry, server *grpc.Server) error, options ...grpc.ServerOption) error { + watchConfigNames = append(watchConfigNames, conf.LogConfigFileName) var ( rpcTcpAddr string netDone = make(chan struct{}, 2) @@ -185,12 +188,17 @@ func Start[T any](ctx context.Context, discovery *conf.Discovery, prometheusConf go func() { err := srv.Serve(listener) - if err != nil { + if err != nil && !errors.Is(err, http.ErrServerClosed) { netErr = errs.WrapMsg(err, "rpc start err: ", rpcTcpAddr) netDone <- struct{}{} } }() + if discovery.Enable == conf.ETCD { + cm := disetcd.NewConfigManager(client.(*etcd.SvcDiscoveryRegistryImpl).GetClient(), watchConfigNames) + cm.Watch(ctx) + } + sigs := make(chan os.Signal, 1) signal.Notify(sigs, syscall.SIGTERM) select { diff --git a/tools/check-component/main.go b/tools/check-component/main.go index 15d8640c7..9df0da7de 100644 --- a/tools/check-component/main.go +++ b/tools/check-component/main.go @@ -24,7 +24,6 @@ import ( "path/filepath" "time" - "github.com/openimsdk/open-im-server/v3/pkg/common/cmd" "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/tools/db/mongoutil" "github.com/openimsdk/tools/db/redisutil" @@ -87,35 +86,35 @@ func initConfig(configDir string) (*config.Mongo, *config.Redis, *config.Kafka, ) runtimeEnv := runtimeenv.PrintRuntimeEnvironment() - err := config.Load(configDir, cmd.MongodbConfigFileName, cmd.ConfigEnvPrefixMap[cmd.MongodbConfigFileName], runtimeEnv, mongoConfig) + err := config.Load(configDir, config.MongodbConfigFileName, config.EnvPrefixMap[config.MongodbConfigFileName], runtimeEnv, mongoConfig) if err != nil { return nil, nil, nil, nil, nil, err } - err = config.Load(configDir, cmd.RedisConfigFileName, cmd.ConfigEnvPrefixMap[cmd.RedisConfigFileName], runtimeEnv, redisConfig) + err = config.Load(configDir, config.RedisConfigFileName, config.EnvPrefixMap[config.RedisConfigFileName], runtimeEnv, redisConfig) if err != nil { return nil, nil, nil, nil, nil, err } - err = config.Load(configDir, cmd.KafkaConfigFileName, cmd.ConfigEnvPrefixMap[cmd.KafkaConfigFileName], runtimeEnv, kafkaConfig) + err = config.Load(configDir, config.KafkaConfigFileName, config.EnvPrefixMap[config.KafkaConfigFileName], runtimeEnv, kafkaConfig) if err != nil { return nil, nil, nil, nil, nil, err } - err = config.Load(configDir, cmd.OpenIMRPCThirdCfgFileName, cmd.ConfigEnvPrefixMap[cmd.OpenIMRPCThirdCfgFileName], runtimeEnv, thirdConfig) + err = config.Load(configDir, config.OpenIMRPCThirdCfgFileName, config.EnvPrefixMap[config.OpenIMRPCThirdCfgFileName], runtimeEnv, thirdConfig) if err != nil { return nil, nil, nil, nil, nil, err } if thirdConfig.Object.Enable == "minio" { - err = config.Load(configDir, cmd.MinioConfigFileName, cmd.ConfigEnvPrefixMap[cmd.MinioConfigFileName], runtimeEnv, minioConfig) + err = config.Load(configDir, config.MinioConfigFileName, config.EnvPrefixMap[config.MinioConfigFileName], runtimeEnv, minioConfig) if err != nil { return nil, nil, nil, nil, nil, err } } else { minioConfig = nil } - err = config.Load(configDir, cmd.DiscoveryConfigFilename, cmd.ConfigEnvPrefixMap[cmd.DiscoveryConfigFilename], runtimeEnv, discovery) + err = config.Load(configDir, config.DiscoveryConfigFilename, config.EnvPrefixMap[config.DiscoveryConfigFilename], runtimeEnv, discovery) if err != nil { return nil, nil, nil, nil, nil, err } diff --git a/tools/seq/internal/seq.go b/tools/seq/internal/seq.go index 62466670e..c931cda5d 100644 --- a/tools/seq/internal/seq.go +++ b/tools/seq/internal/seq.go @@ -15,7 +15,6 @@ import ( "syscall" "time" - "github.com/openimsdk/open-im-server/v3/pkg/common/cmd" "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database/mgo" "github.com/openimsdk/tools/db/mongoutil" @@ -59,11 +58,11 @@ func readConfig[T any](dir string, name string) (*T, error) { } func Main(conf string, del time.Duration) error { - redisConfig, err := readConfig[config.Redis](conf, cmd.RedisConfigFileName) + redisConfig, err := readConfig[config.Redis](conf, config.RedisConfigFileName) if err != nil { return err } - mongodbConfig, err := readConfig[config.Mongo](conf, cmd.MongodbConfigFileName) + mongodbConfig, err := readConfig[config.Mongo](conf, config.MongodbConfigFileName) if err != nil { return err } From 8b79a7685c16f18ace4099c7c0c430f6fa6deb53 Mon Sep 17 00:00:00 2001 From: chao <48119764+withchao@users.noreply.github.com> Date: Thu, 26 Dec 2024 17:49:05 +0800 Subject: [PATCH 088/199] feat: support message cache (#3007) * pb * fix: Modifying other fields while setting IsPrivateChat does not take effect * fix: quote message error revoke * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * upgrading pkg tools * redis msg cache * redis msg cache * redis msg cache * redis msg cache * redis msg cache * redis msg cache * redis msg cache --- internal/msgtransfer/init.go | 2 +- .../online_msg_to_mongo_handler.go | 20 +- internal/rpc/msg/delete.go | 6 +- internal/rpc/msg/revoke.go | 5 +- internal/rpc/msg/server.go | 2 +- pkg/common/storage/cache/cachekey/msg.go | 42 +- pkg/common/storage/cache/msg.go | 19 +- pkg/common/storage/cache/redis/msg.go | 180 ++----- pkg/common/storage/cache/redis/msg_test.go | 133 ----- pkg/common/storage/controller/msg.go | 458 +++++------------- pkg/common/storage/controller/msg_transfer.go | 35 +- pkg/common/storage/database/mgo/msg.go | 422 ++-------------- pkg/common/storage/database/msg.go | 12 +- pkg/common/storage/model/msg.go | 4 + 14 files changed, 277 insertions(+), 1063 deletions(-) delete mode 100644 pkg/common/storage/cache/redis/msg_test.go diff --git a/internal/msgtransfer/init.go b/internal/msgtransfer/init.go index c4a9a89c6..fcd6152dc 100644 --- a/internal/msgtransfer/init.go +++ b/internal/msgtransfer/init.go @@ -108,11 +108,11 @@ func Start(ctx context.Context, index int, config *Config) error { cm.Watch(ctx) } - msgModel := redis.NewMsgCache(rdb) msgDocModel, err := mgo.NewMsgMongo(mgocli.GetDB()) if err != nil { return err } + msgModel := redis.NewMsgCache(rdb, msgDocModel) seqConversation, err := mgo.NewSeqConversationMongo(mgocli.GetDB()) if err != nil { return err diff --git a/internal/msgtransfer/online_msg_to_mongo_handler.go b/internal/msgtransfer/online_msg_to_mongo_handler.go index 82002c26b..d8836d54e 100644 --- a/internal/msgtransfer/online_msg_to_mongo_handler.go +++ b/internal/msgtransfer/online_msg_to_mongo_handler.go @@ -77,27 +77,13 @@ func (mc *OnlineHistoryMongoConsumerHandler) handleChatWs2Mongo(ctx context.Cont for _, msg := range msgFromMQ.MsgData { seqs = append(seqs, msg.Seq) } - err = mc.msgTransferDatabase.DeleteMessagesFromCache(ctx, msgFromMQ.ConversationID, seqs) - if err != nil { - log.ZError( - ctx, - "remove cache msg from redis err", - err, - "msg", - msgFromMQ.MsgData, - "conversationID", - msgFromMQ.ConversationID, - ) - } } -func (*OnlineHistoryMongoConsumerHandler) Setup(_ sarama.ConsumerGroupSession) error { return nil } +func (*OnlineHistoryMongoConsumerHandler) Setup(_ sarama.ConsumerGroupSession) error { return nil } + func (*OnlineHistoryMongoConsumerHandler) Cleanup(_ sarama.ConsumerGroupSession) error { return nil } -func (mc *OnlineHistoryMongoConsumerHandler) ConsumeClaim( - sess sarama.ConsumerGroupSession, - claim sarama.ConsumerGroupClaim, -) error { // an instance in the consumer group +func (mc *OnlineHistoryMongoConsumerHandler) ConsumeClaim(sess sarama.ConsumerGroupSession, claim sarama.ConsumerGroupClaim) error { // an instance in the consumer group log.ZDebug(context.Background(), "online new session msg come", "highWaterMarkOffset", claim.HighWaterMarkOffset(), "topic", claim.Topic(), "partition", claim.Partition()) for msg := range claim.Messages() { diff --git a/internal/rpc/msg/delete.go b/internal/rpc/msg/delete.go index f45713ff1..d3485faaa 100644 --- a/internal/rpc/msg/delete.go +++ b/internal/rpc/msg/delete.go @@ -106,10 +106,8 @@ func (m *msgServer) DeleteMsgPhysical(ctx context.Context, req *msg.DeleteMsgPhy return nil, err } remainTime := timeutil.GetCurrentTimestampBySecond() - req.Timestamp - for _, conversationID := range req.ConversationIDs { - if err := m.MsgDatabase.DeleteConversationMsgsAndSetMinSeq(ctx, conversationID, remainTime); err != nil { - log.ZWarn(ctx, "DeleteConversationMsgsAndSetMinSeq error", err, "conversationID", conversationID, "err", err) - } + if _, err := m.DestructMsgs(ctx, &msg.DestructMsgsReq{Timestamp: remainTime, Limit: 9999}); err != nil { + return nil, err } return &msg.DeleteMsgPhysicalResp{}, nil } diff --git a/internal/rpc/msg/revoke.go b/internal/rpc/msg/revoke.go index b7cc7df62..97de0f48a 100644 --- a/internal/rpc/msg/revoke.go +++ b/internal/rpc/msg/revoke.go @@ -63,7 +63,8 @@ func (m *msgServer) RevokeMsg(ctx context.Context, req *msg.RevokeMsgReq) (*msg. log.ZDebug(ctx, "GetMsgBySeqs", "conversationID", req.ConversationID, "seq", req.Seq, "msg", string(data)) var role int32 if !authverify.IsAppManagerUid(ctx, m.config.Share.IMAdminUserID) { - switch msgs[0].SessionType { + sessionType := msgs[0].SessionType + switch sessionType { case constant.SingleChatType: if err := authverify.CheckAccessV3(ctx, msgs[0].SendID, m.config.Share.IMAdminUserID); err != nil { return nil, err @@ -89,7 +90,7 @@ func (m *msgServer) RevokeMsg(ctx context.Context, req *msg.RevokeMsgReq) (*msg. role = member.RoleLevel } default: - return nil, errs.ErrInternalServer.WrapMsg("msg sessionType not supported") + return nil, errs.ErrInternalServer.WrapMsg("msg sessionType not supported", "sessionType", sessionType) } } now := time.Now().UnixMilli() diff --git a/internal/rpc/msg/server.go b/internal/rpc/msg/server.go index 991fb8704..7ccda6bd5 100644 --- a/internal/rpc/msg/server.go +++ b/internal/rpc/msg/server.go @@ -89,7 +89,7 @@ func Start(ctx context.Context, config *Config, client discovery.SvcDiscoveryReg if err != nil { return err } - msgModel := redis.NewMsgCache(rdb) + msgModel := redis.NewMsgCache(rdb, msgDocModel) seqConversation, err := mgo.NewSeqConversationMongo(mgocli.GetDB()) if err != nil { return err diff --git a/pkg/common/storage/cache/cachekey/msg.go b/pkg/common/storage/cache/cachekey/msg.go index 8e05b64f1..ac449df38 100644 --- a/pkg/common/storage/cache/cachekey/msg.go +++ b/pkg/common/storage/cache/cachekey/msg.go @@ -15,50 +15,16 @@ package cachekey import ( - "github.com/openimsdk/protocol/constant" "strconv" ) const ( - messageCache = "MESSAGE_CACHE:" - messageDelUserList = "MESSAGE_DEL_USER_LIST:" - userDelMessagesList = "USER_DEL_MESSAGES_LIST:" - sendMsgFailedFlag = "SEND_MSG_FAILED_FLAG:" - exTypeKeyLocker = "EX_LOCK:" - reactionExSingle = "EX_SINGLE_" - reactionWriteGroup = "EX_GROUP_" - reactionReadGroup = "EX_SUPER_GROUP_" - reactionNotification = "EX_NOTIFICATION_" + sendMsgFailedFlag = "SEND_MSG_FAILED_FLAG:" + messageCache = "MSG_CACHE:" ) -func GetMessageCacheKey(conversationID string, seq int64) string { - return messageCache + conversationID + "_" + strconv.Itoa(int(seq)) -} - -func GetMessageDelUserListKey(conversationID string, seq int64) string { - return messageDelUserList + conversationID + ":" + strconv.Itoa(int(seq)) -} - -func GetUserDelListKey(conversationID, userID string) string { - return userDelMessagesList + conversationID + ":" + userID -} - -func GetMessageReactionExKey(clientMsgID string, sessionType int32) string { - switch sessionType { - case constant.SingleChatType: - return reactionExSingle + clientMsgID - case constant.WriteGroupChatType: - return reactionWriteGroup + clientMsgID - case constant.ReadGroupChatType: - return reactionReadGroup + clientMsgID - case constant.NotificationChatType: - return reactionNotification + clientMsgID - } - - return "" -} -func GetLockMessageTypeKey(clientMsgID string, TypeKey string) string { - return exTypeKeyLocker + clientMsgID + "_" + TypeKey +func GetMsgCacheKey(conversationID string, seq int64) string { + return messageCache + conversationID + ":" + strconv.Itoa(int(seq)) } func GetSendMsgKey(id string) string { diff --git a/pkg/common/storage/cache/msg.go b/pkg/common/storage/cache/msg.go index 00eb28c02..271ed19fe 100644 --- a/pkg/common/storage/cache/msg.go +++ b/pkg/common/storage/cache/msg.go @@ -16,23 +16,14 @@ package cache import ( "context" - "time" - - "github.com/openimsdk/protocol/sdkws" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" ) type MsgCache interface { - GetMessagesBySeq(ctx context.Context, conversationID string, seqs []int64) (seqMsg []*sdkws.MsgData, failedSeqList []int64, err error) - SetMessagesToCache(ctx context.Context, conversationID string, msgs []*sdkws.MsgData) (int, error) - DeleteMessagesFromCache(ctx context.Context, conversationID string, seqs []int64) error SetSendMsgStatus(ctx context.Context, id string, status int32) error GetSendMsgStatus(ctx context.Context, id string) (int32, error) - JudgeMessageReactionExist(ctx context.Context, clientMsgID string, sessionType int32) (bool, error) - GetOneMessageAllReactionList(ctx context.Context, clientMsgID string, sessionType int32) (map[string]string, error) - DeleteOneMessageKey(ctx context.Context, clientMsgID string, sessionType int32, subKey string) error - SetMessageReactionExpire(ctx context.Context, clientMsgID string, sessionType int32, expiration time.Duration) (bool, error) - GetMessageTypeKeyValue(ctx context.Context, clientMsgID string, sessionType int32, typeKey string) (string, error) - SetMessageTypeKeyValue(ctx context.Context, clientMsgID string, sessionType int32, typeKey, value string) error - LockMessageTypeKey(ctx context.Context, clientMsgID string, TypeKey string) error - UnLockMessageTypeKey(ctx context.Context, clientMsgID string, TypeKey string) error + + GetMessageBySeqs(ctx context.Context, conversationID string, seqs []int64) ([]*model.MsgInfoModel, error) + DelMessageBySeqs(ctx context.Context, conversationID string, seqs []int64) error + SetMessageBySeqs(ctx context.Context, conversationID string, msgs []*model.MsgInfoModel) error } diff --git a/pkg/common/storage/cache/redis/msg.go b/pkg/common/storage/cache/redis/msg.go index b04bc5c35..0651f0283 100644 --- a/pkg/common/storage/cache/redis/msg.go +++ b/pkg/common/storage/cache/redis/msg.go @@ -2,10 +2,12 @@ package redis import ( "context" + "encoding/json" + "github.com/dtm-labs/rockscache" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/cachekey" - "github.com/openimsdk/open-im-server/v3/pkg/msgprocessor" - "github.com/openimsdk/protocol/sdkws" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" "github.com/openimsdk/tools/errs" "github.com/openimsdk/tools/utils/datautil" "github.com/redis/go-redis/v9" @@ -13,76 +15,26 @@ import ( ) // // msgCacheTimeout is expiration time of message cache, 86400 seconds -const msgCacheTimeout = 86400 +const msgCacheTimeout = time.Hour * 24 -func NewMsgCache(client redis.UniversalClient) cache.MsgCache { - return &msgCache{rdb: client} +func NewMsgCache(client redis.UniversalClient, db database.Msg) cache.MsgCache { + return &msgCache{ + rdb: client, + rcClient: rockscache.NewClient(client, *GetRocksCacheOptions()), + msgDocDatabase: db, + } } type msgCache struct { - rdb redis.UniversalClient -} - -func (c *msgCache) getMessageCacheKey(conversationID string, seq int64) string { - return cachekey.GetMessageCacheKey(conversationID, seq) -} -func (c *msgCache) getMessageDelUserListKey(conversationID string, seq int64) string { - return cachekey.GetMessageDelUserListKey(conversationID, seq) -} - -func (c *msgCache) getUserDelList(conversationID, userID string) string { - return cachekey.GetUserDelListKey(conversationID, userID) + rdb redis.UniversalClient + rcClient *rockscache.Client + msgDocDatabase database.Msg } func (c *msgCache) getSendMsgKey(id string) string { return cachekey.GetSendMsgKey(id) } -func (c *msgCache) getLockMessageTypeKey(clientMsgID string, TypeKey string) string { - return cachekey.GetLockMessageTypeKey(clientMsgID, TypeKey) -} - -func (c *msgCache) getMessageReactionExPrefix(clientMsgID string, sessionType int32) string { - return cachekey.GetMessageReactionExKey(clientMsgID, sessionType) -} - -func (c *msgCache) SetMessagesToCache(ctx context.Context, conversationID string, msgs []*sdkws.MsgData) (int, error) { - msgMap := datautil.SliceToMap(msgs, func(msg *sdkws.MsgData) string { - return c.getMessageCacheKey(conversationID, msg.Seq) - }) - keys := datautil.Slice(msgs, func(msg *sdkws.MsgData) string { - return c.getMessageCacheKey(conversationID, msg.Seq) - }) - err := ProcessKeysBySlot(ctx, c.rdb, keys, func(ctx context.Context, slot int64, keys []string) error { - var values []string - for _, key := range keys { - if msg, ok := msgMap[key]; ok { - s, err := msgprocessor.Pb2String(msg) - if err != nil { - return err - } - values = append(values, s) - } - } - return LuaSetBatchWithCommonExpire(ctx, c.rdb, keys, values, msgCacheTimeout) - }) - if err != nil { - return 0, err - } - return len(msgs), nil -} - -func (c *msgCache) DeleteMessagesFromCache(ctx context.Context, conversationID string, seqs []int64) error { - var keys []string - for _, seq := range seqs { - keys = append(keys, c.getMessageCacheKey(conversationID, seq)) - } - - return ProcessKeysBySlot(ctx, c.rdb, keys, func(ctx context.Context, slot int64, keys []string) error { - return LuaDeleteBatch(ctx, c.rdb, keys) - }) -} - func (c *msgCache) SetSendMsgStatus(ctx context.Context, id string, status int32) error { return errs.Wrap(c.rdb.Set(ctx, c.getSendMsgKey(id), status, time.Hour*24).Err()) } @@ -92,81 +44,53 @@ func (c *msgCache) GetSendMsgStatus(ctx context.Context, id string) (int32, erro return int32(result), errs.Wrap(err) } -func (c *msgCache) LockMessageTypeKey(ctx context.Context, clientMsgID string, TypeKey string) error { - key := c.getLockMessageTypeKey(clientMsgID, TypeKey) - return errs.Wrap(c.rdb.SetNX(ctx, key, 1, time.Minute).Err()) -} - -func (c *msgCache) UnLockMessageTypeKey(ctx context.Context, clientMsgID string, TypeKey string) error { - key := c.getLockMessageTypeKey(clientMsgID, TypeKey) - return errs.Wrap(c.rdb.Del(ctx, key).Err()) +func (c *msgCache) GetMessageBySeqs(ctx context.Context, conversationID string, seqs []int64) ([]*model.MsgInfoModel, error) { + if len(seqs) == 0 { + return nil, nil + } + getKey := func(seq int64) string { + return cachekey.GetMsgCacheKey(conversationID, seq) + } + getMsgID := func(msg *model.MsgInfoModel) int64 { + return msg.Msg.Seq + } + find := func(ctx context.Context, seqs []int64) ([]*model.MsgInfoModel, error) { + return c.msgDocDatabase.FindSeqs(ctx, conversationID, seqs) + } + return batchGetCache2(ctx, c.rcClient, msgCacheTimeout, seqs, getKey, getMsgID, find) } -func (c *msgCache) JudgeMessageReactionExist(ctx context.Context, clientMsgID string, sessionType int32) (bool, error) { - n, err := c.rdb.Exists(ctx, c.getMessageReactionExPrefix(clientMsgID, sessionType)).Result() +func (c *msgCache) DelMessageBySeqs(ctx context.Context, conversationID string, seqs []int64) error { + if len(seqs) == 0 { + return nil + } + keys := datautil.Slice(seqs, func(seq int64) string { + return cachekey.GetMsgCacheKey(conversationID, seq) + }) + slotKeys, err := groupKeysBySlot(ctx, getRocksCacheRedisClient(c.rcClient), keys) if err != nil { - return false, errs.Wrap(err) + return err } - - return n > 0, nil -} - -func (c *msgCache) SetMessageTypeKeyValue(ctx context.Context, clientMsgID string, sessionType int32, typeKey, value string) error { - return errs.Wrap(c.rdb.HSet(ctx, c.getMessageReactionExPrefix(clientMsgID, sessionType), typeKey, value).Err()) -} - -func (c *msgCache) SetMessageReactionExpire(ctx context.Context, clientMsgID string, sessionType int32, expiration time.Duration) (bool, error) { - val, err := c.rdb.Expire(ctx, c.getMessageReactionExPrefix(clientMsgID, sessionType), expiration).Result() - return val, errs.Wrap(err) -} - -func (c *msgCache) GetMessageTypeKeyValue(ctx context.Context, clientMsgID string, sessionType int32, typeKey string) (string, error) { - val, err := c.rdb.HGet(ctx, c.getMessageReactionExPrefix(clientMsgID, sessionType), typeKey).Result() - return val, errs.Wrap(err) -} - -func (c *msgCache) GetOneMessageAllReactionList(ctx context.Context, clientMsgID string, sessionType int32) (map[string]string, error) { - val, err := c.rdb.HGetAll(ctx, c.getMessageReactionExPrefix(clientMsgID, sessionType)).Result() - return val, errs.Wrap(err) -} - -func (c *msgCache) DeleteOneMessageKey(ctx context.Context, clientMsgID string, sessionType int32, subKey string) error { - return errs.Wrap(c.rdb.HDel(ctx, c.getMessageReactionExPrefix(clientMsgID, sessionType), subKey).Err()) + for _, keys := range slotKeys { + if err := c.rcClient.TagAsDeletedBatch2(ctx, keys); err != nil { + return err + } + } + return nil } -func (c *msgCache) GetMessagesBySeq(ctx context.Context, conversationID string, seqs []int64) (seqMsgs []*sdkws.MsgData, failedSeqs []int64, err error) { - var keys []string - keySeqMap := make(map[string]int64, 10) - for _, seq := range seqs { - key := c.getMessageCacheKey(conversationID, seq) - keys = append(keys, key) - keySeqMap[key] = seq - } - err = ProcessKeysBySlot(ctx, c.rdb, keys, func(ctx context.Context, slot int64, keys []string) error { - result, err := LuaGetBatch(ctx, c.rdb, keys) +func (c *msgCache) SetMessageBySeqs(ctx context.Context, conversationID string, msgs []*model.MsgInfoModel) error { + for _, msg := range msgs { + if msg == nil || msg.Msg == nil || msg.Msg.Seq <= 0 { + continue + } + data, err := json.Marshal(msg) if err != nil { return err } - for i, value := range result { - seq := keySeqMap[keys[i]] - if value == nil { - failedSeqs = append(failedSeqs, seq) - continue - } - - msg := &sdkws.MsgData{} - msgString, ok := value.(string) - if !ok || msgprocessor.String2Pb(msgString, msg) != nil { - failedSeqs = append(failedSeqs, seq) - continue - } - seqMsgs = append(seqMsgs, msg) - + if err := c.rcClient.RawSet(ctx, cachekey.GetMsgCacheKey(conversationID, msg.Msg.Seq), string(data), msgCacheTimeout); err != nil { + return err } - return nil - }) - if err != nil { - return nil, nil, err } - return seqMsgs, failedSeqs, nil + return nil } diff --git a/pkg/common/storage/cache/redis/msg_test.go b/pkg/common/storage/cache/redis/msg_test.go deleted file mode 100644 index 10b9ce18b..000000000 --- a/pkg/common/storage/cache/redis/msg_test.go +++ /dev/null @@ -1,133 +0,0 @@ -// Copyright © 2023 OpenIM. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package redis - -import ( - "context" - "fmt" - "github.com/openimsdk/protocol/sdkws" - "github.com/redis/go-redis/v9" - "github.com/stretchr/testify/assert" - "google.golang.org/protobuf/proto" - "testing" -) - -func Test_msgCache_SetMessagesToCache(t *testing.T) { - type fields struct { - rdb redis.UniversalClient - } - type args struct { - ctx context.Context - conversationID string - msgs []*sdkws.MsgData - } - tests := []struct { - name string - fields fields - args args - want int - wantErr assert.ErrorAssertionFunc - }{ - {"test1", fields{rdb: redis.NewClient(&redis.Options{Addr: "localhost:16379", Username: "", Password: "openIM123", DB: 0})}, args{context.Background(), - "cid", []*sdkws.MsgData{{Seq: 1}, {Seq: 2}, {Seq: 3}}}, 3, assert.NoError}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - c := &msgCache{ - rdb: tt.fields.rdb, - } - got, err := c.SetMessagesToCache(tt.args.ctx, tt.args.conversationID, tt.args.msgs) - if !tt.wantErr(t, err, fmt.Sprintf("SetMessagesToCache(%v, %v, %v)", tt.args.ctx, tt.args.conversationID, tt.args.msgs)) { - return - } - assert.Equalf(t, tt.want, got, "SetMessagesToCache(%v, %v, %v)", tt.args.ctx, tt.args.conversationID, tt.args.msgs) - }) - } -} - -func Test_msgCache_GetMessagesBySeq(t *testing.T) { - type fields struct { - rdb redis.UniversalClient - } - type args struct { - ctx context.Context - conversationID string - seqs []int64 - } - var failedSeq []int64 - tests := []struct { - name string - fields fields - args args - wantSeqMsgs []*sdkws.MsgData - wantFailedSeqs []int64 - wantErr assert.ErrorAssertionFunc - }{ - {"test1", fields{rdb: redis.NewClient(&redis.Options{Addr: "localhost:16379", Password: "openIM123", DB: 0})}, - args{context.Background(), "cid", []int64{1, 2, 3}}, - []*sdkws.MsgData{{Seq: 1}, {Seq: 2}, {Seq: 3}}, failedSeq, assert.NoError}, - {"test2", fields{rdb: redis.NewClient(&redis.Options{Addr: "localhost:16379", Password: "openIM123", DB: 0})}, - args{context.Background(), "cid", []int64{4, 5, 6}}, - nil, []int64{4, 5, 6}, assert.NoError}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - c := &msgCache{ - rdb: tt.fields.rdb, - } - gotSeqMsgs, gotFailedSeqs, err := c.GetMessagesBySeq(tt.args.ctx, tt.args.conversationID, tt.args.seqs) - if !tt.wantErr(t, err, fmt.Sprintf("GetMessagesBySeq(%v, %v, %v)", tt.args.ctx, tt.args.conversationID, tt.args.seqs)) { - return - } - equalMsgDataSlices(t, tt.wantSeqMsgs, gotSeqMsgs) - assert.Equalf(t, tt.wantFailedSeqs, gotFailedSeqs, "GetMessagesBySeq(%v, %v, %v)", tt.args.ctx, tt.args.conversationID, tt.args.seqs) - }) - } -} - -func equalMsgDataSlices(t *testing.T, expected, actual []*sdkws.MsgData) { - assert.Equal(t, len(expected), len(actual), "Slices have different lengths") - for i := range expected { - assert.True(t, proto.Equal(expected[i], actual[i]), "Element %d not equal: expected %v, got %v", i, expected[i], actual[i]) - } -} - -func Test_msgCache_DeleteMessagesFromCache(t *testing.T) { - type fields struct { - rdb redis.UniversalClient - } - type args struct { - ctx context.Context - conversationID string - seqs []int64 - } - tests := []struct { - name string - fields fields - args args - wantErr assert.ErrorAssertionFunc - }{ - {"test1", fields{rdb: redis.NewClient(&redis.Options{Addr: "localhost:16379", Password: "openIM123"})}, - args{context.Background(), "cid", []int64{1, 2, 3}}, assert.NoError}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - c := &msgCache{ - rdb: tt.fields.rdb, - } - tt.wantErr(t, c.DeleteMessagesFromCache(tt.args.ctx, tt.args.conversationID, tt.args.seqs), - fmt.Sprintf("DeleteMessagesFromCache(%v, %v, %v)", tt.args.ctx, tt.args.conversationID, tt.args.seqs)) - }) - } -} diff --git a/pkg/common/storage/controller/msg.go b/pkg/common/storage/controller/msg.go index c29544c33..d5ad12584 100644 --- a/pkg/common/storage/controller/msg.go +++ b/pkg/common/storage/controller/msg.go @@ -18,6 +18,9 @@ import ( "context" "encoding/json" "errors" + "github.com/openimsdk/tools/utils/jsonutil" + "strconv" + "strings" "time" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" @@ -36,7 +39,6 @@ import ( "github.com/openimsdk/tools/log" "github.com/openimsdk/tools/mq/kafka" "github.com/openimsdk/tools/utils/datautil" - "github.com/openimsdk/tools/utils/timeutil" ) const ( @@ -54,12 +56,8 @@ type CommonMsgDatabase interface { GetMsgBySeqsRange(ctx context.Context, userID string, conversationID string, begin, end, num, userMaxSeq int64) (minSeq int64, maxSeq int64, seqMsg []*sdkws.MsgData, err error) // GetMsgBySeqs retrieves messages for large groups from MongoDB by sequence numbers. GetMsgBySeqs(ctx context.Context, userID string, conversationID string, seqs []int64) (minSeq int64, maxSeq int64, seqMsg []*sdkws.MsgData, err error) - // DeleteConversationMsgsAndSetMinSeq deletes conversation messages and resets the minimum sequence number. If `remainTime` is 0, all messages are deleted (this method does not delete Redis - // cache). + GetMessagesBySeqWithBounds(ctx context.Context, userID string, conversationID string, seqs []int64, pullOrder sdkws.PullOrder) (bool, int64, []*sdkws.MsgData, error) - DeleteConversationMsgsAndSetMinSeq(ctx context.Context, conversationID string, remainTime int64) error - // ClearUserMsgs marks messages for deletion based on clear time and returns a list of sequence numbers for marked messages. - ClearUserMsgs(ctx context.Context, userID string, conversationID string, clearTime int64, lastMsgClearTime time.Time) (seqs []int64, err error) // DeleteUserMsgsBySeqs allows a user to delete messages based on sequence numbers. DeleteUserMsgsBySeqs(ctx context.Context, userID string, conversationID string, seqs []int64) error // DeleteMsgsPhysicalBySeqs physically deletes messages by emptying them based on sequence numbers. @@ -80,8 +78,6 @@ type CommonMsgDatabase interface { GetMaxSeqWithTime(ctx context.Context, conversationID string) (database.SeqTime, error) GetCacheMaxSeqWithTime(ctx context.Context, conversationIDs []string) (map[string]database.SeqTime, error) - //GetMongoMaxAndMinSeq(ctx context.Context, conversationID string) (minSeqMongo, maxSeqMongo int64, err error) - //GetConversationMinMaxSeqInMongoAndCache(ctx context.Context, conversationID string) (minSeqMongo, maxSeqMongo, minSeqCache, maxSeqCache int64, err error) SetSendMsgStatus(ctx context.Context, id string, status int32) error GetSendMsgStatus(ctx context.Context, id string) (int32, error) SearchMessage(ctx context.Context, req *pbmsg.SearchMessageReq) (total int64, msgData []*pbmsg.SearchedMsgData, err error) @@ -92,10 +88,6 @@ type CommonMsgDatabase interface { RangeUserSendCount(ctx context.Context, start time.Time, end time.Time, group bool, ase bool, pageNumber int32, showNumber int32) (msgCount int64, userCount int64, users []*model.UserCount, dateCount map[string]int64, err error) RangeGroupSendCount(ctx context.Context, start time.Time, end time.Time, ase bool, pageNumber int32, showNumber int32) (msgCount int64, userCount int64, groups []*model.GroupCount, dateCount map[string]int64, err error) - ConvertMsgsDocLen(ctx context.Context, conversationIDs []string) - - // get Msg when destruct msg before - //DeleteDocMsgBefore(ctx context.Context, ts int64, doc *model.MsgDocModel) ([]int, error) GetRandBeforeMsg(ctx context.Context, ts int64, limit int) ([]*model.MsgDocModel, error) @@ -118,7 +110,7 @@ func NewCommonMsgDatabase(msgDocModel database.Msg, msg cache.MsgCache, seqUser } return &commonMsgDatabase{ msgDocDatabase: msgDocModel, - msg: msg, + msgCache: msg, seqUser: seqUser, seqConversation: seqConversation, producer: producerToRedis, @@ -128,7 +120,7 @@ func NewCommonMsgDatabase(msgDocModel database.Msg, msg cache.MsgCache, seqUser type commonMsgDatabase struct { msgDocDatabase database.Msg msgTable model.MsgDocModel - msg cache.MsgCache + msgCache cache.MsgCache seqConversation cache.SeqConversationCache seqUser cache.SeqUser producer *kafka.Producer @@ -139,7 +131,7 @@ func (db *commonMsgDatabase) MsgToMQ(ctx context.Context, key string, msg2mq *sd return err } -func (db *commonMsgDatabase) BatchInsertBlock(ctx context.Context, conversationID string, fields []any, key int8, firstSeq int64) error { +func (db *commonMsgDatabase) batchInsertBlock(ctx context.Context, conversationID string, fields []any, key int8, firstSeq int64) error { if len(fields) == 0 { return nil } @@ -237,11 +229,15 @@ func (db *commonMsgDatabase) BatchInsertBlock(ctx context.Context, conversationI tryUpdate = false // The current block is inserted successfully, and the next block is inserted preferentially i += insert - 1 // Skip the inserted data } + return nil } func (db *commonMsgDatabase) RevokeMsg(ctx context.Context, conversationID string, seq int64, revoke *model.RevokeModel) error { - return db.BatchInsertBlock(ctx, conversationID, []any{revoke}, updateKeyRevoke, seq) + if err := db.batchInsertBlock(ctx, conversationID, []any{revoke}, updateKeyRevoke, seq); err != nil { + return err + } + return db.msgCache.DelMessageBySeqs(ctx, conversationID, []int64{seq}) } func (db *commonMsgDatabase) MarkSingleChatMsgsAsRead(ctx context.Context, userID string, conversationID string, totalSeqs []int64) error { @@ -256,23 +252,17 @@ func (db *commonMsgDatabase) MarkSingleChatMsgsAsRead(ctx context.Context, userI return err } } - return nil + return db.msgCache.DelMessageBySeqs(ctx, conversationID, totalSeqs) } func (db *commonMsgDatabase) getMsgBySeqs(ctx context.Context, userID, conversationID string, seqs []int64) (totalMsgs []*sdkws.MsgData, err error) { - for docID, seqs := range db.msgTable.GetDocIDSeqsMap(conversationID, seqs) { - // log.ZDebug(ctx, "getMsgBySeqs", "docID", docID, "seqs", seqs) - msgs, err := db.findMsgInfoBySeq(ctx, userID, docID, conversationID, seqs) - if err != nil { - return nil, err - } - for _, msg := range msgs { - totalMsgs = append(totalMsgs, convert.MsgDB2Pb(msg.Msg)) - } - } - return totalMsgs, nil + return db.GetMessageBySeqs(ctx, conversationID, userID, seqs) } + func (db *commonMsgDatabase) handlerDBMsg(ctx context.Context, cache map[int64][]*model.MsgInfoModel, userID, conversationID string, msg *model.MsgInfoModel) { + if msg == nil || msg.Msg == nil { + return + } if msg.IsRead { msg.Msg.IsRead = true } @@ -360,9 +350,6 @@ func (db *commonMsgDatabase) handlerDBMsg(ctx context.Context, cache map[int64][ return } msg.Msg.Content = string(data) - //if _, err := db.msgDocDatabase.UpdateMsg(ctx, db.msgTable.GetDocID(conversationID, msg.Msg.Seq), db.msgTable.GetMsgIndex(msg.Msg.Seq), "msg", msg.Msg); err != nil { - // log.ZError(ctx, "UpdateMsgContent", err) - //} } func (db *commonMsgDatabase) findMsgInfoBySeq(ctx context.Context, userID, docID string, conversationID string, seqs []int64) (totalMsgs []*model.MsgInfoModel, err error) { @@ -377,24 +364,6 @@ func (db *commonMsgDatabase) findMsgInfoBySeq(ctx context.Context, userID, docID return msgs, err } -func (db *commonMsgDatabase) getMsgBySeqsRange(ctx context.Context, userID string, conversationID string, allSeqs []int64, begin, end int64) (seqMsgs []*sdkws.MsgData, err error) { - log.ZDebug(ctx, "getMsgBySeqsRange", "conversationID", conversationID, "allSeqs", allSeqs, "begin", begin, "end", end) - for docID, seqs := range db.msgTable.GetDocIDSeqsMap(conversationID, allSeqs) { - log.ZDebug(ctx, "getMsgBySeqsRange", "docID", docID, "seqs", seqs) - msgs, err := db.findMsgInfoBySeq(ctx, userID, docID, conversationID, seqs) - if err != nil { - return nil, err - } - for _, msg := range msgs { - if msg.IsRead { - msg.Msg.IsRead = true - } - seqMsgs = append(seqMsgs, convert.MsgDB2Pb(msg.Msg)) - } - } - return seqMsgs, nil -} - // GetMsgBySeqsRange In the context of group chat, we have the following parameters: // // "maxSeq" of a conversation: It represents the maximum value of messages in the group conversation. @@ -463,37 +432,10 @@ func (db *commonMsgDatabase) GetMsgBySeqsRange(ctx context.Context, userID strin seqs = append(seqs, i) } } - - if len(seqs) == 0 { - return 0, 0, nil, nil - } - newBegin := seqs[0] - newEnd := seqs[len(seqs)-1] - var successMsgs []*sdkws.MsgData - log.ZDebug(ctx, "GetMsgBySeqsRange", "first seqs", seqs, "newBegin", newBegin, "newEnd", newEnd) - cachedMsgs, failedSeqs, err := db.msg.GetMessagesBySeq(ctx, conversationID, seqs) - if err != nil && !errors.Is(err, redis.Nil) { - log.ZError(ctx, "get message from redis exception", err, "conversationID", conversationID, "seqs", seqs) - } - successMsgs = append(successMsgs, cachedMsgs...) - log.ZDebug(ctx, "get msgs from cache", "cachedMsgs", cachedMsgs) - // get from cache or db - - if len(failedSeqs) > 0 { - log.ZDebug(ctx, "msgs not exist in redis", "seqs", failedSeqs) - mongoMsgs, err := db.getMsgBySeqsRange(ctx, userID, conversationID, failedSeqs, begin, end) - if err != nil { - - return 0, 0, nil, err - } - successMsgs = append(mongoMsgs, successMsgs...) - - //_, err = db.msg.SetMessagesToCache(ctx, conversationID, mongoMsgs) - //if err != nil { - // return 0, 0, nil, err - //} + successMsgs, err := db.GetMessageBySeqs(ctx, conversationID, userID, seqs) + if err != nil { + return 0, 0, nil, err } - return minSeq, maxSeq, successMsgs, nil } @@ -529,31 +471,9 @@ func (db *commonMsgDatabase) GetMsgBySeqs(ctx context.Context, userID string, co newSeqs = append(newSeqs, seq) } } - if len(newSeqs) == 0 { - return minSeq, maxSeq, nil, nil - } - successMsgs, failedSeqs, err := db.msg.GetMessagesBySeq(ctx, conversationID, newSeqs) + successMsgs, err := db.GetMessageBySeqs(ctx, conversationID, userID, newSeqs) if err != nil { - if !errors.Is(err, redis.Nil) { - log.ZWarn(ctx, "get message from redis exception", err, "failedSeqs", failedSeqs, "conversationID", conversationID) - } - } - log.ZDebug(ctx, "db.seq.GetMessagesBySeq", "userID", userID, "conversationID", conversationID, "seqs", - seqs, "len(successMsgs)", len(successMsgs), "failedSeqs", failedSeqs) - - if len(failedSeqs) > 0 { - mongoMsgs, err := db.getMsgBySeqs(ctx, userID, conversationID, failedSeqs) - if err != nil { - - return 0, 0, nil, err - } - - successMsgs = append(successMsgs, mongoMsgs...) - - //_, err = db.msg.SetMessagesToCache(ctx, conversationID, mongoMsgs) - //if err != nil { - // return 0, 0, nil, err - //} + return 0, 0, nil, err } return minSeq, maxSeq, successMsgs, nil } @@ -607,174 +527,14 @@ func (db *commonMsgDatabase) GetMessagesBySeqWithBounds(ctx context.Context, use if len(newSeqs) == 0 { return isEnd, endSeq, nil, nil } - successMsgs, failedSeqs, err := db.msg.GetMessagesBySeq(ctx, conversationID, newSeqs) + successMsgs, err := db.GetMessageBySeqs(ctx, conversationID, userID, newSeqs) if err != nil { - if !errors.Is(err, redis.Nil) { - log.ZWarn(ctx, "get message from redis exception", err, "failedSeqs", failedSeqs, "conversationID", conversationID) - } - } - log.ZDebug(ctx, "db.seq.GetMessagesBySeq", "userID", userID, "conversationID", conversationID, "seqs", - seqs, "len(successMsgs)", len(successMsgs), "failedSeqs", failedSeqs) - - if len(failedSeqs) > 0 { - mongoMsgs, err := db.getMsgBySeqs(ctx, userID, conversationID, failedSeqs) - if err != nil { - - return false, 0, nil, err - } - - successMsgs = append(successMsgs, mongoMsgs...) - - //_, err = db.msg.SetMessagesToCache(ctx, conversationID, mongoMsgs) - //if err != nil { - // return 0, 0, nil, err - //} + return false, 0, nil, err } return isEnd, endSeq, successMsgs, nil } -func (db *commonMsgDatabase) DeleteConversationMsgsAndSetMinSeq(ctx context.Context, conversationID string, remainTime int64) error { - var delStruct delMsgRecursionStruct - var skip int64 - minSeq, err := db.deleteMsgRecursion(ctx, conversationID, skip, &delStruct, remainTime) - if err != nil { - return err - } - log.ZDebug(ctx, "DeleteConversationMsgsAndSetMinSeq", "conversationID", conversationID, "minSeq", minSeq) - if minSeq == 0 { - return nil - } - return db.seqConversation.SetMinSeq(ctx, conversationID, minSeq) -} - -func (db *commonMsgDatabase) ClearUserMsgs(ctx context.Context, userID string, conversationID string, clearTime int64, lastMsgClearTime time.Time) (seqs []int64, err error) { - var index int64 - for { - // from oldest 2 newest, ASC - msgDocModel, err := db.msgDocDatabase.GetMsgDocModelByIndex(ctx, conversationID, index, 1) - if err != nil || msgDocModel.DocID == "" { - if err != nil { - if err == model.ErrMsgListNotExist { - log.ZDebug(ctx, "not doc find", "conversationID", conversationID, "userID", userID, "index", index) - } else { - log.ZError(ctx, "deleteMsgRecursion GetUserMsgListByIndex failed", err, "conversationID", conversationID, "index", index) - } - } - // If an error is reported, or the error cannot be obtained, it is physically deleted and seq delMongoMsgsPhysical(delStruct.delDocIDList) is returned to end the recursion - break - } - - index++ - - // && msgDocModel.Msg[0].Msg.SendTime > lastMsgClearTime.UnixMilli() - if len(msgDocModel.Msg) > 0 { - i := 0 - var over bool - for _, msg := range msgDocModel.Msg { - i++ - // over clear time, need to clear - if msg != nil && msg.Msg != nil && msg.Msg.SendTime+clearTime*1000 <= time.Now().UnixMilli() { - // if msg is not in del list, add to del list - if msg.Msg.SendTime+clearTime*1000 > lastMsgClearTime.UnixMilli() && !datautil.Contain(userID, msg.DelList...) { - seqs = append(seqs, msg.Msg.Seq) - } - } else { - log.ZDebug(ctx, "all msg need destruct is found", "conversationID", conversationID, "userID", userID, "index", index, "stop index", i) - over = true - break - } - } - if over { - break - } - } - } - - log.ZDebug(ctx, "ClearUserMsgs", "conversationID", conversationID, "userID", userID, "seqs", seqs) - - // have msg need to destruct - if len(seqs) > 0 { - // update min seq to clear after - userMinSeq := seqs[len(seqs)-1] + 1 // user min seq when clear after - currentUserMinSeq, err := db.seqUser.GetUserMinSeq(ctx, conversationID, userID) // user min seq when clear before - if err != nil { - return nil, err - } - - // if before < after, update min seq - if currentUserMinSeq < userMinSeq { - if err := db.seqUser.SetUserMinSeq(ctx, conversationID, userID, userMinSeq); err != nil { - return nil, err - } - } - } - return seqs, nil -} - -// this is struct for recursion. -type delMsgRecursionStruct struct { - minSeq int64 - delDocIDs []string -} - -func (d *delMsgRecursionStruct) getSetMinSeq() int64 { - return d.minSeq -} - -// index 0....19(del) 20...69 -// seq 70 -// set minSeq 21 -// recursion deletes the list and returns the set minimum seq. -func (db *commonMsgDatabase) deleteMsgRecursion(ctx context.Context, conversationID string, index int64, delStruct *delMsgRecursionStruct, remainTime int64) (int64, error) { - // find from oldest list - msgDocModel, err := db.msgDocDatabase.GetMsgDocModelByIndex(ctx, conversationID, index, 1) - if err != nil || msgDocModel.DocID == "" { - if err != nil { - if err == model.ErrMsgListNotExist { - log.ZDebug(ctx, "deleteMsgRecursion ErrMsgListNotExist", "conversationID", conversationID, "index:", index) - } else { - log.ZError(ctx, "deleteMsgRecursion GetUserMsgListByIndex failed", err, "conversationID", conversationID, "index", index) - } - } - // If an error is reported, or the error cannot be obtained, it is physically deleted and seq delMongoMsgsPhysical(delStruct.delDocIDList) is returned to end the recursion - err = db.msgDocDatabase.DeleteDocs(ctx, delStruct.delDocIDs) - if err != nil { - return 0, err - } - return delStruct.getSetMinSeq() + 1, nil - } - log.ZDebug(ctx, "doc info", "conversationID", conversationID, "index", index, "docID", msgDocModel.DocID, "len", len(msgDocModel.Msg)) - if int64(len(msgDocModel.Msg)) > db.msgTable.GetSingleGocMsgNum() { - log.ZWarn(ctx, "msgs too large", nil, "length", len(msgDocModel.Msg), "docID:", msgDocModel.DocID) - } - if msgDocModel.IsFull() && msgDocModel.Msg[len(msgDocModel.Msg)-1].Msg.SendTime+(remainTime*1000) < timeutil.GetCurrentTimestampByMill() { - log.ZDebug(ctx, "doc is full and all msg is expired", "docID", msgDocModel.DocID) - delStruct.delDocIDs = append(delStruct.delDocIDs, msgDocModel.DocID) - delStruct.minSeq = msgDocModel.Msg[len(msgDocModel.Msg)-1].Msg.Seq - } else { - var delMsgIndexs []int - for i, MsgInfoModel := range msgDocModel.Msg { - if MsgInfoModel != nil && MsgInfoModel.Msg != nil { - if timeutil.GetCurrentTimestampByMill() > MsgInfoModel.Msg.SendTime+(remainTime*1000) { - delMsgIndexs = append(delMsgIndexs, i) - } - } - } - if len(delMsgIndexs) > 0 { - if err = db.msgDocDatabase.DeleteMsgsInOneDocByIndex(ctx, msgDocModel.DocID, delMsgIndexs); err != nil { - log.ZError(ctx, "deleteMsgRecursion DeleteMsgsInOneDocByIndex failed", err, "conversationID", conversationID, "index", index) - } - delStruct.minSeq = int64(msgDocModel.Msg[delMsgIndexs[len(delMsgIndexs)-1]].Msg.Seq) - } - } - seq, err := db.deleteMsgRecursion(ctx, conversationID, index+1, delStruct, remainTime) - return seq, err -} - func (db *commonMsgDatabase) DeleteMsgsPhysicalBySeqs(ctx context.Context, conversationID string, allSeqs []int64) error { - if err := db.msg.DeleteMessagesFromCache(ctx, conversationID, allSeqs); err != nil { - return err - } for docID, seqs := range db.msgTable.GetDocIDSeqsMap(conversationID, allSeqs) { var indexes []int for _, seq := range seqs { @@ -784,13 +544,10 @@ func (db *commonMsgDatabase) DeleteMsgsPhysicalBySeqs(ctx context.Context, conve return err } } - return nil + return db.msgCache.DelMessageBySeqs(ctx, conversationID, allSeqs) } func (db *commonMsgDatabase) DeleteUserMsgsBySeqs(ctx context.Context, userID string, conversationID string, seqs []int64) error { - if err := db.msg.DeleteMessagesFromCache(ctx, conversationID, seqs); err != nil { - return err - } for docID, seqs := range db.msgTable.GetDocIDSeqsMap(conversationID, seqs) { for _, seq := range seqs { if _, err := db.msgDocDatabase.PushUnique(ctx, docID, db.msgTable.GetMsgIndex(seq), "del_list", []string{userID}); err != nil { @@ -798,7 +555,7 @@ func (db *commonMsgDatabase) DeleteUserMsgsBySeqs(ctx context.Context, userID st } } } - return nil + return db.msgCache.DelMessageBySeqs(ctx, conversationID, seqs) } func (db *commonMsgDatabase) GetMaxSeqs(ctx context.Context, conversationIDs []string) (map[string]int64, error) { @@ -809,11 +566,6 @@ func (db *commonMsgDatabase) GetMaxSeq(ctx context.Context, conversationID strin return db.seqConversation.GetMaxSeq(ctx, conversationID) } -// -//func (db *commonMsgDatabase) SetMinSeq(ctx context.Context, conversationID string, minSeq int64) error { -// return db.seqConversation.SetMinSeq(ctx, conversationID, minSeq) -//} - func (db *commonMsgDatabase) SetMinSeqs(ctx context.Context, seqs map[string]int64) error { return db.seqConversation.SetMinSeqs(ctx, seqs) } @@ -847,11 +599,11 @@ func (db *commonMsgDatabase) GetHasReadSeq(ctx context.Context, userID string, c } func (db *commonMsgDatabase) SetSendMsgStatus(ctx context.Context, id string, status int32) error { - return db.msg.SetSendMsgStatus(ctx, id, status) + return db.msgCache.SetSendMsgStatus(ctx, id, status) } func (db *commonMsgDatabase) GetSendMsgStatus(ctx context.Context, id string) (int32, error) { - return db.msg.GetSendMsgStatus(ctx, id) + return db.msgCache.GetSendMsgStatus(ctx, id) } func (db *commonMsgDatabase) GetConversationMinMaxSeqInMongoAndCache(ctx context.Context, conversationID string) (minSeqMongo, maxSeqMongo, minSeqCache, maxSeqCache int64, err error) { @@ -888,26 +640,11 @@ func (db *commonMsgDatabase) GetMinMaxSeqMongo(ctx context.Context, conversation return } -func (db *commonMsgDatabase) RangeUserSendCount( - ctx context.Context, - start time.Time, - end time.Time, - group bool, - ase bool, - pageNumber int32, - showNumber int32, -) (msgCount int64, userCount int64, users []*model.UserCount, dateCount map[string]int64, err error) { +func (db *commonMsgDatabase) RangeUserSendCount(ctx context.Context, start time.Time, end time.Time, group bool, ase bool, pageNumber int32, showNumber int32) (msgCount int64, userCount int64, users []*model.UserCount, dateCount map[string]int64, err error) { return db.msgDocDatabase.RangeUserSendCount(ctx, start, end, group, ase, pageNumber, showNumber) } -func (db *commonMsgDatabase) RangeGroupSendCount( - ctx context.Context, - start time.Time, - end time.Time, - ase bool, - pageNumber int32, - showNumber int32, -) (msgCount int64, userCount int64, groups []*model.GroupCount, dateCount map[string]int64, err error) { +func (db *commonMsgDatabase) RangeGroupSendCount(ctx context.Context, start time.Time, end time.Time, ase bool, pageNumber int32, showNumber int32) (msgCount int64, userCount int64, groups []*model.GroupCount, dateCount map[string]int64, err error) { return db.msgDocDatabase.RangeGroupSendCount(ctx, start, end, ase, pageNumber, showNumber) } @@ -947,43 +684,10 @@ func (db *commonMsgDatabase) FindOneByDocIDs(ctx context.Context, conversationID return totalMsgs, nil } -func (db *commonMsgDatabase) ConvertMsgsDocLen(ctx context.Context, conversationIDs []string) { - db.msgDocDatabase.ConvertMsgsDocLen(ctx, conversationIDs) -} - func (db *commonMsgDatabase) GetRandBeforeMsg(ctx context.Context, ts int64, limit int) ([]*model.MsgDocModel, error) { return db.msgDocDatabase.GetRandBeforeMsg(ctx, ts, limit) } -// -//func (db *commonMsgDatabase) DeleteDocMsgBefore(ctx context.Context, ts int64, doc *model.MsgDocModel) ([]int, error) { -// var notNull int -// index := make([]int, 0, len(doc.Msg)) -// for i, message := range doc.Msg { -// if message.Msg != nil { -// notNull++ -// if message.Msg.SendTime < ts { -// index = append(index, i) -// } -// } -// } -// if len(index) == 0 { -// return index, nil -// } -// maxSeq := doc.Msg[index[len(index)-1]].Msg.Seq -// conversationID := doc.DocID[:strings.LastIndex(doc.DocID, ":")] -// if err := db.SetMinSeq(ctx, conversationID, maxSeq+1); err != nil { -// return index, err -// } -// if len(index) == notNull { -// log.ZDebug(ctx, "Delete db in Doc", "DocID", doc.DocID, "index", index, "maxSeq", maxSeq) -// return index, db.msgDocDatabase.DeleteDoc(ctx, doc.DocID) -// } else { -// log.ZDebug(ctx, "delete db in index", "DocID", doc.DocID, "index", index, "maxSeq", maxSeq) -// return index, db.msgDocDatabase.DeleteMsgByIndex(ctx, doc.DocID, index) -// } -//} - func (db *commonMsgDatabase) SetMinSeq(ctx context.Context, conversationID string, seq int64) error { dbSeq, err := db.seqConversation.GetMinSeq(ctx, conversationID) if err != nil { @@ -998,10 +702,6 @@ func (db *commonMsgDatabase) SetMinSeq(ctx context.Context, conversationID strin return db.seqConversation.SetMinSeq(ctx, conversationID, seq) } -func (db *commonMsgDatabase) GetRandDocIDs(ctx context.Context, limit int) ([]string, error) { - return db.msgDocDatabase.GetRandDocIDs(ctx, limit) -} - func (db *commonMsgDatabase) GetCacheMaxSeqWithTime(ctx context.Context, conversationIDs []string) (map[string]database.SeqTime, error) { return db.seqConversation.GetCacheMaxSeqWithTime(ctx, conversationIDs) } @@ -1016,9 +716,103 @@ func (db *commonMsgDatabase) GetMaxSeqsWithTime(ctx context.Context, conversatio } func (db *commonMsgDatabase) DeleteDoc(ctx context.Context, docID string) error { - return db.msgDocDatabase.DeleteDoc(ctx, docID) + index := strings.LastIndex(docID, ":") + if index <= 0 { + return errs.ErrInternalServer.WrapMsg("docID is invalid", "docID", docID) + } + index, err := strconv.Atoi(docID[index+1:]) + if err != nil { + return errs.WrapMsg(err, "strconv.Atoi", "docID", docID) + } + conversationID := docID[:index] + seqs := make([]int64, db.msgTable.GetSingleGocMsgNum()) + minSeq := db.msgTable.GetMinSeq(index) + for i := range seqs { + seqs[i] = minSeq + int64(i) + } + if err := db.msgDocDatabase.DeleteDoc(ctx, docID); err != nil { + return err + } + return db.msgCache.DelMessageBySeqs(ctx, conversationID, seqs) } func (db *commonMsgDatabase) GetLastMessageSeqByTime(ctx context.Context, conversationID string, time int64) (int64, error) { return db.msgDocDatabase.GetLastMessageSeqByTime(ctx, conversationID, time) } + +func (db *commonMsgDatabase) handlerDeleteAndRevoked(ctx context.Context, userID string, msgs []*model.MsgInfoModel) { + for i := range msgs { + msg := msgs[i] + if msg == nil || msg.Msg == nil { + continue + } + msg.Msg.IsRead = msg.IsRead + if datautil.Contain(userID, msg.DelList...) { + msg.Msg.Content = "" + msg.Msg.Status = constant.MsgDeleted + } + if msg.Revoke == nil { + continue + } + msg.Msg.ContentType = constant.MsgRevokeNotification + revokeContent := sdkws.MessageRevokedContent{ + RevokerID: msg.Revoke.UserID, + RevokerRole: msg.Revoke.Role, + ClientMsgID: msg.Msg.ClientMsgID, + RevokerNickname: msg.Revoke.Nickname, + RevokeTime: msg.Revoke.Time, + SourceMessageSendTime: msg.Msg.SendTime, + SourceMessageSendID: msg.Msg.SendID, + SourceMessageSenderNickname: msg.Msg.SenderNickname, + SessionType: msg.Msg.SessionType, + Seq: msg.Msg.Seq, + Ex: msg.Msg.Ex, + } + data, err := jsonutil.JsonMarshal(&revokeContent) + if err != nil { + log.ZWarn(ctx, "handlerDeleteAndRevoked JsonMarshal MessageRevokedContent", err, "msg", msg) + continue + } + elem := sdkws.NotificationElem{ + Detail: string(data), + } + content, err := jsonutil.JsonMarshal(&elem) + if err != nil { + log.ZWarn(ctx, "handlerDeleteAndRevoked JsonMarshal NotificationElem", err, "msg", msg) + continue + } + msg.Msg.Content = string(content) + } +} + +func (db *commonMsgDatabase) handlerQuote(ctx context.Context, userID, conversationID string, msgs []*model.MsgInfoModel) { + temp := make(map[int64][]*model.MsgInfoModel) + for i := range msgs { + db.handlerDBMsg(ctx, temp, userID, conversationID, msgs[i]) + } +} + +func (db *commonMsgDatabase) GetMessageBySeqs(ctx context.Context, conversationID string, userID string, seqs []int64) ([]*sdkws.MsgData, error) { + msgs, err := db.msgCache.GetMessageBySeqs(ctx, conversationID, seqs) + if err != nil { + return nil, err + } + db.handlerDeleteAndRevoked(ctx, userID, msgs) + db.handlerQuote(ctx, userID, conversationID, msgs) + seqMsgs := make(map[int64]*model.MsgInfoModel) + for i, msg := range msgs { + if msg.Msg == nil { + continue + } + seqMsgs[msg.Msg.Seq] = msgs[i] + } + res := make([]*sdkws.MsgData, 0, len(seqs)) + for _, seq := range seqs { + if v, ok := seqMsgs[seq]; ok { + res = append(res, convert.MsgDB2Pb(v.Msg)) + } else { + res = append(res, &sdkws.MsgData{Seq: seq}) + } + } + return res, nil +} diff --git a/pkg/common/storage/controller/msg_transfer.go b/pkg/common/storage/controller/msg_transfer.go index 1ecd786aa..f4c0c6270 100644 --- a/pkg/common/storage/controller/msg_transfer.go +++ b/pkg/common/storage/controller/msg_transfer.go @@ -2,10 +2,11 @@ package controller import ( "context" + "github.com/openimsdk/open-im-server/v3/pkg/common/convert" "github.com/openimsdk/protocol/constant" + "github.com/openimsdk/tools/utils/datautil" "github.com/openimsdk/open-im-server/v3/pkg/common/config" - "github.com/openimsdk/open-im-server/v3/pkg/common/prommetrics" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" @@ -50,7 +51,7 @@ func NewMsgTransferDatabase(msgDocModel database.Msg, msg cache.MsgCache, seqUse } return &msgTransferDatabase{ msgDocDatabase: msgDocModel, - msg: msg, + msgCache: msg, seqUser: seqUser, seqConversation: seqConversation, producerToMongo: producerToMongo, @@ -61,7 +62,7 @@ func NewMsgTransferDatabase(msgDocModel database.Msg, msg cache.MsgCache, seqUse type msgTransferDatabase struct { msgDocDatabase database.Msg msgTable model.MsgDocModel - msg cache.MsgCache + msgCache cache.MsgCache seqConversation cache.SeqConversationCache seqUser cache.SeqUser producerToMongo *kafka.Producer @@ -73,10 +74,12 @@ func (db *msgTransferDatabase) BatchInsertChat2DB(ctx context.Context, conversat return errs.ErrArgs.WrapMsg("msgList is empty") } msgs := make([]any, len(msgList)) + seqs := make([]int64, len(msgList)) for i, msg := range msgList { if msg == nil { continue } + seqs[i] = msg.Seq var offlinePushModel *model.OfflinePushModel if msg.OfflinePushInfo != nil { offlinePushModel = &model.OfflinePushModel{ @@ -114,7 +117,11 @@ func (db *msgTransferDatabase) BatchInsertChat2DB(ctx context.Context, conversat Ex: msg.Ex, } } - return db.BatchInsertBlock(ctx, conversationID, msgs, updateKeyMsg, msgList[0].Seq) + if err := db.BatchInsertBlock(ctx, conversationID, msgs, updateKeyMsg, msgList[0].Seq); err != nil { + return err + } + //return db.msgCache.DelMessageBySeqs(ctx, conversationID, seqs) + return nil } func (db *msgTransferDatabase) BatchInsertBlock(ctx context.Context, conversationID string, fields []any, key int8, firstSeq int64) error { @@ -219,7 +226,7 @@ func (db *msgTransferDatabase) BatchInsertBlock(ctx context.Context, conversatio } func (db *msgTransferDatabase) DeleteMessagesFromCache(ctx context.Context, conversationID string, seqs []int64) error { - return db.msg.DeleteMessagesFromCache(ctx, conversationID, seqs) + return db.msgCache.DelMessageBySeqs(ctx, conversationID, seqs) } func (db *msgTransferDatabase) BatchInsertChat2Cache(ctx context.Context, conversationID string, msgs []*sdkws.MsgData) (seq int64, isNew bool, userHasReadMap map[string]int64, err error) { @@ -238,20 +245,22 @@ func (db *msgTransferDatabase) BatchInsertChat2Cache(ctx context.Context, conver isNew = currentMaxSeq == 0 lastMaxSeq := currentMaxSeq userSeqMap := make(map[string]int64) + seqs := make([]int64, 0, lenList) for _, m := range msgs { currentMaxSeq++ m.Seq = currentMaxSeq userSeqMap[m.SendID] = m.Seq + seqs = append(seqs, m.Seq) } - - failedNum, err := db.msg.SetMessagesToCache(ctx, conversationID, msgs) - if err != nil { - prommetrics.MsgInsertRedisFailedCounter.Add(float64(failedNum)) - log.ZError(ctx, "setMessageToCache error", err, "len", len(msgs), "conversationID", conversationID) - } else { - prommetrics.MsgInsertRedisSuccessCounter.Inc() + msgToDB := func(msg *sdkws.MsgData) *model.MsgInfoModel { + return &model.MsgInfoModel{ + Msg: convert.MsgPb2DB(msg), + } + } + if err := db.msgCache.SetMessageBySeqs(ctx, conversationID, datautil.Slice(msgs, msgToDB)); err != nil { + return 0, false, nil, err } - return lastMaxSeq, isNew, userSeqMap, errs.Wrap(err) + return lastMaxSeq, isNew, userSeqMap, nil } func (db *msgTransferDatabase) SetHasReadSeqs(ctx context.Context, conversationID string, userSeqMap map[string]int64) error { diff --git a/pkg/common/storage/database/mgo/msg.go b/pkg/common/storage/database/mgo/msg.go index f37176695..03ebff611 100644 --- a/pkg/common/storage/database/mgo/msg.go +++ b/pkg/common/storage/database/mgo/msg.go @@ -7,15 +7,12 @@ import ( "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" - "github.com/openimsdk/tools/utils/datautil" - "golang.org/x/exp/rand" - "github.com/openimsdk/protocol/constant" "github.com/openimsdk/protocol/msg" "github.com/openimsdk/protocol/sdkws" "github.com/openimsdk/tools/db/mongoutil" "github.com/openimsdk/tools/errs" - "github.com/openimsdk/tools/log" + "github.com/openimsdk/tools/utils/datautil" "github.com/openimsdk/tools/utils/jsonutil" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/bson/primitive" @@ -42,12 +39,6 @@ type MsgMgo struct { model model.MsgDocModel } -func (m *MsgMgo) PushMsgsToDoc(ctx context.Context, docID string, msgsToMongo []model.MsgInfoModel) error { - filter := bson.M{"doc_id": docID} - update := bson.M{"$push": bson.M{"msgs": bson.M{"$each": msgsToMongo}}} - return mongoutil.UpdateOne(ctx, m.coll, filter, update, false) -} - func (m *MsgMgo) Create(ctx context.Context, msg *model.MsgDocModel) error { return mongoutil.InsertMany(ctx, m.coll, []*model.MsgDocModel{msg}) } @@ -80,16 +71,6 @@ func (m *MsgMgo) PushUnique(ctx context.Context, docID string, index int64, key return mongoutil.UpdateOneResult(ctx, m.coll, filter, update) } -func (m *MsgMgo) UpdateMsgContent(ctx context.Context, docID string, index int64, msg []byte) error { - filter := bson.M{"doc_id": docID} - update := bson.M{"$set": bson.M{fmt.Sprintf("msgs.%d.msg", index): msg}} - return mongoutil.UpdateOne(ctx, m.coll, filter, update, false) -} - -func (m *MsgMgo) IsExistDocID(ctx context.Context, docID string) (bool, error) { - return mongoutil.Exist(ctx, m.coll, bson.M{"doc_id": docID}) -} - func (m *MsgMgo) FindOneByDocID(ctx context.Context, docID string) (*model.MsgDocModel, error) { return mongoutil.FindOne[*model.MsgDocModel](ctx, m.coll, bson.M{"doc_id": docID}) } @@ -218,13 +199,6 @@ func (m *MsgMgo) GetOldestMsg(ctx context.Context, conversationID string) (*mode } } -func (m *MsgMgo) DeleteDocs(ctx context.Context, docIDs []string) error { - if len(docIDs) == 0 { - return nil - } - return mongoutil.DeleteMany(ctx, m.coll, bson.M{"doc_id": bson.M{"$in": docIDs}}) -} - func (m *MsgMgo) GetMsgDocModelByIndex(ctx context.Context, conversationID string, index, sort int64) (*model.MsgDocModel, error) { if sort != 1 && sort != -1 { return nil, errs.ErrArgs.WrapMsg("mongo sort must be 1 or -1") @@ -279,95 +253,6 @@ func (m *MsgMgo) MarkSingleChatMsgsAsRead(ctx context.Context, userID string, do return nil } -//func (m *MsgMgo) searchCount(ctx context.Context, filter any) (int64, error) { -// -// return nil, nil -//} - -//func (m *MsgMgo) searchMessage(ctx context.Context, filter any, nextID primitive.ObjectID, content bool, limit int) (int64, []*model.MsgInfoModel, primitive.ObjectID, error) { -// var pipeline bson.A -// if !nextID.IsZero() { -// pipeline = append(pipeline, bson.M{"$match": bson.M{"_id": bson.M{"$gt": nextID}}}) -// } -// pipeline = append(pipeline, -// bson.M{"$match": filter}, -// bson.M{"$limit": limit}, -// bson.M{"$unwind": "$msgs"}, -// bson.M{"$match": filter}, -// bson.M{ -// "$group": bson.M{ -// "_id": "$_id", -// "doc_id": bson.M{ -// "$first": "$doc_id", -// }, -// "msgs": bson.M{"$push": "$msgs"}, -// }, -// }, -// ) -// if !content { -// pipeline = append(pipeline, -// bson.M{ -// "$project": bson.M{ -// "_id": 1, -// "count": bson.M{"$size": "$msgs"}, -// }, -// }, -// ) -// type result struct { -// ID primitive.ObjectID `bson:"_id"` -// Count int64 `bson:"count"` -// } -// res, err := mongoutil.Aggregate[result](ctx, m.coll, pipeline) -// if err != nil { -// return 0, nil, primitive.ObjectID{}, err -// } -// if len(res) == 0 { -// return 0, nil, primitive.ObjectID{}, nil -// } -// var count int64 -// for _, r := range res { -// count += r.Count -// } -// return count, nil, res[len(res)-1].ID, nil -// } -// type result struct { -// ID primitive.ObjectID `bson:"_id"` -// Msg []*model.MsgInfoModel `bson:"msgs"` -// } -// res, err := mongoutil.Aggregate[result](ctx, m.coll, pipeline) -// if err != nil { -// return 0, nil, primitive.ObjectID{}, err -// } -// if len(res) == 0 { -// return 0, nil, primitive.ObjectID{}, err -// } -// var count int -// for _, r := range res { -// count += len(r.Msg) -// } -// msgs := make([]*model.MsgInfoModel, 0, count) -// for _, r := range res { -// msgs = append(msgs, r.Msg...) -// } -// return int64(count), msgs, res[len(res)-1].ID, nil -//} - -/* - -db.msg3.aggregate( - [ - { - "$match": { - "doc_id": "si_7009965934_8710838466:0" - }, - - } - ] -) - - -*/ - type searchMessageIndex struct { ID primitive.ObjectID `bson:"_id"` Index []int64 `bson:"index"` @@ -512,22 +397,6 @@ func (m *MsgMgo) searchMessage(ctx context.Context, req *msg.SearchMessageReq) ( } } -func (m *MsgMgo) getDocRange(ctx context.Context, id primitive.ObjectID, index []int64) ([]*model.MsgInfoModel, error) { - if len(index) == 0 { - return nil, nil - } - - pipeline := bson.A{ - bson.M{"$match": bson.M{"_id": id}}, - bson.M{"$project": "$msgs"}, - } - msgs, err := mongoutil.Aggregate[*model.MsgInfoModel](ctx, m.coll, pipeline) - if err != nil { - return nil, err - } - return msgs, nil -} - func (m *MsgMgo) SearchMessage(ctx context.Context, req *msg.SearchMessageReq) (int64, []*model.MsgInfoModel, error) { count, data, err := m.searchMessage(ctx, req) if err != nil { @@ -556,143 +425,6 @@ func (m *MsgMgo) SearchMessage(ctx context.Context, req *msg.SearchMessageReq) ( return count, msgs, nil } -//func (m *MsgMgo) SearchMessage(ctx context.Context, req *msg.SearchMessageReq) (int32, []*model.MsgInfoModel, error) { -// where := make(bson.A, 0, 6) -// if req.RecvID != "" { -// if req.SessionType == constant.ReadGroupChatType { -// where = append(where, bson.M{ -// "$or": bson.A{ -// bson.M{"doc_id": "^n_" + req.RecvID + ":"}, -// bson.M{"doc_id": "^sg_" + req.RecvID + ":"}, -// }, -// }) -// } else { -// where = append(where, bson.M{"msgs.msg.recv_id": req.RecvID}) -// } -// } -// if req.SendID != "" { -// where = append(where, bson.M{"msgs.msg.send_id": req.SendID}) -// } -// if req.ContentType != 0 { -// where = append(where, bson.M{"msgs.msg.content_type": req.ContentType}) -// } -// if req.SessionType != 0 { -// where = append(where, bson.M{"msgs.msg.session_type": req.SessionType}) -// } -// if req.SendTime != "" { -// sendTime, err := time.Parse(time.DateOnly, req.SendTime) -// if err != nil { -// return 0, nil, errs.ErrArgs.WrapMsg("invalid sendTime", "req", req.SendTime, "format", time.DateOnly, "cause", err.Error()) -// } -// where = append(where, -// bson.M{ -// "msgs.msg.send_time": bson.M{ -// "$gte": sendTime.UnixMilli(), -// }, -// }, -// bson.M{ -// "msgs.msg.send_time": bson.M{ -// "$lt": sendTime.Add(time.Hour * 24).UnixMilli(), -// }, -// }, -// ) -// } -// opt := options.Find().SetLimit(100) -// res, err := mongoutil.Find[model.MsgDocModel](ctx, m.coll, bson.M{"$and": where}, opt) -// if err != nil { -// return 0, nil, err -// } -// _ = res -// fmt.Println() -// -// return 0, nil, nil -// pipeline := bson.A{ -// bson.M{ -// "$unwind": "$msgs", -// }, -// } -// if len(where) > 0 { -// pipeline = append(pipeline, bson.M{ -// "$match": bson.M{"$and": where}, -// }) -// } -// pipeline = append(pipeline, -// bson.M{ -// "$project": bson.M{ -// "_id": 0, -// "msg": "$msgs.msg", -// }, -// }, -// bson.M{ -// "$count": "count", -// }, -// ) -// //count, err := mongoutil.Aggregate[int32](ctx, m.coll, pipeline) -// //if err != nil { -// // return 0, nil, err -// //} -// //if len(count) == 0 || count[0] == 0 { -// // return 0, nil, nil -// //} -// count := []int32{0} -// pipeline = pipeline[:len(pipeline)-1] -// pipeline = append(pipeline, -// bson.M{ -// "$skip": (req.Pagination.GetPageNumber() - 1) * req.Pagination.GetShowNumber(), -// }, -// bson.M{ -// "$limit": req.Pagination.GetShowNumber(), -// }, -// ) -// msgs, err := mongoutil.Aggregate[*model.MsgInfoModel](ctx, m.coll, pipeline) -// if err != nil { -// return 0, nil, err -// } -// for i := range msgs { -// msgInfo := msgs[i] -// if msgInfo == nil || msgInfo.Msg == nil { -// continue -// } -// if msgInfo.Revoke != nil { -// revokeContent := sdkws.MessageRevokedContent{ -// RevokerID: msgInfo.Revoke.UserID, -// RevokerRole: msgInfo.Revoke.Role, -// ClientMsgID: msgInfo.Msg.ClientMsgID, -// RevokerNickname: msgInfo.Revoke.Nickname, -// RevokeTime: msgInfo.Revoke.Time, -// SourceMessageSendTime: msgInfo.Msg.SendTime, -// SourceMessageSendID: msgInfo.Msg.SendID, -// SourceMessageSenderNickname: msgInfo.Msg.SenderNickname, -// SessionType: msgInfo.Msg.SessionType, -// Seq: msgInfo.Msg.Seq, -// Ex: msgInfo.Msg.Ex, -// } -// data, err := jsonutil.JsonMarshal(&revokeContent) -// if err != nil { -// return 0, nil, errs.WrapMsg(err, "json.Marshal revokeContent") -// } -// elem := sdkws.NotificationElem{Detail: string(data)} -// content, err := jsonutil.JsonMarshal(&elem) -// if err != nil { -// return 0, nil, errs.WrapMsg(err, "json.Marshal elem") -// } -// msgInfo.Msg.ContentType = constant.MsgRevokeNotification -// msgInfo.Msg.Content = string(content) -// } -// } -// //start := (req.Pagination.PageNumber - 1) * req.Pagination.ShowNumber -// //n := int32(len(msgs)) -// //if start >= n { -// // return n, []*relation.MsgInfoModel{}, nil -// //} -// //if start+req.Pagination.ShowNumber < n { -// // msgs = msgs[start : start+req.Pagination.ShowNumber] -// //} else { -// // msgs = msgs[start:] -// //} -// return count[0], msgs, nil -//} - func (m *MsgMgo) RangeUserSendCount(ctx context.Context, start time.Time, end time.Time, group bool, ase bool, pageNumber int32, showNumber int32) (msgCount int64, userCount int64, users []*model.UserCount, dateCount map[string]int64, err error) { var sort int if ase { @@ -1178,94 +910,6 @@ func (m *MsgMgo) RangeGroupSendCount(ctx context.Context, start time.Time, end t return result[0].MsgCount, result[0].UserCount, groups, dateCount, nil } -func (m *MsgMgo) ConvertMsgsDocLen(ctx context.Context, conversationIDs []string) { - for _, conversationID := range conversationIDs { - regex := primitive.Regex{Pattern: fmt.Sprintf("^%s:", conversationID)} - msgDocs, err := mongoutil.Find[*model.MsgDocModel](ctx, m.coll, bson.M{"doc_id": regex}) - if err != nil { - log.ZError(ctx, "convertAll find msg doc failed", err, "conversationID", conversationID) - continue - } - if len(msgDocs) < 1 { - continue - } - log.ZDebug(ctx, "msg doc convert", "conversationID", conversationID, "len(msgDocs)", len(msgDocs)) - if len(msgDocs[0].Msg) == int(m.model.GetSingleGocMsgNum5000()) { - if err := mongoutil.DeleteMany(ctx, m.coll, bson.M{"doc_id": regex}); err != nil { - log.ZError(ctx, "convertAll delete many failed", err, "conversationID", conversationID) - continue - } - var newMsgDocs []any - for _, msgDoc := range msgDocs { - if int64(len(msgDoc.Msg)) == m.model.GetSingleGocMsgNum() { - continue - } - var index int64 - for index < int64(len(msgDoc.Msg)) { - msg := msgDoc.Msg[index] - if msg != nil && msg.Msg != nil { - msgDocModel := model.MsgDocModel{DocID: m.model.GetDocID(conversationID, msg.Msg.Seq)} - end := index + m.model.GetSingleGocMsgNum() - if int(end) >= len(msgDoc.Msg) { - msgDocModel.Msg = msgDoc.Msg[index:] - } else { - msgDocModel.Msg = msgDoc.Msg[index:end] - } - newMsgDocs = append(newMsgDocs, msgDocModel) - index = end - } else { - break - } - } - } - if err = mongoutil.InsertMany(ctx, m.coll, newMsgDocs); err != nil { - log.ZError(ctx, "convertAll insert many failed", err, "conversationID", conversationID, "len(newMsgDocs)", len(newMsgDocs)) - } else { - log.ZDebug(ctx, "msg doc convert", "conversationID", conversationID, "len(newMsgDocs)", len(newMsgDocs)) - } - } - } -} - -func (m *MsgMgo) GetRandDocIDs(ctx context.Context, limit int) ([]string, error) { - var skip int - var docIDs []string - var offset int - - count, err := m.coll.CountDocuments(ctx, bson.M{}) - if err != nil { - return nil, err - } - - if count < int64(limit) { - skip = 0 - } else { - rand.Seed(uint64(time.Now().UnixMilli())) - skip = rand.Intn(int(count / int64(limit))) - offset = skip * limit - } - log.ZDebug(ctx, "offset", "skip", skip, "offset", offset) - res, err := mongoutil.Aggregate[*model.MsgDocModel](ctx, m.coll, []bson.M{ - { - "$project": bson.M{ - "doc_id": 1, - }, - }, - { - "$skip": offset, - }, - { - "$limit": limit, - }, - }) - - for _, doc := range res { - docIDs = append(docIDs, doc.DocID) - } - - return docIDs, errs.Wrap(err) -} - func (m *MsgMgo) GetRandBeforeMsg(ctx context.Context, ts int64, limit int) ([]*model.MsgDocModel, error) { return mongoutil.Aggregate[*model.MsgDocModel](ctx, m.coll, []bson.M{ { @@ -1297,18 +941,6 @@ func (m *MsgMgo) GetRandBeforeMsg(ctx context.Context, ts int64, limit int) ([]* }) } -func (m *MsgMgo) DeleteMsgByIndex(ctx context.Context, docID string, index []int) error { - if len(index) == 0 { - return nil - } - model := &model.MsgInfoModel{DelList: []string{}} - set := make(map[string]any) - for i := range index { - set[fmt.Sprintf("msgs.%d", i)] = model - } - return mongoutil.UpdateOne(ctx, m.coll, bson.M{"doc_id": docID}, bson.M{"$set": set}, true) -} - func (m *MsgMgo) DeleteDoc(ctx context.Context, docID string) error { return mongoutil.DeleteOne(ctx, m.coll, bson.M{"doc_id": docID}) } @@ -1364,3 +996,55 @@ func (m *MsgMgo) GetLastMessageSeqByTime(ctx context.Context, conversationID str } return seq, nil } + +func (m *MsgMgo) onlyFindDocIndex(ctx context.Context, docID string, indexes []int64) ([]*model.MsgInfoModel, error) { + if len(indexes) == 0 { + return nil, nil + } + pipeline := mongo.Pipeline{ + bson.D{{Key: "$match", Value: bson.D{ + {Key: "doc_id", Value: docID}, + }}}, + bson.D{{Key: "$project", Value: bson.D{ + {Key: "_id", Value: 0}, + {Key: "doc_id", Value: 1}, + {Key: "msgs", Value: bson.D{ + {Key: "$map", Value: bson.D{ + {Key: "input", Value: indexes}, + {Key: "as", Value: "index"}, + {Key: "in", Value: bson.D{ + {Key: "$arrayElemAt", Value: bson.A{"$msgs", "$$index"}}, + }}, + }}, + }}, + }}}, + } + msgDocModel, err := mongoutil.Aggregate[*model.MsgDocModel](ctx, m.coll, pipeline) + if err != nil { + return nil, err + } + if len(msgDocModel) == 0 { + return nil, nil + } + return msgDocModel[0].Msg, nil +} + +func (m *MsgMgo) FindSeqs(ctx context.Context, conversationID string, seqs []int64) ([]*model.MsgInfoModel, error) { + if len(seqs) == 0 { + return nil, nil + } + result := make([]*model.MsgInfoModel, 0, len(seqs)) + for docID, seqs := range m.model.GetDocIDSeqsMap(conversationID, seqs) { + res, err := m.onlyFindDocIndex(ctx, docID, datautil.Slice(seqs, m.model.GetMsgIndex)) + if err != nil { + return nil, err + } + for i, re := range res { + if re == nil || re.Msg == nil { + continue + } + result = append(result, res[i]) + } + } + return result, nil +} diff --git a/pkg/common/storage/database/msg.go b/pkg/common/storage/database/msg.go index abb2a44c2..b44e70296 100644 --- a/pkg/common/storage/database/msg.go +++ b/pkg/common/storage/database/msg.go @@ -24,30 +24,20 @@ import ( ) type Msg interface { - PushMsgsToDoc(ctx context.Context, docID string, msgsToMongo []model.MsgInfoModel) error Create(ctx context.Context, model *model.MsgDocModel) error UpdateMsg(ctx context.Context, docID string, index int64, key string, value any) (*mongo.UpdateResult, error) PushUnique(ctx context.Context, docID string, index int64, key string, value any) (*mongo.UpdateResult, error) - UpdateMsgContent(ctx context.Context, docID string, index int64, msg []byte) error - IsExistDocID(ctx context.Context, docID string) (bool, error) FindOneByDocID(ctx context.Context, docID string) (*model.MsgDocModel, error) GetMsgBySeqIndexIn1Doc(ctx context.Context, userID, docID string, seqs []int64) ([]*model.MsgInfoModel, error) GetNewestMsg(ctx context.Context, conversationID string) (*model.MsgInfoModel, error) GetOldestMsg(ctx context.Context, conversationID string) (*model.MsgInfoModel, error) - DeleteDocs(ctx context.Context, docIDs []string) error - GetMsgDocModelByIndex(ctx context.Context, conversationID string, index, sort int64) (*model.MsgDocModel, error) DeleteMsgsInOneDocByIndex(ctx context.Context, docID string, indexes []int) error MarkSingleChatMsgsAsRead(ctx context.Context, userID string, docID string, indexes []int64) error SearchMessage(ctx context.Context, req *msg.SearchMessageReq) (int64, []*model.MsgInfoModel, error) RangeUserSendCount(ctx context.Context, start time.Time, end time.Time, group bool, ase bool, pageNumber int32, showNumber int32) (msgCount int64, userCount int64, users []*model.UserCount, dateCount map[string]int64, err error) RangeGroupSendCount(ctx context.Context, start time.Time, end time.Time, ase bool, pageNumber int32, showNumber int32) (msgCount int64, userCount int64, groups []*model.GroupCount, dateCount map[string]int64, err error) - ConvertMsgsDocLen(ctx context.Context, conversationIDs []string) - DeleteDoc(ctx context.Context, docID string) error - DeleteMsgByIndex(ctx context.Context, docID string, index []int) error GetRandBeforeMsg(ctx context.Context, ts int64, limit int) ([]*model.MsgDocModel, error) - - GetRandDocIDs(ctx context.Context, limit int) ([]string, error) - GetLastMessageSeqByTime(ctx context.Context, conversationID string, time int64) (int64, error) + FindSeqs(ctx context.Context, conversationID string, seqs []int64) ([]*model.MsgInfoModel, error) } diff --git a/pkg/common/storage/model/msg.go b/pkg/common/storage/model/msg.go index e16233973..69113032d 100644 --- a/pkg/common/storage/model/msg.go +++ b/pkg/common/storage/model/msg.go @@ -143,3 +143,7 @@ func (*MsgDocModel) GenExceptionMessageBySeqs(seqs []int64) (exceptionMsg []*sdk } return exceptionMsg } + +func (*MsgDocModel) GetMinSeq(index int) int64 { + return int64(index*singleGocMsgNum) + 1 +} From 4c537321b61f6b2b095f8871a0dc34ef974b816a Mon Sep 17 00:00:00 2001 From: Monet Lee Date: Thu, 26 Dec 2024 17:53:14 +0800 Subject: [PATCH 089/199] docs: improve deployment docs in kubernetes. (#2973) * docs: improve deployment docs in kubernetes. * move docs path. * format contents. * update contents. * build: update deployment env. * docs: update deploy docs. * build: add kafka secret and dependencies. * docs: update deployment docs. * Update docs contents. * update docs contents. --- deployments/Readme.md | 189 +++++++++++++++++- deployments/deploy/README.md | 85 -------- deployments/deploy/kafka-secret.yml | 7 + deployments/deploy/minio-secret.yml | 8 + deployments/deploy/minio-statefulset.yml | 14 +- deployments/deploy/mongo-secret.yml | 8 + deployments/deploy/mongo-statefulset.yml | 23 ++- deployments/deploy/openim-api-deployment.yml | 18 +- deployments/deploy/openim-config.yml | 20 +- .../deploy/openim-msggateway-deployment.yml | 2 +- .../deploy/openim-msgtransfer-deployment.yml | 14 +- deployments/deploy/openim-push-deployment.yml | 7 +- .../deploy/openim-rpc-auth-deployment.yml | 2 +- .../openim-rpc-conversation-deployment.yml | 9 +- .../deploy/openim-rpc-friend-deployment.yml | 13 +- .../deploy/openim-rpc-group-deployment.yml | 12 +- .../deploy/openim-rpc-msg-deployment.yml | 18 +- .../deploy/openim-rpc-third-deployment.yml | 15 +- .../deploy/openim-rpc-user-deployment.yml | 14 +- deployments/deploy/redis-secret.yml | 7 + deployments/deploy/redis-statefulset.yml | 11 - 21 files changed, 329 insertions(+), 167 deletions(-) delete mode 100644 deployments/deploy/README.md create mode 100644 deployments/deploy/kafka-secret.yml create mode 100644 deployments/deploy/minio-secret.yml create mode 100644 deployments/deploy/mongo-secret.yml create mode 100644 deployments/deploy/redis-secret.yml diff --git a/deployments/Readme.md b/deployments/Readme.md index 775baaea0..8da4f90aa 100644 --- a/deployments/Readme.md +++ b/deployments/Readme.md @@ -1,3 +1,188 @@ -# OpenIM Application Containerization Deployment Guide +# Kubernetes Deployment -view deploy [README](./deploy/README.md) \ No newline at end of file +## Resource Requests + +- CPU: 2 cores +- Memory: 4 GiB +- Disk usage: 20 GiB (on Node) + +## Preconditions + +ensure that you have already deployed the following components: + +- Redis +- MongoDB +- Kafka +- MinIO + +## Origin Deploy + +### Enter the target dir + +`cd ./deployments/deploy/` + +### Deploy configs and dependencies + +Upate your configMap `openim-config.yml`. **You can check the official docs for more details.** + +In `openim-config.yml`, you need modify the following configurations: + +**discovery.yml** + +- `kubernetes.namespace`: default is `default`, you can change it to your namespace. + +**mongodb.yml** + +- `address`: set to your already mongodb address or mongo Service name and port in your deployed. +- `database`: set to your mongodb database name.(Need have a created database.) +- `authSource`: set to your mongodb authSource. (authSource is specify the database name associated with the user's credentials, user need create in this database.) + +**kafka.yml** + +- `address`: set to your already kafka address or kafka Service name and port in your deployed. + +**redis.yml** + +- `address`: set to your already redis address or redis Service name and port in your deployed. + +**minio.yml** + +- `internalAddress`: set to your minio Service name and port in your deployed. +- `externalAddress`: set to your already expose minio external address. + +### Set the secret + +A Secret is an object that contains a small amount of sensitive data. Such as password and secret. Secret is similar to ConfigMaps. + +#### Redis: + +Update the `redis-password` value in `redis-secret.yml` to your Redis password encoded in base64. + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: openim-redis-secret +type: Opaque +data: + redis-password: b3BlbklNMTIz # update to your redis password encoded in base64, if need empty, you can set to "" +``` + +#### Mongo: + +Update the `mongo_openim_username`, `mongo_openim_password` value in `mongo-secret.yml` to your Mongo username and password encoded in base64. + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: openim-mongo-secret +type: Opaque +data: + mongo_openim_username: b3BlbklN # update to your mongo username encoded in base64, if need empty, you can set to "" (this user credentials need in authSource database). + mongo_openim_password: b3BlbklNMTIz # update to your mongo password encoded in base64, if need empty, you can set to "" +``` + +#### Minio: + +Update the `minio-root-user` and `minio-root-password` value in `minio-secret.yml` to your MinIO accessKeyID and secretAccessKey encoded in base64. + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: openim-minio-secret +type: Opaque +data: + minio-root-user: cm9vdA== # update to your minio accessKeyID encoded in base64, if need empty, you can set to "" + minio-root-password: b3BlbklNMTIz # update to your minio secretAccessKey encoded in base64, if need empty, you can set to "" +``` + +#### Kafka: + +Update the `kafka-password` value in `kafka-secret.yml` to your Kafka password encoded in base64. + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: openim-kafka-secret +type: Opaque +data: + kafka-password: b3BlbklNMTIz # update to your kafka password encoded in base64, if need empty, you can set to "" +``` + +### Apply the secret. + +```shell +kubectl apply -f redis-secret.yml -f minio-secret.yml -f mongo-secret.yml -f kafka-secret.yml +``` + +### Apply all config + +`kubectl apply -f ./openim-config.yml` + +> Attation: If you use `default` namespace, you can excute `clusterRile.yml` to create a cluster role binding for default service account. +> +> Namespace is modify to `discovery.yml` in `openim-config.yml`, you can change `kubernetes.namespace` to your namespace. + +**Excute `clusterRole.yml`** + +`kubectl apply -f ./clusterRole.yml` + +### run all deployments and services + +> Note: Ensure that infrastructure services like MinIO, Redis, and Kafka are running before deploying the main applications. + +```bash +kubectl apply \ + -f openim-api-deployment.yml \ + -f openim-api-service.yml \ + -f openim-crontask-deployment.yml \ + -f openim-rpc-user-deployment.yml \ + -f openim-rpc-user-service.yml \ + -f openim-msggateway-deployment.yml \ + -f openim-msggateway-service.yml \ + -f openim-push-deployment.yml \ + -f openim-push-service.yml \ + -f openim-msgtransfer-service.yml \ + -f openim-msgtransfer-deployment.yml \ + -f openim-rpc-conversation-deployment.yml \ + -f openim-rpc-conversation-service.yml \ + -f openim-rpc-auth-deployment.yml \ + -f openim-rpc-auth-service.yml \ + -f openim-rpc-group-deployment.yml \ + -f openim-rpc-group-service.yml \ + -f openim-rpc-friend-deployment.yml \ + -f openim-rpc-friend-service.yml \ + -f openim-rpc-msg-deployment.yml \ + -f openim-rpc-msg-service.yml \ + -f openim-rpc-third-deployment.yml \ + -f openim-rpc-third-service.yml +``` + +### Verification + +After deploying the services, verify that everything is running smoothly: + +```bash +# Check the status of all pods +kubectl get pods + +# Check the status of services +kubectl get svc + +# Check the status of deployments +kubectl get deployments + +# View all resources +kubectl get all +``` + +### clean all + +`kubectl delete -f ./` + +### Notes: + +- If you use a specific namespace for your deployment, be sure to append the -n flag to your kubectl commands. diff --git a/deployments/deploy/README.md b/deployments/deploy/README.md deleted file mode 100644 index d6b083bc5..000000000 --- a/deployments/deploy/README.md +++ /dev/null @@ -1,85 +0,0 @@ -# Kubernetes Deployment - -## Resource Requests - -- CPU: 2 cores -- Memory: 4 GiB -- Disk usage: 20 GiB (on Node) - -## Origin Deploy - -1. Enter the target dir - `cd ./deployments/deploy/` - -2. Deploy configs and dependencies - Upate your `openim-config.yml` - -Apply all config and dependencies -`kubectl apply -f ./openim-config.yml` - -> Attation: If you use `default` namespace, you can excute `clusterRile.yml` to create a cluster role binding for default service account. -> -> Namespace is modify to `discovery.yml` in `openim-config.yml`, you can change `kubernetes.namespace` to your namespace. - -Excute `clusterRole.yml` -`kubectl apply -f ./clusterRole.yml` - -Run infrasturcture components. - -`kubectl apply -f minio-service.yml -f minio-statefulset.yml -f mongo-service.yml -f mongo-statefulset.yml -f redis-service.yml -f redis-statefulset.yml -f kafka-service.yml -f kafka-statefulset.yml` - -> Note: Ensure that infrastructure services like MinIO, Redis, and Kafka are running before deploying the main applications. - -3. run all deployments and services - -```bash -kubectl apply \ - -f openim-api-deployment.yml \ - -f openim-api-service.yml \ - -f openim-crontask-deployment.yml \ - -f openim-rpc-user-deployment.yml \ - -f openim-rpc-user-service.yml \ - -f openim-msggateway-deployment.yml \ - -f openim-msggateway-service.yml \ - -f openim-push-deployment.yml \ - -f openim-push-service.yml \ - -f openim-msgtransfer-service.yml \ - -f openim-msgtransfer-deployment.yml \ - -f openim-rpc-conversation-deployment.yml \ - -f openim-rpc-conversation-service.yml \ - -f openim-rpc-auth-deployment.yml \ - -f openim-rpc-auth-service.yml \ - -f openim-rpc-group-deployment.yml \ - -f openim-rpc-group-service.yml \ - -f openim-rpc-friend-deployment.yml \ - -f openim-rpc-friend-service.yml \ - -f openim-rpc-msg-deployment.yml \ - -f openim-rpc-msg-service.yml \ - -f openim-rpc-third-deployment.yml \ - -f openim-rpc-third-service.yml -``` - -4. Verification - After deploying the services, verify that everything is running smoothly: - -```bash -# Check the status of all pods -kubectl get pods - -# Check the status of services -kubectl get svc - -# Check the status of deployments -kubectl get deployments - -# View all resources -kubectl get all -``` - -5. clean all - -`kubectl delete -f ./` - -### Notes: - -- If you use a specific namespace for your deployment, be sure to append the -n flag to your kubectl commands. diff --git a/deployments/deploy/kafka-secret.yml b/deployments/deploy/kafka-secret.yml new file mode 100644 index 000000000..dcee689c8 --- /dev/null +++ b/deployments/deploy/kafka-secret.yml @@ -0,0 +1,7 @@ +apiVersion: v1 +kind: Secret +metadata: + name: openim-kafka-secret +type: Opaque +data: + kafka-password: "" diff --git a/deployments/deploy/minio-secret.yml b/deployments/deploy/minio-secret.yml new file mode 100644 index 000000000..3ea09a19f --- /dev/null +++ b/deployments/deploy/minio-secret.yml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: Secret +metadata: + name: openim-minio-secret +type: Opaque +data: + minio-root-user: cm9vdA== # Base64 encoded "root" + minio-root-password: b3BlbklNMTIz # Base64 encoded "openIM123" diff --git a/deployments/deploy/minio-statefulset.yml b/deployments/deploy/minio-statefulset.yml index c8806ff12..9cf0a42d0 100644 --- a/deployments/deploy/minio-statefulset.yml +++ b/deployments/deploy/minio-statefulset.yml @@ -31,12 +31,12 @@ spec: - name: MINIO_ROOT_USER valueFrom: secretKeyRef: - name: minio-secret + name: openim-minio-secret key: minio-root-user - name: MINIO_ROOT_PASSWORD valueFrom: secretKeyRef: - name: minio-secret + name: openim-minio-secret key: minio-root-password command: - "/bin/sh" @@ -76,12 +76,4 @@ spec: requests: storage: 2Gi ---- -apiVersion: v1 -kind: Secret -metadata: - name: minio-secret -type: Opaque -data: - minio-root-user: cm9vdA== # Base64 encoded "root" - minio-root-password: b3BlbklNMTIz # Base64 encoded "openIM123" + diff --git a/deployments/deploy/mongo-secret.yml b/deployments/deploy/mongo-secret.yml new file mode 100644 index 000000000..c3c10af24 --- /dev/null +++ b/deployments/deploy/mongo-secret.yml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: Secret +metadata: + name: openim-mongo-secret +type: Opaque +data: + mongo_openim_username: b3BlbklN # base64 for "openIM", this user credentials need in authSource database. + mongo_openim_password: b3BlbklNMTIz # base64 for "openIM123" diff --git a/deployments/deploy/mongo-statefulset.yml b/deployments/deploy/mongo-statefulset.yml index e8510fdf7..41cd4cb7f 100644 --- a/deployments/deploy/mongo-statefulset.yml +++ b/deployments/deploy/mongo-statefulset.yml @@ -47,27 +47,27 @@ spec: - name: MONGO_INITDB_ROOT_USERNAME valueFrom: secretKeyRef: - name: mongo-secret + name: openim-mongo-init-secret key: mongo_initdb_root_username - name: MONGO_INITDB_ROOT_PASSWORD valueFrom: secretKeyRef: - name: mongo-secret + name: openim-mongo-init-secret key: mongo_initdb_root_password - name: MONGO_INITDB_DATABASE valueFrom: secretKeyRef: - name: mongo-secret + name: openim-mongo-init-secret key: mongo_initdb_database - name: MONGO_OPENIM_USERNAME valueFrom: secretKeyRef: - name: mongo-secret + name: openim-mongo-init-secret key: mongo_openim_username - name: MONGO_OPENIM_PASSWORD valueFrom: secretKeyRef: - name: mongo-secret + name: openim-mongo-init-secret key: mongo_openim_password - name: TZ value: "Asia/Shanghai" @@ -93,3 +93,16 @@ spec: resources: requests: storage: 5Gi + +--- +apiVersion: v1 +kind: Secret +metadata: + name: openim-mongo-init-secret +type: Opaque +data: + mongo_initdb_root_username: cm9vdA== # base64 for "root" + mongo_initdb_root_password: b3BlbklNMTIz # base64 for "openIM123" + mongo_initdb_database: b3BlbmltX3Yz # base64 for "openim_v3" + mongo_openim_username: b3BlbklN # base64 for "openIM" + mongo_openim_password: b3BlbklNMTIz # base64 for "openIM123" diff --git a/deployments/deploy/openim-api-deployment.yml b/deployments/deploy/openim-api-deployment.yml index cb1a68075..d2d27dc0c 100644 --- a/deployments/deploy/openim-api-deployment.yml +++ b/deployments/deploy/openim-api-deployment.yml @@ -21,20 +21,18 @@ spec: - name: IMENV_REDIS_PASSWORD valueFrom: secretKeyRef: - name: redis-secret + name: openim-redis-secret key: redis-password - - - name: IMENV_MONGODB_PASSWORD - valueFrom: - secretKeyRef: - name: mongo-secret - key: mongo_openim_password - - name: IMENV_MONGODB_USERNAME valueFrom: secretKeyRef: - name: mongo-secret + name: openim-mongo-secret key: mongo_openim_username + - name: IMENV_MONGODB_PASSWORD + valueFrom: + secretKeyRef: + name: openim-mongo-secret + key: mongo_openim_password volumeMounts: - name: openim-config @@ -46,4 +44,4 @@ spec: volumes: - name: openim-config configMap: - name: openim-config \ No newline at end of file + name: openim-config diff --git a/deployments/deploy/openim-config.yml b/deployments/deploy/openim-config.yml index d0651bdea..105dd98e3 100644 --- a/deployments/deploy/openim-config.yml +++ b/deployments/deploy/openim-config.yml @@ -4,7 +4,7 @@ metadata: name: openim-config data: discovery.yml: | - enable: "kubernetes" + enable: "kubernetes" # "kubernetes" or "etcd" kubernetes: namespace: default etcd: @@ -26,7 +26,6 @@ data: log.yml: | # Log storage path, default is acceptable, change to a full path if modification is needed - # storageLocation: ../../../../logs/ storageLocation: ./logs/ # Log rotation period (in hours), default is acceptable rotationTime: 24 @@ -49,9 +48,9 @@ data: # Name of the database database: openim_v3 # Username for database authentication - username: openIM + username: '' # openIM # Password for database authentication - password: openIM123 + password: '' # openIM123 # Authentication source for database authentication, if use root user, set it to admin authSource: openim_v3 # Maximum number of connections in the connection pool @@ -1055,16 +1054,3 @@ data: - targets: [ internal_ip:12320 ] labels: namespace: default - ---- -apiVersion: v1 -kind: Secret -metadata: - name: mongo-secret -type: Opaque -data: - mongo_initdb_root_username: cm9vdA== # base64 for "root" - mongo_initdb_root_password: b3BlbklNMTIz # base64 for "openIM123" - mongo_initdb_database: b3BlbmltX3Yz # base64 for "openim_v3" - mongo_openim_username: b3BlbklN # base64 for "openIM" - mongo_openim_password: b3BlbklNMTIz # base64 for "openIM123" \ No newline at end of file diff --git a/deployments/deploy/openim-msggateway-deployment.yml b/deployments/deploy/openim-msggateway-deployment.yml index ba2b1b84e..b1a142e23 100644 --- a/deployments/deploy/openim-msggateway-deployment.yml +++ b/deployments/deploy/openim-msggateway-deployment.yml @@ -21,7 +21,7 @@ spec: - name: IMENV_REDIS_PASSWORD valueFrom: secretKeyRef: - name: redis-secret + name: openim-redis-secret key: redis-password volumeMounts: - name: openim-config diff --git a/deployments/deploy/openim-msgtransfer-deployment.yml b/deployments/deploy/openim-msgtransfer-deployment.yml index be608845c..323ed5660 100644 --- a/deployments/deploy/openim-msgtransfer-deployment.yml +++ b/deployments/deploy/openim-msgtransfer-deployment.yml @@ -21,13 +21,23 @@ spec: - name: IMENV_REDIS_PASSWORD valueFrom: secretKeyRef: - name: redis-secret + name: openim-redis-secret key: redis-password + - name: IMENV_MONGODB_USERNAME + valueFrom: + secretKeyRef: + name: openim-mongo-secret + key: mongo_openim_username - name: IMENV_MONGODB_PASSWORD valueFrom: secretKeyRef: - name: mongo-secret + name: openim-mongo-secret key: mongo_openim_password + - name: IMENV_KAFKA_PASSWORD + valueFrom: + secretKeyRef: + name: openim-kafka-secret + key: kafka-password volumeMounts: - name: openim-config mountPath: "/config" diff --git a/deployments/deploy/openim-push-deployment.yml b/deployments/deploy/openim-push-deployment.yml index 2092b343c..bb36170e9 100644 --- a/deployments/deploy/openim-push-deployment.yml +++ b/deployments/deploy/openim-push-deployment.yml @@ -21,8 +21,13 @@ spec: - name: IMENV_REDIS_PASSWORD valueFrom: secretKeyRef: - name: redis-secret + name: openim-redis-secret key: redis-password + - name: IMENV_KAFKA_PASSWORD + valueFrom: + secretKeyRef: + name: openim-kafka-secret + key: kafka-password volumeMounts: - name: openim-config mountPath: "/config" diff --git a/deployments/deploy/openim-rpc-auth-deployment.yml b/deployments/deploy/openim-rpc-auth-deployment.yml index b785ea92f..a15c901f5 100644 --- a/deployments/deploy/openim-rpc-auth-deployment.yml +++ b/deployments/deploy/openim-rpc-auth-deployment.yml @@ -22,7 +22,7 @@ spec: - name: IMENV_REDIS_PASSWORD valueFrom: secretKeyRef: - name: redis-secret + name: openim-redis-secret key: redis-password volumeMounts: - name: openim-config diff --git a/deployments/deploy/openim-rpc-conversation-deployment.yml b/deployments/deploy/openim-rpc-conversation-deployment.yml index 4d7a32497..2c9bde337 100644 --- a/deployments/deploy/openim-rpc-conversation-deployment.yml +++ b/deployments/deploy/openim-rpc-conversation-deployment.yml @@ -21,12 +21,17 @@ spec: - name: IMENV_REDIS_PASSWORD valueFrom: secretKeyRef: - name: redis-secret + name: openim-redis-secret key: redis-password + - name: IMENV_MONGODB_USERNAME + valueFrom: + secretKeyRef: + name: openim-mongo-secret + key: mongo_openim_username - name: IMENV_MONGODB_PASSWORD valueFrom: secretKeyRef: - name: mongo-secret + name: openim-mongo-secret key: mongo_openim_password volumeMounts: - name: openim-config diff --git a/deployments/deploy/openim-rpc-friend-deployment.yml b/deployments/deploy/openim-rpc-friend-deployment.yml index 5fdd3bf62..e01238888 100644 --- a/deployments/deploy/openim-rpc-friend-deployment.yml +++ b/deployments/deploy/openim-rpc-friend-deployment.yml @@ -14,19 +14,24 @@ spec: spec: containers: - name: friend-rpc-server-container - image: openim/openim-rpc-friend:v3.8.3 + image: openim/openim-rpc-friend:v3.8.3 env: - name: CONFIG_PATH value: "/config" - name: IMENV_REDIS_PASSWORD valueFrom: secretKeyRef: - name: redis-secret + name: openim-redis-secret key: redis-password + - name: IMENV_MONGODB_USERNAME + valueFrom: + secretKeyRef: + name: openim-mongo-secret + key: mongo_openim_username - name: IMENV_MONGODB_PASSWORD valueFrom: secretKeyRef: - name: mongo-secret + name: openim-mongo-secret key: mongo_openim_password volumeMounts: - name: openim-config @@ -38,4 +43,4 @@ spec: volumes: - name: openim-config configMap: - name: openim-config \ No newline at end of file + name: openim-config diff --git a/deployments/deploy/openim-rpc-group-deployment.yml b/deployments/deploy/openim-rpc-group-deployment.yml index 313fec897..4698d60b5 100644 --- a/deployments/deploy/openim-rpc-group-deployment.yml +++ b/deployments/deploy/openim-rpc-group-deployment.yml @@ -15,19 +15,23 @@ spec: containers: - name: group-rpc-server-container image: openim/openim-rpc-group:v3.8.3 - env: - name: CONFIG_PATH value: "/config" - name: IMENV_REDIS_PASSWORD valueFrom: secretKeyRef: - name: redis-secret + name: openim-redis-secret key: redis-password + - name: IMENV_MONGODB_USERNAME + valueFrom: + secretKeyRef: + name: openim-mongo-secret + key: mongo_openim_username - name: IMENV_MONGODB_PASSWORD valueFrom: secretKeyRef: - name: mongo-secret + name: openim-mongo-secret key: mongo_openim_password volumeMounts: - name: openim-config @@ -39,4 +43,4 @@ spec: volumes: - name: openim-config configMap: - name: openim-config \ No newline at end of file + name: openim-config diff --git a/deployments/deploy/openim-rpc-msg-deployment.yml b/deployments/deploy/openim-rpc-msg-deployment.yml index e883f5849..26a833342 100644 --- a/deployments/deploy/openim-rpc-msg-deployment.yml +++ b/deployments/deploy/openim-rpc-msg-deployment.yml @@ -14,20 +14,30 @@ spec: spec: containers: - name: msg-rpc-server-container - image: openim/openim-rpc-msg:v3.8.3 + image: openim/openim-rpc-msg:v3.8.3 env: - name: CONFIG_PATH value: "/config" - name: IMENV_REDIS_PASSWORD valueFrom: secretKeyRef: - name: redis-secret + name: openim-redis-secret key: redis-password + - name: IMENV_MONGODB_USERNAME + valueFrom: + secretKeyRef: + name: openim-mongo-secret + key: mongo_openim_username - name: IMENV_MONGODB_PASSWORD valueFrom: secretKeyRef: - name: mongo-secret + name: openim-mongo-secret key: mongo_openim_password + - name: IMENV_KAFKA_PASSWORD + valueFrom: + secretKeyRef: + name: openim-kafka-secret + key: kafka-password volumeMounts: - name: openim-config mountPath: "/config" @@ -38,4 +48,4 @@ spec: volumes: - name: openim-config configMap: - name: openim-config \ No newline at end of file + name: openim-config diff --git a/deployments/deploy/openim-rpc-third-deployment.yml b/deployments/deploy/openim-rpc-third-deployment.yml index 326aaee03..f6919f510 100644 --- a/deployments/deploy/openim-rpc-third-deployment.yml +++ b/deployments/deploy/openim-rpc-third-deployment.yml @@ -14,29 +14,34 @@ spec: spec: containers: - name: third-rpc-server-container - image: openim/openim-rpc-third:v3.8.3 + image: openim/openim-rpc-third:v3.8.3 env: - name: CONFIG_PATH value: "/config" - name: IMENV_MINIO_ACCESSKEYID valueFrom: secretKeyRef: - name: minio-secret + name: openim-minio-secret key: minio-root-user - name: IMENV_MINIO_SECRETACCESSKEY valueFrom: secretKeyRef: - name: minio-secret + name: openim-minio-secret key: minio-root-password - name: IMENV_REDIS_PASSWORD valueFrom: secretKeyRef: - name: redis-secret + name: openim-redis-secret key: redis-password + - name: IMENV_MONGODB_USERNAME + valueFrom: + secretKeyRef: + name: openim-mongo-secret + key: mongo_openim_username - name: IMENV_MONGODB_PASSWORD valueFrom: secretKeyRef: - name: mongo-secret + name: openim-mongo-secret key: mongo_openim_password volumeMounts: - name: openim-config diff --git a/deployments/deploy/openim-rpc-user-deployment.yml b/deployments/deploy/openim-rpc-user-deployment.yml index c6a47e125..c3e36d1be 100644 --- a/deployments/deploy/openim-rpc-user-deployment.yml +++ b/deployments/deploy/openim-rpc-user-deployment.yml @@ -21,13 +21,23 @@ spec: - name: IMENV_REDIS_PASSWORD valueFrom: secretKeyRef: - name: redis-secret + name: openim-redis-secret key: redis-password + - name: IMENV_MONGODB_USERNAME + valueFrom: + secretKeyRef: + name: openim-mongo-secret + key: mongo_openim_username - name: IMENV_MONGODB_PASSWORD valueFrom: secretKeyRef: - name: mongo-secret + name: openim-mongo-secret key: mongo_openim_password + - name: IMENV_KAFKA_PASSWORD + valueFrom: + secretKeyRef: + name: openim-kafka-secret + key: kafka-password volumeMounts: - name: openim-config mountPath: "/config" diff --git a/deployments/deploy/redis-secret.yml b/deployments/deploy/redis-secret.yml new file mode 100644 index 000000000..463ec9545 --- /dev/null +++ b/deployments/deploy/redis-secret.yml @@ -0,0 +1,7 @@ +apiVersion: v1 +kind: Secret +metadata: + name: openim-redis-secret +type: Opaque +data: + redis-password: b3BlbklNMTIz # "openIM123" in base64 diff --git a/deployments/deploy/redis-statefulset.yml b/deployments/deploy/redis-statefulset.yml index 3f6dc41bc..5668b20cc 100644 --- a/deployments/deploy/redis-statefulset.yml +++ b/deployments/deploy/redis-statefulset.yml @@ -29,9 +29,6 @@ spec: volumeMounts: - name: redis-data mountPath: /data - # - name: redis-config-volume - # mountPath: /usr/local/redis/config/redis.conf - # subPath: redis.conf command: [ "/bin/sh", @@ -56,11 +53,3 @@ spec: resources: requests: storage: 5Gi ---- -apiVersion: v1 -kind: Secret -metadata: - name: redis-secret -type: Opaque -data: - redis-password: b3BlbklNMTIz # "openIM123" in base64 From dec423eeb31ad789d92e87390ea7e3be9a8c04a3 Mon Sep 17 00:00:00 2001 From: OpenIM-Gordon <1432970085@qq.com> Date: Thu, 26 Dec 2024 17:53:54 +0800 Subject: [PATCH 090/199] fix: The message @ information will be set only for members in the group. (#3009) --- go.mod | 2 +- go.sum | 4 ++-- internal/rpc/msg/send.go | 20 +++++++++++++------- 3 files changed, 16 insertions(+), 10 deletions(-) diff --git a/go.mod b/go.mod index 05fe6db15..3cbbff3c9 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/mitchellh/mapstructure v1.5.0 github.com/openimsdk/protocol v0.0.72-alpha.68 - github.com/openimsdk/tools v0.0.50-alpha.61 + github.com/openimsdk/tools v0.0.50-alpha.62 github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 github.com/stretchr/testify v1.9.0 diff --git a/go.sum b/go.sum index 1fc3c33db..846ef721d 100644 --- a/go.sum +++ b/go.sum @@ -349,8 +349,8 @@ github.com/openimsdk/gomake v0.0.15-alpha.2 h1:5Q8yl8ezy2yx+q8/ucU/t4kJnDfCzNOrk github.com/openimsdk/gomake v0.0.15-alpha.2/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= github.com/openimsdk/protocol v0.0.72-alpha.68 h1:Ekn6S9Ftt12Xs/p9kJ39RDr2gSwIczz+MmSHQE4lAek= github.com/openimsdk/protocol v0.0.72-alpha.68/go.mod h1:Iet+piS/jaS+kWWyj6EEr36mk4ISzIRYjoMSVA4dq2M= -github.com/openimsdk/tools v0.0.50-alpha.61 h1:zKEZwrj+fUVuyC6KR3kZp9zFaCCIFgoSbHO0r0mZ6h4= -github.com/openimsdk/tools v0.0.50-alpha.61/go.mod h1:JowL2jYr8tu4vcQe+5hJh4v3BtSx1T0CIS3pgU/Mw+U= +github.com/openimsdk/tools v0.0.50-alpha.62 h1:e/m1XL7+EXbkOoxr/En/612WcOPKOUHPBj0++gG6MuQ= +github.com/openimsdk/tools v0.0.50-alpha.62/go.mod h1:JowL2jYr8tu4vcQe+5hJh4v3BtSx1T0CIS3pgU/Mw+U= github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= diff --git a/internal/rpc/msg/send.go b/internal/rpc/msg/send.go index f01134f8f..19f4e9ffd 100644 --- a/internal/rpc/msg/send.go +++ b/internal/rpc/msg/send.go @@ -101,14 +101,14 @@ func (m *msgServer) setConversationAtInfo(nctx context.Context, msg *sdkws.MsgDa ConversationType: msg.SessionType, GroupID: msg.GroupID, } + memberUserIDList, err := m.GroupLocalCache.GetGroupMemberIDs(ctx, msg.GroupID) + if err != nil { + log.ZWarn(ctx, "GetGroupMemberIDs", err) + return + } tagAll := datautil.Contain(constant.AtAllString, msg.AtUserIDList...) if tagAll { - memberUserIDList, err := m.GroupLocalCache.GetGroupMemberIDs(ctx, msg.GroupID) - if err != nil { - log.ZWarn(ctx, "GetGroupMemberIDs", err) - return - } memberUserIDList = datautil.DeleteElems(memberUserIDList, msg.SendID) @@ -118,6 +118,9 @@ func (m *msgServer) setConversationAtInfo(nctx context.Context, msg *sdkws.MsgDa conversation.GroupAtType = &wrapperspb.Int32Value{Value: constant.AtAll} } else { // @Everyone and @other people conversation.GroupAtType = &wrapperspb.Int32Value{Value: constant.AtAllAtMe} + atUserID = datautil.SliceIntersectFuncs(atUserID, memberUserIDList, func(a string) string { return a }, func(b string) string { + return b + }) if err := m.conversationClient.SetConversations(ctx, atUserID, conversation); err != nil { log.ZWarn(ctx, "SetConversations", err, "userID", atUserID, "conversation", conversation) } @@ -131,10 +134,13 @@ func (m *msgServer) setConversationAtInfo(nctx context.Context, msg *sdkws.MsgDa return } + atUserID = datautil.SliceIntersectFuncs(msg.AtUserIDList, memberUserIDList, func(a string) string { return a }, func(b string) string { + return b + }) conversation.GroupAtType = &wrapperspb.Int32Value{Value: constant.AtMe} - if err := m.conversationClient.SetConversations(ctx, msg.AtUserIDList, conversation); err != nil { - log.ZWarn(ctx, "SetConversations", err, msg.AtUserIDList, conversation) + if err := m.conversationClient.SetConversations(ctx, atUserID, conversation); err != nil { + log.ZWarn(ctx, "SetConversations", err, atUserID, conversation) } } From 1af31847fa1119e513fe68d611762bee875c07a4 Mon Sep 17 00:00:00 2001 From: icey-yu <119291641+icey-yu@users.noreply.github.com> Date: Fri, 27 Dec 2024 16:23:12 +0800 Subject: [PATCH 091/199] fix: restart permission check (#3011) --- internal/api/router.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/api/router.go b/internal/api/router.go index 789b8205d..5c4fa2137 100644 --- a/internal/api/router.go +++ b/internal/api/router.go @@ -314,7 +314,7 @@ func newGinRouter(ctx context.Context, client discovery.SvcDiscoveryRegistry, cf configGroup.POST("/reset_config", cm.ResetConfig) } { - r.POST("/restart", cm.Restart) + r.POST("/restart", cm.CheckAdmin, cm.Restart) } return r, nil } From 930a9fd7d58830902f0ead3da169011cc7344edc Mon Sep 17 00:00:00 2001 From: icey-yu <119291641+icey-yu@users.noreply.github.com> Date: Fri, 27 Dec 2024 16:23:26 +0800 Subject: [PATCH 092/199] fix: The system cannot be restarted the first time the configuration is set. (#3013) --- pkg/common/cmd/root.go | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/pkg/common/cmd/root.go b/pkg/common/cmd/root.go index 20db4126e..4c5256d80 100644 --- a/pkg/common/cmd/root.go +++ b/pkg/common/cmd/root.go @@ -143,9 +143,22 @@ func (r *RootCmd) updateConfigFromEtcd(opts *CmdOpts) error { } update := func(configFileName string, configStruct any) error { + ctx := context.TODO() key := disetcd.BuildKey(configFileName) - etcdRes, err := r.etcdClient.Get(context.TODO(), key) - if err != nil || etcdRes.Count == 0 { + etcdRes, err := r.etcdClient.Get(ctx, key) + if err != nil { + log.ZWarn(ctx, "root cmd updateConfigFromEtcd, etcd Get err: %v", errs.Wrap(err)) + return nil + } + if etcdRes.Count == 0 { + data, err := json.Marshal(configStruct) + if err != nil { + return errs.ErrArgs.WithDetail(err.Error()).Wrap() + } + _, err = r.etcdClient.Put(ctx, disetcd.BuildKey(configFileName), string(data)) + if err != nil { + log.ZWarn(ctx, "root cmd updateConfigFromEtcd, etcd Put err: %v", errs.Wrap(err)) + } return nil } err = json.Unmarshal(etcdRes.Kvs[0].Value, configStruct) From 36c6b41621cf00f338eb33c79211aecfe55ca85c Mon Sep 17 00:00:00 2001 From: chao <48119764+withchao@users.noreply.github.com> Date: Fri, 27 Dec 2024 17:39:23 +0800 Subject: [PATCH 093/199] fix: jssdk not init (#3016) * pb * fix: Modifying other fields while setting IsPrivateChat does not take effect * fix: quote message error revoke * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * upgrading pkg tools * fix --- internal/api/jssdk/jssdk.go | 21 ++++++++++++------- internal/api/router.go | 6 ++++-- .../msgtransfer/online_history_msg_handler.go | 2 +- 3 files changed, 19 insertions(+), 10 deletions(-) diff --git a/internal/api/jssdk/jssdk.go b/internal/api/jssdk/jssdk.go index 036cb027a..3c0911207 100644 --- a/internal/api/jssdk/jssdk.go +++ b/internal/api/jssdk/jssdk.go @@ -20,16 +20,23 @@ const ( defaultGetActiveConversation = 100 ) -func NewJSSdkApi() *JSSdk { - return &JSSdk{} +func NewJSSdkApi(userClient *rpcli.UserClient, relationClient *rpcli.RelationClient, groupClient *rpcli.GroupClient, + conversationClient *rpcli.ConversationClient, msgClient *rpcli.MsgClient) *JSSdk { + return &JSSdk{ + userClient: userClient, + relationClient: relationClient, + groupClient: groupClient, + conversationClient: conversationClient, + msgClient: msgClient, + } } type JSSdk struct { - userClient rpcli.UserClient - relationClient rpcli.RelationClient - groupClient rpcli.GroupClient - conversationClient rpcli.ConversationClient - msgClient rpcli.MsgClient + userClient *rpcli.UserClient + relationClient *rpcli.RelationClient + groupClient *rpcli.GroupClient + conversationClient *rpcli.ConversationClient + msgClient *rpcli.MsgClient } func (x *JSSdk) GetActiveConversations(c *gin.Context) { diff --git a/internal/api/router.go b/internal/api/router.go index 5c4fa2137..e516d8ca8 100644 --- a/internal/api/router.go +++ b/internal/api/router.go @@ -100,8 +100,8 @@ func newGinRouter(ctx context.Context, client discovery.SvcDiscoveryRegistry, cf case BestSpeed: r.Use(gzip.Gzip(gzip.BestSpeed)) } - r.Use(prommetricsGin(), gin.RecoveryWithWriter(gin.DefaultErrorWriter, mw.GinPanicErr), mw.CorsHandler(), mw.GinParseOperationID(), GinParseToken(rpcli.NewAuthClient(authConn))) - j := jssdk.NewJSSdkApi() + r.Use(prommetricsGin(), gin.RecoveryWithWriter(gin.DefaultErrorWriter, mw.GinPanicErr), mw.CorsHandler(), + mw.GinParseOperationID(), GinParseToken(rpcli.NewAuthClient(authConn))) u := NewUserApi(user.NewUserClient(userConn), client, cfg.Discovery.RpcService) { @@ -280,6 +280,8 @@ func newGinRouter(ctx context.Context, client discovery.SvcDiscoveryRegistry, cf } { + j := jssdk.NewJSSdkApi(rpcli.NewUserClient(userConn), rpcli.NewRelationClient(friendConn), + rpcli.NewGroupClient(groupConn), rpcli.NewConversationClient(conversationConn), rpcli.NewMsgClient(msgConn)) jssdk := r.Group("/jssdk") jssdk.POST("/get_conversations", j.GetConversations) jssdk.POST("/get_active_conversations", j.GetActiveConversations) diff --git a/internal/msgtransfer/online_history_msg_handler.go b/internal/msgtransfer/online_history_msg_handler.go index 7b989070b..6334c95fd 100644 --- a/internal/msgtransfer/online_history_msg_handler.go +++ b/internal/msgtransfer/online_history_msg_handler.go @@ -116,7 +116,7 @@ func NewOnlineHistoryRedisConsumerHandler(ctx context.Context, client discovery. och.redisMessageBatches = b och.historyConsumerGroup = historyConsumerGroup - return &och, err + return &och, nil } func (och *OnlineHistoryRedisConsumerHandler) do(ctx context.Context, channelID int, val *batcher.Msg[sarama.ConsumerMessage]) { ctx = mcontext.WithTriggerIDContext(ctx, val.TriggerID()) From d73652119c4eaa69849821e0dd6fa5115c24a642 Mon Sep 17 00:00:00 2001 From: Monet Lee Date: Sun, 29 Dec 2024 12:50:51 +0800 Subject: [PATCH 094/199] build: fix uncorrect path. (#3020) --- .../build-and-release-services-images.yml | 91 ------------------- ...ker-build-and-release-services-images.yml} | 10 +- 2 files changed, 5 insertions(+), 96 deletions(-) delete mode 100644 .github/workflows/build-and-release-services-images.yml rename .github/workflows/{docker-build-and-release.yml => docker-build-and-release-services-images.yml} (97%) diff --git a/.github/workflows/build-and-release-services-images.yml b/.github/workflows/build-and-release-services-images.yml deleted file mode 100644 index d53fd2142..000000000 --- a/.github/workflows/build-and-release-services-images.yml +++ /dev/null @@ -1,91 +0,0 @@ -name: Build and release services Images - -on: - push: - branches: - - release-* - release: - types: [published] - workflow_dispatch: - inputs: - tag: - description: "Tag version to be used for Docker image" - required: true - default: "v3.8.3" - -jobs: - build-and-push: - runs-on: ubuntu-latest - - steps: - - name: Checkout repository - uses: actions/checkout@v3 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - - - name: Log in to Docker Hub - uses: docker/login-action@v2 - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} - - - name: Log in to GitHub Container Registry - uses: docker/login-action@v2 - with: - registry: ghcr.io - username: ${{ github.repository_owner }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Log in to Aliyun Container Registry - uses: docker/login-action@v2 - with: - registry: registry.cn-hangzhou.aliyuncs.com - username: ${{ secrets.ALIREGISTRY_USERNAME }} - password: ${{ secrets.ALIREGISTRY_TOKEN }} - - - name: Extract metadata for Docker (tags, labels) - id: meta - uses: docker/metadata-action@v5 - with: - tags: | - type=ref,event=tag - type=schedule - type=ref,event=branch - type=semver,pattern={{version}} - type=semver,pattern=v{{version}} - # type=semver,pattern={{major}}.{{minor}} - type=semver,pattern=release-{{raw}} - type=sha - type=raw,value=${{ github.event.inputs.tag }} - - - name: Build and push Docker images - run: | - ROOT_DIR="build/images" - for dir in "$ROOT_DIR"/*/; do - # Find Dockerfile or *.dockerfile in a case-insensitive manner - dockerfile=$(find "$dir" -maxdepth 1 -type f \( -iname 'dockerfile' -o -iname '*.dockerfile' \) | head -n 1) - - if [ -n "$dockerfile" ] && [ -f "$dockerfile" ]; then - IMAGE_NAME=$(basename "$dir") - echo "Building Docker image for $IMAGE_NAME with tags:" - - # Initialize tag arguments - tag_args=() - - # Read each tag and append --tag arguments - while IFS= read -r tag; do - tag_args+=(--tag "${{ secrets.DOCKER_USERNAME }}/$IMAGE_NAME:$tag") - tag_args+=(--tag "ghcr.io/${{ github.repository_owner }}/$IMAGE_NAME:$tag") - tag_args+=(--tag "registry.cn-hangzhou.aliyuncs.com/openimsdk/$IMAGE_NAME:$tag") - done <<< "${{ steps.meta.outputs.tags }}" - - # Build and push the Docker image with all tags - docker buildx build --platform linux/amd64,linux/arm64 \ - --file "$dockerfile" \ - "${tag_args[@]}" \ - --push "$dir" - else - echo "No valid Dockerfile found in $dir" - fi - done \ No newline at end of file diff --git a/.github/workflows/docker-build-and-release.yml b/.github/workflows/docker-build-and-release-services-images.yml similarity index 97% rename from .github/workflows/docker-build-and-release.yml rename to .github/workflows/docker-build-and-release-services-images.yml index 76602c4d1..a79fab978 100644 --- a/.github/workflows/docker-build-and-release.yml +++ b/.github/workflows/docker-build-and-release-services-images.yml @@ -61,8 +61,8 @@ jobs: - name: Build and push Docker images run: | - ROOT_DIR="build/images" - for dir in "$ROOT_DIR"/*/; do + IMG_DIR="build/images" + for dir in "$IMG_DIR"/*/; do # Find Dockerfile or *.dockerfile in a case-insensitive manner dockerfile=$(find "$dir" -maxdepth 1 -type f \( -iname 'dockerfile' -o -iname '*.dockerfile' \) | head -n 1) @@ -72,14 +72,14 @@ jobs: # Initialize tag arguments tag_args=() - + # Read each tag and append --tag arguments while IFS= read -r tag; do tag_args+=(--tag "${{ secrets.DOCKER_USERNAME }}/$IMAGE_NAME:$tag") tag_args+=(--tag "ghcr.io/${{ github.repository_owner }}/$IMAGE_NAME:$tag") tag_args+=(--tag "registry.cn-hangzhou.aliyuncs.com/openimsdk/$IMAGE_NAME:$tag") done <<< "${{ steps.meta.outputs.tags }}" - + # Build and push the Docker image with all tags docker buildx build --platform linux/amd64,linux/arm64 \ --file "$dockerfile" \ @@ -88,4 +88,4 @@ jobs: else echo "No valid Dockerfile found in $dir" fi - done \ No newline at end of file + done From 435d8bb8bf69c59d4ab482b0050b57b7124e0475 Mon Sep 17 00:00:00 2001 From: chao <48119764+withchao@users.noreply.github.com> Date: Sun, 29 Dec 2024 14:27:35 +0800 Subject: [PATCH 095/199] fix: online status error (#3022) * pb * fix: Modifying other fields while setting IsPrivateChat does not take effect * fix: quote message error revoke * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * upgrading pkg tools * fix * fix --- go.mod | 4 ++-- go.sum | 4 ++-- pkg/rpccache/online.go | 47 +++--------------------------------------- pkg/rpcli/group.go | 24 +++++++++++++++++++++ 4 files changed, 31 insertions(+), 48 deletions(-) diff --git a/go.mod b/go.mod index 3cbbff3c9..387cb7304 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/gorilla/websocket v1.5.1 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/mitchellh/mapstructure v1.5.0 - github.com/openimsdk/protocol v0.0.72-alpha.68 + github.com/openimsdk/protocol v0.0.72-alpha.69 github.com/openimsdk/tools v0.0.50-alpha.62 github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 @@ -42,7 +42,6 @@ require ( github.com/spf13/viper v1.18.2 go.etcd.io/etcd/client/v3 v3.5.13 go.uber.org/automaxprocs v1.5.3 - golang.org/x/exp v0.0.0-20230905200255-921286631fa9 golang.org/x/sync v0.8.0 k8s.io/api v0.31.2 k8s.io/apimachinery v0.31.2 @@ -187,6 +186,7 @@ require ( go.uber.org/atomic v1.9.0 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/arch v0.7.0 // indirect + golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect golang.org/x/image v0.15.0 // indirect golang.org/x/net v0.29.0 // indirect golang.org/x/oauth2 v0.23.0 // indirect diff --git a/go.sum b/go.sum index 846ef721d..d207325e9 100644 --- a/go.sum +++ b/go.sum @@ -347,8 +347,8 @@ github.com/onsi/gomega v1.25.0 h1:Vw7br2PCDYijJHSfBOWhov+8cAnUf8MfMaIOV323l6Y= github.com/onsi/gomega v1.25.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdMPGhM= github.com/openimsdk/gomake v0.0.15-alpha.2 h1:5Q8yl8ezy2yx+q8/ucU/t4kJnDfCzNOrkXcDACCqtyM= github.com/openimsdk/gomake v0.0.15-alpha.2/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= -github.com/openimsdk/protocol v0.0.72-alpha.68 h1:Ekn6S9Ftt12Xs/p9kJ39RDr2gSwIczz+MmSHQE4lAek= -github.com/openimsdk/protocol v0.0.72-alpha.68/go.mod h1:Iet+piS/jaS+kWWyj6EEr36mk4ISzIRYjoMSVA4dq2M= +github.com/openimsdk/protocol v0.0.72-alpha.69 h1:b22oY2XTdBR/BePqA73KsrM3GDF3Vk8YcBEXZU4ArJc= +github.com/openimsdk/protocol v0.0.72-alpha.69/go.mod h1:Iet+piS/jaS+kWWyj6EEr36mk4ISzIRYjoMSVA4dq2M= github.com/openimsdk/tools v0.0.50-alpha.62 h1:e/m1XL7+EXbkOoxr/En/612WcOPKOUHPBj0++gG6MuQ= github.com/openimsdk/tools v0.0.50-alpha.62/go.mod h1:JowL2jYr8tu4vcQe+5hJh4v3BtSx1T0CIS3pgU/Mw+U= github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= diff --git a/pkg/rpccache/online.go b/pkg/rpccache/online.go index 8f5323477..32b2f2889 100644 --- a/pkg/rpccache/online.go +++ b/pkg/rpccache/online.go @@ -206,16 +206,6 @@ func (o *OnlineCache) GetUserOnlinePlatform(ctx context.Context, userID string) return platformIDs, nil } -// func (o *OnlineCache) GetUserOnlinePlatformBatch(ctx context.Context, userIDs []string) (map[string]int32, error) { -// platformIDs, err := o.getUserOnlinePlatform(ctx, userIDs) -// if err != nil { -// return nil, err -// } -// tmp := make([]int32, len(platformIDs)) -// copy(tmp, platformIDs) -// return platformIDs, nil -// } - func (o *OnlineCache) GetUserOnline(ctx context.Context, userID string) (bool, error) { platformIDs, err := o.getUserOnlinePlatform(ctx, userID) if err != nil { @@ -225,6 +215,9 @@ func (o *OnlineCache) GetUserOnline(ctx context.Context, userID string) (bool, e } func (o *OnlineCache) getUserOnlinePlatformBatch(ctx context.Context, userIDs []string) (map[string][]int32, error) { + if len(userIDs) == 0 { + return nil, nil + } platformIDsMap, err := o.lruCache.GetBatch(userIDs, func(missingUsers []string) (map[string][]int32, error) { platformIDsMap := make(map[string][]int32) usersStatus, err := o.client.GetUsersOnlinePlatform(ctx, missingUsers) @@ -281,40 +274,6 @@ func (o *OnlineCache) GetUsersOnline(ctx context.Context, userIDs []string) ([]s return userIDs, offlineUserIDs, nil } -//func (o *OnlineCache) GetUsersOnline(ctx context.Context, userIDs []string) ([]string, error) { -// onlineUserIDs := make([]string, 0, len(userIDs)) -// for _, userID := range userIDs { -// online, err := o.GetUserOnline(ctx, userID) -// if err != nil { -// return nil, err -// } -// if online { -// onlineUserIDs = append(onlineUserIDs, userID) -// } -// } -// log.ZDebug(ctx, "OnlineCache GetUsersOnline", "userIDs", userIDs, "onlineUserIDs", onlineUserIDs) -// return onlineUserIDs, nil -//} -// -//func (o *OnlineCache) GetGroupOnline(ctx context.Context, groupID string) ([]string, error) { -// userIDs, err := o.group.GetGroupMemberIDs(ctx, groupID) -// if err != nil { -// return nil, err -// } -// var onlineUserIDs []string -// for _, userID := range userIDs { -// online, err := o.GetUserOnline(ctx, userID) -// if err != nil { -// return nil, err -// } -// if online { -// onlineUserIDs = append(onlineUserIDs, userID) -// } -// } -// log.ZDebug(ctx, "OnlineCache GetGroupOnline", "groupID", groupID, "onlineUserIDs", onlineUserIDs, "allUserID", userIDs) -// return onlineUserIDs, nil -//} - func (o *OnlineCache) setUserOnline(userID string, platformIDs []int32) { switch o.fullUserCache { case true: diff --git a/pkg/rpcli/group.go b/pkg/rpcli/group.go index b51283c5d..623c5cc12 100644 --- a/pkg/rpcli/group.go +++ b/pkg/rpcli/group.go @@ -46,3 +46,27 @@ func (x *GroupClient) GetGroupMemberUserIDs(ctx context.Context, groupID string) req := &group.GetGroupMemberUserIDsReq{GroupID: groupID} return extractField(ctx, x.GroupClient.GetGroupMemberUserIDs, req, (*group.GetGroupMemberUserIDsResp).GetUserIDs) } + +func (x *GroupClient) GetGroupMembersInfo(ctx context.Context, groupID string, userIDs []string) ([]*sdkws.GroupMemberFullInfo, error) { + if len(userIDs) == 0 { + return nil, nil + } + req := &group.GetGroupMembersInfoReq{GroupID: groupID, UserIDs: userIDs} + return extractField(ctx, x.GroupClient.GetGroupMembersInfo, req, (*group.GetGroupMembersInfoResp).GetMembers) +} + +func (x *GroupClient) GetGroupMemberInfo(ctx context.Context, groupID string, userID string) (*sdkws.GroupMemberFullInfo, error) { + return firstValue(x.GetGroupMembersInfo(ctx, groupID, []string{userID})) +} + +func (x *GroupClient) GetGroupMemberMapInfo(ctx context.Context, groupID string, userIDs []string) (map[string]*sdkws.GroupMemberFullInfo, error) { + members, err := x.GetGroupMembersInfo(ctx, groupID, userIDs) + if err != nil { + return nil, err + } + memberMap := make(map[string]*sdkws.GroupMemberFullInfo) + for _, member := range members { + memberMap[member.UserID] = member + } + return memberMap, nil +} From 7b02b4b59b5d4dc80bfdfaff1977d322686f3734 Mon Sep 17 00:00:00 2001 From: Monet Lee Date: Mon, 30 Dec 2024 09:56:58 +0800 Subject: [PATCH 096/199] build: fix docker images build. (#3024) * build: fix uncorrect path. * build: fix docker images build. --- .github/workflows/docker-build-and-release-services-images.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/docker-build-and-release-services-images.yml b/.github/workflows/docker-build-and-release-services-images.yml index a79fab978..ed4c7ad38 100644 --- a/.github/workflows/docker-build-and-release-services-images.yml +++ b/.github/workflows/docker-build-and-release-services-images.yml @@ -84,7 +84,8 @@ jobs: docker buildx build --platform linux/amd64,linux/arm64 \ --file "$dockerfile" \ "${tag_args[@]}" \ - --push "$dir" + --push \ + "." else echo "No valid Dockerfile found in $dir" fi From a1b5f054a56ba12aaf5cdfd1f57905bc37768997 Mon Sep 17 00:00:00 2001 From: chao <48119764+withchao@users.noreply.github.com> Date: Mon, 30 Dec 2024 15:59:03 +0800 Subject: [PATCH 097/199] feat: optimize log output (#3026) * pb * fix: Modifying other fields while setting IsPrivateChat does not take effect * fix: quote message error revoke * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * upgrading pkg tools * fix * fix * optimize log output --- go.mod | 2 +- go.sum | 4 ++-- internal/msggateway/hub_server.go | 9 --------- internal/push/offlinepush/dummy/push.go | 7 +++++-- 4 files changed, 8 insertions(+), 14 deletions(-) diff --git a/go.mod b/go.mod index 387cb7304..efdac43f8 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/mitchellh/mapstructure v1.5.0 github.com/openimsdk/protocol v0.0.72-alpha.69 - github.com/openimsdk/tools v0.0.50-alpha.62 + github.com/openimsdk/tools v0.0.50-alpha.63 github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 github.com/stretchr/testify v1.9.0 diff --git a/go.sum b/go.sum index d207325e9..f87e038d0 100644 --- a/go.sum +++ b/go.sum @@ -349,8 +349,8 @@ github.com/openimsdk/gomake v0.0.15-alpha.2 h1:5Q8yl8ezy2yx+q8/ucU/t4kJnDfCzNOrk github.com/openimsdk/gomake v0.0.15-alpha.2/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= github.com/openimsdk/protocol v0.0.72-alpha.69 h1:b22oY2XTdBR/BePqA73KsrM3GDF3Vk8YcBEXZU4ArJc= github.com/openimsdk/protocol v0.0.72-alpha.69/go.mod h1:Iet+piS/jaS+kWWyj6EEr36mk4ISzIRYjoMSVA4dq2M= -github.com/openimsdk/tools v0.0.50-alpha.62 h1:e/m1XL7+EXbkOoxr/En/612WcOPKOUHPBj0++gG6MuQ= -github.com/openimsdk/tools v0.0.50-alpha.62/go.mod h1:JowL2jYr8tu4vcQe+5hJh4v3BtSx1T0CIS3pgU/Mw+U= +github.com/openimsdk/tools v0.0.50-alpha.63 h1:dPoVvg4KWqYX/xtK3j96TwX2A/4jwT5S5XIHvSM9hTY= +github.com/openimsdk/tools v0.0.50-alpha.63/go.mod h1:B+oqV0zdewN7OiEHYJm+hW+8/Te7B8tHHgD8rK5ZLZk= github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= diff --git a/internal/msggateway/hub_server.go b/internal/msggateway/hub_server.go index 52afe495b..490711cfe 100644 --- a/internal/msggateway/hub_server.go +++ b/internal/msggateway/hub_server.go @@ -96,10 +96,6 @@ func NewServer(longConnServer LongConnServer, conf *Config, ready func(srv *Serv return s } -func (s *Server) OnlinePushMsg(context context.Context, req *msggateway.OnlinePushMsgReq) (*msggateway.OnlinePushMsgResp, error) { - panic("implement me") -} - func (s *Server) GetUsersOnlineStatus(ctx context.Context, req *msggateway.GetUsersOnlineStatusReq) (*msggateway.GetUsersOnlineStatusResp, error) { if !authverify.IsAppManagerUid(ctx, s.config.Share.IMAdminUserID) { return nil, errs.ErrNoPermission.WrapMsg("only app manager") @@ -133,11 +129,6 @@ func (s *Server) GetUsersOnlineStatus(ctx context.Context, req *msggateway.GetUs return &resp, nil } -func (s *Server) OnlineBatchPushOneMsg(ctx context.Context, req *msggateway.OnlineBatchPushOneMsgReq) (*msggateway.OnlineBatchPushOneMsgResp, error) { - // todo implement - return nil, nil -} - func (s *Server) pushToUser(ctx context.Context, userID string, msgData *sdkws.MsgData) *msggateway.SingleMsgToUserResults { clients, ok := s.LongConnServer.GetUserAllCons(userID) if !ok { diff --git a/internal/push/offlinepush/dummy/push.go b/internal/push/offlinepush/dummy/push.go index 0bccaf4a4..1babbc798 100644 --- a/internal/push/offlinepush/dummy/push.go +++ b/internal/push/offlinepush/dummy/push.go @@ -18,6 +18,7 @@ import ( "context" "github.com/openimsdk/open-im-server/v3/internal/push/offlinepush/options" "github.com/openimsdk/tools/log" + "sync/atomic" ) func NewClient() *Dummy { @@ -25,10 +26,12 @@ func NewClient() *Dummy { } type Dummy struct { + v atomic.Bool } func (d *Dummy) Push(ctx context.Context, userIDs []string, title, content string, opts *options.Opts) error { - log.ZDebug(ctx, "dummy push") - log.ZWarn(ctx, "Dummy push", nil, "ps", "The offline push is not configured. To configure it, please go to config/openim-push.yml.") + if d.v.CompareAndSwap(false, true) { + log.ZWarn(ctx, "dummy push", nil, "ps", "the offline push is not configured. to configure it, please go to config/openim-push.yml") + } return nil } From ed0a834e2e9d4db7439361e6e0779fcf78337ffb Mon Sep 17 00:00:00 2001 From: chao <48119764+withchao@users.noreply.github.com> Date: Thu, 2 Jan 2025 17:15:38 +0800 Subject: [PATCH 098/199] feat: support GetLastMessage (#3029) * pb * fix: Modifying other fields while setting IsPrivateChat does not take effect * fix: quote message error revoke * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * upgrading pkg tools * fix * fix * optimize log output * feat: support GetLastMessage * feat: support GetLastMessage * feat: s3 switch * feat: s3 switch --- go.mod | 2 +- go.sum | 4 +- internal/msggateway/client.go | 2 + internal/msggateway/constant.go | 1 + internal/msggateway/message_handler.go | 13 ++ internal/rpc/msg/sync_msg.go | 8 + pkg/common/storage/controller/msg.go | 25 ++- pkg/common/storage/database/mgo/msg.go | 62 +++++++ pkg/common/storage/database/mgo/object.go | 12 ++ pkg/common/storage/database/msg.go | 1 + pkg/common/storage/database/object.go | 4 + tools/s3/README.md | 12 ++ tools/s3/internal/conversion.go | 202 ++++++++++++++++++++++ tools/s3/main.go | 23 +++ 14 files changed, 367 insertions(+), 4 deletions(-) create mode 100644 tools/s3/README.md create mode 100644 tools/s3/internal/conversion.go create mode 100644 tools/s3/main.go diff --git a/go.mod b/go.mod index efdac43f8..03a7a4d4d 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/gorilla/websocket v1.5.1 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/mitchellh/mapstructure v1.5.0 - github.com/openimsdk/protocol v0.0.72-alpha.69 + github.com/openimsdk/protocol v0.0.72-alpha.70 github.com/openimsdk/tools v0.0.50-alpha.63 github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 diff --git a/go.sum b/go.sum index f87e038d0..4c297134d 100644 --- a/go.sum +++ b/go.sum @@ -347,8 +347,8 @@ github.com/onsi/gomega v1.25.0 h1:Vw7br2PCDYijJHSfBOWhov+8cAnUf8MfMaIOV323l6Y= github.com/onsi/gomega v1.25.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdMPGhM= github.com/openimsdk/gomake v0.0.15-alpha.2 h1:5Q8yl8ezy2yx+q8/ucU/t4kJnDfCzNOrkXcDACCqtyM= github.com/openimsdk/gomake v0.0.15-alpha.2/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= -github.com/openimsdk/protocol v0.0.72-alpha.69 h1:b22oY2XTdBR/BePqA73KsrM3GDF3Vk8YcBEXZU4ArJc= -github.com/openimsdk/protocol v0.0.72-alpha.69/go.mod h1:Iet+piS/jaS+kWWyj6EEr36mk4ISzIRYjoMSVA4dq2M= +github.com/openimsdk/protocol v0.0.72-alpha.70 h1:j7vB81+rTthijRda2b8tlli9oWvPxr4yXHwZ8nPZIBQ= +github.com/openimsdk/protocol v0.0.72-alpha.70/go.mod h1:Iet+piS/jaS+kWWyj6EEr36mk4ISzIRYjoMSVA4dq2M= github.com/openimsdk/tools v0.0.50-alpha.63 h1:dPoVvg4KWqYX/xtK3j96TwX2A/4jwT5S5XIHvSM9hTY= github.com/openimsdk/tools v0.0.50-alpha.63/go.mod h1:B+oqV0zdewN7OiEHYJm+hW+8/Te7B8tHHgD8rK5ZLZk= github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= diff --git a/internal/msggateway/client.go b/internal/msggateway/client.go index 1040f2be2..bdb62aece 100644 --- a/internal/msggateway/client.go +++ b/internal/msggateway/client.go @@ -236,6 +236,8 @@ func (c *Client) handleMessage(message []byte) error { resp, messageErr = c.longConnServer.GetSeqMessage(ctx, binaryReq) case WSGetConvMaxReadSeq: resp, messageErr = c.longConnServer.GetConversationsHasReadAndMaxSeq(ctx, binaryReq) + case WsPullConvLastMessage: + resp, messageErr = c.longConnServer.GetLastMessage(ctx, binaryReq) case WsLogoutMsg: resp, messageErr = c.longConnServer.UserLogout(ctx, binaryReq) case WsSetBackgroundStatus: diff --git a/internal/msggateway/constant.go b/internal/msggateway/constant.go index a825c0519..1e7ab3bb7 100644 --- a/internal/msggateway/constant.go +++ b/internal/msggateway/constant.go @@ -47,6 +47,7 @@ const ( WSSendSignalMsg = 1004 WSPullMsg = 1005 WSGetConvMaxReadSeq = 1006 + WsPullConvLastMessage = 1007 WSPushMsg = 2001 WSKickOnlineMsg = 2002 WsLogoutMsg = 2003 diff --git a/internal/msggateway/message_handler.go b/internal/msggateway/message_handler.go index 9b59867d6..ca15e1ef6 100644 --- a/internal/msggateway/message_handler.go +++ b/internal/msggateway/message_handler.go @@ -108,6 +108,7 @@ type MessageHandler interface { GetSeqMessage(ctx context.Context, data *Req) ([]byte, error) UserLogout(ctx context.Context, data *Req) ([]byte, error) SetUserDeviceBackground(ctx context.Context, data *Req) ([]byte, bool, error) + GetLastMessage(ctx context.Context, data *Req) ([]byte, error) } var _ MessageHandler = (*GrpcHandler)(nil) @@ -266,3 +267,15 @@ func (g *GrpcHandler) SetUserDeviceBackground(ctx context.Context, data *Req) ([ } return nil, req.IsBackground, nil } + +func (g *GrpcHandler) GetLastMessage(ctx context.Context, data *Req) ([]byte, error) { + var req msg.GetLastMessageReq + if err := proto.Unmarshal(data.Data, &req); err != nil { + return nil, err + } + resp, err := g.msgClient.GetLastMessage(ctx, &req) + if err != nil { + return nil, err + } + return proto.Marshal(resp) +} diff --git a/internal/rpc/msg/sync_msg.go b/internal/rpc/msg/sync_msg.go index 7d4ffa3e6..6cf1c21d3 100644 --- a/internal/rpc/msg/sync_msg.go +++ b/internal/rpc/msg/sync_msg.go @@ -245,3 +245,11 @@ func (m *msgServer) SearchMessage(ctx context.Context, req *msg.SearchMessageReq func (m *msgServer) GetServerTime(ctx context.Context, _ *msg.GetServerTimeReq) (*msg.GetServerTimeResp, error) { return &msg.GetServerTimeResp{ServerTime: timeutil.GetCurrentTimestampByMill()}, nil } + +func (m *msgServer) GetLastMessage(ctx context.Context, req *msg.GetLastMessageReq) (*msg.GetLastMessageResp, error) { + msgs, err := m.MsgDatabase.GetLastMessage(ctx, req.ConversationIDs, req.UserID) + if err != nil { + return nil, err + } + return &msg.GetLastMessageResp{Msgs: msgs}, nil +} diff --git a/pkg/common/storage/controller/msg.go b/pkg/common/storage/controller/msg.go index d5ad12584..a93d581eb 100644 --- a/pkg/common/storage/controller/msg.go +++ b/pkg/common/storage/controller/msg.go @@ -97,6 +97,8 @@ type CommonMsgDatabase interface { DeleteDoc(ctx context.Context, docID string) error GetLastMessageSeqByTime(ctx context.Context, conversationID string, time int64) (int64, error) + + GetLastMessage(ctx context.Context, conversationIDS []string, userID string) (map[string]*sdkws.MsgData, error) } func NewCommonMsgDatabase(msgDocModel database.Msg, msg cache.MsgCache, seqUser cache.SeqUser, seqConversation cache.SeqConversationCache, kafkaConf *config.Kafka) (CommonMsgDatabase, error) { @@ -811,8 +813,29 @@ func (db *commonMsgDatabase) GetMessageBySeqs(ctx context.Context, conversationI if v, ok := seqMsgs[seq]; ok { res = append(res, convert.MsgDB2Pb(v.Msg)) } else { - res = append(res, &sdkws.MsgData{Seq: seq}) + res = append(res, &sdkws.MsgData{Seq: seq, Status: constant.MsgStatusHasDeleted}) + } + } + return res, nil +} + +func (db *commonMsgDatabase) GetLastMessage(ctx context.Context, conversationIDs []string, userID string) (map[string]*sdkws.MsgData, error) { + res := make(map[string]*sdkws.MsgData) + for _, conversationID := range conversationIDs { + if _, ok := res[conversationID]; ok { + continue + } + msg, err := db.msgDocDatabase.GetLastMessage(ctx, conversationID) + if err != nil { + if errs.Unwrap(err) == mongo.ErrNoDocuments { + continue + } + return nil, err } + tmp := []*model.MsgInfoModel{msg} + db.handlerDeleteAndRevoked(ctx, userID, tmp) + db.handlerQuote(ctx, userID, conversationID, tmp) + res[conversationID] = convert.MsgDB2Pb(msg.Msg) } return res, nil } diff --git a/pkg/common/storage/database/mgo/msg.go b/pkg/common/storage/database/mgo/msg.go index 03ebff611..c440d4442 100644 --- a/pkg/common/storage/database/mgo/msg.go +++ b/pkg/common/storage/database/mgo/msg.go @@ -997,6 +997,68 @@ func (m *MsgMgo) GetLastMessageSeqByTime(ctx context.Context, conversationID str return seq, nil } +func (m *MsgMgo) GetLastMessage(ctx context.Context, conversationID string) (*model.MsgInfoModel, error) { + pipeline := []bson.M{ + { + "$match": bson.M{ + "doc_id": bson.M{ + "$regex": fmt.Sprintf("^%s", conversationID), + }, + }, + }, + { + "$match": bson.M{ + "msgs.msg.status": bson.M{ + "$lt": constant.MsgStatusHasDeleted, + }, + }, + }, + { + "$sort": bson.M{ + "_id": -1, + }, + }, + { + "$limit": 1, + }, + { + "$project": bson.M{ + "_id": 0, + "doc_id": 0, + }, + }, + { + "$unwind": "$msgs", + }, + { + "$match": bson.M{ + "msgs.msg.status": bson.M{ + "$lt": constant.MsgStatusHasDeleted, + }, + }, + }, + { + "$sort": bson.M{ + "msgs.msg.seq": -1, + }, + }, + { + "$limit": 1, + }, + } + type Result struct { + Msgs *model.MsgInfoModel `bson:"msgs"` + } + res, err := mongoutil.Aggregate[*Result](ctx, m.coll, pipeline) + if err != nil { + return nil, err + } + if len(res) == 0 { + return nil, errs.Wrap(mongo.ErrNoDocuments) + } + return res[0].Msgs, nil +} + func (m *MsgMgo) onlyFindDocIndex(ctx context.Context, docID string, indexes []int64) ([]*model.MsgInfoModel, error) { if len(indexes) == 0 { return nil, nil diff --git a/pkg/common/storage/database/mgo/object.go b/pkg/common/storage/database/mgo/object.go index 624eb84a0..ee48c6693 100644 --- a/pkg/common/storage/database/mgo/object.go +++ b/pkg/common/storage/database/mgo/object.go @@ -112,3 +112,15 @@ func (o *S3Mongo) FindExpirationObject(ctx context.Context, engine string, expir func (o *S3Mongo) GetKeyCount(ctx context.Context, engine string, key string) (int64, error) { return mongoutil.Count(ctx, o.coll, bson.M{"engine": engine, "key": key}) } + +func (o *S3Mongo) GetEngineCount(ctx context.Context, engine string) (int64, error) { + return mongoutil.Count(ctx, o.coll, bson.M{"engine": engine}) +} + +func (o *S3Mongo) GetEngineInfo(ctx context.Context, engine string, limit int, skip int) ([]*model.Object, error) { + return mongoutil.Find[*model.Object](ctx, o.coll, bson.M{"engine": engine}, options.Find().SetLimit(int64(limit)).SetSkip(int64(skip))) +} + +func (o *S3Mongo) UpdateEngine(ctx context.Context, oldEngine, oldName string, newEngine string) error { + return mongoutil.UpdateOne(ctx, o.coll, bson.M{"engine": oldEngine, "name": oldName}, bson.M{"$set": bson.M{"engine": newEngine}}, false) +} diff --git a/pkg/common/storage/database/msg.go b/pkg/common/storage/database/msg.go index b44e70296..e3c4e8ece 100644 --- a/pkg/common/storage/database/msg.go +++ b/pkg/common/storage/database/msg.go @@ -39,5 +39,6 @@ type Msg interface { DeleteDoc(ctx context.Context, docID string) error GetRandBeforeMsg(ctx context.Context, ts int64, limit int) ([]*model.MsgDocModel, error) GetLastMessageSeqByTime(ctx context.Context, conversationID string, time int64) (int64, error) + GetLastMessage(ctx context.Context, conversationID string) (*model.MsgInfoModel, error) FindSeqs(ctx context.Context, conversationID string, seqs []int64) ([]*model.MsgInfoModel, error) } diff --git a/pkg/common/storage/database/object.go b/pkg/common/storage/database/object.go index 5541a159b..a0e4ebe2b 100644 --- a/pkg/common/storage/database/object.go +++ b/pkg/common/storage/database/object.go @@ -27,4 +27,8 @@ type ObjectInfo interface { Delete(ctx context.Context, engine string, name []string) error FindExpirationObject(ctx context.Context, engine string, expiration time.Time, needDelType []string, count int64) ([]*model.Object, error) GetKeyCount(ctx context.Context, engine string, key string) (int64, error) + + GetEngineCount(ctx context.Context, engine string) (int64, error) + GetEngineInfo(ctx context.Context, engine string, limit int, skip int) ([]*model.Object, error) + UpdateEngine(ctx context.Context, oldEngine, oldName string, newEngine string) error } diff --git a/tools/s3/README.md b/tools/s3/README.md new file mode 100644 index 000000000..ac30347d8 --- /dev/null +++ b/tools/s3/README.md @@ -0,0 +1,12 @@ +# After s3 switches the storage engine, convert the data + +- build +```shell +go build -o s3convert main.go +``` + +- start +```shell +./s3convert -config -name +# ./s3convert -config ./../../config -name minio +``` diff --git a/tools/s3/internal/conversion.go b/tools/s3/internal/conversion.go new file mode 100644 index 000000000..ba2174535 --- /dev/null +++ b/tools/s3/internal/conversion.go @@ -0,0 +1,202 @@ +package internal + +import ( + "context" + "errors" + "fmt" + "github.com/mitchellh/mapstructure" + "github.com/openimsdk/open-im-server/v3/pkg/common/config" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/redis" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database/mgo" + "github.com/openimsdk/tools/db/mongoutil" + "github.com/openimsdk/tools/db/redisutil" + "github.com/openimsdk/tools/s3" + "github.com/openimsdk/tools/s3/aws" + "github.com/openimsdk/tools/s3/cos" + "github.com/openimsdk/tools/s3/kodo" + "github.com/openimsdk/tools/s3/minio" + "github.com/openimsdk/tools/s3/oss" + "github.com/spf13/viper" + "go.mongodb.org/mongo-driver/mongo" + "log" + "net/http" + "path/filepath" + "time" +) + +const defaultTimeout = time.Second * 10 + +func readConf(path string, val any) error { + v := viper.New() + v.SetConfigFile(path) + if err := v.ReadInConfig(); err != nil { + return err + } + fn := func(config *mapstructure.DecoderConfig) { + config.TagName = "mapstructure" + } + return v.Unmarshal(val, fn) +} + +func getS3(path string, name string, thirdConf *config.Third) (s3.Interface, error) { + switch name { + case "minio": + ctx, cancel := context.WithTimeout(context.Background(), defaultTimeout) + defer cancel() + var minioConf config.Minio + if err := readConf(filepath.Join(path, minioConf.GetConfigFileName()), &minioConf); err != nil { + return nil, err + } + var redisConf config.Redis + if err := readConf(filepath.Join(path, redisConf.GetConfigFileName()), &redisConf); err != nil { + return nil, err + } + rdb, err := redisutil.NewRedisClient(ctx, redisConf.Build()) + if err != nil { + return nil, err + } + return minio.NewMinio(ctx, redis.NewMinioCache(rdb), *minioConf.Build()) + case "cos": + return cos.NewCos(*thirdConf.Object.Cos.Build()) + case "oss": + return oss.NewOSS(*thirdConf.Object.Oss.Build()) + case "kodo": + return kodo.NewKodo(*thirdConf.Object.Kodo.Build()) + case "aws": + return aws.NewAws(*thirdConf.Object.Aws.Build()) + default: + return nil, fmt.Errorf("invalid object enable: %s", name) + } +} + +func getMongo(path string) (database.ObjectInfo, error) { + var mongoConf config.Mongo + if err := readConf(filepath.Join(path, mongoConf.GetConfigFileName()), &mongoConf); err != nil { + return nil, err + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTimeout) + defer cancel() + mgocli, err := mongoutil.NewMongoDB(ctx, mongoConf.Build()) + if err != nil { + return nil, err + } + return mgo.NewS3Mongo(mgocli.GetDB()) +} + +func Main(path string, engine string) error { + var thirdConf config.Third + if err := readConf(filepath.Join(path, thirdConf.GetConfigFileName()), &thirdConf); err != nil { + return err + } + if thirdConf.Object.Enable == engine { + return errors.New("same s3 storage") + } + s3db, err := getMongo(path) + if err != nil { + return err + } + oldS3, err := getS3(path, engine, &thirdConf) + if err != nil { + return err + } + newS3, err := getS3(path, thirdConf.Object.Enable, &thirdConf) + if err != nil { + return err + } + count, err := getEngineCount(s3db, oldS3.Engine()) + if err != nil { + return err + } + log.Printf("engine %s count: %d", oldS3.Engine(), count) + var skip int + for i := 1; i <= count+1; i++ { + log.Printf("start %d/%d", i, count) + start := time.Now() + res, err := doObject(s3db, newS3, oldS3, skip) + if err != nil { + log.Printf("end [%s] %d/%d error %s", time.Since(start), i, count, err) + return err + } + log.Printf("end [%s] %d/%d result %+v", time.Since(start), i, count, *res) + if res.Skip { + skip++ + } + if res.End { + break + } + } + return nil +} + +func getEngineCount(db database.ObjectInfo, name string) (int, error) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTimeout) + defer cancel() + count, err := db.GetEngineCount(ctx, name) + if err != nil { + return 0, err + } + return int(count), nil +} + +func doObject(db database.ObjectInfo, newS3, oldS3 s3.Interface, skip int) (*Result, error) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTimeout) + defer cancel() + infos, err := db.GetEngineInfo(ctx, oldS3.Engine(), 1, skip) + if err != nil { + return nil, err + } + if len(infos) == 0 { + return &Result{End: true}, nil + } + obj := infos[0] + if _, err := db.Take(ctx, newS3.Engine(), obj.Name); err == nil { + return &Result{Skip: true}, nil + } else if !errors.Is(err, mongo.ErrNoDocuments) { + return nil, err + } + downloadURL, err := oldS3.AccessURL(ctx, obj.Key, time.Hour, &s3.AccessURLOption{}) + if err != nil { + return nil, err + } + putURL, err := newS3.PresignedPutObject(ctx, obj.Key, time.Hour) + if err != nil { + return nil, err + } + downloadResp, err := http.Get(downloadURL) + if err != nil { + return nil, err + } + defer downloadResp.Body.Close() + switch downloadResp.StatusCode { + case http.StatusNotFound: + return &Result{Skip: true}, nil + case http.StatusOK: + default: + return nil, fmt.Errorf("download object failed %s", downloadResp.Status) + } + log.Printf("file size %d", obj.Size) + request, err := http.NewRequest(http.MethodPut, putURL, downloadResp.Body) + if err != nil { + return nil, err + } + putResp, err := http.DefaultClient.Do(request) + if err != nil { + return nil, err + } + defer putResp.Body.Close() + if putResp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("put object failed %s", putResp.Status) + } + ctx, cancel = context.WithTimeout(context.Background(), defaultTimeout) + defer cancel() + if err := db.UpdateEngine(ctx, obj.Engine, obj.Name, newS3.Engine()); err != nil { + return nil, err + } + return &Result{}, nil +} + +type Result struct { + Skip bool + End bool +} diff --git a/tools/s3/main.go b/tools/s3/main.go new file mode 100644 index 000000000..1e661c9a7 --- /dev/null +++ b/tools/s3/main.go @@ -0,0 +1,23 @@ +package main + +import ( + "flag" + "fmt" + "github.com/openimsdk/open-im-server/v3/tools/s3/internal" + "os" +) + +func main() { + var ( + name string + config string + ) + flag.StringVar(&name, "name", "", "old previous storage name") + flag.StringVar(&config, "config", "", "config directory") + flag.Parse() + if err := internal.Main(config, name); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + fmt.Fprintln(os.Stdout, "success") +} From 07c255ade7f3bd644cbecb24d7a2bd8606719f4b Mon Sep 17 00:00:00 2001 From: icey-yu <119291641+icey-yu@users.noreply.github.com> Date: Sat, 4 Jan 2025 20:21:21 +0800 Subject: [PATCH 099/199] feat: Add enable config center button && fix: grpc connection leakage (#3036) * feat: add enable config center * fix: config * fix: config * fix: config * fix: config --- go.mod | 2 +- go.sum | 4 +- internal/api/config_manager.go | 134 ++++++++++++++------ internal/api/init.go | 4 +- internal/api/router.go | 2 + internal/msggateway/hub_server.go | 6 +- internal/msgtransfer/init.go | 2 +- internal/tools/cron_task.go | 34 +++-- pkg/apistruct/config_manager.go | 8 ++ pkg/common/cmd/auth.go | 3 + pkg/common/cmd/conversation.go | 2 +- pkg/common/cmd/friend.go | 2 +- pkg/common/cmd/group.go | 2 +- pkg/common/cmd/msg.go | 2 +- pkg/common/cmd/push.go | 3 + pkg/common/cmd/root.go | 23 +++- pkg/common/cmd/third.go | 2 +- pkg/common/cmd/user.go | 2 +- pkg/common/discovery/discoveryregister.go | 3 +- pkg/common/discovery/etcd/config_manager.go | 5 - pkg/common/discovery/etcd/const.go | 9 ++ pkg/common/discovery/etcd/doc.go | 15 --- pkg/common/startrpc/start.go | 4 +- 23 files changed, 186 insertions(+), 87 deletions(-) create mode 100644 pkg/common/discovery/etcd/const.go delete mode 100644 pkg/common/discovery/etcd/doc.go diff --git a/go.mod b/go.mod index 03a7a4d4d..10ff7e6d0 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/mitchellh/mapstructure v1.5.0 github.com/openimsdk/protocol v0.0.72-alpha.70 - github.com/openimsdk/tools v0.0.50-alpha.63 + github.com/openimsdk/tools v0.0.50-alpha.64 github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 github.com/stretchr/testify v1.9.0 diff --git a/go.sum b/go.sum index 4c297134d..98ba2e7e2 100644 --- a/go.sum +++ b/go.sum @@ -349,8 +349,8 @@ github.com/openimsdk/gomake v0.0.15-alpha.2 h1:5Q8yl8ezy2yx+q8/ucU/t4kJnDfCzNOrk github.com/openimsdk/gomake v0.0.15-alpha.2/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= github.com/openimsdk/protocol v0.0.72-alpha.70 h1:j7vB81+rTthijRda2b8tlli9oWvPxr4yXHwZ8nPZIBQ= github.com/openimsdk/protocol v0.0.72-alpha.70/go.mod h1:Iet+piS/jaS+kWWyj6EEr36mk4ISzIRYjoMSVA4dq2M= -github.com/openimsdk/tools v0.0.50-alpha.63 h1:dPoVvg4KWqYX/xtK3j96TwX2A/4jwT5S5XIHvSM9hTY= -github.com/openimsdk/tools v0.0.50-alpha.63/go.mod h1:B+oqV0zdewN7OiEHYJm+hW+8/Te7B8tHHgD8rK5ZLZk= +github.com/openimsdk/tools v0.0.50-alpha.64 h1:KmtE8V2K8atQJJg1xq2ySSrPQyf8ldwk8fw6jRnsxCw= +github.com/openimsdk/tools v0.0.50-alpha.64/go.mod h1:B+oqV0zdewN7OiEHYJm+hW+8/Te7B8tHHgD8rK5ZLZk= github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= diff --git a/internal/api/config_manager.go b/internal/api/config_manager.go index c330cad46..7a36bb605 100644 --- a/internal/api/config_manager.go +++ b/internal/api/config_manager.go @@ -19,22 +19,29 @@ import ( clientv3 "go.etcd.io/etcd/client/v3" ) +const ( + // wait for Restart http call return + waitHttp = time.Millisecond * 200 +) + type ConfigManager struct { imAdminUserID []string config *config.AllConfig client *clientv3.Client - configPath string - runtimeEnv string + + configPath string + runtimeEnv string } func NewConfigManager(IMAdminUserID []string, cfg *config.AllConfig, client *clientv3.Client, configPath string, runtimeEnv string) *ConfigManager { - return &ConfigManager{ + cm := &ConfigManager{ imAdminUserID: IMAdminUserID, config: cfg, client: client, configPath: configPath, runtimeEnv: runtimeEnv, } + return cm } func (cm *ConfigManager) CheckAdmin(c *gin.Context) { @@ -85,49 +92,49 @@ func (cm *ConfigManager) SetConfig(c *gin.Context) { var err error switch req.ConfigName { case cm.config.Discovery.GetConfigFileName(): - err = compareAndSave[config.Discovery](c, cm.config.Name2Config(req.ConfigName), &req, cm.client) + err = compareAndSave[config.Discovery](c, cm.config.Name2Config(req.ConfigName), &req, cm) case cm.config.Kafka.GetConfigFileName(): - err = compareAndSave[config.Kafka](c, cm.config.Name2Config(req.ConfigName), &req, cm.client) + err = compareAndSave[config.Kafka](c, cm.config.Name2Config(req.ConfigName), &req, cm) case cm.config.LocalCache.GetConfigFileName(): - err = compareAndSave[config.LocalCache](c, cm.config.Name2Config(req.ConfigName), &req, cm.client) + err = compareAndSave[config.LocalCache](c, cm.config.Name2Config(req.ConfigName), &req, cm) case cm.config.Log.GetConfigFileName(): - err = compareAndSave[config.Log](c, cm.config.Name2Config(req.ConfigName), &req, cm.client) + err = compareAndSave[config.Log](c, cm.config.Name2Config(req.ConfigName), &req, cm) case cm.config.Minio.GetConfigFileName(): - err = compareAndSave[config.Minio](c, cm.config.Name2Config(req.ConfigName), &req, cm.client) + err = compareAndSave[config.Minio](c, cm.config.Name2Config(req.ConfigName), &req, cm) case cm.config.Mongo.GetConfigFileName(): - err = compareAndSave[config.Mongo](c, cm.config.Name2Config(req.ConfigName), &req, cm.client) + err = compareAndSave[config.Mongo](c, cm.config.Name2Config(req.ConfigName), &req, cm) case cm.config.Notification.GetConfigFileName(): - err = compareAndSave[config.Notification](c, cm.config.Name2Config(req.ConfigName), &req, cm.client) + err = compareAndSave[config.Notification](c, cm.config.Name2Config(req.ConfigName), &req, cm) case cm.config.API.GetConfigFileName(): - err = compareAndSave[config.API](c, cm.config.Name2Config(req.ConfigName), &req, cm.client) + err = compareAndSave[config.API](c, cm.config.Name2Config(req.ConfigName), &req, cm) case cm.config.CronTask.GetConfigFileName(): - err = compareAndSave[config.CronTask](c, cm.config.Name2Config(req.ConfigName), &req, cm.client) + err = compareAndSave[config.CronTask](c, cm.config.Name2Config(req.ConfigName), &req, cm) case cm.config.MsgGateway.GetConfigFileName(): - err = compareAndSave[config.MsgGateway](c, cm.config.Name2Config(req.ConfigName), &req, cm.client) + err = compareAndSave[config.MsgGateway](c, cm.config.Name2Config(req.ConfigName), &req, cm) case cm.config.MsgTransfer.GetConfigFileName(): - err = compareAndSave[config.MsgTransfer](c, cm.config.Name2Config(req.ConfigName), &req, cm.client) + err = compareAndSave[config.MsgTransfer](c, cm.config.Name2Config(req.ConfigName), &req, cm) case cm.config.Push.GetConfigFileName(): - err = compareAndSave[config.Push](c, cm.config.Name2Config(req.ConfigName), &req, cm.client) + err = compareAndSave[config.Push](c, cm.config.Name2Config(req.ConfigName), &req, cm) case cm.config.Auth.GetConfigFileName(): - err = compareAndSave[config.Auth](c, cm.config.Name2Config(req.ConfigName), &req, cm.client) + err = compareAndSave[config.Auth](c, cm.config.Name2Config(req.ConfigName), &req, cm) case cm.config.Conversation.GetConfigFileName(): - err = compareAndSave[config.Conversation](c, cm.config.Name2Config(req.ConfigName), &req, cm.client) + err = compareAndSave[config.Conversation](c, cm.config.Name2Config(req.ConfigName), &req, cm) case cm.config.Friend.GetConfigFileName(): - err = compareAndSave[config.Friend](c, cm.config.Name2Config(req.ConfigName), &req, cm.client) + err = compareAndSave[config.Friend](c, cm.config.Name2Config(req.ConfigName), &req, cm) case cm.config.Group.GetConfigFileName(): - err = compareAndSave[config.Group](c, cm.config.Name2Config(req.ConfigName), &req, cm.client) + err = compareAndSave[config.Group](c, cm.config.Name2Config(req.ConfigName), &req, cm) case cm.config.Msg.GetConfigFileName(): - err = compareAndSave[config.Msg](c, cm.config.Name2Config(req.ConfigName), &req, cm.client) + err = compareAndSave[config.Msg](c, cm.config.Name2Config(req.ConfigName), &req, cm) case cm.config.Third.GetConfigFileName(): - err = compareAndSave[config.Third](c, cm.config.Name2Config(req.ConfigName), &req, cm.client) + err = compareAndSave[config.Third](c, cm.config.Name2Config(req.ConfigName), &req, cm) case cm.config.User.GetConfigFileName(): - err = compareAndSave[config.User](c, cm.config.Name2Config(req.ConfigName), &req, cm.client) + err = compareAndSave[config.User](c, cm.config.Name2Config(req.ConfigName), &req, cm) case cm.config.Redis.GetConfigFileName(): - err = compareAndSave[config.Redis](c, cm.config.Name2Config(req.ConfigName), &req, cm.client) + err = compareAndSave[config.Redis](c, cm.config.Name2Config(req.ConfigName), &req, cm) case cm.config.Share.GetConfigFileName(): - err = compareAndSave[config.Share](c, cm.config.Name2Config(req.ConfigName), &req, cm.client) + err = compareAndSave[config.Share](c, cm.config.Name2Config(req.ConfigName), &req, cm) case cm.config.Webhooks.GetConfigFileName(): - err = compareAndSave[config.Webhooks](c, cm.config.Name2Config(req.ConfigName), &req, cm.client) + err = compareAndSave[config.Webhooks](c, cm.config.Name2Config(req.ConfigName), &req, cm) default: apiresp.GinError(c, errs.ErrArgs.Wrap()) return @@ -139,7 +146,7 @@ func (cm *ConfigManager) SetConfig(c *gin.Context) { apiresp.GinSuccess(c, nil) } -func compareAndSave[T any](c *gin.Context, old any, req *apistruct.SetConfigReq, client *clientv3.Client) error { +func compareAndSave[T any](c *gin.Context, old any, req *apistruct.SetConfigReq, cm *ConfigManager) error { conf := new(T) err := json.Unmarshal([]byte(req.Data), &conf) if err != nil { @@ -153,7 +160,7 @@ func compareAndSave[T any](c *gin.Context, old any, req *apistruct.SetConfigReq, if err != nil { return errs.ErrArgs.WithDetail(err.Error()).Wrap() } - _, err = client.Put(c, etcd.BuildKey(req.ConfigName), string(data)) + _, err = cm.client.Put(c, etcd.BuildKey(req.ConfigName), string(data)) if err != nil { return errs.WrapMsg(err, "save to etcd failed") } @@ -161,16 +168,19 @@ func compareAndSave[T any](c *gin.Context, old any, req *apistruct.SetConfigReq, } func (cm *ConfigManager) ResetConfig(c *gin.Context) { - go cm.resetConfig(c) + go func() { + if err := cm.resetConfig(c, true); err != nil { + log.ZError(c, "reset config err", err) + } + }() apiresp.GinSuccess(c, nil) } -func (cm *ConfigManager) resetConfig(c *gin.Context) { +func (cm *ConfigManager) resetConfig(c *gin.Context, checkChange bool, ops ...clientv3.Op) error { txn := cm.client.Txn(c) type initConf struct { - old any - new any - isChanged bool + old any + new any } configMap := map[string]*initConf{ cm.config.Discovery.GetConfigFileName(): {old: &cm.config.Discovery, new: new(config.Discovery)}, @@ -210,13 +220,12 @@ func (cm *ConfigManager) resetConfig(c *gin.Context) { log.ZError(c, "load config failed", err) continue } - v.isChanged = reflect.DeepEqual(v.old, v.new) - if !v.isChanged { + equal := reflect.DeepEqual(v.old, v.new) + if !checkChange || !equal { changedKeys = append(changedKeys, k) } } - ops := make([]clientv3.Op, 0) for _, k := range changedKeys { data, err := json.Marshal(configMap[k].new) if err != nil { @@ -229,10 +238,10 @@ func (cm *ConfigManager) resetConfig(c *gin.Context) { txn.Then(ops...) _, err := txn.Commit() if err != nil { - log.ZError(c, "commit etcd txn failed", err) - return + return errs.WrapMsg(err, "commit etcd txn failed") } } + return nil } func (cm *ConfigManager) Restart(c *gin.Context) { @@ -241,10 +250,59 @@ func (cm *ConfigManager) Restart(c *gin.Context) { } func (cm *ConfigManager) restart(c *gin.Context) { - time.Sleep(time.Millisecond * 200) // wait for Restart http call return + time.Sleep(waitHttp) // wait for Restart http call return t := time.Now().Unix() _, err := cm.client.Put(c, etcd.BuildKey(etcd.RestartKey), strconv.Itoa(int(t))) if err != nil { log.ZError(c, "restart etcd put key failed", err) } } + +func (cm *ConfigManager) SetEnableConfigManager(c *gin.Context) { + var req apistruct.SetEnableConfigManagerReq + if err := c.BindJSON(&req); err != nil { + apiresp.GinError(c, errs.ErrArgs.WithDetail(err.Error()).Wrap()) + return + } + var enableStr string + if req.Enable { + enableStr = etcd.Enable + } else { + enableStr = etcd.Disable + } + resp, err := cm.client.Get(c, etcd.BuildKey(etcd.EnableConfigCenterKey)) + if err != nil { + apiresp.GinError(c, errs.WrapMsg(err, "getEnableConfigManager failed")) + return + } + if !(resp.Count > 0 && string(resp.Kvs[0].Value) == etcd.Enable) && req.Enable { + go func() { + time.Sleep(waitHttp) // wait for Restart http call return + err := cm.resetConfig(c, false, clientv3.OpPut(etcd.BuildKey(etcd.EnableConfigCenterKey), enableStr)) + if err != nil { + log.ZError(c, "resetConfig failed", err) + } + }() + } else { + _, err = cm.client.Put(c, etcd.BuildKey(etcd.EnableConfigCenterKey), enableStr) + if err != nil { + apiresp.GinError(c, errs.WrapMsg(err, "setEnableConfigManager failed")) + return + } + } + + apiresp.GinSuccess(c, nil) +} + +func (cm *ConfigManager) GetEnableConfigManager(c *gin.Context) { + resp, err := cm.client.Get(c, etcd.BuildKey(etcd.EnableConfigCenterKey)) + if err != nil { + apiresp.GinError(c, errs.WrapMsg(err, "getEnableConfigManager failed")) + return + } + var enable bool + if resp.Count > 0 && string(resp.Kvs[0].Value) == etcd.Enable { + enable = true + } + apiresp.GinSuccess(c, &apistruct.GetEnableConfigManagerResp{Enable: enable}) +} diff --git a/internal/api/init.go b/internal/api/init.go index 780ecb913..20237ebc2 100644 --- a/internal/api/init.go +++ b/internal/api/init.go @@ -58,7 +58,9 @@ func Start(ctx context.Context, index int, config *Config) error { config.RuntimeEnv = runtimeenv.PrintRuntimeEnvironment() - client, err := kdisc.NewDiscoveryRegister(&config.Discovery, config.RuntimeEnv) + client, err := kdisc.NewDiscoveryRegister(&config.Discovery, config.RuntimeEnv, []string{ + config.Discovery.RpcService.MessageGateway, + }) if err != nil { return errs.WrapMsg(err, "failed to register discovery service") } diff --git a/internal/api/router.go b/internal/api/router.go index e516d8ca8..0ae0eab05 100644 --- a/internal/api/router.go +++ b/internal/api/router.go @@ -314,6 +314,8 @@ func newGinRouter(ctx context.Context, client discovery.SvcDiscoveryRegistry, cf configGroup.POST("/get_config", cm.GetConfig) configGroup.POST("/set_config", cm.SetConfig) configGroup.POST("/reset_config", cm.ResetConfig) + configGroup.POST("/set_enable_config_manager", cm.SetEnableConfigManager) + configGroup.POST("/get_enable_config_manager", cm.GetEnableConfigManager) } { r.POST("/restart", cm.CheckAdmin, cm.Restart) diff --git a/internal/msggateway/hub_server.go b/internal/msggateway/hub_server.go index 490711cfe..887a90d7a 100644 --- a/internal/msggateway/hub_server.go +++ b/internal/msggateway/hub_server.go @@ -16,9 +16,10 @@ package msggateway import ( "context" - "github.com/openimsdk/open-im-server/v3/pkg/rpcli" "sync/atomic" + "github.com/openimsdk/open-im-server/v3/pkg/rpcli" + "github.com/openimsdk/open-im-server/v3/pkg/authverify" "github.com/openimsdk/open-im-server/v3/pkg/common/servererrs" "github.com/openimsdk/open-im-server/v3/pkg/common/startrpc" @@ -64,6 +65,9 @@ func (s *Server) Start(ctx context.Context, index int, conf *Config) error { conf.WebhooksConfig.GetConfigFileName(), conf.RedisConfig.GetConfigFileName(), }, + []string{ + conf.Discovery.RpcService.MessageGateway, + }, s.InitServer, ) } diff --git a/internal/msgtransfer/init.go b/internal/msgtransfer/init.go index fcd6152dc..96e6bbde0 100644 --- a/internal/msgtransfer/init.go +++ b/internal/msgtransfer/init.go @@ -87,7 +87,7 @@ func Start(ctx context.Context, index int, config *Config) error { if err != nil { return err } - client, err := discRegister.NewDiscoveryRegister(&config.Discovery, runTimeEnv) + client, err := discRegister.NewDiscoveryRegister(&config.Discovery, runTimeEnv, nil) if err != nil { return err } diff --git a/internal/tools/cron_task.go b/internal/tools/cron_task.go index 71fd886f6..da1c6320e 100644 --- a/internal/tools/cron_task.go +++ b/internal/tools/cron_task.go @@ -2,11 +2,14 @@ package tools import ( "context" + "github.com/openimsdk/open-im-server/v3/pkg/common/config" kdisc "github.com/openimsdk/open-im-server/v3/pkg/common/discovery" + disetcd "github.com/openimsdk/open-im-server/v3/pkg/common/discovery/etcd" pbconversation "github.com/openimsdk/protocol/conversation" "github.com/openimsdk/protocol/msg" "github.com/openimsdk/protocol/third" + "github.com/openimsdk/tools/discovery/etcd" "github.com/openimsdk/tools/mcontext" "github.com/openimsdk/tools/mw" @@ -27,38 +30,47 @@ type CronTaskConfig struct { runTimeEnv string } -func Start(ctx context.Context, config *CronTaskConfig) error { - config.runTimeEnv = runtimeenv.PrintRuntimeEnvironment() +func Start(ctx context.Context, conf *CronTaskConfig) error { + conf.runTimeEnv = runtimeenv.PrintRuntimeEnvironment() - log.CInfo(ctx, "CRON-TASK server is initializing", "runTimeEnv", config.runTimeEnv, "chatRecordsClearTime", config.CronTask.CronExecuteTime, "msgDestructTime", config.CronTask.RetainChatRecords) - if config.CronTask.RetainChatRecords < 1 { + log.CInfo(ctx, "CRON-TASK server is initializing", "runTimeEnv", conf.runTimeEnv, "chatRecordsClearTime", conf.CronTask.CronExecuteTime, "msgDestructTime", conf.CronTask.RetainChatRecords) + if conf.CronTask.RetainChatRecords < 1 { return errs.New("msg destruct time must be greater than 1").Wrap() } - client, err := kdisc.NewDiscoveryRegister(&config.Discovery, config.runTimeEnv) + client, err := kdisc.NewDiscoveryRegister(&conf.Discovery, conf.runTimeEnv, nil) if err != nil { return errs.WrapMsg(err, "failed to register discovery service") } client.AddOption(mw.GrpcClient(), grpc.WithTransportCredentials(insecure.NewCredentials())) - ctx = mcontext.SetOpUserID(ctx, config.Share.IMAdminUserID[0]) + ctx = mcontext.SetOpUserID(ctx, conf.Share.IMAdminUserID[0]) - msgConn, err := client.GetConn(ctx, config.Discovery.RpcService.Msg) + msgConn, err := client.GetConn(ctx, conf.Discovery.RpcService.Msg) if err != nil { return err } - thirdConn, err := client.GetConn(ctx, config.Discovery.RpcService.Third) + thirdConn, err := client.GetConn(ctx, conf.Discovery.RpcService.Third) if err != nil { return err } - conversationConn, err := client.GetConn(ctx, config.Discovery.RpcService.Conversation) + conversationConn, err := client.GetConn(ctx, conf.Discovery.RpcService.Conversation) if err != nil { return err } + if conf.Discovery.Enable == config.ETCD { + cm := disetcd.NewConfigManager(client.(*etcd.SvcDiscoveryRegistryImpl).GetClient(), []string{ + conf.CronTask.GetConfigFileName(), + conf.Share.GetConfigFileName(), + conf.Discovery.GetConfigFileName(), + }) + cm.Watch(ctx) + } + srv := &cronServer{ ctx: ctx, - config: config, + config: conf, cron: cron.New(), msgClient: msg.NewMsgClient(msgConn), conversationClient: pbconversation.NewConversationClient(conversationConn), @@ -74,7 +86,7 @@ func Start(ctx context.Context, config *CronTaskConfig) error { if err := srv.registerClearUserMsg(); err != nil { return err } - log.ZDebug(ctx, "start cron task", "CronExecuteTime", config.CronTask.CronExecuteTime) + log.ZDebug(ctx, "start cron task", "CronExecuteTime", conf.CronTask.CronExecuteTime) srv.cron.Start() <-ctx.Done() return nil diff --git a/pkg/apistruct/config_manager.go b/pkg/apistruct/config_manager.go index 84b8fb36b..9b8641c9d 100644 --- a/pkg/apistruct/config_manager.go +++ b/pkg/apistruct/config_manager.go @@ -14,3 +14,11 @@ type SetConfigReq struct { ConfigName string `json:"configName"` Data string `json:"data"` } + +type SetEnableConfigManagerReq struct { + Enable bool `json:"enable"` +} + +type GetEnableConfigManagerResp struct { + Enable bool `json:"enable"` +} diff --git a/pkg/common/cmd/auth.go b/pkg/common/cmd/auth.go index a5ab3fea7..b09d4153f 100644 --- a/pkg/common/cmd/auth.go +++ b/pkg/common/cmd/auth.go @@ -64,5 +64,8 @@ func (a *AuthRpcCmd) runE() error { a.authConfig.RedisConfig.GetConfigFileName(), a.authConfig.Discovery.GetConfigFileName(), }, + []string{ + a.authConfig.Discovery.RpcService.MessageGateway, + }, auth.Start) } diff --git a/pkg/common/cmd/conversation.go b/pkg/common/cmd/conversation.go index 12c29a873..2f8769897 100644 --- a/pkg/common/cmd/conversation.go +++ b/pkg/common/cmd/conversation.go @@ -68,6 +68,6 @@ func (a *ConversationRpcCmd) runE() error { a.conversationConfig.Share.GetConfigFileName(), a.conversationConfig.LocalCacheConfig.GetConfigFileName(), a.conversationConfig.Discovery.GetConfigFileName(), - }, + }, nil, conversation.Start) } diff --git a/pkg/common/cmd/friend.go b/pkg/common/cmd/friend.go index 209d481bb..dd850cf17 100644 --- a/pkg/common/cmd/friend.go +++ b/pkg/common/cmd/friend.go @@ -70,6 +70,6 @@ func (a *FriendRpcCmd) runE() error { a.relationConfig.WebhooksConfig.GetConfigFileName(), a.relationConfig.LocalCacheConfig.GetConfigFileName(), a.relationConfig.Discovery.GetConfigFileName(), - }, + }, nil, relation.Start) } diff --git a/pkg/common/cmd/group.go b/pkg/common/cmd/group.go index 23fd460f7..7a599077f 100644 --- a/pkg/common/cmd/group.go +++ b/pkg/common/cmd/group.go @@ -71,6 +71,6 @@ func (a *GroupRpcCmd) runE() error { a.groupConfig.WebhooksConfig.GetConfigFileName(), a.groupConfig.LocalCacheConfig.GetConfigFileName(), a.groupConfig.Discovery.GetConfigFileName(), - }, + }, nil, group.Start, versionctx.EnableVersionCtx()) } diff --git a/pkg/common/cmd/msg.go b/pkg/common/cmd/msg.go index b6f0b6131..c4049be05 100644 --- a/pkg/common/cmd/msg.go +++ b/pkg/common/cmd/msg.go @@ -72,6 +72,6 @@ func (a *MsgRpcCmd) runE() error { a.msgConfig.WebhooksConfig.GetConfigFileName(), a.msgConfig.LocalCacheConfig.GetConfigFileName(), a.msgConfig.Discovery.GetConfigFileName(), - }, + }, nil, msg.Start) } diff --git a/pkg/common/cmd/push.go b/pkg/common/cmd/push.go index 41b9d56e6..c4ae84952 100644 --- a/pkg/common/cmd/push.go +++ b/pkg/common/cmd/push.go @@ -72,5 +72,8 @@ func (a *PushRpcCmd) runE() error { a.pushConfig.LocalCacheConfig.GetConfigFileName(), a.pushConfig.Discovery.GetConfigFileName(), }, + []string{ + a.pushConfig.Discovery.RpcService.MessageGateway, + }, push.Start) } diff --git a/pkg/common/cmd/root.go b/pkg/common/cmd/root.go index 4c5256d80..0a405fb6e 100644 --- a/pkg/common/cmd/root.go +++ b/pkg/common/cmd/root.go @@ -93,7 +93,7 @@ func (r *RootCmd) initEtcd() error { return err } if disConfig.Enable == config.ETCD { - discov, _ := kdisc.NewDiscoveryRegister(&disConfig, env) + discov, _ := kdisc.NewDiscoveryRegister(&disConfig, env, nil) r.etcdClient = discov.(*etcd.SvcDiscoveryRegistryImpl).GetClient() } return nil @@ -113,7 +113,9 @@ func (r *RootCmd) persistentPreRun(cmd *cobra.Command, opts ...func(*CmdOpts)) e if err := r.initializeLogger(cmdOpts); err != nil { return errs.WrapMsg(err, "failed to initialize logger") } - + if err := r.etcdClient.Close(); err != nil { + return errs.WrapMsg(err, "failed to close etcd client") + } return nil } @@ -141,9 +143,24 @@ func (r *RootCmd) updateConfigFromEtcd(opts *CmdOpts) error { if r.etcdClient == nil { return nil } + ctx := context.TODO() + + res, err := r.etcdClient.Get(ctx, disetcd.BuildKey(disetcd.EnableConfigCenterKey)) + if err != nil { + log.ZWarn(ctx, "root cmd updateConfigFromEtcd, etcd Get EnableConfigCenterKey err: %v", errs.Wrap(err)) + return nil + } + if res.Count == 0 { + return nil + } else { + if string(res.Kvs[0].Value) == disetcd.Disable { + return nil + } else if string(res.Kvs[0].Value) != disetcd.Enable { + return errs.New("unknown EnableConfigCenter value").Wrap() + } + } update := func(configFileName string, configStruct any) error { - ctx := context.TODO() key := disetcd.BuildKey(configFileName) etcdRes, err := r.etcdClient.Get(ctx, key) if err != nil { diff --git a/pkg/common/cmd/third.go b/pkg/common/cmd/third.go index 5086116b5..e567234e4 100644 --- a/pkg/common/cmd/third.go +++ b/pkg/common/cmd/third.go @@ -70,6 +70,6 @@ func (a *ThirdRpcCmd) runE() error { a.thirdConfig.MinioConfig.GetConfigFileName(), a.thirdConfig.LocalCacheConfig.GetConfigFileName(), a.thirdConfig.Discovery.GetConfigFileName(), - }, + }, nil, third.Start) } diff --git a/pkg/common/cmd/user.go b/pkg/common/cmd/user.go index 61125e0c3..190f6f892 100644 --- a/pkg/common/cmd/user.go +++ b/pkg/common/cmd/user.go @@ -72,6 +72,6 @@ func (a *UserRpcCmd) runE() error { a.userConfig.WebhooksConfig.GetConfigFileName(), a.userConfig.LocalCacheConfig.GetConfigFileName(), a.userConfig.Discovery.GetConfigFileName(), - }, + }, nil, user.Start) } diff --git a/pkg/common/discovery/discoveryregister.go b/pkg/common/discovery/discoveryregister.go index bc9fd0f5a..1b64c3e78 100644 --- a/pkg/common/discovery/discoveryregister.go +++ b/pkg/common/discovery/discoveryregister.go @@ -28,7 +28,7 @@ import ( ) // NewDiscoveryRegister creates a new service discovery and registry client based on the provided environment type. -func NewDiscoveryRegister(discovery *config.Discovery, runtimeEnv string) (discovery.SvcDiscoveryRegistry, error) { +func NewDiscoveryRegister(discovery *config.Discovery, runtimeEnv string, watchNames []string) (discovery.SvcDiscoveryRegistry, error) { if runtimeEnv == config.KUBERNETES { return kubernetes.NewKubernetesConnManager(discovery.Kubernetes.Namespace, grpc.WithDefaultCallOptions( @@ -42,6 +42,7 @@ func NewDiscoveryRegister(discovery *config.Discovery, runtimeEnv string) (disco return etcd.NewSvcDiscoveryRegistry( discovery.Etcd.RootDirectory, discovery.Etcd.Address, + watchNames, etcd.WithDialTimeout(10*time.Second), etcd.WithMaxCallSendMsgSize(20*1024*1024), etcd.WithUsernameAndPassword(discovery.Etcd.Username, discovery.Etcd.Password)) diff --git a/pkg/common/discovery/etcd/config_manager.go b/pkg/common/discovery/etcd/config_manager.go index 013e2cce3..70d37c323 100644 --- a/pkg/common/discovery/etcd/config_manager.go +++ b/pkg/common/discovery/etcd/config_manager.go @@ -14,11 +14,6 @@ import ( clientv3 "go.etcd.io/etcd/client/v3" ) -const ( - ConfigKeyPrefix = "/open-im/config/" - RestartKey = "restart" -) - var ( ShutDowns []func() error ) diff --git a/pkg/common/discovery/etcd/const.go b/pkg/common/discovery/etcd/const.go new file mode 100644 index 000000000..c9b00fc2c --- /dev/null +++ b/pkg/common/discovery/etcd/const.go @@ -0,0 +1,9 @@ +package etcd + +const ( + ConfigKeyPrefix = "/open-im/config/" + RestartKey = "restart" + EnableConfigCenterKey = "enable-config-center" + Enable = "enable" + Disable = "disable" +) diff --git a/pkg/common/discovery/etcd/doc.go b/pkg/common/discovery/etcd/doc.go deleted file mode 100644 index fedf5ad51..000000000 --- a/pkg/common/discovery/etcd/doc.go +++ /dev/null @@ -1,15 +0,0 @@ -// Copyright © 2024 OpenIM. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package etcd // import "github.com/openimsdk/open-im-server/v3/pkg/common/discovery/etcd" diff --git a/pkg/common/startrpc/start.go b/pkg/common/startrpc/start.go index 5facc8f73..27aabca95 100644 --- a/pkg/common/startrpc/start.go +++ b/pkg/common/startrpc/start.go @@ -49,7 +49,7 @@ import ( // Start rpc server. func Start[T any](ctx context.Context, discovery *conf.Discovery, prometheusConfig *conf.Prometheus, listenIP, registerIP string, autoSetPorts bool, rpcPorts []int, index int, rpcRegisterName string, notification *conf.Notification, config T, - watchConfigNames []string, + watchConfigNames []string, watchServiceNames []string, rpcFn func(ctx context.Context, config T, client discovery.SvcDiscoveryRegistry, server *grpc.Server) error, options ...grpc.ServerOption) error { @@ -95,7 +95,7 @@ func Start[T any](ctx context.Context, discovery *conf.Discovery, prometheusConf if autoSetPorts && discovery.Enable != conf.ETCD { return errs.New("only etcd support autoSetPorts", "rpcRegisterName", rpcRegisterName).Wrap() } - client, err := kdisc.NewDiscoveryRegister(discovery, runTimeEnv) + client, err := kdisc.NewDiscoveryRegister(discovery, runTimeEnv, watchServiceNames) if err != nil { return err } From 1e8a106320137843ab43039fc43f3bfe91f43d56 Mon Sep 17 00:00:00 2001 From: icey-yu <119291641+icey-yu@users.noreply.github.com> Date: Mon, 6 Jan 2025 22:43:21 +0800 Subject: [PATCH 100/199] feat: change appNotificationAccount to appManagerAccount && fix: enable config center add env check && fix: error return (#3038) * feat: change appNotificationAccount to appManagerAccount && fix: enable config center add env check && fix: error return * fix: err --- go.mod | 4 ++-- go.sum | 8 ++++---- internal/api/config_manager.go | 4 ++++ internal/api/router.go | 5 ----- internal/rpc/auth/auth.go | 7 ++++++- internal/rpc/user/user.go | 33 +++++++++++++++++++++------------ 6 files changed, 37 insertions(+), 24 deletions(-) diff --git a/go.mod b/go.mod index 10ff7e6d0..12d546254 100644 --- a/go.mod +++ b/go.mod @@ -12,8 +12,8 @@ require ( github.com/gorilla/websocket v1.5.1 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/mitchellh/mapstructure v1.5.0 - github.com/openimsdk/protocol v0.0.72-alpha.70 - github.com/openimsdk/tools v0.0.50-alpha.64 + github.com/openimsdk/protocol v0.0.72-alpha.71 + github.com/openimsdk/tools v0.0.50-alpha.65 github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 github.com/stretchr/testify v1.9.0 diff --git a/go.sum b/go.sum index 98ba2e7e2..65b5dcc25 100644 --- a/go.sum +++ b/go.sum @@ -347,10 +347,10 @@ github.com/onsi/gomega v1.25.0 h1:Vw7br2PCDYijJHSfBOWhov+8cAnUf8MfMaIOV323l6Y= github.com/onsi/gomega v1.25.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdMPGhM= github.com/openimsdk/gomake v0.0.15-alpha.2 h1:5Q8yl8ezy2yx+q8/ucU/t4kJnDfCzNOrkXcDACCqtyM= github.com/openimsdk/gomake v0.0.15-alpha.2/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= -github.com/openimsdk/protocol v0.0.72-alpha.70 h1:j7vB81+rTthijRda2b8tlli9oWvPxr4yXHwZ8nPZIBQ= -github.com/openimsdk/protocol v0.0.72-alpha.70/go.mod h1:Iet+piS/jaS+kWWyj6EEr36mk4ISzIRYjoMSVA4dq2M= -github.com/openimsdk/tools v0.0.50-alpha.64 h1:KmtE8V2K8atQJJg1xq2ySSrPQyf8ldwk8fw6jRnsxCw= -github.com/openimsdk/tools v0.0.50-alpha.64/go.mod h1:B+oqV0zdewN7OiEHYJm+hW+8/Te7B8tHHgD8rK5ZLZk= +github.com/openimsdk/protocol v0.0.72-alpha.71 h1:R3utzOlqepaJWTAmnfJi4ccUM/XIoFasSyjQMOipM70= +github.com/openimsdk/protocol v0.0.72-alpha.71/go.mod h1:WF7EuE55vQvpyUAzDXcqg+B+446xQyEba0X35lTINmw= +github.com/openimsdk/tools v0.0.50-alpha.65 h1:BRtxkyWxDWPHuHphSwEyHZj7kJSR98am/fHOH84naK8= +github.com/openimsdk/tools v0.0.50-alpha.65/go.mod h1:B+oqV0zdewN7OiEHYJm+hW+8/Te7B8tHHgD8rK5ZLZk= github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= diff --git a/internal/api/config_manager.go b/internal/api/config_manager.go index 7a36bb605..c61b2cb0b 100644 --- a/internal/api/config_manager.go +++ b/internal/api/config_manager.go @@ -259,6 +259,10 @@ func (cm *ConfigManager) restart(c *gin.Context) { } func (cm *ConfigManager) SetEnableConfigManager(c *gin.Context) { + if cm.config.Discovery.Enable != config.ETCD { + apiresp.GinError(c, errs.New("only etcd support config manager").Wrap()) + return + } var req apistruct.SetEnableConfigManagerReq if err := c.BindJSON(&req); err != nil { apiresp.GinError(c, errs.ErrArgs.WithDetail(err.Error()).Wrap()) diff --git a/internal/api/router.go b/internal/api/router.go index 0ae0eab05..da9d22463 100644 --- a/internal/api/router.go +++ b/internal/api/router.go @@ -2,7 +2,6 @@ package api import ( "context" - "fmt" "net/http" "strings" @@ -29,8 +28,6 @@ import ( "github.com/openimsdk/tools/log" "github.com/openimsdk/tools/mw" clientv3 "go.etcd.io/etcd/client/v3" - "google.golang.org/grpc" - "google.golang.org/grpc/credentials/insecure" ) const ( @@ -56,8 +53,6 @@ func prommetricsGin() gin.HandlerFunc { } func newGinRouter(ctx context.Context, client discovery.SvcDiscoveryRegistry, cfg *Config) (*gin.Engine, error) { - client.AddOption(mw.GrpcClient(), grpc.WithTransportCredentials(insecure.NewCredentials()), - grpc.WithDefaultServiceConfig(fmt.Sprintf(`{"LoadBalancingPolicy": "%s"}`, "round_robin"))) authConn, err := client.GetConn(ctx, cfg.Discovery.RpcService.Auth) if err != nil { return nil, err diff --git a/internal/rpc/auth/auth.go b/internal/rpc/auth/auth.go index a2b14f40d..3e096aa64 100644 --- a/internal/rpc/auth/auth.go +++ b/internal/rpc/auth/auth.go @@ -17,6 +17,7 @@ package auth import ( "context" "errors" + "github.com/openimsdk/open-im-server/v3/pkg/rpcli" "github.com/openimsdk/open-im-server/v3/pkg/common/config" @@ -118,9 +119,13 @@ func (s *authServer) GetUserToken(ctx context.Context, req *pbauth.GetUserTokenR if authverify.IsManagerUserID(req.UserID, s.config.Share.IMAdminUserID) { return nil, errs.ErrNoPermission.WrapMsg("don't get Admin token") } - if err := s.userClient.CheckUser(ctx, []string{req.UserID}); err != nil { + user, err := s.userClient.GetUserInfo(ctx, req.UserID) + if err != nil { return nil, err } + if user.AppMangerLevel >= constant.AppNotificationAdmin { + return nil, errs.ErrArgs.WrapMsg("app account can`t get token") + } token, err := s.authDatabase.CreateToken(ctx, req.UserID, int(req.PlatformID)) if err != nil { return nil, err diff --git a/internal/rpc/user/user.go b/internal/rpc/user/user.go index c5ee3be70..15e93a988 100644 --- a/internal/rpc/user/user.go +++ b/internal/rpc/user/user.go @@ -17,7 +17,6 @@ package user import ( "context" "errors" - "github.com/openimsdk/open-im-server/v3/pkg/rpcli" "math/rand" "strings" "sync" @@ -32,6 +31,7 @@ import ( tablerelation "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" "github.com/openimsdk/open-im-server/v3/pkg/common/webhook" "github.com/openimsdk/open-im-server/v3/pkg/localcache" + "github.com/openimsdk/open-im-server/v3/pkg/rpcli" "github.com/openimsdk/protocol/group" friendpb "github.com/openimsdk/protocol/relation" "github.com/openimsdk/tools/db/redisutil" @@ -480,7 +480,9 @@ func (s *userServer) AddNotificationAccount(ctx context.Context, req *pbuser.Add if err := authverify.CheckAdmin(ctx, s.config.Share.IMAdminUserID); err != nil { return nil, err } - + if req.AppMangerLevel < constant.AppNotificationAdmin { + return nil, errs.ErrArgs.WithDetail("app level not supported") + } if req.UserID == "" { for i := 0; i < 20; i++ { userId := s.genUserID() @@ -506,16 +508,17 @@ func (s *userServer) AddNotificationAccount(ctx context.Context, req *pbuser.Add Nickname: req.NickName, FaceURL: req.FaceURL, CreateTime: time.Now(), - AppMangerLevel: constant.AppNotificationAdmin, + AppMangerLevel: req.AppMangerLevel, } if err := s.db.Create(ctx, []*tablerelation.User{user}); err != nil { return nil, err } return &pbuser.AddNotificationAccountResp{ - UserID: req.UserID, - NickName: req.NickName, - FaceURL: req.FaceURL, + UserID: req.UserID, + NickName: req.NickName, + FaceURL: req.FaceURL, + AppMangerLevel: req.AppMangerLevel, }, nil } @@ -595,8 +598,13 @@ func (s *userServer) GetNotificationAccount(ctx context.Context, req *pbuser.Get if err != nil { return nil, servererrs.ErrUserIDNotFound.Wrap() } - if user.AppMangerLevel == constant.AppAdmin || user.AppMangerLevel == constant.AppNotificationAdmin { - return &pbuser.GetNotificationAccountResp{}, nil + if user.AppMangerLevel == constant.AppAdmin || user.AppMangerLevel >= constant.AppNotificationAdmin { + return &pbuser.GetNotificationAccountResp{Account: &pbuser.NotificationAccountInfo{ + UserID: user.UserID, + FaceURL: user.FaceURL, + NickName: user.Nickname, + AppMangerLevel: user.AppMangerLevel, + }}, nil } return nil, errs.ErrNoPermission.WrapMsg("notification messages cannot be sent for this ID") @@ -621,11 +629,12 @@ func (s *userServer) userModelToResp(users []*tablerelation.User, pagination pag accounts := make([]*pbuser.NotificationAccountInfo, 0) var total int64 for _, v := range users { - if v.AppMangerLevel == constant.AppNotificationAdmin && !datautil.Contain(v.UserID, s.config.Share.IMAdminUserID...) { + if v.AppMangerLevel >= constant.AppNotificationAdmin && !datautil.Contain(v.UserID, s.config.Share.IMAdminUserID...) { temp := &pbuser.NotificationAccountInfo{ - UserID: v.UserID, - FaceURL: v.FaceURL, - NickName: v.Nickname, + UserID: v.UserID, + FaceURL: v.FaceURL, + NickName: v.Nickname, + AppMangerLevel: v.AppMangerLevel, } accounts = append(accounts, temp) total += 1 From 3f58f3e904c8d9ca8d7f309463774b031b12dc47 Mon Sep 17 00:00:00 2001 From: chao <48119764+withchao@users.noreply.github.com> Date: Tue, 7 Jan 2025 11:26:42 +0800 Subject: [PATCH 101/199] fix: GetUsersOnline returns an error in the online list (#3040) * pb * fix: Modifying other fields while setting IsPrivateChat does not take effect * fix: quote message error revoke * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * upgrading pkg tools * fix * fix * optimize log output * feat: support GetLastMessage * feat: support GetLastMessage * feat: s3 switch * feat: s3 switch * fix: GetUsersOnline --- pkg/rpccache/online.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/rpccache/online.go b/pkg/rpccache/online.go index 32b2f2889..b5308bbe8 100644 --- a/pkg/rpccache/online.go +++ b/pkg/rpccache/online.go @@ -270,8 +270,8 @@ func (o *OnlineCache) GetUsersOnline(ctx context.Context, userIDs []string) ([]s } } - log.ZInfo(ctx, "get users online", "online users length", len(userIDs), "offline users length", len(offlineUserIDs), "cost", time.Since(t)) - return userIDs, offlineUserIDs, nil + log.ZInfo(ctx, "get users online", "online users length", len(onlineUserIDs), "offline users length", len(offlineUserIDs), "cost", time.Since(t)) + return onlineUserIDs, offlineUserIDs, nil } func (o *OnlineCache) setUserOnline(userID string, platformIDs []int32) { From 2c747c95a035512d6c5a583e32ec1e6bb298776b Mon Sep 17 00:00:00 2001 From: chao <48119764+withchao@users.noreply.github.com> Date: Thu, 9 Jan 2025 17:16:36 +0800 Subject: [PATCH 102/199] feat: SendBusinessNotification supported configuration parameters (#3048) * pb * fix: Modifying other fields while setting IsPrivateChat does not take effect * fix: quote message error revoke * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * upgrading pkg tools * fix * fix * optimize log output * feat: support GetLastMessage * feat: support GetLastMessage * feat: s3 switch * feat: s3 switch * fix: GetUsersOnline * feat: SendBusinessNotification supported configuration parameters * feat: SendBusinessNotification supported configuration parameters * feat: SendBusinessNotification supported configuration parameters --- internal/api/msg.go | 40 ++++++++++++++++++++++++++++++---------- 1 file changed, 30 insertions(+), 10 deletions(-) diff --git a/internal/api/msg.go b/internal/api/msg.go index fc235354c..b21e792db 100644 --- a/internal/api/msg.go +++ b/internal/api/msg.go @@ -248,24 +248,44 @@ func (m *MessageApi) SendMessage(c *gin.Context) { func (m *MessageApi) SendBusinessNotification(c *gin.Context) { req := struct { - Key string `json:"key"` - Data string `json:"data"` - SendUserID string `json:"sendUserID" binding:"required"` - RecvUserID string `json:"recvUserID" binding:"required"` + Key string `json:"key"` + Data string `json:"data"` + SendUserID string `json:"sendUserID" binding:"required"` + RecvUserID string `json:"recvUserID"` + RecvGroupID string `json:"recvGroupID"` + SendMsg bool `json:"sendMsg"` + ReliabilityLevel *int `json:"reliabilityLevel"` }{} if err := c.BindJSON(&req); err != nil { apiresp.GinError(c, errs.ErrArgs.WithDetail(err.Error()).Wrap()) return } - + if req.RecvUserID == "" && req.RecvGroupID == "" { + apiresp.GinError(c, errs.ErrArgs.WrapMsg("recvUserID and recvGroupID cannot be empty at the same time")) + return + } + if req.RecvUserID != "" && req.RecvGroupID != "" { + apiresp.GinError(c, errs.ErrArgs.WrapMsg("recvUserID and recvGroupID cannot be set at the same time")) + return + } + var sessionType int32 + if req.RecvUserID != "" { + sessionType = constant.SingleChatType + } else { + sessionType = constant.ReadGroupChatType + } + if req.ReliabilityLevel == nil { + req.ReliabilityLevel = datautil.ToPtr(1) + } if !authverify.IsAppManagerUid(c, m.imAdminUserID) { apiresp.GinError(c, errs.ErrNoPermission.WrapMsg("only app manager can send message")) return } sendMsgReq := msg.SendMsgReq{ MsgData: &sdkws.MsgData{ - SendID: req.SendUserID, - RecvID: req.RecvUserID, + SendID: req.SendUserID, + RecvID: req.RecvUserID, + GroupID: req.RecvGroupID, Content: []byte(jsonutil.StructToJsonString(&sdkws.NotificationElem{ Detail: jsonutil.StructToJsonString(&struct { Key string `json:"key"` @@ -274,12 +294,12 @@ func (m *MessageApi) SendBusinessNotification(c *gin.Context) { })), MsgFrom: constant.SysMsgType, ContentType: constant.BusinessNotification, - SessionType: constant.SingleChatType, + SessionType: sessionType, CreateTime: timeutil.GetCurrentTimestampByMill(), ClientMsgID: idutil.GetMsgIDByMD5(mcontext.GetOpUserID(c)), Options: config.GetOptionsByNotification(config.NotificationConfig{ - IsSendMsg: false, - ReliabilityLevel: 1, + IsSendMsg: req.SendMsg, + ReliabilityLevel: *req.ReliabilityLevel, UnreadCount: false, }), }, From 7a1e347776661f6d8d884c2251f8fd5311558861 Mon Sep 17 00:00:00 2001 From: chao <48119764+withchao@users.noreply.github.com> Date: Fri, 10 Jan 2025 11:15:34 +0800 Subject: [PATCH 103/199] fix: seq conversion failed without exiting (#3052) * pb * fix: Modifying other fields while setting IsPrivateChat does not take effect * fix: quote message error revoke * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * upgrading pkg tools * fix * fix * optimize log output * feat: support GetLastMessage * feat: support GetLastMessage * feat: s3 switch * feat: s3 switch * fix: GetUsersOnline * feat: SendBusinessNotification supported configuration parameters * feat: SendBusinessNotification supported configuration parameters * feat: SendBusinessNotification supported configuration parameters * feat: seq conversion failed without exiting --- tools/seq/main.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tools/seq/main.go b/tools/seq/main.go index 16da9f156..ef2875fe1 100644 --- a/tools/seq/main.go +++ b/tools/seq/main.go @@ -3,8 +3,10 @@ package main import ( "flag" "fmt" - "github.com/openimsdk/open-im-server/v3/tools/seq/internal" + "os" "time" + + "github.com/openimsdk/open-im-server/v3/tools/seq/internal" ) func main() { @@ -17,6 +19,8 @@ func main() { flag.Parse() if err := internal.Main(config, time.Duration(second)*time.Second); err != nil { fmt.Println("seq task", err) + os.Exit(1) + return } fmt.Println("seq task success!") } From f7a1d5be3445f36970b5773549cec0adaaa80c8c Mon Sep 17 00:00:00 2001 From: icey-yu <119291641+icey-yu@users.noreply.github.com> Date: Fri, 10 Jan 2025 17:27:36 +0800 Subject: [PATCH 104/199] update: env (#3055) --- .env | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.env b/.env index a615e5096..dd0e89c39 100644 --- a/.env +++ b/.env @@ -8,12 +8,12 @@ ALERTMANAGER_IMAGE=prom/alertmanager:v0.27.0 GRAFANA_IMAGE=grafana/grafana:11.0.1 NODE_EXPORTER_IMAGE=prom/node-exporter:v1.7.0 -OPENIM_WEB_FRONT_IMAGE=openim/openim-web-front:release-v3.8.1 -OPENIM_ADMIN_FRONT_IMAGE=openim/openim-admin-front:release-v1.8.3 +OPENIM_WEB_FRONT_IMAGE=openim/openim-web-front:release-v3.8.3 +OPENIM_ADMIN_FRONT_IMAGE=openim/openim-admin-front:release-v1.8.4 #FRONT_IMAGE: use aliyun images -#OPENIM_WEB_FRONT_IMAGE=registry.cn-hangzhou.aliyuncs.com/openimsdk/openim-web-front:release-v3.8.1 -#OPENIM_ADMIN_FRONT_IMAGE=registry.cn-hangzhou.aliyuncs.com/openimsdk/openim-admin-front:release-v1.8.3 +#OPENIM_WEB_FRONT_IMAGE=registry.cn-hangzhou.aliyuncs.com/openimsdk/openim-web-front:release-v3.8.3 +#OPENIM_ADMIN_FRONT_IMAGE=registry.cn-hangzhou.aliyuncs.com/openimsdk/openim-admin-front:release-v1.8.4 DATA_DIR=./ From 47e916aebedab95995295cd184b391c1c0f9d617 Mon Sep 17 00:00:00 2001 From: icey-yu <119291641+icey-yu@users.noreply.github.com> Date: Fri, 17 Jan 2025 09:38:55 +0800 Subject: [PATCH 105/199] feat: add backup volume && optimize log print (#3066) * feat: backup * feat: backup * feat: backup * feat: optimize log print --- .env | 2 ++ docker-compose.yml | 1 + go.mod | 2 +- go.sum | 4 ++-- internal/push/push_handler.go | 27 ++++++++++++++------------- 5 files changed, 20 insertions(+), 16 deletions(-) diff --git a/.env b/.env index dd0e89c39..0ab998037 100644 --- a/.env +++ b/.env @@ -17,6 +17,8 @@ OPENIM_ADMIN_FRONT_IMAGE=openim/openim-admin-front:release-v1.8.4 DATA_DIR=./ +MONGO_BACKUP_DIR=${DATA_DIR}components/backup/mongo/ + PROMETHEUS_PORT=19091 ALERTMANAGER_PORT=19093 GRAFANA_PORT=13000 \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 0cdeebe43..65b4e6625 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -37,6 +37,7 @@ services: - "${DATA_DIR}/components/mongodb/data/db:/data/db" - "${DATA_DIR}/components/mongodb/data/logs:/data/logs" - "${DATA_DIR}/components/mongodb/data/conf:/etc/mongo" + - "${MONGO_BACKUP_DIR}:/data/backup" environment: - TZ=Asia/Shanghai - wiredTigerCacheSizeGB=1 diff --git a/go.mod b/go.mod index 12d546254..fd188e978 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/mitchellh/mapstructure v1.5.0 github.com/openimsdk/protocol v0.0.72-alpha.71 - github.com/openimsdk/tools v0.0.50-alpha.65 + github.com/openimsdk/tools v0.0.50-alpha.70 github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 github.com/stretchr/testify v1.9.0 diff --git a/go.sum b/go.sum index 65b5dcc25..4f3431133 100644 --- a/go.sum +++ b/go.sum @@ -349,8 +349,8 @@ github.com/openimsdk/gomake v0.0.15-alpha.2 h1:5Q8yl8ezy2yx+q8/ucU/t4kJnDfCzNOrk github.com/openimsdk/gomake v0.0.15-alpha.2/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= github.com/openimsdk/protocol v0.0.72-alpha.71 h1:R3utzOlqepaJWTAmnfJi4ccUM/XIoFasSyjQMOipM70= github.com/openimsdk/protocol v0.0.72-alpha.71/go.mod h1:WF7EuE55vQvpyUAzDXcqg+B+446xQyEba0X35lTINmw= -github.com/openimsdk/tools v0.0.50-alpha.65 h1:BRtxkyWxDWPHuHphSwEyHZj7kJSR98am/fHOH84naK8= -github.com/openimsdk/tools v0.0.50-alpha.65/go.mod h1:B+oqV0zdewN7OiEHYJm+hW+8/Te7B8tHHgD8rK5ZLZk= +github.com/openimsdk/tools v0.0.50-alpha.70 h1:pyqWkJzXbELWU9KKAsWkj3g0flJYNsDTcjR5SLFQAZU= +github.com/openimsdk/tools v0.0.50-alpha.70/go.mod h1:B+oqV0zdewN7OiEHYJm+hW+8/Te7B8tHHgD8rK5ZLZk= github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= diff --git a/internal/push/push_handler.go b/internal/push/push_handler.go index d5d457c0d..80d14499f 100644 --- a/internal/push/push_handler.go +++ b/internal/push/push_handler.go @@ -3,11 +3,12 @@ package push import ( "context" "encoding/json" - "github.com/openimsdk/open-im-server/v3/pkg/rpcli" "math/rand" "strconv" "time" + "github.com/openimsdk/open-im-server/v3/pkg/rpcli" + "github.com/IBM/sarama" "github.com/openimsdk/open-im-server/v3/internal/push/offlinepush" "github.com/openimsdk/open-im-server/v3/internal/push/offlinepush/options" @@ -152,24 +153,24 @@ func (c *ConsumerHandler) Push2User(ctx context.Context, userIDs []string, msg * log.ZInfo(ctx, "Get msg from msg_transfer And push msg", "userIDs", userIDs, "msg", msg.String()) defer func(duration time.Time) { t := time.Since(duration) - log.ZInfo(ctx, "Get msg from msg_transfer And push msg", "msg", msg.String(), "time cost", t) + log.ZInfo(ctx, "Get msg from msg_transfer And push msg end", "msg", msg.String(), "time cost", t) }(time.Now()) if err := c.webhookBeforeOnlinePush(ctx, &c.config.WebhooksConfig.BeforeOnlinePush, userIDs, msg); err != nil { return err } - log.ZInfo(ctx, "webhookBeforeOnlinePush end") wsResults, err := c.GetConnsAndOnlinePush(ctx, msg, userIDs) if err != nil { return err } - log.ZInfo(ctx, "single and notification push result", "result", wsResults, "msg", msg, "push_to_userID", userIDs) + log.ZDebug(ctx, "single and notification push result", "result", wsResults, "msg", msg, "push_to_userID", userIDs) + log.ZInfo(ctx, "single and notification push end") if !c.shouldPushOffline(ctx, msg) { return nil } - log.ZInfo(ctx, "shouldPushOffline end") + log.ZInfo(ctx, "pushOffline start") for _, v := range wsResults { //message sender do not need offline push @@ -188,14 +189,14 @@ func (c *ConsumerHandler) Push2User(ctx context.Context, userIDs []string, msg * if err = c.webhookBeforeOfflinePush(ctx, &c.config.WebhooksConfig.BeforeOfflinePush, needOfflinePushUserID, msg, &offlinePushUserID); err != nil { return err } - log.ZInfo(ctx, "webhookBeforeOfflinePush end") if len(offlinePushUserID) > 0 { needOfflinePushUserID = offlinePushUserID } err = c.offlinePushMsg(ctx, msg, needOfflinePushUserID) if err != nil { - log.ZWarn(ctx, "offlinePushMsg failed", err, "needOfflinePushUserID", needOfflinePushUserID, "msg", msg) + log.ZDebug(ctx, "offlinePushMsg failed", err, "needOfflinePushUserID", needOfflinePushUserID, "msg", msg) + log.ZWarn(ctx, "offlinePushMsg failed", err, "needOfflinePushUserID length", len(needOfflinePushUserID), "msg", msg) return nil } @@ -250,26 +251,24 @@ func (c *ConsumerHandler) Push2Group(ctx context.Context, groupID string, msg *s &pushToUserIDs); err != nil { return err } - log.ZInfo(ctx, "webhookBeforeGroupOnlinePush end") err = c.groupMessagesHandler(ctx, groupID, &pushToUserIDs, msg) if err != nil { return err } - log.ZInfo(ctx, "groupMessagesHandler end") wsResults, err := c.GetConnsAndOnlinePush(ctx, msg, pushToUserIDs) if err != nil { return err } - log.ZInfo(ctx, "group push result", "result", wsResults, "msg", msg) + log.ZDebug(ctx, "group push result", "result", wsResults, "msg", msg) + log.ZInfo(ctx, "online group push end") if !c.shouldPushOffline(ctx, msg) { return nil } needOfflinePushUserIDs := c.onlinePusher.GetOnlinePushFailedUserIDs(ctx, msg, wsResults, &pushToUserIDs) - log.ZInfo(ctx, "GetOnlinePushFailedUserIDs end") //filter some user, like don not disturb or don't need offline push etc. needOfflinePushUserIDs, err = c.filterGroupMessageOfflinePush(ctx, groupID, msg, needOfflinePushUserIDs) if err != nil { @@ -297,9 +296,11 @@ func (c *ConsumerHandler) asyncOfflinePush(ctx context.Context, needOfflinePushU needOfflinePushUserIDs = offlinePushUserIDs } if err := c.pushDatabase.MsgToOfflinePushMQ(ctx, conversationutil.GenConversationUniqueKeyForSingle(msg.SendID, msg.RecvID), needOfflinePushUserIDs, msg); err != nil { - log.ZError(ctx, "Msg To OfflinePush MQ error", err, "needOfflinePushUserIDs", + log.ZDebug(ctx, "Msg To OfflinePush MQ error", err, "needOfflinePushUserIDs", needOfflinePushUserIDs, "msg", msg) - prommetrics.SingleChatMsgProcessFailedCounter.Inc() + log.ZWarn(ctx, "Msg To OfflinePush MQ error", err, "needOfflinePushUserIDs length", + len(needOfflinePushUserIDs), "msg", msg) + prommetrics.GroupChatMsgProcessFailedCounter.Inc() return } } From 96baa5a0ffac3d7c4d4727876ffe418ec6be4e3b Mon Sep 17 00:00:00 2001 From: icey-yu <119291641+icey-yu@users.noreply.github.com> Date: Wed, 22 Jan 2025 14:38:27 +0800 Subject: [PATCH 106/199] fix: check error in BatchSetTokenMapByUidPid (#3076) --- pkg/common/storage/controller/auth.go | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/pkg/common/storage/controller/auth.go b/pkg/common/storage/controller/auth.go index 2885b985a..ee2a06f53 100644 --- a/pkg/common/storage/controller/auth.go +++ b/pkg/common/storage/controller/auth.go @@ -60,16 +60,15 @@ func (a *authDatabase) BatchSetTokenMapByUidPid(ctx context.Context, tokens []st setMap := make(map[string]map[string]any) for _, token := range tokens { claims, err := tokenverify.GetClaimFromToken(token, authverify.Secret(a.accessSecret)) - key := cachekey.GetTokenKey(claims.UserID, claims.PlatformID) if err != nil { continue + } + key := cachekey.GetTokenKey(claims.UserID, claims.PlatformID) + if v, ok := setMap[key]; ok { + v[token] = constant.KickedToken } else { - if v, ok := setMap[key]; ok { - v[token] = constant.KickedToken - } else { - setMap[key] = map[string]any{ - token: constant.KickedToken, - } + setMap[key] = map[string]any{ + token: constant.KickedToken, } } } From bbb5473d26a847a4a35b01a3c9d70da16234591d Mon Sep 17 00:00:00 2001 From: chao <48119764+withchao@users.noreply.github.com> Date: Wed, 22 Jan 2025 15:24:40 +0800 Subject: [PATCH 107/199] fix: DeleteDoc crash (#3078) * pb * fix: Modifying other fields while setting IsPrivateChat does not take effect * fix: quote message error revoke * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * upgrading pkg tools * fix * fix * optimize log output * feat: support GetLastMessage * feat: support GetLastMessage * feat: s3 switch * feat: s3 switch * fix: GetUsersOnline * feat: SendBusinessNotification supported configuration parameters * feat: SendBusinessNotification supported configuration parameters * feat: SendBusinessNotification supported configuration parameters * feat: seq conversion failed without exiting * fix: DeleteDoc crash --- pkg/common/storage/controller/msg.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pkg/common/storage/controller/msg.go b/pkg/common/storage/controller/msg.go index a93d581eb..b92f9b510 100644 --- a/pkg/common/storage/controller/msg.go +++ b/pkg/common/storage/controller/msg.go @@ -18,11 +18,12 @@ import ( "context" "encoding/json" "errors" - "github.com/openimsdk/tools/utils/jsonutil" "strconv" "strings" "time" + "github.com/openimsdk/tools/utils/jsonutil" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" @@ -722,13 +723,13 @@ func (db *commonMsgDatabase) DeleteDoc(ctx context.Context, docID string) error if index <= 0 { return errs.ErrInternalServer.WrapMsg("docID is invalid", "docID", docID) } - index, err := strconv.Atoi(docID[index+1:]) + docIndex, err := strconv.Atoi(docID[index+1:]) if err != nil { return errs.WrapMsg(err, "strconv.Atoi", "docID", docID) } conversationID := docID[:index] seqs := make([]int64, db.msgTable.GetSingleGocMsgNum()) - minSeq := db.msgTable.GetMinSeq(index) + minSeq := db.msgTable.GetMinSeq(docIndex) for i := range seqs { seqs[i] = minSeq + int64(i) } From 83c7b7134c78dffdc470a9a19d6e4be4fb0e7c5e Mon Sep 17 00:00:00 2001 From: Monet Lee Date: Thu, 23 Jan 2025 15:18:39 +0800 Subject: [PATCH 108/199] refactor: improve workflows logic. (#3072) * refactor: improve workflows logic. * update args. * remove unused contents. * update milestone merge contents. * update contents. --- ...cker-build-and-release-services-images.yml | 13 +- .github/workflows/go-build-test.yml | 137 ++++++++++---- .github/workflows/merge-from-milestone.yml | 178 ++++++------------ .github/workflows/publish-docker-image.yml | 85 ++++----- 4 files changed, 197 insertions(+), 216 deletions(-) diff --git a/.github/workflows/docker-build-and-release-services-images.yml b/.github/workflows/docker-build-and-release-services-images.yml index ed4c7ad38..407589f1e 100644 --- a/.github/workflows/docker-build-and-release-services-images.yml +++ b/.github/workflows/docker-build-and-release-services-images.yml @@ -19,26 +19,26 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3.8.0 - name: Log in to Docker Hub - uses: docker/login-action@v2 + uses: docker/login-action@v3.3.0 with: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} - name: Log in to GitHub Container Registry - uses: docker/login-action@v2 + uses: docker/login-action@v3.3.0 with: registry: ghcr.io username: ${{ github.repository_owner }} password: ${{ secrets.GITHUB_TOKEN }} - name: Log in to Aliyun Container Registry - uses: docker/login-action@v2 + uses: docker/login-action@v3.3.0 with: registry: registry.cn-hangzhou.aliyuncs.com username: ${{ secrets.ALIREGISTRY_USERNAME }} @@ -46,7 +46,7 @@ jobs: - name: Extract metadata for Docker (tags, labels) id: meta - uses: docker/metadata-action@v5 + uses: docker/metadata-action@v5.6.0 with: tags: | type=ref,event=tag @@ -54,7 +54,6 @@ jobs: type=ref,event=branch type=semver,pattern={{version}} type=semver,pattern=v{{version}} - # type=semver,pattern={{major}}.{{minor}} type=semver,pattern=release-{{raw}} type=sha type=raw,value=${{ github.event.inputs.tag }} diff --git a/.github/workflows/go-build-test.yml b/.github/workflows/go-build-test.yml index 10a4154d6..4033603e6 100644 --- a/.github/workflows/go-build-test.yml +++ b/.github/workflows/go-build-test.yml @@ -4,7 +4,7 @@ on: push: pull_request: paths-ignore: - - '**/*.md' + - "**/*.md" workflow_dispatch: @@ -18,7 +18,7 @@ jobs: strategy: matrix: os: [ubuntu-latest] - go_version: ["1.21.x", "1.22.x"] + go_version: ["1.22.x"] steps: - name: Checkout Server repository @@ -37,27 +37,20 @@ jobs: - name: Set up infra services uses: hoverkraft-tech/compose-action@v2.0.1 - # Uncomment and set the correct path to your docker-compose file with: compose-file: "./docker-compose.yml" - # run: | - # sudo docker compose up -d - # sudo sleep 30 # Increased sleep time for better stability - # timeout-minutes: 60 # Increased timeout for Docker setup - + # - name: Get Internal IP Address + # id: get-ip + # run: | + # IP=$(hostname -I | awk '{print $1}') + # echo "The IP Address is: $IP" + # echo "::set-output name=ip::$IP" - # - name: Get Internal IP Address - # id: get-ip - # run: | - # IP=$(hostname -I | awk '{print $1}') - # echo "The IP Address is: $IP" - # echo "::set-output name=ip::$IP" - - # - name: Update .env - # run: | - # sed -i 's|externalAddress:.*|externalAddress: "http://${{ steps.get-ip.outputs.ip }}:10005"|' config/minio.yml - # cat config/minio.yml + # - name: Update .env + # run: | + # sed -i 's|externalAddress:.*|externalAddress: "http://${{ steps.get-ip.outputs.ip }}:10005"|' config/minio.yml + # cat config/minio.yml - name: Build and test Server Services run: | @@ -85,6 +78,90 @@ jobs: mage start mage check + - name: Test Server and Chat + run: | + check_error() { + echo "Response: $1" + errCode=$(echo $1 | jq -r '.errCode') + if [ "$errCode" != "0" ]; then + errMsg=$(echo $1 | jq -r '.errMsg') + echo "Error: $errMsg" + exit 1 + fi + } + + # Test register + response1=$(curl -X POST -H "Content-Type: application/json" -H "operationID: imAdmin" -d '{ + "verifyCode": "666666", + "platform": 3, + "autoLogin": true, + "user":{ + "nickname": "test12312", + "areaCode":"+86", + "phoneNumber": "12345678190", + "password":"test123456" + } + }' http://127.0.0.1:10008/account/register) + check_error "$response1" + userID1=$(echo $response1 | jq -r '.data.userID') + echo "userID1: $userID1" + + response2=$(curl -X POST -H "Content-Type: application/json" -H "operationID: imAdmin" -d '{ + "verifyCode": "666666", + "platform": 3, + "autoLogin": true, + "user":{ + "nickname": "test22312", + "areaCode":"+86", + "phoneNumber": "12345678290", + "password":"test123456" + } + }' http://127.0.0.1:10008/account/register) + check_error "$response2" + userID2=$(echo $response2 | jq -r '.data.userID') + echo "userID2: $userID2" + + # Test login + login_response=$(curl -X POST -H "Content-Type: application/json" -H "operationID: imAdmin" -d '{ + "platform": 3, + "areaCode":"+86", + "phoneNumber": "12345678190", + "password":"test123456" + }' http://localhost:10008/account/login) + check_error "$login_response" + + # Test get admin token + get_admin_token_response=$(curl -X POST -H "Content-Type: application/json" -H "operationID: imAdmin" -d '{ + "secret": "openIM123", + "platformID": 2, + "userID": "imAdmin" + }' http://127.0.0.1:10002/auth/get_admin_token) + check_error "$get_admin_token_response" + adminToken=$(echo $get_admin_token_response | jq -r '.data.token') + echo "adminToken: $adminToken" + + # Test send message + send_msg_response=$(curl -X POST -H "Content-Type: application/json" -H "operationID: imAdmin" -H "token: $adminToken" -d '{ + "sendID": "'$userID1'", + "recvID": "'$userID2'", + "senderPlatformID": 3, + "content": { + "content": "hello!!" + }, + "contentType": 101, + "sessionType": 1 + }' http://127.0.0.1:10002/msg/send_msg) + check_error "$send_msg_response" + + # Test get users + get_users_response=$(curl -X POST -H "Content-Type: application/json" -H "operationID: imAdmin" -H "token: $adminToken" -d '{ + "pagination": { + "pageNumber": 1, + "showNumber": 100 + } + }' http://127.0.0.1:10002/user/get_users) + check_error "$get_users_response" + go-test: name: Benchmark Test with go ${{ matrix.go_version }} on ${{ matrix.os }} runs-on: ${{ matrix.os }} @@ -93,11 +170,11 @@ jobs: env: SDK_DIR: openim-sdk-core CONFIG_PATH: config/notification.yml - # pull-requests: write + strategy: matrix: - os: [ ubuntu-latest ] - go_version: [ "1.22.x" ] + os: [ubuntu-latest] + go_version: ["1.22.x"] steps: - name: Checkout Server repository @@ -106,7 +183,8 @@ jobs: - name: Checkout SDK repository uses: actions/checkout@v4 with: - repository: 'openimsdk/openim-sdk-core' + repository: "openimsdk/openim-sdk-core" + ref: "release-v3.8" path: ${{ env.SDK_DIR }} - name: Set up Go ${{ matrix.go_version }} @@ -119,11 +197,6 @@ jobs: go install github.com/magefile/mage@latest go mod download - - name: Install yq - run: | - sudo wget https://github.com/mikefarah/yq/releases/download/v4.34.1/yq_linux_amd64 -O /usr/bin/yq - sudo chmod +x /usr/bin/yq - - name: Modify Server Configuration run: | yq e '.groupCreated.isSendMsg = true' -i ${{ env.CONFIG_PATH }} @@ -183,11 +256,3 @@ jobs: run: | CONTAINER_NAME="${{ github.event.repository.name }}-container" docker logs $CONTAINER_NAME - - # - name: Cleanup Docker Container - # run: | - # CONTAINER_NAME="${{ github.event.repository.name }}-container" - # IMAGE_NAME="${{ github.event.repository.name }}-test" - # docker stop $CONTAINER_NAME - # docker rm $CONTAINER_NAME - # docker rmi $IMAGE_NAME diff --git a/.github/workflows/merge-from-milestone.yml b/.github/workflows/merge-from-milestone.yml index 44b4f81f4..8e09c9ab7 100644 --- a/.github/workflows/merge-from-milestone.yml +++ b/.github/workflows/merge-from-milestone.yml @@ -1,4 +1,4 @@ -name: Create Pre-Release PR from Milestone +name: Create Individual PRs from Milestone permissions: contents: write @@ -9,24 +9,24 @@ on: workflow_dispatch: inputs: milestone_name: - description: 'Milestone name to collect closed PRs from' + description: "Milestone name to collect closed PRs from" required: true - default: 'v3.8.2' + default: "v3.8.4" target_branch: - description: 'Target branch to merge the consolidated PR' + description: "Target branch to merge the consolidated PR" required: true - default: 'pre-release-v3.8.2' + default: "pre-release-v3.8.4" env: - MILESTONE_NAME: ${{ github.event.inputs.milestone_name || 'v3.8.2' }} - TARGET_BRANCH: ${{ github.event.inputs.target_branch || 'pre-release-v3.8.2' }} + MILESTONE_NAME: ${{ github.event.inputs.milestone_name || 'v3.8.4' }} + TARGET_BRANCH: ${{ github.event.inputs.target_branch || 'pre-release-v3.8.4' }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} BOT_TOKEN: ${{ secrets.BOT_TOKEN }} LABEL_NAME: cherry-picked - TEMP_DIR: /tmp # Using /tmp as the temporary directory + TEMP_DIR: /tmp jobs: - cherry_pick_milestone_prs: + merge_milestone_prs: runs-on: ubuntu-latest steps: - name: Setup temp directory @@ -47,7 +47,6 @@ jobs: - name: Setup Git User for OpenIM-Robot run: | - # Set up Git credentials for the bot git config --global user.email "OpenIM-Robot@users.noreply.github.com" git config --global user.name "OpenIM-Robot" @@ -83,136 +82,65 @@ jobs: if ! echo "$labels" | grep -q "${LABEL_NAME}"; then echo "PR #$pr_number does not have the 'cherry-picked' label. Adding to the list." echo "$pr_number" >> ${{ env.TEMP_DIR }}/pr_numbers.txt - else - echo "PR #$pr_number already has the 'cherry-picked' label. Skipping." fi done - # Sort the filtered PR numbers sort -n ${{ env.TEMP_DIR }}/pr_numbers.txt -o ${{ env.TEMP_DIR }}/pr_numbers.txt - echo "Filtered and sorted PR numbers:" - cat ${{ env.TEMP_DIR }}/pr_numbers.txt || echo "No closed PR numbers found for milestone." - - - name: Fetch Merge Commits for PRs and Generate Title and Body + - name: Create Individual PRs run: | - # Ensure the files are initialized - > ${{ env.TEMP_DIR }}/commit_hashes.txt - > ${{ env.TEMP_DIR }}/pr_title.txt - > ${{ env.TEMP_DIR }}/pr_body.txt - - # Write description to the PR body - echo "### Description:" >> ${{ env.TEMP_DIR }}/pr_body.txt - echo "Merging PRs from milestone \`$MILESTONE_NAME\` into target branch \`$TARGET_BRANCH\`." >> ${{ env.TEMP_DIR }}/pr_body.txt - echo "" >> ${{ env.TEMP_DIR }}/pr_body.txt - echo "### Need Merge PRs:" >> ${{ env.TEMP_DIR }}/pr_body.txt - - pr_numbers_in_title="" - - # Process sorted PR numbers and generate commit hashes for pr_number in $(cat ${{ env.TEMP_DIR }}/pr_numbers.txt); do - echo "Processing PR #$pr_number" pr_details=$(curl -s -H "Authorization: token $BOT_TOKEN" \ -H "Accept: application/vnd.github+json" \ "https://api.github.com/repos/${{ github.repository }}/pulls/$pr_number") pr_title=$(echo "$pr_details" | jq -r '.title') + pr_body=$(echo "$pr_details" | jq -r '.body') + pr_creator=$(echo "$pr_details" | jq -r '.user.login') merge_commit=$(echo "$pr_details" | jq -r '.merge_commit_sha') short_commit_hash=$(echo "$merge_commit" | cut -c 1-7) - # Append PR details to the body - echo "- $pr_title: (#$pr_number) ($short_commit_hash)" >> ${{ env.TEMP_DIR }}/pr_body.txt - - if [ "$merge_commit" != "null" ];then - echo "$merge_commit" >> ${{ env.TEMP_DIR }}/commit_hashes.txt - echo "#$pr_number" >> ${{ env.TEMP_DIR }}/pr_title.txt - pr_numbers_in_title="$pr_numbers_in_title #$pr_number" - fi - done + if [ "$merge_commit" != "null" ]; then + git fetch origin + + echo "Checking out target branch: $TARGET_BRANCH" + git checkout $TARGET_BRANCH + + echo "Pulling latest changes from target branch: $TARGET_BRANCH" + git pull origin $TARGET_BRANCH + + cherry_pick_branch="cherry-pick-${short_commit_hash}" + git checkout -b $cherry_pick_branch + + echo "Cherry-picking commit: $merge_commit" + if ! git cherry-pick "$merge_commit" --strategy=recursive -X theirs; then + echo "Cherry-pick encountered conflicts, attempting to continue..." + git cherry-pick --continue || { echo "Cherry-pick failed"; exit 1; } + fi + + git remote set-url origin "https://${BOT_TOKEN}@github.com/${{ github.repository }}.git" + + echo "Pushing branch: $cherry_pick_branch" + git push origin $cherry_pick_branch --force || { echo "Push failed"; exit 1; } + + new_pr_title="$pr_title [Created by @$pr_creator from #$pr_number]" + new_pr_body="$pr_body + > This PR is created from original PR #$pr_number." + + response=$(curl -s -X POST -H "Authorization: token $BOT_TOKEN" \ + -H "Accept: application/vnd.github+json" \ + https://api.github.com/repos/${{ github.repository }}/pulls \ + -d "$(jq -n --arg title "$new_pr_title" \ + --arg head "$cherry_pick_branch" \ + --arg base "$TARGET_BRANCH" \ + --arg body "$new_pr_body" \ + '{title: $title, head: $head, base: $base, body: $body}')") - commit_hashes=$(cat ${{ env.TEMP_DIR }}/commit_hashes.txt | tr '\n' ' ') - first_commit_hash=$(head -n 1 ${{ env.TEMP_DIR }}/commit_hashes.txt) - cherry_pick_branch="cherry-pick-${first_commit_hash:0:7}" - echo "COMMIT_HASHES=$commit_hashes" >> $GITHUB_ENV - echo "CHERRY_PICK_BRANCH=$cherry_pick_branch" >> $GITHUB_ENV - echo "pr_numbers_in_title=$pr_numbers_in_title" >> $GITHUB_ENV + new_pr_number=$(echo "$response" | jq -r '.number') + echo "Created PR #$new_pr_number" - - name: Pull and Cherry-pick Commits, Then Push - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - BOT_TOKEN: ${{ secrets.BOT_TOKEN }} - run: | - # Fetch and pull the latest changes from the target branch - git fetch origin - git checkout $TARGET_BRANCH - git pull origin $TARGET_BRANCH - - # Create a new branch for cherry-picking - git checkout -b $CHERRY_PICK_BRANCH - - # Cherry-pick the commits and handle conflicts - for commit_hash in $COMMIT_HASHES; do - echo "Attempting to cherry-pick commit $commit_hash" - if ! git cherry-pick "$commit_hash" --strategy=recursive -X theirs; then - echo "Conflict detected for $commit_hash. Resolving with incoming changes." - conflict_files=$(git diff --name-only --diff-filter=U) - echo "Conflicting files:" - echo "$conflict_files" - - for file in $conflict_files; do - if [ -f "$file" ]; then - echo "Resolving conflict for $file" - git add "$file" - else - echo "File $file has been deleted. Skipping." - git rm "$file" - fi - done - - echo "Conflicts resolved. Continuing cherry-pick." - git cherry-pick --continue - else - echo "Cherry-pick successful for commit $commit_hash." + curl -s -X POST -H "Authorization: token $GITHUB_TOKEN" \ + -H "Accept: application/vnd.github+json" \ + -d '{"labels": ["milestone-merge"]}' \ + "https://api.github.com/repos/${{ github.repository }}/issues/$new_pr_number/labels" fi done - - # Push the cherry-pick branch to the repository - git remote set-url origin "https://${BOT_TOKEN}@github.com/${{ github.repository }}.git" - git push origin $CHERRY_PICK_BRANCH --force - - - name: Create Pull Request - run: | - # Prepare and create the PR - pr_title="deps: Merge ${{ env.pr_numbers_in_title }} PRs into $TARGET_BRANCH" - pr_body=$(cat ${{ env.TEMP_DIR }}/pr_body.txt) - - echo "Prepared PR title:" - echo "$pr_title" - echo "Prepared PR body:" - echo "$pr_body" - - # Create the PR using the GitHub API - response=$(curl -s -X POST -H "Authorization: token $BOT_TOKEN" \ - -H "Accept: application/vnd.github+json" \ - https://api.github.com/repos/${{ github.repository }}/pulls \ - -d "$(jq -n --arg title "$pr_title" \ - --arg head "$CHERRY_PICK_BRANCH" \ - --arg base "$TARGET_BRANCH" \ - --arg body "$pr_body" \ - '{title: $title, head: $head, base: $base, body: $body}')") - - pr_number=$(echo "$response" | jq -r '.number') - echo "$pr_number" > ${{ env.TEMP_DIR }}/created_pr_number.txt - echo "Created PR #$pr_number" - - - name: Add Label to Created Pull Request - run: | - # Add 'milestone-merge' label to the created PR - pr_number=$(cat ${{ env.TEMP_DIR }}/created_pr_number.txt) - echo "Adding label to PR #$pr_number" - - curl -s -X POST -H "Authorization: token $GITHUB_TOKEN" \ - -H "Accept: application/vnd.github+json" \ - -d '{"labels": ["milestone-merge"]}' \ - "https://api.github.com/repos/${{ github.repository }}/issues/$pr_number/labels" - - echo "Added 'milestone-merge' label to PR #$pr_number." diff --git a/.github/workflows/publish-docker-image.yml b/.github/workflows/publish-docker-image.yml index 40b79e61a..998a11cf3 100644 --- a/.github/workflows/publish-docker-image.yml +++ b/.github/workflows/publish-docker-image.yml @@ -25,11 +25,11 @@ jobs: with: path: main-repo - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 + # - name: Set up QEMU + # uses: docker/setup-qemu-action@v3.3.0 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 + uses: docker/setup-buildx-action@v3.8.0 - name: Build Docker image id: build @@ -38,11 +38,8 @@ jobs: context: ./main-repo load: true tags: "openim/openim-server:local" - cache-from: type=gha - cache-to: type=gha,mode=max - - - name: Save Docker image to file - run: docker save -o image.tar openim/openim-server:local + cache-from: type=gha,scope=build + cache-to: type=gha,mode=max,scope=build - name: Checkout compose repository uses: actions/checkout@v4 @@ -66,43 +63,12 @@ jobs: run: | cd ${{ github.workspace }}/compose-repo docker compose up -d - sleep 60 - - # - name: Check openim-server health - # run: | - # timeout=300 - # interval=30 - # elapsed=0 - # while [[ $elapsed -le $timeout ]]; do - # if ! docker exec openim-server mage check; then - # echo "openim-server is not ready, waiting..." - # sleep $interval - # elapsed=$(($elapsed + $interval)) - # else - # echo "Health check successful" - # exit 0 - # fi - # done - # echo "Health check failed after 5 minutes" - # exit 1 - - # - name: Check openim-chat health - # if: success() - # run: | - # if ! docker exec openim-chat mage check; then - # echo "openim-chat check failed" - # exit 1 - # else - # echo "Health check successful" - # exit 0 - # fi - - - name: Load Docker image from file - run: docker load -i image.tar + + docker compose ps - name: Extract metadata for Docker (tags, labels) id: meta - uses: docker/metadata-action@v5.5.1 + uses: docker/metadata-action@v5.6.0 with: images: | openim/openim-server @@ -112,29 +78,27 @@ jobs: type=ref,event=tag type=schedule type=ref,event=branch - type=semver,pattern={{version}} + # type=semver,pattern={{version}} type=semver,pattern=v{{version}} - type=semver,pattern={{major}}.{{minor}} - type=semver,pattern={{major}} type=semver,pattern=release-{{raw}} type=sha type=raw,value=${{ github.event.inputs.tag }} - name: Log in to Docker Hub - uses: docker/login-action@v2 + uses: docker/login-action@v3.3.0 with: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} - name: Log in to GitHub Container Registry - uses: docker/login-action@v2 + uses: docker/login-action@v3.3.0 with: registry: ghcr.io username: ${{ github.repository_owner }} password: ${{ secrets.GITHUB_TOKEN }} - name: Log in to Aliyun Container Registry - uses: docker/login-action@v2 + uses: docker/login-action@v3.3.0 with: registry: registry.cn-hangzhou.aliyuncs.com username: ${{ secrets.ALIREGISTRY_USERNAME }} @@ -148,3 +112,28 @@ jobs: platforms: linux/amd64,linux/arm64 tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha,scope=build + cache-to: type=gha,mode=max,scope=build + + - name: Verify multi-platform support + run: | + images=("openim/openim-server" "ghcr.io/openimsdk/openim-server" "registry.cn-hangzhou.aliyuncs.com/openimsdk/openim-server") + for image in "${images[@]}"; do + for tag in $(echo "${{ steps.meta.outputs.tags }}" | tr ',' '\n'); do + manifest=$(docker manifest inspect "$image:$tag" || echo "error") + if [[ "$manifest" == "error" ]]; then + echo "Manifest not found for $image:$tag" + exit 1 + fi + amd64_found=$(echo "$manifest" | jq '.manifests[] | select(.platform.architecture == "amd64")') + arm64_found=$(echo "$manifest" | jq '.manifests[] | select(.platform.architecture == "arm64")') + if [[ -z "$amd64_found" ]]; then + echo "Multi-platform support check failed for $image:$tag - missing amd64" + exit 1 + fi + if [[ -z "$arm64_found" ]]; then + echo "Multi-platform support check failed for $image:$tag - missing arm64" + exit 1 + fi + done + done From 274a9bee6577ff080ed803de035f6953069a7804 Mon Sep 17 00:00:00 2001 From: chao <48119764+withchao@users.noreply.github.com> Date: Fri, 24 Jan 2025 16:13:12 +0800 Subject: [PATCH 109/199] fix: the abnormal message has no sending time, causing the SDK to be abnormal (#3087) * pb * fix: Modifying other fields while setting IsPrivateChat does not take effect * fix: quote message error revoke * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * upgrading pkg tools * fix * fix * optimize log output * feat: support GetLastMessage * feat: support GetLastMessage * feat: s3 switch * feat: s3 switch * fix: GetUsersOnline * feat: SendBusinessNotification supported configuration parameters * feat: SendBusinessNotification supported configuration parameters * feat: SendBusinessNotification supported configuration parameters * feat: seq conversion failed without exiting * fix: DeleteDoc crash * fix: fill send time * fix: fill send time --- pkg/common/storage/database/mgo/msg.go | 132 ++++++++++++++++++++++++- pkg/common/storage/model/msg.go | 11 ++- 2 files changed, 139 insertions(+), 4 deletions(-) diff --git a/pkg/common/storage/database/mgo/msg.go b/pkg/common/storage/database/mgo/msg.go index c440d4442..83fefbfe6 100644 --- a/pkg/common/storage/database/mgo/msg.go +++ b/pkg/common/storage/database/mgo/msg.go @@ -1091,22 +1091,148 @@ func (m *MsgMgo) onlyFindDocIndex(ctx context.Context, docID string, indexes []i return msgDocModel[0].Msg, nil } +//func (m *MsgMgo) FindSeqs(ctx context.Context, conversationID string, seqs []int64) ([]*model.MsgInfoModel, error) { +// if len(seqs) == 0 { +// return nil, nil +// } +// result := make([]*model.MsgInfoModel, 0, len(seqs)) +// for docID, seqs := range m.model.GetDocIDSeqsMap(conversationID, seqs) { +// res, err := m.onlyFindDocIndex(ctx, docID, datautil.Slice(seqs, m.model.GetMsgIndex)) +// if err != nil { +// return nil, err +// } +// for i, re := range res { +// if re == nil || re.Msg == nil { +// continue +// } +// result = append(result, res[i]) +// } +// } +// return result, nil +//} + +func (m *MsgMgo) findBeforeDocSendTime(ctx context.Context, docID string, limit int64) (int64, int64, error) { + if limit == 0 { + return 0, 0, nil + } + pipeline := []bson.M{ + { + "$match": bson.M{ + "doc_id": docID, + }, + }, + { + "$project": bson.M{ + "_id": 0, + "doc_id": 0, + }, + }, + { + "$unwind": "$msgs", + }, + { + "$project": bson.M{ + //"_id": 0, + //"doc_id": 0, + "msgs.msg.send_time": 1, + "msgs.msg.seq": 1, + }, + }, + } + if limit > 0 { + pipeline = append(pipeline, bson.M{"$limit": limit}) + } + type Result struct { + Msgs *model.MsgInfoModel `bson:"msgs"` + } + res, err := mongoutil.Aggregate[Result](ctx, m.coll, pipeline) + if err != nil { + return 0, 0, err + } + for i := len(res) - 1; i > 0; i-- { + v := res[i] + if v.Msgs != nil && v.Msgs.Msg != nil && v.Msgs.Msg.SendTime > 0 { + return v.Msgs.Msg.Seq, v.Msgs.Msg.SendTime, nil + } + } + return 0, 0, nil +} + +func (m *MsgMgo) findBeforeSendTime(ctx context.Context, conversationID string, seq int64) (int64, int64, error) { + first := true + for i := m.model.GetDocIndex(seq); i >= 0; i-- { + limit := int64(-1) + if first { + first = false + limit = m.model.GetMsgIndex(seq) + } + docID := m.model.BuildDocIDByIndex(conversationID, i) + msgSeq, msgSendTime, err := m.findBeforeDocSendTime(ctx, docID, limit) + if err != nil { + return 0, 0, err + } + if msgSendTime > 0 { + return msgSeq, msgSendTime, nil + } + } + return 0, 0, nil +} + func (m *MsgMgo) FindSeqs(ctx context.Context, conversationID string, seqs []int64) ([]*model.MsgInfoModel, error) { if len(seqs) == 0 { return nil, nil } + var abnormalSeq []int64 result := make([]*model.MsgInfoModel, 0, len(seqs)) - for docID, seqs := range m.model.GetDocIDSeqsMap(conversationID, seqs) { - res, err := m.onlyFindDocIndex(ctx, docID, datautil.Slice(seqs, m.model.GetMsgIndex)) + for docID, docSeqs := range m.model.GetDocIDSeqsMap(conversationID, seqs) { + res, err := m.onlyFindDocIndex(ctx, docID, datautil.Slice(docSeqs, m.model.GetMsgIndex)) if err != nil { return nil, err } + if len(res) == 0 { + abnormalSeq = append(abnormalSeq, docSeqs...) + continue + } for i, re := range res { - if re == nil || re.Msg == nil { + if re == nil || re.Msg == nil || re.Msg.SendTime == 0 { + abnormalSeq = append(abnormalSeq, docSeqs[i]) continue } result = append(result, res[i]) } } + if len(abnormalSeq) > 0 { + datautil.Sort(abnormalSeq, false) + sendTime := make(map[int64]int64) + var ( + lastSeq int64 + lastSendTime int64 + ) + for _, seq := range abnormalSeq { + if lastSendTime > 0 && lastSeq <= seq { + sendTime[seq] = lastSendTime + continue + } + msgSeq, msgSendTime, err := m.findBeforeSendTime(ctx, conversationID, seq) + if err != nil { + return nil, err + } + if msgSendTime <= 0 { + break + } + sendTime[seq] = msgSendTime + lastSeq = msgSeq + lastSendTime = msgSendTime + } + for _, seq := range abnormalSeq { + result = append(result, &model.MsgInfoModel{ + Msg: &model.MsgDataModel{ + Seq: seq, + Status: constant.MsgStatusHasDeleted, + SendTime: sendTime[seq], + }, + }) + } + } return result, nil } diff --git a/pkg/common/storage/model/msg.go b/pkg/common/storage/model/msg.go index 69113032d..6cf63bfcd 100644 --- a/pkg/common/storage/model/msg.go +++ b/pkg/common/storage/model/msg.go @@ -15,9 +15,10 @@ package model import ( + "strconv" + "github.com/openimsdk/protocol/sdkws" "github.com/openimsdk/tools/errs" - "strconv" ) const ( @@ -108,6 +109,10 @@ func (m *MsgDocModel) IsFull() bool { return m.Msg[len(m.Msg)-1].Msg != nil } +func (m *MsgDocModel) GetDocIndex(seq int64) int64 { + return (seq - 1) / singleGocMsgNum +} + func (m *MsgDocModel) GetDocID(conversationID string, seq int64) string { seqSuffix := (seq - 1) / singleGocMsgNum return m.indexGen(conversationID, seqSuffix) @@ -135,6 +140,10 @@ func (*MsgDocModel) indexGen(conversationID string, seqSuffix int64) string { return conversationID + ":" + strconv.FormatInt(seqSuffix, 10) } +func (*MsgDocModel) BuildDocIDByIndex(conversationID string, index int64) string { + return conversationID + ":" + strconv.FormatInt(index, 10) +} + func (*MsgDocModel) GenExceptionMessageBySeqs(seqs []int64) (exceptionMsg []*sdkws.MsgData) { for _, v := range seqs { msgModel := new(sdkws.MsgData) From d1b4d84628513b800d76404d89c29fc8070117e9 Mon Sep 17 00:00:00 2001 From: skiffer-git <72860476+skiffer-git@users.noreply.github.com> Date: Tue, 4 Feb 2025 23:25:46 +0800 Subject: [PATCH 110/199] Update LICENSE --- LICENSE | 210 ++++++-------------------------------------------------- 1 file changed, 22 insertions(+), 188 deletions(-) diff --git a/LICENSE b/LICENSE index 261eeb9e9..4591ca426 100644 --- a/LICENSE +++ b/LICENSE @@ -1,201 +1,35 @@ - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ +# Open Source License - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION +OpenIM is licensed under the Apache License 2.0, with the following additional conditions: - 1. Definitions. +1. OpenIM may be utilized commercially, including as a backend service for other applications or as an application development platform for enterprises. +A commercial license must be obtained from the producer if: + - Under no circumstances may you operate a multi-tenant or multi-business environment using the OpenIM source code, whether or not you have modified the repository code. In other words, a single instance of OpenIM may not simultaneously serve multiple enterprises, nor may it serve multiple lines of business within the same enterprise. + - If you intend to operate in such a multi-tenant or multi-business manner, you must obtain a commercial license from the producer in advance. - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. +2. As a contributor, you should agree that: - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. + a. The producer can adjust the open-source agreement to be more strict or more relaxed as deemed necessary. + b. Your contributed code may be used for commercial purposes, including but not limited to its cloud business operations. - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. +Apart from the specific conditions mentioned above, all other rights and restrictions follow the Apache License 2.0. Detailed information about the Apache License 2.0 can be found at http://www.apache.org/licenses/LICENSE-2.0. - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. +For any licensing-related questions or to obtain a commercial license, please contact contact@openim.io. - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). +© 2024 OpenIMSDK - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. +---------- - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. + http://www.apache.org/licenses/LICENSE-2.0 - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. From 86aa26ef62957ab72053cc32eb2b43490e8dd5d2 Mon Sep 17 00:00:00 2001 From: skiffer-git <72860476+skiffer-git@users.noreply.github.com> Date: Wed, 5 Feb 2025 08:28:22 +0800 Subject: [PATCH 111/199] Update README.md --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 6b1779b25..3b88935eb 100644 --- a/README.md +++ b/README.md @@ -132,7 +132,8 @@ Thank you for contributing to building a powerful instant messaging solution! ## :closed_book: License -OpenIMSDK is available under the Apache License 2.0. See the [LICENSE file](https://github.com/openimsdk/open-im-server/blob/main/LICENSE) for more information. +For more details, please refer to [here](./LICENSE). + From ed416f83768d76ab4acdd23bae926c0536bb55de Mon Sep 17 00:00:00 2001 From: chao <48119764+withchao@users.noreply.github.com> Date: Thu, 6 Feb 2025 10:38:46 +0800 Subject: [PATCH 112/199] fix: crash caused by withdrawing messages from users who have left the group (#3100) * pb * fix: Modifying other fields while setting IsPrivateChat does not take effect * fix: quote message error revoke * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * upgrading pkg tools * fix * fix * optimize log output * feat: support GetLastMessage * feat: support GetLastMessage * feat: s3 switch * feat: s3 switch * fix: GetUsersOnline * feat: SendBusinessNotification supported configuration parameters * feat: SendBusinessNotification supported configuration parameters * feat: SendBusinessNotification supported configuration parameters * feat: seq conversion failed without exiting * fix: DeleteDoc crash * fix: fill send time * fix: fill send time * fix: crash caused by withdrawing messages from users who have left the group --- internal/rpc/msg/revoke.go | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/internal/rpc/msg/revoke.go b/internal/rpc/msg/revoke.go index 97de0f48a..c2fb5833f 100644 --- a/internal/rpc/msg/revoke.go +++ b/internal/rpc/msg/revoke.go @@ -17,9 +17,10 @@ package msg import ( "context" "encoding/json" - "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" "time" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" + "github.com/openimsdk/open-im-server/v3/pkg/authverify" "github.com/openimsdk/open-im-server/v3/pkg/common/servererrs" "github.com/openimsdk/protocol/constant" @@ -79,8 +80,10 @@ func (m *msgServer) RevokeMsg(ctx context.Context, req *msg.RevokeMsgReq) (*msg. switch members[req.UserID].RoleLevel { case constant.GroupOwner: case constant.GroupAdmin: - if members[msgs[0].SendID].RoleLevel != constant.GroupOrdinaryUsers { - return nil, errs.ErrNoPermission.WrapMsg("no permission") + if sendMember, ok := members[msgs[0].SendID]; ok { + if sendMember.RoleLevel != constant.GroupOrdinaryUsers { + return nil, errs.ErrNoPermission.WrapMsg("no permission") + } } default: return nil, errs.ErrNoPermission.WrapMsg("no permission") From 0bf076bb05bfd2bfcf5ee5e2c95a3fce6a62e3f1 Mon Sep 17 00:00:00 2001 From: chao <48119764+withchao@users.noreply.github.com> Date: Sat, 8 Feb 2025 10:36:00 +0800 Subject: [PATCH 113/199] fix: the user sets the conversation timer cleanup timestamp unit incorrectly (#3102) * pb * fix: Modifying other fields while setting IsPrivateChat does not take effect * fix: quote message error revoke * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * upgrading pkg tools * fix * fix * optimize log output * feat: support GetLastMessage * feat: support GetLastMessage * feat: s3 switch * feat: s3 switch * fix: GetUsersOnline * feat: SendBusinessNotification supported configuration parameters * feat: SendBusinessNotification supported configuration parameters * feat: SendBusinessNotification supported configuration parameters * feat: seq conversion failed without exiting * fix: DeleteDoc crash * fix: fill send time * fix: fill send time * fix: crash caused by withdrawing messages from users who have left the group * fix: user msg timestamp --- internal/rpc/conversation/conversation.go | 5 +++-- pkg/common/storage/database/mgo/conversation.go | 9 ++++++++- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/internal/rpc/conversation/conversation.go b/internal/rpc/conversation/conversation.go index 53364ff86..76dd606aa 100644 --- a/internal/rpc/conversation/conversation.go +++ b/internal/rpc/conversation/conversation.go @@ -16,10 +16,11 @@ package conversation import ( "context" - "github.com/openimsdk/open-im-server/v3/pkg/rpcli" "sort" "time" + "github.com/openimsdk/open-im-server/v3/pkg/rpcli" + "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/redis" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database/mgo" @@ -773,7 +774,7 @@ func (c *conversationServer) ClearUserConversationMsg(ctx context.Context, req * if conversation.IsMsgDestruct == false || conversation.MsgDestructTime == 0 { continue } - seq, err := c.msgClient.GetLastMessageSeqByTime(ctx, conversation.ConversationID, req.Timestamp-conversation.MsgDestructTime) + seq, err := c.msgClient.GetLastMessageSeqByTime(ctx, conversation.ConversationID, req.Timestamp-(conversation.MsgDestructTime*1000)) if err != nil { return nil, err } diff --git a/pkg/common/storage/database/mgo/conversation.go b/pkg/common/storage/database/mgo/conversation.go index 851ec99c4..536827450 100644 --- a/pkg/common/storage/database/mgo/conversation.go +++ b/pkg/common/storage/database/mgo/conversation.go @@ -243,7 +243,14 @@ func (c *ConversationMgo) FindRandConversation(ctx context.Context, ts int64, li "$add": []any{ bson.M{ "$toLong": "$latest_msg_destruct_time", - }, "$msg_destruct_time"}, + }, + bson.M{ + "$multiply": []any{ + "$msg_destruct_time", + 1000, // convert to milliseconds + }, + }, + }, }, }, }, From ad8829c5a6d267f29fe0e506b962d86ad8ab1cdc Mon Sep 17 00:00:00 2001 From: Monet Lee Date: Sat, 8 Feb 2025 12:03:29 +0800 Subject: [PATCH 114/199] build: keep conflict is true. (#3103) --- .github/workflows/merge-from-milestone.yml | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/.github/workflows/merge-from-milestone.yml b/.github/workflows/merge-from-milestone.yml index 8e09c9ab7..67aafcf0d 100644 --- a/.github/workflows/merge-from-milestone.yml +++ b/.github/workflows/merge-from-milestone.yml @@ -113,14 +113,26 @@ jobs: echo "Cherry-picking commit: $merge_commit" if ! git cherry-pick "$merge_commit" --strategy=recursive -X theirs; then - echo "Cherry-pick encountered conflicts, attempting to continue..." - git cherry-pick --continue || { echo "Cherry-pick failed"; exit 1; } + echo "Cherry-pick encountered conflicts, attempting to resolve..." + git status --porcelain | grep '^UU ' | cut -c 4- | while read -r file; do + echo "Resolving conflict in $file" + git add "$file" + done + git status --porcelain | grep '^AA ' | cut -c 4- | while read -r file; do + echo "Adding new file $file" + git add "$file" + done + git status --porcelain | grep '^DD ' | cut -c 4- | while read -r file; do + echo "Removing deleted file $file" + git rm "$file" + done + git cherry-pick --continue || { echo "Cherry-pick failed"; continue; } fi git remote set-url origin "https://${BOT_TOKEN}@github.com/${{ github.repository }}.git" echo "Pushing branch: $cherry_pick_branch" - git push origin $cherry_pick_branch --force || { echo "Push failed"; exit 1; } + git push origin $cherry_pick_branch --force || { echo "Push failed"; continue; } new_pr_title="$pr_title [Created by @$pr_creator from #$pr_number]" new_pr_body="$pr_body From 489571f4b6cdb8c8906a15aab4ad3fefa38ffbda Mon Sep 17 00:00:00 2001 From: Monet Lee Date: Sat, 8 Feb 2025 15:26:35 +0800 Subject: [PATCH 115/199] fix: solve stop when merge failed (#3106) --- .github/workflows/merge-from-milestone.yml | 35 +++++++++++++--------- 1 file changed, 21 insertions(+), 14 deletions(-) diff --git a/.github/workflows/merge-from-milestone.yml b/.github/workflows/merge-from-milestone.yml index 67aafcf0d..1f5762ccb 100644 --- a/.github/workflows/merge-from-milestone.yml +++ b/.github/workflows/merge-from-milestone.yml @@ -113,26 +113,33 @@ jobs: echo "Cherry-picking commit: $merge_commit" if ! git cherry-pick "$merge_commit" --strategy=recursive -X theirs; then - echo "Cherry-pick encountered conflicts, attempting to resolve..." - git status --porcelain | grep '^UU ' | cut -c 4- | while read -r file; do - echo "Resolving conflict in $file" - git add "$file" + echo "Conflict detected for $merge_commit. Resolving with incoming changes." + conflict_files=$(git diff --name-only --diff-filter=U) + echo "Conflicting files:" + echo "$conflict_files" + + for file in $conflict_files; do + if [ -f "$file" ]; then + echo "Resolving conflict for $file" + git add "$file" + else + echo "File $file has been deleted. Skipping." + git rm "$file" + fi done - git status --porcelain | grep '^AA ' | cut -c 4- | while read -r file; do - echo "Adding new file $file" - git add "$file" - done - git status --porcelain | grep '^DD ' | cut -c 4- | while read -r file; do - echo "Removing deleted file $file" - git rm "$file" - done - git cherry-pick --continue || { echo "Cherry-pick failed"; continue; } + + echo "Conflicts resolved. Continuing cherry-pick." + git cherry-pick --continue || { echo "Cherry-pick failed, but continuing to create PR."; } + else + echo "Cherry-pick successful for commit $merge_commit." fi git remote set-url origin "https://${BOT_TOKEN}@github.com/${{ github.repository }}.git" echo "Pushing branch: $cherry_pick_branch" - git push origin $cherry_pick_branch --force || { echo "Push failed"; continue; } + if ! git push origin $cherry_pick_branch --force; then + echo "Push failed, but continuing to create PR..." + fi new_pr_title="$pr_title [Created by @$pr_creator from #$pr_number]" new_pr_body="$pr_body From a3c43a49d829be586454c3793c440e39fb4ae019 Mon Sep 17 00:00:00 2001 From: chao <48119764+withchao@users.noreply.github.com> Date: Mon, 10 Feb 2025 11:44:57 +0800 Subject: [PATCH 116/199] fix: seq conversion not reading env in docker environment (#3130) * pb * fix: Modifying other fields while setting IsPrivateChat does not take effect * fix: quote message error revoke * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * upgrading pkg tools * fix * fix * optimize log output * feat: support GetLastMessage * feat: support GetLastMessage * feat: s3 switch * feat: s3 switch * fix: GetUsersOnline * feat: SendBusinessNotification supported configuration parameters * feat: SendBusinessNotification supported configuration parameters * feat: SendBusinessNotification supported configuration parameters * feat: seq conversion failed without exiting * fix: DeleteDoc crash * fix: fill send time * fix: fill send time * fix: crash caused by withdrawing messages from users who have left the group * fix: user msg timestamp * seq read config * seq read config --- tools/seq/internal/seq.go | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/tools/seq/internal/seq.go b/tools/seq/internal/seq.go index c931cda5d..574e7cef9 100644 --- a/tools/seq/internal/seq.go +++ b/tools/seq/internal/seq.go @@ -15,16 +15,17 @@ import ( "syscall" "time" + "github.com/mitchellh/mapstructure" "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database/mgo" "github.com/openimsdk/tools/db/mongoutil" "github.com/openimsdk/tools/db/redisutil" "github.com/openimsdk/tools/utils/runtimeenv" "github.com/redis/go-redis/v9" + "github.com/spf13/viper" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/mongo" "go.mongodb.org/mongo-driver/mongo/options" - "gopkg.in/yaml.v3" ) const ( @@ -45,13 +46,19 @@ func readConfig[T any](dir string, name string) (*T, error) { if runtimeenv.PrintRuntimeEnvironment() == config.KUBERNETES { dir = os.Getenv(config.MountConfigFilePath) } - - data, err := os.ReadFile(filepath.Join(dir, name)) - if err != nil { + v := viper.New() + v.SetEnvPrefix(config.EnvPrefixMap[name]) + v.AutomaticEnv() + v.SetEnvKeyReplacer(strings.NewReplacer(".", "_")) + v.SetConfigFile(filepath.Join(dir, name)) + if err := v.ReadInConfig(); err != nil { return nil, err } + fn := func(config *mapstructure.DecoderConfig) { + config.TagName = "mapstructure" + } var conf T - if err := yaml.Unmarshal(data, &conf); err != nil { + if err := v.Unmarshal(&conf, fn); err != nil { return nil, err } return &conf, nil From e37ea50b9493e45bc1581bbf74e126050a846303 Mon Sep 17 00:00:00 2001 From: chao <48119764+withchao@users.noreply.github.com> Date: Wed, 12 Feb 2025 18:46:38 +0800 Subject: [PATCH 117/199] fix: the source message of the reference is withdrawn, and the referenced message is deleted (#3137) * pb * fix: Modifying other fields while setting IsPrivateChat does not take effect * fix: quote message error revoke * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * upgrading pkg tools * fix * fix * optimize log output * feat: support GetLastMessage * feat: support GetLastMessage * feat: s3 switch * feat: s3 switch * fix: GetUsersOnline * feat: SendBusinessNotification supported configuration parameters * feat: SendBusinessNotification supported configuration parameters * feat: SendBusinessNotification supported configuration parameters * feat: seq conversion failed without exiting * fix: DeleteDoc crash * fix: fill send time * fix: fill send time * fix: crash caused by withdrawing messages from users who have left the group * fix: user msg timestamp * seq read config * seq read config * fix: the source message of the reference is withdrawn, and the referenced message is deleted --- internal/msgtransfer/online_msg_to_mongo_handler.go | 12 ++++++++---- pkg/common/storage/controller/msg.go | 2 +- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/internal/msgtransfer/online_msg_to_mongo_handler.go b/internal/msgtransfer/online_msg_to_mongo_handler.go index d8836d54e..8405be7fe 100644 --- a/internal/msgtransfer/online_msg_to_mongo_handler.go +++ b/internal/msgtransfer/online_msg_to_mongo_handler.go @@ -73,10 +73,14 @@ func (mc *OnlineHistoryMongoConsumerHandler) handleChatWs2Mongo(ctx context.Cont } else { prommetrics.MsgInsertMongoSuccessCounter.Inc() } - var seqs []int64 - for _, msg := range msgFromMQ.MsgData { - seqs = append(seqs, msg.Seq) - } + //var seqs []int64 + //for _, msg := range msgFromMQ.MsgData { + // seqs = append(seqs, msg.Seq) + //} + //if err := mc.msgTransferDatabase.DeleteMessagesFromCache(ctx, msgFromMQ.ConversationID, seqs); err != nil { + // log.ZError(ctx, "remove cache msg from redis err", err, "msg", + // msgFromMQ.MsgData, "conversationID", msgFromMQ.ConversationID) + //} } func (*OnlineHistoryMongoConsumerHandler) Setup(_ sarama.ConsumerGroupSession) error { return nil } diff --git a/pkg/common/storage/controller/msg.go b/pkg/common/storage/controller/msg.go index b92f9b510..0069dc7cc 100644 --- a/pkg/common/storage/controller/msg.go +++ b/pkg/common/storage/controller/msg.go @@ -310,7 +310,7 @@ func (db *commonMsgDatabase) handlerDBMsg(ctx context.Context, cache map[int64][ log.ZError(ctx, "json.Unmarshal", err) return } - if quoteMsg.QuoteMessage == nil || quoteMsg.QuoteMessage.Content == "" { + if quoteMsg.QuoteMessage == nil { return } if quoteMsg.QuoteMessage.Content == "e30=" { From 9ed6200e457465fd71da8068015f3fe310943853 Mon Sep 17 00:00:00 2001 From: chao <48119764+withchao@users.noreply.github.com> Date: Fri, 14 Feb 2025 16:18:27 +0800 Subject: [PATCH 118/199] feat: optimize code and support running in single process mode (#3142) * pb * fix: Modifying other fields while setting IsPrivateChat does not take effect * fix: quote message error revoke * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * upgrading pkg tools * fix * fix * optimize log output * feat: support GetLastMessage * feat: support GetLastMessage * feat: s3 switch * feat: s3 switch * fix: GetUsersOnline * feat: SendBusinessNotification supported configuration parameters * feat: SendBusinessNotification supported configuration parameters * feat: SendBusinessNotification supported configuration parameters * feat: seq conversion failed without exiting * monolithic * fix: DeleteDoc crash * fix: DeleteDoc crash * fix: monolithic * fix: monolithic * fix: fill send time * fix: fill send time * fix: crash caused by withdrawing messages from users who have left the group * fix: mq * fix: mq * fix: user msg timestamp * fix: mq * 1 * 1 * 1 * 1 * 1 * 1 * 1 * seq read config * seq read config * 1 * 1 * fix: the source message of the reference is withdrawn, and the referenced message is deleted * 1 * 1 * 1 * 1 * 1 * 1 * 1 * 1 * 1 * 1 * 1 * 1 * 1 * 1 --- cmd/main.go | 419 ++++++++++++ go.mod | 6 +- go.sum | 4 +- internal/api/config_manager.go | 14 +- internal/api/init.go | 168 ++--- internal/api/prometheus_discovery.go | 50 +- internal/api/router.go | 7 +- internal/api/user.go | 4 +- internal/msggateway/hub_server.go | 43 +- internal/msggateway/init.go | 69 +- internal/msggateway/ws_server.go | 87 ++- internal/msgtransfer/init.go | 237 +++---- .../msgtransfer/online_history_msg_handler.go | 88 +-- .../online_msg_to_mongo_handler.go | 43 +- internal/push/callback.go | 1 + internal/push/offlinepush_handler.go | 30 +- internal/push/onlinepusher.go | 30 +- internal/push/push.go | 94 ++- internal/push/push_handler.go | 43 +- internal/rpc/auth/auth.go | 31 +- internal/rpc/conversation/conversation.go | 19 +- internal/rpc/group/group.go | 18 +- internal/rpc/msg/server.go | 44 +- internal/rpc/relation/friend.go | 26 +- internal/rpc/third/s3.go | 8 +- internal/rpc/third/third.go | 51 +- internal/rpc/user/user.go | 26 +- internal/tools/{ => cron}/cron_task.go | 36 +- internal/tools/{ => cron}/cron_test.go | 4 +- internal/tools/{ => cron}/msg.go | 7 +- internal/tools/{ => cron}/s3.go | 7 +- internal/tools/{ => cron}/user_msg.go | 7 +- pkg/common/cmd/api.go | 24 +- pkg/common/cmd/auth.go | 1 + pkg/common/cmd/cron_task.go | 22 +- pkg/common/cmd/msg_gateway.go | 18 +- pkg/common/cmd/msg_transfer.go | 17 +- pkg/common/cmd/push.go | 3 +- pkg/common/cmd/root.go | 14 +- pkg/common/config/config.go | 630 +++++++++--------- pkg/common/config/constant.go | 5 +- pkg/common/config/global.go | 11 + pkg/common/config/load_config.go | 7 +- pkg/common/discovery/discoveryregister.go | 9 +- pkg/common/prommetrics/api.go | 15 +- pkg/common/prommetrics/discovery.go | 31 - pkg/common/prommetrics/doc.go | 15 - pkg/common/prommetrics/grpc_auth.go | 4 + pkg/common/prommetrics/grpc_msg.go | 9 + pkg/common/prommetrics/grpc_msggateway.go | 4 + pkg/common/prommetrics/grpc_push.go | 7 + pkg/common/prommetrics/grpc_user.go | 4 + pkg/common/prommetrics/prommetrics.go | 77 ++- pkg/common/prommetrics/prommetrics_test.go | 8 + pkg/common/prommetrics/rpc.go | 9 +- pkg/common/prommetrics/transfer.go | 13 +- pkg/common/redispubsub/doc.go | 15 - pkg/common/redispubsub/redispubliser.go | 30 - pkg/common/redispubsub/redissubscriber.go | 49 -- pkg/common/startrpc/doc.go | 15 - pkg/common/startrpc/start.go | 264 ++++---- pkg/common/storage/cache/mcache/minio.go | 50 ++ pkg/common/storage/cache/mcache/msg_cache.go | 132 ++++ pkg/common/storage/cache/mcache/online.go | 82 +++ .../storage/cache/mcache/seq_conversation.go | 79 +++ pkg/common/storage/cache/mcache/third.go | 98 +++ pkg/common/storage/cache/mcache/token.go | 130 ++++ pkg/common/storage/cache/mcache/tools.go | 63 ++ pkg/common/storage/cache/redis/batch.go | 61 +- .../storage/cache/redis/batch_handler.go | 79 +-- pkg/common/storage/cache/redis/black.go | 31 +- .../storage/cache/redis/conversation.go | 35 +- pkg/common/storage/cache/redis/doc.go | 15 - pkg/common/storage/cache/redis/friend.go | 29 +- pkg/common/storage/cache/redis/group.go | 38 +- pkg/common/storage/cache/redis/minio.go | 59 ++ pkg/common/storage/cache/redis/msg.go | 20 +- pkg/common/storage/cache/redis/online.go | 12 +- .../cache/redis/redis_shard_manager.go | 164 ++--- pkg/common/storage/cache/redis/s3.go | 86 +-- .../storage/cache/redis/seq_conversation.go | 36 +- pkg/common/storage/cache/redis/seq_user.go | 23 +- pkg/common/storage/cache/redis/third.go | 17 +- pkg/common/storage/cache/redis/user.go | 28 +- pkg/common/storage/controller/group.go | 2 +- pkg/common/storage/controller/msg.go | 32 +- pkg/common/storage/controller/msg_transfer.go | 59 +- pkg/common/storage/controller/push.go | 27 +- pkg/common/storage/controller/s3.go | 4 +- pkg/common/storage/database/black.go | 83 +++ pkg/common/storage/database/cache.go | 16 + pkg/common/storage/database/mgo/cache.go | 183 +++++ pkg/common/storage/database/mgo/cache_test.go | 133 ++++ pkg/common/storage/database/mgo/msg_test.go | 52 +- .../database/mgo/seq_conversation_test.go | 7 +- pkg/common/storage/database/name.go | 1 + pkg/common/storage/model/cache.go | 9 + pkg/dbbuild/builder.go | 25 + pkg/dbbuild/microservices.go | 26 + pkg/dbbuild/standalone.go | 76 +++ pkg/mqbuild/builder.go | 60 ++ pkg/rpccache/online.go | 16 +- pkg/tools/batcher/batcher.go | 9 +- tools/check-component/main.go | 14 +- tools/seq/internal/seq.go | 2 +- 105 files changed, 3357 insertions(+), 1897 deletions(-) create mode 100644 cmd/main.go rename internal/tools/{ => cron}/cron_task.go (80%) rename internal/tools/{ => cron}/cron_test.go (95%) rename internal/tools/{ => cron}/msg.go (98%) rename internal/tools/{ => cron}/s3.go (99%) rename internal/tools/{ => cron}/user_msg.go (98%) create mode 100644 pkg/common/config/global.go delete mode 100644 pkg/common/prommetrics/discovery.go delete mode 100644 pkg/common/prommetrics/doc.go delete mode 100644 pkg/common/redispubsub/doc.go delete mode 100644 pkg/common/redispubsub/redispubliser.go delete mode 100644 pkg/common/redispubsub/redissubscriber.go delete mode 100644 pkg/common/startrpc/doc.go create mode 100644 pkg/common/storage/cache/mcache/minio.go create mode 100644 pkg/common/storage/cache/mcache/msg_cache.go create mode 100644 pkg/common/storage/cache/mcache/online.go create mode 100644 pkg/common/storage/cache/mcache/seq_conversation.go create mode 100644 pkg/common/storage/cache/mcache/third.go create mode 100644 pkg/common/storage/cache/mcache/token.go create mode 100644 pkg/common/storage/cache/mcache/tools.go delete mode 100644 pkg/common/storage/cache/redis/doc.go create mode 100644 pkg/common/storage/cache/redis/minio.go create mode 100644 pkg/common/storage/database/cache.go create mode 100644 pkg/common/storage/database/mgo/cache.go create mode 100644 pkg/common/storage/database/mgo/cache_test.go create mode 100644 pkg/common/storage/model/cache.go create mode 100644 pkg/dbbuild/builder.go create mode 100644 pkg/dbbuild/microservices.go create mode 100644 pkg/dbbuild/standalone.go create mode 100644 pkg/mqbuild/builder.go diff --git a/cmd/main.go b/cmd/main.go new file mode 100644 index 000000000..1d0b82be8 --- /dev/null +++ b/cmd/main.go @@ -0,0 +1,419 @@ +package main + +import ( + "bytes" + "context" + "encoding/json" + "flag" + "fmt" + "net" + "os" + "os/signal" + "path" + "path/filepath" + "reflect" + "runtime" + "strings" + "sync" + "syscall" + "time" + + "github.com/mitchellh/mapstructure" + "github.com/openimsdk/open-im-server/v3/internal/api" + "github.com/openimsdk/open-im-server/v3/internal/msggateway" + "github.com/openimsdk/open-im-server/v3/internal/msgtransfer" + "github.com/openimsdk/open-im-server/v3/internal/push" + "github.com/openimsdk/open-im-server/v3/internal/rpc/auth" + "github.com/openimsdk/open-im-server/v3/internal/rpc/conversation" + "github.com/openimsdk/open-im-server/v3/internal/rpc/group" + "github.com/openimsdk/open-im-server/v3/internal/rpc/msg" + "github.com/openimsdk/open-im-server/v3/internal/rpc/relation" + "github.com/openimsdk/open-im-server/v3/internal/rpc/third" + "github.com/openimsdk/open-im-server/v3/internal/rpc/user" + "github.com/openimsdk/open-im-server/v3/internal/tools/cron" + "github.com/openimsdk/open-im-server/v3/pkg/common/config" + "github.com/openimsdk/open-im-server/v3/pkg/common/prommetrics" + "github.com/openimsdk/open-im-server/v3/version" + "github.com/openimsdk/tools/discovery" + "github.com/openimsdk/tools/discovery/standalone" + "github.com/openimsdk/tools/log" + "github.com/openimsdk/tools/system/program" + "github.com/openimsdk/tools/utils/datautil" + "github.com/openimsdk/tools/utils/network" + "github.com/spf13/viper" + "google.golang.org/grpc" +) + +func init() { + config.SetStandalone() + prommetrics.RegistryAll() +} + +func main() { + var configPath string + flag.StringVar(&configPath, "c", "", "config path") + flag.Parse() + if configPath == "" { + _, _ = fmt.Fprintln(os.Stderr, "config path is empty") + os.Exit(1) + return + } + cmd := newCmds(configPath) + putCmd(cmd, false, auth.Start) + putCmd(cmd, false, conversation.Start) + putCmd(cmd, false, relation.Start) + putCmd(cmd, false, group.Start) + putCmd(cmd, false, msg.Start) + putCmd(cmd, false, third.Start) + putCmd(cmd, false, user.Start) + putCmd(cmd, false, push.Start) + putCmd(cmd, true, msggateway.Start) + putCmd(cmd, true, msgtransfer.Start) + putCmd(cmd, true, api.Start) + putCmd(cmd, true, cron.Start) + ctx := context.Background() + if err := cmd.run(ctx); err != nil { + _, _ = fmt.Fprintf(os.Stderr, "server exit %s", err) + os.Exit(1) + return + } +} + +func newCmds(confPath string) *cmds { + return &cmds{confPath: confPath} +} + +type cmdName struct { + Name string + Func func(ctx context.Context) error + Block bool +} +type cmds struct { + confPath string + cmds []cmdName + config config.AllConfig + conf map[string]reflect.Value +} + +func (x *cmds) getTypePath(typ reflect.Type) string { + return path.Join(typ.PkgPath(), typ.Name()) +} + +func (x *cmds) initDiscovery() { + x.config.Discovery.Enable = "standalone" + vof := reflect.ValueOf(&x.config.Discovery.RpcService).Elem() + tof := reflect.TypeOf(&x.config.Discovery.RpcService).Elem() + num := tof.NumField() + for i := 0; i < num; i++ { + field := tof.Field(i) + if !field.IsExported() { + continue + } + if field.Type.Kind() != reflect.String { + continue + } + vof.Field(i).SetString(field.Name) + } +} + +func (x *cmds) initAllConfig() error { + x.conf = make(map[string]reflect.Value) + vof := reflect.ValueOf(&x.config).Elem() + num := vof.NumField() + for i := 0; i < num; i++ { + field := vof.Field(i) + for ptr := true; ptr; { + if field.Kind() == reflect.Ptr { + field = field.Elem() + } else { + ptr = false + } + } + x.conf[x.getTypePath(field.Type())] = field + val := field.Addr().Interface() + name := val.(interface{ GetConfigFileName() string }).GetConfigFileName() + confData, err := os.ReadFile(filepath.Join(x.confPath, name)) + if err != nil { + if os.IsNotExist(err) { + continue + } + return err + } + v := viper.New() + v.SetConfigType("yaml") + if err := v.ReadConfig(bytes.NewReader(confData)); err != nil { + return err + } + opt := func(conf *mapstructure.DecoderConfig) { + conf.TagName = config.StructTagName + } + if err := v.Unmarshal(val, opt); err != nil { + return err + } + } + x.initDiscovery() + x.config.Redis.Disable = false + x.config.LocalCache = config.LocalCache{} + config.InitNotification(&x.config.Notification) + return nil +} + +func (x *cmds) parseConf(conf any) error { + vof := reflect.ValueOf(conf) + for { + if vof.Kind() == reflect.Ptr { + vof = vof.Elem() + } else { + break + } + } + tof := vof.Type() + numField := vof.NumField() + for i := 0; i < numField; i++ { + typeField := tof.Field(i) + if !typeField.IsExported() { + continue + } + field := vof.Field(i) + pkt := x.getTypePath(field.Type()) + val, ok := x.conf[pkt] + if !ok { + switch field.Interface().(type) { + case config.Index: + case config.Path: + field.SetString(x.confPath) + case config.AllConfig: + field.Set(reflect.ValueOf(x.config)) + case *config.AllConfig: + field.Set(reflect.ValueOf(&x.config)) + default: + return fmt.Errorf("config field %s %s not found", vof.Type().Name(), typeField.Name) + } + continue + } + field.Set(val) + } + return nil +} + +func (x *cmds) add(name string, block bool, fn func(ctx context.Context) error) { + x.cmds = append(x.cmds, cmdName{Name: name, Block: block, Func: fn}) +} + +func (x *cmds) initLog() error { + conf := x.config.Log + if err := log.InitLoggerFromConfig( + "openim-server", + program.GetProcessName(), + "", "", + conf.RemainLogLevel, + conf.IsStdout, + conf.IsJson, + conf.StorageLocation, + conf.RemainRotationCount, + conf.RotationTime, + strings.TrimSpace(version.Version), + conf.IsSimplify, + ); err != nil { + return err + } + return nil + +} + +func (x *cmds) run(ctx context.Context) error { + if len(x.cmds) == 0 { + return fmt.Errorf("no command to run") + } + if err := x.initAllConfig(); err != nil { + return err + } + if err := x.initLog(); err != nil { + return err + } + + ctx, cancel := context.WithCancelCause(ctx) + + go func() { + <-ctx.Done() + log.ZError(ctx, "context server exit cause", context.Cause(ctx)) + }() + + if prometheus := x.config.API.Prometheus; prometheus.Enable { + var ( + port int + err error + ) + if !prometheus.AutoSetPorts { + port, err = datautil.GetElemByIndex(prometheus.Ports, 0) + if err != nil { + return err + } + } + ip, err := network.GetLocalIP() + if err != nil { + return err + } + listener, err := net.Listen("tcp", fmt.Sprintf(":%d", port)) + if err != nil { + return fmt.Errorf("prometheus listen %d error %w", port, err) + } + defer listener.Close() + log.ZDebug(ctx, "prometheus start", "addr", listener.Addr()) + target, err := json.Marshal(prommetrics.BuildDefaultTarget(ip, listener.Addr().(*net.TCPAddr).Port)) + if err != nil { + return err + } + if err := standalone.GetKeyValue().SetKey(ctx, prommetrics.BuildDiscoveryKey(prommetrics.APIKeyName), target); err != nil { + return err + } + go func() { + err := prommetrics.Start(listener) + if err == nil { + err = fmt.Errorf("http done") + } + cancel(fmt.Errorf("prometheus %w", err)) + }() + } + + go func() { + sigs := make(chan os.Signal, 1) + signal.Notify(sigs, syscall.SIGTERM, syscall.SIGINT, syscall.SIGKILL) + select { + case <-ctx.Done(): + return + case val := <-sigs: + log.ZDebug(ctx, "recv signal", "signal", val.String()) + cancel(fmt.Errorf("signal %s", val.String())) + } + }() + + for i := range x.cmds { + cmd := x.cmds[i] + if cmd.Block { + continue + } + if err := cmd.Func(ctx); err != nil { + cancel(fmt.Errorf("server %s exit %w", cmd.Name, err)) + return err + } + go func() { + if cmd.Block { + cancel(fmt.Errorf("server %s exit", cmd.Name)) + } + }() + } + + var wait cmdManger + for i := range x.cmds { + cmd := x.cmds[i] + if !cmd.Block { + continue + } + wait.Start(cmd.Name) + go func() { + defer wait.Shutdown(cmd.Name) + if err := cmd.Func(ctx); err != nil { + cancel(fmt.Errorf("server %s exit %w", cmd.Name, err)) + return + } + cancel(fmt.Errorf("server %s exit", cmd.Name)) + }() + } + <-ctx.Done() + exitCause := context.Cause(ctx) + log.ZWarn(ctx, "notification of service closure", exitCause) + done := wait.Wait() + timeout := time.NewTimer(time.Second * 10) + defer timeout.Stop() + for { + select { + case <-timeout.C: + log.ZWarn(ctx, "server exit timeout", nil, "running", wait.Running()) + return exitCause + case _, ok := <-done: + if ok { + log.ZWarn(ctx, "waiting for the service to exit", nil, "running", wait.Running()) + } else { + log.ZInfo(ctx, "all server exit done") + return exitCause + } + } + } +} + +func putCmd[C any](cmd *cmds, block bool, fn func(ctx context.Context, config *C, client discovery.Conn, server grpc.ServiceRegistrar) error) { + name := path.Base(runtime.FuncForPC(reflect.ValueOf(fn).Pointer()).Name()) + if index := strings.Index(name, "."); index >= 0 { + name = name[:index] + } + cmd.add(name, block, func(ctx context.Context) error { + var conf C + if err := cmd.parseConf(&conf); err != nil { + return err + } + return fn(ctx, &conf, standalone.GetDiscoveryConn(), standalone.GetServiceRegistrar()) + }) +} + +type cmdManger struct { + lock sync.Mutex + done chan struct{} + count int + names map[string]struct{} +} + +func (x *cmdManger) Start(name string) { + x.lock.Lock() + defer x.lock.Unlock() + if x.names == nil { + x.names = make(map[string]struct{}) + } + if x.done == nil { + x.done = make(chan struct{}, 1) + } + if _, ok := x.names[name]; ok { + panic(fmt.Errorf("cmd %s already exists", name)) + } + x.count++ + x.names[name] = struct{}{} +} + +func (x *cmdManger) Shutdown(name string) { + x.lock.Lock() + defer x.lock.Unlock() + if _, ok := x.names[name]; !ok { + panic(fmt.Errorf("cmd %s not exists", name)) + } + delete(x.names, name) + x.count-- + if x.count == 0 { + close(x.done) + } else { + select { + case x.done <- struct{}{}: + default: + } + } +} + +func (x *cmdManger) Wait() <-chan struct{} { + x.lock.Lock() + defer x.lock.Unlock() + if x.count == 0 || x.done == nil { + tmp := make(chan struct{}) + close(tmp) + return tmp + } + return x.done +} + +func (x *cmdManger) Running() []string { + x.lock.Lock() + defer x.lock.Unlock() + names := make([]string, 0, len(x.names)) + for name := range x.names { + names = append(names, name) + } + return names +} diff --git a/go.mod b/go.mod index fd188e978..7145519fb 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/mitchellh/mapstructure v1.5.0 github.com/openimsdk/protocol v0.0.72-alpha.71 - github.com/openimsdk/tools v0.0.50-alpha.70 + github.com/openimsdk/tools v0.0.50-alpha.74 github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 github.com/stretchr/testify v1.9.0 @@ -27,7 +27,6 @@ require ( require github.com/google/uuid v1.6.0 require ( - github.com/IBM/sarama v1.43.0 github.com/fatih/color v1.14.1 github.com/gin-contrib/gzip v1.0.1 github.com/go-redis/redis v6.15.9+incompatible @@ -55,6 +54,7 @@ require ( cloud.google.com/go/iam v1.1.7 // indirect cloud.google.com/go/longrunning v0.5.5 // indirect cloud.google.com/go/storage v1.40.0 // indirect + github.com/IBM/sarama v1.43.0 // indirect github.com/MicahParks/keyfunc v1.9.0 // indirect github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible // indirect github.com/aws/aws-sdk-go-v2 v1.32.5 // indirect @@ -219,3 +219,5 @@ require ( golang.org/x/crypto v0.27.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect ) + +//replace github.com/openimsdk/tools => /Users/chao/Desktop/code/tools diff --git a/go.sum b/go.sum index 4f3431133..f67869c0d 100644 --- a/go.sum +++ b/go.sum @@ -349,8 +349,8 @@ github.com/openimsdk/gomake v0.0.15-alpha.2 h1:5Q8yl8ezy2yx+q8/ucU/t4kJnDfCzNOrk github.com/openimsdk/gomake v0.0.15-alpha.2/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= github.com/openimsdk/protocol v0.0.72-alpha.71 h1:R3utzOlqepaJWTAmnfJi4ccUM/XIoFasSyjQMOipM70= github.com/openimsdk/protocol v0.0.72-alpha.71/go.mod h1:WF7EuE55vQvpyUAzDXcqg+B+446xQyEba0X35lTINmw= -github.com/openimsdk/tools v0.0.50-alpha.70 h1:pyqWkJzXbELWU9KKAsWkj3g0flJYNsDTcjR5SLFQAZU= -github.com/openimsdk/tools v0.0.50-alpha.70/go.mod h1:B+oqV0zdewN7OiEHYJm+hW+8/Te7B8tHHgD8rK5ZLZk= +github.com/openimsdk/tools v0.0.50-alpha.74 h1:yh10SiMiivMEjicEQg+QAsH4pvaO+4noMPdlw+ew0Kc= +github.com/openimsdk/tools v0.0.50-alpha.74/go.mod h1:n2poR3asX1e1XZce4O+MOWAp+X02QJRFvhcLCXZdzRo= github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= diff --git a/internal/api/config_manager.go b/internal/api/config_manager.go index c61b2cb0b..15f5e7004 100644 --- a/internal/api/config_manager.go +++ b/internal/api/config_manager.go @@ -30,16 +30,14 @@ type ConfigManager struct { client *clientv3.Client configPath string - runtimeEnv string } -func NewConfigManager(IMAdminUserID []string, cfg *config.AllConfig, client *clientv3.Client, configPath string, runtimeEnv string) *ConfigManager { +func NewConfigManager(IMAdminUserID []string, cfg *config.AllConfig, client *clientv3.Client, configPath string) *ConfigManager { cm := &ConfigManager{ imAdminUserID: IMAdminUserID, config: cfg, client: client, configPath: configPath, - runtimeEnv: runtimeEnv, } return cm } @@ -73,7 +71,7 @@ func (cm *ConfigManager) GetConfig(c *gin.Context) { func (cm *ConfigManager) GetConfigList(c *gin.Context) { var resp apistruct.GetConfigListResp resp.ConfigNames = cm.config.GetConfigNames() - resp.Environment = runtimeenv.PrintRuntimeEnvironment() + resp.Environment = runtimeenv.RuntimeEnvironment() resp.Version = version.Version apiresp.GinSuccess(c, resp) @@ -209,13 +207,7 @@ func (cm *ConfigManager) resetConfig(c *gin.Context, checkChange bool, ops ...cl changedKeys := make([]string, 0, len(configMap)) for k, v := range configMap { - err := config.Load( - cm.configPath, - k, - config.EnvPrefixMap[k], - cm.runtimeEnv, - v.new, - ) + err := config.Load(cm.configPath, k, config.EnvPrefixMap[k], v.new) if err != nil { log.ZError(c, "load config failed", err) continue diff --git a/internal/api/init.go b/internal/api/init.go index 20237ebc2..4a1404ffc 100644 --- a/internal/api/init.go +++ b/internal/api/init.go @@ -20,157 +20,85 @@ import ( "fmt" "net" "net/http" - "os" - "os/signal" "strconv" - "syscall" "time" conf "github.com/openimsdk/open-im-server/v3/pkg/common/config" - kdisc "github.com/openimsdk/open-im-server/v3/pkg/common/discovery" - disetcd "github.com/openimsdk/open-im-server/v3/pkg/common/discovery/etcd" - "github.com/openimsdk/open-im-server/v3/pkg/common/prommetrics" - "github.com/openimsdk/tools/discovery/etcd" - "github.com/openimsdk/tools/errs" + "github.com/openimsdk/tools/discovery" "github.com/openimsdk/tools/log" - "github.com/openimsdk/tools/mw" - "github.com/openimsdk/tools/system/program" "github.com/openimsdk/tools/utils/datautil" - "github.com/openimsdk/tools/utils/jsonutil" "github.com/openimsdk/tools/utils/network" "github.com/openimsdk/tools/utils/runtimeenv" "google.golang.org/grpc" - "google.golang.org/grpc/credentials/insecure" ) type Config struct { - *conf.AllConfig + conf.AllConfig - RuntimeEnv string - ConfigPath string + ConfigPath conf.Path + Index conf.Index } -func Start(ctx context.Context, index int, config *Config) error { - apiPort, err := datautil.GetElemByIndex(config.API.Api.Ports, index) +func Start(ctx context.Context, config *Config, client discovery.Conn, service grpc.ServiceRegistrar) error { + apiPort, err := datautil.GetElemByIndex(config.API.Api.Ports, int(config.Index)) if err != nil { return err } - config.RuntimeEnv = runtimeenv.PrintRuntimeEnvironment() - - client, err := kdisc.NewDiscoveryRegister(&config.Discovery, config.RuntimeEnv, []string{ - config.Discovery.RpcService.MessageGateway, - }) - if err != nil { - return errs.WrapMsg(err, "failed to register discovery service") - } - client.AddOption(mw.GrpcClient(), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithDefaultServiceConfig(fmt.Sprintf(`{"LoadBalancingPolicy": "%s"}`, "round_robin"))) - - var ( - netDone = make(chan struct{}, 1) - netErr error - prometheusPort int - ) - - registerIP, err := network.GetRpcRegisterIP("") - if err != nil { - return err - } - - getAutoPort := func() (net.Listener, int, error) { - registerAddr := net.JoinHostPort(registerIP, "0") - listener, err := net.Listen("tcp", registerAddr) - if err != nil { - return nil, 0, errs.WrapMsg(err, "listen err", "registerAddr", registerAddr) - } - _, portStr, _ := net.SplitHostPort(listener.Addr().String()) - port, _ := strconv.Atoi(portStr) - return listener, port, nil - } - - if config.API.Prometheus.AutoSetPorts && config.Discovery.Enable != conf.ETCD { - return errs.New("only etcd support autoSetPorts", "RegisterName", "api").Wrap() - } - router, err := newGinRouter(ctx, client, config) if err != nil { return err } - if config.API.Prometheus.Enable { - var ( - listener net.Listener - ) - - if config.API.Prometheus.AutoSetPorts { - listener, prometheusPort, err = getAutoPort() - if err != nil { - return err - } - - etcdClient := client.(*etcd.SvcDiscoveryRegistryImpl).GetClient() - _, err = etcdClient.Put(ctx, prommetrics.BuildDiscoveryKey(prommetrics.APIKeyName), jsonutil.StructToJsonString(prommetrics.BuildDefaultTarget(registerIP, prometheusPort))) - if err != nil { - return errs.WrapMsg(err, "etcd put err") - } - } else { - prometheusPort, err = datautil.GetElemByIndex(config.API.Prometheus.Ports, index) - if err != nil { - return err - } - listener, err = net.Listen("tcp", fmt.Sprintf(":%d", prometheusPort)) - if err != nil { - return errs.WrapMsg(err, "listen err", "addr", fmt.Sprintf(":%d", prometheusPort)) - } + apiCtx, apiCancel := context.WithCancelCause(context.Background()) + done := make(chan struct{}) + go func() { + httpServer := &http.Server{ + Handler: router, + Addr: net.JoinHostPort(network.GetListenIP(config.API.Api.ListenIP), strconv.Itoa(apiPort)), } - go func() { - if err := prommetrics.ApiInit(listener); err != nil && !errors.Is(err, http.ErrServerClosed) { - netErr = errs.WrapMsg(err, fmt.Sprintf("api prometheus start err: %d", prometheusPort)) - netDone <- struct{}{} + defer close(done) + select { + case <-ctx.Done(): + apiCancel(fmt.Errorf("recv ctx %w", context.Cause(ctx))) + case <-apiCtx.Done(): + } + log.ZDebug(ctx, "api server is shutting down") + if err := httpServer.Shutdown(context.Background()); err != nil { + log.ZWarn(ctx, "api server shutdown err", err) } }() - - } - address := net.JoinHostPort(network.GetListenIP(config.API.Api.ListenIP), strconv.Itoa(apiPort)) - - server := http.Server{Addr: address, Handler: router} - log.CInfo(ctx, "API server is initializing", "runtimeEnv", config.RuntimeEnv, "address", address, "apiPort", apiPort, "prometheusPort", prometheusPort) - go func() { - err = server.ListenAndServe() - if err != nil && !errors.Is(err, http.ErrServerClosed) { - netErr = errs.WrapMsg(err, fmt.Sprintf("api start err: %s", server.Addr)) - netDone <- struct{}{} + log.CInfo(ctx, "api server is init", "runtimeEnv", runtimeenv.RuntimeEnvironment(), "address", httpServer.Addr, "apiPort", apiPort) + err := httpServer.ListenAndServe() + if err == nil { + err = errors.New("api done") } + apiCancel(err) }() - if config.Discovery.Enable == conf.ETCD { - cm := disetcd.NewConfigManager(client.(*etcd.SvcDiscoveryRegistryImpl).GetClient(), config.GetConfigNames()) - cm.Watch(ctx) - } - - sigs := make(chan os.Signal, 1) - signal.Notify(sigs, syscall.SIGTERM) - - shutdown := func() error { - ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) - defer cancel() - err := server.Shutdown(ctx) - if err != nil { - return errs.WrapMsg(err, "shutdown err") - } - return nil - } - disetcd.RegisterShutDown(shutdown) + //if config.Discovery.Enable == conf.ETCD { + // cm := disetcd.NewConfigManager(client.(*etcd.SvcDiscoveryRegistryImpl).GetClient(), config.GetConfigNames()) + // cm.Watch(ctx) + //} + //sigs := make(chan os.Signal, 1) + //signal.Notify(sigs, syscall.SIGTERM) + //select { + //case val := <-sigs: + // log.ZDebug(ctx, "recv exit", "signal", val.String()) + // cancel(fmt.Errorf("signal %s", val.String())) + //case <-ctx.Done(): + //} + <-apiCtx.Done() + exitCause := context.Cause(ctx) + log.ZWarn(ctx, "api server exit", exitCause) + timer := time.NewTimer(time.Second * 15) + defer timer.Stop() select { - case <-sigs: - program.SIGTERMExit() - if err := shutdown(); err != nil { - return err - } - case <-netDone: - close(netDone) - return netErr + case <-timer.C: + log.ZWarn(ctx, "api server graceful stop timeout", nil) + case <-done: + log.ZDebug(ctx, "api server graceful stop done") } - return nil + return exitCause } diff --git a/internal/api/prometheus_discovery.go b/internal/api/prometheus_discovery.go index 5e1a9cae2..bdcca4e26 100644 --- a/internal/api/prometheus_discovery.go +++ b/internal/api/prometheus_discovery.go @@ -2,6 +2,7 @@ package api import ( "encoding/json" + "errors" "net/http" "github.com/gin-gonic/gin" @@ -11,16 +12,16 @@ import ( "github.com/openimsdk/tools/discovery" "github.com/openimsdk/tools/discovery/etcd" "github.com/openimsdk/tools/errs" - "github.com/openimsdk/tools/log" clientv3 "go.etcd.io/etcd/client/v3" ) type PrometheusDiscoveryApi struct { config *Config client *clientv3.Client + kv discovery.KeyValue } -func NewPrometheusDiscoveryApi(config *Config, client discovery.SvcDiscoveryRegistry) *PrometheusDiscoveryApi { +func NewPrometheusDiscoveryApi(config *Config, client discovery.Conn) *PrometheusDiscoveryApi { api := &PrometheusDiscoveryApi{ config: config, } @@ -30,43 +31,26 @@ func NewPrometheusDiscoveryApi(config *Config, client discovery.SvcDiscoveryRegi return api } -func (p *PrometheusDiscoveryApi) Enable(c *gin.Context) { - if p.config.Discovery.Enable != conf.ETCD { - c.JSON(http.StatusOK, []struct{}{}) - c.Abort() - } -} - func (p *PrometheusDiscoveryApi) discovery(c *gin.Context, key string) { - eResp, err := p.client.Get(c, prommetrics.BuildDiscoveryKey(key)) + value, err := p.kv.GetKey(c, prommetrics.BuildDiscoveryKey(key)) if err != nil { - // Log and respond with an error if preparation fails. - apiresp.GinError(c, errs.WrapMsg(err, "etcd get err")) + if errors.Is(err, discovery.ErrNotSupportedKeyValue) { + c.JSON(http.StatusOK, []struct{}{}) + return + } + apiresp.GinError(c, errs.WrapMsg(err, "get key value")) return } - if len(eResp.Kvs) == 0 { - c.JSON(http.StatusOK, []*prommetrics.Target{}) + if len(value) == 0 { + c.JSON(http.StatusOK, []*prommetrics.RespTarget{}) + return } - - var ( - resp = &prommetrics.RespTarget{ - Targets: make([]string, 0, len(eResp.Kvs)), - } - ) - - for i := range eResp.Kvs { - var target prommetrics.Target - err = json.Unmarshal(eResp.Kvs[i].Value, &target) - if err != nil { - log.ZError(c, "prometheus unmarshal err", errs.Wrap(err)) - } - resp.Targets = append(resp.Targets, target.Target) - if resp.Labels == nil { - resp.Labels = target.Labels - } + var resp prommetrics.RespTarget + if err := json.Unmarshal(value, &resp); err != nil { + apiresp.GinError(c, errs.WrapMsg(err, "json unmarshal err")) + return } - - c.JSON(200, []*prommetrics.RespTarget{resp}) + c.JSON(http.StatusOK, []*prommetrics.RespTarget{&resp}) } func (p *PrometheusDiscoveryApi) Api(c *gin.Context) { diff --git a/internal/api/router.go b/internal/api/router.go index da9d22463..ebaff019c 100644 --- a/internal/api/router.go +++ b/internal/api/router.go @@ -52,7 +52,7 @@ func prommetricsGin() gin.HandlerFunc { } } -func newGinRouter(ctx context.Context, client discovery.SvcDiscoveryRegistry, cfg *Config) (*gin.Engine, error) { +func newGinRouter(ctx context.Context, client discovery.Conn, cfg *Config) (*gin.Engine, error) { authConn, err := client.GetConn(ctx, cfg.Discovery.RpcService.Auth) if err != nil { return nil, err @@ -283,7 +283,7 @@ func newGinRouter(ctx context.Context, client discovery.SvcDiscoveryRegistry, cf } { pd := NewPrometheusDiscoveryApi(cfg, client) - proDiscoveryGroup := r.Group("/prometheus_discovery", pd.Enable) + proDiscoveryGroup := r.Group("/prometheus_discovery") proDiscoveryGroup.GET("/api", pd.Api) proDiscoveryGroup.GET("/user", pd.User) proDiscoveryGroup.GET("/group", pd.Group) @@ -301,9 +301,8 @@ func newGinRouter(ctx context.Context, client discovery.SvcDiscoveryRegistry, cf if cfg.Discovery.Enable == config.ETCD { etcdClient = client.(*etcd.SvcDiscoveryRegistryImpl).GetClient() } - cm := NewConfigManager(cfg.Share.IMAdminUserID, cfg.AllConfig, etcdClient, cfg.ConfigPath, cfg.RuntimeEnv) + cm := NewConfigManager(cfg.Share.IMAdminUserID, &cfg.AllConfig, etcdClient, string(cfg.ConfigPath)) { - configGroup := r.Group("/config", cm.CheckAdmin) configGroup.POST("/get_config_list", cm.GetConfigList) configGroup.POST("/get_config", cm.GetConfig) diff --git a/internal/api/user.go b/internal/api/user.go index a88f8f65a..6427e222e 100644 --- a/internal/api/user.go +++ b/internal/api/user.go @@ -29,11 +29,11 @@ import ( type UserApi struct { Client user.UserClient - discov discovery.SvcDiscoveryRegistry + discov discovery.Conn config config.RpcService } -func NewUserApi(client user.UserClient, discov discovery.SvcDiscoveryRegistry, config config.RpcService) UserApi { +func NewUserApi(client user.UserClient, discov discovery.Conn, config config.RpcService) UserApi { return UserApi{Client: client, discov: discov, config: config} } diff --git a/internal/msggateway/hub_server.go b/internal/msggateway/hub_server.go index 887a90d7a..376a697fd 100644 --- a/internal/msggateway/hub_server.go +++ b/internal/msggateway/hub_server.go @@ -22,7 +22,6 @@ import ( "github.com/openimsdk/open-im-server/v3/pkg/authverify" "github.com/openimsdk/open-im-server/v3/pkg/common/servererrs" - "github.com/openimsdk/open-im-server/v3/pkg/common/startrpc" "github.com/openimsdk/protocol/constant" "github.com/openimsdk/protocol/msggateway" "github.com/openimsdk/protocol/sdkws" @@ -35,7 +34,7 @@ import ( "google.golang.org/grpc" ) -func (s *Server) InitServer(ctx context.Context, config *Config, disCov discovery.SvcDiscoveryRegistry, server *grpc.Server) error { +func (s *Server) InitServer(ctx context.Context, config *Config, disCov discovery.Conn, server grpc.ServiceRegistrar) error { userConn, err := disCov.GetConn(ctx, config.Discovery.RpcService.User) if err != nil { return err @@ -51,26 +50,26 @@ func (s *Server) InitServer(ctx context.Context, config *Config, disCov discover return nil } -func (s *Server) Start(ctx context.Context, index int, conf *Config) error { - return startrpc.Start(ctx, &conf.Discovery, &conf.MsgGateway.Prometheus, conf.MsgGateway.ListenIP, - conf.MsgGateway.RPC.RegisterIP, - conf.MsgGateway.RPC.AutoSetPorts, conf.MsgGateway.RPC.Ports, index, - conf.Discovery.RpcService.MessageGateway, - nil, - conf, - []string{ - conf.Share.GetConfigFileName(), - conf.Discovery.GetConfigFileName(), - conf.MsgGateway.GetConfigFileName(), - conf.WebhooksConfig.GetConfigFileName(), - conf.RedisConfig.GetConfigFileName(), - }, - []string{ - conf.Discovery.RpcService.MessageGateway, - }, - s.InitServer, - ) -} +//func (s *Server) Start(ctx context.Context, index int, conf *Config) error { +// return startrpc.Start(ctx, &conf.Discovery, &conf.MsgGateway.Prometheus, conf.MsgGateway.ListenIP, +// conf.MsgGateway.RPC.RegisterIP, +// conf.MsgGateway.RPC.AutoSetPorts, conf.MsgGateway.RPC.Ports, index, +// conf.Discovery.RpcService.MessageGateway, +// nil, +// conf, +// []string{ +// conf.Share.GetConfigFileName(), +// conf.Discovery.GetConfigFileName(), +// conf.MsgGateway.GetConfigFileName(), +// conf.WebhooksConfig.GetConfigFileName(), +// conf.RedisConfig.GetConfigFileName(), +// }, +// []string{ +// conf.Discovery.RpcService.MessageGateway, +// }, +// s.InitServer, +// ) +//} type Server struct { msggateway.UnimplementedMsgGatewayServer diff --git a/internal/msggateway/init.go b/internal/msggateway/init.go index 6614b96bd..8772693cc 100644 --- a/internal/msggateway/init.go +++ b/internal/msggateway/init.go @@ -19,10 +19,12 @@ import ( "time" "github.com/openimsdk/open-im-server/v3/pkg/common/config" + "github.com/openimsdk/open-im-server/v3/pkg/dbbuild" "github.com/openimsdk/open-im-server/v3/pkg/rpccache" - "github.com/openimsdk/tools/db/redisutil" + "github.com/openimsdk/tools/discovery" "github.com/openimsdk/tools/utils/datautil" "github.com/openimsdk/tools/utils/runtimeenv" + "google.golang.org/grpc" "github.com/openimsdk/tools/log" ) @@ -33,26 +35,25 @@ type Config struct { RedisConfig config.Redis WebhooksConfig config.Webhooks Discovery config.Discovery - - RuntimeEnv string + Index config.Index } // Start run ws server. -func Start(ctx context.Context, index int, conf *Config) error { - conf.RuntimeEnv = runtimeenv.PrintRuntimeEnvironment() - - log.CInfo(ctx, "MSG-GATEWAY server is initializing", "runtimeEnv", conf.RuntimeEnv, +func Start(ctx context.Context, conf *Config, client discovery.Conn, server grpc.ServiceRegistrar) error { + log.CInfo(ctx, "MSG-GATEWAY server is initializing", "runtimeEnv", runtimeenv.RuntimeEnvironment(), "rpcPorts", conf.MsgGateway.RPC.Ports, "wsPort", conf.MsgGateway.LongConnSvr.Ports, "prometheusPorts", conf.MsgGateway.Prometheus.Ports) - wsPort, err := datautil.GetElemByIndex(conf.MsgGateway.LongConnSvr.Ports, index) + wsPort, err := datautil.GetElemByIndex(conf.MsgGateway.LongConnSvr.Ports, int(conf.Index)) if err != nil { return err } - rdb, err := redisutil.NewRedisClient(ctx, conf.RedisConfig.Build()) + dbb := dbbuild.NewBuilder(nil, &conf.RedisConfig) + rdb, err := dbb.Redis(ctx) if err != nil { return err } + longServer := NewWsServer( conf, WithPort(wsPort), @@ -67,12 +68,50 @@ func Start(ctx context.Context, index int, conf *Config) error { return err }) + if err := hubServer.InitServer(ctx, conf, client, server); err != nil { + return err + } + go longServer.ChangeOnlineStatus(4) - netDone := make(chan error) - go func() { - err = hubServer.Start(ctx, index, conf) - netDone <- err - }() - return hubServer.LongConnServer.Run(netDone) + return hubServer.LongConnServer.Run(ctx) } + +// +//// Start run ws server. +//func Start(ctx context.Context, index int, conf *Config) error { +// log.CInfo(ctx, "MSG-GATEWAY server is initializing", "runtimeEnv", runtimeenv.RuntimeEnvironment(), +// "rpcPorts", conf.MsgGateway.RPC.Ports, +// "wsPort", conf.MsgGateway.LongConnSvr.Ports, "prometheusPorts", conf.MsgGateway.Prometheus.Ports) +// wsPort, err := datautil.GetElemByIndex(conf.MsgGateway.LongConnSvr.Ports, index) +// if err != nil { +// return err +// } +// +// rdb, err := redisutil.NewRedisClient(ctx, conf.RedisConfig.Build()) +// if err != nil { +// return err +// } +// longServer := NewWsServer( +// conf, +// WithPort(wsPort), +// WithMaxConnNum(int64(conf.MsgGateway.LongConnSvr.WebsocketMaxConnNum)), +// WithHandshakeTimeout(time.Duration(conf.MsgGateway.LongConnSvr.WebsocketTimeout)*time.Second), +// WithMessageMaxMsgLength(conf.MsgGateway.LongConnSvr.WebsocketMaxMsgLen), +// ) +// +// hubServer := NewServer(longServer, conf, func(srv *Server) error { +// var err error +// longServer.online, err = rpccache.NewOnlineCache(srv.userClient, nil, rdb, false, longServer.subscriberUserOnlineStatusChanges) +// return err +// }) +// +// go longServer.ChangeOnlineStatus(4) +// +// netDone := make(chan error) +// go func() { +// err = hubServer.Start(ctx, index, conf) +// netDone <- err +// }() +// return hubServer.LongConnServer.Run(netDone) +//} diff --git a/internal/msggateway/ws_server.go b/internal/msggateway/ws_server.go index 24dd823f6..304cb3b5c 100644 --- a/internal/msggateway/ws_server.go +++ b/internal/msggateway/ws_server.go @@ -2,7 +2,6 @@ package msggateway import ( "context" - "errors" "fmt" "net/http" "sync" @@ -11,7 +10,6 @@ import ( "github.com/openimsdk/open-im-server/v3/pkg/rpcli" - "github.com/openimsdk/open-im-server/v3/pkg/common/discovery/etcd" "github.com/openimsdk/open-im-server/v3/pkg/common/webhook" "github.com/openimsdk/open-im-server/v3/pkg/rpccache" pbAuth "github.com/openimsdk/protocol/auth" @@ -23,19 +21,18 @@ import ( "github.com/openimsdk/protocol/constant" "github.com/openimsdk/protocol/msggateway" "github.com/openimsdk/tools/discovery" - "github.com/openimsdk/tools/errs" "github.com/openimsdk/tools/log" "github.com/openimsdk/tools/utils/stringutil" "golang.org/x/sync/errgroup" ) type LongConnServer interface { - Run(done chan error) error + Run(ctx context.Context) error wsHandler(w http.ResponseWriter, r *http.Request) GetUserAllCons(userID string) ([]*Client, bool) GetUserPlatformCons(userID string, platform int) ([]*Client, bool, bool) Validate(s any) error - SetDiscoveryRegistry(ctx context.Context, client discovery.SvcDiscoveryRegistry, config *Config) error + SetDiscoveryRegistry(ctx context.Context, client discovery.Conn, config *Config) error KickUserConn(client *Client) error UnRegister(c *Client) SetKickHandlerInfo(i *kickHandler) @@ -60,7 +57,7 @@ type WsServer struct { handshakeTimeout time.Duration writeBufferSize int validate *validator.Validate - disCov discovery.SvcDiscoveryRegistry + disCov discovery.Conn Compressor //Encoder MessageHandler @@ -75,7 +72,7 @@ type kickHandler struct { newClient *Client } -func (ws *WsServer) SetDiscoveryRegistry(ctx context.Context, disCov discovery.SvcDiscoveryRegistry, config *Config) error { +func (ws *WsServer) SetDiscoveryRegistry(ctx context.Context, disCov discovery.Conn, config *Config) error { userConn, err := disCov.GetConn(ctx, config.Discovery.RpcService.User) if err != nil { return err @@ -158,19 +155,14 @@ func NewWsServer(msgGatewayConfig *Config, opts ...Option) *WsServer { } } -func (ws *WsServer) Run(done chan error) error { - var ( - client *Client - netErr error - shutdownDone = make(chan struct{}, 1) - ) - - server := http.Server{Addr: ":" + stringutil.IntToString(ws.port), Handler: nil} +func (ws *WsServer) Run(ctx context.Context) error { + var client *Client + ctx, cancel := context.WithCancelCause(ctx) go func() { for { select { - case <-shutdownDone: + case <-ctx.Done(): return case client = <-ws.registerChan: ws.registerClient(client) @@ -181,57 +173,56 @@ func (ws *WsServer) Run(done chan error) error { } } }() - netDone := make(chan struct{}, 1) + + done := make(chan struct{}) go func() { + wsServer := http.Server{Addr: fmt.Sprintf(":%d", ws.port), Handler: nil} http.HandleFunc("/", ws.wsHandler) - err := server.ListenAndServe() - if err != nil && !errors.Is(err, http.ErrServerClosed) { - netErr = errs.WrapMsg(err, "ws start err", server.Addr) - netDone <- struct{}{} + go func() { + defer close(done) + <-ctx.Done() + _ = wsServer.Shutdown(context.Background()) + }() + err := wsServer.ListenAndServe() + if err == nil { + err = fmt.Errorf("http server closed") } + cancel(fmt.Errorf("msg gateway %w", err)) }() - ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) - shutDown := func() error { - sErr := server.Shutdown(ctx) - if sErr != nil { - return errs.WrapMsg(sErr, "shutdown err") - } - close(shutdownDone) - return nil - } - etcd.RegisterShutDown(shutDown) - defer cancel() - var err error + + <-ctx.Done() + + timeout := time.NewTimer(time.Second * 15) + defer timeout.Stop() select { - case err = <-done: - if err := shutDown(); err != nil { - return err - } - if err != nil { - return err - } - case <-netDone: + case <-timeout.C: + log.ZWarn(ctx, "msg gateway graceful stop timeout", nil) + case <-done: + log.ZDebug(ctx, "msg gateway graceful stop done") } - return netErr - + return context.Cause(ctx) } -var concurrentRequest = 3 +const concurrentRequest = 3 func (ws *WsServer) sendUserOnlineInfoToOtherNode(ctx context.Context, client *Client) error { conns, err := ws.disCov.GetConns(ctx, ws.msgGatewayConfig.Discovery.RpcService.MessageGateway) if err != nil { return err } + if len(conns) == 0 || (len(conns) == 1 && ws.disCov.IsSelfNode(conns[0])) { + return nil + } + wg := errgroup.Group{} wg.SetLimit(concurrentRequest) // Online push user online message to other node for _, v := range conns { v := v - log.ZDebug(ctx, " sendUserOnlineInfoToOtherNode conn ", "target", v.Target()) - if v.Target() == ws.disCov.GetSelfConnTarget() { - log.ZDebug(ctx, "Filter out this node", "node", v.Target()) + log.ZDebug(ctx, "sendUserOnlineInfoToOtherNode conn") + if ws.disCov.IsSelfNode(v) { + log.ZDebug(ctx, "Filter out this node") continue } @@ -242,7 +233,7 @@ func (ws *WsServer) sendUserOnlineInfoToOtherNode(ctx context.Context, client *C PlatformID: int32(client.PlatformID), Token: client.token, }) if err != nil { - log.ZWarn(ctx, "MultiTerminalLoginCheck err", err, "node", v.Target()) + log.ZWarn(ctx, "MultiTerminalLoginCheck err", err) } return nil }) diff --git a/internal/msgtransfer/init.go b/internal/msgtransfer/init.go index 96e6bbde0..175813552 100644 --- a/internal/msgtransfer/init.go +++ b/internal/msgtransfer/init.go @@ -16,51 +16,35 @@ package msgtransfer import ( "context" - "errors" "fmt" - "net" - "net/http" - "os" - "os/signal" - "strconv" - "syscall" - - disetcd "github.com/openimsdk/open-im-server/v3/pkg/common/discovery/etcd" - "github.com/openimsdk/tools/discovery" - "github.com/openimsdk/tools/discovery/etcd" - "github.com/openimsdk/tools/utils/jsonutil" - "github.com/openimsdk/tools/utils/network" - "github.com/openimsdk/open-im-server/v3/pkg/common/prommetrics" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/mcache" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/redis" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database/mgo" - "github.com/openimsdk/tools/db/mongoutil" - "github.com/openimsdk/tools/db/redisutil" - "github.com/openimsdk/tools/utils/datautil" + "github.com/openimsdk/open-im-server/v3/pkg/dbbuild" + "github.com/openimsdk/open-im-server/v3/pkg/mqbuild" + "github.com/openimsdk/tools/discovery" + "github.com/openimsdk/tools/mq" "github.com/openimsdk/tools/utils/runtimeenv" conf "github.com/openimsdk/open-im-server/v3/pkg/common/config" - discRegister "github.com/openimsdk/open-im-server/v3/pkg/common/discovery" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/controller" - "github.com/openimsdk/tools/errs" "github.com/openimsdk/tools/log" - "github.com/openimsdk/tools/mw" - "github.com/openimsdk/tools/system/program" "google.golang.org/grpc" - "google.golang.org/grpc/credentials/insecure" ) type MsgTransfer struct { + historyConsumer mq.Consumer + historyMongoConsumer mq.Consumer // This consumer aggregated messages, subscribed to the topic:toRedis, // the message is stored in redis, Incr Redis, and then the message is sent to toPush topic for push, // and the message is sent to toMongo topic for persistence - historyCH *OnlineHistoryRedisConsumerHandler + historyHandler *OnlineHistoryRedisConsumerHandler //This consumer handle message to mongo - historyMongoCH *OnlineHistoryMongoConsumerHandler - ctx context.Context - cancel context.CancelFunc - - runTimeEnv string + historyMongoHandler *OnlineHistoryMongoConsumerHandler + ctx context.Context + //cancel context.CancelFunc } type Config struct { @@ -71,48 +55,59 @@ type Config struct { Share conf.Share WebhooksConfig conf.Webhooks Discovery conf.Discovery + Index conf.Index } -func Start(ctx context.Context, index int, config *Config) error { - runTimeEnv := runtimeenv.PrintRuntimeEnvironment() +func Start(ctx context.Context, config *Config, client discovery.Conn, server grpc.ServiceRegistrar) error { + builder := mqbuild.NewBuilder(&config.KafkaConfig) - log.CInfo(ctx, "MSG-TRANSFER server is initializing", "runTimeEnv", runTimeEnv, "prometheusPorts", - config.MsgTransfer.Prometheus.Ports, "index", index) - - mgocli, err := mongoutil.NewMongoDB(ctx, config.MongodbConfig.Build()) + log.CInfo(ctx, "MSG-TRANSFER server is initializing", "runTimeEnv", runtimeenv.RuntimeEnvironment(), "prometheusPorts", + config.MsgTransfer.Prometheus.Ports, "index", config.Index) + dbb := dbbuild.NewBuilder(&config.MongodbConfig, &config.RedisConfig) + mgocli, err := dbb.Mongo(ctx) if err != nil { return err } - rdb, err := redisutil.NewRedisClient(ctx, config.RedisConfig.Build()) + rdb, err := dbb.Redis(ctx) if err != nil { return err } - client, err := discRegister.NewDiscoveryRegister(&config.Discovery, runTimeEnv, nil) + + //if config.Discovery.Enable == conf.ETCD { + // cm := disetcd.NewConfigManager(client.(*etcd.SvcDiscoveryRegistryImpl).GetClient(), []string{ + // config.MsgTransfer.GetConfigFileName(), + // config.RedisConfig.GetConfigFileName(), + // config.MongodbConfig.GetConfigFileName(), + // config.KafkaConfig.GetConfigFileName(), + // config.Share.GetConfigFileName(), + // config.WebhooksConfig.GetConfigFileName(), + // config.Discovery.GetConfigFileName(), + // conf.LogConfigFileName, + // }) + // cm.Watch(ctx) + //} + mongoProducer, err := builder.GetTopicProducer(ctx, config.KafkaConfig.ToMongoTopic) if err != nil { return err } - client.AddOption(mw.GrpcClient(), grpc.WithTransportCredentials(insecure.NewCredentials()), - grpc.WithDefaultServiceConfig(fmt.Sprintf(`{"LoadBalancingPolicy": "%s"}`, "round_robin"))) - - if config.Discovery.Enable == conf.ETCD { - cm := disetcd.NewConfigManager(client.(*etcd.SvcDiscoveryRegistryImpl).GetClient(), []string{ - config.MsgTransfer.GetConfigFileName(), - config.RedisConfig.GetConfigFileName(), - config.MongodbConfig.GetConfigFileName(), - config.KafkaConfig.GetConfigFileName(), - config.Share.GetConfigFileName(), - config.WebhooksConfig.GetConfigFileName(), - config.Discovery.GetConfigFileName(), - conf.LogConfigFileName, - }) - cm.Watch(ctx) + pushProducer, err := builder.GetTopicProducer(ctx, config.KafkaConfig.ToPushTopic) + if err != nil { + return err } - msgDocModel, err := mgo.NewMsgMongo(mgocli.GetDB()) if err != nil { return err } - msgModel := redis.NewMsgCache(rdb, msgDocModel) + var msgModel cache.MsgCache + if rdb == nil { + cm, err := mgo.NewCacheMgo(mgocli.GetDB()) + if err != nil { + return err + } + msgModel = mcache.NewMsgCache(cm, msgDocModel) + } else { + msgModel = redis.NewMsgCache(rdb, msgDocModel) + } seqConversation, err := mgo.NewSeqConversationMongo(mgocli.GetDB()) if err != nil { return err @@ -123,124 +118,68 @@ func Start(ctx context.Context, index int, config *Config) error { return err } seqUserCache := redis.NewSeqUserCacheRedis(rdb, seqUser) - msgTransferDatabase, err := controller.NewMsgTransferDatabase(msgDocModel, msgModel, seqUserCache, seqConversationCache, &config.KafkaConfig) + msgTransferDatabase, err := controller.NewMsgTransferDatabase(msgDocModel, msgModel, seqUserCache, seqConversationCache, mongoProducer, pushProducer) if err != nil { return err } - historyCH, err := NewOnlineHistoryRedisConsumerHandler(ctx, client, config, msgTransferDatabase) + historyConsumer, err := builder.GetTopicConsumer(ctx, config.KafkaConfig.ToRedisTopic) if err != nil { return err } - historyMongoCH, err := NewOnlineHistoryMongoConsumerHandler(&config.KafkaConfig, msgTransferDatabase) + historyMongoConsumer, err := builder.GetTopicConsumer(ctx, config.KafkaConfig.ToMongoTopic) if err != nil { return err } - - msgTransfer := &MsgTransfer{ - historyCH: historyCH, - historyMongoCH: historyMongoCH, - runTimeEnv: runTimeEnv, - } - - return msgTransfer.Start(index, config, client) -} - -func (m *MsgTransfer) Start(index int, config *Config, client discovery.SvcDiscoveryRegistry) error { - m.ctx, m.cancel = context.WithCancel(context.Background()) - var ( - netDone = make(chan struct{}, 1) - netErr error - ) - - go m.historyCH.historyConsumerGroup.RegisterHandleAndConsumer(m.ctx, m.historyCH) - go m.historyMongoCH.historyConsumerGroup.RegisterHandleAndConsumer(m.ctx, m.historyMongoCH) - go m.historyCH.HandleUserHasReadSeqMessages(m.ctx) - err := m.historyCH.redisMessageBatches.Start() + historyHandler, err := NewOnlineHistoryRedisConsumerHandler(ctx, client, config, msgTransferDatabase) if err != nil { return err } + historyMongoHandler := NewOnlineHistoryMongoConsumerHandler(msgTransferDatabase) - registerIP, err := network.GetRpcRegisterIP("") - if err != nil { - return err - } - - getAutoPort := func() (net.Listener, int, error) { - registerAddr := net.JoinHostPort(registerIP, "0") - listener, err := net.Listen("tcp", registerAddr) - if err != nil { - return nil, 0, errs.WrapMsg(err, "listen err", "registerAddr", registerAddr) - } - _, portStr, _ := net.SplitHostPort(listener.Addr().String()) - port, _ := strconv.Atoi(portStr) - return listener, port, nil + msgTransfer := &MsgTransfer{ + historyConsumer: historyConsumer, + historyMongoConsumer: historyMongoConsumer, + historyHandler: historyHandler, + historyMongoHandler: historyMongoHandler, } - if config.MsgTransfer.Prometheus.AutoSetPorts && config.Discovery.Enable != conf.ETCD { - return errs.New("only etcd support autoSetPorts", "RegisterName", "api").Wrap() - } + return msgTransfer.Start(ctx) +} - if config.MsgTransfer.Prometheus.Enable { - var ( - listener net.Listener - prometheusPort int - ) +func (m *MsgTransfer) Start(ctx context.Context) error { + var cancel context.CancelCauseFunc + m.ctx, cancel = context.WithCancelCause(ctx) - if config.MsgTransfer.Prometheus.AutoSetPorts { - listener, prometheusPort, err = getAutoPort() - if err != nil { - return err + go func() { + for { + if err := m.historyConsumer.Subscribe(m.ctx, m.historyHandler.HandlerRedisMessage); err != nil { + cancel(fmt.Errorf("history consumer %w", err)) + log.ZError(m.ctx, "historyConsumer err", err) + return } + } + }() - etcdClient := client.(*etcd.SvcDiscoveryRegistryImpl).GetClient() - - _, err = etcdClient.Put(context.TODO(), prommetrics.BuildDiscoveryKey(prommetrics.MessageTransferKeyName), jsonutil.StructToJsonString(prommetrics.BuildDefaultTarget(registerIP, prometheusPort))) - if err != nil { - return errs.WrapMsg(err, "etcd put err") - } - } else { - prometheusPort, err = datautil.GetElemByIndex(config.MsgTransfer.Prometheus.Ports, index) - if err != nil { - return err - } - listener, err = net.Listen("tcp", fmt.Sprintf(":%d", prometheusPort)) - if err != nil { - return errs.WrapMsg(err, "listen err", "addr", fmt.Sprintf(":%d", prometheusPort)) + go func() { + fn := func(ctx context.Context, key string, value []byte) error { + m.historyMongoHandler.HandleChatWs2Mongo(ctx, key, value) + return nil + } + for { + if err := m.historyMongoConsumer.Subscribe(m.ctx, fn); err != nil { + cancel(fmt.Errorf("history mongo consumer %w", err)) + log.ZError(m.ctx, "historyMongoConsumer err", err) + return } } + }() - go func() { - defer func() { - if r := recover(); r != nil { - log.ZPanic(m.ctx, "MsgTransfer Start Panic", errs.ErrPanic(r)) - } - }() - if err := prommetrics.TransferInit(listener); err != nil && !errors.Is(err, http.ErrServerClosed) { - netErr = errs.WrapMsg(err, "prometheus start error", "prometheusPort", prometheusPort) - netDone <- struct{}{} - } - }() - } + go m.historyHandler.HandleUserHasReadSeqMessages(m.ctx) - sigs := make(chan os.Signal, 1) - signal.Notify(sigs, syscall.SIGTERM) - select { - case <-sigs: - program.SIGTERMExit() - // graceful close kafka client. - m.cancel() - m.historyCH.redisMessageBatches.Close() - m.historyCH.Close() - m.historyCH.historyConsumerGroup.Close() - m.historyMongoCH.historyConsumerGroup.Close() - return nil - case <-netDone: - m.cancel() - m.historyCH.redisMessageBatches.Close() - m.historyCH.Close() - m.historyCH.historyConsumerGroup.Close() - m.historyMongoCH.historyConsumerGroup.Close() - close(netDone) - return netErr + err := m.historyHandler.redisMessageBatches.Start() + if err != nil { + return err } + <-m.ctx.Done() + return context.Cause(m.ctx) } diff --git a/internal/msgtransfer/online_history_msg_handler.go b/internal/msgtransfer/online_history_msg_handler.go index 6334c95fd..a2d0cca67 100644 --- a/internal/msgtransfer/online_history_msg_handler.go +++ b/internal/msgtransfer/online_history_msg_handler.go @@ -18,14 +18,13 @@ import ( "context" "encoding/json" "errors" - "github.com/openimsdk/open-im-server/v3/pkg/rpcli" - "github.com/openimsdk/tools/discovery" - "strconv" - "strings" + "sync" "time" - "github.com/IBM/sarama" + "github.com/openimsdk/open-im-server/v3/pkg/rpcli" + "github.com/openimsdk/tools/discovery" + "github.com/go-redis/redis" "github.com/openimsdk/open-im-server/v3/pkg/common/prommetrics" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/controller" @@ -37,7 +36,6 @@ import ( "github.com/openimsdk/tools/errs" "github.com/openimsdk/tools/log" "github.com/openimsdk/tools/mcontext" - "github.com/openimsdk/tools/mq/kafka" "github.com/openimsdk/tools/utils/stringutil" "google.golang.org/protobuf/proto" ) @@ -64,9 +62,7 @@ type userHasReadSeq struct { } type OnlineHistoryRedisConsumerHandler struct { - historyConsumerGroup *kafka.MConsumerGroup - - redisMessageBatches *batcher.Batcher[sarama.ConsumerMessage] + redisMessageBatches *batcher.Batcher[ConsumerMessage] msgTransferDatabase controller.MsgTransferDatabase conversationUserHasReadChan chan *userHasReadSeq @@ -76,12 +72,13 @@ type OnlineHistoryRedisConsumerHandler struct { conversationClient *rpcli.ConversationClient } -func NewOnlineHistoryRedisConsumerHandler(ctx context.Context, client discovery.SvcDiscoveryRegistry, config *Config, database controller.MsgTransferDatabase) (*OnlineHistoryRedisConsumerHandler, error) { - kafkaConf := config.KafkaConfig - historyConsumerGroup, err := kafka.NewMConsumerGroup(kafkaConf.Build(), kafkaConf.ToRedisGroupID, []string{kafkaConf.ToRedisTopic}, false) - if err != nil { - return nil, err - } +type ConsumerMessage struct { + Ctx context.Context + Key string + Value []byte +} + +func NewOnlineHistoryRedisConsumerHandler(ctx context.Context, client discovery.Conn, config *Config, database controller.MsgTransferDatabase) (*OnlineHistoryRedisConsumerHandler, error) { groupConn, err := client.GetConn(ctx, config.Discovery.RpcService.Group) if err != nil { return nil, err @@ -97,7 +94,7 @@ func NewOnlineHistoryRedisConsumerHandler(ctx context.Context, client discovery. och.conversationClient = rpcli.NewConversationClient(conversationConn) och.wg.Add(1) - b := batcher.New[sarama.ConsumerMessage]( + b := batcher.New[ConsumerMessage]( batcher.WithSize(size), batcher.WithWorker(worker), batcher.WithInterval(interval), @@ -109,16 +106,15 @@ func NewOnlineHistoryRedisConsumerHandler(ctx context.Context, client discovery. hashCode := stringutil.GetHashCode(key) return int(hashCode) % och.redisMessageBatches.Worker() } - b.Key = func(consumerMessage *sarama.ConsumerMessage) string { - return string(consumerMessage.Key) + b.Key = func(consumerMessage *ConsumerMessage) string { + return consumerMessage.Key } b.Do = och.do och.redisMessageBatches = b - och.historyConsumerGroup = historyConsumerGroup return &och, nil } -func (och *OnlineHistoryRedisConsumerHandler) do(ctx context.Context, channelID int, val *batcher.Msg[sarama.ConsumerMessage]) { +func (och *OnlineHistoryRedisConsumerHandler) do(ctx context.Context, channelID int, val *batcher.Msg[ConsumerMessage]) { ctx = mcontext.WithTriggerIDContext(ctx, val.TriggerID()) ctxMessages := och.parseConsumerMessages(ctx, val.Val()) ctx = withAggregationCtx(ctx, ctxMessages) @@ -189,7 +185,7 @@ func (och *OnlineHistoryRedisConsumerHandler) doSetReadSeq(ctx context.Context, } -func (och *OnlineHistoryRedisConsumerHandler) parseConsumerMessages(ctx context.Context, consumerMessages []*sarama.ConsumerMessage) []*ContextMsg { +func (och *OnlineHistoryRedisConsumerHandler) parseConsumerMessages(ctx context.Context, consumerMessages []*ConsumerMessage) []*ContextMsg { var ctxMessages []*ContextMsg for i := 0; i < len(consumerMessages); i++ { ctxMsg := &ContextMsg{} @@ -199,16 +195,9 @@ func (och *OnlineHistoryRedisConsumerHandler) parseConsumerMessages(ctx context. log.ZWarn(ctx, "msg_transfer Unmarshal msg err", err, string(consumerMessages[i].Value)) continue } - var arr []string - for i, header := range consumerMessages[i].Headers { - arr = append(arr, strconv.Itoa(i), string(header.Key), string(header.Value)) - } - log.ZDebug(ctx, "consumer.kafka.GetContextWithMQHeader", "len", len(consumerMessages[i].Headers), - "header", strings.Join(arr, ", ")) - ctxMsg.ctx = kafka.GetContextWithMQHeader(consumerMessages[i].Headers) + ctxMsg.ctx = consumerMessages[i].Ctx ctxMsg.message = msgFromMQ - log.ZDebug(ctx, "message parse finish", "message", msgFromMQ, "key", - string(consumerMessages[i].Key)) + log.ZDebug(ctx, "message parse finish", "message", msgFromMQ, "key", consumerMessages[i].Key) ctxMessages = append(ctxMessages, ctxMsg) } return ctxMessages @@ -383,7 +372,9 @@ func (och *OnlineHistoryRedisConsumerHandler) Close() { func (och *OnlineHistoryRedisConsumerHandler) toPushTopic(ctx context.Context, key, conversationID string, msgs []*ContextMsg) { for _, v := range msgs { log.ZDebug(ctx, "push msg to topic", "msg", v.message.String()) - _, _, _ = och.msgTransferDatabase.MsgToPushMQ(v.ctx, key, conversationID, v.message) + if err := och.msgTransferDatabase.MsgToPushMQ(v.ctx, key, conversationID, v.message); err != nil { + log.ZError(ctx, "msg to push topic error", err, "msg", v.message.String()) + } } } @@ -401,35 +392,10 @@ func withAggregationCtx(ctx context.Context, values []*ContextMsg) context.Conte return mcontext.SetOperationID(ctx, allMessageOperationID) } -func (och *OnlineHistoryRedisConsumerHandler) Setup(_ sarama.ConsumerGroupSession) error { return nil } -func (och *OnlineHistoryRedisConsumerHandler) Cleanup(_ sarama.ConsumerGroupSession) error { - return nil -} - -func (och *OnlineHistoryRedisConsumerHandler) ConsumeClaim(session sarama.ConsumerGroupSession, - claim sarama.ConsumerGroupClaim) error { // a instance in the consumer group - log.ZDebug(context.Background(), "online new session msg come", "highWaterMarkOffset", - claim.HighWaterMarkOffset(), "topic", claim.Topic(), "partition", claim.Partition()) - och.redisMessageBatches.OnComplete = func(lastMessage *sarama.ConsumerMessage, totalCount int) { - session.MarkMessage(lastMessage, "") - session.Commit() - } - for { - select { - case msg, ok := <-claim.Messages(): - if !ok { - return nil - } - - if len(msg.Value) == 0 { - continue - } - err := och.redisMessageBatches.Put(context.Background(), msg) - if err != nil { - log.ZWarn(context.Background(), "put msg to error", err, "msg", msg) - } - case <-session.Context().Done(): - return nil - } +func (och *OnlineHistoryRedisConsumerHandler) HandlerRedisMessage(ctx context.Context, key string, value []byte) error { // a instance in the consumer group + err := och.redisMessageBatches.Put(ctx, &ConsumerMessage{Ctx: ctx, Key: key, Value: value}) + if err != nil { + log.ZWarn(ctx, "put msg to error", err, "key", key, "value", value) } + return nil } diff --git a/internal/msgtransfer/online_msg_to_mongo_handler.go b/internal/msgtransfer/online_msg_to_mongo_handler.go index 8405be7fe..ae14d02a1 100644 --- a/internal/msgtransfer/online_msg_to_mongo_handler.go +++ b/internal/msgtransfer/online_msg_to_mongo_handler.go @@ -17,36 +17,24 @@ package msgtransfer import ( "context" - "github.com/IBM/sarama" - "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/open-im-server/v3/pkg/common/prommetrics" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/controller" pbmsg "github.com/openimsdk/protocol/msg" "github.com/openimsdk/tools/log" - "github.com/openimsdk/tools/mq/kafka" "google.golang.org/protobuf/proto" ) type OnlineHistoryMongoConsumerHandler struct { - historyConsumerGroup *kafka.MConsumerGroup - msgTransferDatabase controller.MsgTransferDatabase + msgTransferDatabase controller.MsgTransferDatabase } -func NewOnlineHistoryMongoConsumerHandler(kafkaConf *config.Kafka, database controller.MsgTransferDatabase) (*OnlineHistoryMongoConsumerHandler, error) { - historyConsumerGroup, err := kafka.NewMConsumerGroup(kafkaConf.Build(), kafkaConf.ToMongoGroupID, []string{kafkaConf.ToMongoTopic}, true) - if err != nil { - return nil, err - } - - mc := &OnlineHistoryMongoConsumerHandler{ - historyConsumerGroup: historyConsumerGroup, - msgTransferDatabase: database, +func NewOnlineHistoryMongoConsumerHandler(database controller.MsgTransferDatabase) *OnlineHistoryMongoConsumerHandler { + return &OnlineHistoryMongoConsumerHandler{ + msgTransferDatabase: database, } - return mc, nil } -func (mc *OnlineHistoryMongoConsumerHandler) handleChatWs2Mongo(ctx context.Context, cMsg *sarama.ConsumerMessage, key string, session sarama.ConsumerGroupSession) { - msg := cMsg.Value +func (mc *OnlineHistoryMongoConsumerHandler) HandleChatWs2Mongo(ctx context.Context, key string, msg []byte) { msgFromMQ := pbmsg.MsgDataToMongoByMQ{} err := proto.Unmarshal(msg, &msgFromMQ) if err != nil { @@ -54,7 +42,7 @@ func (mc *OnlineHistoryMongoConsumerHandler) handleChatWs2Mongo(ctx context.Cont return } if len(msgFromMQ.MsgData) == 0 { - log.ZError(ctx, "msgFromMQ.MsgData is empty", nil, "cMsg", cMsg) + log.ZError(ctx, "msgFromMQ.MsgData is empty", nil, "key", key, "msg", msg) return } log.ZDebug(ctx, "mongo consumer recv msg", "msgs", msgFromMQ.String()) @@ -82,22 +70,3 @@ func (mc *OnlineHistoryMongoConsumerHandler) handleChatWs2Mongo(ctx context.Cont // msgFromMQ.MsgData, "conversationID", msgFromMQ.ConversationID) //} } - -func (*OnlineHistoryMongoConsumerHandler) Setup(_ sarama.ConsumerGroupSession) error { return nil } - -func (*OnlineHistoryMongoConsumerHandler) Cleanup(_ sarama.ConsumerGroupSession) error { return nil } - -func (mc *OnlineHistoryMongoConsumerHandler) ConsumeClaim(sess sarama.ConsumerGroupSession, claim sarama.ConsumerGroupClaim) error { // an instance in the consumer group - log.ZDebug(context.Background(), "online new session msg come", "highWaterMarkOffset", - claim.HighWaterMarkOffset(), "topic", claim.Topic(), "partition", claim.Partition()) - for msg := range claim.Messages() { - ctx := mc.historyConsumerGroup.GetContextFromMsg(msg) - if len(msg.Value) != 0 { - mc.handleChatWs2Mongo(ctx, msg, string(msg.Key), sess) - } else { - log.ZError(ctx, "mongo msg get from kafka but is nil", nil, "conversationID", msg.Key) - } - sess.MarkMessage(msg, "") - } - return nil -} diff --git a/internal/push/callback.go b/internal/push/callback.go index f8e17bb8c..2d1b8090d 100644 --- a/internal/push/callback.go +++ b/internal/push/callback.go @@ -17,6 +17,7 @@ package push import ( "context" "encoding/json" + "github.com/openimsdk/open-im-server/v3/pkg/common/webhook" "github.com/openimsdk/open-im-server/v3/pkg/callbackstruct" diff --git a/internal/push/offlinepush_handler.go b/internal/push/offlinepush_handler.go index 5c69da005..eaf6b8ed8 100644 --- a/internal/push/offlinepush_handler.go +++ b/internal/push/offlinepush_handler.go @@ -3,7 +3,6 @@ package push import ( "context" - "github.com/IBM/sarama" "github.com/openimsdk/open-im-server/v3/internal/push/offlinepush" "github.com/openimsdk/open-im-server/v3/internal/push/offlinepush/options" "github.com/openimsdk/open-im-server/v3/pkg/common/prommetrics" @@ -12,40 +11,21 @@ import ( "github.com/openimsdk/protocol/sdkws" "github.com/openimsdk/tools/errs" "github.com/openimsdk/tools/log" - "github.com/openimsdk/tools/mq/kafka" "github.com/openimsdk/tools/utils/jsonutil" "google.golang.org/protobuf/proto" ) type OfflinePushConsumerHandler struct { - OfflinePushConsumerGroup *kafka.MConsumerGroup - offlinePusher offlinepush.OfflinePusher + offlinePusher offlinepush.OfflinePusher } -func NewOfflinePushConsumerHandler(config *Config, offlinePusher offlinepush.OfflinePusher) (*OfflinePushConsumerHandler, error) { - var offlinePushConsumerHandler OfflinePushConsumerHandler - var err error - offlinePushConsumerHandler.offlinePusher = offlinePusher - offlinePushConsumerHandler.OfflinePushConsumerGroup, err = kafka.NewMConsumerGroup(config.KafkaConfig.Build(), config.KafkaConfig.ToOfflineGroupID, - []string{config.KafkaConfig.ToOfflinePushTopic}, true) - if err != nil { - return nil, err +func NewOfflinePushConsumerHandler(offlinePusher offlinepush.OfflinePusher) *OfflinePushConsumerHandler { + return &OfflinePushConsumerHandler{ + offlinePusher: offlinePusher, } - return &offlinePushConsumerHandler, nil -} - -func (*OfflinePushConsumerHandler) Setup(sarama.ConsumerGroupSession) error { return nil } -func (*OfflinePushConsumerHandler) Cleanup(sarama.ConsumerGroupSession) error { return nil } -func (o *OfflinePushConsumerHandler) ConsumeClaim(sess sarama.ConsumerGroupSession, claim sarama.ConsumerGroupClaim) error { - for msg := range claim.Messages() { - ctx := o.OfflinePushConsumerGroup.GetContextFromMsg(msg) - o.handleMsg2OfflinePush(ctx, msg.Value) - sess.MarkMessage(msg, "") - } - return nil } -func (o *OfflinePushConsumerHandler) handleMsg2OfflinePush(ctx context.Context, msg []byte) { +func (o *OfflinePushConsumerHandler) HandleMsg2OfflinePush(ctx context.Context, msg []byte) { offlinePushMsg := pbpush.PushMsgReq{} if err := proto.Unmarshal(msg, &offlinePushMsg); err != nil { log.ZError(ctx, "offline push Unmarshal msg err", err, "msg", string(msg)) diff --git a/internal/push/onlinepusher.go b/internal/push/onlinepusher.go index 23e68339c..b9d371c55 100644 --- a/internal/push/onlinepusher.go +++ b/internal/push/onlinepusher.go @@ -2,7 +2,7 @@ package push import ( "context" - "errors" + "fmt" "sync" "github.com/openimsdk/protocol/msggateway" @@ -11,6 +11,7 @@ import ( "github.com/openimsdk/tools/errs" "github.com/openimsdk/tools/log" "github.com/openimsdk/tools/utils/datautil" + "github.com/openimsdk/tools/utils/runtimeenv" "golang.org/x/sync/errgroup" "google.golang.org/grpc" @@ -30,37 +31,36 @@ func newEmptyOnlinePusher() *emptyOnlinePusher { return &emptyOnlinePusher{} } -func (emptyOnlinePusher) GetConnsAndOnlinePush(ctx context.Context, msg *sdkws.MsgData, - pushToUserIDs []string) (wsResults []*msggateway.SingleMsgToUserResults, err error) { +func (emptyOnlinePusher) GetConnsAndOnlinePush(ctx context.Context, msg *sdkws.MsgData, pushToUserIDs []string) (wsResults []*msggateway.SingleMsgToUserResults, err error) { log.ZInfo(ctx, "emptyOnlinePusher GetConnsAndOnlinePush", nil) return nil, nil } -func (u emptyOnlinePusher) GetOnlinePushFailedUserIDs(ctx context.Context, msg *sdkws.MsgData, - wsResults []*msggateway.SingleMsgToUserResults, pushToUserIDs *[]string) []string { +func (u emptyOnlinePusher) GetOnlinePushFailedUserIDs(ctx context.Context, msg *sdkws.MsgData, wsResults []*msggateway.SingleMsgToUserResults, pushToUserIDs *[]string) []string { log.ZInfo(ctx, "emptyOnlinePusher GetOnlinePushFailedUserIDs", nil) return nil } -func NewOnlinePusher(disCov discovery.SvcDiscoveryRegistry, config *Config) OnlinePusher { - - if config.runTimeEnv == conf.KUBERNETES { - return NewDefaultAllNode(disCov, config) +func NewOnlinePusher(disCov discovery.Conn, config *Config) (OnlinePusher, error) { + if conf.Standalone() { + return NewDefaultAllNode(disCov, config), nil + } + if runtimeenv.RuntimeEnvironment() == conf.KUBERNETES { + return NewDefaultAllNode(disCov, config), nil } switch config.Discovery.Enable { case conf.ETCD: - return NewDefaultAllNode(disCov, config) + return NewDefaultAllNode(disCov, config), nil default: - log.ZError(context.Background(), "NewOnlinePusher is error", errs.Wrap(errors.New("unsupported discovery type")), "type", config.Discovery.Enable) - return nil + return nil, errs.New(fmt.Sprintf("unsupported discovery type %s", config.Discovery.Enable)) } } type DefaultAllNode struct { - disCov discovery.SvcDiscoveryRegistry + disCov discovery.Conn config *Config } -func NewDefaultAllNode(disCov discovery.SvcDiscoveryRegistry, config *Config) *DefaultAllNode { +func NewDefaultAllNode(disCov discovery.Conn, config *Config) *DefaultAllNode { return &DefaultAllNode{disCov: disCov, config: config} } @@ -166,7 +166,7 @@ func (k *K8sStaticConsistentHash) GetConnsAndOnlinePush(ctx context.Context, msg } } log.ZDebug(ctx, "genUsers send hosts struct:", "usersHost", usersHost) - var usersConns = make(map[*grpc.ClientConn][]string) + var usersConns = make(map[grpc.ClientConnInterface][]string) for host, userIds := range usersHost { tconn, _ := k.disCov.GetConn(ctx, host) usersConns[tconn] = userIds diff --git a/internal/push/push.go b/internal/push/push.go index b7c1ec427..13818e93d 100644 --- a/internal/push/push.go +++ b/internal/push/push.go @@ -2,39 +2,43 @@ package push import ( "context" + "math/rand" + "strconv" "github.com/openimsdk/open-im-server/v3/internal/push/offlinepush" "github.com/openimsdk/open-im-server/v3/pkg/common/config" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/mcache" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/redis" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/controller" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database/mgo" + "github.com/openimsdk/open-im-server/v3/pkg/dbbuild" + "github.com/openimsdk/open-im-server/v3/pkg/mqbuild" pbpush "github.com/openimsdk/protocol/push" - "github.com/openimsdk/tools/db/redisutil" "github.com/openimsdk/tools/discovery" - "github.com/openimsdk/tools/utils/runtimeenv" + "github.com/openimsdk/tools/log" + "github.com/openimsdk/tools/mcontext" "google.golang.org/grpc" ) type pushServer struct { pbpush.UnimplementedPushMsgServiceServer database controller.PushDatabase - disCov discovery.SvcDiscoveryRegistry + disCov discovery.Conn offlinePusher offlinepush.OfflinePusher - pushCh *ConsumerHandler - offlinePushCh *OfflinePushConsumerHandler } type Config struct { RpcConfig config.Push RedisConfig config.Redis + MongoConfig config.Mongo KafkaConfig config.Kafka NotificationConfig config.Notification Share config.Share WebhooksConfig config.Webhooks LocalCacheConfig config.LocalCache Discovery config.Discovery - FcmConfigPath string - - runTimeEnv string + FcmConfigPath config.Path } func (p pushServer) DelUserPushToken(ctx context.Context, @@ -45,42 +49,90 @@ func (p pushServer) DelUserPushToken(ctx context.Context, return &pbpush.DelUserPushTokenResp{}, nil } -func Start(ctx context.Context, config *Config, client discovery.SvcDiscoveryRegistry, server *grpc.Server) error { - config.runTimeEnv = runtimeenv.PrintRuntimeEnvironment() - - rdb, err := redisutil.NewRedisClient(ctx, config.RedisConfig.Build()) +func Start(ctx context.Context, config *Config, client discovery.Conn, server grpc.ServiceRegistrar) error { + dbb := dbbuild.NewBuilder(&config.MongoConfig, &config.RedisConfig) + rdb, err := dbb.Redis(ctx) if err != nil { return err } - cacheModel := redis.NewThirdCache(rdb) - offlinePusher, err := offlinepush.NewOfflinePusher(&config.RpcConfig, cacheModel, config.FcmConfigPath) + var cacheModel cache.ThirdCache + if rdb == nil { + mdb, err := dbb.Mongo(ctx) + if err != nil { + return err + } + mc, err := mgo.NewCacheMgo(mdb.GetDB()) + if err != nil { + return err + } + cacheModel = mcache.NewThirdCache(mc) + } else { + cacheModel = redis.NewThirdCache(rdb) + } + offlinePusher, err := offlinepush.NewOfflinePusher(&config.RpcConfig, cacheModel, string(config.FcmConfigPath)) if err != nil { return err } + builder := mqbuild.NewBuilder(&config.KafkaConfig) - database := controller.NewPushDatabase(cacheModel, &config.KafkaConfig) + offlinePushProducer, err := builder.GetTopicProducer(ctx, config.KafkaConfig.ToOfflinePushTopic) + if err != nil { + return err + } + database := controller.NewPushDatabase(cacheModel, offlinePushProducer) - consumer, err := NewConsumerHandler(ctx, config, database, offlinePusher, rdb, client) + pushConsumer, err := builder.GetTopicConsumer(ctx, config.KafkaConfig.ToPushTopic) + if err != nil { + return err + } + offlinePushConsumer, err := builder.GetTopicConsumer(ctx, config.KafkaConfig.ToOfflinePushTopic) if err != nil { return err } - offlinePushConsumer, err := NewOfflinePushConsumerHandler(config, offlinePusher) + pushHandler, err := NewConsumerHandler(ctx, config, database, offlinePusher, rdb, client) if err != nil { return err } + offlineHandler := NewOfflinePushConsumerHandler(offlinePusher) + pbpush.RegisterPushMsgServiceServer(server, &pushServer{ database: database, disCov: client, offlinePusher: offlinePusher, - pushCh: consumer, - offlinePushCh: offlinePushConsumer, }) - go consumer.pushConsumerGroup.RegisterHandleAndConsumer(ctx, consumer) + go func() { + pushHandler.WaitCache() + fn := func(ctx context.Context, key string, value []byte) error { + pushHandler.HandleMs2PsChat(ctx, value) + return nil + } + consumerCtx := mcontext.SetOperationID(context.Background(), "push_"+strconv.Itoa(int(rand.Uint32()))) + log.ZInfo(consumerCtx, "begin consume messages") + for { + if err := pushConsumer.Subscribe(consumerCtx, fn); err != nil { + log.ZError(consumerCtx, "subscribe err", err) + return + } + } + }() - go offlinePushConsumer.OfflinePushConsumerGroup.RegisterHandleAndConsumer(ctx, offlinePushConsumer) + go func() { + fn := func(ctx context.Context, key string, value []byte) error { + offlineHandler.HandleMsg2OfflinePush(ctx, value) + return nil + } + consumerCtx := mcontext.SetOperationID(context.Background(), "push_"+strconv.Itoa(int(rand.Uint32()))) + log.ZInfo(consumerCtx, "begin consume messages") + for { + if err := offlinePushConsumer.Subscribe(consumerCtx, fn); err != nil { + log.ZError(consumerCtx, "subscribe err", err) + return + } + } + }() return nil } diff --git a/internal/push/push_handler.go b/internal/push/push_handler.go index 80d14499f..707782c70 100644 --- a/internal/push/push_handler.go +++ b/internal/push/push_handler.go @@ -3,13 +3,8 @@ package push import ( "context" "encoding/json" - "math/rand" - "strconv" "time" - "github.com/openimsdk/open-im-server/v3/pkg/rpcli" - - "github.com/IBM/sarama" "github.com/openimsdk/open-im-server/v3/internal/push/offlinepush" "github.com/openimsdk/open-im-server/v3/internal/push/offlinepush/options" "github.com/openimsdk/open-im-server/v3/pkg/common/prommetrics" @@ -17,6 +12,7 @@ import ( "github.com/openimsdk/open-im-server/v3/pkg/common/webhook" "github.com/openimsdk/open-im-server/v3/pkg/msgprocessor" "github.com/openimsdk/open-im-server/v3/pkg/rpccache" + "github.com/openimsdk/open-im-server/v3/pkg/rpcli" "github.com/openimsdk/open-im-server/v3/pkg/util/conversationutil" "github.com/openimsdk/protocol/constant" "github.com/openimsdk/protocol/msggateway" @@ -25,7 +21,6 @@ import ( "github.com/openimsdk/tools/discovery" "github.com/openimsdk/tools/log" "github.com/openimsdk/tools/mcontext" - "github.com/openimsdk/tools/mq/kafka" "github.com/openimsdk/tools/utils/datautil" "github.com/openimsdk/tools/utils/jsonutil" "github.com/openimsdk/tools/utils/timeutil" @@ -34,7 +29,7 @@ import ( ) type ConsumerHandler struct { - pushConsumerGroup *kafka.MConsumerGroup + //pushConsumerGroup mq.Consumer offlinePusher offlinepush.OfflinePusher onlinePusher OnlinePusher pushDatabase controller.PushDatabase @@ -49,15 +44,7 @@ type ConsumerHandler struct { conversationClient *rpcli.ConversationClient } -func NewConsumerHandler(ctx context.Context, config *Config, database controller.PushDatabase, offlinePusher offlinepush.OfflinePusher, rdb redis.UniversalClient, - client discovery.SvcDiscoveryRegistry) (*ConsumerHandler, error) { - var consumerHandler ConsumerHandler - var err error - consumerHandler.pushConsumerGroup, err = kafka.NewMConsumerGroup(config.KafkaConfig.Build(), config.KafkaConfig.ToPushGroupID, - []string{config.KafkaConfig.ToPushTopic}, true) - if err != nil { - return nil, err - } +func NewConsumerHandler(ctx context.Context, config *Config, database controller.PushDatabase, offlinePusher offlinepush.OfflinePusher, rdb redis.UniversalClient, client discovery.Conn) (*ConsumerHandler, error) { userConn, err := client.GetConn(ctx, config.Discovery.RpcService.User) if err != nil { return nil, err @@ -74,13 +61,18 @@ func NewConsumerHandler(ctx context.Context, config *Config, database controller if err != nil { return nil, err } + onlinePusher, err := NewOnlinePusher(client, config) + if err != nil { + return nil, err + } + var consumerHandler ConsumerHandler consumerHandler.userClient = rpcli.NewUserClient(userConn) consumerHandler.groupClient = rpcli.NewGroupClient(groupConn) consumerHandler.msgClient = rpcli.NewMsgClient(msgConn) consumerHandler.conversationClient = rpcli.NewConversationClient(conversationConn) consumerHandler.offlinePusher = offlinePusher - consumerHandler.onlinePusher = NewOnlinePusher(client, config) + consumerHandler.onlinePusher = onlinePusher consumerHandler.groupLocalCache = rpccache.NewGroupLocalCache(consumerHandler.groupClient, &config.LocalCacheConfig, rdb) consumerHandler.conversationLocalCache = rpccache.NewConversationLocalCache(consumerHandler.conversationClient, &config.LocalCacheConfig, rdb) consumerHandler.webhookClient = webhook.NewWebhookClient(config.WebhooksConfig.URL) @@ -93,7 +85,7 @@ func NewConsumerHandler(ctx context.Context, config *Config, database controller return &consumerHandler, nil } -func (c *ConsumerHandler) handleMs2PsChat(ctx context.Context, msg []byte) { +func (c *ConsumerHandler) HandleMs2PsChat(ctx context.Context, msg []byte) { msgFromMQ := pbpush.PushMsgReq{} if err := proto.Unmarshal(msg, &msgFromMQ); err != nil { log.ZError(ctx, "push Unmarshal msg err", err, "msg", string(msg)) @@ -127,25 +119,12 @@ func (c *ConsumerHandler) handleMs2PsChat(ctx context.Context, msg []byte) { } } -func (*ConsumerHandler) Setup(sarama.ConsumerGroupSession) error { return nil } - -func (*ConsumerHandler) Cleanup(sarama.ConsumerGroupSession) error { return nil } - -func (c *ConsumerHandler) ConsumeClaim(sess sarama.ConsumerGroupSession, claim sarama.ConsumerGroupClaim) error { +func (c *ConsumerHandler) WaitCache() { c.onlineCache.Lock.Lock() for c.onlineCache.CurrentPhase.Load() < rpccache.DoSubscribeOver { c.onlineCache.Cond.Wait() } c.onlineCache.Lock.Unlock() - ctx := mcontext.SetOperationID(context.TODO(), strconv.FormatInt(time.Now().UnixNano()+int64(rand.Uint32()), 10)) - log.ZInfo(ctx, "begin consume messages") - - for msg := range claim.Messages() { - ctx := c.pushConsumerGroup.GetContextFromMsg(msg) - c.handleMs2PsChat(ctx, msg.Value) - sess.MarkMessage(msg, "") - } - return nil } // Push2User Suitable for two types of conversations, one is SingleChatType and the other is NotificationChatType. diff --git a/internal/rpc/auth/auth.go b/internal/rpc/auth/auth.go index 3e096aa64..399d0bd4b 100644 --- a/internal/rpc/auth/auth.go +++ b/internal/rpc/auth/auth.go @@ -18,11 +18,14 @@ import ( "context" "errors" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/mcache" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database/mgo" + "github.com/openimsdk/open-im-server/v3/pkg/dbbuild" "github.com/openimsdk/open-im-server/v3/pkg/rpcli" "github.com/openimsdk/open-im-server/v3/pkg/common/config" redis2 "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/redis" - "github.com/openimsdk/tools/db/redisutil" "github.com/openimsdk/tools/utils/datautil" "github.com/redis/go-redis/v9" @@ -43,7 +46,7 @@ import ( type authServer struct { pbauth.UnimplementedAuthServer authDatabase controller.AuthDatabase - RegisterCenter discovery.SvcDiscoveryRegistry + RegisterCenter discovery.Conn config *Config userClient *rpcli.UserClient } @@ -51,15 +54,31 @@ type authServer struct { type Config struct { RpcConfig config.Auth RedisConfig config.Redis + MongoConfig config.Mongo Share config.Share Discovery config.Discovery } -func Start(ctx context.Context, config *Config, client discovery.SvcDiscoveryRegistry, server *grpc.Server) error { - rdb, err := redisutil.NewRedisClient(ctx, config.RedisConfig.Build()) +func Start(ctx context.Context, config *Config, client discovery.Conn, server grpc.ServiceRegistrar) error { + dbb := dbbuild.NewBuilder(&config.MongoConfig, &config.RedisConfig) + rdb, err := dbb.Redis(ctx) if err != nil { return err } + var token cache.TokenModel + if rdb == nil { + mdb, err := dbb.Mongo(ctx) + if err != nil { + return err + } + mc, err := mgo.NewCacheMgo(mdb.GetDB()) + if err != nil { + return err + } + token = mcache.NewTokenCacheModel(mc, config.RpcConfig.TokenPolicy.Expire) + } else { + token = redis2.NewTokenCacheModel(rdb, config.RpcConfig.TokenPolicy.Expire) + } userConn, err := client.GetConn(ctx, config.Discovery.RpcService.User) if err != nil { return err @@ -67,7 +86,7 @@ func Start(ctx context.Context, config *Config, client discovery.SvcDiscoveryReg pbauth.RegisterAuthServer(server, &authServer{ RegisterCenter: client, authDatabase: controller.NewAuthDatabase( - redis2.NewTokenCacheModel(rdb, config.RpcConfig.TokenPolicy.Expire), + token, config.Share.Secret, config.RpcConfig.TokenPolicy.Expire, config.Share.MultiLogin, @@ -192,7 +211,7 @@ func (s *authServer) forceKickOff(ctx context.Context, userID string, platformID return err } for _, v := range conns { - log.ZDebug(ctx, "forceKickOff", "conn", v.Target()) + log.ZDebug(ctx, "forceKickOff", "userID", userID, "platformID", platformID) client := msggateway.NewMsgGatewayClient(v) kickReq := &msggateway.KickUserOfflineReq{KickUserIDList: []string{userID}, PlatformID: platformID} _, err := client.KickUserOffline(ctx, kickReq) diff --git a/internal/rpc/conversation/conversation.go b/internal/rpc/conversation/conversation.go index 76dd606aa..cd7839234 100644 --- a/internal/rpc/conversation/conversation.go +++ b/internal/rpc/conversation/conversation.go @@ -19,24 +19,22 @@ import ( "sort" "time" + "github.com/openimsdk/open-im-server/v3/pkg/dbbuild" "github.com/openimsdk/open-im-server/v3/pkg/rpcli" "github.com/openimsdk/open-im-server/v3/pkg/common/config" + "github.com/openimsdk/open-im-server/v3/pkg/common/convert" + "github.com/openimsdk/open-im-server/v3/pkg/common/servererrs" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/redis" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/controller" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database/mgo" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" dbModel "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" "github.com/openimsdk/open-im-server/v3/pkg/localcache" "github.com/openimsdk/open-im-server/v3/pkg/msgprocessor" - "github.com/openimsdk/tools/db/redisutil" - - "github.com/openimsdk/open-im-server/v3/pkg/common/convert" - "github.com/openimsdk/open-im-server/v3/pkg/common/servererrs" - "github.com/openimsdk/open-im-server/v3/pkg/common/storage/controller" "github.com/openimsdk/protocol/constant" pbconversation "github.com/openimsdk/protocol/conversation" "github.com/openimsdk/protocol/sdkws" - "github.com/openimsdk/tools/db/mongoutil" "github.com/openimsdk/tools/discovery" "github.com/openimsdk/tools/errs" "github.com/openimsdk/tools/log" @@ -66,12 +64,13 @@ type Config struct { Discovery config.Discovery } -func Start(ctx context.Context, config *Config, client discovery.SvcDiscoveryRegistry, server *grpc.Server) error { - mgocli, err := mongoutil.NewMongoDB(ctx, config.MongodbConfig.Build()) +func Start(ctx context.Context, config *Config, client discovery.Conn, server grpc.ServiceRegistrar) error { + dbb := dbbuild.NewBuilder(&config.MongodbConfig, &config.RedisConfig) + mgocli, err := dbb.Mongo(ctx) if err != nil { return err } - rdb, err := redisutil.NewRedisClient(ctx, config.RedisConfig.Build()) + rdb, err := dbb.Redis(ctx) if err != nil { return err } @@ -96,7 +95,7 @@ func Start(ctx context.Context, config *Config, client discovery.SvcDiscoveryReg pbconversation.RegisterConversationServer(server, &conversationServer{ conversationNotificationSender: NewConversationNotificationSender(&config.NotificationConfig, msgClient), conversationDatabase: controller.NewConversationDatabase(conversationDB, - redis.NewConversationRedis(rdb, &config.LocalCacheConfig, redis.GetRocksCacheOptions(), conversationDB), mgocli.GetTx()), + redis.NewConversationRedis(rdb, &config.LocalCacheConfig, conversationDB), mgocli.GetTx()), userClient: rpcli.NewUserClient(userConn), groupClient: rpcli.NewGroupClient(groupConn), msgClient: msgClient, diff --git a/internal/rpc/group/group.go b/internal/rpc/group/group.go index bea0e1af4..8ad4a949f 100644 --- a/internal/rpc/group/group.go +++ b/internal/rpc/group/group.go @@ -17,13 +17,15 @@ package group import ( "context" "fmt" - "github.com/openimsdk/open-im-server/v3/pkg/rpcli" "math/big" "math/rand" "strconv" "strings" "time" + "github.com/openimsdk/open-im-server/v3/pkg/dbbuild" + "github.com/openimsdk/open-im-server/v3/pkg/rpcli" + "github.com/openimsdk/open-im-server/v3/pkg/authverify" "github.com/openimsdk/open-im-server/v3/pkg/callbackstruct" "github.com/openimsdk/open-im-server/v3/pkg/common/config" @@ -42,8 +44,6 @@ import ( pbgroup "github.com/openimsdk/protocol/group" "github.com/openimsdk/protocol/sdkws" "github.com/openimsdk/protocol/wrapperspb" - "github.com/openimsdk/tools/db/mongoutil" - "github.com/openimsdk/tools/db/redisutil" "github.com/openimsdk/tools/discovery" "github.com/openimsdk/tools/errs" "github.com/openimsdk/tools/log" @@ -76,12 +76,13 @@ type Config struct { Discovery config.Discovery } -func Start(ctx context.Context, config *Config, client discovery.SvcDiscoveryRegistry, server *grpc.Server) error { - mgocli, err := mongoutil.NewMongoDB(ctx, config.MongodbConfig.Build()) +func Start(ctx context.Context, config *Config, client discovery.Conn, server grpc.ServiceRegistrar) error { + dbb := dbbuild.NewBuilder(&config.MongodbConfig, &config.RedisConfig) + mgocli, err := dbb.Mongo(ctx) if err != nil { return err } - rdb, err := redisutil.NewRedisClient(ctx, config.RedisConfig.Build()) + rdb, err := dbb.Redis(ctx) if err != nil { return err } @@ -97,11 +98,6 @@ func Start(ctx context.Context, config *Config, client discovery.SvcDiscoveryReg if err != nil { return err } - - //userRpcClient := rpcclient.NewUserRpcClient(client, config.Share.RpcRegisterName.User, config.Share.IMAdminUserID) - //msgRpcClient := rpcclient.NewMessageRpcClient(client, config.Share.RpcRegisterName.Msg) - //conversationRpcClient := rpcclient.NewConversationRpcClient(client, config.Share.RpcRegisterName.Conversation) - userConn, err := client.GetConn(ctx, config.Discovery.RpcService.User) if err != nil { return err diff --git a/internal/rpc/msg/server.go b/internal/rpc/msg/server.go index 7ccda6bd5..7737f7e7f 100644 --- a/internal/rpc/msg/server.go +++ b/internal/rpc/msg/server.go @@ -16,22 +16,24 @@ package msg import ( "context" + + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/mcache" + "github.com/openimsdk/open-im-server/v3/pkg/dbbuild" + "github.com/openimsdk/open-im-server/v3/pkg/mqbuild" "github.com/openimsdk/open-im-server/v3/pkg/rpcli" "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/redis" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/controller" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database/mgo" "github.com/openimsdk/open-im-server/v3/pkg/common/webhook" - "github.com/openimsdk/protocol/sdkws" - "github.com/openimsdk/tools/db/mongoutil" - "github.com/openimsdk/tools/db/redisutil" - - "github.com/openimsdk/open-im-server/v3/pkg/common/storage/controller" "github.com/openimsdk/open-im-server/v3/pkg/notification" "github.com/openimsdk/open-im-server/v3/pkg/rpccache" "github.com/openimsdk/protocol/constant" "github.com/openimsdk/protocol/conversation" "github.com/openimsdk/protocol/msg" + "github.com/openimsdk/protocol/sdkws" "github.com/openimsdk/tools/discovery" "google.golang.org/grpc" ) @@ -56,8 +58,8 @@ type Config struct { // MsgServer encapsulates dependencies required for message handling. type msgServer struct { msg.UnimplementedMsgServer - RegisterCenter discovery.SvcDiscoveryRegistry // Service discovery registry for service registration. - MsgDatabase controller.CommonMsgDatabase // Interface for message database operations. + RegisterCenter discovery.Conn // Service discovery registry for service registration. + MsgDatabase controller.CommonMsgDatabase // Interface for message database operations. StreamMsgDatabase controller.StreamMsgDatabase UserLocalCache *rpccache.UserLocalCache // Local cache for user data. FriendLocalCache *rpccache.FriendLocalCache // Local cache for friend data. @@ -76,12 +78,18 @@ func (m *msgServer) addInterceptorHandler(interceptorFunc ...MessageInterceptorF } -func Start(ctx context.Context, config *Config, client discovery.SvcDiscoveryRegistry, server *grpc.Server) error { - mgocli, err := mongoutil.NewMongoDB(ctx, config.MongodbConfig.Build()) +func Start(ctx context.Context, config *Config, client discovery.Conn, server grpc.ServiceRegistrar) error { + builder := mqbuild.NewBuilder(&config.KafkaConfig) + redisProducer, err := builder.GetTopicProducer(ctx, config.KafkaConfig.ToRedisTopic) + if err != nil { + return err + } + dbb := dbbuild.NewBuilder(&config.MongodbConfig, &config.RedisConfig) + mgocli, err := dbb.Mongo(ctx) if err != nil { return err } - rdb, err := redisutil.NewRedisClient(ctx, config.RedisConfig.Build()) + rdb, err := dbb.Redis(ctx) if err != nil { return err } @@ -89,7 +97,16 @@ func Start(ctx context.Context, config *Config, client discovery.SvcDiscoveryReg if err != nil { return err } - msgModel := redis.NewMsgCache(rdb, msgDocModel) + var msgModel cache.MsgCache + if rdb == nil { + cm, err := mgo.NewCacheMgo(mgocli.GetDB()) + if err != nil { + return err + } + msgModel = mcache.NewMsgCache(cm, msgDocModel) + } else { + msgModel = redis.NewMsgCache(rdb, msgDocModel) + } seqConversation, err := mgo.NewSeqConversationMongo(mgocli.GetDB()) if err != nil { return err @@ -104,10 +121,6 @@ func Start(ctx context.Context, config *Config, client discovery.SvcDiscoveryReg return err } seqUserCache := redis.NewSeqUserCacheRedis(rdb, seqUser) - msgDatabase, err := controller.NewCommonMsgDatabase(msgDocModel, msgModel, seqUserCache, seqConversationCache, &config.KafkaConfig) - if err != nil { - return err - } userConn, err := client.GetConn(ctx, config.Discovery.RpcService.User) if err != nil { return err @@ -125,6 +138,7 @@ func Start(ctx context.Context, config *Config, client discovery.SvcDiscoveryReg return err } conversationClient := rpcli.NewConversationClient(conversationConn) + msgDatabase := controller.NewCommonMsgDatabase(msgDocModel, msgModel, seqUserCache, seqConversationCache, redisProducer) s := &msgServer{ MsgDatabase: msgDatabase, StreamMsgDatabase: controller.NewStreamMsgDatabase(streamMsg), diff --git a/internal/rpc/relation/friend.go b/internal/rpc/relation/friend.go index 79db14970..a4908d924 100644 --- a/internal/rpc/relation/friend.go +++ b/internal/rpc/relation/friend.go @@ -16,26 +16,25 @@ package relation import ( "context" + + "github.com/openimsdk/open-im-server/v3/pkg/dbbuild" "github.com/openimsdk/open-im-server/v3/pkg/rpcli" "github.com/openimsdk/tools/mq/memamq" + "github.com/openimsdk/open-im-server/v3/pkg/authverify" "github.com/openimsdk/open-im-server/v3/pkg/common/config" + "github.com/openimsdk/open-im-server/v3/pkg/common/convert" + "github.com/openimsdk/open-im-server/v3/pkg/common/servererrs" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/redis" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/controller" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database/mgo" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" "github.com/openimsdk/open-im-server/v3/pkg/common/webhook" "github.com/openimsdk/open-im-server/v3/pkg/localcache" - "github.com/openimsdk/tools/db/redisutil" - - "github.com/openimsdk/open-im-server/v3/pkg/authverify" - "github.com/openimsdk/open-im-server/v3/pkg/common/convert" - "github.com/openimsdk/open-im-server/v3/pkg/common/servererrs" - "github.com/openimsdk/open-im-server/v3/pkg/common/storage/controller" "github.com/openimsdk/protocol/constant" "github.com/openimsdk/protocol/relation" "github.com/openimsdk/protocol/sdkws" - "github.com/openimsdk/tools/db/mongoutil" "github.com/openimsdk/tools/discovery" "github.com/openimsdk/tools/errs" "github.com/openimsdk/tools/utils/datautil" @@ -47,7 +46,7 @@ type friendServer struct { db controller.FriendDatabase blackDatabase controller.BlackDatabase notificationSender *FriendNotificationSender - RegisterCenter discovery.SvcDiscoveryRegistry + RegisterCenter discovery.Conn config *Config webhookClient *webhook.Client queue *memamq.MemoryQueue @@ -66,12 +65,13 @@ type Config struct { Discovery config.Discovery } -func Start(ctx context.Context, config *Config, client discovery.SvcDiscoveryRegistry, server *grpc.Server) error { - mgocli, err := mongoutil.NewMongoDB(ctx, config.MongodbConfig.Build()) +func Start(ctx context.Context, config *Config, client discovery.Conn, server grpc.ServiceRegistrar) error { + dbb := dbbuild.NewBuilder(&config.MongodbConfig, &config.RedisConfig) + mgocli, err := dbb.Mongo(ctx) if err != nil { return err } - rdb, err := redisutil.NewRedisClient(ctx, config.RedisConfig.Build()) + rdb, err := dbb.Redis(ctx) if err != nil { return err } @@ -114,12 +114,12 @@ func Start(ctx context.Context, config *Config, client discovery.SvcDiscoveryReg db: controller.NewFriendDatabase( friendMongoDB, friendRequestMongoDB, - redis.NewFriendCacheRedis(rdb, &config.LocalCacheConfig, friendMongoDB, redis.GetRocksCacheOptions()), + redis.NewFriendCacheRedis(rdb, &config.LocalCacheConfig, friendMongoDB), mgocli.GetTx(), ), blackDatabase: controller.NewBlackDatabase( blackMongoDB, - redis.NewBlackCacheRedis(rdb, &config.LocalCacheConfig, blackMongoDB, redis.GetRocksCacheOptions()), + redis.NewBlackCacheRedis(rdb, &config.LocalCacheConfig, blackMongoDB), ), notificationSender: notificationSender, RegisterCenter: client, diff --git a/internal/rpc/third/s3.go b/internal/rpc/third/s3.go index 8796fe824..97206dd6d 100644 --- a/internal/rpc/third/s3.go +++ b/internal/rpc/third/s3.go @@ -19,11 +19,12 @@ import ( "encoding/base64" "encoding/hex" "encoding/json" - "github.com/openimsdk/open-im-server/v3/pkg/authverify" "path" "strconv" "time" + "github.com/openimsdk/open-im-server/v3/pkg/authverify" + "github.com/google/uuid" "github.com/openimsdk/open-im-server/v3/pkg/common/servererrs" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" @@ -37,7 +38,10 @@ import ( ) func (t *thirdServer) PartLimit(ctx context.Context, req *third.PartLimitReq) (*third.PartLimitResp, error) { - limit := t.s3dataBase.PartLimit() + limit, err := t.s3dataBase.PartLimit() + if err != nil { + return nil, err + } return &third.PartLimitResp{ MinPartSize: limit.MinPartSize, MaxPartSize: limit.MaxPartSize, diff --git a/internal/rpc/third/third.go b/internal/rpc/third/third.go index 0b8ca25a8..0afd54014 100644 --- a/internal/rpc/third/third.go +++ b/internal/rpc/third/third.go @@ -17,9 +17,14 @@ package third import ( "context" "fmt" - "github.com/openimsdk/open-im-server/v3/pkg/rpcli" "time" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/mcache" + "github.com/openimsdk/open-im-server/v3/pkg/dbbuild" + "github.com/openimsdk/open-im-server/v3/pkg/rpcli" + "github.com/openimsdk/tools/s3/disable" + "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/redis" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database/mgo" @@ -29,8 +34,6 @@ import ( "github.com/openimsdk/open-im-server/v3/pkg/common/storage/controller" "github.com/openimsdk/protocol/third" - "github.com/openimsdk/tools/db/mongoutil" - "github.com/openimsdk/tools/db/redisutil" "github.com/openimsdk/tools/discovery" "github.com/openimsdk/tools/s3" "github.com/openimsdk/tools/s3/cos" @@ -60,15 +63,17 @@ type Config struct { Discovery config.Discovery } -func Start(ctx context.Context, config *Config, client discovery.SvcDiscoveryRegistry, server *grpc.Server) error { - mgocli, err := mongoutil.NewMongoDB(ctx, config.MongodbConfig.Build()) +func Start(ctx context.Context, config *Config, client discovery.Conn, server grpc.ServiceRegistrar) error { + dbb := dbbuild.NewBuilder(&config.MongodbConfig, &config.RedisConfig) + mgocli, err := dbb.Mongo(ctx) if err != nil { return err } - rdb, err := redisutil.NewRedisClient(ctx, config.RedisConfig.Build()) + rdb, err := dbb.Redis(ctx) if err != nil { return err } + logdb, err := mgo.NewLogMongo(mgocli.GetDB()) if err != nil { return err @@ -77,15 +82,31 @@ func Start(ctx context.Context, config *Config, client discovery.SvcDiscoveryReg if err != nil { return err } - + var thirdCache cache.ThirdCache + if rdb == nil { + tc, err := mgo.NewCacheMgo(mgocli.GetDB()) + if err != nil { + return err + } + thirdCache = mcache.NewThirdCache(tc) + } else { + thirdCache = redis.NewThirdCache(rdb) + } // Select the oss method according to the profile policy - enable := config.RpcConfig.Object.Enable - var ( - o s3.Interface - ) - switch enable { + var o s3.Interface + switch enable := config.RpcConfig.Object.Enable; enable { case "minio": - o, err = minio.NewMinio(ctx, redis.NewMinioCache(rdb), *config.MinioConfig.Build()) + var minioCache minio.Cache + if rdb == nil { + mc, err := mgo.NewCacheMgo(mgocli.GetDB()) + if err != nil { + return err + } + minioCache = mcache.NewMinioCache(mc) + } else { + minioCache = redis.NewMinioCache(rdb) + } + o, err = minio.NewMinio(ctx, minioCache, *config.MinioConfig.Build()) case "cos": o, err = cos.NewCos(*config.RpcConfig.Object.Cos.Build()) case "oss": @@ -94,6 +115,8 @@ func Start(ctx context.Context, config *Config, client discovery.SvcDiscoveryReg o, err = kodo.NewKodo(*config.RpcConfig.Object.Kodo.Build()) case "aws": o, err = aws.NewAws(*config.RpcConfig.Object.Aws.Build()) + case "": + o = disable.NewDisable() default: err = fmt.Errorf("invalid object enable: %s", enable) } @@ -106,7 +129,7 @@ func Start(ctx context.Context, config *Config, client discovery.SvcDiscoveryReg } localcache.InitLocalCache(&config.LocalCacheConfig) third.RegisterThirdServer(server, &thirdServer{ - thirdDatabase: controller.NewThirdDatabase(redis.NewThirdCache(rdb), logdb), + thirdDatabase: controller.NewThirdDatabase(thirdCache, logdb), s3dataBase: controller.NewS3Database(rdb, o, s3db), defaultExpire: time.Hour * 24 * 7, config: config, diff --git a/internal/rpc/user/user.go b/internal/rpc/user/user.go index 15e93a988..bde9a3f8f 100644 --- a/internal/rpc/user/user.go +++ b/internal/rpc/user/user.go @@ -23,29 +23,27 @@ import ( "time" "github.com/openimsdk/open-im-server/v3/internal/rpc/relation" + "github.com/openimsdk/open-im-server/v3/pkg/authverify" "github.com/openimsdk/open-im-server/v3/pkg/common/config" + "github.com/openimsdk/open-im-server/v3/pkg/common/convert" "github.com/openimsdk/open-im-server/v3/pkg/common/prommetrics" + "github.com/openimsdk/open-im-server/v3/pkg/common/servererrs" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/redis" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/controller" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database/mgo" tablerelation "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" "github.com/openimsdk/open-im-server/v3/pkg/common/webhook" + "github.com/openimsdk/open-im-server/v3/pkg/dbbuild" "github.com/openimsdk/open-im-server/v3/pkg/localcache" "github.com/openimsdk/open-im-server/v3/pkg/rpcli" + "github.com/openimsdk/protocol/constant" "github.com/openimsdk/protocol/group" friendpb "github.com/openimsdk/protocol/relation" - "github.com/openimsdk/tools/db/redisutil" - - "github.com/openimsdk/open-im-server/v3/pkg/authverify" - "github.com/openimsdk/open-im-server/v3/pkg/common/convert" - "github.com/openimsdk/open-im-server/v3/pkg/common/servererrs" - "github.com/openimsdk/open-im-server/v3/pkg/common/storage/controller" - "github.com/openimsdk/protocol/constant" "github.com/openimsdk/protocol/sdkws" pbuser "github.com/openimsdk/protocol/user" - "github.com/openimsdk/tools/db/mongoutil" "github.com/openimsdk/tools/db/pagination" - registry "github.com/openimsdk/tools/discovery" + "github.com/openimsdk/tools/discovery" "github.com/openimsdk/tools/errs" "github.com/openimsdk/tools/utils/datautil" "google.golang.org/grpc" @@ -57,7 +55,7 @@ type userServer struct { db controller.UserDatabase friendNotificationSender *relation.FriendNotificationSender userNotificationSender *UserNotificationSender - RegisterCenter registry.SvcDiscoveryRegistry + RegisterCenter discovery.Conn config *Config webhookClient *webhook.Client groupClient *rpcli.GroupClient @@ -76,15 +74,17 @@ type Config struct { Discovery config.Discovery } -func Start(ctx context.Context, config *Config, client registry.SvcDiscoveryRegistry, server *grpc.Server) error { - mgocli, err := mongoutil.NewMongoDB(ctx, config.MongodbConfig.Build()) +func Start(ctx context.Context, config *Config, client discovery.Conn, server grpc.ServiceRegistrar) error { + dbb := dbbuild.NewBuilder(&config.MongodbConfig, &config.RedisConfig) + mgocli, err := dbb.Mongo(ctx) if err != nil { return err } - rdb, err := redisutil.NewRedisClient(ctx, config.RedisConfig.Build()) + rdb, err := dbb.Redis(ctx) if err != nil { return err } + users := make([]*tablerelation.User, 0) for _, v := range config.Share.IMAdminUserID { diff --git a/internal/tools/cron_task.go b/internal/tools/cron/cron_task.go similarity index 80% rename from internal/tools/cron_task.go rename to internal/tools/cron/cron_task.go index da1c6320e..7ae314193 100644 --- a/internal/tools/cron_task.go +++ b/internal/tools/cron/cron_task.go @@ -1,47 +1,37 @@ -package tools +package cron import ( "context" "github.com/openimsdk/open-im-server/v3/pkg/common/config" - kdisc "github.com/openimsdk/open-im-server/v3/pkg/common/discovery" disetcd "github.com/openimsdk/open-im-server/v3/pkg/common/discovery/etcd" pbconversation "github.com/openimsdk/protocol/conversation" "github.com/openimsdk/protocol/msg" "github.com/openimsdk/protocol/third" + "github.com/openimsdk/tools/discovery" "github.com/openimsdk/tools/discovery/etcd" - "github.com/openimsdk/tools/mcontext" - "github.com/openimsdk/tools/mw" - "github.com/openimsdk/tools/utils/runtimeenv" - "google.golang.org/grpc" - "google.golang.org/grpc/credentials/insecure" - "github.com/openimsdk/tools/errs" "github.com/openimsdk/tools/log" + "github.com/openimsdk/tools/mcontext" + "github.com/openimsdk/tools/utils/runtimeenv" "github.com/robfig/cron/v3" + "google.golang.org/grpc" ) -type CronTaskConfig struct { +type Config struct { CronTask config.CronTask Share config.Share Discovery config.Discovery - - runTimeEnv string } -func Start(ctx context.Context, conf *CronTaskConfig) error { - conf.runTimeEnv = runtimeenv.PrintRuntimeEnvironment() - - log.CInfo(ctx, "CRON-TASK server is initializing", "runTimeEnv", conf.runTimeEnv, "chatRecordsClearTime", conf.CronTask.CronExecuteTime, "msgDestructTime", conf.CronTask.RetainChatRecords) +func Start(ctx context.Context, conf *Config, client discovery.Conn, service grpc.ServiceRegistrar) error { + log.CInfo(ctx, "CRON-TASK server is initializing", "runTimeEnv", runtimeenv.RuntimeEnvironment(), "chatRecordsClearTime", conf.CronTask.CronExecuteTime, "msgDestructTime", conf.CronTask.RetainChatRecords) if conf.CronTask.RetainChatRecords < 1 { - return errs.New("msg destruct time must be greater than 1").Wrap() - } - client, err := kdisc.NewDiscoveryRegister(&conf.Discovery, conf.runTimeEnv, nil) - if err != nil { - return errs.WrapMsg(err, "failed to register discovery service") + log.ZInfo(ctx, "disable cron") + <-ctx.Done() + return nil } - client.AddOption(mw.GrpcClient(), grpc.WithTransportCredentials(insecure.NewCredentials())) ctx = mcontext.SetOpUserID(ctx, conf.Share.IMAdminUserID[0]) msgConn, err := client.GetConn(ctx, conf.Discovery.RpcService.Msg) @@ -88,13 +78,15 @@ func Start(ctx context.Context, conf *CronTaskConfig) error { } log.ZDebug(ctx, "start cron task", "CronExecuteTime", conf.CronTask.CronExecuteTime) srv.cron.Start() + log.ZDebug(ctx, "cron task server is running") <-ctx.Done() + log.ZDebug(ctx, "cron task server is shutting down") return nil } type cronServer struct { ctx context.Context - config *CronTaskConfig + config *Config cron *cron.Cron msgClient msg.MsgClient conversationClient pbconversation.ConversationClient diff --git a/internal/tools/cron_test.go b/internal/tools/cron/cron_test.go similarity index 95% rename from internal/tools/cron_test.go rename to internal/tools/cron/cron_test.go index b4082a5a5..b98b14f14 100644 --- a/internal/tools/cron_test.go +++ b/internal/tools/cron/cron_test.go @@ -1,4 +1,4 @@ -package tools +package cron import ( "context" @@ -24,7 +24,7 @@ func TestName(t *testing.T) { Address: []string{"localhost:12379"}, }, } - client, err := kdisc.NewDiscoveryRegister(conf, "source") + client, err := kdisc.NewDiscoveryRegister(conf, nil) if err != nil { panic(err) } diff --git a/internal/tools/msg.go b/internal/tools/cron/msg.go similarity index 98% rename from internal/tools/msg.go rename to internal/tools/cron/msg.go index cc00cc5b8..c75189047 100644 --- a/internal/tools/msg.go +++ b/internal/tools/cron/msg.go @@ -1,12 +1,13 @@ -package tools +package cron import ( "fmt" + "os" + "time" + "github.com/openimsdk/protocol/msg" "github.com/openimsdk/tools/log" "github.com/openimsdk/tools/mcontext" - "os" - "time" ) func (c *cronServer) deleteMsg() { diff --git a/internal/tools/s3.go b/internal/tools/cron/s3.go similarity index 99% rename from internal/tools/s3.go rename to internal/tools/cron/s3.go index 9b6b9c408..c42e223b8 100644 --- a/internal/tools/s3.go +++ b/internal/tools/cron/s3.go @@ -1,12 +1,13 @@ -package tools +package cron import ( "fmt" + "os" + "time" + "github.com/openimsdk/protocol/third" "github.com/openimsdk/tools/log" "github.com/openimsdk/tools/mcontext" - "os" - "time" ) func (c *cronServer) clearS3() { diff --git a/internal/tools/user_msg.go b/internal/tools/cron/user_msg.go similarity index 98% rename from internal/tools/user_msg.go rename to internal/tools/cron/user_msg.go index a4afa769e..0c00640a1 100644 --- a/internal/tools/user_msg.go +++ b/internal/tools/cron/user_msg.go @@ -1,12 +1,13 @@ -package tools +package cron import ( "fmt" + "os" + "time" + pbconversation "github.com/openimsdk/protocol/conversation" "github.com/openimsdk/tools/log" "github.com/openimsdk/tools/mcontext" - "os" - "time" ) func (c *cronServer) clearUserMsg() { diff --git a/pkg/common/cmd/api.go b/pkg/common/cmd/api.go index 050b313ff..484467798 100644 --- a/pkg/common/cmd/api.go +++ b/pkg/common/cmd/api.go @@ -19,6 +19,7 @@ import ( "github.com/openimsdk/open-im-server/v3/internal/api" "github.com/openimsdk/open-im-server/v3/pkg/common/config" + "github.com/openimsdk/open-im-server/v3/pkg/common/startrpc" "github.com/openimsdk/open-im-server/v3/version" "github.com/openimsdk/tools/system/program" "github.com/spf13/cobra" @@ -32,7 +33,7 @@ type ApiCmd struct { } func NewApiCmd() *ApiCmd { - apiConfig := api.Config{AllConfig: &config.AllConfig{}} + var apiConfig api.Config ret := &ApiCmd{apiConfig: &apiConfig} ret.configMap = map[string]any{ config.DiscoveryConfigFilename: &apiConfig.Discovery, @@ -61,7 +62,7 @@ func NewApiCmd() *ApiCmd { ret.RootCmd = NewRootCmd(program.GetProcessName(), WithConfigMap(ret.configMap)) ret.ctx = context.WithValue(context.Background(), "version", version.Version) ret.Command.RunE = func(cmd *cobra.Command, args []string) error { - apiConfig.ConfigPath = ret.configPath + apiConfig.ConfigPath = config.Path(ret.configPath) return ret.runE() } return ret @@ -72,5 +73,22 @@ func (a *ApiCmd) Exec() error { } func (a *ApiCmd) runE() error { - return api.Start(a.ctx, a.Index(), a.apiConfig) + a.apiConfig.Index = config.Index(a.Index()) + prometheus := config.Prometheus{ + Enable: a.apiConfig.API.Prometheus.Enable, + Ports: a.apiConfig.API.Prometheus.Ports, + } + return startrpc.Start( + a.ctx, &a.apiConfig.Discovery, + &prometheus, + a.apiConfig.API.Api.ListenIP, "", + a.apiConfig.API.Prometheus.AutoSetPorts, + nil, int(a.apiConfig.Index), + a.apiConfig.Discovery.RpcService.MessageGateway, + &a.apiConfig.Notification, + a.apiConfig, + []string{}, + []string{}, + api.Start, + ) } diff --git a/pkg/common/cmd/auth.go b/pkg/common/cmd/auth.go index b09d4153f..6e59b6dc6 100644 --- a/pkg/common/cmd/auth.go +++ b/pkg/common/cmd/auth.go @@ -38,6 +38,7 @@ func NewAuthRpcCmd() *AuthRpcCmd { ret.configMap = map[string]any{ config.OpenIMRPCAuthCfgFileName: &authConfig.RpcConfig, config.RedisConfigFileName: &authConfig.RedisConfig, + config.MongodbConfigFileName: &authConfig.MongoConfig, config.ShareFileName: &authConfig.Share, config.DiscoveryConfigFilename: &authConfig.Discovery, } diff --git a/pkg/common/cmd/cron_task.go b/pkg/common/cmd/cron_task.go index e7eb0ce18..af2cbbee9 100644 --- a/pkg/common/cmd/cron_task.go +++ b/pkg/common/cmd/cron_task.go @@ -17,8 +17,9 @@ package cmd import ( "context" - "github.com/openimsdk/open-im-server/v3/internal/tools" + "github.com/openimsdk/open-im-server/v3/internal/tools/cron" "github.com/openimsdk/open-im-server/v3/pkg/common/config" + "github.com/openimsdk/open-im-server/v3/pkg/common/startrpc" "github.com/openimsdk/open-im-server/v3/version" "github.com/openimsdk/tools/system/program" "github.com/spf13/cobra" @@ -28,11 +29,11 @@ type CronTaskCmd struct { *RootCmd ctx context.Context configMap map[string]any - cronTaskConfig *tools.CronTaskConfig + cronTaskConfig *cron.Config } func NewCronTaskCmd() *CronTaskCmd { - var cronTaskConfig tools.CronTaskConfig + var cronTaskConfig cron.Config ret := &CronTaskCmd{cronTaskConfig: &cronTaskConfig} ret.configMap = map[string]any{ config.OpenIMCronTaskCfgFileName: &cronTaskConfig.CronTask, @@ -52,5 +53,18 @@ func (a *CronTaskCmd) Exec() error { } func (a *CronTaskCmd) runE() error { - return tools.Start(a.ctx, a.cronTaskConfig) + var prometheus config.Prometheus + return startrpc.Start( + a.ctx, &a.cronTaskConfig.Discovery, + &prometheus, + "", "", + true, + nil, 0, + "", + nil, + a.cronTaskConfig, + []string{}, + []string{}, + cron.Start, + ) } diff --git a/pkg/common/cmd/msg_gateway.go b/pkg/common/cmd/msg_gateway.go index 3f66ea720..83cfb6272 100644 --- a/pkg/common/cmd/msg_gateway.go +++ b/pkg/common/cmd/msg_gateway.go @@ -19,6 +19,7 @@ import ( "github.com/openimsdk/open-im-server/v3/internal/msggateway" "github.com/openimsdk/open-im-server/v3/pkg/common/config" + "github.com/openimsdk/open-im-server/v3/pkg/common/startrpc" "github.com/openimsdk/open-im-server/v3/version" "github.com/openimsdk/tools/system/program" @@ -55,5 +56,20 @@ func (m *MsgGatewayCmd) Exec() error { } func (m *MsgGatewayCmd) runE() error { - return msggateway.Start(m.ctx, m.Index(), m.msgGatewayConfig) + m.msgGatewayConfig.Index = config.Index(m.Index()) + rpc := m.msgGatewayConfig.MsgGateway.RPC + var prometheus config.Prometheus + return startrpc.Start( + m.ctx, &m.msgGatewayConfig.Discovery, + &prometheus, + rpc.ListenIP, rpc.RegisterIP, + rpc.AutoSetPorts, + rpc.Ports, int(m.msgGatewayConfig.Index), + m.msgGatewayConfig.Discovery.RpcService.MessageGateway, + nil, + m.msgGatewayConfig, + []string{}, + []string{}, + msggateway.Start, + ) } diff --git a/pkg/common/cmd/msg_transfer.go b/pkg/common/cmd/msg_transfer.go index fbb83c65f..9411a2cd0 100644 --- a/pkg/common/cmd/msg_transfer.go +++ b/pkg/common/cmd/msg_transfer.go @@ -19,6 +19,7 @@ import ( "github.com/openimsdk/open-im-server/v3/internal/msgtransfer" "github.com/openimsdk/open-im-server/v3/pkg/common/config" + "github.com/openimsdk/open-im-server/v3/pkg/common/startrpc" "github.com/openimsdk/open-im-server/v3/version" "github.com/openimsdk/tools/system/program" "github.com/spf13/cobra" @@ -56,5 +57,19 @@ func (m *MsgTransferCmd) Exec() error { } func (m *MsgTransferCmd) runE() error { - return msgtransfer.Start(m.ctx, m.Index(), m.msgTransferConfig) + m.msgTransferConfig.Index = config.Index(m.Index()) + var prometheus config.Prometheus + return startrpc.Start( + m.ctx, &m.msgTransferConfig.Discovery, + &prometheus, + "", "", + true, + nil, int(m.msgTransferConfig.Index), + "", + nil, + m.msgTransferConfig, + []string{}, + []string{}, + msgtransfer.Start, + ) } diff --git a/pkg/common/cmd/push.go b/pkg/common/cmd/push.go index c4ae84952..7cd3c481e 100644 --- a/pkg/common/cmd/push.go +++ b/pkg/common/cmd/push.go @@ -38,6 +38,7 @@ func NewPushRpcCmd() *PushRpcCmd { ret.configMap = map[string]any{ config.OpenIMPushCfgFileName: &pushConfig.RpcConfig, config.RedisConfigFileName: &pushConfig.RedisConfig, + config.MongodbConfigFileName: &pushConfig.MongoConfig, config.KafkaConfigFileName: &pushConfig.KafkaConfig, config.ShareFileName: &pushConfig.Share, config.NotificationFileName: &pushConfig.NotificationConfig, @@ -48,7 +49,7 @@ func NewPushRpcCmd() *PushRpcCmd { ret.RootCmd = NewRootCmd(program.GetProcessName(), WithConfigMap(ret.configMap)) ret.ctx = context.WithValue(context.Background(), "version", version.Version) ret.Command.RunE = func(cmd *cobra.Command, args []string) error { - ret.pushConfig.FcmConfigPath = ret.ConfigPath() + ret.pushConfig.FcmConfigPath = config.Path(ret.ConfigPath()) return ret.runE() } return ret diff --git a/pkg/common/cmd/root.go b/pkg/common/cmd/root.go index 0a405fb6e..35b1c7dfb 100644 --- a/pkg/common/cmd/root.go +++ b/pkg/common/cmd/root.go @@ -12,7 +12,6 @@ import ( "github.com/openimsdk/tools/discovery/etcd" "github.com/openimsdk/tools/errs" "github.com/openimsdk/tools/log" - "github.com/openimsdk/tools/utils/runtimeenv" "github.com/spf13/cobra" clientv3 "go.etcd.io/etcd/client/v3" ) @@ -86,14 +85,12 @@ func (r *RootCmd) initEtcd() error { return err } disConfig := config.Discovery{} - env := runtimeenv.PrintRuntimeEnvironment() - err = config.Load(configDirectory, config.DiscoveryConfigFilename, config.EnvPrefixMap[config.DiscoveryConfigFilename], - env, &disConfig) + err = config.Load(configDirectory, config.DiscoveryConfigFilename, config.EnvPrefixMap[config.DiscoveryConfigFilename], &disConfig) if err != nil { return err } if disConfig.Enable == config.ETCD { - discov, _ := kdisc.NewDiscoveryRegister(&disConfig, env, nil) + discov, _ := kdisc.NewDiscoveryRegister(&disConfig, nil) r.etcdClient = discov.(*etcd.SvcDiscoveryRegistryImpl).GetClient() } return nil @@ -125,18 +122,16 @@ func (r *RootCmd) initializeConfiguration(cmd *cobra.Command, opts *CmdOpts) err return err } - runtimeEnv := runtimeenv.PrintRuntimeEnvironment() - // Load common configuration file //opts.configMap[ShareFileName] = StructEnvPrefix{EnvPrefix: shareEnvPrefix, ConfigStruct: &r.share} for configFileName, configStruct := range opts.configMap { - err := config.Load(configDirectory, configFileName, config.EnvPrefixMap[configFileName], runtimeEnv, configStruct) + err := config.Load(configDirectory, configFileName, config.EnvPrefixMap[configFileName], configStruct) if err != nil { return err } } // Load common log configuration file - return config.Load(configDirectory, config.LogConfigFileName, config.EnvPrefixMap[config.LogConfigFileName], runtimeEnv, &r.log) + return config.Load(configDirectory, config.LogConfigFileName, config.EnvPrefixMap[config.LogConfigFileName], &r.log) } func (r *RootCmd) updateConfigFromEtcd(opts *CmdOpts) error { @@ -208,7 +203,6 @@ func (r *RootCmd) applyOptions(opts ...func(*CmdOpts)) *CmdOpts { func (r *RootCmd) initializeLogger(cmdOpts *CmdOpts) error { err := log.InitLoggerFromConfig( - cmdOpts.loggerPrefixName, r.processName, "", "", diff --git a/pkg/common/config/config.go b/pkg/common/config/config.go index 1b9121b7a..c7a7b2d95 100644 --- a/pkg/common/config/config.go +++ b/pkg/common/config/config.go @@ -28,378 +28,348 @@ import ( "github.com/openimsdk/tools/s3/oss" ) +const StructTagName = "yaml" + +type Path string + +type Index int + type CacheConfig struct { - Topic string `mapstructure:"topic"` - SlotNum int `mapstructure:"slotNum"` - SlotSize int `mapstructure:"slotSize"` - SuccessExpire int `mapstructure:"successExpire"` - FailedExpire int `mapstructure:"failedExpire"` + Topic string `yaml:"topic"` + SlotNum int `yaml:"slotNum"` + SlotSize int `yaml:"slotSize"` + SuccessExpire int `yaml:"successExpire"` + FailedExpire int `yaml:"failedExpire"` } type LocalCache struct { - User CacheConfig `mapstructure:"user"` - Group CacheConfig `mapstructure:"group"` - Friend CacheConfig `mapstructure:"friend"` - Conversation CacheConfig `mapstructure:"conversation"` + User CacheConfig `yaml:"user"` + Group CacheConfig `yaml:"group"` + Friend CacheConfig `yaml:"friend"` + Conversation CacheConfig `yaml:"conversation"` } type Log struct { - StorageLocation string `mapstructure:"storageLocation"` - RotationTime uint `mapstructure:"rotationTime"` - RemainRotationCount uint `mapstructure:"remainRotationCount"` - RemainLogLevel int `mapstructure:"remainLogLevel"` - IsStdout bool `mapstructure:"isStdout"` - IsJson bool `mapstructure:"isJson"` - IsSimplify bool `mapstructure:"isSimplify"` - WithStack bool `mapstructure:"withStack"` + StorageLocation string `yaml:"storageLocation"` + RotationTime uint `yaml:"rotationTime"` + RemainRotationCount uint `yaml:"remainRotationCount"` + RemainLogLevel int `yaml:"remainLogLevel"` + IsStdout bool `yaml:"isStdout"` + IsJson bool `yaml:"isJson"` + IsSimplify bool `yaml:"isSimplify"` + WithStack bool `yaml:"withStack"` } type Minio struct { - Bucket string `mapstructure:"bucket"` - AccessKeyID string `mapstructure:"accessKeyID"` - SecretAccessKey string `mapstructure:"secretAccessKey"` - SessionToken string `mapstructure:"sessionToken"` - InternalAddress string `mapstructure:"internalAddress"` - ExternalAddress string `mapstructure:"externalAddress"` - PublicRead bool `mapstructure:"publicRead"` + Bucket string `yaml:"bucket"` + AccessKeyID string `yaml:"accessKeyID"` + SecretAccessKey string `yaml:"secretAccessKey"` + SessionToken string `yaml:"sessionToken"` + InternalAddress string `yaml:"internalAddress"` + ExternalAddress string `yaml:"externalAddress"` + PublicRead bool `yaml:"publicRead"` } type Mongo struct { - URI string `mapstructure:"uri"` - Address []string `mapstructure:"address"` - Database string `mapstructure:"database"` - Username string `mapstructure:"username"` - Password string `mapstructure:"password"` - AuthSource string `mapstructure:"authSource"` - MaxPoolSize int `mapstructure:"maxPoolSize"` - MaxRetry int `mapstructure:"maxRetry"` + URI string `yaml:"uri"` + Address []string `yaml:"address"` + Database string `yaml:"database"` + Username string `yaml:"username"` + Password string `yaml:"password"` + AuthSource string `yaml:"authSource"` + MaxPoolSize int `yaml:"maxPoolSize"` + MaxRetry int `yaml:"maxRetry"` } type Kafka struct { - Username string `mapstructure:"username"` - Password string `mapstructure:"password"` - ProducerAck string `mapstructure:"producerAck"` - CompressType string `mapstructure:"compressType"` - Address []string `mapstructure:"address"` - ToRedisTopic string `mapstructure:"toRedisTopic"` - ToMongoTopic string `mapstructure:"toMongoTopic"` - ToPushTopic string `mapstructure:"toPushTopic"` - ToOfflinePushTopic string `mapstructure:"toOfflinePushTopic"` - ToRedisGroupID string `mapstructure:"toRedisGroupID"` - ToMongoGroupID string `mapstructure:"toMongoGroupID"` - ToPushGroupID string `mapstructure:"toPushGroupID"` - ToOfflineGroupID string `mapstructure:"toOfflinePushGroupID"` - - Tls TLSConfig `mapstructure:"tls"` + Username string `yaml:"username"` + Password string `yaml:"password"` + ProducerAck string `yaml:"producerAck"` + CompressType string `yaml:"compressType"` + Address []string `yaml:"address"` + ToRedisTopic string `yaml:"toRedisTopic"` + ToMongoTopic string `yaml:"toMongoTopic"` + ToPushTopic string `yaml:"toPushTopic"` + ToOfflinePushTopic string `yaml:"toOfflinePushTopic"` + ToRedisGroupID string `yaml:"toRedisGroupID"` + ToMongoGroupID string `yaml:"toMongoGroupID"` + ToPushGroupID string `yaml:"toPushGroupID"` + ToOfflineGroupID string `yaml:"toOfflinePushGroupID"` + + Tls TLSConfig `yaml:"tls"` } type TLSConfig struct { - EnableTLS bool `mapstructure:"enableTLS"` - CACrt string `mapstructure:"caCrt"` - ClientCrt string `mapstructure:"clientCrt"` - ClientKey string `mapstructure:"clientKey"` - ClientKeyPwd string `mapstructure:"clientKeyPwd"` - InsecureSkipVerify bool `mapstructure:"insecureSkipVerify"` + EnableTLS bool `yaml:"enableTLS"` + CACrt string `yaml:"caCrt"` + ClientCrt string `yaml:"clientCrt"` + ClientKey string `yaml:"clientKey"` + ClientKeyPwd string `yaml:"clientKeyPwd"` + InsecureSkipVerify bool `yaml:"insecureSkipVerify"` } type API struct { Api struct { - ListenIP string `mapstructure:"listenIP"` - Ports []int `mapstructure:"ports"` - CompressionLevel int `mapstructure:"compressionLevel"` - } `mapstructure:"api"` + ListenIP string `yaml:"listenIP"` + Ports []int `yaml:"ports"` + CompressionLevel int `yaml:"compressionLevel"` + } `yaml:"api"` Prometheus struct { - Enable bool `mapstructure:"enable"` - AutoSetPorts bool `mapstructure:"autoSetPorts"` - Ports []int `mapstructure:"ports"` - GrafanaURL string `mapstructure:"grafanaURL"` - } `mapstructure:"prometheus"` + Enable bool `yaml:"enable"` + AutoSetPorts bool `yaml:"autoSetPorts"` + Ports []int `yaml:"ports"` + GrafanaURL string `yaml:"grafanaURL"` + } `yaml:"prometheus"` } type CronTask struct { - CronExecuteTime string `mapstructure:"cronExecuteTime"` - RetainChatRecords int `mapstructure:"retainChatRecords"` - FileExpireTime int `mapstructure:"fileExpireTime"` - DeleteObjectType []string `mapstructure:"deleteObjectType"` + CronExecuteTime string `yaml:"cronExecuteTime"` + RetainChatRecords int `yaml:"retainChatRecords"` + FileExpireTime int `yaml:"fileExpireTime"` + DeleteObjectType []string `yaml:"deleteObjectType"` } type OfflinePushConfig struct { - Enable bool `mapstructure:"enable"` - Title string `mapstructure:"title"` - Desc string `mapstructure:"desc"` - Ext string `mapstructure:"ext"` + Enable bool `yaml:"enable"` + Title string `yaml:"title"` + Desc string `yaml:"desc"` + Ext string `yaml:"ext"` } type NotificationConfig struct { - IsSendMsg bool `mapstructure:"isSendMsg"` - ReliabilityLevel int `mapstructure:"reliabilityLevel"` - UnreadCount bool `mapstructure:"unreadCount"` - OfflinePush OfflinePushConfig `mapstructure:"offlinePush"` + IsSendMsg bool `yaml:"isSendMsg"` + ReliabilityLevel int `yaml:"reliabilityLevel"` + UnreadCount bool `yaml:"unreadCount"` + OfflinePush OfflinePushConfig `yaml:"offlinePush"` } type Notification struct { - GroupCreated NotificationConfig `mapstructure:"groupCreated"` - GroupInfoSet NotificationConfig `mapstructure:"groupInfoSet"` - JoinGroupApplication NotificationConfig `mapstructure:"joinGroupApplication"` - MemberQuit NotificationConfig `mapstructure:"memberQuit"` - GroupApplicationAccepted NotificationConfig `mapstructure:"groupApplicationAccepted"` - GroupApplicationRejected NotificationConfig `mapstructure:"groupApplicationRejected"` - GroupOwnerTransferred NotificationConfig `mapstructure:"groupOwnerTransferred"` - MemberKicked NotificationConfig `mapstructure:"memberKicked"` - MemberInvited NotificationConfig `mapstructure:"memberInvited"` - MemberEnter NotificationConfig `mapstructure:"memberEnter"` - GroupDismissed NotificationConfig `mapstructure:"groupDismissed"` - GroupMuted NotificationConfig `mapstructure:"groupMuted"` - GroupCancelMuted NotificationConfig `mapstructure:"groupCancelMuted"` - GroupMemberMuted NotificationConfig `mapstructure:"groupMemberMuted"` - GroupMemberCancelMuted NotificationConfig `mapstructure:"groupMemberCancelMuted"` - GroupMemberInfoSet NotificationConfig `mapstructure:"groupMemberInfoSet"` + GroupCreated NotificationConfig `yaml:"groupCreated"` + GroupInfoSet NotificationConfig `yaml:"groupInfoSet"` + JoinGroupApplication NotificationConfig `yaml:"joinGroupApplication"` + MemberQuit NotificationConfig `yaml:"memberQuit"` + GroupApplicationAccepted NotificationConfig `yaml:"groupApplicationAccepted"` + GroupApplicationRejected NotificationConfig `yaml:"groupApplicationRejected"` + GroupOwnerTransferred NotificationConfig `yaml:"groupOwnerTransferred"` + MemberKicked NotificationConfig `yaml:"memberKicked"` + MemberInvited NotificationConfig `yaml:"memberInvited"` + MemberEnter NotificationConfig `yaml:"memberEnter"` + GroupDismissed NotificationConfig `yaml:"groupDismissed"` + GroupMuted NotificationConfig `yaml:"groupMuted"` + GroupCancelMuted NotificationConfig `yaml:"groupCancelMuted"` + GroupMemberMuted NotificationConfig `yaml:"groupMemberMuted"` + GroupMemberCancelMuted NotificationConfig `yaml:"groupMemberCancelMuted"` + GroupMemberInfoSet NotificationConfig `yaml:"groupMemberInfoSet"` GroupMemberSetToAdmin NotificationConfig `yaml:"groupMemberSetToAdmin"` GroupMemberSetToOrdinary NotificationConfig `yaml:"groupMemberSetToOrdinaryUser"` - GroupInfoSetAnnouncement NotificationConfig `mapstructure:"groupInfoSetAnnouncement"` - GroupInfoSetName NotificationConfig `mapstructure:"groupInfoSetName"` - FriendApplicationAdded NotificationConfig `mapstructure:"friendApplicationAdded"` - FriendApplicationApproved NotificationConfig `mapstructure:"friendApplicationApproved"` - FriendApplicationRejected NotificationConfig `mapstructure:"friendApplicationRejected"` - FriendAdded NotificationConfig `mapstructure:"friendAdded"` - FriendDeleted NotificationConfig `mapstructure:"friendDeleted"` - FriendRemarkSet NotificationConfig `mapstructure:"friendRemarkSet"` - BlackAdded NotificationConfig `mapstructure:"blackAdded"` - BlackDeleted NotificationConfig `mapstructure:"blackDeleted"` - FriendInfoUpdated NotificationConfig `mapstructure:"friendInfoUpdated"` - UserInfoUpdated NotificationConfig `mapstructure:"userInfoUpdated"` - UserStatusChanged NotificationConfig `mapstructure:"userStatusChanged"` - ConversationChanged NotificationConfig `mapstructure:"conversationChanged"` - ConversationSetPrivate NotificationConfig `mapstructure:"conversationSetPrivate"` + GroupInfoSetAnnouncement NotificationConfig `yaml:"groupInfoSetAnnouncement"` + GroupInfoSetName NotificationConfig `yaml:"groupInfoSetName"` + FriendApplicationAdded NotificationConfig `yaml:"friendApplicationAdded"` + FriendApplicationApproved NotificationConfig `yaml:"friendApplicationApproved"` + FriendApplicationRejected NotificationConfig `yaml:"friendApplicationRejected"` + FriendAdded NotificationConfig `yaml:"friendAdded"` + FriendDeleted NotificationConfig `yaml:"friendDeleted"` + FriendRemarkSet NotificationConfig `yaml:"friendRemarkSet"` + BlackAdded NotificationConfig `yaml:"blackAdded"` + BlackDeleted NotificationConfig `yaml:"blackDeleted"` + FriendInfoUpdated NotificationConfig `yaml:"friendInfoUpdated"` + UserInfoUpdated NotificationConfig `yaml:"userInfoUpdated"` + UserStatusChanged NotificationConfig `yaml:"userStatusChanged"` + ConversationChanged NotificationConfig `yaml:"conversationChanged"` + ConversationSetPrivate NotificationConfig `yaml:"conversationSetPrivate"` } type Prometheus struct { - Enable bool `mapstructure:"enable"` - Ports []int `mapstructure:"ports"` + Enable bool `yaml:"enable"` + Ports []int `yaml:"ports"` } type MsgGateway struct { - RPC struct { - RegisterIP string `mapstructure:"registerIP"` - AutoSetPorts bool `mapstructure:"autoSetPorts"` - Ports []int `mapstructure:"ports"` - } `mapstructure:"rpc"` - Prometheus Prometheus `mapstructure:"prometheus"` - ListenIP string `mapstructure:"listenIP"` + RPC RPC `yaml:"rpc"` + Prometheus Prometheus `yaml:"prometheus"` + ListenIP string `yaml:"listenIP"` LongConnSvr struct { - Ports []int `mapstructure:"ports"` - WebsocketMaxConnNum int `mapstructure:"websocketMaxConnNum"` - WebsocketMaxMsgLen int `mapstructure:"websocketMaxMsgLen"` - WebsocketTimeout int `mapstructure:"websocketTimeout"` - } `mapstructure:"longConnSvr"` + Ports []int `yaml:"ports"` + WebsocketMaxConnNum int `yaml:"websocketMaxConnNum"` + WebsocketMaxMsgLen int `yaml:"websocketMaxMsgLen"` + WebsocketTimeout int `yaml:"websocketTimeout"` + } `yaml:"longConnSvr"` } type MsgTransfer struct { Prometheus struct { - Enable bool `mapstructure:"enable"` - AutoSetPorts bool `mapstructure:"autoSetPorts"` - Ports []int `mapstructure:"ports"` - } `mapstructure:"prometheus"` + Enable bool `yaml:"enable"` + AutoSetPorts bool `yaml:"autoSetPorts"` + Ports []int `yaml:"ports"` + } `yaml:"prometheus"` } type Push struct { - RPC struct { - RegisterIP string `mapstructure:"registerIP"` - ListenIP string `mapstructure:"listenIP"` - AutoSetPorts bool `mapstructure:"autoSetPorts"` - Ports []int `mapstructure:"ports"` - } `mapstructure:"rpc"` - Prometheus Prometheus `mapstructure:"prometheus"` - MaxConcurrentWorkers int `mapstructure:"maxConcurrentWorkers"` - Enable string `mapstructure:"enable"` + RPC RPC `yaml:"rpc"` + Prometheus Prometheus `yaml:"prometheus"` + MaxConcurrentWorkers int `yaml:"maxConcurrentWorkers"` + Enable string `yaml:"enable"` GeTui struct { - PushUrl string `mapstructure:"pushUrl"` - MasterSecret string `mapstructure:"masterSecret"` - AppKey string `mapstructure:"appKey"` - Intent string `mapstructure:"intent"` - ChannelID string `mapstructure:"channelID"` - ChannelName string `mapstructure:"channelName"` - } `mapstructure:"geTui"` + PushUrl string `yaml:"pushUrl"` + MasterSecret string `yaml:"masterSecret"` + AppKey string `yaml:"appKey"` + Intent string `yaml:"intent"` + ChannelID string `yaml:"channelID"` + ChannelName string `yaml:"channelName"` + } `yaml:"geTui"` FCM struct { - FilePath string `mapstructure:"filePath"` - AuthURL string `mapstructure:"authURL"` - } `mapstructure:"fcm"` + FilePath string `yaml:"filePath"` + AuthURL string `yaml:"authURL"` + } `yaml:"fcm"` JPush struct { - AppKey string `mapstructure:"appKey"` - MasterSecret string `mapstructure:"masterSecret"` - PushURL string `mapstructure:"pushURL"` - PushIntent string `mapstructure:"pushIntent"` - } `mapstructure:"jpush"` + AppKey string `yaml:"appKey"` + MasterSecret string `yaml:"masterSecret"` + PushURL string `yaml:"pushURL"` + PushIntent string `yaml:"pushIntent"` + } `yaml:"jpush"` IOSPush struct { - PushSound string `mapstructure:"pushSound"` - BadgeCount bool `mapstructure:"badgeCount"` - Production bool `mapstructure:"production"` - } `mapstructure:"iosPush"` - FullUserCache bool `mapstructure:"fullUserCache"` + PushSound string `yaml:"pushSound"` + BadgeCount bool `yaml:"badgeCount"` + Production bool `yaml:"production"` + } `yaml:"iosPush"` + FullUserCache bool `yaml:"fullUserCache"` } type Auth struct { - RPC struct { - RegisterIP string `mapstructure:"registerIP"` - ListenIP string `mapstructure:"listenIP"` - AutoSetPorts bool `mapstructure:"autoSetPorts"` - Ports []int `mapstructure:"ports"` - } `mapstructure:"rpc"` - Prometheus Prometheus `mapstructure:"prometheus"` + RPC RPC `yaml:"rpc"` + Prometheus Prometheus `yaml:"prometheus"` TokenPolicy struct { - Expire int64 `mapstructure:"expire"` - } `mapstructure:"tokenPolicy"` + Expire int64 `yaml:"expire"` + } `yaml:"tokenPolicy"` } type Conversation struct { - RPC struct { - RegisterIP string `mapstructure:"registerIP"` - ListenIP string `mapstructure:"listenIP"` - AutoSetPorts bool `mapstructure:"autoSetPorts"` - Ports []int `mapstructure:"ports"` - } `mapstructure:"rpc"` - Prometheus Prometheus `mapstructure:"prometheus"` + RPC RPC `yaml:"rpc"` + Prometheus Prometheus `yaml:"prometheus"` } type Friend struct { - RPC struct { - RegisterIP string `mapstructure:"registerIP"` - ListenIP string `mapstructure:"listenIP"` - AutoSetPorts bool `mapstructure:"autoSetPorts"` - Ports []int `mapstructure:"ports"` - } `mapstructure:"rpc"` - Prometheus Prometheus `mapstructure:"prometheus"` + RPC RPC `yaml:"rpc"` + Prometheus Prometheus `yaml:"prometheus"` } type Group struct { - RPC struct { - RegisterIP string `mapstructure:"registerIP"` - ListenIP string `mapstructure:"listenIP"` - AutoSetPorts bool `mapstructure:"autoSetPorts"` - Ports []int `mapstructure:"ports"` - } `mapstructure:"rpc"` - Prometheus Prometheus `mapstructure:"prometheus"` - EnableHistoryForNewMembers bool `mapstructure:"enableHistoryForNewMembers"` + RPC RPC `yaml:"rpc"` + Prometheus Prometheus `yaml:"prometheus"` + EnableHistoryForNewMembers bool `yaml:"enableHistoryForNewMembers"` } type Msg struct { - RPC struct { - RegisterIP string `mapstructure:"registerIP"` - ListenIP string `mapstructure:"listenIP"` - AutoSetPorts bool `mapstructure:"autoSetPorts"` - Ports []int `mapstructure:"ports"` - } `mapstructure:"rpc"` - Prometheus Prometheus `mapstructure:"prometheus"` - FriendVerify bool `mapstructure:"friendVerify"` + RPC RPC `yaml:"rpc"` + Prometheus Prometheus `yaml:"prometheus"` + FriendVerify bool `yaml:"friendVerify"` } type Third struct { - RPC struct { - RegisterIP string `mapstructure:"registerIP"` - ListenIP string `mapstructure:"listenIP"` - AutoSetPorts bool `mapstructure:"autoSetPorts"` - Ports []int `mapstructure:"ports"` - } `mapstructure:"rpc"` - Prometheus Prometheus `mapstructure:"prometheus"` + RPC RPC `yaml:"rpc"` + Prometheus Prometheus `yaml:"prometheus"` Object struct { - Enable string `mapstructure:"enable"` - Cos Cos `mapstructure:"cos"` - Oss Oss `mapstructure:"oss"` - Kodo Kodo `mapstructure:"kodo"` - Aws Aws `mapstructure:"aws"` - } `mapstructure:"object"` + Enable string `yaml:"enable"` + Cos Cos `yaml:"cos"` + Oss Oss `yaml:"oss"` + Kodo Kodo `yaml:"kodo"` + Aws Aws `yaml:"aws"` + } `yaml:"object"` } type Cos struct { - BucketURL string `mapstructure:"bucketURL"` - SecretID string `mapstructure:"secretID"` - SecretKey string `mapstructure:"secretKey"` - SessionToken string `mapstructure:"sessionToken"` - PublicRead bool `mapstructure:"publicRead"` + BucketURL string `yaml:"bucketURL"` + SecretID string `yaml:"secretID"` + SecretKey string `yaml:"secretKey"` + SessionToken string `yaml:"sessionToken"` + PublicRead bool `yaml:"publicRead"` } type Oss struct { - Endpoint string `mapstructure:"endpoint"` - Bucket string `mapstructure:"bucket"` - BucketURL string `mapstructure:"bucketURL"` - AccessKeyID string `mapstructure:"accessKeyID"` - AccessKeySecret string `mapstructure:"accessKeySecret"` - SessionToken string `mapstructure:"sessionToken"` - PublicRead bool `mapstructure:"publicRead"` + Endpoint string `yaml:"endpoint"` + Bucket string `yaml:"bucket"` + BucketURL string `yaml:"bucketURL"` + AccessKeyID string `yaml:"accessKeyID"` + AccessKeySecret string `yaml:"accessKeySecret"` + SessionToken string `yaml:"sessionToken"` + PublicRead bool `yaml:"publicRead"` } type Kodo struct { - Endpoint string `mapstructure:"endpoint"` - Bucket string `mapstructure:"bucket"` - BucketURL string `mapstructure:"bucketURL"` - AccessKeyID string `mapstructure:"accessKeyID"` - AccessKeySecret string `mapstructure:"accessKeySecret"` - SessionToken string `mapstructure:"sessionToken"` - PublicRead bool `mapstructure:"publicRead"` + Endpoint string `yaml:"endpoint"` + Bucket string `yaml:"bucket"` + BucketURL string `yaml:"bucketURL"` + AccessKeyID string `yaml:"accessKeyID"` + AccessKeySecret string `yaml:"accessKeySecret"` + SessionToken string `yaml:"sessionToken"` + PublicRead bool `yaml:"publicRead"` } type Aws struct { - Region string `mapstructure:"region"` - Bucket string `mapstructure:"bucket"` - AccessKeyID string `mapstructure:"accessKeyID"` - SecretAccessKey string `mapstructure:"secretAccessKey"` - SessionToken string `mapstructure:"sessionToken"` - PublicRead bool `mapstructure:"publicRead"` + Region string `yaml:"region"` + Bucket string `yaml:"bucket"` + AccessKeyID string `yaml:"accessKeyID"` + SecretAccessKey string `yaml:"secretAccessKey"` + SessionToken string `yaml:"sessionToken"` + PublicRead bool `yaml:"publicRead"` } type User struct { - RPC struct { - RegisterIP string `mapstructure:"registerIP"` - ListenIP string `mapstructure:"listenIP"` - AutoSetPorts bool `mapstructure:"autoSetPorts"` - Ports []int `mapstructure:"ports"` - } `mapstructure:"rpc"` - Prometheus Prometheus `mapstructure:"prometheus"` + RPC RPC `yaml:"rpc"` + Prometheus Prometheus `yaml:"prometheus"` +} + +type RPC struct { + RegisterIP string `yaml:"registerIP"` + ListenIP string `yaml:"listenIP"` + AutoSetPorts bool `yaml:"autoSetPorts"` + Ports []int `yaml:"ports"` } type Redis struct { - Address []string `mapstructure:"address"` - Username string `mapstructure:"username"` - Password string `mapstructure:"password"` - ClusterMode bool `mapstructure:"clusterMode"` - DB int `mapstructure:"storage"` - MaxRetry int `mapstructure:"maxRetry"` - PoolSize int `mapstructure:"poolSize"` + Disable bool `yaml:"-"` + Address []string `yaml:"address"` + Username string `yaml:"username"` + Password string `yaml:"password"` + ClusterMode bool `yaml:"clusterMode"` + DB int `yaml:"storage"` + MaxRetry int `yaml:"maxRetry"` + PoolSize int `yaml:"poolSize"` } type BeforeConfig struct { - Enable bool `mapstructure:"enable"` - Timeout int `mapstructure:"timeout"` - FailedContinue bool `mapstructure:"failedContinue"` - AllowedTypes []string `mapstructure:"allowedTypes"` - DeniedTypes []string `mapstructure:"deniedTypes"` + Enable bool `yaml:"enable"` + Timeout int `yaml:"timeout"` + FailedContinue bool `yaml:"failedContinue"` + AllowedTypes []string `yaml:"allowedTypes"` + DeniedTypes []string `yaml:"deniedTypes"` } type AfterConfig struct { - Enable bool `mapstructure:"enable"` - Timeout int `mapstructure:"timeout"` - AttentionIds []string `mapstructure:"attentionIds"` - AllowedTypes []string `mapstructure:"allowedTypes"` - DeniedTypes []string `mapstructure:"deniedTypes"` + Enable bool `yaml:"enable"` + Timeout int `yaml:"timeout"` + AttentionIds []string `yaml:"attentionIds"` + AllowedTypes []string `yaml:"allowedTypes"` + DeniedTypes []string `yaml:"deniedTypes"` } type Share struct { - Secret string `mapstructure:"secret"` - IMAdminUserID []string `mapstructure:"imAdminUserID"` - MultiLogin MultiLogin `mapstructure:"multiLogin"` + Secret string `yaml:"secret"` + IMAdminUserID []string `yaml:"imAdminUserID"` + MultiLogin MultiLogin `yaml:"multiLogin"` } type MultiLogin struct { - Policy int `mapstructure:"policy"` - MaxNumOneEnd int `mapstructure:"maxNumOneEnd"` + Policy int `yaml:"policy"` + MaxNumOneEnd int `yaml:"maxNumOneEnd"` } type RpcService struct { - User string `mapstructure:"user"` - Friend string `mapstructure:"friend"` - Msg string `mapstructure:"msg"` - Push string `mapstructure:"push"` - MessageGateway string `mapstructure:"messageGateway"` - Group string `mapstructure:"group"` - Auth string `mapstructure:"auth"` - Conversation string `mapstructure:"conversation"` - Third string `mapstructure:"third"` + User string `yaml:"user"` + Friend string `yaml:"friend"` + Msg string `yaml:"msg"` + Push string `yaml:"push"` + MessageGateway string `yaml:"messageGateway"` + Group string `yaml:"group"` + Auth string `yaml:"auth"` + Conversation string `yaml:"conversation"` + Third string `yaml:"third"` } func (r *RpcService) GetServiceNames() []string { @@ -418,80 +388,80 @@ func (r *RpcService) GetServiceNames() []string { // FullConfig stores all configurations for before and after events type Webhooks struct { - URL string `mapstructure:"url"` - BeforeSendSingleMsg BeforeConfig `mapstructure:"beforeSendSingleMsg"` - BeforeUpdateUserInfoEx BeforeConfig `mapstructure:"beforeUpdateUserInfoEx"` - AfterUpdateUserInfoEx AfterConfig `mapstructure:"afterUpdateUserInfoEx"` - AfterSendSingleMsg AfterConfig `mapstructure:"afterSendSingleMsg"` - BeforeSendGroupMsg BeforeConfig `mapstructure:"beforeSendGroupMsg"` - BeforeMsgModify BeforeConfig `mapstructure:"beforeMsgModify"` - AfterSendGroupMsg AfterConfig `mapstructure:"afterSendGroupMsg"` - AfterUserOnline AfterConfig `mapstructure:"afterUserOnline"` - AfterUserOffline AfterConfig `mapstructure:"afterUserOffline"` - AfterUserKickOff AfterConfig `mapstructure:"afterUserKickOff"` - BeforeOfflinePush BeforeConfig `mapstructure:"beforeOfflinePush"` - BeforeOnlinePush BeforeConfig `mapstructure:"beforeOnlinePush"` - BeforeGroupOnlinePush BeforeConfig `mapstructure:"beforeGroupOnlinePush"` - BeforeAddFriend BeforeConfig `mapstructure:"beforeAddFriend"` - BeforeUpdateUserInfo BeforeConfig `mapstructure:"beforeUpdateUserInfo"` - AfterUpdateUserInfo AfterConfig `mapstructure:"afterUpdateUserInfo"` - BeforeCreateGroup BeforeConfig `mapstructure:"beforeCreateGroup"` - AfterCreateGroup AfterConfig `mapstructure:"afterCreateGroup"` - BeforeMemberJoinGroup BeforeConfig `mapstructure:"beforeMemberJoinGroup"` - BeforeSetGroupMemberInfo BeforeConfig `mapstructure:"beforeSetGroupMemberInfo"` - AfterSetGroupMemberInfo AfterConfig `mapstructure:"afterSetGroupMemberInfo"` - AfterQuitGroup AfterConfig `mapstructure:"afterQuitGroup"` - AfterKickGroupMember AfterConfig `mapstructure:"afterKickGroupMember"` - AfterDismissGroup AfterConfig `mapstructure:"afterDismissGroup"` - BeforeApplyJoinGroup BeforeConfig `mapstructure:"beforeApplyJoinGroup"` - AfterGroupMsgRead AfterConfig `mapstructure:"afterGroupMsgRead"` - AfterSingleMsgRead AfterConfig `mapstructure:"afterSingleMsgRead"` - BeforeUserRegister BeforeConfig `mapstructure:"beforeUserRegister"` - AfterUserRegister AfterConfig `mapstructure:"afterUserRegister"` - AfterTransferGroupOwner AfterConfig `mapstructure:"afterTransferGroupOwner"` - BeforeSetFriendRemark BeforeConfig `mapstructure:"beforeSetFriendRemark"` - AfterSetFriendRemark AfterConfig `mapstructure:"afterSetFriendRemark"` - AfterGroupMsgRevoke AfterConfig `mapstructure:"afterGroupMsgRevoke"` - AfterJoinGroup AfterConfig `mapstructure:"afterJoinGroup"` - BeforeInviteUserToGroup BeforeConfig `mapstructure:"beforeInviteUserToGroup"` - AfterSetGroupInfo AfterConfig `mapstructure:"afterSetGroupInfo"` - BeforeSetGroupInfo BeforeConfig `mapstructure:"beforeSetGroupInfo"` - AfterSetGroupInfoEx AfterConfig `mapstructure:"afterSetGroupInfoEx"` - BeforeSetGroupInfoEx BeforeConfig `mapstructure:"beforeSetGroupInfoEx"` - AfterRevokeMsg AfterConfig `mapstructure:"afterRevokeMsg"` - BeforeAddBlack BeforeConfig `mapstructure:"beforeAddBlack"` - AfterAddFriend AfterConfig `mapstructure:"afterAddFriend"` - BeforeAddFriendAgree BeforeConfig `mapstructure:"beforeAddFriendAgree"` - AfterAddFriendAgree AfterConfig `mapstructure:"afterAddFriendAgree"` - AfterDeleteFriend AfterConfig `mapstructure:"afterDeleteFriend"` - BeforeImportFriends BeforeConfig `mapstructure:"beforeImportFriends"` - AfterImportFriends AfterConfig `mapstructure:"afterImportFriends"` - AfterRemoveBlack AfterConfig `mapstructure:"afterRemoveBlack"` + URL string `yaml:"url"` + BeforeSendSingleMsg BeforeConfig `yaml:"beforeSendSingleMsg"` + BeforeUpdateUserInfoEx BeforeConfig `yaml:"beforeUpdateUserInfoEx"` + AfterUpdateUserInfoEx AfterConfig `yaml:"afterUpdateUserInfoEx"` + AfterSendSingleMsg AfterConfig `yaml:"afterSendSingleMsg"` + BeforeSendGroupMsg BeforeConfig `yaml:"beforeSendGroupMsg"` + BeforeMsgModify BeforeConfig `yaml:"beforeMsgModify"` + AfterSendGroupMsg AfterConfig `yaml:"afterSendGroupMsg"` + AfterUserOnline AfterConfig `yaml:"afterUserOnline"` + AfterUserOffline AfterConfig `yaml:"afterUserOffline"` + AfterUserKickOff AfterConfig `yaml:"afterUserKickOff"` + BeforeOfflinePush BeforeConfig `yaml:"beforeOfflinePush"` + BeforeOnlinePush BeforeConfig `yaml:"beforeOnlinePush"` + BeforeGroupOnlinePush BeforeConfig `yaml:"beforeGroupOnlinePush"` + BeforeAddFriend BeforeConfig `yaml:"beforeAddFriend"` + BeforeUpdateUserInfo BeforeConfig `yaml:"beforeUpdateUserInfo"` + AfterUpdateUserInfo AfterConfig `yaml:"afterUpdateUserInfo"` + BeforeCreateGroup BeforeConfig `yaml:"beforeCreateGroup"` + AfterCreateGroup AfterConfig `yaml:"afterCreateGroup"` + BeforeMemberJoinGroup BeforeConfig `yaml:"beforeMemberJoinGroup"` + BeforeSetGroupMemberInfo BeforeConfig `yaml:"beforeSetGroupMemberInfo"` + AfterSetGroupMemberInfo AfterConfig `yaml:"afterSetGroupMemberInfo"` + AfterQuitGroup AfterConfig `yaml:"afterQuitGroup"` + AfterKickGroupMember AfterConfig `yaml:"afterKickGroupMember"` + AfterDismissGroup AfterConfig `yaml:"afterDismissGroup"` + BeforeApplyJoinGroup BeforeConfig `yaml:"beforeApplyJoinGroup"` + AfterGroupMsgRead AfterConfig `yaml:"afterGroupMsgRead"` + AfterSingleMsgRead AfterConfig `yaml:"afterSingleMsgRead"` + BeforeUserRegister BeforeConfig `yaml:"beforeUserRegister"` + AfterUserRegister AfterConfig `yaml:"afterUserRegister"` + AfterTransferGroupOwner AfterConfig `yaml:"afterTransferGroupOwner"` + BeforeSetFriendRemark BeforeConfig `yaml:"beforeSetFriendRemark"` + AfterSetFriendRemark AfterConfig `yaml:"afterSetFriendRemark"` + AfterGroupMsgRevoke AfterConfig `yaml:"afterGroupMsgRevoke"` + AfterJoinGroup AfterConfig `yaml:"afterJoinGroup"` + BeforeInviteUserToGroup BeforeConfig `yaml:"beforeInviteUserToGroup"` + AfterSetGroupInfo AfterConfig `yaml:"afterSetGroupInfo"` + BeforeSetGroupInfo BeforeConfig `yaml:"beforeSetGroupInfo"` + AfterSetGroupInfoEx AfterConfig `yaml:"afterSetGroupInfoEx"` + BeforeSetGroupInfoEx BeforeConfig `yaml:"beforeSetGroupInfoEx"` + AfterRevokeMsg AfterConfig `yaml:"afterRevokeMsg"` + BeforeAddBlack BeforeConfig `yaml:"beforeAddBlack"` + AfterAddFriend AfterConfig `yaml:"afterAddFriend"` + BeforeAddFriendAgree BeforeConfig `yaml:"beforeAddFriendAgree"` + AfterAddFriendAgree AfterConfig `yaml:"afterAddFriendAgree"` + AfterDeleteFriend AfterConfig `yaml:"afterDeleteFriend"` + BeforeImportFriends BeforeConfig `yaml:"beforeImportFriends"` + AfterImportFriends AfterConfig `yaml:"afterImportFriends"` + AfterRemoveBlack AfterConfig `yaml:"afterRemoveBlack"` } type ZooKeeper struct { - Schema string `mapstructure:"schema"` - Address []string `mapstructure:"address"` - Username string `mapstructure:"username"` - Password string `mapstructure:"password"` + Schema string `yaml:"schema"` + Address []string `yaml:"address"` + Username string `yaml:"username"` + Password string `yaml:"password"` } type Discovery struct { - Enable string `mapstructure:"enable"` - Etcd Etcd `mapstructure:"etcd"` - Kubernetes Kubernetes `mapstructure:"kubernetes"` - RpcService RpcService `mapstructure:"rpcService"` + Enable string `yaml:"enable"` + Etcd Etcd `yaml:"etcd"` + Kubernetes Kubernetes `yaml:"kubernetes"` + RpcService RpcService `yaml:"rpcService"` } type Kubernetes struct { - Namespace string `mapstructure:"namespace"` + Namespace string `yaml:"namespace"` } type Etcd struct { - RootDirectory string `mapstructure:"rootDirectory"` - Address []string `mapstructure:"address"` - Username string `mapstructure:"username"` - Password string `mapstructure:"password"` + RootDirectory string `yaml:"rootDirectory"` + Address []string `yaml:"address"` + Username string `yaml:"username"` + Password string `yaml:"password"` } func (m *Mongo) Build() *mongoutil.Config { @@ -783,7 +753,7 @@ func (a *AllConfig) GetConfigNames() []string { } } -var ( +const ( FileName = "config.yaml" DiscoveryConfigFilename = "discovery.yml" KafkaConfigFileName = "kafka.yml" diff --git a/pkg/common/config/constant.go b/pkg/common/config/constant.go index f3fcc67ef..fa3f0ca05 100644 --- a/pkg/common/config/constant.go +++ b/pkg/common/config/constant.go @@ -14,13 +14,16 @@ package config +import "github.com/openimsdk/tools/utils/runtimeenv" + const ConfKey = "conf" const ( MountConfigFilePath = "CONFIG_PATH" DeploymentType = "DEPLOYMENT_TYPE" - KUBERNETES = "kubernetes" + KUBERNETES = runtimeenv.Kubernetes ETCD = "etcd" + //Standalone = "standalone" ) const ( diff --git a/pkg/common/config/global.go b/pkg/common/config/global.go new file mode 100644 index 000000000..19f74b0a9 --- /dev/null +++ b/pkg/common/config/global.go @@ -0,0 +1,11 @@ +package config + +var standalone bool + +func SetStandalone() { + standalone = true +} + +func Standalone() bool { + return standalone +} diff --git a/pkg/common/config/load_config.go b/pkg/common/config/load_config.go index aa87211f9..142b704e1 100644 --- a/pkg/common/config/load_config.go +++ b/pkg/common/config/load_config.go @@ -7,11 +7,12 @@ import ( "github.com/mitchellh/mapstructure" "github.com/openimsdk/tools/errs" + "github.com/openimsdk/tools/utils/runtimeenv" "github.com/spf13/viper" ) -func Load(configDirectory string, configFileName string, envPrefix string, runtimeEnv string, config any) error { - if runtimeEnv == KUBERNETES { +func Load(configDirectory string, configFileName string, envPrefix string, config any) error { + if runtimeenv.RuntimeEnvironment() == KUBERNETES { mountPath := os.Getenv(MountConfigFilePath) if mountPath == "" { return errs.ErrArgs.WrapMsg(MountConfigFilePath + " env is empty") @@ -35,7 +36,7 @@ func loadConfig(path string, envPrefix string, config any) error { } if err := v.Unmarshal(config, func(config *mapstructure.DecoderConfig) { - config.TagName = "mapstructure" + config.TagName = StructTagName }); err != nil { return errs.WrapMsg(err, "failed to unmarshal config", "path", path, "envPrefix", envPrefix) } diff --git a/pkg/common/discovery/discoveryregister.go b/pkg/common/discovery/discoveryregister.go index 1b64c3e78..dc100be5c 100644 --- a/pkg/common/discovery/discoveryregister.go +++ b/pkg/common/discovery/discoveryregister.go @@ -19,6 +19,8 @@ import ( "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/tools/discovery" + "github.com/openimsdk/tools/discovery/standalone" + "github.com/openimsdk/tools/utils/runtimeenv" "google.golang.org/grpc" "github.com/openimsdk/tools/discovery/kubernetes" @@ -28,8 +30,11 @@ import ( ) // NewDiscoveryRegister creates a new service discovery and registry client based on the provided environment type. -func NewDiscoveryRegister(discovery *config.Discovery, runtimeEnv string, watchNames []string) (discovery.SvcDiscoveryRegistry, error) { - if runtimeEnv == config.KUBERNETES { +func NewDiscoveryRegister(discovery *config.Discovery, watchNames []string) (discovery.SvcDiscoveryRegistry, error) { + if config.Standalone() { + return standalone.GetSvcDiscoveryRegistry(), nil + } + if runtimeenv.RuntimeEnvironment() == config.KUBERNETES { return kubernetes.NewKubernetesConnManager(discovery.Kubernetes.Namespace, grpc.WithDefaultCallOptions( grpc.MaxCallSendMsgSize(1024*1024*20), diff --git a/pkg/common/prommetrics/api.go b/pkg/common/prommetrics/api.go index 2dc5cb65d..b1368f511 100644 --- a/pkg/common/prommetrics/api.go +++ b/pkg/common/prommetrics/api.go @@ -1,10 +1,11 @@ package prommetrics import ( - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/promhttp" "net" "strconv" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" ) var ( @@ -24,6 +25,10 @@ var ( ) ) +func RegistryApi() { + registry.MustRegister(apiCounter, httpCounter) +} + func ApiInit(listener net.Listener) error { apiRegistry := prometheus.NewRegistry() cs := append( @@ -41,9 +46,3 @@ func APICall(path string, method string, apiCode int) { func HttpCall(path string, method string, status int) { httpCounter.With(prometheus.Labels{"path": path, "method": method, "status": strconv.Itoa(status)}).Inc() } - -//func ApiHandler() http.Handler { -// return promhttp.InstrumentMetricHandler( -// apiRegistry, promhttp.HandlerFor(apiRegistry, promhttp.HandlerOpts{}), -// ) -//} diff --git a/pkg/common/prommetrics/discovery.go b/pkg/common/prommetrics/discovery.go deleted file mode 100644 index 8f03bc2ae..000000000 --- a/pkg/common/prommetrics/discovery.go +++ /dev/null @@ -1,31 +0,0 @@ -package prommetrics - -import "fmt" - -const ( - APIKeyName = "api" - MessageTransferKeyName = "message-transfer" -) - -type Target struct { - Target string `json:"target"` - Labels map[string]string `json:"labels"` -} - -type RespTarget struct { - Targets []string `json:"targets"` - Labels map[string]string `json:"labels"` -} - -func BuildDiscoveryKey(name string) string { - return fmt.Sprintf("%s/%s/%s", "openim", "prometheus_discovery", name) -} - -func BuildDefaultTarget(host string, ip int) Target { - return Target{ - Target: fmt.Sprintf("%s:%d", host, ip), - Labels: map[string]string{ - "namespace": "default", - }, - } -} diff --git a/pkg/common/prommetrics/doc.go b/pkg/common/prommetrics/doc.go deleted file mode 100644 index c5108b4cb..000000000 --- a/pkg/common/prommetrics/doc.go +++ /dev/null @@ -1,15 +0,0 @@ -// Copyright © 2024 OpenIM. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package prommetrics // import "github.com/openimsdk/open-im-server/v3/pkg/common/prommetrics" diff --git a/pkg/common/prommetrics/grpc_auth.go b/pkg/common/prommetrics/grpc_auth.go index 30dd5f1b1..a102c5d26 100644 --- a/pkg/common/prommetrics/grpc_auth.go +++ b/pkg/common/prommetrics/grpc_auth.go @@ -24,3 +24,7 @@ var ( Help: "The number of user login", }) ) + +func RegistryAuth() { + registry.MustRegister(UserLoginCounter) +} diff --git a/pkg/common/prommetrics/grpc_msg.go b/pkg/common/prommetrics/grpc_msg.go index 758879b90..909fddd3d 100644 --- a/pkg/common/prommetrics/grpc_msg.go +++ b/pkg/common/prommetrics/grpc_msg.go @@ -36,3 +36,12 @@ var ( Help: "The number of group chat msg failed processed", }) ) + +func RegistryMsg() { + registry.MustRegister( + SingleChatMsgProcessSuccessCounter, + SingleChatMsgProcessFailedCounter, + GroupChatMsgProcessSuccessCounter, + GroupChatMsgProcessFailedCounter, + ) +} diff --git a/pkg/common/prommetrics/grpc_msggateway.go b/pkg/common/prommetrics/grpc_msggateway.go index 98d5a3267..0377b2fa9 100644 --- a/pkg/common/prommetrics/grpc_msggateway.go +++ b/pkg/common/prommetrics/grpc_msggateway.go @@ -24,3 +24,7 @@ var ( Help: "The number of online user num", }) ) + +func RegistryMsgGateway() { + registry.MustRegister(OnlineUserGauge) +} diff --git a/pkg/common/prommetrics/grpc_push.go b/pkg/common/prommetrics/grpc_push.go index 5c966310f..c6280ec76 100644 --- a/pkg/common/prommetrics/grpc_push.go +++ b/pkg/common/prommetrics/grpc_push.go @@ -28,3 +28,10 @@ var ( Help: "The number of messages with a push time exceeding 10 seconds", }) ) + +func RegistryPush() { + registry.MustRegister( + MsgOfflinePushFailedCounter, + MsgLoneTimePushCounter, + ) +} diff --git a/pkg/common/prommetrics/grpc_user.go b/pkg/common/prommetrics/grpc_user.go index cc2fc42e6..1c8c94c7a 100644 --- a/pkg/common/prommetrics/grpc_user.go +++ b/pkg/common/prommetrics/grpc_user.go @@ -8,3 +8,7 @@ var ( Help: "The number of user login", }) ) + +func RegistryUser() { + registry.MustRegister(UserRegisterCounter) +} diff --git a/pkg/common/prommetrics/prommetrics.go b/pkg/common/prommetrics/prommetrics.go index 2fc5d76b4..153314bbb 100644 --- a/pkg/common/prommetrics/prommetrics.go +++ b/pkg/common/prommetrics/prommetrics.go @@ -15,14 +15,42 @@ package prommetrics import ( - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/collectors" + "errors" + "fmt" "net" "net/http" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/collectors" + "github.com/prometheus/client_golang/prometheus/promhttp" ) const commonPath = "/metrics" +var registry = &prometheusRegistry{prometheus.NewRegistry()} + +type prometheusRegistry struct { + *prometheus.Registry +} + +func (x *prometheusRegistry) MustRegister(cs ...prometheus.Collector) { + for _, c := range cs { + if err := x.Registry.Register(c); err != nil { + if errors.As(err, &prometheus.AlreadyRegisteredError{}) { + continue + } + panic(err) + } + } +} + +func init() { + registry.MustRegister( + collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}), + collectors.NewGoCollector(), + ) +} + var ( baseCollector = []prometheus.Collector{ collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}), @@ -36,3 +64,48 @@ func Init(registry *prometheus.Registry, listener net.Listener, path string, han srv.Handle(path, handler) return http.Serve(listener, srv) } + +func RegistryAll() { + RegistryApi() + RegistryAuth() + RegistryMsg() + RegistryMsgGateway() + RegistryPush() + RegistryUser() + RegistryRpc() + RegistryTransfer() +} + +func Start(listener net.Listener) error { + srv := http.NewServeMux() + srv.Handle(commonPath, promhttp.HandlerFor(registry, promhttp.HandlerOpts{})) + return http.Serve(listener, srv) +} + +const ( + APIKeyName = "api" + MessageTransferKeyName = "message-transfer" +) + +type Target struct { + Target string `json:"target"` + Labels map[string]string `json:"labels"` +} + +type RespTarget struct { + Targets []string `json:"targets"` + Labels map[string]string `json:"labels"` +} + +func BuildDiscoveryKey(name string) string { + return fmt.Sprintf("%s/%s/%s", "openim", "prometheus_discovery", name) +} + +func BuildDefaultTarget(host string, ip int) Target { + return Target{ + Target: fmt.Sprintf("%s:%d", host, ip), + Labels: map[string]string{ + "namespace": "default", + }, + } +} diff --git a/pkg/common/prommetrics/prommetrics_test.go b/pkg/common/prommetrics/prommetrics_test.go index 14b1aaff3..be2dff7f2 100644 --- a/pkg/common/prommetrics/prommetrics_test.go +++ b/pkg/common/prommetrics/prommetrics_test.go @@ -14,6 +14,8 @@ package prommetrics +import "testing" + //func TestNewGrpcPromObj(t *testing.T) { // // Create a custom metric to pass into the NewGrpcPromObj function. // customMetric := prometheus.NewCounter(prometheus.CounterOpts{ @@ -67,3 +69,9 @@ package prommetrics // }) // } //} + +func TestName(t *testing.T) { + RegistryApi() + RegistryApi() + +} diff --git a/pkg/common/prommetrics/rpc.go b/pkg/common/prommetrics/rpc.go index 3f115d30b..a04e4fd5e 100644 --- a/pkg/common/prommetrics/rpc.go +++ b/pkg/common/prommetrics/rpc.go @@ -1,12 +1,13 @@ package prommetrics import ( + "net" + "strconv" + gp "github.com/grpc-ecosystem/go-grpc-prometheus" "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" - "net" - "strconv" ) const rpcPath = commonPath @@ -22,6 +23,10 @@ var ( ) ) +func RegistryRpc() { + registry.MustRegister(rpcCounter) +} + func RpcInit(cs []prometheus.Collector, listener net.Listener) error { reg := prometheus.NewRegistry() cs = append(append( diff --git a/pkg/common/prommetrics/transfer.go b/pkg/common/prommetrics/transfer.go index 36fe1d568..51a4ca872 100644 --- a/pkg/common/prommetrics/transfer.go +++ b/pkg/common/prommetrics/transfer.go @@ -15,9 +15,10 @@ package prommetrics import ( + "net" + "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" - "net" ) var ( @@ -43,6 +44,16 @@ var ( }) ) +func RegistryTransfer() { + registry.MustRegister( + MsgInsertRedisSuccessCounter, + MsgInsertRedisFailedCounter, + MsgInsertMongoSuccessCounter, + MsgInsertMongoFailedCounter, + SeqSetFailedCounter, + ) +} + func TransferInit(listener net.Listener) error { reg := prometheus.NewRegistry() cs := append( diff --git a/pkg/common/redispubsub/doc.go b/pkg/common/redispubsub/doc.go deleted file mode 100644 index 19b2e38f2..000000000 --- a/pkg/common/redispubsub/doc.go +++ /dev/null @@ -1,15 +0,0 @@ -// Copyright © 2024 OpenIM. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package redispubsub // import "github.com/openimsdk/open-im-server/v3/pkg/common/redispubsub" diff --git a/pkg/common/redispubsub/redispubliser.go b/pkg/common/redispubsub/redispubliser.go deleted file mode 100644 index 6e41af73a..000000000 --- a/pkg/common/redispubsub/redispubliser.go +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright © 2024 OpenIM. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package redispubsub - -import "github.com/redis/go-redis/v9" - -type Publisher struct { - client redis.UniversalClient - channel string -} - -func NewPublisher(client redis.UniversalClient, channel string) *Publisher { - return &Publisher{client: client, channel: channel} -} - -func (p *Publisher) Publish(message string) error { - return p.client.Publish(ctx, p.channel, message).Err() -} diff --git a/pkg/common/redispubsub/redissubscriber.go b/pkg/common/redispubsub/redissubscriber.go deleted file mode 100644 index aea99b318..000000000 --- a/pkg/common/redispubsub/redissubscriber.go +++ /dev/null @@ -1,49 +0,0 @@ -// Copyright © 2024 OpenIM. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package redispubsub - -import ( - "context" - - "github.com/redis/go-redis/v9" -) - -var ctx = context.Background() - -type Subscriber struct { - client redis.UniversalClient - channel string -} - -func NewSubscriber(client redis.UniversalClient, channel string) *Subscriber { - return &Subscriber{client: client, channel: channel} -} - -func (s *Subscriber) OnMessage(ctx context.Context, callback func(string)) error { - messageChannel := s.client.Subscribe(ctx, s.channel).Channel() - - go func() { - for { - select { - case <-ctx.Done(): - return - case msg := <-messageChannel: - callback(msg.Payload) - } - } - }() - - return nil -} diff --git a/pkg/common/startrpc/doc.go b/pkg/common/startrpc/doc.go deleted file mode 100644 index fce7309f4..000000000 --- a/pkg/common/startrpc/doc.go +++ /dev/null @@ -1,15 +0,0 @@ -// Copyright © 2024 OpenIM. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package startrpc // import "github.com/openimsdk/open-im-server/v3/pkg/common/startrpc" diff --git a/pkg/common/startrpc/start.go b/pkg/common/startrpc/start.go index 27aabca95..a69edae20 100644 --- a/pkg/common/startrpc/start.go +++ b/pkg/common/startrpc/start.go @@ -19,7 +19,6 @@ import ( "errors" "fmt" "net" - "net/http" "os" "os/signal" "strconv" @@ -27,205 +26,186 @@ import ( "time" conf "github.com/openimsdk/open-im-server/v3/pkg/common/config" - disetcd "github.com/openimsdk/open-im-server/v3/pkg/common/discovery/etcd" - "github.com/openimsdk/tools/discovery/etcd" "github.com/openimsdk/tools/utils/datautil" "github.com/openimsdk/tools/utils/jsonutil" + "github.com/openimsdk/tools/utils/network" "google.golang.org/grpc/status" - "github.com/openimsdk/tools/utils/runtimeenv" - kdisc "github.com/openimsdk/open-im-server/v3/pkg/common/discovery" "github.com/openimsdk/open-im-server/v3/pkg/common/prommetrics" "github.com/openimsdk/tools/discovery" "github.com/openimsdk/tools/errs" "github.com/openimsdk/tools/log" "github.com/openimsdk/tools/mw" - "github.com/openimsdk/tools/utils/network" "google.golang.org/grpc" "google.golang.org/grpc/credentials/insecure" ) -// Start rpc server. -func Start[T any](ctx context.Context, discovery *conf.Discovery, prometheusConfig *conf.Prometheus, listenIP, +func init() { + prommetrics.RegistryAll() +} + +func Start[T any](ctx context.Context, disc *conf.Discovery, prometheusConfig *conf.Prometheus, listenIP, registerIP string, autoSetPorts bool, rpcPorts []int, index int, rpcRegisterName string, notification *conf.Notification, config T, watchConfigNames []string, watchServiceNames []string, - rpcFn func(ctx context.Context, config T, client discovery.SvcDiscoveryRegistry, server *grpc.Server) error, + rpcFn func(ctx context.Context, config T, client discovery.Conn, server grpc.ServiceRegistrar) error, options ...grpc.ServerOption) error { - watchConfigNames = append(watchConfigNames, conf.LogConfigFileName) - var ( - rpcTcpAddr string - netDone = make(chan struct{}, 2) - netErr error - prometheusPort int - ) - if notification != nil { conf.InitNotification(notification) } + options = append(options, mw.GrpcServer()) + registerIP, err := network.GetRpcRegisterIP(registerIP) if err != nil { return err } - - runTimeEnv := runtimeenv.PrintRuntimeEnvironment() - - if !autoSetPorts { - rpcPort, err := datautil.GetElemByIndex(rpcPorts, index) + var prometheusListenAddr string + if autoSetPorts { + prometheusListenAddr = net.JoinHostPort(listenIP, "0") + } else { + prometheusPort, err := datautil.GetElemByIndex(prometheusConfig.Ports, index) if err != nil { return err } - rpcTcpAddr = net.JoinHostPort(network.GetListenIP(listenIP), strconv.Itoa(rpcPort)) - } else { - rpcTcpAddr = net.JoinHostPort(network.GetListenIP(listenIP), "0") + prometheusListenAddr = net.JoinHostPort(listenIP, strconv.Itoa(prometheusPort)) } - getAutoPort := func() (net.Listener, int, error) { - listener, err := net.Listen("tcp", rpcTcpAddr) - if err != nil { - return nil, 0, errs.WrapMsg(err, "listen err", "rpcTcpAddr", rpcTcpAddr) - } - _, portStr, _ := net.SplitHostPort(listener.Addr().String()) - port, _ := strconv.Atoi(portStr) - return listener, port, nil - } + watchConfigNames = append(watchConfigNames, conf.LogConfigFileName) - if autoSetPorts && discovery.Enable != conf.ETCD { - return errs.New("only etcd support autoSetPorts", "rpcRegisterName", rpcRegisterName).Wrap() - } - client, err := kdisc.NewDiscoveryRegister(discovery, runTimeEnv, watchServiceNames) + client, err := kdisc.NewDiscoveryRegister(disc, watchServiceNames) if err != nil { return err } defer client.Close() - client.AddOption(mw.GrpcClient(), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithDefaultServiceConfig(fmt.Sprintf(`{"LoadBalancingPolicy": "%s"}`, "round_robin"))) - - // var reg *prometheus.Registry - // var metric *grpcprometheus.ServerMetrics - if prometheusConfig.Enable { - // cusMetrics := prommetrics.GetGrpcCusMetrics(rpcRegisterName, share) - // reg, metric, _ = prommetrics.NewGrpcPromObj(cusMetrics) - // options = append(options, mw.GrpcServer(), grpc.StreamInterceptor(metric.StreamServerInterceptor()), - // grpc.UnaryInterceptor(metric.UnaryServerInterceptor())) + client.AddOption( + mw.GrpcClient(), grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithDefaultServiceConfig(fmt.Sprintf(`{"LoadBalancingPolicy": "%s"}`, "round_robin")), + ) + + ctx, cancel := context.WithCancelCause(ctx) + + go func() { + sigs := make(chan os.Signal, 1) + signal.Notify(sigs, syscall.SIGTERM, syscall.SIGINT, syscall.SIGKILL) + select { + case <-ctx.Done(): + return + case val := <-sigs: + log.ZDebug(ctx, "recv signal", "signal", val.String()) + cancel(fmt.Errorf("signal %s", val.String())) + } + }() + + if prometheusListenAddr != "" { options = append( - options, mw.GrpcServer(), + options, prommetricsUnaryInterceptor(rpcRegisterName), prommetricsStreamInterceptor(rpcRegisterName), ) - - var ( - listener net.Listener - ) - - if autoSetPorts { - listener, prometheusPort, err = getAutoPort() - if err != nil { - return err - } - - etcdClient := client.(*etcd.SvcDiscoveryRegistryImpl).GetClient() - - _, err = etcdClient.Put(ctx, prommetrics.BuildDiscoveryKey(rpcRegisterName), jsonutil.StructToJsonString(prommetrics.BuildDefaultTarget(registerIP, prometheusPort))) - if err != nil { - return errs.WrapMsg(err, "etcd put err") - } - } else { - prometheusPort, err = datautil.GetElemByIndex(prometheusConfig.Ports, index) - if err != nil { + prometheusListener, prometheusPort, err := listenTCP(prometheusListenAddr) + if err != nil { + return err + } + log.ZDebug(ctx, "prometheus start", "addr", prometheusListener.Addr(), "rpcRegisterName", rpcRegisterName) + target, err := jsonutil.JsonMarshal(prommetrics.BuildDefaultTarget(registerIP, prometheusPort)) + if err != nil { + return err + } + if err := client.SetKey(ctx, prommetrics.BuildDiscoveryKey(prommetrics.APIKeyName), target); err != nil { + if !errors.Is(err, discovery.ErrNotSupportedKeyValue) { return err } - listener, err = net.Listen("tcp", fmt.Sprintf(":%d", prometheusPort)) - if err != nil { - return errs.WrapMsg(err, "listen err", "rpcTcpAddr", rpcTcpAddr) - } } - cs := prommetrics.GetGrpcCusMetrics(rpcRegisterName, discovery) go func() { - if err := prommetrics.RpcInit(cs, listener); err != nil && !errors.Is(err, http.ErrServerClosed) { - netErr = errs.WrapMsg(err, fmt.Sprintf("rpc %s prometheus start err: %d", rpcRegisterName, prometheusPort)) - netDone <- struct{}{} + err := prommetrics.Start(prometheusListener) + if err == nil { + err = fmt.Errorf("listener done") } - //metric.InitializeMetrics(srv) - // Create a HTTP server for prometheus. - // httpServer = &http.Server{Handler: promhttp.HandlerFor(reg, promhttp.HandlerOpts{}), Addr: fmt.Sprintf("0.0.0.0:%d", prometheusPort)} - // if err := httpServer.ListenAndServe(); err != nil && err != http.ErrServerClosed { - // netErr = errs.WrapMsg(err, "prometheus start err", httpServer.Addr) - // netDone <- struct{}{} - // } + cancel(fmt.Errorf("prommetrics %s %w", rpcRegisterName, err)) }() - } else { - options = append(options, mw.GrpcServer()) } - listener, port, err := getAutoPort() - if err != nil { - return err - } + var ( + rpcServer *grpc.Server + rpcGracefulStop chan struct{} + ) - log.CInfo(ctx, "RPC server is initializing", "rpcRegisterName", rpcRegisterName, "rpcPort", port, - "prometheusPort", prometheusPort) + onGrpcServiceRegistrar := func(desc *grpc.ServiceDesc, impl any) { + if rpcServer != nil { + rpcServer.RegisterService(desc, impl) + return + } + var rpcListenAddr string + if autoSetPorts { + rpcListenAddr = net.JoinHostPort(listenIP, "0") + } else { + rpcPort, err := datautil.GetElemByIndex(rpcPorts, index) + if err != nil { + cancel(fmt.Errorf("rpcPorts index out of range %s %w", rpcRegisterName, err)) + return + } + rpcListenAddr = net.JoinHostPort(listenIP, strconv.Itoa(rpcPort)) + } + rpcListener, err := net.Listen("tcp", rpcListenAddr) + if err != nil { + cancel(fmt.Errorf("listen rpc %s %s %w", rpcRegisterName, rpcListenAddr, err)) + return + } - defer listener.Close() - srv := grpc.NewServer(options...) + rpcServer = grpc.NewServer(options...) + rpcServer.RegisterService(desc, impl) + rpcGracefulStop = make(chan struct{}) + rpcPort := rpcListener.Addr().(*net.TCPAddr).Port + log.ZDebug(ctx, "rpc start register", "rpcRegisterName", rpcRegisterName, "registerIP", registerIP, "rpcPort", rpcPort) + grpcOpt := grpc.WithTransportCredentials(insecure.NewCredentials()) + rpcGracefulStop = make(chan struct{}) + go func() { + <-ctx.Done() + rpcServer.GracefulStop() + close(rpcGracefulStop) + }() + if err := client.Register(ctx, rpcRegisterName, registerIP, rpcListener.Addr().(*net.TCPAddr).Port, grpcOpt); err != nil { + cancel(fmt.Errorf("rpc register %s %w", rpcRegisterName, err)) + return + } - err = rpcFn(ctx, config, client, srv) - if err != nil { - return err + go func() { + err := rpcServer.Serve(rpcListener) + if err == nil { + err = fmt.Errorf("serve end") + } + cancel(fmt.Errorf("rpc %s %w", rpcRegisterName, err)) + }() } - err = client.Register( - rpcRegisterName, - registerIP, - port, - grpc.WithTransportCredentials(insecure.NewCredentials()), - ) + err = rpcFn(ctx, config, client, &grpcServiceRegistrar{onRegisterService: onGrpcServiceRegistrar}) if err != nil { return err } - - go func() { - err := srv.Serve(listener) - if err != nil && !errors.Is(err, http.ErrServerClosed) { - netErr = errs.WrapMsg(err, "rpc start err: ", rpcTcpAddr) - netDone <- struct{}{} + <-ctx.Done() + log.ZDebug(ctx, "cmd wait done", "err", context.Cause(ctx)) + if rpcGracefulStop != nil { + timeout := time.NewTimer(time.Second * 15) + defer timeout.Stop() + select { + case <-timeout.C: + log.ZWarn(ctx, "rcp graceful stop timeout", nil) + case <-rpcGracefulStop: + log.ZDebug(ctx, "rcp graceful stop done") } - }() - - if discovery.Enable == conf.ETCD { - cm := disetcd.NewConfigManager(client.(*etcd.SvcDiscoveryRegistryImpl).GetClient(), watchConfigNames) - cm.Watch(ctx) - } - - sigs := make(chan os.Signal, 1) - signal.Notify(sigs, syscall.SIGTERM) - select { - case <-sigs: - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - if err := gracefulStopWithCtx(ctx, srv.GracefulStop); err != nil { - return err - } - return nil - case <-netDone: - return netErr } + return context.Cause(ctx) } -func gracefulStopWithCtx(ctx context.Context, f func()) error { - done := make(chan struct{}, 1) - go func() { - f() - close(done) - }() - select { - case <-ctx.Done(): - return errs.New("timeout, ctx graceful stop") - case <-done: - return nil +func listenTCP(addr string) (net.Listener, int, error) { + listener, err := net.Listen("tcp", addr) + if err != nil { + return nil, 0, errs.WrapMsg(err, "listen err", "addr", addr) } + return listener, listener.Addr().(*net.TCPAddr).Port, nil } func prommetricsUnaryInterceptor(rpcRegisterName string) grpc.ServerOption { @@ -249,3 +229,11 @@ func prommetricsUnaryInterceptor(rpcRegisterName string) grpc.ServerOption { func prommetricsStreamInterceptor(rpcRegisterName string) grpc.ServerOption { return grpc.ChainStreamInterceptor() } + +type grpcServiceRegistrar struct { + onRegisterService func(desc *grpc.ServiceDesc, impl any) +} + +func (x *grpcServiceRegistrar) RegisterService(desc *grpc.ServiceDesc, impl any) { + x.onRegisterService(desc, impl) +} diff --git a/pkg/common/storage/cache/mcache/minio.go b/pkg/common/storage/cache/mcache/minio.go new file mode 100644 index 000000000..f07203cc2 --- /dev/null +++ b/pkg/common/storage/cache/mcache/minio.go @@ -0,0 +1,50 @@ +package mcache + +import ( + "context" + "time" + + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/cachekey" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" + "github.com/openimsdk/tools/s3/minio" +) + +func NewMinioCache(cache database.Cache) minio.Cache { + return &minioCache{ + cache: cache, + expireTime: time.Hour * 24 * 7, + } +} + +type minioCache struct { + cache database.Cache + expireTime time.Duration +} + +func (g *minioCache) getObjectImageInfoKey(key string) string { + return cachekey.GetObjectImageInfoKey(key) +} + +func (g *minioCache) getMinioImageThumbnailKey(key string, format string, width int, height int) string { + return cachekey.GetMinioImageThumbnailKey(key, format, width, height) +} + +func (g *minioCache) DelObjectImageInfoKey(ctx context.Context, keys ...string) error { + ks := make([]string, 0, len(keys)) + for _, key := range keys { + ks = append(ks, g.getObjectImageInfoKey(key)) + } + return g.cache.Del(ctx, ks) +} + +func (g *minioCache) DelImageThumbnailKey(ctx context.Context, key string, format string, width int, height int) error { + return g.cache.Del(ctx, []string{g.getMinioImageThumbnailKey(key, format, width, height)}) +} + +func (g *minioCache) GetImageObjectKeyInfo(ctx context.Context, key string, fn func(ctx context.Context) (*minio.ImageInfo, error)) (*minio.ImageInfo, error) { + return getCache[*minio.ImageInfo](ctx, g.cache, g.getObjectImageInfoKey(key), g.expireTime, fn) +} + +func (g *minioCache) GetThumbnailKey(ctx context.Context, key string, format string, width int, height int, minioCache func(ctx context.Context) (string, error)) (string, error) { + return getCache[string](ctx, g.cache, g.getMinioImageThumbnailKey(key, format, width, height), g.expireTime, minioCache) +} diff --git a/pkg/common/storage/cache/mcache/msg_cache.go b/pkg/common/storage/cache/mcache/msg_cache.go new file mode 100644 index 000000000..3846be3f8 --- /dev/null +++ b/pkg/common/storage/cache/mcache/msg_cache.go @@ -0,0 +1,132 @@ +package mcache + +import ( + "context" + "strconv" + "sync" + "time" + + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/cachekey" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" + "github.com/openimsdk/open-im-server/v3/pkg/localcache" + "github.com/openimsdk/open-im-server/v3/pkg/localcache/lru" + "github.com/openimsdk/tools/errs" + "github.com/openimsdk/tools/utils/datautil" + "github.com/redis/go-redis/v9" +) + +var ( + memMsgCache lru.LRU[string, *model.MsgInfoModel] + initMemMsgCache sync.Once +) + +func NewMsgCache(cache database.Cache, msgDocDatabase database.Msg) cache.MsgCache { + initMemMsgCache.Do(func() { + memMsgCache = lru.NewLayLRU[string, *model.MsgInfoModel](1024*8, time.Hour, time.Second*10, localcache.EmptyTarget{}, nil) + }) + return &msgCache{ + cache: cache, + msgDocDatabase: msgDocDatabase, + memMsgCache: memMsgCache, + } +} + +type msgCache struct { + cache database.Cache + msgDocDatabase database.Msg + memMsgCache lru.LRU[string, *model.MsgInfoModel] +} + +func (x *msgCache) getSendMsgKey(id string) string { + return cachekey.GetSendMsgKey(id) +} + +func (x *msgCache) SetSendMsgStatus(ctx context.Context, id string, status int32) error { + return x.cache.Set(ctx, x.getSendMsgKey(id), strconv.Itoa(int(status)), time.Hour*24) +} + +func (x *msgCache) GetSendMsgStatus(ctx context.Context, id string) (int32, error) { + key := x.getSendMsgKey(id) + res, err := x.cache.Get(ctx, []string{key}) + if err != nil { + return 0, err + } + val, ok := res[key] + if !ok { + return 0, errs.Wrap(redis.Nil) + } + status, err := strconv.Atoi(val) + if err != nil { + return 0, errs.WrapMsg(err, "GetSendMsgStatus strconv.Atoi error", "val", val) + } + return int32(status), nil +} + +func (x *msgCache) getMsgCacheKey(conversationID string, seq int64) string { + return cachekey.GetMsgCacheKey(conversationID, seq) + +} + +func (x *msgCache) GetMessageBySeqs(ctx context.Context, conversationID string, seqs []int64) ([]*model.MsgInfoModel, error) { + if len(seqs) == 0 { + return nil, nil + } + keys := make([]string, 0, len(seqs)) + keySeq := make(map[string]int64, len(seqs)) + for _, seq := range seqs { + key := x.getMsgCacheKey(conversationID, seq) + keys = append(keys, key) + keySeq[key] = seq + } + res, err := x.memMsgCache.GetBatch(keys, func(keys []string) (map[string]*model.MsgInfoModel, error) { + findSeqs := make([]int64, 0, len(keys)) + for _, key := range keys { + seq, ok := keySeq[key] + if !ok { + continue + } + findSeqs = append(findSeqs, seq) + } + res, err := x.msgDocDatabase.FindSeqs(ctx, conversationID, seqs) + if err != nil { + return nil, err + } + kv := make(map[string]*model.MsgInfoModel) + for i := range res { + msg := res[i] + if msg == nil || msg.Msg == nil || msg.Msg.Seq <= 0 { + continue + } + key := x.getMsgCacheKey(conversationID, msg.Msg.Seq) + kv[key] = msg + } + return kv, nil + }) + if err != nil { + return nil, err + } + return datautil.Values(res), nil +} + +func (x msgCache) DelMessageBySeqs(ctx context.Context, conversationID string, seqs []int64) error { + if len(seqs) == 0 { + return nil + } + for _, seq := range seqs { + x.memMsgCache.Del(x.getMsgCacheKey(conversationID, seq)) + } + return nil +} + +func (x *msgCache) SetMessageBySeqs(ctx context.Context, conversationID string, msgs []*model.MsgInfoModel) error { + for i := range msgs { + msg := msgs[i] + if msg == nil || msg.Msg == nil || msg.Msg.Seq <= 0 { + continue + } + x.memMsgCache.Set(x.getMsgCacheKey(conversationID, msg.Msg.Seq), msg) + } + return nil +} diff --git a/pkg/common/storage/cache/mcache/online.go b/pkg/common/storage/cache/mcache/online.go new file mode 100644 index 000000000..f018da03e --- /dev/null +++ b/pkg/common/storage/cache/mcache/online.go @@ -0,0 +1,82 @@ +package mcache + +import ( + "context" + "sync" + + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache" +) + +var ( + globalOnlineCache cache.OnlineCache + globalOnlineOnce sync.Once +) + +func NewOnlineCache() cache.OnlineCache { + globalOnlineOnce.Do(func() { + globalOnlineCache = &onlineCache{ + user: make(map[string]map[int32]struct{}), + } + }) + return globalOnlineCache +} + +type onlineCache struct { + lock sync.RWMutex + user map[string]map[int32]struct{} +} + +func (x *onlineCache) GetOnline(ctx context.Context, userID string) ([]int32, error) { + x.lock.RLock() + defer x.lock.RUnlock() + pSet, ok := x.user[userID] + if !ok { + return nil, nil + } + res := make([]int32, 0, len(pSet)) + for k := range pSet { + res = append(res, k) + } + return res, nil +} + +func (x *onlineCache) SetUserOnline(ctx context.Context, userID string, online, offline []int32) error { + x.lock.Lock() + defer x.lock.Unlock() + pSet, ok := x.user[userID] + if ok { + for _, p := range offline { + delete(pSet, p) + } + } + if len(online) > 0 { + if !ok { + pSet = make(map[int32]struct{}) + x.user[userID] = pSet + } + for _, p := range online { + pSet[p] = struct{}{} + } + } + if len(pSet) == 0 { + delete(x.user, userID) + } + return nil +} + +func (x *onlineCache) GetAllOnlineUsers(ctx context.Context, cursor uint64) (map[string][]int32, uint64, error) { + if cursor != 0 { + return nil, 0, nil + } + x.lock.RLock() + defer x.lock.RUnlock() + res := make(map[string][]int32) + for k, v := range x.user { + pSet := make([]int32, 0, len(v)) + for p := range v { + pSet = append(pSet, p) + } + res[k] = pSet + } + return res, 0, nil +} diff --git a/pkg/common/storage/cache/mcache/seq_conversation.go b/pkg/common/storage/cache/mcache/seq_conversation.go new file mode 100644 index 000000000..27f00de15 --- /dev/null +++ b/pkg/common/storage/cache/mcache/seq_conversation.go @@ -0,0 +1,79 @@ +package mcache + +import ( + "context" + + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" +) + +func NewSeqConversationCache(sc database.SeqConversation) cache.SeqConversationCache { + return &seqConversationCache{ + sc: sc, + } +} + +type seqConversationCache struct { + sc database.SeqConversation +} + +func (x *seqConversationCache) Malloc(ctx context.Context, conversationID string, size int64) (int64, error) { + return x.sc.Malloc(ctx, conversationID, size) +} + +func (x *seqConversationCache) SetMinSeq(ctx context.Context, conversationID string, seq int64) error { + return x.sc.SetMinSeq(ctx, conversationID, seq) +} + +func (x *seqConversationCache) GetMinSeq(ctx context.Context, conversationID string) (int64, error) { + return x.sc.GetMinSeq(ctx, conversationID) +} + +func (x *seqConversationCache) GetMaxSeqs(ctx context.Context, conversationIDs []string) (map[string]int64, error) { + res := make(map[string]int64) + for _, conversationID := range conversationIDs { + seq, err := x.GetMinSeq(ctx, conversationID) + if err != nil { + return nil, err + } + res[conversationID] = seq + } + return res, nil +} + +func (x *seqConversationCache) GetMaxSeqsWithTime(ctx context.Context, conversationIDs []string) (map[string]database.SeqTime, error) { + res := make(map[string]database.SeqTime) + for _, conversationID := range conversationIDs { + seq, err := x.GetMinSeq(ctx, conversationID) + if err != nil { + return nil, err + } + res[conversationID] = database.SeqTime{Seq: seq} + } + return res, nil +} + +func (x *seqConversationCache) GetMaxSeq(ctx context.Context, conversationID string) (int64, error) { + return x.sc.GetMaxSeq(ctx, conversationID) +} + +func (x *seqConversationCache) GetMaxSeqWithTime(ctx context.Context, conversationID string) (database.SeqTime, error) { + seq, err := x.GetMinSeq(ctx, conversationID) + if err != nil { + return database.SeqTime{}, err + } + return database.SeqTime{Seq: seq}, nil +} + +func (x *seqConversationCache) SetMinSeqs(ctx context.Context, seqs map[string]int64) error { + for conversationID, seq := range seqs { + if err := x.sc.SetMinSeq(ctx, conversationID, seq); err != nil { + return err + } + } + return nil +} + +func (x *seqConversationCache) GetCacheMaxSeqWithTime(ctx context.Context, conversationIDs []string) (map[string]database.SeqTime, error) { + return x.GetMaxSeqsWithTime(ctx, conversationIDs) +} diff --git a/pkg/common/storage/cache/mcache/third.go b/pkg/common/storage/cache/mcache/third.go new file mode 100644 index 000000000..6918ae784 --- /dev/null +++ b/pkg/common/storage/cache/mcache/third.go @@ -0,0 +1,98 @@ +package mcache + +import ( + "context" + "strconv" + "time" + + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/cachekey" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" + "github.com/openimsdk/tools/errs" + "github.com/redis/go-redis/v9" +) + +func NewThirdCache(cache database.Cache) cache.ThirdCache { + return &thirdCache{ + cache: cache, + } +} + +type thirdCache struct { + cache database.Cache +} + +func (c *thirdCache) getGetuiTokenKey() string { + return cachekey.GetGetuiTokenKey() +} + +func (c *thirdCache) getGetuiTaskIDKey() string { + return cachekey.GetGetuiTaskIDKey() +} + +func (c *thirdCache) getUserBadgeUnreadCountSumKey(userID string) string { + return cachekey.GetUserBadgeUnreadCountSumKey(userID) +} + +func (c *thirdCache) getFcmAccountTokenKey(account string, platformID int) string { + return cachekey.GetFcmAccountTokenKey(account, platformID) +} + +func (c *thirdCache) get(ctx context.Context, key string) (string, error) { + res, err := c.cache.Get(ctx, []string{key}) + if err != nil { + return "", err + } + if val, ok := res[key]; ok { + return val, nil + } + return "", errs.Wrap(redis.Nil) +} + +func (c *thirdCache) SetFcmToken(ctx context.Context, account string, platformID int, fcmToken string, expireTime int64) (err error) { + return errs.Wrap(c.cache.Set(ctx, c.getFcmAccountTokenKey(account, platformID), fcmToken, time.Duration(expireTime)*time.Second)) +} + +func (c *thirdCache) GetFcmToken(ctx context.Context, account string, platformID int) (string, error) { + return c.get(ctx, c.getFcmAccountTokenKey(account, platformID)) +} + +func (c *thirdCache) DelFcmToken(ctx context.Context, account string, platformID int) error { + return c.cache.Del(ctx, []string{c.getFcmAccountTokenKey(account, platformID)}) +} + +func (c *thirdCache) IncrUserBadgeUnreadCountSum(ctx context.Context, userID string) (int, error) { + return c.cache.Incr(ctx, c.getUserBadgeUnreadCountSumKey(userID), 1) +} + +func (c *thirdCache) SetUserBadgeUnreadCountSum(ctx context.Context, userID string, value int) error { + return c.cache.Set(ctx, c.getUserBadgeUnreadCountSumKey(userID), strconv.Itoa(value), 0) +} + +func (c *thirdCache) GetUserBadgeUnreadCountSum(ctx context.Context, userID string) (int, error) { + str, err := c.get(ctx, c.getUserBadgeUnreadCountSumKey(userID)) + if err != nil { + return 0, err + } + val, err := strconv.Atoi(str) + if err != nil { + return 0, errs.WrapMsg(err, "strconv.Atoi", "str", str) + } + return val, nil +} + +func (c *thirdCache) SetGetuiToken(ctx context.Context, token string, expireTime int64) error { + return c.cache.Set(ctx, c.getGetuiTokenKey(), token, time.Duration(expireTime)*time.Second) +} + +func (c *thirdCache) GetGetuiToken(ctx context.Context) (string, error) { + return c.get(ctx, c.getGetuiTokenKey()) +} + +func (c *thirdCache) SetGetuiTaskID(ctx context.Context, taskID string, expireTime int64) error { + return c.cache.Set(ctx, c.getGetuiTaskIDKey(), taskID, time.Duration(expireTime)*time.Second) +} + +func (c *thirdCache) GetGetuiTaskID(ctx context.Context) (string, error) { + return c.get(ctx, c.getGetuiTaskIDKey()) +} diff --git a/pkg/common/storage/cache/mcache/token.go b/pkg/common/storage/cache/mcache/token.go new file mode 100644 index 000000000..d7ae29cfc --- /dev/null +++ b/pkg/common/storage/cache/mcache/token.go @@ -0,0 +1,130 @@ +package mcache + +import ( + "context" + "fmt" + "strconv" + "strings" + "time" + + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/cachekey" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" + "github.com/openimsdk/tools/errs" + "github.com/openimsdk/tools/log" +) + +func NewTokenCacheModel(cache database.Cache, accessExpire int64) cache.TokenModel { + c := &tokenCache{cache: cache} + c.accessExpire = c.getExpireTime(accessExpire) + return c +} + +type tokenCache struct { + cache database.Cache + accessExpire time.Duration +} + +func (x *tokenCache) getTokenKey(userID string, platformID int, token string) string { + return cachekey.GetTokenKey(userID, platformID) + ":" + token + +} + +func (x *tokenCache) SetTokenFlag(ctx context.Context, userID string, platformID int, token string, flag int) error { + return x.cache.Set(ctx, x.getTokenKey(userID, platformID, token), strconv.Itoa(flag), x.accessExpire) +} + +// SetTokenFlagEx set token and flag with expire time +func (x *tokenCache) SetTokenFlagEx(ctx context.Context, userID string, platformID int, token string, flag int) error { + return x.SetTokenFlag(ctx, userID, platformID, token, flag) +} + +func (x *tokenCache) GetTokensWithoutError(ctx context.Context, userID string, platformID int) (map[string]int, error) { + prefix := x.getTokenKey(userID, platformID, "") + m, err := x.cache.Prefix(ctx, prefix) + if err != nil { + return nil, errs.Wrap(err) + } + mm := make(map[string]int) + for k, v := range m { + state, err := strconv.Atoi(v) + if err != nil { + log.ZError(ctx, "token value is not int", err, "value", v, "userID", userID, "platformID", platformID) + continue + } + mm[strings.TrimPrefix(k, prefix)] = state + } + return mm, nil +} + +func (x *tokenCache) GetAllTokensWithoutError(ctx context.Context, userID string) (map[int]map[string]int, error) { + prefix := cachekey.UidPidToken + userID + ":" + tokens, err := x.cache.Prefix(ctx, prefix) + if err != nil { + return nil, err + } + res := make(map[int]map[string]int) + for key, flagStr := range tokens { + flag, err := strconv.Atoi(flagStr) + if err != nil { + log.ZError(ctx, "token value is not int", err, "key", key, "value", flagStr, "userID", userID) + continue + } + arr := strings.SplitN(strings.TrimPrefix(key, prefix), ":", 2) + if len(arr) != 2 { + log.ZError(ctx, "token value is not int", err, "key", key, "value", flagStr, "userID", userID) + continue + } + platformID, err := strconv.Atoi(arr[0]) + if err != nil { + log.ZError(ctx, "token value is not int", err, "key", key, "value", flagStr, "userID", userID) + continue + } + token := arr[1] + if token == "" { + log.ZError(ctx, "token value is not int", err, "key", key, "value", flagStr, "userID", userID) + continue + } + tk, ok := res[platformID] + if !ok { + tk = make(map[string]int) + res[platformID] = tk + } + tk[token] = flag + } + return res, nil +} + +func (x *tokenCache) SetTokenMapByUidPid(ctx context.Context, userID string, platformID int, m map[string]int) error { + for token, flag := range m { + err := x.SetTokenFlag(ctx, userID, platformID, token, flag) + if err != nil { + return err + } + } + return nil +} + +func (x *tokenCache) BatchSetTokenMapByUidPid(ctx context.Context, tokens map[string]map[string]any) error { + for prefix, tokenFlag := range tokens { + for token, flag := range tokenFlag { + flagStr := fmt.Sprintf("%v", flag) + if err := x.cache.Set(ctx, prefix+":"+token, flagStr, x.accessExpire); err != nil { + return err + } + } + } + return nil +} + +func (x *tokenCache) DeleteTokenByUidPid(ctx context.Context, userID string, platformID int, fields []string) error { + keys := make([]string, 0, len(fields)) + for _, token := range fields { + keys = append(keys, x.getTokenKey(userID, platformID, token)) + } + return x.cache.Del(ctx, keys) +} + +func (x *tokenCache) getExpireTime(t int64) time.Duration { + return time.Hour * 24 * time.Duration(t) +} diff --git a/pkg/common/storage/cache/mcache/tools.go b/pkg/common/storage/cache/mcache/tools.go new file mode 100644 index 000000000..f3c4265cd --- /dev/null +++ b/pkg/common/storage/cache/mcache/tools.go @@ -0,0 +1,63 @@ +package mcache + +import ( + "context" + "encoding/json" + "time" + + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" + "github.com/openimsdk/tools/log" +) + +func getCache[V any](ctx context.Context, cache database.Cache, key string, expireTime time.Duration, fn func(ctx context.Context) (V, error)) (V, error) { + getDB := func() (V, bool, error) { + res, err := cache.Get(ctx, []string{key}) + if err != nil { + var val V + return val, false, err + } + var val V + if str, ok := res[key]; ok { + if json.Unmarshal([]byte(str), &val) != nil { + return val, false, err + } + return val, true, nil + } + return val, false, nil + } + dbVal, ok, err := getDB() + if err != nil { + return dbVal, err + } + if ok { + return dbVal, nil + } + lockValue, err := cache.Lock(ctx, key, time.Minute) + if err != nil { + return dbVal, err + } + defer func() { + if err := cache.Unlock(ctx, key, lockValue); err != nil { + log.ZError(ctx, "unlock cache key", err, "key", key, "value", lockValue) + } + }() + dbVal, ok, err = getDB() + if err != nil { + return dbVal, err + } + if ok { + return dbVal, nil + } + val, err := fn(ctx) + if err != nil { + return val, err + } + data, err := json.Marshal(val) + if err != nil { + return val, err + } + if err := cache.Set(ctx, key, string(data), expireTime); err != nil { + return val, err + } + return val, nil +} diff --git a/pkg/common/storage/cache/redis/batch.go b/pkg/common/storage/cache/redis/batch.go index 1810ac993..7f9205bab 100644 --- a/pkg/common/storage/cache/redis/batch.go +++ b/pkg/common/storage/cache/redis/batch.go @@ -3,28 +3,65 @@ package redis import ( "context" "encoding/json" + "time" + "github.com/dtm-labs/rockscache" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache" "github.com/openimsdk/tools/errs" "github.com/openimsdk/tools/log" "github.com/redis/go-redis/v9" - "golang.org/x/sync/singleflight" - "time" - "unsafe" ) -func getRocksCacheRedisClient(cli *rockscache.Client) redis.UniversalClient { - type Client struct { - rdb redis.UniversalClient - _ rockscache.Options - _ singleflight.Group +// GetRocksCacheOptions returns the default configuration options for RocksCache. +func GetRocksCacheOptions() *rockscache.Options { + opts := rockscache.NewDefaultOptions() + opts.LockExpire = rocksCacheTimeout + opts.WaitReplicasTimeout = rocksCacheTimeout + opts.StrongConsistency = true + opts.RandomExpireAdjustment = 0.2 + + return &opts +} + +func newRocksCacheClient(rdb redis.UniversalClient) *rocksCacheClient { + if rdb == nil { + return &rocksCacheClient{} + } + rc := &rocksCacheClient{ + rdb: rdb, + client: rockscache.NewClient(rdb, *GetRocksCacheOptions()), } - return (*Client)(unsafe.Pointer(cli)).rdb + return rc +} + +type rocksCacheClient struct { + rdb redis.UniversalClient + client *rockscache.Client } -func batchGetCache2[K comparable, V any](ctx context.Context, rcClient *rockscache.Client, expire time.Duration, ids []K, idKey func(id K) string, vId func(v *V) K, fn func(ctx context.Context, ids []K) ([]*V, error)) ([]*V, error) { +func (x *rocksCacheClient) GetClient() *rockscache.Client { + return x.client +} + +func (x *rocksCacheClient) Disable() bool { + return x.client == nil +} + +func (x *rocksCacheClient) GetRedis() redis.UniversalClient { + return x.rdb +} + +func (x *rocksCacheClient) GetBatchDeleter(topics ...string) cache.BatchDeleter { + return NewBatchDeleterRedis(x, topics) +} + +func batchGetCache2[K comparable, V any](ctx context.Context, rcClient *rocksCacheClient, expire time.Duration, ids []K, idKey func(id K) string, vId func(v *V) K, fn func(ctx context.Context, ids []K) ([]*V, error)) ([]*V, error) { if len(ids) == 0 { return nil, nil } + if rcClient.Disable() { + return fn(ctx, ids) + } findKeys := make([]string, 0, len(ids)) keyId := make(map[string]K) for _, id := range ids { @@ -35,13 +72,13 @@ func batchGetCache2[K comparable, V any](ctx context.Context, rcClient *rockscac keyId[key] = id findKeys = append(findKeys, key) } - slotKeys, err := groupKeysBySlot(ctx, getRocksCacheRedisClient(rcClient), findKeys) + slotKeys, err := groupKeysBySlot(ctx, rcClient.GetRedis(), findKeys) if err != nil { return nil, err } result := make([]*V, 0, len(findKeys)) for _, keys := range slotKeys { - indexCache, err := rcClient.FetchBatch2(ctx, keys, expire, func(idx []int) (map[int]string, error) { + indexCache, err := rcClient.GetClient().FetchBatch2(ctx, keys, expire, func(idx []int) (map[int]string, error) { queryIds := make([]K, 0, len(idx)) idIndex := make(map[K]int) for _, index := range idx { diff --git a/pkg/common/storage/cache/redis/batch_handler.go b/pkg/common/storage/cache/redis/batch_handler.go index 420ebdf77..893ba8abb 100644 --- a/pkg/common/storage/cache/redis/batch_handler.go +++ b/pkg/common/storage/cache/redis/batch_handler.go @@ -1,23 +1,11 @@ -// Copyright © 2023 OpenIM. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - package redis import ( "context" "encoding/json" "fmt" + "time" + "github.com/dtm-labs/rockscache" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache" "github.com/openimsdk/open-im-server/v3/pkg/localcache" @@ -25,7 +13,6 @@ import ( "github.com/openimsdk/tools/log" "github.com/openimsdk/tools/utils/datautil" "github.com/redis/go-redis/v9" - "time" ) const ( @@ -41,10 +28,10 @@ type BatchDeleterRedis struct { } // NewBatchDeleterRedis creates a new BatchDeleterRedis instance. -func NewBatchDeleterRedis(redisClient redis.UniversalClient, options *rockscache.Options, redisPubTopics []string) *BatchDeleterRedis { +func NewBatchDeleterRedis(rcClient *rocksCacheClient, redisPubTopics []string) *BatchDeleterRedis { return &BatchDeleterRedis{ - redisClient: redisClient, - rocksClient: rockscache.NewClient(redisClient, *options), + redisClient: rcClient.GetRedis(), + rocksClient: rcClient.GetClient(), redisPubTopics: redisPubTopics, } } @@ -107,21 +94,29 @@ func (c *BatchDeleterRedis) AddKeys(keys ...string) { c.keys = append(c.keys, keys...) } -// GetRocksCacheOptions returns the default configuration options for RocksCache. -func GetRocksCacheOptions() *rockscache.Options { - opts := rockscache.NewDefaultOptions() - opts.LockExpire = rocksCacheTimeout - opts.WaitReplicasTimeout = rocksCacheTimeout - opts.StrongConsistency = true - opts.RandomExpireAdjustment = 0.2 +type disableBatchDeleter struct{} - return &opts +func (x disableBatchDeleter) ChainExecDel(ctx context.Context) error { + return nil } -func getCache[T any](ctx context.Context, rcClient *rockscache.Client, key string, expire time.Duration, fn func(ctx context.Context) (T, error)) (T, error) { +func (x disableBatchDeleter) ExecDelWithKeys(ctx context.Context, keys []string) error { + return nil +} + +func (x disableBatchDeleter) Clone() cache.BatchDeleter { + return x +} + +func (x disableBatchDeleter) AddKeys(keys ...string) {} + +func getCache[T any](ctx context.Context, rcClient *rocksCacheClient, key string, expire time.Duration, fn func(ctx context.Context) (T, error)) (T, error) { + if rcClient.Disable() { + return fn(ctx) + } var t T var write bool - v, err := rcClient.Fetch2(ctx, key, expire, func() (s string, err error) { + v, err := rcClient.GetClient().Fetch2(ctx, key, expire, func() (s string, err error) { t, err = fn(ctx) if err != nil { //log.ZError(ctx, "getCache query database failed", err, "key", key) @@ -152,31 +147,3 @@ func getCache[T any](ctx context.Context, rcClient *rockscache.Client, key strin return t, nil } - -//func batchGetCache[T any, K comparable]( -// ctx context.Context, -// rcClient *rockscache.Client, -// expire time.Duration, -// keys []K, -// keyFn func(key K) string, -// fns func(ctx context.Context, key K) (T, error), -//) ([]T, error) { -// if len(keys) == 0 { -// return nil, nil -// } -// res := make([]T, 0, len(keys)) -// for _, key := range keys { -// val, err := getCache(ctx, rcClient, keyFn(key), expire, func(ctx context.Context) (T, error) { -// return fns(ctx, key) -// }) -// if err != nil { -// if errs.ErrRecordNotFound.Is(specialerror.ErrCode(errs.Unwrap(err))) { -// continue -// } -// return nil, errs.Wrap(err) -// } -// res = append(res, val) -// } -// -// return res, nil -//} diff --git a/pkg/common/storage/cache/redis/black.go b/pkg/common/storage/cache/redis/black.go index fac6dbe6f..f83399ee1 100644 --- a/pkg/common/storage/cache/redis/black.go +++ b/pkg/common/storage/cache/redis/black.go @@ -1,29 +1,14 @@ -// Copyright © 2023 OpenIM. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - package redis import ( "context" - "github.com/dtm-labs/rockscache" + "time" + "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/cachekey" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" - "github.com/openimsdk/tools/log" "github.com/redis/go-redis/v9" - "time" ) const ( @@ -33,18 +18,16 @@ const ( type BlackCacheRedis struct { cache.BatchDeleter expireTime time.Duration - rcClient *rockscache.Client + rcClient *rocksCacheClient blackDB database.Black } -func NewBlackCacheRedis(rdb redis.UniversalClient, localCache *config.LocalCache, blackDB database.Black, options *rockscache.Options) cache.BlackCache { - batchHandler := NewBatchDeleterRedis(rdb, options, []string{localCache.Friend.Topic}) - b := localCache.Friend - log.ZDebug(context.Background(), "black local cache init", "Topic", b.Topic, "SlotNum", b.SlotNum, "SlotSize", b.SlotSize, "enable", b.Enable()) +func NewBlackCacheRedis(rdb redis.UniversalClient, localCache *config.LocalCache, blackDB database.Black) cache.BlackCache { + rc := newRocksCacheClient(rdb) return &BlackCacheRedis{ - BatchDeleter: batchHandler, + BatchDeleter: rc.GetBatchDeleter(localCache.Friend.Topic), expireTime: blackExpireTime, - rcClient: rockscache.NewClient(rdb, *options), + rcClient: rc, blackDB: blackDB, } } diff --git a/pkg/common/storage/cache/redis/conversation.go b/pkg/common/storage/cache/redis/conversation.go index 91d8ed69d..3453b570d 100644 --- a/pkg/common/storage/cache/redis/conversation.go +++ b/pkg/common/storage/cache/redis/conversation.go @@ -1,47 +1,30 @@ -// Copyright © 2023 OpenIM. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - package redis import ( "context" - "github.com/dtm-labs/rockscache" + "math/big" + "strings" + "time" + "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/cachekey" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" - "github.com/openimsdk/tools/log" "github.com/openimsdk/tools/utils/datautil" "github.com/openimsdk/tools/utils/encrypt" "github.com/redis/go-redis/v9" - "math/big" - "strings" - "time" ) const ( conversationExpireTime = time.Second * 60 * 60 * 12 ) -func NewConversationRedis(rdb redis.UniversalClient, localCache *config.LocalCache, opts *rockscache.Options, db database.Conversation) cache.ConversationCache { - batchHandler := NewBatchDeleterRedis(rdb, opts, []string{localCache.Conversation.Topic}) - c := localCache.Conversation - log.ZDebug(context.Background(), "conversation local cache init", "Topic", c.Topic, "SlotNum", c.SlotNum, "SlotSize", c.SlotSize, "enable", c.Enable()) +func NewConversationRedis(rdb redis.UniversalClient, localCache *config.LocalCache, db database.Conversation) cache.ConversationCache { + rc := newRocksCacheClient(rdb) return &ConversationRedisCache{ - BatchDeleter: batchHandler, - rcClient: rockscache.NewClient(rdb, *opts), + BatchDeleter: rc.GetBatchDeleter(localCache.Conversation.Topic), + rcClient: rc, conversationDB: db, expireTime: conversationExpireTime, } @@ -49,7 +32,7 @@ func NewConversationRedis(rdb redis.UniversalClient, localCache *config.LocalCac type ConversationRedisCache struct { cache.BatchDeleter - rcClient *rockscache.Client + rcClient *rocksCacheClient conversationDB database.Conversation expireTime time.Duration } diff --git a/pkg/common/storage/cache/redis/doc.go b/pkg/common/storage/cache/redis/doc.go deleted file mode 100644 index 4c2fcacd1..000000000 --- a/pkg/common/storage/cache/redis/doc.go +++ /dev/null @@ -1,15 +0,0 @@ -// Copyright © 2023 OpenIM. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package redis diff --git a/pkg/common/storage/cache/redis/friend.go b/pkg/common/storage/cache/redis/friend.go index be4687794..7618f361f 100644 --- a/pkg/common/storage/cache/redis/friend.go +++ b/pkg/common/storage/cache/redis/friend.go @@ -1,30 +1,14 @@ -// Copyright © 2023 OpenIM. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - package redis import ( "context" "time" - "github.com/dtm-labs/rockscache" "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/cachekey" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" - "github.com/openimsdk/tools/log" "github.com/openimsdk/tools/utils/datautil" "github.com/redis/go-redis/v9" ) @@ -38,21 +22,18 @@ type FriendCacheRedis struct { cache.BatchDeleter friendDB database.Friend expireTime time.Duration - rcClient *rockscache.Client + rcClient *rocksCacheClient syncCount int } // NewFriendCacheRedis creates a new instance of FriendCacheRedis. -func NewFriendCacheRedis(rdb redis.UniversalClient, localCache *config.LocalCache, friendDB database.Friend, - options *rockscache.Options) cache.FriendCache { - batchHandler := NewBatchDeleterRedis(rdb, options, []string{localCache.Friend.Topic}) - f := localCache.Friend - log.ZDebug(context.Background(), "friend local cache init", "Topic", f.Topic, "SlotNum", f.SlotNum, "SlotSize", f.SlotSize, "enable", f.Enable()) +func NewFriendCacheRedis(rdb redis.UniversalClient, localCache *config.LocalCache, friendDB database.Friend) cache.FriendCache { + rc := newRocksCacheClient(rdb) return &FriendCacheRedis{ - BatchDeleter: batchHandler, + BatchDeleter: rc.GetBatchDeleter(localCache.Friend.Topic), friendDB: friendDB, expireTime: friendExpireTime, - rcClient: rockscache.NewClient(rdb, *options), + rcClient: rc, } } diff --git a/pkg/common/storage/cache/redis/group.go b/pkg/common/storage/cache/redis/group.go index 736111df3..d66716404 100644 --- a/pkg/common/storage/cache/redis/group.go +++ b/pkg/common/storage/cache/redis/group.go @@ -1,17 +1,3 @@ -// Copyright © 2023 OpenIM. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - package redis import ( @@ -19,7 +5,6 @@ import ( "fmt" "time" - "github.com/dtm-labs/rockscache" "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/cachekey" @@ -36,34 +21,21 @@ const ( groupExpireTime = time.Second * 60 * 60 * 12 ) -var errIndex = errs.New("err index") - type GroupCacheRedis struct { cache.BatchDeleter groupDB database.Group groupMemberDB database.GroupMember groupRequestDB database.GroupRequest expireTime time.Duration - rcClient *rockscache.Client + rcClient *rocksCacheClient groupHash cache.GroupHash } -func NewGroupCacheRedis( - rdb redis.UniversalClient, - localCache *config.LocalCache, - groupDB database.Group, - groupMemberDB database.GroupMember, - groupRequestDB database.GroupRequest, - hashCode cache.GroupHash, - opts *rockscache.Options, -) cache.GroupCache { - batchHandler := NewBatchDeleterRedis(rdb, opts, []string{localCache.Group.Topic}) - g := localCache.Group - log.ZDebug(context.Background(), "group local cache init", "Topic", g.Topic, "SlotNum", g.SlotNum, "SlotSize", g.SlotSize, "enable", g.Enable()) - +func NewGroupCacheRedis(rdb redis.UniversalClient, localCache *config.LocalCache, groupDB database.Group, groupMemberDB database.GroupMember, groupRequestDB database.GroupRequest, hashCode cache.GroupHash) cache.GroupCache { + rc := newRocksCacheClient(rdb) return &GroupCacheRedis{ - BatchDeleter: batchHandler, - rcClient: rockscache.NewClient(rdb, *opts), + BatchDeleter: rc.GetBatchDeleter(localCache.Group.Topic), + rcClient: rc, expireTime: groupExpireTime, groupDB: groupDB, groupMemberDB: groupMemberDB, diff --git a/pkg/common/storage/cache/redis/minio.go b/pkg/common/storage/cache/redis/minio.go new file mode 100644 index 000000000..17bd6ec03 --- /dev/null +++ b/pkg/common/storage/cache/redis/minio.go @@ -0,0 +1,59 @@ +package redis + +import ( + "context" + "time" + + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/cachekey" + "github.com/openimsdk/tools/s3/minio" + "github.com/redis/go-redis/v9" +) + +func NewMinioCache(rdb redis.UniversalClient) minio.Cache { + rc := newRocksCacheClient(rdb) + return &minioCacheRedis{ + BatchDeleter: rc.GetBatchDeleter(), + rcClient: rc, + expireTime: time.Hour * 24 * 7, + } +} + +type minioCacheRedis struct { + cache.BatchDeleter + rcClient *rocksCacheClient + expireTime time.Duration +} + +func (g *minioCacheRedis) getObjectImageInfoKey(key string) string { + return cachekey.GetObjectImageInfoKey(key) +} + +func (g *minioCacheRedis) getMinioImageThumbnailKey(key string, format string, width int, height int) string { + return cachekey.GetMinioImageThumbnailKey(key, format, width, height) +} + +func (g *minioCacheRedis) DelObjectImageInfoKey(ctx context.Context, keys ...string) error { + ks := make([]string, 0, len(keys)) + for _, key := range keys { + ks = append(ks, g.getObjectImageInfoKey(key)) + } + return g.BatchDeleter.ExecDelWithKeys(ctx, ks) +} + +func (g *minioCacheRedis) DelImageThumbnailKey(ctx context.Context, key string, format string, width int, height int) error { + return g.BatchDeleter.ExecDelWithKeys(ctx, []string{g.getMinioImageThumbnailKey(key, format, width, height)}) + +} + +func (g *minioCacheRedis) GetImageObjectKeyInfo(ctx context.Context, key string, fn func(ctx context.Context) (*minio.ImageInfo, error)) (*minio.ImageInfo, error) { + info, err := getCache(ctx, g.rcClient, g.getObjectImageInfoKey(key), g.expireTime, fn) + if err != nil { + return nil, err + } + return info, nil +} + +func (g *minioCacheRedis) GetThumbnailKey(ctx context.Context, key string, format string, width int, height int, minioCache func(ctx context.Context) (string, error)) (string, error) { + return getCache(ctx, g.rcClient, g.getMinioImageThumbnailKey(key, format, width, height), g.expireTime, minioCache) +} diff --git a/pkg/common/storage/cache/redis/msg.go b/pkg/common/storage/cache/redis/msg.go index 0651f0283..dfe6ca04d 100644 --- a/pkg/common/storage/cache/redis/msg.go +++ b/pkg/common/storage/cache/redis/msg.go @@ -3,7 +3,8 @@ package redis import ( "context" "encoding/json" - "github.com/dtm-labs/rockscache" + "time" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/cachekey" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" @@ -11,7 +12,6 @@ import ( "github.com/openimsdk/tools/errs" "github.com/openimsdk/tools/utils/datautil" "github.com/redis/go-redis/v9" - "time" ) // // msgCacheTimeout is expiration time of message cache, 86400 seconds @@ -19,15 +19,13 @@ const msgCacheTimeout = time.Hour * 24 func NewMsgCache(client redis.UniversalClient, db database.Msg) cache.MsgCache { return &msgCache{ - rdb: client, - rcClient: rockscache.NewClient(client, *GetRocksCacheOptions()), + rcClient: newRocksCacheClient(client), msgDocDatabase: db, } } type msgCache struct { - rdb redis.UniversalClient - rcClient *rockscache.Client + rcClient *rocksCacheClient msgDocDatabase database.Msg } @@ -36,11 +34,11 @@ func (c *msgCache) getSendMsgKey(id string) string { } func (c *msgCache) SetSendMsgStatus(ctx context.Context, id string, status int32) error { - return errs.Wrap(c.rdb.Set(ctx, c.getSendMsgKey(id), status, time.Hour*24).Err()) + return errs.Wrap(c.rcClient.GetRedis().Set(ctx, c.getSendMsgKey(id), status, time.Hour*24).Err()) } func (c *msgCache) GetSendMsgStatus(ctx context.Context, id string) (int32, error) { - result, err := c.rdb.Get(ctx, c.getSendMsgKey(id)).Int() + result, err := c.rcClient.GetRedis().Get(ctx, c.getSendMsgKey(id)).Int() return int32(result), errs.Wrap(err) } @@ -67,12 +65,12 @@ func (c *msgCache) DelMessageBySeqs(ctx context.Context, conversationID string, keys := datautil.Slice(seqs, func(seq int64) string { return cachekey.GetMsgCacheKey(conversationID, seq) }) - slotKeys, err := groupKeysBySlot(ctx, getRocksCacheRedisClient(c.rcClient), keys) + slotKeys, err := groupKeysBySlot(ctx, c.rcClient.GetRedis(), keys) if err != nil { return err } for _, keys := range slotKeys { - if err := c.rcClient.TagAsDeletedBatch2(ctx, keys); err != nil { + if err := c.rcClient.GetClient().TagAsDeletedBatch2(ctx, keys); err != nil { return err } } @@ -88,7 +86,7 @@ func (c *msgCache) SetMessageBySeqs(ctx context.Context, conversationID string, if err != nil { return err } - if err := c.rcClient.RawSet(ctx, cachekey.GetMsgCacheKey(conversationID, msg.Msg.Seq), string(data), msgCacheTimeout); err != nil { + if err := c.rcClient.GetClient().RawSet(ctx, cachekey.GetMsgCacheKey(conversationID, msg.Msg.Seq), string(data), msgCacheTimeout); err != nil { return err } } diff --git a/pkg/common/storage/cache/redis/online.go b/pkg/common/storage/cache/redis/online.go index b6c90264e..c11473695 100644 --- a/pkg/common/storage/cache/redis/online.go +++ b/pkg/common/storage/cache/redis/online.go @@ -3,18 +3,24 @@ package redis import ( "context" "fmt" + "strconv" + "strings" + "time" + + "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/cachekey" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/mcache" "github.com/openimsdk/protocol/constant" "github.com/openimsdk/tools/errs" "github.com/openimsdk/tools/log" "github.com/redis/go-redis/v9" - "strconv" - "strings" - "time" ) func NewUserOnline(rdb redis.UniversalClient) cache.OnlineCache { + if rdb == nil || config.Standalone() { + return mcache.NewOnlineCache() + } return &userOnline{ rdb: rdb, expire: cachekey.OnlineExpire, diff --git a/pkg/common/storage/cache/redis/redis_shard_manager.go b/pkg/common/storage/cache/redis/redis_shard_manager.go index 17e5fecf6..0a0263892 100644 --- a/pkg/common/storage/cache/redis/redis_shard_manager.go +++ b/pkg/common/storage/cache/redis/redis_shard_manager.go @@ -2,7 +2,7 @@ package redis import ( "context" - "github.com/dtm-labs/rockscache" + "github.com/openimsdk/tools/errs" "github.com/openimsdk/tools/log" "github.com/redis/go-redis/v9" @@ -28,83 +28,83 @@ type Config struct { // Option is a function type for configuring Config type Option func(c *Config) -// NewRedisShardManager creates a new RedisShardManager instance -func NewRedisShardManager(redisClient redis.UniversalClient, opts ...Option) *RedisShardManager { - config := &Config{ - batchSize: defaultBatchSize, // Default batch size is 50 keys - continueOnError: false, - concurrentLimit: defaultConcurrentLimit, // Default concurrent limit is 3 - } - for _, opt := range opts { - opt(config) - } - rsm := &RedisShardManager{ - redisClient: redisClient, - config: config, - } - return rsm -} - -// WithBatchSize sets the number of keys to process per batch -func WithBatchSize(size int) Option { - return func(c *Config) { - c.batchSize = size - } -} - -// WithContinueOnError sets whether to continue processing on error -func WithContinueOnError(continueOnError bool) Option { - return func(c *Config) { - c.continueOnError = continueOnError - } -} - -// WithConcurrentLimit sets the concurrency limit -func WithConcurrentLimit(limit int) Option { - return func(c *Config) { - c.concurrentLimit = limit - } -} - -// ProcessKeysBySlot groups keys by their Redis cluster hash slots and processes them using the provided function. -func (rsm *RedisShardManager) ProcessKeysBySlot( - ctx context.Context, - keys []string, - processFunc func(ctx context.Context, slot int64, keys []string) error, -) error { - - // Group keys by slot - slots, err := groupKeysBySlot(ctx, rsm.redisClient, keys) - if err != nil { - return err - } - - g, ctx := errgroup.WithContext(ctx) - g.SetLimit(rsm.config.concurrentLimit) - - // Process keys in each slot using the provided function - for slot, singleSlotKeys := range slots { - batches := splitIntoBatches(singleSlotKeys, rsm.config.batchSize) - for _, batch := range batches { - slot, batch := slot, batch // Avoid closure capture issue - g.Go(func() error { - err := processFunc(ctx, slot, batch) - if err != nil { - log.ZWarn(ctx, "Batch processFunc failed", err, "slot", slot, "keys", batch) - if !rsm.config.continueOnError { - return err - } - } - return nil - }) - } - } - - if err := g.Wait(); err != nil { - return err - } - return nil -} +//// NewRedisShardManager creates a new RedisShardManager instance +//func NewRedisShardManager(redisClient redis.UniversalClient, opts ...Option) *RedisShardManager { +// config := &Config{ +// batchSize: defaultBatchSize, // Default batch size is 50 keys +// continueOnError: false, +// concurrentLimit: defaultConcurrentLimit, // Default concurrent limit is 3 +// } +// for _, opt := range opts { +// opt(config) +// } +// rsm := &RedisShardManager{ +// redisClient: redisClient, +// config: config, +// } +// return rsm +//} +// +//// WithBatchSize sets the number of keys to process per batch +//func WithBatchSize(size int) Option { +// return func(c *Config) { +// c.batchSize = size +// } +//} +// +//// WithContinueOnError sets whether to continue processing on error +//func WithContinueOnError(continueOnError bool) Option { +// return func(c *Config) { +// c.continueOnError = continueOnError +// } +//} +// +//// WithConcurrentLimit sets the concurrency limit +//func WithConcurrentLimit(limit int) Option { +// return func(c *Config) { +// c.concurrentLimit = limit +// } +//} +// +//// ProcessKeysBySlot groups keys by their Redis cluster hash slots and processes them using the provided function. +//func (rsm *RedisShardManager) ProcessKeysBySlot( +// ctx context.Context, +// keys []string, +// processFunc func(ctx context.Context, slot int64, keys []string) error, +//) error { +// +// // Group keys by slot +// slots, err := groupKeysBySlot(ctx, rsm.redisClient, keys) +// if err != nil { +// return err +// } +// +// g, ctx := errgroup.WithContext(ctx) +// g.SetLimit(rsm.config.concurrentLimit) +// +// // Process keys in each slot using the provided function +// for slot, singleSlotKeys := range slots { +// batches := splitIntoBatches(singleSlotKeys, rsm.config.batchSize) +// for _, batch := range batches { +// slot, batch := slot, batch // Avoid closure capture issue +// g.Go(func() error { +// err := processFunc(ctx, slot, batch) +// if err != nil { +// log.ZWarn(ctx, "Batch processFunc failed", err, "slot", slot, "keys", batch) +// if !rsm.config.continueOnError { +// return err +// } +// } +// return nil +// }) +// } +// } +// +// if err := g.Wait(); err != nil { +// return err +// } +// return nil +//} // groupKeysBySlot groups keys by their Redis cluster hash slots. func groupKeysBySlot(ctx context.Context, redisClient redis.UniversalClient, keys []string) (map[int64][]string, error) { @@ -197,15 +197,15 @@ func ProcessKeysBySlot( return nil } -func DeleteCacheBySlot(ctx context.Context, rcClient *rockscache.Client, keys []string) error { +func DeleteCacheBySlot(ctx context.Context, rcClient *rocksCacheClient, keys []string) error { switch len(keys) { case 0: return nil case 1: - return rcClient.TagAsDeletedBatch2(ctx, keys) + return rcClient.GetClient().TagAsDeletedBatch2(ctx, keys) default: - return ProcessKeysBySlot(ctx, getRocksCacheRedisClient(rcClient), keys, func(ctx context.Context, slot int64, keys []string) error { - return rcClient.TagAsDeletedBatch2(ctx, keys) + return ProcessKeysBySlot(ctx, rcClient.GetRedis(), keys, func(ctx context.Context, slot int64, keys []string) error { + return rcClient.GetClient().TagAsDeletedBatch2(ctx, keys) }) } } diff --git a/pkg/common/storage/cache/redis/s3.go b/pkg/common/storage/cache/redis/s3.go index 954557aca..37f80a477 100644 --- a/pkg/common/storage/cache/redis/s3.go +++ b/pkg/common/storage/cache/redis/s3.go @@ -1,39 +1,23 @@ -// Copyright © 2023 OpenIM. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - package redis import ( "context" - "github.com/dtm-labs/rockscache" + "time" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/cachekey" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" "github.com/openimsdk/tools/s3" "github.com/openimsdk/tools/s3/cont" - "github.com/openimsdk/tools/s3/minio" "github.com/redis/go-redis/v9" - "time" ) func NewObjectCacheRedis(rdb redis.UniversalClient, objDB database.ObjectInfo) cache.ObjectCache { - opts := rockscache.NewDefaultOptions() - batchHandler := NewBatchDeleterRedis(rdb, &opts, nil) + rc := newRocksCacheClient(rdb) return &objectCacheRedis{ - BatchDeleter: batchHandler, - rcClient: rockscache.NewClient(rdb, opts), + BatchDeleter: rc.GetBatchDeleter(), + rcClient: rc, expireTime: time.Hour * 12, objDB: objDB, } @@ -42,7 +26,7 @@ func NewObjectCacheRedis(rdb redis.UniversalClient, objDB database.ObjectInfo) c type objectCacheRedis struct { cache.BatchDeleter objDB database.ObjectInfo - rcClient *rockscache.Client + rcClient *rocksCacheClient expireTime time.Duration } @@ -76,11 +60,10 @@ func (g *objectCacheRedis) GetName(ctx context.Context, engine string, name stri } func NewS3Cache(rdb redis.UniversalClient, s3 s3.Interface) cont.S3Cache { - opts := rockscache.NewDefaultOptions() - batchHandler := NewBatchDeleterRedis(rdb, &opts, nil) + rc := newRocksCacheClient(rdb) return &s3CacheRedis{ - BatchDeleter: batchHandler, - rcClient: rockscache.NewClient(rdb, opts), + BatchDeleter: rc.GetBatchDeleter(), + rcClient: rc, expireTime: time.Hour * 12, s3: s3, } @@ -89,7 +72,7 @@ func NewS3Cache(rdb redis.UniversalClient, s3 s3.Interface) cont.S3Cache { type s3CacheRedis struct { cache.BatchDeleter s3 s3.Interface - rcClient *rockscache.Client + rcClient *rocksCacheClient expireTime time.Duration } @@ -110,52 +93,3 @@ func (g *s3CacheRedis) GetKey(ctx context.Context, engine string, name string) ( return g.s3.StatObject(ctx, name) }) } - -func NewMinioCache(rdb redis.UniversalClient) minio.Cache { - opts := rockscache.NewDefaultOptions() - batchHandler := NewBatchDeleterRedis(rdb, &opts, nil) - return &minioCacheRedis{ - BatchDeleter: batchHandler, - rcClient: rockscache.NewClient(rdb, opts), - expireTime: time.Hour * 24 * 7, - } -} - -type minioCacheRedis struct { - cache.BatchDeleter - rcClient *rockscache.Client - expireTime time.Duration -} - -func (g *minioCacheRedis) getObjectImageInfoKey(key string) string { - return cachekey.GetObjectImageInfoKey(key) -} - -func (g *minioCacheRedis) getMinioImageThumbnailKey(key string, format string, width int, height int) string { - return cachekey.GetMinioImageThumbnailKey(key, format, width, height) -} - -func (g *minioCacheRedis) DelObjectImageInfoKey(ctx context.Context, keys ...string) error { - ks := make([]string, 0, len(keys)) - for _, key := range keys { - ks = append(ks, g.getObjectImageInfoKey(key)) - } - return g.BatchDeleter.ExecDelWithKeys(ctx, ks) -} - -func (g *minioCacheRedis) DelImageThumbnailKey(ctx context.Context, key string, format string, width int, height int) error { - return g.BatchDeleter.ExecDelWithKeys(ctx, []string{g.getMinioImageThumbnailKey(key, format, width, height)}) - -} - -func (g *minioCacheRedis) GetImageObjectKeyInfo(ctx context.Context, key string, fn func(ctx context.Context) (*minio.ImageInfo, error)) (*minio.ImageInfo, error) { - info, err := getCache(ctx, g.rcClient, g.getObjectImageInfoKey(key), g.expireTime, fn) - if err != nil { - return nil, err - } - return info, nil -} - -func (g *minioCacheRedis) GetThumbnailKey(ctx context.Context, key string, format string, width int, height int, minioCache func(ctx context.Context) (string, error)) (string, error) { - return getCache(ctx, g.rcClient, g.getMinioImageThumbnailKey(key, format, width, height), g.expireTime, minioCache) -} diff --git a/pkg/common/storage/cache/redis/seq_conversation.go b/pkg/common/storage/cache/redis/seq_conversation.go index 71705cef7..604826598 100644 --- a/pkg/common/storage/cache/redis/seq_conversation.go +++ b/pkg/common/storage/cache/redis/seq_conversation.go @@ -4,33 +4,35 @@ import ( "context" "errors" "fmt" - "github.com/dtm-labs/rockscache" + "strconv" + "time" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/cachekey" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/mcache" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" "github.com/openimsdk/open-im-server/v3/pkg/msgprocessor" "github.com/openimsdk/tools/errs" "github.com/openimsdk/tools/log" "github.com/redis/go-redis/v9" - "strconv" - "time" ) func NewSeqConversationCacheRedis(rdb redis.UniversalClient, mgo database.SeqConversation) cache.SeqConversationCache { + if rdb == nil { + return mcache.NewSeqConversationCache(mgo) + } return &seqConversationCacheRedis{ - rdb: rdb, mgo: mgo, lockTime: time.Second * 3, dataTime: time.Hour * 24 * 365, minSeqExpireTime: time.Hour, - rocks: rockscache.NewClient(rdb, *GetRocksCacheOptions()), + rcClient: newRocksCacheClient(rdb), } } type seqConversationCacheRedis struct { - rdb redis.UniversalClient mgo database.SeqConversation - rocks *rockscache.Client + rcClient *rocksCacheClient lockTime time.Duration dataTime time.Duration minSeqExpireTime time.Duration @@ -45,7 +47,7 @@ func (s *seqConversationCacheRedis) SetMinSeq(ctx context.Context, conversationI } func (s *seqConversationCacheRedis) GetMinSeq(ctx context.Context, conversationID string) (int64, error) { - return getCache(ctx, s.rocks, s.getMinSeqKey(conversationID), s.minSeqExpireTime, func(ctx context.Context) (int64, error) { + return getCache(ctx, s.rcClient, s.getMinSeqKey(conversationID), s.minSeqExpireTime, func(ctx context.Context) (int64, error) { return s.mgo.GetMinSeq(ctx, conversationID) }) } @@ -68,7 +70,7 @@ func (s *seqConversationCacheRedis) getSingleMaxSeqWithTime(ctx context.Context, func (s *seqConversationCacheRedis) batchGetMaxSeq(ctx context.Context, keys []string, keyConversationID map[string]string, seqs map[string]int64) error { result := make([]*redis.StringCmd, len(keys)) - pipe := s.rdb.Pipeline() + pipe := s.rcClient.GetRedis().Pipeline() for i, key := range keys { result[i] = pipe.HGet(ctx, key, "CURR") } @@ -99,7 +101,7 @@ func (s *seqConversationCacheRedis) batchGetMaxSeq(ctx context.Context, keys []s func (s *seqConversationCacheRedis) batchGetMaxSeqWithTime(ctx context.Context, keys []string, keyConversationID map[string]string, seqs map[string]database.SeqTime) error { result := make([]*redis.SliceCmd, len(keys)) - pipe := s.rdb.Pipeline() + pipe := s.rcClient.GetRedis().Pipeline() for i, key := range keys { result[i] = pipe.HMGet(ctx, key, "CURR", "TIME") } @@ -157,7 +159,7 @@ func (s *seqConversationCacheRedis) GetMaxSeqs(ctx context.Context, conversation if len(keys) == 1 { return s.getSingleMaxSeq(ctx, conversationIDs[0]) } - slotKeys, err := groupKeysBySlot(ctx, s.rdb, keys) + slotKeys, err := groupKeysBySlot(ctx, s.rcClient.GetRedis(), keys) if err != nil { return nil, err } @@ -190,7 +192,7 @@ func (s *seqConversationCacheRedis) GetMaxSeqsWithTime(ctx context.Context, conv if len(keys) == 1 { return s.getSingleMaxSeqWithTime(ctx, conversationIDs[0]) } - slotKeys, err := groupKeysBySlot(ctx, s.rdb, keys) + slotKeys, err := groupKeysBySlot(ctx, s.rcClient.GetRedis(), keys) if err != nil { return nil, err } @@ -234,7 +236,7 @@ redis.call("HSET", key, "CURR", curr_seq, "LAST", last_seq, "TIME", mallocTime) redis.call("EXPIRE", key, dataSecond) return 0 ` - result, err := s.rdb.Eval(ctx, script, []string{key}, owner, int64(s.dataTime/time.Second), currSeq, lastSeq, mill).Int64() + result, err := s.rcClient.GetRedis().Eval(ctx, script, []string{key}, owner, int64(s.dataTime/time.Second), currSeq, lastSeq, mill).Int64() if err != nil { return 0, errs.Wrap(err) } @@ -305,7 +307,7 @@ table.insert(result, last_seq) table.insert(result, mallocTime) return result ` - result, err := s.rdb.Eval(ctx, script, []string{key}, size, int64(s.lockTime/time.Second), int64(s.dataTime/time.Second), time.Now().UnixMilli()).Int64Slice() + result, err := s.rcClient.GetRedis().Eval(ctx, script, []string{key}, size, int64(s.lockTime/time.Second), int64(s.dataTime/time.Second), time.Now().UnixMilli()).Int64Slice() if err != nil { return nil, errs.Wrap(err) } @@ -438,7 +440,7 @@ func (s *seqConversationCacheRedis) SetMinSeqs(ctx context.Context, seqs map[str return err } } - return DeleteCacheBySlot(ctx, s.rocks, keys) + return DeleteCacheBySlot(ctx, s.rcClient, keys) } // GetCacheMaxSeqWithTime only get the existing cache, if there is no cache, no cache will be generated @@ -456,7 +458,7 @@ func (s *seqConversationCacheRedis) GetCacheMaxSeqWithTime(ctx context.Context, key2conversationID[key] = conversationID keys = append(keys, key) } - slotKeys, err := groupKeysBySlot(ctx, s.rdb, keys) + slotKeys, err := groupKeysBySlot(ctx, s.rcClient.GetRedis(), keys) if err != nil { return nil, err } @@ -465,7 +467,7 @@ func (s *seqConversationCacheRedis) GetCacheMaxSeqWithTime(ctx context.Context, if len(keys) == 0 { continue } - pipe := s.rdb.Pipeline() + pipe := s.rcClient.GetRedis().Pipeline() cmds := make([]*redis.SliceCmd, 0, len(keys)) for _, key := range keys { cmds = append(cmds, pipe.HMGet(ctx, key, "CURR", "TIME")) diff --git a/pkg/common/storage/cache/redis/seq_user.go b/pkg/common/storage/cache/redis/seq_user.go index 0cedfeee1..af9cbef5a 100644 --- a/pkg/common/storage/cache/redis/seq_user.go +++ b/pkg/common/storage/cache/redis/seq_user.go @@ -2,31 +2,29 @@ package redis import ( "context" - "github.com/dtm-labs/rockscache" + "strconv" + "time" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/cachekey" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" "github.com/openimsdk/tools/errs" "github.com/redis/go-redis/v9" - "strconv" - "time" ) func NewSeqUserCacheRedis(rdb redis.UniversalClient, mgo database.SeqUser) cache.SeqUser { return &seqUserCacheRedis{ - rdb: rdb, mgo: mgo, readSeqWriteRatio: 100, expireTime: time.Hour * 24 * 7, readExpireTime: time.Hour * 24 * 30, - rocks: rockscache.NewClient(rdb, *GetRocksCacheOptions()), + rocks: newRocksCacheClient(rdb), } } type seqUserCacheRedis struct { - rdb redis.UniversalClient mgo database.SeqUser - rocks *rockscache.Client + rocks *rocksCacheClient expireTime time.Duration readExpireTime time.Duration readSeqWriteRatio int64 @@ -54,7 +52,7 @@ func (s *seqUserCacheRedis) SetUserMaxSeq(ctx context.Context, conversationID st if err := s.mgo.SetUserMaxSeq(ctx, conversationID, userID, seq); err != nil { return err } - return s.rocks.TagAsDeleted2(ctx, s.getSeqUserMaxSeqKey(conversationID, userID)) + return s.rocks.GetClient().TagAsDeleted2(ctx, s.getSeqUserMaxSeqKey(conversationID, userID)) } func (s *seqUserCacheRedis) GetUserMinSeq(ctx context.Context, conversationID string, userID string) (int64, error) { @@ -74,12 +72,15 @@ func (s *seqUserCacheRedis) GetUserReadSeq(ctx context.Context, conversationID s } func (s *seqUserCacheRedis) SetUserReadSeq(ctx context.Context, conversationID string, userID string, seq int64) error { + if s.rocks.GetRedis() == nil { + return s.SetUserReadSeqToDB(ctx, conversationID, userID, seq) + } dbSeq, err := s.GetUserReadSeq(ctx, conversationID, userID) if err != nil { return err } if dbSeq < seq { - if err := s.rocks.RawSet(ctx, s.getSeqUserReadSeqKey(conversationID, userID), strconv.Itoa(int(seq)), s.readExpireTime); err != nil { + if err := s.rocks.GetClient().RawSet(ctx, s.getSeqUserReadSeqKey(conversationID, userID), strconv.Itoa(int(seq)), s.readExpireTime); err != nil { return errs.Wrap(err) } } @@ -109,12 +110,12 @@ func (s *seqUserCacheRedis) setUserRedisReadSeqs(ctx context.Context, userID str keys = append(keys, key) keySeq[key] = seq } - slotKeys, err := groupKeysBySlot(ctx, s.rdb, keys) + slotKeys, err := groupKeysBySlot(ctx, s.rocks.GetRedis(), keys) if err != nil { return err } for _, keys := range slotKeys { - pipe := s.rdb.Pipeline() + pipe := s.rocks.GetRedis().Pipeline() for _, key := range keys { pipe.HSet(ctx, key, "value", strconv.FormatInt(keySeq[key], 10)) pipe.Expire(ctx, key, s.readExpireTime) diff --git a/pkg/common/storage/cache/redis/third.go b/pkg/common/storage/cache/redis/third.go index 3288cecb8..1ee6576c7 100644 --- a/pkg/common/storage/cache/redis/third.go +++ b/pkg/common/storage/cache/redis/third.go @@ -1,26 +1,13 @@ -// Copyright © 2023 OpenIM. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - package redis import ( "context" + "time" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/cachekey" "github.com/openimsdk/tools/errs" "github.com/redis/go-redis/v9" - "time" ) func NewThirdCache(rdb redis.UniversalClient) cache.ThirdCache { diff --git a/pkg/common/storage/cache/redis/user.go b/pkg/common/storage/cache/redis/user.go index f6b490730..6729650b0 100644 --- a/pkg/common/storage/cache/redis/user.go +++ b/pkg/common/storage/cache/redis/user.go @@ -1,30 +1,16 @@ -// Copyright © 2023 OpenIM. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - package redis import ( "context" + "time" + "github.com/dtm-labs/rockscache" "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/cachekey" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" - "github.com/openimsdk/tools/log" "github.com/redis/go-redis/v9" - "time" ) const ( @@ -38,19 +24,17 @@ type UserCacheRedis struct { rdb redis.UniversalClient userDB database.User expireTime time.Duration - rcClient *rockscache.Client + rcClient *rocksCacheClient } func NewUserCacheRedis(rdb redis.UniversalClient, localCache *config.LocalCache, userDB database.User, options *rockscache.Options) cache.UserCache { - batchHandler := NewBatchDeleterRedis(rdb, options, []string{localCache.User.Topic}) - u := localCache.User - log.ZDebug(context.Background(), "user local cache init", "Topic", u.Topic, "SlotNum", u.SlotNum, "SlotSize", u.SlotSize, "enable", u.Enable()) + rc := newRocksCacheClient(rdb) return &UserCacheRedis{ - BatchDeleter: batchHandler, + BatchDeleter: rc.GetBatchDeleter(localCache.User.Topic), rdb: rdb, userDB: userDB, expireTime: userExpireTime, - rcClient: rockscache.NewClient(rdb, *options), + rcClient: rc, } } diff --git a/pkg/common/storage/controller/group.go b/pkg/common/storage/controller/group.go index 072429ed0..6de0432a3 100644 --- a/pkg/common/storage/controller/group.go +++ b/pkg/common/storage/controller/group.go @@ -140,7 +140,7 @@ func NewGroupDatabase( groupMemberDB: groupMemberDB, groupRequestDB: groupRequestDB, ctxTx: ctxTx, - cache: redis2.NewGroupCacheRedis(rdb, localCache, groupDB, groupMemberDB, groupRequestDB, groupHash, redis2.GetRocksCacheOptions()), + cache: redis2.NewGroupCacheRedis(rdb, localCache, groupDB, groupMemberDB, groupRequestDB, groupHash), } } diff --git a/pkg/common/storage/controller/msg.go b/pkg/common/storage/controller/msg.go index 0069dc7cc..53dd7f13d 100644 --- a/pkg/common/storage/controller/msg.go +++ b/pkg/common/storage/controller/msg.go @@ -18,19 +18,21 @@ import ( "context" "encoding/json" "errors" + + "github.com/openimsdk/tools/mq" + "github.com/openimsdk/tools/utils/jsonutil" + "google.golang.org/protobuf/proto" + "strconv" "strings" "time" - "github.com/openimsdk/tools/utils/jsonutil" - "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" "github.com/redis/go-redis/v9" "go.mongodb.org/mongo-driver/mongo" - "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/open-im-server/v3/pkg/common/convert" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache" "github.com/openimsdk/protocol/constant" @@ -38,7 +40,6 @@ import ( "github.com/openimsdk/protocol/sdkws" "github.com/openimsdk/tools/errs" "github.com/openimsdk/tools/log" - "github.com/openimsdk/tools/mq/kafka" "github.com/openimsdk/tools/utils/datautil" ) @@ -102,22 +103,14 @@ type CommonMsgDatabase interface { GetLastMessage(ctx context.Context, conversationIDS []string, userID string) (map[string]*sdkws.MsgData, error) } -func NewCommonMsgDatabase(msgDocModel database.Msg, msg cache.MsgCache, seqUser cache.SeqUser, seqConversation cache.SeqConversationCache, kafkaConf *config.Kafka) (CommonMsgDatabase, error) { - conf, err := kafka.BuildProducerConfig(*kafkaConf.Build()) - if err != nil { - return nil, err - } - producerToRedis, err := kafka.NewKafkaProducer(conf, kafkaConf.Address, kafkaConf.ToRedisTopic) - if err != nil { - return nil, err - } +func NewCommonMsgDatabase(msgDocModel database.Msg, msg cache.MsgCache, seqUser cache.SeqUser, seqConversation cache.SeqConversationCache, producer mq.Producer) CommonMsgDatabase { return &commonMsgDatabase{ msgDocDatabase: msgDocModel, msgCache: msg, seqUser: seqUser, seqConversation: seqConversation, - producer: producerToRedis, - }, nil + producer: producer, + } } type commonMsgDatabase struct { @@ -126,12 +119,15 @@ type commonMsgDatabase struct { msgCache cache.MsgCache seqConversation cache.SeqConversationCache seqUser cache.SeqUser - producer *kafka.Producer + producer mq.Producer } func (db *commonMsgDatabase) MsgToMQ(ctx context.Context, key string, msg2mq *sdkws.MsgData) error { - _, _, err := db.producer.SendMessage(ctx, key, msg2mq) - return err + data, err := proto.Marshal(msg2mq) + if err != nil { + return err + } + return db.producer.SendMessage(ctx, key, data) } func (db *commonMsgDatabase) batchInsertBlock(ctx context.Context, conversationID string, fields []any, key int8, firstSeq int64) error { diff --git a/pkg/common/storage/controller/msg_transfer.go b/pkg/common/storage/controller/msg_transfer.go index f4c0c6270..260daf5c4 100644 --- a/pkg/common/storage/controller/msg_transfer.go +++ b/pkg/common/storage/controller/msg_transfer.go @@ -2,11 +2,13 @@ package controller import ( "context" + "github.com/openimsdk/open-im-server/v3/pkg/common/convert" "github.com/openimsdk/protocol/constant" + "github.com/openimsdk/tools/mq" "github.com/openimsdk/tools/utils/datautil" + "google.golang.org/protobuf/proto" - "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" @@ -14,7 +16,6 @@ import ( "github.com/openimsdk/protocol/sdkws" "github.com/openimsdk/tools/errs" "github.com/openimsdk/tools/log" - "github.com/openimsdk/tools/mq/kafka" "go.mongodb.org/mongo-driver/mongo" ) @@ -32,30 +33,30 @@ type MsgTransferDatabase interface { SetHasReadSeqToDB(ctx context.Context, conversationID string, userSeqMap map[string]int64) error // to mq - MsgToPushMQ(ctx context.Context, key, conversationID string, msg2mq *sdkws.MsgData) (int32, int64, error) + MsgToPushMQ(ctx context.Context, key, conversationID string, msg2mq *sdkws.MsgData) error MsgToMongoMQ(ctx context.Context, key, conversationID string, msgs []*sdkws.MsgData, lastSeq int64) error } -func NewMsgTransferDatabase(msgDocModel database.Msg, msg cache.MsgCache, seqUser cache.SeqUser, seqConversation cache.SeqConversationCache, kafkaConf *config.Kafka) (MsgTransferDatabase, error) { - conf, err := kafka.BuildProducerConfig(*kafkaConf.Build()) - if err != nil { - return nil, err - } - producerToMongo, err := kafka.NewKafkaProducer(conf, kafkaConf.Address, kafkaConf.ToMongoTopic) - if err != nil { - return nil, err - } - producerToPush, err := kafka.NewKafkaProducer(conf, kafkaConf.Address, kafkaConf.ToPushTopic) - if err != nil { - return nil, err - } +func NewMsgTransferDatabase(msgDocModel database.Msg, msg cache.MsgCache, seqUser cache.SeqUser, seqConversation cache.SeqConversationCache, mongoProducer, pushProducer mq.Producer) (MsgTransferDatabase, error) { + //conf, err := kafka.BuildProducerConfig(*kafkaConf.Build()) + //if err != nil { + // return nil, err + //} + //producerToMongo, err := kafka.NewKafkaProducerV2(conf, kafkaConf.Address, kafkaConf.ToMongoTopic) + //if err != nil { + // return nil, err + //} + //producerToPush, err := kafka.NewKafkaProducerV2(conf, kafkaConf.Address, kafkaConf.ToPushTopic) + //if err != nil { + // return nil, err + //} return &msgTransferDatabase{ msgDocDatabase: msgDocModel, msgCache: msg, seqUser: seqUser, seqConversation: seqConversation, - producerToMongo: producerToMongo, - producerToPush: producerToPush, + producerToMongo: mongoProducer, + producerToPush: pushProducer, }, nil } @@ -65,8 +66,8 @@ type msgTransferDatabase struct { msgCache cache.MsgCache seqConversation cache.SeqConversationCache seqUser cache.SeqUser - producerToMongo *kafka.Producer - producerToPush *kafka.Producer + producerToMongo mq.Producer + producerToPush mq.Producer } func (db *msgTransferDatabase) BatchInsertChat2DB(ctx context.Context, conversationID string, msgList []*sdkws.MsgData, currentMaxSeq int64) error { @@ -281,19 +282,25 @@ func (db *msgTransferDatabase) SetHasReadSeqToDB(ctx context.Context, conversati return nil } -func (db *msgTransferDatabase) MsgToPushMQ(ctx context.Context, key, conversationID string, msg2mq *sdkws.MsgData) (int32, int64, error) { - partition, offset, err := db.producerToPush.SendMessage(ctx, key, &pbmsg.PushMsgDataToMQ{MsgData: msg2mq, ConversationID: conversationID}) +func (db *msgTransferDatabase) MsgToPushMQ(ctx context.Context, key, conversationID string, msg2mq *sdkws.MsgData) error { + data, err := proto.Marshal(&pbmsg.PushMsgDataToMQ{MsgData: msg2mq, ConversationID: conversationID}) if err != nil { - log.ZError(ctx, "MsgToPushMQ", err, "key", key, "msg2mq", msg2mq) - return 0, 0, err + return err + } + if err := db.producerToPush.SendMessage(ctx, key, data); err != nil { + log.ZError(ctx, "MsgToPushMQ", err, "key", key, "conversationID", conversationID) + return err } - return partition, offset, nil + return nil } func (db *msgTransferDatabase) MsgToMongoMQ(ctx context.Context, key, conversationID string, messages []*sdkws.MsgData, lastSeq int64) error { if len(messages) > 0 { - _, _, err := db.producerToMongo.SendMessage(ctx, key, &pbmsg.MsgDataToMongoByMQ{LastSeq: lastSeq, ConversationID: conversationID, MsgData: messages}) + data, err := proto.Marshal(&pbmsg.MsgDataToMongoByMQ{LastSeq: lastSeq, ConversationID: conversationID, MsgData: messages}) if err != nil { + return err + } + if err := db.producerToMongo.SendMessage(ctx, key, data); err != nil { log.ZError(ctx, "MsgToMongoMQ", err, "key", key, "conversationID", conversationID, "lastSeq", lastSeq) return err } diff --git a/pkg/common/storage/controller/push.go b/pkg/common/storage/controller/push.go index 91ef126fe..ce62a7258 100644 --- a/pkg/common/storage/controller/push.go +++ b/pkg/common/storage/controller/push.go @@ -17,12 +17,12 @@ package controller import ( "context" - "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache" "github.com/openimsdk/protocol/push" "github.com/openimsdk/protocol/sdkws" "github.com/openimsdk/tools/log" - "github.com/openimsdk/tools/mq/kafka" + "github.com/openimsdk/tools/mq" + "google.golang.org/protobuf/proto" ) type PushDatabase interface { @@ -32,21 +32,13 @@ type PushDatabase interface { type pushDataBase struct { cache cache.ThirdCache - producerToOfflinePush *kafka.Producer + producerToOfflinePush mq.Producer } -func NewPushDatabase(cache cache.ThirdCache, kafkaConf *config.Kafka) PushDatabase { - conf, err := kafka.BuildProducerConfig(*kafkaConf.Build()) - if err != nil { - return nil - } - producerToOfflinePush, err := kafka.NewKafkaProducer(conf, kafkaConf.Address, kafkaConf.ToOfflinePushTopic) - if err != nil { - return nil - } +func NewPushDatabase(cache cache.ThirdCache, offlinePushProducer mq.Producer) PushDatabase { return &pushDataBase{ cache: cache, - producerToOfflinePush: producerToOfflinePush, + producerToOfflinePush: offlinePushProducer, } } @@ -55,7 +47,12 @@ func (p *pushDataBase) DelFcmToken(ctx context.Context, userID string, platformI } func (p *pushDataBase) MsgToOfflinePushMQ(ctx context.Context, key string, userIDs []string, msg2mq *sdkws.MsgData) error { - _, _, err := p.producerToOfflinePush.SendMessage(ctx, key, &push.PushMsgReq{MsgData: msg2mq, UserIDs: userIDs}) - log.ZInfo(ctx, "message is push to offlinePush topic", "key", key, "userIDs", userIDs, "msg", msg2mq.String()) + data, err := proto.Marshal(&push.PushMsgReq{MsgData: msg2mq, UserIDs: userIDs}) + if err != nil { + return err + } + if err := p.producerToOfflinePush.SendMessage(ctx, key, data); err != nil { + log.ZError(ctx, "message is push to offlinePush topic", err, "key", key, "userIDs", userIDs, "msg", msg2mq.String()) + } return err } diff --git a/pkg/common/storage/controller/s3.go b/pkg/common/storage/controller/s3.go index 6693d2dde..30d8d20ec 100644 --- a/pkg/common/storage/controller/s3.go +++ b/pkg/common/storage/controller/s3.go @@ -30,7 +30,7 @@ import ( ) type S3Database interface { - PartLimit() *s3.PartLimit + PartLimit() (*s3.PartLimit, error) PartSize(ctx context.Context, size int64) (int64, error) AuthSign(ctx context.Context, uploadID string, partNumbers []int) (*s3.AuthSignResult, error) InitiateMultipartUpload(ctx context.Context, hash string, size int64, expire time.Duration, maxParts int) (*cont.InitiateUploadResult, error) @@ -65,7 +65,7 @@ func (s *s3Database) PartSize(ctx context.Context, size int64) (int64, error) { return s.s3.PartSize(ctx, size) } -func (s *s3Database) PartLimit() *s3.PartLimit { +func (s *s3Database) PartLimit() (*s3.PartLimit, error) { return s.s3.PartLimit() } diff --git a/pkg/common/storage/database/black.go b/pkg/common/storage/database/black.go index b53fdd14d..5af960abe 100644 --- a/pkg/common/storage/database/black.go +++ b/pkg/common/storage/database/black.go @@ -16,6 +16,7 @@ package database import ( "context" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" "github.com/openimsdk/tools/db/pagination" ) @@ -29,3 +30,85 @@ type Black interface { FindOwnerBlackInfos(ctx context.Context, ownerUserID string, userIDs []string) (blacks []*model.Black, err error) FindBlackUserIDs(ctx context.Context, ownerUserID string) (blackUserIDs []string, err error) } + +var ( + _ Black = (*mgoImpl)(nil) + _ Black = (*redisImpl)(nil) +) + +type mgoImpl struct { +} + +func (m *mgoImpl) Create(ctx context.Context, blacks []*model.Black) (err error) { + //TODO implement me + panic("implement me") +} + +func (m *mgoImpl) Delete(ctx context.Context, blacks []*model.Black) (err error) { + //TODO implement me + panic("implement me") +} + +func (m *mgoImpl) Find(ctx context.Context, blacks []*model.Black) (blackList []*model.Black, err error) { + //TODO implement me + panic("implement me") +} + +func (m *mgoImpl) Take(ctx context.Context, ownerUserID, blockUserID string) (black *model.Black, err error) { + //TODO implement me + panic("implement me") +} + +func (m *mgoImpl) FindOwnerBlacks(ctx context.Context, ownerUserID string, pagination pagination.Pagination) (total int64, blacks []*model.Black, err error) { + //TODO implement me + panic("implement me") +} + +func (m *mgoImpl) FindOwnerBlackInfos(ctx context.Context, ownerUserID string, userIDs []string) (blacks []*model.Black, err error) { + //TODO implement me + panic("implement me") +} + +func (m *mgoImpl) FindBlackUserIDs(ctx context.Context, ownerUserID string) (blackUserIDs []string, err error) { + //TODO implement me + panic("implement me") +} + +type redisImpl struct { +} + +func (r *redisImpl) Create(ctx context.Context, blacks []*model.Black) (err error) { + + //TODO implement me + panic("implement me") +} + +func (r *redisImpl) Delete(ctx context.Context, blacks []*model.Black) (err error) { + //TODO implement me + panic("implement me") +} + +func (r *redisImpl) Find(ctx context.Context, blacks []*model.Black) (blackList []*model.Black, err error) { + //TODO implement me + panic("implement me") +} + +func (r *redisImpl) Take(ctx context.Context, ownerUserID, blockUserID string) (black *model.Black, err error) { + //TODO implement me + panic("implement me") +} + +func (r *redisImpl) FindOwnerBlacks(ctx context.Context, ownerUserID string, pagination pagination.Pagination) (total int64, blacks []*model.Black, err error) { + //TODO implement me + panic("implement me") +} + +func (r *redisImpl) FindOwnerBlackInfos(ctx context.Context, ownerUserID string, userIDs []string) (blacks []*model.Black, err error) { + //TODO implement me + panic("implement me") +} + +func (r *redisImpl) FindBlackUserIDs(ctx context.Context, ownerUserID string) (blackUserIDs []string, err error) { + //TODO implement me + panic("implement me") +} diff --git a/pkg/common/storage/database/cache.go b/pkg/common/storage/database/cache.go new file mode 100644 index 000000000..c57aea86d --- /dev/null +++ b/pkg/common/storage/database/cache.go @@ -0,0 +1,16 @@ +package database + +import ( + "context" + "time" +) + +type Cache interface { + Get(ctx context.Context, key []string) (map[string]string, error) + Prefix(ctx context.Context, prefix string) (map[string]string, error) + Set(ctx context.Context, key string, value string, expireAt time.Duration) error + Incr(ctx context.Context, key string, value int) (int, error) + Del(ctx context.Context, key []string) error + Lock(ctx context.Context, key string, duration time.Duration) (string, error) + Unlock(ctx context.Context, key string, value string) error +} diff --git a/pkg/common/storage/database/mgo/cache.go b/pkg/common/storage/database/mgo/cache.go new file mode 100644 index 000000000..bcf86cd56 --- /dev/null +++ b/pkg/common/storage/database/mgo/cache.go @@ -0,0 +1,183 @@ +package mgo + +import ( + "context" + "strconv" + "time" + + "github.com/google/uuid" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" + "github.com/openimsdk/tools/db/mongoutil" + "github.com/openimsdk/tools/errs" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" +) + +func NewCacheMgo(db *mongo.Database) (*CacheMgo, error) { + coll := db.Collection(database.CacheName) + _, err := coll.Indexes().CreateMany(context.Background(), []mongo.IndexModel{ + { + Keys: bson.D{ + {Key: "key", Value: 1}, + }, + Options: options.Index().SetUnique(true), + }, + { + Keys: bson.D{ + {Key: "expire_at", Value: 1}, + }, + Options: options.Index().SetExpireAfterSeconds(0), + }, + }) + if err != nil { + return nil, errs.Wrap(err) + } + return &CacheMgo{coll: coll}, nil +} + +type CacheMgo struct { + coll *mongo.Collection +} + +func (x *CacheMgo) findToMap(res []model.Cache, now time.Time) map[string]string { + kv := make(map[string]string) + for _, re := range res { + if re.ExpireAt != nil && re.ExpireAt.Before(now) { + continue + } + kv[re.Key] = re.Value + } + return kv + +} + +func (x *CacheMgo) Get(ctx context.Context, key []string) (map[string]string, error) { + if len(key) == 0 { + return nil, nil + } + now := time.Now() + res, err := mongoutil.Find[model.Cache](ctx, x.coll, bson.M{ + "key": bson.M{"$in": key}, + "$or": []bson.M{ + {"expire_at": bson.M{"$gt": now}}, + {"expire_at": nil}, + }, + }) + if err != nil { + return nil, err + } + return x.findToMap(res, now), nil +} + +func (x *CacheMgo) Prefix(ctx context.Context, prefix string) (map[string]string, error) { + now := time.Now() + res, err := mongoutil.Find[model.Cache](ctx, x.coll, bson.M{ + "key": bson.M{"$regex": "^" + prefix}, + "$or": []bson.M{ + {"expire_at": bson.M{"$gt": now}}, + {"expire_at": nil}, + }, + }) + if err != nil { + return nil, err + } + return x.findToMap(res, now), nil +} + +func (x *CacheMgo) Set(ctx context.Context, key string, value string, expireAt time.Duration) error { + cv := &model.Cache{ + Key: key, + Value: value, + } + if expireAt > 0 { + now := time.Now().Add(expireAt) + cv.ExpireAt = &now + } + opt := options.Update().SetUpsert(true) + return mongoutil.UpdateOne(ctx, x.coll, bson.M{"key": key}, bson.M{"$set": cv}, false, opt) +} + +func (x *CacheMgo) Incr(ctx context.Context, key string, value int) (int, error) { + pipeline := mongo.Pipeline{ + { + {"$set", bson.M{ + "value": bson.M{ + "$toString": bson.M{ + "$add": bson.A{ + bson.M{"$toInt": "$value"}, + value, + }, + }, + }, + }}, + }, + } + opt := options.FindOneAndUpdate().SetReturnDocument(options.After) + res, err := mongoutil.FindOneAndUpdate[model.Cache](ctx, x.coll, bson.M{"key": key}, pipeline, opt) + if err != nil { + return 0, err + } + return strconv.Atoi(res.Value) +} + +func (x *CacheMgo) Del(ctx context.Context, key []string) error { + if len(key) == 0 { + return nil + } + _, err := x.coll.DeleteMany(ctx, bson.M{"key": bson.M{"$in": key}}) + return err +} + +func (x *CacheMgo) lockKey(key string) string { + return "LOCK_" + key +} + +func (x *CacheMgo) Lock(ctx context.Context, key string, duration time.Duration) (string, error) { + tmp, err := uuid.NewUUID() + if err != nil { + return "", err + } + if duration <= 0 || duration > time.Minute*10 { + duration = time.Minute * 10 + } + cv := &model.Cache{ + Key: x.lockKey(key), + Value: tmp.String(), + ExpireAt: nil, + } + ctx, cancel := context.WithTimeout(ctx, time.Second*30) + defer cancel() + wait := func() error { + timeout := time.NewTimer(time.Millisecond * 100) + defer timeout.Stop() + select { + case <-ctx.Done(): + return ctx.Err() + case <-timeout.C: + return nil + } + } + for { + if err := mongoutil.DeleteOne(ctx, x.coll, bson.M{"key": key, "expire_at": bson.M{"$lt": time.Now()}}); err != nil { + return "", err + } + expireAt := time.Now().Add(duration) + cv.ExpireAt = &expireAt + if err := mongoutil.InsertMany[*model.Cache](ctx, x.coll, []*model.Cache{cv}); err != nil { + if mongo.IsDuplicateKeyError(err) { + if err := wait(); err != nil { + return "", err + } + continue + } + return "", err + } + return cv.Value, nil + } +} + +func (x *CacheMgo) Unlock(ctx context.Context, key string, value string) error { + return mongoutil.DeleteOne(ctx, x.coll, bson.M{"key": x.lockKey(key), "value": value}) +} diff --git a/pkg/common/storage/database/mgo/cache_test.go b/pkg/common/storage/database/mgo/cache_test.go new file mode 100644 index 000000000..ac8b22951 --- /dev/null +++ b/pkg/common/storage/database/mgo/cache_test.go @@ -0,0 +1,133 @@ +package mgo + +import ( + "context" + "strings" + "sync" + "testing" + "time" + + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" + "github.com/openimsdk/tools/db/mongoutil" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" +) + +func TestName1111(t *testing.T) { + coll := Mongodb().Collection("temp") + + //updatePipeline := mongo.Pipeline{ + // { + // {"$set", bson.M{ + // "age": bson.M{ + // "$toString": bson.M{ + // "$add": bson.A{ + // bson.M{"$toInt": "$age"}, + // 1, + // }, + // }, + // }, + // }}, + // }, + //} + + pipeline := mongo.Pipeline{ + { + {"$set", bson.M{ + "value": bson.M{ + "$toString": bson.M{ + "$add": bson.A{ + bson.M{"$toInt": "$value"}, + 1, + }, + }, + }, + }}, + }, + } + + opt := options.FindOneAndUpdate().SetUpsert(false).SetReturnDocument(options.After) + res, err := mongoutil.FindOneAndUpdate[model.Cache](context.Background(), coll, bson.M{"key": "123456"}, pipeline, opt) + if err != nil { + panic(err) + } + t.Log(res) +} + +func TestName33333(t *testing.T) { + c, err := NewCacheMgo(Mongodb()) + if err != nil { + panic(err) + } + if err := c.Set(context.Background(), "123456", "123456", time.Hour); err != nil { + panic(err) + } + + if err := c.Set(context.Background(), "123666", "123666", time.Hour); err != nil { + panic(err) + } + + res1, err := c.Get(context.Background(), []string{"123456"}) + if err != nil { + panic(err) + } + t.Log(res1) + + res2, err := c.Prefix(context.Background(), "123") + if err != nil { + panic(err) + } + t.Log(res2) +} + +func TestName1111aa(t *testing.T) { + + c, err := NewCacheMgo(Mongodb()) + if err != nil { + panic(err) + } + var count int + + key := "123456" + + doFunc := func() { + value, err := c.Lock(context.Background(), key, time.Second*30) + if err != nil { + t.Log("Lock error", err) + return + } + tmp := count + tmp++ + count = tmp + t.Log("count", tmp) + if err := c.Unlock(context.Background(), key, value); err != nil { + t.Log("Unlock error", err) + return + } + } + + if _, err := c.Lock(context.Background(), key, time.Second*10); err != nil { + t.Log(err) + return + } + + var wg sync.WaitGroup + for i := 0; i < 32; i++ { + wg.Add(1) + go func() { + defer wg.Done() + for i := 0; i < 100; i++ { + doFunc() + } + }() + } + + wg.Wait() + +} + +func TestName111111a(t *testing.T) { + arr := strings.SplitN("1:testkakskdask:1111", ":", 2) + t.Log(arr) +} diff --git a/pkg/common/storage/database/mgo/msg_test.go b/pkg/common/storage/database/mgo/msg_test.go index 992090552..2ced0210a 100644 --- a/pkg/common/storage/database/mgo/msg_test.go +++ b/pkg/common/storage/database/mgo/msg_test.go @@ -2,16 +2,17 @@ package mgo import ( "context" - "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" - "github.com/openimsdk/tools/db/mongoutil" - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/mongo" - "go.mongodb.org/mongo-driver/mongo/options" "math" "math/rand" "strconv" "testing" "time" + + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" + "github.com/openimsdk/tools/db/mongoutil" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" ) func TestName1(t *testing.T) { @@ -93,7 +94,7 @@ func TestName3(t *testing.T) { func TestName4(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), time.Second*300) defer cancel() - cli := Result(mongo.Connect(ctx, options.Client().ApplyURI("mongodb://openIM:openIM123@172.16.8.66:37017/openim_v3?maxPoolSize=100").SetConnectTimeout(5*time.Second))) + cli := Result(mongo.Connect(ctx, options.Client().ApplyURI("mongodb://openIM:openIM123@172.16.8.135:37017/openim_v3?maxPoolSize=100").SetConnectTimeout(5*time.Second))) msg, err := NewMsgMongo(cli.Database("openim_v3")) if err != nil { @@ -109,6 +110,41 @@ func TestName4(t *testing.T) { } func TestName5(t *testing.T) { - var v time.Time - t.Log(v.UnixMilli()) + ctx, cancel := context.WithTimeout(context.Background(), time.Second*300) + defer cancel() + cli := Result(mongo.Connect(ctx, options.Client().ApplyURI("mongodb://openIM:openIM123@172.16.8.135:37017/openim_v3?maxPoolSize=100").SetConnectTimeout(5*time.Second))) + + tmp, err := NewMsgMongo(cli.Database("openim_v3")) + if err != nil { + panic(err) + } + msg := tmp.(*MsgMgo) + ts := time.Now().Add(-time.Hour * 24 * 5).UnixMilli() + t.Log(ts) + var seqs []int64 + for i := 1; i < 256; i++ { + seqs = append(seqs, int64(i)) + } + res, err := msg.FindSeqs(ctx, "si_4924054191_9511766539", seqs) + if err != nil { + panic(err) + } + t.Log(res) } + +//func TestName6(t *testing.T) { +// ctx, cancel := context.WithTimeout(context.Background(), time.Second*300) +// defer cancel() +// cli := Result(mongo.Connect(ctx, options.Client().ApplyURI("mongodb://openIM:openIM123@172.16.8.135:37017/openim_v3?maxPoolSize=100").SetConnectTimeout(5*time.Second))) +// +// tmp, err := NewMsgMongo(cli.Database("openim_v3")) +// if err != nil { +// panic(err) +// } +// msg := tmp.(*MsgMgo) +// seq, sendTime, err := msg.findBeforeSendTime(ctx, "si_4924054191_9511766539", 1144) +// if err != nil { +// panic(err) +// } +// t.Log(seq, sendTime) +//} diff --git a/pkg/common/storage/database/mgo/seq_conversation_test.go b/pkg/common/storage/database/mgo/seq_conversation_test.go index 42507a693..dd30286e1 100644 --- a/pkg/common/storage/database/mgo/seq_conversation_test.go +++ b/pkg/common/storage/database/mgo/seq_conversation_test.go @@ -2,10 +2,11 @@ package mgo import ( "context" - "go.mongodb.org/mongo-driver/mongo" - "go.mongodb.org/mongo-driver/mongo/options" "testing" "time" + + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" ) func Result[V any](val V, err error) V { @@ -19,7 +20,7 @@ func Mongodb() *mongo.Database { return Result( mongo.Connect(context.Background(), options.Client(). - ApplyURI("mongodb://openIM:openIM123@172.16.8.48:37017/openim_v3?maxPoolSize=100"). + ApplyURI("mongodb://openIM:openIM123@172.16.8.135:37017/openim_v3?maxPoolSize=100"). SetConnectTimeout(5*time.Second)), ).Database("openim_v3") } diff --git a/pkg/common/storage/database/name.go b/pkg/common/storage/database/name.go index 9742f933f..4d74c7ddc 100644 --- a/pkg/common/storage/database/name.go +++ b/pkg/common/storage/database/name.go @@ -18,4 +18,5 @@ const ( SeqConversationName = "seq" SeqUserName = "seq_user" StreamMsgName = "stream_msg" + CacheName = "cache" ) diff --git a/pkg/common/storage/model/cache.go b/pkg/common/storage/model/cache.go new file mode 100644 index 000000000..4bbc55e65 --- /dev/null +++ b/pkg/common/storage/model/cache.go @@ -0,0 +1,9 @@ +package model + +import "time" + +type Cache struct { + Key string `bson:"key"` + Value string `bson:"value"` + ExpireAt *time.Time `bson:"expire_at"` +} diff --git a/pkg/dbbuild/builder.go b/pkg/dbbuild/builder.go new file mode 100644 index 000000000..7712b8117 --- /dev/null +++ b/pkg/dbbuild/builder.go @@ -0,0 +1,25 @@ +package dbbuild + +import ( + "context" + + "github.com/openimsdk/open-im-server/v3/pkg/common/config" + "github.com/openimsdk/tools/db/mongoutil" + "github.com/redis/go-redis/v9" +) + +type Builder interface { + Mongo(ctx context.Context) (*mongoutil.Client, error) + Redis(ctx context.Context) (redis.UniversalClient, error) +} + +func NewBuilder(mongoConf *config.Mongo, redisConf *config.Redis) Builder { + if config.Standalone() { + globalStandalone.setConfig(mongoConf, redisConf) + return globalStandalone + } + return µservices{ + mongo: mongoConf, + redis: redisConf, + } +} diff --git a/pkg/dbbuild/microservices.go b/pkg/dbbuild/microservices.go new file mode 100644 index 000000000..f96760b1f --- /dev/null +++ b/pkg/dbbuild/microservices.go @@ -0,0 +1,26 @@ +package dbbuild + +import ( + "context" + + "github.com/openimsdk/open-im-server/v3/pkg/common/config" + "github.com/openimsdk/tools/db/mongoutil" + "github.com/openimsdk/tools/db/redisutil" + "github.com/redis/go-redis/v9" +) + +type microservices struct { + mongo *config.Mongo + redis *config.Redis +} + +func (x *microservices) Mongo(ctx context.Context) (*mongoutil.Client, error) { + return mongoutil.NewMongoDB(ctx, x.mongo.Build()) +} + +func (x *microservices) Redis(ctx context.Context) (redis.UniversalClient, error) { + if x.redis.Disable { + return nil, nil + } + return redisutil.NewRedisClient(ctx, x.redis.Build()) +} diff --git a/pkg/dbbuild/standalone.go b/pkg/dbbuild/standalone.go new file mode 100644 index 000000000..37bd377de --- /dev/null +++ b/pkg/dbbuild/standalone.go @@ -0,0 +1,76 @@ +package dbbuild + +import ( + "context" + "sync" + + "github.com/openimsdk/open-im-server/v3/pkg/common/config" + "github.com/openimsdk/tools/db/mongoutil" + "github.com/openimsdk/tools/db/redisutil" + "github.com/redis/go-redis/v9" +) + +const ( + standaloneMongo = "mongo" + standaloneRedis = "redis" +) + +var globalStandalone = &standalone{} + +type standaloneConn[C any] struct { + Conn C + Err error +} + +func (x *standaloneConn[C]) result() (C, error) { + return x.Conn, x.Err +} + +type standalone struct { + lock sync.Mutex + mongo *config.Mongo + redis *config.Redis + conn map[string]any +} + +func (x *standalone) setConfig(mongoConf *config.Mongo, redisConf *config.Redis) { + x.lock.Lock() + defer x.lock.Unlock() + x.mongo = mongoConf + x.redis = redisConf +} + +func (x *standalone) Mongo(ctx context.Context) (*mongoutil.Client, error) { + x.lock.Lock() + defer x.lock.Unlock() + if x.conn == nil { + x.conn = make(map[string]any) + } + v, ok := x.conn[standaloneMongo] + if !ok { + var val standaloneConn[*mongoutil.Client] + val.Conn, val.Err = mongoutil.NewMongoDB(ctx, x.mongo.Build()) + v = &val + x.conn[standaloneMongo] = v + } + return v.(*standaloneConn[*mongoutil.Client]).result() +} + +func (x *standalone) Redis(ctx context.Context) (redis.UniversalClient, error) { + x.lock.Lock() + defer x.lock.Unlock() + if x.redis.Disable { + return nil, nil + } + if x.conn == nil { + x.conn = make(map[string]any) + } + v, ok := x.conn[standaloneRedis] + if !ok { + var val standaloneConn[redis.UniversalClient] + val.Conn, val.Err = redisutil.NewRedisClient(ctx, x.redis.Build()) + v = &val + x.conn[standaloneRedis] = v + } + return v.(*standaloneConn[redis.UniversalClient]).result() +} diff --git a/pkg/mqbuild/builder.go b/pkg/mqbuild/builder.go new file mode 100644 index 000000000..938159372 --- /dev/null +++ b/pkg/mqbuild/builder.go @@ -0,0 +1,60 @@ +package mqbuild + +import ( + "context" + "fmt" + + "github.com/openimsdk/open-im-server/v3/pkg/common/config" + "github.com/openimsdk/tools/mq" + "github.com/openimsdk/tools/mq/kafka" + "github.com/openimsdk/tools/mq/simmq" +) + +type Builder interface { + GetTopicProducer(ctx context.Context, topic string) (mq.Producer, error) + GetTopicConsumer(ctx context.Context, topic string) (mq.Consumer, error) +} + +func NewBuilder(kafka *config.Kafka) Builder { + if config.Standalone() { + return standaloneBuilder{} + } + return &kafkaBuilder{ + addr: kafka.Address, + config: kafka.Build(), + topicGroupID: map[string]string{ + kafka.ToRedisTopic: kafka.ToRedisGroupID, + kafka.ToMongoTopic: kafka.ToMongoGroupID, + kafka.ToPushTopic: kafka.ToPushGroupID, + kafka.ToOfflinePushTopic: kafka.ToOfflineGroupID, + }, + } +} + +type standaloneBuilder struct{} + +func (standaloneBuilder) GetTopicProducer(ctx context.Context, topic string) (mq.Producer, error) { + return simmq.GetTopicProducer(topic), nil +} + +func (standaloneBuilder) GetTopicConsumer(ctx context.Context, topic string) (mq.Consumer, error) { + return simmq.GetTopicConsumer(topic), nil +} + +type kafkaBuilder struct { + addr []string + config *kafka.Config + topicGroupID map[string]string +} + +func (x *kafkaBuilder) GetTopicProducer(ctx context.Context, topic string) (mq.Producer, error) { + return kafka.NewKafkaProducerV2(x.config, x.addr, topic) +} + +func (x *kafkaBuilder) GetTopicConsumer(ctx context.Context, topic string) (mq.Consumer, error) { + groupID, ok := x.topicGroupID[topic] + if !ok { + return nil, fmt.Errorf("topic %s groupID not found", topic) + } + return kafka.NewMConsumerGroupV2(ctx, x.config, groupID, []string{topic}, true) +} diff --git a/pkg/rpccache/online.go b/pkg/rpccache/online.go index b5308bbe8..959b054a2 100644 --- a/pkg/rpccache/online.go +++ b/pkg/rpccache/online.go @@ -3,15 +3,16 @@ package rpccache import ( "context" "fmt" - "github.com/openimsdk/open-im-server/v3/pkg/rpcli" - "github.com/openimsdk/protocol/constant" - "github.com/openimsdk/protocol/user" "math/rand" "strconv" "sync" "sync/atomic" "time" + "github.com/openimsdk/open-im-server/v3/pkg/rpcli" + "github.com/openimsdk/protocol/constant" + "github.com/openimsdk/protocol/user" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/cachekey" "github.com/openimsdk/open-im-server/v3/pkg/localcache" "github.com/openimsdk/open-im-server/v3/pkg/localcache/lru" @@ -51,10 +52,11 @@ func NewOnlineCache(client *rpcli.UserClient, group *GroupLocalCache, rdb redis. x.CurrentPhase.Store(DoSubscribeOver) x.Cond.Broadcast() } - - go func() { - x.doSubscribe(ctx, rdb, fn) - }() + if rdb != nil { + go func() { + x.doSubscribe(ctx, rdb, fn) + }() + } return x, nil } diff --git a/pkg/tools/batcher/batcher.go b/pkg/tools/batcher/batcher.go index 163aeed39..dcf5d07ad 100644 --- a/pkg/tools/batcher/batcher.go +++ b/pkg/tools/batcher/batcher.go @@ -3,11 +3,12 @@ package batcher import ( "context" "fmt" - "github.com/openimsdk/tools/errs" - "github.com/openimsdk/tools/utils/idutil" "strings" "sync" "time" + + "github.com/openimsdk/tools/errs" + "github.com/openimsdk/tools/utils/idutil" ) var ( @@ -245,7 +246,9 @@ func (b *Batcher[T]) distributeMessage(messages map[string][]*T, totalCount int, if b.config.syncWait { b.counter.Wait() } - b.OnComplete(lastMessage, totalCount) + if b.OnComplete != nil { + b.OnComplete(lastMessage, totalCount) + } } func (b *Batcher[T]) run(channelID int, ch <-chan *Msg[T]) { diff --git a/tools/check-component/main.go b/tools/check-component/main.go index 9df0da7de..993f549be 100644 --- a/tools/check-component/main.go +++ b/tools/check-component/main.go @@ -32,7 +32,6 @@ import ( "github.com/openimsdk/tools/mq/kafka" "github.com/openimsdk/tools/s3/minio" "github.com/openimsdk/tools/system/program" - "github.com/openimsdk/tools/utils/runtimeenv" ) const maxRetry = 180 @@ -84,37 +83,36 @@ func initConfig(configDir string) (*config.Mongo, *config.Redis, *config.Kafka, discovery = &config.Discovery{} thirdConfig = &config.Third{} ) - runtimeEnv := runtimeenv.PrintRuntimeEnvironment() - err := config.Load(configDir, config.MongodbConfigFileName, config.EnvPrefixMap[config.MongodbConfigFileName], runtimeEnv, mongoConfig) + err := config.Load(configDir, config.MongodbConfigFileName, config.EnvPrefixMap[config.MongodbConfigFileName], mongoConfig) if err != nil { return nil, nil, nil, nil, nil, err } - err = config.Load(configDir, config.RedisConfigFileName, config.EnvPrefixMap[config.RedisConfigFileName], runtimeEnv, redisConfig) + err = config.Load(configDir, config.RedisConfigFileName, config.EnvPrefixMap[config.RedisConfigFileName], redisConfig) if err != nil { return nil, nil, nil, nil, nil, err } - err = config.Load(configDir, config.KafkaConfigFileName, config.EnvPrefixMap[config.KafkaConfigFileName], runtimeEnv, kafkaConfig) + err = config.Load(configDir, config.KafkaConfigFileName, config.EnvPrefixMap[config.KafkaConfigFileName], kafkaConfig) if err != nil { return nil, nil, nil, nil, nil, err } - err = config.Load(configDir, config.OpenIMRPCThirdCfgFileName, config.EnvPrefixMap[config.OpenIMRPCThirdCfgFileName], runtimeEnv, thirdConfig) + err = config.Load(configDir, config.OpenIMRPCThirdCfgFileName, config.EnvPrefixMap[config.OpenIMRPCThirdCfgFileName], thirdConfig) if err != nil { return nil, nil, nil, nil, nil, err } if thirdConfig.Object.Enable == "minio" { - err = config.Load(configDir, config.MinioConfigFileName, config.EnvPrefixMap[config.MinioConfigFileName], runtimeEnv, minioConfig) + err = config.Load(configDir, config.MinioConfigFileName, config.EnvPrefixMap[config.MinioConfigFileName], minioConfig) if err != nil { return nil, nil, nil, nil, nil, err } } else { minioConfig = nil } - err = config.Load(configDir, config.DiscoveryConfigFilename, config.EnvPrefixMap[config.DiscoveryConfigFilename], runtimeEnv, discovery) + err = config.Load(configDir, config.DiscoveryConfigFilename, config.EnvPrefixMap[config.DiscoveryConfigFilename], discovery) if err != nil { return nil, nil, nil, nil, nil, err } diff --git a/tools/seq/internal/seq.go b/tools/seq/internal/seq.go index 574e7cef9..7e5d5598c 100644 --- a/tools/seq/internal/seq.go +++ b/tools/seq/internal/seq.go @@ -43,7 +43,7 @@ const ( ) func readConfig[T any](dir string, name string) (*T, error) { - if runtimeenv.PrintRuntimeEnvironment() == config.KUBERNETES { + if runtimeenv.RuntimeEnvironment() == config.KUBERNETES { dir = os.Getenv(config.MountConfigFilePath) } v := viper.New() From 14393b0f5316aa35d1949bcafd0e73afe0263560 Mon Sep 17 00:00:00 2001 From: icey-yu <119291641+icey-yu@users.noreply.github.com> Date: Mon, 17 Feb 2025 18:20:31 +0800 Subject: [PATCH 119/199] fix: Offline push does not have a badge && Android offline push (#3146) * fix: offline push can display badge * feat: strategy * feat: log * feat: log * chore: offlinepush * fix: offlinepush * fix: log --- internal/push/offlinepush/fcm/push.go | 8 +++-- internal/push/offlinepush/getui/body.go | 38 ++++++++++++++++++-- internal/push/offlinepush/getui/push.go | 8 +++-- pkg/common/storage/cache/redis/lua_script.go | 4 ++- 4 files changed, 49 insertions(+), 9 deletions(-) diff --git a/internal/push/offlinepush/fcm/push.go b/internal/push/offlinepush/fcm/push.go index 6e8355af3..463b72759 100644 --- a/internal/push/offlinepush/fcm/push.go +++ b/internal/push/offlinepush/fcm/push.go @@ -16,12 +16,14 @@ package fcm import ( "context" + "errors" "fmt" - "github.com/openimsdk/open-im-server/v3/internal/push/offlinepush/options" - "github.com/openimsdk/tools/utils/httputil" "path/filepath" "strings" + "github.com/openimsdk/open-im-server/v3/internal/push/offlinepush/options" + "github.com/openimsdk/tools/utils/httputil" + firebase "firebase.google.com/go/v4" "firebase.google.com/go/v4/messaging" "github.com/openimsdk/open-im-server/v3/pkg/common/config" @@ -133,7 +135,7 @@ func (f *Fcm) Push(ctx context.Context, userIDs []string, title, content string, unreadCountSum, err := f.cache.GetUserBadgeUnreadCountSum(ctx, userID) if err == nil && unreadCountSum != 0 { apns.Payload.Aps.Badge = &unreadCountSum - } else if err == redis.Nil || unreadCountSum == 0 { + } else if errors.Is(err, redis.Nil) || unreadCountSum == 0 { zero := 1 apns.Payload.Aps.Badge = &zero } else { diff --git a/internal/push/offlinepush/getui/body.go b/internal/push/offlinepush/getui/body.go index a96ff4efc..2f2469b4f 100644 --- a/internal/push/offlinepush/getui/body.go +++ b/internal/push/offlinepush/getui/body.go @@ -18,6 +18,16 @@ import ( "fmt" "github.com/openimsdk/open-im-server/v3/pkg/common/config" + "github.com/openimsdk/tools/utils/datautil" +) + +var ( + incOne = datautil.ToPtr("+1") + addNum = "1" + defaultStrategy = strategy{ + Default: 1, + } + msgCategory = "CATEGORY_MESSAGE" ) type Resp struct { @@ -58,7 +68,24 @@ type TaskResp struct { } type Settings struct { - TTL *int64 `json:"ttl"` + TTL *int64 `json:"ttl"` + Strategy strategy `json:"strategy"` +} + +type strategy struct { + Default int64 `json:"default"` + //IOS int64 `json:"ios"` + //St int64 `json:"st"` + //Hw int64 `json:"hw"` + //Ho int64 `json:"ho"` + //XM int64 `json:"xm"` + //XMG int64 `json:"xmg"` + //VV int64 `json:"vv"` + //Op int64 `json:"op"` + //OpG int64 `json:"opg"` + //MZ int64 `json:"mz"` + //HosHw int64 `json:"hoshw"` + //WX int64 `json:"wx"` } type Audience struct { @@ -112,6 +139,8 @@ type Notification struct { ChannelID string `json:"channelID"` ChannelName string `json:"ChannelName"` ClickType string `json:"click_type"` + BadgeAddNum string `json:"badge_add_num"` + Category string `json:"category"` } type Options struct { @@ -120,6 +149,7 @@ type Options struct { ChannelID string `json:"/message/android/notification/channel_id"` Sound string `json:"/message/android/notification/sound"` Importance string `json:"/message/android/notification/importance"` + Category string `json:"/message/android/category"` } `json:"HW"` XM struct { ChannelID string `json:"/extra.channel_id"` @@ -140,6 +170,8 @@ func newPushReq(pushConf *config.Push, title, content string) PushReq { ClickType: "startapp", ChannelID: pushConf.GeTui.ChannelID, ChannelName: pushConf.GeTui.ChannelName, + BadgeAddNum: addNum, + Category: msgCategory, }}} return pushReq } @@ -156,6 +188,7 @@ func (pushReq *PushReq) setPushChannel(title string, body string) { notify := "notify" pushReq.PushChannel.Ios.NotificationType = ¬ify pushReq.PushChannel.Ios.Aps.Sound = "default" + pushReq.PushChannel.Ios.AutoBadge = incOne pushReq.PushChannel.Ios.Aps.Alert = Alert{ Title: title, Body: body, @@ -172,7 +205,8 @@ func (pushReq *PushReq) setPushChannel(title string, body string) { ChannelID string `json:"/message/android/notification/channel_id"` Sound string `json:"/message/android/notification/sound"` Importance string `json:"/message/android/notification/importance"` - }{ChannelID: "RingRing4", Sound: "/raw/ring001", Importance: "NORMAL"}, + Category string `json:"/message/android/category"` + }{ChannelID: "RingRing4", Sound: "/raw/ring001", Importance: "NORMAL", Category: "IM"}, XM: struct { ChannelID string `json:"/extra.channel_id"` }{ChannelID: "high_system"}, diff --git a/internal/push/offlinepush/getui/push.go b/internal/push/offlinepush/getui/push.go index e266f9c46..e82b62c7a 100644 --- a/internal/push/offlinepush/getui/push.go +++ b/internal/push/offlinepush/getui/push.go @@ -18,6 +18,7 @@ import ( "context" "crypto/sha256" "encoding/hex" + "errors" "strconv" "sync" "time" @@ -70,7 +71,7 @@ func NewClient(pushConf *config.Push, cache cache.ThirdCache) *Client { func (g *Client) Push(ctx context.Context, userIDs []string, title, content string, opts *options.Opts) error { token, err := g.cache.GetGetuiToken(ctx) if err != nil { - if errs.Unwrap(err) == redis.Nil { + if errors.Is(err, redis.Nil) { log.ZDebug(ctx, "getui token not exist in redis") token, err = g.getTokenAndSave2Redis(ctx) if err != nil { @@ -144,7 +145,7 @@ func (g *Client) Auth(ctx context.Context, timeStamp int64) (token string, expir func (g *Client) GetTaskID(ctx context.Context, token string, pushReq PushReq) (string, error) { respTask := TaskResp{} ttl := int64(1000 * 60 * 5) - pushReq.Settings = &Settings{TTL: &ttl} + pushReq.Settings = &Settings{TTL: &ttl, Strategy: defaultStrategy} err := g.request(ctx, taskURL, pushReq, token, &respTask) if err != nil { return "", errs.Wrap(err) @@ -188,6 +189,7 @@ func (g *Client) postReturn( if err != nil { return err } + log.ZDebug(ctx, "postReturn", "url", url, "header", header, "input", input, "timeout", timeout, "output", output) return output.parseError() } @@ -204,7 +206,7 @@ func (g *Client) getTokenAndSave2Redis(ctx context.Context) (token string, err e } func (g *Client) GetTaskIDAndSave2Redis(ctx context.Context, token string, pushReq PushReq) (taskID string, err error) { - pushReq.Settings = &Settings{TTL: &g.taskIDTTL} + pushReq.Settings = &Settings{TTL: &g.taskIDTTL, Strategy: defaultStrategy} taskID, err = g.GetTaskID(ctx, token, pushReq) if err != nil { return diff --git a/pkg/common/storage/cache/redis/lua_script.go b/pkg/common/storage/cache/redis/lua_script.go index c7609cb44..acf7eaa79 100644 --- a/pkg/common/storage/cache/redis/lua_script.go +++ b/pkg/common/storage/cache/redis/lua_script.go @@ -2,7 +2,9 @@ package redis import ( "context" + "errors" "fmt" + "github.com/openimsdk/open-im-server/v3/pkg/common/servererrs" "github.com/openimsdk/tools/errs" "github.com/openimsdk/tools/log" @@ -56,7 +58,7 @@ func callLua(ctx context.Context, rdb redis.Scripter, script *redis.Script, keys } } v, err := r.Result() - if err == redis.Nil { + if errors.Is(err, redis.Nil) { err = nil } return v, errs.WrapMsg(err, "call lua err", "scriptHash", script.Hash(), "keys", keys, "args", args) From df1c8df693afefd3cb7f4868eb6b56a01cdf2a34 Mon Sep 17 00:00:00 2001 From: icey-yu <119291641+icey-yu@users.noreply.github.com> Date: Thu, 20 Feb 2025 16:13:47 +0800 Subject: [PATCH 120/199] feat: Change after webhook filter && feat SendSimpleMsg (#3151) * feat: msg filter and search system account * feat: search system account * chore: msg * chore: msg * chore: msg * chore: webhook filter && sendSimpleMessage --- config/webhooks.yml | 16 ++--- go.mod | 4 +- go.sum | 4 +- internal/api/msg.go | 81 +++++++++++++++++++++++++ internal/rpc/msg/callback.go | 21 ++++++- internal/rpc/msg/filter.go | 46 ++++++++++---- internal/rpc/user/user.go | 26 +++++--- pkg/apistruct/manage.go | 15 +++++ pkg/common/config/config.go | 12 ++-- pkg/common/storage/controller/user.go | 9 ++- pkg/common/storage/database/mgo/user.go | 7 ++- pkg/common/storage/database/user.go | 4 +- pkg/common/webhook/http_client.go | 50 ++++++++++++++- 13 files changed, 247 insertions(+), 48 deletions(-) diff --git a/config/webhooks.yml b/config/webhooks.yml index 854d2dc2c..41c60e7e2 100644 --- a/config/webhooks.yml +++ b/config/webhooks.yml @@ -3,14 +3,7 @@ beforeSendSingleMsg: enable: false timeout: 5 failedContinue: true - # Only the contentType in allowedTypes will send the callback. - # Supports two formats: a single type or a range. The range is defined by the lower and upper bounds connected with a hyphen ("-"). - # e.g. allowedTypes: [1, 100, 200-500, 600-700] means that only contentType within the range - # {1, 100} ∪ [200, 500] ∪ [600, 700] will be allowed through the filter. - # If not set, all contentType messages will through this filter. - allowedTypes: [] # Only the contentType not in deniedTypes will send the callback. - # Supports two formats, same as allowedTypes. # If not set, all contentType messages will through this filter. deniedTypes: [] beforeUpdateUserInfoEx: @@ -23,31 +16,30 @@ afterUpdateUserInfoEx: afterSendSingleMsg: enable: false timeout: 5 - # Only the senID/recvID specified in attentionIds will send the callback + # Only the recvID specified in attentionIds will send the callback # if not set, all user messages will be callback attentionIds: [] # See beforeSendSingleMsg comment. - allowedTypes: [] deniedTypes: [] beforeSendGroupMsg: enable: false timeout: 5 failedContinue: true # See beforeSendSingleMsg comment. - allowedTypes: [] deniedTypes: [] beforeMsgModify: enable: false timeout: 5 failedContinue: true # See beforeSendSingleMsg comment. - allowedTypes: [] deniedTypes: [] afterSendGroupMsg: enable: false timeout: 5 + # Only the recvID specified in attentionIds will send the callback + # if not set, all user messages will be callback + attentionIds: [] # See beforeSendSingleMsg comment. - allowedTypes: [] deniedTypes: [] afterUserOnline: enable: false diff --git a/go.mod b/go.mod index 7145519fb..20cf3b202 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/gorilla/websocket v1.5.1 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/mitchellh/mapstructure v1.5.0 - github.com/openimsdk/protocol v0.0.72-alpha.71 + github.com/openimsdk/protocol v0.0.72-alpha.74 github.com/openimsdk/tools v0.0.50-alpha.74 github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 @@ -219,5 +219,3 @@ require ( golang.org/x/crypto v0.27.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect ) - -//replace github.com/openimsdk/tools => /Users/chao/Desktop/code/tools diff --git a/go.sum b/go.sum index f67869c0d..63dfe9236 100644 --- a/go.sum +++ b/go.sum @@ -347,8 +347,8 @@ github.com/onsi/gomega v1.25.0 h1:Vw7br2PCDYijJHSfBOWhov+8cAnUf8MfMaIOV323l6Y= github.com/onsi/gomega v1.25.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdMPGhM= github.com/openimsdk/gomake v0.0.15-alpha.2 h1:5Q8yl8ezy2yx+q8/ucU/t4kJnDfCzNOrkXcDACCqtyM= github.com/openimsdk/gomake v0.0.15-alpha.2/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= -github.com/openimsdk/protocol v0.0.72-alpha.71 h1:R3utzOlqepaJWTAmnfJi4ccUM/XIoFasSyjQMOipM70= -github.com/openimsdk/protocol v0.0.72-alpha.71/go.mod h1:WF7EuE55vQvpyUAzDXcqg+B+446xQyEba0X35lTINmw= +github.com/openimsdk/protocol v0.0.72-alpha.74 h1:cGycdzEOxjPuaeoQhIWEKKVf5zp1I+wx7ZnBemjCJJI= +github.com/openimsdk/protocol v0.0.72-alpha.74/go.mod h1:WF7EuE55vQvpyUAzDXcqg+B+446xQyEba0X35lTINmw= github.com/openimsdk/tools v0.0.50-alpha.74 h1:yh10SiMiivMEjicEQg+QAsH4pvaO+4noMPdlw+ew0Kc= github.com/openimsdk/tools v0.0.50-alpha.74/go.mod h1:n2poR3asX1e1XZce4O+MOWAp+X02QJRFvhcLCXZdzRo= github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= diff --git a/internal/api/msg.go b/internal/api/msg.go index b21e792db..927247cca 100644 --- a/internal/api/msg.go +++ b/internal/api/msg.go @@ -15,12 +15,16 @@ package api import ( + "encoding/base64" + "encoding/json" + "github.com/gin-gonic/gin" "github.com/go-playground/validator/v10" "github.com/mitchellh/mapstructure" "github.com/openimsdk/open-im-server/v3/pkg/apistruct" "github.com/openimsdk/open-im-server/v3/pkg/authverify" "github.com/openimsdk/open-im-server/v3/pkg/common/config" + "github.com/openimsdk/open-im-server/v3/pkg/common/webhook" "github.com/openimsdk/open-im-server/v3/pkg/rpcli" "github.com/openimsdk/protocol/constant" "github.com/openimsdk/protocol/msg" @@ -368,6 +372,83 @@ func (m *MessageApi) BatchSendMsg(c *gin.Context) { apiresp.GinSuccess(c, resp) } +func (m *MessageApi) SendSimpleMessage(c *gin.Context) { + encodedKey, ok := c.GetQuery(webhook.Key) + if !ok { + apiresp.GinError(c, errs.ErrArgs.WithDetail("missing key in query").Wrap()) + return + } + + decodedData, err := base64.StdEncoding.DecodeString(encodedKey) + if err != nil { + apiresp.GinError(c, errs.ErrArgs.WithDetail(err.Error()).Wrap()) + return + } + var ( + req apistruct.SendSingleMsgReq + keyMsgData apistruct.KeyMsgData + + sendID string + sessionType int32 + recvID string + ) + err = json.Unmarshal(decodedData, &keyMsgData) + if err != nil { + apiresp.GinError(c, errs.ErrArgs.WithDetail(err.Error()).Wrap()) + return + } + if keyMsgData.GroupID != "" { + sessionType = constant.ReadGroupChatType + sendID = req.SendID + } else { + sessionType = constant.SingleChatType + sendID = keyMsgData.RecvID + recvID = keyMsgData.SendID + } + // check param + if keyMsgData.SendID == "" { + apiresp.GinError(c, errs.ErrArgs.WithDetail("missing recvID or GroupID").Wrap()) + return + } + if sendID == "" { + apiresp.GinError(c, errs.ErrArgs.WithDetail("missing sendID").Wrap()) + return + } + + msgData := &sdkws.MsgData{ + SendID: sendID, + RecvID: recvID, + GroupID: keyMsgData.GroupID, + ClientMsgID: idutil.GetMsgIDByMD5(sendID), + SenderPlatformID: constant.AdminPlatformID, + SessionType: sessionType, + MsgFrom: constant.UserMsgType, + ContentType: constant.Text, + Content: []byte(req.Content), + OfflinePushInfo: req.OfflinePushInfo, + Ex: req.Ex, + } + + respPb, err := m.Client.SendMsg(c, &msg.SendMsgReq{MsgData: msgData}) + if err != nil { + apiresp.GinError(c, err) + return + } + + var status = constant.MsgSendSuccessed + + _, err = m.Client.SetSendMsgStatus(c, &msg.SetSendMsgStatusReq{ + Status: int32(status), + }) + + if err != nil { + apiresp.GinError(c, err) + return + } + + apiresp.GinSuccess(c, respPb) +} + func (m *MessageApi) CheckMsgIsSendSuccess(c *gin.Context) { a2r.Call(c, msg.MsgClient.GetSendMsgStatus, m.Client) } diff --git a/internal/rpc/msg/callback.go b/internal/rpc/msg/callback.go index c66dd6ca9..adf1ff735 100644 --- a/internal/rpc/msg/callback.go +++ b/internal/rpc/msg/callback.go @@ -16,7 +16,11 @@ package msg import ( "context" + "encoding/base64" + + "github.com/openimsdk/open-im-server/v3/pkg/apistruct" "github.com/openimsdk/open-im-server/v3/pkg/common/webhook" + "github.com/openimsdk/tools/utils/stringutil" cbapi "github.com/openimsdk/open-im-server/v3/pkg/callbackstruct" "github.com/openimsdk/open-im-server/v3/pkg/common/config" @@ -94,7 +98,7 @@ func (m *msgServer) webhookAfterSendSingleMsg(ctx context.Context, after *config CommonCallbackReq: toCommonCallback(ctx, msg, cbapi.CallbackAfterSendSingleMsgCommand), RecvID: msg.MsgData.RecvID, } - m.webhookClient.AsyncPost(ctx, cbReq.GetCallbackCommand(), cbReq, &cbapi.CallbackAfterSendSingleMsgResp{}, after) + m.webhookClient.AsyncPostWithQuery(ctx, cbReq.GetCallbackCommand(), cbReq, &cbapi.CallbackAfterSendSingleMsgResp{}, after, buildKeyMsgDataQuery(msg.MsgData)) } func (m *msgServer) webhookBeforeSendGroupMsg(ctx context.Context, before *config.BeforeConfig, msg *pbchat.SendMsgReq) error { @@ -128,7 +132,8 @@ func (m *msgServer) webhookAfterSendGroupMsg(ctx context.Context, after *config. CommonCallbackReq: toCommonCallback(ctx, msg, cbapi.CallbackAfterSendGroupMsgCommand), GroupID: msg.MsgData.GroupID, } - m.webhookClient.AsyncPost(ctx, cbReq.GetCallbackCommand(), cbReq, &cbapi.CallbackAfterSendGroupMsgResp{}, after) + + m.webhookClient.AsyncPostWithQuery(ctx, cbReq.GetCallbackCommand(), cbReq, &cbapi.CallbackAfterSendGroupMsgResp{}, after, buildKeyMsgDataQuery(msg.MsgData)) } func (m *msgServer) webhookBeforeMsgModify(ctx context.Context, before *config.BeforeConfig, msg *pbchat.SendMsgReq) error { @@ -192,3 +197,15 @@ func (m *msgServer) webhookAfterRevokeMsg(ctx context.Context, after *config.Aft } m.webhookClient.AsyncPost(ctx, callbackReq.GetCallbackCommand(), callbackReq, &cbapi.CallbackAfterRevokeMsgResp{}, after) } + +func buildKeyMsgDataQuery(msg *sdkws.MsgData) map[string]string { + keyMsgData := apistruct.KeyMsgData{ + SendID: msg.SendID, + RecvID: msg.RecvID, + GroupID: msg.GroupID, + } + + return map[string]string{ + webhook.Key: base64.StdEncoding.EncodeToString(stringutil.StructToJsonBytes(keyMsgData)), + } +} diff --git a/internal/rpc/msg/filter.go b/internal/rpc/msg/filter.go index ed1a488f1..36511ec7b 100644 --- a/internal/rpc/msg/filter.go +++ b/internal/rpc/msg/filter.go @@ -1,11 +1,13 @@ package msg import ( + "strconv" + "strings" + "github.com/openimsdk/open-im-server/v3/pkg/common/config" + "github.com/openimsdk/protocol/constant" pbchat "github.com/openimsdk/protocol/msg" "github.com/openimsdk/tools/utils/datautil" - "strconv" - "strings" ) const ( @@ -13,28 +15,50 @@ const ( ) func filterAfterMsg(msg *pbchat.SendMsgReq, after *config.AfterConfig) bool { - return filterMsg(msg, after.AttentionIds, after.AllowedTypes, after.DeniedTypes) + return filterMsg(msg, after.AttentionIds, after.DeniedTypes) } func filterBeforeMsg(msg *pbchat.SendMsgReq, before *config.BeforeConfig) bool { - return filterMsg(msg, nil, before.AllowedTypes, before.DeniedTypes) + return filterMsg(msg, nil, before.DeniedTypes) } -func filterMsg(msg *pbchat.SendMsgReq, attentionIds, allowedTypes, deniedTypes []string) bool { +func filterMsg(msg *pbchat.SendMsgReq, attentionIds []string, deniedTypes []int32) bool { // According to the attentionIds configuration, only some users are sent - if len(attentionIds) != 0 && !datautil.Contains([]string{msg.MsgData.SendID, msg.MsgData.RecvID}, attentionIds...) { + if len(attentionIds) != 0 && !datautil.Contain(msg.MsgData.RecvID, attentionIds...) { return false } - if len(allowedTypes) != 0 && !isInInterval(msg.MsgData.ContentType, allowedTypes) { + + if defaultDeniedTypes(msg.MsgData.ContentType) { return false } - if len(deniedTypes) != 0 && isInInterval(msg.MsgData.ContentType, deniedTypes) { + + if len(deniedTypes) != 0 && datautil.Contain(msg.MsgData.ContentType, deniedTypes...) { return false } + //if len(allowedTypes) != 0 && !isInInterval(msg.MsgData.ContentType, allowedTypes) { + // return false + //} + //if len(deniedTypes) != 0 && isInInterval(msg.MsgData.ContentType, deniedTypes) { + // return false + //} return true } -func isInInterval(contentType int32, interval []string) bool { +func defaultDeniedTypes(contentType int32) bool { + if contentType >= constant.NotificationBegin && contentType <= constant.NotificationEnd { + return true + } + if contentType == constant.Typing { + return true + } + return false +} + +// isInInterval if data is in interval +// Supports two formats: a single type or a range. The range is defined by the lower and upper bounds connected with a hyphen ("-") +// e.g. [1, 100, 200-500, 600-700] means that only data within the range +// {1, 100} ∪ [200, 500] ∪ [600, 700] will return true. +func isInInterval(data int32, interval []string) bool { for _, v := range interval { if strings.Contains(v, separator) { // is interval @@ -50,7 +74,7 @@ func isInInterval(contentType int32, interval []string) bool { if err != nil { continue } - if datautil.BetweenEq(int(contentType), bottom, top) { + if datautil.BetweenEq(int(data), bottom, top) { return true } } else { @@ -58,7 +82,7 @@ func isInInterval(contentType int32, interval []string) bool { if err != nil { continue } - if int(contentType) == iv { + if int(data) == iv { return true } } diff --git a/internal/rpc/user/user.go b/internal/rpc/user/user.go index bde9a3f8f..07e3c6201 100644 --- a/internal/rpc/user/user.go +++ b/internal/rpc/user/user.go @@ -566,7 +566,7 @@ func (s *userServer) SearchNotificationAccount(ctx context.Context, req *pbuser. } // Convert users to response format - resp := s.userModelToResp(users, req.Pagination) + resp := s.userModelToResp(users, req.Pagination, req.AppManagerLevel) if resp.Total != 0 { return resp, nil } @@ -576,17 +576,24 @@ func (s *userServer) SearchNotificationAccount(ctx context.Context, req *pbuser. if err != nil { return nil, err } - resp = s.userModelToResp(users, req.Pagination) + resp = s.userModelToResp(users, req.Pagination, req.AppManagerLevel) return resp, nil } // If no keyword, find users with notification settings - users, err = s.db.FindNotification(ctx, constant.AppNotificationAdmin) - if err != nil { - return nil, err + if req.AppManagerLevel != nil { + users, err = s.db.FindNotification(ctx, int64(*req.AppManagerLevel)) + if err != nil { + return nil, err + } + } else { + users, err = s.db.FindSystemAccount(ctx) + if err != nil { + return nil, err + } } - resp := s.userModelToResp(users, req.Pagination) + resp := s.userModelToResp(users, req.Pagination, req.AppManagerLevel) return resp, nil } @@ -625,11 +632,16 @@ func (s *userServer) genUserID() string { return string(data) } -func (s *userServer) userModelToResp(users []*tablerelation.User, pagination pagination.Pagination) *pbuser.SearchNotificationAccountResp { +func (s *userServer) userModelToResp(users []*tablerelation.User, pagination pagination.Pagination, appManagerLevel *int32) *pbuser.SearchNotificationAccountResp { accounts := make([]*pbuser.NotificationAccountInfo, 0) var total int64 for _, v := range users { if v.AppMangerLevel >= constant.AppNotificationAdmin && !datautil.Contain(v.UserID, s.config.Share.IMAdminUserID...) { + if appManagerLevel != nil { + if v.AppMangerLevel != *appManagerLevel { + continue + } + } temp := &pbuser.NotificationAccountInfo{ UserID: v.UserID, FaceURL: v.FaceURL, diff --git a/pkg/apistruct/manage.go b/pkg/apistruct/manage.go index f4deb9fb1..4f1d5863f 100644 --- a/pkg/apistruct/manage.go +++ b/pkg/apistruct/manage.go @@ -111,6 +111,21 @@ type BatchSendMsgResp struct { FailedIDs []string `json:"failedUserIDs"` } +// SendSingleMsgReq defines the structure for sending a message to multiple recipients. +type SendSingleMsgReq struct { + // groupMsg should appoint sendID + SendID string `json:"sendID"` + Content string `json:"content" binding:"required"` + OfflinePushInfo *sdkws.OfflinePushInfo `json:"offlinePushInfo"` + Ex string `json:"ex"` +} + +type KeyMsgData struct { + SendID string `json:"sendID"` + RecvID string `json:"recvID"` + GroupID string `json:"groupID"` +} + // SingleReturnResult encapsulates the result of a single message send attempt. type SingleReturnResult struct { // ServerMsgID is the message identifier on the server-side. diff --git a/pkg/common/config/config.go b/pkg/common/config/config.go index c7a7b2d95..b47e3db68 100644 --- a/pkg/common/config/config.go +++ b/pkg/common/config/config.go @@ -334,19 +334,17 @@ type Redis struct { } type BeforeConfig struct { - Enable bool `yaml:"enable"` - Timeout int `yaml:"timeout"` - FailedContinue bool `yaml:"failedContinue"` - AllowedTypes []string `yaml:"allowedTypes"` - DeniedTypes []string `yaml:"deniedTypes"` + Enable bool `yaml:"enable"` + Timeout int `yaml:"timeout"` + FailedContinue bool `yaml:"failedContinue"` + DeniedTypes []int32 `yaml:"deniedTypes"` } type AfterConfig struct { Enable bool `yaml:"enable"` Timeout int `yaml:"timeout"` AttentionIds []string `yaml:"attentionIds"` - AllowedTypes []string `yaml:"allowedTypes"` - DeniedTypes []string `yaml:"deniedTypes"` + DeniedTypes []int32 `yaml:"deniedTypes"` } type Share struct { diff --git a/pkg/common/storage/controller/user.go b/pkg/common/storage/controller/user.go index 3f34481a3..a8ef1033e 100644 --- a/pkg/common/storage/controller/user.go +++ b/pkg/common/storage/controller/user.go @@ -20,6 +20,7 @@ import ( "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" + "github.com/openimsdk/protocol/constant" "github.com/openimsdk/tools/db/pagination" "github.com/openimsdk/tools/db/tx" "github.com/openimsdk/tools/utils/datautil" @@ -37,8 +38,10 @@ type UserDatabase interface { Find(ctx context.Context, userIDs []string) (users []*model.User, err error) // Find userInfo By Nickname FindByNickname(ctx context.Context, nickname string) (users []*model.User, err error) - // Find notificationAccounts + // FindNotification find system account by level FindNotification(ctx context.Context, level int64) (users []*model.User, err error) + // FindSystemAccount find all system account + FindSystemAccount(ctx context.Context) (users []*model.User, err error) // Create Insert multiple external guarantees that the userID is not repeated and does not exist in the storage Create(ctx context.Context, users []*model.User) (err error) // UpdateByMap update (zero value) external guarantee userID exists @@ -139,6 +142,10 @@ func (u *userDatabase) FindNotification(ctx context.Context, level int64) (users return u.userDB.TakeNotification(ctx, level) } +func (u *userDatabase) FindSystemAccount(ctx context.Context) (users []*model.User, err error) { + return u.userDB.TakeGTEAppManagerLevel(ctx, constant.AppNotificationAdmin) +} + // Create Insert multiple external guarantees that the userID is not repeated and does not exist in the storage. func (u *userDatabase) Create(ctx context.Context, users []*model.User) (err error) { return u.tx.Transaction(ctx, func(ctx context.Context) error { diff --git a/pkg/common/storage/database/mgo/user.go b/pkg/common/storage/database/mgo/user.go index ee92b7554..a08765baf 100644 --- a/pkg/common/storage/database/mgo/user.go +++ b/pkg/common/storage/database/mgo/user.go @@ -16,9 +16,10 @@ package mgo import ( "context" + "time" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" - "time" "github.com/openimsdk/protocol/user" "github.com/openimsdk/tools/db/mongoutil" @@ -71,6 +72,10 @@ func (u *UserMgo) TakeNotification(ctx context.Context, level int64) (user []*mo return mongoutil.Find[*model.User](ctx, u.coll, bson.M{"app_manger_level": level}) } +func (u *UserMgo) TakeGTEAppManagerLevel(ctx context.Context, level int64) (user []*model.User, err error) { + return mongoutil.Find[*model.User](ctx, u.coll, bson.M{"app_manager_level": bson.M{"$gte": level}}) +} + func (u *UserMgo) TakeByNickname(ctx context.Context, nickname string) (user []*model.User, err error) { return mongoutil.Find[*model.User](ctx, u.coll, bson.M{"nickname": nickname}) } diff --git a/pkg/common/storage/database/user.go b/pkg/common/storage/database/user.go index 4ddc8285f..c597424b9 100644 --- a/pkg/common/storage/database/user.go +++ b/pkg/common/storage/database/user.go @@ -16,10 +16,11 @@ package database import ( "context" + "time" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" "github.com/openimsdk/protocol/user" "github.com/openimsdk/tools/db/pagination" - "time" ) type User interface { @@ -28,6 +29,7 @@ type User interface { Find(ctx context.Context, userIDs []string) (users []*model.User, err error) Take(ctx context.Context, userID string) (user *model.User, err error) TakeNotification(ctx context.Context, level int64) (user []*model.User, err error) + TakeGTEAppManagerLevel(ctx context.Context, level int64) (user []*model.User, err error) TakeByNickname(ctx context.Context, nickname string) (user []*model.User, err error) Page(ctx context.Context, pagination pagination.Pagination) (count int64, users []*model.User, err error) PageFindUser(ctx context.Context, level1 int64, level2 int64, pagination pagination.Pagination) (count int64, users []*model.User, err error) diff --git a/pkg/common/webhook/http_client.go b/pkg/common/webhook/http_client.go index e46f08806..0cd13f6e2 100644 --- a/pkg/common/webhook/http_client.go +++ b/pkg/common/webhook/http_client.go @@ -17,6 +17,9 @@ package webhook import ( "context" "encoding/json" + "net/http" + "net/url" + "github.com/openimsdk/open-im-server/v3/pkg/callbackstruct" "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/open-im-server/v3/pkg/common/servererrs" @@ -25,7 +28,6 @@ import ( "github.com/openimsdk/tools/mcontext" "github.com/openimsdk/tools/mq/memamq" "github.com/openimsdk/tools/utils/httputil" - "net/http" ) type Client struct { @@ -37,6 +39,8 @@ type Client struct { const ( webhookWorkerCount = 2 webhookBufferSize = 100 + + Key = "key" ) func NewWebhookClient(url string, options ...*memamq.MemoryQueue) *Client { @@ -66,6 +70,12 @@ func (c *Client) AsyncPost(ctx context.Context, command string, req callbackstru } } +func (c *Client) AsyncPostWithQuery(ctx context.Context, command string, req callbackstruct.CallbackReq, resp callbackstruct.CallbackResp, after *config.AfterConfig, queryParams map[string]string) { + if after.Enable { + c.queue.Push(func() { c.postWithQuery(ctx, command, req, resp, after.Timeout, queryParams) }) + } +} + func (c *Client) post(ctx context.Context, command string, input interface{}, output callbackstruct.CallbackResp, timeout int) error { ctx = mcontext.WithMustInfoCtx([]string{mcontext.GetOperationID(ctx), mcontext.GetOpUserID(ctx), mcontext.GetOpUserPlatform(ctx), mcontext.GetConnID(ctx)}) fullURL := c.url + "/" + command @@ -84,3 +94,41 @@ func (c *Client) post(ctx context.Context, command string, input interface{}, ou log.ZInfo(ctx, "webhook success", "url", fullURL, "input", input, "response", string(b)) return nil } + +func (c *Client) postWithQuery(ctx context.Context, command string, input interface{}, output callbackstruct.CallbackResp, timeout int, queryParams map[string]string) error { + ctx = mcontext.WithMustInfoCtx([]string{mcontext.GetOperationID(ctx), mcontext.GetOpUserID(ctx), mcontext.GetOpUserPlatform(ctx), mcontext.GetConnID(ctx)}) + fullURL := c.url + "/" + command + + parsedURL, err := url.Parse(fullURL) + if err != nil { + return servererrs.ErrNetwork.WrapMsg(err.Error(), "failed to parse URL", fullURL) + } + + query := parsedURL.Query() + + operationID, _ := ctx.Value(constant.OperationID).(string) + + for key, value := range queryParams { + query.Set(key, value) + } + + parsedURL.RawQuery = query.Encode() + + fullURL = parsedURL.String() + log.ZInfo(ctx, "webhook", "url", fullURL, "input", input, "config", timeout) + + b, err := c.client.Post(ctx, fullURL, map[string]string{constant.OperationID: operationID}, input, timeout) + if err != nil { + return servererrs.ErrNetwork.WrapMsg(err.Error(), "post url", fullURL) + } + + if err = json.Unmarshal(b, output); err != nil { + return servererrs.ErrData.WithDetail(err.Error() + " response format error") + } + if err := output.Parse(); err != nil { + return err + } + + log.ZInfo(ctx, "webhook success", "url", fullURL, "input", input, "response", string(b)) + return nil +} From 46f1a9c7a3bcfc602741c1cae81422cfcc8ed9b5 Mon Sep 17 00:00:00 2001 From: icey-yu <119291641+icey-yu@users.noreply.github.com> Date: Mon, 24 Feb 2025 15:29:39 +0800 Subject: [PATCH 121/199] fix: PCAndOther multi login policy can`t get old clients correctly (#3158) --- internal/msggateway/ws_server.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/internal/msggateway/ws_server.go b/internal/msggateway/ws_server.go index 304cb3b5c..d69062bda 100644 --- a/internal/msggateway/ws_server.go +++ b/internal/msggateway/ws_server.go @@ -340,6 +340,15 @@ func (ws *WsServer) multiTerminalLoginChecker(clientOK bool, oldClients []*Clien if constant.PlatformIDToClass(newClient.PlatformID) == constant.TerminalPC { return } + clients, ok := ws.clients.GetAll(newClient.UserID) + clientOK = ok + oldClients = make([]*Client, 0, len(clients)) + for _, c := range clients { + if constant.PlatformIDToClass(c.PlatformID) == constant.TerminalPC { + continue + } + oldClients = append(oldClients, c) + } fallthrough case constant.AllLoginButSameTermKick: if !clientOK { From bf6d77a26715df973203be9a28616a2f6ae105ee Mon Sep 17 00:00:00 2001 From: chao <48119764+withchao@users.noreply.github.com> Date: Thu, 27 Feb 2025 16:17:00 +0800 Subject: [PATCH 122/199] feat: the default notification.yml is not configured properly (#3168) * pb * fix: Modifying other fields while setting IsPrivateChat does not take effect * fix: quote message error revoke * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * upgrading pkg tools * fix * fix * optimize log output * feat: support GetLastMessage * feat: support GetLastMessage * feat: s3 switch * feat: s3 switch * fix: GetUsersOnline * feat: SendBusinessNotification supported configuration parameters * feat: SendBusinessNotification supported configuration parameters * feat: SendBusinessNotification supported configuration parameters * feat: seq conversion failed without exiting * fix: DeleteDoc crash * fix: fill send time * fix: fill send time * fix: crash caused by withdrawing messages from users who have left the group * fix: user msg timestamp * seq read config * seq read config * fix: the source message of the reference is withdrawn, and the referenced message is deleted * feat: optimize the default notification.yml * fix: shouldPushOffline --- config/notification.yml | 26 +++++++++++++------------- internal/push/push_handler.go | 5 ++++- 2 files changed, 17 insertions(+), 14 deletions(-) diff --git a/config/notification.yml b/config/notification.yml index c3ae01539..4f58219be 100644 --- a/config/notification.yml +++ b/config/notification.yml @@ -31,7 +31,7 @@ joinGroupApplication: reliabilityLevel: 1 unreadCount: false offlinePush: - enable: false + enable: true title: joinGroupApplication title desc: joinGroupApplication desc ext: joinGroupApplication ext @@ -51,7 +51,7 @@ groupApplicationAccepted: reliabilityLevel: 1 unreadCount: false offlinePush: - enable: false + enable: true title: groupApplicationAccepted title desc: groupApplicationAccepted desc ext: groupApplicationAccepted ext @@ -61,7 +61,7 @@ groupApplicationRejected: reliabilityLevel: 1 unreadCount: false offlinePush: - enable: false + enable: true title: groupApplicationRejected title desc: groupApplicationRejected desc ext: groupApplicationRejected ext @@ -198,7 +198,7 @@ friendApplicationAdded: reliabilityLevel: 1 unreadCount: false offlinePush: - enable: false + enable: true title: Somebody applies to add you as a friend desc: Somebody applies to add you as a friend ext: Somebody applies to add you as a friend @@ -228,7 +228,7 @@ friendAdded: reliabilityLevel: 1 unreadCount: false offlinePush: - enable: true + enable: false title: We have become friends desc: We have become friends ext: We have become friends @@ -238,7 +238,7 @@ friendDeleted: reliabilityLevel: 1 unreadCount: false offlinePush: - enable: true + enable: false title: deleted a friend desc: deleted a friend ext: deleted a friend @@ -248,7 +248,7 @@ friendRemarkSet: reliabilityLevel: 1 unreadCount: false offlinePush: - enable: true + enable: false title: Your friend's profile has been changed desc: Your friend's profile has been changed ext: Your friend's profile has been changed @@ -258,7 +258,7 @@ blackAdded: reliabilityLevel: 1 unreadCount: false offlinePush: - enable: true + enable: false title: blocked a user desc: blocked a user ext: blocked a user @@ -268,7 +268,7 @@ blackDeleted: reliabilityLevel: 1 unreadCount: false offlinePush: - enable: true + enable: false title: Remove a blocked user desc: Remove a blocked user ext: Remove a blocked user @@ -278,7 +278,7 @@ friendInfoUpdated: reliabilityLevel: 1 unreadCount: false offlinePush: - enable: true + enable: false title: friend info updated desc: friend info updated ext: friend info updated @@ -289,7 +289,7 @@ userInfoUpdated: reliabilityLevel: 1 unreadCount: false offlinePush: - enable: true + enable: false title: userInfo updated desc: userInfo updated ext: userInfo updated @@ -310,7 +310,7 @@ conversationChanged: reliabilityLevel: 1 unreadCount: false offlinePush: - enable: true + enable: false title: conversation changed desc: conversation changed ext: conversation changed @@ -320,7 +320,7 @@ conversationSetPrivate: reliabilityLevel: 1 unreadCount: false offlinePush: - enable: true + enable: false title: burn after reading desc: burn after reading ext: burn after reading diff --git a/internal/push/push_handler.go b/internal/push/push_handler.go index 707782c70..418c4c7f2 100644 --- a/internal/push/push_handler.go +++ b/internal/push/push_handler.go @@ -187,7 +187,10 @@ func (c *ConsumerHandler) shouldPushOffline(_ context.Context, msg *sdkws.MsgDat if !isOfflinePush { return false } - if msg.ContentType == constant.SignalingNotification { + switch msg.ContentType { + case constant.RoomParticipantsConnectedNotification: + return false + case constant.RoomParticipantsDisconnectedNotification: return false } return true From b7f88138bdc3560669b2074e566e52e32b72fe03 Mon Sep 17 00:00:00 2001 From: OpenIM-Gordon <1432970085@qq.com> Date: Fri, 28 Feb 2025 16:22:00 +0800 Subject: [PATCH 123/199] feat: add a new message type: Markdown text (#3162) --- go.mod | 2 +- go.sum | 4 ++-- internal/api/msg.go | 3 +++ internal/rpc/msg/verify.go | 31 +++++++------------------------ pkg/apistruct/msg.go | 4 ++++ 5 files changed, 17 insertions(+), 27 deletions(-) diff --git a/go.mod b/go.mod index 20cf3b202..0bdf09340 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/gorilla/websocket v1.5.1 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/mitchellh/mapstructure v1.5.0 - github.com/openimsdk/protocol v0.0.72-alpha.74 + github.com/openimsdk/protocol v0.0.72-alpha.75 github.com/openimsdk/tools v0.0.50-alpha.74 github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 diff --git a/go.sum b/go.sum index 63dfe9236..e8f1300f4 100644 --- a/go.sum +++ b/go.sum @@ -347,8 +347,8 @@ github.com/onsi/gomega v1.25.0 h1:Vw7br2PCDYijJHSfBOWhov+8cAnUf8MfMaIOV323l6Y= github.com/onsi/gomega v1.25.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdMPGhM= github.com/openimsdk/gomake v0.0.15-alpha.2 h1:5Q8yl8ezy2yx+q8/ucU/t4kJnDfCzNOrkXcDACCqtyM= github.com/openimsdk/gomake v0.0.15-alpha.2/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= -github.com/openimsdk/protocol v0.0.72-alpha.74 h1:cGycdzEOxjPuaeoQhIWEKKVf5zp1I+wx7ZnBemjCJJI= -github.com/openimsdk/protocol v0.0.72-alpha.74/go.mod h1:WF7EuE55vQvpyUAzDXcqg+B+446xQyEba0X35lTINmw= +github.com/openimsdk/protocol v0.0.72-alpha.75 h1:WlmBn8g2Fvv21g8TEFVbolmbw2rU0sN9kj6sQaDK7cA= +github.com/openimsdk/protocol v0.0.72-alpha.75/go.mod h1:WF7EuE55vQvpyUAzDXcqg+B+446xQyEba0X35lTINmw= github.com/openimsdk/tools v0.0.50-alpha.74 h1:yh10SiMiivMEjicEQg+QAsH4pvaO+4noMPdlw+ew0Kc= github.com/openimsdk/tools v0.0.50-alpha.74/go.mod h1:n2poR3asX1e1XZce4O+MOWAp+X02QJRFvhcLCXZdzRo= github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= diff --git a/internal/api/msg.go b/internal/api/msg.go index 927247cca..1316de943 100644 --- a/internal/api/msg.go +++ b/internal/api/msg.go @@ -21,6 +21,7 @@ import ( "github.com/gin-gonic/gin" "github.com/go-playground/validator/v10" "github.com/mitchellh/mapstructure" + "github.com/openimsdk/open-im-server/v3/pkg/apistruct" "github.com/openimsdk/open-im-server/v3/pkg/authverify" "github.com/openimsdk/open-im-server/v3/pkg/common/config" @@ -175,6 +176,8 @@ func (m *MessageApi) getSendMsgReq(c *gin.Context, req apistruct.SendMsg) (sendM data = apistruct.AtElem{} case constant.Custom: data = apistruct.CustomElem{} + case constant.MarkdownText: + data = apistruct.MarkdownTextElem{} case constant.OANotification: data = apistruct.OANotificationElem{} req.SessionType = constant.NotificationChatType diff --git a/internal/rpc/msg/verify.go b/internal/rpc/msg/verify.go index f6c3147ba..a492232a4 100644 --- a/internal/rpc/msg/verify.go +++ b/internal/rpc/msg/verify.go @@ -16,13 +16,14 @@ package msg import ( "context" + "math/rand" + "strconv" + "time" + "github.com/openimsdk/open-im-server/v3/pkg/common/servererrs" "github.com/openimsdk/tools/utils/datautil" "github.com/openimsdk/tools/utils/encrypt" "github.com/openimsdk/tools/utils/timeutil" - "math/rand" - "strconv" - "time" "github.com/openimsdk/protocol/constant" "github.com/openimsdk/protocol/msg" @@ -137,27 +138,9 @@ func (m *msgServer) encapsulateMsgData(msg *sdkws.MsgData) { msg.SendTime = timeutil.GetCurrentTimestampByMill() } switch msg.ContentType { - case constant.Text: - fallthrough - case constant.Picture: - fallthrough - case constant.Voice: - fallthrough - case constant.Video: - fallthrough - case constant.File: - fallthrough - case constant.AtText: - fallthrough - case constant.Merger: - fallthrough - case constant.Card: - fallthrough - case constant.Location: - fallthrough - case constant.Custom: - fallthrough - case constant.Quote: + case constant.Text, constant.Picture, constant.Voice, constant.Video, + constant.File, constant.AtText, constant.Merger, constant.Card, + constant.Location, constant.Custom, constant.Quote, constant.AdvancedText, constant.MarkdownText: case constant.Revoke: datautil.SetSwitchFromOptions(msg.Options, constant.IsUnreadCount, false) datautil.SetSwitchFromOptions(msg.Options, constant.IsOfflinePush, false) diff --git a/pkg/apistruct/msg.go b/pkg/apistruct/msg.go index 44f157b6d..d818e769c 100644 --- a/pkg/apistruct/msg.go +++ b/pkg/apistruct/msg.go @@ -83,6 +83,10 @@ type TextElem struct { Content string `json:"content" validate:"required"` } +type MarkdownTextElem struct { + Content string `mapstructure:"content" validate:"required"` +} + type StreamMsgElem struct { Type string `mapstructure:"type" validate:"required"` Content string `mapstructure:"content" validate:"required"` From 4d2fce0e52e9ffb8bb988c026c32bd10c33ebc0f Mon Sep 17 00:00:00 2001 From: OpenIM-Gordon <1432970085@qq.com> Date: Fri, 28 Feb 2025 16:38:57 +0800 Subject: [PATCH 124/199] =?UTF-8?q?feat:=20add=20a=20field=20to=20specify?= =?UTF-8?q?=20whether=20to=20send=20a=20notification=20message=20w?= =?UTF-8?q?=E2=80=A6=20(#3163)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: add a field to specify whether to send a notification message when creating a group. * feat: add a field to specify whether to send a notification message when creating a group. --------- Co-authored-by: Monet Lee --- go.mod | 2 +- go.sum | 4 +-- internal/api/msg.go | 2 +- internal/rpc/conversation/notification.go | 5 ++-- internal/rpc/group/group.go | 13 +++++----- internal/rpc/group/notification.go | 31 ++++++++++++----------- internal/rpc/msg/notification.go | 6 ++--- internal/rpc/msg/server.go | 9 ++++--- internal/rpc/relation/notification.go | 5 ++-- internal/rpc/user/notification.go | 5 ++-- pkg/common/config/parse.go | 8 ++++-- pkg/notification/msg.go | 17 ++++++++----- 12 files changed, 61 insertions(+), 46 deletions(-) diff --git a/go.mod b/go.mod index 0bdf09340..0b677d1b0 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/gorilla/websocket v1.5.1 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/mitchellh/mapstructure v1.5.0 - github.com/openimsdk/protocol v0.0.72-alpha.75 + github.com/openimsdk/protocol v0.0.72-alpha.77 github.com/openimsdk/tools v0.0.50-alpha.74 github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 diff --git a/go.sum b/go.sum index e8f1300f4..1b132db13 100644 --- a/go.sum +++ b/go.sum @@ -347,8 +347,8 @@ github.com/onsi/gomega v1.25.0 h1:Vw7br2PCDYijJHSfBOWhov+8cAnUf8MfMaIOV323l6Y= github.com/onsi/gomega v1.25.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdMPGhM= github.com/openimsdk/gomake v0.0.15-alpha.2 h1:5Q8yl8ezy2yx+q8/ucU/t4kJnDfCzNOrkXcDACCqtyM= github.com/openimsdk/gomake v0.0.15-alpha.2/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= -github.com/openimsdk/protocol v0.0.72-alpha.75 h1:WlmBn8g2Fvv21g8TEFVbolmbw2rU0sN9kj6sQaDK7cA= -github.com/openimsdk/protocol v0.0.72-alpha.75/go.mod h1:WF7EuE55vQvpyUAzDXcqg+B+446xQyEba0X35lTINmw= +github.com/openimsdk/protocol v0.0.72-alpha.77 h1:ka7TeOpNKKqYTz8aWR8nZnz2rnzgsO9mw6xOQcVXwbE= +github.com/openimsdk/protocol v0.0.72-alpha.77/go.mod h1:WF7EuE55vQvpyUAzDXcqg+B+446xQyEba0X35lTINmw= github.com/openimsdk/tools v0.0.50-alpha.74 h1:yh10SiMiivMEjicEQg+QAsH4pvaO+4noMPdlw+ew0Kc= github.com/openimsdk/tools v0.0.50-alpha.74/go.mod h1:n2poR3asX1e1XZce4O+MOWAp+X02QJRFvhcLCXZdzRo= github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= diff --git a/internal/api/msg.go b/internal/api/msg.go index 1316de943..a7c2c888f 100644 --- a/internal/api/msg.go +++ b/internal/api/msg.go @@ -308,7 +308,7 @@ func (m *MessageApi) SendBusinessNotification(c *gin.Context) { IsSendMsg: req.SendMsg, ReliabilityLevel: *req.ReliabilityLevel, UnreadCount: false, - }), + }, nil), }, } respPb, err := m.Client.SendMsg(c, &sendMsgReq) diff --git a/internal/rpc/conversation/notification.go b/internal/rpc/conversation/notification.go index c6368b916..6e865ba42 100644 --- a/internal/rpc/conversation/notification.go +++ b/internal/rpc/conversation/notification.go @@ -16,6 +16,7 @@ package conversation import ( "context" + "github.com/openimsdk/open-im-server/v3/pkg/rpcli" "github.com/openimsdk/protocol/msg" @@ -26,11 +27,11 @@ import ( ) type ConversationNotificationSender struct { - *rpcclient.NotificationSender + *notification.NotificationSender } func NewConversationNotificationSender(conf *config.Notification, msgClient *rpcli.MsgClient) *ConversationNotificationSender { - return &ConversationNotificationSender{rpcclient.NewNotificationSender(conf, rpcclient.WithRpcClient(func(ctx context.Context, req *msg.SendMsgReq) (*msg.SendMsgResp, error) { + return &ConversationNotificationSender{notification.NewNotificationSender(conf, notification.WithRpcClient(func(ctx context.Context, req *msg.SendMsgReq) (*msg.SendMsgResp, error) { return msgClient.SendMsg(ctx, req) }))} } diff --git a/internal/rpc/group/group.go b/internal/rpc/group/group.go index 8ad4a949f..535eed52a 100644 --- a/internal/rpc/group/group.go +++ b/internal/rpc/group/group.go @@ -26,6 +26,8 @@ import ( "github.com/openimsdk/open-im-server/v3/pkg/dbbuild" "github.com/openimsdk/open-im-server/v3/pkg/rpcli" + "google.golang.org/grpc" + "github.com/openimsdk/open-im-server/v3/pkg/authverify" "github.com/openimsdk/open-im-server/v3/pkg/callbackstruct" "github.com/openimsdk/open-im-server/v3/pkg/common/config" @@ -51,7 +53,6 @@ import ( "github.com/openimsdk/tools/mw/specialerror" "github.com/openimsdk/tools/utils/datautil" "github.com/openimsdk/tools/utils/encrypt" - "google.golang.org/grpc" ) type groupServer struct { @@ -284,7 +285,7 @@ func (g *groupServer) CreateGroup(ctx context.Context, req *pbgroup.CreateGroupR break } } - g.notification.GroupCreatedNotification(ctx, tips) + g.notification.GroupCreatedNotification(ctx, tips, req.SendNotification) if req.GroupInfo.Notification != "" { g.notification.GroupInfoSetAnnouncementNotification(ctx, &sdkws.GroupInfoSetAnnouncementTips{ @@ -447,7 +448,7 @@ func (g *groupServer) InviteUserToGroup(ctx context.Context, req *pbgroup.Invite return nil, err } - if err = g.notification.GroupApplicationAgreeMemberEnterNotification(ctx, req.GroupID, opUserID, req.InvitedUserIDs...); err != nil { + if err = g.notification.GroupApplicationAgreeMemberEnterNotification(ctx, req.GroupID, req.SendNotification, opUserID, req.InvitedUserIDs...); err != nil { return nil, err } return &pbgroup.InviteUserToGroupResp{}, nil @@ -613,7 +614,7 @@ func (g *groupServer) KickGroupMember(ctx context.Context, req *pbgroup.KickGrou for _, userID := range req.KickedUserIDs { tips.KickedUserList = append(tips.KickedUserList, convert.Db2PbGroupMember(memberMap[userID])) } - g.notification.MemberKickedNotification(ctx, tips) + g.notification.MemberKickedNotification(ctx, tips, req.SendNotification) if err := g.deleteMemberAndSetConversationSeq(ctx, req.GroupID, req.KickedUserIDs); err != nil { return nil, err } @@ -822,7 +823,7 @@ func (g *groupServer) GroupApplicationResponse(ctx context.Context, req *pbgroup if member == nil { log.ZDebug(ctx, "GroupApplicationResponse", "member is nil") } else { - if err = g.notification.GroupApplicationAgreeMemberEnterNotification(ctx, req.GroupID, groupRequest.InviterUserID, req.FromUserID); err != nil { + if err = g.notification.GroupApplicationAgreeMemberEnterNotification(ctx, req.GroupID, nil, groupRequest.InviterUserID, req.FromUserID); err != nil { return nil, err } } @@ -1368,7 +1369,7 @@ func (g *groupServer) DismissGroup(ctx context.Context, req *pbgroup.DismissGrou if mcontext.GetOpUserID(ctx) == owner.UserID { tips.OpUser = g.groupMemberDB2PB(owner, 0) } - g.notification.GroupDismissedNotification(ctx, tips) + g.notification.GroupDismissedNotification(ctx, tips, req.SendNotification) } membersID, err := g.db.FindGroupMemberUserID(ctx, group.GroupID) if err != nil { diff --git a/internal/rpc/group/notification.go b/internal/rpc/group/notification.go index 1aa5333b4..a879d3b4a 100644 --- a/internal/rpc/group/notification.go +++ b/internal/rpc/group/notification.go @@ -22,6 +22,8 @@ import ( "github.com/openimsdk/open-im-server/v3/pkg/rpcli" + "go.mongodb.org/mongo-driver/mongo" + "github.com/openimsdk/open-im-server/v3/pkg/authverify" "github.com/openimsdk/open-im-server/v3/pkg/common/convert" "github.com/openimsdk/open-im-server/v3/pkg/common/servererrs" @@ -41,7 +43,6 @@ import ( "github.com/openimsdk/tools/mcontext" "github.com/openimsdk/tools/utils/datautil" "github.com/openimsdk/tools/utils/stringutil" - "go.mongodb.org/mongo-driver/mongo" ) // GroupApplicationReceiver @@ -52,11 +53,11 @@ const ( func NewNotificationSender(db controller.GroupDatabase, config *Config, userClient *rpcli.UserClient, msgClient *rpcli.MsgClient, conversationClient *rpcli.ConversationClient) *NotificationSender { return &NotificationSender{ - NotificationSender: rpcclient.NewNotificationSender(&config.NotificationConfig, - rpcclient.WithRpcClient(func(ctx context.Context, req *msg.SendMsgReq) (*msg.SendMsgResp, error) { + NotificationSender: notification.NewNotificationSender(&config.NotificationConfig, + notification.WithRpcClient(func(ctx context.Context, req *msg.SendMsgReq) (*msg.SendMsgResp, error) { return msgClient.SendMsg(ctx, req) }), - rpcclient.WithUserRpcClient(userClient.GetUserInfo), + notification.WithUserRpcClient(userClient.GetUserInfo), ), getUsersInfo: func(ctx context.Context, userIDs []string) ([]common_user.CommonUser, error) { users, err := userClient.GetUsersInfo(ctx, userIDs) @@ -73,7 +74,7 @@ func NewNotificationSender(db controller.GroupDatabase, config *Config, userClie } type NotificationSender struct { - *rpcclient.NotificationSender + *notification.NotificationSender getUsersInfo func(ctx context.Context, userIDs []string) ([]common_user.CommonUser, error) db controller.GroupDatabase config *Config @@ -307,7 +308,7 @@ func (g *NotificationSender) setSortVersion(ctx context.Context, version *uint64 } } -func (g *NotificationSender) GroupCreatedNotification(ctx context.Context, tips *sdkws.GroupCreatedTips) { +func (g *NotificationSender) GroupCreatedNotification(ctx context.Context, tips *sdkws.GroupCreatedTips, sendNotification *bool) { var err error defer func() { if err != nil { @@ -318,7 +319,7 @@ func (g *NotificationSender) GroupCreatedNotification(ctx context.Context, tips return } g.setVersion(ctx, &tips.GroupMemberVersion, &tips.GroupMemberVersionID, database.GroupMemberVersionName, tips.Group.GroupID) - g.Notification(ctx, mcontext.GetOpUserID(ctx), tips.Group.GroupID, constant.GroupCreatedNotification, tips) + g.Notification(ctx, mcontext.GetOpUserID(ctx), tips.Group.GroupID, constant.GroupCreatedNotification, tips, notification.WithSendNotification(sendNotification)) } func (g *NotificationSender) GroupInfoSetNotification(ctx context.Context, tips *sdkws.GroupInfoSetTips) { @@ -332,7 +333,7 @@ func (g *NotificationSender) GroupInfoSetNotification(ctx context.Context, tips return } g.setVersion(ctx, &tips.GroupMemberVersion, &tips.GroupMemberVersionID, database.GroupMemberVersionName, tips.Group.GroupID) - g.Notification(ctx, mcontext.GetOpUserID(ctx), tips.Group.GroupID, constant.GroupInfoSetNotification, tips, rpcclient.WithRpcGetUserName()) + g.Notification(ctx, mcontext.GetOpUserID(ctx), tips.Group.GroupID, constant.GroupInfoSetNotification, tips, notification.WithRpcGetUserName()) } func (g *NotificationSender) GroupInfoSetNameNotification(ctx context.Context, tips *sdkws.GroupInfoSetNameTips) { @@ -360,7 +361,7 @@ func (g *NotificationSender) GroupInfoSetAnnouncementNotification(ctx context.Co return } g.setVersion(ctx, &tips.GroupMemberVersion, &tips.GroupMemberVersionID, database.GroupMemberVersionName, tips.Group.GroupID) - g.Notification(ctx, mcontext.GetOpUserID(ctx), tips.Group.GroupID, constant.GroupInfoSetAnnouncementNotification, tips, rpcclient.WithRpcGetUserName()) + g.Notification(ctx, mcontext.GetOpUserID(ctx), tips.Group.GroupID, constant.GroupInfoSetAnnouncementNotification, tips, notification.WithRpcGetUserName()) } func (g *NotificationSender) JoinGroupApplicationNotification(ctx context.Context, req *pbgroup.JoinGroupReq) { @@ -505,7 +506,7 @@ func (g *NotificationSender) GroupOwnerTransferredNotification(ctx context.Conte g.Notification(ctx, mcontext.GetOpUserID(ctx), group.GroupID, constant.GroupOwnerTransferredNotification, tips) } -func (g *NotificationSender) MemberKickedNotification(ctx context.Context, tips *sdkws.MemberKickedTips) { +func (g *NotificationSender) MemberKickedNotification(ctx context.Context, tips *sdkws.MemberKickedTips, sendNotification *bool) { var err error defer func() { if err != nil { @@ -516,10 +517,10 @@ func (g *NotificationSender) MemberKickedNotification(ctx context.Context, tips return } g.setVersion(ctx, &tips.GroupMemberVersion, &tips.GroupMemberVersionID, database.GroupMemberVersionName, tips.Group.GroupID) - g.Notification(ctx, mcontext.GetOpUserID(ctx), tips.Group.GroupID, constant.MemberKickedNotification, tips) + g.Notification(ctx, mcontext.GetOpUserID(ctx), tips.Group.GroupID, constant.MemberKickedNotification, tips, notification.WithSendNotification(sendNotification)) } -func (g *NotificationSender) GroupApplicationAgreeMemberEnterNotification(ctx context.Context, groupID string, invitedOpUserID string, entrantUserID ...string) error { +func (g *NotificationSender) GroupApplicationAgreeMemberEnterNotification(ctx context.Context, groupID string, sendNotification *bool, invitedOpUserID string, entrantUserID ...string) error { var err error defer func() { if err != nil { @@ -569,7 +570,7 @@ func (g *NotificationSender) GroupApplicationAgreeMemberEnterNotification(ctx co } } g.setVersion(ctx, &tips.GroupMemberVersion, &tips.GroupMemberVersionID, database.GroupMemberVersionName, tips.Group.GroupID) - g.Notification(ctx, mcontext.GetOpUserID(ctx), group.GroupID, constant.MemberInvitedNotification, tips) + g.Notification(ctx, mcontext.GetOpUserID(ctx), group.GroupID, constant.MemberInvitedNotification, tips, notification.WithSendNotification(sendNotification)) return nil } @@ -614,7 +615,7 @@ func (g *NotificationSender) MemberEnterNotification(ctx context.Context, groupI return nil } -func (g *NotificationSender) GroupDismissedNotification(ctx context.Context, tips *sdkws.GroupDismissedTips) { +func (g *NotificationSender) GroupDismissedNotification(ctx context.Context, tips *sdkws.GroupDismissedTips, sendNotification *bool) { var err error defer func() { if err != nil { @@ -624,7 +625,7 @@ func (g *NotificationSender) GroupDismissedNotification(ctx context.Context, tip if err = g.fillOpUser(ctx, &tips.OpUser, tips.Group.GroupID); err != nil { return } - g.Notification(ctx, mcontext.GetOpUserID(ctx), tips.Group.GroupID, constant.GroupDismissedNotification, tips) + g.Notification(ctx, mcontext.GetOpUserID(ctx), tips.Group.GroupID, constant.GroupDismissedNotification, tips, notification.WithSendNotification(sendNotification)) } func (g *NotificationSender) GroupMemberMutedNotification(ctx context.Context, groupID, groupMemberUserID string, mutedSeconds uint32) { diff --git a/internal/rpc/msg/notification.go b/internal/rpc/msg/notification.go index 14576f693..0daafbe6c 100644 --- a/internal/rpc/msg/notification.go +++ b/internal/rpc/msg/notification.go @@ -23,11 +23,11 @@ import ( ) type MsgNotificationSender struct { - *rpcclient.NotificationSender + *notification.NotificationSender } -func NewMsgNotificationSender(config *Config, opts ...rpcclient.NotificationSenderOptions) *MsgNotificationSender { - return &MsgNotificationSender{rpcclient.NewNotificationSender(&config.NotificationConfig, opts...)} +func NewMsgNotificationSender(config *Config, opts ...notification.NotificationSenderOptions) *MsgNotificationSender { + return &MsgNotificationSender{notification.NewNotificationSender(&config.NotificationConfig, opts...)} } func (m *MsgNotificationSender) UserDeleteMsgsNotification(ctx context.Context, userID, conversationID string, seqs []int64) { diff --git a/internal/rpc/msg/server.go b/internal/rpc/msg/server.go index 7737f7e7f..d1002cca3 100644 --- a/internal/rpc/msg/server.go +++ b/internal/rpc/msg/server.go @@ -23,6 +23,8 @@ import ( "github.com/openimsdk/open-im-server/v3/pkg/mqbuild" "github.com/openimsdk/open-im-server/v3/pkg/rpcli" + "google.golang.org/grpc" + "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/redis" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/controller" @@ -35,7 +37,6 @@ import ( "github.com/openimsdk/protocol/msg" "github.com/openimsdk/protocol/sdkws" "github.com/openimsdk/tools/discovery" - "google.golang.org/grpc" ) type MessageInterceptorFunc func(ctx context.Context, globalConfig *Config, req *msg.SendMsgReq) (*sdkws.MsgData, error) @@ -66,7 +67,7 @@ type msgServer struct { GroupLocalCache *rpccache.GroupLocalCache // Local cache for group data. ConversationLocalCache *rpccache.ConversationLocalCache // Local cache for conversation data. Handlers MessageInterceptorChain // Chain of handlers for processing messages. - notificationSender *rpcclient.NotificationSender // RPC client for sending notifications. + notificationSender *notification.NotificationSender // RPC client for sending notifications. msgNotificationSender *MsgNotificationSender // RPC client for sending msg notifications. config *Config // Global configuration settings. webhookClient *webhook.Client @@ -152,8 +153,8 @@ func Start(ctx context.Context, config *Config, client discovery.Conn, server gr conversationClient: conversationClient, } - s.notificationSender = rpcclient.NewNotificationSender(&config.NotificationConfig, rpcclient.WithLocalSendMsg(s.SendMsg)) - s.msgNotificationSender = NewMsgNotificationSender(config, rpcclient.WithLocalSendMsg(s.SendMsg)) + s.notificationSender = notification.NewNotificationSender(&config.NotificationConfig, notification.WithLocalSendMsg(s.SendMsg)) + s.msgNotificationSender = NewMsgNotificationSender(config, notification.WithLocalSendMsg(s.SendMsg)) msg.RegisterMsgServer(server, s) diff --git a/internal/rpc/relation/notification.go b/internal/rpc/relation/notification.go index a34a4d322..caf2dafe1 100644 --- a/internal/rpc/relation/notification.go +++ b/internal/rpc/relation/notification.go @@ -16,6 +16,7 @@ package relation import ( "context" + "github.com/openimsdk/open-im-server/v3/pkg/rpcli" "github.com/openimsdk/protocol/msg" @@ -36,7 +37,7 @@ import ( ) type FriendNotificationSender struct { - *rpcclient.NotificationSender + *notification.NotificationSender // Target not found err getUsersInfo func(ctx context.Context, userIDs []string) ([]common_user.CommonUser, error) // db controller @@ -89,7 +90,7 @@ func WithRpcFunc( func NewFriendNotificationSender(conf *config.Notification, msgClient *rpcli.MsgClient, opts ...friendNotificationSenderOptions) *FriendNotificationSender { f := &FriendNotificationSender{ - NotificationSender: rpcclient.NewNotificationSender(conf, rpcclient.WithRpcClient(func(ctx context.Context, req *msg.SendMsgReq) (*msg.SendMsgResp, error) { + NotificationSender: notification.NewNotificationSender(conf, notification.WithRpcClient(func(ctx context.Context, req *msg.SendMsgReq) (*msg.SendMsgResp, error) { return msgClient.SendMsg(ctx, req) })), } diff --git a/internal/rpc/user/notification.go b/internal/rpc/user/notification.go index 03fdf95bd..4fb214f74 100644 --- a/internal/rpc/user/notification.go +++ b/internal/rpc/user/notification.go @@ -16,6 +16,7 @@ package user import ( "context" + "github.com/openimsdk/open-im-server/v3/pkg/rpcli" "github.com/openimsdk/protocol/msg" @@ -29,7 +30,7 @@ import ( ) type UserNotificationSender struct { - *rpcclient.NotificationSender + *notification.NotificationSender getUsersInfo func(ctx context.Context, userIDs []string) ([]common_user.CommonUser, error) // db controller db controller.UserDatabase @@ -63,7 +64,7 @@ func WithUserFunc( func NewUserNotificationSender(config *Config, msgClient *rpcli.MsgClient, opts ...userNotificationSenderOptions) *UserNotificationSender { f := &UserNotificationSender{ - NotificationSender: rpcclient.NewNotificationSender(&config.NotificationConfig, rpcclient.WithRpcClient(func(ctx context.Context, req *msg.SendMsgReq) (*msg.SendMsgResp, error) { + NotificationSender: notification.NewNotificationSender(&config.NotificationConfig, notification.WithRpcClient(func(ctx context.Context, req *msg.SendMsgReq) (*msg.SendMsgResp, error) { return msgClient.SendMsg(ctx, req) })), } diff --git a/pkg/common/config/parse.go b/pkg/common/config/parse.go index 8fa8812ef..1687cae30 100644 --- a/pkg/common/config/parse.go +++ b/pkg/common/config/parse.go @@ -18,11 +18,12 @@ import ( "os" "path/filepath" + "gopkg.in/yaml.v3" + "github.com/openimsdk/open-im-server/v3/pkg/msgprocessor" "github.com/openimsdk/protocol/constant" "github.com/openimsdk/tools/errs" "github.com/openimsdk/tools/field" - "gopkg.in/yaml.v3" ) const ( @@ -56,9 +57,12 @@ func GetProjectRoot() (string, error) { return projectRoot, nil } -func GetOptionsByNotification(cfg NotificationConfig) msgprocessor.Options { +func GetOptionsByNotification(cfg NotificationConfig, sendNotification *bool) msgprocessor.Options { opts := msgprocessor.NewOptions() + if sendNotification != nil { + cfg.IsSendMsg = *sendNotification + } if cfg.IsSendMsg { opts = msgprocessor.WithOptions(opts, msgprocessor.WithUnreadCount(true)) } diff --git a/pkg/notification/msg.go b/pkg/notification/msg.go index 0795982c8..1d2365b5a 100644 --- a/pkg/notification/msg.go +++ b/pkg/notification/msg.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package rpcclient +package notification import ( "context" @@ -179,19 +179,24 @@ func NewNotificationSender(conf *config.Notification, opts ...NotificationSender } type notificationOpt struct { - WithRpcGetUsername bool + RpcGetUsername bool + SendNotification *bool } type NotificationOptions func(*notificationOpt) func WithRpcGetUserName() NotificationOptions { return func(opt *notificationOpt) { - opt.WithRpcGetUsername = true + opt.RpcGetUsername = true + } +} +func WithSendNotification(send *bool) NotificationOptions { + return func(opt *notificationOpt) { + opt.SendNotification = send } } func (s *NotificationSender) send(ctx context.Context, sendID, recvID string, contentType, sessionType int32, m proto.Message, opts ...NotificationOptions) { - //ctx = mcontext.WithMustInfoCtx([]string{mcontext.GetOperationID(ctx), mcontext.GetOpUserID(ctx), mcontext.GetOpUserPlatform(ctx), mcontext.GetConnID(ctx)}) ctx = context.WithoutCancel(ctx) ctx, cancel := context.WithTimeout(ctx, time.Second*time.Duration(5)) defer cancel() @@ -208,7 +213,7 @@ func (s *NotificationSender) send(ctx context.Context, sendID, recvID string, co var req msg.SendMsgReq var msg sdkws.MsgData var userInfo *sdkws.UserInfo - if notificationOpt.WithRpcGetUsername && s.getUserInfo != nil { + if notificationOpt.RpcGetUsername && s.getUserInfo != nil { userInfo, err = s.getUserInfo(ctx, sendID) if err != nil { log.ZWarn(ctx, "getUserInfo failed", err, "sendID", sendID) @@ -233,7 +238,7 @@ func (s *NotificationSender) send(ctx context.Context, sendID, recvID string, co if sendID == recvID && contentType == constant.HasReadReceipt { optionsConfig.ReliabilityLevel = constant.UnreliableNotification } - options := config.GetOptionsByNotification(optionsConfig) + options := config.GetOptionsByNotification(optionsConfig, notificationOpt.SendNotification) s.SetOptionsByContentType(ctx, options, contentType) msg.Options = options // fill Notification OfflinePush by config From 1df02692bf473e329a854d72a6c9fc1a0ce00472 Mon Sep 17 00:00:00 2001 From: OpenIM-Gordon <1432970085@qq.com> Date: Fri, 28 Feb 2025 16:59:04 +0800 Subject: [PATCH 125/199] refactor: change sendNotification to sendMessage to avoid ambiguity regarding message sending behavior. (#3173) * feat: add a field to specify whether to send a notification message when creating a group. * feat: add a field to specify whether to send a notification message when creating a group. * refactor: change sendNotification to sendMessage to avoid ambiguity regarding message sending behavior. --------- Co-authored-by: Monet Lee --- go.mod | 2 +- go.sum | 4 ++-- internal/rpc/group/group.go | 8 ++++---- internal/rpc/group/notification.go | 16 ++++++++-------- pkg/common/config/parse.go | 6 +++--- pkg/notification/msg.go | 10 +++++----- 6 files changed, 23 insertions(+), 23 deletions(-) diff --git a/go.mod b/go.mod index 0b677d1b0..57b2b6f7e 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/gorilla/websocket v1.5.1 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/mitchellh/mapstructure v1.5.0 - github.com/openimsdk/protocol v0.0.72-alpha.77 + github.com/openimsdk/protocol v0.0.72-alpha.78 github.com/openimsdk/tools v0.0.50-alpha.74 github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 diff --git a/go.sum b/go.sum index 1b132db13..0f870fa45 100644 --- a/go.sum +++ b/go.sum @@ -347,8 +347,8 @@ github.com/onsi/gomega v1.25.0 h1:Vw7br2PCDYijJHSfBOWhov+8cAnUf8MfMaIOV323l6Y= github.com/onsi/gomega v1.25.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdMPGhM= github.com/openimsdk/gomake v0.0.15-alpha.2 h1:5Q8yl8ezy2yx+q8/ucU/t4kJnDfCzNOrkXcDACCqtyM= github.com/openimsdk/gomake v0.0.15-alpha.2/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= -github.com/openimsdk/protocol v0.0.72-alpha.77 h1:ka7TeOpNKKqYTz8aWR8nZnz2rnzgsO9mw6xOQcVXwbE= -github.com/openimsdk/protocol v0.0.72-alpha.77/go.mod h1:WF7EuE55vQvpyUAzDXcqg+B+446xQyEba0X35lTINmw= +github.com/openimsdk/protocol v0.0.72-alpha.78 h1:n9HVj5olMPiGLF3Z4apPvvYzn2yOpyrsn2/YiAaIsxw= +github.com/openimsdk/protocol v0.0.72-alpha.78/go.mod h1:WF7EuE55vQvpyUAzDXcqg+B+446xQyEba0X35lTINmw= github.com/openimsdk/tools v0.0.50-alpha.74 h1:yh10SiMiivMEjicEQg+QAsH4pvaO+4noMPdlw+ew0Kc= github.com/openimsdk/tools v0.0.50-alpha.74/go.mod h1:n2poR3asX1e1XZce4O+MOWAp+X02QJRFvhcLCXZdzRo= github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= diff --git a/internal/rpc/group/group.go b/internal/rpc/group/group.go index 535eed52a..89ab35d2f 100644 --- a/internal/rpc/group/group.go +++ b/internal/rpc/group/group.go @@ -285,7 +285,7 @@ func (g *groupServer) CreateGroup(ctx context.Context, req *pbgroup.CreateGroupR break } } - g.notification.GroupCreatedNotification(ctx, tips, req.SendNotification) + g.notification.GroupCreatedNotification(ctx, tips, req.SendMessage) if req.GroupInfo.Notification != "" { g.notification.GroupInfoSetAnnouncementNotification(ctx, &sdkws.GroupInfoSetAnnouncementTips{ @@ -448,7 +448,7 @@ func (g *groupServer) InviteUserToGroup(ctx context.Context, req *pbgroup.Invite return nil, err } - if err = g.notification.GroupApplicationAgreeMemberEnterNotification(ctx, req.GroupID, req.SendNotification, opUserID, req.InvitedUserIDs...); err != nil { + if err = g.notification.GroupApplicationAgreeMemberEnterNotification(ctx, req.GroupID, req.SendMessage, opUserID, req.InvitedUserIDs...); err != nil { return nil, err } return &pbgroup.InviteUserToGroupResp{}, nil @@ -614,7 +614,7 @@ func (g *groupServer) KickGroupMember(ctx context.Context, req *pbgroup.KickGrou for _, userID := range req.KickedUserIDs { tips.KickedUserList = append(tips.KickedUserList, convert.Db2PbGroupMember(memberMap[userID])) } - g.notification.MemberKickedNotification(ctx, tips, req.SendNotification) + g.notification.MemberKickedNotification(ctx, tips, req.SendMessage) if err := g.deleteMemberAndSetConversationSeq(ctx, req.GroupID, req.KickedUserIDs); err != nil { return nil, err } @@ -1369,7 +1369,7 @@ func (g *groupServer) DismissGroup(ctx context.Context, req *pbgroup.DismissGrou if mcontext.GetOpUserID(ctx) == owner.UserID { tips.OpUser = g.groupMemberDB2PB(owner, 0) } - g.notification.GroupDismissedNotification(ctx, tips, req.SendNotification) + g.notification.GroupDismissedNotification(ctx, tips, req.SendMessage) } membersID, err := g.db.FindGroupMemberUserID(ctx, group.GroupID) if err != nil { diff --git a/internal/rpc/group/notification.go b/internal/rpc/group/notification.go index a879d3b4a..8bd8a9503 100644 --- a/internal/rpc/group/notification.go +++ b/internal/rpc/group/notification.go @@ -308,7 +308,7 @@ func (g *NotificationSender) setSortVersion(ctx context.Context, version *uint64 } } -func (g *NotificationSender) GroupCreatedNotification(ctx context.Context, tips *sdkws.GroupCreatedTips, sendNotification *bool) { +func (g *NotificationSender) GroupCreatedNotification(ctx context.Context, tips *sdkws.GroupCreatedTips, SendMessage *bool) { var err error defer func() { if err != nil { @@ -319,7 +319,7 @@ func (g *NotificationSender) GroupCreatedNotification(ctx context.Context, tips return } g.setVersion(ctx, &tips.GroupMemberVersion, &tips.GroupMemberVersionID, database.GroupMemberVersionName, tips.Group.GroupID) - g.Notification(ctx, mcontext.GetOpUserID(ctx), tips.Group.GroupID, constant.GroupCreatedNotification, tips, notification.WithSendNotification(sendNotification)) + g.Notification(ctx, mcontext.GetOpUserID(ctx), tips.Group.GroupID, constant.GroupCreatedNotification, tips, notification.WithSendMessage(SendMessage)) } func (g *NotificationSender) GroupInfoSetNotification(ctx context.Context, tips *sdkws.GroupInfoSetTips) { @@ -506,7 +506,7 @@ func (g *NotificationSender) GroupOwnerTransferredNotification(ctx context.Conte g.Notification(ctx, mcontext.GetOpUserID(ctx), group.GroupID, constant.GroupOwnerTransferredNotification, tips) } -func (g *NotificationSender) MemberKickedNotification(ctx context.Context, tips *sdkws.MemberKickedTips, sendNotification *bool) { +func (g *NotificationSender) MemberKickedNotification(ctx context.Context, tips *sdkws.MemberKickedTips, SendMessage *bool) { var err error defer func() { if err != nil { @@ -517,10 +517,10 @@ func (g *NotificationSender) MemberKickedNotification(ctx context.Context, tips return } g.setVersion(ctx, &tips.GroupMemberVersion, &tips.GroupMemberVersionID, database.GroupMemberVersionName, tips.Group.GroupID) - g.Notification(ctx, mcontext.GetOpUserID(ctx), tips.Group.GroupID, constant.MemberKickedNotification, tips, notification.WithSendNotification(sendNotification)) + g.Notification(ctx, mcontext.GetOpUserID(ctx), tips.Group.GroupID, constant.MemberKickedNotification, tips, notification.WithSendMessage(SendMessage)) } -func (g *NotificationSender) GroupApplicationAgreeMemberEnterNotification(ctx context.Context, groupID string, sendNotification *bool, invitedOpUserID string, entrantUserID ...string) error { +func (g *NotificationSender) GroupApplicationAgreeMemberEnterNotification(ctx context.Context, groupID string, SendMessage *bool, invitedOpUserID string, entrantUserID ...string) error { var err error defer func() { if err != nil { @@ -570,7 +570,7 @@ func (g *NotificationSender) GroupApplicationAgreeMemberEnterNotification(ctx co } } g.setVersion(ctx, &tips.GroupMemberVersion, &tips.GroupMemberVersionID, database.GroupMemberVersionName, tips.Group.GroupID) - g.Notification(ctx, mcontext.GetOpUserID(ctx), group.GroupID, constant.MemberInvitedNotification, tips, notification.WithSendNotification(sendNotification)) + g.Notification(ctx, mcontext.GetOpUserID(ctx), group.GroupID, constant.MemberInvitedNotification, tips, notification.WithSendMessage(SendMessage)) return nil } @@ -615,7 +615,7 @@ func (g *NotificationSender) MemberEnterNotification(ctx context.Context, groupI return nil } -func (g *NotificationSender) GroupDismissedNotification(ctx context.Context, tips *sdkws.GroupDismissedTips, sendNotification *bool) { +func (g *NotificationSender) GroupDismissedNotification(ctx context.Context, tips *sdkws.GroupDismissedTips, SendMessage *bool) { var err error defer func() { if err != nil { @@ -625,7 +625,7 @@ func (g *NotificationSender) GroupDismissedNotification(ctx context.Context, tip if err = g.fillOpUser(ctx, &tips.OpUser, tips.Group.GroupID); err != nil { return } - g.Notification(ctx, mcontext.GetOpUserID(ctx), tips.Group.GroupID, constant.GroupDismissedNotification, tips, notification.WithSendNotification(sendNotification)) + g.Notification(ctx, mcontext.GetOpUserID(ctx), tips.Group.GroupID, constant.GroupDismissedNotification, tips, notification.WithSendMessage(SendMessage)) } func (g *NotificationSender) GroupMemberMutedNotification(ctx context.Context, groupID, groupMemberUserID string, mutedSeconds uint32) { diff --git a/pkg/common/config/parse.go b/pkg/common/config/parse.go index 1687cae30..df4ecc5a0 100644 --- a/pkg/common/config/parse.go +++ b/pkg/common/config/parse.go @@ -57,11 +57,11 @@ func GetProjectRoot() (string, error) { return projectRoot, nil } -func GetOptionsByNotification(cfg NotificationConfig, sendNotification *bool) msgprocessor.Options { +func GetOptionsByNotification(cfg NotificationConfig, sendMessage *bool) msgprocessor.Options { opts := msgprocessor.NewOptions() - if sendNotification != nil { - cfg.IsSendMsg = *sendNotification + if sendMessage != nil { + cfg.IsSendMsg = *sendMessage } if cfg.IsSendMsg { opts = msgprocessor.WithOptions(opts, msgprocessor.WithUnreadCount(true)) diff --git a/pkg/notification/msg.go b/pkg/notification/msg.go index 1d2365b5a..ba8a9185a 100644 --- a/pkg/notification/msg.go +++ b/pkg/notification/msg.go @@ -179,8 +179,8 @@ func NewNotificationSender(conf *config.Notification, opts ...NotificationSender } type notificationOpt struct { - RpcGetUsername bool - SendNotification *bool + RpcGetUsername bool + SendMessage *bool } type NotificationOptions func(*notificationOpt) @@ -190,9 +190,9 @@ func WithRpcGetUserName() NotificationOptions { opt.RpcGetUsername = true } } -func WithSendNotification(send *bool) NotificationOptions { +func WithSendMessage(sendMessage *bool) NotificationOptions { return func(opt *notificationOpt) { - opt.SendNotification = send + opt.SendMessage = sendMessage } } @@ -238,7 +238,7 @@ func (s *NotificationSender) send(ctx context.Context, sendID, recvID string, co if sendID == recvID && contentType == constant.HasReadReceipt { optionsConfig.ReliabilityLevel = constant.UnreliableNotification } - options := config.GetOptionsByNotification(optionsConfig, notificationOpt.SendNotification) + options := config.GetOptionsByNotification(optionsConfig, notificationOpt.SendMessage) s.SetOptionsByContentType(ctx, options, contentType) msg.Options = options // fill Notification OfflinePush by config From 887e0b7314d10fdecc6b2ff443fa57e32f9e2b99 Mon Sep 17 00:00:00 2001 From: Monet Lee Date: Mon, 3 Mar 2025 10:39:53 +0800 Subject: [PATCH 126/199] fix: solve uncorrect notification when set group info (#3172) * fix: setGroupInfoEx uncorrect call. * update notification logic. * update group notification logic. * update update group announcement notication. * fix errror. * refresh * solve conflict. * update args. --- internal/rpc/group/db_map.go | 24 +++++++++---- internal/rpc/group/group.go | 55 +++++++++++++++--------------- internal/rpc/group/notification.go | 4 +-- 3 files changed, 47 insertions(+), 36 deletions(-) diff --git a/internal/rpc/group/db_map.go b/internal/rpc/group/db_map.go index e99f9e772..7504bc851 100644 --- a/internal/rpc/group/db_map.go +++ b/internal/rpc/group/db_map.go @@ -16,6 +16,7 @@ package group import ( "context" + "strings" "time" pbgroup "github.com/openimsdk/protocol/group" @@ -55,41 +56,52 @@ func UpdateGroupInfoMap(ctx context.Context, group *sdkws.GroupInfoForSet) map[s return m } -func UpdateGroupInfoExMap(ctx context.Context, group *pbgroup.SetGroupInfoExReq) (map[string]any, error) { - m := make(map[string]any) +func UpdateGroupInfoExMap(ctx context.Context, group *pbgroup.SetGroupInfoExReq) (m map[string]any, normalFlag, groupNameFlag, notificationFlag bool, err error) { + m = make(map[string]any) if group.GroupName != nil { - if group.GroupName.Value != "" { + if strings.TrimSpace(group.GroupName.Value) != "" { m["group_name"] = group.GroupName.Value + groupNameFlag = true } else { - return nil, errs.ErrArgs.WrapMsg("group name is empty") + return nil, normalFlag, notificationFlag, groupNameFlag, errs.ErrArgs.WrapMsg("group name is empty") } } + if group.Notification != nil { + notificationFlag = true + group.Notification.Value = strings.TrimSpace(group.Notification.Value) // if Notification only contains spaces, set it to empty string + m["notification"] = group.Notification.Value - m["notification_update_time"] = time.Now() m["notification_user_id"] = mcontext.GetOpUserID(ctx) + m["notification_update_time"] = time.Now() } if group.Introduction != nil { m["introduction"] = group.Introduction.Value + normalFlag = true } if group.FaceURL != nil { m["face_url"] = group.FaceURL.Value + normalFlag = true } if group.NeedVerification != nil { m["need_verification"] = group.NeedVerification.Value + normalFlag = true } if group.LookMemberInfo != nil { m["look_member_info"] = group.LookMemberInfo.Value + normalFlag = true } if group.ApplyMemberFriend != nil { m["apply_member_friend"] = group.ApplyMemberFriend.Value + normalFlag = true } if group.Ex != nil { m["ex"] = group.Ex.Value + normalFlag = true } - return m, nil + return m, normalFlag, groupNameFlag, notificationFlag, nil } func UpdateGroupStatusMap(status int) map[string]any { diff --git a/internal/rpc/group/group.go b/internal/rpc/group/group.go index 89ab35d2f..db852c944 100644 --- a/internal/rpc/group/group.go +++ b/internal/rpc/group/group.go @@ -288,10 +288,11 @@ func (g *groupServer) CreateGroup(ctx context.Context, req *pbgroup.CreateGroupR g.notification.GroupCreatedNotification(ctx, tips, req.SendMessage) if req.GroupInfo.Notification != "" { + notificationFlag := true g.notification.GroupInfoSetAnnouncementNotification(ctx, &sdkws.GroupInfoSetAnnouncementTips{ Group: tips.Group, OpUser: tips.OpUser, - }) + }, ¬ificationFlag) } reqCallBackAfter := &pbgroup.CreateGroupReq{ @@ -1026,7 +1027,8 @@ func (g *groupServer) SetGroupInfo(ctx context.Context, req *pbgroup.SetGroupInf log.ZWarn(ctx, "SetConversations", err, "UserIDs", resp.UserIDs, "conversation", conversation) } }() - g.notification.GroupInfoSetAnnouncementNotification(ctx, &sdkws.GroupInfoSetAnnouncementTips{Group: tips.Group, OpUser: tips.OpUser}) + notficationFlag := true + g.notification.GroupInfoSetAnnouncementNotification(ctx, &sdkws.GroupInfoSetAnnouncementTips{Group: tips.Group, OpUser: tips.OpUser}, ¬ficationFlag) } if req.GroupInfoForSet.GroupName != "" { num-- @@ -1087,7 +1089,7 @@ func (g *groupServer) SetGroupInfoEx(ctx context.Context, req *pbgroup.SetGroupI return nil, err } - updatedData, err := UpdateGroupInfoExMap(ctx, req) + updatedData, normalFlag, groupNameFlag, notificationFlag, err := UpdateGroupInfoExMap(ctx, req) if len(updatedData) == 0 { return &pbgroup.SetGroupInfoExResp{}, nil } @@ -1115,41 +1117,38 @@ func (g *groupServer) SetGroupInfoEx(ctx context.Context, req *pbgroup.SetGroupI tips.OpUser = g.groupMemberDB2PB(opMember, 0) } - num := len(updatedData) - - if req.Notification != nil { - num -= 3 - + if notificationFlag { if req.Notification.Value != "" { - func() { - conversation := &pbconv.ConversationReq{ - ConversationID: msgprocessor.GetConversationIDBySessionType(constant.ReadGroupChatType, req.GroupID), - ConversationType: constant.ReadGroupChatType, - GroupID: req.GroupID, - } + conversation := &pbconv.ConversationReq{ + ConversationID: msgprocessor.GetConversationIDBySessionType(constant.ReadGroupChatType, req.GroupID), + ConversationType: constant.ReadGroupChatType, + GroupID: req.GroupID, + } - resp, err := g.GetGroupMemberUserIDs(ctx, &pbgroup.GetGroupMemberUserIDsReq{GroupID: req.GroupID}) - if err != nil { - log.ZWarn(ctx, "GetGroupMemberIDs is failed.", err) - return - } + resp, err := g.GetGroupMemberUserIDs(ctx, &pbgroup.GetGroupMemberUserIDsReq{GroupID: req.GroupID}) + if err != nil { + log.ZWarn(ctx, "GetGroupMemberIDs is failed.", err) + return nil, err + } - conversation.GroupAtType = &wrapperspb.Int32Value{Value: constant.GroupNotification} - if err := g.conversationClient.SetConversations(ctx, resp.UserIDs, conversation); err != nil { - log.ZWarn(ctx, "SetConversations", err, "UserIDs", resp.UserIDs, "conversation", conversation) - } - }() + conversation.GroupAtType = &wrapperspb.Int32Value{Value: constant.GroupNotification} + if err := g.conversationClient.SetConversations(ctx, resp.UserIDs, conversation); err != nil { + log.ZWarn(ctx, "SetConversations", err, "UserIDs", resp.UserIDs, "conversation", conversation) + } - g.notification.GroupInfoSetAnnouncementNotification(ctx, &sdkws.GroupInfoSetAnnouncementTips{Group: tips.Group, OpUser: tips.OpUser}) + g.notification.GroupInfoSetAnnouncementNotification(ctx, &sdkws.GroupInfoSetAnnouncementTips{Group: tips.Group, OpUser: tips.OpUser}, ¬ificationFlag) + } else { + notificationFlag = false + g.notification.GroupInfoSetAnnouncementNotification(ctx, &sdkws.GroupInfoSetAnnouncementTips{Group: tips.Group, OpUser: tips.OpUser}, ¬ificationFlag) } } - if req.GroupName != nil { - num-- + if groupNameFlag { g.notification.GroupInfoSetNameNotification(ctx, &sdkws.GroupInfoSetNameTips{Group: tips.Group, OpUser: tips.OpUser}) } - if num > 0 { + // if updatedData > 0, send the normal notification + if normalFlag { g.notification.GroupInfoSetNotification(ctx, tips) } diff --git a/internal/rpc/group/notification.go b/internal/rpc/group/notification.go index 8bd8a9503..5fbb78d64 100644 --- a/internal/rpc/group/notification.go +++ b/internal/rpc/group/notification.go @@ -350,7 +350,7 @@ func (g *NotificationSender) GroupInfoSetNameNotification(ctx context.Context, t g.Notification(ctx, mcontext.GetOpUserID(ctx), tips.Group.GroupID, constant.GroupInfoSetNameNotification, tips) } -func (g *NotificationSender) GroupInfoSetAnnouncementNotification(ctx context.Context, tips *sdkws.GroupInfoSetAnnouncementTips) { +func (g *NotificationSender) GroupInfoSetAnnouncementNotification(ctx context.Context, tips *sdkws.GroupInfoSetAnnouncementTips, sendMessage *bool) { var err error defer func() { if err != nil { @@ -361,7 +361,7 @@ func (g *NotificationSender) GroupInfoSetAnnouncementNotification(ctx context.Co return } g.setVersion(ctx, &tips.GroupMemberVersion, &tips.GroupMemberVersionID, database.GroupMemberVersionName, tips.Group.GroupID) - g.Notification(ctx, mcontext.GetOpUserID(ctx), tips.Group.GroupID, constant.GroupInfoSetAnnouncementNotification, tips, notification.WithRpcGetUserName()) + g.Notification(ctx, mcontext.GetOpUserID(ctx), tips.Group.GroupID, constant.GroupInfoSetAnnouncementNotification, tips, notification.WithRpcGetUserName(), notification.WithSendMessage(sendMessage)) } func (g *NotificationSender) JoinGroupApplicationNotification(ctx context.Context, req *pbgroup.JoinGroupReq) { From ab6c9dca713877264f5c44652b661710e8dc9ad3 Mon Sep 17 00:00:00 2001 From: chao <48119764+withchao@users.noreply.github.com> Date: Mon, 3 Mar 2025 17:59:35 +0800 Subject: [PATCH 127/199] feat: optimizing BatchGetIncrementalGroupMember (#3180) * pb * fix: Modifying other fields while setting IsPrivateChat does not take effect * fix: quote message error revoke * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * upgrading pkg tools * fix * fix * optimize log output * feat: support GetLastMessage * feat: support GetLastMessage * feat: s3 switch * feat: s3 switch * fix: GetUsersOnline * feat: SendBusinessNotification supported configuration parameters * feat: SendBusinessNotification supported configuration parameters * feat: SendBusinessNotification supported configuration parameters * feat: seq conversion failed without exiting * fix: DeleteDoc crash * fix: fill send time * fix: fill send time * fix: crash caused by withdrawing messages from users who have left the group * fix: user msg timestamp * seq read config * seq read config * fix: the source message of the reference is withdrawn, and the referenced message is deleted * feat: optimize the default notification.yml * fix: shouldPushOffline * fix: optimizing BatchGetIncrementalGroupMember --- internal/rpc/group/sync.go | 168 +++++-------------------------------- 1 file changed, 22 insertions(+), 146 deletions(-) diff --git a/internal/rpc/group/sync.go b/internal/rpc/group/sync.go index 50ac8252f..d311ae076 100644 --- a/internal/rpc/group/sync.go +++ b/internal/rpc/group/sync.go @@ -11,10 +11,10 @@ import ( "github.com/openimsdk/protocol/constant" pbgroup "github.com/openimsdk/protocol/group" "github.com/openimsdk/protocol/sdkws" - "github.com/openimsdk/tools/errs" - "github.com/openimsdk/tools/log" ) +const versionSyncLimit = 500 + func (g *groupServer) GetFullGroupMemberUserIDs(ctx context.Context, req *pbgroup.GetFullGroupMemberUserIDsReq) (*pbgroup.GetFullGroupMemberUserIDsResp, error) { vl, err := g.db.FindMaxGroupMemberVersionCache(ctx, req.GroupID) if err != nil { @@ -132,150 +132,6 @@ func (g *groupServer) GetIncrementalGroupMember(ctx context.Context, req *pbgrou return resp, nil } -func (g *groupServer) BatchGetIncrementalGroupMember(ctx context.Context, req *pbgroup.BatchGetIncrementalGroupMemberReq) (resp *pbgroup.BatchGetIncrementalGroupMemberResp, err error) { - type VersionInfo struct { - GroupID string - VersionID string - VersionNumber uint64 - } - - var groupIDs []string - - groupsVersionMap := make(map[string]*VersionInfo) - groupsMap := make(map[string]*model.Group) - hasGroupUpdateMap := make(map[string]bool) - sortVersionMap := make(map[string]uint64) - - var targetKeys, versionIDs []string - var versionNumbers []uint64 - - var requestBodyLen int - - for _, group := range req.ReqList { - groupsVersionMap[group.GroupID] = &VersionInfo{ - GroupID: group.GroupID, - VersionID: group.VersionID, - VersionNumber: group.Version, - } - - groupIDs = append(groupIDs, group.GroupID) - } - - groups, err := g.db.FindGroup(ctx, groupIDs) - if err != nil { - return nil, errs.Wrap(err) - } - - for _, group := range groups { - if group.Status == constant.GroupStatusDismissed { - err = servererrs.ErrDismissedAlready.Wrap() - log.ZError(ctx, "This group is Dismissed Already", err, "group is", group.GroupID) - - delete(groupsVersionMap, group.GroupID) - } else { - groupsMap[group.GroupID] = group - } - } - - for groupID, vInfo := range groupsVersionMap { - targetKeys = append(targetKeys, groupID) - versionIDs = append(versionIDs, vInfo.VersionID) - versionNumbers = append(versionNumbers, vInfo.VersionNumber) - } - - opt := incrversion.BatchOption[[]*sdkws.GroupMemberFullInfo, pbgroup.BatchGetIncrementalGroupMemberResp]{ - Ctx: ctx, - TargetKeys: targetKeys, - VersionIDs: versionIDs, - VersionNumbers: versionNumbers, - Versions: func(ctx context.Context, groupIDs []string, versions []uint64, limits []int) (map[string]*model.VersionLog, error) { - vLogs, err := g.db.BatchFindMemberIncrVersion(ctx, groupIDs, versions, limits) - if err != nil { - return nil, errs.Wrap(err) - } - - for groupID, vlog := range vLogs { - vlogElems := make([]model.VersionLogElem, 0, len(vlog.Logs)) - for i, log := range vlog.Logs { - switch log.EID { - case model.VersionGroupChangeID: - vlog.LogLen-- - hasGroupUpdateMap[groupID] = true - case model.VersionSortChangeID: - vlog.LogLen-- - sortVersionMap[groupID] = uint64(log.Version) - default: - vlogElems = append(vlogElems, vlog.Logs[i]) - } - } - vlog.Logs = vlogElems - if vlog.LogLen > 0 { - hasGroupUpdateMap[groupID] = true - } - } - - return vLogs, nil - }, - CacheMaxVersions: g.db.BatchFindMaxGroupMemberVersionCache, - Find: func(ctx context.Context, groupID string, ids []string) ([]*sdkws.GroupMemberFullInfo, error) { - memberInfo, err := g.getGroupMembersInfo(ctx, groupID, ids) - if err != nil { - return nil, err - } - - return memberInfo, err - }, - Resp: func(versions map[string]*model.VersionLog, deleteIdsMap map[string][]string, insertListMap, updateListMap map[string][]*sdkws.GroupMemberFullInfo, fullMap map[string]bool) *pbgroup.BatchGetIncrementalGroupMemberResp { - resList := make(map[string]*pbgroup.GetIncrementalGroupMemberResp) - - for groupID, versionLog := range versions { - resList[groupID] = &pbgroup.GetIncrementalGroupMemberResp{ - VersionID: versionLog.ID.Hex(), - Version: uint64(versionLog.Version), - Full: fullMap[groupID], - Delete: deleteIdsMap[groupID], - Insert: insertListMap[groupID], - Update: updateListMap[groupID], - SortVersion: sortVersionMap[groupID], - } - - requestBodyLen += len(insertListMap[groupID]) + len(updateListMap[groupID]) + len(deleteIdsMap[groupID]) - if requestBodyLen > 200 { - break - } - } - - return &pbgroup.BatchGetIncrementalGroupMemberResp{ - RespList: resList, - } - }, - } - - resp, err = opt.Build() - if err != nil { - return nil, errs.Wrap(err) - } - - for groupID, val := range resp.RespList { - if val.Full || hasGroupUpdateMap[groupID] { - count, err := g.db.FindGroupMemberNum(ctx, groupID) - if err != nil { - return nil, err - } - - owner, err := g.db.TakeGroupOwner(ctx, groupID) - if err != nil { - return nil, err - } - - resp.RespList[groupID].Group = g.groupDB2PB(groupsMap[groupID], owner.UserID, count) - } - } - - return resp, nil - -} - func (g *groupServer) GetIncrementalJoinGroup(ctx context.Context, req *pbgroup.GetIncrementalJoinGroupReq) (*pbgroup.GetIncrementalJoinGroupResp, error) { if err := authverify.CheckAccessV3(ctx, req.UserID, g.config.Share.IMAdminUserID); err != nil { return nil, err @@ -301,3 +157,23 @@ func (g *groupServer) GetIncrementalJoinGroup(ctx context.Context, req *pbgroup. } return opt.Build() } + +func (g *groupServer) BatchGetIncrementalGroupMember(ctx context.Context, req *pbgroup.BatchGetIncrementalGroupMemberReq) (*pbgroup.BatchGetIncrementalGroupMemberResp, error) { + var num int + resp := make(map[string]*pbgroup.GetIncrementalGroupMemberResp) + for _, memberReq := range req.ReqList { + if _, ok := resp[memberReq.GroupID]; ok { + continue + } + memberResp, err := g.GetIncrementalGroupMember(ctx, memberReq) + if err != nil { + return nil, err + } + resp[memberReq.GroupID] = memberResp + num += len(memberResp.Insert) + len(memberResp.Update) + len(memberResp.Delete) + if num >= versionSyncLimit { + break + } + } + return &pbgroup.BatchGetIncrementalGroupMemberResp{RespList: resp}, nil +} From 68ee86c1d9e28cdc746bed691d8985a6cfc999c9 Mon Sep 17 00:00:00 2001 From: chao <48119764+withchao@users.noreply.github.com> Date: Tue, 4 Mar 2025 16:09:29 +0800 Subject: [PATCH 128/199] fix: the sorting is wrong after canceling the administrator in group settings (#3185) * pb * fix: Modifying other fields while setting IsPrivateChat does not take effect * fix: quote message error revoke * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * upgrading pkg tools * fix * fix * optimize log output * feat: support GetLastMessage * feat: support GetLastMessage * feat: s3 switch * feat: s3 switch * fix: GetUsersOnline * feat: SendBusinessNotification supported configuration parameters * feat: SendBusinessNotification supported configuration parameters * feat: SendBusinessNotification supported configuration parameters * feat: seq conversion failed without exiting * fix: DeleteDoc crash * fix: fill send time * fix: fill send time * fix: crash caused by withdrawing messages from users who have left the group * fix: user msg timestamp * seq read config * seq read config * fix: the source message of the reference is withdrawn, and the referenced message is deleted * feat: optimize the default notification.yml * fix: shouldPushOffline * fix: the sorting is wrong after canceling the administrator in group settings --- internal/rpc/group/notification.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/rpc/group/notification.go b/internal/rpc/group/notification.go index 5fbb78d64..cc70a86f6 100644 --- a/internal/rpc/group/notification.go +++ b/internal/rpc/group/notification.go @@ -782,7 +782,7 @@ func (g *NotificationSender) GroupMemberSetToAdminNotification(ctx context.Conte if err = g.fillOpUser(ctx, &tips.OpUser, tips.Group.GroupID); err != nil { return } - g.setVersion(ctx, &tips.GroupMemberVersion, &tips.GroupMemberVersionID, database.GroupMemberVersionName, tips.Group.GroupID) + g.setSortVersion(ctx, &tips.GroupMemberVersion, &tips.GroupMemberVersionID, database.GroupMemberVersionName, tips.Group.GroupID, &tips.GroupSortVersion) g.Notification(ctx, mcontext.GetOpUserID(ctx), group.GroupID, constant.GroupMemberSetToAdminNotification, tips) } @@ -807,6 +807,6 @@ func (g *NotificationSender) GroupMemberSetToOrdinaryUserNotification(ctx contex if err = g.fillOpUser(ctx, &tips.OpUser, tips.Group.GroupID); err != nil { return } - g.setVersion(ctx, &tips.GroupMemberVersion, &tips.GroupMemberVersionID, database.GroupMemberVersionName, tips.Group.GroupID) + g.setSortVersion(ctx, &tips.GroupMemberVersion, &tips.GroupMemberVersionID, database.GroupMemberVersionName, tips.Group.GroupID, &tips.GroupSortVersion) g.Notification(ctx, mcontext.GetOpUserID(ctx), group.GroupID, constant.GroupMemberSetToOrdinaryUserNotification, tips) } From 737169a466d4381bc4c20ce402fefb4ae808e224 Mon Sep 17 00:00:00 2001 From: icey-yu <119291641+icey-yu@users.noreply.github.com> Date: Tue, 4 Mar 2025 18:04:21 +0800 Subject: [PATCH 129/199] feat: system account send msg doesn't need friend verify (#3187) --- internal/rpc/msg/verify.go | 8 ++++++++ pkg/authverify/token.go | 5 +++++ 2 files changed, 13 insertions(+) diff --git a/internal/rpc/msg/verify.go b/internal/rpc/msg/verify.go index a492232a4..213545d83 100644 --- a/internal/rpc/msg/verify.go +++ b/internal/rpc/msg/verify.go @@ -20,6 +20,7 @@ import ( "strconv" "time" + "github.com/openimsdk/open-im-server/v3/pkg/authverify" "github.com/openimsdk/open-im-server/v3/pkg/common/servererrs" "github.com/openimsdk/tools/utils/datautil" "github.com/openimsdk/tools/utils/encrypt" @@ -63,6 +64,13 @@ func (m *msgServer) messageVerification(ctx context.Context, data *msg.SendMsgRe if err := m.webhookBeforeSendSingleMsg(ctx, &m.config.WebhooksConfig.BeforeSendSingleMsg, data); err != nil { return err } + u, err := m.UserLocalCache.GetUserInfo(ctx, data.MsgData.SendID) + if err != nil { + return err + } + if authverify.CheckSystemAccount(ctx, u.AppMangerLevel) { + return nil + } black, err := m.FriendLocalCache.IsBlack(ctx, data.MsgData.SendID, data.MsgData.RecvID) if err != nil { return err diff --git a/pkg/authverify/token.go b/pkg/authverify/token.go index f1b377bad..872feb1cf 100644 --- a/pkg/authverify/token.go +++ b/pkg/authverify/token.go @@ -20,6 +20,7 @@ import ( "github.com/golang-jwt/jwt/v4" "github.com/openimsdk/open-im-server/v3/pkg/common/servererrs" + "github.com/openimsdk/protocol/constant" "github.com/openimsdk/tools/mcontext" "github.com/openimsdk/tools/utils/datautil" ) @@ -55,3 +56,7 @@ func CheckAdmin(ctx context.Context, imAdminUserID []string) error { func IsManagerUserID(opUserID string, imAdminUserID []string) bool { return datautil.Contain(opUserID, imAdminUserID...) } + +func CheckSystemAccount(ctx context.Context, level int32) bool { + return level >= constant.AppAdmin +} From 0541d0bf06165cb5c066605e105be9620efcae17 Mon Sep 17 00:00:00 2001 From: Monet Lee Date: Tue, 4 Mar 2025 18:12:35 +0800 Subject: [PATCH 130/199] fix: solve uncorrect GroupMember enter group notification type. (#3188) --- internal/rpc/group/group.go | 10 +++++++-- internal/rpc/group/notification.go | 34 ++++++++++++++---------------- 2 files changed, 24 insertions(+), 20 deletions(-) diff --git a/internal/rpc/group/group.go b/internal/rpc/group/group.go index db852c944..0a877a03e 100644 --- a/internal/rpc/group/group.go +++ b/internal/rpc/group/group.go @@ -824,8 +824,14 @@ func (g *groupServer) GroupApplicationResponse(ctx context.Context, req *pbgroup if member == nil { log.ZDebug(ctx, "GroupApplicationResponse", "member is nil") } else { - if err = g.notification.GroupApplicationAgreeMemberEnterNotification(ctx, req.GroupID, nil, groupRequest.InviterUserID, req.FromUserID); err != nil { - return nil, err + if groupRequest.InviterUserID == "" { + if err = g.notification.MemberEnterNotification(ctx, req.GroupID, req.FromUserID); err != nil { + return nil, err + } + } else { + if err = g.notification.GroupApplicationAgreeMemberEnterNotification(ctx, req.GroupID, nil, groupRequest.InviterUserID, req.FromUserID); err != nil { + return nil, err + } } } case constant.GroupResponseRefuse: diff --git a/internal/rpc/group/notification.go b/internal/rpc/group/notification.go index cc70a86f6..0a135af23 100644 --- a/internal/rpc/group/notification.go +++ b/internal/rpc/group/notification.go @@ -234,17 +234,17 @@ func (g *NotificationSender) groupMemberDB2PB(member *model.GroupMember, appMang return result, nil } */ -func (g *NotificationSender) fillOpUser(ctx context.Context, opUser **sdkws.GroupMemberFullInfo, groupID string) (err error) { - return g.fillOpUserByUserID(ctx, mcontext.GetOpUserID(ctx), opUser, groupID) +func (g *NotificationSender) fillOpUser(ctx context.Context, targetUser **sdkws.GroupMemberFullInfo, groupID string) (err error) { + return g.fillUserByUserID(ctx, mcontext.GetOpUserID(ctx), targetUser, groupID) } -func (g *NotificationSender) fillOpUserByUserID(ctx context.Context, userID string, opUser **sdkws.GroupMemberFullInfo, groupID string) error { - if opUser == nil { +func (g *NotificationSender) fillUserByUserID(ctx context.Context, userID string, targetUser **sdkws.GroupMemberFullInfo, groupID string) error { + if targetUser == nil { return errs.ErrInternalServer.WrapMsg("**sdkws.GroupMemberFullInfo is nil") } if groupID != "" { if authverify.IsManagerUserID(userID, g.config.Share.IMAdminUserID) { - *opUser = &sdkws.GroupMemberFullInfo{ + *targetUser = &sdkws.GroupMemberFullInfo{ GroupID: groupID, UserID: userID, RoleLevel: constant.GroupAdmin, @@ -253,7 +253,7 @@ func (g *NotificationSender) fillOpUserByUserID(ctx context.Context, userID stri } else { member, err := g.db.TakeGroupMember(ctx, groupID, userID) if err == nil { - *opUser = g.groupMemberDB2PB(member, 0) + *targetUser = g.groupMemberDB2PB(member, 0) } else if !(errors.Is(err, mongo.ErrNoDocuments) || errs.ErrRecordNotFound.Is(err)) { return err } @@ -263,8 +263,8 @@ func (g *NotificationSender) fillOpUserByUserID(ctx context.Context, userID stri if err != nil { return err } - if *opUser == nil { - *opUser = &sdkws.GroupMemberFullInfo{ + if *targetUser == nil { + *targetUser = &sdkws.GroupMemberFullInfo{ GroupID: groupID, UserID: userID, Nickname: user.Nickname, @@ -272,11 +272,11 @@ func (g *NotificationSender) fillOpUserByUserID(ctx context.Context, userID stri OperatorUserID: userID, } } else { - if (*opUser).Nickname == "" { - (*opUser).Nickname = user.Nickname + if (*targetUser).Nickname == "" { + (*targetUser).Nickname = user.Nickname } - if (*opUser).FaceURL == "" { - (*opUser).FaceURL = user.FaceURL + if (*targetUser).FaceURL == "" { + (*targetUser).FaceURL = user.FaceURL } } return nil @@ -557,15 +557,13 @@ func (g *NotificationSender) GroupApplicationAgreeMemberEnterNotification(ctx co InvitedUserList: users, } opUserID := mcontext.GetOpUserID(ctx) - if err = g.fillOpUserByUserID(ctx, opUserID, &tips.OpUser, tips.Group.GroupID); err != nil { + if err = g.fillUserByUserID(ctx, opUserID, &tips.OpUser, tips.Group.GroupID); err != nil { return nil } - switch { - case invitedOpUserID == "": - case invitedOpUserID == opUserID: + if invitedOpUserID == opUserID { tips.InviterUser = tips.OpUser - default: - if err = g.fillOpUserByUserID(ctx, invitedOpUserID, &tips.InviterUser, tips.Group.GroupID); err != nil { + } else { + if err = g.fillUserByUserID(ctx, invitedOpUserID, &tips.InviterUser, tips.Group.GroupID); err != nil { return err } } From 964ee7a8dd6b673f6800f7f243462570d595130c Mon Sep 17 00:00:00 2001 From: chao <48119764+withchao@users.noreply.github.com> Date: Wed, 5 Mar 2025 17:04:57 +0800 Subject: [PATCH 131/199] feat: sending messages supports returning fields modified by webhook (#3192) * pb * fix: Modifying other fields while setting IsPrivateChat does not take effect * fix: quote message error revoke * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * upgrading pkg tools * fix * fix * optimize log output * feat: support GetLastMessage * feat: support GetLastMessage * feat: s3 switch * feat: s3 switch * fix: GetUsersOnline * feat: SendBusinessNotification supported configuration parameters * feat: SendBusinessNotification supported configuration parameters * feat: SendBusinessNotification supported configuration parameters * feat: seq conversion failed without exiting * fix: DeleteDoc crash * fix: fill send time * fix: fill send time * fix: crash caused by withdrawing messages from users who have left the group * fix: user msg timestamp * seq read config * seq read config * fix: the source message of the reference is withdrawn, and the referenced message is deleted * feat: optimize the default notification.yml * fix: shouldPushOffline * fix: the sorting is wrong after canceling the administrator in group settings * feat: Sending messages supports returning fields modified by webhook * feat: Sending messages supports returning fields modified by webhook * feat: Sending messages supports returning fields modified by webhook --- go.mod | 2 +- go.sum | 4 +- internal/api/msg.go | 84 ++++++++++++++++- internal/rpc/msg/callback.go | 17 +++- internal/rpc/msg/send.go | 60 +++++++----- pkg/apistruct/manage.go | 12 +++ pkg/apistruct/msg_test.go | 1 - test/webhook/msgmodify/main.go | 65 +++++++++++++ tools/streammsg/main.go | 161 --------------------------------- 9 files changed, 208 insertions(+), 198 deletions(-) delete mode 100644 pkg/apistruct/msg_test.go create mode 100644 test/webhook/msgmodify/main.go delete mode 100644 tools/streammsg/main.go diff --git a/go.mod b/go.mod index 57b2b6f7e..7bf9e6ef6 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/gorilla/websocket v1.5.1 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/mitchellh/mapstructure v1.5.0 - github.com/openimsdk/protocol v0.0.72-alpha.78 + github.com/openimsdk/protocol v0.0.72-alpha.79 github.com/openimsdk/tools v0.0.50-alpha.74 github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 diff --git a/go.sum b/go.sum index 0f870fa45..2a86d97ea 100644 --- a/go.sum +++ b/go.sum @@ -347,8 +347,8 @@ github.com/onsi/gomega v1.25.0 h1:Vw7br2PCDYijJHSfBOWhov+8cAnUf8MfMaIOV323l6Y= github.com/onsi/gomega v1.25.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdMPGhM= github.com/openimsdk/gomake v0.0.15-alpha.2 h1:5Q8yl8ezy2yx+q8/ucU/t4kJnDfCzNOrkXcDACCqtyM= github.com/openimsdk/gomake v0.0.15-alpha.2/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= -github.com/openimsdk/protocol v0.0.72-alpha.78 h1:n9HVj5olMPiGLF3Z4apPvvYzn2yOpyrsn2/YiAaIsxw= -github.com/openimsdk/protocol v0.0.72-alpha.78/go.mod h1:WF7EuE55vQvpyUAzDXcqg+B+446xQyEba0X35lTINmw= +github.com/openimsdk/protocol v0.0.72-alpha.79 h1:e46no8WVAsmTzyy405klrdoUiG7u+1ohDsXvQuFng4s= +github.com/openimsdk/protocol v0.0.72-alpha.79/go.mod h1:WF7EuE55vQvpyUAzDXcqg+B+446xQyEba0X35lTINmw= github.com/openimsdk/tools v0.0.50-alpha.74 h1:yh10SiMiivMEjicEQg+QAsH4pvaO+4noMPdlw+ew0Kc= github.com/openimsdk/tools v0.0.50-alpha.74/go.mod h1:n2poR3asX1e1XZce4O+MOWAp+X02QJRFvhcLCXZdzRo= github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= diff --git a/internal/api/msg.go b/internal/api/msg.go index a7c2c888f..1ec1f44a7 100644 --- a/internal/api/msg.go +++ b/internal/api/msg.go @@ -17,10 +17,12 @@ package api import ( "encoding/base64" "encoding/json" + "sync" "github.com/gin-gonic/gin" "github.com/go-playground/validator/v10" "github.com/mitchellh/mapstructure" + "google.golang.org/protobuf/reflect/protoreflect" "github.com/openimsdk/open-im-server/v3/pkg/apistruct" "github.com/openimsdk/open-im-server/v3/pkg/authverify" @@ -41,6 +43,39 @@ import ( "github.com/openimsdk/tools/utils/timeutil" ) +var ( + msgDataDescriptor []protoreflect.FieldDescriptor + msgDataDescriptorOnce sync.Once +) + +func getMsgDataDescriptor() []protoreflect.FieldDescriptor { + msgDataDescriptorOnce.Do(func() { + skip := make(map[string]struct{}) + respFields := new(msg.SendMsgResp).ProtoReflect().Descriptor().Fields() + for i := 0; i < respFields.Len(); i++ { + field := respFields.Get(i) + if !field.HasJSONName() { + continue + } + skip[field.JSONName()] = struct{}{} + } + fields := new(sdkws.MsgData).ProtoReflect().Descriptor().Fields() + num := fields.Len() + msgDataDescriptor = make([]protoreflect.FieldDescriptor, 0, num) + for i := 0; i < num; i++ { + field := fields.Get(i) + if !field.HasJSONName() { + continue + } + if _, ok := skip[field.JSONName()]; ok { + continue + } + msgDataDescriptor = append(msgDataDescriptor, fields.Get(i)) + } + }) + return msgDataDescriptor +} + type MessageApi struct { Client msg.MsgClient userClient *rpcli.UserClient @@ -197,6 +232,42 @@ func (m *MessageApi) getSendMsgReq(c *gin.Context, req apistruct.SendMsg) (sendM return m.newUserSendMsgReq(c, &req), nil } +func (m *MessageApi) getModifyFields(req, respModify *sdkws.MsgData) map[string]any { + if req == nil || respModify == nil { + return nil + } + fields := make(map[string]any) + reqProtoReflect := req.ProtoReflect() + respProtoReflect := respModify.ProtoReflect() + for _, descriptor := range getMsgDataDescriptor() { + reqValue := reqProtoReflect.Get(descriptor) + respValue := respProtoReflect.Get(descriptor) + if !reqValue.Equal(respValue) { + val := respValue.Interface() + name := descriptor.JSONName() + if name == "content" { + if bs, ok := val.([]byte); ok { + val = string(bs) + } + } + fields[name] = val + } + } + if len(fields) == 0 { + fields = nil + } + return fields +} + +func (m *MessageApi) ginRespSendMsg(c *gin.Context, req *msg.SendMsgReq, resp *msg.SendMsgResp) { + res := m.getModifyFields(req.MsgData, resp.Modify) + resp.Modify = nil + apiresp.GinSuccess(c, &apistruct.SendMsgResp{ + SendMsgResp: resp, + Modify: res, + }) +} + // SendMessage handles the sending of a message. It's an HTTP handler function to be used with Gin framework. func (m *MessageApi) SendMessage(c *gin.Context) { // Initialize a request struct for sending a message. @@ -250,7 +321,7 @@ func (m *MessageApi) SendMessage(c *gin.Context) { } // Respond with a success message and the response payload. - apiresp.GinSuccess(c, respPb) + m.ginRespSendMsg(c, sendMsgReq, respPb) } func (m *MessageApi) SendBusinessNotification(c *gin.Context) { @@ -316,7 +387,7 @@ func (m *MessageApi) SendBusinessNotification(c *gin.Context) { apiresp.GinError(c, err) return } - apiresp.GinSuccess(c, respPb) + m.ginRespSendMsg(c, &sendMsgReq, respPb) } func (m *MessageApi) BatchSendMsg(c *gin.Context) { @@ -370,6 +441,7 @@ func (m *MessageApi) BatchSendMsg(c *gin.Context) { ClientMsgID: rpcResp.ClientMsgID, SendTime: rpcResp.SendTime, RecvID: recvID, + Modify: m.getModifyFields(sendMsgReq.MsgData, rpcResp.Modify), }) } apiresp.GinSuccess(c, resp) @@ -432,7 +504,11 @@ func (m *MessageApi) SendSimpleMessage(c *gin.Context) { Ex: req.Ex, } - respPb, err := m.Client.SendMsg(c, &msg.SendMsgReq{MsgData: msgData}) + sendReq := &msg.SendMsgReq{ + MsgData: msgData, + } + + respPb, err := m.Client.SendMsg(c, sendReq) if err != nil { apiresp.GinError(c, err) return @@ -449,7 +525,7 @@ func (m *MessageApi) SendSimpleMessage(c *gin.Context) { return } - apiresp.GinSuccess(c, respPb) + m.ginRespSendMsg(c, sendReq, respPb) } func (m *MessageApi) CheckMsgIsSendSuccess(c *gin.Context) { diff --git a/internal/rpc/msg/callback.go b/internal/rpc/msg/callback.go index adf1ff735..5bc98de0c 100644 --- a/internal/rpc/msg/callback.go +++ b/internal/rpc/msg/callback.go @@ -17,9 +17,11 @@ package msg import ( "context" "encoding/base64" + "encoding/json" "github.com/openimsdk/open-im-server/v3/pkg/apistruct" "github.com/openimsdk/open-im-server/v3/pkg/common/webhook" + "github.com/openimsdk/tools/errs" "github.com/openimsdk/tools/utils/stringutil" cbapi "github.com/openimsdk/open-im-server/v3/pkg/callbackstruct" @@ -136,11 +138,11 @@ func (m *msgServer) webhookAfterSendGroupMsg(ctx context.Context, after *config. m.webhookClient.AsyncPostWithQuery(ctx, cbReq.GetCallbackCommand(), cbReq, &cbapi.CallbackAfterSendGroupMsgResp{}, after, buildKeyMsgDataQuery(msg.MsgData)) } -func (m *msgServer) webhookBeforeMsgModify(ctx context.Context, before *config.BeforeConfig, msg *pbchat.SendMsgReq) error { +func (m *msgServer) webhookBeforeMsgModify(ctx context.Context, before *config.BeforeConfig, msg *pbchat.SendMsgReq, beforeMsgData **sdkws.MsgData) error { return webhook.WithCondition(ctx, before, func(ctx context.Context) error { - if msg.MsgData.ContentType != constant.Text { - return nil - } + //if msg.MsgData.ContentType != constant.Text { + // return nil + //} if !filterBeforeMsg(msg, before) { return nil } @@ -151,9 +153,14 @@ func (m *msgServer) webhookBeforeMsgModify(ctx context.Context, before *config.B if err := m.webhookClient.SyncPost(ctx, cbReq.GetCallbackCommand(), cbReq, resp, before); err != nil { return err } - + if beforeMsgData != nil { + *beforeMsgData = proto.Clone(msg.MsgData).(*sdkws.MsgData) + } if resp.Content != nil { msg.MsgData.Content = []byte(*resp.Content) + if err := json.Unmarshal(msg.MsgData.Content, &struct{}{}); err != nil { + return errs.ErrArgs.WrapMsg("webhook msg modify content is not json", "content", string(msg.MsgData.Content)) + } } datautil.NotNilReplace(msg.MsgData.OfflinePushInfo, resp.OfflinePushInfo) datautil.NotNilReplace(&msg.MsgData.RecvID, resp.RecvID) diff --git a/internal/rpc/msg/send.go b/internal/rpc/msg/send.go index 19f4e9ffd..f226c4921 100644 --- a/internal/rpc/msg/send.go +++ b/internal/rpc/msg/send.go @@ -29,31 +29,44 @@ import ( "github.com/openimsdk/tools/log" "github.com/openimsdk/tools/mcontext" "github.com/openimsdk/tools/utils/datautil" + "google.golang.org/protobuf/proto" ) func (m *msgServer) SendMsg(ctx context.Context, req *pbmsg.SendMsgReq) (*pbmsg.SendMsgResp, error) { - if req.MsgData != nil { - m.encapsulateMsgData(req.MsgData) - if req.MsgData.ContentType == constant.Stream { - if err := m.handlerStreamMsg(ctx, req.MsgData); err != nil { - return nil, err - } - } - switch req.MsgData.SessionType { - case constant.SingleChatType: - return m.sendMsgSingleChat(ctx, req) - case constant.NotificationChatType: - return m.sendMsgNotification(ctx, req) - case constant.ReadGroupChatType: - return m.sendMsgGroupChat(ctx, req) - default: - return nil, errs.ErrArgs.WrapMsg("unknown sessionType") + if req.MsgData == nil { + return nil, errs.ErrArgs.WrapMsg("msgData is nil") + } + before := new(*sdkws.MsgData) + resp, err := m.sendMsg(ctx, req, before) + if err != nil { + return nil, err + } + if *before != nil && proto.Equal(*before, req.MsgData) == false { + resp.Modify = req.MsgData + } + return resp, nil +} + +func (m *msgServer) sendMsg(ctx context.Context, req *pbmsg.SendMsgReq, before **sdkws.MsgData) (*pbmsg.SendMsgResp, error) { + m.encapsulateMsgData(req.MsgData) + if req.MsgData.ContentType == constant.Stream { + if err := m.handlerStreamMsg(ctx, req.MsgData); err != nil { + return nil, err } } - return nil, errs.ErrArgs.WrapMsg("msgData is nil") + switch req.MsgData.SessionType { + case constant.SingleChatType: + return m.sendMsgSingleChat(ctx, req, before) + case constant.NotificationChatType: + return m.sendMsgNotification(ctx, req, before) + case constant.ReadGroupChatType: + return m.sendMsgGroupChat(ctx, req, before) + default: + return nil, errs.ErrArgs.WrapMsg("unknown sessionType") + } } -func (m *msgServer) sendMsgGroupChat(ctx context.Context, req *pbmsg.SendMsgReq) (resp *pbmsg.SendMsgResp, err error) { +func (m *msgServer) sendMsgGroupChat(ctx context.Context, req *pbmsg.SendMsgReq, before **sdkws.MsgData) (resp *pbmsg.SendMsgResp, err error) { if err = m.messageVerification(ctx, req); err != nil { prommetrics.GroupChatMsgProcessFailedCounter.Inc() return nil, err @@ -62,7 +75,7 @@ func (m *msgServer) sendMsgGroupChat(ctx context.Context, req *pbmsg.SendMsgReq) if err = m.webhookBeforeSendGroupMsg(ctx, &m.config.WebhooksConfig.BeforeSendGroupMsg, req); err != nil { return nil, err } - if err := m.webhookBeforeMsgModify(ctx, &m.config.WebhooksConfig.BeforeMsgModify, req); err != nil { + if err := m.webhookBeforeMsgModify(ctx, &m.config.WebhooksConfig.BeforeMsgModify, req, before); err != nil { return nil, err } err = m.MsgDatabase.MsgToMQ(ctx, conversationutil.GenConversationUniqueKeyForGroup(req.MsgData.GroupID), req.MsgData) @@ -144,7 +157,7 @@ func (m *msgServer) setConversationAtInfo(nctx context.Context, msg *sdkws.MsgDa } } -func (m *msgServer) sendMsgNotification(ctx context.Context, req *pbmsg.SendMsgReq) (resp *pbmsg.SendMsgResp, err error) { +func (m *msgServer) sendMsgNotification(ctx context.Context, req *pbmsg.SendMsgReq, before **sdkws.MsgData) (resp *pbmsg.SendMsgResp, err error) { if err := m.MsgDatabase.MsgToMQ(ctx, conversationutil.GenConversationUniqueKeyForSingle(req.MsgData.SendID, req.MsgData.RecvID), req.MsgData); err != nil { return nil, err } @@ -156,7 +169,7 @@ func (m *msgServer) sendMsgNotification(ctx context.Context, req *pbmsg.SendMsgR return resp, nil } -func (m *msgServer) sendMsgSingleChat(ctx context.Context, req *pbmsg.SendMsgReq) (resp *pbmsg.SendMsgResp, err error) { +func (m *msgServer) sendMsgSingleChat(ctx context.Context, req *pbmsg.SendMsgReq, before **sdkws.MsgData) (resp *pbmsg.SendMsgResp, err error) { if err := m.messageVerification(ctx, req); err != nil { return nil, err } @@ -176,12 +189,11 @@ func (m *msgServer) sendMsgSingleChat(ctx context.Context, req *pbmsg.SendMsgReq } if !isSend { prommetrics.SingleChatMsgProcessFailedCounter.Inc() - return nil, nil + return nil, errs.ErrArgs.WrapMsg("message is not sent") } else { - if err := m.webhookBeforeMsgModify(ctx, &m.config.WebhooksConfig.BeforeMsgModify, req); err != nil { + if err := m.webhookBeforeMsgModify(ctx, &m.config.WebhooksConfig.BeforeMsgModify, req, before); err != nil { return nil, err } - if err := m.MsgDatabase.MsgToMQ(ctx, conversationutil.GenConversationUniqueKeyForSingle(req.MsgData.SendID, req.MsgData.RecvID), req.MsgData); err != nil { prommetrics.SingleChatMsgProcessFailedCounter.Inc() return nil, err diff --git a/pkg/apistruct/manage.go b/pkg/apistruct/manage.go index 4f1d5863f..cb2b1756f 100644 --- a/pkg/apistruct/manage.go +++ b/pkg/apistruct/manage.go @@ -15,6 +15,7 @@ package apistruct import ( + pbmsg "github.com/openimsdk/protocol/msg" "github.com/openimsdk/protocol/sdkws" ) @@ -139,4 +140,15 @@ type SingleReturnResult struct { // RecvID uniquely identifies the receiver of the message. RecvID string `json:"recvID"` + + // Modify fields modified via webhook. + Modify map[string]any `json:"modify,omitempty"` +} + +type SendMsgResp struct { + // SendMsgResp original response. + *pbmsg.SendMsgResp + + // Modify fields modified via webhook. + Modify map[string]any `json:"modify,omitempty"` } diff --git a/pkg/apistruct/msg_test.go b/pkg/apistruct/msg_test.go deleted file mode 100644 index 28f878a9f..000000000 --- a/pkg/apistruct/msg_test.go +++ /dev/null @@ -1 +0,0 @@ -package apistruct diff --git a/test/webhook/msgmodify/main.go b/test/webhook/msgmodify/main.go new file mode 100644 index 000000000..3bdfa0ec5 --- /dev/null +++ b/test/webhook/msgmodify/main.go @@ -0,0 +1,65 @@ +package main + +import ( + "encoding/json" + "fmt" + "io" + "net/http" + "strings" + + "github.com/gin-gonic/gin" + cbapi "github.com/openimsdk/open-im-server/v3/pkg/callbackstruct" + "github.com/openimsdk/protocol/constant" +) + +func main() { + g := gin.Default() + g.POST("/callbackExample/callbackBeforeMsgModifyCommand", toGin(handlerMsg)) + if err := g.Run(":10006"); err != nil { + panic(err) + } +} + +func toGin[R any](fn func(c *gin.Context, req *R)) gin.HandlerFunc { + return func(c *gin.Context) { + body, err := io.ReadAll(c.Request.Body) + if err != nil { + c.String(http.StatusInternalServerError, err.Error()) + return + } + fmt.Printf("HTTP %s %s %s\n", c.Request.Method, c.Request.URL, body) + var req R + if err := json.Unmarshal(body, &req); err != nil { + c.String(http.StatusInternalServerError, err.Error()) + return + } + fn(c, &req) + } +} + +func handlerMsg(c *gin.Context, req *cbapi.CallbackMsgModifyCommandReq) { + var resp cbapi.CallbackMsgModifyCommandResp + if req.ContentType != constant.Text { + c.JSON(http.StatusOK, &resp) + return + } + var textElem struct { + Content string `json:"content"` + } + if err := json.Unmarshal([]byte(req.Content), &textElem); err != nil { + c.String(http.StatusInternalServerError, err.Error()) + return + } + const word = "xxx" + if strings.Contains(textElem.Content, word) { + textElem.Content = strings.ReplaceAll(textElem.Content, word, strings.Repeat("*", len(word))) + content, err := json.Marshal(&textElem) + if err != nil { + c.String(http.StatusInternalServerError, err.Error()) + return + } + tmp := string(content) + resp.Content = &tmp + } + c.JSON(http.StatusOK, &resp) +} diff --git a/tools/streammsg/main.go b/tools/streammsg/main.go deleted file mode 100644 index bb567e233..000000000 --- a/tools/streammsg/main.go +++ /dev/null @@ -1,161 +0,0 @@ -package main - -import ( - "bytes" - "context" - "encoding/json" - "fmt" - "github.com/gin-gonic/gin" - "github.com/google/uuid" - "github.com/openimsdk/open-im-server/v3/pkg/apistruct" - cbapi "github.com/openimsdk/open-im-server/v3/pkg/callbackstruct" - "github.com/openimsdk/protocol/auth" - "github.com/openimsdk/protocol/constant" - "github.com/openimsdk/protocol/msg" - "github.com/openimsdk/tools/apiresp" - "github.com/openimsdk/tools/errs" - "io" - "net/http" - "strings" - "time" -) - -const ( - getAdminToken = "/auth/get_admin_token" - sendMsgApi = "/msg/send_msg" - appendStreamMsg = "/msg/append_stream_msg" -) - -var ( - ApiAddr = "http://127.0.0.1:10002" - Token string -) - -func ApiCall[R any](api string, req any) (*R, error) { - data, err := json.Marshal(req) - if err != nil { - return nil, err - } - ctx, cancel := context.WithTimeout(context.Background(), time.Second*3) - defer cancel() - request, err := http.NewRequestWithContext(ctx, http.MethodPost, ApiAddr+api, bytes.NewBuffer(data)) - if err != nil { - return nil, err - } - if Token != "" { - request.Header.Set("token", Token) - } - request.Header.Set(constant.OperationID, uuid.New().String()) - response, err := http.DefaultClient.Do(request) - if err != nil { - return nil, err - } - defer response.Body.Close() - var resp R - apiResponse := apiresp.ApiResponse{ - Data: &resp, - } - if err := json.NewDecoder(response.Body).Decode(&apiResponse); err != nil { - return nil, err - } - if apiResponse.ErrCode != 0 { - return nil, errs.NewCodeError(apiResponse.ErrCode, apiResponse.ErrMsg) - } - return &resp, nil -} - -func main() { - resp, err := ApiCall[auth.GetAdminTokenResp](getAdminToken, &auth.GetAdminTokenReq{ - Secret: "openIM123", - UserID: "imAdmin", - }) - if err != nil { - fmt.Println("get admin token failed", err) - return - } - Token = resp.Token - g := gin.Default() - g.POST("/callbackExample/callbackAfterSendSingleMsgCommand", toGin(handlerUserMsg)) - if err := g.Run(":10006"); err != nil { - panic(err) - } -} - -func toGin[R any](fn func(c *gin.Context, req *R) error) gin.HandlerFunc { - return func(c *gin.Context) { - body, err := io.ReadAll(c.Request.Body) - if err != nil { - c.String(http.StatusInternalServerError, err.Error()) - return - } - fmt.Printf("HTTP %s %s %s\n", c.Request.Method, c.Request.URL, body) - var req R - if err := json.Unmarshal(body, &req); err != nil { - c.String(http.StatusInternalServerError, err.Error()) - return - } - if err := fn(c, &req); err != nil { - c.String(http.StatusInternalServerError, err.Error()) - return - } - c.String(http.StatusOK, "{}") - } -} - -func handlerUserMsg(c *gin.Context, req *cbapi.CallbackAfterSendSingleMsgReq) error { - if req.ContentType != constant.Text { - return nil - } - if !strings.Contains(req.Content, "stream") { - return nil - } - apiReq := apistruct.SendMsgReq{ - RecvID: req.SendID, - SendMsg: apistruct.SendMsg{ - SendID: req.RecvID, - SenderNickname: "xxx", - SenderFaceURL: "", - SenderPlatformID: constant.AdminPlatformID, - ContentType: constant.Stream, - SessionType: req.SessionType, - SendTime: time.Now().UnixMilli(), - Content: map[string]any{ - "type": "xxx", - "content": "server test stream msg", - }, - }, - } - go func() { - if err := doPushStreamMsg(&apiReq); err != nil { - fmt.Println("doPushStreamMsg failed", err) - return - } - fmt.Println("doPushStreamMsg success") - }() - return nil -} - -func doPushStreamMsg(sendReq *apistruct.SendMsgReq) error { - resp, err := ApiCall[msg.SendMsgResp](sendMsgApi, sendReq) - if err != nil { - return err - } - const num = 5 - for i := 1; i <= num; i++ { - _, err := ApiCall[msg.AppendStreamMsgResp](appendStreamMsg, &msg.AppendStreamMsgReq{ - ClientMsgID: resp.ClientMsgID, - StartIndex: int64(i - 1), - Packets: []string{ - fmt.Sprintf("stream_msg_packet_%03d", i), - }, - End: i == num, - }) - if err != nil { - fmt.Println("append stream msg failed", "clientMsgID", resp.ClientMsgID, "index", fmt.Sprintf("%d/%d", i, num), "error", err) - return err - } - fmt.Println("append stream msg success", "clientMsgID", resp.ClientMsgID, "index", fmt.Sprintf("%d/%d", i, num)) - time.Sleep(time.Second * 10) - } - return nil -} From 7859f2ccb37d1b6eebf8e4139c6e4baf3e58bfd1 Mon Sep 17 00:00:00 2001 From: Monet Lee Date: Fri, 7 Mar 2025 17:03:40 +0800 Subject: [PATCH 132/199] refactor: improve setConversations method. (#3194) --- internal/rpc/conversation/conversation.go | 99 +++-------------------- internal/rpc/conversation/db_map.go | 85 +++++++++++++++++++ 2 files changed, 95 insertions(+), 89 deletions(-) create mode 100644 internal/rpc/conversation/db_map.go diff --git a/internal/rpc/conversation/conversation.go b/internal/rpc/conversation/conversation.go index cd7839234..165d1bdc2 100644 --- a/internal/rpc/conversation/conversation.go +++ b/internal/rpc/conversation/conversation.go @@ -237,6 +237,7 @@ func (c *conversationServer) SetConversations(ctx context.Context, req *pbconver if req.Conversation == nil { return nil, errs.ErrArgs.WrapMsg("conversation must not be nil") } + if req.Conversation.ConversationType == constant.WriteGroupChatType { groupInfo, err := c.groupClient.GetGroupInfo(ctx, req.Conversation.GroupID) if err != nil { @@ -271,109 +272,29 @@ func (c *conversationServer) SetConversations(ctx context.Context, req *pbconver conversation.UserID = req.Conversation.UserID conversation.GroupID = req.Conversation.GroupID - m := make(map[string]any) - - setConversationFieldsFunc := func() { - if req.Conversation.RecvMsgOpt != nil { - conversation.RecvMsgOpt = req.Conversation.RecvMsgOpt.Value - m["recv_msg_opt"] = req.Conversation.RecvMsgOpt.Value - } - if req.Conversation.AttachedInfo != nil { - conversation.AttachedInfo = req.Conversation.AttachedInfo.Value - m["attached_info"] = req.Conversation.AttachedInfo.Value - } - if req.Conversation.Ex != nil { - conversation.Ex = req.Conversation.Ex.Value - m["ex"] = req.Conversation.Ex.Value - } - if req.Conversation.IsPinned != nil { - conversation.IsPinned = req.Conversation.IsPinned.Value - m["is_pinned"] = req.Conversation.IsPinned.Value - } - if req.Conversation.GroupAtType != nil { - conversation.GroupAtType = req.Conversation.GroupAtType.Value - m["group_at_type"] = req.Conversation.GroupAtType.Value - } - if req.Conversation.MsgDestructTime != nil { - conversation.MsgDestructTime = req.Conversation.MsgDestructTime.Value - m["msg_destruct_time"] = req.Conversation.MsgDestructTime.Value - } - if req.Conversation.IsMsgDestruct != nil { - conversation.IsMsgDestruct = req.Conversation.IsMsgDestruct.Value - m["is_msg_destruct"] = req.Conversation.IsMsgDestruct.Value - } - if req.Conversation.BurnDuration != nil { - conversation.BurnDuration = req.Conversation.BurnDuration.Value - m["burn_duration"] = req.Conversation.BurnDuration.Value - } + m, conversation, err := UpdateConversationsMap(ctx, req) + if err != nil { + return nil, err } - // set need set field in conversation - setConversationFieldsFunc() - for userID := range conversationMap { - unequal := len(m) - - if req.Conversation.RecvMsgOpt != nil { - if req.Conversation.RecvMsgOpt.Value == conversationMap[userID].RecvMsgOpt { - unequal-- - } - } - - if req.Conversation.AttachedInfo != nil { - if req.Conversation.AttachedInfo.Value == conversationMap[userID].AttachedInfo { - unequal-- - } - } + unequal := UserUpdateCheckMap(ctx, userID, req.Conversation, conversationMap[userID]) - if req.Conversation.Ex != nil { - if req.Conversation.Ex.Value == conversationMap[userID].Ex { - unequal-- - } - } - if req.Conversation.IsPinned != nil { - if req.Conversation.IsPinned.Value == conversationMap[userID].IsPinned { - unequal-- - } - } - - if req.Conversation.GroupAtType != nil { - if req.Conversation.GroupAtType.Value == conversationMap[userID].GroupAtType { - unequal-- - } - } - - if req.Conversation.MsgDestructTime != nil { - if req.Conversation.MsgDestructTime.Value == conversationMap[userID].MsgDestructTime { - unequal-- - } - } - - if req.Conversation.IsMsgDestruct != nil { - if req.Conversation.IsMsgDestruct.Value == conversationMap[userID].IsMsgDestruct { - unequal-- - } - } - - if req.Conversation.BurnDuration != nil { - if req.Conversation.BurnDuration.Value == conversationMap[userID].BurnDuration { - unequal-- - } - } - - if unequal > 0 { + if unequal { needUpdateUsersList = append(needUpdateUsersList, userID) } } + if len(m) != 0 && len(needUpdateUsersList) != 0 { if err := c.conversationDatabase.SetUsersConversationFieldTx(ctx, needUpdateUsersList, &conversation, m); err != nil { return nil, err } - for _, v := range needUpdateUsersList { - c.conversationNotificationSender.ConversationChangeNotification(ctx, v, []string{req.Conversation.ConversationID}) + for _, userID := range needUpdateUsersList { + c.conversationNotificationSender.ConversationChangeNotification(ctx, userID, []string{req.Conversation.ConversationID}) } } + if req.Conversation.IsPrivateChat != nil && req.Conversation.ConversationType != constant.ReadGroupChatType { var conversations []*dbModel.Conversation for _, ownerUserID := range req.UserIDs { diff --git a/internal/rpc/conversation/db_map.go b/internal/rpc/conversation/db_map.go new file mode 100644 index 000000000..4acb16e4b --- /dev/null +++ b/internal/rpc/conversation/db_map.go @@ -0,0 +1,85 @@ +package conversation + +import ( + "context" + + dbModel "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" + "github.com/openimsdk/protocol/conversation" +) + +func UpdateConversationsMap(ctx context.Context, req *conversation.SetConversationsReq) (m map[string]any, conversation dbModel.Conversation, err error) { + m = make(map[string]any) + + conversation.ConversationID = req.Conversation.ConversationID + conversation.ConversationType = req.Conversation.ConversationType + conversation.UserID = req.Conversation.UserID + conversation.GroupID = req.Conversation.GroupID + + if req.Conversation.RecvMsgOpt != nil { + conversation.RecvMsgOpt = req.Conversation.RecvMsgOpt.Value + m["recv_msg_opt"] = req.Conversation.RecvMsgOpt.Value + } + + if req.Conversation.AttachedInfo != nil { + conversation.AttachedInfo = req.Conversation.AttachedInfo.Value + m["attached_info"] = req.Conversation.AttachedInfo.Value + } + + if req.Conversation.Ex != nil { + conversation.Ex = req.Conversation.Ex.Value + m["ex"] = req.Conversation.Ex.Value + } + if req.Conversation.IsPinned != nil { + conversation.IsPinned = req.Conversation.IsPinned.Value + m["is_pinned"] = req.Conversation.IsPinned.Value + } + if req.Conversation.GroupAtType != nil { + conversation.GroupAtType = req.Conversation.GroupAtType.Value + m["group_at_type"] = req.Conversation.GroupAtType.Value + } + if req.Conversation.MsgDestructTime != nil { + conversation.MsgDestructTime = req.Conversation.MsgDestructTime.Value + m["msg_destruct_time"] = req.Conversation.MsgDestructTime.Value + } + if req.Conversation.IsMsgDestruct != nil { + conversation.IsMsgDestruct = req.Conversation.IsMsgDestruct.Value + m["is_msg_destruct"] = req.Conversation.IsMsgDestruct.Value + } + if req.Conversation.BurnDuration != nil { + conversation.BurnDuration = req.Conversation.BurnDuration.Value + m["burn_duration"] = req.Conversation.BurnDuration.Value + } + + return m, conversation, nil +} + +func UserUpdateCheckMap(ctx context.Context, userID string, req *conversation.ConversationReq, conversation *dbModel.Conversation) (unequal bool) { + unequal = false + + if req.RecvMsgOpt != nil && conversation.RecvMsgOpt != req.RecvMsgOpt.Value { + unequal = true + } + if req.AttachedInfo != nil && conversation.AttachedInfo != req.AttachedInfo.Value { + unequal = true + } + if req.Ex != nil && conversation.Ex != req.Ex.Value { + unequal = true + } + if req.IsPinned != nil && conversation.IsPinned != req.IsPinned.Value { + unequal = true + } + if req.GroupAtType != nil && conversation.GroupAtType != req.GroupAtType.Value { + unequal = true + } + if req.MsgDestructTime != nil && conversation.MsgDestructTime != req.MsgDestructTime.Value { + unequal = true + } + if req.IsMsgDestruct != nil && conversation.IsMsgDestruct != req.IsMsgDestruct.Value { + unequal = true + } + if req.BurnDuration != nil && conversation.BurnDuration != req.BurnDuration.Value { + unequal = true + } + + return unequal +} From c0c4ba21a8e53547385b403906d48e4378a809e5 Mon Sep 17 00:00:00 2001 From: Monet Lee Date: Wed, 12 Mar 2025 11:38:18 +0800 Subject: [PATCH 133/199] fix: solve unocrrect invite notification (#3213) --- internal/rpc/group/group.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/internal/rpc/group/group.go b/internal/rpc/group/group.go index 0a877a03e..e1604e7e5 100644 --- a/internal/rpc/group/group.go +++ b/internal/rpc/group/group.go @@ -390,6 +390,8 @@ func (g *groupServer) InviteUserToGroup(ctx context.Context, req *pbgroup.Invite if err := g.PopulateGroupMember(ctx, groupMember); err != nil { return nil, err } + } else { + opUserID = mcontext.GetOpUserID(ctx) } if err := g.webhookBeforeInviteUserToGroup(ctx, &g.config.WebhooksConfig.BeforeInviteUserToGroup, req); err != nil && err != servererrs.ErrCallbackContinue { @@ -425,6 +427,7 @@ func (g *groupServer) InviteUserToGroup(ctx context.Context, req *pbgroup.Invite } } } + var groupMembers []*model.GroupMember for _, userID := range req.InvitedUserIDs { member := &model.GroupMember{ From e76e02fdba3f03a87b91bbbc1acb3c659ee11d7b Mon Sep 17 00:00:00 2001 From: icey-yu <119291641+icey-yu@users.noreply.github.com> Date: Wed, 12 Mar 2025 14:54:37 +0800 Subject: [PATCH 134/199] feat: set configs (#3183) --- internal/api/config_manager.go | 105 ++++++++++++++++++++++++++++++++ internal/api/router.go | 1 + pkg/apistruct/config_manager.go | 4 ++ 3 files changed, 110 insertions(+) diff --git a/internal/api/config_manager.go b/internal/api/config_manager.go index 15f5e7004..aca02c52c 100644 --- a/internal/api/config_manager.go +++ b/internal/api/config_manager.go @@ -15,6 +15,7 @@ import ( "github.com/openimsdk/tools/apiresp" "github.com/openimsdk/tools/errs" "github.com/openimsdk/tools/log" + "github.com/openimsdk/tools/utils/datautil" "github.com/openimsdk/tools/utils/runtimeenv" clientv3 "go.etcd.io/etcd/client/v3" ) @@ -144,6 +145,110 @@ func (cm *ConfigManager) SetConfig(c *gin.Context) { apiresp.GinSuccess(c, nil) } +func (cm *ConfigManager) SetConfigs(c *gin.Context) { + if cm.config.Discovery.Enable != config.ETCD { + apiresp.GinError(c, errs.New("only etcd support set config").Wrap()) + return + } + var req apistruct.SetConfigsReq + if err := c.BindJSON(&req); err != nil { + apiresp.GinError(c, errs.ErrArgs.WithDetail(err.Error()).Wrap()) + return + } + var ( + err error + ops []*clientv3.Op + ) + + for _, cf := range req.Configs { + var op *clientv3.Op + switch cf.ConfigName { + case cm.config.Discovery.GetConfigFileName(): + op, err = compareAndOp[config.Discovery](c, cm.config.Name2Config(cf.ConfigName), &cf, cm) + case cm.config.Kafka.GetConfigFileName(): + op, err = compareAndOp[config.Kafka](c, cm.config.Name2Config(cf.ConfigName), &cf, cm) + case cm.config.LocalCache.GetConfigFileName(): + op, err = compareAndOp[config.LocalCache](c, cm.config.Name2Config(cf.ConfigName), &cf, cm) + case cm.config.Log.GetConfigFileName(): + op, err = compareAndOp[config.Log](c, cm.config.Name2Config(cf.ConfigName), &cf, cm) + case cm.config.Minio.GetConfigFileName(): + op, err = compareAndOp[config.Minio](c, cm.config.Name2Config(cf.ConfigName), &cf, cm) + case cm.config.Mongo.GetConfigFileName(): + op, err = compareAndOp[config.Mongo](c, cm.config.Name2Config(cf.ConfigName), &cf, cm) + case cm.config.Notification.GetConfigFileName(): + op, err = compareAndOp[config.Notification](c, cm.config.Name2Config(cf.ConfigName), &cf, cm) + case cm.config.API.GetConfigFileName(): + op, err = compareAndOp[config.API](c, cm.config.Name2Config(cf.ConfigName), &cf, cm) + case cm.config.CronTask.GetConfigFileName(): + op, err = compareAndOp[config.CronTask](c, cm.config.Name2Config(cf.ConfigName), &cf, cm) + case cm.config.MsgGateway.GetConfigFileName(): + op, err = compareAndOp[config.MsgGateway](c, cm.config.Name2Config(cf.ConfigName), &cf, cm) + case cm.config.MsgTransfer.GetConfigFileName(): + op, err = compareAndOp[config.MsgTransfer](c, cm.config.Name2Config(cf.ConfigName), &cf, cm) + case cm.config.Push.GetConfigFileName(): + op, err = compareAndOp[config.Push](c, cm.config.Name2Config(cf.ConfigName), &cf, cm) + case cm.config.Auth.GetConfigFileName(): + op, err = compareAndOp[config.Auth](c, cm.config.Name2Config(cf.ConfigName), &cf, cm) + case cm.config.Conversation.GetConfigFileName(): + op, err = compareAndOp[config.Conversation](c, cm.config.Name2Config(cf.ConfigName), &cf, cm) + case cm.config.Friend.GetConfigFileName(): + op, err = compareAndOp[config.Friend](c, cm.config.Name2Config(cf.ConfigName), &cf, cm) + case cm.config.Group.GetConfigFileName(): + op, err = compareAndOp[config.Group](c, cm.config.Name2Config(cf.ConfigName), &cf, cm) + case cm.config.Msg.GetConfigFileName(): + op, err = compareAndOp[config.Msg](c, cm.config.Name2Config(cf.ConfigName), &cf, cm) + case cm.config.Third.GetConfigFileName(): + op, err = compareAndOp[config.Third](c, cm.config.Name2Config(cf.ConfigName), &cf, cm) + case cm.config.User.GetConfigFileName(): + op, err = compareAndOp[config.User](c, cm.config.Name2Config(cf.ConfigName), &cf, cm) + case cm.config.Redis.GetConfigFileName(): + op, err = compareAndOp[config.Redis](c, cm.config.Name2Config(cf.ConfigName), &cf, cm) + case cm.config.Share.GetConfigFileName(): + op, err = compareAndOp[config.Share](c, cm.config.Name2Config(cf.ConfigName), &cf, cm) + case cm.config.Webhooks.GetConfigFileName(): + op, err = compareAndOp[config.Webhooks](c, cm.config.Name2Config(cf.ConfigName), &cf, cm) + default: + apiresp.GinError(c, errs.ErrArgs.Wrap()) + return + } + if err != nil { + apiresp.GinError(c, errs.ErrArgs.WithDetail(err.Error()).Wrap()) + return + } + if op != nil { + ops = append(ops, op) + } + } + if len(ops) > 0 { + tx := cm.client.Txn(c) + if _, err = tx.Then(datautil.Batch(func(op *clientv3.Op) clientv3.Op { return *op }, ops)...).Commit(); err != nil { + apiresp.GinError(c, errs.WrapMsg(err, "save to etcd failed")) + return + } + + } + + apiresp.GinSuccess(c, nil) +} + +func compareAndOp[T any](c *gin.Context, old any, req *apistruct.SetConfigReq, cm *ConfigManager) (*clientv3.Op, error) { + conf := new(T) + err := json.Unmarshal([]byte(req.Data), &conf) + if err != nil { + return nil, errs.ErrArgs.WithDetail(err.Error()).Wrap() + } + eq := reflect.DeepEqual(old, conf) + if eq { + return nil, nil + } + data, err := json.Marshal(conf) + if err != nil { + return nil, errs.ErrArgs.WithDetail(err.Error()).Wrap() + } + op := clientv3.OpPut(etcd.BuildKey(req.ConfigName), string(data)) + return &op, nil +} + func compareAndSave[T any](c *gin.Context, old any, req *apistruct.SetConfigReq, cm *ConfigManager) error { conf := new(T) err := json.Unmarshal([]byte(req.Data), &conf) diff --git a/internal/api/router.go b/internal/api/router.go index ebaff019c..850c23972 100644 --- a/internal/api/router.go +++ b/internal/api/router.go @@ -307,6 +307,7 @@ func newGinRouter(ctx context.Context, client discovery.Conn, cfg *Config) (*gin configGroup.POST("/get_config_list", cm.GetConfigList) configGroup.POST("/get_config", cm.GetConfig) configGroup.POST("/set_config", cm.SetConfig) + configGroup.POST("/set_configs", cm.SetConfigs) configGroup.POST("/reset_config", cm.ResetConfig) configGroup.POST("/set_enable_config_manager", cm.SetEnableConfigManager) configGroup.POST("/get_enable_config_manager", cm.GetEnableConfigManager) diff --git a/pkg/apistruct/config_manager.go b/pkg/apistruct/config_manager.go index 9b8641c9d..ca5683a3e 100644 --- a/pkg/apistruct/config_manager.go +++ b/pkg/apistruct/config_manager.go @@ -15,6 +15,10 @@ type SetConfigReq struct { Data string `json:"data"` } +type SetConfigsReq struct { + Configs []SetConfigReq `json:"configs"` +} + type SetEnableConfigManagerReq struct { Enable bool `json:"enable"` } From 3516f843dbd0d9d421436fc10b6af8f8a0bf2427 Mon Sep 17 00:00:00 2001 From: icey-yu <119291641+icey-yu@users.noreply.github.com> Date: Thu, 13 Mar 2025 17:17:50 +0800 Subject: [PATCH 135/199] fix: AdminToken save to redis && limit 1 for each userID (#3224) --- internal/rpc/user/user.go | 4 +-- pkg/common/storage/controller/auth.go | 45 ++++++++++++--------------- pkg/common/storage/controller/user.go | 5 ++- 3 files changed, 24 insertions(+), 30 deletions(-) diff --git a/internal/rpc/user/user.go b/internal/rpc/user/user.go index 07e3c6201..cd239aae3 100644 --- a/internal/rpc/user/user.go +++ b/internal/rpc/user/user.go @@ -88,7 +88,7 @@ func Start(ctx context.Context, config *Config, client discovery.Conn, server gr users := make([]*tablerelation.User, 0) for _, v := range config.Share.IMAdminUserID { - users = append(users, &tablerelation.User{UserID: v, Nickname: v, AppMangerLevel: constant.AppNotificationAdmin}) + users = append(users, &tablerelation.User{UserID: v, Nickname: v, AppMangerLevel: constant.AppAdmin}) } userDB, err := mgo.NewUserMongo(mgocli.GetDB()) if err != nil { @@ -605,7 +605,7 @@ func (s *userServer) GetNotificationAccount(ctx context.Context, req *pbuser.Get if err != nil { return nil, servererrs.ErrUserIDNotFound.Wrap() } - if user.AppMangerLevel == constant.AppAdmin || user.AppMangerLevel >= constant.AppNotificationAdmin { + if user.AppMangerLevel >= constant.AppAdmin { return &pbuser.GetNotificationAccountResp{Account: &pbuser.NotificationAccountInfo{ UserID: user.UserID, FaceURL: user.FaceURL, diff --git a/pkg/common/storage/controller/auth.go b/pkg/common/storage/controller/auth.go index ee2a06f53..013d8b155 100644 --- a/pkg/common/storage/controller/auth.go +++ b/pkg/common/storage/controller/auth.go @@ -80,31 +80,28 @@ func (a *authDatabase) BatchSetTokenMapByUidPid(ctx context.Context, tokens []st // Create Token. func (a *authDatabase) CreateToken(ctx context.Context, userID string, platformID int) (string, error) { - isAdmin := authverify.IsManagerUserID(userID, a.adminUserIDs) - if !isAdmin { - tokens, err := a.cache.GetAllTokensWithoutError(ctx, userID) - if err != nil { - return "", err - } + tokens, err := a.cache.GetAllTokensWithoutError(ctx, userID) + if err != nil { + return "", err + } - deleteTokenKey, kickedTokenKey, err := a.checkToken(ctx, tokens, platformID) + deleteTokenKey, kickedTokenKey, err := a.checkToken(ctx, tokens, platformID) + if err != nil { + return "", err + } + if len(deleteTokenKey) != 0 { + err = a.cache.DeleteTokenByUidPid(ctx, userID, platformID, deleteTokenKey) if err != nil { return "", err } - if len(deleteTokenKey) != 0 { - err = a.cache.DeleteTokenByUidPid(ctx, userID, platformID, deleteTokenKey) + } + if len(kickedTokenKey) != 0 { + for _, k := range kickedTokenKey { + err := a.cache.SetTokenFlagEx(ctx, userID, platformID, k, constant.KickedToken) if err != nil { return "", err } - } - if len(kickedTokenKey) != 0 { - for _, k := range kickedTokenKey { - err := a.cache.SetTokenFlagEx(ctx, userID, platformID, k, constant.KickedToken) - if err != nil { - return "", err - } - log.ZDebug(ctx, "kicked token in create token", "token", k) - } + log.ZDebug(ctx, "kicked token in create token", "token", k) } } @@ -115,10 +112,8 @@ func (a *authDatabase) CreateToken(ctx context.Context, userID string, platformI return "", errs.WrapMsg(err, "token.SignedString") } - if !isAdmin { - if err = a.cache.SetTokenFlagEx(ctx, userID, platformID, tokenString, constant.NormalToken); err != nil { - return "", err - } + if err = a.cache.SetTokenFlagEx(ctx, userID, platformID, tokenString, constant.NormalToken); err != nil { + return "", err } return tokenString, nil @@ -224,9 +219,6 @@ func (a *authDatabase) checkToken(ctx context.Context, tokens map[int]map[string } //var adminTokenMaxNum = a.multiLogin.MaxNumOneEnd - //if a.multiLogin.Policy == constant.Customize { - // adminTokenMaxNum = a.multiLogin.CustomizeLoginNum[constant.AdminPlatformID] - //} //l := len(adminToken) //if platformID == constant.AdminPlatformID { // l++ @@ -234,5 +226,8 @@ func (a *authDatabase) checkToken(ctx context.Context, tokens map[int]map[string //if l > adminTokenMaxNum { // kickToken = append(kickToken, adminToken[:l-adminTokenMaxNum]...) //} + if platformID == constant.AdminPlatformID { + kickToken = append(kickToken, adminToken...) + } return deleteToken, kickToken, nil } diff --git a/pkg/common/storage/controller/user.go b/pkg/common/storage/controller/user.go index a8ef1033e..f97c5330b 100644 --- a/pkg/common/storage/controller/user.go +++ b/pkg/common/storage/controller/user.go @@ -21,12 +21,11 @@ import ( "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" "github.com/openimsdk/protocol/constant" + "github.com/openimsdk/protocol/user" "github.com/openimsdk/tools/db/pagination" "github.com/openimsdk/tools/db/tx" - "github.com/openimsdk/tools/utils/datautil" - - "github.com/openimsdk/protocol/user" "github.com/openimsdk/tools/errs" + "github.com/openimsdk/tools/utils/datautil" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache" ) From 0b9dbd301c211c6ec27bcda9f7aaa311fe5064c5 Mon Sep 17 00:00:00 2001 From: icey-yu <119291641+icey-yu@users.noreply.github.com> Date: Thu, 13 Mar 2025 18:21:48 +0800 Subject: [PATCH 136/199] feat: check if the secret in config/share.yml has been changed during registration (#3223) * feat: check if the secret in config/share.yml has been changed during registration. * fix: cicd * fix: code * fix: cicd * fix: cicd * fix: cicd * fix: cicd * fix: cicd --- .github/workflows/go-build-test.yml | 25 ++++++++++++++++++++----- internal/rpc/user/user.go | 8 ++++++++ pkg/common/servererrs/code.go | 14 ++++++++------ pkg/common/servererrs/predefine.go | 2 ++ 4 files changed, 38 insertions(+), 11 deletions(-) diff --git a/.github/workflows/go-build-test.yml b/.github/workflows/go-build-test.yml index 4033603e6..9e2aa3f1c 100644 --- a/.github/workflows/go-build-test.yml +++ b/.github/workflows/go-build-test.yml @@ -12,6 +12,10 @@ jobs: go-build: name: Test with go ${{ matrix.go_version }} on ${{ matrix.os }} runs-on: ${{ matrix.os }} + + env: + SHARE_CONFIG_PATH: config/share.yml + permissions: contents: write pull-requests: write @@ -40,6 +44,10 @@ jobs: with: compose-file: "./docker-compose.yml" + - name: Modify Server Configuration + run: | + yq e '.secret = 123456' -i ${{ env.SHARE_CONFIG_PATH }} + # - name: Get Internal IP Address # id: get-ip # run: | @@ -71,6 +79,11 @@ jobs: go mod download go install github.com/magefile/mage@latest + - name: Modify Chat Configuration + run: | + cd ${{ github.workspace }}/chat-repo + yq e '.openIM.secret = 123456' -i ${{ env.SHARE_CONFIG_PATH }} + - name: Build and test Chat Services run: | cd ${{ github.workspace }}/chat-repo @@ -132,7 +145,7 @@ jobs: # Test get admin token get_admin_token_response=$(curl -X POST -H "Content-Type: application/json" -H "operationID: imAdmin" -d '{ - "secret": "openIM123", + "secret": "123456", "platformID": 2, "userID": "imAdmin" }' http://127.0.0.1:10002/auth/get_admin_token) @@ -169,7 +182,8 @@ jobs: contents: write env: SDK_DIR: openim-sdk-core - CONFIG_PATH: config/notification.yml + NOTIFICATION_CONFIG_PATH: config/notification.yml + SHARE_CONFIG_PATH: config/share.yml strategy: matrix: @@ -184,7 +198,7 @@ jobs: uses: actions/checkout@v4 with: repository: "openimsdk/openim-sdk-core" - ref: "release-v3.8" + ref: "main" path: ${{ env.SDK_DIR }} - name: Set up Go ${{ matrix.go_version }} @@ -199,8 +213,9 @@ jobs: - name: Modify Server Configuration run: | - yq e '.groupCreated.isSendMsg = true' -i ${{ env.CONFIG_PATH }} - yq e '.friendApplicationApproved.isSendMsg = true' -i ${{ env.CONFIG_PATH }} + yq e '.groupCreated.isSendMsg = true' -i ${{ env.NOTIFICATION_CONFIG_PATH }} + yq e '.friendApplicationApproved.isSendMsg = true' -i ${{ env.NOTIFICATION_CONFIG_PATH }} + yq e '.secret = 123456' -i ${{ env.SHARE_CONFIG_PATH }} - name: Start Server Services run: | diff --git a/internal/rpc/user/user.go b/internal/rpc/user/user.go index cd239aae3..3e8ec3537 100644 --- a/internal/rpc/user/user.go +++ b/internal/rpc/user/user.go @@ -49,6 +49,10 @@ import ( "google.golang.org/grpc" ) +const ( + defaultSecret = "openIM123" +) + type userServer struct { pbuser.UnimplementedUserServer online cache.OnlineCache @@ -273,6 +277,10 @@ func (s *userServer) UserRegister(ctx context.Context, req *pbuser.UserRegisterR if len(req.Users) == 0 { return nil, errs.ErrArgs.WrapMsg("users is empty") } + // check if secret is changed + if s.config.Share.Secret == defaultSecret { + return nil, servererrs.ErrSecretNotChanged.Wrap() + } if err = authverify.CheckAdmin(ctx, s.config.Share.IMAdminUserID); err != nil { return nil, err diff --git a/pkg/common/servererrs/code.go b/pkg/common/servererrs/code.go index 3d0aa4a71..906f890a5 100644 --- a/pkg/common/servererrs/code.go +++ b/pkg/common/servererrs/code.go @@ -37,7 +37,8 @@ const ( // General error codes. const ( - NoError = 0 // No error + NoError = 0 // No error + DatabaseError = 90002 // Database error (redis/mysql, etc.) NetworkError = 90004 // Network error DataError = 90007 // Data error @@ -45,11 +46,12 @@ const ( CallbackError = 80000 // General error codes. - ServerInternalError = 500 // Server internal error - ArgsError = 1001 // Input parameter error - NoPermissionError = 1002 // Insufficient permission - DuplicateKeyError = 1003 - RecordNotFoundError = 1004 // Record does not exist + ServerInternalError = 500 // Server internal error + ArgsError = 1001 // Input parameter error + NoPermissionError = 1002 // Insufficient permission + DuplicateKeyError = 1003 + RecordNotFoundError = 1004 // Record does not exist + SecretNotChangedError = 1050 // secret not changed // Account error codes. UserIDNotFoundError = 1101 // UserID does not exist or is not registered diff --git a/pkg/common/servererrs/predefine.go b/pkg/common/servererrs/predefine.go index ab09aa512..b1d6b06a9 100644 --- a/pkg/common/servererrs/predefine.go +++ b/pkg/common/servererrs/predefine.go @@ -17,6 +17,8 @@ package servererrs import "github.com/openimsdk/tools/errs" var ( + ErrSecretNotChanged = errs.NewCodeError(SecretNotChangedError, "secret not changed, please change secret in config/share.yml for security reasons") + ErrDatabase = errs.NewCodeError(DatabaseError, "DatabaseError") ErrNetwork = errs.NewCodeError(NetworkError, "NetworkError") ErrCallback = errs.NewCodeError(CallbackError, "CallbackError") From b969827b9af9120b28f267b7f600e3b97c928e59 Mon Sep 17 00:00:00 2001 From: Monet Lee Date: Fri, 14 Mar 2025 16:46:29 +0800 Subject: [PATCH 137/199] feat: Implement webhook in createConversation (#3228) * update test method args. * feat: implement createConversations webhook function. * improve webhookCreateConversations Implement * implement createconversation webhook. * remove unused paramaters. --- config/webhooks.yml | 20 ++- internal/rpc/conversation/callback.go | 117 ++++++++++++++++++ internal/rpc/conversation/conversation.go | 73 ++++++++--- internal/tools/cron/cron_test.go | 2 +- pkg/callbackstruct/constant.go | 98 ++++++++------- pkg/callbackstruct/conversation.go | 91 ++++++++++++++ pkg/common/cmd/conversation.go | 2 + pkg/common/config/config.go | 102 +++++++-------- pkg/common/config/load_config_test.go | 16 +-- pkg/common/storage/controller/conversation.go | 19 ++- 10 files changed, 410 insertions(+), 130 deletions(-) create mode 100644 internal/rpc/conversation/callback.go create mode 100644 pkg/callbackstruct/conversation.go diff --git a/config/webhooks.yml b/config/webhooks.yml index 41c60e7e2..283a23ed4 100644 --- a/config/webhooks.yml +++ b/config/webhooks.yml @@ -7,11 +7,11 @@ beforeSendSingleMsg: # If not set, all contentType messages will through this filter. deniedTypes: [] beforeUpdateUserInfoEx: - enable: false + enable: false timeout: 5 failedContinue: true afterUpdateUserInfoEx: - enable: false + enable: false timeout: 5 afterSendSingleMsg: enable: false @@ -181,3 +181,19 @@ afterImportFriends: afterRemoveBlack: enable: false timeout: 5 +beforeCreateSingleChatConversations: + enable: false + timeout: 5 + failedContinue: false +afterCreateSingleChatConversations: + enable: false + timeout: 5 + failedContinue: false +beforeCreateGroupChatConversations: + enable: false + timeout: 5 + failedContinue: false +afterCreateGroupChatConversations: + enable: false + timeout: 5 + failedContinue: false diff --git a/internal/rpc/conversation/callback.go b/internal/rpc/conversation/callback.go new file mode 100644 index 000000000..93e925afd --- /dev/null +++ b/internal/rpc/conversation/callback.go @@ -0,0 +1,117 @@ +package conversation + +import ( + "context" + + "github.com/openimsdk/open-im-server/v3/pkg/callbackstruct" + "github.com/openimsdk/open-im-server/v3/pkg/common/config" + dbModel "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" + "github.com/openimsdk/open-im-server/v3/pkg/common/webhook" + "github.com/openimsdk/tools/utils/datautil" +) + +func (c *conversationServer) webhookBeforeCreateSingleChatConversations(ctx context.Context, before *config.BeforeConfig, req *dbModel.Conversation) error { + return webhook.WithCondition(ctx, before, func(ctx context.Context) error { + cbReq := &callbackstruct.CallbackBeforeCreateSingleChatConversationsReq{ + CallbackCommand: callbackstruct.CallbackBeforeCreateSingleChatConversationsCommand, + OwnerUserID: req.OwnerUserID, + ConversationID: req.ConversationID, + ConversationType: req.ConversationType, + UserID: req.UserID, + RecvMsgOpt: req.RecvMsgOpt, + IsPinned: req.IsPinned, + IsPrivateChat: req.IsPrivateChat, + BurnDuration: req.BurnDuration, + GroupAtType: req.GroupAtType, + AttachedInfo: req.AttachedInfo, + Ex: req.Ex, + } + + resp := &callbackstruct.CallbackBeforeCreateSingleChatConversationsResp{} + + if err := c.webhookClient.SyncPost(ctx, cbReq.GetCallbackCommand(), cbReq, resp, before); err != nil { + return err + } + + datautil.NotNilReplace(&req.RecvMsgOpt, resp.RecvMsgOpt) + datautil.NotNilReplace(&req.IsPinned, resp.IsPinned) + datautil.NotNilReplace(&req.IsPrivateChat, resp.IsPrivateChat) + datautil.NotNilReplace(&req.BurnDuration, resp.BurnDuration) + datautil.NotNilReplace(&req.GroupAtType, resp.GroupAtType) + datautil.NotNilReplace(&req.AttachedInfo, resp.AttachedInfo) + datautil.NotNilReplace(&req.Ex, resp.Ex) + return nil + }) +} + +func (c *conversationServer) webhookAfterCreateSingleChatConversations(ctx context.Context, after *config.AfterConfig, req *dbModel.Conversation) error { + cbReq := &callbackstruct.CallbackAfterCreateSingleChatConversationsReq{ + CallbackCommand: callbackstruct.CallbackAfterCreateSingleChatConversationsCommand, + OwnerUserID: req.OwnerUserID, + ConversationID: req.ConversationID, + ConversationType: req.ConversationType, + UserID: req.UserID, + RecvMsgOpt: req.RecvMsgOpt, + IsPinned: req.IsPinned, + IsPrivateChat: req.IsPrivateChat, + BurnDuration: req.BurnDuration, + GroupAtType: req.GroupAtType, + AttachedInfo: req.AttachedInfo, + Ex: req.Ex, + } + + c.webhookClient.AsyncPost(ctx, cbReq.GetCallbackCommand(), cbReq, &callbackstruct.CallbackAfterCreateSingleChatConversationsResp{}, after) + return nil +} + +func (c *conversationServer) webhookBeforeCreateGroupChatConversations(ctx context.Context, before *config.BeforeConfig, req *dbModel.Conversation) error { + return webhook.WithCondition(ctx, before, func(ctx context.Context) error { + cbReq := &callbackstruct.CallbackBeforeCreateGroupChatConversationsReq{ + CallbackCommand: callbackstruct.CallbackBeforeCreateGroupChatConversationsCommand, + ConversationID: req.ConversationID, + ConversationType: req.ConversationType, + GroupID: req.GroupID, + RecvMsgOpt: req.RecvMsgOpt, + IsPinned: req.IsPinned, + IsPrivateChat: req.IsPrivateChat, + BurnDuration: req.BurnDuration, + GroupAtType: req.GroupAtType, + AttachedInfo: req.AttachedInfo, + Ex: req.Ex, + } + + resp := &callbackstruct.CallbackBeforeCreateGroupChatConversationsResp{} + + if err := c.webhookClient.SyncPost(ctx, cbReq.GetCallbackCommand(), cbReq, resp, before); err != nil { + return err + } + + datautil.NotNilReplace(&req.RecvMsgOpt, resp.RecvMsgOpt) + datautil.NotNilReplace(&req.IsPinned, resp.IsPinned) + datautil.NotNilReplace(&req.IsPrivateChat, resp.IsPrivateChat) + datautil.NotNilReplace(&req.BurnDuration, resp.BurnDuration) + datautil.NotNilReplace(&req.GroupAtType, resp.GroupAtType) + datautil.NotNilReplace(&req.AttachedInfo, resp.AttachedInfo) + datautil.NotNilReplace(&req.Ex, resp.Ex) + return nil + }) +} + +func (c *conversationServer) webhookAfterCreateGroupChatConversations(ctx context.Context, after *config.AfterConfig, req *dbModel.Conversation) error { + cbReq := &callbackstruct.CallbackAfterCreateGroupChatConversationsReq{ + CallbackCommand: callbackstruct.CallbackAfterCreateGroupChatConversationsCommand, + ConversationID: req.ConversationID, + ConversationType: req.ConversationType, + GroupID: req.GroupID, + RecvMsgOpt: req.RecvMsgOpt, + IsPinned: req.IsPinned, + IsPrivateChat: req.IsPrivateChat, + BurnDuration: req.BurnDuration, + GroupAtType: req.GroupAtType, + AttachedInfo: req.AttachedInfo, + Ex: req.Ex, + } + + c.webhookClient.AsyncPost(ctx, cbReq.GetCallbackCommand(), cbReq, &callbackstruct.CallbackAfterCreateGroupChatConversationsResp{}, after) + return nil +} diff --git a/internal/rpc/conversation/conversation.go b/internal/rpc/conversation/conversation.go index 165d1bdc2..8bc3fd2a9 100644 --- a/internal/rpc/conversation/conversation.go +++ b/internal/rpc/conversation/conversation.go @@ -30,6 +30,7 @@ import ( "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database/mgo" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" dbModel "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" + "github.com/openimsdk/open-im-server/v3/pkg/common/webhook" "github.com/openimsdk/open-im-server/v3/pkg/localcache" "github.com/openimsdk/open-im-server/v3/pkg/msgprocessor" "github.com/openimsdk/protocol/constant" @@ -49,9 +50,10 @@ type conversationServer struct { conversationNotificationSender *ConversationNotificationSender config *Config - userClient *rpcli.UserClient - msgClient *rpcli.MsgClient - groupClient *rpcli.GroupClient + webhookClient *webhook.Client + userClient *rpcli.UserClient + msgClient *rpcli.MsgClient + groupClient *rpcli.GroupClient } type Config struct { @@ -60,6 +62,7 @@ type Config struct { MongodbConfig config.Mongo NotificationConfig config.Notification Share config.Share + WebhooksConfig config.Webhooks LocalCacheConfig config.LocalCache Discovery config.Discovery } @@ -90,16 +93,25 @@ func Start(ctx context.Context, config *Config, client discovery.Conn, server gr if err != nil { return err } + msgClient := rpcli.NewMsgClient(msgConn) + + cs := conversationServer{ + config: config, + webhookClient: webhook.NewWebhookClient(config.WebhooksConfig.URL), + userClient: rpcli.NewUserClient(userConn), + groupClient: rpcli.NewGroupClient(groupConn), + msgClient: msgClient, + } + + cs.conversationNotificationSender = NewConversationNotificationSender(&config.NotificationConfig, msgClient) + cs.conversationDatabase = controller.NewConversationDatabase( + conversationDB, + redis.NewConversationRedis(rdb, &config.LocalCacheConfig, conversationDB), + mgocli.GetTx()) + localcache.InitLocalCache(&config.LocalCacheConfig) - pbconversation.RegisterConversationServer(server, &conversationServer{ - conversationNotificationSender: NewConversationNotificationSender(&config.NotificationConfig, msgClient), - conversationDatabase: controller.NewConversationDatabase(conversationDB, - redis.NewConversationRedis(rdb, &config.LocalCacheConfig, conversationDB), mgocli.GetTx()), - userClient: rpcli.NewUserClient(userConn), - groupClient: rpcli.NewGroupClient(groupConn), - msgClient: msgClient, - }) + pbconversation.RegisterConversationServer(server, &cs) return nil } @@ -326,49 +338,76 @@ func (c *conversationServer) GetRecvMsgNotNotifyUserIDs(ctx context.Context, req func (c *conversationServer) CreateSingleChatConversations(ctx context.Context, req *pbconversation.CreateSingleChatConversationsReq, ) (*pbconversation.CreateSingleChatConversationsResp, error) { + var conversation dbModel.Conversation switch req.ConversationType { case constant.SingleChatType: - var conversation dbModel.Conversation + // sendUser create conversation.ConversationID = req.ConversationID conversation.ConversationType = req.ConversationType conversation.OwnerUserID = req.SendID conversation.UserID = req.RecvID + if err := c.webhookBeforeCreateSingleChatConversations(ctx, &c.config.WebhooksConfig.BeforeCreateSingleChatConversations, &conversation); err != nil && err != servererrs.ErrCallbackContinue { + return nil, err + } + err := c.conversationDatabase.CreateConversation(ctx, []*dbModel.Conversation{&conversation}) if err != nil { log.ZWarn(ctx, "create conversation failed", err, "conversation", conversation) } + c.webhookAfterCreateSingleChatConversations(ctx, &c.config.WebhooksConfig.AfterCreateSingleChatConversations, &conversation) + + // recvUser create conversation2 := conversation conversation2.OwnerUserID = req.RecvID conversation2.UserID = req.SendID + if err := c.webhookBeforeCreateSingleChatConversations(ctx, &c.config.WebhooksConfig.BeforeCreateSingleChatConversations, &conversation); err != nil && err != servererrs.ErrCallbackContinue { + return nil, err + } + err = c.conversationDatabase.CreateConversation(ctx, []*dbModel.Conversation{&conversation2}) if err != nil { log.ZWarn(ctx, "create conversation failed", err, "conversation2", conversation) } + + c.webhookAfterCreateSingleChatConversations(ctx, &c.config.WebhooksConfig.AfterCreateSingleChatConversations, &conversation2) case constant.NotificationChatType: - var conversation dbModel.Conversation conversation.ConversationID = req.ConversationID conversation.ConversationType = req.ConversationType conversation.OwnerUserID = req.RecvID conversation.UserID = req.SendID + if err := c.webhookBeforeCreateSingleChatConversations(ctx, &c.config.WebhooksConfig.BeforeCreateSingleChatConversations, &conversation); err != nil && err != servererrs.ErrCallbackContinue { + return nil, err + } + err := c.conversationDatabase.CreateConversation(ctx, []*dbModel.Conversation{&conversation}) if err != nil { log.ZWarn(ctx, "create conversation failed", err, "conversation2", conversation) } + + c.webhookAfterCreateSingleChatConversations(ctx, &c.config.WebhooksConfig.AfterCreateSingleChatConversations, &conversation) } return &pbconversation.CreateSingleChatConversationsResp{}, nil } func (c *conversationServer) CreateGroupChatConversations(ctx context.Context, req *pbconversation.CreateGroupChatConversationsReq) (*pbconversation.CreateGroupChatConversationsResp, error) { - err := c.conversationDatabase.CreateGroupChatConversation(ctx, req.GroupID, req.UserIDs) - if err != nil { + var conversation dbModel.Conversation + + conversation.ConversationID = msgprocessor.GetConversationIDBySessionType(constant.ReadGroupChatType, req.GroupID) + conversation.GroupID = req.GroupID + conversation.ConversationType = constant.ReadGroupChatType + + if err := c.webhookBeforeCreateGroupChatConversations(ctx, &c.config.WebhooksConfig.BeforeCreateGroupChatConversations, &conversation); err != nil { return nil, err } - conversationID := msgprocessor.GetConversationIDBySessionType(constant.ReadGroupChatType, req.GroupID) - if err := c.msgClient.SetUserConversationMaxSeq(ctx, conversationID, req.UserIDs, 0); err != nil { + + err := c.conversationDatabase.CreateGroupChatConversation(ctx, req.GroupID, req.UserIDs, &conversation) + if err != nil { return nil, err } + + c.webhookAfterCreateGroupChatConversations(ctx, &c.config.WebhooksConfig.AfterCreateGroupChatConversations, &conversation) return &pbconversation.CreateGroupChatConversationsResp{}, nil } diff --git a/internal/tools/cron/cron_test.go b/internal/tools/cron/cron_test.go index b98b14f14..af827fc38 100644 --- a/internal/tools/cron/cron_test.go +++ b/internal/tools/cron/cron_test.go @@ -46,7 +46,7 @@ func TestName(t *testing.T) { srv := &cronServer{ ctx: ctx, - config: &CronTaskConfig{ + config: &Config{ CronTask: config.CronTask{ RetainChatRecords: 1, FileExpireTime: 1, diff --git a/pkg/callbackstruct/constant.go b/pkg/callbackstruct/constant.go index 73f89a719..bbc34e71f 100644 --- a/pkg/callbackstruct/constant.go +++ b/pkg/callbackstruct/constant.go @@ -15,51 +15,55 @@ package callbackstruct const ( - CallbackBeforeInviteJoinGroupCommand = "callbackBeforeInviteJoinGroupCommand" - CallbackAfterJoinGroupCommand = "callbackAfterJoinGroupCommand" - CallbackAfterSetGroupInfoCommand = "callbackAfterSetGroupInfoCommand" - CallbackAfterSetGroupInfoExCommand = "callbackAfterSetGroupInfoExCommand" - CallbackBeforeSetGroupInfoCommand = "callbackBeforeSetGroupInfoCommand" - CallbackBeforeSetGroupInfoExCommand = "callbackBeforeSetGroupInfoExCommand" - CallbackAfterRevokeMsgCommand = "callbackBeforeAfterMsgCommand" - CallbackBeforeAddBlackCommand = "callbackBeforeAddBlackCommand" - CallbackAfterAddFriendCommand = "callbackAfterAddFriendCommand" - CallbackBeforeAddFriendAgreeCommand = "callbackBeforeAddFriendAgreeCommand" - CallbackAfterAddFriendAgreeCommand = "callbackAfterAddFriendAgreeCommand" - CallbackAfterDeleteFriendCommand = "callbackAfterDeleteFriendCommand" - CallbackBeforeImportFriendsCommand = "callbackBeforeImportFriendsCommand" - CallbackAfterImportFriendsCommand = "callbackAfterImportFriendsCommand" - CallbackAfterRemoveBlackCommand = "callbackAfterRemoveBlackCommand" - CallbackAfterQuitGroupCommand = "callbackAfterQuitGroupCommand" - CallbackAfterKickGroupCommand = "callbackAfterKickGroupCommand" - CallbackAfterDisMissGroupCommand = "callbackAfterDisMissGroupCommand" - CallbackBeforeJoinGroupCommand = "callbackBeforeJoinGroupCommand" - CallbackAfterGroupMsgReadCommand = "callbackAfterGroupMsgReadCommand" - CallbackBeforeMsgModifyCommand = "callbackBeforeMsgModifyCommand" - CallbackAfterUpdateUserInfoCommand = "callbackAfterUpdateUserInfoCommand" - CallbackAfterUpdateUserInfoExCommand = "callbackAfterUpdateUserInfoExCommand" - CallbackBeforeUpdateUserInfoExCommand = "callbackBeforeUpdateUserInfoExCommand" - CallbackBeforeUserRegisterCommand = "callbackBeforeUserRegisterCommand" - CallbackAfterUserRegisterCommand = "callbackAfterUserRegisterCommand" - CallbackAfterTransferGroupOwnerCommand = "callbackAfterTransferGroupOwnerCommand" - CallbackBeforeSetFriendRemarkCommand = "callbackBeforeSetFriendRemarkCommand" - CallbackAfterSetFriendRemarkCommand = "callbackAfterSetFriendRemarkCommand" - CallbackAfterSingleMsgReadCommand = "callbackAfterSingleMsgReadCommand" - CallbackBeforeSendSingleMsgCommand = "callbackBeforeSendSingleMsgCommand" - CallbackAfterSendSingleMsgCommand = "callbackAfterSendSingleMsgCommand" - CallbackBeforeSendGroupMsgCommand = "callbackBeforeSendGroupMsgCommand" - CallbackAfterSendGroupMsgCommand = "callbackAfterSendGroupMsgCommand" - CallbackAfterUserOnlineCommand = "callbackAfterUserOnlineCommand" - CallbackAfterUserOfflineCommand = "callbackAfterUserOfflineCommand" - CallbackAfterUserKickOffCommand = "callbackAfterUserKickOffCommand" - CallbackBeforeOfflinePushCommand = "callbackBeforeOfflinePushCommand" - CallbackBeforeOnlinePushCommand = "callbackBeforeOnlinePushCommand" - CallbackBeforeGroupOnlinePushCommand = "callbackBeforeGroupOnlinePushCommand" - CallbackBeforeAddFriendCommand = "callbackBeforeAddFriendCommand" - CallbackBeforeUpdateUserInfoCommand = "callbackBeforeUpdateUserInfoCommand" - CallbackBeforeCreateGroupCommand = "callbackBeforeCreateGroupCommand" - CallbackAfterCreateGroupCommand = "callbackAfterCreateGroupCommand" - CallbackBeforeMembersJoinGroupCommand = "callbackBeforeMembersJoinGroupCommand" - CallbackBeforeSetGroupMemberInfoCommand = "callbackBeforeSetGroupMemberInfoCommand" - CallbackAfterSetGroupMemberInfoCommand = "callbackAfterSetGroupMemberInfoCommand" + CallbackBeforeInviteJoinGroupCommand = "callbackBeforeInviteJoinGroupCommand" + CallbackAfterJoinGroupCommand = "callbackAfterJoinGroupCommand" + CallbackAfterSetGroupInfoCommand = "callbackAfterSetGroupInfoCommand" + CallbackAfterSetGroupInfoExCommand = "callbackAfterSetGroupInfoExCommand" + CallbackBeforeSetGroupInfoCommand = "callbackBeforeSetGroupInfoCommand" + CallbackBeforeSetGroupInfoExCommand = "callbackBeforeSetGroupInfoExCommand" + CallbackAfterRevokeMsgCommand = "callbackBeforeAfterMsgCommand" + CallbackBeforeAddBlackCommand = "callbackBeforeAddBlackCommand" + CallbackAfterAddFriendCommand = "callbackAfterAddFriendCommand" + CallbackBeforeAddFriendAgreeCommand = "callbackBeforeAddFriendAgreeCommand" + CallbackAfterAddFriendAgreeCommand = "callbackAfterAddFriendAgreeCommand" + CallbackAfterDeleteFriendCommand = "callbackAfterDeleteFriendCommand" + CallbackBeforeImportFriendsCommand = "callbackBeforeImportFriendsCommand" + CallbackAfterImportFriendsCommand = "callbackAfterImportFriendsCommand" + CallbackAfterRemoveBlackCommand = "callbackAfterRemoveBlackCommand" + CallbackAfterQuitGroupCommand = "callbackAfterQuitGroupCommand" + CallbackAfterKickGroupCommand = "callbackAfterKickGroupCommand" + CallbackAfterDisMissGroupCommand = "callbackAfterDisMissGroupCommand" + CallbackBeforeJoinGroupCommand = "callbackBeforeJoinGroupCommand" + CallbackAfterGroupMsgReadCommand = "callbackAfterGroupMsgReadCommand" + CallbackBeforeMsgModifyCommand = "callbackBeforeMsgModifyCommand" + CallbackAfterUpdateUserInfoCommand = "callbackAfterUpdateUserInfoCommand" + CallbackAfterUpdateUserInfoExCommand = "callbackAfterUpdateUserInfoExCommand" + CallbackBeforeUpdateUserInfoExCommand = "callbackBeforeUpdateUserInfoExCommand" + CallbackBeforeUserRegisterCommand = "callbackBeforeUserRegisterCommand" + CallbackAfterUserRegisterCommand = "callbackAfterUserRegisterCommand" + CallbackAfterTransferGroupOwnerCommand = "callbackAfterTransferGroupOwnerCommand" + CallbackBeforeSetFriendRemarkCommand = "callbackBeforeSetFriendRemarkCommand" + CallbackAfterSetFriendRemarkCommand = "callbackAfterSetFriendRemarkCommand" + CallbackAfterSingleMsgReadCommand = "callbackAfterSingleMsgReadCommand" + CallbackBeforeSendSingleMsgCommand = "callbackBeforeSendSingleMsgCommand" + CallbackAfterSendSingleMsgCommand = "callbackAfterSendSingleMsgCommand" + CallbackBeforeSendGroupMsgCommand = "callbackBeforeSendGroupMsgCommand" + CallbackAfterSendGroupMsgCommand = "callbackAfterSendGroupMsgCommand" + CallbackAfterUserOnlineCommand = "callbackAfterUserOnlineCommand" + CallbackAfterUserOfflineCommand = "callbackAfterUserOfflineCommand" + CallbackAfterUserKickOffCommand = "callbackAfterUserKickOffCommand" + CallbackBeforeOfflinePushCommand = "callbackBeforeOfflinePushCommand" + CallbackBeforeOnlinePushCommand = "callbackBeforeOnlinePushCommand" + CallbackBeforeGroupOnlinePushCommand = "callbackBeforeGroupOnlinePushCommand" + CallbackBeforeAddFriendCommand = "callbackBeforeAddFriendCommand" + CallbackBeforeUpdateUserInfoCommand = "callbackBeforeUpdateUserInfoCommand" + CallbackBeforeCreateGroupCommand = "callbackBeforeCreateGroupCommand" + CallbackAfterCreateGroupCommand = "callbackAfterCreateGroupCommand" + CallbackBeforeMembersJoinGroupCommand = "callbackBeforeMembersJoinGroupCommand" + CallbackBeforeSetGroupMemberInfoCommand = "callbackBeforeSetGroupMemberInfoCommand" + CallbackAfterSetGroupMemberInfoCommand = "callbackAfterSetGroupMemberInfoCommand" + CallbackBeforeCreateSingleChatConversationsCommand = "callbackBeforeCreateSingleChatConversationsCommand" + CallbackAfterCreateSingleChatConversationsCommand = "callbackAfterCreateSingleChatConversationsCommand" + CallbackBeforeCreateGroupChatConversationsCommand = "callbackBeforeCreateGroupChatConversationsCommand" + CallbackAfterCreateGroupChatConversationsCommand = "callbackAfterCreateGroupChatConversationsCommand" ) diff --git a/pkg/callbackstruct/conversation.go b/pkg/callbackstruct/conversation.go new file mode 100644 index 000000000..14e78094c --- /dev/null +++ b/pkg/callbackstruct/conversation.go @@ -0,0 +1,91 @@ +package callbackstruct + +type CallbackBeforeCreateSingleChatConversationsReq struct { + CallbackCommand `json:"callbackCommand"` + OwnerUserID string `json:"owner_user_id"` + ConversationID string `json:"conversation_id"` + ConversationType int32 `json:"conversation_type"` + UserID string `json:"user_id"` + RecvMsgOpt int32 `json:"recv_msg_opt"` + IsPinned bool `json:"is_pinned"` + IsPrivateChat bool `json:"is_private_chat"` + BurnDuration int32 `json:"burn_duration"` + GroupAtType int32 `json:"group_at_type"` + AttachedInfo string `json:"attached_info"` + Ex string `json:"ex"` +} + +type CallbackBeforeCreateSingleChatConversationsResp struct { + CommonCallbackResp + RecvMsgOpt *int32 `json:"recv_msg_opt"` + IsPinned *bool `json:"is_pinned"` + IsPrivateChat *bool `json:"is_private_chat"` + BurnDuration *int32 `json:"burn_duration"` + GroupAtType *int32 `json:"group_at_type"` + AttachedInfo *string `json:"attached_info"` + Ex *string `json:"ex"` +} + +type CallbackAfterCreateSingleChatConversationsReq struct { + CallbackCommand `json:"callbackCommand"` + OwnerUserID string `json:"owner_user_id"` + ConversationID string `json:"conversation_id"` + ConversationType int32 `json:"conversation_type"` + UserID string `json:"user_id"` + RecvMsgOpt int32 `json:"recv_msg_opt"` + IsPinned bool `json:"is_pinned"` + IsPrivateChat bool `json:"is_private_chat"` + BurnDuration int32 `json:"burn_duration"` + GroupAtType int32 `json:"group_at_type"` + AttachedInfo string `json:"attached_info"` + Ex string `json:"ex"` +} + +type CallbackAfterCreateSingleChatConversationsResp struct { + CommonCallbackResp +} + +type CallbackBeforeCreateGroupChatConversationsReq struct { + CallbackCommand `json:"callbackCommand"` + OwnerUserID string `json:"owner_user_id"` + ConversationID string `json:"conversation_id"` + ConversationType int32 `json:"conversation_type"` + GroupID string `json:"group_id"` + RecvMsgOpt int32 `json:"recv_msg_opt"` + IsPinned bool `json:"is_pinned"` + IsPrivateChat bool `json:"is_private_chat"` + BurnDuration int32 `json:"burn_duration"` + GroupAtType int32 `json:"group_at_type"` + AttachedInfo string `json:"attached_info"` + Ex string `json:"ex"` +} + +type CallbackBeforeCreateGroupChatConversationsResp struct { + CommonCallbackResp + RecvMsgOpt *int32 `json:"recv_msg_opt"` + IsPinned *bool `json:"is_pinned"` + IsPrivateChat *bool `json:"is_private_chat"` + BurnDuration *int32 `json:"burn_duration"` + GroupAtType *int32 `json:"group_at_type"` + AttachedInfo *string `json:"attached_info"` + Ex *string `json:"ex"` +} + +type CallbackAfterCreateGroupChatConversationsReq struct { + CallbackCommand `json:"callbackCommand"` + OwnerUserID string `json:"owner_user_id"` + ConversationID string `json:"conversation_id"` + ConversationType int32 `json:"conversation_type"` + GroupID string `json:"group_id"` + RecvMsgOpt int32 `json:"recv_msg_opt"` + IsPinned bool `json:"is_pinned"` + IsPrivateChat bool `json:"is_private_chat"` + BurnDuration int32 `json:"burn_duration"` + GroupAtType int32 `json:"group_at_type"` + AttachedInfo string `json:"attached_info"` + Ex string `json:"ex"` +} + +type CallbackAfterCreateGroupChatConversationsResp struct { + CommonCallbackResp +} diff --git a/pkg/common/cmd/conversation.go b/pkg/common/cmd/conversation.go index 2f8769897..428c442da 100644 --- a/pkg/common/cmd/conversation.go +++ b/pkg/common/cmd/conversation.go @@ -41,6 +41,7 @@ func NewConversationRpcCmd() *ConversationRpcCmd { config.MongodbConfigFileName: &conversationConfig.MongodbConfig, config.ShareFileName: &conversationConfig.Share, config.NotificationFileName: &conversationConfig.NotificationConfig, + config.WebhooksConfigFileName: &conversationConfig.WebhooksConfig, config.LocalCacheConfigFileName: &conversationConfig.LocalCacheConfig, config.DiscoveryConfigFilename: &conversationConfig.Discovery, } @@ -67,6 +68,7 @@ func (a *ConversationRpcCmd) runE() error { a.conversationConfig.NotificationConfig.GetConfigFileName(), a.conversationConfig.Share.GetConfigFileName(), a.conversationConfig.LocalCacheConfig.GetConfigFileName(), + a.conversationConfig.WebhooksConfig.GetConfigFileName(), a.conversationConfig.Discovery.GetConfigFileName(), }, nil, conversation.Start) diff --git a/pkg/common/config/config.go b/pkg/common/config/config.go index b47e3db68..fa93e3406 100644 --- a/pkg/common/config/config.go +++ b/pkg/common/config/config.go @@ -386,55 +386,59 @@ func (r *RpcService) GetServiceNames() []string { // FullConfig stores all configurations for before and after events type Webhooks struct { - URL string `yaml:"url"` - BeforeSendSingleMsg BeforeConfig `yaml:"beforeSendSingleMsg"` - BeforeUpdateUserInfoEx BeforeConfig `yaml:"beforeUpdateUserInfoEx"` - AfterUpdateUserInfoEx AfterConfig `yaml:"afterUpdateUserInfoEx"` - AfterSendSingleMsg AfterConfig `yaml:"afterSendSingleMsg"` - BeforeSendGroupMsg BeforeConfig `yaml:"beforeSendGroupMsg"` - BeforeMsgModify BeforeConfig `yaml:"beforeMsgModify"` - AfterSendGroupMsg AfterConfig `yaml:"afterSendGroupMsg"` - AfterUserOnline AfterConfig `yaml:"afterUserOnline"` - AfterUserOffline AfterConfig `yaml:"afterUserOffline"` - AfterUserKickOff AfterConfig `yaml:"afterUserKickOff"` - BeforeOfflinePush BeforeConfig `yaml:"beforeOfflinePush"` - BeforeOnlinePush BeforeConfig `yaml:"beforeOnlinePush"` - BeforeGroupOnlinePush BeforeConfig `yaml:"beforeGroupOnlinePush"` - BeforeAddFriend BeforeConfig `yaml:"beforeAddFriend"` - BeforeUpdateUserInfo BeforeConfig `yaml:"beforeUpdateUserInfo"` - AfterUpdateUserInfo AfterConfig `yaml:"afterUpdateUserInfo"` - BeforeCreateGroup BeforeConfig `yaml:"beforeCreateGroup"` - AfterCreateGroup AfterConfig `yaml:"afterCreateGroup"` - BeforeMemberJoinGroup BeforeConfig `yaml:"beforeMemberJoinGroup"` - BeforeSetGroupMemberInfo BeforeConfig `yaml:"beforeSetGroupMemberInfo"` - AfterSetGroupMemberInfo AfterConfig `yaml:"afterSetGroupMemberInfo"` - AfterQuitGroup AfterConfig `yaml:"afterQuitGroup"` - AfterKickGroupMember AfterConfig `yaml:"afterKickGroupMember"` - AfterDismissGroup AfterConfig `yaml:"afterDismissGroup"` - BeforeApplyJoinGroup BeforeConfig `yaml:"beforeApplyJoinGroup"` - AfterGroupMsgRead AfterConfig `yaml:"afterGroupMsgRead"` - AfterSingleMsgRead AfterConfig `yaml:"afterSingleMsgRead"` - BeforeUserRegister BeforeConfig `yaml:"beforeUserRegister"` - AfterUserRegister AfterConfig `yaml:"afterUserRegister"` - AfterTransferGroupOwner AfterConfig `yaml:"afterTransferGroupOwner"` - BeforeSetFriendRemark BeforeConfig `yaml:"beforeSetFriendRemark"` - AfterSetFriendRemark AfterConfig `yaml:"afterSetFriendRemark"` - AfterGroupMsgRevoke AfterConfig `yaml:"afterGroupMsgRevoke"` - AfterJoinGroup AfterConfig `yaml:"afterJoinGroup"` - BeforeInviteUserToGroup BeforeConfig `yaml:"beforeInviteUserToGroup"` - AfterSetGroupInfo AfterConfig `yaml:"afterSetGroupInfo"` - BeforeSetGroupInfo BeforeConfig `yaml:"beforeSetGroupInfo"` - AfterSetGroupInfoEx AfterConfig `yaml:"afterSetGroupInfoEx"` - BeforeSetGroupInfoEx BeforeConfig `yaml:"beforeSetGroupInfoEx"` - AfterRevokeMsg AfterConfig `yaml:"afterRevokeMsg"` - BeforeAddBlack BeforeConfig `yaml:"beforeAddBlack"` - AfterAddFriend AfterConfig `yaml:"afterAddFriend"` - BeforeAddFriendAgree BeforeConfig `yaml:"beforeAddFriendAgree"` - AfterAddFriendAgree AfterConfig `yaml:"afterAddFriendAgree"` - AfterDeleteFriend AfterConfig `yaml:"afterDeleteFriend"` - BeforeImportFriends BeforeConfig `yaml:"beforeImportFriends"` - AfterImportFriends AfterConfig `yaml:"afterImportFriends"` - AfterRemoveBlack AfterConfig `yaml:"afterRemoveBlack"` + URL string `yaml:"url"` + BeforeSendSingleMsg BeforeConfig `yaml:"beforeSendSingleMsg"` + BeforeUpdateUserInfoEx BeforeConfig `yaml:"beforeUpdateUserInfoEx"` + AfterUpdateUserInfoEx AfterConfig `yaml:"afterUpdateUserInfoEx"` + AfterSendSingleMsg AfterConfig `yaml:"afterSendSingleMsg"` + BeforeSendGroupMsg BeforeConfig `yaml:"beforeSendGroupMsg"` + BeforeMsgModify BeforeConfig `yaml:"beforeMsgModify"` + AfterSendGroupMsg AfterConfig `yaml:"afterSendGroupMsg"` + AfterUserOnline AfterConfig `yaml:"afterUserOnline"` + AfterUserOffline AfterConfig `yaml:"afterUserOffline"` + AfterUserKickOff AfterConfig `yaml:"afterUserKickOff"` + BeforeOfflinePush BeforeConfig `yaml:"beforeOfflinePush"` + BeforeOnlinePush BeforeConfig `yaml:"beforeOnlinePush"` + BeforeGroupOnlinePush BeforeConfig `yaml:"beforeGroupOnlinePush"` + BeforeAddFriend BeforeConfig `yaml:"beforeAddFriend"` + BeforeUpdateUserInfo BeforeConfig `yaml:"beforeUpdateUserInfo"` + AfterUpdateUserInfo AfterConfig `yaml:"afterUpdateUserInfo"` + BeforeCreateGroup BeforeConfig `yaml:"beforeCreateGroup"` + AfterCreateGroup AfterConfig `yaml:"afterCreateGroup"` + BeforeMemberJoinGroup BeforeConfig `yaml:"beforeMemberJoinGroup"` + BeforeSetGroupMemberInfo BeforeConfig `yaml:"beforeSetGroupMemberInfo"` + AfterSetGroupMemberInfo AfterConfig `yaml:"afterSetGroupMemberInfo"` + AfterQuitGroup AfterConfig `yaml:"afterQuitGroup"` + AfterKickGroupMember AfterConfig `yaml:"afterKickGroupMember"` + AfterDismissGroup AfterConfig `yaml:"afterDismissGroup"` + BeforeApplyJoinGroup BeforeConfig `yaml:"beforeApplyJoinGroup"` + AfterGroupMsgRead AfterConfig `yaml:"afterGroupMsgRead"` + AfterSingleMsgRead AfterConfig `yaml:"afterSingleMsgRead"` + BeforeUserRegister BeforeConfig `yaml:"beforeUserRegister"` + AfterUserRegister AfterConfig `yaml:"afterUserRegister"` + AfterTransferGroupOwner AfterConfig `yaml:"afterTransferGroupOwner"` + BeforeSetFriendRemark BeforeConfig `yaml:"beforeSetFriendRemark"` + AfterSetFriendRemark AfterConfig `yaml:"afterSetFriendRemark"` + AfterGroupMsgRevoke AfterConfig `yaml:"afterGroupMsgRevoke"` + AfterJoinGroup AfterConfig `yaml:"afterJoinGroup"` + BeforeInviteUserToGroup BeforeConfig `yaml:"beforeInviteUserToGroup"` + AfterSetGroupInfo AfterConfig `yaml:"afterSetGroupInfo"` + BeforeSetGroupInfo BeforeConfig `yaml:"beforeSetGroupInfo"` + AfterSetGroupInfoEx AfterConfig `yaml:"afterSetGroupInfoEx"` + BeforeSetGroupInfoEx BeforeConfig `yaml:"beforeSetGroupInfoEx"` + AfterRevokeMsg AfterConfig `yaml:"afterRevokeMsg"` + BeforeAddBlack BeforeConfig `yaml:"beforeAddBlack"` + AfterAddFriend AfterConfig `yaml:"afterAddFriend"` + BeforeAddFriendAgree BeforeConfig `yaml:"beforeAddFriendAgree"` + AfterAddFriendAgree AfterConfig `yaml:"afterAddFriendAgree"` + AfterDeleteFriend AfterConfig `yaml:"afterDeleteFriend"` + BeforeImportFriends BeforeConfig `yaml:"beforeImportFriends"` + AfterImportFriends AfterConfig `yaml:"afterImportFriends"` + AfterRemoveBlack AfterConfig `yaml:"afterRemoveBlack"` + BeforeCreateSingleChatConversations BeforeConfig `yaml:"beforeCreateSingleChatConversations"` + AfterCreateSingleChatConversations AfterConfig `yaml:"afterCreateSingleChatConversations"` + BeforeCreateGroupChatConversations BeforeConfig `yaml:"beforeCreateGroupChatConversations"` + AfterCreateGroupChatConversations AfterConfig `yaml:"afterCreateGroupChatConversations"` } type ZooKeeper struct { diff --git a/pkg/common/config/load_config_test.go b/pkg/common/config/load_config_test.go index 612d44335..f11d91dad 100644 --- a/pkg/common/config/load_config_test.go +++ b/pkg/common/config/load_config_test.go @@ -10,7 +10,7 @@ import ( func TestLoadLogConfig(t *testing.T) { var log Log os.Setenv("IMENV_LOG_REMAINLOGLEVEL", "5") - err := Load("../../../config/", "log.yml", "IMENV_LOG", "source", &log) + err := Load("../../../config/", "log.yml", "IMENV_LOG", &log) assert.Nil(t, err) t.Log(log.RemainLogLevel) // assert.Equal(t, "../../../../logs/", log.StorageLocation) @@ -22,7 +22,7 @@ func TestLoadMongoConfig(t *testing.T) { os.Setenv("IMENV_MONGODB_PASSWORD", "openIM1231231") // os.Setenv("IMENV_MONGODB_URI", "openIM123") // os.Setenv("IMENV_MONGODB_USERNAME", "openIM123") - err := Load("../../../config/", "mongodb.yml", "IMENV_MONGODB", "source", &mongo) + err := Load("../../../config/", "mongodb.yml", "IMENV_MONGODB", &mongo) // err := LoadApiConfig("../../../config/mongodb.yml", "IMENV_MONGODB", &mongo) assert.Nil(t, err) @@ -38,14 +38,14 @@ func TestLoadMongoConfig(t *testing.T) { func TestLoadMinioConfig(t *testing.T) { var storageConfig Minio - err := Load("../../../config/minio.yml", "IMENV_MINIO", "", "source", &storageConfig) + err := Load("../../../config/minio.yml", "IMENV_MINIO", "", &storageConfig) assert.Nil(t, err) assert.Equal(t, "openim", storageConfig.Bucket) } func TestLoadWebhooksConfig(t *testing.T) { var webhooks Webhooks - err := Load("../../../config/webhooks.yml", "IMENV_WEBHOOKS", "", "source", &webhooks) + err := Load("../../../config/webhooks.yml", "IMENV_WEBHOOKS", "", &webhooks) assert.Nil(t, err) assert.Equal(t, 5, webhooks.BeforeAddBlack.Timeout) @@ -53,7 +53,7 @@ func TestLoadWebhooksConfig(t *testing.T) { func TestLoadOpenIMRpcUserConfig(t *testing.T) { var user User - err := Load("../../../config/openim-rpc-user.yml", "IMENV_OPENIM_RPC_USER", "", "source", &user) + err := Load("../../../config/openim-rpc-user.yml", "IMENV_OPENIM_RPC_USER", "", &user) assert.Nil(t, err) //export IMENV_OPENIM_RPC_USER_RPC_LISTENIP="0.0.0.0" assert.Equal(t, "0.0.0.0", user.RPC.ListenIP) @@ -63,14 +63,14 @@ func TestLoadOpenIMRpcUserConfig(t *testing.T) { func TestLoadNotificationConfig(t *testing.T) { var noti Notification - err := Load("../../../config/notification.yml", "IMENV_NOTIFICATION", "", "source", ¬i) + err := Load("../../../config/notification.yml", "IMENV_NOTIFICATION", "", ¬i) assert.Nil(t, err) assert.Equal(t, "Your friend's profile has been changed", noti.FriendRemarkSet.OfflinePush.Title) } func TestLoadOpenIMThirdConfig(t *testing.T) { var third Third - err := Load("../../../config/openim-rpc-third.yml", "IMENV_OPENIM_RPC_THIRD", "", "source", &third) + err := Load("../../../config/openim-rpc-third.yml", "IMENV_OPENIM_RPC_THIRD", "", &third) assert.Nil(t, err) assert.Equal(t, "enabled", third.Object.Enable) assert.Equal(t, "https://oss-cn-chengdu.aliyuncs.com", third.Object.Oss.Endpoint) @@ -86,7 +86,7 @@ func TestLoadOpenIMThirdConfig(t *testing.T) { func TestTransferConfig(t *testing.T) { var tran MsgTransfer - err := Load("../../../config/openim-msgtransfer.yml", "IMENV_OPENIM-MSGTRANSFER", "", "source", &tran) + err := Load("../../../config/openim-msgtransfer.yml", "IMENV_OPENIM-MSGTRANSFER", "", &tran) assert.Nil(t, err) assert.Equal(t, true, tran.Prometheus.Enable) assert.Equal(t, true, tran.Prometheus.AutoSetPorts) diff --git a/pkg/common/storage/controller/conversation.go b/pkg/common/storage/controller/conversation.go index d4088e0c0..7578394b5 100644 --- a/pkg/common/storage/controller/conversation.go +++ b/pkg/common/storage/controller/conversation.go @@ -22,7 +22,6 @@ import ( relationtb "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache" - "github.com/openimsdk/open-im-server/v3/pkg/msgprocessor" "github.com/openimsdk/protocol/constant" "github.com/openimsdk/tools/db/pagination" "github.com/openimsdk/tools/db/tx" @@ -48,7 +47,7 @@ type ConversationDatabase interface { // transactional. SetUsersConversationFieldTx(ctx context.Context, userIDs []string, conversation *relationtb.Conversation, fieldMap map[string]any) error // CreateGroupChatConversation creates a group chat conversation for the specified group ID and user IDs. - CreateGroupChatConversation(ctx context.Context, groupID string, userIDs []string) error + CreateGroupChatConversation(ctx context.Context, groupID string, userIDs []string, conversations *relationtb.Conversation) error // GetConversationIDs retrieves conversation IDs for a given user. GetConversationIDs(ctx context.Context, userID string) ([]string, error) // GetUserConversationIDsHash gets the hash of conversation IDs for a given user. @@ -298,10 +297,10 @@ func (c *conversationDatabase) SetUserConversations(ctx context.Context, ownerUs // return c.cache.GetSuperGroupRecvMsgNotNotifyUserIDs(ctx, groupID) //} -func (c *conversationDatabase) CreateGroupChatConversation(ctx context.Context, groupID string, userIDs []string) error { +func (c *conversationDatabase) CreateGroupChatConversation(ctx context.Context, groupID string, userIDs []string, conversation *relationtb.Conversation) error { return c.tx.Transaction(ctx, func(ctx context.Context) error { cache := c.cache.CloneConversationCache() - conversationID := msgprocessor.GetConversationIDBySessionType(constant.ReadGroupChatType, groupID) + conversationID := conversation.ConversationID existConversationUserIDs, err := c.conversationDB.FindUserID(ctx, userIDs, []string{conversationID}) if err != nil { return err @@ -309,7 +308,15 @@ func (c *conversationDatabase) CreateGroupChatConversation(ctx context.Context, notExistUserIDs := stringutil.DifferenceString(userIDs, existConversationUserIDs) var conversations []*relationtb.Conversation for _, v := range notExistUserIDs { - conversation := relationtb.Conversation{ConversationType: constant.ReadGroupChatType, GroupID: groupID, OwnerUserID: v, ConversationID: conversationID} + conversation := relationtb.Conversation{ + ConversationType: conversation.ConversationType, GroupID: groupID, OwnerUserID: v, ConversationID: conversationID, + // the parameters have default value + RecvMsgOpt: conversation.RecvMsgOpt, IsPinned: conversation.IsPinned, IsPrivateChat: conversation.IsPrivateChat, + BurnDuration: conversation.BurnDuration, GroupAtType: conversation.GroupAtType, AttachedInfo: conversation.AttachedInfo, + Ex: conversation.Ex, MaxSeq: conversation.MaxSeq, MinSeq: conversation.MinSeq, CreateTime: conversation.CreateTime, + MsgDestructTime: conversation.MsgDestructTime, IsMsgDestruct: conversation.IsMsgDestruct, LatestMsgDestructTime: conversation.LatestMsgDestructTime, + } + conversations = append(conversations, &conversation) cache = cache.DelConversations(v, conversationID).DelConversationNotReceiveMessageUserIDs(conversationID) } @@ -320,7 +327,7 @@ func (c *conversationDatabase) CreateGroupChatConversation(ctx context.Context, return err } } - _, err = c.conversationDB.UpdateByMap(ctx, existConversationUserIDs, conversationID, map[string]any{"max_seq": 0}) + _, err = c.conversationDB.UpdateByMap(ctx, existConversationUserIDs, conversationID, map[string]any{"max_seq": conversation.MaxSeq}) if err != nil { return err } From 11044eac586ee2bec8870cc638cb0688631cfdb5 Mon Sep 17 00:00:00 2001 From: OpenIM-Gordon <1432970085@qq.com> Date: Fri, 21 Mar 2025 15:10:31 +0800 Subject: [PATCH 138/199] =?UTF-8?q?feat:=20add=20a=20function=20for=20busi?= =?UTF-8?q?ness=20info=20change=20to=20update=20related=20conve=E2=80=A6?= =?UTF-8?q?=20(#3225)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: add a function for business info change to update related conversation's ex info. * feat: add a function for business info change to update related conversation's ex info. * feat: add a function for business info change to update related conversation's ex info. * feat: add a function for business info change to update related conversation's ex info. --- go.mod | 2 +- go.sum | 4 +- internal/api/conversation.go | 5 + internal/api/msg.go | 8 -- internal/api/router.go | 6 +- internal/rpc/conversation/conversation.go | 16 ++- internal/rpc/msg/notification.go | 4 - internal/rpc/msg/send.go | 8 +- internal/rpc/msg/server.go | 10 +- internal/rpc/msg/stream_msg.go | 115 ------------------ pkg/common/storage/controller/conversation.go | 15 +++ pkg/common/storage/controller/stream_msg.go | 34 ------ pkg/common/storage/database/conversation.go | 1 + .../storage/database/mgo/conversation.go | 57 +++++++-- pkg/common/storage/database/mgo/stream_msg.go | 60 --------- pkg/common/storage/database/stream_msg.go | 13 -- pkg/common/storage/model/stream_msg.go | 21 ---- 17 files changed, 95 insertions(+), 284 deletions(-) delete mode 100644 internal/rpc/msg/stream_msg.go delete mode 100644 pkg/common/storage/controller/stream_msg.go delete mode 100644 pkg/common/storage/database/mgo/stream_msg.go delete mode 100644 pkg/common/storage/database/stream_msg.go delete mode 100644 pkg/common/storage/model/stream_msg.go diff --git a/go.mod b/go.mod index 7bf9e6ef6..d762f9fae 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/gorilla/websocket v1.5.1 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/mitchellh/mapstructure v1.5.0 - github.com/openimsdk/protocol v0.0.72-alpha.79 + github.com/openimsdk/protocol v0.0.72-alpha.81 github.com/openimsdk/tools v0.0.50-alpha.74 github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 diff --git a/go.sum b/go.sum index 2a86d97ea..ea5a6e5b7 100644 --- a/go.sum +++ b/go.sum @@ -347,8 +347,8 @@ github.com/onsi/gomega v1.25.0 h1:Vw7br2PCDYijJHSfBOWhov+8cAnUf8MfMaIOV323l6Y= github.com/onsi/gomega v1.25.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdMPGhM= github.com/openimsdk/gomake v0.0.15-alpha.2 h1:5Q8yl8ezy2yx+q8/ucU/t4kJnDfCzNOrkXcDACCqtyM= github.com/openimsdk/gomake v0.0.15-alpha.2/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= -github.com/openimsdk/protocol v0.0.72-alpha.79 h1:e46no8WVAsmTzyy405klrdoUiG7u+1ohDsXvQuFng4s= -github.com/openimsdk/protocol v0.0.72-alpha.79/go.mod h1:WF7EuE55vQvpyUAzDXcqg+B+446xQyEba0X35lTINmw= +github.com/openimsdk/protocol v0.0.72-alpha.81 h1:6tDuZ3Anfi1uhX/V5mWxITqJnGQPnvgeaxeqJlEHIVE= +github.com/openimsdk/protocol v0.0.72-alpha.81/go.mod h1:WF7EuE55vQvpyUAzDXcqg+B+446xQyEba0X35lTINmw= github.com/openimsdk/tools v0.0.50-alpha.74 h1:yh10SiMiivMEjicEQg+QAsH4pvaO+4noMPdlw+ew0Kc= github.com/openimsdk/tools v0.0.50-alpha.74/go.mod h1:n2poR3asX1e1XZce4O+MOWAp+X02QJRFvhcLCXZdzRo= github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= diff --git a/internal/api/conversation.go b/internal/api/conversation.go index f7dbc133c..f6eadf15a 100644 --- a/internal/api/conversation.go +++ b/internal/api/conversation.go @@ -16,6 +16,7 @@ package api import ( "github.com/gin-gonic/gin" + "github.com/openimsdk/protocol/conversation" "github.com/openimsdk/tools/a2r" ) @@ -71,3 +72,7 @@ func (o *ConversationApi) GetNotNotifyConversationIDs(c *gin.Context) { func (o *ConversationApi) GetPinnedConversationIDs(c *gin.Context) { a2r.Call(c, conversation.ConversationClient.GetPinnedConversationIDs, o.Client) } + +func (o *ConversationApi) UpdateConversationsByUser(c *gin.Context) { + a2r.Call(c, conversation.ConversationClient.UpdateConversationsByUser, o.Client) +} diff --git a/internal/api/msg.go b/internal/api/msg.go index 1ec1f44a7..090f3329b 100644 --- a/internal/api/msg.go +++ b/internal/api/msg.go @@ -551,11 +551,3 @@ func (m *MessageApi) SearchMsg(c *gin.Context) { func (m *MessageApi) GetServerTime(c *gin.Context) { a2r.Call(c, msg.MsgClient.GetServerTime, m.Client) } - -func (m *MessageApi) GetStreamMsg(c *gin.Context) { - a2r.Call(c, msg.MsgClient.GetServerTime, m.Client) -} - -func (m *MessageApi) AppendStreamMsg(c *gin.Context) { - a2r.Call(c, msg.MsgClient.GetServerTime, m.Client) -} diff --git a/internal/api/router.go b/internal/api/router.go index 850c23972..657493b23 100644 --- a/internal/api/router.go +++ b/internal/api/router.go @@ -9,6 +9,8 @@ import ( "github.com/gin-gonic/gin" "github.com/gin-gonic/gin/binding" "github.com/go-playground/validator/v10" + clientv3 "go.etcd.io/etcd/client/v3" + "github.com/openimsdk/open-im-server/v3/internal/api/jssdk" "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/open-im-server/v3/pkg/common/prommetrics" @@ -27,7 +29,6 @@ import ( "github.com/openimsdk/tools/discovery/etcd" "github.com/openimsdk/tools/log" "github.com/openimsdk/tools/mw" - clientv3 "go.etcd.io/etcd/client/v3" ) const ( @@ -246,8 +247,6 @@ func newGinRouter(ctx context.Context, client discovery.Conn, cfg *Config) (*gin msgGroup.POST("/batch_send_msg", m.BatchSendMsg) msgGroup.POST("/check_msg_is_send_success", m.CheckMsgIsSendSuccess) msgGroup.POST("/get_server_time", m.GetServerTime) - msgGroup.POST("/get_stream_msg", m.GetStreamMsg) - msgGroup.POST("/append_stream_msg", m.AppendStreamMsg) } // Conversation { @@ -264,6 +263,7 @@ func newGinRouter(ctx context.Context, client discovery.Conn, cfg *Config) (*gin conversationGroup.POST("/get_owner_conversation", c.GetOwnerConversation) conversationGroup.POST("/get_not_notify_conversation_ids", c.GetNotNotifyConversationIDs) conversationGroup.POST("/get_pinned_conversation_ids", c.GetPinnedConversationIDs) + conversationGroup.POST("/update_conversations_by_user", c.UpdateConversationsByUser) } { diff --git a/internal/rpc/conversation/conversation.go b/internal/rpc/conversation/conversation.go index 8bc3fd2a9..ca2c58878 100644 --- a/internal/rpc/conversation/conversation.go +++ b/internal/rpc/conversation/conversation.go @@ -22,6 +22,8 @@ import ( "github.com/openimsdk/open-im-server/v3/pkg/dbbuild" "github.com/openimsdk/open-im-server/v3/pkg/rpcli" + "google.golang.org/grpc" + "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/open-im-server/v3/pkg/common/convert" "github.com/openimsdk/open-im-server/v3/pkg/common/servererrs" @@ -40,7 +42,6 @@ import ( "github.com/openimsdk/tools/errs" "github.com/openimsdk/tools/log" "github.com/openimsdk/tools/utils/datautil" - "google.golang.org/grpc" ) type conversationServer struct { @@ -329,6 +330,19 @@ func (c *conversationServer) SetConversations(ctx context.Context, req *pbconver return &pbconversation.SetConversationsResp{}, nil } +func (c *conversationServer) UpdateConversationsByUser(ctx context.Context, req *pbconversation.UpdateConversationsByUserReq) (*pbconversation.UpdateConversationsByUserResp, error) { + m := make(map[string]any) + if req.Ex != nil { + m["ex"] = req.Ex.Value + } + if len(m) > 0 { + if err := c.conversationDatabase.UpdateUserConversations(ctx, req.UserID, m); err != nil { + return nil, err + } + } + return &pbconversation.UpdateConversationsByUserResp{}, nil +} + // Get user IDs with "Do Not Disturb" enabled in super large groups. func (c *conversationServer) GetRecvMsgNotNotifyUserIDs(ctx context.Context, req *pbconversation.GetRecvMsgNotNotifyUserIDsReq) (*pbconversation.GetRecvMsgNotNotifyUserIDsResp, error) { return nil, errs.New("deprecated") diff --git a/internal/rpc/msg/notification.go b/internal/rpc/msg/notification.go index 0daafbe6c..0418823d6 100644 --- a/internal/rpc/msg/notification.go +++ b/internal/rpc/msg/notification.go @@ -48,7 +48,3 @@ func (m *MsgNotificationSender) MarkAsReadNotification(ctx context.Context, conv } m.NotificationWithSessionType(ctx, sendID, recvID, constant.HasReadReceipt, sessionType, tips) } - -func (m *MsgNotificationSender) StreamMsgNotification(ctx context.Context, sendID string, recvID string, sessionType int32, tips *sdkws.StreamMsgTips) { - m.NotificationWithSessionType(ctx, sendID, recvID, constant.StreamMsgNotification, sessionType, tips) -} diff --git a/internal/rpc/msg/send.go b/internal/rpc/msg/send.go index f226c4921..6b2ec30b5 100644 --- a/internal/rpc/msg/send.go +++ b/internal/rpc/msg/send.go @@ -17,6 +17,8 @@ package msg import ( "context" + "google.golang.org/protobuf/proto" + "github.com/openimsdk/open-im-server/v3/pkg/common/prommetrics" "github.com/openimsdk/open-im-server/v3/pkg/msgprocessor" "github.com/openimsdk/open-im-server/v3/pkg/util/conversationutil" @@ -29,7 +31,6 @@ import ( "github.com/openimsdk/tools/log" "github.com/openimsdk/tools/mcontext" "github.com/openimsdk/tools/utils/datautil" - "google.golang.org/protobuf/proto" ) func (m *msgServer) SendMsg(ctx context.Context, req *pbmsg.SendMsgReq) (*pbmsg.SendMsgResp, error) { @@ -49,11 +50,6 @@ func (m *msgServer) SendMsg(ctx context.Context, req *pbmsg.SendMsgReq) (*pbmsg. func (m *msgServer) sendMsg(ctx context.Context, req *pbmsg.SendMsgReq, before **sdkws.MsgData) (*pbmsg.SendMsgResp, error) { m.encapsulateMsgData(req.MsgData) - if req.MsgData.ContentType == constant.Stream { - if err := m.handlerStreamMsg(ctx, req.MsgData); err != nil { - return nil, err - } - } switch req.MsgData.SessionType { case constant.SingleChatType: return m.sendMsgSingleChat(ctx, req, before) diff --git a/internal/rpc/msg/server.go b/internal/rpc/msg/server.go index d1002cca3..cfc750c5b 100644 --- a/internal/rpc/msg/server.go +++ b/internal/rpc/msg/server.go @@ -59,9 +59,8 @@ type Config struct { // MsgServer encapsulates dependencies required for message handling. type msgServer struct { msg.UnimplementedMsgServer - RegisterCenter discovery.Conn // Service discovery registry for service registration. - MsgDatabase controller.CommonMsgDatabase // Interface for message database operations. - StreamMsgDatabase controller.StreamMsgDatabase + RegisterCenter discovery.Conn // Service discovery registry for service registration. + MsgDatabase controller.CommonMsgDatabase // Interface for message database operations. UserLocalCache *rpccache.UserLocalCache // Local cache for user data. FriendLocalCache *rpccache.FriendLocalCache // Local cache for friend data. GroupLocalCache *rpccache.GroupLocalCache // Local cache for group data. @@ -117,10 +116,6 @@ func Start(ctx context.Context, config *Config, client discovery.Conn, server gr if err != nil { return err } - streamMsg, err := mgo.NewStreamMsgMongo(mgocli.GetDB()) - if err != nil { - return err - } seqUserCache := redis.NewSeqUserCacheRedis(rdb, seqUser) userConn, err := client.GetConn(ctx, config.Discovery.RpcService.User) if err != nil { @@ -142,7 +137,6 @@ func Start(ctx context.Context, config *Config, client discovery.Conn, server gr msgDatabase := controller.NewCommonMsgDatabase(msgDocModel, msgModel, seqUserCache, seqConversationCache, redisProducer) s := &msgServer{ MsgDatabase: msgDatabase, - StreamMsgDatabase: controller.NewStreamMsgDatabase(streamMsg), RegisterCenter: client, UserLocalCache: rpccache.NewUserLocalCache(rpcli.NewUserClient(userConn), &config.LocalCacheConfig, rdb), GroupLocalCache: rpccache.NewGroupLocalCache(rpcli.NewGroupClient(groupConn), &config.LocalCacheConfig, rdb), diff --git a/internal/rpc/msg/stream_msg.go b/internal/rpc/msg/stream_msg.go deleted file mode 100644 index 688d766c8..000000000 --- a/internal/rpc/msg/stream_msg.go +++ /dev/null @@ -1,115 +0,0 @@ -package msg - -import ( - "context" - "fmt" - "time" - - "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" - "github.com/openimsdk/open-im-server/v3/pkg/msgprocessor" - "github.com/openimsdk/protocol/constant" - "github.com/openimsdk/protocol/msg" - "github.com/openimsdk/protocol/sdkws" - "github.com/openimsdk/tools/errs" -) - -const StreamDeadlineTime = time.Second * 60 * 10 - -func (m *msgServer) handlerStreamMsg(ctx context.Context, msgData *sdkws.MsgData) error { - now := time.Now() - val := &model.StreamMsg{ - ClientMsgID: msgData.ClientMsgID, - ConversationID: msgprocessor.GetConversationIDByMsg(msgData), - UserID: msgData.SendID, - CreateTime: now, - DeadlineTime: now.Add(StreamDeadlineTime), - } - return m.StreamMsgDatabase.CreateStreamMsg(ctx, val) -} - -func (m *msgServer) getStreamMsg(ctx context.Context, clientMsgID string) (*model.StreamMsg, error) { - res, err := m.StreamMsgDatabase.GetStreamMsg(ctx, clientMsgID) - if err != nil { - return nil, err - } - now := time.Now() - if !res.End && res.DeadlineTime.Before(now) { - res.End = true - res.DeadlineTime = now - _ = m.StreamMsgDatabase.AppendStreamMsg(ctx, res.ClientMsgID, 0, nil, true, now) - } - return res, nil -} - -func (m *msgServer) AppendStreamMsg(ctx context.Context, req *msg.AppendStreamMsgReq) (*msg.AppendStreamMsgResp, error) { - res, err := m.getStreamMsg(ctx, req.ClientMsgID) - if err != nil { - return nil, err - } - if res.End { - return nil, errs.ErrNoPermission.WrapMsg("stream msg is end") - } - if len(res.Packets) < int(req.StartIndex) { - return nil, errs.ErrNoPermission.WrapMsg("start index is invalid") - } - if val := len(res.Packets) - int(req.StartIndex); val > 0 { - exist := res.Packets[int(req.StartIndex):] - for i, s := range exist { - if len(req.Packets) == 0 { - break - } - if s != req.Packets[i] { - return nil, errs.ErrNoPermission.WrapMsg(fmt.Sprintf("packet %d has been written and is inconsistent", i)) - } - req.StartIndex++ - req.Packets = req.Packets[1:] - } - } - if len(req.Packets) == 0 && res.End == req.End { - return &msg.AppendStreamMsgResp{}, nil - } - deadlineTime := time.Now().Add(StreamDeadlineTime) - if err := m.StreamMsgDatabase.AppendStreamMsg(ctx, req.ClientMsgID, int(req.StartIndex), req.Packets, req.End, deadlineTime); err != nil { - return nil, err - } - conversation, err := m.conversationClient.GetConversation(ctx, res.ConversationID, res.UserID) - if err != nil { - return nil, err - } - tips := &sdkws.StreamMsgTips{ - ConversationID: res.ConversationID, - ClientMsgID: res.ClientMsgID, - StartIndex: req.StartIndex, - Packets: req.Packets, - End: req.End, - } - var ( - recvID string - sessionType int32 - ) - if conversation.GroupID == "" { - sessionType = constant.SingleChatType - recvID = conversation.UserID - } else { - sessionType = constant.ReadGroupChatType - recvID = conversation.GroupID - } - m.msgNotificationSender.StreamMsgNotification(ctx, res.UserID, recvID, sessionType, tips) - return &msg.AppendStreamMsgResp{}, nil -} - -func (m *msgServer) GetStreamMsg(ctx context.Context, req *msg.GetStreamMsgReq) (*msg.GetStreamMsgResp, error) { - res, err := m.getStreamMsg(ctx, req.ClientMsgID) - if err != nil { - return nil, err - } - return &msg.GetStreamMsgResp{ - ClientMsgID: res.ClientMsgID, - ConversationID: res.ConversationID, - UserID: res.UserID, - Packets: res.Packets, - End: res.End, - CreateTime: res.CreateTime.UnixMilli(), - DeadlineTime: res.DeadlineTime.UnixMilli(), - }, nil -} diff --git a/pkg/common/storage/controller/conversation.go b/pkg/common/storage/controller/conversation.go index 7578394b5..27442ca66 100644 --- a/pkg/common/storage/controller/conversation.go +++ b/pkg/common/storage/controller/conversation.go @@ -46,6 +46,9 @@ type ConversationDatabase interface { // SetUsersConversationFieldTx updates a specific field for multiple users' conversations, creating new conversations if they do not exist, or updates them otherwise. This operation is // transactional. SetUsersConversationFieldTx(ctx context.Context, userIDs []string, conversation *relationtb.Conversation, fieldMap map[string]any) error + // UpdateUserConversations updates all conversations related to a specified user. + // This function does NOT update the user's own conversations but rather the conversations where this user is involved (e.g., other users' conversations referencing this user). + UpdateUserConversations(ctx context.Context, userID string, args map[string]any) error // CreateGroupChatConversation creates a group chat conversation for the specified group ID and user IDs. CreateGroupChatConversation(ctx context.Context, groupID string, userIDs []string, conversations *relationtb.Conversation) error // GetConversationIDs retrieves conversation IDs for a given user. @@ -145,6 +148,18 @@ func (c *conversationDatabase) SetUsersConversationFieldTx(ctx context.Context, }) } +func (c *conversationDatabase) UpdateUserConversations(ctx context.Context, userID string, args map[string]any) error { + conversations, err := c.conversationDB.UpdateUserConversations(ctx, userID, args) + if err != nil { + return err + } + cache := c.cache.CloneConversationCache() + for _, conversation := range conversations { + cache = cache.DelUsersConversation(conversation.ConversationID, conversation.OwnerUserID).DelConversationVersionUserIDs(conversation.OwnerUserID) + } + return cache.ChainExecDel(ctx) +} + func (c *conversationDatabase) UpdateUsersConversationField(ctx context.Context, userIDs []string, conversationID string, args map[string]any) error { _, err := c.conversationDB.UpdateByMap(ctx, userIDs, conversationID, args) if err != nil { diff --git a/pkg/common/storage/controller/stream_msg.go b/pkg/common/storage/controller/stream_msg.go deleted file mode 100644 index 3409ccd93..000000000 --- a/pkg/common/storage/controller/stream_msg.go +++ /dev/null @@ -1,34 +0,0 @@ -package controller - -import ( - "context" - "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" - "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" - "time" -) - -type StreamMsgDatabase interface { - CreateStreamMsg(ctx context.Context, model *model.StreamMsg) error - AppendStreamMsg(ctx context.Context, clientMsgID string, startIndex int, packets []string, end bool, deadlineTime time.Time) error - GetStreamMsg(ctx context.Context, clientMsgID string) (*model.StreamMsg, error) -} - -func NewStreamMsgDatabase(db database.StreamMsg) StreamMsgDatabase { - return &streamMsgDatabase{db: db} -} - -type streamMsgDatabase struct { - db database.StreamMsg -} - -func (m *streamMsgDatabase) CreateStreamMsg(ctx context.Context, model *model.StreamMsg) error { - return m.db.CreateStreamMsg(ctx, model) -} - -func (m *streamMsgDatabase) AppendStreamMsg(ctx context.Context, clientMsgID string, startIndex int, packets []string, end bool, deadlineTime time.Time) error { - return m.db.AppendStreamMsg(ctx, clientMsgID, startIndex, packets, end, deadlineTime) -} - -func (m *streamMsgDatabase) GetStreamMsg(ctx context.Context, clientMsgID string) (*model.StreamMsg, error) { - return m.db.GetStreamMsg(ctx, clientMsgID) -} diff --git a/pkg/common/storage/database/conversation.go b/pkg/common/storage/database/conversation.go index 1fb53cfed..d612dfc2d 100644 --- a/pkg/common/storage/database/conversation.go +++ b/pkg/common/storage/database/conversation.go @@ -24,6 +24,7 @@ import ( type Conversation interface { Create(ctx context.Context, conversations []*model.Conversation) (err error) UpdateByMap(ctx context.Context, userIDs []string, conversationID string, args map[string]any) (rows int64, err error) + UpdateUserConversations(ctx context.Context, userID string, args map[string]any) ([]*model.Conversation, error) Update(ctx context.Context, conversation *model.Conversation) (err error) Find(ctx context.Context, ownerUserID string, conversationIDs []string) (conversations []*model.Conversation, err error) FindUserID(ctx context.Context, userIDs []string, conversationIDs []string) ([]string, error) diff --git a/pkg/common/storage/database/mgo/conversation.go b/pkg/common/storage/database/mgo/conversation.go index 536827450..89f13ea3d 100644 --- a/pkg/common/storage/database/mgo/conversation.go +++ b/pkg/common/storage/database/mgo/conversation.go @@ -21,23 +21,32 @@ import ( "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" + "github.com/openimsdk/protocol/constant" "github.com/openimsdk/tools/db/mongoutil" "github.com/openimsdk/tools/db/pagination" "github.com/openimsdk/tools/errs" - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/mongo" - "go.mongodb.org/mongo-driver/mongo/options" ) func NewConversationMongo(db *mongo.Database) (*ConversationMgo, error) { coll := db.Collection(database.ConversationName) - _, err := coll.Indexes().CreateOne(context.Background(), mongo.IndexModel{ - Keys: bson.D{ - {Key: "owner_user_id", Value: 1}, - {Key: "conversation_id", Value: 1}, + _, err := coll.Indexes().CreateMany(context.Background(), []mongo.IndexModel{ + { + Keys: bson.D{ + {Key: "owner_user_id", Value: 1}, + {Key: "conversation_id", Value: 1}, + }, + Options: options.Index().SetUnique(true), + }, + { + Keys: bson.D{ + {Key: "user_id", Value: 1}, + }, + Options: options.Index(), }, - Options: options.Index().SetUnique(true), }) if err != nil { return nil, errs.Wrap(err) @@ -101,6 +110,38 @@ func (c *ConversationMgo) UpdateByMap(ctx context.Context, userIDs []string, con return rows, nil } +func (c *ConversationMgo) UpdateUserConversations(ctx context.Context, userID string, args map[string]any) ([]*model.Conversation, error) { + if len(args) == 0 { + return nil, nil + } + filter := bson.M{ + "user_id": userID, + } + + conversations, err := mongoutil.Find[*model.Conversation](ctx, c.coll, filter, options.Find().SetProjection(bson.M{"_id": 0, "owner_user_id": 1, "conversation_id": 1})) + if err != nil { + return nil, err + } + err = mongoutil.IncrVersion(func() error { + _, err := mongoutil.UpdateMany(ctx, c.coll, filter, bson.M{"$set": args}) + if err != nil { + return err + } + return nil + }, func() error { + for _, conversation := range conversations { + if err := c.version.IncrVersion(ctx, conversation.OwnerUserID, []string{conversation.ConversationID}, model.VersionStateUpdate); err != nil { + return err + } + } + return nil + }) + if err != nil { + return nil, err + } + return conversations, nil +} + func (c *ConversationMgo) Update(ctx context.Context, conversation *model.Conversation) (err error) { return mongoutil.IncrVersion(func() error { return mongoutil.UpdateOne(ctx, c.coll, bson.M{"owner_user_id": conversation.OwnerUserID, "conversation_id": conversation.ConversationID}, bson.M{"$set": conversation}, true) diff --git a/pkg/common/storage/database/mgo/stream_msg.go b/pkg/common/storage/database/mgo/stream_msg.go deleted file mode 100644 index c57798daa..000000000 --- a/pkg/common/storage/database/mgo/stream_msg.go +++ /dev/null @@ -1,60 +0,0 @@ -package mgo - -import ( - "context" - "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" - "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" - "github.com/openimsdk/tools/db/mongoutil" - "github.com/openimsdk/tools/errs" - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/mongo" - "go.mongodb.org/mongo-driver/mongo/options" - "time" -) - -func NewStreamMsgMongo(db *mongo.Database) (*StreamMsgMongo, error) { - coll := db.Collection(database.StreamMsgName) - _, err := coll.Indexes().CreateOne(context.Background(), mongo.IndexModel{ - Keys: bson.D{ - {Key: "client_msg_id", Value: 1}, - }, - Options: options.Index().SetUnique(true), - }) - if err != nil { - return nil, errs.Wrap(err) - } - return &StreamMsgMongo{coll: coll}, nil -} - -type StreamMsgMongo struct { - coll *mongo.Collection -} - -func (m *StreamMsgMongo) CreateStreamMsg(ctx context.Context, val *model.StreamMsg) error { - if val.Packets == nil { - val.Packets = []string{} - } - return mongoutil.InsertMany(ctx, m.coll, []*model.StreamMsg{val}) -} - -func (m *StreamMsgMongo) AppendStreamMsg(ctx context.Context, clientMsgID string, startIndex int, packets []string, end bool, deadlineTime time.Time) error { - update := bson.M{ - "$set": bson.M{ - "end": end, - "deadline_time": deadlineTime, - }, - } - if len(packets) > 0 { - update["$push"] = bson.M{ - "packets": bson.M{ - "$each": packets, - "$position": startIndex, - }, - } - } - return mongoutil.UpdateOne(ctx, m.coll, bson.M{"client_msg_id": clientMsgID, "end": false}, update, true) -} - -func (m *StreamMsgMongo) GetStreamMsg(ctx context.Context, clientMsgID string) (*model.StreamMsg, error) { - return mongoutil.FindOne[*model.StreamMsg](ctx, m.coll, bson.M{"client_msg_id": clientMsgID}) -} diff --git a/pkg/common/storage/database/stream_msg.go b/pkg/common/storage/database/stream_msg.go deleted file mode 100644 index e83fffbaa..000000000 --- a/pkg/common/storage/database/stream_msg.go +++ /dev/null @@ -1,13 +0,0 @@ -package database - -import ( - "context" - "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" - "time" -) - -type StreamMsg interface { - CreateStreamMsg(ctx context.Context, model *model.StreamMsg) error - AppendStreamMsg(ctx context.Context, clientMsgID string, startIndex int, packets []string, end bool, deadlineTime time.Time) error - GetStreamMsg(ctx context.Context, clientMsgID string) (*model.StreamMsg, error) -} diff --git a/pkg/common/storage/model/stream_msg.go b/pkg/common/storage/model/stream_msg.go deleted file mode 100644 index c040426a4..000000000 --- a/pkg/common/storage/model/stream_msg.go +++ /dev/null @@ -1,21 +0,0 @@ -package model - -import ( - "time" -) - -const ( - StreamMsgStatusWait = 0 - StreamMsgStatusDone = 1 - StreamMsgStatusFail = 2 -) - -type StreamMsg struct { - ClientMsgID string `bson:"client_msg_id"` - ConversationID string `bson:"conversation_id"` - UserID string `bson:"user_id"` - Packets []string `bson:"packets"` - End bool `bson:"end"` - CreateTime time.Time `bson:"create_time"` - DeadlineTime time.Time `bson:"deadline_time"` -} From 73934fd9553fd5848a0409766c3306b3e729a93b Mon Sep 17 00:00:00 2001 From: OpenIM-Gordon <1432970085@qq.com> Date: Fri, 21 Mar 2025 15:19:21 +0800 Subject: [PATCH 139/199] feat: add filtering for invalid messages and invalid conversations to prevent data-fetching exceptions after conversations are deleted. (#3239) --- internal/api/jssdk/jssdk.go | 80 +++++++++++++++++++++++++++---------- pkg/rpcli/msg.go | 4 +- 2 files changed, 62 insertions(+), 22 deletions(-) diff --git a/internal/api/jssdk/jssdk.go b/internal/api/jssdk/jssdk.go index 3c0911207..0d30b1ea0 100644 --- a/internal/api/jssdk/jssdk.go +++ b/internal/api/jssdk/jssdk.go @@ -2,10 +2,14 @@ package jssdk import ( "context" - "github.com/openimsdk/open-im-server/v3/pkg/rpcli" "sort" + "github.com/openimsdk/open-im-server/v3/pkg/rpcli" + "github.com/openimsdk/protocol/constant" + "github.com/openimsdk/tools/log" + "github.com/gin-gonic/gin" + "github.com/openimsdk/protocol/conversation" "github.com/openimsdk/protocol/jssdk" "github.com/openimsdk/protocol/msg" @@ -109,10 +113,7 @@ func (x *JSSdk) getActiveConversations(ctx context.Context, req *jssdk.GetActive if len(conversationIDs) == 0 { return &jssdk.GetActiveConversationsResp{}, nil } - readSeq, err := x.msgClient.GetHasReadSeqs(ctx, conversationIDs, req.OwnerUserID) - if err != nil { - return nil, err - } + activeConversation, err := x.msgClient.GetActiveConversation(ctx, conversationIDs) if err != nil { return nil, err @@ -120,6 +121,10 @@ func (x *JSSdk) getActiveConversations(ctx context.Context, req *jssdk.GetActive if len(activeConversation) == 0 { return &jssdk.GetActiveConversationsResp{}, nil } + readSeq, err := x.msgClient.GetHasReadSeqs(ctx, conversationIDs, req.OwnerUserID) + if err != nil { + return nil, err + } sortConversations := sortActiveConversations{ Conversation: activeConversation, } @@ -147,6 +152,7 @@ func (x *JSSdk) getActiveConversations(ctx context.Context, req *jssdk.GetActive if err != nil { return nil, err } + x.checkMessagesAndGetLastMessage(ctx, req.OwnerUserID, msgs) conversationMap := datautil.SliceToMap(conversations, func(c *conversation.Conversation) string { return c.ConversationID }) @@ -156,16 +162,15 @@ func (x *JSSdk) getActiveConversations(ctx context.Context, req *jssdk.GetActive if !ok { continue } - var lastMsg *sdkws.MsgData if msgList, ok := msgs[c.ConversationID]; ok && len(msgList.Msgs) > 0 { - lastMsg = msgList.Msgs[0] + resp = append(resp, &jssdk.ConversationMsg{ + Conversation: conv, + LastMsg: msgList.Msgs[0], + MaxSeq: c.MaxSeq, + ReadSeq: readSeq[c.ConversationID], + }) } - resp = append(resp, &jssdk.ConversationMsg{ - Conversation: conv, - LastMsg: lastMsg, - MaxSeq: c.MaxSeq, - ReadSeq: readSeq[c.ConversationID], - }) + } if err := x.fillConversations(ctx, resp); err != nil { return nil, err @@ -219,18 +224,18 @@ func (x *JSSdk) getConversations(ctx context.Context, req *jssdk.GetConversation return nil, err } } + x.checkMessagesAndGetLastMessage(ctx, req.OwnerUserID, msgs) resp := make([]*jssdk.ConversationMsg, 0, len(conversations)) for _, c := range conversations { - var lastMsg *sdkws.MsgData if msgList, ok := msgs[c.ConversationID]; ok && len(msgList.Msgs) > 0 { - lastMsg = msgList.Msgs[0] + resp = append(resp, &jssdk.ConversationMsg{ + Conversation: c, + LastMsg: msgList.Msgs[0], + MaxSeq: maxSeqs[c.ConversationID], + ReadSeq: readSeqs[c.ConversationID], + }) } - resp = append(resp, &jssdk.ConversationMsg{ - Conversation: c, - LastMsg: lastMsg, - MaxSeq: maxSeqs[c.ConversationID], - ReadSeq: readSeqs[c.ConversationID], - }) + } if err := x.fillConversations(ctx, resp); err != nil { return nil, err @@ -247,3 +252,36 @@ func (x *JSSdk) getConversations(ctx context.Context, req *jssdk.GetConversation UnreadCount: unreadCount, }, nil } + +// This function checks whether the latest MaxSeq message is valid. +// If not, it needs to fetch a valid message again. +func (x *JSSdk) checkMessagesAndGetLastMessage(ctx context.Context, userID string, messages map[string]*sdkws.PullMsgs) { + var conversationIDs []string + + for conversationID, message := range messages { + allInValid := true + for _, data := range message.Msgs { + if data.Status < constant.MsgStatusHasDeleted { + allInValid = false + break + } + } + if allInValid { + conversationIDs = append(conversationIDs, conversationID) + } + } + if len(conversationIDs) > 0 { + resp, err := x.msgClient.GetLastMessage(ctx, &msg.GetLastMessageReq{ + UserID: userID, + ConversationIDs: conversationIDs, + }) + if err != nil { + log.ZError(ctx, "fetchLatestValidMessages", err, "conversationIDs", conversationIDs) + return + } + for conversationID, message := range resp.Msgs { + messages[conversationID] = &sdkws.PullMsgs{Msgs: []*sdkws.MsgData{message}} + } + } + +} diff --git a/pkg/rpcli/msg.go b/pkg/rpcli/msg.go index 0c44b7c8b..e4d1ece6e 100644 --- a/pkg/rpcli/msg.go +++ b/pkg/rpcli/msg.go @@ -2,9 +2,11 @@ package rpcli import ( "context" + + "google.golang.org/grpc" + "github.com/openimsdk/protocol/msg" "github.com/openimsdk/protocol/sdkws" - "google.golang.org/grpc" ) func NewMsgClient(cc grpc.ClientConnInterface) *MsgClient { From 4dc9b4586188302d0a93edf3d1f9f8f4f8380db4 Mon Sep 17 00:00:00 2001 From: Monet Lee Date: Fri, 28 Mar 2025 15:46:42 +0800 Subject: [PATCH 140/199] feat: implement stress-test tools. (#3261) * feat: implement stress-test tools. * revert config file. --- config/share.yml | 4 +- tools/stress-test/README.md | 25 ++ tools/stress-test/main.go | 452 ++++++++++++++++++++++++++++++++++++ 3 files changed, 479 insertions(+), 2 deletions(-) create mode 100644 tools/stress-test/README.md create mode 100755 tools/stress-test/main.go diff --git a/config/share.yml b/config/share.yml index a5fbeac75..0913c1e88 100644 --- a/config/share.yml +++ b/config/share.yml @@ -1,9 +1,9 @@ secret: openIM123 -imAdminUserID: [ imAdmin ] +imAdminUserID: [imAdmin] # 1: For Android, iOS, Windows, Mac, and web platforms, only one instance can be online at a time multiLogin: policy: 1 # max num of tokens in one end - maxNumOneEnd: 30 \ No newline at end of file + maxNumOneEnd: 30 diff --git a/tools/stress-test/README.md b/tools/stress-test/README.md new file mode 100644 index 000000000..531233a20 --- /dev/null +++ b/tools/stress-test/README.md @@ -0,0 +1,25 @@ +# Stress Test + +## Usage + +You need set `TestTargetUserList` and `DefaultGroupID` variables. + +### Build + +```bash +go build -o _output/bin/tools/linux/amd64/stress-test tools/stress-test/main.go + +# or + +go build -o tools/stress-test/stress-test tools/stress-test/main.go +``` + +### Excute + +```bash +_output/bin/tools/linux/amd64/stress-test -c config/ + +#or + +tools/stress-test/stress-test -c config/ +``` diff --git a/tools/stress-test/main.go b/tools/stress-test/main.go new file mode 100755 index 000000000..ee58c9749 --- /dev/null +++ b/tools/stress-test/main.go @@ -0,0 +1,452 @@ +package main + +import ( + "bytes" + "context" + "encoding/json" + "flag" + "fmt" + "io" + "net/http" + "os" + "os/signal" + "sync" + "syscall" + "time" + + "github.com/openimsdk/open-im-server/v3/pkg/common/config" + "github.com/openimsdk/protocol/auth" + "github.com/openimsdk/protocol/constant" + "github.com/openimsdk/protocol/group" + "github.com/openimsdk/protocol/relation" + "github.com/openimsdk/protocol/sdkws" + pbuser "github.com/openimsdk/protocol/user" + "github.com/openimsdk/tools/log" + "github.com/openimsdk/tools/system/program" +) + +/* + 1. Create one user every minute + 2. Import target users as friends + 3. Add users to the default group + 4. Send a message to the default group every second, containing index and current timestamp + 5. Create a new group every minute and invite target users to join +*/ + +// !!! ATTENTION: This variable is must be added! +var ( + // Use default userIDs List for testing, need to be created. + TestTargetUserList = []string{ + "", + } + DefaultGroupID = "" // Use default group ID for testing, need to be created. +) + +var ( + ApiAddress string + + // API method + GetAdminToken = "/auth/get_admin_token" + CreateUser = "/user/user_register" + ImportFriend = "/friend/import_friend" + InviteToGroup = "/group/invite_user_to_group" + SendMsg = "/msg/send_msg" + CreateGroup = "/group/create_group" + GetUserToken = "/auth/user_token" +) + +const ( + MaxUser = 10000 + MaxGroup = 1000 + + CreateUserTicker = 1 * time.Minute // Ticker is 1min in create user + SendMessageTicker = 1 * time.Second // Ticker is 1s in send message + CreateGroupTicker = 1 * time.Minute +) + +type BaseResp struct { + ErrCode int `json:"errCode"` + ErrMsg string `json:"errMsg"` + Data json.RawMessage `json:"data"` +} + +type StressTest struct { + Conf *conf + AdminUserID string + AdminToken string + DefaultGroupID string + DefaultSendUserID string + UserCounter int + GroupCounter int + MsgCounter int + CreatedUsers []string + CreatedGroups []string + Mutex sync.Mutex + Ctx context.Context + Cancel context.CancelFunc + HttpClient *http.Client + Wg sync.WaitGroup + Once sync.Once +} + +type conf struct { + Share config.Share + Api config.API +} + +func initConfig(configDir string) (*config.Share, *config.API, error) { + var ( + share = &config.Share{} + apiConfig = &config.API{} + ) + + err := config.Load(configDir, config.ShareFileName, config.EnvPrefixMap[config.ShareFileName], share) + if err != nil { + return nil, nil, err + } + + err = config.Load(configDir, config.OpenIMAPICfgFileName, config.EnvPrefixMap[config.OpenIMAPICfgFileName], apiConfig) + if err != nil { + return nil, nil, err + } + + return share, apiConfig, nil +} + +// Post Request +func (st *StressTest) PostRequest(ctx context.Context, url string, reqbody any) ([]byte, error) { + // Marshal body + jsonBody, err := json.Marshal(reqbody) + if err != nil { + log.ZError(ctx, "Failed to marshal request body", err, "url", url, "reqbody", reqbody) + return nil, err + } + + req, err := http.NewRequest(http.MethodPost, url, bytes.NewReader(jsonBody)) + if err != nil { + return nil, err + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("operationID", st.AdminUserID) + if st.AdminToken != "" { + req.Header.Set("token", st.AdminToken) + } + + // log.ZInfo(ctx, "Header info is ", "Content-Type", "application/json", "operationID", st.AdminUserID, "token", st.AdminToken) + + resp, err := st.HttpClient.Do(req) + if err != nil { + log.ZError(ctx, "Failed to send request", err, "url", url, "reqbody", reqbody) + return nil, err + } + defer resp.Body.Close() + + respBody, err := io.ReadAll(resp.Body) + if err != nil { + log.ZError(ctx, "Failed to read response body", err, "url", url) + return nil, err + } + + var baseResp BaseResp + if err := json.Unmarshal(respBody, &baseResp); err != nil { + log.ZError(ctx, "Failed to unmarshal response body", err, "url", url, "respBody", string(respBody)) + return nil, err + } + + if baseResp.ErrCode != 0 { + err = fmt.Errorf(baseResp.ErrMsg) + log.ZError(ctx, "Failed to send request", err, "url", url, "reqbody", reqbody, "resp", baseResp) + return nil, err + } + + return baseResp.Data, nil +} + +func (st *StressTest) GetAdminToken(ctx context.Context) (string, error) { + req := auth.GetAdminTokenReq{ + Secret: st.Conf.Share.Secret, + UserID: st.AdminUserID, + } + + resp, err := st.PostRequest(ctx, ApiAddress+GetAdminToken, &req) + if err != nil { + return "", err + } + + data := &auth.GetAdminTokenResp{} + if err := json.Unmarshal(resp, &data); err != nil { + return "", err + } + + return data.Token, nil +} + +func (st *StressTest) CreateUser(ctx context.Context, userID string) (string, error) { + user := &sdkws.UserInfo{ + UserID: userID, + Nickname: userID, + } + + req := pbuser.UserRegisterReq{ + Users: []*sdkws.UserInfo{user}, + } + + _, err := st.PostRequest(ctx, ApiAddress+CreateUser, &req) + if err != nil { + return "", err + } + + st.UserCounter++ + return userID, nil +} + +func (st *StressTest) ImportFriend(ctx context.Context, userID string) error { + req := relation.ImportFriendReq{ + OwnerUserID: userID, + FriendUserIDs: TestTargetUserList, + } + + _, err := st.PostRequest(ctx, ApiAddress+ImportFriend, &req) + if err != nil { + return err + } + + return nil +} + +func (st *StressTest) InviteToGroup(ctx context.Context, userID string) error { + req := group.InviteUserToGroupReq{ + GroupID: st.DefaultGroupID, + InvitedUserIDs: []string{userID}, + } + _, err := st.PostRequest(ctx, ApiAddress+InviteToGroup, &req) + if err != nil { + return err + } + + return nil +} + +func (st *StressTest) SendMsg(ctx context.Context, userID string) error { + contentObj := map[string]any{ + "content": fmt.Sprintf("index %d. The current time is %s", st.MsgCounter, time.Now().Format("2006-01-02 15:04:05.000")), + } + + req := map[string]any{ + "sendID": userID, + "groupID": st.DefaultGroupID, + "contentType": constant.Text, + "sessionType": constant.ReadGroupChatType, + "content": contentObj, + } + + _, err := st.PostRequest(ctx, ApiAddress+SendMsg, &req) + if err != nil { + log.ZError(ctx, "Failed to send message", err, "userID", userID, "req", &req) + return err + } + + st.MsgCounter++ + + return nil +} + +func (st *StressTest) CreateGroup(ctx context.Context, userID string) (string, error) { + groupID := fmt.Sprintf("StressTestGroup_%d_%s", st.GroupCounter, time.Now().Format("20060102150405")) + + req := map[string]any{ + "memberUserIDs": TestTargetUserList, + "ownerUserID": userID, + "groupInfo": map[string]any{ + "groupID": groupID, + "groupName": groupID, + "groupType": constant.WorkingGroup, + }, + } + resp := group.CreateGroupResp{} + + response, err := st.PostRequest(ctx, ApiAddress+CreateGroup, &req) + if err != nil { + return "", err + } + + if err := json.Unmarshal(response, &resp); err != nil { + return "", err + } + + st.GroupCounter++ + + return resp.GroupInfo.GroupID, nil +} + +func main() { + var configPath string + // defaultConfigDir := filepath.Join("..", "..", "..", "..", "..", "config") + // flag.StringVar(&configPath, "c", defaultConfigDir, "config path") + flag.StringVar(&configPath, "c", "", "config path") + flag.Parse() + + if configPath == "" { + _, _ = fmt.Fprintln(os.Stderr, "config path is empty") + os.Exit(1) + return + } + + fmt.Printf(" Config Path: %s\n", configPath) + + share, apiConfig, err := initConfig(configPath) + if err != nil { + program.ExitWithError(err) + return + } + + ApiAddress = fmt.Sprintf("http://%s:%s", "127.0.0.1", fmt.Sprint(apiConfig.Api.Ports[0])) + + ctx, cancel := context.WithCancel(context.Background()) + ch := make(chan struct{}) + + defer cancel() + + st := &StressTest{ + Conf: &conf{ + Share: *share, + Api: *apiConfig, + }, + AdminUserID: share.IMAdminUserID[0], + Ctx: ctx, + Cancel: cancel, + HttpClient: &http.Client{ + Timeout: 50 * time.Second, + }, + } + + c := make(chan os.Signal, 1) + signal.Notify(c, os.Interrupt, syscall.SIGTERM) + go func() { + <-c + fmt.Println("\nReceived stop signal, stopping...") + + select { + case <-ch: + default: + close(ch) + } + + st.Cancel() + }() + + token, err := st.GetAdminToken(st.Ctx) + if err != nil { + log.ZError(ctx, "Get Admin Token failed.", err, "AdminUserID", st.AdminUserID) + } + + st.AdminToken = token + fmt.Println("Admin Token:", st.AdminToken) + fmt.Println("ApiAddress:", ApiAddress) + + st.DefaultGroupID = DefaultGroupID + + st.Wg.Add(1) + go func() { + defer st.Wg.Done() + + ticker := time.NewTicker(CreateUserTicker) + defer ticker.Stop() + + for st.UserCounter < MaxUser { + select { + case <-st.Ctx.Done(): + log.ZInfo(st.Ctx, "Stop Create user", "reason", "context done") + return + + case <-ticker.C: + // Create User + userID := fmt.Sprintf("%d_Stresstest_%s", st.UserCounter, time.Now().Format("0102150405")) + + userCreatedID, err := st.CreateUser(st.Ctx, userID) + if err != nil { + log.ZError(st.Ctx, "Create User failed.", err, "UserID", userID) + os.Exit(1) + return + } + // fmt.Println("User Created ID:", userCreatedID) + + // Import Friend + if err = st.ImportFriend(st.Ctx, userCreatedID); err != nil { + log.ZError(st.Ctx, "Import Friend failed.", err, "UserID", userCreatedID) + os.Exit(1) + return + } + + // Invite To Group + if err = st.InviteToGroup(st.Ctx, userCreatedID); err != nil { + log.ZError(st.Ctx, "Invite To Group failed.", err, "UserID", userCreatedID) + os.Exit(1) + return + } + + st.Once.Do(func() { + st.DefaultSendUserID = userCreatedID + fmt.Println("Default Send User Created ID:", userCreatedID) + close(ch) + }) + } + } + }() + + st.Wg.Add(1) + go func() { + defer st.Wg.Done() + + ticker := time.NewTicker(SendMessageTicker) + defer ticker.Stop() + <-ch + + for { + select { + case <-st.Ctx.Done(): + log.ZInfo(st.Ctx, "Stop Send message", "reason", "context done") + return + + case <-ticker.C: + // Send Message + if err = st.SendMsg(st.Ctx, st.DefaultSendUserID); err != nil { + log.ZError(st.Ctx, "Send Message failed.", err, "UserID", st.DefaultSendUserID) + continue + } + } + } + }() + + st.Wg.Add(1) + go func() { + defer st.Wg.Done() + + ticker := time.NewTicker(CreateGroupTicker) + defer ticker.Stop() + <-ch + + for st.GroupCounter < MaxGroup { + + select { + case <-st.Ctx.Done(): + log.ZInfo(st.Ctx, "Stop Create Group", "reason", "context done") + return + + case <-ticker.C: + + // Create Group + _, err := st.CreateGroup(st.Ctx, st.DefaultSendUserID) + if err != nil { + log.ZError(st.Ctx, "Create Group failed.", err, "UserID", st.DefaultSendUserID) + os.Exit(1) + return + } + + // fmt.Println("Group Created ID:", groupID) + } + } + }() + + st.Wg.Wait() +} From 3b710fdfdbcc825dabd29afea637df9f484299ae Mon Sep 17 00:00:00 2001 From: Monet Lee Date: Mon, 31 Mar 2025 16:45:52 +0800 Subject: [PATCH 141/199] fix: improve stress test tools parms. (#3265) * feat: implement stress-test tools. * revert config file. * fix: improve tools parms. * fix modify args. --- tools/stress-test/main.go | 35 +++++++++++++++++++++-------------- 1 file changed, 21 insertions(+), 14 deletions(-) diff --git a/tools/stress-test/main.go b/tools/stress-test/main.go index ee58c9749..aa52b69ed 100755 --- a/tools/stress-test/main.go +++ b/tools/stress-test/main.go @@ -14,6 +14,7 @@ import ( "syscall" "time" + "github.com/openimsdk/open-im-server/v3/pkg/apistruct" "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/protocol/auth" "github.com/openimsdk/protocol/constant" @@ -232,12 +233,15 @@ func (st *StressTest) SendMsg(ctx context.Context, userID string) error { "content": fmt.Sprintf("index %d. The current time is %s", st.MsgCounter, time.Now().Format("2006-01-02 15:04:05.000")), } - req := map[string]any{ - "sendID": userID, - "groupID": st.DefaultGroupID, - "contentType": constant.Text, - "sessionType": constant.ReadGroupChatType, - "content": contentObj, + req := &apistruct.SendMsgReq{ + SendMsg: apistruct.SendMsg{ + SendID: userID, + SenderNickname: userID, + GroupID: st.DefaultGroupID, + ContentType: constant.Text, + SessionType: constant.ReadGroupChatType, + Content: contentObj, + }, } _, err := st.PostRequest(ctx, ApiAddress+SendMsg, &req) @@ -254,15 +258,18 @@ func (st *StressTest) SendMsg(ctx context.Context, userID string) error { func (st *StressTest) CreateGroup(ctx context.Context, userID string) (string, error) { groupID := fmt.Sprintf("StressTestGroup_%d_%s", st.GroupCounter, time.Now().Format("20060102150405")) - req := map[string]any{ - "memberUserIDs": TestTargetUserList, - "ownerUserID": userID, - "groupInfo": map[string]any{ - "groupID": groupID, - "groupName": groupID, - "groupType": constant.WorkingGroup, - }, + groupInfo := &sdkws.GroupInfo{ + GroupID: groupID, + GroupName: groupID, + GroupType: constant.WorkingGroup, + } + + req := group.CreateGroupReq{ + OwnerUserID: userID, + MemberUserIDs: TestTargetUserList, + GroupInfo: groupInfo, } + resp := group.CreateGroupResp{} response, err := st.PostRequest(ctx, ApiAddress+CreateGroup, &req) From aca0eac955555cfd9103ee9d7612ec0a621ea225 Mon Sep 17 00:00:00 2001 From: chao <48119764+withchao@users.noreply.github.com> Date: Tue, 1 Apr 2025 17:13:32 +0800 Subject: [PATCH 142/199] fix: oss specifies content-type when uploading (#3267) * pb * fix: Modifying other fields while setting IsPrivateChat does not take effect * fix: quote message error revoke * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * upgrading pkg tools * fix * fix * optimize log output * feat: support GetLastMessage * feat: support GetLastMessage * feat: s3 switch * feat: s3 switch * fix: GetUsersOnline * feat: SendBusinessNotification supported configuration parameters * feat: SendBusinessNotification supported configuration parameters * feat: SendBusinessNotification supported configuration parameters * feat: seq conversion failed without exiting * fix: DeleteDoc crash * fix: fill send time * fix: fill send time * fix: crash caused by withdrawing messages from users who have left the group * fix: user msg timestamp * seq read config * seq read config * fix: the source message of the reference is withdrawn, and the referenced message is deleted * feat: optimize the default notification.yml * fix: shouldPushOffline * fix: the sorting is wrong after canceling the administrator in group settings * feat: Sending messages supports returning fields modified by webhook * feat: Sending messages supports returning fields modified by webhook * feat: Sending messages supports returning fields modified by webhook * fix: oss specifies content-type when uploading * fix: the version number contains a line break * fix: the version number contains a line break --- go.mod | 2 +- go.sum | 4 ++-- internal/api/init.go | 2 +- internal/rpc/third/s3.go | 2 +- pkg/common/storage/controller/s3.go | 6 +++--- tools/s3/internal/conversion.go | 13 +++++++------ version/version.go | 10 +++++++++- 7 files changed, 24 insertions(+), 15 deletions(-) diff --git a/go.mod b/go.mod index d762f9fae..782a306f4 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/mitchellh/mapstructure v1.5.0 github.com/openimsdk/protocol v0.0.72-alpha.81 - github.com/openimsdk/tools v0.0.50-alpha.74 + github.com/openimsdk/tools v0.0.50-alpha.79 github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 github.com/stretchr/testify v1.9.0 diff --git a/go.sum b/go.sum index ea5a6e5b7..aa0dfa6ac 100644 --- a/go.sum +++ b/go.sum @@ -349,8 +349,8 @@ github.com/openimsdk/gomake v0.0.15-alpha.2 h1:5Q8yl8ezy2yx+q8/ucU/t4kJnDfCzNOrk github.com/openimsdk/gomake v0.0.15-alpha.2/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= github.com/openimsdk/protocol v0.0.72-alpha.81 h1:6tDuZ3Anfi1uhX/V5mWxITqJnGQPnvgeaxeqJlEHIVE= github.com/openimsdk/protocol v0.0.72-alpha.81/go.mod h1:WF7EuE55vQvpyUAzDXcqg+B+446xQyEba0X35lTINmw= -github.com/openimsdk/tools v0.0.50-alpha.74 h1:yh10SiMiivMEjicEQg+QAsH4pvaO+4noMPdlw+ew0Kc= -github.com/openimsdk/tools v0.0.50-alpha.74/go.mod h1:n2poR3asX1e1XZce4O+MOWAp+X02QJRFvhcLCXZdzRo= +github.com/openimsdk/tools v0.0.50-alpha.79 h1:jxYEbrzaze4Z2r4NrKad816buZ690ix0L9MTOOOH3ik= +github.com/openimsdk/tools v0.0.50-alpha.79/go.mod h1:n2poR3asX1e1XZce4O+MOWAp+X02QJRFvhcLCXZdzRo= github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= diff --git a/internal/api/init.go b/internal/api/init.go index 4a1404ffc..378f03eda 100644 --- a/internal/api/init.go +++ b/internal/api/init.go @@ -90,7 +90,7 @@ func Start(ctx context.Context, config *Config, client discovery.Conn, service g //case <-ctx.Done(): //} <-apiCtx.Done() - exitCause := context.Cause(ctx) + exitCause := context.Cause(apiCtx) log.ZWarn(ctx, "api server exit", exitCause) timer := time.NewTimer(time.Second * 15) defer timer.Stop() diff --git a/internal/rpc/third/s3.go b/internal/rpc/third/s3.go index 97206dd6d..757320dac 100644 --- a/internal/rpc/third/s3.go +++ b/internal/rpc/third/s3.go @@ -62,7 +62,7 @@ func (t *thirdServer) InitiateMultipartUpload(ctx context.Context, req *third.In return nil, err } expireTime := time.Now().Add(t.defaultExpire) - result, err := t.s3dataBase.InitiateMultipartUpload(ctx, req.Hash, req.Size, t.defaultExpire, int(req.MaxParts)) + result, err := t.s3dataBase.InitiateMultipartUpload(ctx, req.Hash, req.Size, t.defaultExpire, int(req.MaxParts), req.ContentType) if err != nil { if haErr, ok := errs.Unwrap(err).(*cont.HashAlreadyExistsError); ok { obj := &model.Object{ diff --git a/pkg/common/storage/controller/s3.go b/pkg/common/storage/controller/s3.go index 30d8d20ec..9ab31c5a6 100644 --- a/pkg/common/storage/controller/s3.go +++ b/pkg/common/storage/controller/s3.go @@ -33,7 +33,7 @@ type S3Database interface { PartLimit() (*s3.PartLimit, error) PartSize(ctx context.Context, size int64) (int64, error) AuthSign(ctx context.Context, uploadID string, partNumbers []int) (*s3.AuthSignResult, error) - InitiateMultipartUpload(ctx context.Context, hash string, size int64, expire time.Duration, maxParts int) (*cont.InitiateUploadResult, error) + InitiateMultipartUpload(ctx context.Context, hash string, size int64, expire time.Duration, maxParts int, contentType string) (*cont.InitiateUploadResult, error) CompleteMultipartUpload(ctx context.Context, uploadID string, parts []string) (*cont.UploadResult, error) AccessURL(ctx context.Context, name string, expire time.Duration, opt *s3.AccessURLOption) (time.Time, string, error) SetObject(ctx context.Context, info *model.Object) error @@ -73,8 +73,8 @@ func (s *s3Database) AuthSign(ctx context.Context, uploadID string, partNumbers return s.s3.AuthSign(ctx, uploadID, partNumbers) } -func (s *s3Database) InitiateMultipartUpload(ctx context.Context, hash string, size int64, expire time.Duration, maxParts int) (*cont.InitiateUploadResult, error) { - return s.s3.InitiateUpload(ctx, hash, size, expire, maxParts) +func (s *s3Database) InitiateMultipartUpload(ctx context.Context, hash string, size int64, expire time.Duration, maxParts int, contentType string) (*cont.InitiateUploadResult, error) { + return s.s3.InitiateUploadContentType(ctx, hash, size, expire, maxParts, contentType) } func (s *s3Database) CompleteMultipartUpload(ctx context.Context, uploadID string, parts []string) (*cont.UploadResult, error) { diff --git a/tools/s3/internal/conversion.go b/tools/s3/internal/conversion.go index ba2174535..af391ec42 100644 --- a/tools/s3/internal/conversion.go +++ b/tools/s3/internal/conversion.go @@ -4,6 +4,11 @@ import ( "context" "errors" "fmt" + "log" + "net/http" + "path/filepath" + "time" + "github.com/mitchellh/mapstructure" "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/redis" @@ -19,10 +24,6 @@ import ( "github.com/openimsdk/tools/s3/oss" "github.com/spf13/viper" "go.mongodb.org/mongo-driver/mongo" - "log" - "net/http" - "path/filepath" - "time" ) const defaultTimeout = time.Second * 10 @@ -159,7 +160,7 @@ func doObject(db database.ObjectInfo, newS3, oldS3 s3.Interface, skip int) (*Res if err != nil { return nil, err } - putURL, err := newS3.PresignedPutObject(ctx, obj.Key, time.Hour) + putURL, err := newS3.PresignedPutObject(ctx, obj.Key, time.Hour, &s3.PutOption{ContentType: obj.ContentType}) if err != nil { return nil, err } @@ -176,7 +177,7 @@ func doObject(db database.ObjectInfo, newS3, oldS3 s3.Interface, skip int) (*Res return nil, fmt.Errorf("download object failed %s", downloadResp.Status) } log.Printf("file size %d", obj.Size) - request, err := http.NewRequest(http.MethodPut, putURL, downloadResp.Body) + request, err := http.NewRequest(http.MethodPut, putURL.URL, downloadResp.Body) if err != nil { return nil, err } diff --git a/version/version.go b/version/version.go index 23b3a82f5..32ad27808 100644 --- a/version/version.go +++ b/version/version.go @@ -1,6 +1,14 @@ package version -import _ "embed" +import ( + _ "embed" + "strings" +) //go:embed version var Version string + +func init() { + Version = strings.Trim(Version, "\n") + Version = strings.TrimSpace(Version) +} From d385fdd0aa51a689a3f735991159557ea7549829 Mon Sep 17 00:00:00 2001 From: chao <48119764+withchao@users.noreply.github.com> Date: Wed, 2 Apr 2025 18:18:06 +0800 Subject: [PATCH 143/199] feat: support server-issued configuration, which can be set for individual users (#3271) * pb * fix: Modifying other fields while setting IsPrivateChat does not take effect * fix: quote message error revoke * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * upgrading pkg tools * fix * fix * optimize log output * feat: support GetLastMessage * feat: support GetLastMessage * feat: s3 switch * feat: s3 switch * fix: GetUsersOnline * feat: SendBusinessNotification supported configuration parameters * feat: SendBusinessNotification supported configuration parameters * feat: SendBusinessNotification supported configuration parameters * feat: seq conversion failed without exiting * fix: DeleteDoc crash * fix: fill send time * fix: fill send time * fix: crash caused by withdrawing messages from users who have left the group * fix: user msg timestamp * seq read config * seq read config * fix: the source message of the reference is withdrawn, and the referenced message is deleted * feat: optimize the default notification.yml * fix: shouldPushOffline * fix: the sorting is wrong after canceling the administrator in group settings * feat: Sending messages supports returning fields modified by webhook * feat: Sending messages supports returning fields modified by webhook * feat: Sending messages supports returning fields modified by webhook * fix: oss specifies content-type when uploading * fix: the version number contains a line break * fix: the version number contains a line break * feat: support client config * feat: support client config --- go.mod | 4 +- go.sum | 4 +- internal/api/router.go | 5 + internal/api/user.go | 16 +++ internal/rpc/user/config.go | 71 +++++++++++++ internal/rpc/user/user.go | 11 ++- .../storage/cache/cachekey/client_config.go | 10 ++ pkg/common/storage/cache/client_config.go | 8 ++ .../storage/cache/redis/client_config.go | 69 +++++++++++++ .../storage/controller/client_config.go | 58 +++++++++++ pkg/common/storage/database/client_config.go | 15 +++ .../storage/database/mgo/client_config.go | 99 +++++++++++++++++++ pkg/common/storage/model/client_config.go | 7 ++ 13 files changed, 371 insertions(+), 6 deletions(-) create mode 100644 internal/rpc/user/config.go create mode 100644 pkg/common/storage/cache/cachekey/client_config.go create mode 100644 pkg/common/storage/cache/client_config.go create mode 100644 pkg/common/storage/cache/redis/client_config.go create mode 100644 pkg/common/storage/controller/client_config.go create mode 100644 pkg/common/storage/database/client_config.go create mode 100644 pkg/common/storage/database/mgo/client_config.go create mode 100644 pkg/common/storage/model/client_config.go diff --git a/go.mod b/go.mod index 782a306f4..b002ac377 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/gorilla/websocket v1.5.1 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/mitchellh/mapstructure v1.5.0 - github.com/openimsdk/protocol v0.0.72-alpha.81 + github.com/openimsdk/protocol v0.0.73-alpha.3 github.com/openimsdk/tools v0.0.50-alpha.79 github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 @@ -219,3 +219,5 @@ require ( golang.org/x/crypto v0.27.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect ) + +//replace github.com/openimsdk/protocol => /Users/chao/Desktop/code/protocol diff --git a/go.sum b/go.sum index aa0dfa6ac..e6408cfcd 100644 --- a/go.sum +++ b/go.sum @@ -347,8 +347,8 @@ github.com/onsi/gomega v1.25.0 h1:Vw7br2PCDYijJHSfBOWhov+8cAnUf8MfMaIOV323l6Y= github.com/onsi/gomega v1.25.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdMPGhM= github.com/openimsdk/gomake v0.0.15-alpha.2 h1:5Q8yl8ezy2yx+q8/ucU/t4kJnDfCzNOrkXcDACCqtyM= github.com/openimsdk/gomake v0.0.15-alpha.2/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= -github.com/openimsdk/protocol v0.0.72-alpha.81 h1:6tDuZ3Anfi1uhX/V5mWxITqJnGQPnvgeaxeqJlEHIVE= -github.com/openimsdk/protocol v0.0.72-alpha.81/go.mod h1:WF7EuE55vQvpyUAzDXcqg+B+446xQyEba0X35lTINmw= +github.com/openimsdk/protocol v0.0.73-alpha.3 h1:mf/REUZA5in2gk8ggwqJD8444xLvB7WlF7M97oXN78g= +github.com/openimsdk/protocol v0.0.73-alpha.3/go.mod h1:WF7EuE55vQvpyUAzDXcqg+B+446xQyEba0X35lTINmw= github.com/openimsdk/tools v0.0.50-alpha.79 h1:jxYEbrzaze4Z2r4NrKad816buZ690ix0L9MTOOOH3ik= github.com/openimsdk/tools v0.0.50-alpha.79/go.mod h1:n2poR3asX1e1XZce4O+MOWAp+X02QJRFvhcLCXZdzRo= github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= diff --git a/internal/api/router.go b/internal/api/router.go index 657493b23..920bd5366 100644 --- a/internal/api/router.go +++ b/internal/api/router.go @@ -125,6 +125,11 @@ func newGinRouter(ctx context.Context, client discovery.Conn, cfg *Config) (*gin userRouterGroup.POST("/add_notification_account", u.AddNotificationAccount) userRouterGroup.POST("/update_notification_account", u.UpdateNotificationAccountInfo) userRouterGroup.POST("/search_notification_account", u.SearchNotificationAccount) + + userRouterGroup.POST("/get_user_client_config", u.GetUserClientConfig) + userRouterGroup.POST("/set_user_client_config", u.SetUserClientConfig) + userRouterGroup.POST("/del_user_client_config", u.DelUserClientConfig) + userRouterGroup.POST("/page_user_client_config", u.PageUserClientConfig) } // friend routing group { diff --git a/internal/api/user.go b/internal/api/user.go index 6427e222e..93a311a9b 100644 --- a/internal/api/user.go +++ b/internal/api/user.go @@ -242,3 +242,19 @@ func (u *UserApi) UpdateNotificationAccountInfo(c *gin.Context) { func (u *UserApi) SearchNotificationAccount(c *gin.Context) { a2r.Call(c, user.UserClient.SearchNotificationAccount, u.Client) } + +func (u *UserApi) GetUserClientConfig(c *gin.Context) { + a2r.Call(c, user.UserClient.GetUserClientConfig, u.Client) +} + +func (u *UserApi) SetUserClientConfig(c *gin.Context) { + a2r.Call(c, user.UserClient.SetUserClientConfig, u.Client) +} + +func (u *UserApi) DelUserClientConfig(c *gin.Context) { + a2r.Call(c, user.UserClient.DelUserClientConfig, u.Client) +} + +func (u *UserApi) PageUserClientConfig(c *gin.Context) { + a2r.Call(c, user.UserClient.PageUserClientConfig, u.Client) +} diff --git a/internal/rpc/user/config.go b/internal/rpc/user/config.go new file mode 100644 index 000000000..5a9a46359 --- /dev/null +++ b/internal/rpc/user/config.go @@ -0,0 +1,71 @@ +package user + +import ( + "context" + + "github.com/openimsdk/open-im-server/v3/pkg/authverify" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" + pbuser "github.com/openimsdk/protocol/user" + "github.com/openimsdk/tools/utils/datautil" +) + +func (s *userServer) GetUserClientConfig(ctx context.Context, req *pbuser.GetUserClientConfigReq) (*pbuser.GetUserClientConfigResp, error) { + if req.UserID != "" { + if err := authverify.CheckAccessV3(ctx, req.UserID, s.config.Share.IMAdminUserID); err != nil { + return nil, err + } + if _, err := s.db.GetUserByID(ctx, req.UserID); err != nil { + return nil, err + } + } + res, err := s.clientConfig.GetUserConfig(ctx, req.UserID) + if err != nil { + return nil, err + } + return &pbuser.GetUserClientConfigResp{Configs: res}, nil +} + +func (s *userServer) SetUserClientConfig(ctx context.Context, req *pbuser.SetUserClientConfigReq) (*pbuser.SetUserClientConfigResp, error) { + if err := authverify.CheckAdmin(ctx, s.config.Share.IMAdminUserID); err != nil { + return nil, err + } + if req.UserID != "" { + if _, err := s.db.GetUserByID(ctx, req.UserID); err != nil { + return nil, err + } + } + if err := s.clientConfig.SetUserConfig(ctx, req.UserID, req.Configs); err != nil { + return nil, err + } + return &pbuser.SetUserClientConfigResp{}, nil +} + +func (s *userServer) DelUserClientConfig(ctx context.Context, req *pbuser.DelUserClientConfigReq) (*pbuser.DelUserClientConfigResp, error) { + if err := authverify.CheckAdmin(ctx, s.config.Share.IMAdminUserID); err != nil { + return nil, err + } + if err := s.clientConfig.DelUserConfig(ctx, req.UserID, req.Keys); err != nil { + return nil, err + } + return &pbuser.DelUserClientConfigResp{}, nil +} + +func (s *userServer) PageUserClientConfig(ctx context.Context, req *pbuser.PageUserClientConfigReq) (*pbuser.PageUserClientConfigResp, error) { + if err := authverify.CheckAdmin(ctx, s.config.Share.IMAdminUserID); err != nil { + return nil, err + } + total, res, err := s.clientConfig.GetUserConfigPage(ctx, req.UserID, req.Key, req.Pagination) + if err != nil { + return nil, err + } + return &pbuser.PageUserClientConfigResp{ + Total: total, + Configs: datautil.Slice(res, func(e *model.ClientConfig) *pbuser.ClientConfig { + return &pbuser.ClientConfig{ + UserID: e.UserID, + Key: e.Key, + Value: e.Value, + } + }), + }, nil +} diff --git a/internal/rpc/user/user.go b/internal/rpc/user/user.go index 3e8ec3537..6ef61f773 100644 --- a/internal/rpc/user/user.go +++ b/internal/rpc/user/user.go @@ -64,6 +64,7 @@ type userServer struct { webhookClient *webhook.Client groupClient *rpcli.GroupClient relationClient *rpcli.RelationClient + clientConfig controller.ClientConfigDatabase } type Config struct { @@ -98,6 +99,10 @@ func Start(ctx context.Context, config *Config, client discovery.Conn, server gr if err != nil { return err } + clientConfigDB, err := mgo.NewClientConfig(mgocli.GetDB()) + if err != nil { + return err + } msgConn, err := client.GetConn(ctx, config.Discovery.RpcService.Msg) if err != nil { return err @@ -122,9 +127,9 @@ func Start(ctx context.Context, config *Config, client discovery.Conn, server gr userNotificationSender: NewUserNotificationSender(config, msgClient, WithUserFunc(database.FindWithError)), config: config, webhookClient: webhook.NewWebhookClient(config.WebhooksConfig.URL), - - groupClient: rpcli.NewGroupClient(groupConn), - relationClient: rpcli.NewRelationClient(friendConn), + clientConfig: controller.NewClientConfigDatabase(clientConfigDB, redis.NewClientConfigCache(rdb, clientConfigDB), mgocli.GetTx()), + groupClient: rpcli.NewGroupClient(groupConn), + relationClient: rpcli.NewRelationClient(friendConn), } pbuser.RegisterUserServer(server, u) return u.db.InitOnce(context.Background(), users) diff --git a/pkg/common/storage/cache/cachekey/client_config.go b/pkg/common/storage/cache/cachekey/client_config.go new file mode 100644 index 000000000..16770adef --- /dev/null +++ b/pkg/common/storage/cache/cachekey/client_config.go @@ -0,0 +1,10 @@ +package cachekey + +const ClientConfig = "CLIENT_CONFIG" + +func GetClientConfigKey(userID string) string { + if userID == "" { + return ClientConfig + } + return ClientConfig + ":" + userID +} diff --git a/pkg/common/storage/cache/client_config.go b/pkg/common/storage/cache/client_config.go new file mode 100644 index 000000000..329f25c59 --- /dev/null +++ b/pkg/common/storage/cache/client_config.go @@ -0,0 +1,8 @@ +package cache + +import "context" + +type ClientConfigCache interface { + DeleteUserCache(ctx context.Context, userIDs []string) error + GetUserConfig(ctx context.Context, userID string) (map[string]string, error) +} diff --git a/pkg/common/storage/cache/redis/client_config.go b/pkg/common/storage/cache/redis/client_config.go new file mode 100644 index 000000000..c5a455146 --- /dev/null +++ b/pkg/common/storage/cache/redis/client_config.go @@ -0,0 +1,69 @@ +package redis + +import ( + "context" + "time" + + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/cachekey" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" + "github.com/redis/go-redis/v9" +) + +func NewClientConfigCache(rdb redis.UniversalClient, mgo database.ClientConfig) cache.ClientConfigCache { + rc := newRocksCacheClient(rdb) + return &ClientConfigCache{ + mgo: mgo, + rcClient: rc, + delete: rc.GetBatchDeleter(), + } +} + +type ClientConfigCache struct { + mgo database.ClientConfig + rcClient *rocksCacheClient + delete cache.BatchDeleter +} + +func (x *ClientConfigCache) getExpireTime(userID string) time.Duration { + if userID == "" { + return time.Hour * 24 + } else { + return time.Hour + } +} + +func (x *ClientConfigCache) getClientConfigKey(userID string) string { + return cachekey.GetClientConfigKey(userID) +} + +func (x *ClientConfigCache) GetConfig(ctx context.Context, userID string) (map[string]string, error) { + return getCache(ctx, x.rcClient, x.getClientConfigKey(userID), x.getExpireTime(userID), func(ctx context.Context) (map[string]string, error) { + return x.mgo.Get(ctx, userID) + }) +} + +func (x *ClientConfigCache) DeleteUserCache(ctx context.Context, userIDs []string) error { + keys := make([]string, 0, len(userIDs)) + for _, userID := range userIDs { + keys = append(keys, x.getClientConfigKey(userID)) + } + return x.delete.ExecDelWithKeys(ctx, keys) +} + +func (x *ClientConfigCache) GetUserConfig(ctx context.Context, userID string) (map[string]string, error) { + config, err := x.GetConfig(ctx, "") + if err != nil { + return nil, err + } + if userID != "" { + userConfig, err := x.GetConfig(ctx, userID) + if err != nil { + return nil, err + } + for k, v := range userConfig { + config[k] = v + } + } + return config, nil +} diff --git a/pkg/common/storage/controller/client_config.go b/pkg/common/storage/controller/client_config.go new file mode 100644 index 000000000..1c3787634 --- /dev/null +++ b/pkg/common/storage/controller/client_config.go @@ -0,0 +1,58 @@ +package controller + +import ( + "context" + + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" + "github.com/openimsdk/tools/db/pagination" + "github.com/openimsdk/tools/db/tx" +) + +type ClientConfigDatabase interface { + SetUserConfig(ctx context.Context, userID string, config map[string]string) error + GetUserConfig(ctx context.Context, userID string) (map[string]string, error) + DelUserConfig(ctx context.Context, userID string, keys []string) error + GetUserConfigPage(ctx context.Context, userID string, key string, pagination pagination.Pagination) (int64, []*model.ClientConfig, error) +} + +func NewClientConfigDatabase(db database.ClientConfig, cache cache.ClientConfigCache, tx tx.Tx) ClientConfigDatabase { + return &clientConfigDatabase{ + tx: tx, + db: db, + cache: cache, + } +} + +type clientConfigDatabase struct { + tx tx.Tx + db database.ClientConfig + cache cache.ClientConfigCache +} + +func (x *clientConfigDatabase) SetUserConfig(ctx context.Context, userID string, config map[string]string) error { + return x.tx.Transaction(ctx, func(ctx context.Context) error { + if err := x.db.Set(ctx, userID, config); err != nil { + return err + } + return x.cache.DeleteUserCache(ctx, []string{userID}) + }) +} + +func (x *clientConfigDatabase) GetUserConfig(ctx context.Context, userID string) (map[string]string, error) { + return x.cache.GetUserConfig(ctx, userID) +} + +func (x *clientConfigDatabase) DelUserConfig(ctx context.Context, userID string, keys []string) error { + return x.tx.Transaction(ctx, func(ctx context.Context) error { + if err := x.db.Del(ctx, userID, keys); err != nil { + return err + } + return x.cache.DeleteUserCache(ctx, []string{userID}) + }) +} + +func (x *clientConfigDatabase) GetUserConfigPage(ctx context.Context, userID string, key string, pagination pagination.Pagination) (int64, []*model.ClientConfig, error) { + return x.db.GetPage(ctx, userID, key, pagination) +} diff --git a/pkg/common/storage/database/client_config.go b/pkg/common/storage/database/client_config.go new file mode 100644 index 000000000..7fa888d24 --- /dev/null +++ b/pkg/common/storage/database/client_config.go @@ -0,0 +1,15 @@ +package database + +import ( + "context" + + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" + "github.com/openimsdk/tools/db/pagination" +) + +type ClientConfig interface { + Set(ctx context.Context, userID string, config map[string]string) error + Get(ctx context.Context, userID string) (map[string]string, error) + Del(ctx context.Context, userID string, keys []string) error + GetPage(ctx context.Context, userID string, key string, pagination pagination.Pagination) (int64, []*model.ClientConfig, error) +} diff --git a/pkg/common/storage/database/mgo/client_config.go b/pkg/common/storage/database/mgo/client_config.go new file mode 100644 index 000000000..0aa462899 --- /dev/null +++ b/pkg/common/storage/database/mgo/client_config.go @@ -0,0 +1,99 @@ +// Copyright © 2023 OpenIM open source community. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package mgo + +import ( + "context" + + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" + "github.com/openimsdk/tools/db/mongoutil" + "github.com/openimsdk/tools/db/pagination" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" + + "github.com/openimsdk/tools/errs" +) + +func NewClientConfig(db *mongo.Database) (database.ClientConfig, error) { + coll := db.Collection("config") + _, err := coll.Indexes().CreateMany(context.Background(), []mongo.IndexModel{ + { + Keys: bson.D{ + {Key: "key", Value: 1}, + {Key: "user_id", Value: 1}, + }, + Options: options.Index().SetUnique(true), + }, + }) + if err != nil { + return nil, errs.Wrap(err) + } + return &ClientConfig{ + coll: coll, + }, nil +} + +type ClientConfig struct { + coll *mongo.Collection +} + +func (x *ClientConfig) Set(ctx context.Context, userID string, config map[string]string) error { + if len(config) == 0 { + return nil + } + for key, value := range config { + filter := bson.M{"key": key, "user_id": userID} + update := bson.M{ + "value": value, + } + err := mongoutil.UpdateOne(ctx, x.coll, filter, bson.M{"$set": update}, false, options.Update().SetUpsert(true)) + if err != nil { + return err + } + } + return nil +} + +func (x *ClientConfig) Get(ctx context.Context, userID string) (map[string]string, error) { + cs, err := mongoutil.Find[*model.ClientConfig](ctx, x.coll, bson.M{"user_id": userID}) + if err != nil { + return nil, err + } + cm := make(map[string]string) + for _, config := range cs { + cm[config.Key] = config.Value + } + return cm, nil +} + +func (x *ClientConfig) Del(ctx context.Context, userID string, keys []string) error { + if len(keys) == 0 { + return nil + } + return mongoutil.DeleteMany(ctx, x.coll, bson.M{"key": bson.M{"$in": keys}, "user_id": userID}) +} + +func (x *ClientConfig) GetPage(ctx context.Context, userID string, key string, pagination pagination.Pagination) (int64, []*model.ClientConfig, error) { + filter := bson.M{} + if userID != "" { + filter["user_id"] = userID + } + if key != "" { + filter["key"] = key + } + return mongoutil.FindPage[*model.ClientConfig](ctx, x.coll, filter, pagination) +} diff --git a/pkg/common/storage/model/client_config.go b/pkg/common/storage/model/client_config.go new file mode 100644 index 000000000..f06e29102 --- /dev/null +++ b/pkg/common/storage/model/client_config.go @@ -0,0 +1,7 @@ +package model + +type ClientConfig struct { + Key string `bson:"key"` + UserID string `bson:"user_id"` + Value string `bson:"value"` +} From 16976511002c75c8a4c5fd879741d7535401ad66 Mon Sep 17 00:00:00 2001 From: chao <48119764+withchao@users.noreply.github.com> Date: Mon, 14 Apr 2025 11:18:07 +0800 Subject: [PATCH 144/199] feat: GetConversationsHasReadAndMaxSeq support pinned (#3281) * pb * fix: Modifying other fields while setting IsPrivateChat does not take effect * fix: quote message error revoke * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * upgrading pkg tools * fix * fix * optimize log output * feat: support GetLastMessage * feat: support GetLastMessage * feat: s3 switch * feat: s3 switch * fix: GetUsersOnline * feat: SendBusinessNotification supported configuration parameters * feat: SendBusinessNotification supported configuration parameters * feat: SendBusinessNotification supported configuration parameters * feat: seq conversion failed without exiting * fix: DeleteDoc crash * fix: fill send time * fix: fill send time * fix: crash caused by withdrawing messages from users who have left the group * fix: user msg timestamp * seq read config * seq read config * fix: the source message of the reference is withdrawn, and the referenced message is deleted * feat: optimize the default notification.yml * fix: shouldPushOffline * fix: the sorting is wrong after canceling the administrator in group settings * feat: Sending messages supports returning fields modified by webhook * feat: Sending messages supports returning fields modified by webhook * feat: Sending messages supports returning fields modified by webhook * fix: oss specifies content-type when uploading * fix: the version number contains a line break * fix: the version number contains a line break * feat: GetConversationsHasReadAndMaxSeq support pinned * feat: GetConversationsHasReadAndMaxSeq support pinned * feat: GetConversationsHasReadAndMaxSeq support pinned --- go.mod | 4 +--- go.sum | 4 ++-- internal/rpc/msg/as_read.go | 7 +++++++ pkg/rpccache/conversation.go | 25 +++++++++++++++++++++++++ 4 files changed, 35 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index b002ac377..fbaa19434 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/gorilla/websocket v1.5.1 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/mitchellh/mapstructure v1.5.0 - github.com/openimsdk/protocol v0.0.73-alpha.3 + github.com/openimsdk/protocol v0.0.73-alpha.6 github.com/openimsdk/tools v0.0.50-alpha.79 github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 @@ -219,5 +219,3 @@ require ( golang.org/x/crypto v0.27.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect ) - -//replace github.com/openimsdk/protocol => /Users/chao/Desktop/code/protocol diff --git a/go.sum b/go.sum index e6408cfcd..390a51c4a 100644 --- a/go.sum +++ b/go.sum @@ -347,8 +347,8 @@ github.com/onsi/gomega v1.25.0 h1:Vw7br2PCDYijJHSfBOWhov+8cAnUf8MfMaIOV323l6Y= github.com/onsi/gomega v1.25.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdMPGhM= github.com/openimsdk/gomake v0.0.15-alpha.2 h1:5Q8yl8ezy2yx+q8/ucU/t4kJnDfCzNOrkXcDACCqtyM= github.com/openimsdk/gomake v0.0.15-alpha.2/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= -github.com/openimsdk/protocol v0.0.73-alpha.3 h1:mf/REUZA5in2gk8ggwqJD8444xLvB7WlF7M97oXN78g= -github.com/openimsdk/protocol v0.0.73-alpha.3/go.mod h1:WF7EuE55vQvpyUAzDXcqg+B+446xQyEba0X35lTINmw= +github.com/openimsdk/protocol v0.0.73-alpha.6 h1:sna9coWG7HN1zObBPtvG0Ki/vzqHXiB4qKbA5P3w7kc= +github.com/openimsdk/protocol v0.0.73-alpha.6/go.mod h1:WF7EuE55vQvpyUAzDXcqg+B+446xQyEba0X35lTINmw= github.com/openimsdk/tools v0.0.50-alpha.79 h1:jxYEbrzaze4Z2r4NrKad816buZ690ix0L9MTOOOH3ik= github.com/openimsdk/tools v0.0.50-alpha.79/go.mod h1:n2poR3asX1e1XZce4O+MOWAp+X02QJRFvhcLCXZdzRo= github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= diff --git a/internal/rpc/msg/as_read.go b/internal/rpc/msg/as_read.go index de1879438..b25eae6b1 100644 --- a/internal/rpc/msg/as_read.go +++ b/internal/rpc/msg/as_read.go @@ -61,6 +61,13 @@ func (m *msgServer) GetConversationsHasReadAndMaxSeq(ctx context.Context, req *m return nil, err } resp := &msg.GetConversationsHasReadAndMaxSeqResp{Seqs: make(map[string]*msg.Seqs)} + if req.ReturnPinned { + pinnedConversationIDs, err := m.ConversationLocalCache.GetPinnedConversationIDs(ctx, req.UserID) + if err != nil { + return nil, err + } + resp.PinnedConversationIDs = pinnedConversationIDs + } for conversationID, maxSeq := range maxSeqs { resp.Seqs[conversationID] = &msg.Seqs{ HasReadSeq: hasReadSeqs[conversationID], diff --git a/pkg/rpccache/conversation.go b/pkg/rpccache/conversation.go index 70f5acfd1..162fda596 100644 --- a/pkg/rpccache/conversation.go +++ b/pkg/rpccache/conversation.go @@ -16,6 +16,7 @@ package rpccache import ( "context" + "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/cachekey" "github.com/openimsdk/open-im-server/v3/pkg/localcache" @@ -153,6 +154,26 @@ func (c *ConversationLocalCache) getConversationNotReceiveMessageUserIDs(ctx con })) } +func (c *ConversationLocalCache) getPinnedConversationIDs(ctx context.Context, userID string) (val []string, err error) { + log.ZDebug(ctx, "ConversationLocalCache getPinnedConversations req", "userID", userID) + defer func() { + if err == nil { + log.ZDebug(ctx, "ConversationLocalCache getPinnedConversations return", "userID", userID, "value", val) + } else { + log.ZError(ctx, "ConversationLocalCache getPinnedConversations return", err, "userID", userID) + } + }() + var cache cacheProto[pbconversation.GetPinnedConversationIDsResp] + resp, err := cache.Unmarshal(c.local.Get(ctx, cachekey.GetPinnedConversationIDs(userID), func(ctx context.Context) ([]byte, error) { + log.ZDebug(ctx, "ConversationLocalCache getConversationNotReceiveMessageUserIDs rpc", "userID", userID) + return cache.Marshal(c.client.ConversationClient.GetPinnedConversationIDs(ctx, &pbconversation.GetPinnedConversationIDsReq{UserID: userID})) + })) + if err != nil { + return nil, err + } + return resp.ConversationIDs, nil +} + func (c *ConversationLocalCache) GetConversationNotReceiveMessageUserIDs(ctx context.Context, conversationID string) ([]string, error) { res, err := c.getConversationNotReceiveMessageUserIDs(ctx, conversationID) if err != nil { @@ -168,3 +189,7 @@ func (c *ConversationLocalCache) GetConversationNotReceiveMessageUserIDMap(ctx c } return datautil.SliceSet(res.UserIDs), nil } + +func (c *ConversationLocalCache) GetPinnedConversationIDs(ctx context.Context, userID string) ([]string, error) { + return c.getPinnedConversationIDs(ctx, userID) +} From 52bd5e8be9027de8ca9aca2407469b915eb8a97d Mon Sep 17 00:00:00 2001 From: chao <48119764+withchao@users.noreply.github.com> Date: Tue, 15 Apr 2025 18:27:39 +0800 Subject: [PATCH 145/199] fix: transferring the group owner to a muted member, incremental version error (#3284) * pb * fix: Modifying other fields while setting IsPrivateChat does not take effect * fix: quote message error revoke * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * upgrading pkg tools * fix * fix * optimize log output * feat: support GetLastMessage * feat: support GetLastMessage * feat: s3 switch * feat: s3 switch * fix: GetUsersOnline * feat: SendBusinessNotification supported configuration parameters * feat: SendBusinessNotification supported configuration parameters * feat: SendBusinessNotification supported configuration parameters * feat: seq conversion failed without exiting * fix: DeleteDoc crash * fix: fill send time * fix: fill send time * fix: crash caused by withdrawing messages from users who have left the group * fix: user msg timestamp * seq read config * seq read config * fix: the source message of the reference is withdrawn, and the referenced message is deleted * feat: optimize the default notification.yml * fix: shouldPushOffline * fix: the sorting is wrong after canceling the administrator in group settings * feat: Sending messages supports returning fields modified by webhook * feat: Sending messages supports returning fields modified by webhook * feat: Sending messages supports returning fields modified by webhook * fix: oss specifies content-type when uploading * fix: the version number contains a line break * fix: the version number contains a line break * feat: GetConversationsHasReadAndMaxSeq support pinned * feat: GetConversationsHasReadAndMaxSeq support pinned * feat: GetConversationsHasReadAndMaxSeq support pinned * fix: transferring the group owner to a muted member, incremental version error --- internal/rpc/group/notification.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/internal/rpc/group/notification.go b/internal/rpc/group/notification.go index 0a135af23..f9da88863 100644 --- a/internal/rpc/group/notification.go +++ b/internal/rpc/group/notification.go @@ -284,7 +284,8 @@ func (g *NotificationSender) fillUserByUserID(ctx context.Context, userID string func (g *NotificationSender) setVersion(ctx context.Context, version *uint64, versionID *string, collName string, id string) { versions := versionctx.GetVersionLog(ctx).Get() - for _, coll := range versions { + for i := len(versions) - 1; i >= 0; i-- { + coll := versions[i] if coll.Name == collName && coll.Doc.DID == id { *version = uint64(coll.Doc.Version) *versionID = coll.Doc.ID.Hex() From 8e824c7e8eb58265a6e4c32edb5ad2bed2f1be67 Mon Sep 17 00:00:00 2001 From: icey-yu <119291641+icey-yu@users.noreply.github.com> Date: Thu, 17 Apr 2025 17:28:23 +0800 Subject: [PATCH 146/199] fix: group status in GroupDismissedNotification (#3286) --- internal/rpc/group/group.go | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/rpc/group/group.go b/internal/rpc/group/group.go index e1604e7e5..63d9336d5 100644 --- a/internal/rpc/group/group.go +++ b/internal/rpc/group/group.go @@ -1370,6 +1370,7 @@ func (g *groupServer) DismissGroup(ctx context.Context, req *pbgroup.DismissGrou if err != nil { return nil, err } + group.Status = constant.GroupStatusDismissed tips := &sdkws.GroupDismissedTips{ Group: g.groupDB2PB(group, owner.UserID, num), OpUser: &sdkws.GroupMemberFullInfo{}, From 045afd74b9049bba137ff8df6ef48ebc6d4f33b2 Mon Sep 17 00:00:00 2001 From: skiffer-git <72860476+skiffer-git@users.noreply.github.com> Date: Tue, 22 Apr 2025 14:35:50 +0800 Subject: [PATCH 147/199] Update LICENSE --- LICENSE | 670 ++++++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 648 insertions(+), 22 deletions(-) diff --git a/LICENSE b/LICENSE index 4591ca426..0ad25db4b 100644 --- a/LICENSE +++ b/LICENSE @@ -1,35 +1,661 @@ -# Open Source License + GNU AFFERO GENERAL PUBLIC LICENSE + Version 3, 19 November 2007 -OpenIM is licensed under the Apache License 2.0, with the following additional conditions: + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. -1. OpenIM may be utilized commercially, including as a backend service for other applications or as an application development platform for enterprises. -A commercial license must be obtained from the producer if: - - Under no circumstances may you operate a multi-tenant or multi-business environment using the OpenIM source code, whether or not you have modified the repository code. In other words, a single instance of OpenIM may not simultaneously serve multiple enterprises, nor may it serve multiple lines of business within the same enterprise. - - If you intend to operate in such a multi-tenant or multi-business manner, you must obtain a commercial license from the producer in advance. + Preamble -2. As a contributor, you should agree that: + The GNU Affero General Public License is a free, copyleft license for +software and other kinds of works, specifically designed to ensure +cooperation with the community in the case of network server software. - a. The producer can adjust the open-source agreement to be more strict or more relaxed as deemed necessary. - b. Your contributed code may be used for commercial purposes, including but not limited to its cloud business operations. + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +our General Public Licenses are intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. -Apart from the specific conditions mentioned above, all other rights and restrictions follow the Apache License 2.0. Detailed information about the Apache License 2.0 can be found at http://www.apache.org/licenses/LICENSE-2.0. + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + Developers that use our General Public Licenses protect your rights +with two steps: (1) assert copyright on the software, and (2) offer +you this License which gives you legal permission to copy, distribute +and/or modify the software. -For any licensing-related questions or to obtain a commercial license, please contact contact@openim.io. + A secondary benefit of defending all users' freedom is that +improvements made in alternate versions of the program, if they +receive widespread use, become available for other developers to +incorporate. Many developers of free software are heartened and +encouraged by the resulting cooperation. However, in the case of +software used on network servers, this result may fail to come about. +The GNU General Public License permits making a modified version and +letting the public access it on a server without ever releasing its +source code to the public. + The GNU Affero General Public License is designed specifically to +ensure that, in such cases, the modified source code becomes available +to the community. It requires the operator of a network server to +provide the source code of the modified version running there to the +users of that server. Therefore, public use of a modified version, on +a publicly accessible server, gives the public access to the source +code of the modified version. -© 2024 OpenIMSDK + An older license, called the Affero General Public License and +published by Affero, was designed to accomplish similar goals. This is +a different license, not a version of the Affero GPL, but Affero has +released a new version of the Affero GPL which permits relicensing under +this license. ----------- + The precise terms and conditions for copying, distribution and +modification follow. -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at + TERMS AND CONDITIONS - http://www.apache.org/licenses/LICENSE-2.0 + 0. Definitions. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. + "This License" refers to version 3 of the GNU Affero General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Remote Network Interaction; Use with the GNU General Public License. + + Notwithstanding any other provision of this License, if you modify the +Program, your modified version must prominently offer all users +interacting with it remotely through a computer network (if your version +supports such interaction) an opportunity to receive the Corresponding +Source of your version by providing access to the Corresponding Source +from a network server at no charge, through some standard or customary +means of facilitating copying of software. This Corresponding Source +shall include the Corresponding Source for any work covered by version 3 +of the GNU General Public License that is incorporated pursuant to the +following paragraph. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the work with which it is combined will remain governed by version +3 of the GNU General Public License. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU Affero General Public License from time to time. Such new versions +will be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU Affero General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU Affero General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU Affero General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published + by the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + + If your software can interact with users remotely through a computer +network, you should also make sure that it provides a way for users to +get its source. For example, if your program is a web application, its +interface could display a "Source" link that leads users to an archive +of the code. There are many ways you could offer source, and different +solutions will be better for different programs; see section 13 for the +specific requirements. + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU AGPL, see +. From 97b8c07d57337baf2d4b6b7d89c7ac4236135a84 Mon Sep 17 00:00:00 2001 From: Monet Lee Date: Tue, 22 Apr 2025 15:52:55 +0800 Subject: [PATCH 148/199] feat: Implement stress test v2. (#3292) * feat: improve stress test code. * feat: Implement stress test v2. --- tools/stress-test-v2/main.go | 736 +++++++++++++++++++++++++++++++++++ tools/stress-test/main.go | 43 +- 2 files changed, 757 insertions(+), 22 deletions(-) create mode 100644 tools/stress-test-v2/main.go diff --git a/tools/stress-test-v2/main.go b/tools/stress-test-v2/main.go new file mode 100644 index 000000000..0c309b9c9 --- /dev/null +++ b/tools/stress-test-v2/main.go @@ -0,0 +1,736 @@ +package main + +import ( + "bytes" + "context" + "encoding/json" + "flag" + "fmt" + "io" + "net/http" + "os" + "os/signal" + "sync" + "syscall" + "time" + + "github.com/openimsdk/open-im-server/v3/pkg/apistruct" + "github.com/openimsdk/open-im-server/v3/pkg/common/config" + "github.com/openimsdk/protocol/auth" + "github.com/openimsdk/protocol/constant" + "github.com/openimsdk/protocol/group" + "github.com/openimsdk/protocol/sdkws" + pbuser "github.com/openimsdk/protocol/user" + "github.com/openimsdk/tools/log" + "github.com/openimsdk/tools/system/program" +) + +// 1. Create 100K New Users +// 2. Create 100 100K Groups +// 3. Create 1000 999 Groups +// 4. Send message to 100K Groups every second +// 5. Send message to 999 Groups every minute + +var ( + // Use default userIDs List for testing, need to be created. + TestTargetUserList = []string{ + // "", + } + // DefaultGroupID = "" // Use default group ID for testing, need to be created. +) + +var ( + ApiAddress string + + // API method + GetAdminToken = "/auth/get_admin_token" + UserCheck = "/user/account_check" + CreateUser = "/user/user_register" + ImportFriend = "/friend/import_friend" + InviteToGroup = "/group/invite_user_to_group" + GetGroupMemberInfo = "/group/get_group_members_info" + SendMsg = "/msg/send_msg" + CreateGroup = "/group/create_group" + GetUserToken = "/auth/user_token" +) + +const ( + MaxUser = 100000 + Max100KGroup = 100 + Max999Group = 1000 + MaxInviteUserLimit = 999 + + CreateUserTicker = 1 * time.Second + CreateGroupTicker = 1 * time.Second + Create100KGroupTicker = 1 * time.Second + Create999GroupTicker = 1 * time.Second + SendMsgTo100KGroupTicker = 1 * time.Second + SendMsgTo999GroupTicker = 1 * time.Minute +) + +type BaseResp struct { + ErrCode int `json:"errCode"` + ErrMsg string `json:"errMsg"` + Data json.RawMessage `json:"data"` +} + +type StressTest struct { + Conf *conf + AdminUserID string + AdminToken string + DefaultGroupID string + DefaultUserID string + UserCounter int + CreateUserCounter int + Create100kGroupCounter int + Create999GroupCounter int + MsgCounter int + CreatedUsers []string + CreatedGroups []string + Mutex sync.Mutex + Ctx context.Context + Cancel context.CancelFunc + HttpClient *http.Client + Wg sync.WaitGroup + Once sync.Once +} + +type conf struct { + Share config.Share + Api config.API +} + +func initConfig(configDir string) (*config.Share, *config.API, error) { + var ( + share = &config.Share{} + apiConfig = &config.API{} + ) + + err := config.Load(configDir, config.ShareFileName, config.EnvPrefixMap[config.ShareFileName], share) + if err != nil { + return nil, nil, err + } + + err = config.Load(configDir, config.OpenIMAPICfgFileName, config.EnvPrefixMap[config.OpenIMAPICfgFileName], apiConfig) + if err != nil { + return nil, nil, err + } + + return share, apiConfig, nil +} + +// Post Request +func (st *StressTest) PostRequest(ctx context.Context, url string, reqbody any) ([]byte, error) { + // Marshal body + jsonBody, err := json.Marshal(reqbody) + if err != nil { + log.ZError(ctx, "Failed to marshal request body", err, "url", url, "reqbody", reqbody) + return nil, err + } + + req, err := http.NewRequest(http.MethodPost, url, bytes.NewReader(jsonBody)) + if err != nil { + return nil, err + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("operationID", st.AdminUserID) + if st.AdminToken != "" { + req.Header.Set("token", st.AdminToken) + } + + // log.ZInfo(ctx, "Header info is ", "Content-Type", "application/json", "operationID", st.AdminUserID, "token", st.AdminToken) + + resp, err := st.HttpClient.Do(req) + if err != nil { + log.ZError(ctx, "Failed to send request", err, "url", url, "reqbody", reqbody) + return nil, err + } + defer resp.Body.Close() + + respBody, err := io.ReadAll(resp.Body) + if err != nil { + log.ZError(ctx, "Failed to read response body", err, "url", url) + return nil, err + } + + var baseResp BaseResp + if err := json.Unmarshal(respBody, &baseResp); err != nil { + log.ZError(ctx, "Failed to unmarshal response body", err, "url", url, "respBody", string(respBody)) + return nil, err + } + + if baseResp.ErrCode != 0 { + err = fmt.Errorf(baseResp.ErrMsg) + log.ZError(ctx, "Failed to send request", err, "url", url, "reqbody", reqbody, "resp", baseResp) + return nil, err + } + + return baseResp.Data, nil +} + +func (st *StressTest) GetAdminToken(ctx context.Context) (string, error) { + req := auth.GetAdminTokenReq{ + Secret: st.Conf.Share.Secret, + UserID: st.AdminUserID, + } + + resp, err := st.PostRequest(ctx, ApiAddress+GetAdminToken, &req) + if err != nil { + return "", err + } + + data := &auth.GetAdminTokenResp{} + if err := json.Unmarshal(resp, &data); err != nil { + return "", err + } + + return data.Token, nil +} + +func (st *StressTest) CheckUser(ctx context.Context, userIDs []string) ([]string, error) { + req := pbuser.AccountCheckReq{ + CheckUserIDs: userIDs, + } + + resp, err := st.PostRequest(ctx, ApiAddress+UserCheck, &req) + if err != nil { + return nil, err + } + + data := &pbuser.AccountCheckResp{} + if err := json.Unmarshal(resp, &data); err != nil { + return nil, err + } + + unRegisteredUserIDs := make([]string, 0) + + for _, res := range data.Results { + if res.AccountStatus == constant.UnRegistered { + unRegisteredUserIDs = append(unRegisteredUserIDs, res.UserID) + } + } + + return unRegisteredUserIDs, nil +} + +func (st *StressTest) CreateUser(ctx context.Context, userID string) (string, error) { + user := &sdkws.UserInfo{ + UserID: userID, + Nickname: userID, + } + + req := pbuser.UserRegisterReq{ + Users: []*sdkws.UserInfo{user}, + } + + _, err := st.PostRequest(ctx, ApiAddress+CreateUser, &req) + if err != nil { + return "", err + } + + st.UserCounter++ + return userID, nil +} + +func (st *StressTest) CreateUserBatch(ctx context.Context, userIDs []string) error { + // The method can import a large number of users at once. + var userList []*sdkws.UserInfo + + defer st.Once.Do( + func() { + st.DefaultUserID = userIDs[0] + fmt.Println("Default Send User Created ID:", st.DefaultUserID) + }) + + needUserIDs, err := st.CheckUser(ctx, userIDs) + if err != nil { + return err + } + + for _, userID := range needUserIDs { + user := &sdkws.UserInfo{ + UserID: userID, + Nickname: userID, + } + userList = append(userList, user) + } + + req := pbuser.UserRegisterReq{ + Users: userList, + } + + _, err = st.PostRequest(ctx, ApiAddress+CreateUser, &req) + if err != nil { + return err + } + + st.UserCounter += len(userList) + return nil +} + +func (st *StressTest) GetGroupMembersInfo(ctx context.Context, groupID string, userIDs []string) ([]string, error) { + needInviteUserIDs := make([]string, 0) + + const maxBatchSize = 500 + if len(userIDs) > maxBatchSize { + for i := 0; i < len(userIDs); i += maxBatchSize { + end := min(i+maxBatchSize, len(userIDs)) + batchUserIDs := userIDs[i:end] + + // log.ZInfo(ctx, "Processing group members batch", "groupID", groupID, "batch", i/maxBatchSize+1, + // "batchUserCount", len(batchUserIDs)) + + // Process a single batch + batchReq := group.GetGroupMembersInfoReq{ + GroupID: groupID, + UserIDs: batchUserIDs, + } + + resp, err := st.PostRequest(ctx, ApiAddress+GetGroupMemberInfo, &batchReq) + if err != nil { + log.ZError(ctx, "Batch query failed", err, "batch", i/maxBatchSize+1) + continue + } + + data := &group.GetGroupMembersInfoResp{} + if err := json.Unmarshal(resp, &data); err != nil { + log.ZError(ctx, "Failed to parse batch response", err, "batch", i/maxBatchSize+1) + continue + } + + // Process the batch results + existingMembers := make(map[string]bool) + for _, member := range data.Members { + existingMembers[member.UserID] = true + } + + for _, userID := range batchUserIDs { + if !existingMembers[userID] { + needInviteUserIDs = append(needInviteUserIDs, userID) + } + } + } + + return needInviteUserIDs, nil + } + + req := group.GetGroupMembersInfoReq{ + GroupID: groupID, + UserIDs: userIDs, + } + + resp, err := st.PostRequest(ctx, ApiAddress+GetGroupMemberInfo, &req) + if err != nil { + return nil, err + } + + data := &group.GetGroupMembersInfoResp{} + if err := json.Unmarshal(resp, &data); err != nil { + return nil, err + } + + existingMembers := make(map[string]bool) + for _, member := range data.Members { + existingMembers[member.UserID] = true + } + + for _, userID := range userIDs { + if !existingMembers[userID] { + needInviteUserIDs = append(needInviteUserIDs, userID) + } + } + + return needInviteUserIDs, nil +} + +func (st *StressTest) InviteToGroup(ctx context.Context, groupID string, userIDs []string) error { + req := group.InviteUserToGroupReq{ + GroupID: groupID, + InvitedUserIDs: userIDs, + } + _, err := st.PostRequest(ctx, ApiAddress+InviteToGroup, &req) + if err != nil { + return err + } + + return nil +} + +func (st *StressTest) SendMsg(ctx context.Context, userID string, groupID string) error { + contentObj := map[string]any{ + // "content": fmt.Sprintf("index %d. The current time is %s", st.MsgCounter, time.Now().Format("2006-01-02 15:04:05.000")), + "content": fmt.Sprintf("The current time is %s", time.Now().Format("2006-01-02 15:04:05.000")), + } + + req := &apistruct.SendMsgReq{ + SendMsg: apistruct.SendMsg{ + SendID: userID, + SenderNickname: userID, + GroupID: groupID, + ContentType: constant.Text, + SessionType: constant.ReadGroupChatType, + Content: contentObj, + }, + } + + _, err := st.PostRequest(ctx, ApiAddress+SendMsg, &req) + if err != nil { + log.ZError(ctx, "Failed to send message", err, "userID", userID, "req", &req) + return err + } + + st.MsgCounter++ + + return nil +} + +// Max userIDs number is 1000 +func (st *StressTest) CreateGroup(ctx context.Context, groupID string, userID string, userIDsList []string) (string, error) { + groupInfo := &sdkws.GroupInfo{ + GroupID: groupID, + GroupName: groupID, + GroupType: constant.WorkingGroup, + } + + req := group.CreateGroupReq{ + OwnerUserID: userID, + MemberUserIDs: userIDsList, + GroupInfo: groupInfo, + } + + resp := group.CreateGroupResp{} + + response, err := st.PostRequest(ctx, ApiAddress+CreateGroup, &req) + if err != nil { + return "", err + } + + if err := json.Unmarshal(response, &resp); err != nil { + return "", err + } + + // st.GroupCounter++ + + return resp.GroupInfo.GroupID, nil +} + +func main() { + var configPath string + // defaultConfigDir := filepath.Join("..", "..", "..", "..", "..", "config") + // flag.StringVar(&configPath, "c", defaultConfigDir, "config path") + flag.StringVar(&configPath, "c", "", "config path") + flag.Parse() + + if configPath == "" { + _, _ = fmt.Fprintln(os.Stderr, "config path is empty") + os.Exit(1) + return + } + + fmt.Printf(" Config Path: %s\n", configPath) + + share, apiConfig, err := initConfig(configPath) + if err != nil { + program.ExitWithError(err) + return + } + + ApiAddress = fmt.Sprintf("http://%s:%s", "127.0.0.1", fmt.Sprint(apiConfig.Api.Ports[0])) + + ctx, cancel := context.WithCancel(context.Background()) + // ch := make(chan struct{}) + + st := &StressTest{ + Conf: &conf{ + Share: *share, + Api: *apiConfig, + }, + AdminUserID: share.IMAdminUserID[0], + Ctx: ctx, + Cancel: cancel, + HttpClient: &http.Client{ + Timeout: 50 * time.Second, + }, + } + + c := make(chan os.Signal, 1) + signal.Notify(c, os.Interrupt, syscall.SIGTERM) + go func() { + <-c + fmt.Println("\nReceived stop signal, stopping...") + + go func() { + // time.Sleep(5 * time.Second) + fmt.Println("Force exit") + os.Exit(0) + }() + + st.Cancel() + }() + + token, err := st.GetAdminToken(st.Ctx) + if err != nil { + log.ZError(ctx, "Get Admin Token failed.", err, "AdminUserID", st.AdminUserID) + } + + st.AdminToken = token + fmt.Println("Admin Token:", st.AdminToken) + fmt.Println("ApiAddress:", ApiAddress) + + for i := range MaxUser { + userID := fmt.Sprintf("v2_StressTest_User_%d", i) + st.CreatedUsers = append(st.CreatedUsers, userID) + st.CreateUserCounter++ + } + + // err = st.CreateUserBatch(st.Ctx, st.CreatedUsers) + // if err != nil { + // log.ZError(ctx, "Create user failed.", err) + // } + + const batchSize = 1000 + totalUsers := len(st.CreatedUsers) + successCount := 0 + + if st.DefaultUserID == "" && len(st.CreatedUsers) > 0 { + st.DefaultUserID = st.CreatedUsers[0] + } + + for i := 0; i < totalUsers; i += batchSize { + end := min(i+batchSize, totalUsers) + + userBatch := st.CreatedUsers[i:end] + log.ZInfo(st.Ctx, "Creating user batch", "batch", i/batchSize+1, "count", len(userBatch)) + + err = st.CreateUserBatch(st.Ctx, userBatch) + if err != nil { + log.ZError(st.Ctx, "Batch user creation failed", err, "batch", i/batchSize+1) + } else { + successCount += len(userBatch) + log.ZInfo(st.Ctx, "Batch user creation succeeded", "batch", i/batchSize+1, + "progress", fmt.Sprintf("%d/%d", successCount, totalUsers)) + } + } + + // Execute create 100k group + st.Wg.Add(1) + go func() { + defer st.Wg.Done() + + create100kGroupTicker := time.NewTicker(Create100KGroupTicker) + defer create100kGroupTicker.Stop() + + for i := range Max100KGroup { + select { + case <-st.Ctx.Done(): + log.ZInfo(st.Ctx, "Stop Create 100K Group") + return + + case <-create100kGroupTicker.C: + // Create 100K groups + st.Wg.Add(1) + go func(idx int) { + defer st.Wg.Done() + defer func() { + st.Create100kGroupCounter++ + }() + + groupID := fmt.Sprintf("v2_StressTest_Group_100K_%d", idx) + + if _, err = st.CreateGroup(st.Ctx, groupID, st.DefaultUserID, TestTargetUserList); err != nil { + log.ZError(st.Ctx, "Create group failed.", err) + // continue + } + + for i := 0; i < MaxUser/MaxInviteUserLimit; i++ { + InviteUserIDs := make([]string, 0) + // ensure TargetUserList is in group + InviteUserIDs = append(InviteUserIDs, TestTargetUserList...) + + startIdx := max(i*MaxInviteUserLimit, 1) + endIdx := min((i+1)*MaxInviteUserLimit, MaxUser) + + for j := startIdx; j < endIdx; j++ { + userCreatedID := fmt.Sprintf("v2_StressTest_User_%d", j) + InviteUserIDs = append(InviteUserIDs, userCreatedID) + } + + if len(InviteUserIDs) == 0 { + log.ZWarn(st.Ctx, "InviteUserIDs is empty", nil, "groupID", groupID) + continue + } + + InviteUserIDs, err := st.GetGroupMembersInfo(ctx, groupID, InviteUserIDs) + if err != nil { + log.ZError(st.Ctx, "GetGroupMembersInfo failed.", err, "groupID", groupID) + continue + } + + if len(InviteUserIDs) == 0 { + log.ZWarn(st.Ctx, "InviteUserIDs is empty", nil, "groupID", groupID) + continue + } + + // Invite To Group + if err = st.InviteToGroup(st.Ctx, groupID, InviteUserIDs); err != nil { + log.ZError(st.Ctx, "Invite To Group failed.", err, "UserID", InviteUserIDs) + continue + // os.Exit(1) + // return + } + } + }(i) + } + } + }() + + // create 999 groups + st.Wg.Add(1) + go func() { + defer st.Wg.Done() + + create999GroupTicker := time.NewTicker(Create999GroupTicker) + defer create999GroupTicker.Stop() + + for i := range Max999Group { + select { + case <-st.Ctx.Done(): + log.ZInfo(st.Ctx, "Stop Create 999 Group") + return + + case <-create999GroupTicker.C: + // Create 999 groups + st.Wg.Add(1) + go func(idx int) { + defer st.Wg.Done() + defer func() { + st.Create999GroupCounter++ + }() + + groupID := fmt.Sprintf("v2_StressTest_Group_1K_%d", idx) + + if _, err = st.CreateGroup(st.Ctx, groupID, st.DefaultUserID, TestTargetUserList); err != nil { + log.ZError(st.Ctx, "Create group failed.", err) + // continue + } + for i := 0; i < MaxUser/MaxInviteUserLimit; i++ { + InviteUserIDs := make([]string, 0) + // ensure TargetUserList is in group + InviteUserIDs = append(InviteUserIDs, TestTargetUserList...) + + startIdx := max(i*MaxInviteUserLimit, 1) + endIdx := min((i+1)*MaxInviteUserLimit, MaxUser) + + for j := startIdx; j < endIdx; j++ { + userCreatedID := fmt.Sprintf("v2_StressTest_User_%d", j) + InviteUserIDs = append(InviteUserIDs, userCreatedID) + } + + if len(InviteUserIDs) == 0 { + log.ZWarn(st.Ctx, "InviteUserIDs is empty", nil, "groupID", groupID) + continue + } + + InviteUserIDs, err := st.GetGroupMembersInfo(ctx, groupID, InviteUserIDs) + if err != nil { + log.ZError(st.Ctx, "GetGroupMembersInfo failed.", err, "groupID", groupID) + continue + } + + if len(InviteUserIDs) == 0 { + log.ZWarn(st.Ctx, "InviteUserIDs is empty", nil, "groupID", groupID) + continue + } + + // Invite To Group + if err = st.InviteToGroup(st.Ctx, groupID, InviteUserIDs); err != nil { + log.ZError(st.Ctx, "Invite To Group failed.", err, "UserID", InviteUserIDs) + continue + // os.Exit(1) + // return + } + } + }(i) + } + } + }() + + // Send message to 100K groups + st.Wg.Wait() + fmt.Println("All groups created successfully, starting to send messages...") + log.ZInfo(ctx, "All groups created successfully, starting to send messages...") + + var groups100K []string + var groups999 []string + + for i := range Max100KGroup { + groupID := fmt.Sprintf("v2_StressTest_Group_100K_%d", i) + groups100K = append(groups100K, groupID) + } + + for i := range Max999Group { + groupID := fmt.Sprintf("v2_StressTest_Group_1K_%d", i) + groups999 = append(groups999, groupID) + } + + send100kGroupLimiter := make(chan struct{}, 20) + send999GroupLimiter := make(chan struct{}, 100) + + // execute Send message to 100K groups + go func() { + ticker := time.NewTicker(SendMsgTo100KGroupTicker) + defer ticker.Stop() + + for { + select { + case <-st.Ctx.Done(): + log.ZInfo(st.Ctx, "Stop Send Message to 100K Group") + return + + case <-ticker.C: + // Send message to 100K groups + for _, groupID := range groups100K { + send100kGroupLimiter <- struct{}{} + go func(groupID string) { + defer func() { <-send100kGroupLimiter }() + if err := st.SendMsg(st.Ctx, st.DefaultUserID, groupID); err != nil { + log.ZError(st.Ctx, "Send message to 100K group failed.", err) + } + }(groupID) + } + // log.ZInfo(st.Ctx, "Send message to 100K groups successfully.") + } + } + }() + + // execute Send message to 999 groups + go func() { + ticker := time.NewTicker(SendMsgTo999GroupTicker) + defer ticker.Stop() + + for { + select { + case <-st.Ctx.Done(): + log.ZInfo(st.Ctx, "Stop Send Message to 999 Group") + return + + case <-ticker.C: + // Send message to 999 groups + for _, groupID := range groups999 { + send999GroupLimiter <- struct{}{} + go func(groupID string) { + defer func() { <-send999GroupLimiter }() + + if err := st.SendMsg(st.Ctx, st.DefaultUserID, groupID); err != nil { + log.ZError(st.Ctx, "Send message to 999 group failed.", err) + } + }(groupID) + } + // log.ZInfo(st.Ctx, "Send message to 999 groups successfully.") + } + } + }() + + <-st.Ctx.Done() + fmt.Println("Received signal to exit, shutting down...") +} diff --git a/tools/stress-test/main.go b/tools/stress-test/main.go index aa52b69ed..6adbd12ee 100755 --- a/tools/stress-test/main.go +++ b/tools/stress-test/main.go @@ -72,22 +72,22 @@ type BaseResp struct { } type StressTest struct { - Conf *conf - AdminUserID string - AdminToken string - DefaultGroupID string - DefaultSendUserID string - UserCounter int - GroupCounter int - MsgCounter int - CreatedUsers []string - CreatedGroups []string - Mutex sync.Mutex - Ctx context.Context - Cancel context.CancelFunc - HttpClient *http.Client - Wg sync.WaitGroup - Once sync.Once + Conf *conf + AdminUserID string + AdminToken string + DefaultGroupID string + DefaultUserID string + UserCounter int + GroupCounter int + MsgCounter int + CreatedUsers []string + CreatedGroups []string + Mutex sync.Mutex + Ctx context.Context + Cancel context.CancelFunc + HttpClient *http.Client + Wg sync.WaitGroup + Once sync.Once } type conf struct { @@ -384,7 +384,6 @@ func main() { os.Exit(1) return } - // Invite To Group if err = st.InviteToGroup(st.Ctx, userCreatedID); err != nil { log.ZError(st.Ctx, "Invite To Group failed.", err, "UserID", userCreatedID) @@ -393,7 +392,7 @@ func main() { } st.Once.Do(func() { - st.DefaultSendUserID = userCreatedID + st.DefaultUserID = userCreatedID fmt.Println("Default Send User Created ID:", userCreatedID) close(ch) }) @@ -417,8 +416,8 @@ func main() { case <-ticker.C: // Send Message - if err = st.SendMsg(st.Ctx, st.DefaultSendUserID); err != nil { - log.ZError(st.Ctx, "Send Message failed.", err, "UserID", st.DefaultSendUserID) + if err = st.SendMsg(st.Ctx, st.DefaultUserID); err != nil { + log.ZError(st.Ctx, "Send Message failed.", err, "UserID", st.DefaultUserID) continue } } @@ -443,9 +442,9 @@ func main() { case <-ticker.C: // Create Group - _, err := st.CreateGroup(st.Ctx, st.DefaultSendUserID) + _, err := st.CreateGroup(st.Ctx, st.DefaultUserID) if err != nil { - log.ZError(st.Ctx, "Create Group failed.", err, "UserID", st.DefaultSendUserID) + log.ZError(st.Ctx, "Create Group failed.", err, "UserID", st.DefaultUserID) os.Exit(1) return } From 58994095a5e9ba78067dca16bf86e0d587dc6346 Mon Sep 17 00:00:00 2001 From: skiffer-git <72860476+skiffer-git@users.noreply.github.com> Date: Tue, 22 Apr 2025 16:48:12 +0800 Subject: [PATCH 149/199] License (#3293) * 3.6.1 code conventions (#2203) * Adjust configuration settings * Adjust configuration settings * Adjust configuration settings * refactor: webhooks update. * refactor: kafka update. * Simplify the Docker Compose configuration, remove unnecessary environment variables, and eliminate the gateway service. * refactor: kafka update. * refactor: kafka update. * Simplify the Docker Compose configuration, remove unnecessary environment variables, and eliminate the gateway service. * Simplify the Docker Compose configuration, remove unnecessary environment variables, and eliminate the gateway service. * Windows can compile and run. * Windows can compile and run. * refactor: kafka update. * feat: msg cache split * refactor: webhooks update * refactor: webhooks update * refactor: friends update * refactor: group update * refactor: third update * refactor: api update * refactor: crontab update * refactor: msggateway update * mage * mage * refactor: all module update. * check * refactor: all module update. * load config * load config * load config * load config * refactor: all module update. * refactor: all module update. * refactor: all module update. * refactor: all module update. * refactor: all module update. * Optimize Docker configuration and script. * refactor: all module update. * Optimize Docker configuration and script. * Optimize Docker configuration and script. * Optimize Docker configuration and script. * refactor: all module update. * Optimize Docker configuration and script. * refactor: all module update. * refactor: all module update. * Optimize Docker configuration and script. * Optimize Docker configuration and script. * Optimize Docker configuration and script. * Optimize Docker configuration and script. * Optimize Docker configuration and script. * Optimize Docker configuration and script. * update tools * update tools * Optimize Docker configuration and script. * Optimize Docker configuration and script. * Optimize Docker configuration and script. * Optimize Docker configuration and script. * Optimize Docker configuration and script. * Optimize Docker configuration and script. * Optimize Docker configuration and script. * Optimize Docker configuration and script. * Optimize Docker configuration and script. * Optimize Docker configuration and script. * Optimize Docker configuration and script. * update protocol * Optimize Docker configuration and script. * Optimize Docker configuration and script. * refactor: all module update. * Optimize Docker configuration and script. * Optimize Docker configuration and script. * Optimize Docker configuration and script. * Optimize Docker configuration and script. * refactor: api remove token auth by redis directly. * Code Refactoring * refactor: websocket auth change to call rpc of auth. * refactor: kick online user and remove token change to call auth rpc. * refactor: kick online user and remove token change to call auth rpc. * refactor: remove msggateway redis. * refactor: cmd update. * refactor: cmd update. * refactor: cmd update. * refactor: cmd update. * refactor: cmd update. * refactor: cmd update. * refactor: cmd update. * refactor: cmd update. * refactor: cmd update. * refactor: cmd update. * refactor: cmd update. * refactor: cmd update. * refactor: cmd update. * refactor: cmd update. * refactor: cmd update. * refactor: cmd update. * refactor: cmd update. * refactor: cmd update. * refactor: cmd update. * refactor: cmd update. * refactor: cmd update. * refactor: cmd update. * refactor: cmd update. * refactor: cmd update. * refactor: cmd update. * refactor: cmd update. * refactor webhook * refactor: cmd update. * refactor: cmd update. * refactor: cmd update. * refactor: cmd update. * refactor webhook * refactor: cmd update. * refactor: cmd update. * fix: runtime: goroutine stack exceeds * refactor: cmd update. * refactor notification * refactor notification * refactor * refactor: cmd update. * refactor: cmd update. * refactor * refactor * refactor * protojson * protojson * protojson * go mod * wrapperspb * refactor: cmd update. * refactor: cmd update. * refactor: cmd update. * refactor: context update. * refactor: websocket update info. * refactor: websocket update info. * refactor: websocket update info. * refactor: websocket update info. * refactor: api name change. * refactor: debug info. * refactor: debug info. * refactor: debug info. * fix: update file * refactor * refactor * refactor: debug info. * refactor: debug info. * refactor: debug info. * refactor: debug info. * refactor: debug info. * refactor: debug info. * fix: callback update. * fix: callback update. * refactor * fix: update message. * fix: msg cache timeout. * refactor * refactor * fix: push update. * fix: push update. * fix: push update. * fix: push update. * fix: push update. * fix: push update. * refactor * refactor * fix: push update. * fix: websocket handle error remove when upgrade error. * fix: priority url * fix: minio config * refactor: add zk logger. * refactor * fix: minio config * refactor * remove \r * remove \r * remove \r * remove \r * remove \r * remove \r * remove \r * remove \r * remove \r * remove \r * fix bug: get localIP * refactor * refactor * refactor * refactor: remove zk logger. * refactor: update tools version. * refactor * refactor: update server version to 3.7.0. * refactor * refactor * refactor * refactor * refactor * refactor * refactor * refactor * refactor * refactor * refactor * refactor * refactor * refactor: zk log debug. * refactor: zk log debug. * refactor: zk log debug. * refactor: zk log debug. * refactor: zk log debug. * refactor * refactor * refactor * refactor: log level change. * refactor: 3.7.0 code conventions. --------- Co-authored-by: skiffer-git <44203734@qq.com> Co-authored-by: withchao <993506633@qq.com> Co-authored-by: root * update go.mod go.sum (#2209) * remove \r * remove \r * remove \r * remove \r * remove \r * remove \r * remove \r * remove \r * remove \r * remove \r * fix bug: get localIP * update some ci file (#2200) * Update openimci.yml * Update golangci-lint.yml * Update e2e-test.yml * 3.6.1 code conventions (#2202) * refactor: webhooks update. * Adjust configuration settings * Adjust configuration settings * Adjust configuration settings * feat: s3 api addr * refactor: webhooks update. * Adjust configuration settings * Adjust configuration settings * Adjust configuration settings * Adjust configuration settings * Adjust configuration settings * Adjust configuration settings * Adjust configuration settings * refactor: webhooks update. * refactor: kafka update. * Simplify the Docker Compose configuration, remove unnecessary environment variables, and eliminate the gateway service. * refactor: kafka update. * refactor: kafka update. * Simplify the Docker Compose configuration, remove unnecessary environment variables, and eliminate the gateway service. * Simplify the Docker Compose configuration, remove unnecessary environment variables, and eliminate the gateway service. * Windows can compile and run. * Windows can compile and run. * refactor: kafka update. * feat: msg cache split * refactor: webhooks update * refactor: webhooks update * refactor: friends update * refactor: group update * refactor: third update * refactor: api update * refactor: crontab update * refactor: msggateway update * mage * mage * refactor: all module update. * check * refactor: all module update. * load config * load config * load config * load config * refactor: all module update. * refactor: all module update. * refactor: all module update. * refactor: all module update. * refactor: all module update. * Optimize Docker configuration and script. * refactor: all module update. * Optimize Docker configuration and script. * Optimize Docker configuration and script. * Optimize Docker configuration and script. * refactor: all module update. * Optimize Docker configuration and script. * refactor: all module update. * refactor: all module update. * Optimize Docker configuration and script. * Optimize Docker configuration and script. * Optimize Docker configuration and script. * Optimize Docker configuration and script. * Optimize Docker configuration and script. * Optimize Docker configuration and script. * update tools * update tools * Optimize Docker configuration and script. * Optimize Docker configuration and script. * Optimize Docker configuration and script. * Optimize Docker configuration and script. * Optimize Docker configuration and script. * Optimize Docker configuration and script. * Optimize Docker configuration and script. * Optimize Docker configuration and script. * Optimize Docker configuration and script. * Optimize Docker configuration and script. * Optimize Docker configuration and script. * update protocol * Optimize Docker configuration and script. * Optimize Docker configuration and script. * refactor: all module update. * Optimize Docker configuration and script. * Optimize Docker configuration and script. * Optimize Docker configuration and script. * Optimize Docker configuration and script. * refactor: api remove token auth by redis directly. * Code Refactoring * refactor: websocket auth change to call rpc of auth. * refactor: kick online user and remove token change to call auth rpc. * refactor: kick online user and remove token change to call auth rpc. * refactor: remove msggateway redis. * refactor: cmd update. * refactor: cmd update. * refactor: cmd update. * refactor: cmd update. * refactor: cmd update. * refactor: cmd update. * refactor: cmd update. * refactor: cmd update. * refactor: cmd update. * refactor: cmd update. * refactor: cmd update. * refactor: cmd update. * refactor: cmd update. * refactor: cmd update. * refactor: cmd update. * refactor: cmd update. * refactor: cmd update. * refactor: cmd update. * refactor: cmd update. * refactor: cmd update. * refactor: cmd update. * refactor: cmd update. * refactor: cmd update. * refactor: cmd update. * refactor: cmd update. * refactor: cmd update. * refactor webhook * refactor: cmd update. * refactor: cmd update. * refactor: cmd update. * refactor: cmd update. * refactor webhook * refactor: cmd update. * refactor: cmd update. * fix: runtime: goroutine stack exceeds * refactor: cmd update. * refactor notification * refactor notification * refactor * refactor: cmd update. * refactor: cmd update. * refactor * refactor * refactor * protojson * protojson * protojson * go mod * wrapperspb * refactor: cmd update. * refactor: cmd update. * refactor: cmd update. * refactor: context update. * refactor: websocket update info. * refactor: websocket update info. * refactor: websocket update info. * refactor: websocket update info. * refactor: api name change. * refactor: debug info. * refactor: debug info. * refactor: debug info. * fix: update file * refactor * refactor * refactor: debug info. * refactor: debug info. * refactor: debug info. * refactor: debug info. * refactor: debug info. * refactor: debug info. * fix: callback update. * fix: callback update. * refactor * fix: update message. * fix: msg cache timeout. * refactor * refactor * fix: push update. * fix: push update. * fix: push update. * fix: push update. * fix: push update. * fix: push update. * refactor * refactor * fix: push update. * fix: websocket handle error remove when upgrade error. * fix: priority url * fix: minio config * refactor: add zk logger. * refactor * fix: minio config * refactor * refactor * refactor * refactor * refactor: remove zk logger. * refactor: update tools version. * refactor * refactor: update server version to 3.7.0. * refactor * refactor * refactor * refactor * refactor * refactor * refactor * refactor * refactor * refactor * refactor * refactor * refactor * refactor: zk log debug. * refactor: zk log debug. * refactor: zk log debug. * refactor: zk log debug. * refactor: zk log debug. * refactor * refactor * refactor * refactor: log level change. * refactor: 3.7.0 code conventions. --------- Co-authored-by: skiffer-git <44203734@qq.com> Co-authored-by: withchao <993506633@qq.com> * update go.mod go.sum * Remove Chinese comments * user localhost for minio * user localhost for minio * Remove Chinese comments * Remove Chinese comments * Remove Chinese comments * Set up 4 instances of transfer * Set up 4 instances of transfer * Add comments to the configuration file * Add comments to the configuration file --------- Co-authored-by: root Co-authored-by: xuan <146319162+wxuanF@users.noreply.github.com> Co-authored-by: OpenIM-Gordon <46924906+FGadvancer@users.noreply.github.com> Co-authored-by: withchao <993506633@qq.com> * Update the document (#2221) * Update the document * Update the document * use openim/openim-admin openim/openim-web image * Update .golangci.yml * Add etcd as a service discovery mechanism * Add etcd as a service discovery mechanism * update * update license * update license * update license * update license * update license --------- Co-authored-by: OpenIM-Gordon <46924906+FGadvancer@users.noreply.github.com> Co-authored-by: withchao <993506633@qq.com> Co-authored-by: root Co-authored-by: xuan <146319162+wxuanF@users.noreply.github.com> --- .golangci.yml | 2 ++ README.md | 9 ++++++++- README_zh_CN.md | 12 ++++++++++-- 3 files changed, 20 insertions(+), 3 deletions(-) diff --git a/.golangci.yml b/.golangci.yml index a95e980f8..7d6c6b596 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -28,6 +28,8 @@ run: # - util # - .*~ # - api/swagger/docs + + # - server/docs # - components/mnt/config/certs # - logs diff --git a/README.md b/README.md index 3b88935eb..a23c72c61 100644 --- a/README.md +++ b/README.md @@ -132,7 +132,14 @@ Thank you for contributing to building a powerful instant messaging solution! ## :closed_book: License -For more details, please refer to [here](./LICENSE). +This software is licensed under a dual-license model: + +- The GNU Affero General Public License (AGPL), Version 3 or later; **OR** +- Commercial license terms from OpenIMSDK. + +If you wish to use this software under commercial terms, please contact us at: contact@openim.io + +For more information, see: https://www.openim.io/en/licensing diff --git a/README_zh_CN.md b/README_zh_CN.md index 59198eafb..2340ad09a 100644 --- a/README_zh_CN.md +++ b/README_zh_CN.md @@ -131,9 +131,17 @@ 感谢您的贡献，一起来打造强大的即时通讯解决方案！ -## :closed_book: 许可证 +## :closed_book: 开源许可证 License + +本软件采用双重授权模型： + +GNU Affero 通用公共许可证（AGPL）第 3 版或更高版本；或 + +来自 OpenIMSDK 的商业授权条款。 + +如需商用，请联系：contact@openim.io +详见：https://www.openim.io/en/licensing - OpenIMSDK 在 Apache License 2.0 许可下可用。查看[LICENSE 文件](https://github.com/openimsdk/open-im-server/blob/main/LICENSE)了解更多信息。 ## 🔮 Thanks to our contributors! From fa3d251dcb3bd063289b37e3b8530ff53c6a6af8 Mon Sep 17 00:00:00 2001 From: chao <48119764+withchao@users.noreply.github.com> Date: Tue, 22 Apr 2025 17:02:45 +0800 Subject: [PATCH 150/199] feat: GroupApplicationAgreeMemberEnterNotification splitting (#3297) * pb * fix: Modifying other fields while setting IsPrivateChat does not take effect * fix: quote message error revoke * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * upgrading pkg tools * fix * fix * optimize log output * feat: support GetLastMessage * feat: support GetLastMessage * feat: s3 switch * feat: s3 switch * fix: GetUsersOnline * feat: SendBusinessNotification supported configuration parameters * feat: SendBusinessNotification supported configuration parameters * feat: SendBusinessNotification supported configuration parameters * feat: seq conversion failed without exiting * fix: DeleteDoc crash * fix: fill send time * fix: fill send time * fix: crash caused by withdrawing messages from users who have left the group * fix: user msg timestamp * seq read config * seq read config * fix: the source message of the reference is withdrawn, and the referenced message is deleted * feat: optimize the default notification.yml * fix: shouldPushOffline * fix: the sorting is wrong after canceling the administrator in group settings * feat: Sending messages supports returning fields modified by webhook * feat: Sending messages supports returning fields modified by webhook * feat: Sending messages supports returning fields modified by webhook * fix: oss specifies content-type when uploading * fix: the version number contains a line break * fix: the version number contains a line break * feat: GetConversationsHasReadAndMaxSeq support pinned * feat: GetConversationsHasReadAndMaxSeq support pinned * feat: GetConversationsHasReadAndMaxSeq support pinned * fix: transferring the group owner to a muted member, incremental version error * feat: GroupApplicationAgreeMemberEnterNotification splitting, rpc body size limit * feat: GroupApplicationAgreeMemberEnterNotification splitting, rpc body size limit --- config/share.yml | 4 +++ internal/rpc/group/group.go | 23 ++++++++++--- internal/rpc/group/notification.go | 4 +++ pkg/common/config/config.go | 12 +++++-- pkg/common/startrpc/start.go | 53 +++++++++++++++++++++++++++++- 5 files changed, 87 insertions(+), 9 deletions(-) diff --git a/config/share.yml b/config/share.yml index 0913c1e88..2e9821436 100644 --- a/config/share.yml +++ b/config/share.yml @@ -7,3 +7,7 @@ multiLogin: policy: 1 # max num of tokens in one end maxNumOneEnd: 30 + +rpcMaxBodySize: + requestMaxBodySize: 8388608 + responseMaxBodySize: 8388608 diff --git a/internal/rpc/group/group.go b/internal/rpc/group/group.go index 63d9336d5..455b77635 100644 --- a/internal/rpc/group/group.go +++ b/internal/rpc/group/group.go @@ -448,12 +448,25 @@ func (g *groupServer) InviteUserToGroup(ctx context.Context, req *pbgroup.Invite return nil, err } - if err := g.db.CreateGroup(ctx, nil, groupMembers); err != nil { - return nil, err - } + const singleQuantity = 50 + for start := 0; start < len(groupMembers); start += singleQuantity { + end := start + singleQuantity + if end > len(groupMembers) { + end = len(groupMembers) + } + currentMembers := groupMembers[start:end] - if err = g.notification.GroupApplicationAgreeMemberEnterNotification(ctx, req.GroupID, req.SendMessage, opUserID, req.InvitedUserIDs...); err != nil { - return nil, err + if err := g.db.CreateGroup(ctx, nil, currentMembers); err != nil { + return nil, err + } + + userIDs := datautil.Slice(currentMembers, func(e *model.GroupMember) string { + return e.UserID + }) + + if err = g.notification.GroupApplicationAgreeMemberEnterNotification(ctx, req.GroupID, req.SendMessage, opUserID, userIDs...); err != nil { + return nil, err + } } return &pbgroup.InviteUserToGroupResp{}, nil } diff --git a/internal/rpc/group/notification.go b/internal/rpc/group/notification.go index f9da88863..43becf4f2 100644 --- a/internal/rpc/group/notification.go +++ b/internal/rpc/group/notification.go @@ -522,6 +522,10 @@ func (g *NotificationSender) MemberKickedNotification(ctx context.Context, tips } func (g *NotificationSender) GroupApplicationAgreeMemberEnterNotification(ctx context.Context, groupID string, SendMessage *bool, invitedOpUserID string, entrantUserID ...string) error { + return g.groupApplicationAgreeMemberEnterNotification(ctx, groupID, SendMessage, invitedOpUserID, entrantUserID...) +} + +func (g *NotificationSender) groupApplicationAgreeMemberEnterNotification(ctx context.Context, groupID string, SendMessage *bool, invitedOpUserID string, entrantUserID ...string) error { var err error defer func() { if err != nil { diff --git a/pkg/common/config/config.go b/pkg/common/config/config.go index fa93e3406..d5ae68ec0 100644 --- a/pkg/common/config/config.go +++ b/pkg/common/config/config.go @@ -348,9 +348,15 @@ type AfterConfig struct { } type Share struct { - Secret string `yaml:"secret"` - IMAdminUserID []string `yaml:"imAdminUserID"` - MultiLogin MultiLogin `yaml:"multiLogin"` + Secret string `yaml:"secret"` + IMAdminUserID []string `yaml:"imAdminUserID"` + MultiLogin MultiLogin `yaml:"multiLogin"` + RPCMaxBodySize MaxRequestBody `yaml:"rpcMaxBodySize"` +} + +type MaxRequestBody struct { + RequestMaxBodySize int `yaml:"requestMaxBodySize"` + ResponseMaxBodySize int `yaml:"responseMaxBodySize"` } type MultiLogin struct { diff --git a/pkg/common/startrpc/start.go b/pkg/common/startrpc/start.go index a69edae20..af50c408d 100644 --- a/pkg/common/startrpc/start.go +++ b/pkg/common/startrpc/start.go @@ -21,6 +21,7 @@ import ( "net" "os" "os/signal" + "reflect" "strconv" "syscall" "time" @@ -45,6 +46,36 @@ func init() { prommetrics.RegistryAll() } +func getConfigRpcMaxRequestBody(value reflect.Value) *conf.MaxRequestBody { + for value.Kind() == reflect.Pointer { + value = value.Elem() + } + if value.Kind() == reflect.Struct { + num := value.NumField() + for i := 0; i < num; i++ { + field := value.Field(i) + if !field.CanInterface() { + continue + } + for field.Kind() == reflect.Pointer { + field = field.Elem() + } + switch elem := field.Interface().(type) { + case conf.Share: + return &elem.RPCMaxBodySize + case conf.MaxRequestBody: + return &elem + } + if field.Kind() == reflect.Struct { + if elem := getConfigRpcMaxRequestBody(field); elem != nil { + return elem + } + } + } + } + return nil +} + func Start[T any](ctx context.Context, disc *conf.Discovery, prometheusConfig *conf.Prometheus, listenIP, registerIP string, autoSetPorts bool, rpcPorts []int, index int, rpcRegisterName string, notification *conf.Notification, config T, watchConfigNames []string, watchServiceNames []string, @@ -55,7 +86,24 @@ func Start[T any](ctx context.Context, disc *conf.Discovery, prometheusConfig *c conf.InitNotification(notification) } - options = append(options, mw.GrpcServer()) + maxRequestBody := getConfigRpcMaxRequestBody(reflect.ValueOf(config)) + + log.ZDebug(ctx, "rpc start", "rpcMaxRequestBody", maxRequestBody, "rpcRegisterName", rpcRegisterName, "registerIP", registerIP, "listenIP", listenIP) + + options = append(options, + mw.GrpcServer(), + ) + var clientOptions []grpc.DialOption + if maxRequestBody != nil { + if maxRequestBody.RequestMaxBodySize > 0 { + options = append(options, grpc.MaxRecvMsgSize(maxRequestBody.RequestMaxBodySize)) + clientOptions = append(clientOptions, grpc.WithDefaultCallOptions(grpc.MaxCallSendMsgSize(maxRequestBody.RequestMaxBodySize))) + } + if maxRequestBody.ResponseMaxBodySize > 0 { + options = append(options, grpc.MaxSendMsgSize(maxRequestBody.ResponseMaxBodySize)) + clientOptions = append(clientOptions, grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(maxRequestBody.ResponseMaxBodySize))) + } + } registerIP, err := network.GetRpcRegisterIP(registerIP) if err != nil { @@ -84,6 +132,9 @@ func Start[T any](ctx context.Context, disc *conf.Discovery, prometheusConfig *c mw.GrpcClient(), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithDefaultServiceConfig(fmt.Sprintf(`{"LoadBalancingPolicy": "%s"}`, "round_robin")), ) + if len(clientOptions) > 0 { + client.AddOption(clientOptions...) + } ctx, cancel := context.WithCancelCause(ctx) From c54b948f1a6b35a4aa1f2f924443ba44524b2c84 Mon Sep 17 00:00:00 2001 From: skiffer-git <72860476+skiffer-git@users.noreply.github.com> Date: Tue, 22 Apr 2025 21:47:13 +0800 Subject: [PATCH 151/199] Update LICENSE --- LICENSE | 862 +++++++++++++------------------------------------------- 1 file changed, 201 insertions(+), 661 deletions(-) diff --git a/LICENSE b/LICENSE index 0ad25db4b..261eeb9e9 100644 --- a/LICENSE +++ b/LICENSE @@ -1,661 +1,201 @@ - GNU AFFERO GENERAL PUBLIC LICENSE - Version 3, 19 November 2007 - - Copyright (C) 2007 Free Software Foundation, Inc. - Everyone is permitted to copy and distribute verbatim copies - of this license document, but changing it is not allowed. - - Preamble - - The GNU Affero General Public License is a free, copyleft license for -software and other kinds of works, specifically designed to ensure -cooperation with the community in the case of network server software. - - The licenses for most software and other practical works are designed -to take away your freedom to share and change the works. By contrast, -our General Public Licenses are intended to guarantee your freedom to -share and change all versions of a program--to make sure it remains free -software for all its users. - - When we speak of free software, we are referring to freedom, not -price. Our General Public Licenses are designed to make sure that you -have the freedom to distribute copies of free software (and charge for -them if you wish), that you receive source code or can get it if you -want it, that you can change the software or use pieces of it in new -free programs, and that you know you can do these things. - - Developers that use our General Public Licenses protect your rights -with two steps: (1) assert copyright on the software, and (2) offer -you this License which gives you legal permission to copy, distribute -and/or modify the software. - - A secondary benefit of defending all users' freedom is that -improvements made in alternate versions of the program, if they -receive widespread use, become available for other developers to -incorporate. Many developers of free software are heartened and -encouraged by the resulting cooperation. However, in the case of -software used on network servers, this result may fail to come about. -The GNU General Public License permits making a modified version and -letting the public access it on a server without ever releasing its -source code to the public. - - The GNU Affero General Public License is designed specifically to -ensure that, in such cases, the modified source code becomes available -to the community. It requires the operator of a network server to -provide the source code of the modified version running there to the -users of that server. Therefore, public use of a modified version, on -a publicly accessible server, gives the public access to the source -code of the modified version. - - An older license, called the Affero General Public License and -published by Affero, was designed to accomplish similar goals. This is -a different license, not a version of the Affero GPL, but Affero has -released a new version of the Affero GPL which permits relicensing under -this license. - - The precise terms and conditions for copying, distribution and -modification follow. - - TERMS AND CONDITIONS - - 0. Definitions. - - "This License" refers to version 3 of the GNU Affero General Public License. - - "Copyright" also means copyright-like laws that apply to other kinds of -works, such as semiconductor masks. - - "The Program" refers to any copyrightable work licensed under this -License. Each licensee is addressed as "you". "Licensees" and -"recipients" may be individuals or organizations. - - To "modify" a work means to copy from or adapt all or part of the work -in a fashion requiring copyright permission, other than the making of an -exact copy. The resulting work is called a "modified version" of the -earlier work or a work "based on" the earlier work. - - A "covered work" means either the unmodified Program or a work based -on the Program. - - To "propagate" a work means to do anything with it that, without -permission, would make you directly or secondarily liable for -infringement under applicable copyright law, except executing it on a -computer or modifying a private copy. Propagation includes copying, -distribution (with or without modification), making available to the -public, and in some countries other activities as well. - - To "convey" a work means any kind of propagation that enables other -parties to make or receive copies. Mere interaction with a user through -a computer network, with no transfer of a copy, is not conveying. - - An interactive user interface displays "Appropriate Legal Notices" -to the extent that it includes a convenient and prominently visible -feature that (1) displays an appropriate copyright notice, and (2) -tells the user that there is no warranty for the work (except to the -extent that warranties are provided), that licensees may convey the -work under this License, and how to view a copy of this License. If -the interface presents a list of user commands or options, such as a -menu, a prominent item in the list meets this criterion. - - 1. Source Code. - - The "source code" for a work means the preferred form of the work -for making modifications to it. "Object code" means any non-source -form of a work. - - A "Standard Interface" means an interface that either is an official -standard defined by a recognized standards body, or, in the case of -interfaces specified for a particular programming language, one that -is widely used among developers working in that language. - - The "System Libraries" of an executable work include anything, other -than the work as a whole, that (a) is included in the normal form of -packaging a Major Component, but which is not part of that Major -Component, and (b) serves only to enable use of the work with that -Major Component, or to implement a Standard Interface for which an -implementation is available to the public in source code form. A -"Major Component", in this context, means a major essential component -(kernel, window system, and so on) of the specific operating system -(if any) on which the executable work runs, or a compiler used to -produce the work, or an object code interpreter used to run it. - - The "Corresponding Source" for a work in object code form means all -the source code needed to generate, install, and (for an executable -work) run the object code and to modify the work, including scripts to -control those activities. However, it does not include the work's -System Libraries, or general-purpose tools or generally available free -programs which are used unmodified in performing those activities but -which are not part of the work. For example, Corresponding Source -includes interface definition files associated with source files for -the work, and the source code for shared libraries and dynamically -linked subprograms that the work is specifically designed to require, -such as by intimate data communication or control flow between those -subprograms and other parts of the work. - - The Corresponding Source need not include anything that users -can regenerate automatically from other parts of the Corresponding -Source. - - The Corresponding Source for a work in source code form is that -same work. - - 2. Basic Permissions. - - All rights granted under this License are granted for the term of -copyright on the Program, and are irrevocable provided the stated -conditions are met. This License explicitly affirms your unlimited -permission to run the unmodified Program. The output from running a -covered work is covered by this License only if the output, given its -content, constitutes a covered work. This License acknowledges your -rights of fair use or other equivalent, as provided by copyright law. - - You may make, run and propagate covered works that you do not -convey, without conditions so long as your license otherwise remains -in force. You may convey covered works to others for the sole purpose -of having them make modifications exclusively for you, or provide you -with facilities for running those works, provided that you comply with -the terms of this License in conveying all material for which you do -not control copyright. Those thus making or running the covered works -for you must do so exclusively on your behalf, under your direction -and control, on terms that prohibit them from making any copies of -your copyrighted material outside their relationship with you. - - Conveying under any other circumstances is permitted solely under -the conditions stated below. Sublicensing is not allowed; section 10 -makes it unnecessary. - - 3. Protecting Users' Legal Rights From Anti-Circumvention Law. - - No covered work shall be deemed part of an effective technological -measure under any applicable law fulfilling obligations under article -11 of the WIPO copyright treaty adopted on 20 December 1996, or -similar laws prohibiting or restricting circumvention of such -measures. - - When you convey a covered work, you waive any legal power to forbid -circumvention of technological measures to the extent such circumvention -is effected by exercising rights under this License with respect to -the covered work, and you disclaim any intention to limit operation or -modification of the work as a means of enforcing, against the work's -users, your or third parties' legal rights to forbid circumvention of -technological measures. - - 4. Conveying Verbatim Copies. - - You may convey verbatim copies of the Program's source code as you -receive it, in any medium, provided that you conspicuously and -appropriately publish on each copy an appropriate copyright notice; -keep intact all notices stating that this License and any -non-permissive terms added in accord with section 7 apply to the code; -keep intact all notices of the absence of any warranty; and give all -recipients a copy of this License along with the Program. - - You may charge any price or no price for each copy that you convey, -and you may offer support or warranty protection for a fee. - - 5. Conveying Modified Source Versions. - - You may convey a work based on the Program, or the modifications to -produce it from the Program, in the form of source code under the -terms of section 4, provided that you also meet all of these conditions: - - a) The work must carry prominent notices stating that you modified - it, and giving a relevant date. - - b) The work must carry prominent notices stating that it is - released under this License and any conditions added under section - 7. This requirement modifies the requirement in section 4 to - "keep intact all notices". - - c) You must license the entire work, as a whole, under this - License to anyone who comes into possession of a copy. This - License will therefore apply, along with any applicable section 7 - additional terms, to the whole of the work, and all its parts, - regardless of how they are packaged. This License gives no - permission to license the work in any other way, but it does not - invalidate such permission if you have separately received it. - - d) If the work has interactive user interfaces, each must display - Appropriate Legal Notices; however, if the Program has interactive - interfaces that do not display Appropriate Legal Notices, your - work need not make them do so. - - A compilation of a covered work with other separate and independent -works, which are not by their nature extensions of the covered work, -and which are not combined with it such as to form a larger program, -in or on a volume of a storage or distribution medium, is called an -"aggregate" if the compilation and its resulting copyright are not -used to limit the access or legal rights of the compilation's users -beyond what the individual works permit. Inclusion of a covered work -in an aggregate does not cause this License to apply to the other -parts of the aggregate. - - 6. Conveying Non-Source Forms. - - You may convey a covered work in object code form under the terms -of sections 4 and 5, provided that you also convey the -machine-readable Corresponding Source under the terms of this License, -in one of these ways: - - a) Convey the object code in, or embodied in, a physical product - (including a physical distribution medium), accompanied by the - Corresponding Source fixed on a durable physical medium - customarily used for software interchange. - - b) Convey the object code in, or embodied in, a physical product - (including a physical distribution medium), accompanied by a - written offer, valid for at least three years and valid for as - long as you offer spare parts or customer support for that product - model, to give anyone who possesses the object code either (1) a - copy of the Corresponding Source for all the software in the - product that is covered by this License, on a durable physical - medium customarily used for software interchange, for a price no - more than your reasonable cost of physically performing this - conveying of source, or (2) access to copy the - Corresponding Source from a network server at no charge. - - c) Convey individual copies of the object code with a copy of the - written offer to provide the Corresponding Source. This - alternative is allowed only occasionally and noncommercially, and - only if you received the object code with such an offer, in accord - with subsection 6b. - - d) Convey the object code by offering access from a designated - place (gratis or for a charge), and offer equivalent access to the - Corresponding Source in the same way through the same place at no - further charge. You need not require recipients to copy the - Corresponding Source along with the object code. If the place to - copy the object code is a network server, the Corresponding Source - may be on a different server (operated by you or a third party) - that supports equivalent copying facilities, provided you maintain - clear directions next to the object code saying where to find the - Corresponding Source. Regardless of what server hosts the - Corresponding Source, you remain obligated to ensure that it is - available for as long as needed to satisfy these requirements. - - e) Convey the object code using peer-to-peer transmission, provided - you inform other peers where the object code and Corresponding - Source of the work are being offered to the general public at no - charge under subsection 6d. - - A separable portion of the object code, whose source code is excluded -from the Corresponding Source as a System Library, need not be -included in conveying the object code work. - - A "User Product" is either (1) a "consumer product", which means any -tangible personal property which is normally used for personal, family, -or household purposes, or (2) anything designed or sold for incorporation -into a dwelling. In determining whether a product is a consumer product, -doubtful cases shall be resolved in favor of coverage. For a particular -product received by a particular user, "normally used" refers to a -typical or common use of that class of product, regardless of the status -of the particular user or of the way in which the particular user -actually uses, or expects or is expected to use, the product. A product -is a consumer product regardless of whether the product has substantial -commercial, industrial or non-consumer uses, unless such uses represent -the only significant mode of use of the product. - - "Installation Information" for a User Product means any methods, -procedures, authorization keys, or other information required to install -and execute modified versions of a covered work in that User Product from -a modified version of its Corresponding Source. The information must -suffice to ensure that the continued functioning of the modified object -code is in no case prevented or interfered with solely because -modification has been made. - - If you convey an object code work under this section in, or with, or -specifically for use in, a User Product, and the conveying occurs as -part of a transaction in which the right of possession and use of the -User Product is transferred to the recipient in perpetuity or for a -fixed term (regardless of how the transaction is characterized), the -Corresponding Source conveyed under this section must be accompanied -by the Installation Information. But this requirement does not apply -if neither you nor any third party retains the ability to install -modified object code on the User Product (for example, the work has -been installed in ROM). - - The requirement to provide Installation Information does not include a -requirement to continue to provide support service, warranty, or updates -for a work that has been modified or installed by the recipient, or for -the User Product in which it has been modified or installed. Access to a -network may be denied when the modification itself materially and -adversely affects the operation of the network or violates the rules and -protocols for communication across the network. - - Corresponding Source conveyed, and Installation Information provided, -in accord with this section must be in a format that is publicly -documented (and with an implementation available to the public in -source code form), and must require no special password or key for -unpacking, reading or copying. - - 7. Additional Terms. - - "Additional permissions" are terms that supplement the terms of this -License by making exceptions from one or more of its conditions. -Additional permissions that are applicable to the entire Program shall -be treated as though they were included in this License, to the extent -that they are valid under applicable law. If additional permissions -apply only to part of the Program, that part may be used separately -under those permissions, but the entire Program remains governed by -this License without regard to the additional permissions. - - When you convey a copy of a covered work, you may at your option -remove any additional permissions from that copy, or from any part of -it. (Additional permissions may be written to require their own -removal in certain cases when you modify the work.) You may place -additional permissions on material, added by you to a covered work, -for which you have or can give appropriate copyright permission. - - Notwithstanding any other provision of this License, for material you -add to a covered work, you may (if authorized by the copyright holders of -that material) supplement the terms of this License with terms: - - a) Disclaiming warranty or limiting liability differently from the - terms of sections 15 and 16 of this License; or - - b) Requiring preservation of specified reasonable legal notices or - author attributions in that material or in the Appropriate Legal - Notices displayed by works containing it; or - - c) Prohibiting misrepresentation of the origin of that material, or - requiring that modified versions of such material be marked in - reasonable ways as different from the original version; or - - d) Limiting the use for publicity purposes of names of licensors or - authors of the material; or - - e) Declining to grant rights under trademark law for use of some - trade names, trademarks, or service marks; or - - f) Requiring indemnification of licensors and authors of that - material by anyone who conveys the material (or modified versions of - it) with contractual assumptions of liability to the recipient, for - any liability that these contractual assumptions directly impose on - those licensors and authors. - - All other non-permissive additional terms are considered "further -restrictions" within the meaning of section 10. If the Program as you -received it, or any part of it, contains a notice stating that it is -governed by this License along with a term that is a further -restriction, you may remove that term. If a license document contains -a further restriction but permits relicensing or conveying under this -License, you may add to a covered work material governed by the terms -of that license document, provided that the further restriction does -not survive such relicensing or conveying. - - If you add terms to a covered work in accord with this section, you -must place, in the relevant source files, a statement of the -additional terms that apply to those files, or a notice indicating -where to find the applicable terms. - - Additional terms, permissive or non-permissive, may be stated in the -form of a separately written license, or stated as exceptions; -the above requirements apply either way. - - 8. Termination. - - You may not propagate or modify a covered work except as expressly -provided under this License. Any attempt otherwise to propagate or -modify it is void, and will automatically terminate your rights under -this License (including any patent licenses granted under the third -paragraph of section 11). - - However, if you cease all violation of this License, then your -license from a particular copyright holder is reinstated (a) -provisionally, unless and until the copyright holder explicitly and -finally terminates your license, and (b) permanently, if the copyright -holder fails to notify you of the violation by some reasonable means -prior to 60 days after the cessation. - - Moreover, your license from a particular copyright holder is -reinstated permanently if the copyright holder notifies you of the -violation by some reasonable means, this is the first time you have -received notice of violation of this License (for any work) from that -copyright holder, and you cure the violation prior to 30 days after -your receipt of the notice. - - Termination of your rights under this section does not terminate the -licenses of parties who have received copies or rights from you under -this License. If your rights have been terminated and not permanently -reinstated, you do not qualify to receive new licenses for the same -material under section 10. - - 9. Acceptance Not Required for Having Copies. - - You are not required to accept this License in order to receive or -run a copy of the Program. Ancillary propagation of a covered work -occurring solely as a consequence of using peer-to-peer transmission -to receive a copy likewise does not require acceptance. However, -nothing other than this License grants you permission to propagate or -modify any covered work. These actions infringe copyright if you do -not accept this License. Therefore, by modifying or propagating a -covered work, you indicate your acceptance of this License to do so. - - 10. Automatic Licensing of Downstream Recipients. - - Each time you convey a covered work, the recipient automatically -receives a license from the original licensors, to run, modify and -propagate that work, subject to this License. You are not responsible -for enforcing compliance by third parties with this License. - - An "entity transaction" is a transaction transferring control of an -organization, or substantially all assets of one, or subdividing an -organization, or merging organizations. If propagation of a covered -work results from an entity transaction, each party to that -transaction who receives a copy of the work also receives whatever -licenses to the work the party's predecessor in interest had or could -give under the previous paragraph, plus a right to possession of the -Corresponding Source of the work from the predecessor in interest, if -the predecessor has it or can get it with reasonable efforts. - - You may not impose any further restrictions on the exercise of the -rights granted or affirmed under this License. For example, you may -not impose a license fee, royalty, or other charge for exercise of -rights granted under this License, and you may not initiate litigation -(including a cross-claim or counterclaim in a lawsuit) alleging that -any patent claim is infringed by making, using, selling, offering for -sale, or importing the Program or any portion of it. - - 11. Patents. - - A "contributor" is a copyright holder who authorizes use under this -License of the Program or a work on which the Program is based. The -work thus licensed is called the contributor's "contributor version". - - A contributor's "essential patent claims" are all patent claims -owned or controlled by the contributor, whether already acquired or -hereafter acquired, that would be infringed by some manner, permitted -by this License, of making, using, or selling its contributor version, -but do not include claims that would be infringed only as a -consequence of further modification of the contributor version. For -purposes of this definition, "control" includes the right to grant -patent sublicenses in a manner consistent with the requirements of -this License. - - Each contributor grants you a non-exclusive, worldwide, royalty-free -patent license under the contributor's essential patent claims, to -make, use, sell, offer for sale, import and otherwise run, modify and -propagate the contents of its contributor version. - - In the following three paragraphs, a "patent license" is any express -agreement or commitment, however denominated, not to enforce a patent -(such as an express permission to practice a patent or covenant not to -sue for patent infringement). To "grant" such a patent license to a -party means to make such an agreement or commitment not to enforce a -patent against the party. - - If you convey a covered work, knowingly relying on a patent license, -and the Corresponding Source of the work is not available for anyone -to copy, free of charge and under the terms of this License, through a -publicly available network server or other readily accessible means, -then you must either (1) cause the Corresponding Source to be so -available, or (2) arrange to deprive yourself of the benefit of the -patent license for this particular work, or (3) arrange, in a manner -consistent with the requirements of this License, to extend the patent -license to downstream recipients. "Knowingly relying" means you have -actual knowledge that, but for the patent license, your conveying the -covered work in a country, or your recipient's use of the covered work -in a country, would infringe one or more identifiable patents in that -country that you have reason to believe are valid. - - If, pursuant to or in connection with a single transaction or -arrangement, you convey, or propagate by procuring conveyance of, a -covered work, and grant a patent license to some of the parties -receiving the covered work authorizing them to use, propagate, modify -or convey a specific copy of the covered work, then the patent license -you grant is automatically extended to all recipients of the covered -work and works based on it. - - A patent license is "discriminatory" if it does not include within -the scope of its coverage, prohibits the exercise of, or is -conditioned on the non-exercise of one or more of the rights that are -specifically granted under this License. You may not convey a covered -work if you are a party to an arrangement with a third party that is -in the business of distributing software, under which you make payment -to the third party based on the extent of your activity of conveying -the work, and under which the third party grants, to any of the -parties who would receive the covered work from you, a discriminatory -patent license (a) in connection with copies of the covered work -conveyed by you (or copies made from those copies), or (b) primarily -for and in connection with specific products or compilations that -contain the covered work, unless you entered into that arrangement, -or that patent license was granted, prior to 28 March 2007. - - Nothing in this License shall be construed as excluding or limiting -any implied license or other defenses to infringement that may -otherwise be available to you under applicable patent law. - - 12. No Surrender of Others' Freedom. - - If conditions are imposed on you (whether by court order, agreement or -otherwise) that contradict the conditions of this License, they do not -excuse you from the conditions of this License. If you cannot convey a -covered work so as to satisfy simultaneously your obligations under this -License and any other pertinent obligations, then as a consequence you may -not convey it at all. For example, if you agree to terms that obligate you -to collect a royalty for further conveying from those to whom you convey -the Program, the only way you could satisfy both those terms and this -License would be to refrain entirely from conveying the Program. - - 13. Remote Network Interaction; Use with the GNU General Public License. - - Notwithstanding any other provision of this License, if you modify the -Program, your modified version must prominently offer all users -interacting with it remotely through a computer network (if your version -supports such interaction) an opportunity to receive the Corresponding -Source of your version by providing access to the Corresponding Source -from a network server at no charge, through some standard or customary -means of facilitating copying of software. This Corresponding Source -shall include the Corresponding Source for any work covered by version 3 -of the GNU General Public License that is incorporated pursuant to the -following paragraph. - - Notwithstanding any other provision of this License, you have -permission to link or combine any covered work with a work licensed -under version 3 of the GNU General Public License into a single -combined work, and to convey the resulting work. The terms of this -License will continue to apply to the part which is the covered work, -but the work with which it is combined will remain governed by version -3 of the GNU General Public License. - - 14. Revised Versions of this License. - - The Free Software Foundation may publish revised and/or new versions of -the GNU Affero General Public License from time to time. Such new versions -will be similar in spirit to the present version, but may differ in detail to -address new problems or concerns. - - Each version is given a distinguishing version number. If the -Program specifies that a certain numbered version of the GNU Affero General -Public License "or any later version" applies to it, you have the -option of following the terms and conditions either of that numbered -version or of any later version published by the Free Software -Foundation. If the Program does not specify a version number of the -GNU Affero General Public License, you may choose any version ever published -by the Free Software Foundation. - - If the Program specifies that a proxy can decide which future -versions of the GNU Affero General Public License can be used, that proxy's -public statement of acceptance of a version permanently authorizes you -to choose that version for the Program. - - Later license versions may give you additional or different -permissions. However, no additional obligations are imposed on any -author or copyright holder as a result of your choosing to follow a -later version. - - 15. Disclaimer of Warranty. - - THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY -APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT -HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY -OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, -THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM -IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF -ALL NECESSARY SERVICING, REPAIR OR CORRECTION. - - 16. Limitation of Liability. - - IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING -WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS -THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY -GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE -USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF -DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD -PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), -EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF -SUCH DAMAGES. - - 17. Interpretation of Sections 15 and 16. - - If the disclaimer of warranty and limitation of liability provided -above cannot be given local legal effect according to their terms, -reviewing courts shall apply local law that most closely approximates -an absolute waiver of all civil liability in connection with the -Program, unless a warranty or assumption of liability accompanies a -copy of the Program in return for a fee. - - END OF TERMS AND CONDITIONS - - How to Apply These Terms to Your New Programs - - If you develop a new program, and you want it to be of the greatest -possible use to the public, the best way to achieve this is to make it -free software which everyone can redistribute and change under these terms. - - To do so, attach the following notices to the program. It is safest -to attach them to the start of each source file to most effectively -state the exclusion of warranty; and each file should have at least -the "copyright" line and a pointer to where the full notice is found. - - - Copyright (C) - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU Affero General Public License as published - by the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . - -Also add information on how to contact you by electronic and paper mail. - - If your software can interact with users remotely through a computer -network, you should also make sure that it provides a way for users to -get its source. For example, if your program is a web application, its -interface could display a "Source" link that leads users to an archive -of the code. There are many ways you could offer source, and different -solutions will be better for different programs; see section 13 for the -specific requirements. - - You should also get your employer (if you work as a programmer) or school, -if any, to sign a "copyright disclaimer" for the program, if necessary. -For more information on this, and how to apply and follow the GNU AGPL, see -. + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. From 986237af2e767590e1481991dbbd698e3a8ee29a Mon Sep 17 00:00:00 2001 From: skiffer-git <72860476+skiffer-git@users.noreply.github.com> Date: Tue, 22 Apr 2025 21:52:02 +0800 Subject: [PATCH 152/199] Update README.md --- README.md | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/README.md b/README.md index a23c72c61..e59bf20c4 100644 --- a/README.md +++ b/README.md @@ -132,14 +132,7 @@ Thank you for contributing to building a powerful instant messaging solution! ## :closed_book: License -This software is licensed under a dual-license model: - -- The GNU Affero General Public License (AGPL), Version 3 or later; **OR** -- Commercial license terms from OpenIMSDK. - -If you wish to use this software under commercial terms, please contact us at: contact@openim.io - -For more information, see: https://www.openim.io/en/licensing +This software is licensed under the Apache License 2.0 From bed81af75c46078303102c57dd805a13569777d0 Mon Sep 17 00:00:00 2001 From: skiffer-git <72860476+skiffer-git@users.noreply.github.com> Date: Tue, 22 Apr 2025 21:52:32 +0800 Subject: [PATCH 153/199] Update README_zh_CN.md --- README_zh_CN.md | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/README_zh_CN.md b/README_zh_CN.md index 2340ad09a..2147c0243 100644 --- a/README_zh_CN.md +++ b/README_zh_CN.md @@ -133,14 +133,7 @@ ## :closed_book: 开源许可证 License -本软件采用双重授权模型： - -GNU Affero 通用公共许可证（AGPL）第 3 版或更高版本；或 - -来自 OpenIMSDK 的商业授权条款。 - -如需商用，请联系：contact@openim.io -详见：https://www.openim.io/en/licensing +This software is licensed under the Apache License 2.0 ## 🔮 Thanks to our contributors! From 91db18168d74874f5637609af6df1758270e8a16 Mon Sep 17 00:00:00 2001 From: icey-yu <119291641+icey-yu@users.noreply.github.com> Date: Thu, 24 Apr 2025 17:26:52 +0800 Subject: [PATCH 154/199] fix: data version SetVersion will add record (#3304) --- tools/seq/internal/seq.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/seq/internal/seq.go b/tools/seq/internal/seq.go index 7e5d5598c..9fd352a96 100644 --- a/tools/seq/internal/seq.go +++ b/tools/seq/internal/seq.go @@ -337,7 +337,7 @@ func SetVersion(coll *mongo.Collection, key string, version int) error { ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) defer cancel() option := options.Update().SetUpsert(true) - filter := bson.M{"key": key, "value": strconv.Itoa(version)} + filter := bson.M{"key": key} update := bson.M{"$set": bson.M{"key": key, "value": strconv.Itoa(version)}} return mongoutil.UpdateOne(ctx, coll, filter, update, false, option) } From dd981b976dbf2f84f09b1e34c91a532320d4dacc Mon Sep 17 00:00:00 2001 From: Monet Lee Date: Fri, 25 Apr 2025 15:57:23 +0800 Subject: [PATCH 155/199] refactor: move stress-test tools location. (#3295) * refactor: move stress-test tools location. * improve stress tools. * improve stress_test-v2 --- test/stress-test-v2/README.md | 19 +++++++++++++ {tools => test}/stress-test-v2/main.go | 39 ++++++++++++++++++++------ test/stress-test/README.md | 19 +++++++++++++ {tools => test}/stress-test/main.go | 0 tools/stress-test/README.md | 25 ----------------- 5 files changed, 69 insertions(+), 33 deletions(-) create mode 100644 test/stress-test-v2/README.md rename {tools => test}/stress-test-v2/main.go (93%) create mode 100644 test/stress-test/README.md rename {tools => test}/stress-test/main.go (100%) delete mode 100644 tools/stress-test/README.md diff --git a/test/stress-test-v2/README.md b/test/stress-test-v2/README.md new file mode 100644 index 000000000..cbd4bdbde --- /dev/null +++ b/test/stress-test-v2/README.md @@ -0,0 +1,19 @@ +# Stress Test V2 + +## Usage + +You need set `TestTargetUserList` variables. + +### Build + +```bash + +go build -o test/stress-test-v2/stress-test-v2 test/stress-test-v2/main.go +``` + +### Excute + +```bash + +tools/stress-test-v2/stress-test-v2 -c config/ +``` diff --git a/tools/stress-test-v2/main.go b/test/stress-test-v2/main.go similarity index 93% rename from tools/stress-test-v2/main.go rename to test/stress-test-v2/main.go index 0c309b9c9..0e4609964 100644 --- a/tools/stress-test-v2/main.go +++ b/test/stress-test-v2/main.go @@ -56,6 +56,7 @@ var ( const ( MaxUser = 100000 + Max1kUser = 1000 Max100KGroup = 100 Max999Group = 1000 MaxInviteUserLimit = 999 @@ -161,7 +162,7 @@ func (st *StressTest) PostRequest(ctx context.Context, url string, reqbody any) if baseResp.ErrCode != 0 { err = fmt.Errorf(baseResp.ErrMsg) - log.ZError(ctx, "Failed to send request", err, "url", url, "reqbody", reqbody, "resp", baseResp) + // log.ZError(ctx, "Failed to send request", err, "url", url, "reqbody", reqbody, "resp", baseResp) return nil, err } @@ -530,9 +531,20 @@ func main() { // Create 100K groups st.Wg.Add(1) go func(idx int) { + startTime := time.Now() + defer func() { + elapsedTime := time.Since(startTime) + log.ZInfo(st.Ctx, "100K group creation completed", + "groupID", fmt.Sprintf("v2_StressTest_Group_100K_%d", idx), + "index", idx, + "duration", elapsedTime.String()) + }() + defer st.Wg.Done() defer func() { + st.Mutex.Lock() st.Create100kGroupCounter++ + st.Mutex.Unlock() }() groupID := fmt.Sprintf("v2_StressTest_Group_100K_%d", idx) @@ -542,7 +554,7 @@ func main() { // continue } - for i := 0; i < MaxUser/MaxInviteUserLimit; i++ { + for i := 0; i <= MaxUser/MaxInviteUserLimit; i++ { InviteUserIDs := make([]string, 0) // ensure TargetUserList is in group InviteUserIDs = append(InviteUserIDs, TestTargetUserList...) @@ -556,7 +568,7 @@ func main() { } if len(InviteUserIDs) == 0 { - log.ZWarn(st.Ctx, "InviteUserIDs is empty", nil, "groupID", groupID) + // log.ZWarn(st.Ctx, "InviteUserIDs is empty", nil, "groupID", groupID) continue } @@ -567,7 +579,7 @@ func main() { } if len(InviteUserIDs) == 0 { - log.ZWarn(st.Ctx, "InviteUserIDs is empty", nil, "groupID", groupID) + // log.ZWarn(st.Ctx, "InviteUserIDs is empty", nil, "groupID", groupID) continue } @@ -602,9 +614,20 @@ func main() { // Create 999 groups st.Wg.Add(1) go func(idx int) { + startTime := time.Now() + defer func() { + elapsedTime := time.Since(startTime) + log.ZInfo(st.Ctx, "999 group creation completed", + "groupID", fmt.Sprintf("v2_StressTest_Group_1K_%d", idx), + "index", idx, + "duration", elapsedTime.String()) + }() + defer st.Wg.Done() defer func() { + st.Mutex.Lock() st.Create999GroupCounter++ + st.Mutex.Unlock() }() groupID := fmt.Sprintf("v2_StressTest_Group_1K_%d", idx) @@ -613,13 +636,13 @@ func main() { log.ZError(st.Ctx, "Create group failed.", err) // continue } - for i := 0; i < MaxUser/MaxInviteUserLimit; i++ { + for i := 0; i <= Max1kUser/MaxInviteUserLimit; i++ { InviteUserIDs := make([]string, 0) // ensure TargetUserList is in group InviteUserIDs = append(InviteUserIDs, TestTargetUserList...) startIdx := max(i*MaxInviteUserLimit, 1) - endIdx := min((i+1)*MaxInviteUserLimit, MaxUser) + endIdx := min((i+1)*MaxInviteUserLimit, Max1kUser) for j := startIdx; j < endIdx; j++ { userCreatedID := fmt.Sprintf("v2_StressTest_User_%d", j) @@ -627,7 +650,7 @@ func main() { } if len(InviteUserIDs) == 0 { - log.ZWarn(st.Ctx, "InviteUserIDs is empty", nil, "groupID", groupID) + // log.ZWarn(st.Ctx, "InviteUserIDs is empty", nil, "groupID", groupID) continue } @@ -638,7 +661,7 @@ func main() { } if len(InviteUserIDs) == 0 { - log.ZWarn(st.Ctx, "InviteUserIDs is empty", nil, "groupID", groupID) + // log.ZWarn(st.Ctx, "InviteUserIDs is empty", nil, "groupID", groupID) continue } diff --git a/test/stress-test/README.md b/test/stress-test/README.md new file mode 100644 index 000000000..cba93e279 --- /dev/null +++ b/test/stress-test/README.md @@ -0,0 +1,19 @@ +# Stress Test + +## Usage + +You need set `TestTargetUserList` and `DefaultGroupID` variables. + +### Build + +```bash + +go build -o test/stress-test/stress-test test/stress-test/main.go +``` + +### Excute + +```bash + +tools/stress-test/stress-test -c config/ +``` diff --git a/tools/stress-test/main.go b/test/stress-test/main.go similarity index 100% rename from tools/stress-test/main.go rename to test/stress-test/main.go diff --git a/tools/stress-test/README.md b/tools/stress-test/README.md deleted file mode 100644 index 531233a20..000000000 --- a/tools/stress-test/README.md +++ /dev/null @@ -1,25 +0,0 @@ -# Stress Test - -## Usage - -You need set `TestTargetUserList` and `DefaultGroupID` variables. - -### Build - -```bash -go build -o _output/bin/tools/linux/amd64/stress-test tools/stress-test/main.go - -# or - -go build -o tools/stress-test/stress-test tools/stress-test/main.go -``` - -### Excute - -```bash -_output/bin/tools/linux/amd64/stress-test -c config/ - -#or - -tools/stress-test/stress-test -c config/ -``` From 56c5c1f01511064cb0ef787cf4fa15dbdb1834f9 Mon Sep 17 00:00:00 2001 From: icey-yu <119291641+icey-yu@users.noreply.github.com> Date: Tue, 6 May 2025 15:10:10 +0800 Subject: [PATCH 156/199] fix: delete token by correct platformID && feat: adminToken can be retained for five minutes after deleting (#3313) --- internal/rpc/auth/auth.go | 17 +++-- pkg/common/storage/cache/cachekey/token.go | 7 +- pkg/common/storage/cache/mcache/token.go | 38 +++++++++- pkg/common/storage/cache/redis/token.go | 73 +++++++++++++++++-- pkg/common/storage/cache/token.go | 3 + pkg/common/storage/controller/auth.go | 82 ++++++++++++++-------- pkg/common/storage/database/mgo/cache.go | 2 +- 7 files changed, 177 insertions(+), 45 deletions(-) diff --git a/internal/rpc/auth/auth.go b/internal/rpc/auth/auth.go index 399d0bd4b..6697e27c0 100644 --- a/internal/rpc/auth/auth.go +++ b/internal/rpc/auth/auth.go @@ -159,15 +159,17 @@ func (s *authServer) parseToken(ctx context.Context, tokensString string) (claim if err != nil { return nil, err } - isAdmin := authverify.IsManagerUserID(claims.UserID, s.config.Share.IMAdminUserID) - if isAdmin { - return claims, nil - } m, err := s.authDatabase.GetTokensWithoutError(ctx, claims.UserID, claims.PlatformID) if err != nil { return nil, err } if len(m) == 0 { + isAdmin := authverify.IsManagerUserID(claims.UserID, s.config.Share.IMAdminUserID) + if isAdmin { + if err = s.authDatabase.GetTemporaryTokensWithoutError(ctx, claims.UserID, claims.PlatformID, tokensString); err == nil { + return claims, nil + } + } return nil, servererrs.ErrTokenNotExist.Wrap() } if v, ok := m[tokensString]; ok { @@ -179,6 +181,13 @@ func (s *authServer) parseToken(ctx context.Context, tokensString string) (claim default: return nil, errs.Wrap(errs.ErrTokenUnknown) } + } else { + isAdmin := authverify.IsManagerUserID(claims.UserID, s.config.Share.IMAdminUserID) + if isAdmin { + if err = s.authDatabase.GetTemporaryTokensWithoutError(ctx, claims.UserID, claims.PlatformID, tokensString); err == nil { + return claims, nil + } + } } return nil, servererrs.ErrTokenNotExist.Wrap() } diff --git a/pkg/common/storage/cache/cachekey/token.go b/pkg/common/storage/cache/cachekey/token.go index 83ba2f211..6fe1bdfef 100644 --- a/pkg/common/storage/cache/cachekey/token.go +++ b/pkg/common/storage/cache/cachekey/token.go @@ -1,8 +1,9 @@ package cachekey import ( - "github.com/openimsdk/protocol/constant" "strings" + + "github.com/openimsdk/protocol/constant" ) const ( @@ -13,6 +14,10 @@ func GetTokenKey(userID string, platformID int) string { return UidPidToken + userID + ":" + constant.PlatformIDToName(platformID) } +func GetTemporaryTokenKey(userID string, platformID int, token string) string { + return UidPidToken + ":TEMPORARY:" + userID + ":" + constant.PlatformIDToName(platformID) + ":" + token +} + func GetAllPlatformTokenKey(userID string) []string { res := make([]string, len(constant.PlatformID2Name)) for k := range constant.PlatformID2Name { diff --git a/pkg/common/storage/cache/mcache/token.go b/pkg/common/storage/cache/mcache/token.go index d7ae29cfc..98b9cc066 100644 --- a/pkg/common/storage/cache/mcache/token.go +++ b/pkg/common/storage/cache/mcache/token.go @@ -27,7 +27,6 @@ type tokenCache struct { func (x *tokenCache) getTokenKey(userID string, platformID int, token string) string { return cachekey.GetTokenKey(userID, platformID) + ":" + token - } func (x *tokenCache) SetTokenFlag(ctx context.Context, userID string, platformID int, token string, flag int) error { @@ -57,6 +56,14 @@ func (x *tokenCache) GetTokensWithoutError(ctx context.Context, userID string, p return mm, nil } +func (x *tokenCache) HasTemporaryToken(ctx context.Context, userID string, platformID int, token string) error { + key := cachekey.GetTemporaryTokenKey(userID, platformID, token) + if _, err := x.cache.Get(ctx, []string{key}); err != nil { + return err + } + return nil +} + func (x *tokenCache) GetAllTokensWithoutError(ctx context.Context, userID string) (map[int]map[string]int, error) { prefix := cachekey.UidPidToken + userID + ":" tokens, err := x.cache.Prefix(ctx, prefix) @@ -128,3 +135,32 @@ func (x *tokenCache) DeleteTokenByUidPid(ctx context.Context, userID string, pla func (x *tokenCache) getExpireTime(t int64) time.Duration { return time.Hour * 24 * time.Duration(t) } + +func (x *tokenCache) DeleteTokenByTokenMap(ctx context.Context, userID string, tokens map[int][]string) error { + keys := make([]string, 0, len(tokens)) + for platformID, ts := range tokens { + for _, t := range ts { + keys = append(keys, x.getTokenKey(userID, platformID, t)) + } + } + return x.cache.Del(ctx, keys) +} + +func (x *tokenCache) DeleteAndSetTemporary(ctx context.Context, userID string, platformID int, fields []string) error { + keys := make([]string, 0, len(fields)) + for _, f := range fields { + keys = append(keys, x.getTokenKey(userID, platformID, f)) + } + if err := x.cache.Del(ctx, keys); err != nil { + return err + } + + for _, f := range fields { + k := cachekey.GetTemporaryTokenKey(userID, platformID, f) + if err := x.cache.Set(ctx, k, "", time.Minute*5); err != nil { + return errs.Wrap(err) + } + } + + return nil +} diff --git a/pkg/common/storage/cache/redis/token.go b/pkg/common/storage/cache/redis/token.go index 510da43e3..b3870daee 100644 --- a/pkg/common/storage/cache/redis/token.go +++ b/pkg/common/storage/cache/redis/token.go @@ -9,6 +9,7 @@ import ( "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/cachekey" "github.com/openimsdk/tools/errs" + "github.com/openimsdk/tools/utils/datautil" "github.com/redis/go-redis/v9" ) @@ -55,6 +56,14 @@ func (c *tokenCache) GetTokensWithoutError(ctx context.Context, userID string, p return mm, nil } +func (c *tokenCache) HasTemporaryToken(ctx context.Context, userID string, platformID int, token string) error { + err := c.rdb.Get(ctx, cachekey.GetTemporaryTokenKey(userID, platformID, token)).Err() + if err != nil { + return errs.Wrap(err) + } + return nil +} + func (c *tokenCache) GetAllTokensWithoutError(ctx context.Context, userID string) (map[int]map[string]int, error) { var ( res = make(map[int]map[string]int) @@ -101,13 +110,19 @@ func (c *tokenCache) SetTokenMapByUidPid(ctx context.Context, userID string, pla } func (c *tokenCache) BatchSetTokenMapByUidPid(ctx context.Context, tokens map[string]map[string]any) error { - pipe := c.rdb.Pipeline() - for k, v := range tokens { - pipe.HSet(ctx, k, v) - } - _, err := pipe.Exec(ctx) - if err != nil { - return errs.Wrap(err) + keys := datautil.Keys(tokens) + if err := ProcessKeysBySlot(ctx, c.rdb, keys, func(ctx context.Context, slot int64, keys []string) error { + pipe := c.rdb.Pipeline() + for k, v := range tokens { + pipe.HSet(ctx, k, v) + } + _, err := pipe.Exec(ctx) + if err != nil { + return errs.Wrap(err) + } + return nil + }); err != nil { + return err } return nil } @@ -119,3 +134,47 @@ func (c *tokenCache) DeleteTokenByUidPid(ctx context.Context, userID string, pla func (c *tokenCache) getExpireTime(t int64) time.Duration { return time.Hour * 24 * time.Duration(t) } + +// DeleteTokenByTokenMap tokens key is platformID, value is token slice +func (c *tokenCache) DeleteTokenByTokenMap(ctx context.Context, userID string, tokens map[int][]string) error { + var ( + keys = make([]string, 0, len(tokens)) + keyMap = make(map[string][]string) + ) + for k, v := range tokens { + k1 := cachekey.GetTokenKey(userID, k) + keys = append(keys, k1) + keyMap[k1] = v + } + + if err := ProcessKeysBySlot(ctx, c.rdb, keys, func(ctx context.Context, slot int64, keys []string) error { + pipe := c.rdb.Pipeline() + for k, v := range tokens { + pipe.HDel(ctx, cachekey.GetTokenKey(userID, k), v...) + } + _, err := pipe.Exec(ctx) + if err != nil { + return errs.Wrap(err) + } + return nil + }); err != nil { + return err + } + + return nil +} + +func (c *tokenCache) DeleteAndSetTemporary(ctx context.Context, userID string, platformID int, fields []string) error { + key := cachekey.GetTokenKey(userID, platformID) + if err := c.rdb.HDel(ctx, key, fields...).Err(); err != nil { + return errs.Wrap(err) + } + for _, f := range fields { + k := cachekey.GetTemporaryTokenKey(userID, platformID, f) + if err := c.rdb.Set(ctx, k, "", time.Minute*5).Err(); err != nil { + return errs.Wrap(err) + } + } + + return nil +} diff --git a/pkg/common/storage/cache/token.go b/pkg/common/storage/cache/token.go index e5e0a9383..441c08939 100644 --- a/pkg/common/storage/cache/token.go +++ b/pkg/common/storage/cache/token.go @@ -9,8 +9,11 @@ type TokenModel interface { // SetTokenFlagEx set token and flag with expire time SetTokenFlagEx(ctx context.Context, userID string, platformID int, token string, flag int) error GetTokensWithoutError(ctx context.Context, userID string, platformID int) (map[string]int, error) + HasTemporaryToken(ctx context.Context, userID string, platformID int, token string) error GetAllTokensWithoutError(ctx context.Context, userID string) (map[int]map[string]int, error) SetTokenMapByUidPid(ctx context.Context, userID string, platformID int, m map[string]int) error BatchSetTokenMapByUidPid(ctx context.Context, tokens map[string]map[string]any) error DeleteTokenByUidPid(ctx context.Context, userID string, platformID int, fields []string) error + DeleteTokenByTokenMap(ctx context.Context, userID string, tokens map[int][]string) error + DeleteAndSetTemporary(ctx context.Context, userID string, platformID int, fields []string) error } diff --git a/pkg/common/storage/controller/auth.go b/pkg/common/storage/controller/auth.go index 013d8b155..488a116c3 100644 --- a/pkg/common/storage/controller/auth.go +++ b/pkg/common/storage/controller/auth.go @@ -17,6 +17,8 @@ import ( type AuthDatabase interface { // If the result is empty, no error is returned. GetTokensWithoutError(ctx context.Context, userID string, platformID int) (map[string]int, error) + + GetTemporaryTokensWithoutError(ctx context.Context, userID string, platformID int, token string) error // Create token CreateToken(ctx context.Context, userID string, platformID int) (string, error) @@ -52,6 +54,10 @@ func (a *authDatabase) GetTokensWithoutError(ctx context.Context, userID string, return a.cache.GetTokensWithoutError(ctx, userID, platformID) } +func (a *authDatabase) GetTemporaryTokensWithoutError(ctx context.Context, userID string, platformID int, token string) error { + return a.cache.HasTemporaryToken(ctx, userID, platformID, token) +} + func (a *authDatabase) SetTokenMapByUidPid(ctx context.Context, userID string, platformID int, m map[string]int) error { return a.cache.SetTokenMapByUidPid(ctx, userID, platformID, m) } @@ -85,23 +91,30 @@ func (a *authDatabase) CreateToken(ctx context.Context, userID string, platformI return "", err } - deleteTokenKey, kickedTokenKey, err := a.checkToken(ctx, tokens, platformID) + deleteTokenKey, kickedTokenKey, adminTokens, err := a.checkToken(ctx, tokens, platformID) if err != nil { return "", err } if len(deleteTokenKey) != 0 { - err = a.cache.DeleteTokenByUidPid(ctx, userID, platformID, deleteTokenKey) + err = a.cache.DeleteTokenByTokenMap(ctx, userID, deleteTokenKey) if err != nil { return "", err } } if len(kickedTokenKey) != 0 { - for _, k := range kickedTokenKey { - err := a.cache.SetTokenFlagEx(ctx, userID, platformID, k, constant.KickedToken) - if err != nil { - return "", err + for plt, ks := range kickedTokenKey { + for _, k := range ks { + err := a.cache.SetTokenFlagEx(ctx, userID, plt, k, constant.KickedToken) + if err != nil { + return "", err + } + log.ZDebug(ctx, "kicked token in create token", "token", k) } - log.ZDebug(ctx, "kicked token in create token", "token", k) + } + } + if len(adminTokens) != 0 { + if err = a.cache.DeleteAndSetTemporary(ctx, userID, constant.AdminPlatformID, adminTokens); err != nil { + return "", err } } @@ -119,12 +132,13 @@ func (a *authDatabase) CreateToken(ctx context.Context, userID string, platformI return tokenString, nil } -func (a *authDatabase) checkToken(ctx context.Context, tokens map[int]map[string]int, platformID int) ([]string, []string, error) { - // todo: Move the logic for handling old data to another location. +// checkToken will check token by tokenPolicy and return deleteToken,kickToken,deleteAdminToken +func (a *authDatabase) checkToken(ctx context.Context, tokens map[int]map[string]int, platformID int) (map[int][]string, map[int][]string, []string, error) { + // todo: Asynchronous deletion of old data. var ( loginTokenMap = make(map[int][]string) // The length of the value of the map must be greater than 0 - deleteToken = make([]string, 0) - kickToken = make([]string, 0) + deleteToken = make(map[int][]string) + kickToken = make(map[int][]string) adminToken = make([]string, 0) unkickTerminal = "" ) @@ -133,7 +147,7 @@ func (a *authDatabase) checkToken(ctx context.Context, tokens map[int]map[string for k, v := range tks { _, err := tokenverify.GetClaimFromToken(k, authverify.Secret(a.accessSecret)) if err != nil || v != constant.NormalToken { - deleteToken = append(deleteToken, k) + deleteToken[plfID] = append(deleteToken[plfID], k) } else { if plfID != constant.AdminPlatformID { loginTokenMap[plfID] = append(loginTokenMap[plfID], k) @@ -153,14 +167,15 @@ func (a *authDatabase) checkToken(ctx context.Context, tokens map[int]map[string } limit := a.multiLogin.MaxNumOneEnd if l > limit { - kickToken = append(kickToken, ts[:l-limit]...) + kickToken[plt] = ts[:l-limit] } } case constant.AllLoginButSameTermKick: for plt, ts := range loginTokenMap { - kickToken = append(kickToken, ts[:len(ts)-1]...) + kickToken[plt] = ts[:len(ts)-1] + if plt == platformID { - kickToken = append(kickToken, ts[len(ts)-1]) + kickToken[plt] = append(kickToken[plt], ts[len(ts)-1]) } } case constant.PCAndOther: @@ -168,29 +183,33 @@ func (a *authDatabase) checkToken(ctx context.Context, tokens map[int]map[string if constant.PlatformIDToClass(platformID) != unkickTerminal { for plt, ts := range loginTokenMap { if constant.PlatformIDToClass(plt) != unkickTerminal { - kickToken = append(kickToken, ts...) + kickToken[plt] = ts } } } else { var ( - preKick []string - isReserve = true + preKickToken string + preKickPlt int + reserveToken = false ) for plt, ts := range loginTokenMap { if constant.PlatformIDToClass(plt) != unkickTerminal { // Keep a token from another end - if isReserve { - isReserve = false - kickToken = append(kickToken, ts[:len(ts)-1]...) - preKick = append(preKick, ts[len(ts)-1]) + if !reserveToken { + reserveToken = true + kickToken[plt] = ts[:len(ts)-1] + preKickToken = ts[len(ts)-1] + preKickPlt = plt continue } else { // Prioritize keeping Android if plt == constant.AndroidPlatformID { - kickToken = append(kickToken, preKick...) - kickToken = append(kickToken, ts[:len(ts)-1]...) + if preKickToken != "" { + kickToken[preKickPlt] = append(kickToken[preKickPlt], preKickToken) + } + kickToken[plt] = ts[:len(ts)-1] } else { - kickToken = append(kickToken, ts...) + kickToken[plt] = ts } } } @@ -203,19 +222,19 @@ func (a *authDatabase) checkToken(ctx context.Context, tokens map[int]map[string for plt, ts := range loginTokenMap { if constant.PlatformIDToClass(plt) == constant.PlatformIDToClass(platformID) { - kickToken = append(kickToken, ts...) + kickToken[plt] = ts } else { if _, ok := reserved[constant.PlatformIDToClass(plt)]; !ok { reserved[constant.PlatformIDToClass(plt)] = struct{}{} - kickToken = append(kickToken, ts[:len(ts)-1]...) + kickToken[plt] = ts[:len(ts)-1] continue } else { - kickToken = append(kickToken, ts...) + kickToken[plt] = ts } } } default: - return nil, nil, errs.New("unknown multiLogin policy").Wrap() + return nil, nil, nil, errs.New("unknown multiLogin policy").Wrap() } //var adminTokenMaxNum = a.multiLogin.MaxNumOneEnd @@ -226,8 +245,9 @@ func (a *authDatabase) checkToken(ctx context.Context, tokens map[int]map[string //if l > adminTokenMaxNum { // kickToken = append(kickToken, adminToken[:l-adminTokenMaxNum]...) //} + var deleteAdminToken []string if platformID == constant.AdminPlatformID { - kickToken = append(kickToken, adminToken...) + deleteAdminToken = adminToken } - return deleteToken, kickToken, nil + return deleteToken, kickToken, deleteAdminToken, nil } diff --git a/pkg/common/storage/database/mgo/cache.go b/pkg/common/storage/database/mgo/cache.go index bcf86cd56..991dfa874 100644 --- a/pkg/common/storage/database/mgo/cache.go +++ b/pkg/common/storage/database/mgo/cache.go @@ -127,7 +127,7 @@ func (x *CacheMgo) Del(ctx context.Context, key []string) error { return nil } _, err := x.coll.DeleteMany(ctx, bson.M{"key": bson.M{"$in": key}}) - return err + return errs.Wrap(err) } func (x *CacheMgo) lockKey(key string) string { From 1178808ba7d7798cfc13aada0b47c4a1e130cdce Mon Sep 17 00:00:00 2001 From: chao <48119764+withchao@users.noreply.github.com> Date: Fri, 9 May 2025 09:31:30 +0800 Subject: [PATCH 157/199] feat: optimize server code (#3319) * pb * fix: Modifying other fields while setting IsPrivateChat does not take effect * fix: quote message error revoke * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * upgrading pkg tools * fix * fix * optimize log output * feat: support GetLastMessage * feat: support GetLastMessage * feat: s3 switch * feat: s3 switch * fix: GetUsersOnline * feat: SendBusinessNotification supported configuration parameters * feat: SendBusinessNotification supported configuration parameters * feat: SendBusinessNotification supported configuration parameters * feat: seq conversion failed without exiting * fix: DeleteDoc crash * fix: fill send time * fix: fill send time * fix: crash caused by withdrawing messages from users who have left the group * fix: user msg timestamp * seq read config * seq read config * fix: the source message of the reference is withdrawn, and the referenced message is deleted * feat: optimize the default notification.yml * fix: shouldPushOffline * fix: the sorting is wrong after canceling the administrator in group settings * feat: Sending messages supports returning fields modified by webhook * feat: Sending messages supports returning fields modified by webhook * feat: Sending messages supports returning fields modified by webhook * fix: oss specifies content-type when uploading * fix: the version number contains a line break * fix: the version number contains a line break * feat: GetConversationsHasReadAndMaxSeq support pinned * feat: GetConversationsHasReadAndMaxSeq support pinned * feat: GetConversationsHasReadAndMaxSeq support pinned * fix: transferring the group owner to a muted member, incremental version error * feat: unified conversion code * feat: update gomake --- go.mod | 2 +- go.sum | 4 +-- pkg/common/storage/controller/msg_transfer.go | 34 +------------------ 3 files changed, 4 insertions(+), 36 deletions(-) diff --git a/go.mod b/go.mod index fbaa19434..3863fde1f 100644 --- a/go.mod +++ b/go.mod @@ -34,7 +34,7 @@ require ( github.com/hashicorp/golang-lru/v2 v2.0.7 github.com/kelindar/bitmap v1.5.2 github.com/likexian/gokit v0.25.13 - github.com/openimsdk/gomake v0.0.15-alpha.2 + github.com/openimsdk/gomake v0.0.15-alpha.5 github.com/redis/go-redis/v9 v9.4.0 github.com/robfig/cron/v3 v3.0.1 github.com/shirou/gopsutil v3.21.11+incompatible diff --git a/go.sum b/go.sum index 390a51c4a..c80690f80 100644 --- a/go.sum +++ b/go.sum @@ -345,8 +345,8 @@ github.com/onsi/ginkgo/v2 v2.19.0 h1:9Cnnf7UHo57Hy3k6/m5k3dRfGTMXGvxhHFvkDTCTpvA github.com/onsi/ginkgo/v2 v2.19.0/go.mod h1:rlwLi9PilAFJ8jCg9UE1QP6VBpd6/xj3SRC0d6TU0To= github.com/onsi/gomega v1.25.0 h1:Vw7br2PCDYijJHSfBOWhov+8cAnUf8MfMaIOV323l6Y= github.com/onsi/gomega v1.25.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdMPGhM= -github.com/openimsdk/gomake v0.0.15-alpha.2 h1:5Q8yl8ezy2yx+q8/ucU/t4kJnDfCzNOrkXcDACCqtyM= -github.com/openimsdk/gomake v0.0.15-alpha.2/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= +github.com/openimsdk/gomake v0.0.15-alpha.5 h1:eEZCEHm+NsmcO3onXZPIUbGFCYPYbsX5beV3ZyOsGhY= +github.com/openimsdk/gomake v0.0.15-alpha.5/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= github.com/openimsdk/protocol v0.0.73-alpha.6 h1:sna9coWG7HN1zObBPtvG0Ki/vzqHXiB4qKbA5P3w7kc= github.com/openimsdk/protocol v0.0.73-alpha.6/go.mod h1:WF7EuE55vQvpyUAzDXcqg+B+446xQyEba0X35lTINmw= github.com/openimsdk/tools v0.0.50-alpha.79 h1:jxYEbrzaze4Z2r4NrKad816buZ690ix0L9MTOOOH3ik= diff --git a/pkg/common/storage/controller/msg_transfer.go b/pkg/common/storage/controller/msg_transfer.go index 260daf5c4..28392d66e 100644 --- a/pkg/common/storage/controller/msg_transfer.go +++ b/pkg/common/storage/controller/msg_transfer.go @@ -81,42 +81,10 @@ func (db *msgTransferDatabase) BatchInsertChat2DB(ctx context.Context, conversat continue } seqs[i] = msg.Seq - var offlinePushModel *model.OfflinePushModel - if msg.OfflinePushInfo != nil { - offlinePushModel = &model.OfflinePushModel{ - Title: msg.OfflinePushInfo.Title, - Desc: msg.OfflinePushInfo.Desc, - Ex: msg.OfflinePushInfo.Ex, - IOSPushSound: msg.OfflinePushInfo.IOSPushSound, - IOSBadgeCount: msg.OfflinePushInfo.IOSBadgeCount, - } - } if msg.Status == constant.MsgStatusSending { msg.Status = constant.MsgStatusSendSuccess } - msgs[i] = &model.MsgDataModel{ - SendID: msg.SendID, - RecvID: msg.RecvID, - GroupID: msg.GroupID, - ClientMsgID: msg.ClientMsgID, - ServerMsgID: msg.ServerMsgID, - SenderPlatformID: msg.SenderPlatformID, - SenderNickname: msg.SenderNickname, - SenderFaceURL: msg.SenderFaceURL, - SessionType: msg.SessionType, - MsgFrom: msg.MsgFrom, - ContentType: msg.ContentType, - Content: string(msg.Content), - Seq: msg.Seq, - SendTime: msg.SendTime, - CreateTime: msg.CreateTime, - Status: msg.Status, - Options: msg.Options, - OfflinePush: offlinePushModel, - AtUserIDList: msg.AtUserIDList, - AttachedInfo: msg.AttachedInfo, - Ex: msg.Ex, - } + msgs[i] = convert.MsgPb2DB(msg) } if err := db.BatchInsertBlock(ctx, conversationID, msgs, updateKeyMsg, msgList[0].Seq); err != nil { return err From 1d7660bedb98c692cdc1f09d84428acaf9cb7ea2 Mon Sep 17 00:00:00 2001 From: chao <48119764+withchao@users.noreply.github.com> Date: Wed, 14 May 2025 16:07:05 +0800 Subject: [PATCH 158/199] fix: optimize grpc option and fix some interface permission checks (#3327) * pb * fix: Modifying other fields while setting IsPrivateChat does not take effect * fix: quote message error revoke * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * upgrading pkg tools * fix * fix * optimize log output * feat: support GetLastMessage * feat: support GetLastMessage * feat: s3 switch * feat: s3 switch * fix: GetUsersOnline * feat: SendBusinessNotification supported configuration parameters * feat: SendBusinessNotification supported configuration parameters * feat: SendBusinessNotification supported configuration parameters * feat: seq conversion failed without exiting * fix: DeleteDoc crash * fix: fill send time * fix: fill send time * fix: crash caused by withdrawing messages from users who have left the group * fix: user msg timestamp * seq read config * seq read config * fix: the source message of the reference is withdrawn, and the referenced message is deleted * feat: optimize the default notification.yml * fix: shouldPushOffline * fix: the sorting is wrong after canceling the administrator in group settings * feat: Sending messages supports returning fields modified by webhook * feat: Sending messages supports returning fields modified by webhook * feat: Sending messages supports returning fields modified by webhook * fix: oss specifies content-type when uploading * fix: the version number contains a line break * fix: the version number contains a line break * feat: GetConversationsHasReadAndMaxSeq support pinned * feat: GetConversationsHasReadAndMaxSeq support pinned * feat: GetConversationsHasReadAndMaxSeq support pinned * fix: transferring the group owner to a muted member, incremental version error * feat: unified conversion code * feat: update gomake * feat: grpc mw * fix: permission verification * fix: optimizing the code * fix: optimize grpc option and fix some interface permission checks --- go.mod | 2 +- go.sum | 4 +-- internal/api/config_manager.go | 2 +- internal/api/msg.go | 6 ++-- internal/api/router.go | 13 ++++++- internal/msggateway/hub_server.go | 2 +- internal/rpc/auth/auth.go | 10 +++--- internal/rpc/conversation/sync.go | 4 +++ internal/rpc/group/group.go | 58 +++++++++++------------------- internal/rpc/group/notification.go | 2 +- internal/rpc/group/sync.go | 20 +++++++++-- internal/rpc/msg/clear.go | 5 +-- internal/rpc/msg/delete.go | 8 ++--- internal/rpc/msg/revoke.go | 6 ++-- internal/rpc/msg/sync_msg.go | 2 +- internal/rpc/relation/black.go | 9 +++-- internal/rpc/relation/friend.go | 18 +++++----- internal/rpc/relation/sync.go | 8 +++-- internal/rpc/third/log.go | 4 +-- internal/rpc/third/s3.go | 4 +-- internal/rpc/third/tool.go | 6 +--- internal/rpc/user/config.go | 8 ++--- internal/rpc/user/user.go | 33 ++++++++--------- pkg/authverify/token.go | 53 +++++++++++++++++---------- pkg/common/startrpc/mw.go | 15 ++++++++ pkg/common/startrpc/start.go | 47 ++++++++++++++++++++++-- 26 files changed, 216 insertions(+), 133 deletions(-) create mode 100644 pkg/common/startrpc/mw.go diff --git a/go.mod b/go.mod index 3863fde1f..4af0695db 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/mitchellh/mapstructure v1.5.0 github.com/openimsdk/protocol v0.0.73-alpha.6 - github.com/openimsdk/tools v0.0.50-alpha.79 + github.com/openimsdk/tools v0.0.50-alpha.81 github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 github.com/stretchr/testify v1.9.0 diff --git a/go.sum b/go.sum index c80690f80..6bc410a2d 100644 --- a/go.sum +++ b/go.sum @@ -349,8 +349,8 @@ github.com/openimsdk/gomake v0.0.15-alpha.5 h1:eEZCEHm+NsmcO3onXZPIUbGFCYPYbsX5b github.com/openimsdk/gomake v0.0.15-alpha.5/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= github.com/openimsdk/protocol v0.0.73-alpha.6 h1:sna9coWG7HN1zObBPtvG0Ki/vzqHXiB4qKbA5P3w7kc= github.com/openimsdk/protocol v0.0.73-alpha.6/go.mod h1:WF7EuE55vQvpyUAzDXcqg+B+446xQyEba0X35lTINmw= -github.com/openimsdk/tools v0.0.50-alpha.79 h1:jxYEbrzaze4Z2r4NrKad816buZ690ix0L9MTOOOH3ik= -github.com/openimsdk/tools v0.0.50-alpha.79/go.mod h1:n2poR3asX1e1XZce4O+MOWAp+X02QJRFvhcLCXZdzRo= +github.com/openimsdk/tools v0.0.50-alpha.81 h1:VbuJKtigNXLkCKB/Q6f2UHsqoSaTOAwS8F51c1nhOCA= +github.com/openimsdk/tools v0.0.50-alpha.81/go.mod h1:n2poR3asX1e1XZce4O+MOWAp+X02QJRFvhcLCXZdzRo= github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= diff --git a/internal/api/config_manager.go b/internal/api/config_manager.go index aca02c52c..8786257b3 100644 --- a/internal/api/config_manager.go +++ b/internal/api/config_manager.go @@ -44,7 +44,7 @@ func NewConfigManager(IMAdminUserID []string, cfg *config.AllConfig, client *cli } func (cm *ConfigManager) CheckAdmin(c *gin.Context) { - if err := authverify.CheckAdmin(c, cm.imAdminUserID); err != nil { + if err := authverify.CheckAdmin(c); err != nil { apiresp.GinError(c, err) c.Abort() } diff --git a/internal/api/msg.go b/internal/api/msg.go index 090f3329b..5349faf87 100644 --- a/internal/api/msg.go +++ b/internal/api/msg.go @@ -281,7 +281,7 @@ func (m *MessageApi) SendMessage(c *gin.Context) { } // Check if the user has the app manager role. - if !authverify.IsAppManagerUid(c, m.imAdminUserID) { + if !authverify.IsAdmin(c) { // Respond with a permission error if the user is not an app manager. apiresp.GinError(c, errs.ErrNoPermission.WrapMsg("only app manager can send message")) return @@ -355,7 +355,7 @@ func (m *MessageApi) SendBusinessNotification(c *gin.Context) { if req.ReliabilityLevel == nil { req.ReliabilityLevel = datautil.ToPtr(1) } - if !authverify.IsAppManagerUid(c, m.imAdminUserID) { + if !authverify.IsAdmin(c) { apiresp.GinError(c, errs.ErrNoPermission.WrapMsg("only app manager can send message")) return } @@ -399,7 +399,7 @@ func (m *MessageApi) BatchSendMsg(c *gin.Context) { apiresp.GinError(c, errs.ErrArgs.WithDetail(err.Error()).Wrap()) return } - if err := authverify.CheckAdmin(c, m.imAdminUserID); err != nil { + if err := authverify.CheckAdmin(c); err != nil { apiresp.GinError(c, errs.ErrNoPermission.WrapMsg("only app manager can send message")) return } diff --git a/internal/api/router.go b/internal/api/router.go index 920bd5366..7c2c78c7d 100644 --- a/internal/api/router.go +++ b/internal/api/router.go @@ -9,6 +9,9 @@ import ( "github.com/gin-gonic/gin" "github.com/gin-gonic/gin/binding" "github.com/go-playground/validator/v10" + "github.com/openimsdk/open-im-server/v3/pkg/authverify" + "github.com/openimsdk/tools/mcontext" + "github.com/openimsdk/tools/utils/datautil" clientv3 "go.etcd.io/etcd/client/v3" "github.com/openimsdk/open-im-server/v3/internal/api/jssdk" @@ -97,7 +100,7 @@ func newGinRouter(ctx context.Context, client discovery.Conn, cfg *Config) (*gin r.Use(gzip.Gzip(gzip.BestSpeed)) } r.Use(prommetricsGin(), gin.RecoveryWithWriter(gin.DefaultErrorWriter, mw.GinPanicErr), mw.CorsHandler(), - mw.GinParseOperationID(), GinParseToken(rpcli.NewAuthClient(authConn))) + mw.GinParseOperationID(), GinParseToken(rpcli.NewAuthClient(authConn)), setGinIsAdmin(cfg.Share.IMAdminUserID)) u := NewUserApi(user.NewUserClient(userConn), client, cfg.Discovery.RpcService) { @@ -354,6 +357,14 @@ func GinParseToken(authClient *rpcli.AuthClient) gin.HandlerFunc { } } +func setGinIsAdmin(imAdminUserID []string) gin.HandlerFunc { + return func(c *gin.Context) { + opUserID := mcontext.GetOpUserID(c) + admin := datautil.Contain(opUserID, imAdminUserID...) + c.Set(authverify.CtxIsAdminKey, admin) + } +} + // Whitelist api not parse token var Whitelist = []string{ "/auth/get_admin_token", diff --git a/internal/msggateway/hub_server.go b/internal/msggateway/hub_server.go index 376a697fd..3d0d1e3f9 100644 --- a/internal/msggateway/hub_server.go +++ b/internal/msggateway/hub_server.go @@ -100,7 +100,7 @@ func NewServer(longConnServer LongConnServer, conf *Config, ready func(srv *Serv } func (s *Server) GetUsersOnlineStatus(ctx context.Context, req *msggateway.GetUsersOnlineStatusReq) (*msggateway.GetUsersOnlineStatusResp, error) { - if !authverify.IsAppManagerUid(ctx, s.config.Share.IMAdminUserID) { + if !authverify.IsAdmin(ctx) { return nil, errs.ErrNoPermission.WrapMsg("only app manager") } var resp msggateway.GetUsersOnlineStatusResp diff --git a/internal/rpc/auth/auth.go b/internal/rpc/auth/auth.go index 6697e27c0..2c2691d1d 100644 --- a/internal/rpc/auth/auth.go +++ b/internal/rpc/auth/auth.go @@ -125,7 +125,7 @@ func (s *authServer) GetAdminToken(ctx context.Context, req *pbauth.GetAdminToke } func (s *authServer) GetUserToken(ctx context.Context, req *pbauth.GetUserTokenReq) (*pbauth.GetUserTokenResp, error) { - if err := authverify.CheckAdmin(ctx, s.config.Share.IMAdminUserID); err != nil { + if err := authverify.CheckAdmin(ctx); err != nil { return nil, err } @@ -135,7 +135,7 @@ func (s *authServer) GetUserToken(ctx context.Context, req *pbauth.GetUserTokenR resp := pbauth.GetUserTokenResp{} - if authverify.IsManagerUserID(req.UserID, s.config.Share.IMAdminUserID) { + if authverify.CheckUserIsAdmin(ctx, req.UserID) { return nil, errs.ErrNoPermission.WrapMsg("don't get Admin token") } user, err := s.userClient.GetUserInfo(ctx, req.UserID) @@ -164,7 +164,7 @@ func (s *authServer) parseToken(ctx context.Context, tokensString string) (claim return nil, err } if len(m) == 0 { - isAdmin := authverify.IsManagerUserID(claims.UserID, s.config.Share.IMAdminUserID) + isAdmin := authverify.CheckUserIsAdmin(ctx, claims.UserID) if isAdmin { if err = s.authDatabase.GetTemporaryTokensWithoutError(ctx, claims.UserID, claims.PlatformID, tokensString); err == nil { return claims, nil @@ -182,7 +182,7 @@ func (s *authServer) parseToken(ctx context.Context, tokensString string) (claim return nil, errs.Wrap(errs.ErrTokenUnknown) } } else { - isAdmin := authverify.IsManagerUserID(claims.UserID, s.config.Share.IMAdminUserID) + isAdmin := authverify.CheckUserIsAdmin(ctx, claims.UserID) if isAdmin { if err = s.authDatabase.GetTemporaryTokensWithoutError(ctx, claims.UserID, claims.PlatformID, tokensString); err == nil { return claims, nil @@ -205,7 +205,7 @@ func (s *authServer) ParseToken(ctx context.Context, req *pbauth.ParseTokenReq) } func (s *authServer) ForceLogout(ctx context.Context, req *pbauth.ForceLogoutReq) (*pbauth.ForceLogoutResp, error) { - if err := authverify.CheckAdmin(ctx, s.config.Share.IMAdminUserID); err != nil { + if err := authverify.CheckAdmin(ctx); err != nil { return nil, err } if err := s.forceKickOff(ctx, req.UserID, req.PlatformID); err != nil { diff --git a/internal/rpc/conversation/sync.go b/internal/rpc/conversation/sync.go index ad88b2bbd..cee74b319 100644 --- a/internal/rpc/conversation/sync.go +++ b/internal/rpc/conversation/sync.go @@ -4,12 +4,16 @@ import ( "context" "github.com/openimsdk/open-im-server/v3/internal/rpc/incrversion" + "github.com/openimsdk/open-im-server/v3/pkg/authverify" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" "github.com/openimsdk/open-im-server/v3/pkg/util/hashutil" "github.com/openimsdk/protocol/conversation" ) func (c *conversationServer) GetFullOwnerConversationIDs(ctx context.Context, req *conversation.GetFullOwnerConversationIDsReq) (*conversation.GetFullOwnerConversationIDsResp, error) { + if err := authverify.CheckAccess(ctx, req.UserID); err != nil { + return nil, err + } vl, err := c.conversationDatabase.FindMaxConversationUserVersionCache(ctx, req.UserID) if err != nil { return nil, err diff --git a/internal/rpc/group/group.go b/internal/rpc/group/group.go index 455b77635..10cdc2546 100644 --- a/internal/rpc/group/group.go +++ b/internal/rpc/group/group.go @@ -152,7 +152,7 @@ func (g *groupServer) NotificationUserInfoUpdate(ctx context.Context, req *pbgro } func (g *groupServer) CheckGroupAdmin(ctx context.Context, groupID string) error { - if !authverify.IsAppManagerUid(ctx, g.config.Share.IMAdminUserID) { + if !authverify.IsAdmin(ctx) { groupMember, err := g.db.TakeGroupMember(ctx, groupID, mcontext.GetOpUserID(ctx)) if err != nil { return err @@ -204,7 +204,7 @@ func (g *groupServer) CreateGroup(ctx context.Context, req *pbgroup.CreateGroupR if req.OwnerUserID == "" { return nil, errs.ErrArgs.WrapMsg("no group owner") } - if err := authverify.CheckAccessV3(ctx, req.OwnerUserID, g.config.Share.IMAdminUserID); err != nil { + if err := authverify.CheckAccess(ctx, req.OwnerUserID); err != nil { return nil, err } userIDs := append(append(req.MemberUserIDs, req.AdminUserIDs...), req.OwnerUserID) @@ -308,7 +308,7 @@ func (g *groupServer) CreateGroup(ctx context.Context, req *pbgroup.CreateGroupR } func (g *groupServer) GetJoinedGroupList(ctx context.Context, req *pbgroup.GetJoinedGroupListReq) (*pbgroup.GetJoinedGroupListResp, error) { - if err := authverify.CheckAccessV3(ctx, req.FromUserID, g.config.Share.IMAdminUserID); err != nil { + if err := authverify.CheckAccess(ctx, req.FromUserID); err != nil { return nil, err } total, members, err := g.db.PageGetJoinGroup(ctx, req.FromUserID, req.Pagination) @@ -380,7 +380,7 @@ func (g *groupServer) InviteUserToGroup(ctx context.Context, req *pbgroup.Invite var groupMember *model.GroupMember var opUserID string - if !authverify.IsAppManagerUid(ctx, g.config.Share.IMAdminUserID) { + if !authverify.IsAdmin(ctx) { opUserID = mcontext.GetOpUserID(ctx) var err error groupMember, err = g.db.TakeGroupMember(ctx, req.GroupID, opUserID) @@ -399,7 +399,7 @@ func (g *groupServer) InviteUserToGroup(ctx context.Context, req *pbgroup.Invite } if group.NeedVerification == constant.AllNeedVerification { - if !authverify.IsAppManagerUid(ctx, g.config.Share.IMAdminUserID) { + if !authverify.IsAdmin(ctx) { if !(groupMember.RoleLevel == constant.GroupOwner || groupMember.RoleLevel == constant.GroupAdmin) { var requests []*model.GroupRequest for _, userID := range req.InvitedUserIDs { @@ -487,6 +487,11 @@ func (g *groupServer) GetGroupAllMember(ctx context.Context, req *pbgroup.GetGro } func (g *groupServer) GetGroupMemberList(ctx context.Context, req *pbgroup.GetGroupMemberListReq) (*pbgroup.GetGroupMemberListResp, error) { + if opUserID := mcontext.GetOpUserID(ctx); !datautil.Contain(opUserID, g.config.Share.IMAdminUserID...) { + if _, err := g.db.TakeGroupMember(ctx, req.GroupID, opUserID); err != nil { + return nil, err + } + } var ( total int64 members []*model.GroupMember @@ -495,7 +500,7 @@ func (g *groupServer) GetGroupMemberList(ctx context.Context, req *pbgroup.GetGr if req.Keyword == "" { total, members, err = g.db.PageGetGroupMember(ctx, req.GroupID, req.Pagination) } else { - members, err = g.db.FindGroupMemberAll(ctx, req.GroupID) + total, members, err = g.db.SearchGroupMember(ctx, req.GroupID, req.Keyword, req.Pagination) } if err != nil { return nil, err @@ -503,27 +508,6 @@ func (g *groupServer) GetGroupMemberList(ctx context.Context, req *pbgroup.GetGr if err := g.PopulateGroupMember(ctx, members...); err != nil { return nil, err } - if req.Keyword != "" { - groupMembers := make([]*model.GroupMember, 0) - for _, member := range members { - if member.UserID == req.Keyword { - groupMembers = append(groupMembers, member) - total++ - continue - } - if member.Nickname == req.Keyword { - groupMembers = append(groupMembers, member) - total++ - continue - } - } - - members := datautil.Paginate(groupMembers, int(req.Pagination.GetPageNumber()), int(req.Pagination.GetShowNumber())) - return &pbgroup.GetGroupMemberListResp{ - Total: uint32(total), - Members: datautil.Batch(convert.Db2PbGroupMember, members), - }, nil - } return &pbgroup.GetGroupMemberListResp{ Total: uint32(total), Members: datautil.Batch(convert.Db2PbGroupMember, members), @@ -564,7 +548,7 @@ func (g *groupServer) KickGroupMember(ctx context.Context, req *pbgroup.KickGrou for i, member := range members { memberMap[member.UserID] = members[i] } - isAppManagerUid := authverify.IsAppManagerUid(ctx, g.config.Share.IMAdminUserID) + isAppManagerUid := authverify.IsAdmin(ctx) opMember := memberMap[opUserID] for _, userID := range req.KickedUserIDs { member, ok := memberMap[userID] @@ -782,7 +766,7 @@ func (g *groupServer) GroupApplicationResponse(ctx context.Context, req *pbgroup if !datautil.Contain(req.HandleResult, constant.GroupResponseAgree, constant.GroupResponseRefuse) { return nil, errs.ErrArgs.WrapMsg("HandleResult unknown") } - if !authverify.IsAppManagerUid(ctx, g.config.Share.IMAdminUserID) { + if !authverify.IsAdmin(ctx) { groupMember, err := g.db.TakeGroupMember(ctx, req.GroupID, mcontext.GetOpUserID(ctx)) if err != nil { return nil, err @@ -936,7 +920,7 @@ func (g *groupServer) QuitGroup(ctx context.Context, req *pbgroup.QuitGroupReq) if req.UserID == "" { req.UserID = mcontext.GetOpUserID(ctx) } else { - if err := authverify.CheckAccessV3(ctx, req.UserID, g.config.Share.IMAdminUserID); err != nil { + if err := authverify.CheckAccess(ctx, req.UserID); err != nil { return nil, err } } @@ -974,7 +958,7 @@ func (g *groupServer) deleteMemberAndSetConversationSeq(ctx context.Context, gro func (g *groupServer) SetGroupInfo(ctx context.Context, req *pbgroup.SetGroupInfoReq) (*pbgroup.SetGroupInfoResp, error) { var opMember *model.GroupMember - if !authverify.IsAppManagerUid(ctx, g.config.Share.IMAdminUserID) { + if !authverify.IsAdmin(ctx) { var err error opMember, err = g.db.TakeGroupMember(ctx, req.GroupInfoForSet.GroupID, mcontext.GetOpUserID(ctx)) if err != nil { @@ -1068,7 +1052,7 @@ func (g *groupServer) SetGroupInfo(ctx context.Context, req *pbgroup.SetGroupInf func (g *groupServer) SetGroupInfoEx(ctx context.Context, req *pbgroup.SetGroupInfoExReq) (*pbgroup.SetGroupInfoExResp, error) { var opMember *model.GroupMember - if !authverify.IsAppManagerUid(ctx, g.config.Share.IMAdminUserID) { + if !authverify.IsAdmin(ctx) { var err error opMember, err = g.db.TakeGroupMember(ctx, req.GroupID, mcontext.GetOpUserID(ctx)) @@ -1217,7 +1201,7 @@ func (g *groupServer) TransferGroupOwner(ctx context.Context, req *pbgroup.Trans return nil, errs.ErrArgs.WrapMsg("NewOwnerUser not in group " + req.NewOwnerUserID) } - if !authverify.IsAppManagerUid(ctx, g.config.Share.IMAdminUserID) { + if !authverify.IsAdmin(ctx) { if !(mcontext.GetOpUserID(ctx) == oldOwner.UserID && oldOwner.RoleLevel == constant.GroupOwner) { return nil, errs.ErrNoPermission.WrapMsg("no permission transfer group owner") } @@ -1360,7 +1344,7 @@ func (g *groupServer) DismissGroup(ctx context.Context, req *pbgroup.DismissGrou if err != nil { return nil, err } - if !authverify.IsAppManagerUid(ctx, g.config.Share.IMAdminUserID) { + if !authverify.IsAdmin(ctx) { if owner.UserID != mcontext.GetOpUserID(ctx) { return nil, errs.ErrNoPermission.WrapMsg("not group owner") } @@ -1417,7 +1401,7 @@ func (g *groupServer) MuteGroupMember(ctx context.Context, req *pbgroup.MuteGrou if err := g.PopulateGroupMember(ctx, member); err != nil { return nil, err } - if !authverify.IsAppManagerUid(ctx, g.config.Share.IMAdminUserID) { + if !authverify.IsAdmin(ctx) { opMember, err := g.db.TakeGroupMember(ctx, req.GroupID, mcontext.GetOpUserID(ctx)) if err != nil { return nil, err @@ -1453,7 +1437,7 @@ func (g *groupServer) CancelMuteGroupMember(ctx context.Context, req *pbgroup.Ca return nil, err } - if !authverify.IsAppManagerUid(ctx, g.config.Share.IMAdminUserID) { + if !authverify.IsAdmin(ctx) { opMember, err := g.db.TakeGroupMember(ctx, req.GroupID, mcontext.GetOpUserID(ctx)) if err != nil { return nil, err @@ -1513,7 +1497,7 @@ func (g *groupServer) SetGroupMemberInfo(ctx context.Context, req *pbgroup.SetGr if opUserID == "" { return nil, errs.ErrNoPermission.WrapMsg("no op user id") } - isAppManagerUid := authverify.IsAppManagerUid(ctx, g.config.Share.IMAdminUserID) + isAppManagerUid := authverify.IsAdmin(ctx) groupMembers := make(map[string][]*pbgroup.SetGroupMemberInfo) for i, member := range req.Members { if member.RoleLevel != nil { diff --git a/internal/rpc/group/notification.go b/internal/rpc/group/notification.go index 43becf4f2..0a18371f9 100644 --- a/internal/rpc/group/notification.go +++ b/internal/rpc/group/notification.go @@ -243,7 +243,7 @@ func (g *NotificationSender) fillUserByUserID(ctx context.Context, userID string return errs.ErrInternalServer.WrapMsg("**sdkws.GroupMemberFullInfo is nil") } if groupID != "" { - if authverify.IsManagerUserID(userID, g.config.Share.IMAdminUserID) { + if authverify.CheckUserIsAdmin(ctx, userID) { *targetUser = &sdkws.GroupMemberFullInfo{ GroupID: groupID, UserID: userID, diff --git a/internal/rpc/group/sync.go b/internal/rpc/group/sync.go index d311ae076..822b15307 100644 --- a/internal/rpc/group/sync.go +++ b/internal/rpc/group/sync.go @@ -11,16 +11,24 @@ import ( "github.com/openimsdk/protocol/constant" pbgroup "github.com/openimsdk/protocol/group" "github.com/openimsdk/protocol/sdkws" + "github.com/openimsdk/tools/errs" + "github.com/openimsdk/tools/mcontext" + "github.com/openimsdk/tools/utils/datautil" ) const versionSyncLimit = 500 func (g *groupServer) GetFullGroupMemberUserIDs(ctx context.Context, req *pbgroup.GetFullGroupMemberUserIDsReq) (*pbgroup.GetFullGroupMemberUserIDsResp, error) { - vl, err := g.db.FindMaxGroupMemberVersionCache(ctx, req.GroupID) + userIDs, err := g.db.FindGroupMemberUserID(ctx, req.GroupID) if err != nil { return nil, err } - userIDs, err := g.db.FindGroupMemberUserID(ctx, req.GroupID) + if opUserID := mcontext.GetOpUserID(ctx); !datautil.Contain(opUserID, g.config.Share.IMAdminUserID...) { + if !datautil.Contain(opUserID, userIDs...) { + return nil, errs.ErrNoPermission.WrapMsg("user not in group") + } + } + vl, err := g.db.FindMaxGroupMemberVersionCache(ctx, req.GroupID) if err != nil { return nil, err } @@ -37,6 +45,9 @@ func (g *groupServer) GetFullGroupMemberUserIDs(ctx context.Context, req *pbgrou } func (g *groupServer) GetFullJoinGroupIDs(ctx context.Context, req *pbgroup.GetFullJoinGroupIDsReq) (*pbgroup.GetFullJoinGroupIDsResp, error) { + if err := authverify.CheckAccess(ctx, req.UserID); err != nil { + return nil, err + } vl, err := g.db.FindMaxJoinGroupVersionCache(ctx, req.UserID) if err != nil { return nil, err @@ -65,6 +76,9 @@ func (g *groupServer) GetIncrementalGroupMember(ctx context.Context, req *pbgrou if group.Status == constant.GroupStatusDismissed { return nil, servererrs.ErrDismissedAlready.Wrap() } + if _, err := g.db.TakeGroupMember(ctx, req.GroupID, mcontext.GetOpUserID(ctx)); err != nil { + return nil, err + } var ( hasGroupUpdate bool sortVersion uint64 @@ -133,7 +147,7 @@ func (g *groupServer) GetIncrementalGroupMember(ctx context.Context, req *pbgrou } func (g *groupServer) GetIncrementalJoinGroup(ctx context.Context, req *pbgroup.GetIncrementalJoinGroupReq) (*pbgroup.GetIncrementalJoinGroupResp, error) { - if err := authverify.CheckAccessV3(ctx, req.UserID, g.config.Share.IMAdminUserID); err != nil { + if err := authverify.CheckAccess(ctx, req.UserID); err != nil { return nil, err } opt := incrversion.Option[*sdkws.GroupInfo, pbgroup.GetIncrementalJoinGroupResp]{ diff --git a/internal/rpc/msg/clear.go b/internal/rpc/msg/clear.go index 8e14b281e..96eb99aed 100644 --- a/internal/rpc/msg/clear.go +++ b/internal/rpc/msg/clear.go @@ -2,15 +2,16 @@ package msg import ( "context" + "strings" + "github.com/openimsdk/open-im-server/v3/pkg/authverify" "github.com/openimsdk/protocol/msg" "github.com/openimsdk/tools/log" - "strings" ) // DestructMsgs hard delete in Database. func (m *msgServer) DestructMsgs(ctx context.Context, req *msg.DestructMsgsReq) (*msg.DestructMsgsResp, error) { - if err := authverify.CheckAdmin(ctx, m.config.Share.IMAdminUserID); err != nil { + if err := authverify.CheckAdmin(ctx); err != nil { return nil, err } docs, err := m.MsgDatabase.GetRandBeforeMsg(ctx, req.Timestamp, int(req.Limit)) diff --git a/internal/rpc/msg/delete.go b/internal/rpc/msg/delete.go index d3485faaa..4590523d5 100644 --- a/internal/rpc/msg/delete.go +++ b/internal/rpc/msg/delete.go @@ -42,7 +42,7 @@ func (m *msgServer) validateDeleteSyncOpt(opt *msg.DeleteSyncOpt) (isSyncSelf, i } func (m *msgServer) ClearConversationsMsg(ctx context.Context, req *msg.ClearConversationsMsgReq) (*msg.ClearConversationsMsgResp, error) { - if err := authverify.CheckAccessV3(ctx, req.UserID, m.config.Share.IMAdminUserID); err != nil { + if err := authverify.CheckAccess(ctx, req.UserID); err != nil { return nil, err } if err := m.clearConversation(ctx, req.ConversationIDs, req.UserID, req.DeleteSyncOpt); err != nil { @@ -52,7 +52,7 @@ func (m *msgServer) ClearConversationsMsg(ctx context.Context, req *msg.ClearCon } func (m *msgServer) UserClearAllMsg(ctx context.Context, req *msg.UserClearAllMsgReq) (*msg.UserClearAllMsgResp, error) { - if err := authverify.CheckAccessV3(ctx, req.UserID, m.config.Share.IMAdminUserID); err != nil { + if err := authverify.CheckAccess(ctx, req.UserID); err != nil { return nil, err } conversationIDs, err := m.ConversationLocalCache.GetConversationIDs(ctx, req.UserID) @@ -66,7 +66,7 @@ func (m *msgServer) UserClearAllMsg(ctx context.Context, req *msg.UserClearAllMs } func (m *msgServer) DeleteMsgs(ctx context.Context, req *msg.DeleteMsgsReq) (*msg.DeleteMsgsResp, error) { - if err := authverify.CheckAccessV3(ctx, req.UserID, m.config.Share.IMAdminUserID); err != nil { + if err := authverify.CheckAccess(ctx, req.UserID); err != nil { return nil, err } isSyncSelf, isSyncOther := m.validateDeleteSyncOpt(req.DeleteSyncOpt) @@ -102,7 +102,7 @@ func (m *msgServer) DeleteMsgPhysicalBySeq(ctx context.Context, req *msg.DeleteM } func (m *msgServer) DeleteMsgPhysical(ctx context.Context, req *msg.DeleteMsgPhysicalReq) (*msg.DeleteMsgPhysicalResp, error) { - if err := authverify.CheckAdmin(ctx, m.config.Share.IMAdminUserID); err != nil { + if err := authverify.CheckAdmin(ctx); err != nil { return nil, err } remainTime := timeutil.GetCurrentTimestampBySecond() - req.Timestamp diff --git a/internal/rpc/msg/revoke.go b/internal/rpc/msg/revoke.go index c2fb5833f..bd1d66ba1 100644 --- a/internal/rpc/msg/revoke.go +++ b/internal/rpc/msg/revoke.go @@ -42,7 +42,7 @@ func (m *msgServer) RevokeMsg(ctx context.Context, req *msg.RevokeMsgReq) (*msg. if req.Seq < 0 { return nil, errs.ErrArgs.WrapMsg("seq is invalid") } - if err := authverify.CheckAccessV3(ctx, req.UserID, m.config.Share.IMAdminUserID); err != nil { + if err := authverify.CheckAccess(ctx, req.UserID); err != nil { return nil, err } user, err := m.UserLocalCache.GetUserInfo(ctx, req.UserID) @@ -63,11 +63,11 @@ func (m *msgServer) RevokeMsg(ctx context.Context, req *msg.RevokeMsgReq) (*msg. data, _ := json.Marshal(msgs[0]) log.ZDebug(ctx, "GetMsgBySeqs", "conversationID", req.ConversationID, "seq", req.Seq, "msg", string(data)) var role int32 - if !authverify.IsAppManagerUid(ctx, m.config.Share.IMAdminUserID) { + if !authverify.IsAdmin(ctx) { sessionType := msgs[0].SessionType switch sessionType { case constant.SingleChatType: - if err := authverify.CheckAccessV3(ctx, msgs[0].SendID, m.config.Share.IMAdminUserID); err != nil { + if err := authverify.CheckAccess(ctx, msgs[0].SendID); err != nil { return nil, err } role = user.AppMangerLevel diff --git a/internal/rpc/msg/sync_msg.go b/internal/rpc/msg/sync_msg.go index 6cf1c21d3..38eed93bc 100644 --- a/internal/rpc/msg/sync_msg.go +++ b/internal/rpc/msg/sync_msg.go @@ -118,7 +118,7 @@ func (m *msgServer) GetSeqMessage(ctx context.Context, req *msg.GetSeqMessageReq } func (m *msgServer) GetMaxSeq(ctx context.Context, req *sdkws.GetMaxSeqReq) (*sdkws.GetMaxSeqResp, error) { - if err := authverify.CheckAccessV3(ctx, req.UserID, m.config.Share.IMAdminUserID); err != nil { + if err := authverify.CheckAccess(ctx, req.UserID); err != nil { return nil, err } conversationIDs, err := m.ConversationLocalCache.GetConversationIDs(ctx, req.UserID) diff --git a/internal/rpc/relation/black.go b/internal/rpc/relation/black.go index 2108d7dc5..4a0e60ecd 100644 --- a/internal/rpc/relation/black.go +++ b/internal/rpc/relation/black.go @@ -29,10 +29,9 @@ import ( ) func (s *friendServer) GetPaginationBlacks(ctx context.Context, req *relation.GetPaginationBlacksReq) (resp *relation.GetPaginationBlacksResp, err error) { - if err := authverify.CheckAccessV3(ctx, req.UserID, s.config.Share.IMAdminUserID); err != nil { + if err := authverify.CheckAccess(ctx, req.UserID); err != nil { return nil, err } - total, blacks, err := s.blackDatabase.FindOwnerBlacks(ctx, req.UserID, req.Pagination) if err != nil { return nil, err @@ -58,7 +57,7 @@ func (s *friendServer) IsBlack(ctx context.Context, req *relation.IsBlackReq) (* } func (s *friendServer) RemoveBlack(ctx context.Context, req *relation.RemoveBlackReq) (*relation.RemoveBlackResp, error) { - if err := authverify.CheckAccessV3(ctx, req.OwnerUserID, s.config.Share.IMAdminUserID); err != nil { + if err := authverify.CheckAccess(ctx, req.OwnerUserID); err != nil { return nil, err } @@ -73,7 +72,7 @@ func (s *friendServer) RemoveBlack(ctx context.Context, req *relation.RemoveBlac } func (s *friendServer) AddBlack(ctx context.Context, req *relation.AddBlackReq) (*relation.AddBlackResp, error) { - if err := authverify.CheckAccessV3(ctx, req.OwnerUserID, s.config.Share.IMAdminUserID); err != nil { + if err := authverify.CheckAccess(ctx, req.OwnerUserID); err != nil { return nil, err } @@ -99,7 +98,7 @@ func (s *friendServer) AddBlack(ctx context.Context, req *relation.AddBlackReq) } func (s *friendServer) GetSpecifiedBlacks(ctx context.Context, req *relation.GetSpecifiedBlacksReq) (*relation.GetSpecifiedBlacksResp, error) { - if err := authverify.CheckAccessV3(ctx, req.OwnerUserID, s.config.Share.IMAdminUserID); err != nil { + if err := authverify.CheckAccess(ctx, req.OwnerUserID); err != nil { return nil, err } diff --git a/internal/rpc/relation/friend.go b/internal/rpc/relation/friend.go index a4908d924..050cd5ffe 100644 --- a/internal/rpc/relation/friend.go +++ b/internal/rpc/relation/friend.go @@ -134,7 +134,7 @@ func Start(ctx context.Context, config *Config, client discovery.Conn, server gr // ok. func (s *friendServer) ApplyToAddFriend(ctx context.Context, req *relation.ApplyToAddFriendReq) (resp *relation.ApplyToAddFriendResp, err error) { resp = &relation.ApplyToAddFriendResp{} - if err := authverify.CheckAccessV3(ctx, req.FromUserID, s.config.Share.IMAdminUserID); err != nil { + if err := authverify.CheckAccess(ctx, req.FromUserID); err != nil { return nil, err } if req.ToUserID == req.FromUserID { @@ -164,7 +164,7 @@ func (s *friendServer) ApplyToAddFriend(ctx context.Context, req *relation.Apply // ok. func (s *friendServer) ImportFriends(ctx context.Context, req *relation.ImportFriendReq) (resp *relation.ImportFriendResp, err error) { - if err := authverify.CheckAdmin(ctx, s.config.Share.IMAdminUserID); err != nil { + if err := authverify.CheckAdmin(ctx); err != nil { return nil, err } @@ -200,7 +200,7 @@ func (s *friendServer) ImportFriends(ctx context.Context, req *relation.ImportFr // ok. func (s *friendServer) RespondFriendApply(ctx context.Context, req *relation.RespondFriendApplyReq) (resp *relation.RespondFriendApplyResp, err error) { resp = &relation.RespondFriendApplyResp{} - if err := authverify.CheckAccessV3(ctx, req.ToUserID, s.config.Share.IMAdminUserID); err != nil { + if err := authverify.CheckAccess(ctx, req.ToUserID); err != nil { return nil, err } @@ -235,7 +235,7 @@ func (s *friendServer) RespondFriendApply(ctx context.Context, req *relation.Res // ok. func (s *friendServer) DeleteFriend(ctx context.Context, req *relation.DeleteFriendReq) (resp *relation.DeleteFriendResp, err error) { - if err := authverify.CheckAccessV3(ctx, req.OwnerUserID, s.config.Share.IMAdminUserID); err != nil { + if err := authverify.CheckAccess(ctx, req.OwnerUserID); err != nil { return nil, err } @@ -260,7 +260,7 @@ func (s *friendServer) SetFriendRemark(ctx context.Context, req *relation.SetFri return nil, err } - if err := authverify.CheckAccessV3(ctx, req.OwnerUserID, s.config.Share.IMAdminUserID); err != nil { + if err := authverify.CheckAccess(ctx, req.OwnerUserID); err != nil { return nil, err } @@ -330,7 +330,7 @@ func (s *friendServer) GetDesignatedFriendsApply(ctx context.Context, // Get received friend requests (i.e., those initiated by others). func (s *friendServer) GetPaginationFriendsApplyTo(ctx context.Context, req *relation.GetPaginationFriendsApplyToReq) (resp *relation.GetPaginationFriendsApplyToResp, err error) { - if err := authverify.CheckAccessV3(ctx, req.UserID, s.config.Share.IMAdminUserID); err != nil { + if err := authverify.CheckAccess(ctx, req.UserID); err != nil { return nil, err } @@ -353,7 +353,7 @@ func (s *friendServer) GetPaginationFriendsApplyTo(ctx context.Context, req *rel func (s *friendServer) GetPaginationFriendsApplyFrom(ctx context.Context, req *relation.GetPaginationFriendsApplyFromReq) (resp *relation.GetPaginationFriendsApplyFromResp, err error) { resp = &relation.GetPaginationFriendsApplyFromResp{} - if err := authverify.CheckAccessV3(ctx, req.UserID, s.config.Share.IMAdminUserID); err != nil { + if err := authverify.CheckAccess(ctx, req.UserID); err != nil { return nil, err } @@ -383,7 +383,7 @@ func (s *friendServer) IsFriend(ctx context.Context, req *relation.IsFriendReq) } func (s *friendServer) GetPaginationFriends(ctx context.Context, req *relation.GetPaginationFriendsReq) (resp *relation.GetPaginationFriendsResp, err error) { - if err := authverify.CheckAccessV3(ctx, req.UserID, s.config.Share.IMAdminUserID); err != nil { + if err := authverify.CheckAccess(ctx, req.UserID); err != nil { return nil, err } @@ -404,7 +404,7 @@ func (s *friendServer) GetPaginationFriends(ctx context.Context, req *relation.G } func (s *friendServer) GetFriendIDs(ctx context.Context, req *relation.GetFriendIDsReq) (resp *relation.GetFriendIDsResp, err error) { - if err := authverify.CheckAccessV3(ctx, req.UserID, s.config.Share.IMAdminUserID); err != nil { + if err := authverify.CheckAccess(ctx, req.UserID); err != nil { return nil, err } diff --git a/internal/rpc/relation/sync.go b/internal/rpc/relation/sync.go index 0ad94fe82..79fa0858c 100644 --- a/internal/rpc/relation/sync.go +++ b/internal/rpc/relation/sync.go @@ -2,10 +2,11 @@ package relation import ( "context" + "slices" + "github.com/openimsdk/open-im-server/v3/pkg/util/hashutil" "github.com/openimsdk/protocol/sdkws" "github.com/openimsdk/tools/log" - "slices" "github.com/openimsdk/open-im-server/v3/internal/rpc/incrversion" "github.com/openimsdk/open-im-server/v3/pkg/authverify" @@ -39,6 +40,9 @@ func (s *friendServer) NotificationUserInfoUpdate(ctx context.Context, req *rela } func (s *friendServer) GetFullFriendUserIDs(ctx context.Context, req *relation.GetFullFriendUserIDsReq) (*relation.GetFullFriendUserIDsResp, error) { + if err := authverify.CheckAccess(ctx, req.UserID); err != nil { + return nil, err + } vl, err := s.db.FindMaxFriendVersionCache(ctx, req.UserID) if err != nil { return nil, err @@ -60,7 +64,7 @@ func (s *friendServer) GetFullFriendUserIDs(ctx context.Context, req *relation.G } func (s *friendServer) GetIncrementalFriends(ctx context.Context, req *relation.GetIncrementalFriendsReq) (*relation.GetIncrementalFriendsResp, error) { - if err := authverify.CheckAccessV3(ctx, req.UserID, s.config.Share.IMAdminUserID); err != nil { + if err := authverify.CheckAccess(ctx, req.UserID); err != nil { return nil, err } var sortVersion uint64 diff --git a/internal/rpc/third/log.go b/internal/rpc/third/log.go index 4d8cbc0bb..fba3ecb88 100644 --- a/internal/rpc/third/log.go +++ b/internal/rpc/third/log.go @@ -82,7 +82,7 @@ func (t *thirdServer) UploadLogs(ctx context.Context, req *third.UploadLogsReq) } func (t *thirdServer) DeleteLogs(ctx context.Context, req *third.DeleteLogsReq) (*third.DeleteLogsResp, error) { - if err := authverify.CheckAdmin(ctx, t.config.Share.IMAdminUserID); err != nil { + if err := authverify.CheckAdmin(ctx); err != nil { return nil, err } userID := "" @@ -123,7 +123,7 @@ func dbToPbLogInfos(logs []*relationtb.Log) []*third.LogInfo { } func (t *thirdServer) SearchLogs(ctx context.Context, req *third.SearchLogsReq) (*third.SearchLogsResp, error) { - if err := authverify.CheckAdmin(ctx, t.config.Share.IMAdminUserID); err != nil { + if err := authverify.CheckAdmin(ctx); err != nil { return nil, err } var ( diff --git a/internal/rpc/third/s3.go b/internal/rpc/third/s3.go index 757320dac..bfdd9f1b8 100644 --- a/internal/rpc/third/s3.go +++ b/internal/rpc/third/s3.go @@ -198,7 +198,7 @@ func (t *thirdServer) InitiateFormData(ctx context.Context, req *third.InitiateF var duration time.Duration opUserID := mcontext.GetOpUserID(ctx) var key string - if t.IsManagerUserID(opUserID) { + if authverify.CheckUserIsAdmin(ctx, opUserID) { if req.Millisecond <= 0 { duration = time.Minute * 10 } else { @@ -289,7 +289,7 @@ func (t *thirdServer) apiAddress(prefix, name string) string { } func (t *thirdServer) DeleteOutdatedData(ctx context.Context, req *third.DeleteOutdatedDataReq) (*third.DeleteOutdatedDataResp, error) { - if err := authverify.CheckAdmin(ctx, t.config.Share.IMAdminUserID); err != nil { + if err := authverify.CheckAdmin(ctx); err != nil { return nil, err } engine := t.config.RpcConfig.Object.Enable diff --git a/internal/rpc/third/tool.go b/internal/rpc/third/tool.go index 4e22ffbf9..a063fa654 100644 --- a/internal/rpc/third/tool.go +++ b/internal/rpc/third/tool.go @@ -54,7 +54,7 @@ func (t *thirdServer) checkUploadName(ctx context.Context, name string) error { if opUserID == "" { return errs.ErrNoPermission.WrapMsg("opUserID is empty") } - if !authverify.IsManagerUserID(opUserID, t.config.Share.IMAdminUserID) { + if !authverify.CheckUserIsAdmin(ctx, opUserID) { if !strings.HasPrefix(name, opUserID+"/") { return errs.ErrNoPermission.WrapMsg(fmt.Sprintf("name must start with `%s/`", opUserID)) } @@ -79,10 +79,6 @@ func checkValidObjectName(objectName string) error { return checkValidObjectNamePrefix(objectName) } -func (t *thirdServer) IsManagerUserID(opUserID string) bool { - return authverify.IsManagerUserID(opUserID, t.config.Share.IMAdminUserID) -} - func putUpdate[T any](update map[string]any, name string, val interface{ GetValuePtr() *T }) { ptrVal := val.GetValuePtr() if ptrVal == nil { diff --git a/internal/rpc/user/config.go b/internal/rpc/user/config.go index 5a9a46359..f3f5a7a96 100644 --- a/internal/rpc/user/config.go +++ b/internal/rpc/user/config.go @@ -11,7 +11,7 @@ import ( func (s *userServer) GetUserClientConfig(ctx context.Context, req *pbuser.GetUserClientConfigReq) (*pbuser.GetUserClientConfigResp, error) { if req.UserID != "" { - if err := authverify.CheckAccessV3(ctx, req.UserID, s.config.Share.IMAdminUserID); err != nil { + if err := authverify.CheckAccess(ctx, req.UserID); err != nil { return nil, err } if _, err := s.db.GetUserByID(ctx, req.UserID); err != nil { @@ -26,7 +26,7 @@ func (s *userServer) GetUserClientConfig(ctx context.Context, req *pbuser.GetUse } func (s *userServer) SetUserClientConfig(ctx context.Context, req *pbuser.SetUserClientConfigReq) (*pbuser.SetUserClientConfigResp, error) { - if err := authverify.CheckAdmin(ctx, s.config.Share.IMAdminUserID); err != nil { + if err := authverify.CheckAdmin(ctx); err != nil { return nil, err } if req.UserID != "" { @@ -41,7 +41,7 @@ func (s *userServer) SetUserClientConfig(ctx context.Context, req *pbuser.SetUse } func (s *userServer) DelUserClientConfig(ctx context.Context, req *pbuser.DelUserClientConfigReq) (*pbuser.DelUserClientConfigResp, error) { - if err := authverify.CheckAdmin(ctx, s.config.Share.IMAdminUserID); err != nil { + if err := authverify.CheckAdmin(ctx); err != nil { return nil, err } if err := s.clientConfig.DelUserConfig(ctx, req.UserID, req.Keys); err != nil { @@ -51,7 +51,7 @@ func (s *userServer) DelUserClientConfig(ctx context.Context, req *pbuser.DelUse } func (s *userServer) PageUserClientConfig(ctx context.Context, req *pbuser.PageUserClientConfigReq) (*pbuser.PageUserClientConfigResp, error) { - if err := authverify.CheckAdmin(ctx, s.config.Share.IMAdminUserID); err != nil { + if err := authverify.CheckAdmin(ctx); err != nil { return nil, err } total, res, err := s.clientConfig.GetUserConfigPage(ctx, req.UserID, req.Key, req.Pagination) diff --git a/internal/rpc/user/user.go b/internal/rpc/user/user.go index 6ef61f773..7f082f784 100644 --- a/internal/rpc/user/user.go +++ b/internal/rpc/user/user.go @@ -150,7 +150,7 @@ func (s *userServer) GetDesignateUsers(ctx context.Context, req *pbuser.GetDesig // UpdateUserInfo func (s *userServer) UpdateUserInfo(ctx context.Context, req *pbuser.UpdateUserInfoReq) (resp *pbuser.UpdateUserInfoResp, err error) { resp = &pbuser.UpdateUserInfoResp{} - err = authverify.CheckAccessV3(ctx, req.UserInfo.UserID, s.config.Share.IMAdminUserID) + err = authverify.CheckAccess(ctx, req.UserInfo.UserID) if err != nil { return nil, err } @@ -177,7 +177,7 @@ func (s *userServer) UpdateUserInfo(ctx context.Context, req *pbuser.UpdateUserI func (s *userServer) UpdateUserInfoEx(ctx context.Context, req *pbuser.UpdateUserInfoExReq) (resp *pbuser.UpdateUserInfoExResp, err error) { resp = &pbuser.UpdateUserInfoExResp{} - err = authverify.CheckAccessV3(ctx, req.UserInfo.UserID, s.config.Share.IMAdminUserID) + err = authverify.CheckAccess(ctx, req.UserInfo.UserID) if err != nil { return nil, err } @@ -235,8 +235,7 @@ func (s *userServer) AccountCheck(ctx context.Context, req *pbuser.AccountCheckR if datautil.Duplicate(req.CheckUserIDs) { return nil, errs.ErrArgs.WrapMsg("userID repeated") } - err = authverify.CheckAdmin(ctx, s.config.Share.IMAdminUserID) - if err != nil { + if err = authverify.CheckAdmin(ctx); err != nil { return nil, err } users, err := s.db.Find(ctx, req.CheckUserIDs) @@ -283,14 +282,12 @@ func (s *userServer) UserRegister(ctx context.Context, req *pbuser.UserRegisterR return nil, errs.ErrArgs.WrapMsg("users is empty") } // check if secret is changed - if s.config.Share.Secret == defaultSecret { - return nil, servererrs.ErrSecretNotChanged.Wrap() - } - - if err = authverify.CheckAdmin(ctx, s.config.Share.IMAdminUserID); err != nil { + //if s.config.Share.Secret == defaultSecret { + // return nil, servererrs.ErrSecretNotChanged.Wrap() + //} + if err = authverify.CheckAdmin(ctx); err != nil { return nil, err } - if datautil.DuplicateAny(req.Users, func(e *sdkws.UserInfo) string { return e.UserID }) { return nil, errs.ErrArgs.WrapMsg("userID repeated") } @@ -356,7 +353,7 @@ func (s *userServer) GetAllUserID(ctx context.Context, req *pbuser.GetAllUserIDR // ProcessUserCommandAdd user general function add. func (s *userServer) ProcessUserCommandAdd(ctx context.Context, req *pbuser.ProcessUserCommandAddReq) (*pbuser.ProcessUserCommandAddResp, error) { - err := authverify.CheckAccessV3(ctx, req.UserID, s.config.Share.IMAdminUserID) + err := authverify.CheckAccess(ctx, req.UserID) if err != nil { return nil, err } @@ -384,7 +381,7 @@ func (s *userServer) ProcessUserCommandAdd(ctx context.Context, req *pbuser.Proc // ProcessUserCommandDelete user general function delete. func (s *userServer) ProcessUserCommandDelete(ctx context.Context, req *pbuser.ProcessUserCommandDeleteReq) (*pbuser.ProcessUserCommandDeleteResp, error) { - err := authverify.CheckAccessV3(ctx, req.UserID, s.config.Share.IMAdminUserID) + err := authverify.CheckAccess(ctx, req.UserID) if err != nil { return nil, err } @@ -403,7 +400,7 @@ func (s *userServer) ProcessUserCommandDelete(ctx context.Context, req *pbuser.P // ProcessUserCommandUpdate user general function update. func (s *userServer) ProcessUserCommandUpdate(ctx context.Context, req *pbuser.ProcessUserCommandUpdateReq) (*pbuser.ProcessUserCommandUpdateResp, error) { - err := authverify.CheckAccessV3(ctx, req.UserID, s.config.Share.IMAdminUserID) + err := authverify.CheckAccess(ctx, req.UserID) if err != nil { return nil, err } @@ -432,7 +429,7 @@ func (s *userServer) ProcessUserCommandUpdate(ctx context.Context, req *pbuser.P func (s *userServer) ProcessUserCommandGet(ctx context.Context, req *pbuser.ProcessUserCommandGetReq) (*pbuser.ProcessUserCommandGetResp, error) { - err := authverify.CheckAccessV3(ctx, req.UserID, s.config.Share.IMAdminUserID) + err := authverify.CheckAccess(ctx, req.UserID) if err != nil { return nil, err } @@ -461,7 +458,7 @@ func (s *userServer) ProcessUserCommandGet(ctx context.Context, req *pbuser.Proc } func (s *userServer) ProcessUserCommandGetAll(ctx context.Context, req *pbuser.ProcessUserCommandGetAllReq) (*pbuser.ProcessUserCommandGetAllResp, error) { - err := authverify.CheckAccessV3(ctx, req.UserID, s.config.Share.IMAdminUserID) + err := authverify.CheckAccess(ctx, req.UserID) if err != nil { return nil, err } @@ -490,7 +487,7 @@ func (s *userServer) ProcessUserCommandGetAll(ctx context.Context, req *pbuser.P } func (s *userServer) AddNotificationAccount(ctx context.Context, req *pbuser.AddNotificationAccountReq) (*pbuser.AddNotificationAccountResp, error) { - if err := authverify.CheckAdmin(ctx, s.config.Share.IMAdminUserID); err != nil { + if err := authverify.CheckAdmin(ctx); err != nil { return nil, err } if req.AppMangerLevel < constant.AppNotificationAdmin { @@ -536,7 +533,7 @@ func (s *userServer) AddNotificationAccount(ctx context.Context, req *pbuser.Add } func (s *userServer) UpdateNotificationAccountInfo(ctx context.Context, req *pbuser.UpdateNotificationAccountInfoReq) (*pbuser.UpdateNotificationAccountInfoResp, error) { - if err := authverify.CheckAdmin(ctx, s.config.Share.IMAdminUserID); err != nil { + if err := authverify.CheckAdmin(ctx); err != nil { return nil, err } @@ -563,7 +560,7 @@ func (s *userServer) UpdateNotificationAccountInfo(ctx context.Context, req *pbu func (s *userServer) SearchNotificationAccount(ctx context.Context, req *pbuser.SearchNotificationAccountReq) (*pbuser.SearchNotificationAccountResp, error) { // Check if user is an admin - if err := authverify.CheckAdmin(ctx, s.config.Share.IMAdminUserID); err != nil { + if err := authverify.CheckAdmin(ctx); err != nil { return nil, err } diff --git a/pkg/authverify/token.go b/pkg/authverify/token.go index 872feb1cf..75fb1448b 100644 --- a/pkg/authverify/token.go +++ b/pkg/authverify/token.go @@ -31,32 +31,49 @@ func Secret(secret string) jwt.Keyfunc { } } -func CheckAccessV3(ctx context.Context, ownerUserID string, imAdminUserID []string) (err error) { - opUserID := mcontext.GetOpUserID(ctx) - if datautil.Contain(opUserID, imAdminUserID...) { - return nil - } - if opUserID == ownerUserID { +func CheckAdmin(ctx context.Context) error { + if IsAdmin(ctx) { return nil } - return servererrs.ErrNoPermission.WrapMsg("ownerUserID", ownerUserID) + return servererrs.ErrNoPermission.WrapMsg(fmt.Sprintf("user %s is not admin userID", mcontext.GetOpUserID(ctx))) } -func IsAppManagerUid(ctx context.Context, imAdminUserID []string) bool { - return datautil.Contain(mcontext.GetOpUserID(ctx), imAdminUserID...) +//func IsManagerUserID(opUserID string, imAdminUserID []string) bool { +// return datautil.Contain(opUserID, imAdminUserID...) +//} + +func CheckUserIsAdmin(ctx context.Context, userID string) bool { + return datautil.Contain(userID, GetIMAdminUserIDs(ctx)...) } -func CheckAdmin(ctx context.Context, imAdminUserID []string) error { - if datautil.Contain(mcontext.GetOpUserID(ctx), imAdminUserID...) { - return nil - } - return servererrs.ErrNoPermission.WrapMsg(fmt.Sprintf("user %s is not admin userID", mcontext.GetOpUserID(ctx))) +func CheckSystemAccount(ctx context.Context, level int32) bool { + return level >= constant.AppAdmin } -func IsManagerUserID(opUserID string, imAdminUserID []string) bool { - return datautil.Contain(opUserID, imAdminUserID...) +const ( + CtxIsAdminKey = "CtxIsAdminKey" +) + +func WithIMAdminUserIDs(ctx context.Context, imAdminUserID []string) context.Context { + return context.WithValue(ctx, CtxIsAdminKey, imAdminUserID) } -func CheckSystemAccount(ctx context.Context, level int32) bool { - return level >= constant.AppAdmin +func GetIMAdminUserIDs(ctx context.Context) []string { + imAdminUserID, _ := ctx.Value(CtxIsAdminKey).([]string) + return imAdminUserID +} + +func IsAdmin(ctx context.Context) bool { + return datautil.Contain(mcontext.GetOpUserID(ctx), GetIMAdminUserIDs(ctx)...) +} + +func CheckAccess(ctx context.Context, ownerUserID string) error { + opUserID := mcontext.GetOpUserID(ctx) + if opUserID == ownerUserID { + return nil + } + if datautil.Contain(mcontext.GetOpUserID(ctx), GetIMAdminUserIDs(ctx)...) { + return nil + } + return servererrs.ErrNoPermission.WrapMsg("ownerUserID", ownerUserID) } diff --git a/pkg/common/startrpc/mw.go b/pkg/common/startrpc/mw.go new file mode 100644 index 000000000..c6cd55380 --- /dev/null +++ b/pkg/common/startrpc/mw.go @@ -0,0 +1,15 @@ +package startrpc + +import ( + "context" + + "github.com/openimsdk/open-im-server/v3/pkg/authverify" + "google.golang.org/grpc" +) + +func grpcServerIMAdminUserID(imAdminUserID []string) grpc.ServerOption { + return grpc.ChainUnaryInterceptor(func(ctx context.Context, req any, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (resp any, err error) { + ctx = authverify.WithIMAdminUserIDs(ctx, imAdminUserID) + return handler(ctx, req) + }) +} diff --git a/pkg/common/startrpc/start.go b/pkg/common/startrpc/start.go index af50c408d..03621343b 100644 --- a/pkg/common/startrpc/start.go +++ b/pkg/common/startrpc/start.go @@ -37,7 +37,8 @@ import ( "github.com/openimsdk/tools/discovery" "github.com/openimsdk/tools/errs" "github.com/openimsdk/tools/log" - "github.com/openimsdk/tools/mw" + grpccli "github.com/openimsdk/tools/mw/grpc/client" + grpcsrv "github.com/openimsdk/tools/mw/grpc/server" "google.golang.org/grpc" "google.golang.org/grpc/credentials/insecure" ) @@ -76,6 +77,34 @@ func getConfigRpcMaxRequestBody(value reflect.Value) *conf.MaxRequestBody { return nil } +func getConfigShare(value reflect.Value) *conf.Share { + for value.Kind() == reflect.Pointer { + value = value.Elem() + } + if value.Kind() == reflect.Struct { + num := value.NumField() + for i := 0; i < num; i++ { + field := value.Field(i) + if !field.CanInterface() { + continue + } + for field.Kind() == reflect.Pointer { + field = field.Elem() + } + switch elem := field.Interface().(type) { + case conf.Share: + return &elem + } + if field.Kind() == reflect.Struct { + if elem := getConfigShare(field); elem != nil { + return elem + } + } + } + } + return nil +} + func Start[T any](ctx context.Context, disc *conf.Discovery, prometheusConfig *conf.Prometheus, listenIP, registerIP string, autoSetPorts bool, rpcPorts []int, index int, rpcRegisterName string, notification *conf.Notification, config T, watchConfigNames []string, watchServiceNames []string, @@ -87,12 +116,20 @@ func Start[T any](ctx context.Context, disc *conf.Discovery, prometheusConfig *c } maxRequestBody := getConfigRpcMaxRequestBody(reflect.ValueOf(config)) + shareConfig := getConfigShare(reflect.ValueOf(config)) log.ZDebug(ctx, "rpc start", "rpcMaxRequestBody", maxRequestBody, "rpcRegisterName", rpcRegisterName, "registerIP", registerIP, "listenIP", listenIP) options = append(options, - mw.GrpcServer(), + grpcsrv.GrpcServerMetadataContext(), + grpcsrv.GrpcServerLogger(), + grpcsrv.GrpcServerErrorConvert(), + grpcsrv.GrpcServerRequestValidate(), + grpcsrv.GrpcServerPanicCapture(), ) + if shareConfig != nil && len(shareConfig.IMAdminUserID) > 0 { + options = append(options, grpcServerIMAdminUserID(shareConfig.IMAdminUserID)) + } var clientOptions []grpc.DialOption if maxRequestBody != nil { if maxRequestBody.RequestMaxBodySize > 0 { @@ -129,8 +166,12 @@ func Start[T any](ctx context.Context, disc *conf.Discovery, prometheusConfig *c defer client.Close() client.AddOption( - mw.GrpcClient(), grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithDefaultServiceConfig(fmt.Sprintf(`{"LoadBalancingPolicy": "%s"}`, "round_robin")), + + grpccli.GrpcClientLogger(), + grpccli.GrpcClientContext(), + grpccli.GrpcClientErrorConvert(), ) if len(clientOptions) > 0 { client.AddOption(clientOptions...) From 632a65303c88b5465e4c3f8c4e9fb37d632b1911 Mon Sep 17 00:00:00 2001 From: chao <48119764+withchao@users.noreply.github.com> Date: Fri, 16 May 2025 17:30:38 +0800 Subject: [PATCH 159/199] fix: standalone mode cannot be used (#3360) * pb * fix: Modifying other fields while setting IsPrivateChat does not take effect * fix: quote message error revoke * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * upgrading pkg tools * fix * fix * optimize log output * feat: support GetLastMessage * feat: support GetLastMessage * feat: s3 switch * feat: s3 switch * fix: GetUsersOnline * feat: SendBusinessNotification supported configuration parameters * feat: SendBusinessNotification supported configuration parameters * feat: SendBusinessNotification supported configuration parameters * feat: seq conversion failed without exiting * fix: DeleteDoc crash * fix: fill send time * fix: fill send time * fix: crash caused by withdrawing messages from users who have left the group * fix: user msg timestamp * seq read config * seq read config * fix: the source message of the reference is withdrawn, and the referenced message is deleted * feat: optimize the default notification.yml * fix: shouldPushOffline * fix: the sorting is wrong after canceling the administrator in group settings * feat: Sending messages supports returning fields modified by webhook * feat: Sending messages supports returning fields modified by webhook * feat: Sending messages supports returning fields modified by webhook * fix: oss specifies content-type when uploading * fix: the version number contains a line break * fix: the version number contains a line break * feat: GetConversationsHasReadAndMaxSeq support pinned * feat: GetConversationsHasReadAndMaxSeq support pinned * feat: GetConversationsHasReadAndMaxSeq support pinned * fix: transferring the group owner to a muted member, incremental version error * feat: unified conversion code * feat: update gomake * fix: in standalone mode, the user online status is wrong --- go.mod | 2 +- go.sum | 4 +- internal/api/router.go | 13 +-- internal/msggateway/ws_server.go | 2 +- .../online_msg_to_mongo_handler.go | 10 +- internal/push/push_handler.go | 8 +- pkg/authverify/token.go | 6 +- pkg/common/startrpc/start.go | 60 +----------- pkg/common/startrpc/tools.go | 39 ++++++++ pkg/rpccache/online.go | 92 +++++++++++++++---- 10 files changed, 127 insertions(+), 109 deletions(-) create mode 100644 pkg/common/startrpc/tools.go diff --git a/go.mod b/go.mod index 4af0695db..d35252536 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/mitchellh/mapstructure v1.5.0 github.com/openimsdk/protocol v0.0.73-alpha.6 - github.com/openimsdk/tools v0.0.50-alpha.81 + github.com/openimsdk/tools v0.0.50-alpha.83 github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 github.com/stretchr/testify v1.9.0 diff --git a/go.sum b/go.sum index 6bc410a2d..35fc3d4c6 100644 --- a/go.sum +++ b/go.sum @@ -349,8 +349,8 @@ github.com/openimsdk/gomake v0.0.15-alpha.5 h1:eEZCEHm+NsmcO3onXZPIUbGFCYPYbsX5b github.com/openimsdk/gomake v0.0.15-alpha.5/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= github.com/openimsdk/protocol v0.0.73-alpha.6 h1:sna9coWG7HN1zObBPtvG0Ki/vzqHXiB4qKbA5P3w7kc= github.com/openimsdk/protocol v0.0.73-alpha.6/go.mod h1:WF7EuE55vQvpyUAzDXcqg+B+446xQyEba0X35lTINmw= -github.com/openimsdk/tools v0.0.50-alpha.81 h1:VbuJKtigNXLkCKB/Q6f2UHsqoSaTOAwS8F51c1nhOCA= -github.com/openimsdk/tools v0.0.50-alpha.81/go.mod h1:n2poR3asX1e1XZce4O+MOWAp+X02QJRFvhcLCXZdzRo= +github.com/openimsdk/tools v0.0.50-alpha.83 h1:7c1D40YGqIWUmGfCII5pduETGC/8c2DyS9SQ4LvoplU= +github.com/openimsdk/tools v0.0.50-alpha.83/go.mod h1:n2poR3asX1e1XZce4O+MOWAp+X02QJRFvhcLCXZdzRo= github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= diff --git a/internal/api/router.go b/internal/api/router.go index 7c2c78c7d..add8ef36b 100644 --- a/internal/api/router.go +++ b/internal/api/router.go @@ -9,12 +9,8 @@ import ( "github.com/gin-gonic/gin" "github.com/gin-gonic/gin/binding" "github.com/go-playground/validator/v10" - "github.com/openimsdk/open-im-server/v3/pkg/authverify" - "github.com/openimsdk/tools/mcontext" - "github.com/openimsdk/tools/utils/datautil" - clientv3 "go.etcd.io/etcd/client/v3" - "github.com/openimsdk/open-im-server/v3/internal/api/jssdk" + "github.com/openimsdk/open-im-server/v3/pkg/authverify" "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/open-im-server/v3/pkg/common/prommetrics" "github.com/openimsdk/open-im-server/v3/pkg/common/servererrs" @@ -32,6 +28,7 @@ import ( "github.com/openimsdk/tools/discovery/etcd" "github.com/openimsdk/tools/log" "github.com/openimsdk/tools/mw" + clientv3 "go.etcd.io/etcd/client/v3" ) const ( @@ -271,7 +268,6 @@ func newGinRouter(ctx context.Context, client discovery.Conn, cfg *Config) (*gin conversationGroup.POST("/get_owner_conversation", c.GetOwnerConversation) conversationGroup.POST("/get_not_notify_conversation_ids", c.GetNotNotifyConversationIDs) conversationGroup.POST("/get_pinned_conversation_ids", c.GetPinnedConversationIDs) - conversationGroup.POST("/update_conversations_by_user", c.UpdateConversationsByUser) } { @@ -315,7 +311,6 @@ func newGinRouter(ctx context.Context, client discovery.Conn, cfg *Config) (*gin configGroup.POST("/get_config_list", cm.GetConfigList) configGroup.POST("/get_config", cm.GetConfig) configGroup.POST("/set_config", cm.SetConfig) - configGroup.POST("/set_configs", cm.SetConfigs) configGroup.POST("/reset_config", cm.ResetConfig) configGroup.POST("/set_enable_config_manager", cm.SetEnableConfigManager) configGroup.POST("/get_enable_config_manager", cm.GetEnableConfigManager) @@ -359,9 +354,7 @@ func GinParseToken(authClient *rpcli.AuthClient) gin.HandlerFunc { func setGinIsAdmin(imAdminUserID []string) gin.HandlerFunc { return func(c *gin.Context) { - opUserID := mcontext.GetOpUserID(c) - admin := datautil.Contain(opUserID, imAdminUserID...) - c.Set(authverify.CtxIsAdminKey, admin) + c.Set(authverify.CtxAdminUserIDsKey, imAdminUserID) } } diff --git a/internal/msggateway/ws_server.go b/internal/msggateway/ws_server.go index d69062bda..211482cb1 100644 --- a/internal/msggateway/ws_server.go +++ b/internal/msggateway/ws_server.go @@ -49,7 +49,7 @@ type WsServer struct { unregisterChan chan *Client kickHandlerChan chan *kickHandler clients UserMap - online *rpccache.OnlineCache + online rpccache.OnlineCache subscription *Subscription clientPool sync.Pool onlineUserNum atomic.Int64 diff --git a/internal/msgtransfer/online_msg_to_mongo_handler.go b/internal/msgtransfer/online_msg_to_mongo_handler.go index ae14d02a1..a895bb9c4 100644 --- a/internal/msgtransfer/online_msg_to_mongo_handler.go +++ b/internal/msgtransfer/online_msg_to_mongo_handler.go @@ -48,15 +48,7 @@ func (mc *OnlineHistoryMongoConsumerHandler) HandleChatWs2Mongo(ctx context.Cont log.ZDebug(ctx, "mongo consumer recv msg", "msgs", msgFromMQ.String()) err = mc.msgTransferDatabase.BatchInsertChat2DB(ctx, msgFromMQ.ConversationID, msgFromMQ.MsgData, msgFromMQ.LastSeq) if err != nil { - log.ZError( - ctx, - "single data insert to mongo err", - err, - "msg", - msgFromMQ.MsgData, - "conversationID", - msgFromMQ.ConversationID, - ) + log.ZError(ctx, "single data insert to mongo err", err, "msg", msgFromMQ.MsgData, "conversationID", msgFromMQ.ConversationID) prommetrics.MsgInsertMongoFailedCounter.Inc() } else { prommetrics.MsgInsertMongoSuccessCounter.Inc() diff --git a/internal/push/push_handler.go b/internal/push/push_handler.go index 418c4c7f2..7b1efe3bf 100644 --- a/internal/push/push_handler.go +++ b/internal/push/push_handler.go @@ -33,7 +33,7 @@ type ConsumerHandler struct { offlinePusher offlinepush.OfflinePusher onlinePusher OnlinePusher pushDatabase controller.PushDatabase - onlineCache *rpccache.OnlineCache + onlineCache rpccache.OnlineCache groupLocalCache *rpccache.GroupLocalCache conversationLocalCache *rpccache.ConversationLocalCache webhookClient *webhook.Client @@ -120,11 +120,7 @@ func (c *ConsumerHandler) HandleMs2PsChat(ctx context.Context, msg []byte) { } func (c *ConsumerHandler) WaitCache() { - c.onlineCache.Lock.Lock() - for c.onlineCache.CurrentPhase.Load() < rpccache.DoSubscribeOver { - c.onlineCache.Cond.Wait() - } - c.onlineCache.Lock.Unlock() + c.onlineCache.WaitCache() } // Push2User Suitable for two types of conversations, one is SingleChatType and the other is NotificationChatType. diff --git a/pkg/authverify/token.go b/pkg/authverify/token.go index 75fb1448b..2e3639776 100644 --- a/pkg/authverify/token.go +++ b/pkg/authverify/token.go @@ -51,15 +51,15 @@ func CheckSystemAccount(ctx context.Context, level int32) bool { } const ( - CtxIsAdminKey = "CtxIsAdminKey" + CtxAdminUserIDsKey = "CtxAdminUserIDsKey" ) func WithIMAdminUserIDs(ctx context.Context, imAdminUserID []string) context.Context { - return context.WithValue(ctx, CtxIsAdminKey, imAdminUserID) + return context.WithValue(ctx, CtxAdminUserIDsKey, imAdminUserID) } func GetIMAdminUserIDs(ctx context.Context) []string { - imAdminUserID, _ := ctx.Value(CtxIsAdminKey).([]string) + imAdminUserID, _ := ctx.Value(CtxAdminUserIDsKey).([]string) return imAdminUserID } diff --git a/pkg/common/startrpc/start.go b/pkg/common/startrpc/start.go index 03621343b..b99d32db1 100644 --- a/pkg/common/startrpc/start.go +++ b/pkg/common/startrpc/start.go @@ -47,64 +47,6 @@ func init() { prommetrics.RegistryAll() } -func getConfigRpcMaxRequestBody(value reflect.Value) *conf.MaxRequestBody { - for value.Kind() == reflect.Pointer { - value = value.Elem() - } - if value.Kind() == reflect.Struct { - num := value.NumField() - for i := 0; i < num; i++ { - field := value.Field(i) - if !field.CanInterface() { - continue - } - for field.Kind() == reflect.Pointer { - field = field.Elem() - } - switch elem := field.Interface().(type) { - case conf.Share: - return &elem.RPCMaxBodySize - case conf.MaxRequestBody: - return &elem - } - if field.Kind() == reflect.Struct { - if elem := getConfigRpcMaxRequestBody(field); elem != nil { - return elem - } - } - } - } - return nil -} - -func getConfigShare(value reflect.Value) *conf.Share { - for value.Kind() == reflect.Pointer { - value = value.Elem() - } - if value.Kind() == reflect.Struct { - num := value.NumField() - for i := 0; i < num; i++ { - field := value.Field(i) - if !field.CanInterface() { - continue - } - for field.Kind() == reflect.Pointer { - field = field.Elem() - } - switch elem := field.Interface().(type) { - case conf.Share: - return &elem - } - if field.Kind() == reflect.Struct { - if elem := getConfigShare(field); elem != nil { - return elem - } - } - } - } - return nil -} - func Start[T any](ctx context.Context, disc *conf.Discovery, prometheusConfig *conf.Prometheus, listenIP, registerIP string, autoSetPorts bool, rpcPorts []int, index int, rpcRegisterName string, notification *conf.Notification, config T, watchConfigNames []string, watchServiceNames []string, @@ -122,8 +64,8 @@ func Start[T any](ctx context.Context, disc *conf.Discovery, prometheusConfig *c options = append(options, grpcsrv.GrpcServerMetadataContext(), - grpcsrv.GrpcServerLogger(), grpcsrv.GrpcServerErrorConvert(), + grpcsrv.GrpcServerLogger(), grpcsrv.GrpcServerRequestValidate(), grpcsrv.GrpcServerPanicCapture(), ) diff --git a/pkg/common/startrpc/tools.go b/pkg/common/startrpc/tools.go new file mode 100644 index 000000000..54b518181 --- /dev/null +++ b/pkg/common/startrpc/tools.go @@ -0,0 +1,39 @@ +package startrpc + +import ( + "reflect" + + conf "github.com/openimsdk/open-im-server/v3/pkg/common/config" +) + +func getConfig[T any](value reflect.Value) *T { + for value.Kind() == reflect.Pointer { + value = value.Elem() + } + if value.Kind() == reflect.Struct { + num := value.NumField() + for i := 0; i < num; i++ { + field := value.Field(i) + for field.Kind() == reflect.Pointer { + field = field.Elem() + } + if field.Kind() == reflect.Struct { + if elem, ok := field.Interface().(T); ok { + return &elem + } + if elem := getConfig[T](field); elem != nil { + return elem + } + } + } + } + return nil +} + +func getConfigRpcMaxRequestBody(value reflect.Value) *conf.MaxRequestBody { + return getConfig[conf.MaxRequestBody](value) +} + +func getConfigShare(value reflect.Value) *conf.Share { + return getConfig[conf.Share](value) +} diff --git a/pkg/rpccache/online.go b/pkg/rpccache/online.go index 959b054a2..9b8c03101 100644 --- a/pkg/rpccache/online.go +++ b/pkg/rpccache/online.go @@ -9,6 +9,7 @@ import ( "sync/atomic" "time" + "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/open-im-server/v3/pkg/rpcli" "github.com/openimsdk/protocol/constant" "github.com/openimsdk/protocol/user" @@ -23,9 +24,25 @@ import ( "github.com/redis/go-redis/v9" ) -func NewOnlineCache(client *rpcli.UserClient, group *GroupLocalCache, rdb redis.UniversalClient, fullUserCache bool, fn func(ctx context.Context, userID string, platformIDs []int32)) (*OnlineCache, error) { +const ( + Begin uint32 = iota + DoOnlineStatusOver + DoSubscribeOver +) + +type OnlineCache interface { + GetUserOnlinePlatform(ctx context.Context, userID string) ([]int32, error) + GetUserOnline(ctx context.Context, userID string) (bool, error) + GetUsersOnline(ctx context.Context, userIDs []string) ([]string, []string, error) + WaitCache() +} + +func NewOnlineCache(client *rpcli.UserClient, group *GroupLocalCache, rdb redis.UniversalClient, fullUserCache bool, fn func(ctx context.Context, userID string, platformIDs []int32)) (OnlineCache, error) { + if config.Standalone() { + return disableOnlineCache{client: client}, nil + } l := &sync.Mutex{} - x := &OnlineCache{ + x := &defaultOnlineCache{ client: client, group: group, fullUserCache: fullUserCache, @@ -60,13 +77,7 @@ func NewOnlineCache(client *rpcli.UserClient, group *GroupLocalCache, rdb redis. return x, nil } -const ( - Begin uint32 = iota - DoOnlineStatusOver - DoSubscribeOver -) - -type OnlineCache struct { +type defaultOnlineCache struct { client *rpcli.UserClient group *GroupLocalCache @@ -82,7 +93,7 @@ type OnlineCache struct { CurrentPhase atomic.Uint32 } -func (o *OnlineCache) initUsersOnlineStatus(ctx context.Context) (err error) { +func (o *defaultOnlineCache) initUsersOnlineStatus(ctx context.Context) (err error) { log.ZDebug(ctx, "init users online status begin") var ( @@ -135,7 +146,7 @@ func (o *OnlineCache) initUsersOnlineStatus(ctx context.Context) (err error) { return nil } -func (o *OnlineCache) doSubscribe(ctx context.Context, rdb redis.UniversalClient, fn func(ctx context.Context, userID string, platformIDs []int32)) { +func (o *defaultOnlineCache) doSubscribe(ctx context.Context, rdb redis.UniversalClient, fn func(ctx context.Context, userID string, platformIDs []int32)) { o.Lock.Lock() ch := rdb.Subscribe(ctx, cachekey.OnlineChannel).Channel() for o.CurrentPhase.Load() < DoOnlineStatusOver { @@ -186,7 +197,7 @@ func (o *OnlineCache) doSubscribe(ctx context.Context, rdb redis.UniversalClient } } -func (o *OnlineCache) getUserOnlinePlatform(ctx context.Context, userID string) ([]int32, error) { +func (o *defaultOnlineCache) getUserOnlinePlatform(ctx context.Context, userID string) ([]int32, error) { platformIDs, err := o.lruCache.Get(userID, func() ([]int32, error) { return o.client.GetUserOnlinePlatform(ctx, userID) }) @@ -198,7 +209,7 @@ func (o *OnlineCache) getUserOnlinePlatform(ctx context.Context, userID string) return platformIDs, nil } -func (o *OnlineCache) GetUserOnlinePlatform(ctx context.Context, userID string) ([]int32, error) { +func (o *defaultOnlineCache) GetUserOnlinePlatform(ctx context.Context, userID string) ([]int32, error) { platformIDs, err := o.getUserOnlinePlatform(ctx, userID) if err != nil { return nil, err @@ -208,7 +219,7 @@ func (o *OnlineCache) GetUserOnlinePlatform(ctx context.Context, userID string) return platformIDs, nil } -func (o *OnlineCache) GetUserOnline(ctx context.Context, userID string) (bool, error) { +func (o *defaultOnlineCache) GetUserOnline(ctx context.Context, userID string) (bool, error) { platformIDs, err := o.getUserOnlinePlatform(ctx, userID) if err != nil { return false, err @@ -216,7 +227,7 @@ func (o *OnlineCache) GetUserOnline(ctx context.Context, userID string) (bool, e return len(platformIDs) > 0, nil } -func (o *OnlineCache) getUserOnlinePlatformBatch(ctx context.Context, userIDs []string) (map[string][]int32, error) { +func (o *defaultOnlineCache) getUserOnlinePlatformBatch(ctx context.Context, userIDs []string) (map[string][]int32, error) { if len(userIDs) == 0 { return nil, nil } @@ -240,7 +251,7 @@ func (o *OnlineCache) getUserOnlinePlatformBatch(ctx context.Context, userIDs [] return platformIDsMap, nil } -func (o *OnlineCache) GetUsersOnline(ctx context.Context, userIDs []string) ([]string, []string, error) { +func (o *defaultOnlineCache) GetUsersOnline(ctx context.Context, userIDs []string) ([]string, []string, error) { t := time.Now() var ( @@ -276,7 +287,7 @@ func (o *OnlineCache) GetUsersOnline(ctx context.Context, userIDs []string) ([]s return onlineUserIDs, offlineUserIDs, nil } -func (o *OnlineCache) setUserOnline(userID string, platformIDs []int32) { +func (o *defaultOnlineCache) setUserOnline(userID string, platformIDs []int32) { switch o.fullUserCache { case true: o.mapCache.Store(userID, platformIDs) @@ -285,6 +296,51 @@ func (o *OnlineCache) setUserOnline(userID string, platformIDs []int32) { } } -func (o *OnlineCache) setHasUserOnline(userID string, platformIDs []int32) bool { +func (o *defaultOnlineCache) setHasUserOnline(userID string, platformIDs []int32) bool { return o.lruCache.SetHas(userID, platformIDs) } + +func (o *defaultOnlineCache) WaitCache() { + o.Lock.Lock() + for o.CurrentPhase.Load() < DoSubscribeOver { + o.Cond.Wait() + } + o.Lock.Unlock() +} + +type disableOnlineCache struct { + client *rpcli.UserClient +} + +func (o disableOnlineCache) GetUserOnlinePlatform(ctx context.Context, userID string) ([]int32, error) { + return o.client.GetUserOnlinePlatform(ctx, userID) +} + +func (o disableOnlineCache) GetUserOnline(ctx context.Context, userID string) (bool, error) { + onlinePlatform, err := o.client.GetUserOnlinePlatform(ctx, userID) + if err != nil { + return false, err + } + return len(onlinePlatform) > 0, err +} + +func (o disableOnlineCache) GetUsersOnline(ctx context.Context, userIDs []string) ([]string, []string, error) { + var ( + onlineUserIDs = make([]string, 0, len(userIDs)) + offlineUserIDs = make([]string, 0, len(userIDs)) + ) + for _, userID := range userIDs { + online, err := o.GetUserOnline(ctx, userID) + if err != nil { + return nil, nil, err + } + if online { + onlineUserIDs = append(onlineUserIDs, userID) + } else { + offlineUserIDs = append(offlineUserIDs, userID) + } + } + return onlineUserIDs, offlineUserIDs, nil +} + +func (o disableOnlineCache) WaitCache() {} From 748d783d36fc3f1e4eacaee64203b10e8e2b8744 Mon Sep 17 00:00:00 2001 From: chao <48119764+withchao@users.noreply.github.com> Date: Tue, 20 May 2025 11:30:00 +0800 Subject: [PATCH 160/199] feat: add rpc interface permission check (#3366) * pb * fix: Modifying other fields while setting IsPrivateChat does not take effect * fix: quote message error revoke * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * upgrading pkg tools * fix * fix * optimize log output * feat: support GetLastMessage * feat: support GetLastMessage * feat: s3 switch * feat: s3 switch * fix: GetUsersOnline * feat: SendBusinessNotification supported configuration parameters * feat: SendBusinessNotification supported configuration parameters * feat: SendBusinessNotification supported configuration parameters * feat: seq conversion failed without exiting * fix: DeleteDoc crash * fix: fill send time * fix: fill send time * fix: crash caused by withdrawing messages from users who have left the group * fix: user msg timestamp * seq read config * seq read config * fix: the source message of the reference is withdrawn, and the referenced message is deleted * feat: optimize the default notification.yml * fix: shouldPushOffline * fix: the sorting is wrong after canceling the administrator in group settings * feat: Sending messages supports returning fields modified by webhook * feat: Sending messages supports returning fields modified by webhook * feat: Sending messages supports returning fields modified by webhook * fix: oss specifies content-type when uploading * fix: the version number contains a line break * fix: the version number contains a line break * feat: GetConversationsHasReadAndMaxSeq support pinned * feat: GetConversationsHasReadAndMaxSeq support pinned * feat: GetConversationsHasReadAndMaxSeq support pinned * fix: transferring the group owner to a muted member, incremental version error * feat: unified conversion code * feat: update gomake * fix: in standalone mode, the user online status is wrong * fix: add permission check * fix: add permission check --- internal/api/conversation.go | 6 +-- internal/api/router.go | 2 +- internal/rpc/conversation/conversation.go | 66 ++++++++++++++++------- internal/rpc/conversation/sync.go | 3 ++ internal/rpc/group/cache.go | 4 ++ internal/rpc/group/group.go | 48 +++++++++++++++-- internal/rpc/group/statistics.go | 4 ++ internal/rpc/group/sync.go | 15 ++---- internal/rpc/msg/as_read.go | 15 ++++-- internal/rpc/msg/delete.go | 3 ++ internal/rpc/msg/send.go | 12 ++--- internal/rpc/msg/seq.go | 3 +- internal/rpc/msg/statistics.go | 10 +++- internal/rpc/msg/sync_msg.go | 3 ++ internal/rpc/relation/black.go | 3 ++ internal/rpc/relation/friend.go | 28 +++++++--- internal/rpc/third/log.go | 3 +- internal/rpc/third/third.go | 4 ++ pkg/authverify/token.go | 49 +++++++++++++++-- pkg/tools/batcher/batcher.go | 4 +- 20 files changed, 222 insertions(+), 63 deletions(-) diff --git a/internal/api/conversation.go b/internal/api/conversation.go index f6eadf15a..5a191c0ec 100644 --- a/internal/api/conversation.go +++ b/internal/api/conversation.go @@ -49,9 +49,9 @@ func (o *ConversationApi) SetConversations(c *gin.Context) { a2r.Call(c, conversation.ConversationClient.SetConversations, o.Client) } -func (o *ConversationApi) GetConversationOfflinePushUserIDs(c *gin.Context) { - a2r.Call(c, conversation.ConversationClient.GetConversationOfflinePushUserIDs, o.Client) -} +//func (o *ConversationApi) GetConversationOfflinePushUserIDs(c *gin.Context) { +// a2r.Call(c, conversation.ConversationClient.GetConversationOfflinePushUserIDs, o.Client) +//} func (o *ConversationApi) GetFullOwnerConversationIDs(c *gin.Context) { a2r.Call(c, conversation.ConversationClient.GetFullOwnerConversationIDs, o.Client) diff --git a/internal/api/router.go b/internal/api/router.go index add8ef36b..700d8392e 100644 --- a/internal/api/router.go +++ b/internal/api/router.go @@ -262,7 +262,7 @@ func newGinRouter(ctx context.Context, client discovery.Conn, cfg *Config) (*gin conversationGroup.POST("/get_conversation", c.GetConversation) conversationGroup.POST("/get_conversations", c.GetConversations) conversationGroup.POST("/set_conversations", c.SetConversations) - conversationGroup.POST("/get_conversation_offline_push_user_ids", c.GetConversationOfflinePushUserIDs) + //conversationGroup.POST("/get_conversation_offline_push_user_ids", c.GetConversationOfflinePushUserIDs) conversationGroup.POST("/get_full_conversation_ids", c.GetFullOwnerConversationIDs) conversationGroup.POST("/get_incremental_conversations", c.GetIncrementalConversation) conversationGroup.POST("/get_owner_conversation", c.GetOwnerConversation) diff --git a/internal/rpc/conversation/conversation.go b/internal/rpc/conversation/conversation.go index ca2c58878..b0b1053ed 100644 --- a/internal/rpc/conversation/conversation.go +++ b/internal/rpc/conversation/conversation.go @@ -19,6 +19,7 @@ import ( "sort" "time" + "github.com/openimsdk/open-im-server/v3/pkg/authverify" "github.com/openimsdk/open-im-server/v3/pkg/dbbuild" "github.com/openimsdk/open-im-server/v3/pkg/rpcli" @@ -117,6 +118,9 @@ func Start(ctx context.Context, config *Config, client discovery.Conn, server gr } func (c *conversationServer) GetConversation(ctx context.Context, req *pbconversation.GetConversationReq) (*pbconversation.GetConversationResp, error) { + if err := authverify.CheckAccess(ctx, req.OwnerUserID); err != nil { + return nil, err + } conversations, err := c.conversationDatabase.FindConversations(ctx, req.OwnerUserID, []string{req.ConversationID}) if err != nil { return nil, err @@ -130,7 +134,9 @@ func (c *conversationServer) GetConversation(ctx context.Context, req *pbconvers } func (c *conversationServer) GetSortedConversationList(ctx context.Context, req *pbconversation.GetSortedConversationListReq) (resp *pbconversation.GetSortedConversationListResp, err error) { - log.ZDebug(ctx, "GetSortedConversationList", "seqs", req, "userID", req.UserID) + if err := authverify.CheckAccess(ctx, req.UserID); err != nil { + return nil, err + } var conversationIDs []string if len(req.ConversationIDs) == 0 { conversationIDs, err = c.conversationDatabase.GetConversationIDs(ctx, req.UserID) @@ -203,6 +209,9 @@ func (c *conversationServer) GetSortedConversationList(ctx context.Context, req } func (c *conversationServer) GetAllConversations(ctx context.Context, req *pbconversation.GetAllConversationsReq) (*pbconversation.GetAllConversationsResp, error) { + if err := authverify.CheckAccess(ctx, req.OwnerUserID); err != nil { + return nil, err + } conversations, err := c.conversationDatabase.GetUserAllConversation(ctx, req.OwnerUserID) if err != nil { return nil, err @@ -213,6 +222,9 @@ func (c *conversationServer) GetAllConversations(ctx context.Context, req *pbcon } func (c *conversationServer) GetConversations(ctx context.Context, req *pbconversation.GetConversationsReq) (*pbconversation.GetConversationsResp, error) { + if err := authverify.CheckAccess(ctx, req.OwnerUserID); err != nil { + return nil, err + } conversations, err := c.getConversations(ctx, req.OwnerUserID, req.ConversationIDs) if err != nil { return nil, err @@ -233,6 +245,9 @@ func (c *conversationServer) getConversations(ctx context.Context, ownerUserID s } func (c *conversationServer) SetConversation(ctx context.Context, req *pbconversation.SetConversationReq) (*pbconversation.SetConversationResp, error) { + if err := authverify.CheckAccess(ctx, req.GetConversation().GetUserID()); err != nil { + return nil, err + } var conversation dbModel.Conversation if err := datautil.CopyStructFields(&conversation, req.Conversation); err != nil { return nil, err @@ -247,10 +262,11 @@ func (c *conversationServer) SetConversation(ctx context.Context, req *pbconvers } func (c *conversationServer) SetConversations(ctx context.Context, req *pbconversation.SetConversationsReq) (*pbconversation.SetConversationsResp, error) { - if req.Conversation == nil { - return nil, errs.ErrArgs.WrapMsg("conversation must not be nil") + for _, userID := range req.UserIDs { + if err := authverify.CheckAccess(ctx, userID); err != nil { + return nil, err + } } - if req.Conversation.ConversationType == constant.WriteGroupChatType { groupInfo, err := c.groupClient.GetGroupInfo(ctx, req.Conversation.GroupID) if err != nil { @@ -331,6 +347,9 @@ func (c *conversationServer) SetConversations(ctx context.Context, req *pbconver } func (c *conversationServer) UpdateConversationsByUser(ctx context.Context, req *pbconversation.UpdateConversationsByUserReq) (*pbconversation.UpdateConversationsByUserResp, error) { + if err := authverify.CheckAccess(ctx, req.UserID); err != nil { + return nil, err + } m := make(map[string]any) if req.Ex != nil { m["ex"] = req.Ex.Value @@ -343,15 +362,8 @@ func (c *conversationServer) UpdateConversationsByUser(ctx context.Context, req return &pbconversation.UpdateConversationsByUserResp{}, nil } -// Get user IDs with "Do Not Disturb" enabled in super large groups. -func (c *conversationServer) GetRecvMsgNotNotifyUserIDs(ctx context.Context, req *pbconversation.GetRecvMsgNotNotifyUserIDsReq) (*pbconversation.GetRecvMsgNotNotifyUserIDsResp, error) { - return nil, errs.New("deprecated") -} - // create conversation without notification for msg redis transfer. -func (c *conversationServer) CreateSingleChatConversations(ctx context.Context, - req *pbconversation.CreateSingleChatConversationsReq, -) (*pbconversation.CreateSingleChatConversationsResp, error) { +func (c *conversationServer) CreateSingleChatConversations(ctx context.Context, req *pbconversation.CreateSingleChatConversationsReq) (*pbconversation.CreateSingleChatConversationsResp, error) { var conversation dbModel.Conversation switch req.ConversationType { case constant.SingleChatType: @@ -454,6 +466,9 @@ func (c *conversationServer) SetConversationMinSeq(ctx context.Context, req *pbc } func (c *conversationServer) GetConversationIDs(ctx context.Context, req *pbconversation.GetConversationIDsReq) (*pbconversation.GetConversationIDsResp, error) { + if err := authverify.CheckAccess(ctx, req.UserID); err != nil { + return nil, err + } conversationIDs, err := c.conversationDatabase.GetConversationIDs(ctx, req.UserID) if err != nil { return nil, err @@ -462,6 +477,9 @@ func (c *conversationServer) GetConversationIDs(ctx context.Context, req *pbconv } func (c *conversationServer) GetUserConversationIDsHash(ctx context.Context, req *pbconversation.GetUserConversationIDsHashReq) (*pbconversation.GetUserConversationIDsHashResp, error) { + if err := authverify.CheckAccess(ctx, req.OwnerUserID); err != nil { + return nil, err + } hash, err := c.conversationDatabase.GetUserConversationIDsHash(ctx, req.OwnerUserID) if err != nil { return nil, err @@ -469,10 +487,7 @@ func (c *conversationServer) GetUserConversationIDsHash(ctx context.Context, req return &pbconversation.GetUserConversationIDsHashResp{Hash: hash}, nil } -func (c *conversationServer) GetConversationsByConversationID( - ctx context.Context, - req *pbconversation.GetConversationsByConversationIDReq, -) (*pbconversation.GetConversationsByConversationIDResp, error) { +func (c *conversationServer) GetConversationsByConversationID(ctx context.Context, req *pbconversation.GetConversationsByConversationIDReq) (*pbconversation.GetConversationsByConversationIDResp, error) { conversations, err := c.conversationDatabase.GetConversationsByConversationID(ctx, req.ConversationIDs) if err != nil { return nil, err @@ -526,10 +541,7 @@ func (c *conversationServer) conversationSort(conversations map[int64]string, re resp.ConversationElems = append(resp.ConversationElems, cons...) } -func (c *conversationServer) getConversationInfo( - ctx context.Context, - chatLogs map[string]*sdkws.MsgData, - userID string) (map[string]*pbconversation.ConversationElem, error) { +func (c *conversationServer) getConversationInfo(ctx context.Context, chatLogs map[string]*sdkws.MsgData, userID string) (map[string]*pbconversation.ConversationElem, error) { var ( sendIDs []string groupIDs []string @@ -615,6 +627,11 @@ func (c *conversationServer) GetConversationNotReceiveMessageUserIDs(ctx context } func (c *conversationServer) UpdateConversation(ctx context.Context, req *pbconversation.UpdateConversationReq) (*pbconversation.UpdateConversationResp, error) { + for _, userID := range req.UserIDs { + if err := authverify.CheckAccess(ctx, userID); err != nil { + return nil, err + } + } m := make(map[string]any) if req.RecvMsgOpt != nil { m["recv_msg_opt"] = req.RecvMsgOpt.Value @@ -661,6 +678,9 @@ func (c *conversationServer) UpdateConversation(ctx context.Context, req *pbconv } func (c *conversationServer) GetOwnerConversation(ctx context.Context, req *pbconversation.GetOwnerConversationReq) (*pbconversation.GetOwnerConversationResp, error) { + if err := authverify.CheckAccess(ctx, req.UserID); err != nil { + return nil, err + } total, conversations, err := c.conversationDatabase.GetOwnerConversation(ctx, req.UserID, req.Pagination) if err != nil { return nil, err @@ -722,6 +742,9 @@ func (c *conversationServer) GetConversationsNeedClearMsg(ctx context.Context, _ } func (c *conversationServer) GetNotNotifyConversationIDs(ctx context.Context, req *pbconversation.GetNotNotifyConversationIDsReq) (*pbconversation.GetNotNotifyConversationIDsResp, error) { + if err := authverify.CheckAccess(ctx, req.UserID); err != nil { + return nil, err + } conversationIDs, err := c.conversationDatabase.GetNotNotifyConversationIDs(ctx, req.UserID) if err != nil { return nil, err @@ -730,6 +753,9 @@ func (c *conversationServer) GetNotNotifyConversationIDs(ctx context.Context, re } func (c *conversationServer) GetPinnedConversationIDs(ctx context.Context, req *pbconversation.GetPinnedConversationIDsReq) (*pbconversation.GetPinnedConversationIDsResp, error) { + if err := authverify.CheckAccess(ctx, req.UserID); err != nil { + return nil, err + } conversationIDs, err := c.conversationDatabase.GetPinnedConversationIDs(ctx, req.UserID) if err != nil { return nil, err diff --git a/internal/rpc/conversation/sync.go b/internal/rpc/conversation/sync.go index cee74b319..a24dd85c6 100644 --- a/internal/rpc/conversation/sync.go +++ b/internal/rpc/conversation/sync.go @@ -35,6 +35,9 @@ func (c *conversationServer) GetFullOwnerConversationIDs(ctx context.Context, re } func (c *conversationServer) GetIncrementalConversation(ctx context.Context, req *conversation.GetIncrementalConversationReq) (*conversation.GetIncrementalConversationResp, error) { + if err := authverify.CheckAccess(ctx, req.UserID); err != nil { + return nil, err + } opt := incrversion.Option[*conversation.Conversation, conversation.GetIncrementalConversationResp]{ Ctx: ctx, VersionKey: req.UserID, diff --git a/internal/rpc/group/cache.go b/internal/rpc/group/cache.go index bcd246267..ec0e5b566 100644 --- a/internal/rpc/group/cache.go +++ b/internal/rpc/group/cache.go @@ -17,6 +17,7 @@ package group import ( "context" + "github.com/openimsdk/open-im-server/v3/pkg/authverify" "github.com/openimsdk/open-im-server/v3/pkg/common/convert" pbgroup "github.com/openimsdk/protocol/group" ) @@ -33,6 +34,9 @@ func (g *groupServer) GetGroupInfoCache(ctx context.Context, req *pbgroup.GetGro } func (g *groupServer) GetGroupMemberCache(ctx context.Context, req *pbgroup.GetGroupMemberCacheReq) (*pbgroup.GetGroupMemberCacheResp, error) { + if err := authverify.CheckAccess(ctx, req.GroupMemberID); err != nil { + return nil, err + } members, err := g.db.TakeGroupMember(ctx, req.GroupID, req.GroupMemberID) if err != nil { return nil, err diff --git a/internal/rpc/group/group.go b/internal/rpc/group/group.go index 10cdc2546..2026ba71b 100644 --- a/internal/rpc/group/group.go +++ b/internal/rpc/group/group.go @@ -476,6 +476,19 @@ func (g *groupServer) GetGroupAllMember(ctx context.Context, req *pbgroup.GetGro if err != nil { return nil, err } + if !authverify.IsAdmin(ctx) { + var inGroup bool + opUserID := mcontext.GetOpUserID(ctx) + for _, member := range members { + if member.UserID == opUserID { + inGroup = true + break + } + } + if !inGroup { + return nil, errs.ErrNoPermission.WrapMsg("opuser not in group") + } + } if err := g.PopulateGroupMember(ctx, members...); err != nil { return nil, err } @@ -486,11 +499,24 @@ func (g *groupServer) GetGroupAllMember(ctx context.Context, req *pbgroup.GetGro return &resp, nil } +func (g *groupServer) checkAdminOrInGroup(ctx context.Context, groupID string) error { + if authverify.IsAdmin(ctx) { + return nil + } + opUserID := mcontext.GetOpUserID(ctx) + members, err := g.db.FindGroupMembers(ctx, groupID, []string{opUserID}) + if err != nil { + return err + } + if len(members) == 0 { + return errs.ErrNoPermission.WrapMsg("op user not in group") + } + return nil +} + func (g *groupServer) GetGroupMemberList(ctx context.Context, req *pbgroup.GetGroupMemberListReq) (*pbgroup.GetGroupMemberListResp, error) { - if opUserID := mcontext.GetOpUserID(ctx); !datautil.Contain(opUserID, g.config.Share.IMAdminUserID...) { - if _, err := g.db.TakeGroupMember(ctx, req.GroupID, opUserID); err != nil { - return nil, err - } + if err := g.checkAdminOrInGroup(ctx, req.GroupID); err != nil { + return nil, err } var ( total int64 @@ -631,6 +657,9 @@ func (g *groupServer) GetGroupMembersInfo(ctx context.Context, req *pbgroup.GetG if req.GroupID == "" { return nil, errs.ErrArgs.WrapMsg("groupID empty") } + if err := g.checkAdminOrInGroup(ctx, req.GroupID); err != nil { + return nil, err + } members, err := g.getGroupMembersInfo(ctx, req.GroupID, req.UserIDs) if err != nil { return nil, err @@ -658,6 +687,9 @@ func (g *groupServer) getGroupMembersInfo(ctx context.Context, groupID string, u // GetGroupApplicationList handles functions that get a list of group requests. func (g *groupServer) GetGroupApplicationList(ctx context.Context, req *pbgroup.GetGroupApplicationListReq) (*pbgroup.GetGroupApplicationListResp, error) { + if err := authverify.CheckAccess(ctx, req.FromUserID); err != nil { + return nil, err + } groupIDs, err := g.db.FindUserManagedGroupID(ctx, req.FromUserID) if err != nil { return nil, err @@ -1652,6 +1684,11 @@ func (g *groupServer) GetGroupAbstractInfo(ctx context.Context, req *pbgroup.Get if datautil.Duplicate(req.GroupIDs) { return nil, errs.ErrArgs.WrapMsg("groupIDs duplicate") } + for _, groupID := range req.GroupIDs { + if err := g.checkAdminOrInGroup(ctx, groupID); err != nil { + return nil, err + } + } groups, err := g.db.FindGroup(ctx, req.GroupIDs) if err != nil { return nil, err @@ -1699,6 +1736,9 @@ func (g *groupServer) GetGroupMemberUserIDs(ctx context.Context, req *pbgroup.Ge if err != nil { return nil, err } + if err := authverify.CheckAccessIn(ctx, userIDs...); err != nil { + return nil, err + } return &pbgroup.GetGroupMemberUserIDsResp{ UserIDs: userIDs, }, nil diff --git a/internal/rpc/group/statistics.go b/internal/rpc/group/statistics.go index 1c582fda1..4ee3396da 100644 --- a/internal/rpc/group/statistics.go +++ b/internal/rpc/group/statistics.go @@ -18,6 +18,7 @@ import ( "context" "time" + "github.com/openimsdk/open-im-server/v3/pkg/authverify" "github.com/openimsdk/protocol/group" "github.com/openimsdk/tools/errs" ) @@ -26,6 +27,9 @@ func (g *groupServer) GroupCreateCount(ctx context.Context, req *group.GroupCrea if req.Start > req.End { return nil, errs.ErrArgs.WrapMsg("start > end: %d > %d", req.Start, req.End) } + if err := authverify.CheckAdmin(ctx); err != nil { + return nil, err + } total, err := g.db.CountTotal(ctx, nil) if err != nil { return nil, err diff --git a/internal/rpc/group/sync.go b/internal/rpc/group/sync.go index 822b15307..baee2f2d4 100644 --- a/internal/rpc/group/sync.go +++ b/internal/rpc/group/sync.go @@ -11,9 +11,6 @@ import ( "github.com/openimsdk/protocol/constant" pbgroup "github.com/openimsdk/protocol/group" "github.com/openimsdk/protocol/sdkws" - "github.com/openimsdk/tools/errs" - "github.com/openimsdk/tools/mcontext" - "github.com/openimsdk/tools/utils/datautil" ) const versionSyncLimit = 500 @@ -23,10 +20,8 @@ func (g *groupServer) GetFullGroupMemberUserIDs(ctx context.Context, req *pbgrou if err != nil { return nil, err } - if opUserID := mcontext.GetOpUserID(ctx); !datautil.Contain(opUserID, g.config.Share.IMAdminUserID...) { - if !datautil.Contain(opUserID, userIDs...) { - return nil, errs.ErrNoPermission.WrapMsg("user not in group") - } + if err := authverify.CheckAccessIn(ctx, userIDs...); err != nil { + return nil, err } vl, err := g.db.FindMaxGroupMemberVersionCache(ctx, req.GroupID) if err != nil { @@ -69,6 +64,9 @@ func (g *groupServer) GetFullJoinGroupIDs(ctx context.Context, req *pbgroup.GetF } func (g *groupServer) GetIncrementalGroupMember(ctx context.Context, req *pbgroup.GetIncrementalGroupMemberReq) (*pbgroup.GetIncrementalGroupMemberResp, error) { + if err := g.checkAdminOrInGroup(ctx, req.GroupID); err != nil { + return nil, err + } group, err := g.db.TakeGroup(ctx, req.GroupID) if err != nil { return nil, err @@ -76,9 +74,6 @@ func (g *groupServer) GetIncrementalGroupMember(ctx context.Context, req *pbgrou if group.Status == constant.GroupStatusDismissed { return nil, servererrs.ErrDismissedAlready.Wrap() } - if _, err := g.db.TakeGroupMember(ctx, req.GroupID, mcontext.GetOpUserID(ctx)); err != nil { - return nil, err - } var ( hasGroupUpdate bool sortVersion uint64 diff --git a/internal/rpc/msg/as_read.go b/internal/rpc/msg/as_read.go index b25eae6b1..c52ce9c07 100644 --- a/internal/rpc/msg/as_read.go +++ b/internal/rpc/msg/as_read.go @@ -18,6 +18,7 @@ import ( "context" "errors" + "github.com/openimsdk/open-im-server/v3/pkg/authverify" cbapi "github.com/openimsdk/open-im-server/v3/pkg/callbackstruct" "github.com/openimsdk/protocol/constant" "github.com/openimsdk/protocol/msg" @@ -29,6 +30,9 @@ import ( ) func (m *msgServer) GetConversationsHasReadAndMaxSeq(ctx context.Context, req *msg.GetConversationsHasReadAndMaxSeqReq) (*msg.GetConversationsHasReadAndMaxSeqResp, error) { + if err := authverify.CheckAccess(ctx, req.UserID); err != nil { + return nil, err + } var conversationIDs []string if len(req.ConversationIDs) == 0 { var err error @@ -82,6 +86,9 @@ func (m *msgServer) GetConversationsHasReadAndMaxSeq(ctx context.Context, req *m } func (m *msgServer) SetConversationHasReadSeq(ctx context.Context, req *msg.SetConversationHasReadSeqReq) (*msg.SetConversationHasReadSeqResp, error) { + if err := authverify.CheckAccess(ctx, req.UserID); err != nil { + return nil, err + } maxSeq, err := m.MsgDatabase.GetMaxSeq(ctx, req.ConversationID) if err != nil { return nil, err @@ -97,8 +104,8 @@ func (m *msgServer) SetConversationHasReadSeq(ctx context.Context, req *msg.SetC } func (m *msgServer) MarkMsgsAsRead(ctx context.Context, req *msg.MarkMsgsAsReadReq) (*msg.MarkMsgsAsReadResp, error) { - if len(req.Seqs) < 1 { - return nil, errs.ErrArgs.WrapMsg("seqs must not be empty") + if err := authverify.CheckAccess(ctx, req.UserID); err != nil { + return nil, err } maxSeq, err := m.MsgDatabase.GetMaxSeq(ctx, req.ConversationID) if err != nil { @@ -139,6 +146,9 @@ func (m *msgServer) MarkMsgsAsRead(ctx context.Context, req *msg.MarkMsgsAsReadR } func (m *msgServer) MarkConversationAsRead(ctx context.Context, req *msg.MarkConversationAsReadReq) (*msg.MarkConversationAsReadResp, error) { + if err := authverify.CheckAccess(ctx, req.UserID); err != nil { + return nil, err + } conversation, err := m.ConversationLocalCache.GetConversation(ctx, req.UserID, req.ConversationID) if err != nil { return nil, err @@ -216,5 +226,4 @@ func (m *msgServer) sendMarkAsReadNotification(ctx context.Context, conversation HasReadSeq: hasReadSeq, } m.notificationSender.NotificationWithSessionType(ctx, sendID, recvID, constant.HasReadReceipt, sessionType, tips) - } diff --git a/internal/rpc/msg/delete.go b/internal/rpc/msg/delete.go index 4590523d5..d24420ebb 100644 --- a/internal/rpc/msg/delete.go +++ b/internal/rpc/msg/delete.go @@ -94,6 +94,9 @@ func (m *msgServer) DeleteMsgs(ctx context.Context, req *msg.DeleteMsgsReq) (*ms } func (m *msgServer) DeleteMsgPhysicalBySeq(ctx context.Context, req *msg.DeleteMsgPhysicalBySeqReq) (*msg.DeleteMsgPhysicalBySeqResp, error) { + if err := authverify.CheckAdmin(ctx); err != nil { + return nil, err + } err := m.MsgDatabase.DeleteMsgsPhysicalBySeqs(ctx, req.ConversationID, req.Seqs) if err != nil { return nil, err diff --git a/internal/rpc/msg/send.go b/internal/rpc/msg/send.go index 6b2ec30b5..0e3a9950b 100644 --- a/internal/rpc/msg/send.go +++ b/internal/rpc/msg/send.go @@ -17,6 +17,7 @@ package msg import ( "context" + "github.com/openimsdk/open-im-server/v3/pkg/authverify" "google.golang.org/protobuf/proto" "github.com/openimsdk/open-im-server/v3/pkg/common/prommetrics" @@ -37,6 +38,9 @@ func (m *msgServer) SendMsg(ctx context.Context, req *pbmsg.SendMsgReq) (*pbmsg. if req.MsgData == nil { return nil, errs.ErrArgs.WrapMsg("msgData is nil") } + if err := authverify.CheckAccess(ctx, req.MsgData.SendID); err != nil { + return nil, err + } before := new(*sdkws.MsgData) resp, err := m.sendMsg(ctx, req, before) if err != nil { @@ -172,13 +176,7 @@ func (m *msgServer) sendMsgSingleChat(ctx context.Context, req *pbmsg.SendMsgReq isSend := true isNotification := msgprocessor.IsNotificationByMsg(req.MsgData) if !isNotification { - isSend, err = m.modifyMessageByUserMessageReceiveOpt( - ctx, - req.MsgData.RecvID, - conversationutil.GenConversationIDForSingle(req.MsgData.SendID, req.MsgData.RecvID), - constant.SingleChatType, - req, - ) + isSend, err = m.modifyMessageByUserMessageReceiveOpt(authverify.WithTempAdmin(ctx), req.MsgData.RecvID, conversationutil.GenConversationIDForSingle(req.MsgData.SendID, req.MsgData.RecvID), constant.SingleChatType, req) if err != nil { return nil, err } diff --git a/internal/rpc/msg/seq.go b/internal/rpc/msg/seq.go index bd68138fb..6a0461dc8 100644 --- a/internal/rpc/msg/seq.go +++ b/internal/rpc/msg/seq.go @@ -17,9 +17,10 @@ package msg import ( "context" "errors" + "sort" + pbmsg "github.com/openimsdk/protocol/msg" "github.com/redis/go-redis/v9" - "sort" ) func (m *msgServer) GetConversationMaxSeq(ctx context.Context, req *pbmsg.GetConversationMaxSeqReq) (*pbmsg.GetConversationMaxSeqResp, error) { diff --git a/internal/rpc/msg/statistics.go b/internal/rpc/msg/statistics.go index 01c0f1c46..b1f90cae4 100644 --- a/internal/rpc/msg/statistics.go +++ b/internal/rpc/msg/statistics.go @@ -16,15 +16,20 @@ package msg import ( "context" - "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" "time" + "github.com/openimsdk/open-im-server/v3/pkg/authverify" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" + "github.com/openimsdk/protocol/msg" "github.com/openimsdk/protocol/sdkws" "github.com/openimsdk/tools/utils/datautil" ) func (m *msgServer) GetActiveUser(ctx context.Context, req *msg.GetActiveUserReq) (*msg.GetActiveUserResp, error) { + if err := authverify.CheckAdmin(ctx); err != nil { + return nil, err + } msgCount, userCount, users, dateCount, err := m.MsgDatabase.RangeUserSendCount(ctx, time.UnixMilli(req.Start), time.UnixMilli(req.End), req.Group, req.Ase, req.Pagination.PageNumber, req.Pagination.ShowNumber) if err != nil { return nil, err @@ -60,6 +65,9 @@ func (m *msgServer) GetActiveUser(ctx context.Context, req *msg.GetActiveUserReq } func (m *msgServer) GetActiveGroup(ctx context.Context, req *msg.GetActiveGroupReq) (*msg.GetActiveGroupResp, error) { + if err := authverify.CheckAdmin(ctx); err != nil { + return nil, err + } msgCount, groupCount, groups, dateCount, err := m.MsgDatabase.RangeGroupSendCount(ctx, time.UnixMilli(req.Start), time.UnixMilli(req.End), req.Ase, req.Pagination.PageNumber, req.Pagination.ShowNumber) if err != nil { return nil, err diff --git a/internal/rpc/msg/sync_msg.go b/internal/rpc/msg/sync_msg.go index 38eed93bc..259c7f85d 100644 --- a/internal/rpc/msg/sync_msg.go +++ b/internal/rpc/msg/sync_msg.go @@ -29,6 +29,9 @@ import ( ) func (m *msgServer) PullMessageBySeqs(ctx context.Context, req *sdkws.PullMessageBySeqsReq) (*sdkws.PullMessageBySeqsResp, error) { + if err := authverify.CheckAccess(ctx, req.UserID); err != nil { + return nil, err + } resp := &sdkws.PullMessageBySeqsResp{} resp.Msgs = make(map[string]*sdkws.PullMsgs) resp.NotificationMsgs = make(map[string]*sdkws.PullMsgs) diff --git a/internal/rpc/relation/black.go b/internal/rpc/relation/black.go index 4a0e60ecd..07a1537b1 100644 --- a/internal/rpc/relation/black.go +++ b/internal/rpc/relation/black.go @@ -46,6 +46,9 @@ func (s *friendServer) GetPaginationBlacks(ctx context.Context, req *relation.Ge } func (s *friendServer) IsBlack(ctx context.Context, req *relation.IsBlackReq) (*relation.IsBlackResp, error) { + if err := authverify.CheckAccessIn(ctx, req.UserID1, req.UserID2); err != nil { + return nil, err + } in1, in2, err := s.blackDatabase.CheckIn(ctx, req.UserID1, req.UserID2) if err != nil { return nil, err diff --git a/internal/rpc/relation/friend.go b/internal/rpc/relation/friend.go index 050cd5ffe..06661c79d 100644 --- a/internal/rpc/relation/friend.go +++ b/internal/rpc/relation/friend.go @@ -280,6 +280,9 @@ func (s *friendServer) SetFriendRemark(ctx context.Context, req *relation.SetFri } func (s *friendServer) GetFriendInfo(ctx context.Context, req *relation.GetFriendInfoReq) (*relation.GetFriendInfoResp, error) { + if err := authverify.CheckAccess(ctx, req.OwnerUserID); err != nil { + return nil, err + } friends, err := s.db.FindFriendsWithError(ctx, req.OwnerUserID, req.FriendUserIDs) if err != nil { return nil, err @@ -288,6 +291,9 @@ func (s *friendServer) GetFriendInfo(ctx context.Context, req *relation.GetFrien } func (s *friendServer) GetDesignatedFriends(ctx context.Context, req *relation.GetDesignatedFriendsReq) (resp *relation.GetDesignatedFriendsResp, err error) { + if err := authverify.CheckAccess(ctx, req.OwnerUserID); err != nil { + return nil, err + } resp = &relation.GetDesignatedFriendsResp{} if datautil.Duplicate(req.FriendUserIDs) { return nil, errs.ErrArgs.WrapMsg("friend userID repeated") @@ -313,9 +319,10 @@ func (s *friendServer) getFriend(ctx context.Context, ownerUserID string, friend } // Get the list of friend requests sent out proactively. -func (s *friendServer) GetDesignatedFriendsApply(ctx context.Context, - req *relation.GetDesignatedFriendsApplyReq, -) (resp *relation.GetDesignatedFriendsApplyResp, err error) { +func (s *friendServer) GetDesignatedFriendsApply(ctx context.Context, req *relation.GetDesignatedFriendsApplyReq) (resp *relation.GetDesignatedFriendsApplyResp, err error) { + if err := authverify.CheckAccessIn(ctx, req.FromUserID, req.ToUserID); err != nil { + return nil, err + } friendRequests, err := s.db.FindBothFriendRequests(ctx, req.FromUserID, req.ToUserID) if err != nil { return nil, err @@ -374,6 +381,9 @@ func (s *friendServer) GetPaginationFriendsApplyFrom(ctx context.Context, req *r // ok. func (s *friendServer) IsFriend(ctx context.Context, req *relation.IsFriendReq) (resp *relation.IsFriendResp, err error) { + if err := authverify.CheckAccessIn(ctx, req.UserID1, req.UserID2); err != nil { + return nil, err + } resp = &relation.IsFriendResp{} resp.InUser1Friends, resp.InUser2Friends, err = s.db.CheckIn(ctx, req.UserID1, req.UserID2) if err != nil { @@ -426,6 +436,9 @@ func (s *friendServer) GetSpecifiedFriendsInfo(ctx context.Context, req *relatio return nil, errs.ErrArgs.WrapMsg("userIDList repeated") } + if err := authverify.CheckAccess(ctx, req.OwnerUserID); err != nil { + return nil, err + } userMap, err := s.userClient.GetUsersInfoMap(ctx, req.UserIDList) if err != nil { return nil, err @@ -494,10 +507,7 @@ func (s *friendServer) GetSpecifiedFriendsInfo(ctx context.Context, req *relatio return resp, nil } -func (s *friendServer) UpdateFriends( - ctx context.Context, - req *relation.UpdateFriendsReq, -) (*relation.UpdateFriendsResp, error) { +func (s *friendServer) UpdateFriends(ctx context.Context, req *relation.UpdateFriendsReq) (*relation.UpdateFriendsResp, error) { if len(req.FriendUserIDs) == 0 { return nil, errs.ErrArgs.WrapMsg("friendIDList is empty") } @@ -505,6 +515,10 @@ func (s *friendServer) UpdateFriends( return nil, errs.ErrArgs.WrapMsg("friendIDList repeated") } + if err := authverify.CheckAccess(ctx, req.OwnerUserID); err != nil { + return nil, err + } + _, err := s.db.FindFriendsWithError(ctx, req.OwnerUserID, req.FriendUserIDs) if err != nil { return nil, err diff --git a/internal/rpc/third/log.go b/internal/rpc/third/log.go index fba3ecb88..9a4995ace 100644 --- a/internal/rpc/third/log.go +++ b/internal/rpc/third/log.go @@ -25,6 +25,7 @@ import ( "github.com/openimsdk/protocol/constant" "github.com/openimsdk/protocol/third" "github.com/openimsdk/tools/errs" + "github.com/openimsdk/tools/mcontext" "github.com/openimsdk/tools/utils/datautil" ) @@ -45,7 +46,7 @@ func genLogID() string { func (t *thirdServer) UploadLogs(ctx context.Context, req *third.UploadLogsReq) (*third.UploadLogsResp, error) { var dbLogs []*relationtb.Log - userID := ctx.Value(constant.OpUserID).(string) + userID := mcontext.GetOpUserID(ctx) platform := constant.PlatformID2Name[int(req.Platform)] for _, fileURL := range req.FileURLs { log := relationtb.Log{ diff --git a/internal/rpc/third/third.go b/internal/rpc/third/third.go index 0afd54014..c6dcb2ea4 100644 --- a/internal/rpc/third/third.go +++ b/internal/rpc/third/third.go @@ -19,6 +19,7 @@ import ( "fmt" "time" + "github.com/openimsdk/open-im-server/v3/pkg/authverify" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/mcache" "github.com/openimsdk/open-im-server/v3/pkg/dbbuild" @@ -148,6 +149,9 @@ func (t *thirdServer) FcmUpdateToken(ctx context.Context, req *third.FcmUpdateTo } func (t *thirdServer) SetAppBadge(ctx context.Context, req *third.SetAppBadgeReq) (resp *third.SetAppBadgeResp, err error) { + if err := authverify.CheckAccess(ctx, req.UserID); err != nil { + return nil, err + } err = t.thirdDatabase.SetAppBadge(ctx, req.UserID, int(req.AppUnreadCount)) if err != nil { return nil, err diff --git a/pkg/authverify/token.go b/pkg/authverify/token.go index 2e3639776..a82eba30a 100644 --- a/pkg/authverify/token.go +++ b/pkg/authverify/token.go @@ -64,16 +64,57 @@ func GetIMAdminUserIDs(ctx context.Context) []string { } func IsAdmin(ctx context.Context) bool { - return datautil.Contain(mcontext.GetOpUserID(ctx), GetIMAdminUserIDs(ctx)...) + return IsTempAdmin(ctx) || IsSystemAdmin(ctx) } func CheckAccess(ctx context.Context, ownerUserID string) error { - opUserID := mcontext.GetOpUserID(ctx) - if opUserID == ownerUserID { + if mcontext.GetOpUserID(ctx) == ownerUserID { return nil } - if datautil.Contain(mcontext.GetOpUserID(ctx), GetIMAdminUserIDs(ctx)...) { + if IsAdmin(ctx) { return nil } return servererrs.ErrNoPermission.WrapMsg("ownerUserID", ownerUserID) } + +func CheckAccessIn(ctx context.Context, ownerUserIDs ...string) error { + opUserID := mcontext.GetOpUserID(ctx) + for _, userID := range ownerUserIDs { + if opUserID == userID { + return nil + } + } + if IsAdmin(ctx) { + return nil + } + return servererrs.ErrNoPermission.WrapMsg("opUser in ownerUserIDs") +} + +var tempAdminValue = []string{"1"} + +const ctxTempAdminKey = "ctxImTempAdminKey" + +func WithTempAdmin(ctx context.Context) context.Context { + keys, _ := ctx.Value(constant.RpcCustomHeader).([]string) + if datautil.Contain(ctxTempAdminKey, keys...) { + return ctx + } + if len(keys) > 0 { + temp := make([]string, 0, len(keys)+1) + temp = append(temp, keys...) + keys = append(temp, ctxTempAdminKey) + } else { + keys = []string{ctxTempAdminKey} + } + ctx = context.WithValue(ctx, constant.RpcCustomHeader, keys) + return context.WithValue(ctx, ctxTempAdminKey, tempAdminValue) +} + +func IsTempAdmin(ctx context.Context) bool { + values, _ := ctx.Value(ctxTempAdminKey).([]string) + return datautil.Equal(tempAdminValue, values) +} + +func IsSystemAdmin(ctx context.Context) bool { + return datautil.Contain(mcontext.GetOpUserID(ctx), GetIMAdminUserIDs(ctx)...) +} diff --git a/pkg/tools/batcher/batcher.go b/pkg/tools/batcher/batcher.go index dcf5d07ad..93a31ed8f 100644 --- a/pkg/tools/batcher/batcher.go +++ b/pkg/tools/batcher/batcher.go @@ -7,6 +7,7 @@ import ( "sync" "time" + "github.com/openimsdk/open-im-server/v3/pkg/authverify" "github.com/openimsdk/tools/errs" "github.com/openimsdk/tools/utils/idutil" ) @@ -253,13 +254,14 @@ func (b *Batcher[T]) distributeMessage(messages map[string][]*T, totalCount int, func (b *Batcher[T]) run(channelID int, ch <-chan *Msg[T]) { defer b.wait.Done() + ctx := authverify.WithTempAdmin(context.Background()) for { select { case messages, ok := <-ch: if !ok { return } - b.Do(context.Background(), channelID, messages) + b.Do(ctx, channelID, messages) if b.config.syncWait { b.counter.Done() } From 8b23d4f5bb3a80e1e636495b80ee0db856f1d31a Mon Sep 17 00:00:00 2001 From: Monet Lee Date: Thu, 22 May 2025 14:21:44 +0800 Subject: [PATCH 161/199] fix: solve user not found when notification invitedUserID is zero in InviteUserToGroup. (#3375) --- internal/rpc/group/group.go | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/internal/rpc/group/group.go b/internal/rpc/group/group.go index 2026ba71b..5dbba1146 100644 --- a/internal/rpc/group/group.go +++ b/internal/rpc/group/group.go @@ -379,9 +379,9 @@ func (g *groupServer) InviteUserToGroup(ctx context.Context, req *pbgroup.Invite } var groupMember *model.GroupMember - var opUserID string + opUserID := mcontext.GetOpUserID(ctx) + if !authverify.IsAdmin(ctx) { - opUserID = mcontext.GetOpUserID(ctx) var err error groupMember, err = g.db.TakeGroupMember(ctx, req.GroupID, opUserID) if err != nil { @@ -390,8 +390,6 @@ func (g *groupServer) InviteUserToGroup(ctx context.Context, req *pbgroup.Invite if err := g.PopulateGroupMember(ctx, groupMember); err != nil { return nil, err } - } else { - opUserID = mcontext.GetOpUserID(ctx) } if err := g.webhookBeforeInviteUserToGroup(ctx, &g.config.WebhooksConfig.BeforeInviteUserToGroup, req); err != nil && err != servererrs.ErrCallbackContinue { @@ -450,10 +448,7 @@ func (g *groupServer) InviteUserToGroup(ctx context.Context, req *pbgroup.Invite const singleQuantity = 50 for start := 0; start < len(groupMembers); start += singleQuantity { - end := start + singleQuantity - if end > len(groupMembers) { - end = len(groupMembers) - } + end := min(start+singleQuantity, len(groupMembers)) currentMembers := groupMembers[start:end] if err := g.db.CreateGroup(ctx, nil, currentMembers); err != nil { @@ -464,8 +459,8 @@ func (g *groupServer) InviteUserToGroup(ctx context.Context, req *pbgroup.Invite return e.UserID }) - if err = g.notification.GroupApplicationAgreeMemberEnterNotification(ctx, req.GroupID, req.SendMessage, opUserID, userIDs...); err != nil { - return nil, err + if len(userIDs) != 0 { + g.notification.GroupApplicationAgreeMemberEnterNotification(ctx, req.GroupID, req.SendMessage, opUserID, userIDs...) } } return &pbgroup.InviteUserToGroupResp{}, nil From 4d69194f62f86bb73d7ff5e563c4891f340d3d1f Mon Sep 17 00:00:00 2001 From: icey-yu <119291641+icey-yu@users.noreply.github.com> Date: Thu, 22 May 2025 14:23:43 +0800 Subject: [PATCH 162/199] fix: send simple msg (#3362) * fix: content in sendSimpleMessage * fix: send simple msg * fix: send simple msg --- internal/api/msg.go | 24 +++++++++++++++++++----- internal/api/router.go | 1 + internal/rpc/msg/send.go | 22 ++++++++++++++++++++++ 3 files changed, 42 insertions(+), 5 deletions(-) diff --git a/internal/api/msg.go b/internal/api/msg.go index 5349faf87..d832bfdcd 100644 --- a/internal/api/msg.go +++ b/internal/api/msg.go @@ -467,6 +467,10 @@ func (m *MessageApi) SendSimpleMessage(c *gin.Context) { sessionType int32 recvID string ) + if err = c.BindJSON(&req); err != nil { + apiresp.GinError(c, errs.ErrArgs.WithDetail(err.Error()).Wrap()) + return + } err = json.Unmarshal(decodedData, &keyMsgData) if err != nil { apiresp.GinError(c, errs.ErrArgs.WithDetail(err.Error()).Wrap()) @@ -490,6 +494,11 @@ func (m *MessageApi) SendSimpleMessage(c *gin.Context) { return } + content, err := jsonutil.JsonMarshal(apistruct.MarkdownTextElem{Content: req.Content}) + if err != nil { + apiresp.GinError(c, errs.Wrap(err)) + return + } msgData := &sdkws.MsgData{ SendID: sendID, RecvID: recvID, @@ -498,17 +507,17 @@ func (m *MessageApi) SendSimpleMessage(c *gin.Context) { SenderPlatformID: constant.AdminPlatformID, SessionType: sessionType, MsgFrom: constant.UserMsgType, - ContentType: constant.Text, - Content: []byte(req.Content), + ContentType: constant.MarkdownText, + Content: content, OfflinePushInfo: req.OfflinePushInfo, Ex: req.Ex, } - sendReq := &msg.SendMsgReq{ + sendReq := &msg.SendSimpleMsgReq{ MsgData: msgData, } - respPb, err := m.Client.SendMsg(c, sendReq) + respPb, err := m.Client.SendSimpleMsg(c, sendReq) if err != nil { apiresp.GinError(c, err) return @@ -525,7 +534,12 @@ func (m *MessageApi) SendSimpleMessage(c *gin.Context) { return } - m.ginRespSendMsg(c, sendReq, respPb) + m.ginRespSendMsg(c, &msg.SendMsgReq{MsgData: sendReq.MsgData}, &msg.SendMsgResp{ + ServerMsgID: respPb.ServerMsgID, + ClientMsgID: respPb.ClientMsgID, + SendTime: respPb.SendTime, + Modify: respPb.Modify, + }) } func (m *MessageApi) CheckMsgIsSendSuccess(c *gin.Context) { diff --git a/internal/api/router.go b/internal/api/router.go index 700d8392e..476aafa48 100644 --- a/internal/api/router.go +++ b/internal/api/router.go @@ -250,6 +250,7 @@ func newGinRouter(ctx context.Context, client discovery.Conn, cfg *Config) (*gin msgGroup.POST("/delete_msg_physical", m.DeleteMsgPhysical) msgGroup.POST("/batch_send_msg", m.BatchSendMsg) + msgGroup.POST("/send_simple_msg", m.SendSimpleMessage) msgGroup.POST("/check_msg_is_send_success", m.CheckMsgIsSendSuccess) msgGroup.POST("/get_server_time", m.GetServerTime) } diff --git a/internal/rpc/msg/send.go b/internal/rpc/msg/send.go index 0e3a9950b..f13b80708 100644 --- a/internal/rpc/msg/send.go +++ b/internal/rpc/msg/send.go @@ -201,3 +201,25 @@ func (m *msgServer) sendMsgSingleChat(ctx context.Context, req *pbmsg.SendMsgReq }, nil } } + +func (m *msgServer) SendSimpleMsg(ctx context.Context, req *pbmsg.SendSimpleMsgReq) (*pbmsg.SendSimpleMsgResp, error) { + if req.MsgData == nil { + return nil, errs.ErrArgs.WrapMsg("msg data is nil") + } + sender, err := m.UserLocalCache.GetUserInfo(ctx, req.MsgData.SendID) + if err != nil { + return nil, err + } + req.MsgData.SenderFaceURL = sender.FaceURL + req.MsgData.SenderNickname = sender.Nickname + resp, err := m.SendMsg(ctx, &pbmsg.SendMsgReq{MsgData: req.MsgData}) + if err != nil { + return nil, err + } + return &pbmsg.SendSimpleMsgResp{ + ServerMsgID: resp.ServerMsgID, + ClientMsgID: resp.ClientMsgID, + SendTime: resp.SendTime, + Modify: resp.Modify, + }, nil +} From 6ae00dfab98da26b1f59ae3acfef23ab03845009 Mon Sep 17 00:00:00 2001 From: Monet Lee Date: Thu, 22 May 2025 14:27:30 +0800 Subject: [PATCH 163/199] fix: solve updateUserInfoEx null pointer. (#3326) --- go.mod | 2 +- go.sum | 4 ++-- internal/rpc/user/user.go | 2 ++ 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index d35252536..dd4034ca5 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/gorilla/websocket v1.5.1 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/mitchellh/mapstructure v1.5.0 - github.com/openimsdk/protocol v0.0.73-alpha.6 + github.com/openimsdk/protocol v0.0.73-alpha.8 github.com/openimsdk/tools v0.0.50-alpha.83 github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 diff --git a/go.sum b/go.sum index 35fc3d4c6..8278480c8 100644 --- a/go.sum +++ b/go.sum @@ -347,8 +347,8 @@ github.com/onsi/gomega v1.25.0 h1:Vw7br2PCDYijJHSfBOWhov+8cAnUf8MfMaIOV323l6Y= github.com/onsi/gomega v1.25.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdMPGhM= github.com/openimsdk/gomake v0.0.15-alpha.5 h1:eEZCEHm+NsmcO3onXZPIUbGFCYPYbsX5beV3ZyOsGhY= github.com/openimsdk/gomake v0.0.15-alpha.5/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= -github.com/openimsdk/protocol v0.0.73-alpha.6 h1:sna9coWG7HN1zObBPtvG0Ki/vzqHXiB4qKbA5P3w7kc= -github.com/openimsdk/protocol v0.0.73-alpha.6/go.mod h1:WF7EuE55vQvpyUAzDXcqg+B+446xQyEba0X35lTINmw= +github.com/openimsdk/protocol v0.0.73-alpha.8 h1:GqksOHXWZSqRQaGYvuVQ4IzA7kFhIXSk7NZk0LGk35A= +github.com/openimsdk/protocol v0.0.73-alpha.8/go.mod h1:WF7EuE55vQvpyUAzDXcqg+B+446xQyEba0X35lTINmw= github.com/openimsdk/tools v0.0.50-alpha.83 h1:7c1D40YGqIWUmGfCII5pduETGC/8c2DyS9SQ4LvoplU= github.com/openimsdk/tools v0.0.50-alpha.83/go.mod h1:n2poR3asX1e1XZce4O+MOWAp+X02QJRFvhcLCXZdzRo= github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= diff --git a/internal/rpc/user/user.go b/internal/rpc/user/user.go index 7f082f784..5639baab9 100644 --- a/internal/rpc/user/user.go +++ b/internal/rpc/user/user.go @@ -197,6 +197,7 @@ func (s *userServer) UpdateUserInfoEx(ctx context.Context, req *pbuser.UpdateUse } s.friendNotificationSender.UserInfoUpdatedNotification(ctx, req.UserInfo.UserID) + //friends, err := s.friendRpcClient.GetFriendIDs(ctx, req.UserInfo.UserID) //if err != nil { // return nil, err @@ -209,6 +210,7 @@ func (s *userServer) UpdateUserInfoEx(ctx context.Context, req *pbuser.UpdateUse //for _, friendID := range friends { // s.friendNotificationSender.FriendInfoUpdatedNotification(ctx, req.UserInfo.UserID, friendID) //} + s.webhookAfterUpdateUserInfoEx(ctx, &s.config.WebhooksConfig.AfterUpdateUserInfoEx, req) if err := s.NotificationUserInfoUpdate(ctx, req.UserInfo.UserID, oldUser); err != nil { return nil, err From 4d3ec5367f6304ae415f80e181464af30a5c8111 Mon Sep 17 00:00:00 2001 From: chao <48119764+withchao@users.noreply.github.com> Date: Thu, 22 May 2025 17:14:30 +0800 Subject: [PATCH 164/199] fix: add rpc interface permission check (#3377) * pb * fix: Modifying other fields while setting IsPrivateChat does not take effect * fix: quote message error revoke * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * upgrading pkg tools * fix * fix * optimize log output * feat: support GetLastMessage * feat: support GetLastMessage * feat: s3 switch * feat: s3 switch * fix: GetUsersOnline * feat: SendBusinessNotification supported configuration parameters * feat: SendBusinessNotification supported configuration parameters * feat: SendBusinessNotification supported configuration parameters * feat: seq conversion failed without exiting * fix: DeleteDoc crash * fix: fill send time * fix: fill send time * fix: crash caused by withdrawing messages from users who have left the group * fix: user msg timestamp * seq read config * seq read config * fix: the source message of the reference is withdrawn, and the referenced message is deleted * feat: optimize the default notification.yml * fix: shouldPushOffline * fix: the sorting is wrong after canceling the administrator in group settings * feat: Sending messages supports returning fields modified by webhook * feat: Sending messages supports returning fields modified by webhook * feat: Sending messages supports returning fields modified by webhook * fix: oss specifies content-type when uploading * fix: the version number contains a line break * fix: the version number contains a line break * feat: GetConversationsHasReadAndMaxSeq support pinned * feat: GetConversationsHasReadAndMaxSeq support pinned * feat: GetConversationsHasReadAndMaxSeq support pinned * fix: transferring the group owner to a muted member, incremental version error * feat: unified conversion code * feat: update gomake * fix: in standalone mode, the user online status is wrong * fix: add permission check * fix: add permission check * fix: add rpc interface permission check * fix: CreateGroupChatConversations --- internal/push/push.go | 3 ++- internal/rpc/conversation/conversation.go | 3 +++ internal/rpc/group/cache.go | 3 +-- internal/rpc/group/group.go | 9 +++++++++ 4 files changed, 15 insertions(+), 3 deletions(-) diff --git a/internal/push/push.go b/internal/push/push.go index 13818e93d..f720a52ac 100644 --- a/internal/push/push.go +++ b/internal/push/push.go @@ -6,6 +6,7 @@ import ( "strconv" "github.com/openimsdk/open-im-server/v3/internal/push/offlinepush" + "github.com/openimsdk/open-im-server/v3/pkg/authverify" "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/mcache" @@ -106,7 +107,7 @@ func Start(ctx context.Context, config *Config, client discovery.Conn, server gr go func() { pushHandler.WaitCache() fn := func(ctx context.Context, key string, value []byte) error { - pushHandler.HandleMs2PsChat(ctx, value) + pushHandler.HandleMs2PsChat(authverify.WithTempAdmin(ctx), value) return nil } consumerCtx := mcontext.SetOperationID(context.Background(), "push_"+strconv.Itoa(int(rand.Uint32()))) diff --git a/internal/rpc/conversation/conversation.go b/internal/rpc/conversation/conversation.go index b0b1053ed..ba9e7746b 100644 --- a/internal/rpc/conversation/conversation.go +++ b/internal/rpc/conversation/conversation.go @@ -432,6 +432,9 @@ func (c *conversationServer) CreateGroupChatConversations(ctx context.Context, r if err != nil { return nil, err } + if err := c.msgClient.SetUserConversationMaxSeq(ctx, conversation.ConversationID, req.UserIDs, 0); err != nil { + return nil, err + } c.webhookAfterCreateGroupChatConversations(ctx, &c.config.WebhooksConfig.AfterCreateGroupChatConversations, &conversation) return &pbconversation.CreateGroupChatConversationsResp{}, nil diff --git a/internal/rpc/group/cache.go b/internal/rpc/group/cache.go index ec0e5b566..27b9eb126 100644 --- a/internal/rpc/group/cache.go +++ b/internal/rpc/group/cache.go @@ -17,7 +17,6 @@ package group import ( "context" - "github.com/openimsdk/open-im-server/v3/pkg/authverify" "github.com/openimsdk/open-im-server/v3/pkg/common/convert" pbgroup "github.com/openimsdk/protocol/group" ) @@ -34,7 +33,7 @@ func (g *groupServer) GetGroupInfoCache(ctx context.Context, req *pbgroup.GetGro } func (g *groupServer) GetGroupMemberCache(ctx context.Context, req *pbgroup.GetGroupMemberCacheReq) (*pbgroup.GetGroupMemberCacheResp, error) { - if err := authverify.CheckAccess(ctx, req.GroupMemberID); err != nil { + if err := g.checkAdminOrInGroup(ctx, req.GroupID); err != nil { return nil, err } members, err := g.db.TakeGroupMember(ctx, req.GroupID, req.GroupMemberID) diff --git a/internal/rpc/group/group.go b/internal/rpc/group/group.go index 5dbba1146..f4a186594 100644 --- a/internal/rpc/group/group.go +++ b/internal/rpc/group/group.go @@ -1303,6 +1303,9 @@ func (g *groupServer) GetGroups(ctx context.Context, req *pbgroup.GetGroupsReq) } func (g *groupServer) GetGroupMembersCMS(ctx context.Context, req *pbgroup.GetGroupMembersCMSReq) (*pbgroup.GetGroupMembersCMSResp, error) { + if err := g.checkAdminOrInGroup(ctx, req.GroupID); err != nil { + return nil, err + } total, members, err := g.db.SearchGroupMember(ctx, req.UserName, req.GroupID, req.Pagination) if err != nil { return nil, err @@ -1712,6 +1715,9 @@ func (g *groupServer) GetUserInGroupMembers(ctx context.Context, req *pbgroup.Ge if len(req.GroupIDs) == 0 { return nil, errs.ErrArgs.WrapMsg("groupIDs empty") } + if err := authverify.CheckAccess(ctx, req.UserID); err != nil { + return nil, err + } members, err := g.db.FindGroupMemberUser(ctx, req.GroupIDs, req.UserID) if err != nil { return nil, err @@ -1743,6 +1749,9 @@ func (g *groupServer) GetGroupMemberRoleLevel(ctx context.Context, req *pbgroup. if len(req.RoleLevels) == 0 { return nil, errs.ErrArgs.WrapMsg("RoleLevels empty") } + if err := g.checkAdminOrInGroup(ctx, req.GroupID); err != nil { + return nil, err + } members, err := g.db.FindGroupMemberRoleLevels(ctx, req.GroupID, req.RoleLevels) if err != nil { return nil, err From 8e61f30e9c59916671e4a75844dcc0a4dfcd5d8a Mon Sep 17 00:00:00 2001 From: chao <48119764+withchao@users.noreply.github.com> Date: Mon, 26 May 2025 10:06:35 +0800 Subject: [PATCH 165/199] feat: optimize friend and group applications (#3384) * pb * fix: Modifying other fields while setting IsPrivateChat does not take effect * fix: quote message error revoke * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * upgrading pkg tools * fix * fix * optimize log output * feat: support GetLastMessage * feat: support GetLastMessage * feat: s3 switch * feat: s3 switch * fix: GetUsersOnline * feat: SendBusinessNotification supported configuration parameters * feat: SendBusinessNotification supported configuration parameters * feat: SendBusinessNotification supported configuration parameters * feat: seq conversion failed without exiting * fix: DeleteDoc crash * fix: fill send time * fix: fill send time * fix: crash caused by withdrawing messages from users who have left the group * fix: user msg timestamp * seq read config * seq read config * fix: the source message of the reference is withdrawn, and the referenced message is deleted * feat: optimize the default notification.yml * fix: shouldPushOffline * fix: the sorting is wrong after canceling the administrator in group settings * feat: Sending messages supports returning fields modified by webhook * feat: Sending messages supports returning fields modified by webhook * feat: Sending messages supports returning fields modified by webhook * fix: oss specifies content-type when uploading * fix: the version number contains a line break * fix: the version number contains a line break * feat: GetConversationsHasReadAndMaxSeq support pinned * feat: GetConversationsHasReadAndMaxSeq support pinned * feat: GetConversationsHasReadAndMaxSeq support pinned * fix: transferring the group owner to a muted member, incremental version error * feat: unified conversion code * feat: update gomake * fix: in standalone mode, the user online status is wrong * fix: add permission check * fix: add permission check * fix: add rpc interface permission check * fix: CreateGroupChatConversations * feat: optimize friend and group applications * feat: optimize friend and group applications * feat: optimize friend and group applications * feat: optimize friend and group applications --- go.mod | 4 +- go.sum | 8 +- internal/api/friend.go | 4 + internal/api/group.go | 4 + internal/api/router.go | 2 + internal/rpc/group/group.go | 70 ++++++-- internal/rpc/group/notification.go | 70 +++++++- internal/rpc/relation/friend.go | 60 +++++-- internal/rpc/relation/notification.go | 152 +++++++++++------- pkg/common/convert/friend.go | 12 +- pkg/common/storage/controller/friend.go | 21 ++- pkg/common/storage/controller/group.go | 24 +-- pkg/common/storage/database/friend_request.go | 6 +- pkg/common/storage/database/group_request.go | 6 +- .../storage/database/mgo/friend_request.go | 48 ++++-- .../storage/database/mgo/group_request.go | 56 +++++-- 16 files changed, 412 insertions(+), 135 deletions(-) diff --git a/go.mod b/go.mod index dd4034ca5..221e28b72 100644 --- a/go.mod +++ b/go.mod @@ -12,8 +12,8 @@ require ( github.com/gorilla/websocket v1.5.1 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/mitchellh/mapstructure v1.5.0 - github.com/openimsdk/protocol v0.0.73-alpha.8 - github.com/openimsdk/tools v0.0.50-alpha.83 + github.com/openimsdk/protocol v0.0.73-alpha.12 + github.com/openimsdk/tools v0.0.50-alpha.84 github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 github.com/stretchr/testify v1.9.0 diff --git a/go.sum b/go.sum index 8278480c8..6298f98c9 100644 --- a/go.sum +++ b/go.sum @@ -347,10 +347,10 @@ github.com/onsi/gomega v1.25.0 h1:Vw7br2PCDYijJHSfBOWhov+8cAnUf8MfMaIOV323l6Y= github.com/onsi/gomega v1.25.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdMPGhM= github.com/openimsdk/gomake v0.0.15-alpha.5 h1:eEZCEHm+NsmcO3onXZPIUbGFCYPYbsX5beV3ZyOsGhY= github.com/openimsdk/gomake v0.0.15-alpha.5/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= -github.com/openimsdk/protocol v0.0.73-alpha.8 h1:GqksOHXWZSqRQaGYvuVQ4IzA7kFhIXSk7NZk0LGk35A= -github.com/openimsdk/protocol v0.0.73-alpha.8/go.mod h1:WF7EuE55vQvpyUAzDXcqg+B+446xQyEba0X35lTINmw= -github.com/openimsdk/tools v0.0.50-alpha.83 h1:7c1D40YGqIWUmGfCII5pduETGC/8c2DyS9SQ4LvoplU= -github.com/openimsdk/tools v0.0.50-alpha.83/go.mod h1:n2poR3asX1e1XZce4O+MOWAp+X02QJRFvhcLCXZdzRo= +github.com/openimsdk/protocol v0.0.73-alpha.12 h1:2NYawXeHChYUeSme6QJ9pOLh+Empce2WmwEtbP4JvKk= +github.com/openimsdk/protocol v0.0.73-alpha.12/go.mod h1:WF7EuE55vQvpyUAzDXcqg+B+446xQyEba0X35lTINmw= +github.com/openimsdk/tools v0.0.50-alpha.84 h1:jN60Ys/0edZjL/TDmm/5VSJFP4pGYRipkWqhILJbq/8= +github.com/openimsdk/tools v0.0.50-alpha.84/go.mod h1:n2poR3asX1e1XZce4O+MOWAp+X02QJRFvhcLCXZdzRo= github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= diff --git a/internal/api/friend.go b/internal/api/friend.go index 7d84ff0dc..0943e8a5d 100644 --- a/internal/api/friend.go +++ b/internal/api/friend.go @@ -114,3 +114,7 @@ func (o *FriendApi) GetIncrementalBlacks(c *gin.Context) { func (o *FriendApi) GetFullFriendUserIDs(c *gin.Context) { a2r.Call(c, relation.FriendClient.GetFullFriendUserIDs, o.Client) } + +func (o *FriendApi) GetSelfUnhandledApplyCount(c *gin.Context) { + a2r.Call(c, relation.FriendClient.GetSelfUnhandledApplyCount, o.Client) +} diff --git a/internal/api/group.go b/internal/api/group.go index 97b6b73f0..926d19a8a 100644 --- a/internal/api/group.go +++ b/internal/api/group.go @@ -165,3 +165,7 @@ func (o *GroupApi) GetFullGroupMemberUserIDs(c *gin.Context) { func (o *GroupApi) GetFullJoinGroupIDs(c *gin.Context) { a2r.Call(c, group.GroupClient.GetFullJoinGroupIDs, o.Client) } + +func (o *GroupApi) GetGroupApplicationUnhandledCount(c *gin.Context) { + a2r.Call(c, group.GroupClient.GetGroupApplicationUnhandledCount, o.Client) +} diff --git a/internal/api/router.go b/internal/api/router.go index 476aafa48..5e5f35a60 100644 --- a/internal/api/router.go +++ b/internal/api/router.go @@ -156,6 +156,7 @@ func newGinRouter(ctx context.Context, client discovery.Conn, cfg *Config) (*gin friendRouterGroup.POST("/update_friends", f.UpdateFriends) friendRouterGroup.POST("/get_incremental_friends", f.GetIncrementalFriends) friendRouterGroup.POST("/get_full_friend_user_ids", f.GetFullFriendUserIDs) + friendRouterGroup.POST("/get_self_unhandled_apply_count", f.GetSelfUnhandledApplyCount) } g := NewGroupApi(group.NewGroupClient(groupConn)) @@ -192,6 +193,7 @@ func newGinRouter(ctx context.Context, client discovery.Conn, cfg *Config) (*gin groupRouterGroup.POST("/get_incremental_group_members_batch", g.GetIncrementalGroupMemberBatch) groupRouterGroup.POST("/get_full_group_member_user_ids", g.GetFullGroupMemberUserIDs) groupRouterGroup.POST("/get_full_join_group_ids", g.GetFullJoinGroupIDs) + groupRouterGroup.POST("/get_group_application_unhandled_count", g.GetGroupApplicationUnhandledCount) } // certificate { diff --git a/internal/rpc/group/group.go b/internal/rpc/group/group.go index f4a186594..5219546b7 100644 --- a/internal/rpc/group/group.go +++ b/internal/rpc/group/group.go @@ -25,7 +25,6 @@ import ( "github.com/openimsdk/open-im-server/v3/pkg/dbbuild" "github.com/openimsdk/open-im-server/v3/pkg/rpcli" - "google.golang.org/grpc" "github.com/openimsdk/open-im-server/v3/pkg/authverify" @@ -153,10 +152,14 @@ func (g *groupServer) NotificationUserInfoUpdate(ctx context.Context, req *pbgro func (g *groupServer) CheckGroupAdmin(ctx context.Context, groupID string) error { if !authverify.IsAdmin(ctx) { - groupMember, err := g.db.TakeGroupMember(ctx, groupID, mcontext.GetOpUserID(ctx)) + members, err := g.db.FindGroupMembers(ctx, groupID, []string{mcontext.GetOpUserID(ctx)}) if err != nil { return err } + if len(members) == 0 { + return errs.ErrNoPermission.WrapMsg("op user not in group") + } + groupMember := members[0] if !(groupMember.RoleLevel == constant.GroupOwner || groupMember.RoleLevel == constant.GroupAdmin) { return errs.ErrNoPermission.WrapMsg("no group owner or admin") } @@ -369,6 +372,10 @@ func (g *groupServer) InviteUserToGroup(ctx context.Context, req *pbgroup.Invite return nil, servererrs.ErrDismissedAlready.WrapMsg("group dismissed checking group status found it dismissed") } + if err := g.checkAdminOrInGroup(ctx, req.GroupID); err != nil { + return nil, err + } + userMap, err := g.userClient.GetUsersInfoMap(ctx, req.InvitedUserIDs) if err != nil { return nil, err @@ -419,7 +426,7 @@ func (g *groupServer) InviteUserToGroup(ctx context.Context, req *pbgroup.Invite ReqMessage: request.ReqMsg, JoinSource: request.JoinSource, InviterUserID: request.InviterUserID, - }) + }, request) } return &pbgroup.InviteUserToGroupResp{}, nil } @@ -685,15 +692,34 @@ func (g *groupServer) GetGroupApplicationList(ctx context.Context, req *pbgroup. if err := authverify.CheckAccess(ctx, req.FromUserID); err != nil { return nil, err } - groupIDs, err := g.db.FindUserManagedGroupID(ctx, req.FromUserID) - if err != nil { - return nil, err + var ( + groupIDs []string + err error + ) + if len(req.GroupIDs) == 0 { + groupIDs, err = g.db.FindUserManagedGroupID(ctx, req.FromUserID) + if err != nil { + return nil, err + } + } else { + req.GroupIDs = datautil.Distinct(req.GroupIDs) + if !authverify.IsAdmin(ctx) { + for _, groupID := range req.GroupIDs { + if err := g.CheckGroupAdmin(ctx, groupID); err != nil { + return nil, err + } + } + } + groupIDs = req.GroupIDs } resp := &pbgroup.GetGroupApplicationListResp{} if len(groupIDs) == 0 { return resp, nil } - total, groupRequests, err := g.db.PageGroupRequest(ctx, groupIDs, req.Pagination) + handleResults := datautil.Slice(req.HandleResults, func(e int32) int { + return int(e) + }) + total, groupRequests, err := g.db.PageGroupRequest(ctx, groupIDs, handleResults, req.Pagination) if err != nil { return nil, err } @@ -758,6 +784,23 @@ func (g *groupServer) GetGroupsInfo(ctx context.Context, req *pbgroup.GetGroupsI }, nil } +func (g *groupServer) GetGroupApplicationUnhandledCount(ctx context.Context, req *pbgroup.GetGroupApplicationUnhandledCountReq) (*pbgroup.GetGroupApplicationUnhandledCountResp, error) { + if err := authverify.CheckAccess(ctx, req.UserID); err != nil { + return nil, err + } + groupIDs, err := g.db.FindUserManagedGroupID(ctx, req.UserID) + if err != nil { + return nil, err + } + count, err := g.db.GetGroupApplicationUnhandledCount(ctx, groupIDs, req.Time) + if err != nil { + return nil, err + } + return &pbgroup.GetGroupApplicationUnhandledCountResp{ + Count: count, + }, nil +} + func (g *groupServer) getGroupsInfo(ctx context.Context, groupIDs []string) ([]*sdkws.GroupInfo, error) { if len(groupIDs) == 0 { return nil, nil @@ -939,7 +982,7 @@ func (g *groupServer) JoinGroup(ctx context.Context, req *pbgroup.JoinGroupReq) if err = g.db.CreateGroupRequest(ctx, []*model.GroupRequest{&groupRequest}); err != nil { return nil, err } - g.notification.JoinGroupApplicationNotification(ctx, req) + g.notification.JoinGroupApplicationNotification(ctx, req, &groupRequest) return &pbgroup.JoinGroupResp{}, nil } @@ -1322,11 +1365,17 @@ func (g *groupServer) GetGroupMembersCMS(ctx context.Context, req *pbgroup.GetGr } func (g *groupServer) GetUserReqApplicationList(ctx context.Context, req *pbgroup.GetUserReqApplicationListReq) (*pbgroup.GetUserReqApplicationListResp, error) { + if err := authverify.CheckAccess(ctx, req.UserID); err != nil { + return nil, err + } user, err := g.userClient.GetUserInfo(ctx, req.UserID) if err != nil { return nil, err } - total, requests, err := g.db.PageGroupRequestUser(ctx, req.UserID, req.Pagination) + handleResults := datautil.Slice(req.HandleResults, func(e int32) int { + return int(e) + }) + total, requests, err := g.db.PageGroupRequestUser(ctx, req.UserID, req.GroupIDs, handleResults, req.Pagination) if err != nil { return nil, err } @@ -1767,6 +1816,9 @@ func (g *groupServer) GetGroupMemberRoleLevel(ctx context.Context, req *pbgroup. } func (g *groupServer) GetGroupUsersReqApplicationList(ctx context.Context, req *pbgroup.GetGroupUsersReqApplicationListReq) (*pbgroup.GetGroupUsersReqApplicationListResp, error) { + if err := g.CheckGroupAdmin(ctx, req.GroupID); err != nil { + return nil, err + } requests, err := g.db.FindGroupRequests(ctx, req.GroupID, req.UserIDs) if err != nil { return nil, err diff --git a/internal/rpc/group/notification.go b/internal/rpc/group/notification.go index 0a18371f9..5a3cbbe26 100644 --- a/internal/rpc/group/notification.go +++ b/internal/rpc/group/notification.go @@ -20,6 +20,7 @@ import ( "fmt" "time" + "github.com/google/uuid" "github.com/openimsdk/open-im-server/v3/pkg/rpcli" "go.mongodb.org/mongo-driver/mongo" @@ -365,13 +366,46 @@ func (g *NotificationSender) GroupInfoSetAnnouncementNotification(ctx context.Co g.Notification(ctx, mcontext.GetOpUserID(ctx), tips.Group.GroupID, constant.GroupInfoSetAnnouncementNotification, tips, notification.WithRpcGetUserName(), notification.WithSendMessage(sendMessage)) } -func (g *NotificationSender) JoinGroupApplicationNotification(ctx context.Context, req *pbgroup.JoinGroupReq) { +func (g *NotificationSender) uuid() string { + return uuid.New().String() +} + +func (g *NotificationSender) getGroupRequest(ctx context.Context, groupID string, userID string) (*sdkws.GroupRequest, error) { + request, err := g.db.TakeGroupRequest(ctx, groupID, userID) + if err != nil { + return nil, err + } + users, err := g.getUsersInfo(ctx, []string{userID}) + if err != nil { + return nil, err + } + if len(users) == 0 { + return nil, servererrs.ErrUserIDNotFound.WrapMsg(fmt.Sprintf("user %s not found", userID)) + } + info, ok := users[0].(*sdkws.UserInfo) + if !ok { + info = &sdkws.UserInfo{ + UserID: users[0].GetUserID(), + Nickname: users[0].GetNickname(), + FaceURL: users[0].GetFaceURL(), + Ex: users[0].GetEx(), + } + } + return convert.Db2PbGroupRequest(request, info, nil), nil +} + +func (g *NotificationSender) JoinGroupApplicationNotification(ctx context.Context, req *pbgroup.JoinGroupReq, dbReq *model.GroupRequest) { var err error defer func() { if err != nil { log.ZError(ctx, stringutil.GetFuncName(1)+" failed", err) } }() + request, err := g.getGroupRequest(ctx, dbReq.GroupID, dbReq.UserID) + if err != nil { + log.ZError(ctx, "JoinGroupApplicationNotification getGroupRequest", err, "dbReq", dbReq) + return + } var group *sdkws.GroupInfo group, err = g.getGroupInfo(ctx, req.GroupID) if err != nil { @@ -387,7 +421,13 @@ func (g *NotificationSender) JoinGroupApplicationNotification(ctx context.Contex return } userIDs = append(userIDs, req.InviterUserID, mcontext.GetOpUserID(ctx)) - tips := &sdkws.JoinGroupApplicationTips{Group: group, Applicant: user, ReqMsg: req.ReqMessage} + tips := &sdkws.JoinGroupApplicationTips{ + Group: group, + Applicant: user, + ReqMsg: req.ReqMessage, + Uuid: g.uuid(), + Request: request, + } for _, userID := range datautil.Distinct(userIDs) { g.Notification(ctx, mcontext.GetOpUserID(ctx), userID, constant.JoinGroupApplicationNotification, tips) } @@ -417,6 +457,11 @@ func (g *NotificationSender) GroupApplicationAcceptedNotification(ctx context.Co log.ZError(ctx, stringutil.GetFuncName(1)+" failed", err) } }() + request, err := g.getGroupRequest(ctx, req.GroupID, req.FromUserID) + if err != nil { + log.ZError(ctx, "GroupApplicationAcceptedNotification getGroupRequest", err, "req", req) + return + } var group *sdkws.GroupInfo group, err = g.getGroupInfo(ctx, req.GroupID) if err != nil { @@ -432,8 +477,14 @@ func (g *NotificationSender) GroupApplicationAcceptedNotification(ctx context.Co if err = g.fillOpUser(ctx, &opUser, group.GroupID); err != nil { return } + tips := &sdkws.GroupApplicationAcceptedTips{ + Group: group, + OpUser: opUser, + HandleMsg: req.HandledMsg, + Uuid: g.uuid(), + Request: request, + } for _, userID := range append(userIDs, req.FromUserID) { - tips := &sdkws.GroupApplicationAcceptedTips{Group: group, OpUser: opUser, HandleMsg: req.HandledMsg} if userID == req.FromUserID { tips.ReceiverAs = applicantReceiver } else { @@ -450,6 +501,11 @@ func (g *NotificationSender) GroupApplicationRejectedNotification(ctx context.Co log.ZError(ctx, stringutil.GetFuncName(1)+" failed", err) } }() + request, err := g.getGroupRequest(ctx, req.GroupID, req.FromUserID) + if err != nil { + log.ZError(ctx, "GroupApplicationAcceptedNotification getGroupRequest", err, "req", req) + return + } var group *sdkws.GroupInfo group, err = g.getGroupInfo(ctx, req.GroupID) if err != nil { @@ -465,8 +521,14 @@ func (g *NotificationSender) GroupApplicationRejectedNotification(ctx context.Co if err = g.fillOpUser(ctx, &opUser, group.GroupID); err != nil { return } + tips := &sdkws.GroupApplicationRejectedTips{ + Group: group, + OpUser: opUser, + HandleMsg: req.HandledMsg, + Uuid: g.uuid(), + Request: request, + } for _, userID := range append(userIDs, req.FromUserID) { - tips := &sdkws.GroupApplicationAcceptedTips{Group: group, OpUser: opUser, HandleMsg: req.HandledMsg} if userID == req.FromUserID { tips.ReceiverAs = applicantReceiver } else { diff --git a/internal/rpc/relation/friend.go b/internal/rpc/relation/friend.go index 06661c79d..8c7c40536 100644 --- a/internal/rpc/relation/friend.go +++ b/internal/rpc/relation/friend.go @@ -18,6 +18,7 @@ import ( "context" "github.com/openimsdk/open-im-server/v3/pkg/dbbuild" + "github.com/openimsdk/open-im-server/v3/pkg/notification/common_user" "github.com/openimsdk/open-im-server/v3/pkg/rpcli" "github.com/openimsdk/tools/mq/memamq" @@ -100,23 +101,24 @@ func Start(ctx context.Context, config *Config, client discovery.Conn, server gr return err } userClient := rpcli.NewUserClient(userConn) - + database := controller.NewFriendDatabase( + friendMongoDB, + friendRequestMongoDB, + redis.NewFriendCacheRedis(rdb, &config.LocalCacheConfig, friendMongoDB), + mgocli.GetTx(), + ) // Initialize notification sender notificationSender := NewFriendNotificationSender( &config.NotificationConfig, rpcli.NewMsgClient(msgConn), WithRpcFunc(userClient.GetUsersInfo), + WithFriendDB(database), ) localcache.InitLocalCache(&config.LocalCacheConfig) // Register Friend server with refactored MongoDB and Redis integrations relation.RegisterFriendServer(server, &friendServer{ - db: controller.NewFriendDatabase( - friendMongoDB, - friendRequestMongoDB, - redis.NewFriendCacheRedis(rdb, &config.LocalCacheConfig, friendMongoDB), - mgocli.GetTx(), - ), + db: database, blackDatabase: controller.NewBlackDatabase( blackMongoDB, redis.NewBlackCacheRedis(rdb, &config.LocalCacheConfig, blackMongoDB), @@ -328,7 +330,7 @@ func (s *friendServer) GetDesignatedFriendsApply(ctx context.Context, req *relat return nil, err } resp = &relation.GetDesignatedFriendsApplyResp{} - resp.FriendRequests, err = convert.FriendRequestDB2Pb(ctx, friendRequests, s.userClient.GetUsersInfoMap) + resp.FriendRequests, err = convert.FriendRequestDB2Pb(ctx, friendRequests, s.getCommonUserMap) if err != nil { return nil, err } @@ -341,13 +343,16 @@ func (s *friendServer) GetPaginationFriendsApplyTo(ctx context.Context, req *rel return nil, err } - total, friendRequests, err := s.db.PageFriendRequestToMe(ctx, req.UserID, req.Pagination) + handleResults := datautil.Slice(req.HandleResults, func(e int32) int { + return int(e) + }) + total, friendRequests, err := s.db.PageFriendRequestToMe(ctx, req.UserID, handleResults, req.Pagination) if err != nil { return nil, err } resp = &relation.GetPaginationFriendsApplyToResp{} - resp.FriendRequests, err = convert.FriendRequestDB2Pb(ctx, friendRequests, s.userClient.GetUsersInfoMap) + resp.FriendRequests, err = convert.FriendRequestDB2Pb(ctx, friendRequests, s.getCommonUserMap) if err != nil { return nil, err } @@ -358,18 +363,20 @@ func (s *friendServer) GetPaginationFriendsApplyTo(ctx context.Context, req *rel } func (s *friendServer) GetPaginationFriendsApplyFrom(ctx context.Context, req *relation.GetPaginationFriendsApplyFromReq) (resp *relation.GetPaginationFriendsApplyFromResp, err error) { - resp = &relation.GetPaginationFriendsApplyFromResp{} - if err := authverify.CheckAccess(ctx, req.UserID); err != nil { return nil, err } - total, friendRequests, err := s.db.PageFriendRequestFromMe(ctx, req.UserID, req.Pagination) + handleResults := datautil.Slice(req.HandleResults, func(e int32) int { + return int(e) + }) + total, friendRequests, err := s.db.PageFriendRequestFromMe(ctx, req.UserID, handleResults, req.Pagination) if err != nil { return nil, err } - resp.FriendRequests, err = convert.FriendRequestDB2Pb(ctx, friendRequests, s.userClient.GetUsersInfoMap) + resp = &relation.GetPaginationFriendsApplyFromResp{} + resp.FriendRequests, err = convert.FriendRequestDB2Pb(ctx, friendRequests, s.getCommonUserMap) if err != nil { return nil, err } @@ -544,3 +551,28 @@ func (s *friendServer) UpdateFriends(ctx context.Context, req *relation.UpdateFr s.notificationSender.FriendsInfoUpdateNotification(ctx, req.OwnerUserID, req.FriendUserIDs) return resp, nil } + +func (s *friendServer) GetSelfUnhandledApplyCount(ctx context.Context, req *relation.GetSelfUnhandledApplyCountReq) (*relation.GetSelfUnhandledApplyCountResp, error) { + if err := authverify.CheckAccess(ctx, req.UserID); err != nil { + return nil, err + } + + count, err := s.db.GetUnhandledCount(ctx, req.UserID, req.Time) + if err != nil { + return nil, err + } + + return &relation.GetSelfUnhandledApplyCountResp{ + Count: count, + }, nil +} + +func (s *friendServer) getCommonUserMap(ctx context.Context, userIDs []string) (map[string]common_user.CommonUser, error) { + users, err := s.userClient.GetUsersInfo(ctx, userIDs) + if err != nil { + return nil, err + } + return datautil.SliceToMapAny(users, func(e *sdkws.UserInfo) (string, common_user.CommonUser) { + return e.UserID, e + }), nil +} diff --git a/internal/rpc/relation/notification.go b/internal/rpc/relation/notification.go index caf2dafe1..d6a03003e 100644 --- a/internal/rpc/relation/notification.go +++ b/internal/rpc/relation/notification.go @@ -19,6 +19,9 @@ import ( "github.com/openimsdk/open-im-server/v3/pkg/rpcli" "github.com/openimsdk/protocol/msg" + "github.com/openimsdk/tools/errs" + "github.com/openimsdk/tools/log" + "github.com/openimsdk/tools/utils/datautil" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/versionctx" @@ -52,9 +55,7 @@ func WithFriendDB(db controller.FriendDatabase) friendNotificationSenderOptions } } -func WithDBFunc( - fn func(ctx context.Context, userIDs []string) (users []*relationtb.User, err error), -) friendNotificationSenderOptions { +func WithDBFunc(fn func(ctx context.Context, userIDs []string) (users []*relationtb.User, err error)) friendNotificationSenderOptions { return func(s *FriendNotificationSender) { f := func(ctx context.Context, userIDs []string) (result []common_user.CommonUser, err error) { users, err := fn(ctx, userIDs) @@ -70,9 +71,7 @@ func WithDBFunc( } } -func WithRpcFunc( - fn func(ctx context.Context, userIDs []string) ([]*sdkws.UserInfo, error), -) friendNotificationSenderOptions { +func WithRpcFunc(fn func(ctx context.Context, userIDs []string) ([]*sdkws.UserInfo, error)) friendNotificationSenderOptions { return func(s *FriendNotificationSender) { f := func(ctx context.Context, userIDs []string) (result []common_user.CommonUser, err error) { users, err := fn(ctx, userIDs) @@ -100,10 +99,7 @@ func NewFriendNotificationSender(conf *config.Notification, msgClient *rpcli.Msg return f } -func (f *FriendNotificationSender) getUsersInfoMap( - ctx context.Context, - userIDs []string, -) (map[string]*sdkws.UserInfo, error) { +func (f *FriendNotificationSender) getUsersInfoMap(ctx context.Context, userIDs []string) (map[string]*sdkws.UserInfo, error) { users, err := f.getUsersInfo(ctx, userIDs) if err != nil { return nil, err @@ -116,10 +112,7 @@ func (f *FriendNotificationSender) getUsersInfoMap( } //nolint:unused -func (f *FriendNotificationSender) getFromToUserNickname( - ctx context.Context, - fromUserID, toUserID string, -) (string, string, error) { +func (f *FriendNotificationSender) getFromToUserNickname(ctx context.Context, fromUserID, toUserID string) (string, string, error) { users, err := f.getUsersInfoMap(ctx, []string{fromUserID, toUserID}) if err != nil { return "", "", nil @@ -132,61 +125,108 @@ func (f *FriendNotificationSender) UserInfoUpdatedNotification(ctx context.Conte f.Notification(ctx, mcontext.GetOpUserID(ctx), changedUserID, constant.UserInfoUpdatedNotification, &tips) } -func (f *FriendNotificationSender) FriendApplicationAddNotification(ctx context.Context, req *relation.ApplyToAddFriendReq) { - tips := sdkws.FriendApplicationTips{FromToUserID: &sdkws.FromToUserID{ - FromUserID: req.FromUserID, - ToUserID: req.ToUserID, - }} - f.Notification(ctx, req.FromUserID, req.ToUserID, constant.FriendApplicationNotification, &tips) +func (f *FriendNotificationSender) getCommonUserMap(ctx context.Context, userIDs []string) (map[string]common_user.CommonUser, error) { + users, err := f.getUsersInfo(ctx, userIDs) + if err != nil { + return nil, err + } + return datautil.SliceToMap(users, func(e common_user.CommonUser) string { + return e.GetUserID() + }), nil } -func (f *FriendNotificationSender) FriendApplicationAgreedNotification( - ctx context.Context, - req *relation.RespondFriendApplyReq, -) { - tips := sdkws.FriendApplicationApprovedTips{FromToUserID: &sdkws.FromToUserID{ - FromUserID: req.FromUserID, - ToUserID: req.ToUserID, - }, HandleMsg: req.HandleMsg} - f.Notification(ctx, req.ToUserID, req.FromUserID, constant.FriendApplicationApprovedNotification, &tips) +func (f *FriendNotificationSender) getFriendRequests(ctx context.Context, fromUserID, toUserID string) (*sdkws.FriendRequest, error) { + if f.db == nil { + return nil, errs.ErrInternalServer.WithDetail("db is nil") + } + friendRequests, err := f.db.FindBothFriendRequests(ctx, fromUserID, toUserID) + if err != nil { + return nil, err + } + requests, err := convert.FriendRequestDB2Pb(ctx, friendRequests, f.getCommonUserMap) + if err != nil { + return nil, err + } + for _, request := range requests { + if request.FromUserID == fromUserID && request.ToUserID == toUserID { + return request, nil + } + } + return nil, errs.ErrRecordNotFound.WrapMsg("friend request not found", "fromUserID", fromUserID, "toUserID", toUserID) } -func (f *FriendNotificationSender) FriendApplicationRefusedNotification( - ctx context.Context, - req *relation.RespondFriendApplyReq, -) { - tips := sdkws.FriendApplicationApprovedTips{FromToUserID: &sdkws.FromToUserID{ - FromUserID: req.FromUserID, - ToUserID: req.ToUserID, - }, HandleMsg: req.HandleMsg} - f.Notification(ctx, req.ToUserID, req.FromUserID, constant.FriendApplicationRejectedNotification, &tips) +func (f *FriendNotificationSender) FriendApplicationAddNotification(ctx context.Context, req *relation.ApplyToAddFriendReq) { + request, err := f.getFriendRequests(ctx, req.FromUserID, req.ToUserID) + if err != nil { + log.ZError(ctx, "FriendApplicationAddNotification get friend request", err, "fromUserID", req.FromUserID, "toUserID", req.ToUserID) + return + } + tips := sdkws.FriendApplicationTips{ + FromToUserID: &sdkws.FromToUserID{ + FromUserID: req.FromUserID, + ToUserID: req.ToUserID, + }, + Request: request, + } + f.Notification(ctx, req.FromUserID, req.ToUserID, constant.FriendApplicationNotification, &tips) } -func (f *FriendNotificationSender) FriendAddedNotification( - ctx context.Context, - operationID, opUserID, fromUserID, toUserID string, -) error { - tips := sdkws.FriendAddedTips{Friend: &sdkws.FriendInfo{}, OpUser: &sdkws.PublicUserInfo{}} - user, err := f.getUsersInfo(ctx, []string{opUserID}) +func (f *FriendNotificationSender) FriendApplicationAgreedNotification(ctx context.Context, req *relation.RespondFriendApplyReq) { + request, err := f.getFriendRequests(ctx, req.FromUserID, req.ToUserID) if err != nil { - return err + log.ZError(ctx, "FriendApplicationAgreedNotification get friend request", err, "fromUserID", req.FromUserID, "toUserID", req.ToUserID) + return } - tips.OpUser.UserID = user[0].GetUserID() - tips.OpUser.Ex = user[0].GetEx() - tips.OpUser.Nickname = user[0].GetNickname() - tips.OpUser.FaceURL = user[0].GetFaceURL() - friends, err := f.db.FindFriendsWithError(ctx, fromUserID, []string{toUserID}) - if err != nil { - return err + tips := sdkws.FriendApplicationApprovedTips{ + FromToUserID: &sdkws.FromToUserID{ + FromUserID: req.FromUserID, + ToUserID: req.ToUserID, + }, + HandleMsg: req.HandleMsg, + Request: request, } - tips.Friend, err = convert.FriendDB2Pb(ctx, friends[0], f.getUsersInfoMap) + f.Notification(ctx, req.ToUserID, req.FromUserID, constant.FriendApplicationApprovedNotification, &tips) +} + +func (f *FriendNotificationSender) FriendApplicationRefusedNotification(ctx context.Context, req *relation.RespondFriendApplyReq) { + request, err := f.getFriendRequests(ctx, req.FromUserID, req.ToUserID) if err != nil { - return err + log.ZError(ctx, "FriendApplicationRefusedNotification get friend request", err, "fromUserID", req.FromUserID, "toUserID", req.ToUserID) + return } - f.Notification(ctx, fromUserID, toUserID, constant.FriendAddedNotification, &tips) - return nil + tips := sdkws.FriendApplicationRejectedTips{ + FromToUserID: &sdkws.FromToUserID{ + FromUserID: req.FromUserID, + ToUserID: req.ToUserID, + }, + HandleMsg: req.HandleMsg, + Request: request, + } + f.Notification(ctx, req.ToUserID, req.FromUserID, constant.FriendApplicationRejectedNotification, &tips) } +//func (f *FriendNotificationSender) FriendAddedNotification(ctx context.Context, operationID, opUserID, fromUserID, toUserID string) error { +// tips := sdkws.FriendAddedTips{Friend: &sdkws.FriendInfo{}, OpUser: &sdkws.PublicUserInfo{}} +// user, err := f.getUsersInfo(ctx, []string{opUserID}) +// if err != nil { +// return err +// } +// tips.OpUser.UserID = user[0].GetUserID() +// tips.OpUser.Ex = user[0].GetEx() +// tips.OpUser.Nickname = user[0].GetNickname() +// tips.OpUser.FaceURL = user[0].GetFaceURL() +// friends, err := f.db.FindFriendsWithError(ctx, fromUserID, []string{toUserID}) +// if err != nil { +// return err +// } +// tips.Friend, err = convert.FriendDB2Pb(ctx, friends[0], f.getUsersInfoMap) +// if err != nil { +// return err +// } +// f.Notification(ctx, fromUserID, toUserID, constant.FriendAddedNotification, &tips) +// return nil +//} + func (f *FriendNotificationSender) FriendDeletedNotification(ctx context.Context, req *relation.DeleteFriendReq) { tips := sdkws.FriendDeletedTips{FromToUserID: &sdkws.FromToUserID{ FromUserID: req.OwnerUserID, diff --git a/pkg/common/convert/friend.go b/pkg/common/convert/friend.go index 6d346b0f4..e783ecb24 100644 --- a/pkg/common/convert/friend.go +++ b/pkg/common/convert/friend.go @@ -17,7 +17,9 @@ package convert import ( "context" "fmt" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" + "github.com/openimsdk/open-im-server/v3/pkg/notification/common_user" "github.com/openimsdk/protocol/relation" "github.com/openimsdk/protocol/sdkws" @@ -98,7 +100,7 @@ func FriendOnlyDB2PbOnly(friendsDB []*model.Friend) []*relation.FriendInfoOnly { }) } -func FriendRequestDB2Pb(ctx context.Context, friendRequests []*model.FriendRequest, getUsers func(ctx context.Context, userIDs []string) (map[string]*sdkws.UserInfo, error)) ([]*sdkws.FriendRequest, error) { +func FriendRequestDB2Pb(ctx context.Context, friendRequests []*model.FriendRequest, getUsers func(ctx context.Context, userIDs []string) (map[string]common_user.CommonUser, error)) ([]*sdkws.FriendRequest, error) { if len(friendRequests) == 0 { return nil, nil } @@ -117,11 +119,11 @@ func FriendRequestDB2Pb(ctx context.Context, friendRequests []*model.FriendReque fromUser := users[friendRequest.FromUserID] res = append(res, &sdkws.FriendRequest{ FromUserID: friendRequest.FromUserID, - FromNickname: fromUser.Nickname, - FromFaceURL: fromUser.FaceURL, + FromNickname: fromUser.GetNickname(), + FromFaceURL: fromUser.GetFaceURL(), ToUserID: friendRequest.ToUserID, - ToNickname: toUser.Nickname, - ToFaceURL: toUser.FaceURL, + ToNickname: toUser.GetNickname(), + ToFaceURL: toUser.GetFaceURL(), HandleResult: friendRequest.HandleResult, ReqMsg: friendRequest.ReqMsg, CreateTime: friendRequest.CreateTime.UnixMilli(), diff --git a/pkg/common/storage/controller/friend.go b/pkg/common/storage/controller/friend.go index 88a5fc863..806468ea1 100644 --- a/pkg/common/storage/controller/friend.go +++ b/pkg/common/storage/controller/friend.go @@ -17,10 +17,11 @@ package controller import ( "context" "fmt" + "time" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database/mgo" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" - "time" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache" "github.com/openimsdk/protocol/constant" @@ -61,10 +62,10 @@ type FriendDatabase interface { PageInWhoseFriends(ctx context.Context, friendUserID string, pagination pagination.Pagination) (total int64, friends []*model.Friend, err error) // PageFriendRequestFromMe retrieves the friend requests sent by the user with pagination - PageFriendRequestFromMe(ctx context.Context, userID string, pagination pagination.Pagination) (total int64, friends []*model.FriendRequest, err error) + PageFriendRequestFromMe(ctx context.Context, userID string, handleResults []int, pagination pagination.Pagination) (total int64, friends []*model.FriendRequest, err error) // PageFriendRequestToMe retrieves the friend requests received by the user with pagination - PageFriendRequestToMe(ctx context.Context, userID string, pagination pagination.Pagination) (total int64, friends []*model.FriendRequest, err error) + PageFriendRequestToMe(ctx context.Context, userID string, handleResults []int, pagination pagination.Pagination) (total int64, friends []*model.FriendRequest, err error) // FindFriendsWithError fetches specified friends of a user and returns an error if any do not exist FindFriendsWithError(ctx context.Context, ownerUserID string, friendUserIDs []string) (friends []*model.Friend, err error) @@ -87,6 +88,8 @@ type FriendDatabase interface { FindFriendUserID(ctx context.Context, friendUserID string) ([]string, error) OwnerIncrVersion(ctx context.Context, ownerUserID string, friendUserIDs []string, state int32) error + + GetUnhandledCount(ctx context.Context, userID string, ts int64) (int64, error) } type friendDatabase struct { @@ -334,13 +337,13 @@ func (f *friendDatabase) PageInWhoseFriends(ctx context.Context, friendUserID st } // PageFriendRequestFromMe retrieves friend requests sent by me. It does not return an error if the result is empty. -func (f *friendDatabase) PageFriendRequestFromMe(ctx context.Context, userID string, pagination pagination.Pagination) (total int64, friends []*model.FriendRequest, err error) { - return f.friendRequest.FindFromUserID(ctx, userID, pagination) +func (f *friendDatabase) PageFriendRequestFromMe(ctx context.Context, userID string, handleResults []int, pagination pagination.Pagination) (total int64, friends []*model.FriendRequest, err error) { + return f.friendRequest.FindFromUserID(ctx, userID, handleResults, pagination) } // PageFriendRequestToMe retrieves friend requests received by me. It does not return an error if the result is empty. -func (f *friendDatabase) PageFriendRequestToMe(ctx context.Context, userID string, pagination pagination.Pagination) (total int64, friends []*model.FriendRequest, err error) { - return f.friendRequest.FindToUserID(ctx, userID, pagination) +func (f *friendDatabase) PageFriendRequestToMe(ctx context.Context, userID string, handleResults []int, pagination pagination.Pagination) (total int64, friends []*model.FriendRequest, err error) { + return f.friendRequest.FindToUserID(ctx, userID, handleResults, pagination) } // FindFriendsWithError retrieves specified friends' information for ownerUserID. Returns an error if any friend does not exist. @@ -397,3 +400,7 @@ func (f *friendDatabase) OwnerIncrVersion(ctx context.Context, ownerUserID strin } return f.cache.DelMaxFriendVersion(ownerUserID).ChainExecDel(ctx) } + +func (f *friendDatabase) GetUnhandledCount(ctx context.Context, userID string, ts int64) (int64, error) { + return f.friendRequest.GetUnhandledCount(ctx, userID, ts) +} diff --git a/pkg/common/storage/controller/group.go b/pkg/common/storage/controller/group.go index 6de0432a3..037ab1c39 100644 --- a/pkg/common/storage/controller/group.go +++ b/pkg/common/storage/controller/group.go @@ -68,7 +68,7 @@ type GroupDatabase interface { // FindUserManagedGroupID retrieves group IDs managed by a user. FindUserManagedGroupID(ctx context.Context, userID string) (groupIDs []string, err error) // PageGroupRequest paginates through group requests for specified groups. - PageGroupRequest(ctx context.Context, groupIDs []string, pagination pagination.Pagination) (int64, []*model.GroupRequest, error) + PageGroupRequest(ctx context.Context, groupIDs []string, handleResults []int, pagination pagination.Pagination) (int64, []*model.GroupRequest, error) // GetGroupRoleLevelMemberIDs retrieves user IDs of group members with a specific role level. GetGroupRoleLevelMemberIDs(ctx context.Context, groupID string, roleLevel int32) ([]string, error) @@ -100,7 +100,7 @@ type GroupDatabase interface { // FindGroupRequests retrieves multiple group join requests. FindGroupRequests(ctx context.Context, groupID string, userIDs []string) ([]*model.GroupRequest, error) // PageGroupRequestUser paginates through group join requests made by a user. - PageGroupRequestUser(ctx context.Context, userID string, pagination pagination.Pagination) (int64, []*model.GroupRequest, error) + PageGroupRequestUser(ctx context.Context, userID string, groupIDs []string, handleResults []int, pagination pagination.Pagination) (int64, []*model.GroupRequest, error) // CountTotal counts the total number of groups as of a certain date. CountTotal(ctx context.Context, before *time.Time) (count int64, err error) @@ -124,6 +124,8 @@ type GroupDatabase interface { SearchJoinGroup(ctx context.Context, userID string, keyword string, pagination pagination.Pagination) (int64, []*model.Group, error) FindJoinGroupID(ctx context.Context, userID string) ([]string, error) + + GetGroupApplicationUnhandledCount(ctx context.Context, groupIDs []string, ts int64) (int64, error) } func NewGroupDatabase( @@ -304,8 +306,8 @@ func (g *groupDatabase) FindUserManagedGroupID(ctx context.Context, userID strin return g.groupMemberDB.FindUserManagedGroupID(ctx, userID) } -func (g *groupDatabase) PageGroupRequest(ctx context.Context, groupIDs []string, pagination pagination.Pagination) (int64, []*model.GroupRequest, error) { - return g.groupRequestDB.PageGroup(ctx, groupIDs, pagination) +func (g *groupDatabase) PageGroupRequest(ctx context.Context, groupIDs []string, handleResults []int, pagination pagination.Pagination) (int64, []*model.GroupRequest, error) { + return g.groupRequestDB.PageGroup(ctx, groupIDs, handleResults, pagination) } func (g *groupDatabase) PageGetJoinGroup(ctx context.Context, userID string, pagination pagination.Pagination) (total int64, totalGroupMembers []*model.GroupMember, err error) { @@ -463,16 +465,12 @@ func (g *groupDatabase) CreateGroupRequest(ctx context.Context, requests []*mode }) } -func (g *groupDatabase) TakeGroupRequest( - ctx context.Context, - groupID string, - userID string, -) (*model.GroupRequest, error) { +func (g *groupDatabase) TakeGroupRequest(ctx context.Context, groupID string, userID string) (*model.GroupRequest, error) { return g.groupRequestDB.Take(ctx, groupID, userID) } -func (g *groupDatabase) PageGroupRequestUser(ctx context.Context, userID string, pagination pagination.Pagination) (int64, []*model.GroupRequest, error) { - return g.groupRequestDB.Page(ctx, userID, pagination) +func (g *groupDatabase) PageGroupRequestUser(ctx context.Context, userID string, groupIDs []string, handleResults []int, pagination pagination.Pagination) (int64, []*model.GroupRequest, error) { + return g.groupRequestDB.Page(ctx, userID, groupIDs, handleResults, pagination) } func (g *groupDatabase) CountTotal(ctx context.Context, before *time.Time) (count int64, err error) { @@ -565,3 +563,7 @@ func (g *groupDatabase) MemberGroupIncrVersion(ctx context.Context, groupID stri } return g.cache.DelMaxGroupMemberVersion(groupID).ChainExecDel(ctx) } + +func (g *groupDatabase) GetGroupApplicationUnhandledCount(ctx context.Context, groupIDs []string, ts int64) (int64, error) { + return g.groupRequestDB.GetUnhandledCount(ctx, groupIDs, ts) +} diff --git a/pkg/common/storage/database/friend_request.go b/pkg/common/storage/database/friend_request.go index f163b4831..c0df77823 100644 --- a/pkg/common/storage/database/friend_request.go +++ b/pkg/common/storage/database/friend_request.go @@ -16,6 +16,7 @@ package database import ( "context" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" "github.com/openimsdk/tools/db/pagination" ) @@ -33,8 +34,9 @@ type FriendRequest interface { Find(ctx context.Context, fromUserID, toUserID string) (friendRequest *model.FriendRequest, err error) Take(ctx context.Context, fromUserID, toUserID string) (friendRequest *model.FriendRequest, err error) // Get list of friend requests received by toUserID - FindToUserID(ctx context.Context, toUserID string, pagination pagination.Pagination) (total int64, friendRequests []*model.FriendRequest, err error) + FindToUserID(ctx context.Context, toUserID string, handleResults []int, pagination pagination.Pagination) (total int64, friendRequests []*model.FriendRequest, err error) // Get list of friend requests sent by fromUserID - FindFromUserID(ctx context.Context, fromUserID string, pagination pagination.Pagination) (total int64, friendRequests []*model.FriendRequest, err error) + FindFromUserID(ctx context.Context, fromUserID string, handleResults []int, pagination pagination.Pagination) (total int64, friendRequests []*model.FriendRequest, err error) FindBothFriendRequests(ctx context.Context, fromUserID, toUserID string) (friends []*model.FriendRequest, err error) + GetUnhandledCount(ctx context.Context, userID string, ts int64) (int64, error) } diff --git a/pkg/common/storage/database/group_request.go b/pkg/common/storage/database/group_request.go index 7309584f0..766ea2cd5 100644 --- a/pkg/common/storage/database/group_request.go +++ b/pkg/common/storage/database/group_request.go @@ -16,6 +16,7 @@ package database import ( "context" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" "github.com/openimsdk/tools/db/pagination" ) @@ -26,6 +27,7 @@ type GroupRequest interface { UpdateHandler(ctx context.Context, groupID string, userID string, handledMsg string, handleResult int32) (err error) Take(ctx context.Context, groupID string, userID string) (groupRequest *model.GroupRequest, err error) FindGroupRequests(ctx context.Context, groupID string, userIDs []string) ([]*model.GroupRequest, error) - Page(ctx context.Context, userID string, pagination pagination.Pagination) (total int64, groups []*model.GroupRequest, err error) - PageGroup(ctx context.Context, groupIDs []string, pagination pagination.Pagination) (total int64, groups []*model.GroupRequest, err error) + Page(ctx context.Context, userID string, groupIDs []string, handleResults []int, pagination pagination.Pagination) (total int64, groups []*model.GroupRequest, err error) + PageGroup(ctx context.Context, groupIDs []string, handleResults []int, pagination pagination.Pagination) (total int64, groups []*model.GroupRequest, err error) + GetUnhandledCount(ctx context.Context, groupIDs []string, ts int64) (int64, error) } diff --git a/pkg/common/storage/database/mgo/friend_request.go b/pkg/common/storage/database/mgo/friend_request.go index 4eed2f4a2..95edb77df 100644 --- a/pkg/common/storage/database/mgo/friend_request.go +++ b/pkg/common/storage/database/mgo/friend_request.go @@ -16,24 +16,32 @@ package mgo import ( "context" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" + "go.mongodb.org/mongo-driver/mongo/options" "github.com/openimsdk/tools/db/mongoutil" "github.com/openimsdk/tools/db/pagination" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/mongo" - "go.mongodb.org/mongo-driver/mongo/options" ) func NewFriendRequestMongo(db *mongo.Database) (database.FriendRequest, error) { coll := db.Collection(database.FriendRequestName) - _, err := coll.Indexes().CreateOne(context.Background(), mongo.IndexModel{ - Keys: bson.D{ - {Key: "from_user_id", Value: 1}, - {Key: "to_user_id", Value: 1}, + _, err := coll.Indexes().CreateMany(context.Background(), []mongo.IndexModel{ + { + Keys: bson.D{ + {Key: "from_user_id", Value: 1}, + {Key: "to_user_id", Value: 1}, + }, + Options: options.Index().SetUnique(true), + }, + { + Keys: bson.D{ + {Key: "create_time", Value: -1}, + }, }, - Options: options.Index().SetUnique(true), }) if err != nil { return nil, err @@ -45,12 +53,24 @@ type FriendRequestMgo struct { coll *mongo.Collection } -func (f *FriendRequestMgo) FindToUserID(ctx context.Context, toUserID string, pagination pagination.Pagination) (total int64, friendRequests []*model.FriendRequest, err error) { - return mongoutil.FindPage[*model.FriendRequest](ctx, f.coll, bson.M{"to_user_id": toUserID}, pagination) +func (f *FriendRequestMgo) sort() any { + return bson.D{{Key: "create_time", Value: -1}} } -func (f *FriendRequestMgo) FindFromUserID(ctx context.Context, fromUserID string, pagination pagination.Pagination) (total int64, friendRequests []*model.FriendRequest, err error) { - return mongoutil.FindPage[*model.FriendRequest](ctx, f.coll, bson.M{"from_user_id": fromUserID}, pagination) +func (f *FriendRequestMgo) FindToUserID(ctx context.Context, toUserID string, handleResults []int, pagination pagination.Pagination) (total int64, friendRequests []*model.FriendRequest, err error) { + filter := bson.M{"to_user_id": toUserID} + if len(handleResults) > 0 { + filter["handle_result"] = bson.M{"$in": handleResults} + } + return mongoutil.FindPage[*model.FriendRequest](ctx, f.coll, filter, pagination, options.Find().SetSort(f.sort())) +} + +func (f *FriendRequestMgo) FindFromUserID(ctx context.Context, fromUserID string, handleResults []int, pagination pagination.Pagination) (total int64, friendRequests []*model.FriendRequest, err error) { + filter := bson.M{"from_user_id": fromUserID} + if len(handleResults) > 0 { + filter["handle_result"] = bson.M{"$in": handleResults} + } + return mongoutil.FindPage[*model.FriendRequest](ctx, f.coll, filter, pagination, options.Find().SetSort(f.sort())) } func (f *FriendRequestMgo) FindBothFriendRequests(ctx context.Context, fromUserID, toUserID string) (friends []*model.FriendRequest, err error) { @@ -110,3 +130,11 @@ func (f *FriendRequestMgo) Find(ctx context.Context, fromUserID, toUserID string func (f *FriendRequestMgo) Take(ctx context.Context, fromUserID, toUserID string) (friendRequest *model.FriendRequest, err error) { return f.Find(ctx, fromUserID, toUserID) } + +func (f *FriendRequestMgo) GetUnhandledCount(ctx context.Context, userID string, ts int64) (int64, error) { + filter := bson.M{"to_user_id": userID, "handle_result": 0} + if ts != 0 { + filter["req_time"] = bson.M{"$gt": ts} + } + return mongoutil.Count(ctx, f.coll, filter) +} diff --git a/pkg/common/storage/database/mgo/group_request.go b/pkg/common/storage/database/mgo/group_request.go index b1942b708..c67bbb3ea 100644 --- a/pkg/common/storage/database/mgo/group_request.go +++ b/pkg/common/storage/database/mgo/group_request.go @@ -16,8 +16,10 @@ package mgo import ( "context" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" + "github.com/openimsdk/tools/utils/datautil" "github.com/openimsdk/tools/db/mongoutil" "github.com/openimsdk/tools/db/pagination" @@ -29,12 +31,19 @@ import ( func NewGroupRequestMgo(db *mongo.Database) (database.GroupRequest, error) { coll := db.Collection(database.GroupRequestName) - _, err := coll.Indexes().CreateOne(context.Background(), mongo.IndexModel{ - Keys: bson.D{ - {Key: "group_id", Value: 1}, - {Key: "user_id", Value: 1}, + _, err := coll.Indexes().CreateMany(context.Background(), []mongo.IndexModel{ + { + Keys: bson.D{ + {Key: "group_id", Value: 1}, + {Key: "user_id", Value: 1}, + }, + Options: options.Index().SetUnique(true), + }, + { + Keys: bson.D{ + {Key: "req_time", Value: -1}, + }, }, - Options: options.Index().SetUnique(true), }) if err != nil { return nil, errs.Wrap(err) @@ -66,10 +75,39 @@ func (g *GroupRequestMgo) FindGroupRequests(ctx context.Context, groupID string, return mongoutil.Find[*model.GroupRequest](ctx, g.coll, bson.M{"group_id": groupID, "user_id": bson.M{"$in": userIDs}}) } -func (g *GroupRequestMgo) Page(ctx context.Context, userID string, pagination pagination.Pagination) (total int64, groups []*model.GroupRequest, err error) { - return mongoutil.FindPage[*model.GroupRequest](ctx, g.coll, bson.M{"user_id": userID}, pagination) +func (g *GroupRequestMgo) sort() any { + return bson.D{{Key: "req_time", Value: -1}} +} + +func (g *GroupRequestMgo) Page(ctx context.Context, userID string, groupIDs []string, handleResults []int, pagination pagination.Pagination) (total int64, groups []*model.GroupRequest, err error) { + filter := bson.M{"user_id": userID} + if len(groupIDs) > 0 { + filter["group_id"] = bson.M{"$in": datautil.Distinct(groupIDs)} + } + if len(handleResults) > 0 { + filter["handle_result"] = bson.M{"$in": handleResults} + } + return mongoutil.FindPage[*model.GroupRequest](ctx, g.coll, filter, pagination, options.Find().SetSort(g.sort())) +} + +func (g *GroupRequestMgo) PageGroup(ctx context.Context, groupIDs []string, handleResults []int, pagination pagination.Pagination) (total int64, groups []*model.GroupRequest, err error) { + if len(groupIDs) == 0 { + return 0, nil, nil + } + filter := bson.M{"group_id": bson.M{"$in": groupIDs}} + if len(handleResults) > 0 { + filter["handle_result"] = bson.M{"$in": handleResults} + } + return mongoutil.FindPage[*model.GroupRequest](ctx, g.coll, filter, pagination, options.Find().SetSort(g.sort())) } -func (g *GroupRequestMgo) PageGroup(ctx context.Context, groupIDs []string, pagination pagination.Pagination) (total int64, groups []*model.GroupRequest, err error) { - return mongoutil.FindPage[*model.GroupRequest](ctx, g.coll, bson.M{"group_id": bson.M{"$in": groupIDs}}, pagination) +func (g *GroupRequestMgo) GetUnhandledCount(ctx context.Context, groupIDs []string, ts int64) (int64, error) { + if len(groupIDs) == 0 { + return 0, nil + } + filter := bson.M{"group_id": bson.M{"$in": groupIDs}, "handle_result": 0} + if ts != 0 { + filter["req_time"] = bson.M{"$gt": ts} + } + return mongoutil.Count(ctx, g.coll, filter) } From 812c1e41271750306c3b4705c555a0bc7913aaa4 Mon Sep 17 00:00:00 2001 From: chao <48119764+withchao@users.noreply.github.com> Date: Tue, 27 May 2025 15:01:09 +0800 Subject: [PATCH 166/199] fix: optimize friend and group applications (#3389) * pb * fix: Modifying other fields while setting IsPrivateChat does not take effect * fix: quote message error revoke * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * upgrading pkg tools * fix * fix * optimize log output * feat: support GetLastMessage * feat: support GetLastMessage * feat: s3 switch * feat: s3 switch * fix: GetUsersOnline * feat: SendBusinessNotification supported configuration parameters * feat: SendBusinessNotification supported configuration parameters * feat: SendBusinessNotification supported configuration parameters * feat: seq conversion failed without exiting * fix: DeleteDoc crash * fix: fill send time * fix: fill send time * fix: crash caused by withdrawing messages from users who have left the group * fix: user msg timestamp * seq read config * seq read config * fix: the source message of the reference is withdrawn, and the referenced message is deleted * feat: optimize the default notification.yml * fix: shouldPushOffline * fix: the sorting is wrong after canceling the administrator in group settings * feat: Sending messages supports returning fields modified by webhook * feat: Sending messages supports returning fields modified by webhook * feat: Sending messages supports returning fields modified by webhook * fix: oss specifies content-type when uploading * fix: the version number contains a line break * fix: the version number contains a line break * feat: GetConversationsHasReadAndMaxSeq support pinned * feat: GetConversationsHasReadAndMaxSeq support pinned * feat: GetConversationsHasReadAndMaxSeq support pinned * fix: transferring the group owner to a muted member, incremental version error * feat: unified conversion code * feat: update gomake * fix: in standalone mode, the user online status is wrong * fix: add permission check * fix: add permission check * fix: add rpc interface permission check * fix: CreateGroupChatConversations * feat: optimize friend and group applications * feat: optimize friend and group applications * feat: optimize friend and group applications * feat: optimize friend and group applications * fix: optimize friend and group applications --- pkg/common/storage/database/mgo/friend_request.go | 11 +++++++---- pkg/common/storage/database/mgo/group_request.go | 10 ++++++---- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/pkg/common/storage/database/mgo/friend_request.go b/pkg/common/storage/database/mgo/friend_request.go index 95edb77df..12e63155b 100644 --- a/pkg/common/storage/database/mgo/friend_request.go +++ b/pkg/common/storage/database/mgo/friend_request.go @@ -16,15 +16,18 @@ package mgo import ( "context" + "time" + + "go.mongodb.org/mongo-driver/mongo/options" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" - "go.mongodb.org/mongo-driver/mongo/options" - "github.com/openimsdk/tools/db/mongoutil" - "github.com/openimsdk/tools/db/pagination" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/mongo" + + "github.com/openimsdk/tools/db/mongoutil" + "github.com/openimsdk/tools/db/pagination" ) func NewFriendRequestMongo(db *mongo.Database) (database.FriendRequest, error) { @@ -134,7 +137,7 @@ func (f *FriendRequestMgo) Take(ctx context.Context, fromUserID, toUserID string func (f *FriendRequestMgo) GetUnhandledCount(ctx context.Context, userID string, ts int64) (int64, error) { filter := bson.M{"to_user_id": userID, "handle_result": 0} if ts != 0 { - filter["req_time"] = bson.M{"$gt": ts} + filter["create_time"] = bson.M{"$gt": time.Unix(ts, 0)} } return mongoutil.Count(ctx, f.coll, filter) } diff --git a/pkg/common/storage/database/mgo/group_request.go b/pkg/common/storage/database/mgo/group_request.go index c67bbb3ea..fb57f890f 100644 --- a/pkg/common/storage/database/mgo/group_request.go +++ b/pkg/common/storage/database/mgo/group_request.go @@ -16,17 +16,19 @@ package mgo import ( "context" + "time" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" "github.com/openimsdk/tools/utils/datautil" - "github.com/openimsdk/tools/db/mongoutil" - "github.com/openimsdk/tools/db/pagination" - "github.com/openimsdk/tools/errs" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/mongo" "go.mongodb.org/mongo-driver/mongo/options" + + "github.com/openimsdk/tools/db/mongoutil" + "github.com/openimsdk/tools/db/pagination" + "github.com/openimsdk/tools/errs" ) func NewGroupRequestMgo(db *mongo.Database) (database.GroupRequest, error) { @@ -107,7 +109,7 @@ func (g *GroupRequestMgo) GetUnhandledCount(ctx context.Context, groupIDs []stri } filter := bson.M{"group_id": bson.M{"$in": groupIDs}, "handle_result": 0} if ts != 0 { - filter["req_time"] = bson.M{"$gt": ts} + filter["req_time"] = bson.M{"$gt": time.Unix(ts, 0)} } return mongoutil.Count(ctx, g.coll, filter) } From b7ca3bd95f580fc16d6de1caf8d9708e97c42dc3 Mon Sep 17 00:00:00 2001 From: HonQi <14954524+HonQii@users.noreply.github.com> Date: Wed, 4 Jun 2025 10:46:20 +0800 Subject: [PATCH 167/199] fix redis config db field (#3395) --- pkg/common/config/config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/common/config/config.go b/pkg/common/config/config.go index d5ae68ec0..cd57a11cf 100644 --- a/pkg/common/config/config.go +++ b/pkg/common/config/config.go @@ -328,7 +328,7 @@ type Redis struct { Username string `yaml:"username"` Password string `yaml:"password"` ClusterMode bool `yaml:"clusterMode"` - DB int `yaml:"storage"` + DB int `yaml:"db"` MaxRetry int `yaml:"maxRetry"` PoolSize int `yaml:"poolSize"` } From 04ee509b68b4ad1579d948cbaf82b37d2a2e518c Mon Sep 17 00:00:00 2001 From: Monet Lee Date: Fri, 6 Jun 2025 11:29:45 +0800 Subject: [PATCH 168/199] fix: solve incorrect when sendMsg webhook callback after. (#3409) * fix: solve incorrect when sendMsg webhook callback after. * remove print. --- internal/msgtransfer/callback.go | 124 ++++++++++++++++++ internal/msgtransfer/init.go | 2 +- .../online_msg_to_mongo_handler.go | 18 ++- internal/rpc/msg/send.go | 6 +- 4 files changed, 146 insertions(+), 4 deletions(-) create mode 100644 internal/msgtransfer/callback.go diff --git a/internal/msgtransfer/callback.go b/internal/msgtransfer/callback.go new file mode 100644 index 000000000..ea51c2839 --- /dev/null +++ b/internal/msgtransfer/callback.go @@ -0,0 +1,124 @@ +package msgtransfer + +import ( + "context" + "encoding/base64" + + "github.com/openimsdk/open-im-server/v3/pkg/apistruct" + "github.com/openimsdk/open-im-server/v3/pkg/common/config" + "github.com/openimsdk/open-im-server/v3/pkg/common/webhook" + "github.com/openimsdk/protocol/constant" + "github.com/openimsdk/protocol/sdkws" + "github.com/openimsdk/tools/mcontext" + "github.com/openimsdk/tools/utils/datautil" + "github.com/openimsdk/tools/utils/stringutil" + "google.golang.org/protobuf/proto" + + cbapi "github.com/openimsdk/open-im-server/v3/pkg/callbackstruct" +) + +func toCommonCallback(ctx context.Context, msg *sdkws.MsgData, command string) cbapi.CommonCallbackReq { + return cbapi.CommonCallbackReq{ + SendID: msg.SendID, + ServerMsgID: msg.ServerMsgID, + CallbackCommand: command, + ClientMsgID: msg.ClientMsgID, + OperationID: mcontext.GetOperationID(ctx), + SenderPlatformID: msg.SenderPlatformID, + SenderNickname: msg.SenderNickname, + SessionType: msg.SessionType, + MsgFrom: msg.MsgFrom, + ContentType: msg.ContentType, + Status: msg.Status, + SendTime: msg.SendTime, + CreateTime: msg.CreateTime, + AtUserIDList: msg.AtUserIDList, + SenderFaceURL: msg.SenderFaceURL, + Content: GetContent(msg), + Seq: uint32(msg.Seq), + Ex: msg.Ex, + } +} + +func GetContent(msg *sdkws.MsgData) string { + if msg.ContentType >= constant.NotificationBegin && msg.ContentType <= constant.NotificationEnd { + var tips sdkws.TipsComm + _ = proto.Unmarshal(msg.Content, &tips) + content := tips.JsonDetail + return content + } else { + return string(msg.Content) + } +} + +func (mc *OnlineHistoryMongoConsumerHandler) webhookAfterSendSingleMsg(ctx context.Context, after *config.AfterConfig, msg *sdkws.MsgData) { + if msg.ContentType == constant.Typing { + return + } + if !filterAfterMsg(msg, after) { + return + } + cbReq := &cbapi.CallbackAfterSendSingleMsgReq{ + CommonCallbackReq: toCommonCallback(ctx, msg, cbapi.CallbackAfterSendSingleMsgCommand), + RecvID: msg.RecvID, + } + mc.webhookClient.AsyncPostWithQuery(ctx, cbReq.GetCallbackCommand(), cbReq, &cbapi.CallbackAfterSendSingleMsgResp{}, after, buildKeyMsgDataQuery(msg)) +} + +func (mc *OnlineHistoryMongoConsumerHandler) webhookAfterSendGroupMsg(ctx context.Context, after *config.AfterConfig, msg *sdkws.MsgData) { + if msg.ContentType == constant.Typing { + return + } + if !filterAfterMsg(msg, after) { + return + } + cbReq := &cbapi.CallbackAfterSendGroupMsgReq{ + CommonCallbackReq: toCommonCallback(ctx, msg, cbapi.CallbackAfterSendGroupMsgCommand), + GroupID: msg.GroupID, + } + + mc.webhookClient.AsyncPostWithQuery(ctx, cbReq.GetCallbackCommand(), cbReq, &cbapi.CallbackAfterSendGroupMsgResp{}, after, buildKeyMsgDataQuery(msg)) +} + +func buildKeyMsgDataQuery(msg *sdkws.MsgData) map[string]string { + keyMsgData := apistruct.KeyMsgData{ + SendID: msg.SendID, + RecvID: msg.RecvID, + GroupID: msg.GroupID, + } + + return map[string]string{ + webhook.Key: base64.StdEncoding.EncodeToString(stringutil.StructToJsonBytes(keyMsgData)), + } +} + +func filterAfterMsg(msg *sdkws.MsgData, after *config.AfterConfig) bool { + return filterMsg(msg, after.AttentionIds, after.DeniedTypes) +} + +func filterMsg(msg *sdkws.MsgData, attentionIds []string, deniedTypes []int32) bool { + // According to the attentionIds configuration, only some users are sent + if len(attentionIds) != 0 && !datautil.Contain(msg.RecvID, attentionIds...) { + return false + } + + if defaultDeniedTypes(msg.ContentType) { + return false + } + + if len(deniedTypes) != 0 && datautil.Contain(msg.ContentType, deniedTypes...) { + return false + } + + return true +} + +func defaultDeniedTypes(contentType int32) bool { + if contentType >= constant.NotificationBegin && contentType <= constant.NotificationEnd { + return true + } + if contentType == constant.Typing { + return true + } + return false +} diff --git a/internal/msgtransfer/init.go b/internal/msgtransfer/init.go index 175813552..b07ec6f1d 100644 --- a/internal/msgtransfer/init.go +++ b/internal/msgtransfer/init.go @@ -134,7 +134,7 @@ func Start(ctx context.Context, config *Config, client discovery.Conn, server gr if err != nil { return err } - historyMongoHandler := NewOnlineHistoryMongoConsumerHandler(msgTransferDatabase) + historyMongoHandler := NewOnlineHistoryMongoConsumerHandler(msgTransferDatabase,config) msgTransfer := &MsgTransfer{ historyConsumer: historyConsumer, diff --git a/internal/msgtransfer/online_msg_to_mongo_handler.go b/internal/msgtransfer/online_msg_to_mongo_handler.go index a895bb9c4..6c1498f82 100644 --- a/internal/msgtransfer/online_msg_to_mongo_handler.go +++ b/internal/msgtransfer/online_msg_to_mongo_handler.go @@ -19,6 +19,8 @@ import ( "github.com/openimsdk/open-im-server/v3/pkg/common/prommetrics" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/controller" + "github.com/openimsdk/open-im-server/v3/pkg/common/webhook" + "github.com/openimsdk/protocol/constant" pbmsg "github.com/openimsdk/protocol/msg" "github.com/openimsdk/tools/log" "google.golang.org/protobuf/proto" @@ -26,11 +28,15 @@ import ( type OnlineHistoryMongoConsumerHandler struct { msgTransferDatabase controller.MsgTransferDatabase + config *Config + webhookClient *webhook.Client } -func NewOnlineHistoryMongoConsumerHandler(database controller.MsgTransferDatabase) *OnlineHistoryMongoConsumerHandler { +func NewOnlineHistoryMongoConsumerHandler(database controller.MsgTransferDatabase, config *Config) *OnlineHistoryMongoConsumerHandler { return &OnlineHistoryMongoConsumerHandler{ msgTransferDatabase: database, + config: config, + webhookClient: webhook.NewWebhookClient(config.WebhooksConfig.URL), } } @@ -53,6 +59,16 @@ func (mc *OnlineHistoryMongoConsumerHandler) HandleChatWs2Mongo(ctx context.Cont } else { prommetrics.MsgInsertMongoSuccessCounter.Inc() } + + for _, msgData := range msgFromMQ.MsgData { + switch msgData.SessionType { + case constant.SingleChatType: + mc.webhookAfterSendSingleMsg(ctx, &mc.config.WebhooksConfig.AfterSendSingleMsg, msgData) + case constant.ReadGroupChatType: + mc.webhookAfterSendGroupMsg(ctx, &mc.config.WebhooksConfig.AfterSendGroupMsg, msgData) + } + } + //var seqs []int64 //for _, msg := range msgFromMQ.MsgData { // seqs = append(seqs, msg.Seq) diff --git a/internal/rpc/msg/send.go b/internal/rpc/msg/send.go index f13b80708..d97905bff 100644 --- a/internal/rpc/msg/send.go +++ b/internal/rpc/msg/send.go @@ -86,7 +86,8 @@ func (m *msgServer) sendMsgGroupChat(ctx context.Context, req *pbmsg.SendMsgReq, go m.setConversationAtInfo(ctx, req.MsgData) } - m.webhookAfterSendGroupMsg(ctx, &m.config.WebhooksConfig.AfterSendGroupMsg, req) + // m.webhookAfterSendGroupMsg(ctx, &m.config.WebhooksConfig.AfterSendGroupMsg, req) + prommetrics.GroupChatMsgProcessSuccessCounter.Inc() resp = &pbmsg.SendMsgResp{} resp.SendTime = req.MsgData.SendTime @@ -192,7 +193,8 @@ func (m *msgServer) sendMsgSingleChat(ctx context.Context, req *pbmsg.SendMsgReq prommetrics.SingleChatMsgProcessFailedCounter.Inc() return nil, err } - m.webhookAfterSendSingleMsg(ctx, &m.config.WebhooksConfig.AfterSendSingleMsg, req) + + // m.webhookAfterSendSingleMsg(ctx, &m.config.WebhooksConfig.AfterSendSingleMsg, req) prommetrics.SingleChatMsgProcessSuccessCounter.Inc() return &pbmsg.SendMsgResp{ ServerMsgID: req.MsgData.ServerMsgID, From 75367545ea330bdb67a6e4242cb31e5652ce44dc Mon Sep 17 00:00:00 2001 From: Monet Lee Date: Fri, 6 Jun 2025 14:25:58 +0800 Subject: [PATCH 169/199] feat: support distributed lock in crontask. (#3401) * feat: support distributed lock in crontask. * remove space. * remove comment. * remove log. * Update logic. * Update contents. --- internal/tools/cron/cron_task.go | 21 ++++++-- internal/tools/cron/dist_look.go | 89 ++++++++++++++++++++++++++++++++ start-config.yml | 2 +- 3 files changed, 108 insertions(+), 4 deletions(-) create mode 100644 internal/tools/cron/dist_look.go diff --git a/internal/tools/cron/cron_task.go b/internal/tools/cron/cron_task.go index 7ae314193..a4de309d4 100644 --- a/internal/tools/cron/cron_task.go +++ b/internal/tools/cron/cron_task.go @@ -58,6 +58,11 @@ func Start(ctx context.Context, conf *Config, client discovery.Conn, service grp cm.Watch(ctx) } + locker, err := NewEtcdLocker(client.(*etcd.SvcDiscoveryRegistryImpl).GetClient()) + if err != nil { + return err + } + srv := &cronServer{ ctx: ctx, config: conf, @@ -65,6 +70,7 @@ func Start(ctx context.Context, conf *Config, client discovery.Conn, service grp msgClient: msg.NewMsgClient(msgConn), conversationClient: pbconversation.NewConversationClient(conversationConn), thirdClient: third.NewThirdClient(thirdConn), + locker: locker, } if err := srv.registerClearS3(); err != nil { @@ -81,6 +87,8 @@ func Start(ctx context.Context, conf *Config, client discovery.Conn, service grp log.ZDebug(ctx, "cron task server is running") <-ctx.Done() log.ZDebug(ctx, "cron task server is shutting down") + srv.cron.Stop() + return nil } @@ -91,6 +99,7 @@ type cronServer struct { msgClient msg.MsgClient conversationClient pbconversation.ConversationClient thirdClient third.ThirdClient + locker *EtcdLocker } func (c *cronServer) registerClearS3() error { @@ -98,7 +107,9 @@ func (c *cronServer) registerClearS3() error { log.ZInfo(c.ctx, "disable scheduled cleanup of s3", "fileExpireTime", c.config.CronTask.FileExpireTime, "deleteObjectType", c.config.CronTask.DeleteObjectType) return nil } - _, err := c.cron.AddFunc(c.config.CronTask.CronExecuteTime, c.clearS3) + _, err := c.cron.AddFunc(c.config.CronTask.CronExecuteTime, func() { + c.locker.ExecuteWithLock(c.ctx, "clearS3", c.clearS3) + }) return errs.WrapMsg(err, "failed to register clear s3 cron task") } @@ -107,11 +118,15 @@ func (c *cronServer) registerDeleteMsg() error { log.ZInfo(c.ctx, "disable scheduled cleanup of chat records", "retainChatRecords", c.config.CronTask.RetainChatRecords) return nil } - _, err := c.cron.AddFunc(c.config.CronTask.CronExecuteTime, c.deleteMsg) + _, err := c.cron.AddFunc(c.config.CronTask.CronExecuteTime, func() { + c.locker.ExecuteWithLock(c.ctx, "deleteMsg", c.deleteMsg) + }) return errs.WrapMsg(err, "failed to register delete msg cron task") } func (c *cronServer) registerClearUserMsg() error { - _, err := c.cron.AddFunc(c.config.CronTask.CronExecuteTime, c.clearUserMsg) + _, err := c.cron.AddFunc(c.config.CronTask.CronExecuteTime, func() { + c.locker.ExecuteWithLock(c.ctx, "clearUserMsg", c.clearUserMsg) + }) return errs.WrapMsg(err, "failed to register clear user msg cron task") } diff --git a/internal/tools/cron/dist_look.go b/internal/tools/cron/dist_look.go new file mode 100644 index 000000000..d1d2d1cb0 --- /dev/null +++ b/internal/tools/cron/dist_look.go @@ -0,0 +1,89 @@ +package cron + +import ( + "context" + "fmt" + "os" + "time" + + "github.com/openimsdk/tools/log" + clientv3 "go.etcd.io/etcd/client/v3" + "go.etcd.io/etcd/client/v3/concurrency" +) + +const ( + lockLeaseTTL = 300 +) + +type EtcdLocker struct { + client *clientv3.Client + instanceID string +} + +// NewEtcdLocker creates a new etcd distributed lock +func NewEtcdLocker(client *clientv3.Client) (*EtcdLocker, error) { + hostname, _ := os.Hostname() + pid := os.Getpid() + instanceID := fmt.Sprintf("%s-pid-%d-%d", hostname, pid, time.Now().UnixNano()) + + locker := &EtcdLocker{ + client: client, + instanceID: instanceID, + } + + return locker, nil +} + +func (e *EtcdLocker) ExecuteWithLock(ctx context.Context, taskName string, task func()) { + session, err := concurrency.NewSession(e.client, concurrency.WithTTL(lockLeaseTTL)) + if err != nil { + log.ZWarn(ctx, "Failed to create etcd session", err, + "taskName", taskName, + "instanceID", e.instanceID) + return + } + defer session.Close() + + lockKey := fmt.Sprintf("openim/crontask/%s", taskName) + mutex := concurrency.NewMutex(session, lockKey) + + ctxWithTimeout, cancel := context.WithTimeout(ctx, 100*time.Millisecond) + defer cancel() + + err = mutex.TryLock(ctxWithTimeout) + if err != nil { + if err == context.DeadlineExceeded { + log.ZDebug(ctx, "Task is being executed by another instance, skipping", + "taskName", taskName, + "instanceID", e.instanceID) + } else { + log.ZWarn(ctx, "Failed to acquire task lock", err, + "taskName", taskName, + "instanceID", e.instanceID) + } + return + } + + defer func() { + if err := mutex.Unlock(ctx); err != nil { + log.ZWarn(ctx, "Failed to release task lock", err, + "taskName", taskName, + "instanceID", e.instanceID) + } else { + log.ZInfo(ctx, "Successfully released task lock", + "taskName", taskName, + "instanceID", e.instanceID) + } + }() + + log.ZInfo(ctx, "Successfully acquired task lock, starting execution", + "taskName", taskName, + "instanceID", e.instanceID, + "sessionID", session.Lease()) + + task() + + log.ZInfo(ctx, "Task execution completed", + "taskName", taskName, + "instanceID", e.instanceID) +} diff --git a/start-config.yml b/start-config.yml index 1231b5d0d..da959044d 100644 --- a/start-config.yml +++ b/start-config.yml @@ -1,6 +1,6 @@ serviceBinaries: openim-api: 1 - openim-crontask: 1 + openim-crontask: 4 openim-rpc-user: 1 openim-msggateway: 1 openim-push: 8 From d156e1e5199cbc923a919ef422a589c9ca73b385 Mon Sep 17 00:00:00 2001 From: icey-yu <119291641+icey-yu@users.noreply.github.com> Date: Wed, 11 Jun 2025 16:29:44 +0800 Subject: [PATCH 170/199] fix: prometheus discovery (#3408) --- cmd/main.go | 17 ++------------ go.mod | 2 +- go.sum | 4 ++-- internal/api/init.go | 2 +- internal/api/prometheus_discovery.go | 27 ++++++++++++----------- internal/api/router.go | 2 +- internal/msggateway/init.go | 2 +- internal/msgtransfer/init.go | 2 +- internal/push/push.go | 2 +- internal/rpc/auth/auth.go | 2 +- internal/rpc/conversation/conversation.go | 2 +- internal/rpc/group/group.go | 2 +- internal/rpc/msg/server.go | 2 +- internal/rpc/relation/friend.go | 2 +- internal/rpc/third/third.go | 2 +- internal/rpc/user/user.go | 2 +- internal/tools/cron/cron_task.go | 2 +- pkg/common/cmd/api.go | 3 ++- pkg/common/cmd/msg_transfer.go | 3 ++- pkg/common/prommetrics/prommetrics.go | 8 ++++++- pkg/common/startrpc/start.go | 10 +++++---- 21 files changed, 49 insertions(+), 51 deletions(-) diff --git a/cmd/main.go b/cmd/main.go index 1d0b82be8..7e19f1c98 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -3,7 +3,6 @@ package main import ( "bytes" "context" - "encoding/json" "flag" "fmt" "net" @@ -39,7 +38,6 @@ import ( "github.com/openimsdk/tools/log" "github.com/openimsdk/tools/system/program" "github.com/openimsdk/tools/utils/datautil" - "github.com/openimsdk/tools/utils/network" "github.com/spf13/viper" "google.golang.org/grpc" ) @@ -250,23 +248,12 @@ func (x *cmds) run(ctx context.Context) error { return err } } - ip, err := network.GetLocalIP() - if err != nil { - return err - } listener, err := net.Listen("tcp", fmt.Sprintf(":%d", port)) if err != nil { return fmt.Errorf("prometheus listen %d error %w", port, err) } defer listener.Close() log.ZDebug(ctx, "prometheus start", "addr", listener.Addr()) - target, err := json.Marshal(prommetrics.BuildDefaultTarget(ip, listener.Addr().(*net.TCPAddr).Port)) - if err != nil { - return err - } - if err := standalone.GetKeyValue().SetKey(ctx, prommetrics.BuildDiscoveryKey(prommetrics.APIKeyName), target); err != nil { - return err - } go func() { err := prommetrics.Start(listener) if err == nil { @@ -342,7 +329,7 @@ func (x *cmds) run(ctx context.Context) error { } } -func putCmd[C any](cmd *cmds, block bool, fn func(ctx context.Context, config *C, client discovery.Conn, server grpc.ServiceRegistrar) error) { +func putCmd[C any](cmd *cmds, block bool, fn func(ctx context.Context, config *C, client discovery.SvcDiscoveryRegistry, server grpc.ServiceRegistrar) error) { name := path.Base(runtime.FuncForPC(reflect.ValueOf(fn).Pointer()).Name()) if index := strings.Index(name, "."); index >= 0 { name = name[:index] @@ -352,7 +339,7 @@ func putCmd[C any](cmd *cmds, block bool, fn func(ctx context.Context, config *C if err := cmd.parseConf(&conf); err != nil { return err } - return fn(ctx, &conf, standalone.GetDiscoveryConn(), standalone.GetServiceRegistrar()) + return fn(ctx, &conf, standalone.GetSvcDiscoveryRegistry(), standalone.GetServiceRegistrar()) }) } diff --git a/go.mod b/go.mod index 221e28b72..845f75bb2 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/mitchellh/mapstructure v1.5.0 github.com/openimsdk/protocol v0.0.73-alpha.12 - github.com/openimsdk/tools v0.0.50-alpha.84 + github.com/openimsdk/tools v0.0.50-alpha.85 github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 github.com/stretchr/testify v1.9.0 diff --git a/go.sum b/go.sum index 6298f98c9..b775a1056 100644 --- a/go.sum +++ b/go.sum @@ -349,8 +349,8 @@ github.com/openimsdk/gomake v0.0.15-alpha.5 h1:eEZCEHm+NsmcO3onXZPIUbGFCYPYbsX5b github.com/openimsdk/gomake v0.0.15-alpha.5/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= github.com/openimsdk/protocol v0.0.73-alpha.12 h1:2NYawXeHChYUeSme6QJ9pOLh+Empce2WmwEtbP4JvKk= github.com/openimsdk/protocol v0.0.73-alpha.12/go.mod h1:WF7EuE55vQvpyUAzDXcqg+B+446xQyEba0X35lTINmw= -github.com/openimsdk/tools v0.0.50-alpha.84 h1:jN60Ys/0edZjL/TDmm/5VSJFP4pGYRipkWqhILJbq/8= -github.com/openimsdk/tools v0.0.50-alpha.84/go.mod h1:n2poR3asX1e1XZce4O+MOWAp+X02QJRFvhcLCXZdzRo= +github.com/openimsdk/tools v0.0.50-alpha.85 h1:OqTUYx6r7Zp/eH8FKB08XeNjPV405TUIG9QT6QQ+F+s= +github.com/openimsdk/tools v0.0.50-alpha.85/go.mod h1:n2poR3asX1e1XZce4O+MOWAp+X02QJRFvhcLCXZdzRo= github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= diff --git a/internal/api/init.go b/internal/api/init.go index 378f03eda..f3548e29a 100644 --- a/internal/api/init.go +++ b/internal/api/init.go @@ -39,7 +39,7 @@ type Config struct { Index conf.Index } -func Start(ctx context.Context, config *Config, client discovery.Conn, service grpc.ServiceRegistrar) error { +func Start(ctx context.Context, config *Config, client discovery.SvcDiscoveryRegistry, service grpc.ServiceRegistrar) error { apiPort, err := datautil.GetElemByIndex(config.API.Api.Ports, int(config.Index)) if err != nil { return err diff --git a/internal/api/prometheus_discovery.go b/internal/api/prometheus_discovery.go index bdcca4e26..c861003a0 100644 --- a/internal/api/prometheus_discovery.go +++ b/internal/api/prometheus_discovery.go @@ -6,35 +6,29 @@ import ( "net/http" "github.com/gin-gonic/gin" - conf "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/open-im-server/v3/pkg/common/prommetrics" "github.com/openimsdk/tools/apiresp" "github.com/openimsdk/tools/discovery" - "github.com/openimsdk/tools/discovery/etcd" "github.com/openimsdk/tools/errs" - clientv3 "go.etcd.io/etcd/client/v3" ) type PrometheusDiscoveryApi struct { config *Config - client *clientv3.Client kv discovery.KeyValue } -func NewPrometheusDiscoveryApi(config *Config, client discovery.Conn) *PrometheusDiscoveryApi { +func NewPrometheusDiscoveryApi(config *Config, client discovery.SvcDiscoveryRegistry) *PrometheusDiscoveryApi { api := &PrometheusDiscoveryApi{ config: config, - } - if config.Discovery.Enable == conf.ETCD { - api.client = client.(*etcd.SvcDiscoveryRegistryImpl).GetClient() + kv: client, } return api } func (p *PrometheusDiscoveryApi) discovery(c *gin.Context, key string) { - value, err := p.kv.GetKey(c, prommetrics.BuildDiscoveryKey(key)) + value, err := p.kv.GetKeyWithPrefix(c, prommetrics.BuildDiscoveryKeyPrefix(key)) if err != nil { - if errors.Is(err, discovery.ErrNotSupportedKeyValue) { + if errors.Is(err, discovery.ErrNotSupported) { c.JSON(http.StatusOK, []struct{}{}) return } @@ -46,10 +40,17 @@ func (p *PrometheusDiscoveryApi) discovery(c *gin.Context, key string) { return } var resp prommetrics.RespTarget - if err := json.Unmarshal(value, &resp); err != nil { - apiresp.GinError(c, errs.WrapMsg(err, "json unmarshal err")) - return + for i := range value { + var tmp prommetrics.Target + if err = json.Unmarshal(value[i], &tmp); err != nil { + apiresp.GinError(c, errs.WrapMsg(err, "json unmarshal err")) + return + } + + resp.Targets = append(resp.Targets, tmp.Target) + resp.Labels = tmp.Labels // default label is fixed. See prommetrics.BuildDefaultTarget } + c.JSON(http.StatusOK, []*prommetrics.RespTarget{&resp}) } diff --git a/internal/api/router.go b/internal/api/router.go index 5e5f35a60..fcad104b8 100644 --- a/internal/api/router.go +++ b/internal/api/router.go @@ -53,7 +53,7 @@ func prommetricsGin() gin.HandlerFunc { } } -func newGinRouter(ctx context.Context, client discovery.Conn, cfg *Config) (*gin.Engine, error) { +func newGinRouter(ctx context.Context, client discovery.SvcDiscoveryRegistry, cfg *Config) (*gin.Engine, error) { authConn, err := client.GetConn(ctx, cfg.Discovery.RpcService.Auth) if err != nil { return nil, err diff --git a/internal/msggateway/init.go b/internal/msggateway/init.go index 8772693cc..40a57b1da 100644 --- a/internal/msggateway/init.go +++ b/internal/msggateway/init.go @@ -39,7 +39,7 @@ type Config struct { } // Start run ws server. -func Start(ctx context.Context, conf *Config, client discovery.Conn, server grpc.ServiceRegistrar) error { +func Start(ctx context.Context, conf *Config, client discovery.SvcDiscoveryRegistry, server grpc.ServiceRegistrar) error { log.CInfo(ctx, "MSG-GATEWAY server is initializing", "runtimeEnv", runtimeenv.RuntimeEnvironment(), "rpcPorts", conf.MsgGateway.RPC.Ports, "wsPort", conf.MsgGateway.LongConnSvr.Ports, "prometheusPorts", conf.MsgGateway.Prometheus.Ports) diff --git a/internal/msgtransfer/init.go b/internal/msgtransfer/init.go index b07ec6f1d..bbec3f9a2 100644 --- a/internal/msgtransfer/init.go +++ b/internal/msgtransfer/init.go @@ -58,7 +58,7 @@ type Config struct { Index conf.Index } -func Start(ctx context.Context, config *Config, client discovery.Conn, server grpc.ServiceRegistrar) error { +func Start(ctx context.Context, config *Config, client discovery.SvcDiscoveryRegistry, server grpc.ServiceRegistrar) error { builder := mqbuild.NewBuilder(&config.KafkaConfig) log.CInfo(ctx, "MSG-TRANSFER server is initializing", "runTimeEnv", runtimeenv.RuntimeEnvironment(), "prometheusPorts", diff --git a/internal/push/push.go b/internal/push/push.go index f720a52ac..1d6f8cb30 100644 --- a/internal/push/push.go +++ b/internal/push/push.go @@ -50,7 +50,7 @@ func (p pushServer) DelUserPushToken(ctx context.Context, return &pbpush.DelUserPushTokenResp{}, nil } -func Start(ctx context.Context, config *Config, client discovery.Conn, server grpc.ServiceRegistrar) error { +func Start(ctx context.Context, config *Config, client discovery.SvcDiscoveryRegistry, server grpc.ServiceRegistrar) error { dbb := dbbuild.NewBuilder(&config.MongoConfig, &config.RedisConfig) rdb, err := dbb.Redis(ctx) if err != nil { diff --git a/internal/rpc/auth/auth.go b/internal/rpc/auth/auth.go index 2c2691d1d..5ed9cdf12 100644 --- a/internal/rpc/auth/auth.go +++ b/internal/rpc/auth/auth.go @@ -59,7 +59,7 @@ type Config struct { Discovery config.Discovery } -func Start(ctx context.Context, config *Config, client discovery.Conn, server grpc.ServiceRegistrar) error { +func Start(ctx context.Context, config *Config, client discovery.SvcDiscoveryRegistry, server grpc.ServiceRegistrar) error { dbb := dbbuild.NewBuilder(&config.MongoConfig, &config.RedisConfig) rdb, err := dbb.Redis(ctx) if err != nil { diff --git a/internal/rpc/conversation/conversation.go b/internal/rpc/conversation/conversation.go index ba9e7746b..cf5a2b9c6 100644 --- a/internal/rpc/conversation/conversation.go +++ b/internal/rpc/conversation/conversation.go @@ -69,7 +69,7 @@ type Config struct { Discovery config.Discovery } -func Start(ctx context.Context, config *Config, client discovery.Conn, server grpc.ServiceRegistrar) error { +func Start(ctx context.Context, config *Config, client discovery.SvcDiscoveryRegistry, server grpc.ServiceRegistrar) error { dbb := dbbuild.NewBuilder(&config.MongodbConfig, &config.RedisConfig) mgocli, err := dbb.Mongo(ctx) if err != nil { diff --git a/internal/rpc/group/group.go b/internal/rpc/group/group.go index 5219546b7..ce8a2c7aa 100644 --- a/internal/rpc/group/group.go +++ b/internal/rpc/group/group.go @@ -76,7 +76,7 @@ type Config struct { Discovery config.Discovery } -func Start(ctx context.Context, config *Config, client discovery.Conn, server grpc.ServiceRegistrar) error { +func Start(ctx context.Context, config *Config, client discovery.SvcDiscoveryRegistry, server grpc.ServiceRegistrar) error { dbb := dbbuild.NewBuilder(&config.MongodbConfig, &config.RedisConfig) mgocli, err := dbb.Mongo(ctx) if err != nil { diff --git a/internal/rpc/msg/server.go b/internal/rpc/msg/server.go index cfc750c5b..9d5391cc9 100644 --- a/internal/rpc/msg/server.go +++ b/internal/rpc/msg/server.go @@ -78,7 +78,7 @@ func (m *msgServer) addInterceptorHandler(interceptorFunc ...MessageInterceptorF } -func Start(ctx context.Context, config *Config, client discovery.Conn, server grpc.ServiceRegistrar) error { +func Start(ctx context.Context, config *Config, client discovery.SvcDiscoveryRegistry, server grpc.ServiceRegistrar) error { builder := mqbuild.NewBuilder(&config.KafkaConfig) redisProducer, err := builder.GetTopicProducer(ctx, config.KafkaConfig.ToRedisTopic) if err != nil { diff --git a/internal/rpc/relation/friend.go b/internal/rpc/relation/friend.go index 8c7c40536..d05cf7d77 100644 --- a/internal/rpc/relation/friend.go +++ b/internal/rpc/relation/friend.go @@ -66,7 +66,7 @@ type Config struct { Discovery config.Discovery } -func Start(ctx context.Context, config *Config, client discovery.Conn, server grpc.ServiceRegistrar) error { +func Start(ctx context.Context, config *Config, client discovery.SvcDiscoveryRegistry, server grpc.ServiceRegistrar) error { dbb := dbbuild.NewBuilder(&config.MongodbConfig, &config.RedisConfig) mgocli, err := dbb.Mongo(ctx) if err != nil { diff --git a/internal/rpc/third/third.go b/internal/rpc/third/third.go index c6dcb2ea4..cea6a8522 100644 --- a/internal/rpc/third/third.go +++ b/internal/rpc/third/third.go @@ -64,7 +64,7 @@ type Config struct { Discovery config.Discovery } -func Start(ctx context.Context, config *Config, client discovery.Conn, server grpc.ServiceRegistrar) error { +func Start(ctx context.Context, config *Config, client discovery.SvcDiscoveryRegistry, server grpc.ServiceRegistrar) error { dbb := dbbuild.NewBuilder(&config.MongodbConfig, &config.RedisConfig) mgocli, err := dbb.Mongo(ctx) if err != nil { diff --git a/internal/rpc/user/user.go b/internal/rpc/user/user.go index 5639baab9..28461ae0a 100644 --- a/internal/rpc/user/user.go +++ b/internal/rpc/user/user.go @@ -79,7 +79,7 @@ type Config struct { Discovery config.Discovery } -func Start(ctx context.Context, config *Config, client discovery.Conn, server grpc.ServiceRegistrar) error { +func Start(ctx context.Context, config *Config, client discovery.SvcDiscoveryRegistry, server grpc.ServiceRegistrar) error { dbb := dbbuild.NewBuilder(&config.MongodbConfig, &config.RedisConfig) mgocli, err := dbb.Mongo(ctx) if err != nil { diff --git a/internal/tools/cron/cron_task.go b/internal/tools/cron/cron_task.go index a4de309d4..abe596192 100644 --- a/internal/tools/cron/cron_task.go +++ b/internal/tools/cron/cron_task.go @@ -25,7 +25,7 @@ type Config struct { Discovery config.Discovery } -func Start(ctx context.Context, conf *Config, client discovery.Conn, service grpc.ServiceRegistrar) error { +func Start(ctx context.Context, conf *Config, client discovery.SvcDiscoveryRegistry, service grpc.ServiceRegistrar) error { log.CInfo(ctx, "CRON-TASK server is initializing", "runTimeEnv", runtimeenv.RuntimeEnvironment(), "chatRecordsClearTime", conf.CronTask.CronExecuteTime, "msgDestructTime", conf.CronTask.RetainChatRecords) if conf.CronTask.RetainChatRecords < 1 { log.ZInfo(ctx, "disable cron") diff --git a/pkg/common/cmd/api.go b/pkg/common/cmd/api.go index 484467798..7b7dbc89b 100644 --- a/pkg/common/cmd/api.go +++ b/pkg/common/cmd/api.go @@ -19,6 +19,7 @@ import ( "github.com/openimsdk/open-im-server/v3/internal/api" "github.com/openimsdk/open-im-server/v3/pkg/common/config" + "github.com/openimsdk/open-im-server/v3/pkg/common/prommetrics" "github.com/openimsdk/open-im-server/v3/pkg/common/startrpc" "github.com/openimsdk/open-im-server/v3/version" "github.com/openimsdk/tools/system/program" @@ -84,7 +85,7 @@ func (a *ApiCmd) runE() error { a.apiConfig.API.Api.ListenIP, "", a.apiConfig.API.Prometheus.AutoSetPorts, nil, int(a.apiConfig.Index), - a.apiConfig.Discovery.RpcService.MessageGateway, + prommetrics.APIKeyName, &a.apiConfig.Notification, a.apiConfig, []string{}, diff --git a/pkg/common/cmd/msg_transfer.go b/pkg/common/cmd/msg_transfer.go index 9411a2cd0..fe6c27e54 100644 --- a/pkg/common/cmd/msg_transfer.go +++ b/pkg/common/cmd/msg_transfer.go @@ -19,6 +19,7 @@ import ( "github.com/openimsdk/open-im-server/v3/internal/msgtransfer" "github.com/openimsdk/open-im-server/v3/pkg/common/config" + "github.com/openimsdk/open-im-server/v3/pkg/common/prommetrics" "github.com/openimsdk/open-im-server/v3/pkg/common/startrpc" "github.com/openimsdk/open-im-server/v3/version" "github.com/openimsdk/tools/system/program" @@ -65,7 +66,7 @@ func (m *MsgTransferCmd) runE() error { "", "", true, nil, int(m.msgTransferConfig.Index), - "", + prommetrics.MessageTransferKeyName, nil, m.msgTransferConfig, []string{}, diff --git a/pkg/common/prommetrics/prommetrics.go b/pkg/common/prommetrics/prommetrics.go index 153314bbb..3f683a50e 100644 --- a/pkg/common/prommetrics/prommetrics.go +++ b/pkg/common/prommetrics/prommetrics.go @@ -85,6 +85,8 @@ func Start(listener net.Listener) error { const ( APIKeyName = "api" MessageTransferKeyName = "message-transfer" + + TTL = 300 ) type Target struct { @@ -97,10 +99,14 @@ type RespTarget struct { Labels map[string]string `json:"labels"` } -func BuildDiscoveryKey(name string) string { +func BuildDiscoveryKeyPrefix(name string) string { return fmt.Sprintf("%s/%s/%s", "openim", "prometheus_discovery", name) } +func BuildDiscoveryKey(name string, index int) string { + return fmt.Sprintf("%s/%s/%s/%d", "openim", "prometheus_discovery", name, index) +} + func BuildDefaultTarget(host string, ip int) Target { return Target{ Target: fmt.Sprintf("%s:%d", host, ip), diff --git a/pkg/common/startrpc/start.go b/pkg/common/startrpc/start.go index b99d32db1..06e19d8d2 100644 --- a/pkg/common/startrpc/start.go +++ b/pkg/common/startrpc/start.go @@ -50,7 +50,7 @@ func init() { func Start[T any](ctx context.Context, disc *conf.Discovery, prometheusConfig *conf.Prometheus, listenIP, registerIP string, autoSetPorts bool, rpcPorts []int, index int, rpcRegisterName string, notification *conf.Notification, config T, watchConfigNames []string, watchServiceNames []string, - rpcFn func(ctx context.Context, config T, client discovery.Conn, server grpc.ServiceRegistrar) error, + rpcFn func(ctx context.Context, config T, client discovery.SvcDiscoveryRegistry, server grpc.ServiceRegistrar) error, options ...grpc.ServerOption) error { if notification != nil { @@ -148,9 +148,11 @@ func Start[T any](ctx context.Context, disc *conf.Discovery, prometheusConfig *c if err != nil { return err } - if err := client.SetKey(ctx, prommetrics.BuildDiscoveryKey(prommetrics.APIKeyName), target); err != nil { - if !errors.Is(err, discovery.ErrNotSupportedKeyValue) { - return err + if autoSetPorts { + if err = client.SetWithLease(ctx, prommetrics.BuildDiscoveryKey(rpcRegisterName, index), target, prommetrics.TTL); err != nil { + if !errors.Is(err, discovery.ErrNotSupported) { + return err + } } } go func() { From 545125884e8b1dbfe4de71a34ad1a5a5663bef7c Mon Sep 17 00:00:00 2001 From: icey-yu <119291641+icey-yu@users.noreply.github.com> Date: Mon, 16 Jun 2025 10:46:07 +0800 Subject: [PATCH 171/199] fix: import friends send notification (#3420) --- internal/rpc/relation/friend.go | 4 ++-- internal/rpc/relation/notification.go | 16 +++++++++++----- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/internal/rpc/relation/friend.go b/internal/rpc/relation/friend.go index d05cf7d77..43909c8e3 100644 --- a/internal/rpc/relation/friend.go +++ b/internal/rpc/relation/friend.go @@ -192,7 +192,7 @@ func (s *friendServer) ImportFriends(ctx context.Context, req *relation.ImportFr FromUserID: req.OwnerUserID, ToUserID: userID, HandleResult: constant.FriendResponseAgree, - }) + }, false) } s.webhookAfterImportFriends(ctx, &s.config.WebhooksConfig.AfterImportFriends, req) @@ -221,7 +221,7 @@ func (s *friendServer) RespondFriendApply(ctx context.Context, req *relation.Res return nil, err } s.webhookAfterAddFriendAgree(ctx, &s.config.WebhooksConfig.AfterAddFriendAgree, req) - s.notificationSender.FriendApplicationAgreedNotification(ctx, req) + s.notificationSender.FriendApplicationAgreedNotification(ctx, req, true) return resp, nil } if req.HandleResult == constant.FriendResponseRefuse { diff --git a/internal/rpc/relation/notification.go b/internal/rpc/relation/notification.go index d6a03003e..4ee45e197 100644 --- a/internal/rpc/relation/notification.go +++ b/internal/rpc/relation/notification.go @@ -171,11 +171,17 @@ func (f *FriendNotificationSender) FriendApplicationAddNotification(ctx context. f.Notification(ctx, req.FromUserID, req.ToUserID, constant.FriendApplicationNotification, &tips) } -func (f *FriendNotificationSender) FriendApplicationAgreedNotification(ctx context.Context, req *relation.RespondFriendApplyReq) { - request, err := f.getFriendRequests(ctx, req.FromUserID, req.ToUserID) - if err != nil { - log.ZError(ctx, "FriendApplicationAgreedNotification get friend request", err, "fromUserID", req.FromUserID, "toUserID", req.ToUserID) - return +func (f *FriendNotificationSender) FriendApplicationAgreedNotification(ctx context.Context, req *relation.RespondFriendApplyReq, checkReq bool) { + var ( + request *sdkws.FriendRequest + err error + ) + if checkReq { + request, err = f.getFriendRequests(ctx, req.FromUserID, req.ToUserID) + if err != nil { + log.ZError(ctx, "FriendApplicationAgreedNotification get friend request", err, "fromUserID", req.FromUserID, "toUserID", req.ToUserID) + return + } } tips := sdkws.FriendApplicationApprovedTips{ FromToUserID: &sdkws.FromToUserID{ From 1e2375faca1cb2ee3b08723f9627856db950cad4 Mon Sep 17 00:00:00 2001 From: Monet Lee Date: Tue, 17 Jun 2025 15:12:25 +0800 Subject: [PATCH 172/199] fix: improve mileston PR workflows contents. (#3382) --- .github/workflows/merge-from-milestone.yml | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/.github/workflows/merge-from-milestone.yml b/.github/workflows/merge-from-milestone.yml index 1f5762ccb..a0ec2ac16 100644 --- a/.github/workflows/merge-from-milestone.yml +++ b/.github/workflows/merge-from-milestone.yml @@ -155,11 +155,27 @@ jobs: '{title: $title, head: $head, base: $base, body: $body}')") new_pr_number=$(echo "$response" | jq -r '.number') - echo "Created PR #$new_pr_number" - curl -s -X POST -H "Authorization: token $GITHUB_TOKEN" \ + if [[ "$new_pr_number" == "null" || -z "$new_pr_number" ]]; then + echo "Failed to create PR. Response: $response" + + git checkout $TARGET_BRANCH + + git branch -D $cherry_pick_branch + + echo "Deleted branch: $cherry_pick_branch" + git push origin --delete $cherry_pick_branch + else + echo "Created PR #$new_pr_number" + + curl -s -X POST -H "Authorization: token $GITHUB_TOKEN" \ -H "Accept: application/vnd.github+json" \ -d '{"labels": ["milestone-merge"]}' \ "https://api.github.com/repos/${{ github.repository }}/issues/$new_pr_number/labels" + fi + + echo "" + echo "----------------------------------------" + echo "" fi done From 1baf9a8e0f11cc94313eec2165e273482d1bdb0d Mon Sep 17 00:00:00 2001 From: Monet Lee Date: Tue, 17 Jun 2025 15:33:25 +0800 Subject: [PATCH 173/199] feat: Implement etcd and kafka auth. (#3394) * feat: Implement etcd and kafka auth. * Update etcd command contents. * update contents. * feat: update auth logic to compatible old version. * update comment. * update contents. --- .env | 2 +- config/discovery.yml | 10 ++-- config/kafka.yml | 20 +++---- docker-compose.yml | 128 ++++++++++++++++++++++++++++++++++++++----- 4 files changed, 131 insertions(+), 29 deletions(-) diff --git a/.env b/.env index 0ab998037..2d4dfd4c7 100644 --- a/.env +++ b/.env @@ -2,7 +2,7 @@ MONGO_IMAGE=mongo:7.0 REDIS_IMAGE=redis:7.0.0 KAFKA_IMAGE=bitnami/kafka:3.5.1 MINIO_IMAGE=minio/minio:RELEASE.2024-01-11T07-46-16Z -ETCD_IMAGE=quay.io/coreos/etcd:v3.5.13 +ETCD_IMAGE=bitnami/etcd:3.5.13 PROMETHEUS_IMAGE=prom/prometheus:v2.45.6 ALERTMANAGER_IMAGE=prom/alertmanager:v0.27.0 GRAFANA_IMAGE=grafana/grafana:11.0.1 diff --git a/config/discovery.yml b/config/discovery.yml index e8d733e9f..2251dceb7 100644 --- a/config/discovery.yml +++ b/config/discovery.yml @@ -1,9 +1,11 @@ enable: etcd etcd: rootDirectory: openim - address: [ localhost:12379 ] - username: '' - password: '' + address: [localhost:12379] + ## Attention: If you set auth in etcd + ## you must also update the username and password in Chat project. + username: + password: kubernetes: namespace: default @@ -17,4 +19,4 @@ rpcService: group: group-rpc-service auth: auth-rpc-service conversation: conversation-rpc-service - third: third-rpc-service \ No newline at end of file + third: third-rpc-service diff --git a/config/kafka.yml b/config/kafka.yml index fd06ae2bb..2e9b5296c 100644 --- a/config/kafka.yml +++ b/config/kafka.yml @@ -1,13 +1,13 @@ -# Username for authentication -username: '' -# Password for authentication -password: '' +## Kafka authentication +username: +password: + # Producer acknowledgment settings -producerAck: +producerAck: # Compression type to use (e.g., none, gzip, snappy) compressType: none # List of Kafka broker addresses -address: [ localhost:19094 ] +address: [localhost:19094] # Kafka topic for Redis integration toRedisTopic: toRedis # Kafka topic for MongoDB integration @@ -29,12 +29,12 @@ tls: # Enable or disable TLS enableTLS: false # CA certificate file path - caCrt: + caCrt: # Client certificate file path - clientCrt: + clientCrt: # Client key file path - clientKey: + clientKey: # Client key password - clientKeyPwd: + clientKeyPwd: # Whether to skip TLS verification (not recommended for production) insecureSkipVerify: false diff --git a/docker-compose.yml b/docker-compose.yml index 65b4e6625..60fc865f2 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -83,8 +83,83 @@ services: - ETCD_INITIAL_CLUSTER=s1=http://0.0.0.0:2380 - ETCD_INITIAL_CLUSTER_TOKEN=tkn - ETCD_INITIAL_CLUSTER_STATE=new + - ALLOW_NONE_AUTHENTICATION=no + + ## Optional: Enable etcd authentication by setting the following credentials + # - ETCD_ROOT_USER=root + # - ETCD_ROOT_PASSWORD=openIM123 + # - ETCD_USERNAME=openIM + # - ETCD_PASSWORD=openIM123 volumes: - "${DATA_DIR}/components/etcd:/etcd-data" + command: > + /bin/sh -c ' + etcd & + export ETCDCTL_API=3 + echo "Waiting for etcd to become healthy..." + until etcdctl --endpoints=http://127.0.0.1:2379 endpoint health &>/dev/null; do + echo "Waiting for ETCD to start..." + sleep 1 + done + + echo "etcd is healthy." + + if [ -n "$${ETCD_ROOT_USER}" ] && [ -n "$${ETCD_ROOT_PASSWORD}" ] && [ -n "$${ETCD_USERNAME}" ] && [ -n "$${ETCD_PASSWORD}" ]; then + echo "Authentication credentials provided. Setting up authentication..." + + echo "Checking authentication status..." + if ! etcdctl --endpoints=http://127.0.0.1:2379 auth status | grep -q "Authentication Status: true"; then + echo "Authentication is disabled. Creating users and enabling..." + + # Create users and setup permissions + etcdctl --endpoints=http://127.0.0.1:2379 user add $${ETCD_ROOT_USER} --new-user-password=$${ETCD_ROOT_PASSWORD} || true + etcdctl --endpoints=http://127.0.0.1:2379 user add $${ETCD_USERNAME} --new-user-password=$${ETCD_PASSWORD} || true + + etcdctl --endpoints=http://127.0.0.1:2379 role add openim-role || true + etcdctl --endpoints=http://127.0.0.1:2379 role grant-permission openim-role --prefix=true readwrite / || true + etcdctl --endpoints=http://127.0.0.1:2379 role grant-permission openim-role --prefix=true readwrite "" || true + etcdctl --endpoints=http://127.0.0.1:2379 user grant-role $${ETCD_USERNAME} openim-role || true + + etcdctl --endpoints=http://127.0.0.1:2379 user grant-role $${ETCD_ROOT_USER} $${ETCD_USERNAME} root || true + + echo "Enabling authentication..." + etcdctl --endpoints=http://127.0.0.1:2379 auth enable + echo "Authentication enabled successfully" + else + echo "Authentication is already enabled. Checking OpenIM user..." + + # Check if openIM user exists and can perform operations + if ! etcdctl --endpoints=http://127.0.0.1:2379 --user=$${ETCD_USERNAME}:$${ETCD_PASSWORD} put /test/auth "auth-check" &>/dev/null; then + echo "OpenIM user test failed. Recreating user with root credentials..." + + # Try to create/update the openIM user using root credentials + etcdctl --endpoints=http://127.0.0.1:2379 --user=$${ETCD_ROOT_USER}:$${ETCD_ROOT_PASSWORD} user add $${ETCD_USERNAME} --new-user-password=$${ETCD_PASSWORD} --no-password-file || true + etcdctl --endpoints=http://127.0.0.1:2379 --user=$${ETCD_ROOT_USER}:$${ETCD_ROOT_PASSWORD} role add openim-role || true + etcdctl --endpoints=http://127.0.0.1:2379 --user=$${ETCD_ROOT_USER}:$${ETCD_ROOT_PASSWORD} role grant-permission openim-role --prefix=true readwrite / || true + etcdctl --endpoints=http://127.0.0.1:2379 --user=$${ETCD_ROOT_USER}:$${ETCD_ROOT_PASSWORD} role grant-permission openim-role --prefix=true readwrite "" || true + etcdctl --endpoints=http://127.0.0.1:2379 --user=$${ETCD_ROOT_USER}:$${ETCD_ROOT_PASSWORD} user grant-role $${ETCD_USERNAME} openim-role || true + etcdctl --endpoints=http://127.0.0.1:2379 user grant-role $${ETCD_ROOT_USER} $${ETCD_USERNAME} root || true + + echo "OpenIM user recreated with required permissions" + else + echo "OpenIM user exists and has correct permissions" + etcdctl --endpoints=http://127.0.0.1:2379 --user=$${ETCD_USERNAME}:$${ETCD_PASSWORD} del /test/auth &>/dev/null + fi + fi + echo "Testing authentication with OpenIM user..." + if etcdctl --endpoints=http://127.0.0.1:2379 --user=$${ETCD_USERNAME}:$${ETCD_PASSWORD} put /test/auth "auth-works"; then + echo "Authentication working properly" + etcdctl --endpoints=http://127.0.0.1:2379 --user=$${ETCD_USERNAME}:$${ETCD_PASSWORD} del /test/auth + else + echo "WARNING: Authentication test failed" + fi + else + echo "No authentication credentials provided. Running in no-auth mode." + echo "To enable authentication, set ETCD_ROOT_USER, ETCD_ROOT_PASSWORD, ETCD_USERNAME, and ETCD_PASSWORD environment variables." + fi + + tail -f /dev/null + ' restart: always networks: - openim @@ -104,12 +179,38 @@ services: KAFKA_CFG_NODE_ID: 0 KAFKA_CFG_PROCESS_ROLES: controller,broker KAFKA_CFG_CONTROLLER_QUORUM_VOTERS: 0@kafka:9093 - KAFKA_CFG_LISTENERS: PLAINTEXT://:9092,CONTROLLER://:9093,EXTERNAL://:9094 - KAFKA_CFG_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092,EXTERNAL://localhost:19094 - KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,EXTERNAL:PLAINTEXT,PLAINTEXT:PLAINTEXT KAFKA_CFG_CONTROLLER_LISTENER_NAMES: CONTROLLER KAFKA_NUM_PARTITIONS: 8 KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE: "true" + + KAFKA_CFG_LISTENERS: "PLAINTEXT://:9092,CONTROLLER://:9093,EXTERNAL://:9094" + KAFKA_CFG_ADVERTISED_LISTENERS: "PLAINTEXT://kafka:9092,EXTERNAL://localhost:19094" + KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: "CONTROLLER:PLAINTEXT,EXTERNAL:PLAINTEXT,PLAINTEXT:PLAINTEXT" + KAFKA_CFG_INTER_BROKER_LISTENER_NAME: "PLAINTEXT" + + # Authentication configuration variables - comment out to disable auth + # KAFKA_USERNAME: "openIM" + # KAFKA_PASSWORD: "openIM123" + command: > + /bin/sh -c ' + if [ -n "$${KAFKA_USERNAME}" ] && [ -n "$${KAFKA_PASSWORD}" ]; then + echo "=== Kafka SASL Authentication ENABLED ===" + echo "Username: $${KAFKA_USERNAME}" + + # Set environment variables for SASL authentication + export KAFKA_CFG_LISTENERS="SASL_PLAINTEXT://:9092,CONTROLLER://:9093,EXTERNAL://:9094" + export KAFKA_CFG_ADVERTISED_LISTENERS="SASL_PLAINTEXT://kafka:9092,EXTERNAL://localhost:19094" + export KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP="CONTROLLER:PLAINTEXT,EXTERNAL:SASL_PLAINTEXT,SASL_PLAINTEXT:SASL_PLAINTEXT" + export KAFKA_CFG_SASL_ENABLED_MECHANISMS="PLAIN" + export KAFKA_CFG_SASL_MECHANISM_INTER_BROKER_PROTOCOL="PLAIN" + export KAFKA_CFG_INTER_BROKER_LISTENER_NAME="SASL_PLAINTEXT" + export KAFKA_CLIENT_USERS="$${KAFKA_USERNAME}" + export KAFKA_CLIENT_PASSWORDS="$${KAFKA_PASSWORD}" + fi + + # Start Kafka with the configured environment + exec /opt/bitnami/scripts/kafka/entrypoint.sh /opt/bitnami/scripts/kafka/run.sh + ' networks: - openim @@ -148,7 +249,7 @@ services: - "11002:80" networks: - openim - + prometheus: image: ${PROMETHEUS_IMAGE} container_name: prometheus @@ -161,9 +262,9 @@ services: - ./config/instance-down-rules.yml:/etc/prometheus/instance-down-rules.yml - ${DATA_DIR}/components/prometheus/data:/prometheus command: - - '--config.file=/etc/prometheus/prometheus.yml' - - '--storage.tsdb.path=/prometheus' - - '--web.listen-address=:${PROMETHEUS_PORT}' + - "--config.file=/etc/prometheus/prometheus.yml" + - "--storage.tsdb.path=/prometheus" + - "--web.listen-address=:${PROMETHEUS_PORT}" network_mode: host alertmanager: @@ -176,8 +277,8 @@ services: - ./config/alertmanager.yml:/etc/alertmanager/alertmanager.yml - ./config/email.tmpl:/etc/alertmanager/email.tmpl command: - - '--config.file=/etc/alertmanager/alertmanager.yml' - - '--web.listen-address=:${ALERTMANAGER_PORT}' + - "--config.file=/etc/alertmanager/alertmanager.yml" + - "--web.listen-address=:${ALERTMANAGER_PORT}" network_mode: host grafana: @@ -209,9 +310,8 @@ services: - /sys:/host/sys:ro - /:/rootfs:ro command: - - '--path.procfs=/host/proc' - - '--path.sysfs=/host/sys' - - '--path.rootfs=/rootfs' - - '--web.listen-address=:19100' + - "--path.procfs=/host/proc" + - "--path.sysfs=/host/sys" + - "--path.rootfs=/rootfs" + - "--web.listen-address=:19100" network_mode: host - From 8f7b02979d3ffb7530005e5352bce53f6101fe18 Mon Sep 17 00:00:00 2001 From: Monet Lee Date: Wed, 18 Jun 2025 14:31:09 +0800 Subject: [PATCH 174/199] feat: support redis sentinel. (#3423) * feat: support redis sentinel. * update docker compose contents. * update config contents. * revert content. * supoort redisMode. * update config. * remvove print. --- config/redis.yml | 13 ++++-- go.mod | 2 +- go.sum | 4 +- pkg/common/config/config.go | 44 ++++++++++++------- pkg/common/discovery/discoveryregister.go | 2 +- pkg/common/storage/cache/redis/online_test.go | 2 +- tools/seq/internal/seq.go | 12 +++-- 7 files changed, 52 insertions(+), 27 deletions(-) diff --git a/config/redis.yml b/config/redis.yml index 2448bcb5c..5e62719ae 100644 --- a/config/redis.yml +++ b/config/redis.yml @@ -1,7 +1,14 @@ -address: [ localhost:16379 ] -username: +address: [localhost:16379] +username: password: openIM123 -clusterMode: false +# redis Mode, including "standalone","cluster","sentinel" +redisMode: "standalone" db: 0 maxRetry: 10 poolSize: 100 +# Sentinel configuration (only used when redisMode is "sentinel") +sentinelMode: + masterName: "redis-master" + sentinelsAddrs: ["127.0.0.1:26379", "127.0.0.1:26380", "127.0.0.1:26381"] + routeByLatency: true + routeRandomly: true diff --git a/go.mod b/go.mod index 845f75bb2..0ecd3a113 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/mitchellh/mapstructure v1.5.0 github.com/openimsdk/protocol v0.0.73-alpha.12 - github.com/openimsdk/tools v0.0.50-alpha.85 + github.com/openimsdk/tools v0.0.50-alpha.91 github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 github.com/stretchr/testify v1.9.0 diff --git a/go.sum b/go.sum index b775a1056..d8eb07f80 100644 --- a/go.sum +++ b/go.sum @@ -349,8 +349,8 @@ github.com/openimsdk/gomake v0.0.15-alpha.5 h1:eEZCEHm+NsmcO3onXZPIUbGFCYPYbsX5b github.com/openimsdk/gomake v0.0.15-alpha.5/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= github.com/openimsdk/protocol v0.0.73-alpha.12 h1:2NYawXeHChYUeSme6QJ9pOLh+Empce2WmwEtbP4JvKk= github.com/openimsdk/protocol v0.0.73-alpha.12/go.mod h1:WF7EuE55vQvpyUAzDXcqg+B+446xQyEba0X35lTINmw= -github.com/openimsdk/tools v0.0.50-alpha.85 h1:OqTUYx6r7Zp/eH8FKB08XeNjPV405TUIG9QT6QQ+F+s= -github.com/openimsdk/tools v0.0.50-alpha.85/go.mod h1:n2poR3asX1e1XZce4O+MOWAp+X02QJRFvhcLCXZdzRo= +github.com/openimsdk/tools v0.0.50-alpha.91 h1:4zXtTwwCIUawet1VDvnD3C/1E4N4ostDfh+RfL5nz90= +github.com/openimsdk/tools v0.0.50-alpha.91/go.mod h1:n2poR3asX1e1XZce4O+MOWAp+X02QJRFvhcLCXZdzRo= github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= diff --git a/pkg/common/config/config.go b/pkg/common/config/config.go index cd57a11cf..619571064 100644 --- a/pkg/common/config/config.go +++ b/pkg/common/config/config.go @@ -323,14 +323,22 @@ type RPC struct { } type Redis struct { - Disable bool `yaml:"-"` - Address []string `yaml:"address"` - Username string `yaml:"username"` - Password string `yaml:"password"` - ClusterMode bool `yaml:"clusterMode"` - DB int `yaml:"db"` - MaxRetry int `yaml:"maxRetry"` - PoolSize int `yaml:"poolSize"` + Disable bool `yaml:"-"` + Address []string `yaml:"address"` + Username string `yaml:"username"` + Password string `yaml:"password"` + RedisMode string `yaml:"redisMode"` + DB int `yaml:"db"` + MaxRetry int `yaml:"maxRetry"` + PoolSize int `yaml:"poolSize"` + SentinelMode Sentinel `yaml:"sentinelMode"` +} + +type Sentinel struct { + MasterName string `yaml:"masterName"` + SentinelAddrs []string `yaml:"sentinelsAddrs"` + RouteByLatency bool `yaml:"routeByLatency"` + RouteRandomly bool `yaml:"routeRandomly"` } type BeforeConfig struct { @@ -487,13 +495,19 @@ func (m *Mongo) Build() *mongoutil.Config { func (r *Redis) Build() *redisutil.Config { return &redisutil.Config{ - ClusterMode: r.ClusterMode, - Address: r.Address, - Username: r.Username, - Password: r.Password, - DB: r.DB, - MaxRetry: r.MaxRetry, - PoolSize: r.PoolSize, + RedisMode: r.RedisMode, + Address: r.Address, + Username: r.Username, + Password: r.Password, + DB: r.DB, + MaxRetry: r.MaxRetry, + PoolSize: r.PoolSize, + Sentinel: &redisutil.Sentinel{ + MasterName: r.SentinelMode.MasterName, + SentinelAddrs: r.SentinelMode.SentinelAddrs, + RouteByLatency: r.SentinelMode.RouteByLatency, + RouteRandomly: r.SentinelMode.RouteRandomly, + }, } } diff --git a/pkg/common/discovery/discoveryregister.go b/pkg/common/discovery/discoveryregister.go index dc100be5c..87333fcac 100644 --- a/pkg/common/discovery/discoveryregister.go +++ b/pkg/common/discovery/discoveryregister.go @@ -35,7 +35,7 @@ func NewDiscoveryRegister(discovery *config.Discovery, watchNames []string) (dis return standalone.GetSvcDiscoveryRegistry(), nil } if runtimeenv.RuntimeEnvironment() == config.KUBERNETES { - return kubernetes.NewKubernetesConnManager(discovery.Kubernetes.Namespace, + return kubernetes.NewConnManager(discovery.Kubernetes.Namespace, nil, grpc.WithDefaultCallOptions( grpc.MaxCallSendMsgSize(1024*1024*20), ), diff --git a/pkg/common/storage/cache/redis/online_test.go b/pkg/common/storage/cache/redis/online_test.go index 0306f6f5d..d2ac283e0 100644 --- a/pkg/common/storage/cache/redis/online_test.go +++ b/pkg/common/storage/cache/redis/online_test.go @@ -26,7 +26,7 @@ func TestName111111(t *testing.T) { "172.16.8.124:7005", "172.16.8.124:7006", }, - ClusterMode: true, + RedisMode: "cluster", Password: "passwd123", //Address: []string{"localhost:16379"}, //Password: "openIM123", diff --git a/tools/seq/internal/seq.go b/tools/seq/internal/seq.go index 9fd352a96..e90c4a41b 100644 --- a/tools/seq/internal/seq.go +++ b/tools/seq/internal/seq.go @@ -28,6 +28,8 @@ import ( "go.mongodb.org/mongo-driver/mongo/options" ) +const StructTagName = "yaml" + const ( MaxSeq = "MAX_SEQ:" MinSeq = "MIN_SEQ:" @@ -54,13 +56,14 @@ func readConfig[T any](dir string, name string) (*T, error) { if err := v.ReadInConfig(); err != nil { return nil, err } - fn := func(config *mapstructure.DecoderConfig) { - config.TagName = "mapstructure" - } + var conf T - if err := v.Unmarshal(&conf, fn); err != nil { + if err := v.Unmarshal(&conf, func(config *mapstructure.DecoderConfig) { + config.TagName = StructTagName + }); err != nil { return nil, err } + return &conf, nil } @@ -69,6 +72,7 @@ func Main(conf string, del time.Duration) error { if err != nil { return err } + mongodbConfig, err := readConfig[config.Mongo](conf, config.MongodbConfigFileName) if err != nil { return err From 53bf8acc213420eca8dcbf94f4978afefcb71153 Mon Sep 17 00:00:00 2001 From: Monet Lee Date: Wed, 18 Jun 2025 14:32:29 +0800 Subject: [PATCH 175/199] fix: solve webhook incorrect attentionID references. (#3411) --- config/webhooks.yml | 4 +- internal/msgtransfer/callback.go | 10 +++- internal/rpc/msg/callback.go | 81 ++++++++++++++++---------------- 3 files changed, 51 insertions(+), 44 deletions(-) diff --git a/config/webhooks.yml b/config/webhooks.yml index 283a23ed4..9fd3eb339 100644 --- a/config/webhooks.yml +++ b/config/webhooks.yml @@ -16,7 +16,7 @@ afterUpdateUserInfoEx: afterSendSingleMsg: enable: false timeout: 5 - # Only the recvID specified in attentionIds will send the callback + # Only the recvIDs specified in attentionIds will send the callback # if not set, all user messages will be callback attentionIds: [] # See beforeSendSingleMsg comment. @@ -36,7 +36,7 @@ beforeMsgModify: afterSendGroupMsg: enable: false timeout: 5 - # Only the recvID specified in attentionIds will send the callback + # Only the GroupIDs specified in attentionIds will send the callback # if not set, all user messages will be callback attentionIds: [] # See beforeSendSingleMsg comment. diff --git a/internal/msgtransfer/callback.go b/internal/msgtransfer/callback.go index ea51c2839..f0d439779 100644 --- a/internal/msgtransfer/callback.go +++ b/internal/msgtransfer/callback.go @@ -55,9 +55,11 @@ func (mc *OnlineHistoryMongoConsumerHandler) webhookAfterSendSingleMsg(ctx conte if msg.ContentType == constant.Typing { return } + if !filterAfterMsg(msg, after) { return } + cbReq := &cbapi.CallbackAfterSendSingleMsgReq{ CommonCallbackReq: toCommonCallback(ctx, msg, cbapi.CallbackAfterSendSingleMsgCommand), RecvID: msg.RecvID, @@ -69,9 +71,11 @@ func (mc *OnlineHistoryMongoConsumerHandler) webhookAfterSendGroupMsg(ctx contex if msg.ContentType == constant.Typing { return } + if !filterAfterMsg(msg, after) { return } + cbReq := &cbapi.CallbackAfterSendGroupMsgReq{ CommonCallbackReq: toCommonCallback(ctx, msg, cbapi.CallbackAfterSendGroupMsgCommand), GroupID: msg.GroupID, @@ -98,7 +102,11 @@ func filterAfterMsg(msg *sdkws.MsgData, after *config.AfterConfig) bool { func filterMsg(msg *sdkws.MsgData, attentionIds []string, deniedTypes []int32) bool { // According to the attentionIds configuration, only some users are sent - if len(attentionIds) != 0 && !datautil.Contain(msg.RecvID, attentionIds...) { + if len(attentionIds) != 0 && msg.ContentType == constant.SingleChatType && !datautil.Contain(msg.RecvID, attentionIds...) { + return false + } + + if len(attentionIds) != 0 && msg.ContentType == constant.ReadGroupChatType && !datautil.Contain(msg.GroupID, attentionIds...) { return false } diff --git a/internal/rpc/msg/callback.go b/internal/rpc/msg/callback.go index 5bc98de0c..2c00efd43 100644 --- a/internal/rpc/msg/callback.go +++ b/internal/rpc/msg/callback.go @@ -16,13 +16,10 @@ package msg import ( "context" - "encoding/base64" "encoding/json" - "github.com/openimsdk/open-im-server/v3/pkg/apistruct" "github.com/openimsdk/open-im-server/v3/pkg/common/webhook" "github.com/openimsdk/tools/errs" - "github.com/openimsdk/tools/utils/stringutil" cbapi "github.com/openimsdk/open-im-server/v3/pkg/callbackstruct" "github.com/openimsdk/open-im-server/v3/pkg/common/config" @@ -89,19 +86,20 @@ func (m *msgServer) webhookBeforeSendSingleMsg(ctx context.Context, before *conf }) } -func (m *msgServer) webhookAfterSendSingleMsg(ctx context.Context, after *config.AfterConfig, msg *pbchat.SendMsgReq) { - if msg.MsgData.ContentType == constant.Typing { - return - } - if !filterAfterMsg(msg, after) { - return - } - cbReq := &cbapi.CallbackAfterSendSingleMsgReq{ - CommonCallbackReq: toCommonCallback(ctx, msg, cbapi.CallbackAfterSendSingleMsgCommand), - RecvID: msg.MsgData.RecvID, - } - m.webhookClient.AsyncPostWithQuery(ctx, cbReq.GetCallbackCommand(), cbReq, &cbapi.CallbackAfterSendSingleMsgResp{}, after, buildKeyMsgDataQuery(msg.MsgData)) -} +// Move to msgtransfer +// func (m *msgServer) webhookAfterSendSingleMsg(ctx context.Context, after *config.AfterConfig, msg *pbchat.SendMsgReq) { +// if msg.MsgData.ContentType == constant.Typing { +// return +// } +// if !filterAfterMsg(msg, after) { +// return +// } +// cbReq := &cbapi.CallbackAfterSendSingleMsgReq{ +// CommonCallbackReq: toCommonCallback(ctx, msg, cbapi.CallbackAfterSendSingleMsgCommand), +// RecvID: msg.MsgData.RecvID, +// } +// m.webhookClient.AsyncPostWithQuery(ctx, cbReq.GetCallbackCommand(), cbReq, &cbapi.CallbackAfterSendSingleMsgResp{}, after, buildKeyMsgDataQuery(msg.MsgData)) +// } func (m *msgServer) webhookBeforeSendGroupMsg(ctx context.Context, before *config.BeforeConfig, msg *pbchat.SendMsgReq) error { return webhook.WithCondition(ctx, before, func(ctx context.Context) error { @@ -123,20 +121,21 @@ func (m *msgServer) webhookBeforeSendGroupMsg(ctx context.Context, before *confi }) } -func (m *msgServer) webhookAfterSendGroupMsg(ctx context.Context, after *config.AfterConfig, msg *pbchat.SendMsgReq) { - if msg.MsgData.ContentType == constant.Typing { - return - } - if !filterAfterMsg(msg, after) { - return - } - cbReq := &cbapi.CallbackAfterSendGroupMsgReq{ - CommonCallbackReq: toCommonCallback(ctx, msg, cbapi.CallbackAfterSendGroupMsgCommand), - GroupID: msg.MsgData.GroupID, - } - - m.webhookClient.AsyncPostWithQuery(ctx, cbReq.GetCallbackCommand(), cbReq, &cbapi.CallbackAfterSendGroupMsgResp{}, after, buildKeyMsgDataQuery(msg.MsgData)) -} +// Move to msgtransfer +// func (m *msgServer) webhookAfterSendGroupMsg(ctx context.Context, after *config.AfterConfig, msg *pbchat.SendMsgReq) { +// if msg.MsgData.ContentType == constant.Typing { +// return +// } +// if !filterAfterMsg(msg, after) { +// return +// } +// cbReq := &cbapi.CallbackAfterSendGroupMsgReq{ +// CommonCallbackReq: toCommonCallback(ctx, msg, cbapi.CallbackAfterSendGroupMsgCommand), +// GroupID: msg.MsgData.GroupID, +// } + +// m.webhookClient.AsyncPostWithQuery(ctx, cbReq.GetCallbackCommand(), cbReq, &cbapi.CallbackAfterSendGroupMsgResp{}, after, buildKeyMsgDataQuery(msg.MsgData)) +// } func (m *msgServer) webhookBeforeMsgModify(ctx context.Context, before *config.BeforeConfig, msg *pbchat.SendMsgReq, beforeMsgData **sdkws.MsgData) error { return webhook.WithCondition(ctx, before, func(ctx context.Context) error { @@ -205,14 +204,14 @@ func (m *msgServer) webhookAfterRevokeMsg(ctx context.Context, after *config.Aft m.webhookClient.AsyncPost(ctx, callbackReq.GetCallbackCommand(), callbackReq, &cbapi.CallbackAfterRevokeMsgResp{}, after) } -func buildKeyMsgDataQuery(msg *sdkws.MsgData) map[string]string { - keyMsgData := apistruct.KeyMsgData{ - SendID: msg.SendID, - RecvID: msg.RecvID, - GroupID: msg.GroupID, - } - - return map[string]string{ - webhook.Key: base64.StdEncoding.EncodeToString(stringutil.StructToJsonBytes(keyMsgData)), - } -} +// func buildKeyMsgDataQuery(msg *sdkws.MsgData) map[string]string { +// keyMsgData := apistruct.KeyMsgData{ +// SendID: msg.SendID, +// RecvID: msg.RecvID, +// GroupID: msg.GroupID, +// } + +// return map[string]string{ +// webhook.Key: base64.StdEncoding.EncodeToString(stringutil.StructToJsonBytes(keyMsgData)), +// } +// } From 6912ad6f332e42b4b729783692e5971fdf00e7da Mon Sep 17 00:00:00 2001 From: chao <48119764+withchao@users.noreply.github.com> Date: Thu, 19 Jun 2025 11:51:35 +0800 Subject: [PATCH 176/199] feat: add api logger (#3427) * pb * fix: Modifying other fields while setting IsPrivateChat does not take effect * fix: quote message error revoke * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * refactoring scheduled tasks * upgrading pkg tools * fix * fix * optimize log output * feat: support GetLastMessage * feat: support GetLastMessage * feat: s3 switch * feat: s3 switch * fix: GetUsersOnline * feat: SendBusinessNotification supported configuration parameters * feat: SendBusinessNotification supported configuration parameters * feat: SendBusinessNotification supported configuration parameters * feat: seq conversion failed without exiting * fix: DeleteDoc crash * fix: fill send time * fix: fill send time * fix: crash caused by withdrawing messages from users who have left the group * fix: user msg timestamp * seq read config * seq read config * fix: the source message of the reference is withdrawn, and the referenced message is deleted * feat: optimize the default notification.yml * fix: shouldPushOffline * fix: the sorting is wrong after canceling the administrator in group settings * feat: Sending messages supports returning fields modified by webhook * feat: Sending messages supports returning fields modified by webhook * feat: Sending messages supports returning fields modified by webhook * fix: oss specifies content-type when uploading * fix: the version number contains a line break * fix: the version number contains a line break * feat: GetConversationsHasReadAndMaxSeq support pinned * feat: GetConversationsHasReadAndMaxSeq support pinned * feat: GetConversationsHasReadAndMaxSeq support pinned * fix: transferring the group owner to a muted member, incremental version error * feat: unified conversion code * feat: update gomake * fix: in standalone mode, the user online status is wrong * fix: add permission check * fix: add permission check * fix: add rpc interface permission check * fix: CreateGroupChatConversations * feat: optimize friend and group applications * feat: optimize friend and group applications * feat: optimize friend and group applications * feat: optimize friend and group applications * fix: optimize friend and group applications * feat: add api logger --- go.mod | 2 +- go.sum | 4 ++-- internal/api/router.go | 3 ++- internal/tools/cron/cron_task.go | 22 ++++++++++++++++++---- 4 files changed, 23 insertions(+), 8 deletions(-) diff --git a/go.mod b/go.mod index 0ecd3a113..2a05b14b6 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/mitchellh/mapstructure v1.5.0 github.com/openimsdk/protocol v0.0.73-alpha.12 - github.com/openimsdk/tools v0.0.50-alpha.91 + github.com/openimsdk/tools v0.0.50-alpha.92 github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 github.com/stretchr/testify v1.9.0 diff --git a/go.sum b/go.sum index d8eb07f80..db68ca960 100644 --- a/go.sum +++ b/go.sum @@ -349,8 +349,8 @@ github.com/openimsdk/gomake v0.0.15-alpha.5 h1:eEZCEHm+NsmcO3onXZPIUbGFCYPYbsX5b github.com/openimsdk/gomake v0.0.15-alpha.5/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= github.com/openimsdk/protocol v0.0.73-alpha.12 h1:2NYawXeHChYUeSme6QJ9pOLh+Empce2WmwEtbP4JvKk= github.com/openimsdk/protocol v0.0.73-alpha.12/go.mod h1:WF7EuE55vQvpyUAzDXcqg+B+446xQyEba0X35lTINmw= -github.com/openimsdk/tools v0.0.50-alpha.91 h1:4zXtTwwCIUawet1VDvnD3C/1E4N4ostDfh+RfL5nz90= -github.com/openimsdk/tools v0.0.50-alpha.91/go.mod h1:n2poR3asX1e1XZce4O+MOWAp+X02QJRFvhcLCXZdzRo= +github.com/openimsdk/tools v0.0.50-alpha.92 h1:hWfykMhmi7EQEiwgQccJqbgggIuhun/PrVkBnjmj9Ec= +github.com/openimsdk/tools v0.0.50-alpha.92/go.mod h1:n2poR3asX1e1XZce4O+MOWAp+X02QJRFvhcLCXZdzRo= github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= diff --git a/internal/api/router.go b/internal/api/router.go index fcad104b8..43456f6d6 100644 --- a/internal/api/router.go +++ b/internal/api/router.go @@ -28,6 +28,7 @@ import ( "github.com/openimsdk/tools/discovery/etcd" "github.com/openimsdk/tools/log" "github.com/openimsdk/tools/mw" + "github.com/openimsdk/tools/mw/api" clientv3 "go.etcd.io/etcd/client/v3" ) @@ -96,7 +97,7 @@ func newGinRouter(ctx context.Context, client discovery.SvcDiscoveryRegistry, cf case BestSpeed: r.Use(gzip.Gzip(gzip.BestSpeed)) } - r.Use(prommetricsGin(), gin.RecoveryWithWriter(gin.DefaultErrorWriter, mw.GinPanicErr), mw.CorsHandler(), + r.Use(api.GinLogger(), prommetricsGin(), gin.RecoveryWithWriter(gin.DefaultErrorWriter, mw.GinPanicErr), mw.CorsHandler(), mw.GinParseOperationID(), GinParseToken(rpcli.NewAuthClient(authConn)), setGinIsAdmin(cfg.Share.IMAdminUserID)) u := NewUserApi(user.NewUserClient(userConn), client, cfg.Discovery.RpcService) diff --git a/internal/tools/cron/cron_task.go b/internal/tools/cron/cron_task.go index abe596192..a6740ab6e 100644 --- a/internal/tools/cron/cron_task.go +++ b/internal/tools/cron/cron_task.go @@ -49,6 +49,7 @@ func Start(ctx context.Context, conf *Config, client discovery.SvcDiscoveryRegis return err } + var locker Locker if conf.Discovery.Enable == config.ETCD { cm := disetcd.NewConfigManager(client.(*etcd.SvcDiscoveryRegistryImpl).GetClient(), []string{ conf.CronTask.GetConfigFileName(), @@ -56,11 +57,14 @@ func Start(ctx context.Context, conf *Config, client discovery.SvcDiscoveryRegis conf.Discovery.GetConfigFileName(), }) cm.Watch(ctx) + locker, err = NewEtcdLocker(client.(*etcd.SvcDiscoveryRegistryImpl).GetClient()) + if err != nil { + return err + } } - locker, err := NewEtcdLocker(client.(*etcd.SvcDiscoveryRegistryImpl).GetClient()) - if err != nil { - return err + if locker == nil { + locker = emptyLocker{} } srv := &cronServer{ @@ -92,6 +96,16 @@ func Start(ctx context.Context, conf *Config, client discovery.SvcDiscoveryRegis return nil } +type Locker interface { + ExecuteWithLock(ctx context.Context, taskName string, task func()) +} + +type emptyLocker struct{} + +func (emptyLocker) ExecuteWithLock(ctx context.Context, taskName string, task func()) { + task() +} + type cronServer struct { ctx context.Context config *Config @@ -99,7 +113,7 @@ type cronServer struct { msgClient msg.MsgClient conversationClient pbconversation.ConversationClient thirdClient third.ThirdClient - locker *EtcdLocker + locker Locker } func (c *cronServer) registerClearS3() error { From a5ac7f2a8117ec6b283c74a9d6a4db578a2bdd37 Mon Sep 17 00:00:00 2001 From: icey-yu <119291641+icey-yu@users.noreply.github.com> Date: Wed, 25 Jun 2025 11:33:19 +0800 Subject: [PATCH 177/199] feat: add nickname for adminUser (#3435) * feat: add nickname for adminUser * feat: add nickname for adminUser * feat: add nickname for adminUser --- config/share.yml | 9 ++++++++- internal/api/router.go | 6 +++--- internal/msggateway/ws_server.go | 2 +- internal/push/push_handler.go | 4 ++-- internal/rpc/auth/auth.go | 12 +++++++----- internal/rpc/group/group.go | 4 +++- internal/rpc/msg/revoke.go | 4 ++-- internal/rpc/msg/server.go | 4 +++- internal/rpc/msg/verify.go | 4 ++-- internal/rpc/user/user.go | 13 ++++++++++--- internal/tools/cron/cron_task.go | 3 +-- pkg/common/config/config.go | 7 +++++-- pkg/common/startrpc/start.go | 4 ++-- pkg/common/storage/controller/user.go | 24 +++++++++++++++++++++--- test/stress-test-v2/main.go | 2 +- test/stress-test/main.go | 2 +- 16 files changed, 72 insertions(+), 32 deletions(-) diff --git a/config/share.yml b/config/share.yml index 2e9821436..a42bdcdd7 100644 --- a/config/share.yml +++ b/config/share.yml @@ -1,6 +1,13 @@ secret: openIM123 -imAdminUserID: [imAdmin] +# imAdminUser: Configuration for instant messaging system administrators +imAdminUser: + # userIDs: List of administrator user IDs. + # Each entry here corresponds by index to the matching entry in the nicknames list below. + userIDs: [imAdmin] + # nicknames: List of administrator display names. + # Each entry here corresponds by index to the matching entry in the userIDs list above. + nicknames: [superAdmin] # 1: For Android, iOS, Windows, Mac, and web platforms, only one instance can be online at a time multiLogin: diff --git a/internal/api/router.go b/internal/api/router.go index 43456f6d6..8a4199581 100644 --- a/internal/api/router.go +++ b/internal/api/router.go @@ -98,7 +98,7 @@ func newGinRouter(ctx context.Context, client discovery.SvcDiscoveryRegistry, cf r.Use(gzip.Gzip(gzip.BestSpeed)) } r.Use(api.GinLogger(), prommetricsGin(), gin.RecoveryWithWriter(gin.DefaultErrorWriter, mw.GinPanicErr), mw.CorsHandler(), - mw.GinParseOperationID(), GinParseToken(rpcli.NewAuthClient(authConn)), setGinIsAdmin(cfg.Share.IMAdminUserID)) + mw.GinParseOperationID(), GinParseToken(rpcli.NewAuthClient(authConn)), setGinIsAdmin(cfg.Share.IMAdminUser.UserIDs)) u := NewUserApi(user.NewUserClient(userConn), client, cfg.Discovery.RpcService) { @@ -232,7 +232,7 @@ func newGinRouter(ctx context.Context, client discovery.SvcDiscoveryRegistry, cf objectGroup.GET("/*name", t.ObjectRedirect) } // Message - m := NewMessageApi(msg.NewMsgClient(msgConn), rpcli.NewUserClient(userConn), cfg.Share.IMAdminUserID) + m := NewMessageApi(msg.NewMsgClient(msgConn), rpcli.NewUserClient(userConn), cfg.Share.IMAdminUser.UserIDs) { msgGroup := r.Group("/msg") msgGroup.POST("/newest_seq", m.GetSeq) @@ -309,7 +309,7 @@ func newGinRouter(ctx context.Context, client discovery.SvcDiscoveryRegistry, cf if cfg.Discovery.Enable == config.ETCD { etcdClient = client.(*etcd.SvcDiscoveryRegistryImpl).GetClient() } - cm := NewConfigManager(cfg.Share.IMAdminUserID, &cfg.AllConfig, etcdClient, string(cfg.ConfigPath)) + cm := NewConfigManager(cfg.Share.IMAdminUser.UserIDs, &cfg.AllConfig, etcdClient, string(cfg.ConfigPath)) { configGroup := r.Group("/config", cm.CheckAdmin) configGroup.POST("/get_config_list", cm.GetConfigList) diff --git a/internal/msggateway/ws_server.go b/internal/msggateway/ws_server.go index 211482cb1..9b7343938 100644 --- a/internal/msggateway/ws_server.go +++ b/internal/msggateway/ws_server.go @@ -130,7 +130,7 @@ func NewWsServer(msgGatewayConfig *Config, opts ...Option) *WsServer { for _, o := range opts { o(&config) } - //userRpcClient := rpcclient.NewUserRpcClient(client, config.Discovery.RpcService.User, config.Share.IMAdminUserID) + //userRpcClient := rpcclient.NewUserRpcClient(client, config.Discovery.RpcService.User, config.Share.IMAdminUser) v := validator.New() return &WsServer{ diff --git a/internal/push/push_handler.go b/internal/push/push_handler.go index 7b1efe3bf..4f1964084 100644 --- a/internal/push/push_handler.go +++ b/internal/push/push_handler.go @@ -317,8 +317,8 @@ func (c *ConsumerHandler) groupMessagesHandler(ctx context.Context, groupID stri return err } log.ZDebug(ctx, "GroupDismissedNotificationInfo****", "groupID", groupID, "num", len(*pushToUserIDs), "list", pushToUserIDs) - if len(c.config.Share.IMAdminUserID) > 0 { - ctx = mcontext.WithOpUserIDContext(ctx, c.config.Share.IMAdminUserID[0]) + if len(c.config.Share.IMAdminUser.UserIDs) > 0 { + ctx = mcontext.WithOpUserIDContext(ctx, c.config.Share.IMAdminUser.UserIDs[0]) } defer func(groupID string) { if err := c.groupClient.DismissGroup(ctx, groupID, true); err != nil { diff --git a/internal/rpc/auth/auth.go b/internal/rpc/auth/auth.go index 5ed9cdf12..7a8607164 100644 --- a/internal/rpc/auth/auth.go +++ b/internal/rpc/auth/auth.go @@ -49,6 +49,7 @@ type authServer struct { RegisterCenter discovery.Conn config *Config userClient *rpcli.UserClient + adminUserIDs []string } type Config struct { @@ -90,10 +91,11 @@ func Start(ctx context.Context, config *Config, client discovery.SvcDiscoveryReg config.Share.Secret, config.RpcConfig.TokenPolicy.Expire, config.Share.MultiLogin, - config.Share.IMAdminUserID, + config.Share.IMAdminUser.UserIDs, ), - config: config, - userClient: rpcli.NewUserClient(userConn), + config: config, + userClient: rpcli.NewUserClient(userConn), + adminUserIDs: config.Share.IMAdminUser.UserIDs, }) return nil } @@ -104,8 +106,8 @@ func (s *authServer) GetAdminToken(ctx context.Context, req *pbauth.GetAdminToke return nil, errs.ErrNoPermission.WrapMsg("secret invalid") } - if !datautil.Contain(req.UserID, s.config.Share.IMAdminUserID...) { - return nil, errs.ErrArgs.WrapMsg("userID is error.", "userID", req.UserID, "adminUserID", s.config.Share.IMAdminUserID) + if !datautil.Contain(req.UserID, s.adminUserIDs...) { + return nil, errs.ErrArgs.WrapMsg("userID is error.", "userID", req.UserID, "adminUserID", s.adminUserIDs) } diff --git a/internal/rpc/group/group.go b/internal/rpc/group/group.go index ce8a2c7aa..3700c054a 100644 --- a/internal/rpc/group/group.go +++ b/internal/rpc/group/group.go @@ -63,6 +63,7 @@ type groupServer struct { userClient *rpcli.UserClient msgClient *rpcli.MsgClient conversationClient *rpcli.ConversationClient + adminUserIDs []string } type Config struct { @@ -116,6 +117,7 @@ func Start(ctx context.Context, config *Config, client discovery.SvcDiscoveryReg userClient: rpcli.NewUserClient(userConn), msgClient: rpcli.NewMsgClient(msgConn), conversationClient: rpcli.NewConversationClient(conversationConn), + adminUserIDs: config.Share.IMAdminUser.UserIDs, } gs.db = controller.NewGroupDatabase(rdb, &config.LocalCacheConfig, groupDB, groupMemberDB, groupRequestDB, mgocli.GetTx(), grouphash.NewGroupHashFromGroupServer(&gs)) gs.notification = NewNotificationSender(gs.db, config, gs.userClient, gs.msgClient, gs.conversationClient) @@ -1901,7 +1903,7 @@ func (g *groupServer) GetSpecifiedUserGroupRequestInfo(ctx context.Context, req } adminIDs = append(adminIDs, owners[0].UserID) - adminIDs = append(adminIDs, g.config.Share.IMAdminUserID...) + adminIDs = append(adminIDs, g.adminUserIDs...) if !datautil.Contain(opUserID, adminIDs...) { return nil, errs.ErrNoPermission.WrapMsg("opUser no permission") diff --git a/internal/rpc/msg/revoke.go b/internal/rpc/msg/revoke.go index bd1d66ba1..1f4d6be68 100644 --- a/internal/rpc/msg/revoke.go +++ b/internal/rpc/msg/revoke.go @@ -109,8 +109,8 @@ func (m *msgServer) RevokeMsg(ctx context.Context, req *msg.RevokeMsgReq) (*msg. revokerUserID := mcontext.GetOpUserID(ctx) var flag bool - if len(m.config.Share.IMAdminUserID) > 0 { - flag = datautil.Contain(revokerUserID, m.config.Share.IMAdminUserID...) + if len(m.config.Share.IMAdminUser.UserIDs) > 0 { + flag = datautil.Contain(revokerUserID, m.adminUserIDs...) } tips := sdkws.RevokeMsgTips{ RevokerUserID: revokerUserID, diff --git a/internal/rpc/msg/server.go b/internal/rpc/msg/server.go index 9d5391cc9..48101cdd7 100644 --- a/internal/rpc/msg/server.go +++ b/internal/rpc/msg/server.go @@ -22,7 +22,6 @@ import ( "github.com/openimsdk/open-im-server/v3/pkg/dbbuild" "github.com/openimsdk/open-im-server/v3/pkg/mqbuild" "github.com/openimsdk/open-im-server/v3/pkg/rpcli" - "google.golang.org/grpc" "github.com/openimsdk/open-im-server/v3/pkg/common/config" @@ -71,6 +70,8 @@ type msgServer struct { config *Config // Global configuration settings. webhookClient *webhook.Client conversationClient *rpcli.ConversationClient + + adminUserIDs []string } func (m *msgServer) addInterceptorHandler(interceptorFunc ...MessageInterceptorFunc) { @@ -145,6 +146,7 @@ func Start(ctx context.Context, config *Config, client discovery.SvcDiscoveryReg config: config, webhookClient: webhook.NewWebhookClient(config.WebhooksConfig.URL), conversationClient: conversationClient, + adminUserIDs: config.Share.IMAdminUser.UserIDs, } s.notificationSender = notification.NewNotificationSender(&config.NotificationConfig, notification.WithLocalSendMsg(s.SendMsg)) diff --git a/internal/rpc/msg/verify.go b/internal/rpc/msg/verify.go index 213545d83..d0f36a388 100644 --- a/internal/rpc/msg/verify.go +++ b/internal/rpc/msg/verify.go @@ -54,7 +54,7 @@ type MessageRevoked struct { func (m *msgServer) messageVerification(ctx context.Context, data *msg.SendMsgReq) error { switch data.MsgData.SessionType { case constant.SingleChatType: - if datautil.Contain(data.MsgData.SendID, m.config.Share.IMAdminUserID...) { + if datautil.Contain(data.MsgData.SendID, m.adminUserIDs...) { return nil } if data.MsgData.ContentType <= constant.NotificationEnd && @@ -102,7 +102,7 @@ func (m *msgServer) messageVerification(ctx context.Context, data *msg.SendMsgRe return nil } - if datautil.Contain(data.MsgData.SendID, m.config.Share.IMAdminUserID...) { + if datautil.Contain(data.MsgData.SendID, m.adminUserIDs...) { return nil } if data.MsgData.ContentType <= constant.NotificationEnd && diff --git a/internal/rpc/user/user.go b/internal/rpc/user/user.go index 28461ae0a..91dfe736e 100644 --- a/internal/rpc/user/user.go +++ b/internal/rpc/user/user.go @@ -65,6 +65,8 @@ type userServer struct { groupClient *rpcli.GroupClient relationClient *rpcli.RelationClient clientConfig controller.ClientConfigDatabase + + adminUserIDs []string } type Config struct { @@ -92,8 +94,12 @@ func Start(ctx context.Context, config *Config, client discovery.SvcDiscoveryReg users := make([]*tablerelation.User, 0) - for _, v := range config.Share.IMAdminUserID { - users = append(users, &tablerelation.User{UserID: v, Nickname: v, AppMangerLevel: constant.AppAdmin}) + for i := range config.Share.IMAdminUser.UserIDs { + users = append(users, &tablerelation.User{ + UserID: config.Share.IMAdminUser.UserIDs[i], + Nickname: config.Share.IMAdminUser.Nicknames[i], + AppMangerLevel: constant.AppAdmin, + }) } userDB, err := mgo.NewUserMongo(mgocli.GetDB()) if err != nil { @@ -130,6 +136,7 @@ func Start(ctx context.Context, config *Config, client discovery.SvcDiscoveryReg clientConfig: controller.NewClientConfigDatabase(clientConfigDB, redis.NewClientConfigCache(rdb, clientConfigDB), mgocli.GetTx()), groupClient: rpcli.NewGroupClient(groupConn), relationClient: rpcli.NewRelationClient(friendConn), + adminUserIDs: config.Share.IMAdminUser.UserIDs, } pbuser.RegisterUserServer(server, u) return u.db.InitOnce(context.Background(), users) @@ -648,7 +655,7 @@ func (s *userServer) userModelToResp(users []*tablerelation.User, pagination pag accounts := make([]*pbuser.NotificationAccountInfo, 0) var total int64 for _, v := range users { - if v.AppMangerLevel >= constant.AppNotificationAdmin && !datautil.Contain(v.UserID, s.config.Share.IMAdminUserID...) { + if v.AppMangerLevel >= constant.AppNotificationAdmin && !datautil.Contain(v.UserID, s.adminUserIDs...) { if appManagerLevel != nil { if v.AppMangerLevel != *appManagerLevel { continue diff --git a/internal/tools/cron/cron_task.go b/internal/tools/cron/cron_task.go index a6740ab6e..2c8655d4a 100644 --- a/internal/tools/cron/cron_task.go +++ b/internal/tools/cron/cron_task.go @@ -10,7 +10,6 @@ import ( "github.com/openimsdk/protocol/third" "github.com/openimsdk/tools/discovery" "github.com/openimsdk/tools/discovery/etcd" - "github.com/openimsdk/tools/errs" "github.com/openimsdk/tools/log" "github.com/openimsdk/tools/mcontext" @@ -32,7 +31,7 @@ func Start(ctx context.Context, conf *Config, client discovery.SvcDiscoveryRegis <-ctx.Done() return nil } - ctx = mcontext.SetOpUserID(ctx, conf.Share.IMAdminUserID[0]) + ctx = mcontext.SetOpUserID(ctx, conf.Share.IMAdminUser.UserIDs[0]) msgConn, err := client.GetConn(ctx, conf.Discovery.RpcService.Msg) if err != nil { diff --git a/pkg/common/config/config.go b/pkg/common/config/config.go index 619571064..43914310e 100644 --- a/pkg/common/config/config.go +++ b/pkg/common/config/config.go @@ -356,8 +356,11 @@ type AfterConfig struct { } type Share struct { - Secret string `yaml:"secret"` - IMAdminUserID []string `yaml:"imAdminUserID"` + Secret string `yaml:"secret"` + IMAdminUser struct { + UserIDs []string `yaml:"userIDs"` + Nicknames []string `yaml:"nicknames"` + } `yaml:"imAdminUser"` MultiLogin MultiLogin `yaml:"multiLogin"` RPCMaxBodySize MaxRequestBody `yaml:"rpcMaxBodySize"` } diff --git a/pkg/common/startrpc/start.go b/pkg/common/startrpc/start.go index 06e19d8d2..9715f2aac 100644 --- a/pkg/common/startrpc/start.go +++ b/pkg/common/startrpc/start.go @@ -69,8 +69,8 @@ func Start[T any](ctx context.Context, disc *conf.Discovery, prometheusConfig *c grpcsrv.GrpcServerRequestValidate(), grpcsrv.GrpcServerPanicCapture(), ) - if shareConfig != nil && len(shareConfig.IMAdminUserID) > 0 { - options = append(options, grpcServerIMAdminUserID(shareConfig.IMAdminUserID)) + if shareConfig != nil && len(shareConfig.IMAdminUser.UserIDs) > 0 { + options = append(options, grpcServerIMAdminUserID(shareConfig.IMAdminUser.UserIDs)) } var clientOptions []grpc.DialOption if maxRequestBody != nil { diff --git a/pkg/common/storage/controller/user.go b/pkg/common/storage/controller/user.go index f97c5330b..d747ce2fa 100644 --- a/pkg/common/storage/controller/user.go +++ b/pkg/common/storage/controller/user.go @@ -97,16 +97,34 @@ func (u *userDatabase) InitOnce(ctx context.Context, users []*model.User) error } // Determine which users are missing from the database. - missingUsers := datautil.SliceAnySub(users, existingUsers, func(e *model.User) string { + var ( + missing, update []*model.User + ) + existMap := datautil.SliceToMap(existingUsers, func(e *model.User) string { return e.UserID }) + orgMap := datautil.SliceToMap(users, func(e *model.User) string { return e.UserID }) + for k, u1 := range orgMap { + if u2, ok := existMap[k]; !ok { + missing = append(missing, u1) + } else if u1.Nickname != u2.Nickname { + update = append(update, u1) + } + } // Create records for missing users. - if len(missingUsers) > 0 { - if err := u.userDB.Create(ctx, missingUsers); err != nil { + if len(missing) > 0 { + if err := u.userDB.Create(ctx, missing); err != nil { return err } } + if len(update) > 0 { + for i := range update { + if err := u.userDB.UpdateByMap(ctx, update[i].UserID, map[string]any{"nickname": update[i].Nickname}); err != nil { + return err + } + } + } return nil } diff --git a/test/stress-test-v2/main.go b/test/stress-test-v2/main.go index 0e4609964..63ce77065 100644 --- a/test/stress-test-v2/main.go +++ b/test/stress-test-v2/main.go @@ -446,7 +446,7 @@ func main() { Share: *share, Api: *apiConfig, }, - AdminUserID: share.IMAdminUserID[0], + AdminUserID: share.IMAdminUser.UserIDs[0], Ctx: ctx, Cancel: cancel, HttpClient: &http.Client{ diff --git a/test/stress-test/main.go b/test/stress-test/main.go index 6adbd12ee..d7822e51b 100755 --- a/test/stress-test/main.go +++ b/test/stress-test/main.go @@ -319,7 +319,7 @@ func main() { Share: *share, Api: *apiConfig, }, - AdminUserID: share.IMAdminUserID[0], + AdminUserID: share.IMAdminUser.UserIDs[0], Ctx: ctx, Cancel: cancel, HttpClient: &http.Client{ From bd56bd23b80796ac40d91e3985f053dc901672b0 Mon Sep 17 00:00:00 2001 From: Monet Lee Date: Tue, 8 Jul 2025 16:33:25 +0800 Subject: [PATCH 178/199] fix: update log level in crontask dist look. (#3440) --- internal/tools/cron/dist_look.go | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/internal/tools/cron/dist_look.go b/internal/tools/cron/dist_look.go index d1d2d1cb0..e46b9206c 100644 --- a/internal/tools/cron/dist_look.go +++ b/internal/tools/cron/dist_look.go @@ -52,15 +52,12 @@ func (e *EtcdLocker) ExecuteWithLock(ctx context.Context, taskName string, task err = mutex.TryLock(ctxWithTimeout) if err != nil { - if err == context.DeadlineExceeded { - log.ZDebug(ctx, "Task is being executed by another instance, skipping", - "taskName", taskName, - "instanceID", e.instanceID) - } else { - log.ZWarn(ctx, "Failed to acquire task lock", err, - "taskName", taskName, - "instanceID", e.instanceID) - } + // errors.Is(err, concurrency.ErrLocked) + log.ZDebug(ctx, "Task is being executed by another instance, skipping", + "taskName", taskName, + "instanceID", e.instanceID, + "error", err.Error()) + return } From 5e813ef07986bfef8ee33a4d9f5f95d015a58774 Mon Sep 17 00:00:00 2001 From: Monet Lee Date: Tue, 8 Jul 2025 16:33:39 +0800 Subject: [PATCH 179/199] fix: solve `createTime` not set in setConversation and Create Conversation. (#3447) --- internal/rpc/conversation/conversation.go | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/internal/rpc/conversation/conversation.go b/internal/rpc/conversation/conversation.go index cf5a2b9c6..a2b72ddfb 100644 --- a/internal/rpc/conversation/conversation.go +++ b/internal/rpc/conversation/conversation.go @@ -31,7 +31,6 @@ import ( "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/redis" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/controller" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database/mgo" - "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" dbModel "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" "github.com/openimsdk/open-im-server/v3/pkg/common/webhook" "github.com/openimsdk/open-im-server/v3/pkg/localcache" @@ -244,11 +243,14 @@ func (c *conversationServer) getConversations(ctx context.Context, ownerUserID s return convert.ConversationsDB2Pb(conversations), nil } +// Deprecated func (c *conversationServer) SetConversation(ctx context.Context, req *pbconversation.SetConversationReq) (*pbconversation.SetConversationResp, error) { if err := authverify.CheckAccess(ctx, req.GetConversation().GetUserID()); err != nil { return nil, err } var conversation dbModel.Conversation + conversation.CreateTime = time.Now() + if err := datautil.CopyStructFields(&conversation, req.Conversation); err != nil { return nil, err } @@ -300,6 +302,7 @@ func (c *conversationServer) SetConversations(ctx context.Context, req *pbconver conversation.ConversationType = req.Conversation.ConversationType conversation.UserID = req.Conversation.UserID conversation.GroupID = req.Conversation.GroupID + conversation.CreateTime = time.Now() m, conversation, err := UpdateConversationsMap(ctx, req) if err != nil { @@ -365,6 +368,8 @@ func (c *conversationServer) UpdateConversationsByUser(ctx context.Context, req // create conversation without notification for msg redis transfer. func (c *conversationServer) CreateSingleChatConversations(ctx context.Context, req *pbconversation.CreateSingleChatConversationsReq) (*pbconversation.CreateSingleChatConversationsResp, error) { var conversation dbModel.Conversation + conversation.CreateTime = time.Now() + switch req.ConversationType { case constant.SingleChatType: // sendUser create @@ -372,6 +377,7 @@ func (c *conversationServer) CreateSingleChatConversations(ctx context.Context, conversation.ConversationType = req.ConversationType conversation.OwnerUserID = req.SendID conversation.UserID = req.RecvID + if err := c.webhookBeforeCreateSingleChatConversations(ctx, &c.config.WebhooksConfig.BeforeCreateSingleChatConversations, &conversation); err != nil && err != servererrs.ErrCallbackContinue { return nil, err } @@ -387,6 +393,7 @@ func (c *conversationServer) CreateSingleChatConversations(ctx context.Context, conversation2 := conversation conversation2.OwnerUserID = req.RecvID conversation2.UserID = req.SendID + if err := c.webhookBeforeCreateSingleChatConversations(ctx, &c.config.WebhooksConfig.BeforeCreateSingleChatConversations, &conversation); err != nil && err != servererrs.ErrCallbackContinue { return nil, err } @@ -402,6 +409,7 @@ func (c *conversationServer) CreateSingleChatConversations(ctx context.Context, conversation.ConversationType = req.ConversationType conversation.OwnerUserID = req.RecvID conversation.UserID = req.SendID + if err := c.webhookBeforeCreateSingleChatConversations(ctx, &c.config.WebhooksConfig.BeforeCreateSingleChatConversations, &conversation); err != nil && err != servererrs.ErrCallbackContinue { return nil, err } @@ -423,6 +431,7 @@ func (c *conversationServer) CreateGroupChatConversations(ctx context.Context, r conversation.ConversationID = msgprocessor.GetConversationIDBySessionType(constant.ReadGroupChatType, req.GroupID) conversation.GroupID = req.GroupID conversation.ConversationType = constant.ReadGroupChatType + conversation.CreateTime = time.Now() if err := c.webhookBeforeCreateGroupChatConversations(ctx, &c.config.WebhooksConfig.BeforeCreateGroupChatConversations, &conversation); err != nil { return nil, err @@ -708,7 +717,7 @@ func (c *conversationServer) GetConversationsNeedClearMsg(ctx context.Context, _ maxPage := (num + batchNum - 1) / batchNum - temp := make([]*model.Conversation, 0, maxPage*batchNum) + temp := make([]*dbModel.Conversation, 0, maxPage*batchNum) for pageNumber := 0; pageNumber < int(maxPage); pageNumber++ { pagination := &sdkws.RequestPagination{ From b1659ef84d0b6459794c37400094ff3101cea48e Mon Sep 17 00:00:00 2001 From: Monet Lee Date: Tue, 22 Jul 2025 10:24:01 +0800 Subject: [PATCH 180/199] refactor: support modified config and args in mage. (#3466) * refactor: support modified config and args in mage. * fix: update go make version. --- go.mod | 2 +- go.sum | 4 ++-- magefile.go | 67 +++++++++++++++++++++++++++++++++++++++++++++++++++-- 3 files changed, 68 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index 2a05b14b6..badf4ffaa 100644 --- a/go.mod +++ b/go.mod @@ -34,7 +34,7 @@ require ( github.com/hashicorp/golang-lru/v2 v2.0.7 github.com/kelindar/bitmap v1.5.2 github.com/likexian/gokit v0.25.13 - github.com/openimsdk/gomake v0.0.15-alpha.5 + github.com/openimsdk/gomake v0.0.15-alpha.11 github.com/redis/go-redis/v9 v9.4.0 github.com/robfig/cron/v3 v3.0.1 github.com/shirou/gopsutil v3.21.11+incompatible diff --git a/go.sum b/go.sum index db68ca960..ab0499ba0 100644 --- a/go.sum +++ b/go.sum @@ -345,8 +345,8 @@ github.com/onsi/ginkgo/v2 v2.19.0 h1:9Cnnf7UHo57Hy3k6/m5k3dRfGTMXGvxhHFvkDTCTpvA github.com/onsi/ginkgo/v2 v2.19.0/go.mod h1:rlwLi9PilAFJ8jCg9UE1QP6VBpd6/xj3SRC0d6TU0To= github.com/onsi/gomega v1.25.0 h1:Vw7br2PCDYijJHSfBOWhov+8cAnUf8MfMaIOV323l6Y= github.com/onsi/gomega v1.25.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdMPGhM= -github.com/openimsdk/gomake v0.0.15-alpha.5 h1:eEZCEHm+NsmcO3onXZPIUbGFCYPYbsX5beV3ZyOsGhY= -github.com/openimsdk/gomake v0.0.15-alpha.5/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= +github.com/openimsdk/gomake v0.0.15-alpha.11 h1:PQudYDRESYeYlUYrrLLJhYIlUPO5x7FAx+o5El9U/Bw= +github.com/openimsdk/gomake v0.0.15-alpha.11/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= github.com/openimsdk/protocol v0.0.73-alpha.12 h1:2NYawXeHChYUeSme6QJ9pOLh+Empce2WmwEtbP4JvKk= github.com/openimsdk/protocol v0.0.73-alpha.12/go.mod h1:WF7EuE55vQvpyUAzDXcqg+B+446xQyEba0X35lTINmw= github.com/openimsdk/tools v0.0.50-alpha.92 h1:hWfykMhmi7EQEiwgQccJqbgggIuhun/PrVkBnjmj9Ec= diff --git a/magefile.go b/magefile.go index d0e77f6fc..b9861bd5e 100644 --- a/magefile.go +++ b/magefile.go @@ -12,15 +12,48 @@ import ( var Default = Build +var Aliases = map[string]any{ + "buildcc": BuildWithCustomConfig, + "startcc": StartWithCustomConfig, +} + +var ( + customRootDir = "." // workDir in mage, default is "./"(project root directory) + customSrcDir = "cmd" // source code directory, default is "cmd" + customOutputDir = "_output" // output directory, default is "_output" + customConfigDir = "config" // configuration directory, default is "config" + customToolsDir = "tools" // tools source code directory, default is "tools" +) + + +// Build support specifical binary build. +// +// Example: `mage build openim-api openim-rpc-user seq` func Build() { flag.Parse() + bin := flag.Args() + if len(bin) != 0 { + bin = bin[1:] + } + mageutil.Build(bin, nil) +} + +func BuildWithCustomConfig() { + flag.Parse() bin := flag.Args() if len(bin) != 0 { bin = bin[1:] } - mageutil.Build(bin) + config := &mageutil.PathOptions{ + RootDir: &customRootDir, + OutputDir: &customOutputDir, + SrcDir: &customSrcDir, + ToolsDir: &customToolsDir, + } + + mageutil.Build(bin, config) } func Start() { @@ -30,7 +63,37 @@ func Start() { mageutil.PrintRed("setMaxOpenFiles failed " + err.Error()) os.Exit(1) } - mageutil.StartToolsAndServices() + + flag.Parse() + bin := flag.Args() + if len(bin) != 0 { + bin = bin[1:] + } + + mageutil.StartToolsAndServices(bin, nil) +} + +func StartWithCustomConfig() { + mageutil.InitForSSC() + err := setMaxOpenFiles() + if err != nil { + mageutil.PrintRed("setMaxOpenFiles failed " + err.Error()) + os.Exit(1) + } + + flag.Parse() + bin := flag.Args() + if len(bin) != 0 { + bin = bin[1:] + } + + config := &mageutil.PathOptions{ + RootDir: &customRootDir, + OutputDir: &customOutputDir, + ConfigDir: &customConfigDir, + } + + mageutil.StartToolsAndServices(bin, config) } func Stop() { From 4ae05d2e38f3f6e3bcc62be59706f7298f7da84a Mon Sep 17 00:00:00 2001 From: Monet Lee Date: Fri, 25 Jul 2025 15:02:49 +0800 Subject: [PATCH 181/199] fix: use safe submodule init in workflows. (#3468) * fix: use safe submodule init in workflows. * update to latest version. --- .../update-version-file-on-release.yml | 101 ++++++++++++------ 1 file changed, 68 insertions(+), 33 deletions(-) diff --git a/.github/workflows/update-version-file-on-release.yml b/.github/workflows/update-version-file-on-release.yml index 113537fd9..b7e2a4409 100644 --- a/.github/workflows/update-version-file-on-release.yml +++ b/.github/workflows/update-version-file-on-release.yml @@ -8,19 +8,40 @@ jobs: update-version: runs-on: ubuntu-latest env: - TAG_VERSION: ${{ github.event.release.tag_name }} + TAG_VERSION: ${{ github.event.release.tag_name }} steps: # Step 1: Checkout the original repository's code - name: Checkout code uses: actions/checkout@v4 with: fetch-depth: 0 + # submodules: "recursive" + + - name: Safe submodule initialization + run: | + echo "Checking for submodules..." + if [ -f .gitmodules ]; then + if [ -s .gitmodules ]; then + echo "Initializing submodules..." + if git submodule sync --recursive 2>/dev/null; then + git submodule update --init --force --recursive || { + echo "Warning: Some submodules failed to initialize, continuing anyway..." + } + else + echo "Warning: Submodule sync failed, continuing without submodules..." + fi + else + echo ".gitmodules exists but is empty, skipping submodule initialization" + fi + else + echo "No .gitmodules file found, no submodules to initialize" + fi # Step 2: Set up Git with official account - name: Set up Git run: | - git config user.name "github-actions[bot]" - git config user.email "github-actions[bot]@users.noreply.github.com" + git config --global user.name "github-actions[bot]" + git config --global user.email "github-actions[bot]@users.noreply.github.com" # Step 3: Check and delete existing tag - name: Check and delete existing tag @@ -33,7 +54,8 @@ jobs: # Step 4: Update version file - name: Update version file run: | - echo "${{ env.TAG_VERSION }}" > version/version + mkdir -p version + echo -n "${{ env.TAG_VERSION }}" > version/version # Step 5: Commit and push changes - name: Commit and push changes @@ -42,43 +64,56 @@ jobs: run: | git add version/version git commit -m "Update version to ${{ env.TAG_VERSION }}" - git push origin HEAD:${{ github.ref }} - # Step 6: Create and push tag - - name: Create and push tag - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + # Step 6: Update tag + - name: Update tag run: | - git tag ${{ env.TAG_VERSION }} - git push origin ${{ env.TAG_VERSION }} + git tag -fa ${{ env.TAG_VERSION }} -m "Update version to ${{ env.TAG_VERSION }}" + git push origin ${{ env.TAG_VERSION }} --force # Step 7: Find and Publish Draft Release - name: Find and Publish Draft Release - uses: actions/github-script@v6 + uses: actions/github-script@v7 with: github-token: ${{ secrets.GITHUB_TOKEN }} script: | - // Get the list of releases - const releases = await github.rest.repos.listReleases({ - owner: context.repo.owner, - repo: context.repo.repo - }); - - // Find the draft release where the title and tag_name are the same - const draftRelease = releases.data.find(release => - release.draft && release.name === release.tag_name - ); + const { owner, repo } = context.repo; + const tagName = process.env.TAG_VERSION; - if (draftRelease) { - // Publish the draft release using the release_id + try { + let release; + try { + const response = await github.rest.repos.getReleaseByTag({ + owner, + repo, + tag: tagName + }); + release = response.data; + } catch (tagError) { + core.info(`Release not found by tag, searching all releases...`); + const releases = await github.rest.repos.listReleases({ + owner, + repo, + per_page: 100 + }); + + release = releases.data.find(r => r.draft && r.tag_name === tagName); + if (!release) { + throw new Error(`No release found with tag ${tagName}`); + } + } + await github.rest.repos.updateRelease({ - owner: context.repo.owner, - repo: context.repo.repo, - release_id: draftRelease.id, // Use release_id - draft: false + owner, + repo, + release_id: release.id, + draft: false, + prerelease: release.prerelease }); - - core.info(`Draft Release ${draftRelease.tag_name} published successfully.`); - } else { - core.info("No matching draft release found."); - } \ No newline at end of file + + const status = release.draft ? "was draft" : "was already published"; + core.info(`Release ${tagName} ensured to be published (${status}).`); + + } catch (error) { + core.warning(`Could not find or update release for tag ${tagName}: ${error.message}`); + } From 230c93cc16d8be8d6196ded250f9d774fa778eae Mon Sep 17 00:00:00 2001 From: Monet Lee Date: Mon, 28 Jul 2025 10:46:23 +0800 Subject: [PATCH 182/199] docs: update slack link. (#3479) --- .github/ISSUE_TEMPLATE/config.yml | 2 +- .github/ISSUE_TEMPLATE/other.yml | 3 +- .github/workflows/auto-invite-comment.yml | 11 ++- .github/workflows/help-comment-issue.yml | 2 +- .github/workflows/user-first-interaction.yml | 10 +-- README.md | 40 +++++----- README_zh_CN.md | 54 ++++++------- docs/readme/README_cs.md | 36 ++++----- docs/readme/README_da.md | 45 +++++------ docs/readme/README_el.md | 39 +++++---- docs/readme/README_es.md | 48 +++++------ docs/readme/README_fa.md | 38 ++++----- docs/readme/README_fr.md | 42 +++++----- docs/readme/README_hu.md | 36 ++++----- docs/readme/README_ja.md | 83 +++++++++----------- docs/readme/README_ko.md | 45 +++++------ docs/readme/README_tr.md | 40 ++++------ docs/readme/README_uk.md | 36 ++++----- docs/readme/README_vi.md | 41 +++++----- install.sh | 2 +- 20 files changed, 288 insertions(+), 365 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml index deb899083..6c91a8522 100644 --- a/.github/ISSUE_TEMPLATE/config.yml +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -4,7 +4,7 @@ contact_links: # description: "Report a bug in the project" # file: "bug-report.yml" - name: 📢 Connect on slack - url: https://join.slack.com/t/openimsdk/shared_invite/zt-1tmoj26uf-_FDy3dowVHBiGvLk9e5Xkg + url: https://join.slack.com/t/openimsdk/shared_invite/zt-2ijy1ys1f-O0aEDCr7ExRZ7mwsHAVg9A about: Support OpenIM-related requests or issues, get in touch with developers and help on slack - name: 🌐 OpenIM Blog url: https://www.openim.io/ diff --git a/.github/ISSUE_TEMPLATE/other.yml b/.github/ISSUE_TEMPLATE/other.yml index 025440229..3a5690e8d 100644 --- a/.github/ISSUE_TEMPLATE/other.yml +++ b/.github/ISSUE_TEMPLATE/other.yml @@ -4,7 +4,6 @@ title: "[Other]: " labels: ["other"] # assignees: [] - body: - type: markdown attributes: @@ -26,5 +25,5 @@ body: - type: markdown attributes: value: | - You can also join our Discord community [here](https://join.slack.com/t/openimsdk/shared_invite/zt-1tmoj26uf-_FDy3dowVHBiGvLk9e5Xkg) + You can also join our Discord community [here](https://join.slack.com/t/openimsdk/shared_invite/zt-2ijy1ys1f-O0aEDCr7ExRZ7mwsHAVg9A) Feel free to check out other cool repositories of the openim Community [here](https://github.com/openimsdk) diff --git a/.github/workflows/auto-invite-comment.yml b/.github/workflows/auto-invite-comment.yml index 76fbcdfd3..dde36eaf1 100644 --- a/.github/workflows/auto-invite-comment.yml +++ b/.github/workflows/auto-invite-comment.yml @@ -11,7 +11,6 @@ jobs: permissions: issues: write steps: - - name: Invite user to join OpenIM Community uses: peter-evans/create-or-update-comment@v4 with: @@ -20,11 +19,11 @@ jobs: body: | We value close connections with our users, developers, and contributors here at Open-IM-Server. With a large community and maintainer team, we're always here to help and support you. Whether you're looking to join our community or have any questions or suggestions, we welcome you to get in touch with us. - Our most recommended way to get in touch is through [Slack](https://join.slack.com/t/openimsdk/shared_invite/zt-22720d66b-o_FvKxMTGXtcnnnHiMqe9Q). Even if you're in China, Slack is usually not blocked by firewalls, making it an easy way to connect with us. Our Slack community is the ideal place to discuss and share ideas and suggestions with other users and developers of Open-IM-Server. You can ask technical questions, seek help, or share your experiences with other users of Open-IM-Server. - + Our most recommended way to get in touch is through [Slack](https://join.slack.com/t/openimsdk/shared_invite/zt-2ijy1ys1f-O0aEDCr7ExRZ7mwsHAVg9A). Even if you're in China, Slack is usually not blocked by firewalls, making it an easy way to connect with us. Our Slack community is the ideal place to discuss and share ideas and suggestions with other users and developers of Open-IM-Server. You can ask technical questions, seek help, or share your experiences with other users of Open-IM-Server. + In addition to Slack, we also offer the following ways to get in touch: - - + We also have Slack channels for you to communicate and discuss. To join, visit https://slack.com/ and join our [👀 Open-IM-Server slack](https://join.slack.com/t/openimsdk/shared_invite/zt-22720d66b-o_FvKxMTGXtcnnnHiMqe9Q) team channel. + + + We also have Slack channels for you to communicate and discuss. To join, visit https://slack.com/ and join our [👀 Open-IM-Server slack](https://join.slack.com/t/openimsdk/shared_invite/zt-2ijy1ys1f-O0aEDCr7ExRZ7mwsHAVg9A) team channel. + Get in touch with us on [Gmail](https://mail.google.com/mail/u/0/?fs=1&tf=cm&to=winxu81@gmail.com). If you have any questions or issues that need resolving, or any suggestions and feedback for our open source projects, please feel free to contact us via email. + Read our [blog](https://doc.rentsoft.cn/). Our blog is a great place to stay up-to-date with Open-IM-Server projects and trends. On the blog, we share our latest developments, tech trends, and other interesting information. + Add [Wechat](https://github.com/OpenIMSDK/OpenIM-Docs/blob/main/docs/images/WechatIMG20.jpeg) and indicate that you are a user or developer of Open-IM-Server. We will process your request as soon as possible. @@ -36,4 +35,4 @@ jobs: # issue-number: ${{ github.event.issue.number }} # comment: 🤖 Auto-closing issue, if you still need help please reopen the issue or ask for help in the community above # labels: | - # accepted \ No newline at end of file + # accepted diff --git a/.github/workflows/help-comment-issue.yml b/.github/workflows/help-comment-issue.yml index b1cc62182..ce8f12b3b 100644 --- a/.github/workflows/help-comment-issue.yml +++ b/.github/workflows/help-comment-issue.yml @@ -32,5 +32,5 @@ jobs: token: ${{ secrets.BOT_TOKEN }} body: | This issue is available for anyone to work on. **Make sure to reference this issue in your pull request.** :sparkles: Thank you for your contribution! :sparkles: - [Join slack 🤖](https://join.slack.com/t/openimsdk/shared_invite/zt-22720d66b-o_FvKxMTGXtcnnnHiMqe9Q) to connect and communicate with our developers. + [Join slack 🤖](https://join.slack.com/t/openimsdk/shared_invite/zt-2ijy1ys1f-O0aEDCr7ExRZ7mwsHAVg9A) to connect and communicate with our developers. If you wish to accept this assignment, please leave a comment in the comments section: `/accept`.🎯 diff --git a/.github/workflows/user-first-interaction.yml b/.github/workflows/user-first-interaction.yml index 6999889eb..4271702a6 100644 --- a/.github/workflows/user-first-interaction.yml +++ b/.github/workflows/user-first-interaction.yml @@ -17,12 +17,12 @@ jobs: repo-token: ${{ secrets.BOT_TOKEN }} pr-message: | Hello! Thank you for your contribution. - + If you are fixing a bug, please reference the issue number in the description. If you are implementing a feature request, please check with the maintainers that the feature will be accepted first. - [Join slack 🤖](https://join.slack.com/t/openimsdk/shared_invite/zt-22720d66b-o_FvKxMTGXtcnnnHiMqe9Q) to connect and communicate with our developers. + [Join slack 🤖](https://join.slack.com/t/openimsdk/shared_invite/zt-2ijy1ys1f-O0aEDCr7ExRZ7mwsHAVg9A) to connect and communicate with our developers. Please leave your information in the [✨ discussions](https://github.com/orgs/OpenIMSDK/discussions/426), we expect anyone to join OpenIM developer community. @@ -30,6 +30,6 @@ jobs: Hello! Thank you for filing an issue. If this is a bug report, please include relevant logs to help us debug the problem. - - [Join slack 🤖](https://join.slack.com/t/openimsdk/shared_invite/zt-22720d66b-o_FvKxMTGXtcnnnHiMqe9Q) to connect and communicate with our developers. - continue-on-error: true \ No newline at end of file + + [Join slack 🤖](https://join.slack.com/t/openimsdk/shared_invite/zt-2ijy1ys1f-O0aEDCr7ExRZ7mwsHAVg9A) to connect and communicate with our developers. + continue-on-error: true diff --git a/README.md b/README.md index e59bf20c4..3463e83b2 100644 --- a/README.md +++ b/README.md @@ -12,13 +12,12 @@ [![Go Report Card](https://goreportcard.com/badge/github.com/openimsdk/open-im-server?style=for-the-badge)](https://goreportcard.com/report/github.com/openimsdk/open-im-server) [![Go Reference](https://img.shields.io/badge/Go%20Reference-blue.svg?style=for-the-badge&logo=go&logoColor=white)](https://pkg.go.dev/github.com/openimsdk/open-im-server/v3) [![License](https://img.shields.io/badge/license-Apache--2.0-green?style=for-the-badge)](https://github.com/openimsdk/open-im-server/blob/main/LICENSE) -[![Slack](https://img.shields.io/badge/Slack-500%2B-blueviolet?style=for-the-badge&logo=slack&logoColor=white)](https://join.slack.com/t/openimsdk/shared_invite/zt-22720d66b-o_FvKxMTGXtcnnnHiMqe9Q) +[![Slack](https://img.shields.io/badge/Slack-500%2B-blueviolet?style=for-the-badge&logo=slack&logoColor=white)](https://join.slack.com/t/openimsdk/shared_invite/zt-2ijy1ys1f-O0aEDCr7ExRZ7mwsHAVg9A) [![Best Practices](https://img.shields.io/badge/Best%20Practices-purple?style=for-the-badge)](https://www.bestpractices.dev/projects/8045) [![Good First Issues](https://img.shields.io/github/issues/openimsdk/open-im-server/good%20first%20issue?style=for-the-badge&logo=github)](https://github.com/openimsdk/open-im-server/issues?q=is%3Aissue+is%3Aopen+sort%3Aupdated-desc+label%3A%22good+first+issue%22) [![Language](https://img.shields.io/badge/Language-Go-blue.svg?style=for-the-badge&logo=go&logoColor=white)](https://golang.org/) [![Gurubase](https://img.shields.io/badge/Gurubase-Ask%20OpenIM%20Guru-006BFF?style=for-the-badge)](https://gurubase.io/g/openim) -

English · 中文 · @@ -47,16 +46,15 @@ Türkçe

-

## :busts_in_silhouette: Join Our Community -+ 💬 [Follow us on Twitter](https://twitter.com/founder_im63606) -+ 🚀 [Join our Slack](https://join.slack.com/t/openimsdk/shared_invite/zt-2ijy1ys1f-O0aEDCr7ExRZ7mwsHAVg9A) -+ :eyes: [Join our WeChat Group](https://openim-1253691595.cos.ap-nanjing.myqcloud.com/WechatIMG20.jpeg) +- 💬 [Follow us on Twitter](https://twitter.com/founder_im63606) +- 🚀 [Join our Slack](https://join.slack.com/t/openimsdk/shared_invite/zt-2ijy1ys1f-O0aEDCr7ExRZ7mwsHAVg9A) +- :eyes: [Join our WeChat Group](https://openim-1253691595.cos.ap-nanjing.myqcloud.com/WechatIMG20.jpeg) ## Ⓜ️ About OpenIM @@ -68,13 +66,14 @@ Unlike standalone chat applications such as Telegram, Signal, and Rocket.Chat, O **OpenIMSDK**, designed for **OpenIMServer**, is an IM SDK created specifically for integration into client applications. It supports various functionalities and modules: -+ 🌟 Main Features: +- 🌟 Main Features: + - 📦 Local Storage - 🔔 Listener Callbacks - 🛡️ API Wrapping - 🌐 Connection Management -+ 📚 Main Modules: +- 📚 Main Modules: 1. 🚀 Initialization and Login 2. 👤 User Management 3. 👫 Friends Management @@ -85,18 +84,18 @@ Built with Golang and supports cross-platform deployment to ensure a consistent 👉 **[Explore the GO SDK](https://github.com/openimsdk/openim-sdk-core)** -## 🌐 Introduction to OpenIMServer +## 🌐 Introduction to OpenIMServer -+ **OpenIMServer** features include: +- **OpenIMServer** features include: - 🌐 Microservices Architecture: Supports cluster mode, including a gateway and multiple rpc services. - 🚀 Diverse Deployment Options: Supports source code, Kubernetes, or Docker deployment. - Massive User Support: Supports large-scale groups with hundreds of thousands, millions of users, and billions of messages. ### Enhanced Business Functions: -+ **REST API**: Provides a REST API for business systems to enhance functionality, such as group creation and message pushing through backend interfaces. +- **REST API**: Provides a REST API for business systems to enhance functionality, such as group creation and message pushing through backend interfaces. -+ **Webhooks**: Expands business forms through callbacks, sending requests to business servers before or after certain events. +- **Webhooks**: Expands business forms through callbacks, sending requests to business servers before or after certain events. ![Overall Architecture](./docs/images/architecture-layers.png) @@ -108,8 +107,8 @@ Experience online for iOS/Android/H5/PC/Web: To facilitate user experience, we offer various deployment solutions. You can choose your preferred deployment method from the list below: -+ **[Source Code Deployment Guide](https://docs.openim.io/guides/gettingStarted/imSourceCodeDeployment)** -+ **[Docker Deployment Guide](https://docs.openim.io/guides/gettingStarted/dockerCompose)** +- **[Source Code Deployment Guide](https://docs.openim.io/guides/gettingStarted/imSourceCodeDeployment)** +- **[Docker Deployment Guide](https://docs.openim.io/guides/gettingStarted/dockerCompose)** ## System Support @@ -117,16 +116,16 @@ Supports Linux, Windows, Mac systems, and ARM and AMD CPU architectures. ## :link: Links - + **[Developer Manual](https://docs.openim.io/)** - + **[Changelog](https://github.com/openimsdk/open-im-server/blob/main/CHANGELOG.md)** +- **[Developer Manual](https://docs.openim.io/)** +- **[Changelog](https://github.com/openimsdk/open-im-server/blob/main/CHANGELOG.md)** ## :writing_hand: How to Contribute We welcome contributions of any kind! Please make sure to read our [Contributor Documentation](https://github.com/openimsdk/open-im-server/blob/main/CONTRIBUTING.md) before submitting a Pull Request. - + **[Report a Bug](https://github.com/openimsdk/open-im-server/issues/new?assignees=&labels=bug&template=bug_report.md&title=)** - + **[Suggest a Feature](https://github.com/openimsdk/open-im-server/issues/new?assignees=&labels=enhancement&template=feature_request.md&title=)** - + **[Submit a Pull Request](https://github.com/openimsdk/open-im-server/pulls)** +- **[Report a Bug](https://github.com/openimsdk/open-im-server/issues/new?assignees=&labels=bug&template=bug_report.md&title=)** +- **[Suggest a Feature](https://github.com/openimsdk/open-im-server/issues/new?assignees=&labels=enhancement&template=feature_request.md&title=)** +- **[Submit a Pull Request](https://github.com/openimsdk/open-im-server/pulls)** Thank you for contributing to building a powerful instant messaging solution! @@ -134,9 +133,6 @@ Thank you for contributing to building a powerful instant messaging solution! This software is licensed under the Apache License 2.0 - - - ## 🔮 Thanks to our contributors! diff --git a/README_zh_CN.md b/README_zh_CN.md index 2147c0243..7728c4904 100644 --- a/README_zh_CN.md +++ b/README_zh_CN.md @@ -12,12 +12,11 @@ [![Go Report Card](https://goreportcard.com/badge/github.com/openimsdk/open-im-server?style=for-the-badge)](https://goreportcard.com/report/github.com/openimsdk/open-im-server) [![Go Reference](https://img.shields.io/badge/Go%20Reference-blue.svg?style=for-the-badge&logo=go&logoColor=white)](https://pkg.go.dev/github.com/openimsdk/open-im-server/v3) [![License](https://img.shields.io/badge/license-Apache--2.0-green?style=for-the-badge)](https://github.com/openimsdk/open-im-server/blob/main/LICENSE) -[![Slack](https://img.shields.io/badge/Slack-500%2B-blueviolet?style=for-the-badge&logo=slack&logoColor=white)](https://join.slack.com/t/openimsdk/shared_invite/zt-22720d66b-o_FvKxMTGXtcnnnHiMqe9Q) +[![Slack](https://img.shields.io/badge/Slack-500%2B-blueviolet?style=for-the-badge&logo=slack&logoColor=white)](https://join.slack.com/t/openimsdk/shared_invite/zt-2ijy1ys1f-O0aEDCr7ExRZ7mwsHAVg9A) [![Best Practices](https://img.shields.io/badge/Best%20Practices-purple?style=for-the-badge)](https://www.bestpractices.dev/projects/8045) [![Good First Issues](https://img.shields.io/github/issues/openimsdk/open-im-server/good%20first%20issue?style=for-the-badge&logo=github)](https://github.com/openimsdk/open-im-server/issues?q=is%3Aissue+is%3Aopen+sort%3Aupdated-desc+label%3A%22good+first+issue%22) [![Language](https://img.shields.io/badge/Language-Go-blue.svg?style=for-the-badge&logo=go&logoColor=white)](https://golang.org/) -

English · 中文 · @@ -46,34 +45,34 @@ Türkçe

-

## :busts_in_silhouette: 加入我们的社区 -+ 💬 [关注我们的 Twitter](https://twitter.com/founder_im63606) -+ 🚀 [加入我们的 Slack](https://join.slack.com/t/openimsdk/shared_invite/zt-2hljfom5u-9ZuzP3NfEKW~BJKbpLm0Hw) -+ :eyes: [加入我们的微信群](https://openim-1253691595.cos.ap-nanjing.myqcloud.com/WechatIMG20.jpeg) +- 💬 [关注我们的 Twitter](https://twitter.com/founder_im63606) +- 🚀 [加入我们的 Slack](https://join.slack.com/t/openimsdk/shared_invite/zt-2hljfom5u-9ZuzP3NfEKW~BJKbpLm0Hw) +- :eyes: [加入我们的微信群](https://openim-1253691595.cos.ap-nanjing.myqcloud.com/WechatIMG20.jpeg) ## Ⓜ️ 关于 OpenIM -与Telegram、Signal、Rocket.Chat等独立聊天应用不同，OpenIM提供了专为开发者设计的开源即时通讯解决方案，而不是直接安装使用的独立聊天应用。OpenIM由OpenIM SDK和OpenIM Server两大部分组成，为开发者提供了一整套集成即时通讯功能的工具和服务，包括消息发送接收、用户管理和群组管理等。总体来说，OpenIM旨在为开发者提供必要的工具和框架，帮助他们在自己的应用中实现高效的即时通讯解决方案。 +与 Telegram、Signal、Rocket.Chat 等独立聊天应用不同，OpenIM 提供了专为开发者设计的开源即时通讯解决方案，而不是直接安装使用的独立聊天应用。OpenIM 由 OpenIM SDK 和 OpenIM Server 两大部分组成，为开发者提供了一整套集成即时通讯功能的工具和服务，包括消息发送接收、用户管理和群组管理等。总体来说，OpenIM 旨在为开发者提供必要的工具和框架，帮助他们在自己的应用中实现高效的即时通讯解决方案。 ![App-OpenIM 关系](./docs/images/oepnim-design.png) ## 🚀 OpenIMSDK 介绍 -**OpenIMSDK** 是为 **OpenIMServer** 设计的IM SDK，专为集成到客户端应用而生。它支持多种功能和模块： +**OpenIMSDK** 是为 **OpenIMServer** 设计的 IM SDK，专为集成到客户端应用而生。它支持多种功能和模块： + +- 🌟 主要功能： -+ 🌟 主要功能： - 📦 本地存储 - 🔔 监听器回调 - - 🛡️ API封装 + - 🛡️ API 封装 - 🌐 连接管理 -+ 📚 主要模块： +- 📚 主要模块： 1. 🚀 初始化及登录 2. 👤 用户管理 3. 👫 好友管理 @@ -86,31 +85,29 @@ ## 🌐 OpenIMServer 介绍 -+ **OpenIMServer** 的特点包括： - - 🌐 微服务架构：支持集群模式，包括网关(gateway)和多个rpc服务。 - - 🚀 多样的部署方式：支持源代码、Kubernetes或Docker部署。 +- **OpenIMServer** 的特点包括： + - 🌐 微服务架构：支持集群模式，包括网关(gateway)和多个 rpc 服务。 + - 🚀 多样的部署方式：支持源代码、Kubernetes 或 Docker 部署。 - 海量用户支持：支持十万级超大群组，千万级用户和百亿级消息。 ### 增强的业务功能： -+ **REST API**：为业务系统提供REST API，增加群组创建、消息推送等后台接口功能。 +- **REST API**：为业务系统提供 REST API，增加群组创建、消息推送等后台接口功能。 -+ **Webhooks**：通过事件前后的回调，向业务服务器发送请求，扩展更多的业务形态。 +- **Webhooks**：通过事件前后的回调，向业务服务器发送请求，扩展更多的业务形态。 ![整体架构](./docs/images/architecture-layers.png) - - ## :rocket: 快速入门 -在线体验iOS/Android/H5/PC/Web： +在线体验 iOS/Android/H5/PC/Web： -👉 **[OpenIM在线演示](https://www.openim.io/en/commercial)** +👉 **[OpenIM 在线演示](https://www.openim.io/en/commercial)** 为了便于用户体验，我们提供了多种部署解决方案，您可以根据以下列表选择适合您的部署方式： -+ **[源代码部署指南](https://docs.openim.io/guides/gettingStarted/imSourceCodeDeployment)** -+ **[Docker 部署指南](https://docs.openim.io/guides/gettingStarted/dockerCompose)** +- **[源代码部署指南](https://docs.openim.io/guides/gettingStarted/imSourceCodeDeployment)** +- **[Docker 部署指南](https://docs.openim.io/guides/gettingStarted/dockerCompose)** ## 系统支持 @@ -118,24 +115,23 @@ ## :link: 相关链接 - + **[开发手册](https://docs.openim.io/)** - + **[更新日志](https://github.com/openimsdk/open-im-server/blob/main/CHANGELOG.md)** +- **[开发手册](https://docs.openim.io/)** +- **[更新日志](https://github.com/openimsdk/open-im-server/blob/main/CHANGELOG.md)** ## :writing_hand: 如何贡献 我们欢迎任何形式的贡献！在提交 Pull Request 之前，请确保阅读我们的[贡献者文档](https://github.com/openimsdk/open-im-server/blob/main/CONTRIBUTING.md) - + **[报告 Bug](https://github.com/openimsdk/open-im-server/issues/new?assignees=&labels=bug&template=bug_report.md&title=)** - + **[提出新特性](https://github.com/openimsdk/open-im-server/issues/new?assignees=&labels=enhancement&template=feature_request.md&title=)** - + **[提交 Pull Request](https://github.com/openimsdk/open-im-server/pulls)** +- **[报告 Bug](https://github.com/openimsdk/open-im-server/issues/new?assignees=&labels=bug&template=bug_report.md&title=)** +- **[提出新特性](https://github.com/openimsdk/open-im-server/issues/new?assignees=&labels=enhancement&template=feature_request.md&title=)** +- **[提交 Pull Request](https://github.com/openimsdk/open-im-server/pulls)** 感谢您的贡献，一起来打造强大的即时通讯解决方案！ -## :closed_book: 开源许可证 License +## :closed_book: 开源许可证 License This software is licensed under the Apache License 2.0 - ## 🔮 Thanks to our contributors! diff --git a/docs/readme/README_cs.md b/docs/readme/README_cs.md index 63f730a51..c771a55e4 100644 --- a/docs/readme/README_cs.md +++ b/docs/readme/README_cs.md @@ -12,12 +12,11 @@ [![Go Report Card](https://goreportcard.com/badge/github.com/openimsdk/open-im-server?style=for-the-badge)](https://goreportcard.com/report/github.com/openimsdk/open-im-server) [![Go Reference](https://img.shields.io/badge/Go%20Reference-blue.svg?style=for-the-badge&logo=go&logoColor=white)](https://pkg.go.dev/github.com/openimsdk/open-im-server/v3) [![License](https://img.shields.io/badge/license-Apache--2.0-green?style=for-the-badge)](https://github.com/openimsdk/open-im-server/blob/main/LICENSE) -[![Slack](https://img.shields.io/badge/Slack-500%2B-blueviolet?style=for-the-badge&logo=slack&logoColor=white)](https://join.slack.com/t/openimsdk/shared_invite/zt-22720d66b-o_FvKxMTGXtcnnnHiMqe9Q) +[![Slack](https://img.shields.io/badge/Slack-500%2B-blueviolet?style=for-the-badge&logo=slack&logoColor=white)](https://join.slack.com/t/openimsdk/shared_invite/zt-2ijy1ys1f-O0aEDCr7ExRZ7mwsHAVg9A) [![Best Practices](https://img.shields.io/badge/Best%20Practices-purple?style=for-the-badge)](https://www.bestpractices.dev/projects/8045) [![Good First Issues](https://img.shields.io/github/issues/openimsdk/open-im-server/good%20first%20issue?style=for-the-badge&logo=github)](https://github.com/openimsdk/open-im-server/issues?q=is%3Aissue+is%3Aopen+sort%3Aupdated-desc+label%3A%22good+first+issue%22) [![Language](https://img.shields.io/badge/Language-Go-blue.svg?style=for-the-badge&logo=go&logoColor=white)](https://golang.org/) -

English · 中文 · @@ -46,7 +45,6 @@ Türkçe

-

@@ -61,14 +59,14 @@ OpenIM je platforma služeb speciálně navržená pro integraci chatu, audio-vi **OpenIMSDK** je IM SDK navržený pro**OpenIMServer**, vytvořený speciálně pro vkládání do klientských aplikací. Jeho hlavní vlastnosti a moduly jsou následující: -+ 🌟 Hlavní vlastnosti: +- 🌟 Hlavní vlastnosti: - 📦 Místní úložiště - 🔔 Zpětná volání posluchačů - 🛡️ API obalování - 🌐 Správa připojení -+ 📚 hlavní moduly: +- 📚 hlavní moduly: 1. 🚀 Inicializace a přihlášení 2. 👤 Správa uživatelů @@ -82,15 +80,15 @@ Je postaven pomocí Golang a podporuje nasazení napříč platformami, což zaj ## 🌐 O OpenIMServeru -+ **OpenIMServer** má následující vlastnosti: +- **OpenIMServer** má následující vlastnosti: - 🌐 Architektura mikroslužeb: Podporuje režim clusteru, včetně brány a více služeb RPC. - 🚀 Různé metody nasazení: Podporuje nasazení prostřednictvím zdrojového kódu, Kubernetes nebo Docker. - Podpora masivní uživatelské základny: Super velké skupiny se stovkami tisíc uživatelů, desítkami milionů uživatelů a miliardami zpráv. ### Vylepšené obchodní funkce: -+ **REST API**: OpenIMServer nabízí REST API pro podnikové systémy, jejichž cílem je poskytnout podnikům více funkcí, jako je vytváření skupin a odesílání push zpráv přes backendová rozhraní. -+ **Webhooks**: OpenIMServer poskytuje možnosti zpětného volání pro rozšíření více obchodních formulářů. Zpětné volání znamená, že OpenIMServer odešle požadavek na obchodní server před nebo po určité události, jako jsou zpětná volání před nebo po odeslání zprávy. +- **REST API**: OpenIMServer nabízí REST API pro podnikové systémy, jejichž cílem je poskytnout podnikům více funkcí, jako je vytváření skupin a odesílání push zpráv přes backendová rozhraní. +- **Webhooks**: OpenIMServer poskytuje možnosti zpětného volání pro rozšíření více obchodních formulářů. Zpětné volání znamená, že OpenIMServer odešle požadavek na obchodní server před nebo po určité události, jako jsou zpětná volání před nebo po odeslání zprávy. 👉 **[Další informace](https://docs.openim.io/guides/introduction/product)** @@ -100,7 +98,6 @@ Ponořte se do srdce funkčnosti Open-IM-Server s naším diagramem architektury ![Overall Architecture](../images/architecture-layers.png) - ## :rocket: Rychlý start Podporujeme mnoho platforem. Zde jsou adresy pro rychlou práci na webové stránce: @@ -109,10 +106,10 @@ Podporujeme mnoho platforem. Zde jsou adresy pro rychlou práci na webové strá 🤲 Pro usnadnění uživatelské zkušenosti nabízíme různá řešení nasazení. Způsob nasazení si můžete vybrat ze seznamu níže: -+ **[Průvodce nasazením zdrojového kódu](https://docs.openim.io/guides/gettingStarted/imSourceCodeDeployment)** -+ **[Docker Deployment Guide](https://docs.openim.io/guides/gettingStarted/dockerCompose)** -+ **[Průvodce nasazením Kubernetes](https://docs.openim.io/guides/gettingStarted/k8s-deployment)** -+ **[Průvodce nasazením pro vývojáře Mac](https://docs.openim.io/guides/gettingstarted/mac-deployment-guide)** +- **[Průvodce nasazením zdrojového kódu](https://docs.openim.io/guides/gettingStarted/imSourceCodeDeployment)** +- **[Docker Deployment Guide](https://docs.openim.io/guides/gettingStarted/dockerCompose)** +- **[Průvodce nasazením Kubernetes](https://docs.openim.io/guides/gettingStarted/k8s-deployment)** +- **[Průvodce nasazením pro vývojáře Mac](https://docs.openim.io/guides/gettingstarted/mac-deployment-guide)** ## :hammer_and_wrench: Chcete-li začít vyvíjet OpenIM @@ -122,7 +119,7 @@ OpenIM Naším cílem je vybudovat špičkovou open source komunitu. Máme soubo Pokud byste chtěli přispět do tohoto úložiště Open-IM-Server, přečtěte si naši [dokumentaci pro přispěvatele](https://github.com/openimsdk/open-im-server/blob/main/CONTRIBUTING.md). -Než začnete, ujistěte se, že jsou vaše změny vyžadovány. Nejlepší pro to je vytvořit [nová diskuze](https://github.com/openimsdk/open-im-server/discussions/new/choose) NEBO [Slack Communication](https://join.slack.com/t/openimsdk/shared_invite/zt-22720d66b-o_FvKxMTGXtcnnnHiMqe9Q), nebo pokud narazíte na problém, [nahlásit jej](https://github.com/openimsdk/open-im-server/issues/new/choose) jako první. +Než začnete, ujistěte se, že jsou vaše změny vyžadovány. Nejlepší pro to je vytvořit [nová diskuze](https://github.com/openimsdk/open-im-server/discussions/new/choose) NEBO [Slack Communication](https://join.slack.com/t/openimsdk/shared_invite/zt-2ijy1ys1f-O0aEDCr7ExRZ7mwsHAVg9A), nebo pokud narazíte na problém, [nahlásit jej](https://github.com/openimsdk/open-im-server/issues/new/choose) jako první. - [OpenIM API Reference](https://github.com/openimsdk/open-im-server/tree/main/docs/contrib/api.md) - [Protokolování OpenIM Bash](https://github.com/openimsdk/open-im-server/tree/main/docs/contrib/bash-log.md) @@ -154,19 +151,18 @@ Než začnete, ujistěte se, že jsou vaše změny vyžadovány. Nejlepší pro - [Spravovat backend a monitorovat nasazení](https://github.com/openimsdk/open-im-server/tree/main/docs/contrib/prometheus-grafana.md) - [Průvodce nasazením pro vývojáře Mac pro OpenIM](https://github.com/openimsdk/open-im-server/tree/main/docs/contrib/mac-developer-deployment-guide.md) - ## :busts_in_silhouette: Společenství -+ 📚 [Komunita OpenIM](https://github.com/OpenIMSDK/community) -+ 💕 [Zájmová skupina OpenIM](https://github.com/Openim-sigs) -+ 🚀 [Připojte se k naší komunitě Slack](https://join.slack.com/t/openimsdk/shared_invite/zt-22720d66b-o_FvKxMTGXtcnnnHiMqe9Q) -+ :eyes: [Připojte se k našemu wechatu](https://openim-1253691595.cos.ap-nanjing.myqcloud.com/WechatIMG20.jpeg) +- 📚 [Komunita OpenIM](https://github.com/OpenIMSDK/community) +- 💕 [Zájmová skupina OpenIM](https://github.com/Openim-sigs) +- 🚀 [Připojte se k naší komunitě Slack](https://join.slack.com/t/openimsdk/shared_invite/zt-2ijy1ys1f-O0aEDCr7ExRZ7mwsHAVg9A) +- :eyes: [Připojte se k našemu wechatu](https://openim-1253691595.cos.ap-nanjing.myqcloud.com/WechatIMG20.jpeg) ## :calendar: Komunitní setkání Chceme, aby se do naší komunity a přispívání kódu zapojil kdokoli, nabízíme dárky a odměny a vítáme vás, abyste se k nám připojili každý čtvrtek večer. -Naše konference je v [OpenIM Slack](https://join.slack.com/t/openimsdk/shared_invite/zt-22720d66b-o_FvKxMTGXtcnnnHiMqe9Q) 🎯, pak můžete vyhledat kanál Open-IM-Server a připojit se +Naše konference je v [OpenIM Slack](https://join.slack.com/t/openimsdk/shared_invite/zt-2ijy1ys1f-O0aEDCr7ExRZ7mwsHAVg9A) 🎯, pak můžete vyhledat kanál Open-IM-Server a připojit se Zaznamenáváme si každou [dvoutýdenní schůzku](https://github.com/orgs/OpenIMSDK/discussions/categories/meeting)do [diskuzí na GitHubu](https://github.com/openimsdk/open-im-server/discussions/categories/meeting), naše historické poznámky ze schůzek a také záznamy schůzek jsou k dispozici na [Dokumenty Google :bookmark_tabs:](https://docs.google.com/document/d/1nx8MDpuG74NASx081JcCpxPgDITNTpIIos0DS6Vr9GU/edit?usp=sharing). diff --git a/docs/readme/README_da.md b/docs/readme/README_da.md index 60d97348a..4ed991ef1 100644 --- a/docs/readme/README_da.md +++ b/docs/readme/README_da.md @@ -12,12 +12,11 @@ [![Go Report Card](https://goreportcard.com/badge/github.com/openimsdk/open-im-server?style=for-the-badge)](https://goreportcard.com/report/github.com/openimsdk/open-im-server) [![Go Reference](https://img.shields.io/badge/Go%20Reference-blue.svg?style=for-the-badge&logo=go&logoColor=white)](https://pkg.go.dev/github.com/openimsdk/open-im-server/v3) [![License](https://img.shields.io/badge/license-Apache--2.0-green?style=for-the-badge)](https://github.com/openimsdk/open-im-server/blob/main/LICENSE) -[![Slack](https://img.shields.io/badge/Slack-500%2B-blueviolet?style=for-the-badge&logo=slack&logoColor=white)](https://join.slack.com/t/openimsdk/shared_invite/zt-22720d66b-o_FvKxMTGXtcnnnHiMqe9Q) +[![Slack](https://img.shields.io/badge/Slack-500%2B-blueviolet?style=for-the-badge&logo=slack&logoColor=white)](https://join.slack.com/t/openimsdk/shared_invite/zt-2ijy1ys1f-O0aEDCr7ExRZ7mwsHAVg9A) [![Best Practices](https://img.shields.io/badge/Best%20Practices-purple?style=for-the-badge)](https://www.bestpractices.dev/projects/8045) [![Good First Issues](https://img.shields.io/github/issues/openimsdk/open-im-server/good%20first%20issue?style=for-the-badge&logo=github)](https://github.com/openimsdk/open-im-server/issues?q=is%3Aissue+is%3Aopen+sort%3Aupdated-desc+label%3A%22good+first+issue%22) [![Language](https://img.shields.io/badge/Language-Go-blue.svg?style=for-the-badge&logo=go&logoColor=white)](https://golang.org/) -

English · 中文 · @@ -46,35 +45,30 @@ Türkçe

-

- ## :busts_in_silhouette: Fællesskab -+ 📚 [OpenIM-fællesskab](https://github.com/OpenIMSDK/community) -+ 💕 [OpenIM-interessegruppe](https://github.com/Openim-sigs) -+ 🚀 [Deltag i vores Slack-fællesskab](https://join.slack.com/t/openimsdk/shared_invite/zt-22720d66b-o_FvKxMTGXtcnnnHiMqe9Q) -+ :eyes: [Deltag i vores WeChat (微信群)](https://openim-1253691595.cos.ap-nanjing.myqcloud.com/WechatIMG20.jpeg) -+ 👫 [Deltag i vores Reddit](https://www.reddit.com/r/OpenIMessaging) -+ 💬 [Følg vores Twitter-konto](https://twitter.com/openimsdk) - +- 📚 [OpenIM-fællesskab](https://github.com/OpenIMSDK/community) +- 💕 [OpenIM-interessegruppe](https://github.com/Openim-sigs) +- 🚀 [Deltag i vores Slack-fællesskab](https://join.slack.com/t/openimsdk/shared_invite/zt-2ijy1ys1f-O0aEDCr7ExRZ7mwsHAVg9A) +- :eyes: [Deltag i vores WeChat (微信群)](https://openim-1253691595.cos.ap-nanjing.myqcloud.com/WechatIMG20.jpeg) +- 👫 [Deltag i vores Reddit](https://www.reddit.com/r/OpenIMessaging) +- 💬 [Følg vores Twitter-konto](https://twitter.com/openimsdk) ## Ⓜ️ Om OpenIM OpenIM er en serviceplatform designet specifikt til integration af chat, lyd-videoopkald, notifikationer og AI-chatbots i applikationer. Den tilbyder en række kraftfulde API'er og Webhooks, som gør det let for udviklere at integrere disse interaktive funktioner i deres applikationer. OpenIM er ikke en selvstændig chatapplikation, men fungerer snarere som en platform, der understøtter andre applikationer i at opnå omfattende kommunikationsfunktionaliteter. Følgende diagram illustrerer interaktionen mellem AppServer, AppClient, OpenIMServer og OpenIMSDK for at forklare detaljeret. - - ![App-OpenIM Relationship](../images/oepnim-design.png) ## 🚀 Om OpenIMSDK - **OpenIMSDK** er en IM SDK designet til **OpenIMServer**, skabt specifikt til indlejring i klientapplikationer. Dens vigtigste funktioner og moduler er som følger: +**OpenIMSDK** er en IM SDK designet til **OpenIMServer**, skabt specifikt til indlejring i klientapplikationer. Dens vigtigste funktioner og moduler er som følger: -+ 🌟 Hovedfunktioner: +- 🌟 Hovedfunktioner: - 📦 Lokal lagring - 🔔 Lytter-callbacks @@ -95,15 +89,15 @@ Det er bygget ved hjælp af Golang og understøtter tværplatformsudrulning, hvi ## 🌐 Om OpenIMServer -+ **OpenIMServer** har følgende karakteristika: +- **OpenIMServer** har følgende karakteristika: - 🌐 Mikroservicarkitektur: Understøtter klyngetilstand, inklusive en gateway og flere rpc-tjenester. - 🚀 Forskellige udrulningsmetoder: Understøtter udrulning via kildekode, Kubernetes eller Docker. - Støtte til massiv brugerbase: Super store grupper med hundredtusinder af brugere, titusinder af brugere og milliarder af beskeder. ### Forbedret forretningsfunktionalitet: -+ **REST API**：OpenIMServer tilbyder REST API'er til forretningssystemer, med det formål at give virksomheder flere funktioner, såsom at oprette grupper og sende push-beskeder gennem backend-grænseflader. -+ **Webhooks**：OpenIMServer giver mulighed for callback-funktionalitet for at udvide flere forretningsformer. Et callback betyder, at OpenIMServer sender en anmodning til forretningsserveren før eller efter en bestemt begivenhed, som callbacks før eller efter at have sendt en besked. +- **REST API**：OpenIMServer tilbyder REST API'er til forretningssystemer, med det formål at give virksomheder flere funktioner, såsom at oprette grupper og sende push-beskeder gennem backend-grænseflader. +- **Webhooks**：OpenIMServer giver mulighed for callback-funktionalitet for at udvide flere forretningsformer. Et callback betyder, at OpenIMServer sender en anmodning til forretningsserveren før eller efter en bestemt begivenhed, som callbacks før eller efter at have sendt en besked. 👉 **[Lær mere](https://docs.openim.io/guides/introduction/product)** @@ -121,10 +115,10 @@ Vi understøtter mange platforme. Her er adresserne for hurtig oplevelse på web 🤲 For at lette brugeroplevelsen tilbyder vi forskellige udrulningsløsninger. Du kan vælge din udrulningsmetode fra listen nedenfor: -+ **[Vejledning til udrulning af kildekode](https://docs.openim.io/guides/gettingStarted/imSourceCodeDeployment)** -+ **[Vejledning til Docker-udrulning](https://docs.openim.io/guides/gettingStarted/dockerCompose)** -+ **[Vejledning til Kubernetes-udrulning](https://docs.openim.io/guides/gettingStarted/k8s-deployment)** -+ **[Vejledning til Mac-udviklerudrulning](https://docs.openim.io/guides/gettingstarted/mac-deployment-guide)** +- **[Vejledning til udrulning af kildekode](https://docs.openim.io/guides/gettingStarted/imSourceCodeDeployment)** +- **[Vejledning til Docker-udrulning](https://docs.openim.io/guides/gettingStarted/dockerCompose)** +- **[Vejledning til Kubernetes-udrulning](https://docs.openim.io/guides/gettingStarted/k8s-deployment)** +- **[Vejledning til Mac-udviklerudrulning](https://docs.openim.io/guides/gettingstarted/mac-deployment-guide)** ## :hammer_and_wrench: For at starte udviklingen af OpenIM @@ -134,7 +128,7 @@ OpenIM Vores mål er at bygge et topniveau åben kildekode-fællesskab. Vi har e Hvis du gerne vil bidrage til dette Open-IM-Server-repositorium, bedes du læse vores [dokumentation for bidragydere](https://github.com/openimsdk/open-im-server/blob/main/CONTRIBUTING.md). -Før du starter, skal du sikre dig, at dine ændringer er efterspurgte. Det bedste for det er at oprette en [ny diskussion](https://github.com/openimsdk/open-im-server/discussions/new/choose) ELLER [Slack-kommunikation](https://join.slack.com/t/openimsdk/shared_invite/zt-22720d66b-o_FvKxMTGXtcnnnHiMqe9Q), eller hvis du finder et problem, [rapportere det](https://github.com/openimsdk/open-im-server/issues/new/choose) først. +Før du starter, skal du sikre dig, at dine ændringer er efterspurgte. Det bedste for det er at oprette en [ny diskussion](https://github.com/openimsdk/open-im-server/discussions/new/choose) ELLER [Slack-kommunikation](https://join.slack.com/t/openimsdk/shared_invite/zt-2ijy1ys1f-O0aEDCr7ExRZ7mwsHAVg9A), eller hvis du finder et problem, [rapportere det](https://github.com/openimsdk/open-im-server/issues/new/choose) først. - [OpenIM API-referencer](https://github.com/openimsdk/open-im-server/tree/main/docs/contrib/api.md) - [OpenIM Bash-logging](https://github.com/openimsdk/open-im-server/tree/main/docs/contrib/bash-log.md) @@ -166,12 +160,11 @@ Før du starter, skal du sikre dig, at dine ændringer er efterspurgte. Det beds - [Administrer backend og overvåg udrulning](https://github.com/openimsdk/open-im-server/tree/main/docs/contrib/prometheus-grafana.md) - [Mac-udviklerudrulningsguide for OpenIM](https://github.com/openimsdk/open-im-server/tree/main/docs/contrib/mac-developer-deployment-guide.md) - ## :calendar: Fællesskabsmøder Vi ønsker, at alle involverer sig i vores fællesskab og bidrager med kode, vi tilbyder gaver og belønninger, og vi byder dig velkommen til at deltage hver torsdag aften. -Vores konference er på [OpenIM Slack](https://join.slack.com/t/openimsdk/shared_invite/zt-22720d66b-o_FvKxMTGXtcnnnHiMqe9Q) 🎯, derefter kan du søge Open-IM-Server pipeline for at deltage. +Vores konference er på [OpenIM Slack](https://join.slack.com/t/openimsdk/shared_invite/zt-2ijy1ys1f-O0aEDCr7ExRZ7mwsHAVg9A) 🎯, derefter kan du søge Open-IM-Server pipeline for at deltage. Vi tager [notater](https://github.com/orgs/OpenIMSDK/discussions/categories/meeting) af hvert fjortendages møde i [GitHub-diskussioner](https://github.com/openimsdk/open-im-server/discussions/categories/meeting), Vores historiske mødenotater samt genudsendelser af møderne er tilgængelige på [Google Docs](https://docs.google.com/document/d/1nx8MDpuG74NASx081JcCpxPgDITNTpIIos0DS6Vr9GU/edit?usp=sharing) 📑. @@ -189,4 +182,4 @@ OpenIM-logoet, inklusive dets variationer og animerede versioner, vist i dette r - \ No newline at end of file + diff --git a/docs/readme/README_el.md b/docs/readme/README_el.md index da01fcb47..9d31e8267 100644 --- a/docs/readme/README_el.md +++ b/docs/readme/README_el.md @@ -12,12 +12,11 @@ [![Go Report Card](https://goreportcard.com/badge/github.com/openimsdk/open-im-server?style=for-the-badge)](https://goreportcard.com/report/github.com/openimsdk/open-im-server) [![Go Reference](https://img.shields.io/badge/Go%20Reference-blue.svg?style=for-the-badge&logo=go&logoColor=white)](https://pkg.go.dev/github.com/openimsdk/open-im-server/v3) [![License](https://img.shields.io/badge/license-Apache--2.0-green?style=for-the-badge)](https://github.com/openimsdk/open-im-server/blob/main/LICENSE) -[![Slack](https://img.shields.io/badge/Slack-500%2B-blueviolet?style=for-the-badge&logo=slack&logoColor=white)](https://join.slack.com/t/openimsdk/shared_invite/zt-22720d66b-o_FvKxMTGXtcnnnHiMqe9Q) +[![Slack](https://img.shields.io/badge/Slack-500%2B-blueviolet?style=for-the-badge&logo=slack&logoColor=white)](https://join.slack.com/t/openimsdk/shared_invite/zt-2ijy1ys1f-O0aEDCr7ExRZ7mwsHAVg9A) [![Best Practices](https://img.shields.io/badge/Best%20Practices-purple?style=for-the-badge)](https://www.bestpractices.dev/projects/8045) [![Good First Issues](https://img.shields.io/github/issues/openimsdk/open-im-server/good%20first%20issue?style=for-the-badge&logo=github)](https://github.com/openimsdk/open-im-server/issues?q=is%3Aissue+is%3Aopen+sort%3Aupdated-desc+label%3A%22good+first+issue%22) [![Language](https://img.shields.io/badge/Language-Go-blue.svg?style=for-the-badge&logo=go&logoColor=white)](https://golang.org/) -

English · 中文 · @@ -46,7 +45,6 @@ Türkçe

-

@@ -61,14 +59,14 @@ Το **OpenIMSDK** είναι ένα SDK για αμεση ανταλλαγή μηνυμάτων σχεδιασμένο για το **OpenIMServer**, δημιουργήθηκε ειδικά για ενσωμάτωση σε εφαρμογές πελατών. Οι κύριες δυνατότητες και μονάδες του είναι οι εξής: -+ 🌟 Κύριες Δυνατότητες: +- 🌟 Κύριες Δυνατότητες: - 📦 Τοπική αποθήκευση - 🔔 Callbacks ακροατών - 🛡️ Περιτύλιγμα API - 🌐 Διαχείριση σύνδεσης -+ 📚 Κύριες Μονάδες: +- 📚 Κύριες Μονάδες: 1. 🚀 Αρχικοποίηση και Σύνδεση 2. 👤 Διαχείριση Χρηστών @@ -82,15 +80,15 @@ ## 🌐 Σχετικά με το OpenIMServer -+ Το **OpenIMServer** έχει τις ακόλουθες χαρακτηριστικές: +- Το **OpenIMServer** έχει τις ακόλουθες χαρακτηριστικές: - 🌐 Αρχιτεκτονική μικροϋπηρεσιών: Υποστηρίζει λειτουργία σε σύμπλεγμα, περιλαμβάνοντας έναν πύλη και πολλαπλές υπηρεσίες rpc. - 🚀 Διάφοροι τρόποι ανάπτυξης: Υποστηρίζει ανάπτυξη μέσω πηγαίου κώδικα, Kubernetes, ή Docker. - Υποστήριξη για τεράστια βάση χρηστών: Πολύ μεγάλες ομάδες με εκατοντάδες χιλιάδες χρήστες, δεκάδες εκατομμύρια χρήστες και δισεκατομμύρια μηνύματα. ### Ενισχυμένη Επιχειρηματική Λειτουργικότητα: -+ **REST API**: Το OpenIMServer προσφέρει REST APIs για επιχειρηματικά συστήματα, με στόχο την ενδυνάμωση των επιχειρήσεων με περισσότερες λειτουργικότητες, όπως η δημιουργία ομάδων και η αποστολή μηνυμάτων push μέσω backend διεπαφών. -+ **Webhooks**: Το OpenIMServer παρέχει δυνατότητες επανάκλησης για την επέκταση περισσότερων επιχειρηματικών μορφών. Μια επανάκληση σημαίνει ότι το OpenIMServer στέλνει ένα αίτημα στον επιχειρηματικό διακομιστή πριν ή μετά από ένα συγκεκριμένο γεγονός, όπως επανακλήσεις πριν ή μετά την αποστολή ενός μηνύματος. +- **REST API**: Το OpenIMServer προσφέρει REST APIs για επιχειρηματικά συστήματα, με στόχο την ενδυνάμωση των επιχειρήσεων με περισσότερες λειτουργικότητες, όπως η δημιουργία ομάδων και η αποστολή μηνυμάτων push μέσω backend διεπαφών. +- **Webhooks**: Το OpenIMServer παρέχει δυνατότητες επανάκλησης για την επέκταση περισσότερων επιχειρηματικών μορφών. Μια επανάκληση σημαίνει ότι το OpenIMServer στέλνει ένα αίτημα στον επιχειρηματικό διακομιστή πριν ή μετά από ένα συγκεκριμένο γεγονός, όπως επανακλήσεις πριν ή μετά την αποστολή ενός μηνύματος. 👉 **[Μάθετε περισσότερα](https://docs.openim.io/guides/introduction/product)** @@ -100,7 +98,6 @@ ![Overall Architecture](../../docs/images/architecture-layers.png) - ## :rocket: Γρήγορη Εκκίνηση Υποστηρίζουμε πολλές πλατφόρμες. Εδώ είναι οι διευθύνσεις για γρήγορη εμπειρία στην πλευρά του διαδικτύου: @@ -109,10 +106,10 @@ 🤲 Για να διευκολύνουμε την εμπειρία του χρήστη, προσφέρουμε διάφορες λύσεις ανάπτυξης. Μπορείτε να επιλέξετε τη μέθοδο ανάπτυξης σας από την παρακάτω λίστα: -+ **[Οδηγός Ανάπτυξης Κώδικα Πηγής](https://docs.openim.io/guides/gettingStarted/imSourceCodeDeployment)** -+ **[δηγός Ανάπτυξης μέσω Docker](https://docs.openim.io/guides/gettingStarted/dockerCompose)** -+ **[Οδηγός Ανάπτυξης Kubernetes](https://docs.openim.io/guides/gettingStarted/k8s-deployment)** -+ **[Οδηγός Ανάπτυξης για Αναπτυξιακούς στο Mac](https://docs.openim.io/guides/gettingstarted/mac-deployment-guide)** +- **[Οδηγός Ανάπτυξης Κώδικα Πηγής](https://docs.openim.io/guides/gettingStarted/imSourceCodeDeployment)** +- **[δηγός Ανάπτυξης μέσω Docker](https://docs.openim.io/guides/gettingStarted/dockerCompose)** +- **[Οδηγός Ανάπτυξης Kubernetes](https://docs.openim.io/guides/gettingStarted/k8s-deployment)** +- **[Οδηγός Ανάπτυξης για Αναπτυξιακούς στο Mac](https://docs.openim.io/guides/gettingstarted/mac-deployment-guide)** ## :hammer_and_wrench: Για να Αρχίσετε την Ανάπτυξη του OpenIM @@ -122,7 +119,7 @@ OpenIM Στόχος μας είναι να δημιουργήσουμε μια Εάν θέλετε να συνεισφέρετε σε αυτό το αποθετήριο Open-IM-Server, παρακαλούμε διαβάστε την [τεκμηρίωση συνεισφέροντος](https://github.com/openimsdk/open-im-server/blob/main/CONTRIBUTING.md). -Πριν ξεκινήσετε, παρακαλούμε βεβαιωθείτε ότι οι αλλαγές σας είναι ζητούμενες. Το καλύτερο για αυτό είναι να δημιουργήσετε ένα [νέα συζήτηση](https://github.com/openimsdk/open-im-server/discussions/new/choose) ή [Επικοινωνία Slack](https://join.slack.com/t/openimsdk/shared_invite/zt-22720d66b-o_FvKxMTGXtcnnnHiMqe9Q), ή αν βρείτε ένα ζήτημα, [αναφέρετέ το](https://github.com/openimsdk/open-im-server/issues/new/choose) πρώτα. +Πριν ξεκινήσετε, παρακαλούμε βεβαιωθείτε ότι οι αλλαγές σας είναι ζητούμενες. Το καλύτερο για αυτό είναι να δημιουργήσετε ένα [νέα συζήτηση](https://github.com/openimsdk/open-im-server/discussions/new/choose) ή [Επικοινωνία Slack](https://join.slack.com/t/openimsdk/shared_invite/zt-2ijy1ys1f-O0aEDCr7ExRZ7mwsHAVg9A), ή αν βρείτε ένα ζήτημα, [αναφέρετέ το](https://github.com/openimsdk/open-im-server/issues/new/choose) πρώτα. - [Αναφορά API του OpenIM](https://github.com/openimsdk/open-im-server/tree/main/docs/contrib/api.md) - [Καταγραφή Bash του OpenIM](https://github.com/openimsdk/open-im-server/tree/main/docs/contrib/bash-log.md) @@ -154,28 +151,28 @@ OpenIM Στόχος μας είναι να δημιουργήσουμε μια - [Διαχείριση backend και παρακολούθηση ανάπτυξης](https://github.com/openimsdk/open-im-server/tree/main/docs/contrib/prometheus-grafana.md) - [Οδηγός Ανάπτυξης για Προγραμματιστές Mac του OpenIM](https://github.com/openimsdk/open-im-server/tree/main/docs/contrib/mac-developer-deployment-guide.md) - ## :busts_in_silhouette: Κοινότητα -+ 📚 [Κοινότητα OpenIM](https://github.com/OpenIMSDK/community) -+ 💕 [Ομάδα Ενδιαφέροντος OpenIM](https://github.com/Openim-sigs) -+ 🚀 [Εγγραφείτε στην κοινότητα Slack μας](https://join.slack.com/t/openimsdk/shared_invite/zt-22720d66b-o_FvKxMTGXtcnnnHiMqe9Q) -+ :eyes: [γγραφείτε στην ομάδα μας wechat (微信群)](https://openim-1253691595.cos.ap-nanjing.myqcloud.com/WechatIMG20.jpeg) +- 📚 [Κοινότητα OpenIM](https://github.com/OpenIMSDK/community) +- 💕 [Ομάδα Ενδιαφέροντος OpenIM](https://github.com/Openim-sigs) +- 🚀 [Εγγραφείτε στην κοινότητα Slack μας](https://join.slack.com/t/openimsdk/shared_invite/zt-2ijy1ys1f-O0aEDCr7ExRZ7mwsHAVg9A) +- :eyes: [γγραφείτε στην ομάδα μας wechat (微信群)](https://openim-1253691595.cos.ap-nanjing.myqcloud.com/WechatIMG20.jpeg) ## :calendar: Συναντήσεις της κοινότητας Θέλουμε οποιονδήποτε να εμπλακεί στην κοινότητά μας και να συνεισφέρει κώδικα. Προσφέρουμε δώρα και ανταμοιβές και σας καλωσορίζουμε να μας ενταχθείτε κάθε Πέμπτη βράδυ. -Η διάσκεψή μας είναι στο [OpenIM Slack](https://join.slack.com/t/openimsdk/shared_invite/zt-22720d66b-o_FvKxMTGXtcnnnHiMqe9Q) 🎯, στη συνέχεια μπορείτε να αναζητήσετε τη διαδικασία Open-IM-Server για να συμμετάσχετε +Η διάσκεψή μας είναι στο [OpenIM Slack](https://join.slack.com/t/openimsdk/shared_invite/zt-2ijy1ys1f-O0aEDCr7ExRZ7mwsHAVg9A) 🎯, στη συνέχεια μπορείτε να αναζητήσετε τη διαδικασία Open-IM-Server για να συμμετάσχετε Κάνουμε σημειώσεις για κάθε μια [Σημειώνουμε κάθε διμηνιαία συνάντηση](https://github.com/orgs/OpenIMSDK/discussions/categories/meeting) στις [συζητήσεις του GitHub](https://github.com/openimsdk/open-im-server/discussions/categories/meeting), Οι ιστορικές μας σημειώσεις συναντήσεων, καθώς και οι επαναλήψεις των συναντήσεων είναι διαθέσιμες στο[Έγγραφα της Google :bookmark_tabs:](https://docs.google.com/document/d/1nx8MDpuG74NASx081JcCpxPgDITNTpIIos0DS6Vr9GU/edit?usp=sharing). ## :eyes: Ποιοί Χρησιμοποιούν το OpenIM Ελέγξτε τη σελίδα με τις [μελέτες περίπτωσης χρήσης ](https://github.com/OpenIMSDK/community/blob/main/ADOPTERS.md) μας για μια λίστα των χρηστών του έργου. Μην διστάσετε να αφήσετε ένα[📝σχόλιο](https://github.com/openimsdk/open-im-server/issues/379) και να μοιραστείτε την περίπτωση χρήσης σας. + ## :page_facing_up: Άδεια Χρήσης -Το OpenIM διατίθεται υπό την άδεια Apache 2.0. Δείτε τη [ΑΔΕΙΑ ΧΡΗΣΗΣ](https://github.com/openimsdk/open-im-server/tree/main/LICENSE) για το πλήρες κείμενο της άδειας. +Το OpenIM διατίθεται υπό την άδεια Apache 2.0. Δείτε τη [ΑΔΕΙΑ ΧΡΗΣΗΣ](https://github.com/openimsdk/open-im-server/tree/main/LICENSE) για το πλήρες κείμενο της άδειας. Το λογότυπο του OpenIM, συμπεριλαμβανομένων των παραλλαγών και των κινούμενων εικόνων, που εμφανίζονται σε αυτό το αποθετήριο[OpenIM](https://github.com/openimsdk/open-im-server) υπό τις διευθύνσεις [assets/logo](../../assets/logo) και [assets/logo-gif](../../assets/logo-gif) προστατεύονται από τους νόμους περί πνευματικής ιδιοκτησίας. diff --git a/docs/readme/README_es.md b/docs/readme/README_es.md index f123b85c3..360aeb190 100644 --- a/docs/readme/README_es.md +++ b/docs/readme/README_es.md @@ -12,12 +12,11 @@ [![Go Report Card](https://goreportcard.com/badge/github.com/openimsdk/open-im-server?style=for-the-badge)](https://goreportcard.com/report/github.com/openimsdk/open-im-server) [![Go Reference](https://img.shields.io/badge/Go%20Reference-blue.svg?style=for-the-badge&logo=go&logoColor=white)](https://pkg.go.dev/github.com/openimsdk/open-im-server/v3) [![License](https://img.shields.io/badge/license-Apache--2.0-green?style=for-the-badge)](https://github.com/openimsdk/open-im-server/blob/main/LICENSE) -[![Slack](https://img.shields.io/badge/Slack-500%2B-blueviolet?style=for-the-badge&logo=slack&logoColor=white)](https://join.slack.com/t/openimsdk/shared_invite/zt-22720d66b-o_FvKxMTGXtcnnnHiMqe9Q) +[![Slack](https://img.shields.io/badge/Slack-500%2B-blueviolet?style=for-the-badge&logo=slack&logoColor=white)](https://join.slack.com/t/openimsdk/shared_invite/zt-2ijy1ys1f-O0aEDCr7ExRZ7mwsHAVg9A) [![Best Practices](https://img.shields.io/badge/Best%20Practices-purple?style=for-the-badge)](https://www.bestpractices.dev/projects/8045) [![Good First Issues](https://img.shields.io/github/issues/openimsdk/open-im-server/good%20first%20issue?style=for-the-badge&logo=github)](https://github.com/openimsdk/open-im-server/issues?q=is%3Aissue+is%3Aopen+sort%3Aupdated-desc+label%3A%22good+first+issue%22) [![Language](https://img.shields.io/badge/Language-Go-blue.svg?style=for-the-badge&logo=go&logoColor=white)](https://golang.org/) -

English · 中文 · @@ -46,7 +45,6 @@ Türkçe

-

@@ -61,14 +59,14 @@ OpenIM es una plataforma de servicio diseñada específicamente para integrar ch **OpenIMSDK** es un SDK de mensajería instantánea diseñado para **OpenIMServer**, creado específicamente para su incorporación en aplicaciones cliente. Sus principales características y módulos son los siguientes: -+ 🌟 Características Principales: +- 🌟 Características Principales: - 📦 Almacenamiento local - 🔔 Callbacks de escuchas - 🛡️ Envoltura de API - 🌐 Gestión de conexiones -+ 📚 Módulos Principales: +- 📚 Módulos Principales: 1. 🚀 Inicialización y acceso 2. 👤 Gestión de usuarios @@ -82,17 +80,15 @@ Está construido con Golang y soporta despliegue multiplataforma, asegurando una ## 🌐 Acerca de OpenIMServer -+ **OpenIMServer** tiene las siguientes características: +- **OpenIMServer** tiene las siguientes características: - 🌐 Arquitectura de microservicios: Soporta modo cluster, incluyendo un gateway y múltiples servicios rpc. - 🚀 Métodos de despliegue diversos: Soporta el despliegue a través de código fuente, Kubernetes o Docker. - Soporte para una base de usuarios masiva: Grupos super grandes con cientos de miles de usuarios, decenas de millones de usuarios y miles de millones de mensajes. - - ### Funcionalidad Empresarial Mejorada: -+ **API REST**: OpenIMServer ofrece APIs REST para sistemas empresariales, destinadas a empoderar a las empresas con más funcionalidades, como la creación de grupos y el envío de mensajes push a través de interfaces de backend. -+ **Webhooks**: OpenIMServer proporciona capacidades de callback para extender más formas de negocio. Un callback significa que OpenIMServer envía una solicitud al servidor empresarial antes o después de un cierto evento, como callbacks antes o después de enviar un mensaje. +- **API REST**: OpenIMServer ofrece APIs REST para sistemas empresariales, destinadas a empoderar a las empresas con más funcionalidades, como la creación de grupos y el envío de mensajes push a través de interfaces de backend. +- **Webhooks**: OpenIMServer proporciona capacidades de callback para extender más formas de negocio. Un callback significa que OpenIMServer envía una solicitud al servidor empresarial antes o después de un cierto evento, como callbacks antes o después de enviar un mensaje. 👉 **[Aprende más](https://docs.openim.io/guides/introduction/product)** @@ -102,10 +98,8 @@ Adéntrate en el corazón de la funcionalidad de Open-IM-Server con nuestro diag ![Arquitectura General](../../docs/images/architecture-layers.png) - ## :rocket: Inicio Rápido - :rocket: Inicio Rápido Apoyamos muchas plataformas. Aquí están las direcciones para una experiencia rápida en el lado web: @@ -113,22 +107,21 @@ Apoyamos muchas plataformas. Aquí están las direcciones para una experiencia r 🤲 Para facilitar la experiencia del usuario, ofrecemos varias soluciones de despliegue. Puedes elegir tu método de despliegue de la lista a continuación: -+ **[Guía de Despliegue de Código Fuente](https://docs.openim.io/guides/gettingStarted/imSourceCodeDeployment)** -+ **[Guía de Despliegue con Docker](https://docs.openim.io/guides/gettingStarted/dockerCompose)** -+ **[Guía de Despliegue con Kubernetes](https://docs.openim.io/guides/gettingStarted/k8s-deployment)** -+ **[Guía de Despliegue para Desarrolladores en Mac](https://docs.openim.io/guides/gettingstarted/mac-deployment-guide)** +- **[Guía de Despliegue de Código Fuente](https://docs.openim.io/guides/gettingStarted/imSourceCodeDeployment)** +- **[Guía de Despliegue con Docker](https://docs.openim.io/guides/gettingStarted/dockerCompose)** +- **[Guía de Despliegue con Kubernetes](https://docs.openim.io/guides/gettingStarted/k8s-deployment)** +- **[Guía de Despliegue para Desarrolladores en Mac](https://docs.openim.io/guides/gettingstarted/mac-deployment-guide)** ## :hammer_and_wrench: Para Comenzar a Desarrollar en OpenIM [![Abrir en Contenedor de Desarrollo](https://img.shields.io/static/v1?label=Dev%20Container&message=Open&color=blue&logo=visualstudiocode)](https://vscode.dev/github/openimsdk/open-im-server) -Nuestro objetivo en OpenIM es construir una comunidad de código abierto de nivel superior. Tenemos un conjunto de estándares, +Nuestro objetivo en OpenIM es construir una comunidad de código abierto de nivel superior. Tenemos un conjunto de estándares, en el [repositorio de la Comunidad.](https://github.com/OpenIMSDK/community). Si te gustaría contribuir a este repositorio de Open-IM-Server, por favor lee nuestra [documentación para colaboradores](https://github.com/openimsdk/open-im-server/blob/main/CONTRIBUTING.md). - -Antes de comenzar, asegúrate de que tus cambios sean demandados. Lo mejor para eso es crear una [nueva discusión](https://github.com/openimsdk/open-im-server/discussions/new/choose) O [Comunicación en Slack](https://join.slack.com/t/openimsdk/shared_invite/zt-22720d66b-o_FvKxMTGXtcnnnHiMqe9Q), o si encuentras un problema, [repórtalo](https://github.com/openimsdk/open-im-server/issues/new/choose) primero. +Antes de comenzar, asegúrate de que tus cambios sean demandados. Lo mejor para eso es crear una [nueva discusión](https://github.com/openimsdk/open-im-server/discussions/new/choose) O [Comunicación en Slack](https://join.slack.com/t/openimsdk/shared_invite/zt-2ijy1ys1f-O0aEDCr7ExRZ7mwsHAVg9A), o si encuentras un problema, [repórtalo](https://github.com/openimsdk/open-im-server/issues/new/choose) primero. - [Referencia de API de OpenIM](https://github.com/openimsdk/open-im-server/tree/main/docs/contrib/api.md) - [Registro de Bash de OpenIM](https://github.com/openimsdk/open-im-server/tree/main/docs/contrib/bash-log.md) @@ -160,32 +153,31 @@ Antes de comenzar, asegúrate de que tus cambios sean demandados. Lo mejor para - [Gestión de backend y despliegue de monitoreo](https://github.com/openimsdk/open-im-server/tree/main/docs/contrib/prometheus-grafana.md) - [Guía de Despliegue para Desarrolladores Mac de OpenIM](https://github.com/openimsdk/open-im-server/tree/main/docs/contrib/mac-developer-deployment-guide.md) - ## :busts_in_silhouette: Comunidad -+ 📚 [Comunidad de OpenIM](https://github.com/OpenIMSDK/community) -+ 💕 [Grupo de Interés de OpenIM](https://github.com/Openim-sigs) -+ 🚀 [Únete a nuestra comunidad de Slack](https://join.slack.com/t/openimsdk/shared_invite/zt-22720d66b-o_FvKxMTGXtcnnnHiMqe9Q) -+ :eyes: [Únete a nuestro wechat (微信群)](https://openim-1253691595.cos.ap-nanjing.myqcloud.com/WechatIMG20.jpeg) +- 📚 [Comunidad de OpenIM](https://github.com/OpenIMSDK/community) +- 💕 [Grupo de Interés de OpenIM](https://github.com/Openim-sigs) +- 🚀 [Únete a nuestra comunidad de Slack](https://join.slack.com/t/openimsdk/shared_invite/zt-2ijy1ys1f-O0aEDCr7ExRZ7mwsHAVg9A) +- :eyes: [Únete a nuestro wechat (微信群)](https://openim-1253691595.cos.ap-nanjing.myqcloud.com/WechatIMG20.jpeg) ## :calendar: Reuniones de la Comunidad Queremos que cualquiera se involucre en nuestra comunidad y contribuya con código, ofrecemos regalos y recompensas, y te damos la bienvenida para que te unas a nosotros cada jueves por la noche. -Nuestra conferencia está en [OpenIM Slack](https://join.slack.com/t/openimsdk/shared_invite/zt-22720d66b-o_FvKxMTGXtcnnnHiMqe9Q) 🎯, luego puedes buscar el pipeline de Open-IM-Server para unirte +Nuestra conferencia está en [OpenIM Slack](https://join.slack.com/t/openimsdk/shared_invite/zt-2ijy1ys1f-O0aEDCr7ExRZ7mwsHAVg9A) 🎯, luego puedes buscar el pipeline de Open-IM-Server para unirte Tomamos notas de cada [reunión quincenal](https://github.com/orgs/OpenIMSDK/discussions/categories/meeting) en [discusiones de GitHub](https://github.com/openimsdk/open-im-server/discussions/categories/meeting), Nuestras notas de reuniones históricas, así como las repeticiones de las reuniones están disponibles en [Google Docs :bookmark_tabs:](https://docs.google.com/document/d/1nx8MDpuG74NASx081JcCpxPgDITNTpIIos0DS6Vr9GU/edit?usp=sharing). ## :eyes: Quiénes Están Usando OpenIM Consulta nuestros [estudios de caso de usuarios](https://github.com/OpenIMSDK/community/blob/main/ADOPTERS.md) página para obtener una lista de los usuarios del proyecto. No dudes en dejar un [📝comentario](https://github.com/openimsdk/open-im-server/issues/379) y compartir tu caso de uso. -## :page_facing_up: Licencia +## :page_facing_up: Licencia -OpenIM está bajo la licencia Apache 2.0. Consulta [LICENSE](https://github.com/openimsdk/open-im-server/tree/main/LICENSE) para ver el texto completo de la licencia. - +OpenIM está bajo la licencia Apache 2.0. Consulta [LICENSE](https://github.com/openimsdk/open-im-server/tree/main/LICENSE) para ver el texto completo de la licencia. El logotipo de OpenIM, incluyendo sus variaciones y versiones animadas, que se muestran en este repositorio [OpenIM](https://github.com/openimsdk/open-im-server) en los directorios [assets/logo](../../assets/logo) y [assets/logo-gif](assets/logo-gif) están protegidos por las leyes de derechos de autor. + ## 🔮 iGracias a nuestros colaboradores! diff --git a/docs/readme/README_fa.md b/docs/readme/README_fa.md index 7a1512e84..9b54a14d8 100644 --- a/docs/readme/README_fa.md +++ b/docs/readme/README_fa.md @@ -12,12 +12,11 @@ [![Go Report Card](https://goreportcard.com/badge/github.com/openimsdk/open-im-server?style=for-the-badge)](https://goreportcard.com/report/github.com/openimsdk/open-im-server) [![Go Reference](https://img.shields.io/badge/Go%20Reference-blue.svg?style=for-the-badge&logo=go&logoColor=white)](https://pkg.go.dev/github.com/openimsdk/open-im-server/v3) [![License](https://img.shields.io/badge/license-Apache--2.0-green?style=for-the-badge)](https://github.com/openimsdk/open-im-server/blob/main/LICENSE) -[![Slack](https://img.shields.io/badge/Slack-500%2B-blueviolet?style=for-the-badge&logo=slack&logoColor=white)](https://join.slack.com/t/openimsdk/shared_invite/zt-22720d66b-o_FvKxMTGXtcnnnHiMqe9Q) +[![Slack](https://img.shields.io/badge/Slack-500%2B-blueviolet?style=for-the-badge&logo=slack&logoColor=white)](https://join.slack.com/t/openimsdk/shared_invite/zt-2ijy1ys1f-O0aEDCr7ExRZ7mwsHAVg9A) [![Best Practices](https://img.shields.io/badge/Best%20Practices-purple?style=for-the-badge)](https://www.bestpractices.dev/projects/8045) [![Good First Issues](https://img.shields.io/github/issues/openimsdk/open-im-server/good%20first%20issue?style=for-the-badge&logo=github)](https://github.com/openimsdk/open-im-server/issues?q=is%3Aissue+is%3Aopen+sort%3Aupdated-desc+label%3A%22good+first+issue%22) [![Language](https://img.shields.io/badge/Language-Go-blue.svg?style=for-the-badge&logo=go&logoColor=white)](https://golang.org/) -

English · 中文 · @@ -46,7 +45,6 @@ Türkçe

-

@@ -61,14 +59,14 @@ OpenIM یک پلتفرم خدماتی است که به طور خاص برای ا **OpenIMSDK** یک IM SDK است که برای **OpenIMServer** طراحی شده است که به طور خاص برای جاسازی در برنامه های مشتری ایجاد شده است. ویژگی ها و ماژول های اصلی آن به شرح زیر است: -+ 🌟 ویژگی های اصلی: +- 🌟 ویژگی های اصلی: - 📦 ذخیره سازی محلی - 🔔 پاسخ تماس شنونده - 🛡️ بسته بندی API - 🌐 مدیریت اتصال -+ 📚 ماژول های اصلی: +- 📚 ماژول های اصلی: 1. 🚀 مقداردهی اولیه و ورود 2. 👤 مدیریت کاربر @@ -82,15 +80,15 @@ OpenIM یک پلتفرم خدماتی است که به طور خاص برای ا ## 🌐 درباره OpenIMServer -+ **OpenIMServer** دارای ویژگی های زیر است: +- **OpenIMServer** دارای ویژگی های زیر است: - 🌐 معماری Microservice: از حالت کلاستر، از جمله یک دروازه و چندین سرویس rpc پشتیبانی می کند. - 🚀 روش‌های استقرار متنوع: از استقرار از طریق کد منبع، Kubernetes یا Docker پشتیبانی می‌کند. - پشتیبانی از پایگاه عظیم کاربران: گروه های فوق العاده بزرگ با صدها هزار کاربر، ده ها میلیون کاربر و میلیاردها پیام. ### عملکردهای تجاری پیشرفته: -+ **REST API**: OpenIMServer APIهای REST را برای سیستم‌های تجاری ارائه می‌کند، با هدف توانمندسازی کسب‌وکارها با قابلیت‌های بیشتر، مانند ایجاد گروه‌ها و ارسال پیام‌های فشار از طریق رابط‌های باطنی. -+ **Webhooks**: OpenIMServer قابلیت های پاسخ به تماس را برای گسترش بیشتر فرم های تجاری ارائه می دهد. پاسخ به تماس به این معنی است که OpenIMServer درخواستی را قبل یا بعد از یک رویداد خاص به سرور تجاری ارسال می کند، مانند تماس های قبل یا بعد از ارسال یک پیام. +- **REST API**: OpenIMServer APIهای REST را برای سیستم‌های تجاری ارائه می‌کند، با هدف توانمندسازی کسب‌وکارها با قابلیت‌های بیشتر، مانند ایجاد گروه‌ها و ارسال پیام‌های فشار از طریق رابط‌های باطنی. +- **Webhooks**: OpenIMServer قابلیت های پاسخ به تماس را برای گسترش بیشتر فرم های تجاری ارائه می دهد. پاسخ به تماس به این معنی است که OpenIMServer درخواستی را قبل یا بعد از یک رویداد خاص به سرور تجاری ارسال می کند، مانند تماس های قبل یا بعد از ارسال یک پیام. 👉 **[بیشتر بدانید](https://docs.openim.io/guides/introduction/product)** @@ -100,7 +98,6 @@ OpenIM یک پلتفرم خدماتی است که به طور خاص برای ا ![Overall Architecture](../images/architecture-layers.png) - ## :rocket: شروع سریع ما از بسیاری از پلتفرم ها پشتیبانی می کنیم. در اینجا آدرس هایی برای تجربه سریع در سمت وب آمده است: @@ -109,10 +106,10 @@ OpenIM یک پلتفرم خدماتی است که به طور خاص برای ا 🤲 برای تسهیل تجربه کاربر، ما راه حل های مختلف استقرار را ارائه می دهیم. می توانید روش استقرار خود را از لیست زیر انتخاب کنید: -+ **[راهنمای استقرار کد منبع](https://docs.openim.io/guides/gettingStarted/imSourceCodeDeployment)** -+ **[راهنمای استقرار داکر](https://docs.openim.io/guides/gettingStarted/dockerCompose)** -+ **[راهنمای استقرار Kubernetes](https://docs.openim.io/guides/gettingStarted/k8s-deployment)** -+ **[راهنمای استقرار توسعه دهنده مک](https://docs.openim.io/guides/gettingstarted/mac-deployment-guide)** +- **[راهنمای استقرار کد منبع](https://docs.openim.io/guides/gettingStarted/imSourceCodeDeployment)** +- **[راهنمای استقرار داکر](https://docs.openim.io/guides/gettingStarted/dockerCompose)** +- **[راهنمای استقرار Kubernetes](https://docs.openim.io/guides/gettingStarted/k8s-deployment)** +- **[راهنمای استقرار توسعه دهنده مک](https://docs.openim.io/guides/gettingstarted/mac-deployment-guide)** ## :hammer_and_wrench: برای شروع توسعه OpenIM @@ -122,7 +119,7 @@ OpenIM هدف ما ایجاد یک جامعه منبع باز سطح بالا ا اگر می‌خواهید در این مخزن Open-IM-Server مشارکت کنید، لطفاً [مستندات مشارکت‌کننده](https://github.com/openimsdk/open-im-server/blob/main/CONTRIBUTING.md) ما را بخوانید. -قبل از شروع، لطفاً مطمئن شوید که تغییرات شما مورد تقاضا هستند. بهترین کار برای آن این است که یک [بحث جدید](https://github.com/openimsdk/open-im-server/discussions/new/choose) یا [ارتباط اسلک](https://join.slack.com/t/openimsdk/shared_invite/zt-22720d66b-o_FvKxMTGXtcnnnHiMqe9Q) ایجاد کنید، یا اگر مشکلی پیدا کردید، ابتدا [آن را گزارش کنید](https://github.com/openimsdk/open-im-server/issues/new/choose). +قبل از شروع، لطفاً مطمئن شوید که تغییرات شما مورد تقاضا هستند. بهترین کار برای آن این است که یک [بحث جدید](https://github.com/openimsdk/open-im-server/discussions/new/choose) یا [ارتباط اسلک](https://join.slack.com/t/openimsdk/shared_invite/zt-2ijy1ys1f-O0aEDCr7ExRZ7mwsHAVg9A) ایجاد کنید، یا اگر مشکلی پیدا کردید، ابتدا [آن را گزارش کنید](https://github.com/openimsdk/open-im-server/issues/new/choose). - [مرجع OpenIM API](https://github.com/openimsdk/open-im-server/tree/main/docs/contrib/api.md) - [OpenIM Bash Logging](https://github.com/openimsdk/open-im-server/tree/main/docs/contrib/bash-log.md) @@ -154,21 +151,20 @@ OpenIM هدف ما ایجاد یک جامعه منبع باز سطح بالا ا - [مدیریت استقرار باطن و نظارت](https://github.com/openimsdk/open-im-server/tree/main/docs/contrib/prometheus-grafana.md) - [راهنمای استقرار توسعه دهنده مک برای OpenIM](https://github.com/openimsdk/open-im-server/tree/main/docs/contrib/mac-developer-deployment-guide.md) - ## :busts_in_silhouette: انجمن -+ 📚 [انجمن OpenIM](https://github.com/OpenIMSDK/community) -+ 💕 [گروه علاقه OpenIM](https://github.com/Openim-sigs) -+ 🚀 [به انجمن Slack ما بپیوندید](https://join.slack.com/t/openimsdk/shared_invite/zt-22720d66b-o_FvKxMTGXtcnnnHiMqe9Q) -+ :eyes: [به وی چت ما بپیوندید](https://openim-1253691595.cos.ap-nanjing.myqcloud.com/WechatIMG20.jpeg) +- 📚 [انجمن OpenIM](https://github.com/OpenIMSDK/community) +- 💕 [گروه علاقه OpenIM](https://github.com/Openim-sigs) +- 🚀 [به انجمن Slack ما بپیوندید](https://join.slack.com/t/openimsdk/shared_invite/zt-2ijy1ys1f-O0aEDCr7ExRZ7mwsHAVg9A) +- :eyes: [به وی چت ما بپیوندید](https://openim-1253691595.cos.ap-nanjing.myqcloud.com/WechatIMG20.jpeg) ## :calendar: جلسات جامعه ما می‌خواهیم هر کسی در انجمن ما مشارکت کند و در کد مشارکت کند، ما هدایا و جوایزی ارائه می‌کنیم، و از شما استقبال می‌کنیم که هر پنجشنبه شب به ما بپیوندید. -کنفرانس ما در [OpenIM Slack](https://join.slack.com/t/openimsdk/shared_invite/zt-22720d66b-o_FvKxMTGXtcnnnHiMqe9Q) 🎯 است، سپس می توانید خط لوله Open-IM-Server را برای پیوستن جستجو کنید. +کنفرانس ما در [OpenIM Slack](https://join.slack.com/t/openimsdk/shared_invite/zt-2ijy1ys1f-O0aEDCr7ExRZ7mwsHAVg9A) 🎯 است، سپس می توانید خط لوله Open-IM-Server را برای پیوستن جستجو کنید. -ما از هر [جلسه دو هفته‌ای](https://github.com/orgs/OpenIMSDK/discussions/categories/meeting) در [بحث‌های GitHub](https://github.com/openimsdk/open-im-server/discussions/categories/meeting) یادداشت‌برداری می‌کنیم، یادداشت‌های جلسه تاریخی ما، و همچنین بازپخش جلسات در [Google Docs :bookmark_tabs:](https://docs.google.com/document/d/1nx8MDpuG74NASx081JcCpxPgDITNTpIIos0DS6Vr9GU/edit?usp=sharing) موجود است. +ما از هر [جلسه دو هفته‌ای](https://github.com/orgs/OpenIMSDK/discussions/categories/meeting) در [بحث‌های GitHub](https://github.com/openimsdk/open-im-server/discussions/categories/meeting) یادداشت‌برداری می‌کنیم، یادداشت‌های جلسه تاریخی ما، و همچنین بازپخش جلسات در [Google Docs :bookmark_tabs:](https://docs.google.com/document/d/1nx8MDpuG74NASx081JcCpxPgDITNTpIIos0DS6Vr9GU/edit?usp=sharing) موجود است. ## :eyes: چه کسانی از OpenIM استفاده می کنند diff --git a/docs/readme/README_fr.md b/docs/readme/README_fr.md index aaf7a9bd4..107fc1e3f 100644 --- a/docs/readme/README_fr.md +++ b/docs/readme/README_fr.md @@ -12,12 +12,11 @@ [![Go Report Card](https://goreportcard.com/badge/github.com/openimsdk/open-im-server?style=for-the-badge)](https://goreportcard.com/report/github.com/openimsdk/open-im-server) [![Go Reference](https://img.shields.io/badge/Go%20Reference-blue.svg?style=for-the-badge&logo=go&logoColor=white)](https://pkg.go.dev/github.com/openimsdk/open-im-server/v3) [![License](https://img.shields.io/badge/license-Apache--2.0-green?style=for-the-badge)](https://github.com/openimsdk/open-im-server/blob/main/LICENSE) -[![Slack](https://img.shields.io/badge/Slack-500%2B-blueviolet?style=for-the-badge&logo=slack&logoColor=white)](https://join.slack.com/t/openimsdk/shared_invite/zt-22720d66b-o_FvKxMTGXtcnnnHiMqe9Q) +[![Slack](https://img.shields.io/badge/Slack-500%2B-blueviolet?style=for-the-badge&logo=slack&logoColor=white)](https://join.slack.com/t/openimsdk/shared_invite/zt-2ijy1ys1f-O0aEDCr7ExRZ7mwsHAVg9A) [![Best Practices](https://img.shields.io/badge/Best%20Practices-purple?style=for-the-badge)](https://www.bestpractices.dev/projects/8045) [![Good First Issues](https://img.shields.io/github/issues/openimsdk/open-im-server/good%20first%20issue?style=for-the-badge&logo=github)](https://github.com/openimsdk/open-im-server/issues?q=is%3Aissue+is%3Aopen+sort%3Aupdated-desc+label%3A%22good+first+issue%22) [![Language](https://img.shields.io/badge/Language-Go-blue.svg?style=for-the-badge&logo=go&logoColor=white)](https://golang.org/) -

English · 中文 · @@ -46,25 +45,21 @@ Türkçe

-

- ## Ⓜ️ À propos de OpenIM OpenIM est une plateforme de services conçue spécifiquement pour intégrer des fonctionnalités de communication telles que le chat, les appels audio et vidéo, les notifications, ainsi que les robots de chat IA dans les applications. Elle offre une série d'API puissantes et de Webhooks, permettant aux développeurs d'incorporer facilement ces caractéristiques interactives dans leurs applications. OpenIM n'est pas en soi une application de chat autonome, mais sert de plateforme supportant d'autres applications pour réaliser des fonctionnalités de communication enrichies. L'image ci-dessous montre les relations d'interaction entre AppServer, AppClient, OpenIMServer et OpenIMSDK pour illustrer spécifiquement. - - ![Relation App-OpenIM](../../images/oepnim-design.png) ## 🚀 À propos de OpenIMSDK -**OpenIMSDK** est un SDK IM conçu pour **OpenIMServer** spécialement créé pour être intégré dans les applications clientes. Ses principales fonctionnalités et modules comprennent : +**OpenIMSDK** est un SDK IM conçu pour **OpenIMServer** spécialement créé pour être intégré dans les applications clientes. Ses principales fonctionnalités et modules comprennent : -+ 🌟 Fonctionnalités clés : +- 🌟 Fonctionnalités clés : - 📦 Stockage local - 🔔 Rappels de l'écouteur @@ -85,15 +80,15 @@ Il est construit avec Golang et supporte le déploiement multiplateforme, assura ## 🌐 À propos de OpenIMServer -+ **OpenIMServer** présente les caractéristiques suivantes ： +- **OpenIMServer** présente les caractéristiques suivantes ： - 🌐 Architecture microservices : prend en charge le mode cluster, incluant le gateway (passerelle) et plusieurs services rpc。 - 🚀 Divers modes de déploiement : supporte le déploiement via le code source, Kubernetes ou Docker。 - Support d'une masse d'utilisateurs : plus de cent mille pour les super grands groupes, des millions d'utilisateurs, et des milliards de messages。 ### Fonctionnalités commerciales améliorées : -+ **REST API**：OpenIMServer fournit une REST API pour les systèmes commerciaux, visant à accorder plus de fonctionnalités, telles que la création de groupes via l'interface backend, l'envoi de messages push, etc。 -+ **Webhooks**：OpenIMServer offre des capacités de rappel pour étendre davantage les formes d'entreprise. Un rappel signifie que OpenIMServer enverra une requête au serveur d'entreprise avant ou après qu'un événement se soit produit, comme un rappel avant ou après l'envoi d'un message。 +- **REST API**：OpenIMServer fournit une REST API pour les systèmes commerciaux, visant à accorder plus de fonctionnalités, telles que la création de groupes via l'interface backend, l'envoi de messages push, etc。 +- **Webhooks**：OpenIMServer offre des capacités de rappel pour étendre davantage les formes d'entreprise. Un rappel signifie que OpenIMServer enverra une requête au serveur d'entreprise avant ou après qu'un événement se soit produit, comme un rappel avant ou après l'envoi d'un message。 👉 **[En savoir plus](https://docs.openim.io/guides/introduction/product)** @@ -103,7 +98,6 @@ Plongez dans le cœur de la fonctionnalité d'Open-IM-Server avec notre diagramm ![Architecture globale](../../images/architecture-layers.png) - ## :rocket: Démarrage rapide Nous prenons en charge de nombreuses plateformes. Voici les adresses pour une expérience rapide du côté web : @@ -112,17 +106,18 @@ Nous prenons en charge de nombreuses plateformes. Voici les adresses pour une ex 🤲 Pour faciliter l'expérience utilisateur, nous proposons plusieurs solutions de déploiement. Vous pouvez choisir votre méthode de déploiement selon la liste ci-dessous ： -+ **[Guide de déploiement du code source](https://docs.openim.io/guides/gettingStarted/imSourceCodeDeployment)** -+ **[Guide de déploiement Docker](https://docs.openim.io/guides/gettingStarted/dockerCompose)** -+ **[Guide de déploiement Kubernetes](https://docs.openim.io/guides/gettingStarted/k8s-deployment)** -+ **[Guide de déploiement pour développeur Mac](https://docs.openim.io/guides/gettingstarted/mac-deployment-guide)** +- **[Guide de déploiement du code source](https://docs.openim.io/guides/gettingStarted/imSourceCodeDeployment)** +- **[Guide de déploiement Docker](https://docs.openim.io/guides/gettingStarted/dockerCompose)** +- **[Guide de déploiement Kubernetes](https://docs.openim.io/guides/gettingStarted/k8s-deployment)** +- **[Guide de déploiement pour développeur Mac](https://docs.openim.io/guides/gettingstarted/mac-deployment-guide)** -## :hammer_and_wrench: Commencer à développer avec OpenIM +## :hammer_and_wrench: Commencer à développer avec OpenIM Chez OpenIM, notre objectif est de construire une communauté open source de premier plan. Nous avons un ensemble de standards, disponibles dans le[ dépôt communautaire](https://github.com/OpenIMSDK/community)。 Si vous souhaitez contribuer à ce dépôt Open-IM-Server, veuillez lire notre[ document pour les contributeurs](https://github.com/openimsdk/open-im-server/blob/main/CONTRIBUTING.md)。 -Avant de commencer, assurez-vous que vos modifications sont nécessaires. La meilleure manière est de créer une[ nouvelle discussion ](https://github.com/openimsdk/open-im-server/discussions/new/choose) ou une [ communication Slack,](https://join.slack.com/t/openimsdk/shared_invite/zt-22720d66b-o_FvKxMTGXtcnnnHiMqe9Q)，ou si vous identifiez un problème, de[ signaler d'abord ](https://github.com/openimsdk/open-im-server/issues/new/choose)。 +Avant de commencer, assurez-vous que vos modifications sont nécessaires. La meilleure manière est de créer une[ nouvelle discussion ](https://github.com/openimsdk/open-im-server/discussions/new/choose) ou une [ communication Slack,](https://join.slack.com/t/openimsdk/shared_invite/zt-2ijy1ys1f-O0aEDCr7ExRZ7mwsHAVg9A)，ou si vous identifiez un problème, de[ signaler d'abord ](https://github.com/openimsdk/open-im-server/issues/new/choose)。 + - [Référence de l'API OpenIM](https://github.com/openimsdk/open-im-server/tree/main/docs/contrib/api.md) - [Journalisation Bash OpenIM](https://github.com/openimsdk/open-im-server/tree/main/docs/contrib/bash-log.md) - [Actions CI/CD OpenIM](https://github.com/openimsdk/open-im-server/tree/main/docs/contrib/cicd-actions.md) @@ -153,13 +148,12 @@ Avant de commencer, assurez-vous que vos modifications sont nécessaires. La mei - [Gérer le déploiement du backend et la surveillance](https://github.com/openimsdk/open-im-server/tree/main/docs/contrib/prometheus-grafana.md) - [Guide de déploiement pour développeur Mac pour OpenIM](https://github.com/openimsdk/open-im-server/tree/main/docs/contrib/mac-developer-deployment-guide.md) - ->## :calendar: Réunions de la Communauté +> ## :calendar: Réunions de la Communauté Nous voulons que tout le monde s'implique dans notre communauté et contribue au code, nous offrons des cadeaux et des récompenses, et nous vous invitons à nous rejoindre chaque jeudi soir. -Notre conférence se trouve dans le [ Slack OpenIM ](https://join.slack.com/t/openimsdk/shared_invite/zt-22720d66b-o_FvKxMTGXtcnnnHiMqe9Q) 🎯, ensuite vous pouvez rechercher le pipeline Open-IM-Server pour rejoindre +Notre conférence se trouve dans le [ Slack OpenIM ](https://join.slack.com/t/openimsdk/shared_invite/zt-2ijy1ys1f-O0aEDCr7ExRZ7mwsHAVg9A) 🎯, ensuite vous pouvez rechercher le pipeline Open-IM-Server pour rejoindre -Nous prenons des notes de chaque [réunion bihebdomadaire ](https://github.com/orgs/OpenIMSDK/discussions/categories/meeting) dans les [discussions GitHub](https://github.com/openimsdk/open-im-server/discussions/categories/meeting), Nos notes de réunion historiques, ainsi que les rediffusions des réunions sont disponibles sur [ Google Docs :bookmark_tabs:](https://docs.google.com/document/d/1nx8MDpuG74NASx081JcCpxPgDITNTpIIos0DS6Vr9GU/edit?usp=sharing). +Nous prenons des notes de chaque [réunion bihebdomadaire ](https://github.com/orgs/OpenIMSDK/discussions/categories/meeting) dans les [discussions GitHub](https://github.com/openimsdk/open-im-server/discussions/categories/meeting), Nos notes de réunion historiques, ainsi que les rediffusions des réunions sont disponibles sur [ Google Docs :bookmark_tabs:](https://docs.google.com/document/d/1nx8MDpuG74NASx081JcCpxPgDITNTpIIos0DS6Vr9GU/edit?usp=sharing). ## :eyes: Qui Utilise OpenIM @@ -167,9 +161,9 @@ Consultez notre page [ études de cas d'utilisateurs ](https://github.com/OpenIM ## :page_facing_up: License -OpenIM est sous licence Apache 2.0. Voir [LICENSE](https://github.com/openimsdk/open-im-server/tree/main/LICENSE) pour le texte complet de la licence. +OpenIM est sous licence Apache 2.0. Voir [LICENSE](https://github.com/openimsdk/open-im-server/tree/main/LICENSE) pour le texte complet de la licence. -Le logo OpenIM, y compris ses variations et versions animées, affiché dans ce dépôt[OpenIM](https://github.com/openimsdk/open-im-server) sous les répertoires [assets/logo](../../assets/logo) et [assets/logo-gif](assets/logo-gif) sont protégés par les lois sur le droit d'auteur. +Le logo OpenIM, y compris ses variations et versions animées, affiché dans ce dépôt[OpenIM](https://github.com/openimsdk/open-im-server) sous les répertoires [assets/logo](../../assets/logo) et [assets/logo-gif](assets/logo-gif) sont protégés par les lois sur le droit d'auteur. ## 🔮 Merci à nos contributeurs ! diff --git a/docs/readme/README_hu.md b/docs/readme/README_hu.md index 61013c334..d6cf340c8 100644 --- a/docs/readme/README_hu.md +++ b/docs/readme/README_hu.md @@ -12,12 +12,11 @@ [![Go Report Card](https://goreportcard.com/badge/github.com/openimsdk/open-im-server?style=for-the-badge)](https://goreportcard.com/report/github.com/openimsdk/open-im-server) [![Go Reference](https://img.shields.io/badge/Go%20Reference-blue.svg?style=for-the-badge&logo=go&logoColor=white)](https://pkg.go.dev/github.com/openimsdk/open-im-server/v3) [![License](https://img.shields.io/badge/license-Apache--2.0-green?style=for-the-badge)](https://github.com/openimsdk/open-im-server/blob/main/LICENSE) -[![Slack](https://img.shields.io/badge/Slack-500%2B-blueviolet?style=for-the-badge&logo=slack&logoColor=white)](https://join.slack.com/t/openimsdk/shared_invite/zt-22720d66b-o_FvKxMTGXtcnnnHiMqe9Q) +[![Slack](https://img.shields.io/badge/Slack-500%2B-blueviolet?style=for-the-badge&logo=slack&logoColor=white)](https://join.slack.com/t/openimsdk/shared_invite/zt-2ijy1ys1f-O0aEDCr7ExRZ7mwsHAVg9A) [![Best Practices](https://img.shields.io/badge/Best%20Practices-purple?style=for-the-badge)](https://www.bestpractices.dev/projects/8045) [![Good First Issues](https://img.shields.io/github/issues/openimsdk/open-im-server/good%20first%20issue?style=for-the-badge&logo=github)](https://github.com/openimsdk/open-im-server/issues?q=is%3Aissue+is%3Aopen+sort%3Aupdated-desc+label%3A%22good+first+issue%22) [![Language](https://img.shields.io/badge/Language-Go-blue.svg?style=for-the-badge&logo=go&logoColor=white)](https://golang.org/) -

English · 中文 · @@ -46,7 +45,6 @@ Türkçe

-

@@ -61,14 +59,14 @@ Az OpenIM egy szolgáltatási platform, amelyet kifejezetten a csevegés, az aud Az **OpenIMSDK** egy **OpenIMServer** számára készült azonnali üzenetküldő SDK, amelyet kifejezetten ügyfélalkalmazásokba való beágyazáshoz hoztak létre. Fő jellemzői és moduljai a következők: -+ 🌟 Főbb jellemzők: +- 🌟 Főbb jellemzők: - 📦 Helyi raktár - 🔔 Hallgatói visszahívások - 🛡️ API-csomagolás - 🌐 Kapcsolatkezelés -+ 📚 Fő modulok: +- 📚 Fő modulok: 1. 🚀 Inicializálás és bejelentkezés 2. 👤 Felhasználókezelés @@ -82,15 +80,15 @@ Golang használatával készült, és támogatja a többplatformos telepítést, ## 🌐 Az OpenIMServerről -+ **OpenIMServer** a következő jellemzőkkel rendelkezik: +- **OpenIMServer** a következő jellemzőkkel rendelkezik: - 🌐 Mikroszolgáltatási architektúra: Támogatja a fürt módot, beleértve az átjárót és több rpc szolgáltatást. - 🚀 Változatos telepítési módszerek: Támogatja a forráskódon, Kubernetesen vagy Dockeren keresztül történő telepítést. - Hatalmas felhasználói bázis támogatása: Szuper nagy csoportok több százezer felhasználóval, több tízmillió felhasználóval és több milliárd üzenettel. ### Továbbfejlesztett üzleti funkcionalitás: -+ **REST API**: Az OpenIMServer REST API-kat kínál az üzleti rendszerek számára, amelyek célja, hogy a vállalkozásokat több funkcióval ruházza fel, mint például csoportok létrehozása és push üzenetek küldése háttérfelületeken keresztül. -+ **Webhooks**: Az OpenIMServer visszahívási lehetőségeket biztosít több üzleti forma kiterjesztéséhez. A visszahívás azt jelenti, hogy az OpenIMServer kérelmet küld az üzleti szervernek egy bizonyos esemény előtt vagy után, például visszahívásokat üzenet küldése előtt vagy után. +- **REST API**: Az OpenIMServer REST API-kat kínál az üzleti rendszerek számára, amelyek célja, hogy a vállalkozásokat több funkcióval ruházza fel, mint például csoportok létrehozása és push üzenetek küldése háttérfelületeken keresztül. +- **Webhooks**: Az OpenIMServer visszahívási lehetőségeket biztosít több üzleti forma kiterjesztéséhez. A visszahívás azt jelenti, hogy az OpenIMServer kérelmet küld az üzleti szervernek egy bizonyos esemény előtt vagy után, például visszahívásokat üzenet küldése előtt vagy után. 👉 **[Tudj meg többet](https://docs.openim.io/guides/introduction/product)** @@ -100,7 +98,6 @@ Merüljön el az Open-IM-Server funkcióinak szívében az architektúra diagram ![Overall Architecture](../images/architecture-layers.png) - ## :rocket: Gyors indítás Számos platformot támogatunk. Íme a címek a gyors weboldali használathoz: @@ -109,10 +106,10 @@ Számos platformot támogatunk. Íme a címek a gyors weboldali használathoz: 🤲 A felhasználói élmény megkönnyítése érdekében különféle telepítési megoldásokat kínálunk. Az alábbi listából választhatja ki a telepítési módot: -+ **[Forráskód-telepítési útmutató](https://docs.openim.io/guides/gettingStarted/imSourceCodeDeployment)** -+ **[Docker telepítési útmutató](https://docs.openim.io/guides/gettingStarted/dockerCompose)** -+ **[Kubernetes telepítési útmutató](https://docs.openim.io/guides/gettingStarted/k8s-deployment)** -+ **[Mac fejlesztői telepítési útmutató](https://docs.openim.io/guides/gettingstarted/mac-deployment-guide)** +- **[Forráskód-telepítési útmutató](https://docs.openim.io/guides/gettingStarted/imSourceCodeDeployment)** +- **[Docker telepítési útmutató](https://docs.openim.io/guides/gettingStarted/dockerCompose)** +- **[Kubernetes telepítési útmutató](https://docs.openim.io/guides/gettingStarted/k8s-deployment)** +- **[Mac fejlesztői telepítési útmutató](https://docs.openim.io/guides/gettingstarted/mac-deployment-guide)** ## :hammer_and_wrench: Az OpenIM fejlesztésének megkezdéséhez @@ -122,7 +119,7 @@ OpenIM Célunk egy felső szintű nyílt forráskódú közösség felépítése Ha hozzá szeretne járulni ehhez az Open-IM-Server adattárhoz, kérjük, olvassa el [közreműködői dokumentációnkat](https://github.com/openimsdk/open-im-server/blob/main/CONTRIBUTING.md). -Mielőtt elkezdené, győződjön meg arról, hogy a változtatásokra van-e igény. Erre a legjobb egy [új beszélgetés](https://github.com/openimsdk/open-im-server/discussions/new/choose) VAGY [Slack Communication](https://join.slack.com/t/openimsdk/shared_invite/zt-22720d66b-o_FvKxMTGXtcnnnHiMqe9Q)létrehozása, vagy ha problémát talál, először [jelentse](https://github.com/openimsdk/open-im-server/issues/new/choose) first. +Mielőtt elkezdené, győződjön meg arról, hogy a változtatásokra van-e igény. Erre a legjobb egy [új beszélgetés](https://github.com/openimsdk/open-im-server/discussions/new/choose) VAGY [Slack Communication](https://join.slack.com/t/openimsdk/shared_invite/zt-2ijy1ys1f-O0aEDCr7ExRZ7mwsHAVg9A)létrehozása, vagy ha problémát talál, először [jelentse](https://github.com/openimsdk/open-im-server/issues/new/choose) first. - [OpenIM API referencia](https://github.com/openimsdk/open-im-server/tree/main/docs/contrib/api.md) - [OpenIM Bash naplózás](https://github.com/openimsdk/open-im-server/tree/main/docs/contrib/bash-log.md) @@ -154,19 +151,18 @@ Mielőtt elkezdené, győződjön meg arról, hogy a változtatásokra van-e ig - [A háttérrendszer kezelése és a telepítés figyelése](https://github.com/openimsdk/open-im-server/tree/main/docs/contrib/prometheus-grafana.md) - [Mac Developer Deployment Guide for OpenIM](https://github.com/openimsdk/open-im-server/tree/main/docs/contrib/mac-developer-deployment-guide.md) - ## :busts_in_silhouette: Közösség -+ 📚 [OpenIM közösség](https://github.com/OpenIMSDK/community) -+ 💕 [OpenIM érdeklődési csoport](https://github.com/Openim-sigs) -+ 🚀 [Csatlakozz a Slack közösségünkhöz](https://join.slack.com/t/openimsdk/shared_invite/zt-22720d66b-o_FvKxMTGXtcnnnHiMqe9Q) -+ :eyes: [Csatlakozz a wechathez](https://openim-1253691595.cos.ap-nanjing.myqcloud.com/WechatIMG20.jpeg) +- 📚 [OpenIM közösség](https://github.com/OpenIMSDK/community) +- 💕 [OpenIM érdeklődési csoport](https://github.com/Openim-sigs) +- 🚀 [Csatlakozz a Slack közösségünkhöz](https://join.slack.com/t/openimsdk/shared_invite/zt-2ijy1ys1f-O0aEDCr7ExRZ7mwsHAVg9A) +- :eyes: [Csatlakozz a wechathez](https://openim-1253691595.cos.ap-nanjing.myqcloud.com/WechatIMG20.jpeg) ## :calendar: Közösségi Találkozók Szeretnénk, ha bárki bekapcsolódna közösségünkbe és hozzájárulna kódunkhoz, ajándékokat és jutalmakat kínálunk, és szeretettel várjuk, hogy csatlakozzon hozzánk minden csütörtök este. -Konferenciánk az [OpenIM Slack](https://join.slack.com/t/openimsdk/shared_invite/zt-22720d66b-o_FvKxMTGXtcnnnHiMqe9Q) 🎯alatt van, akkor kereshet az Open-IM-Server folyamatban a csatlakozáshoz +Konferenciánk az [OpenIM Slack](https://join.slack.com/t/openimsdk/shared_invite/zt-2ijy1ys1f-O0aEDCr7ExRZ7mwsHAVg9A) 🎯alatt van, akkor kereshet az Open-IM-Server folyamatban a csatlakozáshoz A [GitHub-beszélgetések](https://github.com/orgs/OpenIMSDK/discussions/categories/meeting)minden [kéthetente történő megbeszélésről](https://github.com/openimsdk/open-im-server/discussions/categories/meeting) jegyzeteket készítünk. A találkozók történeti feljegyzései, valamint az értekezletek visszajátszásai a [Google Dokumentumok :bookmark_tabs:](https://docs.google.com/document/d/1nx8MDpuG74NASx081JcCpxPgDITNTpIIos0DS6Vr9GU/edit?usp=sharing) webhelyen érhetők el. diff --git a/docs/readme/README_ja.md b/docs/readme/README_ja.md index 5a083c1bf..3c633e102 100644 --- a/docs/readme/README_ja.md +++ b/docs/readme/README_ja.md @@ -12,12 +12,11 @@ [![Go Report Card](https://goreportcard.com/badge/github.com/openimsdk/open-im-server?style=for-the-badge)](https://goreportcard.com/report/github.com/openimsdk/open-im-server) [![Go Reference](https://img.shields.io/badge/Go%20Reference-blue.svg?style=for-the-badge&logo=go&logoColor=white)](https://pkg.go.dev/github.com/openimsdk/open-im-server/v3) [![License](https://img.shields.io/badge/license-Apache--2.0-green?style=for-the-badge)](https://github.com/openimsdk/open-im-server/blob/main/LICENSE) -[![Slack](https://img.shields.io/badge/Slack-500%2B-blueviolet?style=for-the-badge&logo=slack&logoColor=white)](https://join.slack.com/t/openimsdk/shared_invite/zt-22720d66b-o_FvKxMTGXtcnnnHiMqe9Q) +[![Slack](https://img.shields.io/badge/Slack-500%2B-blueviolet?style=for-the-badge&logo=slack&logoColor=white)](https://join.slack.com/t/openimsdk/shared_invite/zt-2ijy1ys1f-O0aEDCr7ExRZ7mwsHAVg9A) [![Best Practices](https://img.shields.io/badge/Best%20Practices-purple?style=for-the-badge)](https://www.bestpractices.dev/projects/8045) [![Good First Issues](https://img.shields.io/github/issues/openimsdk/open-im-server/good%20first%20issue?style=for-the-badge&logo=github)](https://github.com/openimsdk/open-im-server/issues?q=is%3Aissue+is%3Aopen+sort%3Aupdated-desc+label%3A%22good+first+issue%22) [![Language](https://img.shields.io/badge/Language-Go-blue.svg?style=for-the-badge&logo=go&logoColor=white)](https://golang.org/) -

English · 中文 · @@ -46,29 +45,25 @@ Türkçe

-

+## Ⓜ️ OpenIM について -## Ⓜ️ OpenIMについて - -OpenIMは、アプリケーション内でチャット、音声通話、通知、AIチャットボットなどの通信機能を統合するために特別に設計されたサービスプラットフォームです。一連の強力なAPIとWebhooksを提供することで、開発者はアプリケーションに簡単にこれらの通信機能を統合できます。OpenIM自体は独立したチャットアプリではなく、アプリケーションにサポートを提供し、豊富な通信機能を実現するプラットフォームです。以下の図は、AppServer、AppClient、OpenIMServer、OpenIMSDK間の相互作用を示しています。 - - +OpenIM は、アプリケーション内でチャット、音声通話、通知、AI チャットボットなどの通信機能を統合するために特別に設計されたサービスプラットフォームです。一連の強力な API と Webhooks を提供することで、開発者はアプリケーションに簡単にこれらの通信機能を統合できます。OpenIM 自体は独立したチャットアプリではなく、アプリケーションにサポートを提供し、豊富な通信機能を実現するプラットフォームです。以下の図は、AppServer、AppClient、OpenIMServer、OpenIMSDK 間の相互作用を示しています。 ![App-OpenIM Relationship](../images/oepnim-design.png) -## 🚀 OpenIMSDKについて +## 🚀 OpenIMSDK について - **OpenIMSDK**は、**OpenIMServer**用に設計されたIM SDKで、クライアントアプリケーションに組み込むためのものです。主な機能とモジュールは以下の通りです： +**OpenIMSDK**は、**OpenIMServer**用に設計された IM SDK で、クライアントアプリケーションに組み込むためのものです。主な機能とモジュールは以下の通りです： -+ 🌟 主な機能： +- 🌟 主な機能： - - 📦 ローカルストレージ + - 📦 ローカルストレージ - 🔔 リスナーコールバック - - 🛡️ APIのラッピング + - 🛡️ API のラッピング - 🌐 接続管理 ## 📚 主なモジュール： @@ -79,54 +74,54 @@ OpenIMは、アプリケーション内でチャット、音声通話、通知 4. 🤖 グループ機能 5. 💬 会話処理 -Golangを使用して構築され、クロスプラットフォームの導入をサポートし、すべてのプラットフォームで一貫したアクセス体験を提供します。 +Golang を使用して構築され、クロスプラットフォームの導入をサポートし、すべてのプラットフォームで一貫したアクセス体験を提供します。 -👉 **[GO SDKを探索する](https://github.com/openimsdk/openim-sdk-core)** +👉 **[GO SDK を探索する](https://github.com/openimsdk/openim-sdk-core)** -## 🌐 OpenIMServerについて +## 🌐 OpenIMServer について -+ **OpenIMServer** には以下の特徴があります： - - 🌐 マイクロサービスアーキテクチャ：クラスターモードをサポートし、ゲートウェイ（gateway）と複数のrpcサービスを含みます。 - - 🚀 多様なデプロイメント方法：ソースコード、kubernetes、またはdockerでのデプロイメントをサポートします。 +- **OpenIMServer** には以下の特徴があります： + - 🌐 マイクロサービスアーキテクチャ：クラスターモードをサポートし、ゲートウェイ（gateway）と複数の rpc サービスを含みます。 + - 🚀 多様なデプロイメント方法：ソースコード、kubernetes、または docker でのデプロイメントをサポートします。 - 海量ユーザーサポート：十万人規模の超大型グループ、千万人のユーザー、および百億のメッセージ ### 強化されたビジネス機能： -+ **REST API**：OpenIMServerは、ビジネスシステム用のREST APIを提供しており、ビジネスにさらに多くの機能を提供することを目指しています。たとえば、バックエンドインターフェースを通じてグループを作成したり、プッシュメッセージを送信したりするなどです。 -+ **Webhooks**：OpenIMServerは、より多くのビジネス形態を拡張するためのコールバック機能を提供しています。コールバックとは、特定のイベントが発生する前後に、OpenIMServerがビジネスサーバーにリクエストを送信することを意味します。例えば、メッセージ送信の前後のコールバックなどです。 +- **REST API**：OpenIMServer は、ビジネスシステム用の REST API を提供しており、ビジネスにさらに多くの機能を提供することを目指しています。たとえば、バックエンドインターフェースを通じてグループを作成したり、プッシュメッセージを送信したりするなどです。 +- **Webhooks**：OpenIMServer は、より多くのビジネス形態を拡張するためのコールバック機能を提供しています。コールバックとは、特定のイベントが発生する前後に、OpenIMServer がビジネスサーバーにリクエストを送信することを意味します。例えば、メッセージ送信の前後のコールバックなどです。 👉 **[もっと詳しく知る](https://docs.openim.io/guides/introduction/product)** ## :building_construction: 全体のアーキテクチャ -Open-IM-Serverの機能の核心に迫るために、アーキテクチャダイアグラムをご覧ください。 +Open-IM-Server の機能の核心に迫るために、アーキテクチャダイアグラムをご覧ください。 ![Overall Architecture](../images/architecture-layers.png) ## :rocket: クイックスタート -iOS/Android/H5/PC/Webでのオンライン体験： +iOS/Android/H5/PC/Web でのオンライン体験： 👉 **[OpenIM online demo](https://www.openim.io/zh/commercial)** 🤲 ユーザー体験を容易にするために、私たちは様々なデプロイメントソリューションを提供しています。以下のリストから、ご自身のデプロイメント方法を選択できます： -+ **[ソースコードデプロイメントガイド](https://docs.openim.io/guides/gettingStarted/imSourceCodeDeployment)** -+ **[Docker デプロイメントガイド](https://docs.openim.io/guides/gettingStarted/dockerCompose)** -+ **[Kubernetes デプロイメントガイド](https://docs.openim.io/guides/gettingStarted/k8s-deployment)** -+ **[Mac 開発者向けデプロイメントガイド](https://docs.openim.io/guides/gettingstarted/mac-deployment-guide)** +- **[ソースコードデプロイメントガイド](https://docs.openim.io/guides/gettingStarted/imSourceCodeDeployment)** +- **[Docker デプロイメントガイド](https://docs.openim.io/guides/gettingStarted/dockerCompose)** +- **[Kubernetes デプロイメントガイド](https://docs.openim.io/guides/gettingStarted/k8s-deployment)** +- **[Mac 開発者向けデプロイメントガイド](https://docs.openim.io/guides/gettingstarted/mac-deployment-guide)** -## :hammer_and_wrench: OpenIMの開発を始める +## :hammer_and_wrench: OpenIM の開発を始める [![Open in Dev Container](https://img.shields.io/static/v1?label=Dev%20Container&message=Open&color=blue&logo=visualstudiocode)](https://vscode.dev/github/openimsdk/open-im-server) OpenIM 私たちの目標は、トップレベルのオープンソースコミュニティを構築することです。[コミュニティリポジトリ](https://github.com/OpenIMSDK/community)には一連の基準があります。 -このOpen-IM-Serverリポジトリに貢献したい場合は、[貢献者ドキュメントをお読みください](https://github.com/openimsdk/open-im-server/blob/main/CONTRIBUTING.md)。 +この Open-IM-Server リポジトリに貢献したい場合は、[貢献者ドキュメントをお読みください](https://github.com/openimsdk/open-im-server/blob/main/CONTRIBUTING.md)。 -始める前に、変更に必要があることを確認してください。最良の方法は、[新しいディスカッション](https://github.com/openimsdk/open-im-server/discussions/new/choose)や[Slack](https://join.slack.com/t/openimsdk/shared_invite/zt-22720d66b-o_FvKxMTGXtcnnnHiMqe9Q)での通信を作成すること、または問題を発見した場合は、まずそれを[報告](https://github.com/openimsdk/open-im-server/issues/new/choose)することです。 +始める前に、変更に必要があることを確認してください。最良の方法は、[新しいディスカッション](https://github.com/openimsdk/open-im-server/discussions/new/choose)や[Slack](https://join.slack.com/t/openimsdk/shared_invite/zt-2ijy1ys1f-O0aEDCr7ExRZ7mwsHAVg9A)での通信を作成すること、または問題を発見した場合は、まずそれを[報告](https://github.com/openimsdk/open-im-server/issues/new/choose)することです。 -- [OpenIM APIリファレンス](https://github.com/openimsdk/open-im-server/tree/main/docs/contrib/api.md) +- [OpenIM API リファレンス](https://github.com/openimsdk/open-im-server/tree/main/docs/contrib/api.md) - [OpenIM Bash ロギング](https://github.com/openimsdk/open-im-server/tree/main/docs/contrib/bash-log.md) - [OpenIM CI/CD アクション](https://github.com/openimsdk/open-im-server/tree/main/docs/contrib/cicd-actions.md) - [OpenIM コード規約](https://github.com/openimsdk/open-im-server/tree/main/docs/contrib/code-conventions.md) @@ -149,37 +144,35 @@ OpenIM 私たちの目標は、トップレベルのオープンソースコミ - [OpenIM オフラインデプロイメント](https://github.com/openimsdk/open-im-server/tree/main/docs/contrib/offline-deployment.md) - [OpenIM Protoc ツール](https://github.com/openimsdk/open-im-server/tree/main/docs/contrib/protoc-tools.md) - [OpenIM テスティングガイド](https://github.com/openimsdk/open-im-server/tree/main/docs/contrib/test.md) -- [OpenIM ユーティリティGo](https://github.com/openimsdk/open-im-server/tree/main/docs/contrib/util-go.md) +- [OpenIM ユーティリティ Go](https://github.com/openimsdk/open-im-server/tree/main/docs/contrib/util-go.md) - [OpenIM Makefile ユーティリティ](https://github.com/openimsdk/open-im-server/tree/main/docs/contrib/util-makefile.md) - [OpenIM スクリプトユーティリティ](https://github.com/openimsdk/open-im-server/tree/main/docs/contrib/util-scripts.md) - [OpenIM バージョニング](https://github.com/openimsdk/open-im-server/tree/main/docs/contrib/version.md) - [バックエンド管理とモニターデプロイメント](https://github.com/openimsdk/open-im-server/tree/main/docs/contrib/prometheus-grafana.md) -- [OpenIM用Mac開発者デプロイメントガイド](https://github.com/openimsdk/open-im-server/tree/main/docs/contrib/mac-developer-deployment-guide.md) - +- [OpenIM 用 Mac 開発者デプロイメントガイド](https://github.com/openimsdk/open-im-server/tree/main/docs/contrib/mac-developer-deployment-guide.md) ## :busts_in_silhouette: コミュニティ -+ 📚 [OpenIM コミュニティ](https://github.com/OpenIMSDK/community) -+ 💕 [OpenIM 興味グループ](https://github.com/Openim-sigs) -+ 🚀 [私たちのSlackコミュニティに参加する](https://join.slack.com/t/openimsdk/shared_invite/zt-22720d66b-o_FvKxMTGXtcnnnHiMqe9Q) -+ :eyes: [私たちのWeChat（微信群）に参加する](https://openim-1253691595.cos.ap-nanjing.myqcloud.com/WechatIMG20.jpeg) +- 📚 [OpenIM コミュニティ](https://github.com/OpenIMSDK/community) +- 💕 [OpenIM 興味グループ](https://github.com/Openim-sigs) +- 🚀 [私たちの Slack コミュニティに参加する](https://join.slack.com/t/openimsdk/shared_invite/zt-2ijy1ys1f-O0aEDCr7ExRZ7mwsHAVg9A) +- :eyes: [私たちの WeChat（微信群）に参加する](https://openim-1253691595.cos.ap-nanjing.myqcloud.com/WechatIMG20.jpeg) ## :calendar: コミュニティミーティング 私たちは、誰もがコミュニティに参加し、コードに貢献してもらいたいと考えています。私たちは、ギフトや報酬を提供し、毎週木曜日の夜に参加していただくことを歓迎します。 -私たちの会議は[OpenIM Slack](https://join.slack.com/t/openimsdk/shared_invite/zt-22720d66b-o_FvKxMTGXtcnnnHiMqe9Q)🎯で行われます。そこでOpen-IM-Serverパイプラインを検索して参加できます。 - +私たちの会議は[OpenIM Slack](https://join.slack.com/t/openimsdk/shared_invite/zt-2ijy1ys1f-O0aEDCr7ExRZ7mwsHAVg9A)🎯 で行われます。そこで Open-IM-Server パイプラインを検索して参加できます。 -私たちは[隔週の会議](https://github.com/orgs/OpenIMSDK/discussions/categories/meeting)のメモを[GitHubディスカッション](https://github.com/openimsdk/open-im-server/discussions/categories/meeting)に記録しています。歴史的な会議のメモや会議のリプレイは[Google Docs📑](https://docs.google.com/document/d/1nx8MDpuG74NASx081JcCpxPgDITNTpIIos0DS6Vr9GU/edit?usp=sharing)で利用可能です。 +私たちは[隔週の会議](https://github.com/orgs/OpenIMSDK/discussions/categories/meeting)のメモを[GitHub ディスカッション](https://github.com/openimsdk/open-im-server/discussions/categories/meeting)に記録しています。歴史的な会議のメモや会議のリプレイは[Google Docs📑](https://docs.google.com/document/d/1nx8MDpuG74NASx081JcCpxPgDITNTpIIos0DS6Vr9GU/edit?usp=sharing)で利用可能です。 -## :eyes: OpenIMを使用している人たち +## :eyes: OpenIM を使用している人たち -プロジェクトユーザーのリストについては、[ユーザーケーススタディ](https://github.com/OpenIMSDK/community/blob/main/ADOPTERS.md)ページをご覧ください。[コメント📝](https://github.com/openimsdk/open-im-server/issues/379)を残して、あなたの使用例を共有することを躊躇しないでください。 +プロジェクトユーザーのリストについては、[ユーザーケーススタディ](https://github.com/OpenIMSDK/community/blob/main/ADOPTERS.md)ページをご覧ください。[コメント 📝](https://github.com/openimsdk/open-im-server/issues/379)を残して、あなたの使用例を共有することを躊躇しないでください。 ## :page_facing_up: ライセンス -OpenIMはApache 2.0ライセンスの下でライセンスされています。完全なライセンステキストについては、[LICENSE](https://github.com/openimsdk/open-im-server/tree/main/LICENSE)を参照してください。 +OpenIM は Apache 2.0 ライセンスの下でライセンスされています。完全なライセンステキストについては、[LICENSE](https://github.com/openimsdk/open-im-server/tree/main/LICENSE)を参照してください。 このリポジトリに表示される[OpenIM](https://github.com/openimsdk/open-im-server)ロゴ、そのバリエーション、およびアニメーションバージョン（[assets/logo](./assets/logo)および[assets/logo-gif](assets/logo-gif)ディレクトリ内）は、著作権法によって保護されています。 @@ -187,4 +180,4 @@ OpenIMはApache 2.0ライセンスの下でライセンスされています。 - \ No newline at end of file + diff --git a/docs/readme/README_ko.md b/docs/readme/README_ko.md index ebcdd71ee..9b61e260d 100644 --- a/docs/readme/README_ko.md +++ b/docs/readme/README_ko.md @@ -12,12 +12,11 @@ [![Go Report Card](https://goreportcard.com/badge/github.com/openimsdk/open-im-server?style=for-the-badge)](https://goreportcard.com/report/github.com/openimsdk/open-im-server) [![Go Reference](https://img.shields.io/badge/Go%20Reference-blue.svg?style=for-the-badge&logo=go&logoColor=white)](https://pkg.go.dev/github.com/openimsdk/open-im-server/v3) [![License](https://img.shields.io/badge/license-Apache--2.0-green?style=for-the-badge)](https://github.com/openimsdk/open-im-server/blob/main/LICENSE) -[![Slack](https://img.shields.io/badge/Slack-500%2B-blueviolet?style=for-the-badge&logo=slack&logoColor=white)](https://join.slack.com/t/openimsdk/shared_invite/zt-22720d66b-o_FvKxMTGXtcnnnHiMqe9Q) +[![Slack](https://img.shields.io/badge/Slack-500%2B-blueviolet?style=for-the-badge&logo=slack&logoColor=white)](https://join.slack.com/t/openimsdk/shared_invite/zt-2ijy1ys1f-O0aEDCr7ExRZ7mwsHAVg9A) [![Best Practices](https://img.shields.io/badge/Best%20Practices-purple?style=for-the-badge)](https://www.bestpractices.dev/projects/8045) [![Good First Issues](https://img.shields.io/github/issues/openimsdk/open-im-server/good%20first%20issue?style=for-the-badge&logo=github)](https://github.com/openimsdk/open-im-server/issues?q=is%3Aissue+is%3Aopen+sort%3Aupdated-desc+label%3A%22good+first+issue%22) [![Language](https://img.shields.io/badge/Language-Go-blue.svg?style=for-the-badge&logo=go&logoColor=white)](https://golang.org/) -

English · 中文 · @@ -46,26 +45,21 @@ Türkçe

-

- ## Ⓜ️ OpenIM에 대하여 OpenIM은 채팅, 오디오-비디오 통화, 알림 및 AI 챗봇을 애플리케이션에 통합하기 위해 특별히 설계된 서비스 플랫폼입니다. 이 플랫폼은 강력한 API와 웹훅을 제공하여 개발자가 이러한 상호작용 기능을 애플리케이션에 쉽게 통합할 수 있게 합니다. OpenIM은 독립 실행형 채팅 애플리케이션이 아니라, 다른 애플리케이션들이 풍부한 커뮤니케이션 기능을 달성할 수 있도록 지원하는 플랫폼으로서의 역할을 합니다. 다음 다이어그램은 AppServer, AppClient, OpenIMServer, 및 OpenIMSDK 간의 상호작용을 자세히 설명하기 위해 제시되었습니다. - - ![App-OpenIM Relationship](../images/oepnim-design.png) ## 🚀 OpenIMSDK에 대하여 - **OpenIMSDK**는**OpenIMServer**를 위해 특별히 제작된 IM SDK로, 클라이언트 애플리케이션 내에 내장하기 위해 설계되었습니다. 그 주요 기능 및 모듈은 다음과 같습니다: +**OpenIMSDK**는**OpenIMServer**를 위해 특별히 제작된 IM SDK로, 클라이언트 애플리케이션 내에 내장하기 위해 설계되었습니다. 그 주요 기능 및 모듈은 다음과 같습니다: - -+ 🌟 주요 기능: +- 🌟 주요 기능: - 📦 로컬 스토리지 - 🔔 리스너 콜백 @@ -86,15 +80,15 @@ OpenIM은 채팅, 오디오-비디오 통화, 알림 및 AI 챗봇을 애플리 ## 🌐 OpenIMServer에 대하여 -+ **OpenIMServer** 는 다음과 같은 특성을 가지고 있습니다: +- **OpenIMServer** 는 다음과 같은 특성을 가지고 있습니다: - 🌐 마이크로서비스 아키텍처: 게이트웨이 및 다수의 rpc 서비스를 포함하는 클러스터 모드를 지원합니다. - - 🚀 다양한 배포 방법: 소스 코드, 쿠버네티스 또는 도커를 통한 배포를 지원합니다. + - 🚀 다양한 배포 방법: 소스 코드, 쿠버네티스 또는 도커를 통한 배포를 지원합니다. - 대규모 사용자 기반 지원: 수십만 명의 사용자를 포함하는 초대형 그룹, 수천만 명의 사용자 및 수십억 건의 메시지를 지원합니다. ### 강화된 비즈니스 기능: -+ **REST API**：OpenIMServer는 비즈니스 시스템을 위한 REST API를 제공하여, 백엔드 인터페이스를 통해 그룹 생성 및 푸시 메시지 전송과 같은 더 많은 기능을 비즈니스에 제공하기 위해 설계되었습니다. -+ **Webhooks**：OpenIMServer는 더 많은 비즈니스 형태를 확장할 수 있는 콜백 기능을 제공합니다. 콜백이란 메시지 전송 전후와 같은 특정 이벤트 전후에 OpenIMServer가 비즈니스 서버로 요청을 보내는 것을 의미합니다. +- **REST API**：OpenIMServer는 비즈니스 시스템을 위한 REST API를 제공하여, 백엔드 인터페이스를 통해 그룹 생성 및 푸시 메시지 전송과 같은 더 많은 기능을 비즈니스에 제공하기 위해 설계되었습니다. +- **Webhooks**：OpenIMServer는 더 많은 비즈니스 형태를 확장할 수 있는 콜백 기능을 제공합니다. 콜백이란 메시지 전송 전후와 같은 특정 이벤트 전후에 OpenIMServer가 비즈니스 서버로 요청을 보내는 것을 의미합니다. 👉 **[더 알아보기](https://docs.openim.io/guides/introduction/product)** @@ -112,10 +106,10 @@ Open-IM-Server의 기능의 핵심으로 들어가 우리의 아키텍처 다이 🤲 사용자 경험을 용이하게 하기 위해, 다양한 배포 솔루션을 제공합니다. 아래 목록에서 배포 방법을 선택할 수 있습니다: -+ **[소스 코드 배포 가이드](https://docs.openim.io/guides/gettingStarted/imSourceCodeDeployment)** -+ **[docker 배포 가이드](https://docs.openim.io/guides/gettingStarted/dockerCompose)** -+ **[Kubernetes 배포 가이드](https://docs.openim.io/guides/gettingStarted/k8s-deployment)** -+ **[Mac 개발자 배포 가이드](https://docs.openim.io/guides/gettingstarted/mac-deployment-guide)** +- **[소스 코드 배포 가이드](https://docs.openim.io/guides/gettingStarted/imSourceCodeDeployment)** +- **[docker 배포 가이드](https://docs.openim.io/guides/gettingStarted/dockerCompose)** +- **[Kubernetes 배포 가이드](https://docs.openim.io/guides/gettingStarted/k8s-deployment)** +- **[Mac 개발자 배포 가이드](https://docs.openim.io/guides/gettingstarted/mac-deployment-guide)** ## :hammer_and_wrench: OpenIM 개발 시작하기 @@ -125,7 +119,7 @@ OpenIM의 목표는 최상위 수준의 오픈 소스 커뮤니티를 구축하 이 Open-IM-Server 리포지토리에 기여하고 싶다면, 우리의 [기여자 문서](https://github.com/openimsdk/open-im-server/blob/main/CONTRIBUTING.md)를 읽어주세요. -시작하기 전에, 변경 사항이 필요한지 확인해 주세요. 가장 좋은 방법은 [새로운 토론](https://github.com/openimsdk/open-im-server/discussions/new/choose)을 생성하거나 [Slack 통신을](https://join.slack.com/t/openimsdk/shared_invite/zt-22720d66b-o_FvKxMTGXtcnnnHiMqe9Q) 하거나, 문제를 발견했다면 먼저 [보고](https://github.com/openimsdk/open-im-server/issues/new/choose)하는 것입니다. +시작하기 전에, 변경 사항이 필요한지 확인해 주세요. 가장 좋은 방법은 [새로운 토론](https://github.com/openimsdk/open-im-server/discussions/new/choose)을 생성하거나 [Slack 통신을](https://join.slack.com/t/openimsdk/shared_invite/zt-2ijy1ys1f-O0aEDCr7ExRZ7mwsHAVg9A) 하거나, 문제를 발견했다면 먼저 [보고](https://github.com/openimsdk/open-im-server/issues/new/choose)하는 것입니다. - [OpenIM API 참조](https://github.com/openimsdk/open-im-server/tree/main/docs/contrib/api.md) - [OpenIM Bash 로깅](https://github.com/openimsdk/open-im-server/tree/main/docs/contrib/bash-log.md) @@ -157,23 +151,21 @@ OpenIM의 목표는 최상위 수준의 오픈 소스 커뮤니티를 구축하 - [백엔드 관리 및 모니터 배포](https://github.com/openimsdk/open-im-server/tree/main/docs/contrib/prometheus-grafana.md) - [맥 개발자 배포 가이드 for OpenIM](https://github.com/openimsdk/open-im-server/tree/main/docs/contrib/mac-developer-deployment-guide.md) - ## :busts_in_silhouette: 커뮤니티 -+ 📚 [OpenIM 커뮤니티](https://github.com/OpenIMSDK/community) -+ 💕 [OpenIM 관심 그룹](https://github.com/Openim-sigs) -+ 🚀 [우리의 Slack 커뮤니티에 가입하기](https://join.slack.com/t/openimsdk/shared_invite/zt-22720d66b-o_FvKxMTGXtcnnnHiMqe9Q) -+ :eyes: [우리의 위챗(微信群)에 가입하기](https://openim-1253691595.cos.ap-nanjing.myqcloud.com/WechatIMG20.jpeg) +- 📚 [OpenIM 커뮤니티](https://github.com/OpenIMSDK/community) +- 💕 [OpenIM 관심 그룹](https://github.com/Openim-sigs) +- 🚀 [우리의 Slack 커뮤니티에 가입하기](https://join.slack.com/t/openimsdk/shared_invite/zt-2ijy1ys1f-O0aEDCr7ExRZ7mwsHAVg9A) +- :eyes: [우리의 위챗(微信群)에 가입하기](https://openim-1253691595.cos.ap-nanjing.myqcloud.com/WechatIMG20.jpeg) ## :calendar: 커뮤니티 미팅 우리는 누구나 커뮤니티에 참여하고 코드를 기여할 수 있도록 하며, 선물과 보상을 제공하며, 매주 목요일 밤에 여러분을 환영합니다. -우리의 회의는 [OpenIM Slack](https://join.slack.com/t/openimsdk/shared_invite/zt-22720d66b-o_FvKxMTGXtcnnnHiMqe9Q) 🎯에서 이루어지며, Open-IM-Server 파이프라인을 검색하여 참여할 수 있습니다. +우리의 회의는 [OpenIM Slack](https://join.slack.com/t/openimsdk/shared_invite/zt-2ijy1ys1f-O0aEDCr7ExRZ7mwsHAVg9A) 🎯에서 이루어지며, Open-IM-Server 파이프라인을 검색하여 참여할 수 있습니다. 우리는 격주 회의의 메모를 [GitHub 토론](https://github.com/openimsdk/open-im-server/discussions/categories/meeting)에서 기록하며, 우리의 역사적 [회의](https://github.com/orgs/OpenIMSDK/discussions/categories/meeting) 노트와 회의 재생은 [Google Docs 📑](https://docs.google.com/document/d/1nx8MDpuG74NASx081JcCpxPgDITNTpIIos0DS6Vr9GU/edit?usp=sharing)에서 이용할 수 있습니다. - ## :eyes: OpenIM을 사용하는 사람들 프로젝트 사용자 목록을 위한 우리의 [사용자 사례 연구](https://github.com/OpenIMSDK/community/blob/main/ADOPTERS.md) 페이지를 확인하세요. 사용 사례를 공유하고 싶다면 주저하지 말고 [📝코멘트](https://github.com/openimsdk/open-im-server/issues/379)를 남겨주세요. @@ -184,9 +176,8 @@ OpenIM은 Apache 2.0 라이선스에 따라 라이선스가 부여됩니다. 전 이 리포지토리 [OpenIM](https://github.com/openimsdk/open-im-server)에 표시된 OpenIM 로고, 그 변형 및 애니메이션 버전은 [assets/logo](../../assets/logo) 및 [assets/logo-gif](../../assets/logo-gif) 디렉토리 아래에 있으며, 저작권 법에 의해 보호됩니다. - ## 🔮 우리의 기여자들에게 감사합니다! - \ No newline at end of file + diff --git a/docs/readme/README_tr.md b/docs/readme/README_tr.md index 3cf19f537..ec2eed8bb 100644 --- a/docs/readme/README_tr.md +++ b/docs/readme/README_tr.md @@ -12,12 +12,11 @@ [![Go Report Card](https://goreportcard.com/badge/github.com/openimsdk/open-im-server?style=for-the-badge)](https://goreportcard.com/report/github.com/openimsdk/open-im-server) [![Go Reference](https://img.shields.io/badge/Go%20Reference-blue.svg?style=for-the-badge&logo=go&logoColor=white)](https://pkg.go.dev/github.com/openimsdk/open-im-server/v3) [![License](https://img.shields.io/badge/license-Apache--2.0-green?style=for-the-badge)](https://github.com/openimsdk/open-im-server/blob/main/LICENSE) -[![Slack](https://img.shields.io/badge/Slack-500%2B-blueviolet?style=for-the-badge&logo=slack&logoColor=white)](https://join.slack.com/t/openimsdk/shared_invite/zt-22720d66b-o_FvKxMTGXtcnnnHiMqe9Q) +[![Slack](https://img.shields.io/badge/Slack-500%2B-blueviolet?style=for-the-badge&logo=slack&logoColor=white)](https://join.slack.com/t/openimsdk/shared_invite/zt-2ijy1ys1f-O0aEDCr7ExRZ7mwsHAVg9A) [![Best Practices](https://img.shields.io/badge/Best%20Practices-purple?style=for-the-badge)](https://www.bestpractices.dev/projects/8045) [![Good First Issues](https://img.shields.io/github/issues/openimsdk/open-im-server/good%20first%20issue?style=for-the-badge&logo=github)](https://github.com/openimsdk/open-im-server/issues?q=is%3Aissue+is%3Aopen+sort%3Aupdated-desc+label%3A%22good+first+issue%22) [![Language](https://img.shields.io/badge/Language-Go-blue.svg?style=for-the-badge&logo=go&logoColor=white)](https://golang.org/) -

English · 中文 · @@ -46,25 +45,21 @@ Türkçe

-

- ## Ⓜ️ OpenIM Hakkında OpenIM, uygulamalara sohbet, sesli-görüntülü aramalar, bildirimler ve AI sohbet robotları entegre etmek için özel olarak tasarlanmış bir hizmet platformudur. Güçlü API'ler ve Webhook'lar sunarak, geliştiricilerin bu etkileşimli özellikleri uygulamalarına kolayca dahil etmelerini sağlar. OpenIM bağımsız bir sohbet uygulaması değildir, ancak zengin iletişim işlevselliği sağlama amacıyla diğer uygulamaları destekleyen bir platform olarak hizmet verir. Aşağıdaki diyagram, AppServer, AppClient, OpenIMServer ve OpenIMSDK arasındaki etkileşimi detaylandırmak için açıklar. - - ![App-OpenIM Relationship](../images/oepnim-design.png) ## 🚀 OpenIMSDK Hakkında - **OpenIMSDK**, müşteri uygulamalarına gömülmek üzere özel olarak oluşturulan **OpenIMServer** için tasarlanmış bir IM SDK'sıdır. Ana özellikleri ve modülleri aşağıdaki gibidir: +**OpenIMSDK**, müşteri uygulamalarına gömülmek üzere özel olarak oluşturulan **OpenIMServer** için tasarlanmış bir IM SDK'sıdır. Ana özellikleri ve modülleri aşağıdaki gibidir: -+ 🌟 Ana Özellikler: +- 🌟 Ana Özellikler: - 📦 Yerel depolama - 🔔 Dinleyici geri çağırmaları @@ -85,15 +80,15 @@ Golang kullanılarak inşa edilmiş ve tüm platformlarda tutarlı bir erişim d ## 🌐 OpenIMServer Hakkında -+ **OpenIMServer** aşağıdaki özelliklere sahiptir: +- **OpenIMServer** aşağıdaki özelliklere sahiptir: - 🌐 Mikroservis mimarisi: Bir kapı ve çoklu rpc servisleri içeren küme modunu destekler. - 🚀 Çeşitli dağıtım yöntemleri: Kaynak kodu, Kubernetes veya Docker aracılığıyla dağıtımı destekler. - Büyük kullanıcı tabanı desteği: Yüz binlerce kullanıcısı olan süper büyük gruplar, on milyonlarca kullanıcı ve milyarlarca mesaj. ### Geliştirilmiş İşlevsellik: -+ **REST API**：OpenIMServer, işletmeleri gruplar oluşturma ve arka plan arayüzleri aracılığıyla itme mesajları gönderme gibi daha fazla işlevsellikle güçlendirmeyi amaçlayan iş sistemleri için REST API'leri sunar. -+ **Webhooks**：OpenIMServer, daha fazla iş formunu genişletme yetenekleri sağlayan geri çağırma özellikleri sunar. Geri çağırma, OpenIMServer'ın belirli bir olaydan önce veya sonra, örneğin bir mesaj göndermeden önce veya sonra iş sunucusuna bir istek göndermesi anlamına gelir. +- **REST API**：OpenIMServer, işletmeleri gruplar oluşturma ve arka plan arayüzleri aracılığıyla itme mesajları gönderme gibi daha fazla işlevsellikle güçlendirmeyi amaçlayan iş sistemleri için REST API'leri sunar. +- **Webhooks**：OpenIMServer, daha fazla iş formunu genişletme yetenekleri sağlayan geri çağırma özellikleri sunar. Geri çağırma, OpenIMServer'ın belirli bir olaydan önce veya sonra, örneğin bir mesaj göndermeden önce veya sonra iş sunucusuna bir istek göndermesi anlamına gelir. 👉 **[Daha fazla bilgi edinin](https://docs.openim.io/guides/introduction/product)** @@ -111,10 +106,10 @@ Birçok platformu destekliyoruz. Web tarafında hızlı deneyim için adresler 🤲 Kullanıcı deneyimini kolaylaştırmak için çeşitli dağıtım çözümleri sunuyoruz. Aşağıdaki listeden dağıtım yönteminizi seçebilirsiniz: -+ **[Kaynak Kodu Dağıtım Kılavuzu](https://docs.openim.io/guides/gettingStarted/imSourceCodeDeployment)** -+ **[Docker Dağıtım Kılavuzu](https://docs.openim.io/guides/gettingStarted/dockerCompose)** -+ **[Kubernetes Dağıtım Kılavuzu](https://docs.openim.io/guides/gettingStarted/k8s-deployment)** -+ **[Mac Geliştirici Dağıtım Kılavuzu](https://docs.openim.io/guides/gettingstarted/mac-deployment-guide)** +- **[Kaynak Kodu Dağıtım Kılavuzu](https://docs.openim.io/guides/gettingStarted/imSourceCodeDeployment)** +- **[Docker Dağıtım Kılavuzu](https://docs.openim.io/guides/gettingStarted/dockerCompose)** +- **[Kubernetes Dağıtım Kılavuzu](https://docs.openim.io/guides/gettingStarted/k8s-deployment)** +- **[Mac Geliştirici Dağıtım Kılavuzu](https://docs.openim.io/guides/gettingstarted/mac-deployment-guide)** ## :hammer_and_wrench: OpenIM Geliştirmeye Başlamak @@ -124,7 +119,7 @@ OpenIM Amacımız, üst düzey bir açık kaynak topluluğu oluşturmaktır. [To Bu Open-IM-Server deposuna katkıda bulunmak istiyorsanız, lütfen katkıda bulunanlar için [dokümantasyonumuzu](https://github.com/openimsdk/open-im-server/blob/main/CONTRIBUTING.md) okuyun. -Başlamadan önce, lütfen değişikliklerinizin talep edildiğinden emin olun. Bunun için en iyisi, [yeni bir tartışma OLUŞTURMAK](https://github.com/openimsdk/open-im-server/discussions/new/choose) veya [Slack İletişimi](https://join.slack.com/t/openimsdk/shared_invite/zt-22720d66b-o_FvKxMTGXtcnnnHiMqe9Q) kurmak, ya da bir sorun bulursanız, önce bunu [rapor](https://github.com/openimsdk/open-im-server/issues/new/choose) etmektir. +Başlamadan önce, lütfen değişikliklerinizin talep edildiğinden emin olun. Bunun için en iyisi, [yeni bir tartışma OLUŞTURMAK](https://github.com/openimsdk/open-im-server/discussions/new/choose) veya [Slack İletişimi](https://join.slack.com/t/openimsdk/shared_invite/zt-2ijy1ys1f-O0aEDCr7ExRZ7mwsHAVg9A) kurmak, ya da bir sorun bulursanız, önce bunu [rapor](https://github.com/openimsdk/open-im-server/issues/new/choose) etmektir. - [OpenIM API Referansı](https://github.com/openimsdk/open-im-server/tree/main/docs/contrib/api.md) - [OpenIM Bash Günlüğü](https://github.com/openimsdk/open-im-server/tree/main/docs/contrib/bash-log.md) @@ -156,19 +151,18 @@ Başlamadan önce, lütfen değişikliklerinizin talep edildiğinden emin olun. - [Arka uç yönetimi ve izleme dağıtımı](https://github.com/openimsdk/open-im-server/tree/main/docs/contrib/prometheus-grafana.md) - [Mac Geliştirici Dağıtım Kılavuzu for OpenIM](https://github.com/openimsdk/open-im-server/tree/main/docs/contrib/mac-developer-deployment-guide.md) - ## :busts_in_silhouette: Topluluk -+ 📚 [OpenIM Topluluğu](https://github.com/OpenIMSDK/community) -+ 💕 [OpenIM İlgi Grubu](https://github.com/Openim-sigs) -+ 🚀 [Slack topluluğumuza katılın](https://join.slack.com/t/openimsdk/shared_invite/zt-22720d66b-o_FvKxMTGXtcnnnHiMqe9Q) -+ :eyes: [Wechat grubumuza katılın (微信群)](https://openim-1253691595.cos.ap-nanjing.myqcloud.com/WechatIMG20.jpeg) +- 📚 [OpenIM Topluluğu](https://github.com/OpenIMSDK/community) +- 💕 [OpenIM İlgi Grubu](https://github.com/Openim-sigs) +- 🚀 [Slack topluluğumuza katılın](https://join.slack.com/t/openimsdk/shared_invite/zt-2ijy1ys1f-O0aEDCr7ExRZ7mwsHAVg9A) +- :eyes: [Wechat grubumuza katılın (微信群)](https://openim-1253691595.cos.ap-nanjing.myqcloud.com/WechatIMG20.jpeg) ## :calendar: Topluluk Toplantıları Topluluğumuza herkesin katılmasını ve kod katkısında bulunmasını istiyoruz, hediyeler ve ödüller sunuyoruz ve sizi her Perşembe gecesi bize katılmaya davet ediyoruz. -Konferansımız [OpenIM Slack'te](https://join.slack.com/t/openimsdk/shared_invite/zt-22720d66b-o_FvKxMTGXtcnnnHiMqe9Q) 🎯, ardından Open-IM-Server boru hattını arayıp katılabilirsiniz. +Konferansımız [OpenIM Slack'te](https://join.slack.com/t/openimsdk/shared_invite/zt-2ijy1ys1f-O0aEDCr7ExRZ7mwsHAVg9A) 🎯, ardından Open-IM-Server boru hattını arayıp katılabilirsiniz. İki haftada bir yapılan toplantının [notlarını](https://github.com/orgs/OpenIMSDK/discussions/categories/meeting) [GitHub tartışmalarında alıyoruz](https://github.com/openimsdk/open-im-server/discussions/categories/meeting), Tarihi toplantı notlarımız ve toplantıların tekrarları [Google Docs'ta](https://docs.google.com/document/d/1nx8MDpuG74NASx081JcCpxPgDITNTpIIos0DS6Vr9GU/edit?usp=sharing) 📑 mevcut. @@ -186,4 +180,4 @@ Bu depoda, [assets/logo](../../assets/logo) ve [assets/logo-gif](../../assets/lo - \ No newline at end of file + diff --git a/docs/readme/README_uk.md b/docs/readme/README_uk.md index 81820590b..a14905bc5 100644 --- a/docs/readme/README_uk.md +++ b/docs/readme/README_uk.md @@ -12,12 +12,11 @@ [![Go Report Card](https://goreportcard.com/badge/github.com/openimsdk/open-im-server?style=for-the-badge)](https://goreportcard.com/report/github.com/openimsdk/open-im-server) [![Go Reference](https://img.shields.io/badge/Go%20Reference-blue.svg?style=for-the-badge&logo=go&logoColor=white)](https://pkg.go.dev/github.com/openimsdk/open-im-server/v3) [![License](https://img.shields.io/badge/license-Apache--2.0-green?style=for-the-badge)](https://github.com/openimsdk/open-im-server/blob/main/LICENSE) -[![Slack](https://img.shields.io/badge/Slack-500%2B-blueviolet?style=for-the-badge&logo=slack&logoColor=white)](https://join.slack.com/t/openimsdk/shared_invite/zt-22720d66b-o_FvKxMTGXtcnnnHiMqe9Q) +[![Slack](https://img.shields.io/badge/Slack-500%2B-blueviolet?style=for-the-badge&logo=slack&logoColor=white)](https://join.slack.com/t/openimsdk/shared_invite/zt-2ijy1ys1f-O0aEDCr7ExRZ7mwsHAVg9A) [![Best Practices](https://img.shields.io/badge/Best%20Practices-purple?style=for-the-badge)](https://www.bestpractices.dev/projects/8045) [![Good First Issues](https://img.shields.io/github/issues/openimsdk/open-im-server/good%20first%20issue?style=for-the-badge&logo=github)](https://github.com/openimsdk/open-im-server/issues?q=is%3Aissue+is%3Aopen+sort%3Aupdated-desc+label%3A%22good+first+issue%22) [![Language](https://img.shields.io/badge/Language-Go-blue.svg?style=for-the-badge&logo=go&logoColor=white)](https://golang.org/) -

English · 中文 · @@ -46,7 +45,6 @@ Türkçe

-

@@ -61,14 +59,14 @@ OpenIM — це сервісна платформа, спеціально роз **OpenIMSDK** – це пакет IM SDK, розроблений для **OpenIMServer**, створений спеціально для вбудовування в клієнтські програми. Його основні функції та модулі такі: -+ 🌟 Основні характеристики: +- 🌟 Основні характеристики: - 📦 Локальне сховище - 🔔 Зворотні виклики слухача - 🛡️ Обгортка API - 🌐 Керування підключенням -+ 📚 Основні модулі: +- 📚 Основні модулі: 1. 🚀 Ініціалізація та вхід 2. 👤 Керування користувачами @@ -82,15 +80,15 @@ OpenIM — це сервісна платформа, спеціально роз ## 🌐 Про OpenIMServer -+ **OpenIMServer** має такі характеристики: +- **OpenIMServer** має такі характеристики: - 🌐 Архітектура мікросервісу: підтримує режим кластера, включаючи шлюз і кілька служб rpc. - 🚀 Різноманітні методи розгортання: підтримує розгортання через вихідний код, Kubernetes або Docker. - Підтримка величезної бази користувачів: надвеликі групи із сотнями тисяч користувачів, десятками мільйонів користувачів і мільярдами повідомлень. ### Розширена бізнес-функціональність: -+ **REST API**: OpenIMServer пропонує REST API для бізнес-систем, спрямованих на надання компаніям додаткових можливостей, таких як створення груп і надсилання push-повідомлень через серверні інтерфейси. -+ **Веб-перехоплення**: OpenIMServer надає можливості зворотного виклику, щоб розширити більше бізнес-форм. Зворотний виклик означає, що OpenIMServer надсилає запит на бізнес-сервер до або після певної події, як зворотні виклики до або після надсилання повідомлення. +- **REST API**: OpenIMServer пропонує REST API для бізнес-систем, спрямованих на надання компаніям додаткових можливостей, таких як створення груп і надсилання push-повідомлень через серверні інтерфейси. +- **Веб-перехоплення**: OpenIMServer надає можливості зворотного виклику, щоб розширити більше бізнес-форм. Зворотний виклик означає, що OpenIMServer надсилає запит на бізнес-сервер до або після певної події, як зворотні виклики до або після надсилання повідомлення. 👉 **[Докладніше](https://docs.openim.io/guides/introduction/product)** @@ -100,7 +98,6 @@ OpenIM — це сервісна платформа, спеціально роз ![Overall Architecture](../images/architecture-layers.png) - ## :rocket: Швидкий початок Ми підтримуємо багато платформ. Ось адреси для швидкого використання веб-сайту: @@ -109,10 +106,10 @@ OpenIM — це сервісна платформа, спеціально роз 🤲 Щоб полегшити роботу користувача, ми пропонуємо різні рішення для розгортання. Ви можете вибрати спосіб розгортання зі списку нижче: -+ **[Посібник із розгортання вихідного коду](https://docs.openim.io/guides/gettingStarted/imSourceCodeDeployment)** -+ **[Посібник із розгортання Docker](https://docs.openim.io/guides/gettingStarted/dockerCompose)** -+ **[Посібник із розгортання Kubernetes](https://docs.openim.io/guides/gettingStarted/k8s-deployment)** -+ **[Посібник із розгортання розробника Mac](https://docs.openim.io/guides/gettingstarted/mac-deployment-guide)** +- **[Посібник із розгортання вихідного коду](https://docs.openim.io/guides/gettingStarted/imSourceCodeDeployment)** +- **[Посібник із розгортання Docker](https://docs.openim.io/guides/gettingStarted/dockerCompose)** +- **[Посібник із розгортання Kubernetes](https://docs.openim.io/guides/gettingStarted/k8s-deployment)** +- **[Посібник із розгортання розробника Mac](https://docs.openim.io/guides/gettingstarted/mac-deployment-guide)** ## :hammer_and_wrench: Щоб розпочати розробку OpenIM @@ -122,7 +119,7 @@ OpenIM. Наша мета — побудувати спільноту з від Якщо ви хочете внести свій внесок у це сховище Open-IM-Server, прочитайте нашу [документацію для учасників](https://github.com/openimsdk/open-im-server/blob/main/CONTRIBUTING.md). -Перш ніж почати, переконайтеся, що ваші зміни затребувані. Найкраще для цього створити [нове обговорення](https://github.com/openimsdk/open-im-server/discussions/new/choose) АБО [Нездійснене спілкування](https://join.slack.com/t/openimsdk/shared_invite/zt-22720d66b-o_FvKxMTGXtcnnnHiMqe9Q)або, якщо ви виявите проблему, спершу [повідомити про неї](https://github.com/openimsdk/open-im-server/issues/new/choose). +Перш ніж почати, переконайтеся, що ваші зміни затребувані. Найкраще для цього створити [нове обговорення](https://github.com/openimsdk/open-im-server/discussions/new/choose) АБО [Нездійснене спілкування](https://join.slack.com/t/openimsdk/shared_invite/zt-2ijy1ys1f-O0aEDCr7ExRZ7mwsHAVg9A)або, якщо ви виявите проблему, спершу [повідомити про неї](https://github.com/openimsdk/open-im-server/issues/new/choose). - [Довідка щодо API OpenIM](https://github.com/openimsdk/open-im-server/tree/main/docs/contrib/api.md) - [Ведення журналу OpenIM Bash](https://github.com/openimsdk/open-im-server/tree/main/docs/contrib/bash-log.md) @@ -154,19 +151,18 @@ OpenIM. Наша мета — побудувати спільноту з від - [Керування серверною частиною та моніторинг розгортання](https://github.com/openimsdk/open-im-server/tree/main/docs/contrib/prometheus-grafana.md) - [Посібник із розгортання розробника Mac для OpenIM](https://github.com/openimsdk/open-im-server/tree/main/docs/contrib/mac-developer-deployment-guide.md) - ## :busts_in_silhouette: Спільнота -+ 📚 [Спільнота OpenIM](https://github.com/OpenIMSDK/community) -+ 💕 [Група інтересів OpenIM](https://github.com/Openim-sigs) -+ 🚀 [Приєднайтеся до нашої спільноти Slack](https://join.slack.com/t/openimsdk/shared_invite/zt-22720d66b-o_FvKxMTGXtcnnnHiMqe9Q) -+ :eyes: [Приєднайтеся до нашого wechat](https://openim-1253691595.cos.ap-nanjing.myqcloud.com/WechatIMG20.jpeg) +- 📚 [Спільнота OpenIM](https://github.com/OpenIMSDK/community) +- 💕 [Група інтересів OpenIM](https://github.com/Openim-sigs) +- 🚀 [Приєднайтеся до нашої спільноти Slack](https://join.slack.com/t/openimsdk/shared_invite/zt-2ijy1ys1f-O0aEDCr7ExRZ7mwsHAVg9A) +- :eyes: [Приєднайтеся до нашого wechat](https://openim-1253691595.cos.ap-nanjing.myqcloud.com/WechatIMG20.jpeg) ## :calendar: Збори громади Ми хочемо, щоб будь-хто долучився до нашої спільноти та додав код, ми пропонуємо подарунки та нагороди, і ми запрошуємо вас приєднатися до нас щочетверга ввечері. -Наша конференція знаходиться в [OpenIM Slack](https://join.slack.com/t/openimsdk/shared_invite/zt-22720d66b-o_FvKxMTGXtcnnnHiMqe9Q) 🎯, тоді ви можете шукати конвеєр Open-IM-Server, щоб приєднатися. +Наша конференція знаходиться в [OpenIM Slack](https://join.slack.com/t/openimsdk/shared_invite/zt-2ijy1ys1f-O0aEDCr7ExRZ7mwsHAVg9A) 🎯, тоді ви можете шукати конвеєр Open-IM-Server, щоб приєднатися. Ми робимо нотатки про кожну [двотижневу зустріч](https://github.com/orgs/OpenIMSDK/discussions/categories/meeting)в [обговореннях GitHub](https://github.com/openimsdk/open-im-server/discussions/categories/meeting). Наші історичні нотатки зустрічей, а також повтори зустрічей доступні в[Google Docs :bookmark_tabs:](https://docs.google.com/document/d/1nx8MDpuG74NASx081JcCpxPgDITNTpIIos0DS6Vr9GU/edit?usp=sharing). diff --git a/docs/readme/README_vi.md b/docs/readme/README_vi.md index a6ab39253..f8894c87f 100644 --- a/docs/readme/README_vi.md +++ b/docs/readme/README_vi.md @@ -12,12 +12,11 @@ [![Go Report Card](https://goreportcard.com/badge/github.com/openimsdk/open-im-server?style=for-the-badge)](https://goreportcard.com/report/github.com/openimsdk/open-im-server) [![Go Reference](https://img.shields.io/badge/Go%20Reference-blue.svg?style=for-the-badge&logo=go&logoColor=white)](https://pkg.go.dev/github.com/openimsdk/open-im-server/v3) [![License](https://img.shields.io/badge/license-Apache--2.0-green?style=for-the-badge)](https://github.com/openimsdk/open-im-server/blob/main/LICENSE) -[![Slack](https://img.shields.io/badge/Slack-500%2B-blueviolet?style=for-the-badge&logo=slack&logoColor=white)](https://join.slack.com/t/openimsdk/shared_invite/zt-22720d66b-o_FvKxMTGXtcnnnHiMqe9Q) +[![Slack](https://img.shields.io/badge/Slack-500%2B-blueviolet?style=for-the-badge&logo=slack&logoColor=white)](https://join.slack.com/t/openimsdk/shared_invite/zt-2ijy1ys1f-O0aEDCr7ExRZ7mwsHAVg9A) [![Best Practices](https://img.shields.io/badge/Best%20Practices-purple?style=for-the-badge)](https://www.bestpractices.dev/projects/8045) [![Good First Issues](https://img.shields.io/github/issues/openimsdk/open-im-server/good%20first%20issue?style=for-the-badge&logo=github)](https://github.com/openimsdk/open-im-server/issues?q=is%3Aissue+is%3Aopen+sort%3Aupdated-desc+label%3A%22good+first+issue%22) [![Language](https://img.shields.io/badge/Language-Go-blue.svg?style=for-the-badge&logo=go&logoColor=white)](https://golang.org/) -

English · 中文 · @@ -46,7 +45,6 @@ Türkçe

-

@@ -61,14 +59,14 @@ OpenIM là một nền tảng dịch vụ được thiết kế đặc biệt ch **OpenIMSDK** là một SDK IM được thiết kế cho **OpenIMServer**, được tạo ra đặc biệt để nhúng vào các ứng dụng khách. Các tính năng chính và các mô-đun của nó như sau: -+ 🌟 Các Tính Năng Chính: +- 🌟 Các Tính Năng Chính: - 📦 Lưu trữ cục bộ - 🔔 Gọi lại sự kiện (Listener callbacks) - 🛡️ Bọc API - 🌐 Quản lý kết nối -+ 📚 Các Mô-đun Chính: +- 📚 Các Mô-đun Chính: 1. 🚀 Khởi tạo và Đăng nhập 2. 👤 Quản lý Người dùng @@ -82,25 +80,24 @@ Nó được xây dựng bằng Golang và hỗ trợ triển khai đa nền t ## 🌐 Về OpenIMServer -+ **OpenIMServer** có những đặc điểm sau: +- **OpenIMServer** có những đặc điểm sau: - 🌐 Kiến trúc vi dịch vụ: Hỗ trợ chế độ cluster, bao gồm một gateway và nhiều dịch vụ rpc. - 🚀 Phương pháp triển khai đa dạng: Hỗ trợ triển khai qua mã nguồn, Kubernetes hoặc Docker. - Hỗ trợ cho cơ sở người dùng lớn: Nhóm siêu lớn với hàng trăm nghìn người dùng, hàng chục triệu người dùng và hàng tỷ tin nhắn. ### Tăng cường Chức năng Kinh doanh: -+ **REST API**: OpenIMServer cung cấp REST APIs cho các hệ thống kinh doanh, nhằm tăng cường khả năng cho doanh nghiệp với nhiều chức năng hơn, như tạo nhóm và gửi tin nhắn đẩy qua giao diện backend. -+ **Webhooks**: OpenIMServer cung cấp khả năng gọi lại để mở rộng thêm hình thức kinh doanh. Một gọi lại có nghĩa là OpenIMServer gửi một yêu cầu đến máy chủ kinh doanh trước hoặc sau một sự kiện nhất định, giống như gọi lại trước hoặc sau khi gửi một tin nhắn. +- **REST API**: OpenIMServer cung cấp REST APIs cho các hệ thống kinh doanh, nhằm tăng cường khả năng cho doanh nghiệp với nhiều chức năng hơn, như tạo nhóm và gửi tin nhắn đẩy qua giao diện backend. +- **Webhooks**: OpenIMServer cung cấp khả năng gọi lại để mở rộng thêm hình thức kinh doanh. Một gọi lại có nghĩa là OpenIMServer gửi một yêu cầu đến máy chủ kinh doanh trước hoặc sau một sự kiện nhất định, giống như gọi lại trước hoặc sau khi gửi một tin nhắn. 👉 **[Learn more](https://docs.openim.io/guides/introduction/product)** ## :building_construction: Kiến trúc tổng thể - Làm sâu sắc vào trái tim của chức năng Open-IM-Server với sơ đồ kiến trúc của chúng tôi. +Làm sâu sắc vào trái tim của chức năng Open-IM-Server với sơ đồ kiến trúc của chúng tôi. ![Overall Architecture](../../docs/images/architecture-layers.png) - ## :rocket: Bắt đầu nhanh Chúng tôi hỗ trợ nhiều nền tảng. Dưới đây là các địa chỉ để trải nghiệm nhanh trên phía web： @@ -109,12 +106,12 @@ Chúng tôi hỗ trợ nhiều nền tảng. Dưới đây là các địa chỉ 🤲 Để tạo thuận lợi cho trải nghiệm người dùng, chúng tôi cung cấp các giải pháp triển khai đa dạng. Bạn có thể chọn phương thức triển khai từ danh sách dưới đây: -+ **[Hướng dẫn Triển khai Mã Nguồn](https://docs.openim.io/guides/gettingStarted/imSourceCodeDeployment)** -+ **[Hướng dẫn Triển khai Docker](https://docs.openim.io/guides/gettingStarted/dockerCompose)** -+ **[Hướng dẫn Triển khai Kubernetes](https://docs.openim.io/guides/gettingStarted/k8s-deployment)** -+ **[Hướng dẫn Triển khai cho Nhà Phát Triển Mac](https://docs.openim.io/guides/gettingstarted/mac-deployment-guide)** +- **[Hướng dẫn Triển khai Mã Nguồn](https://docs.openim.io/guides/gettingStarted/imSourceCodeDeployment)** +- **[Hướng dẫn Triển khai Docker](https://docs.openim.io/guides/gettingStarted/dockerCompose)** +- **[Hướng dẫn Triển khai Kubernetes](https://docs.openim.io/guides/gettingStarted/k8s-deployment)** +- **[Hướng dẫn Triển khai cho Nhà Phát Triển Mac](https://docs.openim.io/guides/gettingstarted/mac-deployment-guide)** -## :hammer_and_wrench: Để Bắt Đầu Phát Triển OpenIM +## :hammer_and_wrench: Để Bắt Đầu Phát Triển OpenIM [![Mở trong Dev Contain](https://img.shields.io/static/v1?label=Dev%20Container&message=Open&color=blue&logo=visualstudiocode)](https://vscode.dev/github/openimsdk/open-im-server) @@ -122,8 +119,7 @@ Mục tiêu của OpenIM là xây dựng một cộng đồng mã nguồn mở c Nếu bạn muốn đóng góp cho kho lưu trữ Open-IM-Server này, vui lòng đọc [tài liệu hướng dẫn cho người đóng góp](https://github.com/openimsdk/open-im-server/blob/main/CONTRIBUTING.md). - -Trước khi bạn bắt đầu, hãy chắc chắn rằng các thay đổi của bạn được yêu cầu. Cách tốt nhất là tạo một [cuộc thảo luận mới](https://github.com/openimsdk/open-im-server/discussions/new/choose) hoặc [Giao tiếp Slack](https://join.slack.com/t/openimsdk/shared_invite/zt-22720d66b-o_FvKxMTGXtcnnnHiMqe9Q), hoặc nếu bạn tìm thấy một vấn đề, [báo cáo nó ](https://github.com/openimsdk/open-im-server/issues/new/choose) trước. +Trước khi bạn bắt đầu, hãy chắc chắn rằng các thay đổi của bạn được yêu cầu. Cách tốt nhất là tạo một [cuộc thảo luận mới](https://github.com/openimsdk/open-im-server/discussions/new/choose) hoặc [Giao tiếp Slack](https://join.slack.com/t/openimsdk/shared_invite/zt-2ijy1ys1f-O0aEDCr7ExRZ7mwsHAVg9A), hoặc nếu bạn tìm thấy một vấn đề, [báo cáo nó ](https://github.com/openimsdk/open-im-server/issues/new/choose) trước. - [Tham khảo API OpenIM](https://github.com/openimsdk/open-im-server/tree/main/docs/contrib/api.md) - [Nhật ký Bash OpenIM](https://github.com/openimsdk/open-im-server/tree/main/docs/contrib/bash-log.md) @@ -155,19 +151,18 @@ Trước khi bạn bắt đầu, hãy chắc chắn rằng các thay đổi củ - [Quản lý triển khai và giám sát backend](https://github.com/openimsdk/open-im-server/tree/main/docs/contrib/prometheus-grafana.md) - [Hướng dẫn Triển khai cho Nhà Phát triển Mac OpenIM](https://github.com/openimsdk/open-im-server/tree/main/docs/contrib/mac-developer-deployment-guide.md) - ## :busts_in_silhouette: Cộng đồng -+ 📚 [Cộng đồng OpenIM](https://github.com/OpenIMSDK/community) -+ 💕 [Nhóm Quan tâm OpenIM](https://github.com/Openim-sigs) -+ 🚀 [Tham gia cộng đồng Slack của chúng tôi](https://join.slack.com/t/openimsdk/shared_invite/zt-22720d66b-o_FvKxMTGXtcnnnHiMqe9Q) -+ :eyes: [Tham gia nhóm WeChat của chúng tôi (微信群)](https://openim-1253691595.cos.ap-nanjing.myqcloud.com/WechatIMG20.jpeg) +- 📚 [Cộng đồng OpenIM](https://github.com/OpenIMSDK/community) +- 💕 [Nhóm Quan tâm OpenIM](https://github.com/Openim-sigs) +- 🚀 [Tham gia cộng đồng Slack của chúng tôi](https://join.slack.com/t/openimsdk/shared_invite/zt-2ijy1ys1f-O0aEDCr7ExRZ7mwsHAVg9A) +- :eyes: [Tham gia nhóm WeChat của chúng tôi (微信群)](https://openim-1253691595.cos.ap-nanjing.myqcloud.com/WechatIMG20.jpeg) ## :calendar: Cuộc họp Cộng đồng Chúng tôi muốn bất kỳ ai cũng có thể tham gia cộng đồng và đóng góp mã nguồn, chúng tôi cung cấp quà tặng và phần thưởng, và chúng tôi chào đón bạn tham gia cùng chúng tôi mỗi tối thứ Năm. -Hội nghị của chúng tôi được tổ chức trên Slack của [OpenIM Slack](https://join.slack.com/t/openimsdk/shared_invite/zt-22720d66b-o_FvKxMTGXtcnnnHiMqe9Q) 🎯, sau đó bạn có thể tìm kiếm pipeline Open-IM-Server để tham gia +Hội nghị của chúng tôi được tổ chức trên Slack của [OpenIM Slack](https://join.slack.com/t/openimsdk/shared_invite/zt-2ijy1ys1f-O0aEDCr7ExRZ7mwsHAVg9A) 🎯, sau đó bạn có thể tìm kiếm pipeline Open-IM-Server để tham gia Chúng tôi ghi chú mỗi [cuộc họp hai tuần một lần](https://github.com/orgs/OpenIMSDK/discussions/categories/meeting) trong [các cuộc thảo luận GitHub](https://github.com/openimsdk/open-im-server/discussions/categories/meeting), ghi chú cuộc họp lịch sử của chúng tôi cũng như các bản ghi lại của cuộc họp có sẵn tại [Google Docs :bookmark_tabs:](https://docs.google.com/document/d/1nx8MDpuG74NASx081JcCpxPgDITNTpIIos0DS6Vr9GU/edit?usp=sharing). diff --git a/install.sh b/install.sh index 4b8e74c55..d07f97338 100755 --- a/install.sh +++ b/install.sh @@ -531,7 +531,7 @@ O:::::::OOO:::::::O p:::::ppppp:::::::pe::::::::e n::::n n::::nII: # Set text color to yellow for the Slack link echo -e "\033[1;33m" - print_with_delay "Join our developer community on Slack: https://join.slack.com/t/openimsdk/shared_invite/zt-22720d66b-o_FvKxMTGXtcnnnHiMqe9Q" 0.01 + print_with_delay "Join our developer community on Slack: https://join.slack.com/t/openimsdk/shared_invite/zt-2ijy1ys1f-O0aEDCr7ExRZ7mwsHAVg9A" 0.01 # Reset text color back to normal echo -e "\033[0m" From 3ad24d95214a910face80fceba3f3f68db073c56 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 28 Jul 2025 10:52:25 +0800 Subject: [PATCH 183/199] Update CHANGELOG for release v3.8.3-patch.1 (#3164) Co-authored-by: withchao <48119764+withchao@users.noreply.github.com> --- CHANGELOG/CHANGELOG-3.8.md | 19 +++++++++++++++++++ CHANGELOG/README.md | 4 ++++ 2 files changed, 23 insertions(+) create mode 100644 CHANGELOG/CHANGELOG-3.8.md create mode 100644 CHANGELOG/README.md diff --git a/CHANGELOG/CHANGELOG-3.8.md b/CHANGELOG/CHANGELOG-3.8.md new file mode 100644 index 000000000..6ebb1de95 --- /dev/null +++ b/CHANGELOG/CHANGELOG-3.8.md @@ -0,0 +1,19 @@ +## [v3.8.3-patch.1](https://github.com/openimsdk/open-im-server/releases/tag/v3.8.3-patch.1) (2025-02-25) + +### New Features +* feat: add backup volume && optimize log print [Created [#3121](https://github.com/openimsdk/open-im-server/pull/3121) + +### Bug Fixes +* fix: seq conversion failed without exiting [Created [#3120](https://github.com/openimsdk/open-im-server/pull/3120) +* fix: check error in BatchSetTokenMapByUidPid [Created [#3123](https://github.com/openimsdk/open-im-server/pull/3123) +* fix: DeleteDoc crash [Created [#3124](https://github.com/openimsdk/open-im-server/pull/3124) +* fix: the abnormal message has no sending time, causing the SDK to be abnormal [Created [#3126](https://github.com/openimsdk/open-im-server/pull/3126) +* fix: crash caused [#3127](https://github.com/openimsdk/open-im-server/pull/3127) +* fix: the user sets the conversation timer cleanup timestamp unit incorrectly [Created [#3128](https://github.com/openimsdk/open-im-server/pull/3128) +* fix: seq conversion not reading env in docker environment [Created [#3131](https://github.com/openimsdk/open-im-server/pull/3131) + +### Builds +* build: improve workflows contents. [Created [#3125](https://github.com/openimsdk/open-im-server/pull/3125) + +**Full Changelog**: [v3.8.3-e-v1.1.5...v3.8.3-patch.1-e-v1.1.5](https://github.com/openimsdk/open-im-server-enterprise/compare/v3.8.3-e-v1.1.5...v3.8.3-patch.1-e-v1.1.5) + diff --git a/CHANGELOG/README.md b/CHANGELOG/README.md new file mode 100644 index 000000000..204194de1 --- /dev/null +++ b/CHANGELOG/README.md @@ -0,0 +1,4 @@ +# CHANGELOGs + +- [CHANGELOG-3.8.md](./CHANGELOG-3.8.md) + From 097e0333476653801abfb18745577d2f6a0ee534 Mon Sep 17 00:00:00 2001 From: Monet Lee Date: Mon, 28 Jul 2025 10:57:44 +0800 Subject: [PATCH 184/199] fix: fix incorrect kicked logic. (#3480) * fix: fix incorrect ws check logic. * optimize checkSameToken logic. --- internal/msggateway/hub_server.go | 1 + internal/msggateway/ws_server.go | 37 +++++++++++++++++++++++++++---- 2 files changed, 34 insertions(+), 4 deletions(-) diff --git a/internal/msggateway/hub_server.go b/internal/msggateway/hub_server.go index 3d0d1e3f9..8374af06d 100644 --- a/internal/msggateway/hub_server.go +++ b/internal/msggateway/hub_server.go @@ -249,6 +249,7 @@ func (s *Server) MultiTerminalLoginCheck(ctx context.Context, req *msggateway.Mu tempUserCtx.SetOperationID(mcontext.GetOperationID(ctx)) client := &Client{} client.ctx = tempUserCtx + client.token = req.Token client.UserID = req.UserID client.PlatformID = int(req.PlatformID) i := &kickHandler{ diff --git a/internal/msggateway/ws_server.go b/internal/msggateway/ws_server.go index 9b7343938..bc7a2fa5f 100644 --- a/internal/msggateway/ws_server.go +++ b/internal/msggateway/ws_server.go @@ -334,6 +334,27 @@ func (ws *WsServer) multiTerminalLoginChecker(clientOK bool, oldClients []*Clien } } + // If reconnect: When multiple msgGateway instances are deployed, a client may disconnect from instance A and reconnect to instance B. + // During this process, instance A might still be executing, resulting in two clients with the same token existing simultaneously. + // This situation needs to be filtered to prevent duplicate clients. + checkSameTokenFunc := func(oldClients []*Client) []*Client { + var clientsNeedToKick []*Client + + for _, c := range oldClients { + if c.token == newClient.token { + log.ZDebug(newClient.ctx, "token is same, not kick", + "userID", newClient.UserID, + "platformID", newClient.PlatformID, + "token", newClient.token) + continue + } + + clientsNeedToKick = append(clientsNeedToKick, c) + } + + return clientsNeedToKick + } + switch ws.msgGatewayConfig.Share.MultiLogin.Policy { case constant.DefalutNotKick: case constant.PCAndOther: @@ -349,11 +370,15 @@ func (ws *WsServer) multiTerminalLoginChecker(clientOK bool, oldClients []*Clien } oldClients = append(oldClients, c) } + fallthrough case constant.AllLoginButSameTermKick: if !clientOK { return } + + oldClients = checkSameTokenFunc(oldClients) + ws.clients.DeleteClients(newClient.UserID, oldClients) for _, c := range oldClients { err := c.KickOnlineMessage() @@ -361,6 +386,7 @@ func (ws *WsServer) multiTerminalLoginChecker(clientOK bool, oldClients []*Clien log.ZWarn(c.ctx, "KickOnlineMessage", err) } } + ctx := mcontext.WithMustInfoCtx( []string{newClient.ctx.GetOperationID(), newClient.ctx.GetUserID(), constant.PlatformIDToName(newClient.PlatformID), newClient.ctx.GetConnID()}, @@ -379,14 +405,17 @@ func (ws *WsServer) multiTerminalLoginChecker(clientOK bool, oldClients []*Clien if !ok { return } - var ( - kickClients []*Client - ) + + var kickClients []*Client for _, client := range clients { if constant.PlatformIDToClass(client.PlatformID) == constant.PlatformIDToClass(newClient.PlatformID) { - kickClients = append(kickClients, client) + { + kickClients = append(kickClients, client) + } } } + kickClients = checkSameTokenFunc(kickClients) + kickTokenFunc(kickClients) } } From 865eaa719fbb38db587610e8f60cff06337a2f81 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 28 Jul 2025 11:07:59 +0800 Subject: [PATCH 185/199] Update CHANGELOG for release v3.8.3-patch.2 (#3175) Co-authored-by: icey-yu <119291641+icey-yu@users.noreply.github.com> Co-authored-by: Monet Lee --- CHANGELOG/CHANGELOG-3.8.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG/CHANGELOG-3.8.md b/CHANGELOG/CHANGELOG-3.8.md index 6ebb1de95..4315d9971 100644 --- a/CHANGELOG/CHANGELOG-3.8.md +++ b/CHANGELOG/CHANGELOG-3.8.md @@ -1,3 +1,10 @@ +## [v3.8.3-patch.2](https://github.com/openimsdk/open-im-server/releases/tag/v3.8.3-patch.2) (2025-02-28) + +### Bug Fixes +* fix: Offline push does not have a badge && Android offline push (#3146) [#3174](https://github.com/openimsdk/open-im-server/pull/3174) + +**Full Changelog**: [v3.8.3-patch.1...v3.8.3-patch.2](https://github.com/openimsdk/open-im-server/compare/v3.8.3-patch.1...v3.8.3-patch.2) + ## [v3.8.3-patch.1](https://github.com/openimsdk/open-im-server/releases/tag/v3.8.3-patch.1) (2025-02-25) ### New Features From 359201a005df72b79fb96a98148115d435bcedff Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 28 Jul 2025 11:10:42 +0800 Subject: [PATCH 186/199] Update CHANGELOG for release v3.8.3-patch.3 (#3206) Co-authored-by: withchao <48119764+withchao@users.noreply.github.com> Co-authored-by: Monet Lee --- CHANGELOG/CHANGELOG-3.8.md | 67 +++++++++++++++++++++++--------------- 1 file changed, 41 insertions(+), 26 deletions(-) diff --git a/CHANGELOG/CHANGELOG-3.8.md b/CHANGELOG/CHANGELOG-3.8.md index 4315d9971..987c9a9b3 100644 --- a/CHANGELOG/CHANGELOG-3.8.md +++ b/CHANGELOG/CHANGELOG-3.8.md @@ -1,26 +1,41 @@ -## [v3.8.3-patch.2](https://github.com/openimsdk/open-im-server/releases/tag/v3.8.3-patch.2) (2025-02-28) - -### Bug Fixes -* fix: Offline push does not have a badge && Android offline push (#3146) [#3174](https://github.com/openimsdk/open-im-server/pull/3174) - -**Full Changelog**: [v3.8.3-patch.1...v3.8.3-patch.2](https://github.com/openimsdk/open-im-server/compare/v3.8.3-patch.1...v3.8.3-patch.2) - -## [v3.8.3-patch.1](https://github.com/openimsdk/open-im-server/releases/tag/v3.8.3-patch.1) (2025-02-25) - -### New Features -* feat: add backup volume && optimize log print [Created [#3121](https://github.com/openimsdk/open-im-server/pull/3121) - -### Bug Fixes -* fix: seq conversion failed without exiting [Created [#3120](https://github.com/openimsdk/open-im-server/pull/3120) -* fix: check error in BatchSetTokenMapByUidPid [Created [#3123](https://github.com/openimsdk/open-im-server/pull/3123) -* fix: DeleteDoc crash [Created [#3124](https://github.com/openimsdk/open-im-server/pull/3124) -* fix: the abnormal message has no sending time, causing the SDK to be abnormal [Created [#3126](https://github.com/openimsdk/open-im-server/pull/3126) -* fix: crash caused [#3127](https://github.com/openimsdk/open-im-server/pull/3127) -* fix: the user sets the conversation timer cleanup timestamp unit incorrectly [Created [#3128](https://github.com/openimsdk/open-im-server/pull/3128) -* fix: seq conversion not reading env in docker environment [Created [#3131](https://github.com/openimsdk/open-im-server/pull/3131) - -### Builds -* build: improve workflows contents. [Created [#3125](https://github.com/openimsdk/open-im-server/pull/3125) - -**Full Changelog**: [v3.8.3-e-v1.1.5...v3.8.3-patch.1-e-v1.1.5](https://github.com/openimsdk/open-im-server-enterprise/compare/v3.8.3-e-v1.1.5...v3.8.3-patch.1-e-v1.1.5) - +## [v3.8.3-patch.3](https://github.com/openimsdk/open-im-server/releases/tag/v3.8.3-patch.3) (2025-03-07) + +### New Features +* feat: optimizing BatchGetIncrementalGroupMember #3180 + +### Bug Fixes +* fix: solve uncorrect notification when set group info #3172 +* fix: the sorting is wrong after canceling the administrator in group settings #3185 +* fix: solve uncorrect GroupMember enter group notification type. #3188 + +### Refactors +* refactor: change sendNotification to sendMessage to avoid ambiguity regarding message sending behavior. #3173 + +**Full Changelog**: [v3.8.3-patch.2...v3.8.3-patch.3](https://github.com/openimsdk/open-im-server/compare/v3.8.3-patch.2...v3.8.3-patch.3) + +## [v3.8.3-patch.2](https://github.com/openimsdk/open-im-server/releases/tag/v3.8.3-patch.2) (2025-02-28) + +### Bug Fixes +* fix: Offline push does not have a badge && Android offline push (#3146) [#3174](https://github.com/openimsdk/open-im-server/pull/3174) + +**Full Changelog**: [v3.8.3-patch.1...v3.8.3-patch.2](https://github.com/openimsdk/open-im-server/compare/v3.8.3-patch.1...v3.8.3-patch.2) + +## [v3.8.3-patch.1](https://github.com/openimsdk/open-im-server/releases/tag/v3.8.3-patch.1) (2025-02-25) + +### New Features +* feat: add backup volume && optimize log print [Created [#3121](https://github.com/openimsdk/open-im-server/pull/3121) + +### Bug Fixes +* fix: seq conversion failed without exiting [Created [#3120](https://github.com/openimsdk/open-im-server/pull/3120) +* fix: check error in BatchSetTokenMapByUidPid [Created [#3123](https://github.com/openimsdk/open-im-server/pull/3123) +* fix: DeleteDoc crash [Created [#3124](https://github.com/openimsdk/open-im-server/pull/3124) +* fix: the abnormal message has no sending time, causing the SDK to be abnormal [Created [#3126](https://github.com/openimsdk/open-im-server/pull/3126) +* fix: crash caused [#3127](https://github.com/openimsdk/open-im-server/pull/3127) +* fix: the user sets the conversation timer cleanup timestamp unit incorrectly [Created [#3128](https://github.com/openimsdk/open-im-server/pull/3128) +* fix: seq conversion not reading env in docker environment [Created [#3131](https://github.com/openimsdk/open-im-server/pull/3131) + +### Builds +* build: improve workflows contents. [Created [#3125](https://github.com/openimsdk/open-im-server/pull/3125) + +**Full Changelog**: [v3.8.3-e-v1.1.5...v3.8.3-patch.1-e-v1.1.5](https://github.com/openimsdk/open-im-server-enterprise/compare/v3.8.3-e-v1.1.5...v3.8.3-patch.1-e-v1.1.5) + From 839d7d90ac978dc8c99da6d9d486a89cc7d02532 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 28 Jul 2025 11:13:35 +0800 Subject: [PATCH 187/199] Update CHANGELOG for release v3.8.3-patch.4 (#3226) Co-authored-by: mo3et <34803812+mo3et@users.noreply.github.com> Co-authored-by: Monet Lee --- CHANGELOG/CHANGELOG-3.8.md | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/CHANGELOG/CHANGELOG-3.8.md b/CHANGELOG/CHANGELOG-3.8.md index 987c9a9b3..c4748e262 100644 --- a/CHANGELOG/CHANGELOG-3.8.md +++ b/CHANGELOG/CHANGELOG-3.8.md @@ -1,3 +1,10 @@ +## [v3.8.3-patch.4](https://github.com/openimsdk/open-im-server/releases/tag/v3.8.3-patch.4) (2025-03-13) + +### Bug Fixes +* fix: solve unocrrect invite notificationfrom #3213 + +**Full Changelog**: [v3.8.3-patch.3...v3.8.3-patch.4](https://github.com/openimsdk/open-im-server/compare/v3.8.3-patch.3...v3.8.3-patch.4) + ## [v3.8.3-patch.3](https://github.com/openimsdk/open-im-server/releases/tag/v3.8.3-patch.3) (2025-03-07) ### New Features @@ -37,5 +44,4 @@ ### Builds * build: improve workflows contents. [Created [#3125](https://github.com/openimsdk/open-im-server/pull/3125) -**Full Changelog**: [v3.8.3-e-v1.1.5...v3.8.3-patch.1-e-v1.1.5](https://github.com/openimsdk/open-im-server-enterprise/compare/v3.8.3-e-v1.1.5...v3.8.3-patch.1-e-v1.1.5) - +**Full Changelog**: [v3.8.3-e-v1.1.5...v3.8.3-patch.1-e-v1.1.5](https://github.com/openimsdk/open-im-server-enterprise/compare/v3.8.3-e-v1.1.5...v3.8.3-patch.1-e-v1.1.5) \ No newline at end of file From f9475bdd87a3116c88f869f673bd91036eae2254 Mon Sep 17 00:00:00 2001 From: OpenIM-Gordon <1432970085@qq.com> Date: Mon, 28 Jul 2025 11:18:49 +0800 Subject: [PATCH 188/199] fix: added AtUserIDList to the @ message for API sending. (#3472) --- internal/api/msg.go | 64 ++++++++++++++++++++++++-------------------- pkg/apistruct/msg.go | 13 ++++++--- 2 files changed, 45 insertions(+), 32 deletions(-) diff --git a/internal/api/msg.go b/internal/api/msg.go index d832bfdcd..a134dfdb3 100644 --- a/internal/api/msg.go +++ b/internal/api/msg.go @@ -94,7 +94,22 @@ func (*MessageApi) SetOptions(options map[string]bool, value bool) { datautil.SetSwitchFromOptions(options, constant.IsConversationUpdate, value) } -func (m *MessageApi) newUserSendMsgReq(_ *gin.Context, params *apistruct.SendMsg) *msg.SendMsgReq { +func (m *MessageApi) newUserSendMsgReq(_ *gin.Context, params *apistruct.SendMsg, data any) *msg.SendMsgReq { + msgData := &sdkws.MsgData{ + SendID: params.SendID, + GroupID: params.GroupID, + ClientMsgID: idutil.GetMsgIDByMD5(params.SendID), + SenderPlatformID: params.SenderPlatformID, + SenderNickname: params.SenderNickname, + SenderFaceURL: params.SenderFaceURL, + SessionType: params.SessionType, + MsgFrom: constant.SysMsgType, + ContentType: params.ContentType, + CreateTime: timeutil.GetCurrentTimestampByMill(), + SendTime: params.SendTime, + OfflinePushInfo: params.OfflinePushInfo, + Ex: params.Ex, + } var newContent string options := make(map[string]bool, 5) switch params.ContentType { @@ -104,6 +119,11 @@ func (m *MessageApi) newUserSendMsgReq(_ *gin.Context, params *apistruct.SendMsg newContent = jsonutil.StructToJsonString(¬ification) case constant.Text: fallthrough + case constant.AtText: + if atElem, ok := data.(*apistruct.AtElem); ok { + msgData.AtUserIDList = atElem.AtUserList + } + fallthrough case constant.Picture: fallthrough case constant.Custom: @@ -123,24 +143,10 @@ func (m *MessageApi) newUserSendMsgReq(_ *gin.Context, params *apistruct.SendMsg if params.NotOfflinePush { datautil.SetSwitchFromOptions(options, constant.IsOfflinePush, false) } + msgData.Content = []byte(newContent) + msgData.Options = options pbData := msg.SendMsgReq{ - MsgData: &sdkws.MsgData{ - SendID: params.SendID, - GroupID: params.GroupID, - ClientMsgID: idutil.GetMsgIDByMD5(params.SendID), - SenderPlatformID: params.SenderPlatformID, - SenderNickname: params.SenderNickname, - SenderFaceURL: params.SenderFaceURL, - SessionType: params.SessionType, - MsgFrom: constant.SysMsgType, - ContentType: params.ContentType, - Content: []byte(newContent), - CreateTime: timeutil.GetCurrentTimestampByMill(), - SendTime: params.SendTime, - Options: options, - OfflinePushInfo: params.OfflinePushInfo, - Ex: params.Ex, - }, + MsgData: msgData, } return &pbData } @@ -198,23 +204,23 @@ func (m *MessageApi) getSendMsgReq(c *gin.Context, req apistruct.SendMsg) (sendM log.ZDebug(c, "getSendMsgReq", "req", req.Content) switch req.ContentType { case constant.Text: - data = apistruct.TextElem{} + data = &apistruct.TextElem{} case constant.Picture: - data = apistruct.PictureElem{} + data = &apistruct.PictureElem{} case constant.Voice: - data = apistruct.SoundElem{} + data = &apistruct.SoundElem{} case constant.Video: - data = apistruct.VideoElem{} + data = &apistruct.VideoElem{} case constant.File: - data = apistruct.FileElem{} + data = &apistruct.FileElem{} case constant.AtText: - data = apistruct.AtElem{} + data = &apistruct.AtElem{} case constant.Custom: - data = apistruct.CustomElem{} + data = &apistruct.CustomElem{} case constant.MarkdownText: - data = apistruct.MarkdownTextElem{} + data = &apistruct.MarkdownTextElem{} case constant.OANotification: - data = apistruct.OANotificationElem{} + data = &apistruct.OANotificationElem{} req.SessionType = constant.NotificationChatType if err = m.userClient.GetNotificationByID(c, req.SendID); err != nil { return nil, err @@ -222,14 +228,14 @@ func (m *MessageApi) getSendMsgReq(c *gin.Context, req apistruct.SendMsg) (sendM default: return nil, errs.WrapMsg(errs.ErrArgs, "unsupported content type", "contentType", req.ContentType) } - if err := mapstructure.WeakDecode(req.Content, &data); err != nil { + if err := mapstructure.WeakDecode(req.Content, data); err != nil { return nil, errs.WrapMsg(err, "failed to decode message content") } log.ZDebug(c, "getSendMsgReq", "decodedContent", data) if err := m.validate.Struct(data); err != nil { return nil, errs.WrapMsg(err, "validation error") } - return m.newUserSendMsgReq(c, &req), nil + return m.newUserSendMsgReq(c, &req, data), nil } func (m *MessageApi) getModifyFields(req, respModify *sdkws.MsgData) map[string]any { diff --git a/pkg/apistruct/msg.go b/pkg/apistruct/msg.go index d818e769c..0e1b356a1 100644 --- a/pkg/apistruct/msg.go +++ b/pkg/apistruct/msg.go @@ -63,9 +63,11 @@ type FileElem struct { FileSize int64 `mapstructure:"fileSize" validate:"required"` } type AtElem struct { - Text string `mapstructure:"text"` - AtUserList []string `mapstructure:"atUserList" validate:"required,max=1000"` - IsAtSelf bool `mapstructure:"isAtSelf"` + Text string `mapstructure:"text"` + AtUserList []string `mapstructure:"atUserList" validate:"required,max=1000"` + AtUsersInfo []*AtInfo `json:"atUsersInfo"` + QuoteMessage *MsgStruct `json:"quoteMessage"` + IsAtSelf bool `mapstructure:"isAtSelf"` } type LocationElem struct { Description string `mapstructure:"description"` @@ -158,3 +160,8 @@ type MsgStruct struct { CustomElem *CustomElem `json:"customElem,omitempty"` QuoteElem *QuoteElem `json:"quoteElem,omitempty"` } + +type AtInfo struct { + AtUserID string `json:"atUserID,omitempty"` + GroupNickname string `json:"groupNickname,omitempty"` +} From becc999d6302805e95df4c8ada636fa3a7f12d61 Mon Sep 17 00:00:00 2001 From: Monet Lee Date: Mon, 28 Jul 2025 11:29:51 +0800 Subject: [PATCH 189/199] fix: solve batch incorrect error in Find DocIDs (#3476) --- pkg/common/storage/controller/msg.go | 10 ++++++++-- pkg/common/storage/model/msg.go | 12 ++++-------- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/pkg/common/storage/controller/msg.go b/pkg/common/storage/controller/msg.go index 53dd7f13d..f833008e8 100644 --- a/pkg/common/storage/controller/msg.go +++ b/pkg/common/storage/controller/msg.go @@ -671,12 +671,18 @@ func (db *commonMsgDatabase) SearchMessage(ctx context.Context, req *pbmsg.Searc func (db *commonMsgDatabase) FindOneByDocIDs(ctx context.Context, conversationIDs []string, seqs map[string]int64) (map[string]*sdkws.MsgData, error) { totalMsgs := make(map[string]*sdkws.MsgData) for _, conversationID := range conversationIDs { - seq := seqs[conversationID] + seq, ok := seqs[conversationID] + if !ok { + log.ZWarn(ctx, "seq not found for conversationID", errs.New("seq not found for conversation"), "conversationID", conversationID) + continue + } docID := db.msgTable.GetDocID(conversationID, seq) msgs, err := db.msgDocDatabase.FindOneByDocID(ctx, docID) if err != nil { - return nil, err + log.ZWarn(ctx, "FindOneByDocID failed", err, "conversationID", conversationID, "docID", docID, "seq", seq) + continue } + index := db.msgTable.GetMsgIndex(seq) totalMsgs[conversationID] = convert.MsgDB2Pb(msgs.Msg[index].Msg) } diff --git a/pkg/common/storage/model/msg.go b/pkg/common/storage/model/msg.go index 6cf63bfcd..9d5b56b42 100644 --- a/pkg/common/storage/model/msg.go +++ b/pkg/common/storage/model/msg.go @@ -120,15 +120,11 @@ func (m *MsgDocModel) GetDocID(conversationID string, seq int64) string { func (m *MsgDocModel) GetDocIDSeqsMap(conversationID string, seqs []int64) map[string][]int64 { t := make(map[string][]int64) - for i := 0; i < len(seqs); i++ { - docID := m.GetDocID(conversationID, seqs[i]) - if value, ok := t[docID]; !ok { - var temp []int64 - t[docID] = append(temp, seqs[i]) - } else { - t[docID] = append(value, seqs[i]) - } + for _, seq := range seqs { + docID := m.GetDocID(conversationID, seq) + t[docID] = append(t[docID], seq) } + return t } From d6cd0258a59f3863b0c5ec77759378f32e44f8df Mon Sep 17 00:00:00 2001 From: OpenIM-Gordon <1432970085@qq.com> Date: Mon, 28 Jul 2025 12:11:38 +0800 Subject: [PATCH 190/199] fix: correctly aggregate read seqs by conversation and user before DB update. (#3442) * build: docker compose file add some comments. * fix: correctly aggregate read seqs by conversation and user before DB update. --- docker-compose.yml | 29 ++++++++-- .../msgtransfer/online_history_msg_handler.go | 56 +++++++++---------- 2 files changed, 51 insertions(+), 34 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 60fc865f2..233348619 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -176,17 +176,38 @@ services: environment: #KAFKA_HEAP_OPTS: "-Xms128m -Xmx256m" TZ: Asia/Shanghai + # Unique identifier for the Kafka node (required in controller mode) KAFKA_CFG_NODE_ID: 0 + # Defines the roles this Kafka node plays: broker, controller, or both KAFKA_CFG_PROCESS_ROLES: controller,broker + # Specifies which nodes are controller nodes for quorum voting. + # The syntax follows the KRaft mode (no ZooKeeper): node.id@host:port + # The controller listener endpoint here is kafka:9093 KAFKA_CFG_CONTROLLER_QUORUM_VOTERS: 0@kafka:9093 + # Specifies which listener is used for controller-to-controller communication KAFKA_CFG_CONTROLLER_LISTENER_NAMES: CONTROLLER + # Default number of partitions for new topics KAFKA_NUM_PARTITIONS: 8 + # Whether to enable automatic topic creation KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE: "true" + # Kafka internal listeners; Kafka supports multiple ports with different protocols + # Each port is used for a specific purpose: INTERNAL for internal broker communication, + # CONTROLLER for controller communication, EXTERNAL for external client connections. + # These logical listener names are mapped to actual protocols via KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP + # In short, Kafka is listening on three logical ports: 9092 for internal communication, + # 9093 for controller traffic, and 9094 for external access. + KAFKA_CFG_LISTENERS: "INTERNAL://:9092,CONTROLLER://:9093,EXTERNAL://:9094" + # Addresses advertised to clients. INTERNAL://kafka:9092 uses the internal Docker service name 'kafka', + # so other containers can access Kafka via kafka:9092. + # EXTERNAL://localhost:19094 is the address external clients (e.g., in the LAN) should use to connect. + # If Kafka is deployed on a different machine than IM, 'localhost' should be replaced with the LAN IP. + KAFKA_CFG_ADVERTISED_LISTENERS: "INTERNAL://kafka:9092,EXTERNAL://localhost:19094" + # Maps logical listener names to actual protocols. + # Supported protocols include: PLAINTEXT, SSL, SASL_PLAINTEXT, SASL_SSL + KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: "CONTROLLER:PLAINTEXT,EXTERNAL:PLAINTEXT,INTERNAL:PLAINTEXT" + # Defines which listener is used for inter-broker communication within the Kafka cluster + KAFKA_CFG_INTER_BROKER_LISTENER_NAME: "INTERNAL" - KAFKA_CFG_LISTENERS: "PLAINTEXT://:9092,CONTROLLER://:9093,EXTERNAL://:9094" - KAFKA_CFG_ADVERTISED_LISTENERS: "PLAINTEXT://kafka:9092,EXTERNAL://localhost:19094" - KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: "CONTROLLER:PLAINTEXT,EXTERNAL:PLAINTEXT,PLAINTEXT:PLAINTEXT" - KAFKA_CFG_INTER_BROKER_LISTENER_NAME: "PLAINTEXT" # Authentication configuration variables - comment out to disable auth # KAFKA_USERNAME: "openIM" diff --git a/internal/msgtransfer/online_history_msg_handler.go b/internal/msgtransfer/online_history_msg_handler.go index a2d0cca67..05775a1e6 100644 --- a/internal/msgtransfer/online_history_msg_handler.go +++ b/internal/msgtransfer/online_history_msg_handler.go @@ -26,6 +26,8 @@ import ( "github.com/openimsdk/tools/discovery" "github.com/go-redis/redis" + "google.golang.org/protobuf/proto" + "github.com/openimsdk/open-im-server/v3/pkg/common/prommetrics" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/controller" "github.com/openimsdk/open-im-server/v3/pkg/msgprocessor" @@ -37,7 +39,6 @@ import ( "github.com/openimsdk/tools/log" "github.com/openimsdk/tools/mcontext" "github.com/openimsdk/tools/utils/stringutil" - "google.golang.org/protobuf/proto" ) const ( @@ -134,53 +135,48 @@ func (och *OnlineHistoryRedisConsumerHandler) do(ctx context.Context, channelID func (och *OnlineHistoryRedisConsumerHandler) doSetReadSeq(ctx context.Context, msgs []*ContextMsg) { - var conversationID string - var userSeqMap map[string]int64 + // Outer map: conversationID -> (userID -> maxHasReadSeq) + conversationUserSeq := make(map[string]map[string]int64) + for _, msg := range msgs { if msg.message.ContentType != constant.HasReadReceipt { continue } var elem sdkws.NotificationElem if err := json.Unmarshal(msg.message.Content, &elem); err != nil { - log.ZWarn(ctx, "handlerConversationRead Unmarshal NotificationElem msg err", err, "msg", msg) + log.ZWarn(ctx, "Unmarshal NotificationElem error", err, "msg", msg) continue } var tips sdkws.MarkAsReadTips if err := json.Unmarshal([]byte(elem.Detail), &tips); err != nil { - log.ZWarn(ctx, "handlerConversationRead Unmarshal MarkAsReadTips msg err", err, "msg", msg) + log.ZWarn(ctx, "Unmarshal MarkAsReadTips error", err, "msg", msg) continue } - //The conversation ID for each batch of messages processed by the batcher is the same. - conversationID = tips.ConversationID - if len(tips.Seqs) > 0 { - for _, seq := range tips.Seqs { - if tips.HasReadSeq < seq { - tips.HasReadSeq = seq - } - } - clear(tips.Seqs) - tips.Seqs = nil - } - if tips.HasReadSeq < 0 { + if len(tips.ConversationID) == 0 || tips.HasReadSeq < 0 { continue } - if userSeqMap == nil { - userSeqMap = make(map[string]int64) + + // Calculate the max seq from tips.Seqs + for _, seq := range tips.Seqs { + if tips.HasReadSeq < seq { + tips.HasReadSeq = seq + } } - if userSeqMap[tips.MarkAsReadUserID] > tips.HasReadSeq { - continue + if _, ok := conversationUserSeq[tips.ConversationID]; !ok { + conversationUserSeq[tips.ConversationID] = make(map[string]int64) + } + if conversationUserSeq[tips.ConversationID][tips.MarkAsReadUserID] < tips.HasReadSeq { + conversationUserSeq[tips.ConversationID][tips.MarkAsReadUserID] = tips.HasReadSeq } - userSeqMap[tips.MarkAsReadUserID] = tips.HasReadSeq - } - if userSeqMap == nil { - return - } - if len(conversationID) == 0 { - log.ZWarn(ctx, "conversation err", nil, "conversationID", conversationID) } - if err := och.msgTransferDatabase.SetHasReadSeqToDB(ctx, conversationID, userSeqMap); err != nil { - log.ZWarn(ctx, "set read seq to db error", err, "conversationID", conversationID, "userSeqMap", userSeqMap) + log.ZInfo(ctx, "doSetReadSeq", "conversationUserSeq", conversationUserSeq) + + // persist to db + for convID, userSeqMap := range conversationUserSeq { + if err := och.msgTransferDatabase.SetHasReadSeqToDB(ctx, convID, userSeqMap); err != nil { + log.ZWarn(ctx, "SetHasReadSeqToDB error", err, "conversationID", convID, "userSeqMap", userSeqMap) + } } } From 4c9fdf70db2a7f2b0cbe4be3857f46a3f056ac60 Mon Sep 17 00:00:00 2001 From: Monet Lee Date: Mon, 28 Jul 2025 14:22:40 +0800 Subject: [PATCH 191/199] feat: support mongo replicaset mode. (#3433) * feat: support mongo replicaset mode. * fix mongo config. --- config/mongodb.yml | 39 +++++++++++++++++++++++-- go.mod | 2 +- go.sum | 4 +-- pkg/common/config/config.go | 57 +++++++++++++++++++++++++++++++------ 4 files changed, 89 insertions(+), 13 deletions(-) diff --git a/config/mongodb.yml b/config/mongodb.yml index 072cb4b8f..ca45fea6f 100644 --- a/config/mongodb.yml +++ b/config/mongodb.yml @@ -1,7 +1,7 @@ # URI for database connection, leave empty if using address and credential settings directly -uri: +uri: # List of MongoDB server addresses -address: [ localhost:37017 ] +address: [localhost:37017] # Name of the database database: openim_v3 # Username for database authentication @@ -14,3 +14,38 @@ authSource: openim_v3 maxPoolSize: 100 # Maximum number of retry attempts for a failed database connection maxRetry: 10 +# MongoDB Mode, including "standalone", "replicaSet" +mongoMode: "standalone" + +# The following configurations only take effect when mongoMode is set to "replicaSet" +replicaSet: + name: rs0 + hosts: [127.0.0.1:37017, 127.0.0.1:37018, 127.0.0.1:37019] + # Read concern level: "local", "available", "majority", "linearizable", "snapshot" + readConcern: majority + # maximum staleness of data in seconds + maxStaleness: 90s + +# The following configurations only take effect when mongoMode is set to "replicaSet" +readPreference: + # Read preference mode, can be "primary", "primaryPreferred", "secondary", "secondaryPreferred", "nearest" + mode: primary + maxStaleness: 90s + # TagSets is an array of maps with priority based on order, empty map must be placed last for fallback tagSets + tagSets: + - datacenter: "cn-east" + rack: "1" + storage: "ssd" + - datacenter: "cn-east" + storage: "ssd" + - datacenter: "cn-east" + - {} # Empty map, indicates any node + +# The following configurations only take effect when mongoMode is set to "replicaSet" +writeConcern: + # Write node count or tag (int, "majority", or custom tag) + w: majority + # Whether to wait for journal confirmation + j: true + # Write timeout duration + wtimeout: 30s diff --git a/go.mod b/go.mod index badf4ffaa..c06451aaa 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/mitchellh/mapstructure v1.5.0 github.com/openimsdk/protocol v0.0.73-alpha.12 - github.com/openimsdk/tools v0.0.50-alpha.92 + github.com/openimsdk/tools v0.0.50-alpha.96 github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 github.com/stretchr/testify v1.9.0 diff --git a/go.sum b/go.sum index ab0499ba0..d29eb0f3f 100644 --- a/go.sum +++ b/go.sum @@ -349,8 +349,8 @@ github.com/openimsdk/gomake v0.0.15-alpha.11 h1:PQudYDRESYeYlUYrrLLJhYIlUPO5x7FA github.com/openimsdk/gomake v0.0.15-alpha.11/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= github.com/openimsdk/protocol v0.0.73-alpha.12 h1:2NYawXeHChYUeSme6QJ9pOLh+Empce2WmwEtbP4JvKk= github.com/openimsdk/protocol v0.0.73-alpha.12/go.mod h1:WF7EuE55vQvpyUAzDXcqg+B+446xQyEba0X35lTINmw= -github.com/openimsdk/tools v0.0.50-alpha.92 h1:hWfykMhmi7EQEiwgQccJqbgggIuhun/PrVkBnjmj9Ec= -github.com/openimsdk/tools v0.0.50-alpha.92/go.mod h1:n2poR3asX1e1XZce4O+MOWAp+X02QJRFvhcLCXZdzRo= +github.com/openimsdk/tools v0.0.50-alpha.96 h1:U44Fq2jHiEvGi9zuYAnTRNx3Xd9T7P/kBAZLHvQ8xg4= +github.com/openimsdk/tools v0.0.50-alpha.96/go.mod h1:n2poR3asX1e1XZce4O+MOWAp+X02QJRFvhcLCXZdzRo= github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= diff --git a/pkg/common/config/config.go b/pkg/common/config/config.go index 43914310e..63716838a 100644 --- a/pkg/common/config/config.go +++ b/pkg/common/config/config.go @@ -71,15 +71,39 @@ type Minio struct { } type Mongo struct { - URI string `yaml:"uri"` - Address []string `yaml:"address"` - Database string `yaml:"database"` - Username string `yaml:"username"` - Password string `yaml:"password"` - AuthSource string `yaml:"authSource"` - MaxPoolSize int `yaml:"maxPoolSize"` - MaxRetry int `yaml:"maxRetry"` + URI string `yaml:"uri"` + Address []string `yaml:"address"` + Database string `yaml:"database"` + Username string `yaml:"username"` + Password string `yaml:"password"` + AuthSource string `yaml:"authSource"` + MaxPoolSize int `yaml:"maxPoolSize"` + MaxRetry int `yaml:"maxRetry"` + MongoMode string `yaml:"mongoMode"` + ReplicaSet ReplicaSetConfig + ReadPreference ReadPrefConfig + WriteConcern WriteConcernConfig } + +type ReplicaSetConfig struct { + Name string `yaml:"name"` + Hosts []string `yaml:"hosts"` + ReadConcern string `yaml:"readConcern"` + MaxStaleness time.Duration `yaml:"maxStaleness"` +} + +type ReadPrefConfig struct { + Mode string `yaml:"mode"` + TagSets []map[string]string `yaml:"tagSets"` + MaxStaleness time.Duration `yaml:"maxStaleness"` +} + +type WriteConcernConfig struct { + W any `yaml:"w"` + J bool `yaml:"j"` + WTimeout time.Duration `yaml:"wtimeout"` +} + type Kafka struct { Username string `yaml:"username"` Password string `yaml:"password"` @@ -493,6 +517,23 @@ func (m *Mongo) Build() *mongoutil.Config { AuthSource: m.AuthSource, MaxPoolSize: m.MaxPoolSize, MaxRetry: m.MaxRetry, + MongoMode: m.MongoMode, + ReplicaSet: &mongoutil.ReplicaSetConfig{ + Name: m.ReplicaSet.Name, + Hosts: m.ReplicaSet.Hosts, + ReadConcern: m.ReplicaSet.ReadConcern, + MaxStaleness: m.ReplicaSet.MaxStaleness, + }, + ReadPreference: &mongoutil.ReadPrefConfig{ + Mode: m.ReadPreference.Mode, + TagSets: m.ReadPreference.TagSets, + MaxStaleness: m.ReadPreference.MaxStaleness, + }, + WriteConcern: &mongoutil.WriteConcernConfig{ + W: m.WriteConcern.W, + J: m.WriteConcern.J, + WTimeout: m.WriteConcern.WTimeout, + }, } } From 80de08e8ed0bb005eb7dead7d5a007858df4a5fd Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 28 Jul 2025 15:29:41 +0800 Subject: [PATCH 192/199] Update CHANGELOG for release v3.8.3-patch.5 (#3405) Co-authored-by: mo3et <34803812+mo3et@users.noreply.github.com> Co-authored-by: Monet Lee --- CHANGELOG/CHANGELOG-3.8.md | 105 +++++++++++++++++++++---------------- 1 file changed, 59 insertions(+), 46 deletions(-) diff --git a/CHANGELOG/CHANGELOG-3.8.md b/CHANGELOG/CHANGELOG-3.8.md index c4748e262..8d87fe07b 100644 --- a/CHANGELOG/CHANGELOG-3.8.md +++ b/CHANGELOG/CHANGELOG-3.8.md @@ -1,47 +1,60 @@ -## [v3.8.3-patch.4](https://github.com/openimsdk/open-im-server/releases/tag/v3.8.3-patch.4) (2025-03-13) - -### Bug Fixes -* fix: solve unocrrect invite notificationfrom #3213 - -**Full Changelog**: [v3.8.3-patch.3...v3.8.3-patch.4](https://github.com/openimsdk/open-im-server/compare/v3.8.3-patch.3...v3.8.3-patch.4) - -## [v3.8.3-patch.3](https://github.com/openimsdk/open-im-server/releases/tag/v3.8.3-patch.3) (2025-03-07) - -### New Features -* feat: optimizing BatchGetIncrementalGroupMember #3180 - -### Bug Fixes -* fix: solve uncorrect notification when set group info #3172 -* fix: the sorting is wrong after canceling the administrator in group settings #3185 -* fix: solve uncorrect GroupMember enter group notification type. #3188 - -### Refactors -* refactor: change sendNotification to sendMessage to avoid ambiguity regarding message sending behavior. #3173 - -**Full Changelog**: [v3.8.3-patch.2...v3.8.3-patch.3](https://github.com/openimsdk/open-im-server/compare/v3.8.3-patch.2...v3.8.3-patch.3) - -## [v3.8.3-patch.2](https://github.com/openimsdk/open-im-server/releases/tag/v3.8.3-patch.2) (2025-02-28) - -### Bug Fixes -* fix: Offline push does not have a badge && Android offline push (#3146) [#3174](https://github.com/openimsdk/open-im-server/pull/3174) - -**Full Changelog**: [v3.8.3-patch.1...v3.8.3-patch.2](https://github.com/openimsdk/open-im-server/compare/v3.8.3-patch.1...v3.8.3-patch.2) - -## [v3.8.3-patch.1](https://github.com/openimsdk/open-im-server/releases/tag/v3.8.3-patch.1) (2025-02-25) - -### New Features -* feat: add backup volume && optimize log print [Created [#3121](https://github.com/openimsdk/open-im-server/pull/3121) - -### Bug Fixes -* fix: seq conversion failed without exiting [Created [#3120](https://github.com/openimsdk/open-im-server/pull/3120) -* fix: check error in BatchSetTokenMapByUidPid [Created [#3123](https://github.com/openimsdk/open-im-server/pull/3123) -* fix: DeleteDoc crash [Created [#3124](https://github.com/openimsdk/open-im-server/pull/3124) -* fix: the abnormal message has no sending time, causing the SDK to be abnormal [Created [#3126](https://github.com/openimsdk/open-im-server/pull/3126) -* fix: crash caused [#3127](https://github.com/openimsdk/open-im-server/pull/3127) -* fix: the user sets the conversation timer cleanup timestamp unit incorrectly [Created [#3128](https://github.com/openimsdk/open-im-server/pull/3128) -* fix: seq conversion not reading env in docker environment [Created [#3131](https://github.com/openimsdk/open-im-server/pull/3131) - -### Builds -* build: improve workflows contents. [Created [#3125](https://github.com/openimsdk/open-im-server/pull/3125) - +## [v3.8.3-patch.5](https://github.com/openimsdk/open-im-server/releases/tag/v3.8.3-patch.5) (2025-06-10) + +### New Features +* feat: optimize friend and group applications [#3396](https://github.com/openimsdk/open-im-server/pull/3396) + +### Bug Fixes +* fix: solve unocrrect invite notification [Created [#3219](https://github.com/openimsdk/open-im-server/pull/3219) + +### Builds +* build: update gomake version in dockerfile.[Patch branch] [#3416](https://github.com/openimsdk/open-im-server/pull/3416) + +**Full Changelog**: [v3.8.3...v3.8.3-patch.5](https://github.com/openimsdk/open-im-server/compare/v3.8.3...v3.8.3-patch.5) + +## [v3.8.3-patch.4](https://github.com/openimsdk/open-im-server/releases/tag/v3.8.3-patch.4) (2025-03-13) + +### Bug Fixes +* fix: solve unocrrect invite notificationfrom #3213 + +**Full Changelog**: [v3.8.3-patch.3...v3.8.3-patch.4](https://github.com/openimsdk/open-im-server/compare/v3.8.3-patch.3...v3.8.3-patch.4) + +## [v3.8.3-patch.3](https://github.com/openimsdk/open-im-server/releases/tag/v3.8.3-patch.3) (2025-03-07) + +### New Features +* feat: optimizing BatchGetIncrementalGroupMember #3180 + +### Bug Fixes +* fix: solve uncorrect notification when set group info #3172 +* fix: the sorting is wrong after canceling the administrator in group settings #3185 +* fix: solve uncorrect GroupMember enter group notification type. #3188 + +### Refactors +* refactor: change sendNotification to sendMessage to avoid ambiguity regarding message sending behavior. #3173 + +**Full Changelog**: [v3.8.3-patch.2...v3.8.3-patch.3](https://github.com/openimsdk/open-im-server/compare/v3.8.3-patch.2...v3.8.3-patch.3) + +## [v3.8.3-patch.2](https://github.com/openimsdk/open-im-server/releases/tag/v3.8.3-patch.2) (2025-02-28) + +### Bug Fixes +* fix: Offline push does not have a badge && Android offline push (#3146) [#3174](https://github.com/openimsdk/open-im-server/pull/3174) + +**Full Changelog**: [v3.8.3-patch.1...v3.8.3-patch.2](https://github.com/openimsdk/open-im-server/compare/v3.8.3-patch.1...v3.8.3-patch.2) + +## [v3.8.3-patch.1](https://github.com/openimsdk/open-im-server/releases/tag/v3.8.3-patch.1) (2025-02-25) + +### New Features +* feat: add backup volume && optimize log print [Created [#3121](https://github.com/openimsdk/open-im-server/pull/3121) + +### Bug Fixes +* fix: seq conversion failed without exiting [Created [#3120](https://github.com/openimsdk/open-im-server/pull/3120) +* fix: check error in BatchSetTokenMapByUidPid [Created [#3123](https://github.com/openimsdk/open-im-server/pull/3123) +* fix: DeleteDoc crash [Created [#3124](https://github.com/openimsdk/open-im-server/pull/3124) +* fix: the abnormal message has no sending time, causing the SDK to be abnormal [Created [#3126](https://github.com/openimsdk/open-im-server/pull/3126) +* fix: crash caused [#3127](https://github.com/openimsdk/open-im-server/pull/3127) +* fix: the user sets the conversation timer cleanup timestamp unit incorrectly [Created [#3128](https://github.com/openimsdk/open-im-server/pull/3128) +* fix: seq conversion not reading env in docker environment [Created [#3131](https://github.com/openimsdk/open-im-server/pull/3131) + +### Builds +* build: improve workflows contents. [Created [#3125](https://github.com/openimsdk/open-im-server/pull/3125) + **Full Changelog**: [v3.8.3-e-v1.1.5...v3.8.3-patch.1-e-v1.1.5](https://github.com/openimsdk/open-im-server-enterprise/compare/v3.8.3-e-v1.1.5...v3.8.3-patch.1-e-v1.1.5) \ No newline at end of file From b19b81b10ece683f838bf6e7d0bf4a1e5f4dbfde Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 28 Jul 2025 15:33:05 +0800 Subject: [PATCH 193/199] Update CHANGELOG for release v3.8.3-patch.6 (#3473) Co-authored-by: FGadvancer <46924906+FGadvancer@users.noreply.github.com> Co-authored-by: Monet Lee --- CHANGELOG/CHANGELOG-3.8.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/CHANGELOG/CHANGELOG-3.8.md b/CHANGELOG/CHANGELOG-3.8.md index 8d87fe07b..9ae41e6a3 100644 --- a/CHANGELOG/CHANGELOG-3.8.md +++ b/CHANGELOG/CHANGELOG-3.8.md @@ -1,3 +1,13 @@ +## [v3.8.3-patch.6](https://github.com/openimsdk/open-im-server/releases/tag/v3.8.3-patch.6) (2025-07-23) + +### Bug Fixes +* fix: Add friend DB in notification sender [#3438](https://github.com/openimsdk/open-im-server/pull/3438) +* fix: remove update version file workflows have new line in 3.8.3-patch branch. [#3452](https://github.com/openimsdk/open-im-server/pull/3452) +* fix: s3 aws init [#3454](https://github.com/openimsdk/open-im-server/pull/3454) +* fix: use safe submodule init in workflows in v3.8.3-patch. [#3469](https://github.com/openimsdk/open-im-server/pull/3469) + +**Full Changelog**: [v3.8.3-patch.5...v3.8.3-patch.6](https://github.com/openimsdk/open-im-server/compare/v3.8.3-patch.5...v3.8.3-patch.6) + ## [v3.8.3-patch.5](https://github.com/openimsdk/open-im-server/releases/tag/v3.8.3-patch.5) (2025-06-10) ### New Features From b44e56b15124601d34b0c13e604d90d66b3ee38b Mon Sep 17 00:00:00 2001 From: OpenIM-Gordon <1432970085@qq.com> Date: Mon, 28 Jul 2025 15:51:26 +0800 Subject: [PATCH 194/199] docs: update readme of config file. (#3356) --- config/README.md | 116 +++++++++++++++++++++++++---------------- config/README_zh_CN.md | 93 ++++++++++++++++++++------------- 2 files changed, 127 insertions(+), 82 deletions(-) diff --git a/config/README.md b/config/README.md index 048d7fe36..eff2bb9e9 100644 --- a/config/README.md +++ b/config/README.md @@ -1,48 +1,66 @@ ---- -title: 'OpenIM Configuration Files and Common Configuration Item Modifications Guide' - -## Configuration Files Explanation - -| Configuration File | Description | -| ------------------------------- | ------------------------------------------------------------ | -| **kafka.yml** | Configurations for Kafka username, password, address, etc. | -| **redis.yml** | Configurations for Redis password, address, etc. | -| **minio.yml** | Configurations for MinIO username, password, address, and external IP/domain; failing to modify external IP or domain may cause image file sending failures | -| **zookeeper.yml** | Configurations for ZooKeeper user, password, address, etc. | -| **mongodb.yml** | Configurations for MongoDB username, password, address, etc. | -| **log.yml** | Configurations for log level and storage directory. | -| **notification.yml** | Configurations for events like adding friends, creating groups, etc. | -| **share.yml** | Common configurations needed by various OpenIM services, such as secret. | -| **webhooks.yml** | Configurations for URLs in Webhook. | -| **local-cache.yml** | Local cache configurations. | -| **openim-rpc-third.yml** | Configurations for listening IP, port, and storage settings for images and videos in openim-rpc-third service. | -| **openim-rpc-user.yml** | Configurations for listening IP and port in openim-rpc-user service. | -| **openim-api.yml** | Configurations for listening IP, port, etc., in openim-api service. | -| **openim-crontask.yml** | Configurations for openim-crontask service. | -| **openim-msggateway.yml** | Configurations for listening IP, port, etc., in openim-msggateway service. | -| **openim-msgtransfer.yml** | Configurations for openim-msgtransfer service. | -| **openim-push.yml** | Configurations for listening IP, port, and offline push settings in openim-push service. | -| **openim-rpc-auth.yml** | Configurations for listening IP, port, and token expiration settings in openim-rpc-auth service. | -| **openim-rpc-conversation.yml** | Configurations for listening IP, port, etc., in openim-rpc-conversation service. | -| **openim-rpc-friend.yml** | Configurations for listening IP, port, etc., in openim-rpc-friend service. | -| **openim-rpc-group.yml** | Configurations for listening IP, port, etc., in openim-rpc-group service. | -| **openim-rpc-msg.yml** | Configurations for listening IP, port, and whether to verify friendship before sending messages in openim-rpc-msg service. | - -## Common Configuration Item Modifications - -| Configuration Item Modification | Configuration File | -| ----------------------------------------------------- | ----------------------- | -| Using MinIO for image and video file object storage | `minio.yml` | -| Adjusting production environment logs | `log.yml` | -| Verifying friendship before sending messages | `openim-rpc-msg.yml` | -| Modifying secret | `share.yml` | -| Using OSS, COS, AWS, Kodo for image and video storage | `openim-rpc-third.yml` | -| Setting multiple login policy | `openim-msggateway.yml` | -| Setting up offline push | `openim-push.yml` | - -## Starting Multiple Instances of an OpenIM Service - -To start multiple instances of an OpenIM service, simply increase the corresponding port numbers and modify the `start-config.yml` file in the project root directory. Restart the service to take effect. For example, the configuration to start 2 instances of `openim-rpc-user` is as follows: +# OpenIM Configuration File Descriptions and Common Configuration Modifications + +## External Component Configurations + +| Configuration File | Description | +| ------------------ |-------------------------------------------------------------| +| **kafka.yml** | Configuration for Kafka username, password, address, etc. | +| **redis.yml** | Configuration for Redis password, address, etc. | +| **minio.yml** | Configuration for MinIO username, password, address, etc. | +| **mongodb.yml** | Configuration for MongoDB username, password, address, etc. | +| **discovery.yml** | Service discovery and etcd credentials and address. | + +## OpenIMServer Related Configurations +| Configuration File | Description | +| ------------------------------- | ---------------------------------------------- | +| **log.yml** | Configuration for logging levels and storage directory | +| **notification.yml** | Event notification settings (e.g., add friend, create group) | +| **share.yml** | Common settings for all services (e.g., secrets) | +| **webhooks.yml** | Webhook URLs and related settings | +| **local-cache.yml** | Local cache settings (generally do not modify) | +| **openim-rpc-third.yml** | openim-rpc-third listen IP, port, and object storage settings | +| **openim-rpc-user.yml** | openim-rpc-user listen IP and port settings | +| **openim-api.yml** | openim-api listen IP, port, and other settings | +| **openim-crontask.yml** | openim-crontask scheduled task settings | +| **openim-msggateway.yml** | openim-msggateway listen IP, port, and other settings | +| **openim-msgtransfer.yml** | Settings for openim-msgtransfer service | +| **openim-push.yml** | openim-push listen IP, port, and offline push settings | +| **openim-rpc-auth.yml** | openim-rpc-auth listen IP, port, token validity settings | +| **openim-rpc-conversation.yml** | openim-rpc-conversation listen IP and port settings | +| **openim-rpc-friend.yml** | openim-rpc-friend listen IP and port settings | +| **openim-rpc-group.yml** | openim-rpc-group listen IP and port settings | +| **openim-rpc-msg.yml** | openim-rpc-msg listen IP and port settings | + + +## Monitoring and Alerting Related Configurations +| Configuration File | Description | +| ------------------------------ | --------------- | +| **prometheus.yml** | Prometheus configuration | +| **instance-down-rules.yml** | Alert rules | +| **alertmanager.yml** | Alertmanager configuration | +| **email.tmpl** | Email alert template | +| **grefana-template/Demo.json** | Default Grafana dashboard | + +## Common Configuration Modifications +| Configuration Item | Configuration File | +| -------------------------------------------------------- | ----------------------- | +| Configure MinIO as object storage (focus on the externalAddress field) | `minio.yml` | +| Adjust log level and number of log files | `log.yml` | +| Enable or disable friend verification when sending messages | `openim-rpc-msg.yml` | +| OpenIMServer secret | `share.yml` | +| Configure OSS, COS, AWS, or Kodo as object storage | `openim-rpc-third.yml` | +| Multi-end mutual kick strategy and max concurrent connections per gateway | `openim-msggateway.yml` | +| Offline message push configuration | `openim-push.yml` | +| Configure webhooks for callback notifications (e.g., before/after message send) | `webhooks.yml` | +| Whether new group members can view historical messages | `openim-rpc-group.yml` | +| Token expiration time settings | `openim-rpc-auth.yml` | +| Scheduled task settings (e.g., how long to retain messages) | `openim-crontask.yml` | + +## Starting Multiple Instances of a Service and Maximum File Descriptors + + +To start multiple instances of an OpenIM service, simply add the corresponding port numbers and modify the `start-config.yml` file in the project’s root directory, +then restart the service. For example, to start 2 instances of `openim-rpc-user`: ```yaml rpc: @@ -55,9 +73,15 @@ prometheus: ports: [ 20100, 20101 ] ``` -Modify `start-config.yml`: +Modify`start-config.yml`: ```yaml serviceBinaries: openim-rpc-user: 2 ``` + +To set the maximum number of open file descriptors (typically one per online user): + +``` +maxFileDescriptors: 10000 +``` diff --git a/config/README_zh_CN.md b/config/README_zh_CN.md index 6ddab0636..679bfe5f5 100644 --- a/config/README_zh_CN.md +++ b/config/README_zh_CN.md @@ -1,45 +1,63 @@ # OpenIM配置文件说明以及常用配置修改说明 -## 配置文件说明 +## 外部组件相关配置 -| Configuration File | Description | -| ------------------------------- | ------------------------------------------------------------ | -| **kafka.yml** | Kafka用户名、密码、地址等配置 | -| **redis.yml** | Redis密码、地址等配置 | -| **minio.yml** | MinIO用户名、密码、地址及外网IP域名等配置；未修改外网IP或域名可能导致图片文件发送失败 | -| **zookeeper.yml** | ZooKeeper用户、密码、地址等配置 | -| **mongodb.yml** | MongoDB用户名、密码、地址等配置 | -| **log.yml** | 日志级别及存储目录等配置 | -| **notification.yml** | 添加好友、创建群组等事件通知配置 | -| **share.yml** | OpenIM各服务所需的公共配置，如secret等 | -| **webhooks.yml** | Webhook中URL等配置 | -| **local-cache.yml** | 本地缓存配置 | -| **openim-rpc-third.yml** | openim-rpc-third服务的监听IP、端口及图片视频对象存储配置 | -| **openim-rpc-user.yml** | openim-rpc-user服务的监听IP、端口配置 | -| **openim-api.yml** | openim-api服务的监听IP、端口等配置项 | -| **openim-crontask.yml** | openim-crontask服务配置 | -| **openim-msggateway.yml** | openim-msggateway服务的监听IP、端口等配置 | -| **openim-msgtransfer.yml** | openim-msgtransfer服务配置 | -| **openim-push.yml** | openim-push服务的监听IP、端口及离线推送配置 | -| **openim-rpc-auth.yml** | openim-rpc-auth服务的监听IP、端口及token有效期等配置 | -| **openim-rpc-conversation.yml** | openim-rpc-conversation服务的监听IP、端口等配置 | -| **openim-rpc-friend.yml** | openim-rpc-friend服务的监听IP、端口等配置 | -| **openim-rpc-group.yml** | openim-rpc-group服务的监听IP、端口等配置 | -| **openim-rpc-msg.yml** | openim-rpc-msg服务的监听IP、端口及消息发送是否验证好友关系等配置 | +| Configuration File | Description | +| ------------------ | ---------------------------------- | +| **kafka.yml** | Kafka用户名、密码、地址等配置 | +| **redis.yml** | Redis密码、地址等配置 | +| **minio.yml** | MinIO用户名、密码、地址等配置 | +| **mongodb.yml** | MongoDB用户名、密码、地址等配置 | +| **discovery.yml** | 服务发现以及etcd用户名、密码、地址 | + +## OpenIMServer相关配置 +| Configuration File | Description | +| ------------------------------- | ---------------------------------------------- | +| **log.yml** | 日志级别及存储目录等配置 | +| **notification.yml** | 添加好友、创建群组等事件通知配置 | +| **share.yml** | 各服务所需的公共配置，如secret等 | +| **webhooks.yml** | Webhook中URL等配置 | +| **local-cache.yml** | 本地缓存配置，一般不用修改 | +| **openim-rpc-third.yml** | openim-rpc-third监听IP、端口及对象存储配置 | +| **openim-rpc-user.yml** | openim-rpc-user监听IP、端口配置 | +| **openim-api.yml** | openim-api监听IP、端口等配置 | +| **openim-crontask.yml** | openim-crontask定时任务配置 | +| **openim-msggateway.yml** | openim-msggateway监听IP、端口等配置 | +| **openim-msgtransfer.yml** | openim-msgtransfer服务配置 | +| **openim-push.yml** | openim-push监听IP、端口及离线推送配置 | +| **openim-rpc-auth.yml** | openim-rpc-auth监听IP、端口及token有效期等配置 | +| **openim-rpc-conversation.yml** | openim-rpc-conversation监听IP、端口等配置 | +| **openim-rpc-friend.yml** | openim-rpc-friend监听IP、端口等配置 | +| **openim-rpc-group.yml** | openim-rpc-group监听IP、端口等配置 | +| **openim-rpc-msg.yml** | openim-rpc-msg服务的监听IP、端口等配置 | + + +## 监控告警相关配置 +| Configuration File | Description | +| ------------------------------ | --------------- | +| **prometheus.yml** | prometheus配置 | +| **instance-down-rules.yml** | 告警规则 | +| **alertmanager.yml** | 告警管理配置 | +| **email.tmpl** | 邮件告警模版 | +| **grefana-template/Demo.json** | 默认的dashboard | ## 常用配置修改 +| 修改配置项 | 配置文件 | +| -------------------------------------------------------- | ----------------------- | +| 使用minio作为对象存储时配置，重点关注externalAddress字段 | `minio.yml` | +| 日志级别及日志文件数量调整 | `log.yml` | +| 发送消息是否需要验证好友关系 | `openim-rpc-msg.yml` | +| OpenIMServer秘钥 | `share.yml` | +| 使用oss, cos, aws, kodo作为对象存储时配置 | `openim-rpc-third.yml` | +| 多端互踢策略，单个gateway同时最大连接数 | `openim-msggateway.yml` | +| 消息离线推送 | `openim-push.yml` | +| 配置webhook来通知回调服务器，如消息发送前后回调 | `webhooks.yml` | +| 新入群用户是否可以查看历史消息 | `openim-rpc-group.yml` | +| token 过期时间设置 | `openim-rpc-auth.yml` | +| 定时任务设置，例如消息保存多长时间 | `openim-crontask.yml` | -| 修改配置项 | 配置文件 | -| ----------------------------------------------- | ----------------------- | -| 使用minio作为图片视频文件对象存储 | `minio.yml` | -| 生产环境日志调整 | `log.yml` | -| 发送消息是否验证好友关系 | `openim-rpc-msg.yml` | -| 修改secret | `share.yml` | -| 使用oss, cos, aws, kodo作为图片视频文件对象存储 | `openim-rpc-third.yml` | -| 设置多端互踢策略 | `openim-msggateway.yml` | -| 设置离线推送 | `openim-push.yml` | +## 启动某个服务的多个实例和最大文件句柄数 -## 启动某个OpenIM服务的多个实例 若要启动某个OpenIM的多个实例，只需增加对应的端口数，并修改项目根目录下的`start-config.yml`文件，重启服务即可生效。例如，启动2个`openim-rpc-user`实例的配置如下： @@ -61,5 +79,8 @@ serviceBinaries: openim-rpc-user: 2 ``` +修改最大同时打开的文件句柄数，一般是每个在线用户占用一个 - +``` +maxFileDescriptors: 10000 +``` From eebff88a29a8412330d446ed23db69b990b29224 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Mon, 28 Jul 2025 16:57:58 +0800 Subject: [PATCH 195/199] fix: performance issues with Kafka caused by encapsulating the MQ interface --- go.mod | 2 +- go.sum | 4 ++-- internal/msgtransfer/init.go | 6 +++--- internal/msgtransfer/online_history_msg_handler.go | 13 ++++++++++--- internal/msgtransfer/online_msg_to_mongo_handler.go | 10 +++++++--- internal/push/push.go | 9 +++++---- 6 files changed, 28 insertions(+), 16 deletions(-) diff --git a/go.mod b/go.mod index c06451aaa..775765706 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/mitchellh/mapstructure v1.5.0 github.com/openimsdk/protocol v0.0.73-alpha.12 - github.com/openimsdk/tools v0.0.50-alpha.96 + github.com/openimsdk/tools v0.0.50-alpha.97 github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 github.com/stretchr/testify v1.9.0 diff --git a/go.sum b/go.sum index d29eb0f3f..329a916ec 100644 --- a/go.sum +++ b/go.sum @@ -349,8 +349,8 @@ github.com/openimsdk/gomake v0.0.15-alpha.11 h1:PQudYDRESYeYlUYrrLLJhYIlUPO5x7FA github.com/openimsdk/gomake v0.0.15-alpha.11/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= github.com/openimsdk/protocol v0.0.73-alpha.12 h1:2NYawXeHChYUeSme6QJ9pOLh+Empce2WmwEtbP4JvKk= github.com/openimsdk/protocol v0.0.73-alpha.12/go.mod h1:WF7EuE55vQvpyUAzDXcqg+B+446xQyEba0X35lTINmw= -github.com/openimsdk/tools v0.0.50-alpha.96 h1:U44Fq2jHiEvGi9zuYAnTRNx3Xd9T7P/kBAZLHvQ8xg4= -github.com/openimsdk/tools v0.0.50-alpha.96/go.mod h1:n2poR3asX1e1XZce4O+MOWAp+X02QJRFvhcLCXZdzRo= +github.com/openimsdk/tools v0.0.50-alpha.97 h1:6ik5w3PpgDG6VjSo3nb3FT/fxN3JX7iIARVxVu9g7VY= +github.com/openimsdk/tools v0.0.50-alpha.97/go.mod h1:n2poR3asX1e1XZce4O+MOWAp+X02QJRFvhcLCXZdzRo= github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= diff --git a/internal/msgtransfer/init.go b/internal/msgtransfer/init.go index bbec3f9a2..35026c79a 100644 --- a/internal/msgtransfer/init.go +++ b/internal/msgtransfer/init.go @@ -134,7 +134,7 @@ func Start(ctx context.Context, config *Config, client discovery.SvcDiscoveryReg if err != nil { return err } - historyMongoHandler := NewOnlineHistoryMongoConsumerHandler(msgTransferDatabase,config) + historyMongoHandler := NewOnlineHistoryMongoConsumerHandler(msgTransferDatabase, config) msgTransfer := &MsgTransfer{ historyConsumer: historyConsumer, @@ -161,8 +161,8 @@ func (m *MsgTransfer) Start(ctx context.Context) error { }() go func() { - fn := func(ctx context.Context, key string, value []byte) error { - m.historyMongoHandler.HandleChatWs2Mongo(ctx, key, value) + fn := func(msg mq.Message) error { + m.historyMongoHandler.HandleChatWs2Mongo(msg) return nil } for { diff --git a/internal/msgtransfer/online_history_msg_handler.go b/internal/msgtransfer/online_history_msg_handler.go index 05775a1e6..8b212774a 100644 --- a/internal/msgtransfer/online_history_msg_handler.go +++ b/internal/msgtransfer/online_history_msg_handler.go @@ -18,6 +18,7 @@ import ( "context" "encoding/json" "errors" + "github.com/openimsdk/tools/mq" "sync" "time" @@ -77,6 +78,7 @@ type ConsumerMessage struct { Ctx context.Context Key string Value []byte + Raw mq.Message } func NewOnlineHistoryRedisConsumerHandler(ctx context.Context, client discovery.Conn, config *Config, database controller.MsgTransferDatabase) (*OnlineHistoryRedisConsumerHandler, error) { @@ -113,6 +115,11 @@ func NewOnlineHistoryRedisConsumerHandler(ctx context.Context, client discovery. b.Do = och.do och.redisMessageBatches = b + och.redisMessageBatches.OnComplete = func(lastMessage *ConsumerMessage, totalCount int) { + lastMessage.Raw.Mark() + lastMessage.Raw.Commit() + } + return &och, nil } func (och *OnlineHistoryRedisConsumerHandler) do(ctx context.Context, channelID int, val *batcher.Msg[ConsumerMessage]) { @@ -388,10 +395,10 @@ func withAggregationCtx(ctx context.Context, values []*ContextMsg) context.Conte return mcontext.SetOperationID(ctx, allMessageOperationID) } -func (och *OnlineHistoryRedisConsumerHandler) HandlerRedisMessage(ctx context.Context, key string, value []byte) error { // a instance in the consumer group - err := och.redisMessageBatches.Put(ctx, &ConsumerMessage{Ctx: ctx, Key: key, Value: value}) +func (och *OnlineHistoryRedisConsumerHandler) HandlerRedisMessage(msg mq.Message) error { // a instance in the consumer group + err := och.redisMessageBatches.Put(msg.Context(), &ConsumerMessage{Ctx: msg.Context(), Key: msg.Key(), Value: msg.Value(), Raw: msg}) if err != nil { - log.ZWarn(ctx, "put msg to error", err, "key", key, "value", value) + log.ZWarn(msg.Context(), "put msg to error", err, "key", msg.Key(), "value", msg.Value()) } return nil } diff --git a/internal/msgtransfer/online_msg_to_mongo_handler.go b/internal/msgtransfer/online_msg_to_mongo_handler.go index 6c1498f82..8611af7ea 100644 --- a/internal/msgtransfer/online_msg_to_mongo_handler.go +++ b/internal/msgtransfer/online_msg_to_mongo_handler.go @@ -15,12 +15,12 @@ package msgtransfer import ( - "context" + "github.com/openimsdk/protocol/constant" + "github.com/openimsdk/tools/mq" "github.com/openimsdk/open-im-server/v3/pkg/common/prommetrics" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/controller" "github.com/openimsdk/open-im-server/v3/pkg/common/webhook" - "github.com/openimsdk/protocol/constant" pbmsg "github.com/openimsdk/protocol/msg" "github.com/openimsdk/tools/log" "google.golang.org/protobuf/proto" @@ -40,7 +40,10 @@ func NewOnlineHistoryMongoConsumerHandler(database controller.MsgTransferDatabas } } -func (mc *OnlineHistoryMongoConsumerHandler) HandleChatWs2Mongo(ctx context.Context, key string, msg []byte) { +func (mc *OnlineHistoryMongoConsumerHandler) HandleChatWs2Mongo(val mq.Message) { + ctx := val.Context() + key := val.Key() + msg := val.Value() msgFromMQ := pbmsg.MsgDataToMongoByMQ{} err := proto.Unmarshal(msg, &msgFromMQ) if err != nil { @@ -58,6 +61,7 @@ func (mc *OnlineHistoryMongoConsumerHandler) HandleChatWs2Mongo(ctx context.Cont prommetrics.MsgInsertMongoFailedCounter.Inc() } else { prommetrics.MsgInsertMongoSuccessCounter.Inc() + val.Mark() } for _, msgData := range msgFromMQ.MsgData { diff --git a/internal/push/push.go b/internal/push/push.go index 1d6f8cb30..bf95b6acc 100644 --- a/internal/push/push.go +++ b/internal/push/push.go @@ -2,6 +2,7 @@ package push import ( "context" + "github.com/openimsdk/tools/mq" "math/rand" "strconv" @@ -106,8 +107,8 @@ func Start(ctx context.Context, config *Config, client discovery.SvcDiscoveryReg go func() { pushHandler.WaitCache() - fn := func(ctx context.Context, key string, value []byte) error { - pushHandler.HandleMs2PsChat(authverify.WithTempAdmin(ctx), value) + fn := func(msg mq.Message) error { + pushHandler.HandleMs2PsChat(authverify.WithTempAdmin(msg.Context()), msg.Value()) return nil } consumerCtx := mcontext.SetOperationID(context.Background(), "push_"+strconv.Itoa(int(rand.Uint32()))) @@ -121,8 +122,8 @@ func Start(ctx context.Context, config *Config, client discovery.SvcDiscoveryReg }() go func() { - fn := func(ctx context.Context, key string, value []byte) error { - offlineHandler.HandleMsg2OfflinePush(ctx, value) + fn := func(msg mq.Message) error { + offlineHandler.HandleMsg2OfflinePush(msg.Context(), msg.Value()) return nil } consumerCtx := mcontext.SetOperationID(context.Background(), "push_"+strconv.Itoa(int(rand.Uint32()))) From 2d4cf9974493c8e15af70339f6a27b8beafc9b6a Mon Sep 17 00:00:00 2001 From: chao <48119764+withchao@users.noreply.github.com> Date: Mon, 28 Jul 2025 17:10:38 +0800 Subject: [PATCH 196/199] fix: performance issues with Kafka caused by encapsulating the MQ interface (#3485) --- go.mod | 2 +- go.sum | 4 ++-- internal/msgtransfer/init.go | 6 +++--- internal/msgtransfer/online_history_msg_handler.go | 13 ++++++++++--- internal/msgtransfer/online_msg_to_mongo_handler.go | 10 +++++++--- internal/push/push.go | 9 +++++---- 6 files changed, 28 insertions(+), 16 deletions(-) diff --git a/go.mod b/go.mod index c06451aaa..775765706 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/mitchellh/mapstructure v1.5.0 github.com/openimsdk/protocol v0.0.73-alpha.12 - github.com/openimsdk/tools v0.0.50-alpha.96 + github.com/openimsdk/tools v0.0.50-alpha.97 github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 github.com/stretchr/testify v1.9.0 diff --git a/go.sum b/go.sum index d29eb0f3f..329a916ec 100644 --- a/go.sum +++ b/go.sum @@ -349,8 +349,8 @@ github.com/openimsdk/gomake v0.0.15-alpha.11 h1:PQudYDRESYeYlUYrrLLJhYIlUPO5x7FA github.com/openimsdk/gomake v0.0.15-alpha.11/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= github.com/openimsdk/protocol v0.0.73-alpha.12 h1:2NYawXeHChYUeSme6QJ9pOLh+Empce2WmwEtbP4JvKk= github.com/openimsdk/protocol v0.0.73-alpha.12/go.mod h1:WF7EuE55vQvpyUAzDXcqg+B+446xQyEba0X35lTINmw= -github.com/openimsdk/tools v0.0.50-alpha.96 h1:U44Fq2jHiEvGi9zuYAnTRNx3Xd9T7P/kBAZLHvQ8xg4= -github.com/openimsdk/tools v0.0.50-alpha.96/go.mod h1:n2poR3asX1e1XZce4O+MOWAp+X02QJRFvhcLCXZdzRo= +github.com/openimsdk/tools v0.0.50-alpha.97 h1:6ik5w3PpgDG6VjSo3nb3FT/fxN3JX7iIARVxVu9g7VY= +github.com/openimsdk/tools v0.0.50-alpha.97/go.mod h1:n2poR3asX1e1XZce4O+MOWAp+X02QJRFvhcLCXZdzRo= github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= diff --git a/internal/msgtransfer/init.go b/internal/msgtransfer/init.go index bbec3f9a2..35026c79a 100644 --- a/internal/msgtransfer/init.go +++ b/internal/msgtransfer/init.go @@ -134,7 +134,7 @@ func Start(ctx context.Context, config *Config, client discovery.SvcDiscoveryReg if err != nil { return err } - historyMongoHandler := NewOnlineHistoryMongoConsumerHandler(msgTransferDatabase,config) + historyMongoHandler := NewOnlineHistoryMongoConsumerHandler(msgTransferDatabase, config) msgTransfer := &MsgTransfer{ historyConsumer: historyConsumer, @@ -161,8 +161,8 @@ func (m *MsgTransfer) Start(ctx context.Context) error { }() go func() { - fn := func(ctx context.Context, key string, value []byte) error { - m.historyMongoHandler.HandleChatWs2Mongo(ctx, key, value) + fn := func(msg mq.Message) error { + m.historyMongoHandler.HandleChatWs2Mongo(msg) return nil } for { diff --git a/internal/msgtransfer/online_history_msg_handler.go b/internal/msgtransfer/online_history_msg_handler.go index 05775a1e6..8b212774a 100644 --- a/internal/msgtransfer/online_history_msg_handler.go +++ b/internal/msgtransfer/online_history_msg_handler.go @@ -18,6 +18,7 @@ import ( "context" "encoding/json" "errors" + "github.com/openimsdk/tools/mq" "sync" "time" @@ -77,6 +78,7 @@ type ConsumerMessage struct { Ctx context.Context Key string Value []byte + Raw mq.Message } func NewOnlineHistoryRedisConsumerHandler(ctx context.Context, client discovery.Conn, config *Config, database controller.MsgTransferDatabase) (*OnlineHistoryRedisConsumerHandler, error) { @@ -113,6 +115,11 @@ func NewOnlineHistoryRedisConsumerHandler(ctx context.Context, client discovery. b.Do = och.do och.redisMessageBatches = b + och.redisMessageBatches.OnComplete = func(lastMessage *ConsumerMessage, totalCount int) { + lastMessage.Raw.Mark() + lastMessage.Raw.Commit() + } + return &och, nil } func (och *OnlineHistoryRedisConsumerHandler) do(ctx context.Context, channelID int, val *batcher.Msg[ConsumerMessage]) { @@ -388,10 +395,10 @@ func withAggregationCtx(ctx context.Context, values []*ContextMsg) context.Conte return mcontext.SetOperationID(ctx, allMessageOperationID) } -func (och *OnlineHistoryRedisConsumerHandler) HandlerRedisMessage(ctx context.Context, key string, value []byte) error { // a instance in the consumer group - err := och.redisMessageBatches.Put(ctx, &ConsumerMessage{Ctx: ctx, Key: key, Value: value}) +func (och *OnlineHistoryRedisConsumerHandler) HandlerRedisMessage(msg mq.Message) error { // a instance in the consumer group + err := och.redisMessageBatches.Put(msg.Context(), &ConsumerMessage{Ctx: msg.Context(), Key: msg.Key(), Value: msg.Value(), Raw: msg}) if err != nil { - log.ZWarn(ctx, "put msg to error", err, "key", key, "value", value) + log.ZWarn(msg.Context(), "put msg to error", err, "key", msg.Key(), "value", msg.Value()) } return nil } diff --git a/internal/msgtransfer/online_msg_to_mongo_handler.go b/internal/msgtransfer/online_msg_to_mongo_handler.go index 6c1498f82..8611af7ea 100644 --- a/internal/msgtransfer/online_msg_to_mongo_handler.go +++ b/internal/msgtransfer/online_msg_to_mongo_handler.go @@ -15,12 +15,12 @@ package msgtransfer import ( - "context" + "github.com/openimsdk/protocol/constant" + "github.com/openimsdk/tools/mq" "github.com/openimsdk/open-im-server/v3/pkg/common/prommetrics" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/controller" "github.com/openimsdk/open-im-server/v3/pkg/common/webhook" - "github.com/openimsdk/protocol/constant" pbmsg "github.com/openimsdk/protocol/msg" "github.com/openimsdk/tools/log" "google.golang.org/protobuf/proto" @@ -40,7 +40,10 @@ func NewOnlineHistoryMongoConsumerHandler(database controller.MsgTransferDatabas } } -func (mc *OnlineHistoryMongoConsumerHandler) HandleChatWs2Mongo(ctx context.Context, key string, msg []byte) { +func (mc *OnlineHistoryMongoConsumerHandler) HandleChatWs2Mongo(val mq.Message) { + ctx := val.Context() + key := val.Key() + msg := val.Value() msgFromMQ := pbmsg.MsgDataToMongoByMQ{} err := proto.Unmarshal(msg, &msgFromMQ) if err != nil { @@ -58,6 +61,7 @@ func (mc *OnlineHistoryMongoConsumerHandler) HandleChatWs2Mongo(ctx context.Cont prommetrics.MsgInsertMongoFailedCounter.Inc() } else { prommetrics.MsgInsertMongoSuccessCounter.Inc() + val.Mark() } for _, msgData := range msgFromMQ.MsgData { diff --git a/internal/push/push.go b/internal/push/push.go index 1d6f8cb30..bf95b6acc 100644 --- a/internal/push/push.go +++ b/internal/push/push.go @@ -2,6 +2,7 @@ package push import ( "context" + "github.com/openimsdk/tools/mq" "math/rand" "strconv" @@ -106,8 +107,8 @@ func Start(ctx context.Context, config *Config, client discovery.SvcDiscoveryReg go func() { pushHandler.WaitCache() - fn := func(ctx context.Context, key string, value []byte) error { - pushHandler.HandleMs2PsChat(authverify.WithTempAdmin(ctx), value) + fn := func(msg mq.Message) error { + pushHandler.HandleMs2PsChat(authverify.WithTempAdmin(msg.Context()), msg.Value()) return nil } consumerCtx := mcontext.SetOperationID(context.Background(), "push_"+strconv.Itoa(int(rand.Uint32()))) @@ -121,8 +122,8 @@ func Start(ctx context.Context, config *Config, client discovery.SvcDiscoveryReg }() go func() { - fn := func(ctx context.Context, key string, value []byte) error { - offlineHandler.HandleMsg2OfflinePush(ctx, value) + fn := func(msg mq.Message) error { + offlineHandler.HandleMsg2OfflinePush(msg.Context(), msg.Value()) return nil } consumerCtx := mcontext.SetOperationID(context.Background(), "push_"+strconv.Itoa(int(rand.Uint32()))) From c208c3789fbbc2b11557330c685ba2576bb0223f Mon Sep 17 00:00:00 2001 From: xuzhijvn Date: Tue, 29 Jul 2025 16:16:30 +0800 Subject: [PATCH 197/199] fix: searchMessage method has potential NPE bug (#3287) (#3289) --- pkg/common/storage/database/mgo/msg.go | 7 +++++- pkg/common/storage/database/mgo/msg_test.go | 28 +++++++++++++++++++++ 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/pkg/common/storage/database/mgo/msg.go b/pkg/common/storage/database/mgo/msg.go index 83fefbfe6..975e7a0ef 100644 --- a/pkg/common/storage/database/mgo/msg.go +++ b/pkg/common/storage/database/mgo/msg.go @@ -321,7 +321,12 @@ func (m *MsgMgo) searchMessageIndex(ctx context.Context, filter any, nextID prim } func (m *MsgMgo) searchMessage(ctx context.Context, req *msg.SearchMessageReq) (int64, []searchMessageIndex, error) { - filter := bson.M{} + filter := bson.M{ + "msgs.msg": bson.M{ + "$exists": true, + "$type": "object", + }, + } if req.RecvID != "" { filter["$or"] = bson.A{ bson.M{"msgs.msg.recv_id": req.RecvID}, diff --git a/pkg/common/storage/database/mgo/msg_test.go b/pkg/common/storage/database/mgo/msg_test.go index 2ced0210a..8f3bd1e8a 100644 --- a/pkg/common/storage/database/mgo/msg_test.go +++ b/pkg/common/storage/database/mgo/msg_test.go @@ -9,6 +9,8 @@ import ( "time" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" + "github.com/openimsdk/protocol/msg" + "github.com/openimsdk/protocol/sdkws" "github.com/openimsdk/tools/db/mongoutil" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/mongo" @@ -148,3 +150,29 @@ func TestName5(t *testing.T) { // } // t.Log(seq, sendTime) //} + +func TestSearchMessage(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*300) + defer cancel() + cli := Result(mongo.Connect(ctx, options.Client().ApplyURI("mongodb://openIM:openIM123@172.16.8.135:37017/openim_v3?maxPoolSize=100").SetConnectTimeout(5*time.Second))) + + msgMongo, err := NewMsgMongo(cli.Database("openim_v3")) + if err != nil { + panic(err) + } + ts := time.Now().Add(-time.Hour * 24 * 5).UnixMilli() + t.Log(ts) + req := &msg.SearchMessageReq{ + //SendID: "yjz", + //RecvID: "aibot", + Pagination: &sdkws.RequestPagination{ + PageNumber: 1, + ShowNumber: 20, + }, + } + count, resp, err := msgMongo.SearchMessage(ctx, req) + if err != nil { + panic(err) + } + t.Log(resp, count) +} From dd5253f015304981f278f87d57087f142d1c392f Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Tue, 29 Jul 2025 17:30:06 +0800 Subject: [PATCH 198/199] feat: merge main branch --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 775765706..0f51303f9 100644 --- a/go.mod +++ b/go.mod @@ -27,6 +27,7 @@ require ( require github.com/google/uuid v1.6.0 require ( + github.com/IBM/sarama v1.43.0 github.com/fatih/color v1.14.1 github.com/gin-contrib/gzip v1.0.1 github.com/go-redis/redis v6.15.9+incompatible @@ -54,7 +55,6 @@ require ( cloud.google.com/go/iam v1.1.7 // indirect cloud.google.com/go/longrunning v0.5.5 // indirect cloud.google.com/go/storage v1.40.0 // indirect - github.com/IBM/sarama v1.43.0 // indirect github.com/MicahParks/keyfunc v1.9.0 // indirect github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible // indirect github.com/aws/aws-sdk-go-v2 v1.32.5 // indirect From e655f6422c769c373ebf3fde0d775dd5a27d50e6 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Tue, 29 Jul 2025 17:35:39 +0800 Subject: [PATCH 199/199] feat: merge main branch --- tools/stress-test-v2/main.go | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/tools/stress-test-v2/main.go b/tools/stress-test-v2/main.go index 0c309b9c9..8adcbd62c 100644 --- a/tools/stress-test-v2/main.go +++ b/tools/stress-test-v2/main.go @@ -445,7 +445,7 @@ func main() { Share: *share, Api: *apiConfig, }, - AdminUserID: share.IMAdminUserID[0], + AdminUserID: share.IMAdminUser.UserIDs[0], Ctx: ctx, Cancel: cancel, HttpClient: &http.Client{ @@ -476,8 +476,7 @@ func main() { st.AdminToken = token fmt.Println("Admin Token:", st.AdminToken) fmt.Println("ApiAddress:", ApiAddress) - - for i := range MaxUser { + for i := 0; i < MaxUser; i++ { userID := fmt.Sprintf("v2_StressTest_User_%d", i) st.CreatedUsers = append(st.CreatedUsers, userID) st.CreateUserCounter++ @@ -520,7 +519,7 @@ func main() { create100kGroupTicker := time.NewTicker(Create100KGroupTicker) defer create100kGroupTicker.Stop() - for i := range Max100KGroup { + for i := 0; i < Max100KGroup; i++ { select { case <-st.Ctx.Done(): log.ZInfo(st.Ctx, "Stop Create 100K Group") @@ -592,7 +591,7 @@ func main() { create999GroupTicker := time.NewTicker(Create999GroupTicker) defer create999GroupTicker.Stop() - for i := range Max999Group { + for i := 0; i < Max999Group; i++ { select { case <-st.Ctx.Done(): log.ZInfo(st.Ctx, "Stop Create 999 Group") @@ -663,12 +662,12 @@ func main() { var groups100K []string var groups999 []string - for i := range Max100KGroup { + for i := 0; i < Max100KGroup; i++ { groupID := fmt.Sprintf("v2_StressTest_Group_100K_%d", i) groups100K = append(groups100K, groupID) } - for i := range Max999Group { + for i := 0; i < Max999Group; i++ { groupID := fmt.Sprintf("v2_StressTest_Group_1K_%d", i) groups999 = append(groups999, groupID) }