From 3b74ad56d4fce97187080a366e618187d7bf8695 Mon Sep 17 00:00:00 2001 From: "Xinwei Xiong(cubxxw)" <3293172751nss@gmail.com> Date: Wed, 29 Nov 2023 16:27:26 +0800 Subject: [PATCH] feat: add format set Signed-off-by: Xinwei Xiong(cubxxw) <3293172751nss@gmail.com> --- docker-compose.yml | 123 +++++++++--------- internal/rpc/conversation/conversaion.go | 2 + internal/rpc/friend/friend.go | 1 + internal/rpc/group/group.go | 12 +- internal/rpc/group/super_group.go | 1 + internal/rpc/third/third.go | 5 +- internal/rpc/user/user.go | 6 +- internal/tools/conversation.go | 3 +- internal/tools/msg.go | 4 +- pkg/common/cmd/rpc.go | 2 +- pkg/common/convert/user_test.go | 1 + pkg/common/db/cache/group.go | 12 +- pkg/common/db/cache/user.go | 3 +- pkg/common/db/controller/black.go | 1 + pkg/common/db/controller/conversation.go | 7 +- pkg/common/db/controller/friend.go | 8 +- pkg/common/db/controller/group.go | 12 +- pkg/common/db/controller/third.go | 3 +- pkg/common/db/controller/user.go | 4 +- pkg/common/db/mgo/black.go | 4 +- pkg/common/db/mgo/conversation.go | 20 ++- pkg/common/db/mgo/friend.go | 4 +- pkg/common/db/mgo/friend_request.go | 4 +- pkg/common/db/mgo/group.go | 6 +- pkg/common/db/mgo/group_member.go | 4 +- pkg/common/db/mgo/group_request.go | 4 +- pkg/common/db/mgo/log.go | 6 +- pkg/common/db/mgo/object.go | 4 +- pkg/common/db/mgo/user.go | 6 +- pkg/common/db/table/relation/black.go | 3 +- pkg/common/db/table/relation/conversation.go | 3 +- pkg/common/db/table/relation/friend.go | 3 +- .../db/table/relation/friend_request.go | 3 +- pkg/common/db/table/relation/group.go | 3 +- pkg/common/db/table/relation/group_member.go | 3 +- pkg/common/db/table/relation/group_request.go | 3 +- pkg/common/db/table/relation/log.go | 3 +- pkg/common/db/table/relation/user.go | 3 +- pkg/rpcclient/grouphash/grouphash.go | 1 + tools/component/component_test.go | 3 +- tools/up35/README.md | 67 ++++++++++ tools/up35/{main.go => up35.go} | 28 ++-- 42 files changed, 276 insertions(+), 122 deletions(-) create mode 100644 tools/up35/README.md rename tools/up35/{main.go => up35.go} (98%) diff --git a/docker-compose.yml b/docker-compose.yml index 1b9b391b2..4c51a3066 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -24,6 +24,7 @@ services: restart: always networks: server: + # open-im-server_server ipv4_address: ${MYSQL_NETWORK_ADDRESS} mongodb: @@ -143,68 +144,68 @@ services: server: ipv4_address: ${OPENIM_WEB_NETWORK_ADDRESS} - openim-admin: - image: ${IMAGE_REGISTRY}/openim-admin-front:v3.4.0 - # image: ghcr.io/openimsdk/openim-admin-front:v3.4.0 - # image: registry.cn-hangzhou.aliyuncs.com/openimsdk/openim-admin-front:v3.4.0 - # image: openim/openim-admin-front:v3.4.0 - container_name: openim-admin - restart: always - ports: - - "${OPENIM_ADMIN_FRONT_PORT}:80" - networks: - server: - ipv4_address: ${OPENIM_ADMIN_FRONT_NETWORK_ADDRESS} + # openim-admin: + # image: ${IMAGE_REGISTRY}/openim-admin-front:v3.4.0 + # # image: ghcr.io/openimsdk/openim-admin-front:v3.4.0 + # # image: registry.cn-hangzhou.aliyuncs.com/openimsdk/openim-admin-front:v3.4.0 + # # image: openim/openim-admin-front:v3.4.0 + # container_name: openim-admin + # restart: always + # ports: + # - "${OPENIM_ADMIN_FRONT_PORT}:80" + # networks: + # server: + # ipv4_address: ${OPENIM_ADMIN_FRONT_NETWORK_ADDRESS} - prometheus: - image: prom/prometheus - container_name: prometheus - hostname: prometheus - restart: always - volumes: - - ./config/prometheus.yml:/etc/prometheus/prometheus.yml - - ./config/instance-down-rules.yml:/etc/prometheus/instance-down-rules.yml - ports: - - "${PROMETHEUS_PORT}:9090" - networks: - server: - ipv4_address: ${PROMETHEUS_NETWORK_ADDRESS} + # prometheus: + # image: prom/prometheus + # container_name: prometheus + # hostname: prometheus + # restart: always + # volumes: + # - ./config/prometheus.yml:/etc/prometheus/prometheus.yml + # - ./config/instance-down-rules.yml:/etc/prometheus/instance-down-rules.yml + # ports: + # - "${PROMETHEUS_PORT}:9090" + # networks: + # server: + # ipv4_address: ${PROMETHEUS_NETWORK_ADDRESS} - alertmanager: - image: prom/alertmanager - container_name: alertmanager - hostname: alertmanager - restart: always - volumes: - - ./config/alertmanager.yml:/etc/alertmanager/alertmanager.yml - - ./config/email.tmpl:/etc/alertmanager/email.tmpl - ports: - - "${ALERT_MANAGER_PORT}:9093" - networks: - server: - ipv4_address: ${ALERT_MANAGER_NETWORK_ADDRESS} + # alertmanager: + # image: prom/alertmanager + # container_name: alertmanager + # hostname: alertmanager + # restart: always + # volumes: + # - ./config/alertmanager.yml:/etc/alertmanager/alertmanager.yml + # - ./config/email.tmpl:/etc/alertmanager/email.tmpl + # ports: + # - "${ALERT_MANAGER_PORT}:9093" + # networks: + # server: + # ipv4_address: ${ALERT_MANAGER_NETWORK_ADDRESS} - grafana: - image: grafana/grafana - container_name: grafana - hostname: grafana - user: root - restart: always - ports: - - "${GRAFANA_PORT}:3000" - volumes: - - ${DATA_DIR}/components/grafana:/var/lib/grafana - networks: - server: - ipv4_address: ${GRAFANA_NETWORK_ADDRESS} + # grafana: + # image: grafana/grafana + # container_name: grafana + # hostname: grafana + # user: root + # restart: always + # ports: + # - "${GRAFANA_PORT}:3000" + # volumes: + # - ${DATA_DIR}/components/grafana:/var/lib/grafana + # networks: + # server: + # ipv4_address: ${GRAFANA_NETWORK_ADDRESS} - node-exporter: - image: quay.io/prometheus/node-exporter - container_name: node-exporter - hostname: node-exporter - restart: always - ports: - - "${NODE_EXPORTER_PORT}:9100" - networks: - server: - ipv4_address: ${NODE_EXPORTER_NETWORK_ADDRESS} + # node-exporter: + # image: quay.io/prometheus/node-exporter + # container_name: node-exporter + # hostname: node-exporter + # restart: always + # ports: + # - "${NODE_EXPORTER_PORT}:9100" + # networks: + # server: + # ipv4_address: ${NODE_EXPORTER_NETWORK_ADDRESS} diff --git a/internal/rpc/conversation/conversaion.go b/internal/rpc/conversation/conversaion.go index ecfe6147d..88c9ff7ff 100644 --- a/internal/rpc/conversation/conversaion.go +++ b/internal/rpc/conversation/conversaion.go @@ -17,7 +17,9 @@ package conversation import ( "context" "errors" + "github.com/OpenIMSDK/tools/tx" + "github.com/openimsdk/open-im-server/v3/pkg/common/db/mgo" "github.com/openimsdk/open-im-server/v3/pkg/common/db/unrelation" diff --git a/internal/rpc/friend/friend.go b/internal/rpc/friend/friend.go index c3c7db039..ddb5008f2 100644 --- a/internal/rpc/friend/friend.go +++ b/internal/rpc/friend/friend.go @@ -16,6 +16,7 @@ package friend import ( "context" + "github.com/OpenIMSDK/tools/tx" "github.com/OpenIMSDK/protocol/sdkws" diff --git a/internal/rpc/group/group.go b/internal/rpc/group/group.go index adf7d2b42..486e1057d 100644 --- a/internal/rpc/group/group.go +++ b/internal/rpc/group/group.go @@ -17,17 +17,19 @@ package group import ( "context" "fmt" - pbconversation "github.com/OpenIMSDK/protocol/conversation" - "github.com/OpenIMSDK/protocol/wrapperspb" - "github.com/OpenIMSDK/tools/tx" - "github.com/openimsdk/open-im-server/v3/pkg/common/db/mgo" - "github.com/openimsdk/open-im-server/v3/pkg/rpcclient/grouphash" "math/big" "math/rand" "strconv" "strings" "time" + pbconversation "github.com/OpenIMSDK/protocol/conversation" + "github.com/OpenIMSDK/protocol/wrapperspb" + "github.com/OpenIMSDK/tools/tx" + + "github.com/openimsdk/open-im-server/v3/pkg/common/db/mgo" + "github.com/openimsdk/open-im-server/v3/pkg/rpcclient/grouphash" + "github.com/openimsdk/open-im-server/v3/pkg/authverify" "github.com/openimsdk/open-im-server/v3/pkg/msgprocessor" diff --git a/internal/rpc/group/super_group.go b/internal/rpc/group/super_group.go index cb4febe7c..f893a79c2 100644 --- a/internal/rpc/group/super_group.go +++ b/internal/rpc/group/super_group.go @@ -17,6 +17,7 @@ package group import ( "context" "errors" + pbgroup "github.com/OpenIMSDK/protocol/group" ) diff --git a/internal/rpc/third/third.go b/internal/rpc/third/third.go index 396a30c8f..7a63d3526 100644 --- a/internal/rpc/third/third.go +++ b/internal/rpc/third/third.go @@ -17,11 +17,12 @@ package third import ( "context" "fmt" - "github.com/openimsdk/open-im-server/v3/pkg/common/db/mgo" - "github.com/openimsdk/open-im-server/v3/pkg/common/db/unrelation" "net/url" "time" + "github.com/openimsdk/open-im-server/v3/pkg/common/db/mgo" + "github.com/openimsdk/open-im-server/v3/pkg/common/db/unrelation" + "github.com/openimsdk/open-im-server/v3/pkg/common/db/s3" "github.com/openimsdk/open-im-server/v3/pkg/common/db/s3/cos" "github.com/openimsdk/open-im-server/v3/pkg/common/db/s3/minio" diff --git a/internal/rpc/user/user.go b/internal/rpc/user/user.go index d672493dc..3aec93443 100644 --- a/internal/rpc/user/user.go +++ b/internal/rpc/user/user.go @@ -17,11 +17,13 @@ package user import ( "context" "errors" - "github.com/OpenIMSDK/tools/tx" - "github.com/openimsdk/open-im-server/v3/pkg/common/db/mgo" "strings" "time" + "github.com/OpenIMSDK/tools/tx" + + "github.com/openimsdk/open-im-server/v3/pkg/common/db/mgo" + "github.com/OpenIMSDK/protocol/constant" "github.com/OpenIMSDK/protocol/sdkws" "github.com/OpenIMSDK/tools/errs" diff --git a/internal/tools/conversation.go b/internal/tools/conversation.go index c91e92078..0d0275339 100644 --- a/internal/tools/conversation.go +++ b/internal/tools/conversation.go @@ -16,10 +16,11 @@ package tools import ( "context" - "github.com/OpenIMSDK/protocol/sdkws" "math/rand" "time" + "github.com/OpenIMSDK/protocol/sdkws" + "github.com/OpenIMSDK/tools/log" "github.com/OpenIMSDK/tools/mcontext" "github.com/OpenIMSDK/tools/utils" diff --git a/internal/tools/msg.go b/internal/tools/msg.go index 32d13c6b7..97bb2988e 100644 --- a/internal/tools/msg.go +++ b/internal/tools/msg.go @@ -17,10 +17,12 @@ package tools import ( "context" "fmt" + "math" + "github.com/OpenIMSDK/protocol/sdkws" "github.com/OpenIMSDK/tools/tx" + "github.com/openimsdk/open-im-server/v3/pkg/common/db/mgo" - "math" "github.com/redis/go-redis/v9" "google.golang.org/grpc" diff --git a/pkg/common/cmd/rpc.go b/pkg/common/cmd/rpc.go index 094a0e60a..ea2a00b07 100644 --- a/pkg/common/cmd/rpc.go +++ b/pkg/common/cmd/rpc.go @@ -46,7 +46,7 @@ func (a *RpcCmd) Exec() error { return a.Execute() } -func (a *RpcCmd) StartSvr(name string, rpcFn func(discov discoveryregistry.SvcDiscoveryRegistry, server *grpc.Server) error,) error { +func (a *RpcCmd) StartSvr(name string, rpcFn func(discov discoveryregistry.SvcDiscoveryRegistry, server *grpc.Server) error) error { if a.GetPortFlag() == 0 { return errors.New("port is required") } diff --git a/pkg/common/convert/user_test.go b/pkg/common/convert/user_test.go index e58fa16c1..a24efb53c 100644 --- a/pkg/common/convert/user_test.go +++ b/pkg/common/convert/user_test.go @@ -19,6 +19,7 @@ import ( "testing" "github.com/OpenIMSDK/protocol/sdkws" + relationtb "github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation" ) diff --git a/pkg/common/db/cache/group.go b/pkg/common/db/cache/group.go index b45cfd091..603d8e534 100644 --- a/pkg/common/db/cache/group.go +++ b/pkg/common/db/cache/group.go @@ -17,11 +17,12 @@ package cache import ( "context" "fmt" - "github.com/OpenIMSDK/protocol/constant" - "github.com/OpenIMSDK/tools/errs" "strconv" "time" + "github.com/OpenIMSDK/protocol/constant" + "github.com/OpenIMSDK/tools/errs" + "github.com/OpenIMSDK/tools/log" "github.com/dtm-labs/rockscache" @@ -318,7 +319,12 @@ func (g *GroupCacheRedis) GetGroupMembersInfo(ctx context.Context, groupID strin }) } -func (g *GroupCacheRedis) GetGroupMembersPage(ctx context.Context, groupID string, userIDs []string, showNumber, pageNumber int32) (total uint32, groupMembers []*relationtb.GroupMemberModel, err error) { +func (g *GroupCacheRedis) GetGroupMembersPage( + ctx context.Context, + groupID string, + userIDs []string, + showNumber, pageNumber int32, +) (total uint32, groupMembers []*relationtb.GroupMemberModel, err error) { groupMemberIDs, err := g.GetGroupMemberIDs(ctx, groupID) if err != nil { return 0, nil, err diff --git a/pkg/common/db/cache/user.go b/pkg/common/db/cache/user.go index e0f22b2f7..979bd06e4 100644 --- a/pkg/common/db/cache/user.go +++ b/pkg/common/db/cache/user.go @@ -18,11 +18,12 @@ import ( "context" "encoding/json" "errors" - relationtb "github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation" "hash/crc32" "strconv" "time" + relationtb "github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation" + "github.com/OpenIMSDK/tools/log" "github.com/OpenIMSDK/protocol/constant" diff --git a/pkg/common/db/controller/black.go b/pkg/common/db/controller/black.go index 4e51e03d4..e68d06b01 100644 --- a/pkg/common/db/controller/black.go +++ b/pkg/common/db/controller/black.go @@ -16,6 +16,7 @@ package controller import ( "context" + "github.com/OpenIMSDK/tools/pagination" "github.com/OpenIMSDK/tools/log" diff --git a/pkg/common/db/controller/conversation.go b/pkg/common/db/controller/conversation.go index 4902540a5..2a0cb63e4 100644 --- a/pkg/common/db/controller/conversation.go +++ b/pkg/common/db/controller/conversation.go @@ -16,9 +16,10 @@ package controller import ( "context" - "github.com/OpenIMSDK/tools/pagination" "time" + "github.com/OpenIMSDK/tools/pagination" + "github.com/openimsdk/open-im-server/v3/pkg/msgprocessor" "github.com/OpenIMSDK/protocol/constant" @@ -237,7 +238,9 @@ func (c *conversationDatabase) SetUserConversations(ctx context.Context, ownerUs if err != nil { return err } - cache = cache.DelConversationIDs(ownerUserID).DelUserConversationIDsHash(ownerUserID).DelConversationNotReceiveMessageUserIDs(utils.Slice(notExistConversations, func(e *relationtb.ConversationModel) string { return e.ConversationID })...) + cache = cache.DelConversationIDs(ownerUserID). + DelUserConversationIDsHash(ownerUserID). + DelConversationNotReceiveMessageUserIDs(utils.Slice(notExistConversations, func(e *relationtb.ConversationModel) string { return e.ConversationID })...) } return cache.ExecDel(ctx) }) diff --git a/pkg/common/db/controller/friend.go b/pkg/common/db/controller/friend.go index 7d32858ad..29b2ef9b1 100644 --- a/pkg/common/db/controller/friend.go +++ b/pkg/common/db/controller/friend.go @@ -16,9 +16,10 @@ package controller import ( "context" - "github.com/OpenIMSDK/tools/pagination" "time" + "github.com/OpenIMSDK/tools/pagination" + "github.com/OpenIMSDK/protocol/constant" "github.com/OpenIMSDK/tools/errs" "github.com/OpenIMSDK/tools/log" @@ -97,7 +98,10 @@ func (f *friendDatabase) AddFriendRequest(ctx context.Context, fromUserID, toUse m["create_time"] = time.Now() return f.friendRequest.UpdateByMap(ctx, fromUserID, toUserID, m) case relation.IsNotFound(err): - return f.friendRequest.Create(ctx, []*relation.FriendRequestModel{{FromUserID: fromUserID, ToUserID: toUserID, ReqMsg: reqMsg, Ex: ex, CreateTime: time.Now(), HandleTime: time.Unix(0, 0)}}) + return f.friendRequest.Create( + ctx, + []*relation.FriendRequestModel{{FromUserID: fromUserID, ToUserID: toUserID, ReqMsg: reqMsg, Ex: ex, CreateTime: time.Now(), HandleTime: time.Unix(0, 0)}}, + ) default: return err } diff --git a/pkg/common/db/controller/group.go b/pkg/common/db/controller/group.go index 0aff9fe81..4147d59c0 100644 --- a/pkg/common/db/controller/group.go +++ b/pkg/common/db/controller/group.go @@ -16,9 +16,10 @@ package controller import ( "context" + "time" + "github.com/OpenIMSDK/tools/pagination" "github.com/dtm-labs/rockscache" - "time" "github.com/OpenIMSDK/protocol/constant" "github.com/OpenIMSDK/tools/tx" @@ -74,7 +75,14 @@ type GroupDatabase interface { DeleteGroupMemberHash(ctx context.Context, groupIDs []string) error } -func NewGroupDatabase(rdb redis.UniversalClient, groupDB relationtb.GroupModelInterface, groupMemberDB relationtb.GroupMemberModelInterface, groupRequestDB relationtb.GroupRequestModelInterface, ctxTx tx.CtxTx, groupHash cache.GroupHash) GroupDatabase { +func NewGroupDatabase( + rdb redis.UniversalClient, + groupDB relationtb.GroupModelInterface, + groupMemberDB relationtb.GroupMemberModelInterface, + groupRequestDB relationtb.GroupRequestModelInterface, + ctxTx tx.CtxTx, + groupHash cache.GroupHash, +) GroupDatabase { rcOptions := rockscache.NewDefaultOptions() rcOptions.StrongConsistency = true rcOptions.RandomExpireAdjustment = 0.2 diff --git a/pkg/common/db/controller/third.go b/pkg/common/db/controller/third.go index cf59491ca..fb5b0ccbe 100644 --- a/pkg/common/db/controller/third.go +++ b/pkg/common/db/controller/third.go @@ -16,9 +16,10 @@ package controller import ( "context" - "github.com/OpenIMSDK/tools/pagination" "time" + "github.com/OpenIMSDK/tools/pagination" + "github.com/openimsdk/open-im-server/v3/pkg/common/db/cache" "github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation" ) diff --git a/pkg/common/db/controller/user.go b/pkg/common/db/controller/user.go index 48a2592a4..ca703b729 100644 --- a/pkg/common/db/controller/user.go +++ b/pkg/common/db/controller/user.go @@ -16,10 +16,12 @@ package controller import ( "context" + "time" + "github.com/OpenIMSDK/tools/pagination" "github.com/OpenIMSDK/tools/tx" + "github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation" - "time" "github.com/OpenIMSDK/protocol/user" diff --git a/pkg/common/db/mgo/black.go b/pkg/common/db/mgo/black.go index 6830ae26e..6235639aa 100644 --- a/pkg/common/db/mgo/black.go +++ b/pkg/common/db/mgo/black.go @@ -2,12 +2,14 @@ package mgo import ( "context" + "github.com/OpenIMSDK/tools/mgoutil" "github.com/OpenIMSDK/tools/pagination" - "github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/mongo" "go.mongodb.org/mongo-driver/mongo/options" + + "github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation" ) func NewBlackMongo(db *mongo.Database) (relation.BlackModelInterface, error) { diff --git a/pkg/common/db/mgo/conversation.go b/pkg/common/db/mgo/conversation.go index 0b2b87eaf..72d04ebb3 100644 --- a/pkg/common/db/mgo/conversation.go +++ b/pkg/common/db/mgo/conversation.go @@ -2,14 +2,16 @@ package mgo import ( "context" + "time" + "github.com/OpenIMSDK/protocol/constant" "github.com/OpenIMSDK/tools/mgoutil" "github.com/OpenIMSDK/tools/pagination" - "github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/mongo" "go.mongodb.org/mongo-driver/mongo/options" - "time" + + "github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation" ) func NewConversationMongo(db *mongo.Database) (*ConversationMgo, error) { @@ -56,7 +58,12 @@ func (c *ConversationMgo) Find(ctx context.Context, ownerUserID string, conversa } func (c *ConversationMgo) FindUserID(ctx context.Context, userIDs []string, conversationIDs []string) ([]string, error) { - return mgoutil.Find[string](ctx, c.coll, bson.M{"owner_user_id": bson.M{"$in": userIDs}, "conversation_id": bson.M{"$in": conversationIDs}}, options.Find().SetProjection(bson.M{"_id": 0, "owner_user_id": 1})) + return mgoutil.Find[string]( + ctx, + c.coll, + bson.M{"owner_user_id": bson.M{"$in": userIDs}, "conversation_id": bson.M{"$in": conversationIDs}}, + options.Find().SetProjection(bson.M{"_id": 0, "owner_user_id": 1}), + ) } func (c *ConversationMgo) FindUserIDAllConversationID(ctx context.Context, userID string) ([]string, error) { @@ -134,5 +141,10 @@ func (c *ConversationMgo) GetConversationIDsNeedDestruct(ctx context.Context) ([ } func (c *ConversationMgo) GetConversationNotReceiveMessageUserIDs(ctx context.Context, conversationID string) ([]string, error) { - return mgoutil.Find[string](ctx, c.coll, bson.M{"conversation_id": conversationID, "recv_msg_opt": bson.M{"$ne": constant.ReceiveMessage}}, options.Find().SetProjection(bson.M{"_id": 0, "owner_user_id": 1})) + return mgoutil.Find[string]( + ctx, + c.coll, + bson.M{"conversation_id": conversationID, "recv_msg_opt": bson.M{"$ne": constant.ReceiveMessage}}, + options.Find().SetProjection(bson.M{"_id": 0, "owner_user_id": 1}), + ) } diff --git a/pkg/common/db/mgo/friend.go b/pkg/common/db/mgo/friend.go index 503cd4cb7..aa9cb0301 100644 --- a/pkg/common/db/mgo/friend.go +++ b/pkg/common/db/mgo/friend.go @@ -2,13 +2,15 @@ package mgo import ( "context" + "github.com/OpenIMSDK/tools/mgoutil" "github.com/OpenIMSDK/tools/pagination" "go.mongodb.org/mongo-driver/mongo/options" - "github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/mongo" + + "github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation" ) // FriendMgo implements FriendModelInterface using MongoDB as the storage backend. diff --git a/pkg/common/db/mgo/friend_request.go b/pkg/common/db/mgo/friend_request.go index b324878fd..c1be87ca2 100644 --- a/pkg/common/db/mgo/friend_request.go +++ b/pkg/common/db/mgo/friend_request.go @@ -2,13 +2,15 @@ package mgo import ( "context" + "github.com/OpenIMSDK/tools/mgoutil" "github.com/OpenIMSDK/tools/pagination" "go.mongodb.org/mongo-driver/mongo/options" - "github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/mongo" + + "github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation" ) func NewFriendRequestMongo(db *mongo.Database) (relation.FriendRequestModelInterface, error) { diff --git a/pkg/common/db/mgo/group.go b/pkg/common/db/mgo/group.go index cf2bb9021..3553b6b56 100644 --- a/pkg/common/db/mgo/group.go +++ b/pkg/common/db/mgo/group.go @@ -2,13 +2,15 @@ package mgo import ( "context" + "time" + "github.com/OpenIMSDK/tools/mgoutil" "github.com/OpenIMSDK/tools/pagination" - "github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/mongo" "go.mongodb.org/mongo-driver/mongo/options" - "time" + + "github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation" ) func NewGroupMongo(db *mongo.Database) (relation.GroupModelInterface, error) { diff --git a/pkg/common/db/mgo/group_member.go b/pkg/common/db/mgo/group_member.go index d4acc0b1e..ed09a028b 100644 --- a/pkg/common/db/mgo/group_member.go +++ b/pkg/common/db/mgo/group_member.go @@ -2,13 +2,15 @@ package mgo import ( "context" + "github.com/OpenIMSDK/protocol/constant" "github.com/OpenIMSDK/tools/mgoutil" "github.com/OpenIMSDK/tools/pagination" - "github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/mongo" "go.mongodb.org/mongo-driver/mongo/options" + + "github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation" ) func NewGroupMember(db *mongo.Database) (relation.GroupMemberModelInterface, error) { diff --git a/pkg/common/db/mgo/group_request.go b/pkg/common/db/mgo/group_request.go index fd68610c3..e88a39bf5 100644 --- a/pkg/common/db/mgo/group_request.go +++ b/pkg/common/db/mgo/group_request.go @@ -2,12 +2,14 @@ package mgo import ( "context" + "github.com/OpenIMSDK/tools/mgoutil" "github.com/OpenIMSDK/tools/pagination" - "github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/mongo" "go.mongodb.org/mongo-driver/mongo/options" + + "github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation" ) func NewGroupRequestMgo(db *mongo.Database) (relation.GroupRequestModelInterface, error) { diff --git a/pkg/common/db/mgo/log.go b/pkg/common/db/mgo/log.go index cde03a217..aa280fcf2 100644 --- a/pkg/common/db/mgo/log.go +++ b/pkg/common/db/mgo/log.go @@ -2,13 +2,15 @@ package mgo import ( "context" + "time" + "github.com/OpenIMSDK/tools/mgoutil" "github.com/OpenIMSDK/tools/pagination" - "github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/mongo" "go.mongodb.org/mongo-driver/mongo/options" - "time" + + "github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation" ) func NewLogMongo(db *mongo.Database) (relation.LogInterface, error) { diff --git a/pkg/common/db/mgo/object.go b/pkg/common/db/mgo/object.go index 395e3b010..5976b2d28 100644 --- a/pkg/common/db/mgo/object.go +++ b/pkg/common/db/mgo/object.go @@ -2,11 +2,13 @@ package mgo import ( "context" + "github.com/OpenIMSDK/tools/mgoutil" - "github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/mongo" "go.mongodb.org/mongo-driver/mongo/options" + + "github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation" ) func NewS3Mongo(db *mongo.Database) (relation.ObjectInfoModelInterface, error) { diff --git a/pkg/common/db/mgo/user.go b/pkg/common/db/mgo/user.go index e8395b6c3..4a53cb3c4 100644 --- a/pkg/common/db/mgo/user.go +++ b/pkg/common/db/mgo/user.go @@ -2,13 +2,15 @@ package mgo import ( "context" + "time" + "github.com/OpenIMSDK/tools/mgoutil" "github.com/OpenIMSDK/tools/pagination" - "github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/mongo" "go.mongodb.org/mongo-driver/mongo/options" - "time" + + "github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation" ) func NewUserMongo(db *mongo.Database) (relation.UserModelInterface, error) { diff --git a/pkg/common/db/table/relation/black.go b/pkg/common/db/table/relation/black.go index 61aaa0980..50499054c 100644 --- a/pkg/common/db/table/relation/black.go +++ b/pkg/common/db/table/relation/black.go @@ -16,8 +16,9 @@ package relation import ( "context" - "github.com/OpenIMSDK/tools/pagination" "time" + + "github.com/OpenIMSDK/tools/pagination" ) type BlackModel struct { diff --git a/pkg/common/db/table/relation/conversation.go b/pkg/common/db/table/relation/conversation.go index a97c511c5..e0a5268ca 100644 --- a/pkg/common/db/table/relation/conversation.go +++ b/pkg/common/db/table/relation/conversation.go @@ -16,8 +16,9 @@ package relation import ( "context" - "github.com/OpenIMSDK/tools/pagination" "time" + + "github.com/OpenIMSDK/tools/pagination" ) type ConversationModel struct { diff --git a/pkg/common/db/table/relation/friend.go b/pkg/common/db/table/relation/friend.go index b0e88e750..75dbea850 100644 --- a/pkg/common/db/table/relation/friend.go +++ b/pkg/common/db/table/relation/friend.go @@ -16,8 +16,9 @@ package relation import ( "context" - "github.com/OpenIMSDK/tools/pagination" "time" + + "github.com/OpenIMSDK/tools/pagination" ) // FriendModel represents the data structure for a friend relationship in MongoDB. diff --git a/pkg/common/db/table/relation/friend_request.go b/pkg/common/db/table/relation/friend_request.go index 68343c15f..8dceb0778 100644 --- a/pkg/common/db/table/relation/friend_request.go +++ b/pkg/common/db/table/relation/friend_request.go @@ -16,8 +16,9 @@ package relation import ( "context" - "github.com/OpenIMSDK/tools/pagination" "time" + + "github.com/OpenIMSDK/tools/pagination" ) type FriendRequestModel struct { diff --git a/pkg/common/db/table/relation/group.go b/pkg/common/db/table/relation/group.go index 0e6e123cd..bb1ddd878 100644 --- a/pkg/common/db/table/relation/group.go +++ b/pkg/common/db/table/relation/group.go @@ -16,8 +16,9 @@ package relation import ( "context" - "github.com/OpenIMSDK/tools/pagination" "time" + + "github.com/OpenIMSDK/tools/pagination" ) type GroupModel struct { diff --git a/pkg/common/db/table/relation/group_member.go b/pkg/common/db/table/relation/group_member.go index 5779ac16e..88ab87739 100644 --- a/pkg/common/db/table/relation/group_member.go +++ b/pkg/common/db/table/relation/group_member.go @@ -16,8 +16,9 @@ package relation import ( "context" - "github.com/OpenIMSDK/tools/pagination" "time" + + "github.com/OpenIMSDK/tools/pagination" ) type GroupMemberModel struct { diff --git a/pkg/common/db/table/relation/group_request.go b/pkg/common/db/table/relation/group_request.go index d0f41aada..39999d799 100644 --- a/pkg/common/db/table/relation/group_request.go +++ b/pkg/common/db/table/relation/group_request.go @@ -16,8 +16,9 @@ package relation import ( "context" - "github.com/OpenIMSDK/tools/pagination" "time" + + "github.com/OpenIMSDK/tools/pagination" ) type GroupRequestModel struct { diff --git a/pkg/common/db/table/relation/log.go b/pkg/common/db/table/relation/log.go index 648e7dfac..f690ff8aa 100644 --- a/pkg/common/db/table/relation/log.go +++ b/pkg/common/db/table/relation/log.go @@ -2,8 +2,9 @@ package relation import ( "context" - "github.com/OpenIMSDK/tools/pagination" "time" + + "github.com/OpenIMSDK/tools/pagination" ) type LogModel struct { diff --git a/pkg/common/db/table/relation/user.go b/pkg/common/db/table/relation/user.go index bec717fea..1f26d751f 100644 --- a/pkg/common/db/table/relation/user.go +++ b/pkg/common/db/table/relation/user.go @@ -16,8 +16,9 @@ package relation import ( "context" - "github.com/OpenIMSDK/tools/pagination" "time" + + "github.com/OpenIMSDK/tools/pagination" ) type UserModel struct { diff --git a/pkg/rpcclient/grouphash/grouphash.go b/pkg/rpcclient/grouphash/grouphash.go index 7765493fd..b5570dccd 100644 --- a/pkg/rpcclient/grouphash/grouphash.go +++ b/pkg/rpcclient/grouphash/grouphash.go @@ -5,6 +5,7 @@ import ( "crypto/md5" "encoding/binary" "encoding/json" + "github.com/OpenIMSDK/protocol/group" "github.com/OpenIMSDK/protocol/sdkws" "github.com/OpenIMSDK/tools/utils" diff --git a/tools/component/component_test.go b/tools/component/component_test.go index c4f7f7e16..08ea0fda8 100644 --- a/tools/component/component_test.go +++ b/tools/component/component_test.go @@ -20,9 +20,10 @@ import ( "testing" "time" - "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/redis/go-redis/v9" "github.com/stretchr/testify/assert" + + "github.com/openimsdk/open-im-server/v3/pkg/common/config" ) func TestCheckMysql(t *testing.T) { diff --git a/tools/up35/README.md b/tools/up35/README.md new file mode 100644 index 000000000..c5bdcd3b6 --- /dev/null +++ b/tools/up35/README.md @@ -0,0 +1,67 @@ +# README for OpenIM Server Data Conversion Tool + +## Overview + +This tool is part of the OpenIM Server suite, specifically designed for data conversion between MySQL and MongoDB databases. It handles the migration of various data types, including user information, friendships, group memberships, and more from a MySQL database to MongoDB, ensuring data consistency and integrity during the transition. + +## Features + ++ **Configurable Database Connections:** Supports connections to both MySQL and MongoDB, configurable through a YAML file. ++ **Data Conversion Tasks:** Converts a range of data models, including user profiles, friend requests, group memberships, and logs. ++ **Version Control:** Maintains data versioning, ensuring only necessary migrations are performed. ++ **Error Handling:** Robust error handling for database connectivity and query execution. + +## Requirements + ++ Go programming environment ++ MySQL and MongoDB servers ++ OpenIM Server dependencies installed + +## Installation + +1. Ensure Go is installed and set up on your system. +2. Clone the OpenIM Server repository. +3. Navigate to the directory containing this tool. +4. Install required dependencies. + +## Configuration + ++ Configuration is managed through a YAML file specified at runtime. ++ Set up the MySQL and MongoDB connection parameters in the config file. + +## Usage + +To run the tool, use the following command from the terminal: + +```go +make build BINS="up35" +``` + +Where `path/to/config.yaml` is the path to your configuration file. + +## Functionality + +The main functions of the script include: + ++ `InitConfig(path string)`: Reads and parses the YAML configuration file. ++ `GetMysql()`: Establishes a connection to the MySQL database. ++ `GetMongo()`: Establishes a connection to the MongoDB database. ++ `Main(path string)`: Orchestrates the data migration process. ++ `SetMongoDataVersion(db *mongo.Database, curver string)`: Updates the data version in MongoDB after migration. ++ `NewTask(...)`: Generic function to handle the migration of different data types. ++ `insertMany(coll *mongo.Collection, objs []any)`: Inserts multiple records into a MongoDB collection. ++ `getColl(obj any)`: Retrieves the MongoDB collection associated with a given object. ++ `convert struct`: Contains methods for converting MySQL models to MongoDB models. + +## Notes + ++ Ensure that the MySQL and MongoDB instances are accessible and that the credentials provided in the config file are correct. ++ It is advisable to backup databases before running the migration to prevent data loss. + +## Contributing + +Contributions to improve the tool or address issues are welcome. Please follow the project's contribution guidelines. + +## License + +Refer to the project's license document for usage and distribution rights. \ No newline at end of file diff --git a/tools/up35/main.go b/tools/up35/up35.go similarity index 98% rename from tools/up35/main.go rename to tools/up35/up35.go index 5d4740fca..2b7daf9f9 100644 --- a/tools/up35/main.go +++ b/tools/up35/up35.go @@ -5,12 +5,13 @@ import ( "errors" "flag" "fmt" + "log" + "os" + "path/filepath" + "reflect" + "strconv" + "github.com/go-sql-driver/mysql" - "github.com/openimsdk/open-im-server/v3/pkg/common/config" - "github.com/openimsdk/open-im-server/v3/pkg/common/db/mgo" - mongoModel "github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation" - "github.com/openimsdk/open-im-server/v3/pkg/common/db/unrelation" - mysqlModel "github.com/openimsdk/open-im-server/v3/tools/data-conversion/openim/mysql/v3" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/mongo" "go.mongodb.org/mongo-driver/mongo/options" @@ -18,10 +19,12 @@ import ( gormMysql "gorm.io/driver/mysql" "gorm.io/gorm" "gorm.io/gorm/logger" - "log" - "os" - "reflect" - "strconv" + + "github.com/openimsdk/open-im-server/v3/pkg/common/config" + "github.com/openimsdk/open-im-server/v3/pkg/common/db/mgo" + mongoModel "github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation" + "github.com/openimsdk/open-im-server/v3/pkg/common/db/unrelation" + mysqlModel "github.com/openimsdk/open-im-server/v3/tools/data-conversion/openim/mysql/v3" ) const ( @@ -31,9 +34,12 @@ const ( ) func main() { - var path string - flag.StringVar(&path, "c", "", "path config file") + var dir string + flag.StringVar(&dir, "c", "../../../../../", "path to config directory") flag.Parse() + + path := filepath.Join(dir, "config.yaml") + if err := Main(path); err != nil { log.Fatal(err) return