feat: add format set

Signed-off-by: Xinwei Xiong(cubxxw) <3293172751nss@gmail.com>
pull/1508/head
Xinwei Xiong(cubxxw) 2 years ago
parent ec840aa0f6
commit 3b74ad56d4

@ -24,6 +24,7 @@ services:
restart: always restart: always
networks: networks:
server: server:
# open-im-server_server
ipv4_address: ${MYSQL_NETWORK_ADDRESS} ipv4_address: ${MYSQL_NETWORK_ADDRESS}
mongodb: mongodb:
@ -143,68 +144,68 @@ services:
server: server:
ipv4_address: ${OPENIM_WEB_NETWORK_ADDRESS} ipv4_address: ${OPENIM_WEB_NETWORK_ADDRESS}
openim-admin: # openim-admin:
image: ${IMAGE_REGISTRY}/openim-admin-front:v3.4.0 # image: ${IMAGE_REGISTRY}/openim-admin-front:v3.4.0
# image: ghcr.io/openimsdk/openim-admin-front:v3.4.0 # # image: ghcr.io/openimsdk/openim-admin-front:v3.4.0
# image: registry.cn-hangzhou.aliyuncs.com/openimsdk/openim-admin-front:v3.4.0 # # image: registry.cn-hangzhou.aliyuncs.com/openimsdk/openim-admin-front:v3.4.0
# image: openim/openim-admin-front:v3.4.0 # # image: openim/openim-admin-front:v3.4.0
container_name: openim-admin # container_name: openim-admin
restart: always # restart: always
ports: # ports:
- "${OPENIM_ADMIN_FRONT_PORT}:80" # - "${OPENIM_ADMIN_FRONT_PORT}:80"
networks: # networks:
server: # server:
ipv4_address: ${OPENIM_ADMIN_FRONT_NETWORK_ADDRESS} # ipv4_address: ${OPENIM_ADMIN_FRONT_NETWORK_ADDRESS}
prometheus: # prometheus:
image: prom/prometheus # image: prom/prometheus
container_name: prometheus # container_name: prometheus
hostname: prometheus # hostname: prometheus
restart: always # restart: always
volumes: # volumes:
- ./config/prometheus.yml:/etc/prometheus/prometheus.yml # - ./config/prometheus.yml:/etc/prometheus/prometheus.yml
- ./config/instance-down-rules.yml:/etc/prometheus/instance-down-rules.yml # - ./config/instance-down-rules.yml:/etc/prometheus/instance-down-rules.yml
ports: # ports:
- "${PROMETHEUS_PORT}:9090" # - "${PROMETHEUS_PORT}:9090"
networks: # networks:
server: # server:
ipv4_address: ${PROMETHEUS_NETWORK_ADDRESS} # ipv4_address: ${PROMETHEUS_NETWORK_ADDRESS}
alertmanager: # alertmanager:
image: prom/alertmanager # image: prom/alertmanager
container_name: alertmanager # container_name: alertmanager
hostname: alertmanager # hostname: alertmanager
restart: always # restart: always
volumes: # volumes:
- ./config/alertmanager.yml:/etc/alertmanager/alertmanager.yml # - ./config/alertmanager.yml:/etc/alertmanager/alertmanager.yml
- ./config/email.tmpl:/etc/alertmanager/email.tmpl # - ./config/email.tmpl:/etc/alertmanager/email.tmpl
ports: # ports:
- "${ALERT_MANAGER_PORT}:9093" # - "${ALERT_MANAGER_PORT}:9093"
networks: # networks:
server: # server:
ipv4_address: ${ALERT_MANAGER_NETWORK_ADDRESS} # ipv4_address: ${ALERT_MANAGER_NETWORK_ADDRESS}
grafana: # grafana:
image: grafana/grafana # image: grafana/grafana
container_name: grafana # container_name: grafana
hostname: grafana # hostname: grafana
user: root # user: root
restart: always # restart: always
ports: # ports:
- "${GRAFANA_PORT}:3000" # - "${GRAFANA_PORT}:3000"
volumes: # volumes:
- ${DATA_DIR}/components/grafana:/var/lib/grafana # - ${DATA_DIR}/components/grafana:/var/lib/grafana
networks: # networks:
server: # server:
ipv4_address: ${GRAFANA_NETWORK_ADDRESS} # ipv4_address: ${GRAFANA_NETWORK_ADDRESS}
node-exporter: # node-exporter:
image: quay.io/prometheus/node-exporter # image: quay.io/prometheus/node-exporter
container_name: node-exporter # container_name: node-exporter
hostname: node-exporter # hostname: node-exporter
restart: always # restart: always
ports: # ports:
- "${NODE_EXPORTER_PORT}:9100" # - "${NODE_EXPORTER_PORT}:9100"
networks: # networks:
server: # server:
ipv4_address: ${NODE_EXPORTER_NETWORK_ADDRESS} # ipv4_address: ${NODE_EXPORTER_NETWORK_ADDRESS}

@ -17,7 +17,9 @@ package conversation
import ( import (
"context" "context"
"errors" "errors"
"github.com/OpenIMSDK/tools/tx" "github.com/OpenIMSDK/tools/tx"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/mgo" "github.com/openimsdk/open-im-server/v3/pkg/common/db/mgo"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/unrelation" "github.com/openimsdk/open-im-server/v3/pkg/common/db/unrelation"

@ -16,6 +16,7 @@ package friend
import ( import (
"context" "context"
"github.com/OpenIMSDK/tools/tx" "github.com/OpenIMSDK/tools/tx"
"github.com/OpenIMSDK/protocol/sdkws" "github.com/OpenIMSDK/protocol/sdkws"

@ -17,17 +17,19 @@ package group
import ( import (
"context" "context"
"fmt" "fmt"
pbconversation "github.com/OpenIMSDK/protocol/conversation"
"github.com/OpenIMSDK/protocol/wrapperspb"
"github.com/OpenIMSDK/tools/tx"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/mgo"
"github.com/openimsdk/open-im-server/v3/pkg/rpcclient/grouphash"
"math/big" "math/big"
"math/rand" "math/rand"
"strconv" "strconv"
"strings" "strings"
"time" "time"
pbconversation "github.com/OpenIMSDK/protocol/conversation"
"github.com/OpenIMSDK/protocol/wrapperspb"
"github.com/OpenIMSDK/tools/tx"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/mgo"
"github.com/openimsdk/open-im-server/v3/pkg/rpcclient/grouphash"
"github.com/openimsdk/open-im-server/v3/pkg/authverify" "github.com/openimsdk/open-im-server/v3/pkg/authverify"
"github.com/openimsdk/open-im-server/v3/pkg/msgprocessor" "github.com/openimsdk/open-im-server/v3/pkg/msgprocessor"

@ -17,6 +17,7 @@ package group
import ( import (
"context" "context"
"errors" "errors"
pbgroup "github.com/OpenIMSDK/protocol/group" pbgroup "github.com/OpenIMSDK/protocol/group"
) )

@ -17,11 +17,12 @@ package third
import ( import (
"context" "context"
"fmt" "fmt"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/mgo"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/unrelation"
"net/url" "net/url"
"time" "time"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/mgo"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/unrelation"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/s3" "github.com/openimsdk/open-im-server/v3/pkg/common/db/s3"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/s3/cos" "github.com/openimsdk/open-im-server/v3/pkg/common/db/s3/cos"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/s3/minio" "github.com/openimsdk/open-im-server/v3/pkg/common/db/s3/minio"

@ -17,11 +17,13 @@ package user
import ( import (
"context" "context"
"errors" "errors"
"github.com/OpenIMSDK/tools/tx"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/mgo"
"strings" "strings"
"time" "time"
"github.com/OpenIMSDK/tools/tx"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/mgo"
"github.com/OpenIMSDK/protocol/constant" "github.com/OpenIMSDK/protocol/constant"
"github.com/OpenIMSDK/protocol/sdkws" "github.com/OpenIMSDK/protocol/sdkws"
"github.com/OpenIMSDK/tools/errs" "github.com/OpenIMSDK/tools/errs"

@ -16,10 +16,11 @@ package tools
import ( import (
"context" "context"
"github.com/OpenIMSDK/protocol/sdkws"
"math/rand" "math/rand"
"time" "time"
"github.com/OpenIMSDK/protocol/sdkws"
"github.com/OpenIMSDK/tools/log" "github.com/OpenIMSDK/tools/log"
"github.com/OpenIMSDK/tools/mcontext" "github.com/OpenIMSDK/tools/mcontext"
"github.com/OpenIMSDK/tools/utils" "github.com/OpenIMSDK/tools/utils"

@ -17,10 +17,12 @@ package tools
import ( import (
"context" "context"
"fmt" "fmt"
"math"
"github.com/OpenIMSDK/protocol/sdkws" "github.com/OpenIMSDK/protocol/sdkws"
"github.com/OpenIMSDK/tools/tx" "github.com/OpenIMSDK/tools/tx"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/mgo" "github.com/openimsdk/open-im-server/v3/pkg/common/db/mgo"
"math"
"github.com/redis/go-redis/v9" "github.com/redis/go-redis/v9"
"google.golang.org/grpc" "google.golang.org/grpc"

@ -46,7 +46,7 @@ func (a *RpcCmd) Exec() error {
return a.Execute() return a.Execute()
} }
func (a *RpcCmd) StartSvr(name string, rpcFn func(discov discoveryregistry.SvcDiscoveryRegistry, server *grpc.Server) error,) error { func (a *RpcCmd) StartSvr(name string, rpcFn func(discov discoveryregistry.SvcDiscoveryRegistry, server *grpc.Server) error) error {
if a.GetPortFlag() == 0 { if a.GetPortFlag() == 0 {
return errors.New("port is required") return errors.New("port is required")
} }

@ -19,6 +19,7 @@ import (
"testing" "testing"
"github.com/OpenIMSDK/protocol/sdkws" "github.com/OpenIMSDK/protocol/sdkws"
relationtb "github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation" relationtb "github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation"
) )

@ -17,11 +17,12 @@ package cache
import ( import (
"context" "context"
"fmt" "fmt"
"github.com/OpenIMSDK/protocol/constant"
"github.com/OpenIMSDK/tools/errs"
"strconv" "strconv"
"time" "time"
"github.com/OpenIMSDK/protocol/constant"
"github.com/OpenIMSDK/tools/errs"
"github.com/OpenIMSDK/tools/log" "github.com/OpenIMSDK/tools/log"
"github.com/dtm-labs/rockscache" "github.com/dtm-labs/rockscache"
@ -318,7 +319,12 @@ func (g *GroupCacheRedis) GetGroupMembersInfo(ctx context.Context, groupID strin
}) })
} }
func (g *GroupCacheRedis) GetGroupMembersPage(ctx context.Context, groupID string, userIDs []string, showNumber, pageNumber int32) (total uint32, groupMembers []*relationtb.GroupMemberModel, err error) { func (g *GroupCacheRedis) GetGroupMembersPage(
ctx context.Context,
groupID string,
userIDs []string,
showNumber, pageNumber int32,
) (total uint32, groupMembers []*relationtb.GroupMemberModel, err error) {
groupMemberIDs, err := g.GetGroupMemberIDs(ctx, groupID) groupMemberIDs, err := g.GetGroupMemberIDs(ctx, groupID)
if err != nil { if err != nil {
return 0, nil, err return 0, nil, err

@ -18,11 +18,12 @@ import (
"context" "context"
"encoding/json" "encoding/json"
"errors" "errors"
relationtb "github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation"
"hash/crc32" "hash/crc32"
"strconv" "strconv"
"time" "time"
relationtb "github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation"
"github.com/OpenIMSDK/tools/log" "github.com/OpenIMSDK/tools/log"
"github.com/OpenIMSDK/protocol/constant" "github.com/OpenIMSDK/protocol/constant"

@ -16,6 +16,7 @@ package controller
import ( import (
"context" "context"
"github.com/OpenIMSDK/tools/pagination" "github.com/OpenIMSDK/tools/pagination"
"github.com/OpenIMSDK/tools/log" "github.com/OpenIMSDK/tools/log"

@ -16,9 +16,10 @@ package controller
import ( import (
"context" "context"
"github.com/OpenIMSDK/tools/pagination"
"time" "time"
"github.com/OpenIMSDK/tools/pagination"
"github.com/openimsdk/open-im-server/v3/pkg/msgprocessor" "github.com/openimsdk/open-im-server/v3/pkg/msgprocessor"
"github.com/OpenIMSDK/protocol/constant" "github.com/OpenIMSDK/protocol/constant"
@ -237,7 +238,9 @@ func (c *conversationDatabase) SetUserConversations(ctx context.Context, ownerUs
if err != nil { if err != nil {
return err return err
} }
cache = cache.DelConversationIDs(ownerUserID).DelUserConversationIDsHash(ownerUserID).DelConversationNotReceiveMessageUserIDs(utils.Slice(notExistConversations, func(e *relationtb.ConversationModel) string { return e.ConversationID })...) cache = cache.DelConversationIDs(ownerUserID).
DelUserConversationIDsHash(ownerUserID).
DelConversationNotReceiveMessageUserIDs(utils.Slice(notExistConversations, func(e *relationtb.ConversationModel) string { return e.ConversationID })...)
} }
return cache.ExecDel(ctx) return cache.ExecDel(ctx)
}) })

@ -16,9 +16,10 @@ package controller
import ( import (
"context" "context"
"github.com/OpenIMSDK/tools/pagination"
"time" "time"
"github.com/OpenIMSDK/tools/pagination"
"github.com/OpenIMSDK/protocol/constant" "github.com/OpenIMSDK/protocol/constant"
"github.com/OpenIMSDK/tools/errs" "github.com/OpenIMSDK/tools/errs"
"github.com/OpenIMSDK/tools/log" "github.com/OpenIMSDK/tools/log"
@ -97,7 +98,10 @@ func (f *friendDatabase) AddFriendRequest(ctx context.Context, fromUserID, toUse
m["create_time"] = time.Now() m["create_time"] = time.Now()
return f.friendRequest.UpdateByMap(ctx, fromUserID, toUserID, m) return f.friendRequest.UpdateByMap(ctx, fromUserID, toUserID, m)
case relation.IsNotFound(err): case relation.IsNotFound(err):
return f.friendRequest.Create(ctx, []*relation.FriendRequestModel{{FromUserID: fromUserID, ToUserID: toUserID, ReqMsg: reqMsg, Ex: ex, CreateTime: time.Now(), HandleTime: time.Unix(0, 0)}}) return f.friendRequest.Create(
ctx,
[]*relation.FriendRequestModel{{FromUserID: fromUserID, ToUserID: toUserID, ReqMsg: reqMsg, Ex: ex, CreateTime: time.Now(), HandleTime: time.Unix(0, 0)}},
)
default: default:
return err return err
} }

@ -16,9 +16,10 @@ package controller
import ( import (
"context" "context"
"time"
"github.com/OpenIMSDK/tools/pagination" "github.com/OpenIMSDK/tools/pagination"
"github.com/dtm-labs/rockscache" "github.com/dtm-labs/rockscache"
"time"
"github.com/OpenIMSDK/protocol/constant" "github.com/OpenIMSDK/protocol/constant"
"github.com/OpenIMSDK/tools/tx" "github.com/OpenIMSDK/tools/tx"
@ -74,7 +75,14 @@ type GroupDatabase interface {
DeleteGroupMemberHash(ctx context.Context, groupIDs []string) error DeleteGroupMemberHash(ctx context.Context, groupIDs []string) error
} }
func NewGroupDatabase(rdb redis.UniversalClient, groupDB relationtb.GroupModelInterface, groupMemberDB relationtb.GroupMemberModelInterface, groupRequestDB relationtb.GroupRequestModelInterface, ctxTx tx.CtxTx, groupHash cache.GroupHash) GroupDatabase { func NewGroupDatabase(
rdb redis.UniversalClient,
groupDB relationtb.GroupModelInterface,
groupMemberDB relationtb.GroupMemberModelInterface,
groupRequestDB relationtb.GroupRequestModelInterface,
ctxTx tx.CtxTx,
groupHash cache.GroupHash,
) GroupDatabase {
rcOptions := rockscache.NewDefaultOptions() rcOptions := rockscache.NewDefaultOptions()
rcOptions.StrongConsistency = true rcOptions.StrongConsistency = true
rcOptions.RandomExpireAdjustment = 0.2 rcOptions.RandomExpireAdjustment = 0.2

@ -16,9 +16,10 @@ package controller
import ( import (
"context" "context"
"github.com/OpenIMSDK/tools/pagination"
"time" "time"
"github.com/OpenIMSDK/tools/pagination"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/cache" "github.com/openimsdk/open-im-server/v3/pkg/common/db/cache"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation" "github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation"
) )

@ -16,10 +16,12 @@ package controller
import ( import (
"context" "context"
"time"
"github.com/OpenIMSDK/tools/pagination" "github.com/OpenIMSDK/tools/pagination"
"github.com/OpenIMSDK/tools/tx" "github.com/OpenIMSDK/tools/tx"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation" "github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation"
"time"
"github.com/OpenIMSDK/protocol/user" "github.com/OpenIMSDK/protocol/user"

@ -2,12 +2,14 @@ package mgo
import ( import (
"context" "context"
"github.com/OpenIMSDK/tools/mgoutil" "github.com/OpenIMSDK/tools/mgoutil"
"github.com/OpenIMSDK/tools/pagination" "github.com/OpenIMSDK/tools/pagination"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation"
"go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/mongo" "go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options" "go.mongodb.org/mongo-driver/mongo/options"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation"
) )
func NewBlackMongo(db *mongo.Database) (relation.BlackModelInterface, error) { func NewBlackMongo(db *mongo.Database) (relation.BlackModelInterface, error) {

@ -2,14 +2,16 @@ package mgo
import ( import (
"context" "context"
"time"
"github.com/OpenIMSDK/protocol/constant" "github.com/OpenIMSDK/protocol/constant"
"github.com/OpenIMSDK/tools/mgoutil" "github.com/OpenIMSDK/tools/mgoutil"
"github.com/OpenIMSDK/tools/pagination" "github.com/OpenIMSDK/tools/pagination"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation"
"go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/mongo" "go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options" "go.mongodb.org/mongo-driver/mongo/options"
"time"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation"
) )
func NewConversationMongo(db *mongo.Database) (*ConversationMgo, error) { func NewConversationMongo(db *mongo.Database) (*ConversationMgo, error) {
@ -56,7 +58,12 @@ func (c *ConversationMgo) Find(ctx context.Context, ownerUserID string, conversa
} }
func (c *ConversationMgo) FindUserID(ctx context.Context, userIDs []string, conversationIDs []string) ([]string, error) { func (c *ConversationMgo) FindUserID(ctx context.Context, userIDs []string, conversationIDs []string) ([]string, error) {
return mgoutil.Find[string](ctx, c.coll, bson.M{"owner_user_id": bson.M{"$in": userIDs}, "conversation_id": bson.M{"$in": conversationIDs}}, options.Find().SetProjection(bson.M{"_id": 0, "owner_user_id": 1})) return mgoutil.Find[string](
ctx,
c.coll,
bson.M{"owner_user_id": bson.M{"$in": userIDs}, "conversation_id": bson.M{"$in": conversationIDs}},
options.Find().SetProjection(bson.M{"_id": 0, "owner_user_id": 1}),
)
} }
func (c *ConversationMgo) FindUserIDAllConversationID(ctx context.Context, userID string) ([]string, error) { func (c *ConversationMgo) FindUserIDAllConversationID(ctx context.Context, userID string) ([]string, error) {
@ -134,5 +141,10 @@ func (c *ConversationMgo) GetConversationIDsNeedDestruct(ctx context.Context) ([
} }
func (c *ConversationMgo) GetConversationNotReceiveMessageUserIDs(ctx context.Context, conversationID string) ([]string, error) { func (c *ConversationMgo) GetConversationNotReceiveMessageUserIDs(ctx context.Context, conversationID string) ([]string, error) {
return mgoutil.Find[string](ctx, c.coll, bson.M{"conversation_id": conversationID, "recv_msg_opt": bson.M{"$ne": constant.ReceiveMessage}}, options.Find().SetProjection(bson.M{"_id": 0, "owner_user_id": 1})) return mgoutil.Find[string](
ctx,
c.coll,
bson.M{"conversation_id": conversationID, "recv_msg_opt": bson.M{"$ne": constant.ReceiveMessage}},
options.Find().SetProjection(bson.M{"_id": 0, "owner_user_id": 1}),
)
} }

@ -2,13 +2,15 @@ package mgo
import ( import (
"context" "context"
"github.com/OpenIMSDK/tools/mgoutil" "github.com/OpenIMSDK/tools/mgoutil"
"github.com/OpenIMSDK/tools/pagination" "github.com/OpenIMSDK/tools/pagination"
"go.mongodb.org/mongo-driver/mongo/options" "go.mongodb.org/mongo-driver/mongo/options"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation"
"go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/mongo" "go.mongodb.org/mongo-driver/mongo"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation"
) )
// FriendMgo implements FriendModelInterface using MongoDB as the storage backend. // FriendMgo implements FriendModelInterface using MongoDB as the storage backend.

@ -2,13 +2,15 @@ package mgo
import ( import (
"context" "context"
"github.com/OpenIMSDK/tools/mgoutil" "github.com/OpenIMSDK/tools/mgoutil"
"github.com/OpenIMSDK/tools/pagination" "github.com/OpenIMSDK/tools/pagination"
"go.mongodb.org/mongo-driver/mongo/options" "go.mongodb.org/mongo-driver/mongo/options"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation"
"go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/mongo" "go.mongodb.org/mongo-driver/mongo"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation"
) )
func NewFriendRequestMongo(db *mongo.Database) (relation.FriendRequestModelInterface, error) { func NewFriendRequestMongo(db *mongo.Database) (relation.FriendRequestModelInterface, error) {

@ -2,13 +2,15 @@ package mgo
import ( import (
"context" "context"
"time"
"github.com/OpenIMSDK/tools/mgoutil" "github.com/OpenIMSDK/tools/mgoutil"
"github.com/OpenIMSDK/tools/pagination" "github.com/OpenIMSDK/tools/pagination"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation"
"go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/mongo" "go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options" "go.mongodb.org/mongo-driver/mongo/options"
"time"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation"
) )
func NewGroupMongo(db *mongo.Database) (relation.GroupModelInterface, error) { func NewGroupMongo(db *mongo.Database) (relation.GroupModelInterface, error) {

@ -2,13 +2,15 @@ package mgo
import ( import (
"context" "context"
"github.com/OpenIMSDK/protocol/constant" "github.com/OpenIMSDK/protocol/constant"
"github.com/OpenIMSDK/tools/mgoutil" "github.com/OpenIMSDK/tools/mgoutil"
"github.com/OpenIMSDK/tools/pagination" "github.com/OpenIMSDK/tools/pagination"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation"
"go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/mongo" "go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options" "go.mongodb.org/mongo-driver/mongo/options"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation"
) )
func NewGroupMember(db *mongo.Database) (relation.GroupMemberModelInterface, error) { func NewGroupMember(db *mongo.Database) (relation.GroupMemberModelInterface, error) {

@ -2,12 +2,14 @@ package mgo
import ( import (
"context" "context"
"github.com/OpenIMSDK/tools/mgoutil" "github.com/OpenIMSDK/tools/mgoutil"
"github.com/OpenIMSDK/tools/pagination" "github.com/OpenIMSDK/tools/pagination"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation"
"go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/mongo" "go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options" "go.mongodb.org/mongo-driver/mongo/options"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation"
) )
func NewGroupRequestMgo(db *mongo.Database) (relation.GroupRequestModelInterface, error) { func NewGroupRequestMgo(db *mongo.Database) (relation.GroupRequestModelInterface, error) {

@ -2,13 +2,15 @@ package mgo
import ( import (
"context" "context"
"time"
"github.com/OpenIMSDK/tools/mgoutil" "github.com/OpenIMSDK/tools/mgoutil"
"github.com/OpenIMSDK/tools/pagination" "github.com/OpenIMSDK/tools/pagination"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation"
"go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/mongo" "go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options" "go.mongodb.org/mongo-driver/mongo/options"
"time"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation"
) )
func NewLogMongo(db *mongo.Database) (relation.LogInterface, error) { func NewLogMongo(db *mongo.Database) (relation.LogInterface, error) {

@ -2,11 +2,13 @@ package mgo
import ( import (
"context" "context"
"github.com/OpenIMSDK/tools/mgoutil" "github.com/OpenIMSDK/tools/mgoutil"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation"
"go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/mongo" "go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options" "go.mongodb.org/mongo-driver/mongo/options"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation"
) )
func NewS3Mongo(db *mongo.Database) (relation.ObjectInfoModelInterface, error) { func NewS3Mongo(db *mongo.Database) (relation.ObjectInfoModelInterface, error) {

@ -2,13 +2,15 @@ package mgo
import ( import (
"context" "context"
"time"
"github.com/OpenIMSDK/tools/mgoutil" "github.com/OpenIMSDK/tools/mgoutil"
"github.com/OpenIMSDK/tools/pagination" "github.com/OpenIMSDK/tools/pagination"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation"
"go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/mongo" "go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options" "go.mongodb.org/mongo-driver/mongo/options"
"time"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation"
) )
func NewUserMongo(db *mongo.Database) (relation.UserModelInterface, error) { func NewUserMongo(db *mongo.Database) (relation.UserModelInterface, error) {

@ -16,8 +16,9 @@ package relation
import ( import (
"context" "context"
"github.com/OpenIMSDK/tools/pagination"
"time" "time"
"github.com/OpenIMSDK/tools/pagination"
) )
type BlackModel struct { type BlackModel struct {

@ -16,8 +16,9 @@ package relation
import ( import (
"context" "context"
"github.com/OpenIMSDK/tools/pagination"
"time" "time"
"github.com/OpenIMSDK/tools/pagination"
) )
type ConversationModel struct { type ConversationModel struct {

@ -16,8 +16,9 @@ package relation
import ( import (
"context" "context"
"github.com/OpenIMSDK/tools/pagination"
"time" "time"
"github.com/OpenIMSDK/tools/pagination"
) )
// FriendModel represents the data structure for a friend relationship in MongoDB. // FriendModel represents the data structure for a friend relationship in MongoDB.

@ -16,8 +16,9 @@ package relation
import ( import (
"context" "context"
"github.com/OpenIMSDK/tools/pagination"
"time" "time"
"github.com/OpenIMSDK/tools/pagination"
) )
type FriendRequestModel struct { type FriendRequestModel struct {

@ -16,8 +16,9 @@ package relation
import ( import (
"context" "context"
"github.com/OpenIMSDK/tools/pagination"
"time" "time"
"github.com/OpenIMSDK/tools/pagination"
) )
type GroupModel struct { type GroupModel struct {

@ -16,8 +16,9 @@ package relation
import ( import (
"context" "context"
"github.com/OpenIMSDK/tools/pagination"
"time" "time"
"github.com/OpenIMSDK/tools/pagination"
) )
type GroupMemberModel struct { type GroupMemberModel struct {

@ -16,8 +16,9 @@ package relation
import ( import (
"context" "context"
"github.com/OpenIMSDK/tools/pagination"
"time" "time"
"github.com/OpenIMSDK/tools/pagination"
) )
type GroupRequestModel struct { type GroupRequestModel struct {

@ -2,8 +2,9 @@ package relation
import ( import (
"context" "context"
"github.com/OpenIMSDK/tools/pagination"
"time" "time"
"github.com/OpenIMSDK/tools/pagination"
) )
type LogModel struct { type LogModel struct {

@ -16,8 +16,9 @@ package relation
import ( import (
"context" "context"
"github.com/OpenIMSDK/tools/pagination"
"time" "time"
"github.com/OpenIMSDK/tools/pagination"
) )
type UserModel struct { type UserModel struct {

@ -5,6 +5,7 @@ import (
"crypto/md5" "crypto/md5"
"encoding/binary" "encoding/binary"
"encoding/json" "encoding/json"
"github.com/OpenIMSDK/protocol/group" "github.com/OpenIMSDK/protocol/group"
"github.com/OpenIMSDK/protocol/sdkws" "github.com/OpenIMSDK/protocol/sdkws"
"github.com/OpenIMSDK/tools/utils" "github.com/OpenIMSDK/tools/utils"

@ -20,9 +20,10 @@ import (
"testing" "testing"
"time" "time"
"github.com/openimsdk/open-im-server/v3/pkg/common/config"
"github.com/redis/go-redis/v9" "github.com/redis/go-redis/v9"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
"github.com/openimsdk/open-im-server/v3/pkg/common/config"
) )
func TestCheckMysql(t *testing.T) { func TestCheckMysql(t *testing.T) {

@ -0,0 +1,67 @@
# README for OpenIM Server Data Conversion Tool
## Overview
This tool is part of the OpenIM Server suite, specifically designed for data conversion between MySQL and MongoDB databases. It handles the migration of various data types, including user information, friendships, group memberships, and more from a MySQL database to MongoDB, ensuring data consistency and integrity during the transition.
## Features
+ **Configurable Database Connections:** Supports connections to both MySQL and MongoDB, configurable through a YAML file.
+ **Data Conversion Tasks:** Converts a range of data models, including user profiles, friend requests, group memberships, and logs.
+ **Version Control:** Maintains data versioning, ensuring only necessary migrations are performed.
+ **Error Handling:** Robust error handling for database connectivity and query execution.
## Requirements
+ Go programming environment
+ MySQL and MongoDB servers
+ OpenIM Server dependencies installed
## Installation
1. Ensure Go is installed and set up on your system.
2. Clone the OpenIM Server repository.
3. Navigate to the directory containing this tool.
4. Install required dependencies.
## Configuration
+ Configuration is managed through a YAML file specified at runtime.
+ Set up the MySQL and MongoDB connection parameters in the config file.
## Usage
To run the tool, use the following command from the terminal:
```go
make build BINS="up35"
```
Where `path/to/config.yaml` is the path to your configuration file.
## Functionality
The main functions of the script include:
+ `InitConfig(path string)`: Reads and parses the YAML configuration file.
+ `GetMysql()`: Establishes a connection to the MySQL database.
+ `GetMongo()`: Establishes a connection to the MongoDB database.
+ `Main(path string)`: Orchestrates the data migration process.
+ `SetMongoDataVersion(db *mongo.Database, curver string)`: Updates the data version in MongoDB after migration.
+ `NewTask(...)`: Generic function to handle the migration of different data types.
+ `insertMany(coll *mongo.Collection, objs []any)`: Inserts multiple records into a MongoDB collection.
+ `getColl(obj any)`: Retrieves the MongoDB collection associated with a given object.
+ `convert struct`: Contains methods for converting MySQL models to MongoDB models.
## Notes
+ Ensure that the MySQL and MongoDB instances are accessible and that the credentials provided in the config file are correct.
+ It is advisable to backup databases before running the migration to prevent data loss.
## Contributing
Contributions to improve the tool or address issues are welcome. Please follow the project's contribution guidelines.
## License
Refer to the project's license document for usage and distribution rights.

@ -5,12 +5,13 @@ import (
"errors" "errors"
"flag" "flag"
"fmt" "fmt"
"log"
"os"
"path/filepath"
"reflect"
"strconv"
"github.com/go-sql-driver/mysql" "github.com/go-sql-driver/mysql"
"github.com/openimsdk/open-im-server/v3/pkg/common/config"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/mgo"
mongoModel "github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/unrelation"
mysqlModel "github.com/openimsdk/open-im-server/v3/tools/data-conversion/openim/mysql/v3"
"go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/mongo" "go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options" "go.mongodb.org/mongo-driver/mongo/options"
@ -18,10 +19,12 @@ import (
gormMysql "gorm.io/driver/mysql" gormMysql "gorm.io/driver/mysql"
"gorm.io/gorm" "gorm.io/gorm"
"gorm.io/gorm/logger" "gorm.io/gorm/logger"
"log"
"os" "github.com/openimsdk/open-im-server/v3/pkg/common/config"
"reflect" "github.com/openimsdk/open-im-server/v3/pkg/common/db/mgo"
"strconv" mongoModel "github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation"
"github.com/openimsdk/open-im-server/v3/pkg/common/db/unrelation"
mysqlModel "github.com/openimsdk/open-im-server/v3/tools/data-conversion/openim/mysql/v3"
) )
const ( const (
@ -31,9 +34,12 @@ const (
) )
func main() { func main() {
var path string var dir string
flag.StringVar(&path, "c", "", "path config file") flag.StringVar(&dir, "c", "../../../../../", "path to config directory")
flag.Parse() flag.Parse()
path := filepath.Join(dir, "config.yaml")
if err := Main(path); err != nil { if err := Main(path); err != nil {
log.Fatal(err) log.Fatal(err)
return return
Loading…
Cancel
Save