From 5b3e977665ec4a9b082f7dbf4f317cd14b1338de Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Fri, 30 Jun 2023 11:24:37 +0800 Subject: [PATCH] add tools --- .gitignore | 1 - internal/tools/cron_task.go | 31 ++++ internal/tools/msg.go | 125 ++++++++++++++ internal/tools/msg_test.go | 322 ++++++++++++++++++++++++++++++++++++ 4 files changed, 478 insertions(+), 1 deletion(-) create mode 100644 internal/tools/cron_task.go create mode 100644 internal/tools/msg.go create mode 100644 internal/tools/msg_test.go diff --git a/.gitignore b/.gitignore index 88708f173..dd3635cbe 100644 --- a/.gitignore +++ b/.gitignore @@ -24,7 +24,6 @@ components out-test ### Makefile ### -tools/ tmp/ bin/ output/ diff --git a/internal/tools/cron_task.go b/internal/tools/cron_task.go new file mode 100644 index 000000000..debf61a6d --- /dev/null +++ b/internal/tools/cron_task.go @@ -0,0 +1,31 @@ +package tools + +import ( + "context" + "fmt" + "sync" + + "github.com/OpenIMSDK/Open-IM-Server/pkg/common/config" + "github.com/OpenIMSDK/Open-IM-Server/pkg/common/log" + "github.com/robfig/cron/v3" +) + +func StartCronTask() error { + log.ZInfo(context.Background(), "start cron task", "cron config", config.Config.ChatRecordsClearTime) + fmt.Println("cron task start, config", config.Config.ChatRecordsClearTime) + msgTool, err := InitMsgTool() + if err != nil { + return err + } + c := cron.New() + var wg sync.WaitGroup + wg.Add(1) + _, err = c.AddFunc(config.Config.ChatRecordsClearTime, msgTool.AllConversationClearMsgAndFixSeq) + if err != nil { + fmt.Println("start cron failed", err.Error(), config.Config.ChatRecordsClearTime) + return err + } + c.Start() + wg.Wait() + return nil +} diff --git a/internal/tools/msg.go b/internal/tools/msg.go new file mode 100644 index 000000000..fc810aa95 --- /dev/null +++ b/internal/tools/msg.go @@ -0,0 +1,125 @@ +package tools + +import ( + "context" + "errors" + "fmt" + "math" + + "github.com/OpenIMSDK/Open-IM-Server/pkg/common/config" + "github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/cache" + "github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/controller" + "github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/relation" + "github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/tx" + "github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/unrelation" + "github.com/OpenIMSDK/Open-IM-Server/pkg/common/log" + "github.com/OpenIMSDK/Open-IM-Server/pkg/common/mcontext" + "github.com/OpenIMSDK/Open-IM-Server/pkg/utils" +) + +type MsgTool struct { + msgDatabase controller.CommonMsgDatabase + conversationDatabase controller.ConversationDatabase + userDatabase controller.UserDatabase + groupDatabase controller.GroupDatabase +} + +var errSeq = errors.New("cache max seq and mongo max seq is diff > 10") + +func NewMsgTool(msgDatabase controller.CommonMsgDatabase, userDatabase controller.UserDatabase, groupDatabase controller.GroupDatabase, conversationDatabase controller.ConversationDatabase) *MsgTool { + return &MsgTool{ + msgDatabase: msgDatabase, + userDatabase: userDatabase, + groupDatabase: groupDatabase, + conversationDatabase: conversationDatabase, + } +} + +func InitMsgTool() (*MsgTool, error) { + rdb, err := cache.NewRedis() + if err != nil { + return nil, err + } + mongo, err := unrelation.NewMongo() + if err != nil { + return nil, err + } + db, err := relation.NewGormDB() + if err != nil { + return nil, err + } + userDB := relation.NewUserGorm(db) + msgDatabase := controller.InitCommonMsgDatabase(rdb, mongo.GetDatabase()) + userDatabase := controller.NewUserDatabase(userDB, cache.NewUserCacheRedis(rdb, relation.NewUserGorm(db), cache.GetDefaultOpt()), tx.NewGorm(db)) + groupDatabase := controller.InitGroupDatabase(db, rdb, mongo.GetDatabase()) + conversationDatabase := controller.NewConversationDatabase(relation.NewConversationGorm(db), cache.NewConversationRedis(rdb, cache.GetDefaultOpt(), relation.NewConversationGorm(db)), tx.NewGorm(db)) + msgTool := NewMsgTool(msgDatabase, userDatabase, groupDatabase, conversationDatabase) + return msgTool, nil +} + +func (c *MsgTool) AllConversationClearMsgAndFixSeq() { + ctx := mcontext.NewCtx(utils.GetSelfFuncName()) + log.ZInfo(ctx, "============================ start del cron task ============================") + conversationIDs, err := c.conversationDatabase.GetAllConversationIDs(ctx) + if err != nil { + log.ZError(ctx, "GetAllConversationIDs failed", err) + return + } + for _, conversationID := range conversationIDs { + conversationIDs = append(conversationIDs, utils.GetNotificationConversationIDByConversationID(conversationID)) + } + c.ClearConversationsMsg(ctx, conversationIDs) + log.ZInfo(ctx, "============================ start del cron finished ============================") +} + +func (c *MsgTool) ClearConversationsMsg(ctx context.Context, conversationIDs []string) { + for _, conversationID := range conversationIDs { + if err := c.msgDatabase.DeleteConversationMsgsAndSetMinSeq(ctx, conversationID, int64(config.Config.RetainChatRecords*24*60*60)); err != nil { + log.ZError(ctx, "DeleteUserSuperGroupMsgsAndSetMinSeq failed", err, "conversationID", conversationID, "DBRetainChatRecords", config.Config.RetainChatRecords) + } + if err := c.checkMaxSeq(ctx, conversationID); err != nil { + log.ZError(ctx, "fixSeq failed", err, "conversationID", conversationID) + } + + } +} + +func (c *MsgTool) checkMaxSeqWithMongo(ctx context.Context, conversationID string, maxSeqCache int64) error { + maxSeqMongo, _, err := c.msgDatabase.GetMongoMaxAndMinSeq(ctx, conversationID) + if err != nil { + return err + } + if math.Abs(float64(maxSeqMongo-maxSeqCache)) > 10 { + return errSeq + } + return nil +} + +func (c *MsgTool) checkMaxSeq(ctx context.Context, conversationID string) error { + maxSeq, err := c.msgDatabase.GetMaxSeq(ctx, conversationID) + if err != nil { + return err + } + if err := c.checkMaxSeqWithMongo(ctx, conversationID, maxSeq); err != nil { + return err + } + return nil +} + +func (c *MsgTool) FixAllSeq(ctx context.Context) error { + conversationIDs, err := c.conversationDatabase.GetAllConversationIDs(ctx) + if err != nil { + log.ZError(ctx, "GetAllConversationIDs failed", err) + return err + } + for _, conversationID := range conversationIDs { + conversationIDs = append(conversationIDs, utils.GetNotificationConversationIDByConversationID(conversationID)) + } + for _, conversationID := range conversationIDs { + if err := c.checkMaxSeq(ctx, conversationID); err != nil { + log.ZWarn(ctx, "fixSeq failed", err, "conversationID", conversationID) + } + } + fmt.Println("fix all seq finished") + return nil +} diff --git a/internal/tools/msg_test.go b/internal/tools/msg_test.go new file mode 100644 index 000000000..bf548903d --- /dev/null +++ b/internal/tools/msg_test.go @@ -0,0 +1,322 @@ +package tools + +import ( + "context" + "strconv" + + "github.com/OpenIMSDK/Open-IM-Server/pkg/common/constant" + "github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/cache" + "github.com/OpenIMSDK/Open-IM-Server/pkg/common/mcontext" + "github.com/OpenIMSDK/Open-IM-Server/pkg/utils" + "go.mongodb.org/mongo-driver/bson" + + unRelationTb "github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/table/unrelation" + "github.com/OpenIMSDK/Open-IM-Server/pkg/common/db/unrelation" + + "testing" + "time" +) + +func GenMsgDoc(startSeq, stopSeq, delSeq, index int64, conversationID string) *unRelationTb.MsgDocModel { + msgDoc := &unRelationTb.MsgDocModel{DocID: conversationID + strconv.Itoa(int(index))} + for i := 0; i < 5000; i++ { + msgDoc.Msg = append(msgDoc.Msg, &unRelationTb.MsgInfoModel{}) + } + for i := startSeq; i <= stopSeq; i++ { + msg := &unRelationTb.MsgDataModel{ + SendID: "sendID1", + RecvID: "recvID1", + GroupID: "", + ClientMsgID: "xxx", + ServerMsgID: "xxx", + SenderPlatformID: 1, + SenderNickname: "testNickName", + SenderFaceURL: "testFaceURL", + SessionType: 1, + MsgFrom: 100, + ContentType: 101, + Content: "testContent", + Seq: i, + CreateTime: time.Now().Unix(), + Status: 1, + } + if i <= delSeq { + msg.SendTime = 10000 + } else { + msg.SendTime = utils.GetCurrentTimestampByMill() + } + msgDoc.Msg[i-1] = &unRelationTb.MsgInfoModel{Msg: msg} + } + return msgDoc +} + +func TestDeleteMongoMsgAndResetRedisSeq(t *testing.T) { + operationID := "test" + rdb, err := cache.NewRedis() + if err != nil { + return + } + mgo, err := unrelation.NewMongo() + if err != nil { + return + } + cacheModel := cache.NewMsgCacheModel(rdb) + mongoClient := mgo.GetDatabase().Collection(unRelationTb.MsgDocModel{}.TableName()) + ctx := context.Background() + ctx = mcontext.SetOperationID(ctx, operationID) + + testUID1 := "test_del_id1" + var conversationID string + conversationID = utils.GetConversationIDBySessionType(constant.SuperGroupChatType, testUID1) + _, err = mongoClient.DeleteOne(ctx, bson.M{"doc_id": conversationID + ":" + strconv.Itoa(0)}) + if err != nil { + t.Error("DeleteOne failed") + return + } + + err = cacheModel.SetMaxSeq(ctx, conversationID, 600) + if err != nil { + t.Error("SetUserMaxSeq failed") + } + msgDoc := GenMsgDoc(1, 600, 200, 0, conversationID) + if _, err := mongoClient.InsertOne(ctx, msgDoc); err != nil { + t.Error("InsertOne failed", conversationID) + } + + msgTools, err := InitMsgTool() + if err != nil { + t.Error("init failed") + return + } + msgTools.ClearConversationsMsg(ctx, []string{conversationID}) + minSeqMongo, maxSeqMongo, minSeqCache, maxSeqCache, err := msgTools.msgDatabase.GetConversationMinMaxSeqInMongoAndCache(ctx, conversationID) + if err != nil { + t.Error("GetSuperGroupMinMaxSeqInMongoAndCache failed") + return + } + if maxSeqCache != maxSeqMongo { + t.Error("checkMaxSeqWithMongo failed", conversationID) + } + if minSeqMongo != minSeqCache { + t.Error("minSeqMongo != minSeqCache", minSeqMongo, minSeqCache) + } + if minSeqCache != 201 { + t.Error("test1 is not the same", "minSeq:", minSeqCache, "targetSeq", 201) + } + + /////// uid2 + + testUID2 := "test_del_id2" + conversationID = utils.GetConversationIDBySessionType(constant.SuperGroupChatType, testUID2) + + _, err = mongoClient.DeleteOne(ctx, bson.M{"uid": conversationID + ":" + strconv.Itoa(0)}) + if err != nil { + t.Error("delete failed") + } + _, err = mongoClient.DeleteOne(ctx, bson.M{"uid": conversationID + ":" + strconv.Itoa(1)}) + if err != nil { + t.Error("delete failed") + } + + err = cacheModel.SetMaxSeq(ctx, conversationID, 7000) + if err != nil { + t.Error("SetUserMaxSeq failed") + } + msgDoc = GenMsgDoc(1, 4999, 5000, 0, conversationID) + msgDoc2 := GenMsgDoc(5000, 7000, 6000, 1, conversationID) + if _, err := mongoClient.InsertOne(ctx, msgDoc); err != nil { + t.Error("InsertOne failed", testUID1) + } + if _, err := mongoClient.InsertOne(ctx, msgDoc2); err != nil { + t.Error("InsertOne failed", testUID1) + } + + msgTools.ClearConversationsMsg(ctx, []string{conversationID}) + minSeqMongo, maxSeqMongo, minSeqCache, maxSeqCache, err = msgTools.msgDatabase.GetConversationMinMaxSeqInMongoAndCache(ctx, conversationID) + if err != nil { + t.Error("GetSuperGroupMinMaxSeqInMongoAndCache failed") + return + } + if maxSeqCache != maxSeqMongo { + t.Error("checkMaxSeqWithMongo failed", conversationID) + } + if minSeqMongo != minSeqCache { + t.Error("minSeqMongo != minSeqCache", minSeqMongo, minSeqCache) + } + if minSeqCache != 6001 { + t.Error("test1 is not the same", "minSeq:", minSeqCache, "targetSeq", 201) + } + + /////// uid3 + testUID3 := "test_del_id3" + conversationID = utils.GetConversationIDBySessionType(constant.SuperGroupChatType, testUID3) + _, err = mongoClient.DeleteOne(ctx, bson.M{"uid": conversationID + ":" + strconv.Itoa(0)}) + if err != nil { + t.Error("delete failed") + } + err = cacheModel.SetMaxSeq(ctx, conversationID, 4999) + if err != nil { + t.Error("SetUserMaxSeq failed") + } + msgDoc = GenMsgDoc(1, 4999, 5000, 0, conversationID) + if _, err := mongoClient.InsertOne(ctx, msgDoc); err != nil { + t.Error("InsertOne failed", conversationID) + } + + msgTools.ClearConversationsMsg(ctx, []string{conversationID}) + minSeqMongo, maxSeqMongo, minSeqCache, maxSeqCache, err = msgTools.msgDatabase.GetConversationMinMaxSeqInMongoAndCache(ctx, conversationID) + if err != nil { + t.Error("GetSuperGroupMinMaxSeqInMongoAndCache failed") + return + } + if maxSeqCache != maxSeqMongo { + t.Error("checkMaxSeqWithMongo failed", conversationID) + } + if minSeqMongo != minSeqCache { + t.Error("minSeqMongo != minSeqCache", minSeqMongo, minSeqCache) + } + if minSeqCache != 5000 { + t.Error("test1 is not the same", "minSeq:", minSeqCache, "targetSeq", 201) + } + + //// uid4 + testUID4 := "test_del_id4" + conversationID = utils.GetConversationIDBySessionType(constant.SuperGroupChatType, testUID4) + _, err = mongoClient.DeleteOne(ctx, bson.M{"uid": conversationID + ":" + strconv.Itoa(0)}) + if err != nil { + t.Error("delete failed") + } + _, err = mongoClient.DeleteOne(ctx, bson.M{"uid": conversationID + ":" + strconv.Itoa(1)}) + if err != nil { + t.Error("delete failed") + } + _, err = mongoClient.DeleteOne(ctx, bson.M{"uid": conversationID + ":" + strconv.Itoa(2)}) + if err != nil { + t.Error("delete failed") + } + + err = cacheModel.SetMaxSeq(ctx, conversationID, 12000) + msgDoc = GenMsgDoc(1, 4999, 5000, 0, conversationID) + msgDoc2 = GenMsgDoc(5000, 9999, 10000, 1, conversationID) + msgDoc3 := GenMsgDoc(10000, 12000, 11000, 2, conversationID) + if _, err := mongoClient.InsertOne(ctx, msgDoc); err != nil { + t.Error("InsertOne failed", conversationID) + } + if _, err := mongoClient.InsertOne(ctx, msgDoc2); err != nil { + t.Error("InsertOne failed", conversationID) + } + if _, err := mongoClient.InsertOne(ctx, msgDoc3); err != nil { + t.Error("InsertOne failed", conversationID) + } + + msgTools.ClearConversationsMsg(ctx, []string{conversationID}) + if err != nil { + t.Error("GetSuperGroupMinMaxSeqInMongoAndCache failed") + return + } + minSeqMongo, maxSeqMongo, minSeqCache, maxSeqCache, err = msgTools.msgDatabase.GetConversationMinMaxSeqInMongoAndCache(ctx, conversationID) + if err != nil { + t.Error("GetSuperGroupMinMaxSeqInMongoAndCache failed") + return + } + if maxSeqCache != maxSeqMongo { + t.Error("checkMaxSeqWithMongo failed", conversationID) + } + if minSeqMongo != minSeqCache { + t.Error("minSeqMongo != minSeqCache", minSeqMongo, minSeqCache) + } + if minSeqCache != 5000 { + t.Error("test1 is not the same", "minSeq:", minSeqCache) + } + + testUID5 := "test_del_id5" + conversationID = utils.GetConversationIDBySessionType(constant.SuperGroupChatType, testUID5) + + _, err = mongoClient.DeleteOne(ctx, bson.M{"uid": conversationID + ":" + strconv.Itoa(0)}) + if err != nil { + t.Error("delete failed") + } + _, err = mongoClient.DeleteOne(ctx, bson.M{"uid": conversationID + ":" + strconv.Itoa(1)}) + if err != nil { + t.Error("delete failed") + } + err = cacheModel.SetMaxSeq(ctx, conversationID, 9999) + msgDoc = GenMsgDoc(1, 4999, 5000, 0, conversationID) + msgDoc2 = GenMsgDoc(5000, 9999, 10000, 1, conversationID) + if _, err := mongoClient.InsertOne(ctx, msgDoc); err != nil { + t.Error("InsertOne failed", conversationID) + } + if _, err := mongoClient.InsertOne(ctx, msgDoc2); err != nil { + t.Error("InsertOne failed", conversationID) + } + + msgTools.ClearConversationsMsg(ctx, []string{conversationID}) + if err != nil { + t.Error("GetSuperGroupMinMaxSeqInMongoAndCache failed") + return + } + minSeqMongo, maxSeqMongo, minSeqCache, maxSeqCache, err = msgTools.msgDatabase.GetConversationMinMaxSeqInMongoAndCache(ctx, conversationID) + if err != nil { + t.Error("GetSuperGroupMinMaxSeqInMongoAndCache failed") + return + } + if maxSeqCache != maxSeqMongo { + t.Error("checkMaxSeqWithMongo failed", conversationID) + } + if minSeqMongo != minSeqCache { + t.Error("minSeqMongo != minSeqCache", minSeqMongo, minSeqCache) + } + if minSeqCache != 10000 { + t.Error("test1 is not the same", "minSeq:", minSeqCache) + } + + testUID6 := "test_del_id6" + conversationID = utils.GetConversationIDBySessionType(constant.SuperGroupChatType, testUID6) + + _, err = mongoClient.DeleteOne(ctx, bson.M{"uid": conversationID + ":" + strconv.Itoa(0)}) + if err != nil { + t.Error("delete failed") + } + _, err = mongoClient.DeleteOne(ctx, bson.M{"uid": conversationID + ":" + strconv.Itoa(1)}) + if err != nil { + t.Error("delete failed") + } + _, err = mongoClient.DeleteOne(ctx, bson.M{"uid": conversationID + ":" + strconv.Itoa(2)}) + if err != nil { + t.Error("delete failed") + } + _, err = mongoClient.DeleteOne(ctx, bson.M{"uid": conversationID + ":" + strconv.Itoa(3)}) + if err != nil { + t.Error("delete failed") + } + msgDoc = GenMsgDoc(1, 4999, 5000, 0, conversationID) + msgDoc2 = GenMsgDoc(5000, 9999, 10000, 1, conversationID) + msgDoc3 = GenMsgDoc(10000, 14999, 13000, 2, conversationID) + msgDoc4 := GenMsgDoc(15000, 19999, 0, 3, conversationID) + if _, err := mongoClient.InsertOne(ctx, msgDoc); err != nil { + t.Error("InsertOne failed", testUID4) + } + if _, err := mongoClient.InsertOne(ctx, msgDoc2); err != nil { + t.Error("InsertOne failed", testUID4) + } + if _, err := mongoClient.InsertOne(ctx, msgDoc3); err != nil { + t.Error("InsertOne failed", testUID4) + } + if _, err := mongoClient.InsertOne(ctx, msgDoc4); err != nil { + t.Error("InsertOne failed", testUID4) + } + minSeqMongo, maxSeqMongo, minSeqCache, maxSeqCache, err = msgTools.msgDatabase.GetConversationMinMaxSeqInMongoAndCache(ctx, conversationID) + if err != nil { + t.Error("GetSuperGroupMinMaxSeqInMongoAndCache failed") + return + } + if maxSeqCache != maxSeqMongo { + t.Error("checkMaxSeqWithMongo failed", conversationID) + } + if minSeqMongo != minSeqCache { + t.Error("minSeqMongo != minSeqCache", minSeqMongo, minSeqCache) + } + if minSeqCache != 13001 { + t.Error("test1 is not the same", "minSeq:", minSeqCache) + } +}