skiffer-git
2 years ago
commit
923a035e6e
@ -0,0 +1,11 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"Open_IM/internal/cron_task"
|
||||
"fmt"
|
||||
)
|
||||
|
||||
func main() {
|
||||
fmt.Println("start cronTask")
|
||||
cronTask.StartCronTask()
|
||||
}
|
||||
@ -0,0 +1,113 @@
|
||||
package cronTask
|
||||
|
||||
import (
|
||||
"Open_IM/pkg/common/config"
|
||||
"Open_IM/pkg/common/db"
|
||||
"Open_IM/pkg/common/log"
|
||||
server_api_params "Open_IM/pkg/proto/sdk_ws"
|
||||
"Open_IM/pkg/utils"
|
||||
"github.com/golang/protobuf/proto"
|
||||
)
|
||||
|
||||
const oldestList = 0
|
||||
const newestList = -1
|
||||
|
||||
func ResetUserGroupMinSeq(operationID, groupID string, userIDList []string) error {
|
||||
var delMsgIDList [][2]interface{}
|
||||
minSeq, err := deleteMongoMsg(operationID, groupID, oldestList, &delMsgIDList)
|
||||
if err != nil {
|
||||
log.NewError(operationID, utils.GetSelfFuncName(), groupID, "deleteMongoMsg failed")
|
||||
return utils.Wrap(err, "")
|
||||
}
|
||||
for _, userID := range userIDList {
|
||||
userMinSeq, err := db.DB.GetGroupUserMinSeq(groupID, userID)
|
||||
if err != nil {
|
||||
log.NewError(operationID, utils.GetSelfFuncName(), "GetGroupUserMinSeq failed", groupID, userID, err.Error())
|
||||
continue
|
||||
}
|
||||
if userMinSeq > uint64(minSeq) {
|
||||
err = db.DB.SetGroupUserMinSeq(groupID, userID, userMinSeq)
|
||||
} else {
|
||||
err = db.DB.SetGroupUserMinSeq(groupID, userID, uint64(minSeq))
|
||||
}
|
||||
if err != nil {
|
||||
log.NewError(operationID, utils.GetSelfFuncName(), err.Error(), groupID, userID, userMinSeq, minSeq)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func DeleteMongoMsgAndResetRedisSeq(operationID, userID string) error {
|
||||
var delMsgIDList [][2]interface{}
|
||||
minSeq, err := deleteMongoMsg(operationID, userID, oldestList, &delMsgIDList)
|
||||
if err != nil {
|
||||
return utils.Wrap(err, "")
|
||||
}
|
||||
log.NewDebug(operationID, utils.GetSelfFuncName(), "delMsgIDMap: ", userID, delMsgIDList)
|
||||
err = db.DB.SetUserMinSeq(userID, minSeq)
|
||||
return err
|
||||
}
|
||||
|
||||
// recursion
|
||||
func deleteMongoMsg(operationID string, ID string, index int64, delMsgIDList *[][2]interface{}) (uint32, error) {
|
||||
// 从最旧的列表开始找
|
||||
msgs, err := db.DB.GetUserMsgListByIndex(ID, index)
|
||||
if err != nil {
|
||||
return 0, utils.Wrap(err, "GetUserMsgListByIndex failed")
|
||||
}
|
||||
if len(msgs.Msg) > db.GetSingleGocMsgNum() {
|
||||
log.NewWarn(operationID, utils.GetSelfFuncName(), "msgs too large", len(msgs.Msg), msgs.UID)
|
||||
}
|
||||
log.NewDebug(operationID, utils.GetSelfFuncName(), "get msgs: ", msgs.UID)
|
||||
for i, msg := range msgs.Msg {
|
||||
// 找到列表中不需要删除的消息了
|
||||
if utils.GetCurrentTimestampByMill() < msg.SendTime+int64(config.Config.Mongo.DBRetainChatRecords)*24*60*60*1000 {
|
||||
if len(*delMsgIDList) > 0 {
|
||||
var IDList []string
|
||||
for _, v := range *delMsgIDList {
|
||||
IDList = append(IDList, v[0].(string))
|
||||
}
|
||||
err := db.DB.DelMongoMsgs(IDList)
|
||||
if err != nil {
|
||||
return 0, utils.Wrap(err, "DelMongoMsgs failed")
|
||||
}
|
||||
}
|
||||
minSeq := getDelMaxSeqByIDList(*delMsgIDList)
|
||||
if i > 0 {
|
||||
msgPb := &server_api_params.MsgData{}
|
||||
err = proto.Unmarshal(msg.Msg, msgPb)
|
||||
if err != nil {
|
||||
log.NewError(operationID, utils.GetSelfFuncName(), err.Error(), ID, index)
|
||||
} else {
|
||||
err = db.DB.ReplaceMsgToBlankByIndex(msgs.UID, i-1)
|
||||
if err != nil {
|
||||
log.NewError(operationID, utils.GetSelfFuncName(), err.Error(), msgs.UID, i)
|
||||
return minSeq, nil
|
||||
}
|
||||
minSeq = msgPb.Seq - 1
|
||||
}
|
||||
}
|
||||
return minSeq, nil
|
||||
}
|
||||
}
|
||||
msgPb := &server_api_params.MsgData{}
|
||||
err = proto.Unmarshal(msgs.Msg[len(msgs.Msg)-1].Msg, msgPb)
|
||||
if err != nil {
|
||||
log.NewError(operationID, utils.GetSelfFuncName(), err.Error(), len(msgs.Msg)-1, msgs.UID)
|
||||
return 0, utils.Wrap(err, "proto.Unmarshal failed")
|
||||
}
|
||||
*delMsgIDList = append(*delMsgIDList, [2]interface{}{msgs.UID, msgPb.Seq})
|
||||
// 没有找到 代表需要全部删除掉 继续递归查找下一个比较旧的列表
|
||||
seq, err := deleteMongoMsg(operationID, utils.GetSelfFuncName(), index+1, delMsgIDList)
|
||||
if err != nil {
|
||||
return 0, utils.Wrap(err, "deleteMongoMsg failed")
|
||||
}
|
||||
return seq, nil
|
||||
}
|
||||
|
||||
func getDelMaxSeqByIDList(delMsgIDList [][2]interface{}) uint32 {
|
||||
if len(delMsgIDList) == 0 {
|
||||
return 0
|
||||
}
|
||||
return delMsgIDList[len(delMsgIDList)-1][1].(uint32)
|
||||
}
|
||||
@ -0,0 +1,65 @@
|
||||
package cronTask
|
||||
|
||||
import (
|
||||
"Open_IM/pkg/common/constant"
|
||||
"Open_IM/pkg/common/db/mysql_model/im_mysql_model"
|
||||
rocksCache "Open_IM/pkg/common/db/rocks_cache"
|
||||
"Open_IM/pkg/common/log"
|
||||
"Open_IM/pkg/utils"
|
||||
"fmt"
|
||||
"github.com/robfig/cron/v3"
|
||||
"time"
|
||||
)
|
||||
|
||||
const cronTaskOperationID = "cronTaskOperationID-"
|
||||
|
||||
func StartCronTask() {
|
||||
log.NewInfo(utils.OperationIDGenerator(), "start cron task")
|
||||
c := cron.New()
|
||||
_, err := c.AddFunc("30 3-6,20-23 * * *", func() {
|
||||
operationID := getCronTaskOperationID()
|
||||
userIDList, err := im_mysql_model.SelectAllUserID()
|
||||
if err == nil {
|
||||
log.NewDebug(operationID, utils.GetSelfFuncName(), "userIDList: ", userIDList)
|
||||
for _, userID := range userIDList {
|
||||
if err := DeleteMongoMsgAndResetRedisSeq(operationID, userID); err != nil {
|
||||
log.NewError(operationID, utils.GetSelfFuncName(), err.Error(), userID)
|
||||
}
|
||||
}
|
||||
} else {
|
||||
log.NewError(operationID, utils.GetSelfFuncName(), err.Error())
|
||||
}
|
||||
|
||||
workingGroupIDList, err := im_mysql_model.GetGroupIDListByGroupType(constant.WorkingGroup)
|
||||
if err == nil {
|
||||
for _, groupID := range workingGroupIDList {
|
||||
userIDList, err = rocksCache.GetGroupMemberIDListFromCache(groupID)
|
||||
if err != nil {
|
||||
log.NewError(operationID, utils.GetSelfFuncName(), err.Error(), groupID)
|
||||
continue
|
||||
}
|
||||
log.NewDebug(operationID, utils.GetSelfFuncName(), "groupID:", groupID, "userIDList:", userIDList)
|
||||
if err := ResetUserGroupMinSeq(operationID, groupID, userIDList); err != nil {
|
||||
log.NewError(operationID, utils.GetSelfFuncName(), err.Error(), groupID, userIDList)
|
||||
}
|
||||
|
||||
}
|
||||
} else {
|
||||
log.NewError(operationID, utils.GetSelfFuncName(), err.Error())
|
||||
return
|
||||
}
|
||||
})
|
||||
if err != nil {
|
||||
fmt.Println("start cron failed", err.Error())
|
||||
panic(err)
|
||||
}
|
||||
c.Start()
|
||||
fmt.Println("start cron task success")
|
||||
for {
|
||||
time.Sleep(time.Second)
|
||||
}
|
||||
}
|
||||
|
||||
func getCronTaskOperationID() string {
|
||||
return cronTaskOperationID + utils.OperationIDGenerator()
|
||||
}
|
||||
@ -1,89 +0,0 @@
|
||||
package timedTask
|
||||
|
||||
import (
|
||||
"Open_IM/pkg/common/config"
|
||||
"Open_IM/pkg/common/constant"
|
||||
"Open_IM/pkg/common/db"
|
||||
"Open_IM/pkg/common/log"
|
||||
server_api_params "Open_IM/pkg/proto/sdk_ws"
|
||||
"Open_IM/pkg/utils"
|
||||
"github.com/golang/protobuf/proto"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
const oldestList = 0
|
||||
const newestList = -1
|
||||
|
||||
func DeleteMongoMsgAndResetRedisSeq(operationID, ID string, diffusionType int) error {
|
||||
// -1 表示从当前最早的一个开始
|
||||
var delMsgIDList []string
|
||||
minSeq, err := deleteMongoMsg(operationID, ID, oldestList, &delMsgIDList)
|
||||
if err != nil {
|
||||
return utils.Wrap(err, "")
|
||||
}
|
||||
log.NewDebug(operationID, utils.GetSelfFuncName(), "delMsgIDList: ", delMsgIDList)
|
||||
if diffusionType == constant.WriteDiffusion {
|
||||
err = db.DB.SetUserMinSeq(ID, minSeq)
|
||||
} else if diffusionType == constant.ReadDiffusion {
|
||||
err = db.DB.SetGroupMinSeq(ID, minSeq)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// recursion
|
||||
func deleteMongoMsg(operationID string, ID string, index int64, IDList *[]string) (uint32, error) {
|
||||
// 从最旧的列表开始找
|
||||
msgs, err := db.DB.GetUserMsgListByIndex(ID, index)
|
||||
if err != nil {
|
||||
return 0, utils.Wrap(err, "GetUserMsgListByIndex failed")
|
||||
}
|
||||
log.NewDebug(operationID, utils.GetSelfFuncName(), "get msgs: ", msgs.UID)
|
||||
for i, msg := range msgs.Msg {
|
||||
// 找到列表中不需要删除的消息了
|
||||
if msg.SendTime+int64(config.Config.Mongo.DBRetainChatRecords) > utils.GetCurrentTimestampByMill() {
|
||||
if len(*IDList) > 0 {
|
||||
err := db.DB.DelMongoMsgs(*IDList)
|
||||
if err != nil {
|
||||
return 0, utils.Wrap(err, "DelMongoMsgs failed")
|
||||
}
|
||||
}
|
||||
minSeq := getDelMaxSeqByIDList(*IDList)
|
||||
if i > 0 {
|
||||
msgPb := &server_api_params.MsgData{}
|
||||
err = proto.Unmarshal(msg.Msg, msgPb)
|
||||
if err != nil {
|
||||
log.NewError(operationID, utils.GetSelfFuncName(), err.Error(), ID, index)
|
||||
} else {
|
||||
err = db.DB.ReplaceMsgToBlankByIndex(msgs.UID, i-1)
|
||||
if err != nil {
|
||||
log.NewError(operationID, utils.GetSelfFuncName(), err.Error(), msgs.UID, i)
|
||||
return minSeq, nil
|
||||
}
|
||||
minSeq = msgPb.Seq - 1
|
||||
}
|
||||
}
|
||||
return minSeq, nil
|
||||
}
|
||||
}
|
||||
*IDList = append(*IDList, msgs.UID)
|
||||
// 没有找到 代表需要全部删除掉 继续查找下一个比较旧的列表
|
||||
seq, err := deleteMongoMsg(operationID, utils.GetSelfFuncName(), index-1, IDList)
|
||||
if err != nil {
|
||||
return 0, utils.Wrap(err, "deleteMongoMsg failed")
|
||||
}
|
||||
return seq, nil
|
||||
}
|
||||
|
||||
func getDelMaxSeqByIDList(IDList []string) uint32 {
|
||||
if len(IDList) == 0 {
|
||||
return 0
|
||||
}
|
||||
l := strings.Split(IDList[len(IDList)-1], ":")
|
||||
index, _ := strconv.Atoi(l[len(l)-1])
|
||||
if index == 0 {
|
||||
// 4999
|
||||
return uint32(db.GetSingleGocMsgNum()) - 1
|
||||
} // 5000
|
||||
return (uint32(db.GetSingleGocMsgNum()) - 1) + uint32(index*db.GetSingleGocMsgNum())
|
||||
}
|
||||
@ -1 +0,0 @@
|
||||
package timedTask
|
||||
@ -1,23 +0,0 @@
|
||||
package timedTask
|
||||
|
||||
import (
|
||||
"Open_IM/pkg/common/constant"
|
||||
"Open_IM/pkg/common/log"
|
||||
"Open_IM/pkg/utils"
|
||||
"github.com/robfig/cron/v3"
|
||||
)
|
||||
|
||||
func main() {
|
||||
log.NewInfo(utils.OperationIDGenerator(), "start cron task")
|
||||
c := cron.New()
|
||||
_, err := c.AddFunc("30 3-6,20-23 * * *", func() {
|
||||
operationID := utils.OperationIDGenerator()
|
||||
if err := DeleteMongoMsgAndResetRedisSeq(operationID, "", constant.ReadDiffusion); err != nil {
|
||||
log.NewError(operationID)
|
||||
}
|
||||
})
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
c.Start()
|
||||
}
|
||||
Loading…
Reference in new issue