msb-public
/
Open-IM-Server
mirror of https://github.com/OpenIMSDK/Open-IM-Server
2
0
Code Issues Projects Releases Wiki Activity

errcode

Browse Source
test-errcode
wangchuxiao 3 years ago
parent 79c30fdc69
commit 0e97285e0a
8 changed files with 93 additions and 205 deletions
Show Stats Download Patch File Download Diff File

5
pkg/common/db/cache/extend_msg_set.go vendored
Unescape View File

@ -9,6 +9,11 @@ import (
"time"
)
const (
extendMsgSetCache = "EXTEND_MSG_SET_CACHE:"
extendMsgCache = "EXTEND_MSG_CACHE:"
)
type ExtendMsgSetCache struct {
expireTime time.Duration
rcClient *rockscache.Client

72
pkg/common/db/cache/rockscache.go vendored
Unescape View File

@ -1,42 +1,15 @@
package cache
import (
"Open_IM/pkg/common/constant"
"Open_IM/pkg/common/db/relation"
"Open_IM/pkg/common/log"
"Open_IM/pkg/common/tracelog"
"Open_IM/pkg/utils"
"context"
"encoding/json"
"math/big"
"sort"
"strconv"
"github.com/dtm-labs/rockscache"
"time"
)
const (
//userInfoCache = "USER_INFO_CACHE:"
//friendRelationCache = "FRIEND_RELATION_CACHE:"
blackListCache = "BLACK_LIST_CACHE:"
//groupCache = "GROUP_CACHE:"
//groupInfoCache = "GROUP_INFO_CACHE:"
//groupOwnerIDCache = "GROUP_OWNER_ID:"
//joinedGroupListCache = "JOINED_GROUP_LIST_CACHE:"
//groupMemberInfoCache = "GROUP_MEMBER_INFO_CACHE:"
//groupAllMemberInfoCache = "GROUP_ALL_MEMBER_INFO_CACHE:"
//allFriendInfoCache = "ALL_FRIEND_INFO_CACHE:"
//joinedSuperGroupListCache = "JOINED_SUPER_GROUP_LIST_CACHE:"
//groupMemberListHashCache = "GROUP_MEMBER_LIST_HASH_CACHE:"
//groupMemberNumCache = "GROUP_MEMBER_NUM_CACHE:"
conversationCache = "CONVERSATION_CACHE:"
conversationIDListCache = "CONVERSATION_ID_LIST_CACHE:"
extendMsgSetCache = "EXTEND_MSG_SET_CACHE:"
extendMsgCache = "EXTEND_MSG_CACHE:"
)
const scanCount = 3000
const RandomExpireAdjustment = 0.2
func (rc *RcClient) DelKeys() {
for _, key := range []string{"GROUP_CACHE:", "FRIEND_RELATION_CACHE", "BLACK_LIST_CACHE:", "USER_INFO_CACHE:", "GROUP_INFO_CACHE", groupOwnerIDCache, joinedGroupListCache,
@ -68,3 +41,44 @@ func (rc *RcClient) DelKeys() {
}
}
}
func GetCache[T any](ctx context.Context, rcClient *rockscache.Client, key string, expire time.Duration, fn func(ctx context.Context) (T, error)) (T, error) {
var t T
var write bool
v, err := rcClient.Fetch(key, expire, func() (s string, err error) {
t, err = fn(ctx)
if err != nil {
return "", err
}
bs, err := json.Marshal(t)
if err != nil {
return "", utils.Wrap(err, "")
}
write = true
return string(bs), nil
})
if err != nil {
return t, err
}
if write {
return t, nil
}
err = json.Unmarshal([]byte(v), &t)
if err != nil {
return t, utils.Wrap(err, "")
}
return t, nil
}
func GetCacheFor[E any, T any](ctx context.Context, list []E, fn func(ctx context.Context, item E) (T, error)) ([]T, error) {
rs := make([]T, 0, len(list))
for _, e := range list {
r, err := fn(ctx, e)
if err != nil {
return nil, err
}
rs = append(rs, r)
}
return rs, nil
}

49
pkg/common/db/cache/utils.go vendored
Unescape View File

@ -1,49 +0,0 @@
package cache
import (
"Open_IM/pkg/utils"
"context"
"encoding/json"
"github.com/dtm-labs/rockscache"
"time"
)
func GetCache[T any](ctx context.Context, rcClient *rockscache.Client, key string, expire time.Duration, fn func(ctx context.Context) (T, error)) (T, error) {
var t T
var write bool
v, err := rcClient.Fetch(key, expire, func() (s string, err error) {
t, err = fn(ctx)
if err != nil {
return "", err
}
bs, err := json.Marshal(t)
if err != nil {
return "", utils.Wrap(err, "")
}
write = true
return string(bs), nil
})
if err != nil {
return t, err
}
if write {
return t, nil
}
err = json.Unmarshal([]byte(v), &t)
if err != nil {
return t, utils.Wrap(err, "")
}
return t, nil
}
func GetCacheFor[E any, T any](ctx context.Context, list []E, fn func(ctx context.Context, item E) (T, error)) ([]T, error) {
rs := make([]T, 0, len(list))
for _, e := range list {
r, err := fn(ctx, e)
if err != nil {
return nil, err
}
rs = append(rs, r)
}
return rs, nil
}

20
pkg/common/db/controller/msg.go
Unescape View File

@ -1 +1,21 @@
package controller
import (
pbMsg "Open_IM/pkg/proto/msg"
"context"
)
type MsgInterface interface {
BatchInsertChat2DB(ctx context.Context, userID string, msgList []*pbMsg.MsgDataToMQ, currentMaxSeq uint64) error
BatchInsertChat2Cache(ctx context.Context, insertID string, msgList []*pbMsg.MsgDataToMQ) (error, uint64)
DelMsgBySeqList(ctx context.Context, userID string, seqList []uint32) (totalUnExistSeqList []uint32, err error)
DelMsgLogic(ctx context.Context, uid string, seqList []uint32) error
DelMsgBySeqListInOneDoc(ctx context.Context, docID string, seqList []uint32) (unExistSeqList []uint32, err error)
ReplaceMsgToBlankByIndex(suffixID string, index int) (replaceMaxSeq uint32, err error)
}
type MsgDatabaseInterface interface {
BatchInsertChat2DB(ctx context.Context, userID string, msgList []*pbMsg.MsgDataToMQ, currentMaxSeq uint64) error
BatchInsertChat2Cache(ctx context.Context, insertID string, msgList []*pbMsg.MsgDataToMQ) (error, uint64)
DelMsgBySeqList(ctx context.Context, userID string, seqList []uint32) (totalUnExistSeqList []uint32, err error)
}

22
pkg/common/db/table/unrelation/msg.go
Unescape View File

@ -7,9 +7,9 @@ const (
CChat = "msg"
)
type UserChatModel struct {
UID string `bson:"uid"`
Msg []MsgInfoModel `bson:"msg"`
type UserMsgDocModel struct {
DocID string `bson:"uid"`
Msg []MsgInfoModel `bson:"msg"`
}
type MsgInfoModel struct {
@ -17,20 +17,20 @@ type MsgInfoModel struct {
Msg []byte `bson:"msg"`
}
func (UserChatModel) TableName() string {
func (UserMsgDocModel) TableName() string {
return CChat
}
func (UserChatModel) GetSingleGocMsgNum() int {
func (UserMsgDocModel) GetSingleDocMsgNum() int {
return singleGocMsgNum
}
func (u UserChatModel) getSeqUid(uid string, seq uint32) string {
func (u UserMsgDocModel) getSeqUid(uid string, seq uint32) string {
seqSuffix := seq / singleGocMsgNum
return u.indexGen(uid, seqSuffix)
}
func (u UserChatModel) getSeqUserIDList(userID string, maxSeq uint32) []string {
func (u UserMsgDocModel) getSeqUserIDList(userID string, maxSeq uint32) []string {
seqMaxSuffix := maxSeq / singleGocMsgNum
var seqUserIDList []string
for i := 0; i <= int(seqMaxSuffix); i++ {
@ -40,16 +40,16 @@ func (u UserChatModel) getSeqUserIDList(userID string, maxSeq uint32) []string {
return seqUserIDList
}
func (UserChatModel) getSeqSuperGroupID(groupID string, seq uint32) string {
func (UserMsgDocModel) getSeqSuperGroupID(groupID string, seq uint32) string {
seqSuffix := seq / singleGocMsgNum
return superGroupIndexGen(groupID, seqSuffix)
}
func (u UserChatModel) GetSeqUid(uid string, seq uint32) string {
func (u UserMsgDocModel) GetSeqUid(uid string, seq uint32) string {
return u.getSeqUid(uid, seq)
}
func (UserChatModel) getMsgIndex(seq uint32) int {
func (UserMsgDocModel) getMsgIndex(seq uint32) int {
seqSuffix := seq / singleGocMsgNum
var index uint32
if seqSuffix == 0 {
@ -60,6 +60,6 @@ func (UserChatModel) getMsgIndex(seq uint32) int {
return int(index)
}
func (UserChatModel) indexGen(uid string, seqSuffix uint32) string {
func (UserMsgDocModel) indexGen(uid string, seqSuffix uint32) string {
return uid + ":" + strconv.FormatInt(int64(seqSuffix), 10)
}

114
pkg/common/db/unrelation/batch_insert_chat.go
Unescape View File

@ -169,117 +169,3 @@ func (d *db.DataBases) BatchInsertChat2Cache(insertID string, msgList []*pbMsg.M
}
return utils.Wrap(err, ""), lastMaxSeq
}
//func (d *DataBases) BatchInsertChatBoth(userID string, msgList []*pbMsg.MsgDataToMQ, operationID string) (error, uint64) {
// err, lastMaxSeq := d.BatchInsertChat2Cache(userID, msgList, operationID)
// if err != nil {
// log.Error(operationID, "BatchInsertChat2Cache failed ", err.Error(), userID, len(msgList))
// return err, 0
// }
// for {
// if runtime.NumGoroutine() > 50000 {
// log.NewWarn(operationID, "too many NumGoroutine ", runtime.NumGoroutine())
// time.Sleep(10 * time.Millisecond)
// } else {
// break
// }
// }
// return nil, lastMaxSeq
//}
//
//func (d *DataBases) BatchInsertChat(userID string, msgList []*pbMsg.MsgDataToMQ, operationID string) error {
// newTime := getCurrentTimestampByMill()
// if len(msgList) > GetSingleGocMsgNum() {
// return errors.New("too large")
// }
// isInit := false
// currentMaxSeq, err := d.GetUserMaxSeq(userID)
// if err == nil {
//
// } else if err == go_redis.Nil {
// isInit = true
// currentMaxSeq = 0
// } else {
// return utils.Wrap(err, "")
// }
// var remain uint64
// //if currentMaxSeq < uint64(GetSingleGocMsgNum()) {
// // remain = uint64(GetSingleGocMsgNum()-1) - (currentMaxSeq % uint64(GetSingleGocMsgNum()))
// //} else {
// // remain = uint64(GetSingleGocMsgNum()) - ((currentMaxSeq - (uint64(GetSingleGocMsgNum()) - 1)) % uint64(GetSingleGocMsgNum()))
// //}
//
// blk0 := uint64(GetSingleGocMsgNum() - 1)
// if currentMaxSeq < uint64(GetSingleGocMsgNum()) {
// remain = blk0 - currentMaxSeq
// } else {
// excludeBlk0 := currentMaxSeq - blk0
// remain = (uint64(GetSingleGocMsgNum()) - (excludeBlk0 % uint64(GetSingleGocMsgNum()))) % uint64(GetSingleGocMsgNum())
// }
//
// insertCounter := uint64(0)
// msgListToMongo := make([]MsgInfo, 0)
// msgListToMongoNext := make([]MsgInfo, 0)
// seqUid := ""
// seqUidNext := ""
// log.Debug(operationID, "remain ", remain, "insertCounter ", insertCounter, "currentMaxSeq ", currentMaxSeq, userID, len(msgList))
// //4998 remain ==1
// //4999
// for _, m := range msgList {
// log.Debug(operationID, "msg node ", m.String(), m.MsgData.ClientMsgID)
// currentMaxSeq++
// sMsg := MsgInfo{}
// sMsg.SendTime = m.MsgData.SendTime
// m.MsgData.Seq = uint32(currentMaxSeq)
// if sMsg.Msg, err = proto.Marshal(m.MsgData); err != nil {
// return utils.Wrap(err, "")
// }
// if isInit {
// msgListToMongoNext = append(msgListToMongoNext, sMsg)
// seqUidNext = getSeqUid(userID, uint32(currentMaxSeq))
// log.Debug(operationID, "msgListToMongoNext ", seqUidNext, m.MsgData.Seq, m.MsgData.ClientMsgID, insertCounter, remain)
// continue
// }
// if insertCounter < remain {
// msgListToMongo = append(msgListToMongo, sMsg)
// insertCounter++
// seqUid = getSeqUid(userID, uint32(currentMaxSeq))
// log.Debug(operationID, "msgListToMongo ", seqUid, m.MsgData.Seq, m.MsgData.ClientMsgID, insertCounter, remain)
// } else {
// msgListToMongoNext = append(msgListToMongoNext, sMsg)
// seqUidNext = getSeqUid(userID, uint32(currentMaxSeq))
// log.Debug(operationID, "msgListToMongoNext ", seqUidNext, m.MsgData.Seq, m.MsgData.ClientMsgID, insertCounter, remain)
// }
// }
// // ctx, _ := context.WithTimeout(context.Background(), time.Duration(config.Config.Mongo.DBTimeout)*time.Second)
//
// ctx := context.Background()
// c := d.mongoClient.Database(config.Config.Mongo.DBDatabase).Collection(cChat)
//
// if seqUid != "" {
// filter := bson.M{"uid": seqUid}
// log.NewDebug(operationID, "filter ", seqUid, "list ", msgListToMongo)
// err := c.FindOneAndUpdate(ctx, filter, bson.M{"$push": bson.M{"msg": bson.M{"$each": msgListToMongo}}}).Err()
// if err != nil {
// log.Error(operationID, "FindOneAndUpdate failed ", err.Error(), filter)
// return utils.Wrap(err, "")
// }
// }
// if seqUidNext != "" {
// filter := bson.M{"uid": seqUidNext}
// sChat := UserChat{}
// sChat.UID = seqUidNext
// sChat.Msg = msgListToMongoNext
// log.NewDebug(operationID, "filter ", seqUidNext, "list ", msgListToMongoNext)
// if _, err = c.InsertOne(ctx, &sChat); err != nil {
// log.NewError(operationID, "InsertOne failed", filter, err.Error(), sChat)
// return utils.Wrap(err, "")
// }
// }
// log.NewWarn(operationID, "batch mgo cost time ", getCurrentTimestampByMill()-newTime, userID, len(msgList))
// return utils.Wrap(d.SetUserMaxSeq(userID, uint64(currentMaxSeq)), "")
//}
//func (d *DataBases)setMessageToCache(msgList []*pbMsg.MsgDataToMQ, uid string) (err error) {
//
//}

4
pkg/common/db/unrelation/mongo.go
Unescape View File

@ -74,8 +74,8 @@ func (m *Mongo) CreateTagIndex() {
}
func (m *Mongo) CreateMsgIndex() {
if err := m.createMongoIndex(cChat, false, "uid"); err != nil {
fmt.Println(err.Error() + " index create failed " + cChat + " uid, please create index by yourself in field uid")
if err := m.createMongoIndex(unrelation.CChat, false, "uid"); err != nil {
fmt.Println(err.Error() + " index create failed " + unrelation.CChat + " uid, please create index by yourself in field uid")
}
}

12
pkg/common/db/unrelation/msg.go
Unescape View File

@ -27,6 +27,16 @@ func NewMsgMongoDriver(mgoDB *mongo.Database) *MsgMongoDriver {
return &MsgMongoDriver{mgoDB: mgoDB, MsgCollection: mgoDB.Collection(unrelation.CChat)}
}
func (m *MsgMongoDriver) FindOneAndUpdate(ctx context.Context, filter, update, output interface{}, opts ...*options.FindOneAndUpdateOptions) error {
return m.MsgCollection.FindOneAndUpdate(ctx, filter, update, opts...).Decode(output)
}
func (m *MsgMongoDriver) UpdateOne(ctx context.Context, filter, update interface{}, opts ...*options.UpdateOptions) error {
_, err := m.MsgCollection.UpdateOne(ctx, filter, update, opts...)
return err
}
// database controller
func (m *MsgMongoDriver) DelMsgBySeqList(ctx context.Context, userID string, seqList []uint32) (totalUnExistSeqList []uint32, err error) {
sortkeys.Uint32s(seqList)
suffixUserID2SubSeqList := func(uid string, seqList []uint32) map[string][]uint32 {
@ -73,6 +83,7 @@ func (m *MsgMongoDriver) DelMsgBySeqListInOneDoc(ctx context.Context, suffixUser
return unexistSeqList, nil
}
// database
func (m *MsgMongoDriver) DelMsgLogic(ctx context.Context, uid string, seqList []uint32) error {
sortkeys.Uint32s(seqList)
seqMsgs, err := d.GetMsgBySeqListMongo2(ctx, uid, seqList)
@ -88,6 +99,7 @@ func (m *MsgMongoDriver) DelMsgLogic(ctx context.Context, uid string, seqList []
return nil
}
// model
func (m *MsgMongoDriver) ReplaceMsgByIndex(ctx context.Context, suffixUserID string, msg *sdkws.MsgData, seqIndex int) error {
log.NewInfo(operationID, utils.GetSelfFuncName(), suffixUserID, *msg)
ctx, _ := context.WithTimeout(context.Background(), time.Duration(config.Config.Mongo.DBTimeout)*time.Second)

Write Preview
Loading…
Cancel
Save