From 5528e8e44d0ed2e283f2f03bb092ef02b6e46dbc Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Wed, 22 May 2024 10:54:25 +0800 Subject: [PATCH 01/59] new mongo --- pkg/common/listdemo/common.go | 182 ++++++++++++++++++++++++++++ pkg/common/listdemo/friend_model.go | 5 + pkg/common/listdemo/friend_table.go | 86 +++++++++++++ 3 files changed, 273 insertions(+) create mode 100644 pkg/common/listdemo/common.go create mode 100644 pkg/common/listdemo/friend_model.go create mode 100644 pkg/common/listdemo/friend_table.go diff --git a/pkg/common/listdemo/common.go b/pkg/common/listdemo/common.go new file mode 100644 index 000000000..2d99cd8c9 --- /dev/null +++ b/pkg/common/listdemo/common.go @@ -0,0 +1,182 @@ +package listdemo + +import ( + "context" + "github.com/openimsdk/tools/db/mongoutil" + "github.com/openimsdk/tools/db/pagination" + "github.com/pkg/errors" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" +) + +var ( + ErrListNotFound = errors.New("list not found") + ErrElemExist = errors.New("elem exist") + ErrNotFound = mongo.ErrNoDocuments +) + +type ListDoc interface { + IDName() string // 外层业务id字段名字 user_id + ElemsName() string // 外层列表名字 friends + VersionName() string // 外层版本号 version + DeleteVersion() string // 删除版本号 + BuildDoc(lid any, e Elem) any // 返回一个组装的doc文档 +} + +type Elem interface { + IDName() string // 业务id名字 friend_user_id + IDValue() any // 业务id值 userID -> "100000000" + VersionName() string // 版本号 + DeletedName() string // 删除字段名字 + ToMap() map[string]any // 把结构体转换为map +} + +type List[D any, E Elem] struct { + coll *mongo.Collection + lf ListDoc +} + +func (l *List[D, E]) zeroE() E { + var t E + return t +} + +func (l *List[D, E]) FindElem(ctx context.Context, lid any, eid any) (E, error) { + res, err := l.FindElems(ctx, lid, []any{eid}) + if err != nil { + return l.zeroE(), err + } + if len(res) == 0 { + return l.zeroE(), ErrNotFound + } + return res[0], nil +} + +// FindElems 查询Elems +func (l *List[D, E]) FindElems(ctx context.Context, lid any, eids []any) ([]E, error) { + //pipeline := []bson.M{ + // { + // "$match": bson.M{ + // l.lf.IDName(): lid, + // l.lf.IDName() + "." + l.lf.ElemsID(): bson.M{ + // "$in": eids, + // }, + // }, + // }, + // { + // "$unwind": "$" + l.lf.ElemsName(), + // }, + // { + // "$match": bson.M{ + // l.lf.IDName() + "." + l.lf.ElemsID(): bson.M{ + // "$in": eids, + // }, + // }, + // }, + //} + panic("todo") +} + +func (l *List[D, E]) Find(ctx context.Context, filter any, opts ...*options.FindOptions) ([]E, error) { + return nil, nil +} + +func (l *List[D, E]) Count(ctx context.Context, filter any, opts ...*options.CountOptions) (int64, error) { + return 0, nil +} + +func (l *List[D, E]) Update(ctx context.Context, lid any, eid any) (*mongo.UpdateResult, error) { + + return nil, nil +} + +func (l *List[D, E]) Delete(ctx context.Context, lid any, eids any) (*mongo.UpdateResult, error) { + + return nil, nil +} + +func (l *List[D, E]) Page(ctx context.Context, filter any, pagination pagination.Pagination, opts ...*options.FindOptions) (int64, []E, error) { + return 0, nil, nil +} + +func (l *List[D, E]) ElemIDs(ctx context.Context, filter any, opts ...*options.FindOptions) ([]E, error) { + + return nil, nil +} + +// InsertElem 插入一个 +func (l *List[D, E]) InsertElem(ctx context.Context, lid any, e Elem) error { + if err := l.insertElem(ctx, lid, e); err == nil { + return nil + } else if !errors.Is(err, ErrListNotFound) { + return err + } + if _, err := l.coll.InsertOne(ctx, l.lf.BuildDoc(lid, e)); err == nil { + return nil + } else if mongo.IsDuplicateKeyError(err) { + return l.insertElem(ctx, lid, e) + } else { + return err + } +} + +func (l *List[D, E]) insertElem(ctx context.Context, lid any, e Elem) error { + data := e.ToMap() + data[e.VersionName()] = "$max_version" + filter := bson.M{ + l.lf.IDName(): lid, + } + pipeline := []bson.M{ + { + "$addFields": bson.M{ + "found_elem": bson.M{ + "$in": bson.A{e.IDValue(), l.lf.ElemsName() + "." + e.IDName()}, + }, + }, + }, + { + "$set": bson.M{ + "max_version": bson.M{ + "$cond": bson.M{ + "if": "$found_elem", + "then": "$max_version", + "else": bson.M{"$add": bson.A{"max_version", 1}}, + }, + }, + }, + }, + { + "$set": bson.M{ + l.lf.ElemsName(): bson.M{ + "$cond": bson.M{ + "if": "$found_elem", + "then": "$" + l.lf.ElemsName(), + "else": bson.M{ + "$concatArrays": bson.A{ + "$" + l.lf.ElemsName(), + bson.A{ + data, + }, + }, + }, + }, + }, + }, + }, + { + "$unset": "found_elem", + }, + } + res, err := mongoutil.UpdateMany(ctx, l.coll, filter, pipeline) + if err != nil { + return err + } + if res.MatchedCount == 0 { + return ErrListNotFound + } + if res.ModifiedCount == 0 { + return ErrElemExist + } + return nil +} diff --git a/pkg/common/listdemo/friend_model.go b/pkg/common/listdemo/friend_model.go new file mode 100644 index 000000000..13507bfa3 --- /dev/null +++ b/pkg/common/listdemo/friend_model.go @@ -0,0 +1,5 @@ +package listdemo + +type friendModel struct { + db *List[*Friend, *FriendElem] +} diff --git a/pkg/common/listdemo/friend_table.go b/pkg/common/listdemo/friend_table.go new file mode 100644 index 000000000..b1ad877a2 --- /dev/null +++ b/pkg/common/listdemo/friend_table.go @@ -0,0 +1,86 @@ +package listdemo + +import ( + "time" +) + +var ( + _ Elem = (*FriendElem)(nil) + _ ListDoc = (*Friend)(nil) +) + +type FriendElem struct { + FriendUserID string `bson:"friend_user_id"` + Nickname string `bson:"nickname"` + FaceURL string `bson:"face_url"` + Remark string `bson:"remark"` + CreateTime time.Time `bson:"create_time"` + AddSource int32 `bson:"add_source"` + OperatorUserID string `bson:"operator_user_id"` + Ex string `bson:"ex"` + IsPinned bool `bson:"is_pinned"` + Version uint `bson:"version"` + DeleteTime *time.Time `bson:"delete_time"` +} + +func (f *FriendElem) IDName() string { + return "friend_user_id" +} + +func (f *FriendElem) IDValue() any { + return f.FriendUserID +} + +func (f *FriendElem) VersionName() string { + return "version" +} + +func (f *FriendElem) DeletedName() string { + return "delete_time" +} + +func (f *FriendElem) ToMap() map[string]any { + return map[string]any{ + "friend_user_id": f.FriendUserID, + "nickname": f.Nickname, + "face_url": f.FaceURL, + "remark": f.Remark, + "create_time": f.CreateTime, + "add_source": f.AddSource, + "operator_user_id": f.OperatorUserID, + "ex": f.Ex, + "is_pinned": f.IsPinned, + "version": f.Version, + "delete_time": f.DeleteTime, + } +} + +type Friend struct { + UserID string `bson:"user_id"` + Friends []*FriendElem `bson:"friends"` + Version uint `bson:"version"` + DeleteVersion uint `bson:"delete_version"` +} + +func (f *Friend) BuildDoc(lid any, e Elem) any { + return &Friend{ + UserID: lid.(string), + Friends: []*FriendElem{e.(*FriendElem)}, + } +} + +func (f *Friend) ElemsID() string { + return "user_id" +} + +func (f *Friend) IDName() string { + return "user_id" +} + +func (f *Friend) ElemsName() string { + return "friends" +} + +func (f *Friend) VersionName() string { + return "version" +} From c6942f07dae378fd3c367ef8fbcdbf9015bb6c03 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Wed, 22 May 2024 18:38:29 +0800 Subject: [PATCH 02/59] new mongo --- pkg/common/listdemo2/common.go | 163 ++++++++++++++++++++++++++++ pkg/common/listdemo2/common_test.go | 61 +++++++++++ pkg/common/listdemo2/demo.js | 86 +++++++++++++++ 3 files changed, 310 insertions(+) create mode 100644 pkg/common/listdemo2/common.go create mode 100644 pkg/common/listdemo2/common_test.go create mode 100644 pkg/common/listdemo2/demo.js diff --git a/pkg/common/listdemo2/common.go b/pkg/common/listdemo2/common.go new file mode 100644 index 000000000..081de8bba --- /dev/null +++ b/pkg/common/listdemo2/common.go @@ -0,0 +1,163 @@ +package listdemo + +import ( + "context" + "github.com/openimsdk/tools/db/mongoutil" + "github.com/openimsdk/tools/errs" + "github.com/pkg/errors" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" + "time" +) + +var ( + ErrListNotFound = errors.New("list not found") + ErrElemExist = errors.New("elem exist") + ErrNeedFull = errors.New("need full") + ErrNotFound = mongo.ErrNoDocuments +) + +type Elem struct { + ID string + Version uint +} + +type ChangeLog struct { + ChangeIDs []Elem + DeleteIDs []Elem +} + +type WriteLog struct { + DID string `bson:"d_id"` + Logs []LogElem `bson:"logs"` + Version uint `bson:"version"` + LastUpdate time.Time `bson:"last_update"` + DeleteVersion uint `bson:"delete_version"` +} + +type LogElem struct { + EID string `bson:"e_id"` + Deleted bool `bson:"deleted"` + Version uint `bson:"version"` + UpdateTime time.Time `bson:"update_time"` +} + +type LogModel struct { + coll *mongo.Collection +} + +func (l *LogModel) InitIndex(ctx context.Context) error { + return nil +} + +func (l *LogModel) WriteLog(ctx context.Context, dId string, eId string, deleted bool) error { + now := time.Now() + res, err := l.writeLog(ctx, dId, eId, deleted, now) + if err != nil { + return err + } + if res.MatchedCount > 0 { + return nil + } + wl := WriteLog{ + DID: dId, + Logs: []LogElem{ + { + EID: eId, + Deleted: deleted, + Version: 1, + UpdateTime: now, + }, + }, + Version: 1, + LastUpdate: now, + DeleteVersion: 0, + } + if _, err := l.coll.InsertOne(ctx, &wl); err == nil { + return nil + } else if !mongo.IsDuplicateKeyError(err) { + return err + } + if res, err := l.writeLog(ctx, dId, eId, deleted, now); err != nil { + return err + } else if res.ModifiedCount == 0 { + return errs.ErrInternalServer.WrapMsg("mongodb return value that should not occur", "coll", l.coll.Name(), "dId", dId, "eId", eId) + } + return nil +} + +func (l *LogModel) writeLog(ctx context.Context, dId string, eId string, deleted bool, now time.Time) (*mongo.UpdateResult, error) { + filter := bson.M{ + "d_id": dId, + } + elem := bson.M{ + "e_id": eId, + "version": "$version", + "deleted": deleted, + "update_time": now, + } + pipeline := []bson.M{ + { + "$addFields": bson.M{ + "elem_index": bson.M{ + "$indexOfArray": []any{"$logs.e_id", eId}, + }, + }, + }, + { + "$set": bson.M{ + "version": bson.M{"$add": []any{"$version", 1}}, + "update_time": now, + }, + }, + { + "$set": bson.M{ + "logs": bson.M{ + "$cond": bson.M{ + "if": bson.M{ + "$lt": []any{"$elem_index", 0}, + }, + "then": bson.M{ + "$concatArrays": []any{ + "$logs", + []bson.M{ + elem, + }, + }, + }, + "else": bson.M{ + "$map": bson.M{ + "input": bson.M{ + "$range": []any{0, bson.M{"$size": "$logs"}}, + }, + "as": "i", + "in": bson.M{ + "$cond": bson.M{ + "if": bson.M{ + "$eq": []any{"$$i", "$elem_index"}, + }, + "then": elem, + "else": bson.M{ + "$arrayElemAt": []any{ + "$logs", + "$$i", + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + { + "$unset": "elem_index", + }, + } + return mongoutil.UpdateMany(ctx, l.coll, filter, pipeline) +} + +func (l *LogModel) FindChangeLog(ctx context.Context, did string, version uint) (*ChangeLog, error) { + return nil, nil +} diff --git a/pkg/common/listdemo2/common_test.go b/pkg/common/listdemo2/common_test.go new file mode 100644 index 000000000..91f8f8143 --- /dev/null +++ b/pkg/common/listdemo2/common_test.go @@ -0,0 +1,61 @@ +package listdemo + +import ( + "context" + "fmt" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" + "testing" + "time" +) + +func Result[V any](val V, err error) V { + if err != nil { + panic(err) + } + return val +} + +func Check(err error) { + if err != nil { + panic(err) + } +} + +func TestName(t *testing.T) { + cli := Result(mongo.Connect(context.Background(), options.Client().ApplyURI("mongodb://openIM:openIM123@172.16.8.48:37017/openim_v3?maxPoolSize=100").SetConnectTimeout(5*time.Second))) + coll := cli.Database("openim_v3").Collection("demo") + _ = coll + //Result(coll.Indexes().CreateMany(context.Background(), []mongo.IndexModel{ + // { + // Keys: map[string]int{"user_id": 1}, + // }, + // { + // Keys: map[string]int{"friends.friend_user_id": 1}, + // }, + //})) + + wl := WriteLog{ + DID: "100", + Logs: []LogElem{ + { + EID: "1000", + Deleted: false, + Version: 1, + UpdateTime: time.Now(), + }, + { + EID: "2000", + Deleted: false, + Version: 1, + UpdateTime: time.Now(), + }, + }, + Version: 2, + DeleteVersion: 0, + LastUpdate: time.Now(), + } + + fmt.Println(Result(coll.InsertOne(context.Background(), wl))) + +} diff --git a/pkg/common/listdemo2/demo.js b/pkg/common/listdemo2/demo.js new file mode 100644 index 000000000..c0044b238 --- /dev/null +++ b/pkg/common/listdemo2/demo.js @@ -0,0 +1,86 @@ +db.demo.updateMany( + { + "d_id": "100" + }, + [ + { + $addFields: { + elem_index: { + $indexOfArray: [ + "$logs.e_id", + "1000" + ] + } + } + }, + { + $set: { + version: { + $add: ["$version", 1] + }, + update_time: new Date(), + + } + }, + { + $set: { + logs: { + $cond: { + if: { + $lt: ["$elem_index", 0] + }, + then: { + $concatArrays: [ + "$logs", + [ + { + e_id: "1000", + update_time: new Date(), + version: "$version", + deleted: false + } + ] + ] + }, + else: { + $map: { + input: { + $range: [0, { + $size: "$logs" + }] + }, + as: "i", + in: { + $cond: { + if: { + $eq: ["$$i", "$elem_index"] + }, + then: { + e_id: "1000", + update_time: new Date(), + version: "$version", + deleted: false + }, + else: { + $arrayElemAt: ["$logs", "$$i"] + } + }, + + }, + + }, + + }, + + }, + + }, + + }, + + }, + { + $unset: ["elem_index"] + }, + ] +) From 82c6b005e93c88652d33123c24173b09ee86f949 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Thu, 23 May 2024 17:55:02 +0800 Subject: [PATCH 03/59] new mongo --- go.mod | 4 +- go.sum | 4 +- pkg/common/db/dataver/common.go | 250 ++++++++++++++++++++++++++++ pkg/common/db/mgo/friend.go | 62 +++++-- pkg/common/listdemo2/common.go | 239 ++++++++++++++++++++++++-- pkg/common/listdemo2/common_test.go | 42 +++-- pkg/common/listdemo2/demo.js | 8 +- pkg/common/listdemo2/demo2.js | 63 +++++++ pkg/common/listdemo2/demo3.js | 59 +++++++ pkg/common/listdemo2/demo5.js | 10 ++ 10 files changed, 677 insertions(+), 64 deletions(-) create mode 100644 pkg/common/db/dataver/common.go create mode 100644 pkg/common/listdemo2/demo2.js create mode 100644 pkg/common/listdemo2/demo3.js create mode 100644 pkg/common/listdemo2/demo5.js diff --git a/go.mod b/go.mod index 54e8a8e0e..8e06e503f 100644 --- a/go.mod +++ b/go.mod @@ -14,8 +14,8 @@ require ( github.com/lestrrat-go/file-rotatelogs v2.4.0+incompatible // indirect github.com/mitchellh/mapstructure v1.5.0 github.com/openimsdk/protocol v0.0.65 - github.com/openimsdk/tools v0.0.49-alpha.19 - github.com/pkg/errors v0.9.1 // indirect + github.com/openimsdk/tools v0.0.49-alpha.23 + github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.18.0 github.com/stretchr/testify v1.9.0 go.mongodb.org/mongo-driver v1.14.0 diff --git a/go.sum b/go.sum index 5611a6ca6..a80b2fb0a 100644 --- a/go.sum +++ b/go.sum @@ -288,8 +288,8 @@ github.com/openimsdk/gomake v0.0.13 h1:xLDe/moqgWpRoptHzI4packAWzs4C16b+sVY+txNJ github.com/openimsdk/gomake v0.0.13/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= github.com/openimsdk/protocol v0.0.65 h1:SPT9qyUsFRTTKSKb/FjpS+xr6sxz/Kbnu+su1bxYagc= github.com/openimsdk/protocol v0.0.65/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= -github.com/openimsdk/tools v0.0.49-alpha.19 h1:CbASL0yefRSVAmWPVeRnhF7wZKd6umLfz31CIhEgrBs= -github.com/openimsdk/tools v0.0.49-alpha.19/go.mod h1:g7mkHXYUPi0/8aAX8VPMHpnb3hqdV69Jph+bXOGvvNM= +github.com/openimsdk/tools v0.0.49-alpha.23 h1:/KkJ7vfx8FAoJhq3veH9PWnxbSkEf+dTSshvDrHBR38= +github.com/openimsdk/tools v0.0.49-alpha.23/go.mod h1:g7mkHXYUPi0/8aAX8VPMHpnb3hqdV69Jph+bXOGvvNM= github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= github.com/pelletier/go-toml/v2 v2.1.0/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= diff --git a/pkg/common/db/dataver/common.go b/pkg/common/db/dataver/common.go new file mode 100644 index 000000000..c146600fb --- /dev/null +++ b/pkg/common/db/dataver/common.go @@ -0,0 +1,250 @@ +package dataver + +import ( + "context" + "github.com/openimsdk/tools/db/mongoutil" + "github.com/openimsdk/tools/errs" + "github.com/openimsdk/tools/utils/datautil" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" + "time" +) + +const ( + FirstVersion = 1 + DefaultDeleteVersion = 0 +) + +type WriteLog struct { + DID string `bson:"d_id"` + Logs []Elem `bson:"logs"` + Version uint `bson:"version"` + Deleted uint `bson:"deleted"` + LastUpdate time.Time `bson:"last_update"` + LogLen int `bson:"log_len"` +} + +type Elem struct { + EID string `bson:"e_id"` + Deleted bool `bson:"deleted"` + Version uint `bson:"version"` + LastUpdate time.Time `bson:"last_update"` +} + +type DataLog interface { + WriteLog(ctx context.Context, dId string, eIds []string, deleted bool) error + FindChangeLog(ctx context.Context, did string, version uint, limit int) (*WriteLog, error) + DeleteAfterUnchangedLog(ctx context.Context, deadline time.Time) error +} + +func NewDataLog(coll *mongo.Collection) (DataLog, error) { + lm := &logModel{coll: coll} + if lm.initIndex(context.Background()) != nil { + return nil, errs.ErrInternalServer.WrapMsg("init index failed", "coll", coll.Name()) + } + return lm, nil +} + +type logModel struct { + coll *mongo.Collection +} + +func (l *logModel) initIndex(ctx context.Context) error { + _, err := l.coll.Indexes().CreateOne(ctx, mongo.IndexModel{ + Keys: bson.M{ + "d_id": 1, + }, + }) + return err +} + +func (l *logModel) WriteLog(ctx context.Context, dId string, eIds []string, deleted bool) error { + if len(eIds) == 0 { + return errs.ErrArgs.WrapMsg("elem id is empty", "dId", dId) + } + if datautil.Duplicate(eIds) { + return errs.ErrArgs.WrapMsg("elem id is duplicate", "dId", dId, "eIds", eIds) + } + now := time.Now() + res, err := l.writeLogBatch(ctx, dId, eIds, deleted, now) + if err != nil { + return err + } + if res.MatchedCount > 0 { + return nil + } + if err := l.initDoc(ctx, dId, eIds, deleted, now); err == nil { + return nil + } else if !mongo.IsDuplicateKeyError(err) { + return err + } + if res, err := l.writeLogBatch(ctx, dId, eIds, deleted, now); err != nil { + return err + } else if res.ModifiedCount == 0 { + return errs.ErrInternalServer.WrapMsg("mongodb return value that should not occur", "coll", l.coll.Name(), "dId", dId, "eIds", eIds) + } + return nil +} + +func (l *logModel) initDoc(ctx context.Context, dId string, eIds []string, deleted bool, now time.Time) error { + type tableWriteLog struct { + DID string `bson:"d_id"` + Logs []Elem `bson:"logs"` + Version uint `bson:"version"` + Deleted uint `bson:"deleted"` + LastUpdate time.Time `bson:"last_update"` + } + wl := tableWriteLog{ + DID: dId, + Logs: make([]Elem, 0, len(eIds)), + Version: FirstVersion, + Deleted: DefaultDeleteVersion, + LastUpdate: now, + } + for _, eId := range eIds { + wl.Logs = append(wl.Logs, Elem{ + EID: eId, + Deleted: deleted, + Version: FirstVersion, + LastUpdate: now, + }) + } + _, err := l.coll.InsertOne(ctx, &wl) + return err +} + +func (l *logModel) writeLogBatch(ctx context.Context, dId string, eIds []string, deleted bool, now time.Time) (*mongo.UpdateResult, error) { + if len(eIds) == 0 { + return nil, errs.ErrArgs.WrapMsg("elem id is empty", "dId", dId) + } + filter := bson.M{ + "d_id": dId, + } + elems := make([]bson.M, 0, len(eIds)) + for _, eId := range eIds { + elems = append(elems, bson.M{ + "e_id": eId, + "version": "$version", + "deleted": deleted, + "last_update": now, + }) + } + pipeline := []bson.M{ + { + "$addFields": bson.M{ + "delete_e_ids": eIds, + }, + }, + { + "$set": bson.M{ + "version": bson.M{"$add": []any{"$version", 1}}, + "last_update": now, + }, + }, + { + "$set": bson.M{ + "logs": bson.M{ + "$filter": bson.M{ + "input": "$logs", + "as": "log", + "cond": bson.M{ + "$not": bson.M{ + "$in": []any{"$$log.e_id", "$delete_e_ids"}, + }, + }, + }, + }, + }, + }, + { + "$set": bson.M{ + "logs": bson.M{ + "$concatArrays": []any{ + "$logs", + elems, + }, + }, + }, + }, + { + "$unset": "delete_e_ids", + }, + } + return mongoutil.UpdateMany(ctx, l.coll, filter, pipeline) +} + +func (l *logModel) FindChangeLog(ctx context.Context, did string, version uint, limit int) (*WriteLog, error) { + pipeline := []bson.M{ + { + "$match": bson.M{ + "d_id": did, + }, + }, + { + "$addFields": bson.M{ + "logs": bson.M{ + "$cond": bson.M{ + "if": bson.M{ + "$or": []bson.M{ + {"$lt": []any{"$version", version}}, + {"$gte": []any{"$deleted", version}}, + }, + }, + "then": []any{}, + "else": "$logs", + }, + }, + }, + }, + { + "$addFields": bson.M{ + "logs": bson.M{ + "$filter": bson.M{ + "input": "$logs", + "as": "l", + "cond": bson.M{ + "$gt": []any{"$$l.version", version}, + }, + }, + }, + }, + }, + { + "$addFields": bson.M{ + "log_len": bson.M{"$size": "$logs"}, + }, + }, + { + "$addFields": bson.M{ + "logs": bson.M{ + "$cond": bson.M{ + "if": bson.M{ + "$gt": []any{"$log_len", limit}, + }, + "then": []any{}, + "else": "$logs", + }, + }, + }, + }, + } + if limit <= 0 { + pipeline = pipeline[:len(pipeline)-1] + } + res, err := mongoutil.Aggregate[*WriteLog](ctx, l.coll, pipeline) + if err != nil { + return nil, err + } + if len(res) == 0 { + return nil, errs.Wrap(mongo.ErrNoDocuments) + } + return res[0], nil +} + +func (l *logModel) DeleteAfterUnchangedLog(ctx context.Context, deadline time.Time) error { + return mongoutil.DeleteMany(ctx, l.coll, bson.M{ + "last_update": bson.M{ + "$lt": deadline, + }, + }) +} diff --git a/pkg/common/db/mgo/friend.go b/pkg/common/db/mgo/friend.go index 269bb594a..6e7a6db98 100644 --- a/pkg/common/db/mgo/friend.go +++ b/pkg/common/db/mgo/friend.go @@ -16,6 +16,7 @@ package mgo import ( "context" + "github.com/openimsdk/open-im-server/v3/pkg/common/db/dataver" "github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation" "github.com/openimsdk/tools/db/mongoutil" @@ -27,7 +28,9 @@ import ( // FriendMgo implements FriendModelInterface using MongoDB as the storage backend. type FriendMgo struct { - coll *mongo.Collection + coll *mongo.Collection + owner dataver.DataLog + friend dataver.DataLog } // NewFriendMongo creates a new instance of FriendMgo with the provided MongoDB database. @@ -43,12 +46,25 @@ func NewFriendMongo(db *mongo.Database) (relation.FriendModelInterface, error) { if err != nil { return nil, err } - return &FriendMgo{coll: coll}, nil + owner, err := dataver.NewDataLog(db.Collection("friend_owner_log")) + if err != nil { + return nil, err + } + friend, err := dataver.NewDataLog(db.Collection("friend_log")) + if err != nil { + return nil, err + } + return &FriendMgo{coll: coll, owner: owner, friend: friend}, nil } // Create inserts multiple friend records. func (f *FriendMgo) Create(ctx context.Context, friends []*relation.FriendModel) error { - return mongoutil.InsertMany(ctx, f.coll, friends) + return Success(func() error { + return mongoutil.InsertMany(ctx, f.coll, friends) + }, func() error { + + return nil + }) } // Delete removes specified friends of the owner user. @@ -57,7 +73,13 @@ func (f *FriendMgo) Delete(ctx context.Context, ownerUserID string, friendUserID "owner_user_id": ownerUserID, "friend_user_id": bson.M{"$in": friendUserIDs}, } - return mongoutil.DeleteOne(ctx, f.coll, filter) + return Success(func() error { + return mongoutil.DeleteOne(ctx, f.coll, filter) + }, func() error { + return f.owner.WriteLog(ctx, ownerUserID, friendUserIDs, true) + }, func() error { + + }) } // UpdateByMap updates specific fields of a friend document using a map. @@ -69,18 +91,13 @@ func (f *FriendMgo) UpdateByMap(ctx context.Context, ownerUserID string, friendU "owner_user_id": ownerUserID, "friend_user_id": friendUserID, } - return mongoutil.UpdateOne(ctx, f.coll, filter, bson.M{"$set": args}, true) + return Success(func() error { + return mongoutil.UpdateOne(ctx, f.coll, filter, bson.M{"$set": args}, true) + }, func() error { + return f.owner.WriteLog(ctx, ownerUserID, []string{friendUserID}, false) + }) } -// Update modifies multiple friend documents. -// func (f *FriendMgo) Update(ctx context.Context, friends []*relation.FriendModel) error { -// filter := bson.M{ -// "owner_user_id": ownerUserID, -// "friend_user_id": friendUserID, -// } -// return mgotool.UpdateMany(ctx, f.coll, filter, friends) -// } - // UpdateRemark updates the remark for a specific friend. func (f *FriendMgo) UpdateRemark(ctx context.Context, ownerUserID, friendUserID, remark string) error { return f.UpdateByMap(ctx, ownerUserID, friendUserID, map[string]any{"remark": remark}) @@ -157,7 +174,18 @@ func (f *FriendMgo) UpdateFriends(ctx context.Context, ownerUserID string, frien // Create an update document update := bson.M{"$set": val} - // Perform the update operation for all matching documents - _, err := mongoutil.UpdateMany(ctx, f.coll, filter, update) - return err + return Success(func() error { + return mongoutil.Ignore(mongoutil.UpdateMany(ctx, f.coll, filter, update)) + }, func() error { + return f.owner.WriteLog(ctx, ownerUserID, friendUserIDs, false) + }) +} + +func Success(fns ...func() error) error { + for _, fn := range fns { + if err := fn(); err != nil { + return err + } + } + return nil } diff --git a/pkg/common/listdemo2/common.go b/pkg/common/listdemo2/common.go index 081de8bba..2267d8630 100644 --- a/pkg/common/listdemo2/common.go +++ b/pkg/common/listdemo2/common.go @@ -4,6 +4,7 @@ import ( "context" "github.com/openimsdk/tools/db/mongoutil" "github.com/openimsdk/tools/errs" + "github.com/openimsdk/tools/utils/datautil" "github.com/pkg/errors" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/mongo" @@ -17,6 +18,11 @@ var ( ErrNotFound = mongo.ErrNoDocuments ) +const ( + FirstVersion = 1 + DefaultDeleteVersion = 0 +) + type Elem struct { ID string Version uint @@ -28,18 +34,27 @@ type ChangeLog struct { } type WriteLog struct { - DID string `bson:"d_id"` - Logs []LogElem `bson:"logs"` - Version uint `bson:"version"` - LastUpdate time.Time `bson:"last_update"` - DeleteVersion uint `bson:"delete_version"` + DID string `bson:"d_id"` + Logs []LogElem `bson:"logs"` + Version uint `bson:"version"` + Deleted uint `bson:"deleted"` + LastUpdate time.Time `bson:"last_update"` +} + +type WriteLogLen struct { + DID string `bson:"d_id"` + Logs []LogElem `bson:"logs"` + Version uint `bson:"version"` + Deleted uint `bson:"deleted"` + LastUpdate time.Time `bson:"last_update"` + LogLen int `bson:"log_len"` } type LogElem struct { EID string `bson:"e_id"` Deleted bool `bson:"deleted"` Version uint `bson:"version"` - UpdateTime time.Time `bson:"update_time"` + LastUpdate time.Time `bson:"last_update"` } type LogModel struct { @@ -47,7 +62,24 @@ type LogModel struct { } func (l *LogModel) InitIndex(ctx context.Context) error { - return nil + _, err := l.coll.Indexes().CreateOne(ctx, mongo.IndexModel{ + Keys: bson.M{ + "d_id": 1, + }, + }) + return err +} + +func (l *LogModel) WriteLog1(ctx context.Context, dId string, eId string, deleted bool) { + if err := l.WriteLog(ctx, dId, eId, deleted); err != nil { + panic(err) + } +} + +func (l *LogModel) WriteLogBatch1(ctx context.Context, dId string, eIds []string, deleted bool) { + if err := l.WriteLogBatch(ctx, dId, eIds, deleted); err != nil { + panic(err) + } } func (l *LogModel) WriteLog(ctx context.Context, dId string, eId string, deleted bool) error { @@ -65,13 +97,13 @@ func (l *LogModel) WriteLog(ctx context.Context, dId string, eId string, deleted { EID: eId, Deleted: deleted, - Version: 1, - UpdateTime: now, + Version: FirstVersion, + LastUpdate: now, }, }, - Version: 1, - LastUpdate: now, - DeleteVersion: 0, + Version: FirstVersion, + Deleted: DefaultDeleteVersion, + LastUpdate: now, } if _, err := l.coll.InsertOne(ctx, &wl); err == nil { return nil @@ -94,7 +126,7 @@ func (l *LogModel) writeLog(ctx context.Context, dId string, eId string, deleted "e_id": eId, "version": "$version", "deleted": deleted, - "update_time": now, + "last_update": now, } pipeline := []bson.M{ { @@ -107,7 +139,7 @@ func (l *LogModel) writeLog(ctx context.Context, dId string, eId string, deleted { "$set": bson.M{ "version": bson.M{"$add": []any{"$version", 1}}, - "update_time": now, + "last_update": now, }, }, { @@ -158,6 +190,181 @@ func (l *LogModel) writeLog(ctx context.Context, dId string, eId string, deleted return mongoutil.UpdateMany(ctx, l.coll, filter, pipeline) } -func (l *LogModel) FindChangeLog(ctx context.Context, did string, version uint) (*ChangeLog, error) { - return nil, nil +func (l *LogModel) WriteLogBatch(ctx context.Context, dId string, eIds []string, deleted bool) error { + if len(eIds) == 0 { + return errs.ErrArgs.WrapMsg("elem id is empty", "dId", dId) + } + if datautil.Duplicate(eIds) { + return errs.ErrArgs.WrapMsg("elem id is duplicate", "dId", dId, "eIds", eIds) + } + now := time.Now() + res, err := l.writeLogBatch(ctx, dId, eIds, deleted, now) + if err != nil { + return err + } + if res.MatchedCount > 0 { + return nil + } + wl := WriteLog{ + DID: dId, + Logs: make([]LogElem, 0, len(eIds)), + Version: FirstVersion, + Deleted: DefaultDeleteVersion, + LastUpdate: now, + } + for _, eId := range eIds { + wl.Logs = append(wl.Logs, LogElem{ + EID: eId, + Deleted: deleted, + Version: FirstVersion, + LastUpdate: now, + }) + } + if _, err := l.coll.InsertOne(ctx, &wl); err == nil { + return nil + } else if !mongo.IsDuplicateKeyError(err) { + return err + } + if res, err := l.writeLogBatch(ctx, dId, eIds, deleted, now); err != nil { + return err + } else if res.ModifiedCount == 0 { + return errs.ErrInternalServer.WrapMsg("mongodb return value that should not occur", "coll", l.coll.Name(), "dId", dId, "eIds", eIds) + } + return nil +} + +func (l *LogModel) writeLogBatch(ctx context.Context, dId string, eIds []string, deleted bool, now time.Time) (*mongo.UpdateResult, error) { + if len(eIds) == 0 { + return nil, errs.ErrArgs.WrapMsg("elem id is empty", "dId", dId) + } + filter := bson.M{ + "d_id": dId, + } + elems := make([]bson.M, 0, len(eIds)) + for _, eId := range eIds { + elems = append(elems, bson.M{ + "e_id": eId, + "version": "$version", + "deleted": deleted, + "last_update": now, + }) + } + pipeline := []bson.M{ + { + "$addFields": bson.M{ + "delete_e_ids": eIds, + }, + }, + { + "$set": bson.M{ + "version": bson.M{"$add": []any{"$version", 1}}, + "last_update": now, + }, + }, + { + "$set": bson.M{ + "logs": bson.M{ + "$filter": bson.M{ + "input": "$logs", + "as": "log", + "cond": bson.M{ + "$not": bson.M{ + "$in": []any{"$$log.e_id", "$delete_e_ids"}, + }, + }, + }, + }, + }, + }, + { + "$set": bson.M{ + "logs": bson.M{ + "$concatArrays": []any{ + "$logs", + elems, + }, + }, + }, + }, + { + "$unset": "delete_e_ids", + }, + } + return mongoutil.UpdateMany(ctx, l.coll, filter, pipeline) +} + +func (l *LogModel) FindChangeLog(ctx context.Context, did string, version uint, limit int) (*WriteLogLen, error) { + pipeline := []bson.M{ + { + "$match": bson.M{ + "d_id": did, + }, + }, + { + "$addFields": bson.M{ + "logs": bson.M{ + "$cond": bson.M{ + "if": bson.M{ + "$or": []bson.M{ + {"$lt": []any{"$version", version}}, + {"$gte": []any{"$deleted", version}}, + }, + }, + "then": []any{}, + "else": "$logs", + }, + }, + }, + }, + { + "$addFields": bson.M{ + "logs": bson.M{ + "$filter": bson.M{ + "input": "$logs", + "as": "l", + "cond": bson.M{ + "$gt": []any{"$$l.version", version}, + }, + }, + }, + }, + }, + { + "$addFields": bson.M{ + "log_len": bson.M{"$size": "$logs"}, + }, + }, + { + "$addFields": bson.M{ + "logs": bson.M{ + "$cond": bson.M{ + "if": bson.M{ + "$gt": []any{"$log_len", limit}, + }, + "then": []any{}, + "else": "$logs", + }, + }, + }, + }, + } + if limit <= 0 { + pipeline = pipeline[:len(pipeline)-1] + } + res, err := mongoutil.Aggregate[*WriteLogLen](ctx, l.coll, pipeline) + if err != nil { + return nil, err + } + if len(res) == 0 { + return nil, ErrNotFound + } + return res[0], nil +} + +func (l *LogModel) DeleteAfterUnchangedLog(ctx context.Context, deadline time.Time) error { + return mongoutil.DeleteMany(ctx, l.coll, bson.M{ + "last_update": bson.M{ + "$lt": deadline, + }, + }) } diff --git a/pkg/common/listdemo2/common_test.go b/pkg/common/listdemo2/common_test.go index 91f8f8143..bcaf4a19a 100644 --- a/pkg/common/listdemo2/common_test.go +++ b/pkg/common/listdemo2/common_test.go @@ -2,7 +2,6 @@ package listdemo import ( "context" - "fmt" "go.mongodb.org/mongo-driver/mongo" "go.mongodb.org/mongo-driver/mongo/options" "testing" @@ -24,7 +23,7 @@ func Check(err error) { func TestName(t *testing.T) { cli := Result(mongo.Connect(context.Background(), options.Client().ApplyURI("mongodb://openIM:openIM123@172.16.8.48:37017/openim_v3?maxPoolSize=100").SetConnectTimeout(5*time.Second))) - coll := cli.Database("openim_v3").Collection("demo") + coll := cli.Database("openim_v3").Collection("friend_version") _ = coll //Result(coll.Indexes().CreateMany(context.Background(), []mongo.IndexModel{ // { @@ -35,27 +34,24 @@ func TestName(t *testing.T) { // }, //})) - wl := WriteLog{ - DID: "100", - Logs: []LogElem{ - { - EID: "1000", - Deleted: false, - Version: 1, - UpdateTime: time.Now(), - }, - { - EID: "2000", - Deleted: false, - Version: 1, - UpdateTime: time.Now(), - }, - }, - Version: 2, - DeleteVersion: 0, - LastUpdate: time.Now(), - } + const num = 1 + lm := &LogModel{coll: coll} - fmt.Println(Result(coll.InsertOne(context.Background(), wl))) + //start := time.Now() + //eIds := make([]string, 0, num) + //for i := 0; i < num; i++ { + // eIds = append(eIds, strconv.Itoa(1000+(i))) + //} + //lm.WriteLogBatch1(context.Background(), "100", eIds, false) + //end := time.Now() + //t.Log(end.Sub(start)) // 509.962208ms + //t.Log(end.Sub(start) / num) // 511.496µs + start := time.Now() + wll, err := lm.FindChangeLog(context.Background(), "100", 3, 100) + if err != nil { + panic(err) + } + t.Log(time.Since(start)) + t.Log(wll) } diff --git a/pkg/common/listdemo2/demo.js b/pkg/common/listdemo2/demo.js index c0044b238..be9c74cb9 100644 --- a/pkg/common/listdemo2/demo.js +++ b/pkg/common/listdemo2/demo.js @@ -1,4 +1,4 @@ -db.demo.updateMany( +db.friend_version.updateMany( { "d_id": "100" }, @@ -18,7 +18,7 @@ db.demo.updateMany( version: { $add: ["$version", 1] }, - update_time: new Date(), + last_update: new Date(), } }, @@ -35,7 +35,7 @@ db.demo.updateMany( [ { e_id: "1000", - update_time: new Date(), + last_update: new Date(), version: "$version", deleted: false } @@ -57,7 +57,7 @@ db.demo.updateMany( }, then: { e_id: "1000", - update_time: new Date(), + last_update: new Date(), version: "$version", deleted: false }, diff --git a/pkg/common/listdemo2/demo2.js b/pkg/common/listdemo2/demo2.js new file mode 100644 index 000000000..15e7abcfe --- /dev/null +++ b/pkg/common/listdemo2/demo2.js @@ -0,0 +1,63 @@ + +db.friend_version.updateMany( + { + "d_id": "100" + }, + [ + { + $addFields: { + update_elem_ids: ["1000", "1001","1003", "2000"] + } + }, + { + $set: { + version: { + $add: ["$version", 1] + }, + last_update: new Date(), + } + }, + { + $set: { + logs: { + $filter: { + input: "$logs", + as: "log", + cond: { + "$not": { + $in: ["$$log.e_id", "$update_elem_ids"] + } + } + } + }, + + }, + + }, + { + $set: { + logs: { + $concatArrays: [ + "$logs", + [ + { + e_id: "1003", + last_update: ISODate("2024-05-25T06:32:10.238Z"), + version: "$version", + deleted: false + }, + + ] + ] + } + } + }, + { + $unset: ["update_elem_ids"] + }, + + ] +) + + + diff --git a/pkg/common/listdemo2/demo3.js b/pkg/common/listdemo2/demo3.js new file mode 100644 index 000000000..5367971cf --- /dev/null +++ b/pkg/common/listdemo2/demo3.js @@ -0,0 +1,59 @@ + +db.friend_version.aggregate([ + { + "$match": { + "d_id": "100", + } + }, + { + "$project": { + "_id": 0, + "d_id": 0, + } + }, + { + "$addFields": { + "logs": { + $cond: { + if: { + $or: [ + {$lt: ["$version", 3]}, + {$gte: ["$deleted", 3]}, + ], + }, + then: [], + else: "$logs", + } + } + }, + }, + { + "$addFields": { + "logs": { + "$filter": { + input: "$logs", + as: "l", + cond: { $gt: ["$$l.version", 3] } + } + } + } + }, + { + "$addFields": { + "log_len": { + $size: "$logs" + } + } + }, + { + "$addFields": { + "logs": { + $cond: { + if: {$gt: ["$log_len", 1]}, + then: [], + else: "$logs", + } + } + } + } +]) diff --git a/pkg/common/listdemo2/demo5.js b/pkg/common/listdemo2/demo5.js new file mode 100644 index 000000000..71a709a81 --- /dev/null +++ b/pkg/common/listdemo2/demo5.js @@ -0,0 +1,10 @@ +db.friend_version.updateMany( + { + "d_id": "100" + }, + [ + + + + ], +) \ No newline at end of file From a2bfc907bb9e68c53b63c9183ff0887a635b0fd7 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Thu, 23 May 2024 18:39:45 +0800 Subject: [PATCH 04/59] new mongo --- pkg/common/db/mgo/friend.go | 37 +++++++++++++++++++++++++++++++++++-- 1 file changed, 35 insertions(+), 2 deletions(-) diff --git a/pkg/common/db/mgo/friend.go b/pkg/common/db/mgo/friend.go index 6e7a6db98..b82d1c3ab 100644 --- a/pkg/common/db/mgo/friend.go +++ b/pkg/common/db/mgo/friend.go @@ -62,7 +62,26 @@ func (f *FriendMgo) Create(ctx context.Context, friends []*relation.FriendModel) return Success(func() error { return mongoutil.InsertMany(ctx, f.coll, friends) }, func() error { - + mp := make(map[string][]string) + for _, friend := range friends { + mp[friend.OwnerUserID] = append(mp[friend.OwnerUserID], friend.FriendUserID) + } + for ownerUserID, friendUserIDs := range mp { + if err := f.owner.WriteLog(ctx, ownerUserID, friendUserIDs, false); err != nil { + return err + } + } + return nil + }, func() error { + mp := make(map[string][]string) + for _, friend := range friends { + mp[friend.FriendUserID] = append(mp[friend.FriendUserID], friend.OwnerUserID) + } + for friendUserID, ownerUserIDs := range mp { + if err := f.friend.WriteLog(ctx, friendUserID, ownerUserIDs, false); err != nil { + return err + } + } return nil }) } @@ -78,7 +97,12 @@ func (f *FriendMgo) Delete(ctx context.Context, ownerUserID string, friendUserID }, func() error { return f.owner.WriteLog(ctx, ownerUserID, friendUserIDs, true) }, func() error { - + for _, userID := range friendUserIDs { + if err := f.friend.WriteLog(ctx, userID, []string{ownerUserID}, true); err != nil { + return err + } + } + return nil }) } @@ -95,6 +119,8 @@ func (f *FriendMgo) UpdateByMap(ctx context.Context, ownerUserID string, friendU return mongoutil.UpdateOne(ctx, f.coll, filter, bson.M{"$set": args}, true) }, func() error { return f.owner.WriteLog(ctx, ownerUserID, []string{friendUserID}, false) + }, func() error { + return f.friend.WriteLog(ctx, friendUserID, []string{ownerUserID}, false) }) } @@ -178,6 +204,13 @@ func (f *FriendMgo) UpdateFriends(ctx context.Context, ownerUserID string, frien return mongoutil.Ignore(mongoutil.UpdateMany(ctx, f.coll, filter, update)) }, func() error { return f.owner.WriteLog(ctx, ownerUserID, friendUserIDs, false) + }, func() error { + for _, userID := range friendUserIDs { + if err := f.friend.WriteLog(ctx, userID, []string{ownerUserID}, true); err != nil { + return err + } + } + return nil }) } From 05cec1b10c8e88dccd24be65839ac4c29bc365d9 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Fri, 24 May 2024 09:54:17 +0800 Subject: [PATCH 05/59] new mongo --- pkg/common/db/mgo/friend.go | 38 +++---------------------------------- 1 file changed, 3 insertions(+), 35 deletions(-) diff --git a/pkg/common/db/mgo/friend.go b/pkg/common/db/mgo/friend.go index b82d1c3ab..a44244fca 100644 --- a/pkg/common/db/mgo/friend.go +++ b/pkg/common/db/mgo/friend.go @@ -28,9 +28,8 @@ import ( // FriendMgo implements FriendModelInterface using MongoDB as the storage backend. type FriendMgo struct { - coll *mongo.Collection - owner dataver.DataLog - friend dataver.DataLog + coll *mongo.Collection + owner dataver.DataLog } // NewFriendMongo creates a new instance of FriendMgo with the provided MongoDB database. @@ -50,11 +49,7 @@ func NewFriendMongo(db *mongo.Database) (relation.FriendModelInterface, error) { if err != nil { return nil, err } - friend, err := dataver.NewDataLog(db.Collection("friend_log")) - if err != nil { - return nil, err - } - return &FriendMgo{coll: coll, owner: owner, friend: friend}, nil + return &FriendMgo{coll: coll, owner: owner}, nil } // Create inserts multiple friend records. @@ -72,17 +67,6 @@ func (f *FriendMgo) Create(ctx context.Context, friends []*relation.FriendModel) } } return nil - }, func() error { - mp := make(map[string][]string) - for _, friend := range friends { - mp[friend.FriendUserID] = append(mp[friend.FriendUserID], friend.OwnerUserID) - } - for friendUserID, ownerUserIDs := range mp { - if err := f.friend.WriteLog(ctx, friendUserID, ownerUserIDs, false); err != nil { - return err - } - } - return nil }) } @@ -96,13 +80,6 @@ func (f *FriendMgo) Delete(ctx context.Context, ownerUserID string, friendUserID return mongoutil.DeleteOne(ctx, f.coll, filter) }, func() error { return f.owner.WriteLog(ctx, ownerUserID, friendUserIDs, true) - }, func() error { - for _, userID := range friendUserIDs { - if err := f.friend.WriteLog(ctx, userID, []string{ownerUserID}, true); err != nil { - return err - } - } - return nil }) } @@ -119,8 +96,6 @@ func (f *FriendMgo) UpdateByMap(ctx context.Context, ownerUserID string, friendU return mongoutil.UpdateOne(ctx, f.coll, filter, bson.M{"$set": args}, true) }, func() error { return f.owner.WriteLog(ctx, ownerUserID, []string{friendUserID}, false) - }, func() error { - return f.friend.WriteLog(ctx, friendUserID, []string{ownerUserID}, false) }) } @@ -204,13 +179,6 @@ func (f *FriendMgo) UpdateFriends(ctx context.Context, ownerUserID string, frien return mongoutil.Ignore(mongoutil.UpdateMany(ctx, f.coll, filter, update)) }, func() error { return f.owner.WriteLog(ctx, ownerUserID, friendUserIDs, false) - }, func() error { - for _, userID := range friendUserIDs { - if err := f.friend.WriteLog(ctx, userID, []string{ownerUserID}, true); err != nil { - return err - } - } - return nil }) } From 11840025fc1f44808df4fbf669860c3f81e6c0af Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Fri, 24 May 2024 15:05:12 +0800 Subject: [PATCH 06/59] new mongo --- pkg/common/db/dataver/common.go | 23 +++++++++++++-- pkg/common/db/dataver/result.go | 32 +++++++++++++++++++++ pkg/common/db/mgo/friend.go | 15 ++++++++++ pkg/common/listdemo2/common.go | 50 +-------------------------------- 4 files changed, 68 insertions(+), 52 deletions(-) create mode 100644 pkg/common/db/dataver/result.go diff --git a/pkg/common/db/dataver/common.go b/pkg/common/db/dataver/common.go index c146600fb..c7e4658ab 100644 --- a/pkg/common/db/dataver/common.go +++ b/pkg/common/db/dataver/common.go @@ -24,6 +24,23 @@ type WriteLog struct { LogLen int `bson:"log_len"` } +func (w *WriteLog) Full() bool { + if w.Version == 0 { + return true + } + return len(w.Logs) != w.LogLen +} + +func (w *WriteLog) DeleteEId() []string { + var eIds []string + for _, l := range w.Logs { + if l.Deleted { + eIds = append(eIds, l.EID) + } + } + return eIds +} + type Elem struct { EID string `bson:"e_id"` Deleted bool `bson:"deleted"` @@ -33,7 +50,7 @@ type Elem struct { type DataLog interface { WriteLog(ctx context.Context, dId string, eIds []string, deleted bool) error - FindChangeLog(ctx context.Context, did string, version uint, limit int) (*WriteLog, error) + FindChangeLog(ctx context.Context, dId string, version uint, limit int) (*WriteLog, error) DeleteAfterUnchangedLog(ctx context.Context, deadline time.Time) error } @@ -173,11 +190,11 @@ func (l *logModel) writeLogBatch(ctx context.Context, dId string, eIds []string, return mongoutil.UpdateMany(ctx, l.coll, filter, pipeline) } -func (l *logModel) FindChangeLog(ctx context.Context, did string, version uint, limit int) (*WriteLog, error) { +func (l *logModel) FindChangeLog(ctx context.Context, dId string, version uint, limit int) (*WriteLog, error) { pipeline := []bson.M{ { "$match": bson.M{ - "d_id": did, + "d_id": dId, }, }, { diff --git a/pkg/common/db/dataver/result.go b/pkg/common/db/dataver/result.go new file mode 100644 index 000000000..b6e6d22b4 --- /dev/null +++ b/pkg/common/db/dataver/result.go @@ -0,0 +1,32 @@ +package dataver + +type SyncResult[T any] struct { + Version uint + DeleteEID []string + Changes []T + Full bool +} + +func NewSyncResult[T any](wl *WriteLog, find func(eIds []string) ([]T, error)) (*SyncResult[T], error) { + var findEIDs []string + var res SyncResult[T] + if wl.Full() { + res.Full = true + } else { + for _, l := range wl.Logs { + if l.Deleted { + res.DeleteEID = append(res.DeleteEID, l.EID) + } else { + findEIDs = append(findEIDs, l.EID) + } + } + } + if res.Full || len(findEIDs) > 0 { + var err error + res.Changes, err = find(findEIDs) + if err != nil { + return nil, err + } + } + return &res, nil +} diff --git a/pkg/common/db/mgo/friend.go b/pkg/common/db/mgo/friend.go index a44244fca..78e74edd3 100644 --- a/pkg/common/db/mgo/friend.go +++ b/pkg/common/db/mgo/friend.go @@ -16,6 +16,7 @@ package mgo import ( "context" + "errors" "github.com/openimsdk/open-im-server/v3/pkg/common/db/dataver" "github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation" @@ -182,6 +183,20 @@ func (f *FriendMgo) UpdateFriends(ctx context.Context, ownerUserID string, frien }) } +func (f *FriendMgo) IncrSync(ctx context.Context, ownerUserID string, version uint, limit int) (*dataver.SyncResult[*relation.FriendModel], error) { + res, err := f.owner.FindChangeLog(ctx, ownerUserID, version, limit) + if err != nil { + return nil, err + } + return dataver.NewSyncResult[*relation.FriendModel](res, func(eIds []string) ([]*relation.FriendModel, error) { + if len(eIds) == 0 { + return nil, errors.New("todo") + } else { + return f.FindFriends(ctx, ownerUserID, eIds) + } + }) +} + func Success(fns ...func() error) error { for _, fn := range fns { if err := fn(); err != nil { diff --git a/pkg/common/listdemo2/common.go b/pkg/common/listdemo2/common.go index 2267d8630..40419d533 100644 --- a/pkg/common/listdemo2/common.go +++ b/pkg/common/listdemo2/common.go @@ -70,54 +70,6 @@ func (l *LogModel) InitIndex(ctx context.Context) error { return err } -func (l *LogModel) WriteLog1(ctx context.Context, dId string, eId string, deleted bool) { - if err := l.WriteLog(ctx, dId, eId, deleted); err != nil { - panic(err) - } -} - -func (l *LogModel) WriteLogBatch1(ctx context.Context, dId string, eIds []string, deleted bool) { - if err := l.WriteLogBatch(ctx, dId, eIds, deleted); err != nil { - panic(err) - } -} - -func (l *LogModel) WriteLog(ctx context.Context, dId string, eId string, deleted bool) error { - now := time.Now() - res, err := l.writeLog(ctx, dId, eId, deleted, now) - if err != nil { - return err - } - if res.MatchedCount > 0 { - return nil - } - wl := WriteLog{ - DID: dId, - Logs: []LogElem{ - { - EID: eId, - Deleted: deleted, - Version: FirstVersion, - LastUpdate: now, - }, - }, - Version: FirstVersion, - Deleted: DefaultDeleteVersion, - LastUpdate: now, - } - if _, err := l.coll.InsertOne(ctx, &wl); err == nil { - return nil - } else if !mongo.IsDuplicateKeyError(err) { - return err - } - if res, err := l.writeLog(ctx, dId, eId, deleted, now); err != nil { - return err - } else if res.ModifiedCount == 0 { - return errs.ErrInternalServer.WrapMsg("mongodb return value that should not occur", "coll", l.coll.Name(), "dId", dId, "eId", eId) - } - return nil -} - func (l *LogModel) writeLog(ctx context.Context, dId string, eId string, deleted bool, now time.Time) (*mongo.UpdateResult, error) { filter := bson.M{ "d_id": dId, @@ -356,7 +308,7 @@ func (l *LogModel) FindChangeLog(ctx context.Context, did string, version uint, return nil, err } if len(res) == 0 { - return nil, ErrNotFound + return &WriteLogLen{}, nil } return res[0], nil } From e99eaf91464d568e806ae4fcecba4df75a2ee633 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Fri, 24 May 2024 15:44:08 +0800 Subject: [PATCH 07/59] new mongo --- pkg/common/db/dataver/common.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/common/db/dataver/common.go b/pkg/common/db/dataver/common.go index c7e4658ab..d5a0c1694 100644 --- a/pkg/common/db/dataver/common.go +++ b/pkg/common/db/dataver/common.go @@ -97,7 +97,7 @@ func (l *logModel) WriteLog(ctx context.Context, dId string, eIds []string, dele } if res, err := l.writeLogBatch(ctx, dId, eIds, deleted, now); err != nil { return err - } else if res.ModifiedCount == 0 { + } else if res.MatchedCount == 0 { return errs.ErrInternalServer.WrapMsg("mongodb return value that should not occur", "coll", l.coll.Name(), "dId", dId, "eIds", eIds) } return nil @@ -253,7 +253,7 @@ func (l *logModel) FindChangeLog(ctx context.Context, dId string, version uint, return nil, err } if len(res) == 0 { - return nil, errs.Wrap(mongo.ErrNoDocuments) + return &WriteLog{}, nil } return res[0], nil } From 2f97933f6ca841f416aeceff188f6116e8e94fb1 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Fri, 24 May 2024 18:11:48 +0800 Subject: [PATCH 08/59] new mongo --- pkg/common/db/dataver/common.go | 14 ++++++------ pkg/common/db/dataver/todo.go | 38 +++++++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+), 6 deletions(-) create mode 100644 pkg/common/db/dataver/todo.go diff --git a/pkg/common/db/dataver/common.go b/pkg/common/db/dataver/common.go index d5a0c1694..bc367d795 100644 --- a/pkg/common/db/dataver/common.go +++ b/pkg/common/db/dataver/common.go @@ -6,6 +6,7 @@ import ( "github.com/openimsdk/tools/errs" "github.com/openimsdk/tools/utils/datautil" "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" "go.mongodb.org/mongo-driver/mongo" "time" ) @@ -16,12 +17,13 @@ const ( ) type WriteLog struct { - DID string `bson:"d_id"` - Logs []Elem `bson:"logs"` - Version uint `bson:"version"` - Deleted uint `bson:"deleted"` - LastUpdate time.Time `bson:"last_update"` - LogLen int `bson:"log_len"` + ID primitive.ObjectID `bson:"_id"` + DID string `bson:"d_id"` + Logs []Elem `bson:"logs"` + Version uint `bson:"version"` + Deleted uint `bson:"deleted"` + LastUpdate time.Time `bson:"last_update"` + LogLen int `bson:"log_len"` } func (w *WriteLog) Full() bool { diff --git a/pkg/common/db/dataver/todo.go b/pkg/common/db/dataver/todo.go new file mode 100644 index 000000000..69b979fe5 --- /dev/null +++ b/pkg/common/db/dataver/todo.go @@ -0,0 +1,38 @@ +package dataver + +/* + +UserIDs 顺序 +前500顺序 + + +1,2,3,4,5,6,7,8,9 + +1,3,5,7,8,9 + + +1.sdk添加一个表记录 docID(后续换名字), version +2.sdk同步，先计算idHash，api调用参数idHash, docID, version +3.服务器先判断version变更记录，没有直接返回同步成功。 + 有变更，先查版本变更记录，在查前500id，变更记录只保留前500id中的 + 根据前500id计算idHash，不一致返回会全量id，不反悔删除id + 全量同步有标识，只返回全量id + 变更记录只包含id，不包括详细信息。 +4.sdk通过变更记录，同步数据不一致进行重试。 + + + + + + + + + + + + + + + + +*/ From 1f02bdc26779b0c56a57d3dd0e00f534538834ec Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Mon, 27 May 2024 15:15:23 +0800 Subject: [PATCH 09/59] friend incr sync --- go.mod | 6 ++- go.sum | 4 +- internal/rpc/friend/convert.go | 26 +++++++++++ internal/rpc/friend/sync.go | 65 ++++++++++++++++++++++++++ pkg/common/cachekey/friend.go | 15 ++++-- pkg/common/config/config.go | 3 +- pkg/common/db/cache/friend.go | 33 +++++++++++++ pkg/common/db/controller/friend.go | 21 +++++++-- pkg/common/db/dataver/common.go | 16 +++++++ pkg/common/db/dataver/result.go | 25 +++++----- pkg/common/db/dataver/todo.go | 2 +- pkg/common/db/mgo/friend.go | 50 ++++++++++++-------- pkg/common/db/table/relation/friend.go | 7 +++ pkg/common/listdemo2/common_test.go | 48 ++++++++++--------- 14 files changed, 253 insertions(+), 68 deletions(-) create mode 100644 internal/rpc/friend/convert.go create mode 100644 internal/rpc/friend/sync.go diff --git a/go.mod b/go.mod index 8e06e503f..c3d69b0fb 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/lestrrat-go/file-rotatelogs v2.4.0+incompatible // indirect github.com/mitchellh/mapstructure v1.5.0 - github.com/openimsdk/protocol v0.0.65 + github.com/openimsdk/protocol v0.0.69-alpha.1 github.com/openimsdk/tools v0.0.49-alpha.23 github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.18.0 @@ -177,3 +177,7 @@ require ( golang.org/x/crypto v0.21.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect ) + +//replace ( +// github.com/openimsdk/protocol => /Users/chao/Desktop/project/protocol +//) diff --git a/go.sum b/go.sum index a80b2fb0a..908c15c87 100644 --- a/go.sum +++ b/go.sum @@ -286,8 +286,8 @@ github.com/onsi/gomega v1.18.1 h1:M1GfJqGRrBrrGGsbxzV5dqM2U2ApXefZCQpkukxYRLE= github.com/onsi/gomega v1.18.1/go.mod h1:0q+aL8jAiMXy9hbwj2mr5GziHiwhAIQpFmmtT5hitRs= github.com/openimsdk/gomake v0.0.13 h1:xLDe/moqgWpRoptHzI4packAWzs4C16b+sVY+txNJp0= github.com/openimsdk/gomake v0.0.13/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= -github.com/openimsdk/protocol v0.0.65 h1:SPT9qyUsFRTTKSKb/FjpS+xr6sxz/Kbnu+su1bxYagc= -github.com/openimsdk/protocol v0.0.65/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= +github.com/openimsdk/protocol v0.0.69-alpha.1 h1:l/PN8mwmh5O7PRoaiMZvey+hUxyuNNnMgPGKOfEMOKs= +github.com/openimsdk/protocol v0.0.69-alpha.1/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= github.com/openimsdk/tools v0.0.49-alpha.23 h1:/KkJ7vfx8FAoJhq3veH9PWnxbSkEf+dTSshvDrHBR38= github.com/openimsdk/tools v0.0.49-alpha.23/go.mod h1:g7mkHXYUPi0/8aAX8VPMHpnb3hqdV69Jph+bXOGvvNM= github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= diff --git a/internal/rpc/friend/convert.go b/internal/rpc/friend/convert.go new file mode 100644 index 000000000..aa6895284 --- /dev/null +++ b/internal/rpc/friend/convert.go @@ -0,0 +1,26 @@ +package friend + +import ( + relationtb "github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation" + "github.com/openimsdk/protocol/friend" + "github.com/openimsdk/tools/utils/datautil" +) + +func friendDB2PB(db *relationtb.FriendModel) *friend.FriendInfo { + return &friend.FriendInfo{ + OwnerUserID: db.OwnerUserID, + FriendUserID: db.FriendUserID, + FriendNickname: db.FriendNickname, + FriendFaceURL: db.FriendFaceURL, + Remark: db.Remark, + CreateTime: db.CreateTime.UnixMilli(), + AddSource: db.AddSource, + OperatorUserID: db.OperatorUserID, + Ex: db.Ex, + IsPinned: db.IsPinned, + } +} + +func friendsDB2PB(db []*relationtb.FriendModel) []*friend.FriendInfo { + return datautil.Slice(db, friendDB2PB) +} diff --git a/internal/rpc/friend/sync.go b/internal/rpc/friend/sync.go new file mode 100644 index 000000000..1e74cbc0d --- /dev/null +++ b/internal/rpc/friend/sync.go @@ -0,0 +1,65 @@ +package friend + +import ( + "context" + "crypto/md5" + "encoding/binary" + "encoding/json" + "github.com/openimsdk/open-im-server/v3/pkg/common/db/dataver" + "github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation" + pbfriend "github.com/openimsdk/protocol/friend" +) + +func (s *friendServer) SearchFriends(ctx context.Context, req *pbfriend.SearchFriendsReq) (*pbfriend.SearchFriendsResp, error) { + //TODO implement me + panic("implement me") +} + +func (s *friendServer) sortFriendUserIDsHash(userIDs []string) uint64 { + data, _ := json.Marshal(userIDs) + sum := md5.Sum(data) + return binary.BigEndian.Uint64(sum[:]) +} + +func (s *friendServer) IncrSyncFriends(ctx context.Context, req *pbfriend.IncrSyncFriendsReq) (*pbfriend.IncrSyncFriendsResp, error) { + var limit int + if req.Version > 0 { + limit = s.config.RpcConfig.FriendSyncCount + } + incrVer, err := s.friendDatabase.FindFriendIncrVersion(ctx, req.UserID, uint(req.Version), limit) + if err != nil { + return nil, err + } + sortUserIDs, err := s.friendDatabase.FindSortFriendUserIDs(ctx, req.UserID) + if err != nil { + return nil, err + } + if len(sortUserIDs) == 0 { + return &pbfriend.IncrSyncFriendsResp{ + Version: uint64(incrVer.Version), + Full: true, + SyncCount: uint32(s.config.RpcConfig.FriendSyncCount), + }, nil + } + var changes []*relation.FriendModel + res := dataver.NewSyncResult(incrVer, sortUserIDs) + if len(res.Changes) > 0 { + changes, err = s.friendDatabase.FindFriendsWithError(ctx, req.UserID, res.Changes) + if err != nil { + return nil, err + } + } + calcHash := s.sortFriendUserIDsHash(sortUserIDs) + if calcHash == req.IdHash { + sortUserIDs = nil + } + return &pbfriend.IncrSyncFriendsResp{ + Version: uint64(res.Version), + Full: res.Full, + SyncCount: uint32(s.config.RpcConfig.FriendSyncCount), + SortUserIdHash: calcHash, + SortUserIds: sortUserIDs, + DeleteUserIds: res.DeleteEID, + Changes: friendsDB2PB(changes), + }, nil +} diff --git a/pkg/common/cachekey/friend.go b/pkg/common/cachekey/friend.go index 9691b1f5c..6a217bdef 100644 --- a/pkg/common/cachekey/friend.go +++ b/pkg/common/cachekey/friend.go @@ -14,11 +14,14 @@ package cachekey +import "strconv" + const ( - FriendIDsKey = "FRIEND_IDS:" - TwoWayFriendsIDsKey = "COMMON_FRIENDS_IDS:" - FriendKey = "FRIEND_INFO:" - IsFriendKey = "IS_FRIEND:" // local cache key + FriendIDsKey = "FRIEND_IDS:" + TwoWayFriendsIDsKey = "COMMON_FRIENDS_IDS:" + FriendKey = "FRIEND_INFO:" + IsFriendKey = "IS_FRIEND:" // local cache key + FriendSyncSortUserIDsKey = "FRIEND_SYNC_SORT_USER_IDS:" ) func GetFriendIDsKey(ownerUserID string) string { @@ -36,3 +39,7 @@ func GetFriendKey(ownerUserID, friendUserID string) string { func GetIsFriendKey(possibleFriendUserID, userID string) string { return IsFriendKey + possibleFriendUserID + "-" + userID } + +func GetFriendSyncSortUserIDsKey(ownerUserID string, count int) string { + return FriendSyncSortUserIDsKey + strconv.Itoa(count) + ":" + ownerUserID +} diff --git a/pkg/common/config/config.go b/pkg/common/config/config.go index 12c4f7f78..881e05598 100644 --- a/pkg/common/config/config.go +++ b/pkg/common/config/config.go @@ -244,7 +244,8 @@ type Friend struct { ListenIP string `mapstructure:"listenIP"` Ports []int `mapstructure:"ports"` } `mapstructure:"rpc"` - Prometheus Prometheus `mapstructure:"prometheus"` + Prometheus Prometheus `mapstructure:"prometheus"` + FriendSyncCount int `mapstructure:"friendSyncCount"` } type Group struct { diff --git a/pkg/common/db/cache/friend.go b/pkg/common/db/cache/friend.go index 73fe5ea69..4a51a8c17 100644 --- a/pkg/common/db/cache/friend.go +++ b/pkg/common/db/cache/friend.go @@ -16,6 +16,7 @@ package cache import ( "context" + "github.com/openimsdk/open-im-server/v3/pkg/common/db/dataver" "time" "github.com/dtm-labs/rockscache" @@ -41,12 +42,19 @@ type FriendCache interface { GetFriendIDs(ctx context.Context, ownerUserID string) (friendIDs []string, err error) // Called when friendID list changed DelFriendIDs(ownerUserID ...string) FriendCache + + DelSortFriendUserIDs(ownerUserIDs ...string) FriendCache + // Get single friendInfo from the cache GetFriend(ctx context.Context, ownerUserID, friendUserID string) (friend *relationtb.FriendModel, err error) // Delete friend when friend info changed DelFriend(ownerUserID, friendUserID string) FriendCache // Delete friends when friends' info changed DelFriends(ownerUserID string, friendUserIDs []string) FriendCache + + FindSortFriendUserIDs(ctx context.Context, ownerUserID string) ([]string, error) + + FindFriendIncrVersion(ctx context.Context, ownerUserID string, version uint, limit int) (*dataver.WriteLog, error) } // FriendCacheRedis is an implementation of the FriendCache interface using Redis. @@ -55,6 +63,7 @@ type FriendCacheRedis struct { friendDB relationtb.FriendModelInterface expireTime time.Duration rcClient *rockscache.Client + syncCount int } // NewFriendCacheRedis creates a new instance of FriendCacheRedis. @@ -89,6 +98,10 @@ func (f *FriendCacheRedis) getFriendIDsKey(ownerUserID string) string { return cachekey.GetFriendIDsKey(ownerUserID) } +func (f *FriendCacheRedis) getFriendSyncSortUserIDsKey(ownerUserID string) string { + return cachekey.GetFriendSyncSortUserIDsKey(ownerUserID, f.syncCount) +} + // getTwoWayFriendsIDsKey returns the key for storing two-way friend IDs in the cache. func (f *FriendCacheRedis) getTwoWayFriendsIDsKey(ownerUserID string) string { return cachekey.GetTwoWayFriendsIDsKey(ownerUserID) @@ -118,6 +131,16 @@ func (f *FriendCacheRedis) DelFriendIDs(ownerUserIDs ...string) FriendCache { return newGroupCache } +func (f *FriendCacheRedis) DelSortFriendUserIDs(ownerUserIDs ...string) FriendCache { + newGroupCache := f.NewCache() + keys := make([]string, 0, len(ownerUserIDs)) + for _, userID := range ownerUserIDs { + keys = append(keys, f.getFriendSyncSortUserIDsKey(userID)) + } + newGroupCache.AddKeys(keys...) + return newGroupCache +} + // GetTwoWayFriendIDs retrieves two-way friend IDs from the cache. func (f *FriendCacheRedis) GetTwoWayFriendIDs(ctx context.Context, ownerUserID string) (twoWayFriendIDs []string, err error) { friendIDs, err := f.GetFriendIDs(ctx, ownerUserID) @@ -172,3 +195,13 @@ func (f *FriendCacheRedis) DelFriends(ownerUserID string, friendUserIDs []string return newFriendCache } + +func (f *FriendCacheRedis) FindSortFriendUserIDs(ctx context.Context, ownerUserID string) ([]string, error) { + return getCache(ctx, f.rcClient, f.getFriendSyncSortUserIDsKey(ownerUserID), f.expireTime, func(ctx context.Context) ([]string, error) { + return f.friendDB.FindOwnerFriendUserIds(ctx, ownerUserID, f.syncCount) + }) +} + +func (f *FriendCacheRedis) FindFriendIncrVersion(ctx context.Context, ownerUserID string, version uint, limit int) (*dataver.WriteLog, error) { + return f.friendDB.FindIncrVersion(ctx, ownerUserID, version, limit) +} diff --git a/pkg/common/db/controller/friend.go b/pkg/common/db/controller/friend.go index 49136f228..49c0cb990 100644 --- a/pkg/common/db/controller/friend.go +++ b/pkg/common/db/controller/friend.go @@ -17,6 +17,7 @@ package controller import ( "context" "fmt" + "github.com/openimsdk/open-im-server/v3/pkg/common/db/dataver" "time" "github.com/openimsdk/open-im-server/v3/pkg/common/db/cache" @@ -75,6 +76,10 @@ type FriendDatabase interface { // UpdateFriends updates fields for friends UpdateFriends(ctx context.Context, ownerUserID string, friendUserIDs []string, val map[string]any) (err error) + + FindSortFriendUserIDs(ctx context.Context, ownerUserID string) ([]string, error) + + FindFriendIncrVersion(ctx context.Context, ownerUserID string, version uint, limit int) (*dataver.WriteLog, error) } type friendDatabase struct { @@ -173,7 +178,7 @@ func (f *friendDatabase) BecomeFriends(ctx context.Context, ownerUserID string, return err } newFriendIDs = append(newFriendIDs, ownerUserID) - cache = cache.DelFriendIDs(newFriendIDs...) + cache = cache.DelFriendIDs(newFriendIDs...).DelSortFriendUserIDs(ownerUserID) return cache.ExecDel(ctx) }) @@ -276,7 +281,7 @@ func (f *friendDatabase) AgreeFriendRequest(ctx context.Context, friendRequest * return err } } - return f.cache.DelFriendIDs(friendRequest.ToUserID, friendRequest.FromUserID).ExecDel(ctx) + return f.cache.DelFriendIDs(friendRequest.ToUserID, friendRequest.FromUserID).DelSortFriendUserIDs(friendRequest.ToUserID, friendRequest.FromUserID).ExecDel(ctx) }) } @@ -285,7 +290,7 @@ func (f *friendDatabase) Delete(ctx context.Context, ownerUserID string, friendU if err := f.friend.Delete(ctx, ownerUserID, friendUserIDs); err != nil { return err } - return f.cache.DelFriendIDs(append(friendUserIDs, ownerUserID)...).ExecDel(ctx) + return f.cache.DelFriendIDs(append(friendUserIDs, ownerUserID)...).DelSortFriendUserIDs(ownerUserID).ExecDel(ctx) } // UpdateRemark updates the remark for a friend. Zero value for remark is also supported. @@ -342,5 +347,13 @@ func (f *friendDatabase) UpdateFriends(ctx context.Context, ownerUserID string, if err := f.friend.UpdateFriends(ctx, ownerUserID, friendUserIDs, val); err != nil { return err } - return f.cache.DelFriends(ownerUserID, friendUserIDs).ExecDel(ctx) + return f.cache.DelFriends(ownerUserID, friendUserIDs).DelSortFriendUserIDs(ownerUserID).ExecDel(ctx) +} + +func (f *friendDatabase) FindSortFriendUserIDs(ctx context.Context, ownerUserID string) ([]string, error) { + return f.cache.FindSortFriendUserIDs(ctx, ownerUserID) +} + +func (f *friendDatabase) FindFriendIncrVersion(ctx context.Context, ownerUserID string, version uint, limit int) (*dataver.WriteLog, error) { + return f.cache.FindFriendIncrVersion(ctx, ownerUserID, version, limit) } diff --git a/pkg/common/db/dataver/common.go b/pkg/common/db/dataver/common.go index bc367d795..c9a4e0419 100644 --- a/pkg/common/db/dataver/common.go +++ b/pkg/common/db/dataver/common.go @@ -2,12 +2,14 @@ package dataver import ( "context" + "errors" "github.com/openimsdk/tools/db/mongoutil" "github.com/openimsdk/tools/errs" "github.com/openimsdk/tools/utils/datautil" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/bson/primitive" "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" "time" ) @@ -192,7 +194,21 @@ func (l *logModel) writeLogBatch(ctx context.Context, dId string, eIds []string, return mongoutil.UpdateMany(ctx, l.coll, filter, pipeline) } +func (l *logModel) findDoc(ctx context.Context, dId string) (*WriteLog, error) { + res, err := mongoutil.FindOne[*WriteLog](ctx, l.coll, bson.M{"d_id": dId}, options.FindOne().SetProjection(bson.M{"logs": 0})) + if err == nil { + return res, nil + } else if errors.Is(err, mongo.ErrNoDocuments) { + return &WriteLog{}, nil + } else { + return nil, err + } +} + func (l *logModel) FindChangeLog(ctx context.Context, dId string, version uint, limit int) (*WriteLog, error) { + if version == 0 && limit == 0 { + return l.findDoc(ctx, dId) + } pipeline := []bson.M{ { "$match": bson.M{ diff --git a/pkg/common/db/dataver/result.go b/pkg/common/db/dataver/result.go index b6e6d22b4..915a618d0 100644 --- a/pkg/common/db/dataver/result.go +++ b/pkg/common/db/dataver/result.go @@ -1,32 +1,31 @@ package dataver -type SyncResult[T any] struct { +import "github.com/openimsdk/tools/utils/datautil" + +type SyncResult struct { Version uint DeleteEID []string - Changes []T + Changes []string Full bool } -func NewSyncResult[T any](wl *WriteLog, find func(eIds []string) ([]T, error)) (*SyncResult[T], error) { +func NewSyncResult(wl *WriteLog, fullIDs []string) *SyncResult { var findEIDs []string - var res SyncResult[T] + var res SyncResult if wl.Full() { + res.Changes = fullIDs res.Full = true } else { + idSet := datautil.SliceSet(fullIDs) for _, l := range wl.Logs { if l.Deleted { res.DeleteEID = append(res.DeleteEID, l.EID) } else { - findEIDs = append(findEIDs, l.EID) + if _, ok := idSet[l.EID]; ok { + findEIDs = append(findEIDs, l.EID) + } } } } - if res.Full || len(findEIDs) > 0 { - var err error - res.Changes, err = find(findEIDs) - if err != nil { - return nil, err - } - } - return &res, nil + return &res } diff --git a/pkg/common/db/dataver/todo.go b/pkg/common/db/dataver/todo.go index 69b979fe5..cd2be14f0 100644 --- a/pkg/common/db/dataver/todo.go +++ b/pkg/common/db/dataver/todo.go @@ -19,7 +19,7 @@ UserIDs 顺序 全量同步有标识，只返回全量id 变更记录只包含id，不包括详细信息。 4.sdk通过变更记录，同步数据不一致进行重试。 - +5.先修改db，在自增版本号，外层加事务 diff --git a/pkg/common/db/mgo/friend.go b/pkg/common/db/mgo/friend.go index 78e74edd3..bca6873c2 100644 --- a/pkg/common/db/mgo/friend.go +++ b/pkg/common/db/mgo/friend.go @@ -16,7 +16,6 @@ package mgo import ( "context" - "errors" "github.com/openimsdk/open-im-server/v3/pkg/common/db/dataver" "github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation" @@ -55,7 +54,7 @@ func NewFriendMongo(db *mongo.Database) (relation.FriendModelInterface, error) { // Create inserts multiple friend records. func (f *FriendMgo) Create(ctx context.Context, friends []*relation.FriendModel) error { - return Success(func() error { + return IncrVersion(func() error { return mongoutil.InsertMany(ctx, f.coll, friends) }, func() error { mp := make(map[string][]string) @@ -77,7 +76,7 @@ func (f *FriendMgo) Delete(ctx context.Context, ownerUserID string, friendUserID "owner_user_id": ownerUserID, "friend_user_id": bson.M{"$in": friendUserIDs}, } - return Success(func() error { + return IncrVersion(func() error { return mongoutil.DeleteOne(ctx, f.coll, filter) }, func() error { return f.owner.WriteLog(ctx, ownerUserID, friendUserIDs, true) @@ -93,7 +92,7 @@ func (f *FriendMgo) UpdateByMap(ctx context.Context, ownerUserID string, friendU "owner_user_id": ownerUserID, "friend_user_id": friendUserID, } - return Success(func() error { + return IncrVersion(func() error { return mongoutil.UpdateOne(ctx, f.coll, filter, bson.M{"$set": args}, true) }, func() error { return f.owner.WriteLog(ctx, ownerUserID, []string{friendUserID}, false) @@ -146,7 +145,14 @@ func (f *FriendMgo) FindReversalFriends(ctx context.Context, friendUserID string // FindOwnerFriends retrieves a paginated list of friends for a given owner. func (f *FriendMgo) FindOwnerFriends(ctx context.Context, ownerUserID string, pagination pagination.Pagination) (int64, []*relation.FriendModel, error) { filter := bson.M{"owner_user_id": ownerUserID} - return mongoutil.FindPage[*relation.FriendModel](ctx, f.coll, filter, pagination) + opt := options.Find().SetSort(bson.A{bson.M{"friend_nickname": 1}, bson.M{"create_time": 1}}) + return mongoutil.FindPage[*relation.FriendModel](ctx, f.coll, filter, pagination, opt) +} + +func (f *FriendMgo) FindOwnerFriendUserIds(ctx context.Context, ownerUserID string, limit int) ([]string, error) { + filter := bson.M{"owner_user_id": ownerUserID} + opt := options.Find().SetProjection(bson.M{"_id": 0, "friend_user_id": 1}).SetSort(bson.A{bson.M{"friend_nickname": 1}, bson.M{"create_time": 1}}).SetLimit(int64(limit)) + return mongoutil.Find[string](ctx, f.coll, filter, opt) } // FindInWhoseFriends finds users who have added the specified user as a friend, with pagination. @@ -176,29 +182,33 @@ func (f *FriendMgo) UpdateFriends(ctx context.Context, ownerUserID string, frien // Create an update document update := bson.M{"$set": val} - return Success(func() error { + return IncrVersion(func() error { return mongoutil.Ignore(mongoutil.UpdateMany(ctx, f.coll, filter, update)) }, func() error { return f.owner.WriteLog(ctx, ownerUserID, friendUserIDs, false) }) } -func (f *FriendMgo) IncrSync(ctx context.Context, ownerUserID string, version uint, limit int) (*dataver.SyncResult[*relation.FriendModel], error) { - res, err := f.owner.FindChangeLog(ctx, ownerUserID, version, limit) - if err != nil { - return nil, err - } - return dataver.NewSyncResult[*relation.FriendModel](res, func(eIds []string) ([]*relation.FriendModel, error) { - if len(eIds) == 0 { - return nil, errors.New("todo") - } else { - return f.FindFriends(ctx, ownerUserID, eIds) - } - }) +func (f *FriendMgo) FindIncrVersion(ctx context.Context, ownerUserID string, version uint, limit int) (*dataver.WriteLog, error) { + return f.owner.FindChangeLog(ctx, ownerUserID, version, limit) } -func Success(fns ...func() error) error { - for _, fn := range fns { +//func (f *FriendMgo) IncrSync(ctx context.Context, ownerUserID string, version uint, limit int) (*dataver.SyncResult[*relation.FriendModel], error) { +// res, err := f.owner.FindChangeLog(ctx, ownerUserID, version, limit) +// if err != nil { +// return nil, err +// } +// return dataver.NewSyncResult[*relation.FriendModel](res, func(eIds []string) ([]*relation.FriendModel, error) { +// if len(eIds) == 0 { +// return nil, errors.New("todo") +// } else { +// return f.FindFriends(ctx, ownerUserID, eIds) +// } +// }) +//} + +func IncrVersion(dbs ...func() error) error { + for _, fn := range dbs { if err := fn(); err != nil { return err } diff --git a/pkg/common/db/table/relation/friend.go b/pkg/common/db/table/relation/friend.go index 4c84e773d..a28b2278d 100644 --- a/pkg/common/db/table/relation/friend.go +++ b/pkg/common/db/table/relation/friend.go @@ -16,6 +16,7 @@ package relation import ( "context" + "github.com/openimsdk/open-im-server/v3/pkg/common/db/dataver" "time" "github.com/openimsdk/tools/db/pagination" @@ -25,6 +26,8 @@ import ( type FriendModel struct { OwnerUserID string `bson:"owner_user_id"` FriendUserID string `bson:"friend_user_id"` + FriendNickname string `bson:"friend_nickname"` + FriendFaceURL string `bson:"friend_face_url"` Remark string `bson:"remark"` CreateTime time.Time `bson:"create_time"` AddSource int32 `bson:"add_source"` @@ -53,10 +56,14 @@ type FriendModelInterface interface { FindReversalFriends(ctx context.Context, friendUserID string, ownerUserIDs []string) (friends []*FriendModel, err error) // FindOwnerFriends retrieves a paginated list of friends for a given owner. FindOwnerFriends(ctx context.Context, ownerUserID string, pagination pagination.Pagination) (total int64, friends []*FriendModel, err error) + + FindOwnerFriendUserIds(ctx context.Context, ownerUserID string, limit int) ([]string, error) // FindInWhoseFriends finds users who have added the specified user as a friend, with pagination. FindInWhoseFriends(ctx context.Context, friendUserID string, pagination pagination.Pagination) (total int64, friends []*FriendModel, err error) // FindFriendUserIDs retrieves a list of friend user IDs for a given owner. FindFriendUserIDs(ctx context.Context, ownerUserID string) (friendUserIDs []string, err error) // UpdateFriends update friends' fields UpdateFriends(ctx context.Context, ownerUserID string, friendUserIDs []string, val map[string]any) (err error) + + FindIncrVersion(ctx context.Context, ownerUserID string, version uint, limit int) (*dataver.WriteLog, error) } diff --git a/pkg/common/listdemo2/common_test.go b/pkg/common/listdemo2/common_test.go index bcaf4a19a..c03cc31bd 100644 --- a/pkg/common/listdemo2/common_test.go +++ b/pkg/common/listdemo2/common_test.go @@ -2,10 +2,9 @@ package listdemo import ( "context" - "go.mongodb.org/mongo-driver/mongo" - "go.mongodb.org/mongo-driver/mongo/options" + "errors" + "github.com/openimsdk/tools/db/mongoutil" "testing" - "time" ) func Result[V any](val V, err error) V { @@ -22,20 +21,25 @@ func Check(err error) { } func TestName(t *testing.T) { - cli := Result(mongo.Connect(context.Background(), options.Client().ApplyURI("mongodb://openIM:openIM123@172.16.8.48:37017/openim_v3?maxPoolSize=100").SetConnectTimeout(5*time.Second))) - coll := cli.Database("openim_v3").Collection("friend_version") - _ = coll - //Result(coll.Indexes().CreateMany(context.Background(), []mongo.IndexModel{ - // { - // Keys: map[string]int{"user_id": 1}, - // }, - // { - // Keys: map[string]int{"friends.friend_user_id": 1}, - // }, - //})) + cli := Result(mongoutil.NewMongoDB(context.Background(), &mongoutil.Config{Uri: "mongodb://openIM:openIM123@172.16.8.48:37017/openim_v3?maxPoolSize=100", Database: "openim_v3"})) + + db := cli.GetDB() + tx := cli.GetTx() const num = 1 - lm := &LogModel{coll: coll} + lm := &LogModel{coll: db.Collection("friend_version")} + + err := tx.Transaction(context.Background(), func(ctx context.Context) error { + err := tx.Transaction(ctx, func(ctx context.Context) error { + return lm.WriteLogBatch(ctx, "100", []string{"1000", "2000"}, true) + }) + if err != nil { + t.Log("--------->") + return err + } + return errors.New("1234") + }) + t.Log(err) //start := time.Now() //eIds := make([]string, 0, num) @@ -47,11 +51,11 @@ func TestName(t *testing.T) { //t.Log(end.Sub(start)) // 509.962208ms //t.Log(end.Sub(start) / num) // 511.496µs - start := time.Now() - wll, err := lm.FindChangeLog(context.Background(), "100", 3, 100) - if err != nil { - panic(err) - } - t.Log(time.Since(start)) - t.Log(wll) + //start := time.Now() + //wll, err := lm.FindChangeLog(context.Background(), "100", 3, 100) + //if err != nil { + // panic(err) + //} + //t.Log(time.Since(start)) + //t.Log(wll) } From 8e37a417db65c8bb666caab6dda714340356a396 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Mon, 27 May 2024 18:24:33 +0800 Subject: [PATCH 10/59] friend incr sync --- go.mod | 6 ++-- internal/api/friend.go | 5 +++ internal/api/router.go | 1 + internal/rpc/friend/sync.go | 14 +++++--- internal/rpc/group/sync.go | 16 +++++++++ internal/rpc/user/user.go | 5 +++ pkg/common/db/dataver/common.go | 61 +++++++++++++++++++++++---------- pkg/common/db/dataver/result.go | 17 +++++++-- pkg/common/db/mgo/friend.go | 14 -------- 9 files changed, 96 insertions(+), 43 deletions(-) create mode 100644 internal/rpc/group/sync.go diff --git a/go.mod b/go.mod index c3d69b0fb..daf6cc0d2 100644 --- a/go.mod +++ b/go.mod @@ -178,6 +178,6 @@ require ( gopkg.in/ini.v1 v1.67.0 // indirect ) -//replace ( -// github.com/openimsdk/protocol => /Users/chao/Desktop/project/protocol -//) +replace ( + github.com/openimsdk/protocol => /Users/chao/Desktop/project/protocol +) diff --git a/internal/api/friend.go b/internal/api/friend.go index 1fea38b31..3af162a53 100644 --- a/internal/api/friend.go +++ b/internal/api/friend.go @@ -90,6 +90,11 @@ func (o *FriendApi) GetFriendIDs(c *gin.Context) { func (o *FriendApi) GetSpecifiedFriendsInfo(c *gin.Context) { a2r.Call(friend.FriendClient.GetSpecifiedFriendsInfo, o.Client, c) } + func (o *FriendApi) UpdateFriends(c *gin.Context) { a2r.Call(friend.FriendClient.UpdateFriends, o.Client, c) } + +func (o *FriendApi) GetIncrementalFriends(c *gin.Context) { + a2r.Call(friend.FriendClient.GetIncrementalFriends, o.Client, c) +} diff --git a/internal/api/router.go b/internal/api/router.go index 600567178..78e049e0f 100644 --- a/internal/api/router.go +++ b/internal/api/router.go @@ -86,6 +86,7 @@ func newGinRouter(disCov discovery.SvcDiscoveryRegistry, config *Config) *gin.En friendRouterGroup.POST("/get_friend_id", f.GetFriendIDs) friendRouterGroup.POST("/get_specified_friends_info", f.GetSpecifiedFriendsInfo) friendRouterGroup.POST("/update_friends", f.UpdateFriends) + friendRouterGroup.POST("/get_incremental_friends", f.GetIncrementalFriends) } g := NewGroupApi(*groupRpc) groupRouterGroup := r.Group("/group") diff --git a/internal/rpc/friend/sync.go b/internal/rpc/friend/sync.go index 1e74cbc0d..9d2a53b58 100644 --- a/internal/rpc/friend/sync.go +++ b/internal/rpc/friend/sync.go @@ -5,6 +5,7 @@ import ( "crypto/md5" "encoding/binary" "encoding/json" + "github.com/openimsdk/open-im-server/v3/pkg/authverify" "github.com/openimsdk/open-im-server/v3/pkg/common/db/dataver" "github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation" pbfriend "github.com/openimsdk/protocol/friend" @@ -21,7 +22,10 @@ func (s *friendServer) sortFriendUserIDsHash(userIDs []string) uint64 { return binary.BigEndian.Uint64(sum[:]) } -func (s *friendServer) IncrSyncFriends(ctx context.Context, req *pbfriend.IncrSyncFriendsReq) (*pbfriend.IncrSyncFriendsResp, error) { +func (s *friendServer) GetIncrementalFriends(ctx context.Context, req *pbfriend.GetIncrementalFriendsReq) (*pbfriend.GetIncrementalFriendsResp, error) { + if err := authverify.CheckAccessV3(ctx, req.UserID, s.config.Share.IMAdminUserID); err != nil { + return nil, err + } var limit int if req.Version > 0 { limit = s.config.RpcConfig.FriendSyncCount @@ -35,14 +39,15 @@ func (s *friendServer) IncrSyncFriends(ctx context.Context, req *pbfriend.IncrSy return nil, err } if len(sortUserIDs) == 0 { - return &pbfriend.IncrSyncFriendsResp{ + return &pbfriend.GetIncrementalFriendsResp{ Version: uint64(incrVer.Version), + VersionID: dataver.VersionIDStr(incrVer.ID), Full: true, SyncCount: uint32(s.config.RpcConfig.FriendSyncCount), }, nil } var changes []*relation.FriendModel - res := dataver.NewSyncResult(incrVer, sortUserIDs) + res := dataver.NewSyncResult(incrVer, sortUserIDs, req.VersionID) if len(res.Changes) > 0 { changes, err = s.friendDatabase.FindFriendsWithError(ctx, req.UserID, res.Changes) if err != nil { @@ -53,8 +58,9 @@ func (s *friendServer) IncrSyncFriends(ctx context.Context, req *pbfriend.IncrSy if calcHash == req.IdHash { sortUserIDs = nil } - return &pbfriend.IncrSyncFriendsResp{ + return &pbfriend.GetIncrementalFriendsResp{ Version: uint64(res.Version), + VersionID: res.VersionID, Full: res.Full, SyncCount: uint32(s.config.RpcConfig.FriendSyncCount), SortUserIdHash: calcHash, diff --git a/internal/rpc/group/sync.go b/internal/rpc/group/sync.go new file mode 100644 index 000000000..765bceb63 --- /dev/null +++ b/internal/rpc/group/sync.go @@ -0,0 +1,16 @@ +package group + +import ( + "context" + pbgroup "github.com/openimsdk/protocol/group" +) + +func (s *groupServer) SearchGroupMember(ctx context.Context, req *pbgroup.SearchGroupMemberReq) (*pbgroup.SearchGroupMemberResp, error) { + //TODO implement me + panic("implement me") +} + +func (s *groupServer) GetGroupMemberHash(ctx context.Context, req *pbgroup.GetGroupMemberHashReq) (*pbgroup.GetGroupMemberHashResp, error) { + //TODO implement me + panic("implement me") +} diff --git a/internal/rpc/user/user.go b/internal/rpc/user/user.go index a28fa24e2..2296ee62f 100644 --- a/internal/rpc/user/user.go +++ b/internal/rpc/user/user.go @@ -56,6 +56,11 @@ type userServer struct { webhookClient *webhook.Client } +func (s *userServer) SearchUser(ctx context.Context, req *pbuser.SearchUserReq) (*pbuser.SearchUserResp, error) { + //TODO implement me + panic("implement me") +} + type Config struct { RpcConfig config.User RedisConfig config.Redis diff --git a/pkg/common/db/dataver/common.go b/pkg/common/db/dataver/common.go index c9a4e0419..3ff9906e5 100644 --- a/pkg/common/db/dataver/common.go +++ b/pkg/common/db/dataver/common.go @@ -52,6 +52,27 @@ type Elem struct { LastUpdate time.Time `bson:"last_update"` } +type tableWriteLog struct { + ID primitive.ObjectID `bson:"_id"` + DID string `bson:"d_id"` + Logs []Elem `bson:"logs"` + Version uint `bson:"version"` + Deleted uint `bson:"deleted"` + LastUpdate time.Time `bson:"last_update"` +} + +func (t *tableWriteLog) WriteLog() *WriteLog { + return &WriteLog{ + ID: t.ID, + DID: t.DID, + Logs: t.Logs, + Version: t.Version, + Deleted: t.Deleted, + LastUpdate: t.LastUpdate, + LogLen: 0, + } +} + type DataLog interface { WriteLog(ctx context.Context, dId string, eIds []string, deleted bool) error FindChangeLog(ctx context.Context, dId string, version uint, limit int) (*WriteLog, error) @@ -94,7 +115,7 @@ func (l *logModel) WriteLog(ctx context.Context, dId string, eIds []string, dele if res.MatchedCount > 0 { return nil } - if err := l.initDoc(ctx, dId, eIds, deleted, now); err == nil { + if _, err := l.initDoc(ctx, dId, eIds, deleted, now); err == nil { return nil } else if !mongo.IsDuplicateKeyError(err) { return err @@ -107,15 +128,9 @@ func (l *logModel) WriteLog(ctx context.Context, dId string, eIds []string, dele return nil } -func (l *logModel) initDoc(ctx context.Context, dId string, eIds []string, deleted bool, now time.Time) error { - type tableWriteLog struct { - DID string `bson:"d_id"` - Logs []Elem `bson:"logs"` - Version uint `bson:"version"` - Deleted uint `bson:"deleted"` - LastUpdate time.Time `bson:"last_update"` - } +func (l *logModel) initDoc(ctx context.Context, dId string, eIds []string, deleted bool, now time.Time) (*tableWriteLog, error) { wl := tableWriteLog{ + ID: primitive.NewObjectID(), DID: dId, Logs: make([]Elem, 0, len(eIds)), Version: FirstVersion, @@ -131,12 +146,12 @@ func (l *logModel) initDoc(ctx context.Context, dId string, eIds []string, delet }) } _, err := l.coll.InsertOne(ctx, &wl) - return err + return &wl, err } func (l *logModel) writeLogBatch(ctx context.Context, dId string, eIds []string, deleted bool, now time.Time) (*mongo.UpdateResult, error) { - if len(eIds) == 0 { - return nil, errs.ErrArgs.WrapMsg("elem id is empty", "dId", dId) + if eIds == nil { + eIds = []string{} } filter := bson.M{ "d_id": dId, @@ -195,17 +210,25 @@ func (l *logModel) writeLogBatch(ctx context.Context, dId string, eIds []string, } func (l *logModel) findDoc(ctx context.Context, dId string) (*WriteLog, error) { - res, err := mongoutil.FindOne[*WriteLog](ctx, l.coll, bson.M{"d_id": dId}, options.FindOne().SetProjection(bson.M{"logs": 0})) - if err == nil { - return res, nil - } else if errors.Is(err, mongo.ErrNoDocuments) { - return &WriteLog{}, nil + return mongoutil.FindOne[*WriteLog](ctx, l.coll, bson.M{"d_id": dId}, options.FindOne().SetProjection(bson.M{"logs": 0})) +} + +func (l *logModel) FindChangeLog(ctx context.Context, dId string, version uint, limit int) (*WriteLog, error) { + if wl, err := l.findChangeLog(ctx, dId, version, limit); err == nil { + return wl, nil + } else if !errors.Is(err, mongo.ErrNoDocuments) { + return nil, err + } + if res, err := l.initDoc(ctx, dId, nil, false, time.Now()); err == nil { + return res.WriteLog(), nil + } else if mongo.IsDuplicateKeyError(err) { + return l.findChangeLog(ctx, dId, version, limit) } else { return nil, err } } -func (l *logModel) FindChangeLog(ctx context.Context, dId string, version uint, limit int) (*WriteLog, error) { +func (l *logModel) findChangeLog(ctx context.Context, dId string, version uint, limit int) (*WriteLog, error) { if version == 0 && limit == 0 { return l.findDoc(ctx, dId) } @@ -271,7 +294,7 @@ func (l *logModel) FindChangeLog(ctx context.Context, dId string, version uint, return nil, err } if len(res) == 0 { - return &WriteLog{}, nil + return nil, mongo.ErrNoDocuments } return res[0], nil } diff --git a/pkg/common/db/dataver/result.go b/pkg/common/db/dataver/result.go index 915a618d0..ec09ffa31 100644 --- a/pkg/common/db/dataver/result.go +++ b/pkg/common/db/dataver/result.go @@ -1,18 +1,29 @@ package dataver -import "github.com/openimsdk/tools/utils/datautil" +import ( + "github.com/openimsdk/tools/utils/datautil" + "go.mongodb.org/mongo-driver/bson/primitive" +) type SyncResult struct { Version uint + VersionID string DeleteEID []string Changes []string Full bool } -func NewSyncResult(wl *WriteLog, fullIDs []string) *SyncResult { +func VersionIDStr(id primitive.ObjectID) string { + if id.IsZero() { + return "" + } + return id.String() +} + +func NewSyncResult(wl *WriteLog, fullIDs []string, versionID string) *SyncResult { var findEIDs []string var res SyncResult - if wl.Full() { + if wl.Full() || VersionIDStr(wl.ID) != versionID { res.Changes = fullIDs res.Full = true } else { diff --git a/pkg/common/db/mgo/friend.go b/pkg/common/db/mgo/friend.go index bca6873c2..37ba2d7ac 100644 --- a/pkg/common/db/mgo/friend.go +++ b/pkg/common/db/mgo/friend.go @@ -193,20 +193,6 @@ func (f *FriendMgo) FindIncrVersion(ctx context.Context, ownerUserID string, ver return f.owner.FindChangeLog(ctx, ownerUserID, version, limit) } -//func (f *FriendMgo) IncrSync(ctx context.Context, ownerUserID string, version uint, limit int) (*dataver.SyncResult[*relation.FriendModel], error) { -// res, err := f.owner.FindChangeLog(ctx, ownerUserID, version, limit) -// if err != nil { -// return nil, err -// } -// return dataver.NewSyncResult[*relation.FriendModel](res, func(eIds []string) ([]*relation.FriendModel, error) { -// if len(eIds) == 0 { -// return nil, errors.New("todo") -// } else { -// return f.FindFriends(ctx, ownerUserID, eIds) -// } -// }) -//} - func IncrVersion(dbs ...func() error) error { for _, fn := range dbs { if err := fn(); err != nil { From f6131c4ce578674777baab10f6730bf4a5194ec4 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Wed, 29 May 2024 15:13:50 +0800 Subject: [PATCH 11/59] friend incr sync --- internal/api/friend.go | 17 ++++++++++++ internal/rpc/friend/sync.go | 48 ++++++++++++++------------------- pkg/common/db/dataver/common.go | 9 ++++--- pkg/common/db/mgo/mongo_test.go | 34 +++++++++++++++++++++++ 4 files changed, 76 insertions(+), 32 deletions(-) create mode 100644 pkg/common/db/mgo/mongo_test.go diff --git a/internal/api/friend.go b/internal/api/friend.go index 3af162a53..5b2dfa0ca 100644 --- a/internal/api/friend.go +++ b/internal/api/friend.go @@ -98,3 +98,20 @@ func (o *FriendApi) UpdateFriends(c *gin.Context) { func (o *FriendApi) GetIncrementalFriends(c *gin.Context) { a2r.Call(friend.FriendClient.GetIncrementalFriends, o.Client, c) } + +//func BatchIncremental[A, B, C any,D comparable](c *gin.Context, rpc func(client C, ctx context.Context, req *A, options ...grpc.CallOption) (*B, error), getID func(req *A)D, setID func(req *A, id D)) { +// req, err := a2r.ParseRequestNotCheck[BatchIncrementalReq[A]](c) +// if err != nil { +// apiresp.GinError(c, err) +// return +// } +// if len(req.List) == 0 { +// apiresp.GinError(c, errs.ErrArgs.WrapMsg("empty versions list")) +// return +// } +//} +// +//type BatchIncrementalReq[A any] struct { +// UserID string `json:"user_id"` +// List []*A `json:"list"` +//} diff --git a/internal/rpc/friend/sync.go b/internal/rpc/friend/sync.go index 9d2a53b58..a2f49b283 100644 --- a/internal/rpc/friend/sync.go +++ b/internal/rpc/friend/sync.go @@ -6,7 +6,6 @@ import ( "encoding/binary" "encoding/json" "github.com/openimsdk/open-im-server/v3/pkg/authverify" - "github.com/openimsdk/open-im-server/v3/pkg/common/db/dataver" "github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation" pbfriend "github.com/openimsdk/protocol/friend" ) @@ -34,38 +33,31 @@ func (s *friendServer) GetIncrementalFriends(ctx context.Context, req *pbfriend. if err != nil { return nil, err } - sortUserIDs, err := s.friendDatabase.FindSortFriendUserIDs(ctx, req.UserID) - if err != nil { - return nil, err - } - if len(sortUserIDs) == 0 { - return &pbfriend.GetIncrementalFriendsResp{ - Version: uint64(incrVer.Version), - VersionID: dataver.VersionIDStr(incrVer.ID), - Full: true, - SyncCount: uint32(s.config.RpcConfig.FriendSyncCount), - }, nil - } - var changes []*relation.FriendModel - res := dataver.NewSyncResult(incrVer, sortUserIDs, req.VersionID) - if len(res.Changes) > 0 { - changes, err = s.friendDatabase.FindFriendsWithError(ctx, req.UserID, res.Changes) + var ( + deleteUserIDs []string + changeUserIDs []string + ) + if incrVer.Full() { + changeUserIDs, err = s.friendDatabase.FindSortFriendUserIDs(ctx, req.UserID) if err != nil { return nil, err } + } else { + deleteUserIDs, changeUserIDs = incrVer.DeleteAndChangeIDs() } - calcHash := s.sortFriendUserIDsHash(sortUserIDs) - if calcHash == req.IdHash { - sortUserIDs = nil + var friends []*relation.FriendModel + if len(changeUserIDs) > 0 { + friends, err = s.friendDatabase.FindFriendsWithError(ctx, req.UserID, changeUserIDs) + if err != nil { + return nil, err + } } return &pbfriend.GetIncrementalFriendsResp{ - Version: uint64(res.Version), - VersionID: res.VersionID, - Full: res.Full, - SyncCount: uint32(s.config.RpcConfig.FriendSyncCount), - SortUserIdHash: calcHash, - SortUserIds: sortUserIDs, - DeleteUserIds: res.DeleteEID, - Changes: friendsDB2PB(changes), + Version: uint64(incrVer.Version), + VersionID: incrVer.ID.String(), + Full: incrVer.Full(), + SyncCount: uint32(s.config.RpcConfig.FriendSyncCount), + DeleteUserIds: deleteUserIDs, + Changes: friendsDB2PB(friends), }, nil } diff --git a/pkg/common/db/dataver/common.go b/pkg/common/db/dataver/common.go index 3ff9906e5..e29ab2b96 100644 --- a/pkg/common/db/dataver/common.go +++ b/pkg/common/db/dataver/common.go @@ -35,14 +35,15 @@ func (w *WriteLog) Full() bool { return len(w.Logs) != w.LogLen } -func (w *WriteLog) DeleteEId() []string { - var eIds []string +func (w *WriteLog) DeleteAndChangeIDs() (delIds []string, changeIds []string) { for _, l := range w.Logs { if l.Deleted { - eIds = append(eIds, l.EID) + delIds = append(delIds, l.EID) + } else { + changeIds = append(changeIds, l.EID) } } - return eIds + return } type Elem struct { diff --git a/pkg/common/db/mgo/mongo_test.go b/pkg/common/db/mgo/mongo_test.go new file mode 100644 index 000000000..294dd1317 --- /dev/null +++ b/pkg/common/db/mgo/mongo_test.go @@ -0,0 +1,34 @@ +package mgo + +import ( + "context" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" + "testing" + "time" +) + +func Result[V any](val V, err error) V { + if err != nil { + panic(err) + } + return val +} + +func Check(err error) { + if err != nil { + panic(err) + } +} + +func TestName(t *testing.T) { + cli := Result(mongo.Connect(context.Background(), options.Client().ApplyURI("mongodb://openIM:openIM123@172.16.8.48:37017/openim_v3?maxPoolSize=100").SetConnectTimeout(5*time.Second))) + conv := Result(NewConversationMongo(cli.Database("openim_v3"))) + num, err := conv.GetAllConversationIDsNumber(context.Background()) + if err != nil { + panic(err) + } + t.Log(num) + ids := Result(conv.GetAllConversationIDs(context.Background())) + t.Log(ids) +} From 0d57f28a3a523e9d819a661fbbc432b2921c9adc Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Wed, 29 May 2024 15:20:17 +0800 Subject: [PATCH 12/59] friend incr sync --- go.mod | 8 ++++---- go.sum | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index daf6cc0d2..0b2f3cb3b 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/lestrrat-go/file-rotatelogs v2.4.0+incompatible // indirect github.com/mitchellh/mapstructure v1.5.0 - github.com/openimsdk/protocol v0.0.69-alpha.1 + github.com/openimsdk/protocol v0.0.69-alpha.2 github.com/openimsdk/tools v0.0.49-alpha.23 github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.18.0 @@ -178,6 +178,6 @@ require ( gopkg.in/ini.v1 v1.67.0 // indirect ) -replace ( - github.com/openimsdk/protocol => /Users/chao/Desktop/project/protocol -) +//replace ( +// github.com/openimsdk/protocol => /Users/chao/Desktop/project/protocol +//) diff --git a/go.sum b/go.sum index 908c15c87..c0bd78645 100644 --- a/go.sum +++ b/go.sum @@ -286,8 +286,8 @@ github.com/onsi/gomega v1.18.1 h1:M1GfJqGRrBrrGGsbxzV5dqM2U2ApXefZCQpkukxYRLE= github.com/onsi/gomega v1.18.1/go.mod h1:0q+aL8jAiMXy9hbwj2mr5GziHiwhAIQpFmmtT5hitRs= github.com/openimsdk/gomake v0.0.13 h1:xLDe/moqgWpRoptHzI4packAWzs4C16b+sVY+txNJp0= github.com/openimsdk/gomake v0.0.13/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= -github.com/openimsdk/protocol v0.0.69-alpha.1 h1:l/PN8mwmh5O7PRoaiMZvey+hUxyuNNnMgPGKOfEMOKs= -github.com/openimsdk/protocol v0.0.69-alpha.1/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= +github.com/openimsdk/protocol v0.0.69-alpha.2 h1:XNc3pmAXyW+PMo7tghr2O+uydYck1hogppHDW3+Y+3k= +github.com/openimsdk/protocol v0.0.69-alpha.2/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= github.com/openimsdk/tools v0.0.49-alpha.23 h1:/KkJ7vfx8FAoJhq3veH9PWnxbSkEf+dTSshvDrHBR38= github.com/openimsdk/tools v0.0.49-alpha.23/go.mod h1:g7mkHXYUPi0/8aAX8VPMHpnb3hqdV69Jph+bXOGvvNM= github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= From 9ba22f30aeaffb5d5961b3cc8cf8b0304305b925 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Thu, 30 May 2024 11:19:32 +0800 Subject: [PATCH 13/59] friend incr sync --- cmd/openim-rpc/openim-rpc-friend/main.go | 4 + go.mod | 6 +- internal/rpc/friend/friend.go | 3 + internal/rpc/friend/sync.go | 50 ++++++++++- internal/rpc/user/user.go | 108 ++++++++++++++++------- pkg/common/cmd/msg_gateway_test.go | 7 ++ pkg/common/convert/user.go | 28 +++--- pkg/common/db/cache/friend.go | 13 +++ pkg/common/db/controller/friend.go | 21 +++++ pkg/common/db/dataver/common.go | 13 +-- pkg/common/db/mgo/friend.go | 39 +++++++- pkg/common/db/table/relation/friend.go | 6 ++ 12 files changed, 239 insertions(+), 59 deletions(-) diff --git a/cmd/openim-rpc/openim-rpc-friend/main.go b/cmd/openim-rpc/openim-rpc-friend/main.go index 745c40553..4589fb30d 100644 --- a/cmd/openim-rpc/openim-rpc-friend/main.go +++ b/cmd/openim-rpc/openim-rpc-friend/main.go @@ -17,9 +17,13 @@ package main import ( "github.com/openimsdk/open-im-server/v3/pkg/common/cmd" "github.com/openimsdk/tools/system/program" + "os" ) func main() { + if len(os.Args) == 1 { + os.Args = []string{os.Args[0], "-i", "0", "-c", "/Users/chao/Desktop/project/open-im-server/config"} + } if err := cmd.NewFriendRpcCmd().Exec(); err != nil { program.ExitWithError(err) } diff --git a/go.mod b/go.mod index 0b2f3cb3b..0ae542cd2 100644 --- a/go.mod +++ b/go.mod @@ -178,6 +178,6 @@ require ( gopkg.in/ini.v1 v1.67.0 // indirect ) -//replace ( -// github.com/openimsdk/protocol => /Users/chao/Desktop/project/protocol -//) +replace ( + github.com/openimsdk/protocol => /Users/chao/Desktop/project/protocol +) diff --git a/internal/rpc/friend/friend.go b/internal/rpc/friend/friend.go index b49490f26..31965adb9 100644 --- a/internal/rpc/friend/friend.go +++ b/internal/rpc/friend/friend.go @@ -62,6 +62,9 @@ type Config struct { } func Start(ctx context.Context, config *Config, client discovery.SvcDiscoveryRegistry, server *grpc.Server) error { + if config.RpcConfig.FriendSyncCount < 1 { + config.RpcConfig.FriendSyncCount = constant.MaxSyncPullNumber + } mgocli, err := mongoutil.NewMongoDB(ctx, config.MongodbConfig.Build()) if err != nil { return err diff --git a/internal/rpc/friend/sync.go b/internal/rpc/friend/sync.go index a2f49b283..dd6e49250 100644 --- a/internal/rpc/friend/sync.go +++ b/internal/rpc/friend/sync.go @@ -8,11 +8,55 @@ import ( "github.com/openimsdk/open-im-server/v3/pkg/authverify" "github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation" pbfriend "github.com/openimsdk/protocol/friend" + "github.com/openimsdk/tools/errs" ) +func (s *friendServer) NotificationUserInfoUpdate(ctx context.Context, req *pbfriend.NotificationUserInfoUpdateReq) (*pbfriend.NotificationUserInfoUpdateResp, error) { + if req.NewUserInfo == nil { + var err error + req.NewUserInfo, err = s.userRpcClient.GetUserInfo(ctx, req.UserID) + if err != nil { + return nil, err + } + } + if req.UserID != req.NewUserInfo.UserID { + return nil, errs.ErrArgs.WrapMsg("req.UserID != req.NewUserInfo.UserID") + } + userIDs, err := s.friendDatabase.FindFriendUserID(ctx, req.UserID) + if err != nil { + return nil, err + } + if len(userIDs) > 0 { + if err := s.friendDatabase.UpdateFriendUserInfo(ctx, req.UserID, userIDs, req.NewUserInfo.Nickname, req.NewUserInfo.FaceURL); err != nil { + return nil, err + } + s.notificationSender.FriendsInfoUpdateNotification(ctx, req.UserID, userIDs) + } + return &pbfriend.NotificationUserInfoUpdateResp{}, nil +} + func (s *friendServer) SearchFriends(ctx context.Context, req *pbfriend.SearchFriendsReq) (*pbfriend.SearchFriendsResp, error) { - //TODO implement me - panic("implement me") + if err := s.userRpcClient.Access(ctx, req.UserID); err != nil { + return nil, err + } + if req.Keyword == "" { + total, friends, err := s.friendDatabase.PageOwnerFriends(ctx, req.UserID, req.Pagination) + if err != nil { + return nil, err + } + return &pbfriend.SearchFriendsResp{ + Total: total, + Friends: friendsDB2PB(friends), + }, nil + } + total, friends, err := s.friendDatabase.SearchFriend(ctx, req.UserID, req.Keyword, req.Pagination) + if err != nil { + return nil, err + } + return &pbfriend.SearchFriendsResp{ + Total: total, + Friends: friendsDB2PB(friends), + }, nil } func (s *friendServer) sortFriendUserIDsHash(userIDs []string) uint64 { @@ -54,7 +98,7 @@ func (s *friendServer) GetIncrementalFriends(ctx context.Context, req *pbfriend. } return &pbfriend.GetIncrementalFriendsResp{ Version: uint64(incrVer.Version), - VersionID: incrVer.ID.String(), + VersionID: incrVer.ID.Hex(), Full: incrVer.Full(), SyncCount: uint32(s.config.RpcConfig.FriendSyncCount), DeleteUserIds: deleteUserIDs, diff --git a/internal/rpc/user/user.go b/internal/rpc/user/user.go index 2296ee62f..92f897ea3 100644 --- a/internal/rpc/user/user.go +++ b/internal/rpc/user/user.go @@ -16,12 +16,16 @@ package user import ( "context" + "errors" "github.com/openimsdk/open-im-server/v3/internal/rpc/friend" "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/open-im-server/v3/pkg/common/webhook" + friendpb "github.com/openimsdk/protocol/friend" + "github.com/openimsdk/protocol/group" "github.com/openimsdk/tools/db/redisutil" "math/rand" "strings" + "sync" "time" "github.com/openimsdk/open-im-server/v3/pkg/authverify" @@ -56,11 +60,6 @@ type userServer struct { webhookClient *webhook.Client } -func (s *userServer) SearchUser(ctx context.Context, req *pbuser.SearchUserReq) (*pbuser.SearchUserResp, error) { - //TODO implement me - panic("implement me") -} - type Config struct { RpcConfig config.User RedisConfig config.Redis @@ -136,26 +135,29 @@ func (s *userServer) UpdateUserInfo(ctx context.Context, req *pbuser.UpdateUserI if err := s.webhookBeforeUpdateUserInfo(ctx, &s.config.WebhooksConfig.BeforeUpdateUserInfo, req); err != nil { return nil, err } - data := convert.UserPb2DBMap(req.UserInfo) - if err := s.db.UpdateByMap(ctx, req.UserInfo.UserID, data); err != nil { - return nil, err - } - s.friendNotificationSender.UserInfoUpdatedNotification(ctx, req.UserInfo.UserID) - friends, err := s.friendRpcClient.GetFriendIDs(ctx, req.UserInfo.UserID) + oldUser, err := s.db.GetUserByID(ctx, req.UserInfo.UserID) if err != nil { return nil, err } - if req.UserInfo.Nickname != "" || req.UserInfo.FaceURL != "" { - if err = s.groupRpcClient.NotificationUserInfoUpdate(ctx, req.UserInfo.UserID); err != nil { - return nil, err - } - } - for _, friendID := range friends { - s.friendNotificationSender.FriendInfoUpdatedNotification(ctx, req.UserInfo.UserID, friendID) + if err := s.db.UpdateByMap(ctx, req.UserInfo.UserID, data); err != nil { + return nil, err } + //s.friendNotificationSender.UserInfoUpdatedNotification(ctx, req.UserInfo.UserID) + //friends, err := s.friendRpcClient.GetFriendIDs(ctx, req.UserInfo.UserID) + //if err != nil { + // return nil, err + //} + //if req.UserInfo.Nickname != "" || req.UserInfo.FaceURL != "" { + // if err = s.NotificationUserInfoUpdate(ctx, req.UserInfo.UserID,oldUser); err != nil { + // return nil, err + // } + //} + //for _, friendID := range friends { + // s.friendNotificationSender.FriendInfoUpdatedNotification(ctx, req.UserInfo.UserID, friendID) + //} s.webhookAfterUpdateUserInfo(ctx, &s.config.WebhooksConfig.AfterUpdateUserInfo, req) - if err = s.groupRpcClient.NotificationUserInfoUpdate(ctx, req.UserInfo.UserID); err != nil { + if err = s.NotificationUserInfoUpdate(ctx, req.UserInfo.UserID, oldUser); err != nil { return nil, err } return resp, nil @@ -170,24 +172,28 @@ func (s *userServer) UpdateUserInfoEx(ctx context.Context, req *pbuser.UpdateUse return nil, err } data := convert.UserPb2DBMapEx(req.UserInfo) - if err = s.db.UpdateByMap(ctx, req.UserInfo.UserID, data); err != nil { - return nil, err - } - s.friendNotificationSender.UserInfoUpdatedNotification(ctx, req.UserInfo.UserID) - friends, err := s.friendRpcClient.GetFriendIDs(ctx, req.UserInfo.UserID) + oldUser, err := s.db.GetUserByID(ctx, req.UserInfo.UserID) if err != nil { return nil, err } - if req.UserInfo.Nickname != nil || req.UserInfo.FaceURL != nil { - if err := s.groupRpcClient.NotificationUserInfoUpdate(ctx, req.UserInfo.UserID); err != nil { - return nil, err - } - } - for _, friendID := range friends { - s.friendNotificationSender.FriendInfoUpdatedNotification(ctx, req.UserInfo.UserID, friendID) + if err = s.db.UpdateByMap(ctx, req.UserInfo.UserID, data); err != nil { + return nil, err } + //s.friendNotificationSender.UserInfoUpdatedNotification(ctx, req.UserInfo.UserID) + //friends, err := s.friendRpcClient.GetFriendIDs(ctx, req.UserInfo.UserID) + //if err != nil { + // return nil, err + //} + //if req.UserInfo.Nickname != nil || req.UserInfo.FaceURL != nil { + // if err := s.NotificationUserInfoUpdate(ctx, req.UserInfo.UserID); err != nil { + // return nil, err + // } + //} + //for _, friendID := range friends { + // s.friendNotificationSender.FriendInfoUpdatedNotification(ctx, req.UserInfo.UserID, friendID) + //} s.webhookAfterUpdateUserInfoEx(ctx, &s.config.WebhooksConfig.AfterUpdateUserInfoEx, req) - if err := s.groupRpcClient.NotificationUserInfoUpdate(ctx, req.UserInfo.UserID); err != nil { + if err := s.NotificationUserInfoUpdate(ctx, req.UserInfo.UserID, oldUser); err != nil { return nil, err } return resp, nil @@ -688,3 +694,41 @@ func (s *userServer) userModelToResp(users []*relation.UserModel, pagination pag return &pbuser.SearchNotificationAccountResp{Total: total, NotificationAccounts: notificationAccounts} } + +func (s *userServer) NotificationUserInfoUpdate(ctx context.Context, userID string, oldUser *relation.UserModel) error { + user, err := s.db.GetUserByID(ctx, userID) + if err != nil { + return err + } + if *user == *oldUser { + return nil + } + s.friendNotificationSender.UserInfoUpdatedNotification(ctx, userID) + if user.Nickname == oldUser.Nickname && user.FaceURL == oldUser.FaceURL { + return nil + } + oldUserInfo := convert.UserDB2Pb(oldUser) + newUserInfo := convert.UserDB2Pb(user) + var wg sync.WaitGroup + var es [2]error + wg.Add(len(es)) + go func() { + defer wg.Done() + _, es[0] = s.groupRpcClient.Client.NotificationUserInfoUpdate(ctx, &group.NotificationUserInfoUpdateReq{ + UserID: userID, + OldUserInfo: oldUserInfo, + NewUserInfo: newUserInfo, + }) + }() + + go func() { + defer wg.Done() + _, es[1] = s.friendRpcClient.Client.NotificationUserInfoUpdate(ctx, &friendpb.NotificationUserInfoUpdateReq{ + UserID: userID, + OldUserInfo: oldUserInfo, + NewUserInfo: newUserInfo, + }) + }() + wg.Wait() + return errors.Join(es[:]...) +} diff --git a/pkg/common/cmd/msg_gateway_test.go b/pkg/common/cmd/msg_gateway_test.go index d820627b5..2b68a3e3a 100644 --- a/pkg/common/cmd/msg_gateway_test.go +++ b/pkg/common/cmd/msg_gateway_test.go @@ -19,6 +19,7 @@ import ( "github.com/openimsdk/tools/apiresp" "github.com/openimsdk/tools/utils/jsonutil" "github.com/stretchr/testify/mock" + "go.mongodb.org/mongo-driver/bson/primitive" "math" "testing" ) @@ -59,3 +60,9 @@ func TestName(t *testing.T) { t.Logf("%+v\n", rReso) } + +func TestName1(t *testing.T) { + t.Log(primitive.NewObjectID().String()) + t.Log(primitive.NewObjectID().Hex()) + +} diff --git a/pkg/common/convert/user.go b/pkg/common/convert/user.go index a9378e1a0..c7c90edb3 100644 --- a/pkg/common/convert/user.go +++ b/pkg/common/convert/user.go @@ -15,27 +15,27 @@ package convert import ( + "github.com/openimsdk/tools/utils/datautil" "time" relationtb "github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation" "github.com/openimsdk/protocol/sdkws" ) -func UsersDB2Pb(users []*relationtb.UserModel) []*sdkws.UserInfo { - result := make([]*sdkws.UserInfo, 0, len(users)) - for _, user := range users { - userPb := &sdkws.UserInfo{ - UserID: user.UserID, - Nickname: user.Nickname, - FaceURL: user.FaceURL, - Ex: user.Ex, - CreateTime: user.CreateTime.UnixMilli(), - AppMangerLevel: user.AppMangerLevel, - GlobalRecvMsgOpt: user.GlobalRecvMsgOpt, - } - result = append(result, userPb) +func UserDB2Pb(user *relationtb.UserModel) *sdkws.UserInfo { + return &sdkws.UserInfo{ + UserID: user.UserID, + Nickname: user.Nickname, + FaceURL: user.FaceURL, + Ex: user.Ex, + CreateTime: user.CreateTime.UnixMilli(), + AppMangerLevel: user.AppMangerLevel, + GlobalRecvMsgOpt: user.GlobalRecvMsgOpt, } - return result +} + +func UsersDB2Pb(users []*relationtb.UserModel) []*sdkws.UserInfo { + return datautil.Slice(users, UserDB2Pb) } func UserPb2DB(user *sdkws.UserInfo) *relationtb.UserModel { diff --git a/pkg/common/db/cache/friend.go b/pkg/common/db/cache/friend.go index 4a51a8c17..eb984d8a4 100644 --- a/pkg/common/db/cache/friend.go +++ b/pkg/common/db/cache/friend.go @@ -52,6 +52,8 @@ type FriendCache interface { // Delete friends when friends' info changed DelFriends(ownerUserID string, friendUserIDs []string) FriendCache + DelOwner(friendUserID string, ownerUserIDs []string) FriendCache + FindSortFriendUserIDs(ctx context.Context, ownerUserID string) ([]string, error) FindFriendIncrVersion(ctx context.Context, ownerUserID string, version uint, limit int) (*dataver.WriteLog, error) @@ -196,6 +198,17 @@ func (f *FriendCacheRedis) DelFriends(ownerUserID string, friendUserIDs []string return newFriendCache } +func (f *FriendCacheRedis) DelOwner(friendUserID string, ownerUserIDs []string) FriendCache { + newFriendCache := f.NewCache() + + for _, ownerUserID := range ownerUserIDs { + key := f.getFriendKey(ownerUserID, friendUserID) + newFriendCache.AddKeys(key) // Assuming AddKeys marks the keys for deletion + } + + return newFriendCache +} + func (f *FriendCacheRedis) FindSortFriendUserIDs(ctx context.Context, ownerUserID string) ([]string, error) { return getCache(ctx, f.rcClient, f.getFriendSyncSortUserIDsKey(ownerUserID), f.expireTime, func(ctx context.Context) ([]string, error) { return f.friendDB.FindOwnerFriendUserIds(ctx, ownerUserID, f.syncCount) diff --git a/pkg/common/db/controller/friend.go b/pkg/common/db/controller/friend.go index 49c0cb990..0eef2bee3 100644 --- a/pkg/common/db/controller/friend.go +++ b/pkg/common/db/controller/friend.go @@ -80,6 +80,12 @@ type FriendDatabase interface { FindSortFriendUserIDs(ctx context.Context, ownerUserID string) ([]string, error) FindFriendIncrVersion(ctx context.Context, ownerUserID string, version uint, limit int) (*dataver.WriteLog, error) + + FindFriendUserID(ctx context.Context, friendUserID string) ([]string, error) + + UpdateFriendUserInfo(ctx context.Context, friendUserID string, ownerUserID []string, nickname string, faceURL string) error + + SearchFriend(ctx context.Context, ownerUserID, keyword string, pagination pagination.Pagination) (int64, []*relation.FriendModel, error) } type friendDatabase struct { @@ -357,3 +363,18 @@ func (f *friendDatabase) FindSortFriendUserIDs(ctx context.Context, ownerUserID func (f *friendDatabase) FindFriendIncrVersion(ctx context.Context, ownerUserID string, version uint, limit int) (*dataver.WriteLog, error) { return f.cache.FindFriendIncrVersion(ctx, ownerUserID, version, limit) } + +func (f *friendDatabase) FindFriendUserID(ctx context.Context, friendUserID string) ([]string, error) { + return f.friend.FindFriendUserID(ctx, friendUserID) +} + +func (f *friendDatabase) UpdateFriendUserInfo(ctx context.Context, friendUserID string, ownerUserIDs []string, nickname string, faceURL string) error { + if err := f.friend.UpdateFriendUserInfo(ctx, friendUserID, nickname, faceURL); err != nil { + return err + } + return f.cache.DelOwner(friendUserID, ownerUserIDs).ExecDel(ctx) +} + +func (f *friendDatabase) SearchFriend(ctx context.Context, ownerUserID, keyword string, pagination pagination.Pagination) (int64, []*relation.FriendModel, error) { + return f.friend.SearchFriend(ctx, ownerUserID, keyword, pagination) +} diff --git a/pkg/common/db/dataver/common.go b/pkg/common/db/dataver/common.go index e29ab2b96..5ade1fd58 100644 --- a/pkg/common/db/dataver/common.go +++ b/pkg/common/db/dataver/common.go @@ -26,13 +26,11 @@ type WriteLog struct { Deleted uint `bson:"deleted"` LastUpdate time.Time `bson:"last_update"` LogLen int `bson:"log_len"` + queryDoc bool `bson:"-"` } func (w *WriteLog) Full() bool { - if w.Version == 0 { - return true - } - return len(w.Logs) != w.LogLen + return w.queryDoc || w.Version == 0 || len(w.Logs) != w.LogLen } func (w *WriteLog) DeleteAndChangeIDs() (delIds []string, changeIds []string) { @@ -211,7 +209,12 @@ func (l *logModel) writeLogBatch(ctx context.Context, dId string, eIds []string, } func (l *logModel) findDoc(ctx context.Context, dId string) (*WriteLog, error) { - return mongoutil.FindOne[*WriteLog](ctx, l.coll, bson.M{"d_id": dId}, options.FindOne().SetProjection(bson.M{"logs": 0})) + res, err := mongoutil.FindOne[*WriteLog](ctx, l.coll, bson.M{"d_id": dId}, options.FindOne().SetProjection(bson.M{"logs": 0})) + if err != nil { + return nil, err + } + res.queryDoc = true + return res, nil } func (l *logModel) FindChangeLog(ctx context.Context, dId string, version uint, limit int) (*WriteLog, error) { diff --git a/pkg/common/db/mgo/friend.go b/pkg/common/db/mgo/friend.go index 37ba2d7ac..cf84acb1c 100644 --- a/pkg/common/db/mgo/friend.go +++ b/pkg/common/db/mgo/friend.go @@ -145,13 +145,20 @@ func (f *FriendMgo) FindReversalFriends(ctx context.Context, friendUserID string // FindOwnerFriends retrieves a paginated list of friends for a given owner. func (f *FriendMgo) FindOwnerFriends(ctx context.Context, ownerUserID string, pagination pagination.Pagination) (int64, []*relation.FriendModel, error) { filter := bson.M{"owner_user_id": ownerUserID} - opt := options.Find().SetSort(bson.A{bson.M{"friend_nickname": 1}, bson.M{"create_time": 1}}) + opt := options.Find().SetSort(bson.D{{"friend_nickname", 1}, {"create_time", 1}}) return mongoutil.FindPage[*relation.FriendModel](ctx, f.coll, filter, pagination, opt) } func (f *FriendMgo) FindOwnerFriendUserIds(ctx context.Context, ownerUserID string, limit int) ([]string, error) { filter := bson.M{"owner_user_id": ownerUserID} - opt := options.Find().SetProjection(bson.M{"_id": 0, "friend_user_id": 1}).SetSort(bson.A{bson.M{"friend_nickname": 1}, bson.M{"create_time": 1}}).SetLimit(int64(limit)) + opt := options.Find().SetProjection(bson.M{"_id": 0, "friend_user_id": 1}).SetSort(bson.D{{"friend_nickname", 1}, {"create_time", 1}}).SetLimit(int64(limit)) + //res, err := mongoutil.Find[string](ctx, f.coll, filter, opt) + //if err != nil { + // errMsg := err.Error() + // _ = errMsg + // return nil, err + //} + //return res, nil return mongoutil.Find[string](ctx, f.coll, filter, opt) } @@ -193,6 +200,34 @@ func (f *FriendMgo) FindIncrVersion(ctx context.Context, ownerUserID string, ver return f.owner.FindChangeLog(ctx, ownerUserID, version, limit) } +func (f *FriendMgo) FindFriendUserID(ctx context.Context, friendUserID string) ([]string, error) { + filter := bson.M{ + "friend_user_id": friendUserID, + } + return mongoutil.Find[string](ctx, f.coll, filter, options.Find().SetProjection(bson.M{"_id": 0, "owner_user_id": 1})) +} + +func (f *FriendMgo) UpdateFriendUserInfo(ctx context.Context, friendUserID string, nickname string, faceURL string) error { + filter := bson.M{ + "friend_user_id": friendUserID, + } + _, err := mongoutil.UpdateMany(ctx, f.coll, filter, bson.M{"$set": bson.M{"nickname": nickname, "face_url": faceURL}}) + return err +} + +func (f *FriendMgo) SearchFriend(ctx context.Context, ownerUserID, keyword string, pagination pagination.Pagination) (int64, []*relation.FriendModel, error) { + //where := bson.M{ + // "owner_user_id": ownerUserID, + // "$or": []bson.M{ + // {"remark": bson.M{"$regex": keyword, "$options": "i"}}, + // {"friend_user_id": bson.M{"$regex": keyword, "$options": "i"}}, + // {"nickname": bson.M{"$regex": keyword, "$options": "i"}}, + // }, + //} + //return f.aggregatePagination(ctx, where, pagination) + panic("todo") +} + func IncrVersion(dbs ...func() error) error { for _, fn := range dbs { if err := fn(); err != nil { diff --git a/pkg/common/db/table/relation/friend.go b/pkg/common/db/table/relation/friend.go index a28b2278d..4970a0420 100644 --- a/pkg/common/db/table/relation/friend.go +++ b/pkg/common/db/table/relation/friend.go @@ -66,4 +66,10 @@ type FriendModelInterface interface { UpdateFriends(ctx context.Context, ownerUserID string, friendUserIDs []string, val map[string]any) (err error) FindIncrVersion(ctx context.Context, ownerUserID string, version uint, limit int) (*dataver.WriteLog, error) + + FindFriendUserID(ctx context.Context, friendUserID string) ([]string, error) + + UpdateFriendUserInfo(ctx context.Context, friendUserID string, nickname string, faceURL string) error + + SearchFriend(ctx context.Context, ownerUserID, keyword string, pagination pagination.Pagination) (int64, []*FriendModel, error) } From eb362daaf28e37502f1bd595a2c4cfd30bff5a33 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Thu, 30 May 2024 15:07:42 +0800 Subject: [PATCH 14/59] mage --- internal/rpc/friend/convert.go | 6 ++-- internal/rpc/friend/sync.go | 4 +-- internal/rpc/user/user.go | 2 +- pkg/common/convert/user.go | 2 +- pkg/common/storage/cache/friend.go | 9 ++++++ pkg/common/storage/cache/redis/friend.go | 31 ++----------------- pkg/common/storage/controller/friend.go | 8 ++--- pkg/common/storage/database/friend.go | 19 ++---------- pkg/common/storage/database/mgo/friend.go | 2 +- pkg/common/storage/database/mgo/mongo_test.go | 0 pkg/common/storage/model/friend.go | 2 ++ 11 files changed, 29 insertions(+), 56 deletions(-) delete mode 100644 pkg/common/storage/database/mgo/mongo_test.go diff --git a/internal/rpc/friend/convert.go b/internal/rpc/friend/convert.go index aa6895284..0730df529 100644 --- a/internal/rpc/friend/convert.go +++ b/internal/rpc/friend/convert.go @@ -1,12 +1,12 @@ package friend import ( - relationtb "github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" "github.com/openimsdk/protocol/friend" "github.com/openimsdk/tools/utils/datautil" ) -func friendDB2PB(db *relationtb.FriendModel) *friend.FriendInfo { +func friendDB2PB(db *model.Friend) *friend.FriendInfo { return &friend.FriendInfo{ OwnerUserID: db.OwnerUserID, FriendUserID: db.FriendUserID, @@ -21,6 +21,6 @@ func friendDB2PB(db *relationtb.FriendModel) *friend.FriendInfo { } } -func friendsDB2PB(db []*relationtb.FriendModel) []*friend.FriendInfo { +func friendsDB2PB(db []*model.Friend) []*friend.FriendInfo { return datautil.Slice(db, friendDB2PB) } diff --git a/internal/rpc/friend/sync.go b/internal/rpc/friend/sync.go index dd6e49250..bfa7e3a57 100644 --- a/internal/rpc/friend/sync.go +++ b/internal/rpc/friend/sync.go @@ -6,7 +6,7 @@ import ( "encoding/binary" "encoding/json" "github.com/openimsdk/open-im-server/v3/pkg/authverify" - "github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" pbfriend "github.com/openimsdk/protocol/friend" "github.com/openimsdk/tools/errs" ) @@ -89,7 +89,7 @@ func (s *friendServer) GetIncrementalFriends(ctx context.Context, req *pbfriend. } else { deleteUserIDs, changeUserIDs = incrVer.DeleteAndChangeIDs() } - var friends []*relation.FriendModel + var friends []*model.Friend if len(changeUserIDs) > 0 { friends, err = s.friendDatabase.FindFriendsWithError(ctx, req.UserID, changeUserIDs) if err != nil { diff --git a/internal/rpc/user/user.go b/internal/rpc/user/user.go index 14ce09ed5..504c08a2c 100644 --- a/internal/rpc/user/user.go +++ b/internal/rpc/user/user.go @@ -695,7 +695,7 @@ func (s *userServer) userModelToResp(users []*tablerelation.User, pagination pag return &pbuser.SearchNotificationAccountResp{Total: total, NotificationAccounts: notificationAccounts} } -func (s *userServer) NotificationUserInfoUpdate(ctx context.Context, userID string, oldUser *relation.UserModel) error { +func (s *userServer) NotificationUserInfoUpdate(ctx context.Context, userID string, oldUser *tablerelation.User) error { user, err := s.db.GetUserByID(ctx, userID) if err != nil { return err diff --git a/pkg/common/convert/user.go b/pkg/common/convert/user.go index 5595e8fe5..d824fa68e 100644 --- a/pkg/common/convert/user.go +++ b/pkg/common/convert/user.go @@ -34,7 +34,7 @@ func UserDB2Pb(user *relationtb.User) *sdkws.UserInfo { } } -func UsersDB2Pb(users []*relationtb.UserModel) []*sdkws.UserInfo { +func UsersDB2Pb(users []*relationtb.User) []*sdkws.UserInfo { return datautil.Slice(users, UserDB2Pb) } diff --git a/pkg/common/storage/cache/friend.go b/pkg/common/storage/cache/friend.go index acff829f8..9dec3d5ab 100644 --- a/pkg/common/storage/cache/friend.go +++ b/pkg/common/storage/cache/friend.go @@ -16,6 +16,7 @@ package cache import ( "context" + "github.com/openimsdk/open-im-server/v3/pkg/common/db/dataver" relationtb "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" ) @@ -32,4 +33,12 @@ type FriendCache interface { DelFriend(ownerUserID, friendUserID string) FriendCache // Delete friends when friends' info changed DelFriends(ownerUserID string, friendUserIDs []string) FriendCache + + DelOwner(friendUserID string, ownerUserIDs []string) FriendCache + + DelSortFriendUserIDs(ownerUserIDs ...string) FriendCache + + FindSortFriendUserIDs(ctx context.Context, ownerUserID string) ([]string, error) + + FindFriendIncrVersion(ctx context.Context, ownerUserID string, version uint, limit int) (*dataver.WriteLog, error) } diff --git a/pkg/common/storage/cache/redis/friend.go b/pkg/common/storage/cache/redis/friend.go index 7b93c74ac..d6754d7bf 100644 --- a/pkg/common/storage/cache/redis/friend.go +++ b/pkg/common/storage/cache/redis/friend.go @@ -28,37 +28,12 @@ import ( "github.com/openimsdk/tools/log" "github.com/openimsdk/tools/utils/datautil" "github.com/redis/go-redis/v9" - "time" ) const ( friendExpireTime = time.Second * 60 * 60 * 12 ) -//// FriendCache is an interface for caching friend-related data. -//type FriendCache interface { -// metaCache -// NewCache() FriendCache -// GetFriendIDs(ctx context.Context, ownerUserID string) (friendIDs []string, err error) -// // Called when friendID list changed -// DelFriendIDs(ownerUserID ...string) FriendCache -// -// DelSortFriendUserIDs(ownerUserIDs ...string) FriendCache -// -// // Get single friendInfo from the cache -// GetFriend(ctx context.Context, ownerUserID, friendUserID string) (friend *relationtb.FriendModel, err error) -// // Delete friend when friend info changed -// DelFriend(ownerUserID, friendUserID string) FriendCache -// // Delete friends when friends' info changed -// DelFriends(ownerUserID string, friendUserIDs []string) FriendCache -// -// DelOwner(friendUserID string, ownerUserIDs []string) FriendCache -// -// FindSortFriendUserIDs(ctx context.Context, ownerUserID string) ([]string, error) -// -// FindFriendIncrVersion(ctx context.Context, ownerUserID string, version uint, limit int) (*dataver.WriteLog, error) -//} - // FriendCacheRedis is an implementation of the FriendCache interface using Redis. type FriendCacheRedis struct { cache.BatchDeleter @@ -129,8 +104,8 @@ func (f *FriendCacheRedis) DelFriendIDs(ownerUserIDs ...string) cache.FriendCach return newFriendCache } -func (f *FriendCacheRedis) DelSortFriendUserIDs(ownerUserIDs ...string) FriendCache { - newGroupCache := f.NewCache() +func (f *FriendCacheRedis) DelSortFriendUserIDs(ownerUserIDs ...string) cache.FriendCache { + newGroupCache := f.CloneFriendCache() keys := make([]string, 0, len(ownerUserIDs)) for _, userID := range ownerUserIDs { keys = append(keys, f.getFriendSyncSortUserIDsKey(userID)) @@ -194,7 +169,7 @@ func (f *FriendCacheRedis) DelFriends(ownerUserID string, friendUserIDs []string return newFriendCache } -func (f *FriendCacheRedis) DelOwner(friendUserID string, ownerUserIDs []string) FriendCache { +func (f *FriendCacheRedis) DelOwner(friendUserID string, ownerUserIDs []string) cache.FriendCache { newFriendCache := f.CloneFriendCache() for _, ownerUserID := range ownerUserIDs { diff --git a/pkg/common/storage/controller/friend.go b/pkg/common/storage/controller/friend.go index 9b177dc45..0c83930b1 100644 --- a/pkg/common/storage/controller/friend.go +++ b/pkg/common/storage/controller/friend.go @@ -87,7 +87,7 @@ type FriendDatabase interface { UpdateFriendUserInfo(ctx context.Context, friendUserID string, ownerUserID []string, nickname string, faceURL string) error - SearchFriend(ctx context.Context, ownerUserID, keyword string, pagination pagination.Pagination) (int64, []*relation.FriendModel, error) + SearchFriend(ctx context.Context, ownerUserID, keyword string, pagination pagination.Pagination) (int64, []*model.Friend, error) } type friendDatabase struct { @@ -289,7 +289,7 @@ func (f *friendDatabase) AgreeFriendRequest(ctx context.Context, friendRequest * return err } } - return f.cache.DelFriendIDs(friendRequest.ToUserID, friendRequest.FromUserID).DelSortFriendUserIDs(friendRequest.ToUserID, friendRequest.FromUserID).ExecDel(ctx) + return f.cache.DelFriendIDs(friendRequest.ToUserID, friendRequest.FromUserID).DelSortFriendUserIDs(friendRequest.ToUserID, friendRequest.FromUserID).ChainExecDel(ctx) }) } @@ -374,9 +374,9 @@ func (f *friendDatabase) UpdateFriendUserInfo(ctx context.Context, friendUserID if err := f.friend.UpdateFriendUserInfo(ctx, friendUserID, nickname, faceURL); err != nil { return err } - return f.cache.DelOwner(friendUserID, ownerUserIDs).ExecDel(ctx) + return f.cache.DelOwner(friendUserID, ownerUserIDs).ChainExecDel(ctx) } -func (f *friendDatabase) SearchFriend(ctx context.Context, ownerUserID, keyword string, pagination pagination.Pagination) (int64, []*relation.FriendModel, error) { +func (f *friendDatabase) SearchFriend(ctx context.Context, ownerUserID, keyword string, pagination pagination.Pagination) (int64, []*model.Friend, error) { return f.friend.SearchFriend(ctx, ownerUserID, keyword, pagination) } diff --git a/pkg/common/storage/database/friend.go b/pkg/common/storage/database/friend.go index 728e19593..f5a999485 100644 --- a/pkg/common/storage/database/friend.go +++ b/pkg/common/storage/database/friend.go @@ -17,28 +17,13 @@ package database import ( "context" "github.com/openimsdk/open-im-server/v3/pkg/common/db/dataver" - "time" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" "github.com/openimsdk/tools/db/pagination" ) -//// FriendModel represents the data structure for a friend relationship in MongoDB. -//type FriendModel struct { -// OwnerUserID string `bson:"owner_user_id"` -// FriendUserID string `bson:"friend_user_id"` -// FriendNickname string `bson:"friend_nickname"` -// FriendFaceURL string `bson:"friend_face_url"` -// Remark string `bson:"remark"` -// CreateTime time.Time `bson:"create_time"` -// AddSource int32 `bson:"add_source"` -// OperatorUserID string `bson:"operator_user_id"` -// Ex string `bson:"ex"` -// IsPinned bool `bson:"is_pinned"` -//} - // Friend defines the operations for managing friends in MongoDB. -type FriendModelInterface interface { +type Friend interface { // Create inserts multiple friend records. Create(ctx context.Context, friends []*model.Friend) (err error) // Delete removes specified friends of the owner user. @@ -71,4 +56,6 @@ type FriendModelInterface interface { UpdateFriendUserInfo(ctx context.Context, friendUserID string, nickname string, faceURL string) error SearchFriend(ctx context.Context, ownerUserID, keyword string, pagination pagination.Pagination) (int64, []*model.Friend, error) + + FindOwnerFriendUserIds(ctx context.Context, ownerUserID string, limit int) ([]string, error) } diff --git a/pkg/common/storage/database/mgo/friend.go b/pkg/common/storage/database/mgo/friend.go index dae5f2103..8993ee829 100644 --- a/pkg/common/storage/database/mgo/friend.go +++ b/pkg/common/storage/database/mgo/friend.go @@ -225,7 +225,7 @@ func (f *FriendMgo) UpdateFriendUserInfo(ctx context.Context, friendUserID strin return err } -func (f *FriendMgo) SearchFriend(ctx context.Context, ownerUserID, keyword string, pagination pagination.Pagination) (int64, []*relation.FriendModel, error) { +func (f *FriendMgo) SearchFriend(ctx context.Context, ownerUserID, keyword string, pagination pagination.Pagination) (int64, []*model.Friend, error) { //where := bson.M{ // "owner_user_id": ownerUserID, // "$or": []bson.M{ diff --git a/pkg/common/storage/database/mgo/mongo_test.go b/pkg/common/storage/database/mgo/mongo_test.go deleted file mode 100644 index e69de29bb..000000000 diff --git a/pkg/common/storage/model/friend.go b/pkg/common/storage/model/friend.go index 60a40d9c2..c7675e466 100644 --- a/pkg/common/storage/model/friend.go +++ b/pkg/common/storage/model/friend.go @@ -22,6 +22,8 @@ import ( type Friend struct { OwnerUserID string `bson:"owner_user_id"` FriendUserID string `bson:"friend_user_id"` + FriendNickname string `bson:"friend_nickname"` + FriendFaceURL string `bson:"friend_face_url"` Remark string `bson:"remark"` CreateTime time.Time `bson:"create_time"` AddSource int32 `bson:"add_source"` From 0aaf8b93a020f5c90e4f17019ebb1bc09e76c58c Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Thu, 30 May 2024 15:41:45 +0800 Subject: [PATCH 15/59] optimization version log --- go.mod | 8 +- go.sum | 6 +- pkg/common/db/dataver/result.go | 42 --- pkg/common/db/dataver/todo.go | 38 --- pkg/common/listdemo/common.go | 182 ---------- pkg/common/listdemo/friend_model.go | 5 - pkg/common/listdemo/friend_table.go | 86 ----- pkg/common/listdemo2/common.go | 322 ------------------ pkg/common/listdemo2/common_test.go | 61 ---- pkg/common/listdemo2/demo.js | 86 ----- pkg/common/listdemo2/demo2.js | 63 ---- pkg/common/listdemo2/demo3.js | 59 ---- pkg/common/listdemo2/demo5.js | 10 - pkg/common/storage/cache/friend.go | 3 +- pkg/common/storage/cache/redis/friend.go | 3 +- pkg/common/storage/controller/friend.go | 5 +- pkg/common/storage/database/friend.go | 3 +- pkg/common/storage/database/mgo/friend.go | 16 +- .../database/mgo/version_log.go} | 116 ++----- pkg/common/storage/database/version_log.go | 18 + pkg/common/storage/model/user.go | 4 +- pkg/common/storage/model/version_log.go | 61 ++++ 22 files changed, 120 insertions(+), 1077 deletions(-) delete mode 100644 pkg/common/db/dataver/result.go delete mode 100644 pkg/common/db/dataver/todo.go delete mode 100644 pkg/common/listdemo/common.go delete mode 100644 pkg/common/listdemo/friend_model.go delete mode 100644 pkg/common/listdemo/friend_table.go delete mode 100644 pkg/common/listdemo2/common.go delete mode 100644 pkg/common/listdemo2/common_test.go delete mode 100644 pkg/common/listdemo2/demo.js delete mode 100644 pkg/common/listdemo2/demo2.js delete mode 100644 pkg/common/listdemo2/demo3.js delete mode 100644 pkg/common/listdemo2/demo5.js rename pkg/common/{db/dataver/common.go => storage/database/mgo/version_log.go} (56%) create mode 100644 pkg/common/storage/database/version_log.go create mode 100644 pkg/common/storage/model/version_log.go diff --git a/go.mod b/go.mod index 0ae542cd2..bd1e16a5b 100644 --- a/go.mod +++ b/go.mod @@ -14,8 +14,8 @@ require ( github.com/lestrrat-go/file-rotatelogs v2.4.0+incompatible // indirect github.com/mitchellh/mapstructure v1.5.0 github.com/openimsdk/protocol v0.0.69-alpha.2 - github.com/openimsdk/tools v0.0.49-alpha.23 - github.com/pkg/errors v0.9.1 + github.com/openimsdk/tools v0.0.49-alpha.24 + github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 github.com/stretchr/testify v1.9.0 go.mongodb.org/mongo-driver v1.14.0 @@ -178,6 +178,4 @@ require ( gopkg.in/ini.v1 v1.67.0 // indirect ) -replace ( - github.com/openimsdk/protocol => /Users/chao/Desktop/project/protocol -) +replace github.com/openimsdk/protocol => /Users/chao/Desktop/project/protocol diff --git a/go.sum b/go.sum index c0bd78645..1cc2cc1bc 100644 --- a/go.sum +++ b/go.sum @@ -286,10 +286,8 @@ github.com/onsi/gomega v1.18.1 h1:M1GfJqGRrBrrGGsbxzV5dqM2U2ApXefZCQpkukxYRLE= github.com/onsi/gomega v1.18.1/go.mod h1:0q+aL8jAiMXy9hbwj2mr5GziHiwhAIQpFmmtT5hitRs= github.com/openimsdk/gomake v0.0.13 h1:xLDe/moqgWpRoptHzI4packAWzs4C16b+sVY+txNJp0= github.com/openimsdk/gomake v0.0.13/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= -github.com/openimsdk/protocol v0.0.69-alpha.2 h1:XNc3pmAXyW+PMo7tghr2O+uydYck1hogppHDW3+Y+3k= -github.com/openimsdk/protocol v0.0.69-alpha.2/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= -github.com/openimsdk/tools v0.0.49-alpha.23 h1:/KkJ7vfx8FAoJhq3veH9PWnxbSkEf+dTSshvDrHBR38= -github.com/openimsdk/tools v0.0.49-alpha.23/go.mod h1:g7mkHXYUPi0/8aAX8VPMHpnb3hqdV69Jph+bXOGvvNM= +github.com/openimsdk/tools v0.0.49-alpha.24 h1:lJsqnjTPujnr91LRQ6QmcTliMIa4fMOBSTri6rFz2ek= +github.com/openimsdk/tools v0.0.49-alpha.24/go.mod h1:g7mkHXYUPi0/8aAX8VPMHpnb3hqdV69Jph+bXOGvvNM= github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= github.com/pelletier/go-toml/v2 v2.1.0/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= diff --git a/pkg/common/db/dataver/result.go b/pkg/common/db/dataver/result.go deleted file mode 100644 index ec09ffa31..000000000 --- a/pkg/common/db/dataver/result.go +++ /dev/null @@ -1,42 +0,0 @@ -package dataver - -import ( - "github.com/openimsdk/tools/utils/datautil" - "go.mongodb.org/mongo-driver/bson/primitive" -) - -type SyncResult struct { - Version uint - VersionID string - DeleteEID []string - Changes []string - Full bool -} - -func VersionIDStr(id primitive.ObjectID) string { - if id.IsZero() { - return "" - } - return id.String() -} - -func NewSyncResult(wl *WriteLog, fullIDs []string, versionID string) *SyncResult { - var findEIDs []string - var res SyncResult - if wl.Full() || VersionIDStr(wl.ID) != versionID { - res.Changes = fullIDs - res.Full = true - } else { - idSet := datautil.SliceSet(fullIDs) - for _, l := range wl.Logs { - if l.Deleted { - res.DeleteEID = append(res.DeleteEID, l.EID) - } else { - if _, ok := idSet[l.EID]; ok { - findEIDs = append(findEIDs, l.EID) - } - } - } - } - return &res -} diff --git a/pkg/common/db/dataver/todo.go b/pkg/common/db/dataver/todo.go deleted file mode 100644 index cd2be14f0..000000000 --- a/pkg/common/db/dataver/todo.go +++ /dev/null @@ -1,38 +0,0 @@ -package dataver - -/* - -UserIDs 顺序 -前500顺序 - - -1,2,3,4,5,6,7,8,9 - -1,3,5,7,8,9 - - -1.sdk添加一个表记录 docID(后续换名字), version -2.sdk同步，先计算idHash，api调用参数idHash, docID, version -3.服务器先判断version变更记录，没有直接返回同步成功。 - 有变更，先查版本变更记录，在查前500id，变更记录只保留前500id中的 - 根据前500id计算idHash，不一致返回会全量id，不反悔删除id - 全量同步有标识，只返回全量id - 变更记录只包含id，不包括详细信息。 -4.sdk通过变更记录，同步数据不一致进行重试。 -5.先修改db，在自增版本号，外层加事务 - - - - - - - - - - - - - - - -*/ diff --git a/pkg/common/listdemo/common.go b/pkg/common/listdemo/common.go deleted file mode 100644 index 2d99cd8c9..000000000 --- a/pkg/common/listdemo/common.go +++ /dev/null @@ -1,182 +0,0 @@ -package listdemo - -import ( - "context" - "github.com/openimsdk/tools/db/mongoutil" - "github.com/openimsdk/tools/db/pagination" - "github.com/pkg/errors" - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/mongo" - "go.mongodb.org/mongo-driver/mongo/options" -) - -var ( - ErrListNotFound = errors.New("list not found") - ErrElemExist = errors.New("elem exist") - ErrNotFound = mongo.ErrNoDocuments -) - -type ListDoc interface { - IDName() string // 外层业务id字段名字 user_id - ElemsName() string // 外层列表名字 friends - VersionName() string // 外层版本号 version - DeleteVersion() string // 删除版本号 - BuildDoc(lid any, e Elem) any // 返回一个组装的doc文档 -} - -type Elem interface { - IDName() string // 业务id名字 friend_user_id - IDValue() any // 业务id值 userID -> "100000000" - VersionName() string // 版本号 - DeletedName() string // 删除字段名字 - ToMap() map[string]any // 把结构体转换为map -} - -type List[D any, E Elem] struct { - coll *mongo.Collection - lf ListDoc -} - -func (l *List[D, E]) zeroE() E { - var t E - return t -} - -func (l *List[D, E]) FindElem(ctx context.Context, lid any, eid any) (E, error) { - res, err := l.FindElems(ctx, lid, []any{eid}) - if err != nil { - return l.zeroE(), err - } - if len(res) == 0 { - return l.zeroE(), ErrNotFound - } - return res[0], nil -} - -// FindElems 查询Elems -func (l *List[D, E]) FindElems(ctx context.Context, lid any, eids []any) ([]E, error) { - //pipeline := []bson.M{ - // { - // "$match": bson.M{ - // l.lf.IDName(): lid, - // l.lf.IDName() + "." + l.lf.ElemsID(): bson.M{ - // "$in": eids, - // }, - // }, - // }, - // { - // "$unwind": "$" + l.lf.ElemsName(), - // }, - // { - // "$match": bson.M{ - // l.lf.IDName() + "." + l.lf.ElemsID(): bson.M{ - // "$in": eids, - // }, - // }, - // }, - //} - panic("todo") -} - -func (l *List[D, E]) Find(ctx context.Context, filter any, opts ...*options.FindOptions) ([]E, error) { - return nil, nil -} - -func (l *List[D, E]) Count(ctx context.Context, filter any, opts ...*options.CountOptions) (int64, error) { - return 0, nil -} - -func (l *List[D, E]) Update(ctx context.Context, lid any, eid any) (*mongo.UpdateResult, error) { - - return nil, nil -} - -func (l *List[D, E]) Delete(ctx context.Context, lid any, eids any) (*mongo.UpdateResult, error) { - - return nil, nil -} - -func (l *List[D, E]) Page(ctx context.Context, filter any, pagination pagination.Pagination, opts ...*options.FindOptions) (int64, []E, error) { - return 0, nil, nil -} - -func (l *List[D, E]) ElemIDs(ctx context.Context, filter any, opts ...*options.FindOptions) ([]E, error) { - - return nil, nil -} - -// InsertElem 插入一个 -func (l *List[D, E]) InsertElem(ctx context.Context, lid any, e Elem) error { - if err := l.insertElem(ctx, lid, e); err == nil { - return nil - } else if !errors.Is(err, ErrListNotFound) { - return err - } - if _, err := l.coll.InsertOne(ctx, l.lf.BuildDoc(lid, e)); err == nil { - return nil - } else if mongo.IsDuplicateKeyError(err) { - return l.insertElem(ctx, lid, e) - } else { - return err - } -} - -func (l *List[D, E]) insertElem(ctx context.Context, lid any, e Elem) error { - data := e.ToMap() - data[e.VersionName()] = "$max_version" - filter := bson.M{ - l.lf.IDName(): lid, - } - pipeline := []bson.M{ - { - "$addFields": bson.M{ - "found_elem": bson.M{ - "$in": bson.A{e.IDValue(), l.lf.ElemsName() + "." + e.IDName()}, - }, - }, - }, - { - "$set": bson.M{ - "max_version": bson.M{ - "$cond": bson.M{ - "if": "$found_elem", - "then": "$max_version", - "else": bson.M{"$add": bson.A{"max_version", 1}}, - }, - }, - }, - }, - { - "$set": bson.M{ - l.lf.ElemsName(): bson.M{ - "$cond": bson.M{ - "if": "$found_elem", - "then": "$" + l.lf.ElemsName(), - "else": bson.M{ - "$concatArrays": bson.A{ - "$" + l.lf.ElemsName(), - bson.A{ - data, - }, - }, - }, - }, - }, - }, - }, - { - "$unset": "found_elem", - }, - } - res, err := mongoutil.UpdateMany(ctx, l.coll, filter, pipeline) - if err != nil { - return err - } - if res.MatchedCount == 0 { - return ErrListNotFound - } - if res.ModifiedCount == 0 { - return ErrElemExist - } - return nil -} diff --git a/pkg/common/listdemo/friend_model.go b/pkg/common/listdemo/friend_model.go deleted file mode 100644 index 13507bfa3..000000000 --- a/pkg/common/listdemo/friend_model.go +++ /dev/null @@ -1,5 +0,0 @@ -package listdemo - -type friendModel struct { - db *List[*Friend, *FriendElem] -} diff --git a/pkg/common/listdemo/friend_table.go b/pkg/common/listdemo/friend_table.go deleted file mode 100644 index b1ad877a2..000000000 --- a/pkg/common/listdemo/friend_table.go +++ /dev/null @@ -1,86 +0,0 @@ -package listdemo - -import ( - "time" -) - -var ( - _ Elem = (*FriendElem)(nil) - _ ListDoc = (*Friend)(nil) -) - -type FriendElem struct { - FriendUserID string `bson:"friend_user_id"` - Nickname string `bson:"nickname"` - FaceURL string `bson:"face_url"` - Remark string `bson:"remark"` - CreateTime time.Time `bson:"create_time"` - AddSource int32 `bson:"add_source"` - OperatorUserID string `bson:"operator_user_id"` - Ex string `bson:"ex"` - IsPinned bool `bson:"is_pinned"` - Version uint `bson:"version"` - DeleteTime *time.Time `bson:"delete_time"` -} - -func (f *FriendElem) IDName() string { - return "friend_user_id" -} - -func (f *FriendElem) IDValue() any { - return f.FriendUserID -} - -func (f *FriendElem) VersionName() string { - return "version" -} - -func (f *FriendElem) DeletedName() string { - return "delete_time" -} - -func (f *FriendElem) ToMap() map[string]any { - return map[string]any{ - "friend_user_id": f.FriendUserID, - "nickname": f.Nickname, - "face_url": f.FaceURL, - "remark": f.Remark, - "create_time": f.CreateTime, - "add_source": f.AddSource, - "operator_user_id": f.OperatorUserID, - "ex": f.Ex, - "is_pinned": f.IsPinned, - "version": f.Version, - "delete_time": f.DeleteTime, - } -} - -type Friend struct { - UserID string `bson:"user_id"` - Friends []*FriendElem `bson:"friends"` - Version uint `bson:"version"` - DeleteVersion uint `bson:"delete_version"` -} - -func (f *Friend) BuildDoc(lid any, e Elem) any { - return &Friend{ - UserID: lid.(string), - Friends: []*FriendElem{e.(*FriendElem)}, - } -} - -func (f *Friend) ElemsID() string { - return "user_id" -} - -func (f *Friend) IDName() string { - return "user_id" -} - -func (f *Friend) ElemsName() string { - return "friends" -} - -func (f *Friend) VersionName() string { - return "version" -} diff --git a/pkg/common/listdemo2/common.go b/pkg/common/listdemo2/common.go deleted file mode 100644 index 40419d533..000000000 --- a/pkg/common/listdemo2/common.go +++ /dev/null @@ -1,322 +0,0 @@ -package listdemo - -import ( - "context" - "github.com/openimsdk/tools/db/mongoutil" - "github.com/openimsdk/tools/errs" - "github.com/openimsdk/tools/utils/datautil" - "github.com/pkg/errors" - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/mongo" - "time" -) - -var ( - ErrListNotFound = errors.New("list not found") - ErrElemExist = errors.New("elem exist") - ErrNeedFull = errors.New("need full") - ErrNotFound = mongo.ErrNoDocuments -) - -const ( - FirstVersion = 1 - DefaultDeleteVersion = 0 -) - -type Elem struct { - ID string - Version uint -} - -type ChangeLog struct { - ChangeIDs []Elem - DeleteIDs []Elem -} - -type WriteLog struct { - DID string `bson:"d_id"` - Logs []LogElem `bson:"logs"` - Version uint `bson:"version"` - Deleted uint `bson:"deleted"` - LastUpdate time.Time `bson:"last_update"` -} - -type WriteLogLen struct { - DID string `bson:"d_id"` - Logs []LogElem `bson:"logs"` - Version uint `bson:"version"` - Deleted uint `bson:"deleted"` - LastUpdate time.Time `bson:"last_update"` - LogLen int `bson:"log_len"` -} - -type LogElem struct { - EID string `bson:"e_id"` - Deleted bool `bson:"deleted"` - Version uint `bson:"version"` - LastUpdate time.Time `bson:"last_update"` -} - -type LogModel struct { - coll *mongo.Collection -} - -func (l *LogModel) InitIndex(ctx context.Context) error { - _, err := l.coll.Indexes().CreateOne(ctx, mongo.IndexModel{ - Keys: bson.M{ - "d_id": 1, - }, - }) - return err -} - -func (l *LogModel) writeLog(ctx context.Context, dId string, eId string, deleted bool, now time.Time) (*mongo.UpdateResult, error) { - filter := bson.M{ - "d_id": dId, - } - elem := bson.M{ - "e_id": eId, - "version": "$version", - "deleted": deleted, - "last_update": now, - } - pipeline := []bson.M{ - { - "$addFields": bson.M{ - "elem_index": bson.M{ - "$indexOfArray": []any{"$logs.e_id", eId}, - }, - }, - }, - { - "$set": bson.M{ - "version": bson.M{"$add": []any{"$version", 1}}, - "last_update": now, - }, - }, - { - "$set": bson.M{ - "logs": bson.M{ - "$cond": bson.M{ - "if": bson.M{ - "$lt": []any{"$elem_index", 0}, - }, - "then": bson.M{ - "$concatArrays": []any{ - "$logs", - []bson.M{ - elem, - }, - }, - }, - "else": bson.M{ - "$map": bson.M{ - "input": bson.M{ - "$range": []any{0, bson.M{"$size": "$logs"}}, - }, - "as": "i", - "in": bson.M{ - "$cond": bson.M{ - "if": bson.M{ - "$eq": []any{"$$i", "$elem_index"}, - }, - "then": elem, - "else": bson.M{ - "$arrayElemAt": []any{ - "$logs", - "$$i", - }, - }, - }, - }, - }, - }, - }, - }, - }, - }, - { - "$unset": "elem_index", - }, - } - return mongoutil.UpdateMany(ctx, l.coll, filter, pipeline) -} - -func (l *LogModel) WriteLogBatch(ctx context.Context, dId string, eIds []string, deleted bool) error { - if len(eIds) == 0 { - return errs.ErrArgs.WrapMsg("elem id is empty", "dId", dId) - } - if datautil.Duplicate(eIds) { - return errs.ErrArgs.WrapMsg("elem id is duplicate", "dId", dId, "eIds", eIds) - } - now := time.Now() - res, err := l.writeLogBatch(ctx, dId, eIds, deleted, now) - if err != nil { - return err - } - if res.MatchedCount > 0 { - return nil - } - wl := WriteLog{ - DID: dId, - Logs: make([]LogElem, 0, len(eIds)), - Version: FirstVersion, - Deleted: DefaultDeleteVersion, - LastUpdate: now, - } - for _, eId := range eIds { - wl.Logs = append(wl.Logs, LogElem{ - EID: eId, - Deleted: deleted, - Version: FirstVersion, - LastUpdate: now, - }) - } - if _, err := l.coll.InsertOne(ctx, &wl); err == nil { - return nil - } else if !mongo.IsDuplicateKeyError(err) { - return err - } - if res, err := l.writeLogBatch(ctx, dId, eIds, deleted, now); err != nil { - return err - } else if res.ModifiedCount == 0 { - return errs.ErrInternalServer.WrapMsg("mongodb return value that should not occur", "coll", l.coll.Name(), "dId", dId, "eIds", eIds) - } - return nil -} - -func (l *LogModel) writeLogBatch(ctx context.Context, dId string, eIds []string, deleted bool, now time.Time) (*mongo.UpdateResult, error) { - if len(eIds) == 0 { - return nil, errs.ErrArgs.WrapMsg("elem id is empty", "dId", dId) - } - filter := bson.M{ - "d_id": dId, - } - elems := make([]bson.M, 0, len(eIds)) - for _, eId := range eIds { - elems = append(elems, bson.M{ - "e_id": eId, - "version": "$version", - "deleted": deleted, - "last_update": now, - }) - } - pipeline := []bson.M{ - { - "$addFields": bson.M{ - "delete_e_ids": eIds, - }, - }, - { - "$set": bson.M{ - "version": bson.M{"$add": []any{"$version", 1}}, - "last_update": now, - }, - }, - { - "$set": bson.M{ - "logs": bson.M{ - "$filter": bson.M{ - "input": "$logs", - "as": "log", - "cond": bson.M{ - "$not": bson.M{ - "$in": []any{"$$log.e_id", "$delete_e_ids"}, - }, - }, - }, - }, - }, - }, - { - "$set": bson.M{ - "logs": bson.M{ - "$concatArrays": []any{ - "$logs", - elems, - }, - }, - }, - }, - { - "$unset": "delete_e_ids", - }, - } - return mongoutil.UpdateMany(ctx, l.coll, filter, pipeline) -} - -func (l *LogModel) FindChangeLog(ctx context.Context, did string, version uint, limit int) (*WriteLogLen, error) { - pipeline := []bson.M{ - { - "$match": bson.M{ - "d_id": did, - }, - }, - { - "$addFields": bson.M{ - "logs": bson.M{ - "$cond": bson.M{ - "if": bson.M{ - "$or": []bson.M{ - {"$lt": []any{"$version", version}}, - {"$gte": []any{"$deleted", version}}, - }, - }, - "then": []any{}, - "else": "$logs", - }, - }, - }, - }, - { - "$addFields": bson.M{ - "logs": bson.M{ - "$filter": bson.M{ - "input": "$logs", - "as": "l", - "cond": bson.M{ - "$gt": []any{"$$l.version", version}, - }, - }, - }, - }, - }, - { - "$addFields": bson.M{ - "log_len": bson.M{"$size": "$logs"}, - }, - }, - { - "$addFields": bson.M{ - "logs": bson.M{ - "$cond": bson.M{ - "if": bson.M{ - "$gt": []any{"$log_len", limit}, - }, - "then": []any{}, - "else": "$logs", - }, - }, - }, - }, - } - if limit <= 0 { - pipeline = pipeline[:len(pipeline)-1] - } - res, err := mongoutil.Aggregate[*WriteLogLen](ctx, l.coll, pipeline) - if err != nil { - return nil, err - } - if len(res) == 0 { - return &WriteLogLen{}, nil - } - return res[0], nil -} - -func (l *LogModel) DeleteAfterUnchangedLog(ctx context.Context, deadline time.Time) error { - return mongoutil.DeleteMany(ctx, l.coll, bson.M{ - "last_update": bson.M{ - "$lt": deadline, - }, - }) -} diff --git a/pkg/common/listdemo2/common_test.go b/pkg/common/listdemo2/common_test.go deleted file mode 100644 index c03cc31bd..000000000 --- a/pkg/common/listdemo2/common_test.go +++ /dev/null @@ -1,61 +0,0 @@ -package listdemo - -import ( - "context" - "errors" - "github.com/openimsdk/tools/db/mongoutil" - "testing" -) - -func Result[V any](val V, err error) V { - if err != nil { - panic(err) - } - return val -} - -func Check(err error) { - if err != nil { - panic(err) - } -} - -func TestName(t *testing.T) { - cli := Result(mongoutil.NewMongoDB(context.Background(), &mongoutil.Config{Uri: "mongodb://openIM:openIM123@172.16.8.48:37017/openim_v3?maxPoolSize=100", Database: "openim_v3"})) - - db := cli.GetDB() - tx := cli.GetTx() - - const num = 1 - lm := &LogModel{coll: db.Collection("friend_version")} - - err := tx.Transaction(context.Background(), func(ctx context.Context) error { - err := tx.Transaction(ctx, func(ctx context.Context) error { - return lm.WriteLogBatch(ctx, "100", []string{"1000", "2000"}, true) - }) - if err != nil { - t.Log("--------->") - return err - } - return errors.New("1234") - }) - t.Log(err) - - //start := time.Now() - //eIds := make([]string, 0, num) - //for i := 0; i < num; i++ { - // eIds = append(eIds, strconv.Itoa(1000+(i))) - //} - //lm.WriteLogBatch1(context.Background(), "100", eIds, false) - //end := time.Now() - //t.Log(end.Sub(start)) // 509.962208ms - //t.Log(end.Sub(start) / num) // 511.496µs - - //start := time.Now() - //wll, err := lm.FindChangeLog(context.Background(), "100", 3, 100) - //if err != nil { - // panic(err) - //} - //t.Log(time.Since(start)) - //t.Log(wll) -} diff --git a/pkg/common/listdemo2/demo.js b/pkg/common/listdemo2/demo.js deleted file mode 100644 index be9c74cb9..000000000 --- a/pkg/common/listdemo2/demo.js +++ /dev/null @@ -1,86 +0,0 @@ -db.friend_version.updateMany( - { - "d_id": "100" - }, - [ - { - $addFields: { - elem_index: { - $indexOfArray: [ - "$logs.e_id", - "1000" - ] - } - } - }, - { - $set: { - version: { - $add: ["$version", 1] - }, - last_update: new Date(), - - } - }, - { - $set: { - logs: { - $cond: { - if: { - $lt: ["$elem_index", 0] - }, - then: { - $concatArrays: [ - "$logs", - [ - { - e_id: "1000", - last_update: new Date(), - version: "$version", - deleted: false - } - ] - ] - }, - else: { - $map: { - input: { - $range: [0, { - $size: "$logs" - }] - }, - as: "i", - in: { - $cond: { - if: { - $eq: ["$$i", "$elem_index"] - }, - then: { - e_id: "1000", - last_update: new Date(), - version: "$version", - deleted: false - }, - else: { - $arrayElemAt: ["$logs", "$$i"] - } - }, - - }, - - }, - - }, - - }, - - }, - - }, - - }, - { - $unset: ["elem_index"] - }, - ] -) diff --git a/pkg/common/listdemo2/demo2.js b/pkg/common/listdemo2/demo2.js deleted file mode 100644 index 15e7abcfe..000000000 --- a/pkg/common/listdemo2/demo2.js +++ /dev/null @@ -1,63 +0,0 @@ - -db.friend_version.updateMany( - { - "d_id": "100" - }, - [ - { - $addFields: { - update_elem_ids: ["1000", "1001","1003", "2000"] - } - }, - { - $set: { - version: { - $add: ["$version", 1] - }, - last_update: new Date(), - } - }, - { - $set: { - logs: { - $filter: { - input: "$logs", - as: "log", - cond: { - "$not": { - $in: ["$$log.e_id", "$update_elem_ids"] - } - } - } - }, - - }, - - }, - { - $set: { - logs: { - $concatArrays: [ - "$logs", - [ - { - e_id: "1003", - last_update: ISODate("2024-05-25T06:32:10.238Z"), - version: "$version", - deleted: false - }, - - ] - ] - } - } - }, - { - $unset: ["update_elem_ids"] - }, - - ] -) - - - diff --git a/pkg/common/listdemo2/demo3.js b/pkg/common/listdemo2/demo3.js deleted file mode 100644 index 5367971cf..000000000 --- a/pkg/common/listdemo2/demo3.js +++ /dev/null @@ -1,59 +0,0 @@ - -db.friend_version.aggregate([ - { - "$match": { - "d_id": "100", - } - }, - { - "$project": { - "_id": 0, - "d_id": 0, - } - }, - { - "$addFields": { - "logs": { - $cond: { - if: { - $or: [ - {$lt: ["$version", 3]}, - {$gte: ["$deleted", 3]}, - ], - }, - then: [], - else: "$logs", - } - } - }, - }, - { - "$addFields": { - "logs": { - "$filter": { - input: "$logs", - as: "l", - cond: { $gt: ["$$l.version", 3] } - } - } - } - }, - { - "$addFields": { - "log_len": { - $size: "$logs" - } - } - }, - { - "$addFields": { - "logs": { - $cond: { - if: {$gt: ["$log_len", 1]}, - then: [], - else: "$logs", - } - } - } - } -]) diff --git a/pkg/common/listdemo2/demo5.js b/pkg/common/listdemo2/demo5.js deleted file mode 100644 index 71a709a81..000000000 --- a/pkg/common/listdemo2/demo5.js +++ /dev/null @@ -1,10 +0,0 @@ -db.friend_version.updateMany( - { - "d_id": "100" - }, - [ - - - - ], -) \ No newline at end of file diff --git a/pkg/common/storage/cache/friend.go b/pkg/common/storage/cache/friend.go index 9dec3d5ab..3fee297ac 100644 --- a/pkg/common/storage/cache/friend.go +++ b/pkg/common/storage/cache/friend.go @@ -16,7 +16,6 @@ package cache import ( "context" - "github.com/openimsdk/open-im-server/v3/pkg/common/db/dataver" relationtb "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" ) @@ -40,5 +39,5 @@ type FriendCache interface { FindSortFriendUserIDs(ctx context.Context, ownerUserID string) ([]string, error) - FindFriendIncrVersion(ctx context.Context, ownerUserID string, version uint, limit int) (*dataver.WriteLog, error) + FindFriendIncrVersion(ctx context.Context, ownerUserID string, version uint, limit int) (*relationtb.VersionLog, error) } diff --git a/pkg/common/storage/cache/redis/friend.go b/pkg/common/storage/cache/redis/friend.go index d6754d7bf..f4edbca9a 100644 --- a/pkg/common/storage/cache/redis/friend.go +++ b/pkg/common/storage/cache/redis/friend.go @@ -16,7 +16,6 @@ package redis import ( "context" - "github.com/openimsdk/open-im-server/v3/pkg/common/db/dataver" "time" "github.com/dtm-labs/rockscache" @@ -186,6 +185,6 @@ func (f *FriendCacheRedis) FindSortFriendUserIDs(ctx context.Context, ownerUserI }) } -func (f *FriendCacheRedis) FindFriendIncrVersion(ctx context.Context, ownerUserID string, version uint, limit int) (*dataver.WriteLog, error) { +func (f *FriendCacheRedis) FindFriendIncrVersion(ctx context.Context, ownerUserID string, version uint, limit int) (*model.VersionLog, error) { return f.friendDB.FindIncrVersion(ctx, ownerUserID, version, limit) } diff --git a/pkg/common/storage/controller/friend.go b/pkg/common/storage/controller/friend.go index 0c83930b1..1af967b9b 100644 --- a/pkg/common/storage/controller/friend.go +++ b/pkg/common/storage/controller/friend.go @@ -17,7 +17,6 @@ package controller import ( "context" "fmt" - "github.com/openimsdk/open-im-server/v3/pkg/common/db/dataver" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database/mgo" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" @@ -81,7 +80,7 @@ type FriendDatabase interface { FindSortFriendUserIDs(ctx context.Context, ownerUserID string) ([]string, error) - FindFriendIncrVersion(ctx context.Context, ownerUserID string, version uint, limit int) (*dataver.WriteLog, error) + FindFriendIncrVersion(ctx context.Context, ownerUserID string, version uint, limit int) (*model.VersionLog, error) FindFriendUserID(ctx context.Context, friendUserID string) ([]string, error) @@ -362,7 +361,7 @@ func (f *friendDatabase) FindSortFriendUserIDs(ctx context.Context, ownerUserID return f.cache.FindSortFriendUserIDs(ctx, ownerUserID) } -func (f *friendDatabase) FindFriendIncrVersion(ctx context.Context, ownerUserID string, version uint, limit int) (*dataver.WriteLog, error) { +func (f *friendDatabase) FindFriendIncrVersion(ctx context.Context, ownerUserID string, version uint, limit int) (*model.VersionLog, error) { return f.cache.FindFriendIncrVersion(ctx, ownerUserID, version, limit) } diff --git a/pkg/common/storage/database/friend.go b/pkg/common/storage/database/friend.go index f5a999485..6ab1185bc 100644 --- a/pkg/common/storage/database/friend.go +++ b/pkg/common/storage/database/friend.go @@ -16,7 +16,6 @@ package database import ( "context" - "github.com/openimsdk/open-im-server/v3/pkg/common/db/dataver" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" "github.com/openimsdk/tools/db/pagination" @@ -49,7 +48,7 @@ type Friend interface { // UpdateFriends update friends' fields UpdateFriends(ctx context.Context, ownerUserID string, friendUserIDs []string, val map[string]any) (err error) - FindIncrVersion(ctx context.Context, ownerUserID string, version uint, limit int) (*dataver.WriteLog, error) + FindIncrVersion(ctx context.Context, ownerUserID string, version uint, limit int) (*model.VersionLog, error) FindFriendUserID(ctx context.Context, friendUserID string) ([]string, error) diff --git a/pkg/common/storage/database/mgo/friend.go b/pkg/common/storage/database/mgo/friend.go index 8993ee829..19b2f4f18 100644 --- a/pkg/common/storage/database/mgo/friend.go +++ b/pkg/common/storage/database/mgo/friend.go @@ -16,7 +16,6 @@ package mgo import ( "context" - "github.com/openimsdk/open-im-server/v3/pkg/common/db/dataver" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" @@ -30,7 +29,7 @@ import ( // FriendMgo implements Friend using MongoDB as the storage backend. type FriendMgo struct { coll *mongo.Collection - owner dataver.DataLog + owner database.VersionLog } // NewFriendMongo creates a new instance of FriendMgo with the provided MongoDB database. @@ -46,7 +45,7 @@ func NewFriendMongo(db *mongo.Database) (database.Friend, error) { if err != nil { return nil, err } - owner, err := dataver.NewDataLog(db.Collection("friend_owner_log")) + owner, err := NewVersionLog(db.Collection("friend_owner_version_log")) if err != nil { return nil, err } @@ -100,15 +99,6 @@ func (f *FriendMgo) UpdateByMap(ctx context.Context, ownerUserID string, friendU }) } -// Update modifies multiple friend documents. -// func (f *FriendMgo) Update(ctx context.Context, friends []*relation.Friend) error { -// filter := bson.M{ -// "owner_user_id": ownerUserID, -// "friend_user_id": friendUserID, -// } -// return mgotool.UpdateMany(ctx, f.coll, filter, friends) -// } - // UpdateRemark updates the remark for a specific friend. func (f *FriendMgo) UpdateRemark(ctx context.Context, ownerUserID, friendUserID, remark string) error { return f.UpdateByMap(ctx, ownerUserID, friendUserID, map[string]any{"remark": remark}) @@ -206,7 +196,7 @@ func (f *FriendMgo) UpdateFriends(ctx context.Context, ownerUserID string, frien }) } -func (f *FriendMgo) FindIncrVersion(ctx context.Context, ownerUserID string, version uint, limit int) (*dataver.WriteLog, error) { +func (f *FriendMgo) FindIncrVersion(ctx context.Context, ownerUserID string, version uint, limit int) (*model.VersionLog, error) { return f.owner.FindChangeLog(ctx, ownerUserID, version, limit) } diff --git a/pkg/common/db/dataver/common.go b/pkg/common/storage/database/mgo/version_log.go similarity index 56% rename from pkg/common/db/dataver/common.go rename to pkg/common/storage/database/mgo/version_log.go index 5ade1fd58..fe123073e 100644 --- a/pkg/common/db/dataver/common.go +++ b/pkg/common/storage/database/mgo/version_log.go @@ -1,8 +1,10 @@ -package dataver +package mgo import ( "context" "errors" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" "github.com/openimsdk/tools/db/mongoutil" "github.com/openimsdk/tools/errs" "github.com/openimsdk/tools/utils/datautil" @@ -13,84 +15,19 @@ import ( "time" ) -const ( - FirstVersion = 1 - DefaultDeleteVersion = 0 -) - -type WriteLog struct { - ID primitive.ObjectID `bson:"_id"` - DID string `bson:"d_id"` - Logs []Elem `bson:"logs"` - Version uint `bson:"version"` - Deleted uint `bson:"deleted"` - LastUpdate time.Time `bson:"last_update"` - LogLen int `bson:"log_len"` - queryDoc bool `bson:"-"` -} - -func (w *WriteLog) Full() bool { - return w.queryDoc || w.Version == 0 || len(w.Logs) != w.LogLen -} - -func (w *WriteLog) DeleteAndChangeIDs() (delIds []string, changeIds []string) { - for _, l := range w.Logs { - if l.Deleted { - delIds = append(delIds, l.EID) - } else { - changeIds = append(changeIds, l.EID) - } - } - return -} - -type Elem struct { - EID string `bson:"e_id"` - Deleted bool `bson:"deleted"` - Version uint `bson:"version"` - LastUpdate time.Time `bson:"last_update"` -} - -type tableWriteLog struct { - ID primitive.ObjectID `bson:"_id"` - DID string `bson:"d_id"` - Logs []Elem `bson:"logs"` - Version uint `bson:"version"` - Deleted uint `bson:"deleted"` - LastUpdate time.Time `bson:"last_update"` -} - -func (t *tableWriteLog) WriteLog() *WriteLog { - return &WriteLog{ - ID: t.ID, - DID: t.DID, - Logs: t.Logs, - Version: t.Version, - Deleted: t.Deleted, - LastUpdate: t.LastUpdate, - LogLen: 0, - } -} - -type DataLog interface { - WriteLog(ctx context.Context, dId string, eIds []string, deleted bool) error - FindChangeLog(ctx context.Context, dId string, version uint, limit int) (*WriteLog, error) - DeleteAfterUnchangedLog(ctx context.Context, deadline time.Time) error -} - -func NewDataLog(coll *mongo.Collection) (DataLog, error) { - lm := &logModel{coll: coll} +func NewVersionLog(coll *mongo.Collection) (database.VersionLog, error) { + lm := &VersionLogMgo{coll: coll} if lm.initIndex(context.Background()) != nil { return nil, errs.ErrInternalServer.WrapMsg("init index failed", "coll", coll.Name()) } return lm, nil } -type logModel struct { +type VersionLogMgo struct { coll *mongo.Collection } -func (l *logModel) initIndex(ctx context.Context) error { +func (l *VersionLogMgo) initIndex(ctx context.Context) error { _, err := l.coll.Indexes().CreateOne(ctx, mongo.IndexModel{ Keys: bson.M{ "d_id": 1, @@ -99,7 +36,7 @@ func (l *logModel) initIndex(ctx context.Context) error { return err } -func (l *logModel) WriteLog(ctx context.Context, dId string, eIds []string, deleted bool) error { +func (l *VersionLogMgo) WriteLog(ctx context.Context, dId string, eIds []string, deleted bool) error { if len(eIds) == 0 { return errs.ErrArgs.WrapMsg("elem id is empty", "dId", dId) } @@ -127,20 +64,20 @@ func (l *logModel) WriteLog(ctx context.Context, dId string, eIds []string, dele return nil } -func (l *logModel) initDoc(ctx context.Context, dId string, eIds []string, deleted bool, now time.Time) (*tableWriteLog, error) { - wl := tableWriteLog{ +func (l *VersionLogMgo) initDoc(ctx context.Context, dId string, eIds []string, deleted bool, now time.Time) (*model.VersionLogTable, error) { + wl := model.VersionLogTable{ ID: primitive.NewObjectID(), DID: dId, - Logs: make([]Elem, 0, len(eIds)), - Version: FirstVersion, - Deleted: DefaultDeleteVersion, + Logs: make([]model.VersionLogElem, 0, len(eIds)), + Version: database.FirstVersion, + Deleted: database.DefaultDeleteVersion, LastUpdate: now, } for _, eId := range eIds { - wl.Logs = append(wl.Logs, Elem{ + wl.Logs = append(wl.Logs, model.VersionLogElem{ EID: eId, Deleted: deleted, - Version: FirstVersion, + Version: database.FirstVersion, LastUpdate: now, }) } @@ -148,7 +85,7 @@ func (l *logModel) initDoc(ctx context.Context, dId string, eIds []string, delet return &wl, err } -func (l *logModel) writeLogBatch(ctx context.Context, dId string, eIds []string, deleted bool, now time.Time) (*mongo.UpdateResult, error) { +func (l *VersionLogMgo) writeLogBatch(ctx context.Context, dId string, eIds []string, deleted bool, now time.Time) (*mongo.UpdateResult, error) { if eIds == nil { eIds = []string{} } @@ -208,23 +145,22 @@ func (l *logModel) writeLogBatch(ctx context.Context, dId string, eIds []string, return mongoutil.UpdateMany(ctx, l.coll, filter, pipeline) } -func (l *logModel) findDoc(ctx context.Context, dId string) (*WriteLog, error) { - res, err := mongoutil.FindOne[*WriteLog](ctx, l.coll, bson.M{"d_id": dId}, options.FindOne().SetProjection(bson.M{"logs": 0})) +func (l *VersionLogMgo) findDoc(ctx context.Context, dId string) (*model.VersionLog, error) { + vl, err := mongoutil.FindOne[*model.VersionLogTable](ctx, l.coll, bson.M{"d_id": dId}, options.FindOne().SetProjection(bson.M{"logs": 0})) if err != nil { return nil, err } - res.queryDoc = true - return res, nil + return vl.VersionLog(), nil } -func (l *logModel) FindChangeLog(ctx context.Context, dId string, version uint, limit int) (*WriteLog, error) { +func (l *VersionLogMgo) FindChangeLog(ctx context.Context, dId string, version uint, limit int) (*model.VersionLog, error) { if wl, err := l.findChangeLog(ctx, dId, version, limit); err == nil { return wl, nil } else if !errors.Is(err, mongo.ErrNoDocuments) { return nil, err } if res, err := l.initDoc(ctx, dId, nil, false, time.Now()); err == nil { - return res.WriteLog(), nil + return res.VersionLog(), nil } else if mongo.IsDuplicateKeyError(err) { return l.findChangeLog(ctx, dId, version, limit) } else { @@ -232,7 +168,7 @@ func (l *logModel) FindChangeLog(ctx context.Context, dId string, version uint, } } -func (l *logModel) findChangeLog(ctx context.Context, dId string, version uint, limit int) (*WriteLog, error) { +func (l *VersionLogMgo) findChangeLog(ctx context.Context, dId string, version uint, limit int) (*model.VersionLog, error) { if version == 0 && limit == 0 { return l.findDoc(ctx, dId) } @@ -293,17 +229,17 @@ func (l *logModel) findChangeLog(ctx context.Context, dId string, version uint, if limit <= 0 { pipeline = pipeline[:len(pipeline)-1] } - res, err := mongoutil.Aggregate[*WriteLog](ctx, l.coll, pipeline) + vl, err := mongoutil.Aggregate[*model.VersionLog](ctx, l.coll, pipeline) if err != nil { return nil, err } - if len(res) == 0 { + if len(vl) == 0 { return nil, mongo.ErrNoDocuments } - return res[0], nil + return vl[0], nil } -func (l *logModel) DeleteAfterUnchangedLog(ctx context.Context, deadline time.Time) error { +func (l *VersionLogMgo) DeleteAfterUnchangedLog(ctx context.Context, deadline time.Time) error { return mongoutil.DeleteMany(ctx, l.coll, bson.M{ "last_update": bson.M{ "$lt": deadline, diff --git a/pkg/common/storage/database/version_log.go b/pkg/common/storage/database/version_log.go new file mode 100644 index 000000000..6783cf4ff --- /dev/null +++ b/pkg/common/storage/database/version_log.go @@ -0,0 +1,18 @@ +package database + +import ( + "context" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" + "time" +) + +const ( + FirstVersion = 1 + DefaultDeleteVersion = 0 +) + +type VersionLog interface { + WriteLog(ctx context.Context, dId string, eIds []string, deleted bool) error + FindChangeLog(ctx context.Context, dId string, version uint, limit int) (*model.VersionLog, error) + DeleteAfterUnchangedLog(ctx context.Context, deadline time.Time) error +} diff --git a/pkg/common/storage/model/user.go b/pkg/common/storage/model/user.go index c6a4f952c..f64d09e79 100644 --- a/pkg/common/storage/model/user.go +++ b/pkg/common/storage/model/user.go @@ -36,10 +36,10 @@ func (u *User) GetFaceURL() string { return u.FaceURL } -func (u User) GetUserID() string { +func (u *User) GetUserID() string { return u.UserID } -func (u User) GetEx() string { +func (u *User) GetEx() string { return u.Ex } diff --git a/pkg/common/storage/model/version_log.go b/pkg/common/storage/model/version_log.go new file mode 100644 index 000000000..044bd42da --- /dev/null +++ b/pkg/common/storage/model/version_log.go @@ -0,0 +1,61 @@ +package model + +import ( + "go.mongodb.org/mongo-driver/bson/primitive" + "time" +) + +type VersionLogElem struct { + EID string `bson:"e_id"` + Deleted bool `bson:"deleted"` + Version uint `bson:"version"` + LastUpdate time.Time `bson:"last_update"` +} + +type VersionLogTable struct { + ID primitive.ObjectID `bson:"_id"` + DID string `bson:"d_id"` + Logs []VersionLogElem `bson:"logs"` + Version uint `bson:"version"` + Deleted uint `bson:"deleted"` + LastUpdate time.Time `bson:"last_update"` +} + +func (v *VersionLogTable) VersionLog() *VersionLog { + return &VersionLog{ + ID: v.ID, + DID: v.DID, + Logs: v.Logs, + Version: v.Version, + Deleted: v.Deleted, + LastUpdate: v.LastUpdate, + LogLen: 0, + queryDoc: true, + } +} + +type VersionLog struct { + ID primitive.ObjectID `bson:"_id"` + DID string `bson:"d_id"` + Logs []VersionLogElem `bson:"logs"` + Version uint `bson:"version"` + Deleted uint `bson:"deleted"` + LastUpdate time.Time `bson:"last_update"` + LogLen int `bson:"log_len"` + queryDoc bool `bson:"-"` +} + +func (w *VersionLog) Full() bool { + return w.queryDoc || w.Version == 0 || len(w.Logs) != w.LogLen +} + +func (w *VersionLog) DeleteAndChangeIDs() (delIds []string, changeIds []string) { + for _, l := range w.Logs { + if l.Deleted { + delIds = append(delIds, l.EID) + } else { + changeIds = append(changeIds, l.EID) + } + } + return +} From 0f72de85b291c50c85f6638a42e445da1c03813c Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Thu, 30 May 2024 15:49:34 +0800 Subject: [PATCH 16/59] optimization version log --- pkg/common/storage/database/mgo/friend.go | 24 ++++------------------- 1 file changed, 4 insertions(+), 20 deletions(-) diff --git a/pkg/common/storage/database/mgo/friend.go b/pkg/common/storage/database/mgo/friend.go index 19b2f4f18..2df85a430 100644 --- a/pkg/common/storage/database/mgo/friend.go +++ b/pkg/common/storage/database/mgo/friend.go @@ -54,7 +54,7 @@ func NewFriendMongo(db *mongo.Database) (database.Friend, error) { // Create inserts multiple friend records. func (f *FriendMgo) Create(ctx context.Context, friends []*model.Friend) error { - return IncrVersion(func() error { + return mongoutil.IncrVersion(func() error { return mongoutil.InsertMany(ctx, f.coll, friends) }, func() error { mp := make(map[string][]string) @@ -76,7 +76,7 @@ func (f *FriendMgo) Delete(ctx context.Context, ownerUserID string, friendUserID "owner_user_id": ownerUserID, "friend_user_id": bson.M{"$in": friendUserIDs}, } - return IncrVersion(func() error { + return mongoutil.IncrVersion(func() error { return mongoutil.DeleteOne(ctx, f.coll, filter) }, func() error { return f.owner.WriteLog(ctx, ownerUserID, friendUserIDs, true) @@ -92,7 +92,7 @@ func (f *FriendMgo) UpdateByMap(ctx context.Context, ownerUserID string, friendU "owner_user_id": ownerUserID, "friend_user_id": friendUserID, } - return IncrVersion(func() error { + return mongoutil.IncrVersion(func() error { return mongoutil.UpdateOne(ctx, f.coll, filter, bson.M{"$set": args}, true) }, func() error { return f.owner.WriteLog(ctx, ownerUserID, []string{friendUserID}, false) @@ -152,13 +152,6 @@ func (f *FriendMgo) FindOwnerFriends(ctx context.Context, ownerUserID string, pa func (f *FriendMgo) FindOwnerFriendUserIds(ctx context.Context, ownerUserID string, limit int) ([]string, error) { filter := bson.M{"owner_user_id": ownerUserID} opt := options.Find().SetProjection(bson.M{"_id": 0, "friend_user_id": 1}).SetSort(bson.D{{"friend_nickname", 1}, {"create_time", 1}}).SetLimit(int64(limit)) - //res, err := mongoutil.Find[string](ctx, f.coll, filter, opt) - //if err != nil { - // errMsg := err.Error() - // _ = errMsg - // return nil, err - //} - //return res, nil return mongoutil.Find[string](ctx, f.coll, filter, opt) } @@ -189,7 +182,7 @@ func (f *FriendMgo) UpdateFriends(ctx context.Context, ownerUserID string, frien // Create an update document update := bson.M{"$set": val} - return IncrVersion(func() error { + return mongoutil.IncrVersion(func() error { return mongoutil.Ignore(mongoutil.UpdateMany(ctx, f.coll, filter, update)) }, func() error { return f.owner.WriteLog(ctx, ownerUserID, friendUserIDs, false) @@ -227,12 +220,3 @@ func (f *FriendMgo) SearchFriend(ctx context.Context, ownerUserID, keyword strin //return f.aggregatePagination(ctx, where, pagination) panic("todo") } - -func IncrVersion(dbs ...func() error) error { - for _, fn := range dbs { - if err := fn(); err != nil { - return err - } - } - return nil -} From cfc01bb3d962d0445d15a8cd7461cef8ec8d92c9 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Fri, 31 May 2024 18:41:34 +0800 Subject: [PATCH 17/59] sync --- go.mod | 4 +- go.sum | 2 + internal/api/friend.go | 17 --- internal/api/group.go | 50 ++++++++ internal/api/router.go | 3 + internal/rpc/friend/sync.go | 114 +++++++++++------- internal/rpc/group/convert.go | 4 + internal/rpc/group/group.go | 61 +++++++--- internal/rpc/group/sync.go | 66 +++++++++- internal/rpc/incrversion/option.go | 58 +++++++++ pkg/common/config/config.go | 3 +- pkg/common/storage/cache/group.go | 3 + pkg/common/storage/cache/redis/group.go | 23 ++++ pkg/common/storage/controller/group.go | 55 +++++++-- pkg/common/storage/database/group_member.go | 3 + pkg/common/storage/database/mgo/friend.go | 10 +- .../storage/database/mgo/group_member.go | 95 ++++++++++++--- .../storage/database/mgo/version_log.go | 2 +- pkg/common/storage/database/version_log.go | 2 +- 19 files changed, 462 insertions(+), 113 deletions(-) create mode 100644 internal/rpc/incrversion/option.go diff --git a/go.mod b/go.mod index bd1e16a5b..db418bf14 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/lestrrat-go/file-rotatelogs v2.4.0+incompatible // indirect github.com/mitchellh/mapstructure v1.5.0 - github.com/openimsdk/protocol v0.0.69-alpha.2 + github.com/openimsdk/protocol v0.0.69-alpha.3 github.com/openimsdk/tools v0.0.49-alpha.24 github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 @@ -178,4 +178,4 @@ require ( gopkg.in/ini.v1 v1.67.0 // indirect ) -replace github.com/openimsdk/protocol => /Users/chao/Desktop/project/protocol +//replace github.com/openimsdk/protocol => /Users/chao/Desktop/project/protocol diff --git a/go.sum b/go.sum index 1cc2cc1bc..4588df18a 100644 --- a/go.sum +++ b/go.sum @@ -286,6 +286,8 @@ github.com/onsi/gomega v1.18.1 h1:M1GfJqGRrBrrGGsbxzV5dqM2U2ApXefZCQpkukxYRLE= github.com/onsi/gomega v1.18.1/go.mod h1:0q+aL8jAiMXy9hbwj2mr5GziHiwhAIQpFmmtT5hitRs= github.com/openimsdk/gomake v0.0.13 h1:xLDe/moqgWpRoptHzI4packAWzs4C16b+sVY+txNJp0= github.com/openimsdk/gomake v0.0.13/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= +github.com/openimsdk/protocol v0.0.69-alpha.3 h1:Uf167FVB5EqYpiy2zBbR63OiK+Njjy99fXYasK6Zi+4= +github.com/openimsdk/protocol v0.0.69-alpha.3/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= github.com/openimsdk/tools v0.0.49-alpha.24 h1:lJsqnjTPujnr91LRQ6QmcTliMIa4fMOBSTri6rFz2ek= github.com/openimsdk/tools v0.0.49-alpha.24/go.mod h1:g7mkHXYUPi0/8aAX8VPMHpnb3hqdV69Jph+bXOGvvNM= github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= diff --git a/internal/api/friend.go b/internal/api/friend.go index 5b2dfa0ca..3af162a53 100644 --- a/internal/api/friend.go +++ b/internal/api/friend.go @@ -98,20 +98,3 @@ func (o *FriendApi) UpdateFriends(c *gin.Context) { func (o *FriendApi) GetIncrementalFriends(c *gin.Context) { a2r.Call(friend.FriendClient.GetIncrementalFriends, o.Client, c) } - -//func BatchIncremental[A, B, C any,D comparable](c *gin.Context, rpc func(client C, ctx context.Context, req *A, options ...grpc.CallOption) (*B, error), getID func(req *A)D, setID func(req *A, id D)) { -// req, err := a2r.ParseRequestNotCheck[BatchIncrementalReq[A]](c) -// if err != nil { -// apiresp.GinError(c, err) -// return -// } -// if len(req.List) == 0 { -// apiresp.GinError(c, errs.ErrArgs.WrapMsg("empty versions list")) -// return -// } -//} -// -//type BatchIncrementalReq[A any] struct { -// UserID string `json:"user_id"` -// List []*A `json:"list"` -//} diff --git a/internal/api/group.go b/internal/api/group.go index 6079c5343..7baa16323 100644 --- a/internal/api/group.go +++ b/internal/api/group.go @@ -19,6 +19,7 @@ import ( "github.com/openimsdk/open-im-server/v3/pkg/rpcclient" "github.com/openimsdk/protocol/group" "github.com/openimsdk/tools/a2r" + "github.com/openimsdk/tools/apiresp" ) type GroupApi rpcclient.Group @@ -134,3 +135,52 @@ func (o *GroupApi) GetGroups(c *gin.Context) { func (o *GroupApi) GetGroupMemberUserIDs(c *gin.Context) { a2r.Call(group.GroupClient.GetGroupMemberUserIDs, o.Client, c) } + +func (o *GroupApi) GetIncrementalJoinGroup(c *gin.Context) { + a2r.Call(group.GroupClient.GetIncrementalJoinGroup, o.Client, c) +} + +func (o *GroupApi) GetIncrementalGroupMember(c *gin.Context) { + a2r.Call(group.GroupClient.GetIncrementalGroupMember, o.Client, c) +} + +func (o *GroupApi) GetIncrementalGroupMemberBatch(c *gin.Context) { + type BatchIncrementalReq[A any] struct { + UserID string `json:"user_id"` + List []*group.GetIncrementalGroupMemberReq `json:"list"` + } + type BatchIncrementalResp struct { + List map[string]*group.GetIncrementalGroupMemberResp `json:"list"` + } + req, err := a2r.ParseRequestNotCheck[BatchIncrementalReq](c) + if err != nil { + apiresp.GinError(c, err) + return + } + resp := &BatchIncrementalResp{ + List: make(map[string]*group.GetIncrementalGroupMemberResp), + } + var ( + changeCount int + ) + for _, req := range req.List { + if _, ok := resp.List[req.GroupID]; ok { + continue + } + res, err := o.Client.GetIncrementalGroupMember(c, req) + if err != nil { + if len(resp.List) == 0 { + apiresp.GinError(c, err) + } else { + apiresp.GinSuccess(c, resp) + } + return + } + resp.List[req.GroupID] = res + changeCount += len(res.Changes) + len(res.DeleteUserIds) + if changeCount > int(res.SyncCount)*4 { + break + } + } + apiresp.GinSuccess(c, resp) +} diff --git a/internal/api/router.go b/internal/api/router.go index 78e049e0f..8f2733604 100644 --- a/internal/api/router.go +++ b/internal/api/router.go @@ -115,6 +115,9 @@ func newGinRouter(disCov discovery.SvcDiscoveryRegistry, config *Config) *gin.En groupRouterGroup.POST("/get_group_abstract_info", g.GetGroupAbstractInfo) groupRouterGroup.POST("/get_groups", g.GetGroups) groupRouterGroup.POST("/get_group_member_user_id", g.GetGroupMemberUserIDs) + groupRouterGroup.POST("/get_incremental_join_group", g.GetIncrementalJoinGroup) + groupRouterGroup.POST("/get_incremental_group_member", g.GetIncrementalGroupMember) + groupRouterGroup.POST("/get_incremental_group_member_batch", g.GetIncrementalGroupMemberBatch) } // certificate authRouterGroup := r.Group("/auth") diff --git a/internal/rpc/friend/sync.go b/internal/rpc/friend/sync.go index bfa7e3a57..bdea7b9eb 100644 --- a/internal/rpc/friend/sync.go +++ b/internal/rpc/friend/sync.go @@ -2,9 +2,7 @@ package friend import ( "context" - "crypto/md5" - "encoding/binary" - "encoding/json" + "github.com/openimsdk/open-im-server/v3/internal/rpc/incrversion" "github.com/openimsdk/open-im-server/v3/pkg/authverify" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" pbfriend "github.com/openimsdk/protocol/friend" @@ -59,49 +57,79 @@ func (s *friendServer) SearchFriends(ctx context.Context, req *pbfriend.SearchFr }, nil } -func (s *friendServer) sortFriendUserIDsHash(userIDs []string) uint64 { - data, _ := json.Marshal(userIDs) - sum := md5.Sum(data) - return binary.BigEndian.Uint64(sum[:]) -} - func (s *friendServer) GetIncrementalFriends(ctx context.Context, req *pbfriend.GetIncrementalFriendsReq) (*pbfriend.GetIncrementalFriendsResp, error) { if err := authverify.CheckAccessV3(ctx, req.UserID, s.config.Share.IMAdminUserID); err != nil { return nil, err } - var limit int - if req.Version > 0 { - limit = s.config.RpcConfig.FriendSyncCount - } - incrVer, err := s.friendDatabase.FindFriendIncrVersion(ctx, req.UserID, uint(req.Version), limit) - if err != nil { - return nil, err + opt := incrversion.Option[*pbfriend.FriendInfo, pbfriend.GetIncrementalFriendsResp]{ + VersionID: req.VersionID, + Version: func() (*model.VersionLog, error) { + return s.friendDatabase.FindFriendIncrVersion(ctx, req.UserID, uint(req.Version), incrversion.Limit(s.config.RpcConfig.FriendSyncCount, req.Version)) + }, + AllID: func() ([]string, error) { + return s.friendDatabase.FindSortFriendUserIDs(ctx, req.UserID) + }, + Find: func(ids []string) ([]*pbfriend.FriendInfo, error) { + friends, err := s.friendDatabase.FindFriendsWithError(ctx, req.UserID, ids) + if err != nil { + return nil, err + } + return friendsDB2PB(friends), nil + }, + ID: func(elem *pbfriend.FriendInfo) string { + return elem.FriendUserID + }, + Resp: func(version *model.VersionLog, delIDs []string, list []*pbfriend.FriendInfo, full bool) *pbfriend.GetIncrementalFriendsResp { + return &pbfriend.GetIncrementalFriendsResp{ + VersionID: version.ID.Hex(), + Version: uint64(version.Version), + Full: full, + SyncCount: uint32(s.config.RpcConfig.FriendSyncCount), + DeleteUserIds: delIDs, + Changes: list, + } + }, } - var ( - deleteUserIDs []string - changeUserIDs []string - ) - if incrVer.Full() { - changeUserIDs, err = s.friendDatabase.FindSortFriendUserIDs(ctx, req.UserID) - if err != nil { - return nil, err - } - } else { - deleteUserIDs, changeUserIDs = incrVer.DeleteAndChangeIDs() - } - var friends []*model.Friend - if len(changeUserIDs) > 0 { - friends, err = s.friendDatabase.FindFriendsWithError(ctx, req.UserID, changeUserIDs) - if err != nil { - return nil, err - } - } - return &pbfriend.GetIncrementalFriendsResp{ - Version: uint64(incrVer.Version), - VersionID: incrVer.ID.Hex(), - Full: incrVer.Full(), - SyncCount: uint32(s.config.RpcConfig.FriendSyncCount), - DeleteUserIds: deleteUserIDs, - Changes: friendsDB2PB(friends), - }, nil + return opt.Build() } + +//func (s *friendServer) GetIncrementalFriends(ctx context.Context, req *pbfriend.GetIncrementalFriendsReq) (*pbfriend.GetIncrementalFriendsResp, error) { +// if err := authverify.CheckAccessV3(ctx, req.UserID, s.config.Share.IMAdminUserID); err != nil { +// return nil, err +// } +// var limit int +// if req.Version > 0 { +// limit = s.config.RpcConfig.FriendSyncCount +// } +// incrVer, err := s.friendDatabase.FindFriendIncrVersion(ctx, req.UserID, uint(req.Version), limit) +// if err != nil { +// return nil, err +// } +// var ( +// deleteUserIDs []string +// changeUserIDs []string +// ) +// if incrVer.Full() { +// changeUserIDs, err = s.friendDatabase.FindSortFriendUserIDs(ctx, req.UserID) +// if err != nil { +// return nil, err +// } +// } else { +// deleteUserIDs, changeUserIDs = incrVer.DeleteAndChangeIDs() +// } +// var friends []*model.Friend +// if len(changeUserIDs) > 0 { +// friends, err = s.friendDatabase.FindFriendsWithError(ctx, req.UserID, changeUserIDs) +// if err != nil { +// return nil, err +// } +// } +// return &pbfriend.GetIncrementalFriendsResp{ +// Version: uint64(incrVer.Version), +// VersionID: incrVer.ID.Hex(), +// Full: incrVer.Full(), +// SyncCount: uint32(s.config.RpcConfig.FriendSyncCount), +// DeleteUserIds: deleteUserIDs, +// Changes: friendsDB2PB(friends), +// }, nil +//} diff --git a/internal/rpc/group/convert.go b/internal/rpc/group/convert.go index a75693904..8026430c3 100644 --- a/internal/rpc/group/convert.go +++ b/internal/rpc/group/convert.go @@ -57,3 +57,7 @@ func (s *groupServer) groupMemberDB2PB(member *model.GroupMember, appMangerLevel InviterUserID: member.InviterUserID, } } + +func (s *groupServer) groupMemberDB2PB2(member *model.GroupMember) *sdkws.GroupMemberFullInfo { + return s.groupMemberDB2PB(member, 0) +} diff --git a/internal/rpc/group/group.go b/internal/rpc/group/group.go index 51fd2d7b6..9b9ef07c1 100644 --- a/internal/rpc/group/group.go +++ b/internal/rpc/group/group.go @@ -77,6 +77,9 @@ type Config struct { } func Start(ctx context.Context, config *Config, client discovery.SvcDiscoveryRegistry, server *grpc.Server) error { + if config.RpcConfig.GroupSyncCount <= 0 { + config.RpcConfig.GroupSyncCount = constant.MaxSyncPullNumber + } mgocli, err := mongoutil.NewMongoDB(ctx, config.MongodbConfig.Build()) if err != nil { return err @@ -642,18 +645,29 @@ func (s *groupServer) GetGroupMembersInfo(ctx context.Context, req *pbgroup.GetG if req.GroupID == "" { return nil, errs.ErrArgs.WrapMsg("groupID empty") } - members, err := s.db.FindGroupMembers(ctx, req.GroupID, req.UserIDs) + members, err := s.getGroupMembersInfo(ctx, req.GroupID, req.UserIDs) + if err != nil { + return nil, err + } + return &pbgroup.GetGroupMembersInfoResp{ + Members: members, + }, nil +} + +func (s *groupServer) getGroupMembersInfo(ctx context.Context, groupID string, userIDs []string) ([]*sdkws.GroupMemberFullInfo, error) { + if len(userIDs) == 0 { + return nil, nil + } + members, err := s.db.FindGroupMembers(ctx, groupID, userIDs) if err != nil { return nil, err } if err := s.PopulateGroupMember(ctx, members...); err != nil { return nil, err } - return &pbgroup.GetGroupMembersInfoResp{ - Members: datautil.Slice(members, func(e *model.GroupMember) *sdkws.GroupMemberFullInfo { - return convert.Db2PbGroupMember(e) - }), - }, nil + return datautil.Slice(members, func(e *model.GroupMember) *sdkws.GroupMemberFullInfo { + return convert.Db2PbGroupMember(e) + }), nil } // GetGroupApplicationList handles functions that get a list of group requests. @@ -722,15 +736,28 @@ func (s *groupServer) GetGroupsInfo(ctx context.Context, req *pbgroup.GetGroupsI if len(req.GroupIDs) == 0 { return nil, errs.ErrArgs.WrapMsg("groupID is empty") } - groups, err := s.db.FindGroup(ctx, req.GroupIDs) + groups, err := s.getGroupsInfo(ctx, req.GroupIDs) + if err != nil { + return nil, err + } + return &pbgroup.GetGroupsInfoResp{ + GroupInfos: groups, + }, nil +} + +func (s *groupServer) getGroupsInfo(ctx context.Context, groupIDs []string) ([]*sdkws.GroupInfo, error) { + if len(groupIDs) == 0 { + return nil, nil + } + groups, err := s.db.FindGroup(ctx, groupIDs) if err != nil { return nil, err } - groupMemberNumMap, err := s.db.MapGroupMemberNum(ctx, req.GroupIDs) + groupMemberNumMap, err := s.db.MapGroupMemberNum(ctx, groupIDs) if err != nil { return nil, err } - owners, err := s.db.FindGroupsOwner(ctx, req.GroupIDs) + owners, err := s.db.FindGroupsOwner(ctx, groupIDs) if err != nil { return nil, err } @@ -740,15 +767,13 @@ func (s *groupServer) GetGroupsInfo(ctx context.Context, req *pbgroup.GetGroupsI ownerMap := datautil.SliceToMap(owners, func(e *model.GroupMember) string { return e.GroupID }) - return &pbgroup.GetGroupsInfoResp{ - GroupInfos: datautil.Slice(groups, func(e *model.Group) *sdkws.GroupInfo { - var ownerUserID string - if owner, ok := ownerMap[e.GroupID]; ok { - ownerUserID = owner.UserID - } - return convert.Db2PbGroupInfo(e, ownerUserID, groupMemberNumMap[e.GroupID]) - }), - }, nil + return datautil.Slice(groups, func(e *model.Group) *sdkws.GroupInfo { + var ownerUserID string + if owner, ok := ownerMap[e.GroupID]; ok { + ownerUserID = owner.UserID + } + return convert.Db2PbGroupInfo(e, ownerUserID, groupMemberNumMap[e.GroupID]) + }), nil } func (s *groupServer) GroupApplicationResponse(ctx context.Context, req *pbgroup.GroupApplicationResponseReq) (*pbgroup.GroupApplicationResponseResp, error) { diff --git a/internal/rpc/group/sync.go b/internal/rpc/group/sync.go index 765bceb63..80cb5a9ea 100644 --- a/internal/rpc/group/sync.go +++ b/internal/rpc/group/sync.go @@ -2,7 +2,11 @@ package group import ( "context" + "github.com/openimsdk/open-im-server/v3/internal/rpc/incrversion" + "github.com/openimsdk/open-im-server/v3/pkg/authverify" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" pbgroup "github.com/openimsdk/protocol/group" + "github.com/openimsdk/protocol/sdkws" ) func (s *groupServer) SearchGroupMember(ctx context.Context, req *pbgroup.SearchGroupMemberReq) (*pbgroup.SearchGroupMemberResp, error) { @@ -10,7 +14,63 @@ func (s *groupServer) SearchGroupMember(ctx context.Context, req *pbgroup.Search panic("implement me") } -func (s *groupServer) GetGroupMemberHash(ctx context.Context, req *pbgroup.GetGroupMemberHashReq) (*pbgroup.GetGroupMemberHashResp, error) { - //TODO implement me - panic("implement me") +func (s *groupServer) GetIncrementalGroupMember(ctx context.Context, req *pbgroup.GetIncrementalGroupMemberReq) (*pbgroup.GetIncrementalGroupMemberResp, error) { + opt := incrversion.Option[*sdkws.GroupMemberFullInfo, pbgroup.GetIncrementalGroupMemberResp]{ + VersionID: req.VersionID, + Version: func() (*model.VersionLog, error) { + return s.db.FindMemberIncrVersion(ctx, req.GroupID, uint(req.Version), incrversion.Limit(s.config.RpcConfig.GroupSyncCount, req.Version)) + }, + AllID: func() ([]string, error) { + return s.db.FindSortGroupMemberUserIDs(ctx, req.GroupID) + }, + Find: func(ids []string) ([]*sdkws.GroupMemberFullInfo, error) { + return s.getGroupMembersInfo(ctx, req.GroupID, ids) + }, + ID: func(elem *sdkws.GroupMemberFullInfo) string { + return elem.UserID + }, + Resp: func(version *model.VersionLog, delIDs []string, list []*sdkws.GroupMemberFullInfo, full bool) *pbgroup.GetIncrementalGroupMemberResp { + return &pbgroup.GetIncrementalGroupMemberResp{ + VersionID: version.ID.Hex(), + Version: uint64(version.Version), + Full: full, + SyncCount: uint32(s.config.RpcConfig.GroupSyncCount), + DeleteUserIds: delIDs, + Changes: list, + } + }, + } + return opt.Build() +} + +func (s *groupServer) GetIncrementalJoinGroup(ctx context.Context, req *pbgroup.GetIncrementalJoinGroupReq) (*pbgroup.GetIncrementalJoinGroupResp, error) { + if err := authverify.CheckAccessV3(ctx, req.UserID, s.config.Share.IMAdminUserID); err != nil { + return nil, err + } + opt := incrversion.Option[*sdkws.GroupInfo, pbgroup.GetIncrementalJoinGroupResp]{ + VersionID: req.VersionID, + Version: func() (*model.VersionLog, error) { + return s.db.FindJoinIncrVersion(ctx, req.UserID, uint(req.Version), incrversion.Limit(s.config.RpcConfig.GroupSyncCount, req.Version)) + }, + AllID: func() ([]string, error) { + return s.db.FindSortJoinGroupIDs(ctx, req.UserID) + }, + Find: func(ids []string) ([]*sdkws.GroupInfo, error) { + return s.getGroupsInfo(ctx, ids) + }, + ID: func(elem *sdkws.GroupInfo) string { + return elem.GroupID + }, + Resp: func(version *model.VersionLog, delIDs []string, list []*sdkws.GroupInfo, full bool) *pbgroup.GetIncrementalJoinGroupResp { + return &pbgroup.GetIncrementalJoinGroupResp{ + VersionID: version.ID.Hex(), + Version: uint64(version.Version), + Full: full, + SyncCount: uint32(s.config.RpcConfig.GroupSyncCount), + DeleteGroupIds: delIDs, + Changes: list, + } + }, + } + return opt.Build() } diff --git a/internal/rpc/incrversion/option.go b/internal/rpc/incrversion/option.go new file mode 100644 index 000000000..3e19056bf --- /dev/null +++ b/internal/rpc/incrversion/option.go @@ -0,0 +1,58 @@ +package incrversion + +import ( + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" + "github.com/openimsdk/tools/utils/datautil" +) + +func Limit(maxSync int, version uint64) int { + if version == 0 { + return 0 + } + return maxSync +} + +type Option[A, B any] struct { + VersionID string + Version func() (*model.VersionLog, error) + AllID func() ([]string, error) + Find func(ids []string) ([]A, error) + ID func(elem A) string + Resp func(version *model.VersionLog, delIDs []string, list []A, full bool) *B +} + +func (o *Option[A, B]) Build() (*B, error) { + version, err := o.Version() + if err != nil { + return nil, err + } + var ( + deleteIDs []string + changeIDs []string + ) + full := o.VersionID != version.ID.Hex() || version.Full() + if full { + changeIDs, err = o.AllID() + if err != nil { + return nil, err + } + } else { + deleteIDs, changeIDs = version.DeleteAndChangeIDs() + } + var list []A + if len(changeIDs) > 0 { + list, err = o.Find(changeIDs) + if err != nil { + return nil, err + } + if (!full) && o.ID != nil && len(changeIDs) != len(list) { + foundIDs := datautil.SliceSetAny(list, o.ID) + for _, id := range changeIDs { + if _, ok := foundIDs[id]; !ok { + deleteIDs = append(deleteIDs, id) + } + } + } + } + return o.Resp(version, deleteIDs, list, full), nil +} diff --git a/pkg/common/config/config.go b/pkg/common/config/config.go index da62acebf..31a54c371 100644 --- a/pkg/common/config/config.go +++ b/pkg/common/config/config.go @@ -254,7 +254,8 @@ type Group struct { ListenIP string `mapstructure:"listenIP"` Ports []int `mapstructure:"ports"` } `mapstructure:"rpc"` - Prometheus Prometheus `mapstructure:"prometheus"` + Prometheus Prometheus `mapstructure:"prometheus"` + GroupSyncCount int `mapstructure:"groupSyncCount"` } type Msg struct { diff --git a/pkg/common/storage/cache/group.go b/pkg/common/storage/cache/group.go index 53e2cd1c7..ff840a4a5 100644 --- a/pkg/common/storage/cache/group.go +++ b/pkg/common/storage/cache/group.go @@ -59,4 +59,7 @@ type GroupCache interface { GetGroupRolesLevelMemberInfo(ctx context.Context, groupID string, roleLevels []int32) ([]*model.GroupMember, error) GetGroupMemberNum(ctx context.Context, groupID string) (memberNum int64, err error) DelGroupsMemberNum(groupID ...string) GroupCache + + FindSortGroupMemberUserIDs(ctx context.Context, groupID string) ([]string, error) + FindSortJoinGroupIDs(ctx context.Context, userID string) ([]string, error) } diff --git a/pkg/common/storage/cache/redis/group.go b/pkg/common/storage/cache/redis/group.go index 2de03906f..7f0ba62e4 100644 --- a/pkg/common/storage/cache/redis/group.go +++ b/pkg/common/storage/cache/redis/group.go @@ -46,6 +46,7 @@ type GroupCacheRedis struct { expireTime time.Duration rcClient *rockscache.Client groupHash cache.GroupHash + syncCount int } func NewGroupCacheRedis( @@ -406,3 +407,25 @@ func (g *GroupCacheRedis) FindGroupMemberUser(ctx context.Context, groupIDs []st return g.groupMemberDB.Take(ctx, groupID, userID) }) } + +func (g *GroupCacheRedis) FindSortGroupMemberUserIDs(ctx context.Context, groupID string) ([]string, error) { + userIDs, err := g.GetGroupMemberIDs(ctx, groupID) + if err != nil { + return nil, err + } + if len(userIDs) > g.syncCount { + userIDs = userIDs[:g.syncCount] + } + return userIDs, nil +} + +func (g *GroupCacheRedis) FindSortJoinGroupIDs(ctx context.Context, userID string) ([]string, error) { + groupIDs, err := g.GetJoinedGroupIDs(ctx, userID) + if err != nil { + return nil, err + } + if len(groupIDs) > g.syncCount { + groupIDs = groupIDs[:g.syncCount] + } + return groupIDs, nil +} diff --git a/pkg/common/storage/controller/group.go b/pkg/common/storage/controller/group.go index f2a135835..c32f009dd 100644 --- a/pkg/common/storage/controller/group.go +++ b/pkg/common/storage/controller/group.go @@ -106,6 +106,13 @@ type GroupDatabase interface { CountRangeEverydayTotal(ctx context.Context, start time.Time, end time.Time) (map[string]int64, error) // DeleteGroupMemberHash deletes the hash entries for group members in specified groups. DeleteGroupMemberHash(ctx context.Context, groupIDs []string) error + + FindMemberIncrVersion(ctx context.Context, groupID string, version uint, limit int) (*model.VersionLog, error) + FindJoinIncrVersion(ctx context.Context, userID string, version uint, limit int) (*model.VersionLog, error) + + FindSortGroupMemberUserIDs(ctx context.Context, groupID string) ([]string, error) + + FindSortJoinGroupIDs(ctx context.Context, userID string) ([]string, error) } func NewGroupDatabase( @@ -219,10 +226,21 @@ func (g *groupDatabase) SearchGroup(ctx context.Context, keyword string, paginat } func (g *groupDatabase) UpdateGroup(ctx context.Context, groupID string, data map[string]any) error { - if err := g.groupDB.UpdateMap(ctx, groupID, data); err != nil { - return err - } - return g.cache.DelGroupsInfo(groupID).ChainExecDel(ctx) + return g.ctxTx.Transaction(ctx, func(ctx context.Context) error { + if err := g.groupDB.UpdateMap(ctx, groupID, data); err != nil { + return err + } + userIDs, err := g.cache.GetGroupMemberIDs(ctx, groupID) + if err != nil { + return err + } + for _, userID := range userIDs { + if err := g.groupMemberDB.JoinGroupIncrVersion(ctx, userID, []string{groupID}, false); err != nil { + return err + } + } + return g.cache.DelGroupsInfo(groupID).ChainExecDel(ctx) + }) } func (g *groupDatabase) DismissGroup(ctx context.Context, groupID string, deleteMember bool) error { @@ -231,11 +249,11 @@ func (g *groupDatabase) DismissGroup(ctx context.Context, groupID string, delete if err := g.groupDB.UpdateStatus(ctx, groupID, constant.GroupStatusDismissed); err != nil { return err } + userIDs, err := g.cache.GetGroupMemberIDs(ctx, groupID) + if err != nil { + return err + } if deleteMember { - userIDs, err := g.cache.GetGroupMemberIDs(ctx, groupID) - if err != nil { - return err - } if err := g.groupMemberDB.Delete(ctx, groupID, nil); err != nil { return err } @@ -246,6 +264,11 @@ func (g *groupDatabase) DismissGroup(ctx context.Context, groupID string, delete DelGroupAllRoleLevel(groupID). DelGroupMembersInfo(groupID, userIDs...) } + if len(userIDs) > 0 { + if err := g.groupMemberDB.JoinGroupIncrVersion(ctx, groupID, userIDs, true); err != nil { + return err + } + } return c.DelGroupsInfo(groupID).ChainExecDel(ctx) }) } @@ -443,3 +466,19 @@ func (g *groupDatabase) DeleteGroupMemberHash(ctx context.Context, groupIDs []st } return c.ChainExecDel(ctx) } + +func (g *groupDatabase) FindMemberIncrVersion(ctx context.Context, groupID string, version uint, limit int) (*model.VersionLog, error) { + return g.groupMemberDB.FindMemberIncrVersion(ctx, groupID, version, limit) +} + +func (g *groupDatabase) FindJoinIncrVersion(ctx context.Context, userID string, version uint, limit int) (*model.VersionLog, error) { + return g.groupMemberDB.FindJoinIncrVersion(ctx, userID, version, limit) +} + +func (g *groupDatabase) FindSortGroupMemberUserIDs(ctx context.Context, groupID string) ([]string, error) { + return g.cache.FindSortGroupMemberUserIDs(ctx, groupID) +} + +func (g *groupDatabase) FindSortJoinGroupIDs(ctx context.Context, userID string) ([]string, error) { + return g.cache.FindSortJoinGroupIDs(ctx, userID) +} diff --git a/pkg/common/storage/database/group_member.go b/pkg/common/storage/database/group_member.go index f57f2c317..c397eda58 100644 --- a/pkg/common/storage/database/group_member.go +++ b/pkg/common/storage/database/group_member.go @@ -34,4 +34,7 @@ type GroupMember interface { TakeGroupMemberNum(ctx context.Context, groupID string) (count int64, err error) FindUserManagedGroupID(ctx context.Context, userID string) (groupIDs []string, err error) IsUpdateRoleLevel(data map[string]any) bool + JoinGroupIncrVersion(ctx context.Context, userID string, groupIDs []string, deleted bool) error + FindMemberIncrVersion(ctx context.Context, groupID string, version uint, limit int) (*model.VersionLog, error) + FindJoinIncrVersion(ctx context.Context, userID string, version uint, limit int) (*model.VersionLog, error) } diff --git a/pkg/common/storage/database/mgo/friend.go b/pkg/common/storage/database/mgo/friend.go index 2df85a430..83daf9ecd 100644 --- a/pkg/common/storage/database/mgo/friend.go +++ b/pkg/common/storage/database/mgo/friend.go @@ -45,7 +45,7 @@ func NewFriendMongo(db *mongo.Database) (database.Friend, error) { if err != nil { return nil, err } - owner, err := NewVersionLog(db.Collection("friend_owner_version_log")) + owner, err := NewVersionLog(db.Collection("friend_version")) if err != nil { return nil, err } @@ -62,7 +62,7 @@ func (f *FriendMgo) Create(ctx context.Context, friends []*model.Friend) error { mp[friend.OwnerUserID] = append(mp[friend.OwnerUserID], friend.FriendUserID) } for ownerUserID, friendUserIDs := range mp { - if err := f.owner.WriteLog(ctx, ownerUserID, friendUserIDs, false); err != nil { + if err := f.owner.IncrVersion(ctx, ownerUserID, friendUserIDs, false); err != nil { return err } } @@ -79,7 +79,7 @@ func (f *FriendMgo) Delete(ctx context.Context, ownerUserID string, friendUserID return mongoutil.IncrVersion(func() error { return mongoutil.DeleteOne(ctx, f.coll, filter) }, func() error { - return f.owner.WriteLog(ctx, ownerUserID, friendUserIDs, true) + return f.owner.IncrVersion(ctx, ownerUserID, friendUserIDs, true) }) } @@ -95,7 +95,7 @@ func (f *FriendMgo) UpdateByMap(ctx context.Context, ownerUserID string, friendU return mongoutil.IncrVersion(func() error { return mongoutil.UpdateOne(ctx, f.coll, filter, bson.M{"$set": args}, true) }, func() error { - return f.owner.WriteLog(ctx, ownerUserID, []string{friendUserID}, false) + return f.owner.IncrVersion(ctx, ownerUserID, []string{friendUserID}, false) }) } @@ -185,7 +185,7 @@ func (f *FriendMgo) UpdateFriends(ctx context.Context, ownerUserID string, frien return mongoutil.IncrVersion(func() error { return mongoutil.Ignore(mongoutil.UpdateMany(ctx, f.coll, filter, update)) }, func() error { - return f.owner.WriteLog(ctx, ownerUserID, friendUserIDs, false) + return f.owner.IncrVersion(ctx, ownerUserID, friendUserIDs, false) }) } diff --git a/pkg/common/storage/database/mgo/group_member.go b/pkg/common/storage/database/mgo/group_member.go index ccca386e5..372babc9f 100644 --- a/pkg/common/storage/database/mgo/group_member.go +++ b/pkg/common/storage/database/mgo/group_member.go @@ -40,15 +40,53 @@ func NewGroupMember(db *mongo.Database) (database.GroupMember, error) { if err != nil { return nil, errs.Wrap(err) } - return &GroupMemberMgo{coll: coll}, nil + member, err := NewVersionLog(db.Collection("group_member_version")) + if err != nil { + return nil, err + } + join, err := NewVersionLog(db.Collection("group_join_version")) + if err != nil { + return nil, err + } + return &GroupMemberMgo{coll: coll, member: member, join: join}, nil } type GroupMemberMgo struct { - coll *mongo.Collection + coll *mongo.Collection + member database.VersionLog + join database.VersionLog +} + +func (g *GroupMemberMgo) sortBson() any { + return bson.D{{"role_level", -1}, {"create_time", -1}} } func (g *GroupMemberMgo) Create(ctx context.Context, groupMembers []*model.GroupMember) (err error) { - return mongoutil.InsertMany(ctx, g.coll, groupMembers) + return mongoutil.IncrVersion(func() error { + return mongoutil.InsertMany(ctx, g.coll, groupMembers) + }, func() error { + gms := make(map[string][]string) + for _, member := range groupMembers { + gms[member.GroupID] = append(gms[member.GroupID], member.UserID) + } + for groupID, userIDs := range gms { + if err := g.member.IncrVersion(ctx, groupID, userIDs, false); err != nil { + return err + } + } + return nil + }, func() error { + gms := make(map[string][]string) + for _, member := range groupMembers { + gms[member.UserID] = append(gms[member.UserID], member.GroupID) + } + for userID, groupIDs := range gms { + if err := g.join.IncrVersion(ctx, userID, groupIDs, false); err != nil { + return err + } + } + return nil + }) } func (g *GroupMemberMgo) Delete(ctx context.Context, groupID string, userIDs []string) (err error) { @@ -56,24 +94,41 @@ func (g *GroupMemberMgo) Delete(ctx context.Context, groupID string, userIDs []s if len(userIDs) > 0 { filter["user_id"] = bson.M{"$in": userIDs} } - return mongoutil.DeleteMany(ctx, g.coll, filter) + return mongoutil.IncrVersion(func() error { + return mongoutil.DeleteMany(ctx, g.coll, filter) + }, func() error { + return g.member.IncrVersion(ctx, groupID, userIDs, true) + }, func() error { + if len(userIDs) == 0 { + return nil + } + return g.member.IncrVersion(ctx, groupID, userIDs, true) + }) } func (g *GroupMemberMgo) UpdateRoleLevel(ctx context.Context, groupID string, userID string, roleLevel int32) error { - return g.Update(ctx, groupID, userID, bson.M{"role_level": roleLevel}) + return mongoutil.IncrVersion(func() error { + return g.Update(ctx, groupID, userID, bson.M{"role_level": roleLevel}) + }, func() error { + return g.member.IncrVersion(ctx, groupID, []string{userID}, true) + }, func() error { + return g.join.IncrVersion(ctx, groupID, []string{userID}, true) + }) } func (g *GroupMemberMgo) Update(ctx context.Context, groupID string, userID string, data map[string]any) (err error) { - return mongoutil.UpdateOne(ctx, g.coll, bson.M{"group_id": groupID, "user_id": userID}, bson.M{"$set": data}, true) -} - -func (g *GroupMemberMgo) Find(ctx context.Context, groupIDs []string, userIDs []string, roleLevels []int32) (groupMembers []*model.GroupMember, err error) { - // TODO implement me - panic("implement me") + if len(data) == 0 { + return nil + } + return mongoutil.IncrVersion(func() error { + return mongoutil.UpdateOne(ctx, g.coll, bson.M{"group_id": groupID, "user_id": userID}, bson.M{"$set": data}, true) + }, func() error { + return g.member.IncrVersion(ctx, groupID, []string{userID}, false) + }) } func (g *GroupMemberMgo) FindMemberUserID(ctx context.Context, groupID string) (userIDs []string, err error) { - return mongoutil.Find[string](ctx, g.coll, bson.M{"group_id": groupID}, options.Find().SetProjection(bson.M{"_id": 0, "user_id": 1})) + return mongoutil.Find[string](ctx, g.coll, bson.M{"group_id": groupID}, options.Find().SetProjection(bson.M{"_id": 0, "user_id": 1}).SetSort(g.sortBson())) } func (g *GroupMemberMgo) Take(ctx context.Context, groupID string, userID string) (groupMember *model.GroupMember, err error) { @@ -90,11 +145,11 @@ func (g *GroupMemberMgo) FindRoleLevelUserIDs(ctx context.Context, groupID strin func (g *GroupMemberMgo) SearchMember(ctx context.Context, keyword string, groupID string, pagination pagination.Pagination) (total int64, groupList []*model.GroupMember, err error) { filter := bson.M{"group_id": groupID, "nickname": bson.M{"$regex": keyword}} - return mongoutil.FindPage[*model.GroupMember](ctx, g.coll, filter, pagination) + return mongoutil.FindPage[*model.GroupMember](ctx, g.coll, filter, pagination, options.Find().SetSort(g.sortBson())) } func (g *GroupMemberMgo) FindUserJoinedGroupID(ctx context.Context, userID string) (groupIDs []string, err error) { - return mongoutil.Find[string](ctx, g.coll, bson.M{"user_id": userID}, options.Find().SetProjection(bson.M{"_id": 0, "group_id": 1})) + return mongoutil.Find[string](ctx, g.coll, bson.M{"user_id": userID}, options.Find().SetProjection(bson.M{"_id": 0, "group_id": 1}).SetSort(g.sortBson())) } func (g *GroupMemberMgo) TakeGroupMemberNum(ctx context.Context, groupID string) (count int64, err error) { @@ -118,3 +173,15 @@ func (g *GroupMemberMgo) IsUpdateRoleLevel(data map[string]any) bool { _, ok := data["role_level"] return ok } + +func (g *GroupMemberMgo) JoinGroupIncrVersion(ctx context.Context, userID string, groupIDs []string, deleted bool) error { + return g.join.IncrVersion(ctx, userID, groupIDs, deleted) +} + +func (g *GroupMemberMgo) FindMemberIncrVersion(ctx context.Context, groupID string, version uint, limit int) (*model.VersionLog, error) { + return g.member.FindChangeLog(ctx, groupID, version, limit) +} + +func (g *GroupMemberMgo) FindJoinIncrVersion(ctx context.Context, userID string, version uint, limit int) (*model.VersionLog, error) { + return g.join.FindChangeLog(ctx, userID, version, limit) +} diff --git a/pkg/common/storage/database/mgo/version_log.go b/pkg/common/storage/database/mgo/version_log.go index fe123073e..5629c5c00 100644 --- a/pkg/common/storage/database/mgo/version_log.go +++ b/pkg/common/storage/database/mgo/version_log.go @@ -36,7 +36,7 @@ func (l *VersionLogMgo) initIndex(ctx context.Context) error { return err } -func (l *VersionLogMgo) WriteLog(ctx context.Context, dId string, eIds []string, deleted bool) error { +func (l *VersionLogMgo) IncrVersion(ctx context.Context, dId string, eIds []string, deleted bool) error { if len(eIds) == 0 { return errs.ErrArgs.WrapMsg("elem id is empty", "dId", dId) } diff --git a/pkg/common/storage/database/version_log.go b/pkg/common/storage/database/version_log.go index 6783cf4ff..3450d2776 100644 --- a/pkg/common/storage/database/version_log.go +++ b/pkg/common/storage/database/version_log.go @@ -12,7 +12,7 @@ const ( ) type VersionLog interface { - WriteLog(ctx context.Context, dId string, eIds []string, deleted bool) error + IncrVersion(ctx context.Context, dId string, eIds []string, deleted bool) error FindChangeLog(ctx context.Context, dId string, version uint, limit int) (*model.VersionLog, error) DeleteAfterUnchangedLog(ctx context.Context, deadline time.Time) error } From e9f46271ba13c5b379f743cad49f1c92c0bb37ec Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Mon, 3 Jun 2024 14:30:26 +0800 Subject: [PATCH 18/59] sync --- internal/api/group.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/internal/api/group.go b/internal/api/group.go index 7baa16323..62e508d68 100644 --- a/internal/api/group.go +++ b/internal/api/group.go @@ -20,6 +20,7 @@ import ( "github.com/openimsdk/protocol/group" "github.com/openimsdk/tools/a2r" "github.com/openimsdk/tools/apiresp" + "github.com/openimsdk/tools/log" ) type GroupApi rpcclient.Group @@ -145,7 +146,7 @@ func (o *GroupApi) GetIncrementalGroupMember(c *gin.Context) { } func (o *GroupApi) GetIncrementalGroupMemberBatch(c *gin.Context) { - type BatchIncrementalReq[A any] struct { + type BatchIncrementalReq struct { UserID string `json:"user_id"` List []*group.GetIncrementalGroupMemberReq `json:"list"` } @@ -172,6 +173,7 @@ func (o *GroupApi) GetIncrementalGroupMemberBatch(c *gin.Context) { if len(resp.List) == 0 { apiresp.GinError(c, err) } else { + log.ZError(c, "group incr sync versopn", err, "groupID", req.GroupID, "success", len(resp.List)) apiresp.GinSuccess(c, resp) } return From 6363358ad41a52e11aa0e9e86d3e7e33d4bd84f5 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Mon, 3 Jun 2024 15:32:20 +0800 Subject: [PATCH 19/59] sync --- cmd/openim-rpc/openim-rpc-friend/main.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/cmd/openim-rpc/openim-rpc-friend/main.go b/cmd/openim-rpc/openim-rpc-friend/main.go index 4589fb30d..745c40553 100644 --- a/cmd/openim-rpc/openim-rpc-friend/main.go +++ b/cmd/openim-rpc/openim-rpc-friend/main.go @@ -17,13 +17,9 @@ package main import ( "github.com/openimsdk/open-im-server/v3/pkg/common/cmd" "github.com/openimsdk/tools/system/program" - "os" ) func main() { - if len(os.Args) == 1 { - os.Args = []string{os.Args[0], "-i", "0", "-c", "/Users/chao/Desktop/project/open-im-server/config"} - } if err := cmd.NewFriendRpcCmd().Exec(); err != nil { program.ExitWithError(err) } From 1b5621bc0933369c028885f1861141439d384298 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Mon, 3 Jun 2024 18:44:42 +0800 Subject: [PATCH 20/59] group sync --- internal/rpc/group/group.go | 2 +- pkg/common/storage/cache/redis/group.go | 2 ++ pkg/common/storage/controller/group.go | 3 ++- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/internal/rpc/group/group.go b/internal/rpc/group/group.go index 9b9ef07c1..94262da2c 100644 --- a/internal/rpc/group/group.go +++ b/internal/rpc/group/group.go @@ -104,7 +104,7 @@ func Start(ctx context.Context, config *Config, client discovery.SvcDiscoveryReg msgRpcClient := rpcclient.NewMessageRpcClient(client, config.Share.RpcRegisterName.Msg) conversationRpcClient := rpcclient.NewConversationRpcClient(client, config.Share.RpcRegisterName.Conversation) var gs groupServer - database := controller.NewGroupDatabase(rdb, &config.LocalCacheConfig, groupDB, groupMemberDB, groupRequestDB, mgocli.GetTx(), grouphash.NewGroupHashFromGroupServer(&gs)) + database := controller.NewGroupDatabase(rdb, &config.LocalCacheConfig, groupDB, groupMemberDB, groupRequestDB, mgocli.GetTx(), grouphash.NewGroupHashFromGroupServer(&gs), config.RpcConfig.GroupSyncCount) gs.db = database gs.user = userRpcClient gs.notification = NewGroupNotificationSender(database, &msgRpcClient, &userRpcClient, config, func(ctx context.Context, userIDs []string) ([]notification.CommonUser, error) { diff --git a/pkg/common/storage/cache/redis/group.go b/pkg/common/storage/cache/redis/group.go index 7f0ba62e4..1dcace5ed 100644 --- a/pkg/common/storage/cache/redis/group.go +++ b/pkg/common/storage/cache/redis/group.go @@ -57,6 +57,7 @@ func NewGroupCacheRedis( groupRequestDB database.GroupRequest, hashCode cache.GroupHash, opts *rockscache.Options, + syncCount int, ) cache.GroupCache { batchHandler := NewBatchDeleterRedis(rdb, opts, []string{localCache.Group.Topic}) g := localCache.Group @@ -70,6 +71,7 @@ func NewGroupCacheRedis( groupMemberDB: groupMemberDB, groupRequestDB: groupRequestDB, groupHash: hashCode, + syncCount: syncCount, } } diff --git a/pkg/common/storage/controller/group.go b/pkg/common/storage/controller/group.go index c32f009dd..640ded6bd 100644 --- a/pkg/common/storage/controller/group.go +++ b/pkg/common/storage/controller/group.go @@ -123,13 +123,14 @@ func NewGroupDatabase( groupRequestDB database.GroupRequest, ctxTx tx.Tx, groupHash cache.GroupHash, + syncCount int, ) GroupDatabase { return &groupDatabase{ groupDB: groupDB, groupMemberDB: groupMemberDB, groupRequestDB: groupRequestDB, ctxTx: ctxTx, - cache: redis2.NewGroupCacheRedis(rdb, localCache, groupDB, groupMemberDB, groupRequestDB, groupHash, redis2.GetRocksCacheOptions()), + cache: redis2.NewGroupCacheRedis(rdb, localCache, groupDB, groupMemberDB, groupRequestDB, groupHash, redis2.GetRocksCacheOptions(), syncCount), } } From 693935237a87a5a46d7aa8e76aac4a370c918569 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Tue, 4 Jun 2024 17:31:49 +0800 Subject: [PATCH 21/59] sync option --- internal/rpc/friend/sync.go | 62 ++------- internal/rpc/group/sync.go | 44 +++---- internal/rpc/incrversion/option.go | 131 +++++++++++++++++--- pkg/common/storage/cache/cachekey/friend.go | 23 ++-- pkg/common/storage/cache/cachekey/group.go | 10 ++ pkg/common/storage/cache/friend.go | 8 +- pkg/common/storage/cache/group.go | 5 + pkg/common/storage/cache/redis/friend.go | 53 +++++--- pkg/common/storage/cache/redis/group.go | 40 ++++++ pkg/common/storage/controller/friend.go | 37 ++++-- pkg/common/storage/controller/group.go | 79 ++++++++---- pkg/common/storage/model/version_log.go | 10 +- 12 files changed, 336 insertions(+), 166 deletions(-) diff --git a/internal/rpc/friend/sync.go b/internal/rpc/friend/sync.go index bdea7b9eb..a99b0f220 100644 --- a/internal/rpc/friend/sync.go +++ b/internal/rpc/friend/sync.go @@ -62,23 +62,22 @@ func (s *friendServer) GetIncrementalFriends(ctx context.Context, req *pbfriend. return nil, err } opt := incrversion.Option[*pbfriend.FriendInfo, pbfriend.GetIncrementalFriendsResp]{ - VersionID: req.VersionID, - Version: func() (*model.VersionLog, error) { - return s.friendDatabase.FindFriendIncrVersion(ctx, req.UserID, uint(req.Version), incrversion.Limit(s.config.RpcConfig.FriendSyncCount, req.Version)) - }, - AllID: func() ([]string, error) { - return s.friendDatabase.FindSortFriendUserIDs(ctx, req.UserID) - }, - Find: func(ids []string) ([]*pbfriend.FriendInfo, error) { + Ctx: ctx, + VersionKey: req.UserID, + VersionID: req.VersionID, + VersionNumber: req.Version, + SyncLimit: s.config.RpcConfig.FriendSyncCount, + Version: s.friendDatabase.FindFriendIncrVersion, + CacheMaxVersion: s.friendDatabase.FindMaxFriendVersionCache, + SortID: s.friendDatabase.FindSortFriendUserIDs, + Find: func(ctx context.Context, ids []string) ([]*pbfriend.FriendInfo, error) { friends, err := s.friendDatabase.FindFriendsWithError(ctx, req.UserID, ids) if err != nil { return nil, err } return friendsDB2PB(friends), nil }, - ID: func(elem *pbfriend.FriendInfo) string { - return elem.FriendUserID - }, + ID: func(elem *pbfriend.FriendInfo) string { return elem.FriendUserID }, Resp: func(version *model.VersionLog, delIDs []string, list []*pbfriend.FriendInfo, full bool) *pbfriend.GetIncrementalFriendsResp { return &pbfriend.GetIncrementalFriendsResp{ VersionID: version.ID.Hex(), @@ -92,44 +91,3 @@ func (s *friendServer) GetIncrementalFriends(ctx context.Context, req *pbfriend. } return opt.Build() } - -//func (s *friendServer) GetIncrementalFriends(ctx context.Context, req *pbfriend.GetIncrementalFriendsReq) (*pbfriend.GetIncrementalFriendsResp, error) { -// if err := authverify.CheckAccessV3(ctx, req.UserID, s.config.Share.IMAdminUserID); err != nil { -// return nil, err -// } -// var limit int -// if req.Version > 0 { -// limit = s.config.RpcConfig.FriendSyncCount -// } -// incrVer, err := s.friendDatabase.FindFriendIncrVersion(ctx, req.UserID, uint(req.Version), limit) -// if err != nil { -// return nil, err -// } -// var ( -// deleteUserIDs []string -// changeUserIDs []string -// ) -// if incrVer.Full() { -// changeUserIDs, err = s.friendDatabase.FindSortFriendUserIDs(ctx, req.UserID) -// if err != nil { -// return nil, err -// } -// } else { -// deleteUserIDs, changeUserIDs = incrVer.DeleteAndChangeIDs() -// } -// var friends []*model.Friend -// if len(changeUserIDs) > 0 { -// friends, err = s.friendDatabase.FindFriendsWithError(ctx, req.UserID, changeUserIDs) -// if err != nil { -// return nil, err -// } -// } -// return &pbfriend.GetIncrementalFriendsResp{ -// Version: uint64(incrVer.Version), -// VersionID: incrVer.ID.Hex(), -// Full: incrVer.Full(), -// SyncCount: uint32(s.config.RpcConfig.FriendSyncCount), -// DeleteUserIds: deleteUserIDs, -// Changes: friendsDB2PB(friends), -// }, nil -//} diff --git a/internal/rpc/group/sync.go b/internal/rpc/group/sync.go index 80cb5a9ea..445ba6874 100644 --- a/internal/rpc/group/sync.go +++ b/internal/rpc/group/sync.go @@ -16,19 +16,18 @@ func (s *groupServer) SearchGroupMember(ctx context.Context, req *pbgroup.Search func (s *groupServer) GetIncrementalGroupMember(ctx context.Context, req *pbgroup.GetIncrementalGroupMemberReq) (*pbgroup.GetIncrementalGroupMemberResp, error) { opt := incrversion.Option[*sdkws.GroupMemberFullInfo, pbgroup.GetIncrementalGroupMemberResp]{ - VersionID: req.VersionID, - Version: func() (*model.VersionLog, error) { - return s.db.FindMemberIncrVersion(ctx, req.GroupID, uint(req.Version), incrversion.Limit(s.config.RpcConfig.GroupSyncCount, req.Version)) - }, - AllID: func() ([]string, error) { - return s.db.FindSortGroupMemberUserIDs(ctx, req.GroupID) - }, - Find: func(ids []string) ([]*sdkws.GroupMemberFullInfo, error) { + Ctx: ctx, + VersionKey: req.GroupID, + VersionID: req.VersionID, + VersionNumber: req.Version, + SyncLimit: s.config.RpcConfig.GroupSyncCount, + Version: s.db.FindMemberIncrVersion, + CacheMaxVersion: s.db.FindMaxGroupMemberVersionCache, + SortID: s.db.FindSortGroupMemberUserIDs, + Find: func(ctx context.Context, ids []string) ([]*sdkws.GroupMemberFullInfo, error) { return s.getGroupMembersInfo(ctx, req.GroupID, ids) }, - ID: func(elem *sdkws.GroupMemberFullInfo) string { - return elem.UserID - }, + ID: func(elem *sdkws.GroupMemberFullInfo) string { return elem.UserID }, Resp: func(version *model.VersionLog, delIDs []string, list []*sdkws.GroupMemberFullInfo, full bool) *pbgroup.GetIncrementalGroupMemberResp { return &pbgroup.GetIncrementalGroupMemberResp{ VersionID: version.ID.Hex(), @@ -48,19 +47,16 @@ func (s *groupServer) GetIncrementalJoinGroup(ctx context.Context, req *pbgroup. return nil, err } opt := incrversion.Option[*sdkws.GroupInfo, pbgroup.GetIncrementalJoinGroupResp]{ - VersionID: req.VersionID, - Version: func() (*model.VersionLog, error) { - return s.db.FindJoinIncrVersion(ctx, req.UserID, uint(req.Version), incrversion.Limit(s.config.RpcConfig.GroupSyncCount, req.Version)) - }, - AllID: func() ([]string, error) { - return s.db.FindSortJoinGroupIDs(ctx, req.UserID) - }, - Find: func(ids []string) ([]*sdkws.GroupInfo, error) { - return s.getGroupsInfo(ctx, ids) - }, - ID: func(elem *sdkws.GroupInfo) string { - return elem.GroupID - }, + Ctx: ctx, + VersionKey: req.UserID, + VersionID: req.VersionID, + VersionNumber: req.Version, + SyncLimit: s.config.RpcConfig.GroupSyncCount, + Version: s.db.FindJoinIncrVersion, + CacheMaxVersion: s.db.FindMaxJoinGroupVersionCache, + SortID: s.db.FindSortJoinGroupIDs, + Find: s.getGroupsInfo, + ID: func(elem *sdkws.GroupInfo) string { return elem.GroupID }, Resp: func(version *model.VersionLog, delIDs []string, list []*sdkws.GroupInfo, full bool) *pbgroup.GetIncrementalJoinGroupResp { return &pbgroup.GetIncrementalJoinGroupResp{ VersionID: version.ID.Hex(), diff --git a/internal/rpc/incrversion/option.go b/internal/rpc/incrversion/option.go index 3e19056bf..3146985aa 100644 --- a/internal/rpc/incrversion/option.go +++ b/internal/rpc/incrversion/option.go @@ -1,38 +1,139 @@ package incrversion import ( + "context" + "fmt" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" + "github.com/openimsdk/tools/errs" "github.com/openimsdk/tools/utils/datautil" + "go.mongodb.org/mongo-driver/bson/primitive" ) -func Limit(maxSync int, version uint64) int { - if version == 0 { - return 0 +//func Limit(maxSync int, version uint64) int { +// if version == 0 { +// return 0 +// } +// return maxSync +//} + +const ( + tagQuery = iota + 1 + tagFull + tageEqual +) + +type Option[A, B any] struct { + Ctx context.Context + VersionKey string + VersionID string + VersionNumber uint64 + SyncLimit int + CacheMaxVersion func(ctx context.Context, dId string) (*model.VersionLog, error) + Version func(ctx context.Context, dId string, version uint, limit int) (*model.VersionLog, error) + SortID func(ctx context.Context, dId string) ([]string, error) + Find func(ctx context.Context, ids []string) ([]A, error) + ID func(elem A) string + Resp func(version *model.VersionLog, delIDs []string, list []A, full bool) *B +} + +func (o *Option[A, B]) newError(msg string) error { + return errs.ErrInternalServer.WrapMsg(msg) +} + +func (o *Option[A, B]) check() error { + if o.Ctx == nil { + return o.newError("opt ctx is nil") } - return maxSync + if o.VersionKey == "" { + return o.newError("versionKey is empty") + } + if o.SyncLimit <= 0 { + return o.newError("invalid synchronization quantity") + } + if o.Version == nil { + return o.newError("func version is nil") + } + if o.SortID == nil { + return o.newError("func allID is nil") + } + if o.Find == nil { + return o.newError("func find is nil") + } + if o.ID == nil { + return o.newError("func id is nil") + } + if o.Resp == nil { + return o.newError("func resp is nil") + } + return nil } -type Option[A, B any] struct { - VersionID string - Version func() (*model.VersionLog, error) - AllID func() ([]string, error) - Find func(ids []string) ([]A, error) - ID func(elem A) string - Resp func(version *model.VersionLog, delIDs []string, list []A, full bool) *B +func (o *Option[A, B]) validVersion() bool { + objID, err := primitive.ObjectIDFromHex(o.VersionID) + return err == nil && (!objID.IsZero()) && o.VersionNumber > 0 +} + +func (o *Option[A, B]) equalID(objID primitive.ObjectID) bool { + return o.VersionID == objID.Hex() +} + +func (o *Option[A, B]) getVersion(tag *int) (*model.VersionLog, error) { + if o.CacheMaxVersion == nil { + if o.validVersion() { + *tag = tagQuery + return o.Version(o.Ctx, o.VersionKey, uint(o.VersionNumber), o.SyncLimit) + } + *tag = tagFull + return o.Version(o.Ctx, o.VersionKey, 0, 0) + } else { + cache, err := o.CacheMaxVersion(o.Ctx, o.VersionKey) + if err != nil { + return nil, err + } + if !o.validVersion() { + *tag = tagFull + return cache, nil + } + if !o.equalID(cache.ID) { + *tag = tagFull + return cache, nil + } + if o.VersionNumber == uint64(cache.Version) { + *tag = tageEqual + return cache, nil + } + *tag = tagQuery + return o.Version(o.Ctx, o.VersionKey, uint(o.VersionNumber), o.SyncLimit) + } } func (o *Option[A, B]) Build() (*B, error) { - version, err := o.Version() + if err := o.check(); err != nil { + return nil, err + } + var tag int + version, err := o.getVersion(&tag) if err != nil { return nil, err } + var full bool + switch tag { + case tagQuery: + full = version.ID.Hex() != o.VersionID || uint64(version.Version) < o.VersionNumber || len(version.Logs) != version.LogLen + case tagFull: + full = true + case tageEqual: + full = false + default: + panic(fmt.Errorf("undefined tag %d", tag)) + } var ( deleteIDs []string changeIDs []string ) - full := o.VersionID != version.ID.Hex() || version.Full() + //full := o.VersionID != version.ID.Hex() || version.Full() if full { - changeIDs, err = o.AllID() + changeIDs, err = o.SortID(o.Ctx, o.VersionKey) if err != nil { return nil, err } @@ -41,7 +142,7 @@ func (o *Option[A, B]) Build() (*B, error) { } var list []A if len(changeIDs) > 0 { - list, err = o.Find(changeIDs) + list, err = o.Find(o.Ctx, changeIDs) if err != nil { return nil, err } diff --git a/pkg/common/storage/cache/cachekey/friend.go b/pkg/common/storage/cache/cachekey/friend.go index 6a217bdef..8a053ca32 100644 --- a/pkg/common/storage/cache/cachekey/friend.go +++ b/pkg/common/storage/cache/cachekey/friend.go @@ -14,14 +14,13 @@ package cachekey -import "strconv" - const ( - FriendIDsKey = "FRIEND_IDS:" - TwoWayFriendsIDsKey = "COMMON_FRIENDS_IDS:" - FriendKey = "FRIEND_INFO:" - IsFriendKey = "IS_FRIEND:" // local cache key - FriendSyncSortUserIDsKey = "FRIEND_SYNC_SORT_USER_IDS:" + FriendIDsKey = "FRIEND_IDS:" + TwoWayFriendsIDsKey = "COMMON_FRIENDS_IDS:" + FriendKey = "FRIEND_INFO:" + IsFriendKey = "IS_FRIEND:" // local cache key + //FriendSyncSortUserIDsKey = "FRIEND_SYNC_SORT_USER_IDS:" + FriendMaxVersionKey = "FRIEND_MAX_VERSION:" ) func GetFriendIDsKey(ownerUserID string) string { @@ -36,10 +35,14 @@ func GetFriendKey(ownerUserID, friendUserID string) string { return FriendKey + ownerUserID + "-" + friendUserID } +func GetFriendMaxVersionKey(ownerUserID string) string { + return FriendMaxVersionKey + ownerUserID +} + func GetIsFriendKey(possibleFriendUserID, userID string) string { return IsFriendKey + possibleFriendUserID + "-" + userID } -func GetFriendSyncSortUserIDsKey(ownerUserID string, count int) string { - return FriendSyncSortUserIDsKey + strconv.Itoa(count) + ":" + ownerUserID -} +//func GetFriendSyncSortUserIDsKey(ownerUserID string, count int) string { +// return FriendSyncSortUserIDsKey + strconv.Itoa(count) + ":" + ownerUserID +//} diff --git a/pkg/common/storage/cache/cachekey/group.go b/pkg/common/storage/cache/cachekey/group.go index 681121ecb..2ef42c0ff 100644 --- a/pkg/common/storage/cache/cachekey/group.go +++ b/pkg/common/storage/cache/cachekey/group.go @@ -28,6 +28,8 @@ const ( JoinedGroupsKey = "JOIN_GROUPS_KEY:" GroupMemberNumKey = "GROUP_MEMBER_NUM_CACHE:" GroupRoleLevelMemberIDsKey = "GROUP_ROLE_LEVEL_MEMBER_IDS:" + GroupMemberMaxVersionKey = "GROUP_MEMBER_MAX_VERSION:" + GroupJoinMaxVersionKey = "GROUP_JOIN_MAX_VERSION:" ) func GetGroupInfoKey(groupID string) string { @@ -57,3 +59,11 @@ func GetGroupMemberNumKey(groupID string) string { func GetGroupRoleLevelMemberIDsKey(groupID string, roleLevel int32) string { return GroupRoleLevelMemberIDsKey + groupID + "-" + strconv.Itoa(int(roleLevel)) } + +func GetGroupMemberMaxVersionKey(groupID string) string { + return GroupMemberMaxVersionKey + groupID +} + +func GetJoinGroupMaxVersionKey(userID string) string { + return GroupJoinMaxVersionKey + userID +} diff --git a/pkg/common/storage/cache/friend.go b/pkg/common/storage/cache/friend.go index 3fee297ac..064e3baae 100644 --- a/pkg/common/storage/cache/friend.go +++ b/pkg/common/storage/cache/friend.go @@ -35,9 +35,13 @@ type FriendCache interface { DelOwner(friendUserID string, ownerUserIDs []string) FriendCache - DelSortFriendUserIDs(ownerUserIDs ...string) FriendCache + DelMaxFriendVersion(ownerUserIDs ...string) FriendCache + + //DelSortFriendUserIDs(ownerUserIDs ...string) FriendCache FindSortFriendUserIDs(ctx context.Context, ownerUserID string) ([]string, error) - FindFriendIncrVersion(ctx context.Context, ownerUserID string, version uint, limit int) (*relationtb.VersionLog, error) + //FindFriendIncrVersion(ctx context.Context, ownerUserID string, version uint, limit int) (*relationtb.VersionLog, error) + + FindMaxFriendVersion(ctx context.Context, ownerUserID string) (*relationtb.VersionLog, error) } diff --git a/pkg/common/storage/cache/group.go b/pkg/common/storage/cache/group.go index ff840a4a5..f02379a7d 100644 --- a/pkg/common/storage/cache/group.go +++ b/pkg/common/storage/cache/group.go @@ -62,4 +62,9 @@ type GroupCache interface { FindSortGroupMemberUserIDs(ctx context.Context, groupID string) ([]string, error) FindSortJoinGroupIDs(ctx context.Context, userID string) ([]string, error) + + DelMaxGroupMemberVersion(groupIDs ...string) GroupCache + DelMaxJoinGroupVersion(userIDs ...string) GroupCache + FindMaxGroupMemberVersion(ctx context.Context, groupID string) (*model.VersionLog, error) + FindMaxJoinGroupVersion(ctx context.Context, userID string) (*model.VersionLog, error) } diff --git a/pkg/common/storage/cache/redis/friend.go b/pkg/common/storage/cache/redis/friend.go index f4edbca9a..62fad91cd 100644 --- a/pkg/common/storage/cache/redis/friend.go +++ b/pkg/common/storage/cache/redis/friend.go @@ -70,8 +70,12 @@ func (f *FriendCacheRedis) getFriendIDsKey(ownerUserID string) string { return cachekey.GetFriendIDsKey(ownerUserID) } -func (f *FriendCacheRedis) getFriendSyncSortUserIDsKey(ownerUserID string) string { - return cachekey.GetFriendSyncSortUserIDsKey(ownerUserID, f.syncCount) +//func (f *FriendCacheRedis) getFriendSyncSortUserIDsKey(ownerUserID string) string { +// return cachekey.GetFriendSyncSortUserIDsKey(ownerUserID, f.syncCount) +//} + +func (f *FriendCacheRedis) getFriendMaxVersionKey(ownerUserID string) string { + return cachekey.GetFriendMaxVersionKey(ownerUserID) } // getTwoWayFriendsIDsKey returns the key for storing two-way friend IDs in the cache. @@ -103,15 +107,15 @@ func (f *FriendCacheRedis) DelFriendIDs(ownerUserIDs ...string) cache.FriendCach return newFriendCache } -func (f *FriendCacheRedis) DelSortFriendUserIDs(ownerUserIDs ...string) cache.FriendCache { - newGroupCache := f.CloneFriendCache() - keys := make([]string, 0, len(ownerUserIDs)) - for _, userID := range ownerUserIDs { - keys = append(keys, f.getFriendSyncSortUserIDsKey(userID)) - } - newGroupCache.AddKeys(keys...) - return newGroupCache -} +//func (f *FriendCacheRedis) DelSortFriendUserIDs(ownerUserIDs ...string) cache.FriendCache { +// newGroupCache := f.CloneFriendCache() +// keys := make([]string, 0, len(ownerUserIDs)) +// for _, userID := range ownerUserIDs { +// keys = append(keys, f.getFriendSyncSortUserIDsKey(userID)) +// } +// newGroupCache.AddKeys(keys...) +// return newGroupCache +//} // GetTwoWayFriendIDs retrieves two-way friend IDs from the cache. func (f *FriendCacheRedis) GetTwoWayFriendIDs(ctx context.Context, ownerUserID string) (twoWayFriendIDs []string, err error) { @@ -179,12 +183,29 @@ func (f *FriendCacheRedis) DelOwner(friendUserID string, ownerUserIDs []string) return newFriendCache } +func (f *FriendCacheRedis) DelMaxFriendVersion(ownerUserIDs ...string) cache.FriendCache { + newFriendCache := f.CloneFriendCache() + for _, ownerUserID := range ownerUserIDs { + key := f.getFriendMaxVersionKey(ownerUserID) + newFriendCache.AddKeys(key) // Assuming AddKeys marks the keys for deletion + } + + return newFriendCache +} + func (f *FriendCacheRedis) FindSortFriendUserIDs(ctx context.Context, ownerUserID string) ([]string, error) { - return getCache(ctx, f.rcClient, f.getFriendSyncSortUserIDsKey(ownerUserID), f.expireTime, func(ctx context.Context) ([]string, error) { - return f.friendDB.FindOwnerFriendUserIds(ctx, ownerUserID, f.syncCount) - }) + userIDs, err := f.GetFriendIDs(ctx, ownerUserID) + if err != nil { + return nil, err + } + if len(userIDs) > f.syncCount { + userIDs = userIDs[:f.syncCount] + } + return userIDs, nil } -func (f *FriendCacheRedis) FindFriendIncrVersion(ctx context.Context, ownerUserID string, version uint, limit int) (*model.VersionLog, error) { - return f.friendDB.FindIncrVersion(ctx, ownerUserID, version, limit) +func (f *FriendCacheRedis) FindMaxFriendVersion(ctx context.Context, ownerUserID string) (*model.VersionLog, error) { + return getCache(ctx, f.rcClient, f.getFriendMaxVersionKey(ownerUserID), f.expireTime, func(ctx context.Context) (*model.VersionLog, error) { + return f.friendDB.FindIncrVersion(ctx, ownerUserID, 0, 0) + }) } diff --git a/pkg/common/storage/cache/redis/group.go b/pkg/common/storage/cache/redis/group.go index 1dcace5ed..cf88a6514 100644 --- a/pkg/common/storage/cache/redis/group.go +++ b/pkg/common/storage/cache/redis/group.go @@ -114,6 +114,14 @@ func (g *GroupCacheRedis) getGroupRoleLevelMemberIDsKey(groupID string, roleLeve return cachekey.GetGroupRoleLevelMemberIDsKey(groupID, roleLevel) } +func (g *GroupCacheRedis) getGroupMemberMaxVersionKey(groupID string) string { + return cachekey.GetGroupMemberMaxVersionKey(groupID) +} + +func (g *GroupCacheRedis) getJoinGroupMaxVersionKey(userID string) string { + return cachekey.GetJoinGroupMaxVersionKey(userID) +} + func (g *GroupCacheRedis) GetGroupIndex(group *model.Group, keys []string) (int, error) { key := g.getGroupInfoKey(group.GroupID) for i, _key := range keys { @@ -431,3 +439,35 @@ func (g *GroupCacheRedis) FindSortJoinGroupIDs(ctx context.Context, userID strin } return groupIDs, nil } + +func (g *GroupCacheRedis) DelMaxGroupMemberVersion(groupIDs ...string) cache.GroupCache { + keys := make([]string, 0, len(groupIDs)) + for _, groupID := range groupIDs { + keys = append(keys, g.getGroupMemberMaxVersionKey(groupID)) + } + cache := g.CloneGroupCache() + cache.AddKeys(keys...) + return cache +} + +func (g *GroupCacheRedis) DelMaxJoinGroupVersion(userIDs ...string) cache.GroupCache { + keys := make([]string, 0, len(userIDs)) + for _, userID := range userIDs { + keys = append(keys, g.getJoinGroupMaxVersionKey(userID)) + } + cache := g.CloneGroupCache() + cache.AddKeys(keys...) + return cache +} + +func (g *GroupCacheRedis) FindMaxGroupMemberVersion(ctx context.Context, groupID string) (*model.VersionLog, error) { + return getCache(ctx, g.rcClient, g.getGroupMemberMaxVersionKey(groupID), g.expireTime, func(ctx context.Context) (*model.VersionLog, error) { + return g.groupMemberDB.FindJoinIncrVersion(ctx, groupID, 0, 0) + }) +} + +func (g *GroupCacheRedis) FindMaxJoinGroupVersion(ctx context.Context, userID string) (*model.VersionLog, error) { + return getCache(ctx, g.rcClient, g.getJoinGroupMaxVersionKey(userID), g.expireTime, func(ctx context.Context) (*model.VersionLog, error) { + return g.groupMemberDB.FindJoinIncrVersion(ctx, userID, 0, 0) + }) +} diff --git a/pkg/common/storage/controller/friend.go b/pkg/common/storage/controller/friend.go index 1af967b9b..8f72703c0 100644 --- a/pkg/common/storage/controller/friend.go +++ b/pkg/common/storage/controller/friend.go @@ -82,6 +82,8 @@ type FriendDatabase interface { FindFriendIncrVersion(ctx context.Context, ownerUserID string, version uint, limit int) (*model.VersionLog, error) + FindMaxFriendVersionCache(ctx context.Context, ownerUserID string) (*model.VersionLog, error) + FindFriendUserID(ctx context.Context, friendUserID string) ([]string, error) UpdateFriendUserInfo(ctx context.Context, friendUserID string, ownerUserID []string, nickname string, faceURL string) error @@ -185,7 +187,7 @@ func (f *friendDatabase) BecomeFriends(ctx context.Context, ownerUserID string, return err } newFriendIDs = append(newFriendIDs, ownerUserID) - cache = cache.DelFriendIDs(newFriendIDs...).DelSortFriendUserIDs(ownerUserID) + cache = cache.DelFriendIDs(newFriendIDs...).DelMaxFriendVersion(newFriendIDs...) return cache.ChainExecDel(ctx) }) @@ -288,7 +290,7 @@ func (f *friendDatabase) AgreeFriendRequest(ctx context.Context, friendRequest * return err } } - return f.cache.DelFriendIDs(friendRequest.ToUserID, friendRequest.FromUserID).DelSortFriendUserIDs(friendRequest.ToUserID, friendRequest.FromUserID).ChainExecDel(ctx) + return f.cache.DelFriendIDs(friendRequest.ToUserID, friendRequest.FromUserID).DelMaxFriendVersion(friendRequest.ToUserID, friendRequest.FromUserID).ChainExecDel(ctx) }) } @@ -297,7 +299,8 @@ func (f *friendDatabase) Delete(ctx context.Context, ownerUserID string, friendU if err := f.friend.Delete(ctx, ownerUserID, friendUserIDs); err != nil { return err } - return f.cache.DelFriendIDs(append(friendUserIDs, ownerUserID)...).DelSortFriendUserIDs(ownerUserID).ChainExecDel(ctx) + userIds := append(friendUserIDs, ownerUserID) + return f.cache.DelFriendIDs(userIds...).DelMaxFriendVersion(userIds...).ChainExecDel(ctx) } // UpdateRemark updates the remark for a friend. Zero value for remark is also supported. @@ -305,7 +308,7 @@ func (f *friendDatabase) UpdateRemark(ctx context.Context, ownerUserID, friendUs if err := f.friend.UpdateRemark(ctx, ownerUserID, friendUserID, remark); err != nil { return err } - return f.cache.DelFriend(ownerUserID, friendUserID).ChainExecDel(ctx) + return f.cache.DelFriend(ownerUserID, friendUserID).DelMaxFriendVersion(ownerUserID).ChainExecDel(ctx) } // PageOwnerFriends retrieves the list of friends for the ownerUserID. It does not return an error if the result is empty. @@ -351,10 +354,12 @@ func (f *friendDatabase) UpdateFriends(ctx context.Context, ownerUserID string, if len(val) == 0 { return nil } - if err := f.friend.UpdateFriends(ctx, ownerUserID, friendUserIDs, val); err != nil { - return err - } - return f.cache.DelFriends(ownerUserID, friendUserIDs).DelSortFriendUserIDs(ownerUserID).ChainExecDel(ctx) + return f.tx.Transaction(ctx, func(ctx context.Context) error { + if err := f.friend.UpdateFriends(ctx, ownerUserID, friendUserIDs, val); err != nil { + return err + } + return f.cache.DelFriends(ownerUserID, friendUserIDs).DelMaxFriendVersion(ownerUserID).ChainExecDel(ctx) + }) } func (f *friendDatabase) FindSortFriendUserIDs(ctx context.Context, ownerUserID string) ([]string, error) { @@ -362,7 +367,11 @@ func (f *friendDatabase) FindSortFriendUserIDs(ctx context.Context, ownerUserID } func (f *friendDatabase) FindFriendIncrVersion(ctx context.Context, ownerUserID string, version uint, limit int) (*model.VersionLog, error) { - return f.cache.FindFriendIncrVersion(ctx, ownerUserID, version, limit) + return f.friend.FindIncrVersion(ctx, ownerUserID, version, limit) +} + +func (f *friendDatabase) FindMaxFriendVersionCache(ctx context.Context, ownerUserID string) (*model.VersionLog, error) { + return f.cache.FindMaxFriendVersion(ctx, ownerUserID) } func (f *friendDatabase) FindFriendUserID(ctx context.Context, friendUserID string) ([]string, error) { @@ -370,10 +379,12 @@ func (f *friendDatabase) FindFriendUserID(ctx context.Context, friendUserID stri } func (f *friendDatabase) UpdateFriendUserInfo(ctx context.Context, friendUserID string, ownerUserIDs []string, nickname string, faceURL string) error { - if err := f.friend.UpdateFriendUserInfo(ctx, friendUserID, nickname, faceURL); err != nil { - return err - } - return f.cache.DelOwner(friendUserID, ownerUserIDs).ChainExecDel(ctx) + return f.tx.Transaction(ctx, func(ctx context.Context) error { + if err := f.friend.UpdateFriendUserInfo(ctx, friendUserID, nickname, faceURL); err != nil { + return err + } + return f.cache.DelOwner(friendUserID, ownerUserIDs).DelMaxFriendVersion(ownerUserIDs...).ChainExecDel(ctx) + }) } func (f *friendDatabase) SearchFriend(ctx context.Context, ownerUserID, keyword string, pagination pagination.Pagination) (int64, []*model.Friend, error) { diff --git a/pkg/common/storage/controller/group.go b/pkg/common/storage/controller/group.go index 640ded6bd..9750b8b65 100644 --- a/pkg/common/storage/controller/group.go +++ b/pkg/common/storage/controller/group.go @@ -113,6 +113,9 @@ type GroupDatabase interface { FindSortGroupMemberUserIDs(ctx context.Context, groupID string) ([]string, error) FindSortJoinGroupIDs(ctx context.Context, userID string) ([]string, error) + + FindMaxGroupMemberVersionCache(ctx context.Context, groupID string) (*model.VersionLog, error) + FindMaxJoinGroupVersionCache(ctx context.Context, userID string) (*model.VersionLog, error) } func NewGroupDatabase( @@ -182,7 +185,8 @@ func (g *groupDatabase) CreateGroup(ctx context.Context, groups []*model.Group, DelGroupMembersHash(group.GroupID). DelGroupsMemberNum(group.GroupID). DelGroupMemberIDs(group.GroupID). - DelGroupAllRoleLevel(group.GroupID) + DelGroupAllRoleLevel(group.GroupID). + DelMaxGroupMemberVersion(group.GroupID) } } if len(groupMembers) > 0 { @@ -195,7 +199,9 @@ func (g *groupDatabase) CreateGroup(ctx context.Context, groups []*model.Group, DelGroupMemberIDs(groupMember.GroupID). DelJoinedGroupID(groupMember.UserID). DelGroupMembersInfo(groupMember.GroupID, groupMember.UserID). - DelGroupAllRoleLevel(groupMember.GroupID) + DelGroupAllRoleLevel(groupMember.GroupID). + DelMaxJoinGroupVersion(groupMember.UserID). + DelMaxGroupMemberVersion(groupMember.GroupID) } } return c.ChainExecDel(ctx) @@ -239,8 +245,9 @@ func (g *groupDatabase) UpdateGroup(ctx context.Context, groupID string, data ma if err := g.groupMemberDB.JoinGroupIncrVersion(ctx, userID, []string{groupID}, false); err != nil { return err } + } - return g.cache.DelGroupsInfo(groupID).ChainExecDel(ctx) + return g.cache.CloneGroupCache().DelGroupsInfo(groupID).DelMaxJoinGroupVersion(userIDs...).ChainExecDel(ctx) }) } @@ -263,8 +270,10 @@ func (g *groupDatabase) DismissGroup(ctx context.Context, groupID string, delete DelGroupsMemberNum(groupID). DelGroupMembersHash(groupID). DelGroupAllRoleLevel(groupID). - DelGroupMembersInfo(groupID, userIDs...) + DelGroupMembersInfo(groupID, userIDs...). + DelMaxGroupMemberVersion(groupID) } + c = c.DelMaxJoinGroupVersion(userIDs...) if len(userIDs) > 0 { if err := g.groupMemberDB.JoinGroupIncrVersion(ctx, groupID, userIDs, true); err != nil { return err @@ -340,7 +349,9 @@ func (g *groupDatabase) HandlerGroupRequest(ctx context.Context, groupID string, DelGroupMemberIDs(groupID). DelGroupsMemberNum(groupID). DelJoinedGroupID(member.UserID). - DelGroupRoleLevel(groupID, []int32{member.RoleLevel}) + DelGroupRoleLevel(groupID, []int32{member.RoleLevel}). + DelMaxJoinGroupVersion(userID). + DelMaxGroupMemberVersion(groupID) if err := c.ChainExecDel(ctx); err != nil { return err } @@ -350,17 +361,20 @@ func (g *groupDatabase) HandlerGroupRequest(ctx context.Context, groupID string, } func (g *groupDatabase) DeleteGroupMember(ctx context.Context, groupID string, userIDs []string) error { - if err := g.groupMemberDB.Delete(ctx, groupID, userIDs); err != nil { - return err - } - c := g.cache.CloneGroupCache() - return c.DelGroupMembersHash(groupID). - DelGroupMemberIDs(groupID). - DelGroupsMemberNum(groupID). - DelJoinedGroupID(userIDs...). - DelGroupMembersInfo(groupID, userIDs...). - DelGroupAllRoleLevel(groupID). - ChainExecDel(ctx) + return g.ctxTx.Transaction(ctx, func(ctx context.Context) error { + if err := g.groupMemberDB.Delete(ctx, groupID, userIDs); err != nil { + return err + } + c := g.cache.CloneGroupCache() + return c.DelGroupMembersHash(groupID). + DelGroupMemberIDs(groupID). + DelGroupsMemberNum(groupID). + DelJoinedGroupID(userIDs...). + DelGroupMembersInfo(groupID, userIDs...). + DelGroupAllRoleLevel(groupID). + DelMaxGroupMemberVersion(groupID). + ChainExecDel(ctx) + }) } func (g *groupDatabase) MapGroupMemberUserID(ctx context.Context, groupIDs []string) (map[string]*common.GroupSimpleUserID, error) { @@ -390,20 +404,25 @@ func (g *groupDatabase) TransferGroupOwner(ctx context.Context, groupID string, c := g.cache.CloneGroupCache() return c.DelGroupMembersInfo(groupID, oldOwnerUserID, newOwnerUserID). DelGroupAllRoleLevel(groupID). - DelGroupMembersHash(groupID).ChainExecDel(ctx) + DelGroupMembersHash(groupID). + DelJoinedGroupID(oldOwnerUserID, newOwnerUserID). + ChainExecDel(ctx) }) } func (g *groupDatabase) UpdateGroupMember(ctx context.Context, groupID string, userID string, data map[string]any) error { - if err := g.groupMemberDB.Update(ctx, groupID, userID, data); err != nil { - return err - } - c := g.cache.CloneGroupCache() - c = c.DelGroupMembersInfo(groupID, userID) - if g.groupMemberDB.IsUpdateRoleLevel(data) { - c = c.DelGroupAllRoleLevel(groupID) - } - return c.ChainExecDel(ctx) + return g.ctxTx.Transaction(ctx, func(ctx context.Context) error { + if err := g.groupMemberDB.Update(ctx, groupID, userID, data); err != nil { + return err + } + c := g.cache.CloneGroupCache() + c = c.DelGroupMembersInfo(groupID, userID) + if g.groupMemberDB.IsUpdateRoleLevel(data) { + c = c.DelGroupAllRoleLevel(groupID) + } + c = c.DelMaxGroupMemberVersion(groupID).DelMaxJoinGroupVersion(userID) + return c.ChainExecDel(ctx) + }) } func (g *groupDatabase) UpdateGroupMembers(ctx context.Context, data []*common.BatchUpdateGroupMember) error { @@ -483,3 +502,11 @@ func (g *groupDatabase) FindSortGroupMemberUserIDs(ctx context.Context, groupID func (g *groupDatabase) FindSortJoinGroupIDs(ctx context.Context, userID string) ([]string, error) { return g.cache.FindSortJoinGroupIDs(ctx, userID) } + +func (g *groupDatabase) FindMaxGroupMemberVersionCache(ctx context.Context, groupID string) (*model.VersionLog, error) { + return g.cache.FindMaxGroupMemberVersion(ctx, groupID) +} + +func (g *groupDatabase) FindMaxJoinGroupVersionCache(ctx context.Context, userID string) (*model.VersionLog, error) { + return g.cache.FindMaxJoinGroupVersion(ctx, userID) +} diff --git a/pkg/common/storage/model/version_log.go b/pkg/common/storage/model/version_log.go index 044bd42da..a09f493a8 100644 --- a/pkg/common/storage/model/version_log.go +++ b/pkg/common/storage/model/version_log.go @@ -30,7 +30,6 @@ func (v *VersionLogTable) VersionLog() *VersionLog { Deleted: v.Deleted, LastUpdate: v.LastUpdate, LogLen: 0, - queryDoc: true, } } @@ -42,15 +41,10 @@ type VersionLog struct { Deleted uint `bson:"deleted"` LastUpdate time.Time `bson:"last_update"` LogLen int `bson:"log_len"` - queryDoc bool `bson:"-"` } -func (w *VersionLog) Full() bool { - return w.queryDoc || w.Version == 0 || len(w.Logs) != w.LogLen -} - -func (w *VersionLog) DeleteAndChangeIDs() (delIds []string, changeIds []string) { - for _, l := range w.Logs { +func (v *VersionLog) DeleteAndChangeIDs() (delIds []string, changeIds []string) { + for _, l := range v.Logs { if l.Deleted { delIds = append(delIds, l.EID) } else { From 7e13faaa984436e2b47dd22b830e7097957a39b7 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Tue, 4 Jun 2024 19:11:16 +0800 Subject: [PATCH 22/59] sync option --- pkg/common/storage/cache/redis/group.go | 10 +++++- pkg/common/storage/controller/group.go | 10 ++++++ pkg/common/storage/database/group.go | 4 +++ pkg/common/storage/database/mgo/friend.go | 32 ++++++++++-------- pkg/common/storage/database/mgo/group.go | 33 +++++++++++++++++++ .../storage/database/mgo/group_member.go | 10 +++--- 6 files changed, 80 insertions(+), 19 deletions(-) diff --git a/pkg/common/storage/cache/redis/group.go b/pkg/common/storage/cache/redis/group.go index cf88a6514..67e246058 100644 --- a/pkg/common/storage/cache/redis/group.go +++ b/pkg/common/storage/cache/redis/group.go @@ -257,9 +257,17 @@ func (g *GroupCacheRedis) DelGroupMemberIDs(groupID string) cache.GroupCache { return cache } +func (g *GroupCacheRedis) findUserJoinedGroupID(ctx context.Context, userID string) ([]string, error) { + groupIDs, err := g.groupMemberDB.FindUserJoinedGroupID(ctx, userID) + if err != nil { + return nil, err + } + return g.groupDB.FindJoinSortGroupID(ctx, groupIDs) +} + func (g *GroupCacheRedis) GetJoinedGroupIDs(ctx context.Context, userID string) (joinedGroupIDs []string, err error) { return getCache(ctx, g.rcClient, g.getJoinedGroupsKey(userID), g.expireTime, func(ctx context.Context) ([]string, error) { - return g.groupMemberDB.FindUserJoinedGroupID(ctx, userID) + return g.findUserJoinedGroupID(ctx, userID) }) } diff --git a/pkg/common/storage/controller/group.go b/pkg/common/storage/controller/group.go index 9750b8b65..f9ee65955 100644 --- a/pkg/common/storage/controller/group.go +++ b/pkg/common/storage/controller/group.go @@ -116,6 +116,8 @@ type GroupDatabase interface { FindMaxGroupMemberVersionCache(ctx context.Context, groupID string) (*model.VersionLog, error) FindMaxJoinGroupVersionCache(ctx context.Context, userID string) (*model.VersionLog, error) + + SearchJoinGroup(ctx context.Context, userID string, keyword string, pagination pagination.Pagination) (int64, []*model.Group, error) } func NewGroupDatabase( @@ -510,3 +512,11 @@ func (g *groupDatabase) FindMaxGroupMemberVersionCache(ctx context.Context, grou func (g *groupDatabase) FindMaxJoinGroupVersionCache(ctx context.Context, userID string) (*model.VersionLog, error) { return g.cache.FindMaxJoinGroupVersion(ctx, userID) } + +func (g *groupDatabase) SearchJoinGroup(ctx context.Context, userID string, keyword string, pagination pagination.Pagination) (int64, []*model.Group, error) { + groupIDs, err := g.cache.GetJoinedGroupIDs(ctx, userID) + if err != nil { + return 0, nil, err + } + return g.groupDB.SearchJoin(ctx, groupIDs, keyword, pagination) +} diff --git a/pkg/common/storage/database/group.go b/pkg/common/storage/database/group.go index 712db09d2..7ef22f6c9 100644 --- a/pkg/common/storage/database/group.go +++ b/pkg/common/storage/database/group.go @@ -32,4 +32,8 @@ type Group interface { CountTotal(ctx context.Context, before *time.Time) (count int64, err error) // Get Group total quantity every day CountRangeEverydayTotal(ctx context.Context, start time.Time, end time.Time) (map[string]int64, error) + + FindJoinSortGroupID(ctx context.Context, groupIDs []string) ([]string, error) + + SearchJoin(ctx context.Context, groupIDs []string, keyword string, pagination pagination.Pagination) (int64, []*model.Group, error) } diff --git a/pkg/common/storage/database/mgo/friend.go b/pkg/common/storage/database/mgo/friend.go index 83daf9ecd..e0e30642c 100644 --- a/pkg/common/storage/database/mgo/friend.go +++ b/pkg/common/storage/database/mgo/friend.go @@ -52,6 +52,10 @@ func NewFriendMongo(db *mongo.Database) (database.Friend, error) { return &FriendMgo{coll: coll, owner: owner}, nil } +func (f *FriendMgo) friendSort() any { + return bson.D{{"is_pinned", -1}, {"friend_nickname", 1}, {"create_time", 1}} +} + // Create inserts multiple friend records. func (f *FriendMgo) Create(ctx context.Context, friends []*model.Friend) error { return mongoutil.IncrVersion(func() error { @@ -145,13 +149,13 @@ func (f *FriendMgo) FindReversalFriends(ctx context.Context, friendUserID string // FindOwnerFriends retrieves a paginated list of friends for a given owner. func (f *FriendMgo) FindOwnerFriends(ctx context.Context, ownerUserID string, pagination pagination.Pagination) (int64, []*model.Friend, error) { filter := bson.M{"owner_user_id": ownerUserID} - opt := options.Find().SetSort(bson.D{{"friend_nickname", 1}, {"create_time", 1}}) + opt := options.Find().SetSort(f.friendSort()) return mongoutil.FindPage[*model.Friend](ctx, f.coll, filter, pagination, opt) } func (f *FriendMgo) FindOwnerFriendUserIds(ctx context.Context, ownerUserID string, limit int) ([]string, error) { filter := bson.M{"owner_user_id": ownerUserID} - opt := options.Find().SetProjection(bson.M{"_id": 0, "friend_user_id": 1}).SetSort(bson.D{{"friend_nickname", 1}, {"create_time", 1}}).SetLimit(int64(limit)) + opt := options.Find().SetProjection(bson.M{"_id": 0, "friend_user_id": 1}).SetSort(f.friendSort()).SetLimit(int64(limit)) return mongoutil.Find[string](ctx, f.coll, filter, opt) } @@ -197,7 +201,7 @@ func (f *FriendMgo) FindFriendUserID(ctx context.Context, friendUserID string) ( filter := bson.M{ "friend_user_id": friendUserID, } - return mongoutil.Find[string](ctx, f.coll, filter, options.Find().SetProjection(bson.M{"_id": 0, "owner_user_id": 1})) + return mongoutil.Find[string](ctx, f.coll, filter, options.Find().SetProjection(bson.M{"_id": 0, "owner_user_id": 1}).SetSort(f.friendSort())) } func (f *FriendMgo) UpdateFriendUserInfo(ctx context.Context, friendUserID string, nickname string, faceURL string) error { @@ -209,14 +213,16 @@ func (f *FriendMgo) UpdateFriendUserInfo(ctx context.Context, friendUserID strin } func (f *FriendMgo) SearchFriend(ctx context.Context, ownerUserID, keyword string, pagination pagination.Pagination) (int64, []*model.Friend, error) { - //where := bson.M{ - // "owner_user_id": ownerUserID, - // "$or": []bson.M{ - // {"remark": bson.M{"$regex": keyword, "$options": "i"}}, - // {"friend_user_id": bson.M{"$regex": keyword, "$options": "i"}}, - // {"nickname": bson.M{"$regex": keyword, "$options": "i"}}, - // }, - //} - //return f.aggregatePagination(ctx, where, pagination) - panic("todo") + filter := bson.M{ + "owner_user_id": ownerUserID, + } + if keyword != "" { + filter["$or"] = []bson.M{ + {"remark": bson.M{"$regex": keyword, "$options": "i"}}, + {"nickname": bson.M{"$regex": keyword, "$options": "i"}}, + {"friend_user_id": bson.M{"$regex": keyword, "$options": "i"}}, + } + } + opt := options.Find().SetSort(f.friendSort()) + return mongoutil.FindPage[*model.Friend](ctx, f.coll, filter, pagination, opt) } diff --git a/pkg/common/storage/database/mgo/group.go b/pkg/common/storage/database/mgo/group.go index 48d24560b..630bc0291 100644 --- a/pkg/common/storage/database/mgo/group.go +++ b/pkg/common/storage/database/mgo/group.go @@ -47,6 +47,10 @@ type GroupMgo struct { coll *mongo.Collection } +func (g *GroupMgo) sortGroup() any { + return bson.D{{"group_name", 1}, {"create_time", 1}} +} + func (g *GroupMgo) Create(ctx context.Context, groups []*model.Group) (err error) { return mongoutil.InsertMany(ctx, g.coll, groups) } @@ -126,3 +130,32 @@ func (g *GroupMgo) CountRangeEverydayTotal(ctx context.Context, start time.Time, } return res, nil } + +func (g *GroupMgo) FindJoinSortGroupID(ctx context.Context, groupIDs []string) ([]string, error) { + if len(groupIDs) < 2 { + return groupIDs, nil + } + filter := bson.M{ + "group_id": bson.M{"$in": groupIDs}, + "status": bson.M{"$ne": constant.GroupStatusDismissed}, + } + opt := options.Find().SetSort(g.sortGroup()).SetProjection(bson.M{"_id": 0, "group_id": 1}) + return mongoutil.Find[string](ctx, g.coll, filter, opt) +} + +func (g *GroupMgo) SearchJoin(ctx context.Context, groupIDs []string, keyword string, pagination pagination.Pagination) (int64, []*model.Group, error) { + if len(groupIDs) == 0 { + return 0, nil, nil + } + filter := bson.M{ + "group_id": bson.M{"$in": groupIDs}, + "status": bson.M{"$ne": constant.GroupStatusDismissed}, + } + if keyword != "" { + filter["group_name"] = bson.M{"$regex": keyword} + } + // Define the sorting options + opts := options.Find().SetSort(g.sortGroup()) + // Perform the search with pagination and sorting + return mongoutil.FindPage[*model.Group](ctx, g.coll, filter, pagination, opts) +} diff --git a/pkg/common/storage/database/mgo/group_member.go b/pkg/common/storage/database/mgo/group_member.go index 372babc9f..cb64c87a4 100644 --- a/pkg/common/storage/database/mgo/group_member.go +++ b/pkg/common/storage/database/mgo/group_member.go @@ -57,7 +57,7 @@ type GroupMemberMgo struct { join database.VersionLog } -func (g *GroupMemberMgo) sortBson() any { +func (g *GroupMemberMgo) memberSort() any { return bson.D{{"role_level", -1}, {"create_time", -1}} } @@ -128,7 +128,7 @@ func (g *GroupMemberMgo) Update(ctx context.Context, groupID string, userID stri } func (g *GroupMemberMgo) FindMemberUserID(ctx context.Context, groupID string) (userIDs []string, err error) { - return mongoutil.Find[string](ctx, g.coll, bson.M{"group_id": groupID}, options.Find().SetProjection(bson.M{"_id": 0, "user_id": 1}).SetSort(g.sortBson())) + return mongoutil.Find[string](ctx, g.coll, bson.M{"group_id": groupID}, options.Find().SetProjection(bson.M{"_id": 0, "user_id": 1}).SetSort(g.memberSort())) } func (g *GroupMemberMgo) Take(ctx context.Context, groupID string, userID string) (groupMember *model.GroupMember, err error) { @@ -143,13 +143,13 @@ func (g *GroupMemberMgo) FindRoleLevelUserIDs(ctx context.Context, groupID strin return mongoutil.Find[string](ctx, g.coll, bson.M{"group_id": groupID, "role_level": roleLevel}, options.Find().SetProjection(bson.M{"_id": 0, "user_id": 1})) } -func (g *GroupMemberMgo) SearchMember(ctx context.Context, keyword string, groupID string, pagination pagination.Pagination) (total int64, groupList []*model.GroupMember, err error) { +func (g *GroupMemberMgo) SearchMember(ctx context.Context, keyword string, groupID string, pagination pagination.Pagination) (int64, []*model.GroupMember, error) { filter := bson.M{"group_id": groupID, "nickname": bson.M{"$regex": keyword}} - return mongoutil.FindPage[*model.GroupMember](ctx, g.coll, filter, pagination, options.Find().SetSort(g.sortBson())) + return mongoutil.FindPage[*model.GroupMember](ctx, g.coll, filter, pagination, options.Find().SetSort(g.memberSort())) } func (g *GroupMemberMgo) FindUserJoinedGroupID(ctx context.Context, userID string) (groupIDs []string, err error) { - return mongoutil.Find[string](ctx, g.coll, bson.M{"user_id": userID}, options.Find().SetProjection(bson.M{"_id": 0, "group_id": 1}).SetSort(g.sortBson())) + return mongoutil.Find[string](ctx, g.coll, bson.M{"user_id": userID}, options.Find().SetProjection(bson.M{"_id": 0, "group_id": 1}).SetSort(g.memberSort())) } func (g *GroupMemberMgo) TakeGroupMemberNum(ctx context.Context, groupID string) (count int64, err error) { From 1796c3a9e4f5a8eff96bc027ad0f27906d8ea4aa Mon Sep 17 00:00:00 2001 From: Monet Lee Date: Wed, 5 Jun 2024 11:18:14 +0800 Subject: [PATCH 23/59] refactor: replace `friend` package with `realtion`. --- go.mod | 2 +- go.sum | 2 + internal/api/friend.go | 42 ++++++++------ internal/api/router.go | 6 +- internal/rpc/friend/black.go | 21 +++---- internal/rpc/friend/callback.go | 24 ++++---- internal/rpc/friend/convert.go | 8 +-- internal/rpc/friend/friend.go | 89 +++++++++++++++++------------ internal/rpc/friend/notification.go | 15 ++--- internal/rpc/friend/sync.go | 29 +++++----- internal/rpc/user/user.go | 29 +++++----- pkg/rpcclient/friend.go | 20 +++---- 12 files changed, 159 insertions(+), 128 deletions(-) diff --git a/go.mod b/go.mod index db418bf14..26c61aed7 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/lestrrat-go/file-rotatelogs v2.4.0+incompatible // indirect github.com/mitchellh/mapstructure v1.5.0 - github.com/openimsdk/protocol v0.0.69-alpha.3 + github.com/openimsdk/protocol v0.0.69-alpha.4 github.com/openimsdk/tools v0.0.49-alpha.24 github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 diff --git a/go.sum b/go.sum index 4588df18a..1b0feaa34 100644 --- a/go.sum +++ b/go.sum @@ -288,6 +288,8 @@ github.com/openimsdk/gomake v0.0.13 h1:xLDe/moqgWpRoptHzI4packAWzs4C16b+sVY+txNJ github.com/openimsdk/gomake v0.0.13/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= github.com/openimsdk/protocol v0.0.69-alpha.3 h1:Uf167FVB5EqYpiy2zBbR63OiK+Njjy99fXYasK6Zi+4= github.com/openimsdk/protocol v0.0.69-alpha.3/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= +github.com/openimsdk/protocol v0.0.69-alpha.4 h1:QJkOFV5Hlu7CbkHG5smeVw+5fx5DVkpNJWqlAOJxuIY= +github.com/openimsdk/protocol v0.0.69-alpha.4/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= github.com/openimsdk/tools v0.0.49-alpha.24 h1:lJsqnjTPujnr91LRQ6QmcTliMIa4fMOBSTri6rFz2ek= github.com/openimsdk/tools v0.0.49-alpha.24/go.mod h1:g7mkHXYUPi0/8aAX8VPMHpnb3hqdV69Jph+bXOGvvNM= github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= diff --git a/internal/api/friend.go b/internal/api/friend.go index 3af162a53..6912fdbbb 100644 --- a/internal/api/friend.go +++ b/internal/api/friend.go @@ -17,7 +17,7 @@ package api import ( "github.com/gin-gonic/gin" "github.com/openimsdk/open-im-server/v3/pkg/rpcclient" - "github.com/openimsdk/protocol/friend" + "github.com/openimsdk/protocol/relation" "github.com/openimsdk/tools/a2r" ) @@ -28,73 +28,77 @@ func NewFriendApi(client rpcclient.Friend) FriendApi { } func (o *FriendApi) ApplyToAddFriend(c *gin.Context) { - a2r.Call(friend.FriendClient.ApplyToAddFriend, o.Client, c) + a2r.Call(relation.FriendClient.ApplyToAddFriend, o.Client, c) } func (o *FriendApi) RespondFriendApply(c *gin.Context) { - a2r.Call(friend.FriendClient.RespondFriendApply, o.Client, c) + a2r.Call(relation.FriendClient.RespondFriendApply, o.Client, c) } func (o *FriendApi) DeleteFriend(c *gin.Context) { - a2r.Call(friend.FriendClient.DeleteFriend, o.Client, c) + a2r.Call(relation.FriendClient.DeleteFriend, o.Client, c) } func (o *FriendApi) GetFriendApplyList(c *gin.Context) { - a2r.Call(friend.FriendClient.GetPaginationFriendsApplyTo, o.Client, c) + a2r.Call(relation.FriendClient.GetPaginationFriendsApplyTo, o.Client, c) } func (o *FriendApi) GetDesignatedFriendsApply(c *gin.Context) { - a2r.Call(friend.FriendClient.GetDesignatedFriendsApply, o.Client, c) + a2r.Call(relation.FriendClient.GetDesignatedFriendsApply, o.Client, c) } func (o *FriendApi) GetSelfApplyList(c *gin.Context) { - a2r.Call(friend.FriendClient.GetPaginationFriendsApplyFrom, o.Client, c) + a2r.Call(relation.FriendClient.GetPaginationFriendsApplyFrom, o.Client, c) } func (o *FriendApi) GetFriendList(c *gin.Context) { - a2r.Call(friend.FriendClient.GetPaginationFriends, o.Client, c) + a2r.Call(relation.FriendClient.GetPaginationFriends, o.Client, c) } func (o *FriendApi) GetDesignatedFriends(c *gin.Context) { - a2r.Call(friend.FriendClient.GetDesignatedFriends, o.Client, c) + a2r.Call(relation.FriendClient.GetDesignatedFriends, o.Client, c) } func (o *FriendApi) SetFriendRemark(c *gin.Context) { - a2r.Call(friend.FriendClient.SetFriendRemark, o.Client, c) + a2r.Call(relation.FriendClient.SetFriendRemark, o.Client, c) } func (o *FriendApi) AddBlack(c *gin.Context) { - a2r.Call(friend.FriendClient.AddBlack, o.Client, c) + a2r.Call(relation.FriendClient.AddBlack, o.Client, c) } func (o *FriendApi) GetPaginationBlacks(c *gin.Context) { - a2r.Call(friend.FriendClient.GetPaginationBlacks, o.Client, c) + a2r.Call(relation.FriendClient.GetPaginationBlacks, o.Client, c) } func (o *FriendApi) RemoveBlack(c *gin.Context) { - a2r.Call(friend.FriendClient.RemoveBlack, o.Client, c) + a2r.Call(relation.FriendClient.RemoveBlack, o.Client, c) } func (o *FriendApi) ImportFriends(c *gin.Context) { - a2r.Call(friend.FriendClient.ImportFriends, o.Client, c) + a2r.Call(relation.FriendClient.ImportFriends, o.Client, c) } func (o *FriendApi) IsFriend(c *gin.Context) { - a2r.Call(friend.FriendClient.IsFriend, o.Client, c) + a2r.Call(relation.FriendClient.IsFriend, o.Client, c) } func (o *FriendApi) GetFriendIDs(c *gin.Context) { - a2r.Call(friend.FriendClient.GetFriendIDs, o.Client, c) + a2r.Call(relation.FriendClient.GetFriendIDs, o.Client, c) } func (o *FriendApi) GetSpecifiedFriendsInfo(c *gin.Context) { - a2r.Call(friend.FriendClient.GetSpecifiedFriendsInfo, o.Client, c) + a2r.Call(relation.FriendClient.GetSpecifiedFriendsInfo, o.Client, c) } func (o *FriendApi) UpdateFriends(c *gin.Context) { - a2r.Call(friend.FriendClient.UpdateFriends, o.Client, c) + a2r.Call(relation.FriendClient.UpdateFriends, o.Client, c) } func (o *FriendApi) GetIncrementalFriends(c *gin.Context) { - a2r.Call(friend.FriendClient.GetIncrementalFriends, o.Client, c) + a2r.Call(relation.FriendClient.GetIncrementalFriends, o.Client, c) +} + +func (o *FriendApi) GetIncrementalBlacks(c *gin.Context) { + a2r.Call(relation.FriendClient.GetIncrementalBlacks, o.Client, c) } diff --git a/internal/api/router.go b/internal/api/router.go index 8f2733604..445a24694 100644 --- a/internal/api/router.go +++ b/internal/api/router.go @@ -2,6 +2,9 @@ package api import ( "fmt" + "net/http" + "strings" + "github.com/gin-gonic/gin" "github.com/gin-gonic/gin/binding" "github.com/go-playground/validator/v10" @@ -14,8 +17,6 @@ import ( "github.com/openimsdk/tools/mw" "google.golang.org/grpc" "google.golang.org/grpc/credentials/insecure" - "net/http" - "strings" ) func newGinRouter(disCov discovery.SvcDiscoveryRegistry, config *Config) *gin.Engine { @@ -81,6 +82,7 @@ func newGinRouter(disCov discovery.SvcDiscoveryRegistry, config *Config) *gin.En friendRouterGroup.POST("/add_black", f.AddBlack) friendRouterGroup.POST("/get_black_list", f.GetPaginationBlacks) friendRouterGroup.POST("/remove_black", f.RemoveBlack) + friendRouterGroup.POST("/get_incremental_blacks", f.GetIncrementalBlacks) friendRouterGroup.POST("/import_friend", f.ImportFriends) friendRouterGroup.POST("/is_friend", f.IsFriend) friendRouterGroup.POST("/get_friend_id", f.GetFriendIDs) diff --git a/internal/rpc/friend/black.go b/internal/rpc/friend/black.go index caec08b7a..218d1e7f8 100644 --- a/internal/rpc/friend/black.go +++ b/internal/rpc/friend/black.go @@ -16,16 +16,17 @@ package friend import ( "context" - "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" "time" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" + "github.com/openimsdk/open-im-server/v3/pkg/authverify" "github.com/openimsdk/open-im-server/v3/pkg/common/convert" - pbfriend "github.com/openimsdk/protocol/friend" + "github.com/openimsdk/protocol/relation" "github.com/openimsdk/tools/mcontext" ) -func (s *friendServer) GetPaginationBlacks(ctx context.Context, req *pbfriend.GetPaginationBlacksReq) (resp *pbfriend.GetPaginationBlacksResp, err error) { +func (s *friendServer) GetPaginationBlacks(ctx context.Context, req *relation.GetPaginationBlacksReq) (resp *relation.GetPaginationBlacksResp, err error) { if err := s.userRpcClient.Access(ctx, req.UserID); err != nil { return nil, err } @@ -33,7 +34,7 @@ func (s *friendServer) GetPaginationBlacks(ctx context.Context, req *pbfriend.Ge if err != nil { return nil, err } - resp = &pbfriend.GetPaginationBlacksResp{} + resp = &relation.GetPaginationBlacksResp{} resp.Blacks, err = convert.BlackDB2Pb(ctx, blacks, s.userRpcClient.GetUsersInfoMap) if err != nil { return nil, err @@ -42,18 +43,18 @@ func (s *friendServer) GetPaginationBlacks(ctx context.Context, req *pbfriend.Ge return resp, nil } -func (s *friendServer) IsBlack(ctx context.Context, req *pbfriend.IsBlackReq) (*pbfriend.IsBlackResp, error) { +func (s *friendServer) IsBlack(ctx context.Context, req *relation.IsBlackReq) (*relation.IsBlackResp, error) { in1, in2, err := s.blackDatabase.CheckIn(ctx, req.UserID1, req.UserID2) if err != nil { return nil, err } - resp := &pbfriend.IsBlackResp{} + resp := &relation.IsBlackResp{} resp.InUser1Blacks = in1 resp.InUser2Blacks = in2 return resp, nil } -func (s *friendServer) RemoveBlack(ctx context.Context, req *pbfriend.RemoveBlackReq) (*pbfriend.RemoveBlackResp, error) { +func (s *friendServer) RemoveBlack(ctx context.Context, req *relation.RemoveBlackReq) (*relation.RemoveBlackResp, error) { if err := s.userRpcClient.Access(ctx, req.OwnerUserID); err != nil { return nil, err } @@ -64,10 +65,10 @@ func (s *friendServer) RemoveBlack(ctx context.Context, req *pbfriend.RemoveBlac s.notificationSender.BlackDeletedNotification(ctx, req) - return &pbfriend.RemoveBlackResp{}, nil + return &relation.RemoveBlackResp{}, nil } -func (s *friendServer) AddBlack(ctx context.Context, req *pbfriend.AddBlackReq) (*pbfriend.AddBlackResp, error) { +func (s *friendServer) AddBlack(ctx context.Context, req *relation.AddBlackReq) (*relation.AddBlackResp, error) { if err := authverify.CheckAccessV3(ctx, req.OwnerUserID, s.config.Share.IMAdminUserID); err != nil { return nil, err } @@ -87,5 +88,5 @@ func (s *friendServer) AddBlack(ctx context.Context, req *pbfriend.AddBlackReq) return nil, err } s.notificationSender.BlackAddedNotification(ctx, req) - return &pbfriend.AddBlackResp{}, nil + return &relation.AddBlackResp{}, nil } diff --git a/internal/rpc/friend/callback.go b/internal/rpc/friend/callback.go index 0610cdb78..746ad21fa 100644 --- a/internal/rpc/friend/callback.go +++ b/internal/rpc/friend/callback.go @@ -16,14 +16,15 @@ package friend import ( "context" + "github.com/openimsdk/open-im-server/v3/pkg/common/webhook" cbapi "github.com/openimsdk/open-im-server/v3/pkg/callbackstruct" "github.com/openimsdk/open-im-server/v3/pkg/common/config" - pbfriend "github.com/openimsdk/protocol/friend" + "github.com/openimsdk/protocol/relation" ) -func (s *friendServer) webhookAfterDeleteFriend(ctx context.Context, after *config.AfterConfig, req *pbfriend.DeleteFriendReq) { +func (s *friendServer) webhookAfterDeleteFriend(ctx context.Context, after *config.AfterConfig, req *relation.DeleteFriendReq) { cbReq := &cbapi.CallbackAfterDeleteFriendReq{ CallbackCommand: cbapi.CallbackAfterDeleteFriendCommand, OwnerUserID: req.OwnerUserID, @@ -32,7 +33,7 @@ func (s *friendServer) webhookAfterDeleteFriend(ctx context.Context, after *conf s.webhookClient.AsyncPost(ctx, cbReq.GetCallbackCommand(), cbReq, &cbapi.CallbackAfterDeleteFriendResp{}, after) } -func (s *friendServer) webhookBeforeAddFriend(ctx context.Context, before *config.BeforeConfig, req *pbfriend.ApplyToAddFriendReq) error { +func (s *friendServer) webhookBeforeAddFriend(ctx context.Context, before *config.BeforeConfig, req *relation.ApplyToAddFriendReq) error { return webhook.WithCondition(ctx, before, func(ctx context.Context) error { cbReq := &cbapi.CallbackBeforeAddFriendReq{ CallbackCommand: cbapi.CallbackBeforeAddFriendCommand, @@ -50,7 +51,7 @@ func (s *friendServer) webhookBeforeAddFriend(ctx context.Context, before *confi }) } -func (s *friendServer) webhookAfterAddFriend(ctx context.Context, after *config.AfterConfig, req *pbfriend.ApplyToAddFriendReq) { +func (s *friendServer) webhookAfterAddFriend(ctx context.Context, after *config.AfterConfig, req *relation.ApplyToAddFriendReq) { cbReq := &cbapi.CallbackAfterAddFriendReq{ CallbackCommand: cbapi.CallbackAfterAddFriendCommand, FromUserID: req.FromUserID, @@ -61,8 +62,7 @@ func (s *friendServer) webhookAfterAddFriend(ctx context.Context, after *config. s.webhookClient.AsyncPost(ctx, cbReq.GetCallbackCommand(), cbReq, resp, after) } -func (s *friendServer) webhookAfterSetFriendRemark(ctx context.Context, after *config.AfterConfig, req *pbfriend.SetFriendRemarkReq) { - +func (s *friendServer) webhookAfterSetFriendRemark(ctx context.Context, after *config.AfterConfig, req *relation.SetFriendRemarkReq) { cbReq := &cbapi.CallbackAfterSetFriendRemarkReq{ CallbackCommand: cbapi.CallbackAfterSetFriendRemarkCommand, OwnerUserID: req.OwnerUserID, @@ -73,7 +73,7 @@ func (s *friendServer) webhookAfterSetFriendRemark(ctx context.Context, after *c s.webhookClient.AsyncPost(ctx, cbReq.GetCallbackCommand(), cbReq, resp, after) } -func (s *friendServer) webhookAfterImportFriends(ctx context.Context, after *config.AfterConfig, req *pbfriend.ImportFriendReq) { +func (s *friendServer) webhookAfterImportFriends(ctx context.Context, after *config.AfterConfig, req *relation.ImportFriendReq) { cbReq := &cbapi.CallbackAfterImportFriendsReq{ CallbackCommand: cbapi.CallbackAfterImportFriendsCommand, OwnerUserID: req.OwnerUserID, @@ -83,7 +83,7 @@ func (s *friendServer) webhookAfterImportFriends(ctx context.Context, after *con s.webhookClient.AsyncPost(ctx, cbReq.GetCallbackCommand(), cbReq, resp, after) } -func (s *friendServer) webhookAfterRemoveBlack(ctx context.Context, after *config.AfterConfig, req *pbfriend.RemoveBlackReq) { +func (s *friendServer) webhookAfterRemoveBlack(ctx context.Context, after *config.AfterConfig, req *relation.RemoveBlackReq) { cbReq := &cbapi.CallbackAfterRemoveBlackReq{ CallbackCommand: cbapi.CallbackAfterRemoveBlackCommand, OwnerUserID: req.OwnerUserID, @@ -93,7 +93,7 @@ func (s *friendServer) webhookAfterRemoveBlack(ctx context.Context, after *confi s.webhookClient.AsyncPost(ctx, cbReq.GetCallbackCommand(), cbReq, resp, after) } -func (s *friendServer) webhookBeforeSetFriendRemark(ctx context.Context, before *config.BeforeConfig, req *pbfriend.SetFriendRemarkReq) error { +func (s *friendServer) webhookBeforeSetFriendRemark(ctx context.Context, before *config.BeforeConfig, req *relation.SetFriendRemarkReq) error { return webhook.WithCondition(ctx, before, func(ctx context.Context) error { cbReq := &cbapi.CallbackBeforeSetFriendRemarkReq{ CallbackCommand: cbapi.CallbackBeforeSetFriendRemarkCommand, @@ -112,7 +112,7 @@ func (s *friendServer) webhookBeforeSetFriendRemark(ctx context.Context, before }) } -func (s *friendServer) webhookBeforeAddBlack(ctx context.Context, before *config.BeforeConfig, req *pbfriend.AddBlackReq) error { +func (s *friendServer) webhookBeforeAddBlack(ctx context.Context, before *config.BeforeConfig, req *relation.AddBlackReq) error { return webhook.WithCondition(ctx, before, func(ctx context.Context) error { cbReq := &cbapi.CallbackBeforeAddBlackReq{ CallbackCommand: cbapi.CallbackBeforeAddBlackCommand, @@ -124,7 +124,7 @@ func (s *friendServer) webhookBeforeAddBlack(ctx context.Context, before *config }) } -func (s *friendServer) webhookBeforeAddFriendAgree(ctx context.Context, before *config.BeforeConfig, req *pbfriend.RespondFriendApplyReq) error { +func (s *friendServer) webhookBeforeAddFriendAgree(ctx context.Context, before *config.BeforeConfig, req *relation.RespondFriendApplyReq) error { return webhook.WithCondition(ctx, before, func(ctx context.Context) error { cbReq := &cbapi.CallbackBeforeAddFriendAgreeReq{ CallbackCommand: cbapi.CallbackBeforeAddFriendAgreeCommand, @@ -138,7 +138,7 @@ func (s *friendServer) webhookBeforeAddFriendAgree(ctx context.Context, before * }) } -func (s *friendServer) webhookBeforeImportFriends(ctx context.Context, before *config.BeforeConfig, req *pbfriend.ImportFriendReq) error { +func (s *friendServer) webhookBeforeImportFriends(ctx context.Context, before *config.BeforeConfig, req *relation.ImportFriendReq) error { return webhook.WithCondition(ctx, before, func(ctx context.Context) error { cbReq := &cbapi.CallbackBeforeImportFriendsReq{ CallbackCommand: cbapi.CallbackBeforeImportFriendsCommand, diff --git a/internal/rpc/friend/convert.go b/internal/rpc/friend/convert.go index 0730df529..1129dab34 100644 --- a/internal/rpc/friend/convert.go +++ b/internal/rpc/friend/convert.go @@ -2,12 +2,12 @@ package friend import ( "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" - "github.com/openimsdk/protocol/friend" + "github.com/openimsdk/protocol/relation" "github.com/openimsdk/tools/utils/datautil" ) -func friendDB2PB(db *model.Friend) *friend.FriendInfo { - return &friend.FriendInfo{ +func friendDB2PB(db *model.Friend) *relation.FriendInfo { + return &relation.FriendInfo{ OwnerUserID: db.OwnerUserID, FriendUserID: db.FriendUserID, FriendNickname: db.FriendNickname, @@ -21,6 +21,6 @@ func friendDB2PB(db *model.Friend) *friend.FriendInfo { } } -func friendsDB2PB(db []*model.Friend) []*friend.FriendInfo { +func friendsDB2PB(db []*model.Friend) []*relation.FriendInfo { return datautil.Slice(db, friendDB2PB) } diff --git a/internal/rpc/friend/friend.go b/internal/rpc/friend/friend.go index 1c6b52b14..a80881278 100644 --- a/internal/rpc/friend/friend.go +++ b/internal/rpc/friend/friend.go @@ -16,6 +16,7 @@ package friend import ( "context" + "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/redis" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database/mgo" @@ -30,7 +31,7 @@ import ( "github.com/openimsdk/open-im-server/v3/pkg/common/storage/controller" "github.com/openimsdk/open-im-server/v3/pkg/rpcclient" "github.com/openimsdk/protocol/constant" - pbfriend "github.com/openimsdk/protocol/friend" + "github.com/openimsdk/protocol/relation" "github.com/openimsdk/protocol/sdkws" "github.com/openimsdk/tools/db/mongoutil" "github.com/openimsdk/tools/discovery" @@ -54,7 +55,7 @@ type Config struct { RpcConfig config.Friend RedisConfig config.Redis MongodbConfig config.Mongo - //ZookeeperConfig config.ZooKeeper + // ZookeeperConfig config.ZooKeeper NotificationConfig config.Notification Share config.Share WebhooksConfig config.Webhooks @@ -103,7 +104,7 @@ func Start(ctx context.Context, config *Config, client discovery.SvcDiscoveryReg localcache.InitLocalCache(&config.LocalCacheConfig) // Register Friend server with refactored MongoDB and Redis integrations - pbfriend.RegisterFriendServer(server, &friendServer{ + relation.RegisterFriendServer(server, &friendServer{ friendDatabase: controller.NewFriendDatabase( friendMongoDB, friendRequestMongoDB, @@ -126,8 +127,8 @@ func Start(ctx context.Context, config *Config, client discovery.SvcDiscoveryReg } // ok. -func (s *friendServer) ApplyToAddFriend(ctx context.Context, req *pbfriend.ApplyToAddFriendReq) (resp *pbfriend.ApplyToAddFriendResp, err error) { - resp = &pbfriend.ApplyToAddFriendResp{} +func (s *friendServer) ApplyToAddFriend(ctx context.Context, req *relation.ApplyToAddFriendReq) (resp *relation.ApplyToAddFriendResp, err error) { + resp = &relation.ApplyToAddFriendResp{} if err := authverify.CheckAccessV3(ctx, req.FromUserID, s.config.Share.IMAdminUserID); err != nil { return nil, err } @@ -157,7 +158,7 @@ func (s *friendServer) ApplyToAddFriend(ctx context.Context, req *pbfriend.Apply } // ok. -func (s *friendServer) ImportFriends(ctx context.Context, req *pbfriend.ImportFriendReq) (resp *pbfriend.ImportFriendResp, err error) { +func (s *friendServer) ImportFriends(ctx context.Context, req *relation.ImportFriendReq) (resp *relation.ImportFriendResp, err error) { if err := authverify.CheckAdmin(ctx, s.config.Share.IMAdminUserID); err != nil { return nil, err } @@ -179,7 +180,7 @@ func (s *friendServer) ImportFriends(ctx context.Context, req *pbfriend.ImportFr return nil, err } for _, userID := range req.FriendUserIDs { - s.notificationSender.FriendApplicationAgreedNotification(ctx, &pbfriend.RespondFriendApplyReq{ + s.notificationSender.FriendApplicationAgreedNotification(ctx, &relation.RespondFriendApplyReq{ FromUserID: req.OwnerUserID, ToUserID: userID, HandleResult: constant.FriendResponseAgree, @@ -187,12 +188,12 @@ func (s *friendServer) ImportFriends(ctx context.Context, req *pbfriend.ImportFr } s.webhookAfterImportFriends(ctx, &s.config.WebhooksConfig.AfterImportFriends, req) - return &pbfriend.ImportFriendResp{}, nil + return &relation.ImportFriendResp{}, nil } // ok. -func (s *friendServer) RespondFriendApply(ctx context.Context, req *pbfriend.RespondFriendApplyReq) (resp *pbfriend.RespondFriendApplyResp, err error) { - resp = &pbfriend.RespondFriendApplyResp{} +func (s *friendServer) RespondFriendApply(ctx context.Context, req *relation.RespondFriendApplyReq) (resp *relation.RespondFriendApplyResp, err error) { + resp = &relation.RespondFriendApplyResp{} if err := authverify.CheckAccessV3(ctx, req.ToUserID, s.config.Share.IMAdminUserID); err != nil { return nil, err } @@ -226,8 +227,8 @@ func (s *friendServer) RespondFriendApply(ctx context.Context, req *pbfriend.Res } // ok. -func (s *friendServer) DeleteFriend(ctx context.Context, req *pbfriend.DeleteFriendReq) (resp *pbfriend.DeleteFriendResp, err error) { - resp = &pbfriend.DeleteFriendResp{} +func (s *friendServer) DeleteFriend(ctx context.Context, req *relation.DeleteFriendReq) (resp *relation.DeleteFriendResp, err error) { + resp = &relation.DeleteFriendResp{} if err := s.userRpcClient.Access(ctx, req.OwnerUserID); err != nil { return nil, err } @@ -244,11 +245,11 @@ func (s *friendServer) DeleteFriend(ctx context.Context, req *pbfriend.DeleteFri } // ok. -func (s *friendServer) SetFriendRemark(ctx context.Context, req *pbfriend.SetFriendRemarkReq) (resp *pbfriend.SetFriendRemarkResp, err error) { +func (s *friendServer) SetFriendRemark(ctx context.Context, req *relation.SetFriendRemarkReq) (resp *relation.SetFriendRemarkResp, err error) { if err = s.webhookBeforeSetFriendRemark(ctx, &s.config.WebhooksConfig.BeforeSetFriendRemark, req); err != nil && err != servererrs.ErrCallbackContinue { return nil, err } - resp = &pbfriend.SetFriendRemarkResp{} + resp = &relation.SetFriendRemarkResp{} if err := s.userRpcClient.Access(ctx, req.OwnerUserID); err != nil { return nil, err } @@ -265,8 +266,8 @@ func (s *friendServer) SetFriendRemark(ctx context.Context, req *pbfriend.SetFri } // ok. -func (s *friendServer) GetDesignatedFriends(ctx context.Context, req *pbfriend.GetDesignatedFriendsReq) (resp *pbfriend.GetDesignatedFriendsResp, err error) { - resp = &pbfriend.GetDesignatedFriendsResp{} +func (s *friendServer) GetDesignatedFriends(ctx context.Context, req *relation.GetDesignatedFriendsReq) (resp *relation.GetDesignatedFriendsResp, err error) { + resp = &relation.GetDesignatedFriendsResp{} if datautil.Duplicate(req.FriendUserIDs) { return nil, errs.ErrArgs.WrapMsg("friend userID repeated") } @@ -282,12 +283,13 @@ func (s *friendServer) GetDesignatedFriends(ctx context.Context, req *pbfriend.G // Get the list of friend requests sent out proactively. func (s *friendServer) GetDesignatedFriendsApply(ctx context.Context, - req *pbfriend.GetDesignatedFriendsApplyReq) (resp *pbfriend.GetDesignatedFriendsApplyResp, err error) { + req *relation.GetDesignatedFriendsApplyReq, +) (resp *relation.GetDesignatedFriendsApplyResp, err error) { friendRequests, err := s.friendDatabase.FindBothFriendRequests(ctx, req.FromUserID, req.ToUserID) if err != nil { return nil, err } - resp = &pbfriend.GetDesignatedFriendsApplyResp{} + resp = &relation.GetDesignatedFriendsApplyResp{} resp.FriendRequests, err = convert.FriendRequestDB2Pb(ctx, friendRequests, s.userRpcClient.GetUsersInfoMap) if err != nil { return nil, err @@ -296,7 +298,7 @@ func (s *friendServer) GetDesignatedFriendsApply(ctx context.Context, } // Get received friend requests (i.e., those initiated by others). -func (s *friendServer) GetPaginationFriendsApplyTo(ctx context.Context, req *pbfriend.GetPaginationFriendsApplyToReq) (resp *pbfriend.GetPaginationFriendsApplyToResp, err error) { +func (s *friendServer) GetPaginationFriendsApplyTo(ctx context.Context, req *relation.GetPaginationFriendsApplyToReq) (resp *relation.GetPaginationFriendsApplyToResp, err error) { if err := s.userRpcClient.Access(ctx, req.UserID); err != nil { return nil, err } @@ -304,7 +306,7 @@ func (s *friendServer) GetPaginationFriendsApplyTo(ctx context.Context, req *pbf if err != nil { return nil, err } - resp = &pbfriend.GetPaginationFriendsApplyToResp{} + resp = &relation.GetPaginationFriendsApplyToResp{} resp.FriendRequests, err = convert.FriendRequestDB2Pb(ctx, friendRequests, s.userRpcClient.GetUsersInfoMap) if err != nil { return nil, err @@ -313,8 +315,8 @@ func (s *friendServer) GetPaginationFriendsApplyTo(ctx context.Context, req *pbf return resp, nil } -func (s *friendServer) GetPaginationFriendsApplyFrom(ctx context.Context, req *pbfriend.GetPaginationFriendsApplyFromReq) (resp *pbfriend.GetPaginationFriendsApplyFromResp, err error) { - resp = &pbfriend.GetPaginationFriendsApplyFromResp{} +func (s *friendServer) GetPaginationFriendsApplyFrom(ctx context.Context, req *relation.GetPaginationFriendsApplyFromReq) (resp *relation.GetPaginationFriendsApplyFromResp, err error) { + resp = &relation.GetPaginationFriendsApplyFromResp{} if err := s.userRpcClient.Access(ctx, req.UserID); err != nil { return nil, err } @@ -331,8 +333,8 @@ func (s *friendServer) GetPaginationFriendsApplyFrom(ctx context.Context, req *p } // ok. -func (s *friendServer) IsFriend(ctx context.Context, req *pbfriend.IsFriendReq) (resp *pbfriend.IsFriendResp, err error) { - resp = &pbfriend.IsFriendResp{} +func (s *friendServer) IsFriend(ctx context.Context, req *relation.IsFriendReq) (resp *relation.IsFriendResp, err error) { + resp = &relation.IsFriendResp{} resp.InUser1Friends, resp.InUser2Friends, err = s.friendDatabase.CheckIn(ctx, req.UserID1, req.UserID2) if err != nil { return nil, err @@ -340,7 +342,7 @@ func (s *friendServer) IsFriend(ctx context.Context, req *pbfriend.IsFriendReq) return resp, nil } -func (s *friendServer) GetPaginationFriends(ctx context.Context, req *pbfriend.GetPaginationFriendsReq) (resp *pbfriend.GetPaginationFriendsResp, err error) { +func (s *friendServer) GetPaginationFriends(ctx context.Context, req *relation.GetPaginationFriendsReq) (resp *relation.GetPaginationFriendsResp, err error) { if err := s.userRpcClient.Access(ctx, req.UserID); err != nil { return nil, err } @@ -348,7 +350,7 @@ func (s *friendServer) GetPaginationFriends(ctx context.Context, req *pbfriend.G if err != nil { return nil, err } - resp = &pbfriend.GetPaginationFriendsResp{} + resp = &relation.GetPaginationFriendsResp{} resp.FriendsInfo, err = convert.FriendsDB2Pb(ctx, friends, s.userRpcClient.GetUsersInfoMap) if err != nil { return nil, err @@ -357,11 +359,11 @@ func (s *friendServer) GetPaginationFriends(ctx context.Context, req *pbfriend.G return resp, nil } -func (s *friendServer) GetFriendIDs(ctx context.Context, req *pbfriend.GetFriendIDsReq) (resp *pbfriend.GetFriendIDsResp, err error) { +func (s *friendServer) GetFriendIDs(ctx context.Context, req *relation.GetFriendIDsReq) (resp *relation.GetFriendIDsResp, err error) { if err := s.userRpcClient.Access(ctx, req.UserID); err != nil { return nil, err } - resp = &pbfriend.GetFriendIDsResp{} + resp = &relation.GetFriendIDsResp{} resp.FriendIDs, err = s.friendDatabase.FindFriendUserIDs(ctx, req.UserID) if err != nil { return nil, err @@ -369,7 +371,7 @@ func (s *friendServer) GetFriendIDs(ctx context.Context, req *pbfriend.GetFriend return resp, nil } -func (s *friendServer) GetSpecifiedFriendsInfo(ctx context.Context, req *pbfriend.GetSpecifiedFriendsInfoReq) (*pbfriend.GetSpecifiedFriendsInfoResp, error) { +func (s *friendServer) GetSpecifiedFriendsInfo(ctx context.Context, req *relation.GetSpecifiedFriendsInfoReq) (*relation.GetSpecifiedFriendsInfoResp, error) { if len(req.UserIDList) == 0 { return nil, errs.ErrArgs.WrapMsg("userIDList is empty") } @@ -394,8 +396,8 @@ func (s *friendServer) GetSpecifiedFriendsInfo(ctx context.Context, req *pbfrien blackMap := datautil.SliceToMap(blacks, func(e *model.Black) string { return e.BlockUserID }) - resp := &pbfriend.GetSpecifiedFriendsInfoResp{ - Infos: make([]*pbfriend.GetSpecifiedFriendsInfoInfo, 0, len(req.UserIDList)), + resp := &relation.GetSpecifiedFriendsInfoResp{ + Infos: make([]*relation.GetSpecifiedFriendsInfoInfo, 0, len(req.UserIDList)), } for _, userID := range req.UserIDList { user := userMap[userID] @@ -404,7 +406,6 @@ func (s *friendServer) GetSpecifiedFriendsInfo(ctx context.Context, req *pbfrien } var friendInfo *sdkws.FriendInfo if friend := friendMap[userID]; friend != nil { - friendInfo = &sdkws.FriendInfo{ OwnerUserID: friend.OwnerUserID, Remark: friend.Remark, @@ -425,7 +426,7 @@ func (s *friendServer) GetSpecifiedFriendsInfo(ctx context.Context, req *pbfrien Ex: black.Ex, } } - resp.Infos = append(resp.Infos, &pbfriend.GetSpecifiedFriendsInfoInfo{ + resp.Infos = append(resp.Infos, &relation.GetSpecifiedFriendsInfoInfo{ UserInfo: user, FriendInfo: friendInfo, BlackInfo: blackInfo, @@ -433,10 +434,11 @@ func (s *friendServer) GetSpecifiedFriendsInfo(ctx context.Context, req *pbfrien } return resp, nil } + func (s *friendServer) UpdateFriends( ctx context.Context, - req *pbfriend.UpdateFriendsReq, -) (*pbfriend.UpdateFriendsResp, error) { + req *relation.UpdateFriendsReq, +) (*relation.UpdateFriendsResp, error) { if len(req.FriendUserIDs) == 0 { return nil, errs.ErrArgs.WrapMsg("friendIDList is empty") } @@ -464,8 +466,23 @@ func (s *friendServer) UpdateFriends( return nil, err } - resp := &pbfriend.UpdateFriendsResp{} + resp := &relation.UpdateFriendsResp{} s.notificationSender.FriendsInfoUpdateNotification(ctx, req.OwnerUserID, req.FriendUserIDs) return resp, nil } + +func (s *friendServer) GetIncrementalFriendsApplyTo(ctx context.Context, req *relation.GetIncrementalFriendsApplyToReq) (*relation.GetIncrementalFriendsApplyToResp, error) { + // TODO implement me + return nil, nil +} + +func (s *friendServer) GetIncrementalFriendsApplyFrom(ctx context.Context, req *relation.GetIncrementalFriendsApplyFromReq) (*relation.GetIncrementalFriendsApplyFromResp, error) { + // TODO implement me + return nil, nil +} + +func (s *friendServer) GetIncrementalBlacks(ctx context.Context, req *relation.GetIncrementalBlacksReq) (*relation.GetIncrementalBlacksResp, error) { + // TODO implement me + return nil, nil +} diff --git a/internal/rpc/friend/notification.go b/internal/rpc/friend/notification.go index 8089a9bdc..ddee025bb 100644 --- a/internal/rpc/friend/notification.go +++ b/internal/rpc/friend/notification.go @@ -16,6 +16,7 @@ package friend import ( "context" + relationtb "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" "github.com/openimsdk/open-im-server/v3/pkg/common/config" @@ -24,7 +25,7 @@ import ( "github.com/openimsdk/open-im-server/v3/pkg/rpcclient" "github.com/openimsdk/open-im-server/v3/pkg/rpcclient/notification" "github.com/openimsdk/protocol/constant" - pbfriend "github.com/openimsdk/protocol/friend" + "github.com/openimsdk/protocol/relation" "github.com/openimsdk/protocol/sdkws" "github.com/openimsdk/tools/mcontext" ) @@ -127,7 +128,7 @@ func (f *FriendNotificationSender) UserInfoUpdatedNotification(ctx context.Conte f.Notification(ctx, mcontext.GetOpUserID(ctx), changedUserID, constant.UserInfoUpdatedNotification, &tips) } -func (f *FriendNotificationSender) FriendApplicationAddNotification(ctx context.Context, req *pbfriend.ApplyToAddFriendReq) { +func (f *FriendNotificationSender) FriendApplicationAddNotification(ctx context.Context, req *relation.ApplyToAddFriendReq) { tips := sdkws.FriendApplicationTips{FromToUserID: &sdkws.FromToUserID{ FromUserID: req.FromUserID, ToUserID: req.ToUserID, @@ -137,7 +138,7 @@ func (f *FriendNotificationSender) FriendApplicationAddNotification(ctx context. func (f *FriendNotificationSender) FriendApplicationAgreedNotification( ctx context.Context, - req *pbfriend.RespondFriendApplyReq, + req *relation.RespondFriendApplyReq, ) { tips := sdkws.FriendApplicationApprovedTips{FromToUserID: &sdkws.FromToUserID{ FromUserID: req.FromUserID, @@ -148,7 +149,7 @@ func (f *FriendNotificationSender) FriendApplicationAgreedNotification( func (f *FriendNotificationSender) FriendApplicationRefusedNotification( ctx context.Context, - req *pbfriend.RespondFriendApplyReq, + req *relation.RespondFriendApplyReq, ) { tips := sdkws.FriendApplicationApprovedTips{FromToUserID: &sdkws.FromToUserID{ FromUserID: req.FromUserID, @@ -182,7 +183,7 @@ func (f *FriendNotificationSender) FriendAddedNotification( return nil } -func (f *FriendNotificationSender) FriendDeletedNotification(ctx context.Context, req *pbfriend.DeleteFriendReq) { +func (f *FriendNotificationSender) FriendDeletedNotification(ctx context.Context, req *relation.DeleteFriendReq) { tips := sdkws.FriendDeletedTips{FromToUserID: &sdkws.FromToUserID{ FromUserID: req.OwnerUserID, ToUserID: req.FriendUserID, @@ -204,14 +205,14 @@ func (f *FriendNotificationSender) FriendsInfoUpdateNotification(ctx context.Con f.Notification(ctx, toUserID, toUserID, constant.FriendsInfoUpdateNotification, &tips) } -func (f *FriendNotificationSender) BlackAddedNotification(ctx context.Context, req *pbfriend.AddBlackReq) { +func (f *FriendNotificationSender) BlackAddedNotification(ctx context.Context, req *relation.AddBlackReq) { tips := sdkws.BlackAddedTips{FromToUserID: &sdkws.FromToUserID{}} tips.FromToUserID.FromUserID = req.OwnerUserID tips.FromToUserID.ToUserID = req.BlackUserID f.Notification(ctx, req.OwnerUserID, req.BlackUserID, constant.BlackAddedNotification, &tips) } -func (f *FriendNotificationSender) BlackDeletedNotification(ctx context.Context, req *pbfriend.RemoveBlackReq) { +func (f *FriendNotificationSender) BlackDeletedNotification(ctx context.Context, req *relation.RemoveBlackReq) { blackDeletedTips := sdkws.BlackDeletedTips{FromToUserID: &sdkws.FromToUserID{ FromUserID: req.OwnerUserID, ToUserID: req.BlackUserID, diff --git a/internal/rpc/friend/sync.go b/internal/rpc/friend/sync.go index bdea7b9eb..ee8effa70 100644 --- a/internal/rpc/friend/sync.go +++ b/internal/rpc/friend/sync.go @@ -2,14 +2,15 @@ package friend import ( "context" + "github.com/openimsdk/open-im-server/v3/internal/rpc/incrversion" "github.com/openimsdk/open-im-server/v3/pkg/authverify" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" - pbfriend "github.com/openimsdk/protocol/friend" + "github.com/openimsdk/protocol/relation" "github.com/openimsdk/tools/errs" ) -func (s *friendServer) NotificationUserInfoUpdate(ctx context.Context, req *pbfriend.NotificationUserInfoUpdateReq) (*pbfriend.NotificationUserInfoUpdateResp, error) { +func (s *friendServer) NotificationUserInfoUpdate(ctx context.Context, req *relation.NotificationUserInfoUpdateReq) (*relation.NotificationUserInfoUpdateResp, error) { if req.NewUserInfo == nil { var err error req.NewUserInfo, err = s.userRpcClient.GetUserInfo(ctx, req.UserID) @@ -30,10 +31,10 @@ func (s *friendServer) NotificationUserInfoUpdate(ctx context.Context, req *pbfr } s.notificationSender.FriendsInfoUpdateNotification(ctx, req.UserID, userIDs) } - return &pbfriend.NotificationUserInfoUpdateResp{}, nil + return &relation.NotificationUserInfoUpdateResp{}, nil } -func (s *friendServer) SearchFriends(ctx context.Context, req *pbfriend.SearchFriendsReq) (*pbfriend.SearchFriendsResp, error) { +func (s *friendServer) SearchFriends(ctx context.Context, req *relation.SearchFriendsReq) (*relation.SearchFriendsResp, error) { if err := s.userRpcClient.Access(ctx, req.UserID); err != nil { return nil, err } @@ -42,7 +43,7 @@ func (s *friendServer) SearchFriends(ctx context.Context, req *pbfriend.SearchFr if err != nil { return nil, err } - return &pbfriend.SearchFriendsResp{ + return &relation.SearchFriendsResp{ Total: total, Friends: friendsDB2PB(friends), }, nil @@ -51,17 +52,17 @@ func (s *friendServer) SearchFriends(ctx context.Context, req *pbfriend.SearchFr if err != nil { return nil, err } - return &pbfriend.SearchFriendsResp{ + return &relation.SearchFriendsResp{ Total: total, Friends: friendsDB2PB(friends), }, nil } -func (s *friendServer) GetIncrementalFriends(ctx context.Context, req *pbfriend.GetIncrementalFriendsReq) (*pbfriend.GetIncrementalFriendsResp, error) { +func (s *friendServer) GetIncrementalFriends(ctx context.Context, req *relation.GetIncrementalFriendsReq) (*relation.GetIncrementalFriendsResp, error) { if err := authverify.CheckAccessV3(ctx, req.UserID, s.config.Share.IMAdminUserID); err != nil { return nil, err } - opt := incrversion.Option[*pbfriend.FriendInfo, pbfriend.GetIncrementalFriendsResp]{ + opt := incrversion.Option[*relation.FriendInfo, relation.GetIncrementalFriendsResp]{ VersionID: req.VersionID, Version: func() (*model.VersionLog, error) { return s.friendDatabase.FindFriendIncrVersion(ctx, req.UserID, uint(req.Version), incrversion.Limit(s.config.RpcConfig.FriendSyncCount, req.Version)) @@ -69,18 +70,18 @@ func (s *friendServer) GetIncrementalFriends(ctx context.Context, req *pbfriend. AllID: func() ([]string, error) { return s.friendDatabase.FindSortFriendUserIDs(ctx, req.UserID) }, - Find: func(ids []string) ([]*pbfriend.FriendInfo, error) { + Find: func(ids []string) ([]*relation.FriendInfo, error) { friends, err := s.friendDatabase.FindFriendsWithError(ctx, req.UserID, ids) if err != nil { return nil, err } return friendsDB2PB(friends), nil }, - ID: func(elem *pbfriend.FriendInfo) string { + ID: func(elem *relation.FriendInfo) string { return elem.FriendUserID }, - Resp: func(version *model.VersionLog, delIDs []string, list []*pbfriend.FriendInfo, full bool) *pbfriend.GetIncrementalFriendsResp { - return &pbfriend.GetIncrementalFriendsResp{ + Resp: func(version *model.VersionLog, delIDs []string, list []*relation.FriendInfo, full bool) *relation.GetIncrementalFriendsResp { + return &relation.GetIncrementalFriendsResp{ VersionID: version.ID.Hex(), Version: uint64(version.Version), Full: full, @@ -93,7 +94,7 @@ func (s *friendServer) GetIncrementalFriends(ctx context.Context, req *pbfriend. return opt.Build() } -//func (s *friendServer) GetIncrementalFriends(ctx context.Context, req *pbfriend.GetIncrementalFriendsReq) (*pbfriend.GetIncrementalFriendsResp, error) { +//func (s *friendServer) GetIncrementalFriends(ctx context.Context, req *relation.GetIncrementalFriendsReq) (*relation.GetIncrementalFriendsResp, error) { // if err := authverify.CheckAccessV3(ctx, req.UserID, s.config.Share.IMAdminUserID); err != nil { // return nil, err // } @@ -124,7 +125,7 @@ func (s *friendServer) GetIncrementalFriends(ctx context.Context, req *pbfriend. // return nil, err // } // } -// return &pbfriend.GetIncrementalFriendsResp{ +// return &relation.GetIncrementalFriendsResp{ // Version: uint64(incrVer.Version), // VersionID: incrVer.ID.Hex(), // Full: incrVer.Full(), diff --git a/internal/rpc/user/user.go b/internal/rpc/user/user.go index 504c08a2c..35870d4d0 100644 --- a/internal/rpc/user/user.go +++ b/internal/rpc/user/user.go @@ -17,6 +17,11 @@ package user import ( "context" "errors" + "math/rand" + "strings" + "sync" + "time" + "github.com/openimsdk/open-im-server/v3/internal/rpc/friend" "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/redis" @@ -24,13 +29,9 @@ import ( tablerelation "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" "github.com/openimsdk/open-im-server/v3/pkg/common/webhook" "github.com/openimsdk/open-im-server/v3/pkg/localcache" - friendpb "github.com/openimsdk/protocol/friend" "github.com/openimsdk/protocol/group" + "github.com/openimsdk/protocol/relation" "github.com/openimsdk/tools/db/redisutil" - "math/rand" - "strings" - "sync" - "time" "github.com/openimsdk/open-im-server/v3/pkg/authverify" "github.com/openimsdk/open-im-server/v3/pkg/common/convert" @@ -123,8 +124,7 @@ func (s *userServer) GetDesignateUsers(ctx context.Context, req *pbuser.GetDesig // deprecated: -//UpdateUserInfo - +// UpdateUserInfo func (s *userServer) UpdateUserInfo(ctx context.Context, req *pbuser.UpdateUserInfoReq) (resp *pbuser.UpdateUserInfoResp, err error) { resp = &pbuser.UpdateUserInfoResp{} err = authverify.CheckAccessV3(ctx, req.UserInfo.UserID, s.config.Share.IMAdminUserID) @@ -162,6 +162,7 @@ func (s *userServer) UpdateUserInfo(ctx context.Context, req *pbuser.UpdateUserI } return resp, nil } + func (s *userServer) UpdateUserInfoEx(ctx context.Context, req *pbuser.UpdateUserInfoExReq) (resp *pbuser.UpdateUserInfoExResp, err error) { resp = &pbuser.UpdateUserInfoExResp{} err = authverify.CheckAccessV3(ctx, req.UserInfo.UserID, s.config.Share.IMAdminUserID) @@ -198,6 +199,7 @@ func (s *userServer) UpdateUserInfoEx(ctx context.Context, req *pbuser.UpdateUse } return resp, nil } + func (s *userServer) SetGlobalRecvMessageOpt(ctx context.Context, req *pbuser.SetGlobalRecvMessageOptReq) (resp *pbuser.SetGlobalRecvMessageOptResp, err error) { resp = &pbuser.SetGlobalRecvMessageOptResp{} if _, err := s.db.FindWithError(ctx, []string{req.UserID}); err != nil { @@ -256,7 +258,6 @@ func (s *userServer) GetPaginationUsers(ctx context.Context, req *pbuser.GetPagi return &pbuser.GetPaginationUsersResp{Total: int32(total), Users: convert.UsersDB2Pb(users)}, err } - } func (s *userServer) UserRegister(ctx context.Context, req *pbuser.UserRegisterReq) (resp *pbuser.UserRegisterResp, err error) { @@ -353,7 +354,8 @@ func (s *userServer) SubscribeOrCancelUsersStatus(ctx context.Context, req *pbus // GetUserStatus Get the online status of the user. func (s *userServer) GetUserStatus(ctx context.Context, req *pbuser.GetUserStatusReq) (resp *pbuser.GetUserStatusResp, - err error) { + err error, +) { onlineStatusList, err := s.db.GetUserStatus(ctx, req.UserIDs) if err != nil { return nil, err @@ -363,7 +365,8 @@ func (s *userServer) GetUserStatus(ctx context.Context, req *pbuser.GetUserStatu // SetUserStatus Synchronize user's online status. func (s *userServer) SetUserStatus(ctx context.Context, req *pbuser.SetUserStatusReq) (resp *pbuser.SetUserStatusResp, - err error) { + err error, +) { err = s.db.SetUserStatus(ctx, req.UserID, req.Status, req.PlatformID) if err != nil { return nil, err @@ -387,7 +390,8 @@ func (s *userServer) SetUserStatus(ctx context.Context, req *pbuser.SetUserStatu // GetSubscribeUsersStatus Get the online status of subscribers. func (s *userServer) GetSubscribeUsersStatus(ctx context.Context, - req *pbuser.GetSubscribeUsersStatusReq) (*pbuser.GetSubscribeUsersStatusResp, error) { + req *pbuser.GetSubscribeUsersStatusReq, +) (*pbuser.GetSubscribeUsersStatusResp, error) { userList, err := s.db.GetAllSubscribeList(ctx, req.UserID) if err != nil { return nil, err @@ -476,7 +480,6 @@ func (s *userServer) ProcessUserCommandUpdate(ctx context.Context, req *pbuser.P } func (s *userServer) ProcessUserCommandGet(ctx context.Context, req *pbuser.ProcessUserCommandGetReq) (*pbuser.ProcessUserCommandGetResp, error) { - err := authverify.CheckAccessV3(ctx, req.UserID, s.config.Share.IMAdminUserID) if err != nil { return nil, err @@ -723,7 +726,7 @@ func (s *userServer) NotificationUserInfoUpdate(ctx context.Context, userID stri go func() { defer wg.Done() - _, es[1] = s.friendRpcClient.Client.NotificationUserInfoUpdate(ctx, &friendpb.NotificationUserInfoUpdateReq{ + _, es[1] = s.friendRpcClient.Client.NotificationUserInfoUpdate(ctx, &relation.NotificationUserInfoUpdateReq{ UserID: userID, OldUserInfo: oldUserInfo, NewUserInfo: newUserInfo, diff --git a/pkg/rpcclient/friend.go b/pkg/rpcclient/friend.go index 5543afe4f..fd00be329 100644 --- a/pkg/rpcclient/friend.go +++ b/pkg/rpcclient/friend.go @@ -17,7 +17,7 @@ package rpcclient import ( "context" - "github.com/openimsdk/protocol/friend" + "github.com/openimsdk/protocol/relation" sdkws "github.com/openimsdk/protocol/sdkws" "github.com/openimsdk/tools/discovery" "github.com/openimsdk/tools/system/program" @@ -26,7 +26,7 @@ import ( type Friend struct { conn grpc.ClientConnInterface - Client friend.FriendClient + Client relation.FriendClient discov discovery.SvcDiscoveryRegistry } @@ -35,7 +35,7 @@ func NewFriend(discov discovery.SvcDiscoveryRegistry, rpcRegisterName string) *F if err != nil { program.ExitWithError(err) } - client := friend.NewFriendClient(conn) + client := relation.NewFriendClient(conn) return &Friend{discov: discov, conn: conn, Client: client} } @@ -47,11 +47,11 @@ func NewFriendRpcClient(discov discovery.SvcDiscoveryRegistry, rpcRegisterName s func (f *FriendRpcClient) GetFriendsInfo( ctx context.Context, - ownerUserID, friendUserID string, + ownerUserID, relationUserID string, ) (resp *sdkws.FriendInfo, err error) { r, err := f.Client.GetDesignatedFriends( ctx, - &friend.GetDesignatedFriendsReq{OwnerUserID: ownerUserID, FriendUserIDs: []string{friendUserID}}, + &relation.GetDesignatedFriendsReq{OwnerUserID: ownerUserID, FriendUserIDs: []string{relationUserID}}, ) if err != nil { return nil, err @@ -60,17 +60,17 @@ func (f *FriendRpcClient) GetFriendsInfo( return } -// possibleFriendUserID Is PossibleFriendUserId's friends. +// possibleFriendUserID Is PossibleFriendUserId's relations. func (f *FriendRpcClient) IsFriend(ctx context.Context, possibleFriendUserID, userID string) (bool, error) { - resp, err := f.Client.IsFriend(ctx, &friend.IsFriendReq{UserID1: userID, UserID2: possibleFriendUserID}) + resp, err := f.Client.IsFriend(ctx, &relation.IsFriendReq{UserID1: userID, UserID2: possibleFriendUserID}) if err != nil { return false, err } return resp.InUser1Friends, nil } -func (f *FriendRpcClient) GetFriendIDs(ctx context.Context, ownerUserID string) (friendIDs []string, err error) { - req := friend.GetFriendIDsReq{UserID: ownerUserID} +func (f *FriendRpcClient) GetFriendIDs(ctx context.Context, ownerUserID string) (relationIDs []string, err error) { + req := relation.GetFriendIDsReq{UserID: ownerUserID} resp, err := f.Client.GetFriendIDs(ctx, &req) if err != nil { return nil, err @@ -79,7 +79,7 @@ func (f *FriendRpcClient) GetFriendIDs(ctx context.Context, ownerUserID string) } func (b *FriendRpcClient) IsBlack(ctx context.Context, possibleBlackUserID, userID string) (bool, error) { - r, err := b.Client.IsBlack(ctx, &friend.IsBlackReq{UserID1: possibleBlackUserID, UserID2: userID}) + r, err := b.Client.IsBlack(ctx, &relation.IsBlackReq{UserID1: possibleBlackUserID, UserID2: userID}) if err != nil { return false, err } From e11807c02874a670fb038a9e45e05d3d37c2c7c0 Mon Sep 17 00:00:00 2001 From: Monet Lee Date: Wed, 5 Jun 2024 11:39:10 +0800 Subject: [PATCH 24/59] refactor: update lastest commit to relation. --- internal/rpc/friend/sync.go | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/internal/rpc/friend/sync.go b/internal/rpc/friend/sync.go index ba689c7ba..685982620 100644 --- a/internal/rpc/friend/sync.go +++ b/internal/rpc/friend/sync.go @@ -63,7 +63,7 @@ func (s *friendServer) GetIncrementalFriends(ctx context.Context, req *relation. return nil, err } - opt := incrversion.Option[*pbfriend.FriendInfo, pbfriend.GetIncrementalFriendsResp]{ + opt := incrversion.Option[*relation.FriendInfo, relation.GetIncrementalFriendsResp]{ Ctx: ctx, VersionKey: req.UserID, VersionID: req.VersionID, @@ -72,8 +72,7 @@ func (s *friendServer) GetIncrementalFriends(ctx context.Context, req *relation. Version: s.friendDatabase.FindFriendIncrVersion, CacheMaxVersion: s.friendDatabase.FindMaxFriendVersionCache, SortID: s.friendDatabase.FindSortFriendUserIDs, - Find: func(ctx context.Context, ids []string) ([]*pbfriend.FriendInfo, error) { - + Find: func(ctx context.Context, ids []string) ([]*relation.FriendInfo, error) { friends, err := s.friendDatabase.FindFriendsWithError(ctx, req.UserID, ids) if err != nil { return nil, err @@ -81,9 +80,9 @@ func (s *friendServer) GetIncrementalFriends(ctx context.Context, req *relation. return friendsDB2PB(friends), nil }, - ID: func(elem *pbfriend.FriendInfo) string { return elem.FriendUserID }, - Resp: func(version *model.VersionLog, delIDs []string, list []*pbfriend.FriendInfo, full bool) *pbfriend.GetIncrementalFriendsResp { - return &pbfriend.GetIncrementalFriendsResp{ + ID: func(elem *relation.FriendInfo) string { return elem.FriendUserID }, + Resp: func(version *model.VersionLog, delIDs []string, list []*relation.FriendInfo, full bool) *relation.GetIncrementalFriendsResp { + return &relation.GetIncrementalFriendsResp{ VersionID: version.ID.Hex(), Version: uint64(version.Version), Full: full, From 6285b688a275460d2b9f814439c2bec8f75be63e Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Thu, 6 Jun 2024 09:45:44 +0800 Subject: [PATCH 25/59] sync option --- go.mod | 2 +- internal/api/group.go | 2 +- internal/rpc/friend/black.go | 2 +- internal/rpc/friend/callback.go | 2 +- internal/rpc/friend/convert.go | 26 ------ internal/rpc/friend/friend.go | 34 ++++++- internal/rpc/friend/notification.go | 2 +- internal/rpc/friend/sync.go | 88 +++++++----------- internal/rpc/group/sync.go | 5 -- internal/rpc/user/user.go | 105 +++++++--------------- pkg/common/storage/cache/group.go | 1 - pkg/common/storage/cache/redis/group.go | 21 ----- pkg/common/storage/controller/friend.go | 19 +--- pkg/common/storage/controller/user.go | 6 ++ pkg/common/storage/database/friend.go | 4 +- pkg/common/storage/database/mgo/friend.go | 40 ++++----- pkg/common/storage/database/mgo/user.go | 66 ++++++++++++++ pkg/common/storage/database/user.go | 3 + 18 files changed, 193 insertions(+), 235 deletions(-) delete mode 100644 internal/rpc/friend/convert.go diff --git a/go.mod b/go.mod index db418bf14..a3777939b 100644 --- a/go.mod +++ b/go.mod @@ -178,4 +178,4 @@ require ( gopkg.in/ini.v1 v1.67.0 // indirect ) -//replace github.com/openimsdk/protocol => /Users/chao/Desktop/project/protocol +replace github.com/openimsdk/protocol => /Users/chao/Desktop/project/protocol diff --git a/internal/api/group.go b/internal/api/group.go index 62e508d68..5b8b9d044 100644 --- a/internal/api/group.go +++ b/internal/api/group.go @@ -180,7 +180,7 @@ func (o *GroupApi) GetIncrementalGroupMemberBatch(c *gin.Context) { } resp.List[req.GroupID] = res changeCount += len(res.Changes) + len(res.DeleteUserIds) - if changeCount > int(res.SyncCount)*4 { + if changeCount >= int(res.SyncCount) { break } } diff --git a/internal/rpc/friend/black.go b/internal/rpc/friend/black.go index caec08b7a..18bb3dc0a 100644 --- a/internal/rpc/friend/black.go +++ b/internal/rpc/friend/black.go @@ -21,7 +21,7 @@ import ( "github.com/openimsdk/open-im-server/v3/pkg/authverify" "github.com/openimsdk/open-im-server/v3/pkg/common/convert" - pbfriend "github.com/openimsdk/protocol/friend" + pbfriend "github.com/openimsdk/protocol/relation" "github.com/openimsdk/tools/mcontext" ) diff --git a/internal/rpc/friend/callback.go b/internal/rpc/friend/callback.go index 0610cdb78..767733734 100644 --- a/internal/rpc/friend/callback.go +++ b/internal/rpc/friend/callback.go @@ -20,7 +20,7 @@ import ( cbapi "github.com/openimsdk/open-im-server/v3/pkg/callbackstruct" "github.com/openimsdk/open-im-server/v3/pkg/common/config" - pbfriend "github.com/openimsdk/protocol/friend" + pbfriend "github.com/openimsdk/protocol/relation" ) func (s *friendServer) webhookAfterDeleteFriend(ctx context.Context, after *config.AfterConfig, req *pbfriend.DeleteFriendReq) { diff --git a/internal/rpc/friend/convert.go b/internal/rpc/friend/convert.go deleted file mode 100644 index 0730df529..000000000 --- a/internal/rpc/friend/convert.go +++ /dev/null @@ -1,26 +0,0 @@ -package friend - -import ( - "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" - "github.com/openimsdk/protocol/friend" - "github.com/openimsdk/tools/utils/datautil" -) - -func friendDB2PB(db *model.Friend) *friend.FriendInfo { - return &friend.FriendInfo{ - OwnerUserID: db.OwnerUserID, - FriendUserID: db.FriendUserID, - FriendNickname: db.FriendNickname, - FriendFaceURL: db.FriendFaceURL, - Remark: db.Remark, - CreateTime: db.CreateTime.UnixMilli(), - AddSource: db.AddSource, - OperatorUserID: db.OperatorUserID, - Ex: db.Ex, - IsPinned: db.IsPinned, - } -} - -func friendsDB2PB(db []*model.Friend) []*friend.FriendInfo { - return datautil.Slice(db, friendDB2PB) -} diff --git a/internal/rpc/friend/friend.go b/internal/rpc/friend/friend.go index 1c6b52b14..7eb10aa17 100644 --- a/internal/rpc/friend/friend.go +++ b/internal/rpc/friend/friend.go @@ -30,7 +30,7 @@ import ( "github.com/openimsdk/open-im-server/v3/pkg/common/storage/controller" "github.com/openimsdk/open-im-server/v3/pkg/rpcclient" "github.com/openimsdk/protocol/constant" - pbfriend "github.com/openimsdk/protocol/friend" + pbfriend "github.com/openimsdk/protocol/relation" "github.com/openimsdk/protocol/sdkws" "github.com/openimsdk/tools/db/mongoutil" "github.com/openimsdk/tools/discovery" @@ -50,6 +50,21 @@ type friendServer struct { webhookClient *webhook.Client } +func (s *friendServer) GetIncrementalFriendsApplyTo(ctx context.Context, req *pbfriend.GetIncrementalFriendsApplyToReq) (*pbfriend.GetIncrementalFriendsApplyToResp, error) { + //TODO implement me + panic("implement me") +} + +func (s *friendServer) GetIncrementalFriendsApplyFrom(ctx context.Context, req *pbfriend.GetIncrementalFriendsApplyFromReq) (*pbfriend.GetIncrementalFriendsApplyFromResp, error) { + //TODO implement me + panic("implement me") +} + +func (s *friendServer) GetIncrementalBlacks(ctx context.Context, req *pbfriend.GetIncrementalBlacksReq) (*pbfriend.GetIncrementalBlacksResp, error) { + //TODO implement me + panic("implement me") +} + type Config struct { RpcConfig config.Friend RedisConfig config.Redis @@ -270,14 +285,24 @@ func (s *friendServer) GetDesignatedFriends(ctx context.Context, req *pbfriend.G if datautil.Duplicate(req.FriendUserIDs) { return nil, errs.ErrArgs.WrapMsg("friend userID repeated") } - friends, err := s.friendDatabase.FindFriendsWithError(ctx, req.OwnerUserID, req.FriendUserIDs) + friends, err := s.getFriend(ctx, req.OwnerUserID, req.FriendUserIDs) if err != nil { return nil, err } - if resp.FriendsInfo, err = convert.FriendsDB2Pb(ctx, friends, s.userRpcClient.GetUsersInfoMap); err != nil { + return &pbfriend.GetDesignatedFriendsResp{ + FriendsInfo: friends, + }, nil +} + +func (s *friendServer) getFriend(ctx context.Context, ownerUserID string, friendUserIDs []string) ([]*sdkws.FriendInfo, error) { + if len(friendUserIDs) == 0 { + return nil, nil + } + friends, err := s.friendDatabase.FindFriendsWithError(ctx, ownerUserID, friendUserIDs) + if err != nil { return nil, err } - return resp, nil + return convert.FriendsDB2Pb(ctx, friends, s.userRpcClient.GetUsersInfoMap) } // Get the list of friend requests sent out proactively. @@ -433,6 +458,7 @@ func (s *friendServer) GetSpecifiedFriendsInfo(ctx context.Context, req *pbfrien } return resp, nil } + func (s *friendServer) UpdateFriends( ctx context.Context, req *pbfriend.UpdateFriendsReq, diff --git a/internal/rpc/friend/notification.go b/internal/rpc/friend/notification.go index 8089a9bdc..9cf434555 100644 --- a/internal/rpc/friend/notification.go +++ b/internal/rpc/friend/notification.go @@ -24,7 +24,7 @@ import ( "github.com/openimsdk/open-im-server/v3/pkg/rpcclient" "github.com/openimsdk/open-im-server/v3/pkg/rpcclient/notification" "github.com/openimsdk/protocol/constant" - pbfriend "github.com/openimsdk/protocol/friend" + pbfriend "github.com/openimsdk/protocol/relation" "github.com/openimsdk/protocol/sdkws" "github.com/openimsdk/tools/mcontext" ) diff --git a/internal/rpc/friend/sync.go b/internal/rpc/friend/sync.go index a99b0f220..213a38151 100644 --- a/internal/rpc/friend/sync.go +++ b/internal/rpc/friend/sync.go @@ -5,63 +5,39 @@ import ( "github.com/openimsdk/open-im-server/v3/internal/rpc/incrversion" "github.com/openimsdk/open-im-server/v3/pkg/authverify" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" - pbfriend "github.com/openimsdk/protocol/friend" - "github.com/openimsdk/tools/errs" + pbfriend "github.com/openimsdk/protocol/relation" + "github.com/openimsdk/protocol/sdkws" ) -func (s *friendServer) NotificationUserInfoUpdate(ctx context.Context, req *pbfriend.NotificationUserInfoUpdateReq) (*pbfriend.NotificationUserInfoUpdateResp, error) { - if req.NewUserInfo == nil { - var err error - req.NewUserInfo, err = s.userRpcClient.GetUserInfo(ctx, req.UserID) - if err != nil { - return nil, err - } - } - if req.UserID != req.NewUserInfo.UserID { - return nil, errs.ErrArgs.WrapMsg("req.UserID != req.NewUserInfo.UserID") - } - userIDs, err := s.friendDatabase.FindFriendUserID(ctx, req.UserID) - if err != nil { - return nil, err - } - if len(userIDs) > 0 { - if err := s.friendDatabase.UpdateFriendUserInfo(ctx, req.UserID, userIDs, req.NewUserInfo.Nickname, req.NewUserInfo.FaceURL); err != nil { - return nil, err - } - s.notificationSender.FriendsInfoUpdateNotification(ctx, req.UserID, userIDs) - } - return &pbfriend.NotificationUserInfoUpdateResp{}, nil -} - -func (s *friendServer) SearchFriends(ctx context.Context, req *pbfriend.SearchFriendsReq) (*pbfriend.SearchFriendsResp, error) { - if err := s.userRpcClient.Access(ctx, req.UserID); err != nil { - return nil, err - } - if req.Keyword == "" { - total, friends, err := s.friendDatabase.PageOwnerFriends(ctx, req.UserID, req.Pagination) - if err != nil { - return nil, err - } - return &pbfriend.SearchFriendsResp{ - Total: total, - Friends: friendsDB2PB(friends), - }, nil - } - total, friends, err := s.friendDatabase.SearchFriend(ctx, req.UserID, req.Keyword, req.Pagination) - if err != nil { - return nil, err - } - return &pbfriend.SearchFriendsResp{ - Total: total, - Friends: friendsDB2PB(friends), - }, nil -} +//func (s *friendServer) SearchFriends(ctx context.Context, req *pbfriend.SearchFriendsReq) (*pbfriend.SearchFriendsResp, error) { +// if err := s.userRpcClient.Access(ctx, req.UserID); err != nil { +// return nil, err +// } +// if req.Keyword == "" { +// total, friends, err := s.friendDatabase.PageOwnerFriends(ctx, req.UserID, req.Pagination) +// if err != nil { +// return nil, err +// } +// return &pbfriend.SearchFriendsResp{ +// Total: total, +// Friends: friendsDB2PB(friends), +// }, nil +// } +// total, friends, err := s.friendDatabase.SearchFriend(ctx, req.UserID, req.Keyword, req.Pagination) +// if err != nil { +// return nil, err +// } +// return &pbfriend.SearchFriendsResp{ +// Total: total, +// Friends: friendsDB2PB(friends), +// }, nil +//} func (s *friendServer) GetIncrementalFriends(ctx context.Context, req *pbfriend.GetIncrementalFriendsReq) (*pbfriend.GetIncrementalFriendsResp, error) { if err := authverify.CheckAccessV3(ctx, req.UserID, s.config.Share.IMAdminUserID); err != nil { return nil, err } - opt := incrversion.Option[*pbfriend.FriendInfo, pbfriend.GetIncrementalFriendsResp]{ + opt := incrversion.Option[*sdkws.FriendInfo, pbfriend.GetIncrementalFriendsResp]{ Ctx: ctx, VersionKey: req.UserID, VersionID: req.VersionID, @@ -70,15 +46,11 @@ func (s *friendServer) GetIncrementalFriends(ctx context.Context, req *pbfriend. Version: s.friendDatabase.FindFriendIncrVersion, CacheMaxVersion: s.friendDatabase.FindMaxFriendVersionCache, SortID: s.friendDatabase.FindSortFriendUserIDs, - Find: func(ctx context.Context, ids []string) ([]*pbfriend.FriendInfo, error) { - friends, err := s.friendDatabase.FindFriendsWithError(ctx, req.UserID, ids) - if err != nil { - return nil, err - } - return friendsDB2PB(friends), nil + Find: func(ctx context.Context, ids []string) ([]*sdkws.FriendInfo, error) { + return s.getFriend(ctx, req.UserID, ids) }, - ID: func(elem *pbfriend.FriendInfo) string { return elem.FriendUserID }, - Resp: func(version *model.VersionLog, delIDs []string, list []*pbfriend.FriendInfo, full bool) *pbfriend.GetIncrementalFriendsResp { + ID: func(elem *sdkws.FriendInfo) string { return elem.FriendUser.UserID }, + Resp: func(version *model.VersionLog, delIDs []string, list []*sdkws.FriendInfo, full bool) *pbfriend.GetIncrementalFriendsResp { return &pbfriend.GetIncrementalFriendsResp{ VersionID: version.ID.Hex(), Version: uint64(version.Version), diff --git a/internal/rpc/group/sync.go b/internal/rpc/group/sync.go index 445ba6874..6e0103c83 100644 --- a/internal/rpc/group/sync.go +++ b/internal/rpc/group/sync.go @@ -9,11 +9,6 @@ import ( "github.com/openimsdk/protocol/sdkws" ) -func (s *groupServer) SearchGroupMember(ctx context.Context, req *pbgroup.SearchGroupMemberReq) (*pbgroup.SearchGroupMemberResp, error) { - //TODO implement me - panic("implement me") -} - func (s *groupServer) GetIncrementalGroupMember(ctx context.Context, req *pbgroup.GetIncrementalGroupMemberReq) (*pbgroup.GetIncrementalGroupMemberResp, error) { opt := incrversion.Option[*sdkws.GroupMemberFullInfo, pbgroup.GetIncrementalGroupMemberResp]{ Ctx: ctx, diff --git a/internal/rpc/user/user.go b/internal/rpc/user/user.go index 504c08a2c..db44be556 100644 --- a/internal/rpc/user/user.go +++ b/internal/rpc/user/user.go @@ -16,7 +16,6 @@ package user import ( "context" - "errors" "github.com/openimsdk/open-im-server/v3/internal/rpc/friend" "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/redis" @@ -24,12 +23,9 @@ import ( tablerelation "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" "github.com/openimsdk/open-im-server/v3/pkg/common/webhook" "github.com/openimsdk/open-im-server/v3/pkg/localcache" - friendpb "github.com/openimsdk/protocol/friend" - "github.com/openimsdk/protocol/group" "github.com/openimsdk/tools/db/redisutil" "math/rand" "strings" - "sync" "time" "github.com/openimsdk/open-im-server/v3/pkg/authverify" @@ -135,29 +131,26 @@ func (s *userServer) UpdateUserInfo(ctx context.Context, req *pbuser.UpdateUserI if err := s.webhookBeforeUpdateUserInfo(ctx, &s.config.WebhooksConfig.BeforeUpdateUserInfo, req); err != nil { return nil, err } + data := convert.UserPb2DBMap(req.UserInfo) - oldUser, err := s.db.GetUserByID(ctx, req.UserInfo.UserID) - if err != nil { + if err := s.db.UpdateByMap(ctx, req.UserInfo.UserID, data); err != nil { return nil, err } - if err := s.db.UpdateByMap(ctx, req.UserInfo.UserID, data); err != nil { + s.friendNotificationSender.UserInfoUpdatedNotification(ctx, req.UserInfo.UserID) + friends, err := s.friendRpcClient.GetFriendIDs(ctx, req.UserInfo.UserID) + if err != nil { return nil, err } - //s.friendNotificationSender.UserInfoUpdatedNotification(ctx, req.UserInfo.UserID) - //friends, err := s.friendRpcClient.GetFriendIDs(ctx, req.UserInfo.UserID) - //if err != nil { - // return nil, err - //} - //if req.UserInfo.Nickname != "" || req.UserInfo.FaceURL != "" { - // if err = s.NotificationUserInfoUpdate(ctx, req.UserInfo.UserID,oldUser); err != nil { - // return nil, err - // } - //} - //for _, friendID := range friends { - // s.friendNotificationSender.FriendInfoUpdatedNotification(ctx, req.UserInfo.UserID, friendID) - //} + if req.UserInfo.Nickname != "" || req.UserInfo.FaceURL != "" { + if err = s.groupRpcClient.NotificationUserInfoUpdate(ctx, req.UserInfo.UserID); err != nil { + return nil, err + } + } + for _, friendID := range friends { + s.friendNotificationSender.FriendInfoUpdatedNotification(ctx, req.UserInfo.UserID, friendID) + } s.webhookAfterUpdateUserInfo(ctx, &s.config.WebhooksConfig.AfterUpdateUserInfo, req) - if err = s.NotificationUserInfoUpdate(ctx, req.UserInfo.UserID, oldUser); err != nil { + if err = s.groupRpcClient.NotificationUserInfoUpdate(ctx, req.UserInfo.UserID); err != nil { return nil, err } return resp, nil @@ -171,29 +164,25 @@ func (s *userServer) UpdateUserInfoEx(ctx context.Context, req *pbuser.UpdateUse if err = s.webhookBeforeUpdateUserInfoEx(ctx, &s.config.WebhooksConfig.BeforeUpdateUserInfoEx, req); err != nil { return nil, err } - oldUser, err := s.db.GetUserByID(ctx, req.UserInfo.UserID) - if err != nil { - return nil, err - } data := convert.UserPb2DBMapEx(req.UserInfo) if err = s.db.UpdateByMap(ctx, req.UserInfo.UserID, data); err != nil { return nil, err } - //s.friendNotificationSender.UserInfoUpdatedNotification(ctx, req.UserInfo.UserID) - //friends, err := s.friendRpcClient.GetFriendIDs(ctx, req.UserInfo.UserID) - //if err != nil { - // return nil, err - //} - //if req.UserInfo.Nickname != nil || req.UserInfo.FaceURL != nil { - // if err := s.NotificationUserInfoUpdate(ctx, req.UserInfo.UserID); err != nil { - // return nil, err - // } - //} - //for _, friendID := range friends { - // s.friendNotificationSender.FriendInfoUpdatedNotification(ctx, req.UserInfo.UserID, friendID) - //} + s.friendNotificationSender.UserInfoUpdatedNotification(ctx, req.UserInfo.UserID) + friends, err := s.friendRpcClient.GetFriendIDs(ctx, req.UserInfo.UserID) + if err != nil { + return nil, err + } + if req.UserInfo.Nickname != nil || req.UserInfo.FaceURL != nil { + if err := s.groupRpcClient.NotificationUserInfoUpdate(ctx, req.UserInfo.UserID); err != nil { + return nil, err + } + } + for _, friendID := range friends { + s.friendNotificationSender.FriendInfoUpdatedNotification(ctx, req.UserInfo.UserID, friendID) + } s.webhookAfterUpdateUserInfoEx(ctx, &s.config.WebhooksConfig.AfterUpdateUserInfoEx, req) - if err := s.NotificationUserInfoUpdate(ctx, req.UserInfo.UserID, oldUser); err != nil { + if err := s.groupRpcClient.NotificationUserInfoUpdate(ctx, req.UserInfo.UserID); err != nil { return nil, err } return resp, nil @@ -695,40 +684,10 @@ func (s *userServer) userModelToResp(users []*tablerelation.User, pagination pag return &pbuser.SearchNotificationAccountResp{Total: total, NotificationAccounts: notificationAccounts} } -func (s *userServer) NotificationUserInfoUpdate(ctx context.Context, userID string, oldUser *tablerelation.User) error { - user, err := s.db.GetUserByID(ctx, userID) +func (s *userServer) SortQuery(ctx context.Context, req *pbuser.SortQueryReq) (*pbuser.SortQueryResp, error) { + users, err := s.db.SortQuery(ctx, req.UserIDName, req.Asc) if err != nil { - return err + return nil, err } - if *user == *oldUser { - return nil - } - s.friendNotificationSender.UserInfoUpdatedNotification(ctx, userID) - if user.Nickname == oldUser.Nickname && user.FaceURL == oldUser.FaceURL { - return nil - } - oldUserInfo := convert.UserDB2Pb(oldUser) - newUserInfo := convert.UserDB2Pb(user) - var wg sync.WaitGroup - var es [2]error - wg.Add(len(es)) - go func() { - defer wg.Done() - _, es[0] = s.groupRpcClient.Client.NotificationUserInfoUpdate(ctx, &group.NotificationUserInfoUpdateReq{ - UserID: userID, - OldUserInfo: oldUserInfo, - NewUserInfo: newUserInfo, - }) - }() - - go func() { - defer wg.Done() - _, es[1] = s.friendRpcClient.Client.NotificationUserInfoUpdate(ctx, &friendpb.NotificationUserInfoUpdateReq{ - UserID: userID, - OldUserInfo: oldUserInfo, - NewUserInfo: newUserInfo, - }) - }() - wg.Wait() - return errors.Join(es[:]...) + return &pbuser.SortQueryResp{Users: convert.UsersDB2Pb(users)}, nil } diff --git a/pkg/common/storage/cache/group.go b/pkg/common/storage/cache/group.go index f02379a7d..00a46aaf8 100644 --- a/pkg/common/storage/cache/group.go +++ b/pkg/common/storage/cache/group.go @@ -46,7 +46,6 @@ type GroupCache interface { GetGroupMemberInfo(ctx context.Context, groupID, userID string) (groupMember *model.GroupMember, err error) GetGroupMembersInfo(ctx context.Context, groupID string, userID []string) (groupMembers []*model.GroupMember, err error) GetAllGroupMembersInfo(ctx context.Context, groupID string) (groupMembers []*model.GroupMember, err error) - GetGroupMembersPage(ctx context.Context, groupID string, userID []string, showNumber, pageNumber int32) (total uint32, groupMembers []*model.GroupMember, err error) FindGroupMemberUser(ctx context.Context, groupIDs []string, userID string) ([]*model.GroupMember, error) GetGroupRoleLevelMemberIDs(ctx context.Context, groupID string, roleLevel int32) ([]string, error) diff --git a/pkg/common/storage/cache/redis/group.go b/pkg/common/storage/cache/redis/group.go index 67e246058..9699230ef 100644 --- a/pkg/common/storage/cache/redis/group.go +++ b/pkg/common/storage/cache/redis/group.go @@ -27,7 +27,6 @@ import ( "github.com/openimsdk/protocol/constant" "github.com/openimsdk/tools/errs" "github.com/openimsdk/tools/log" - "github.com/openimsdk/tools/utils/datautil" "github.com/redis/go-redis/v9" "time" ) @@ -296,26 +295,6 @@ func (g *GroupCacheRedis) GetGroupMembersInfo(ctx context.Context, groupID strin }) } -func (g *GroupCacheRedis) GetGroupMembersPage( - ctx context.Context, - groupID string, - userIDs []string, - showNumber, pageNumber int32, -) (total uint32, groupMembers []*model.GroupMember, err error) { - groupMemberIDs, err := g.GetGroupMemberIDs(ctx, groupID) - if err != nil { - return 0, nil, err - } - if userIDs != nil { - userIDs = datautil.BothExist(userIDs, groupMemberIDs) - } else { - userIDs = groupMemberIDs - } - groupMembers, err = g.GetGroupMembersInfo(ctx, groupID, datautil.Paginate(userIDs, int(showNumber), int(showNumber))) - - return uint32(len(userIDs)), groupMembers, err -} - func (g *GroupCacheRedis) GetAllGroupMembersInfo(ctx context.Context, groupID string) (groupMembers []*model.GroupMember, err error) { groupMemberIDs, err := g.GetGroupMemberIDs(ctx, groupID) if err != nil { diff --git a/pkg/common/storage/controller/friend.go b/pkg/common/storage/controller/friend.go index 8f72703c0..e03208df9 100644 --- a/pkg/common/storage/controller/friend.go +++ b/pkg/common/storage/controller/friend.go @@ -86,9 +86,7 @@ type FriendDatabase interface { FindFriendUserID(ctx context.Context, friendUserID string) ([]string, error) - UpdateFriendUserInfo(ctx context.Context, friendUserID string, ownerUserID []string, nickname string, faceURL string) error - - SearchFriend(ctx context.Context, ownerUserID, keyword string, pagination pagination.Pagination) (int64, []*model.Friend, error) + //SearchFriend(ctx context.Context, ownerUserID, keyword string, pagination pagination.Pagination) (int64, []*model.Friend, error) } type friendDatabase struct { @@ -378,15 +376,6 @@ func (f *friendDatabase) FindFriendUserID(ctx context.Context, friendUserID stri return f.friend.FindFriendUserID(ctx, friendUserID) } -func (f *friendDatabase) UpdateFriendUserInfo(ctx context.Context, friendUserID string, ownerUserIDs []string, nickname string, faceURL string) error { - return f.tx.Transaction(ctx, func(ctx context.Context) error { - if err := f.friend.UpdateFriendUserInfo(ctx, friendUserID, nickname, faceURL); err != nil { - return err - } - return f.cache.DelOwner(friendUserID, ownerUserIDs).DelMaxFriendVersion(ownerUserIDs...).ChainExecDel(ctx) - }) -} - -func (f *friendDatabase) SearchFriend(ctx context.Context, ownerUserID, keyword string, pagination pagination.Pagination) (int64, []*model.Friend, error) { - return f.friend.SearchFriend(ctx, ownerUserID, keyword, pagination) -} +//func (f *friendDatabase) SearchFriend(ctx context.Context, ownerUserID, keyword string, pagination pagination.Pagination) (int64, []*model.Friend, error) { +// return f.friend.SearchFriend(ctx, ownerUserID, keyword, pagination) +//} diff --git a/pkg/common/storage/controller/user.go b/pkg/common/storage/controller/user.go index 09dc2db22..9efe535c0 100644 --- a/pkg/common/storage/controller/user.go +++ b/pkg/common/storage/controller/user.go @@ -60,6 +60,8 @@ type UserDatabase interface { CountTotal(ctx context.Context, before *time.Time) (int64, error) // CountRangeEverydayTotal Get the user increment in the range CountRangeEverydayTotal(ctx context.Context, start time.Time, end time.Time) (map[string]int64, error) + + SortQuery(ctx context.Context, userIDName map[string]string, asc bool) ([]*model.User, error) // SubscribeUsersStatus Subscribe a user's presence status SubscribeUsersStatus(ctx context.Context, userID string, userIDs []string) error // UnsubscribeUsersStatus unsubscribe a user's presence status @@ -210,6 +212,10 @@ func (u *userDatabase) CountRangeEverydayTotal(ctx context.Context, start time.T return u.userDB.CountRangeEverydayTotal(ctx, start, end) } +func (u *userDatabase) SortQuery(ctx context.Context, userIDName map[string]string, asc bool) ([]*model.User, error) { + return u.userDB.SortQuery(ctx, userIDName, asc) +} + // SubscribeUsersStatus Subscribe or unsubscribe a user's presence status. func (u *userDatabase) SubscribeUsersStatus(ctx context.Context, userID string, userIDs []string) error { err := u.mongoDB.AddSubscriptionList(ctx, userID, userIDs) diff --git a/pkg/common/storage/database/friend.go b/pkg/common/storage/database/friend.go index 6ab1185bc..1c9cd1033 100644 --- a/pkg/common/storage/database/friend.go +++ b/pkg/common/storage/database/friend.go @@ -52,9 +52,7 @@ type Friend interface { FindFriendUserID(ctx context.Context, friendUserID string) ([]string, error) - UpdateFriendUserInfo(ctx context.Context, friendUserID string, nickname string, faceURL string) error - - SearchFriend(ctx context.Context, ownerUserID, keyword string, pagination pagination.Pagination) (int64, []*model.Friend, error) + //SearchFriend(ctx context.Context, ownerUserID, keyword string, pagination pagination.Pagination) (int64, []*model.Friend, error) FindOwnerFriendUserIds(ctx context.Context, ownerUserID string, limit int) ([]string, error) } diff --git a/pkg/common/storage/database/mgo/friend.go b/pkg/common/storage/database/mgo/friend.go index e0e30642c..646951d7d 100644 --- a/pkg/common/storage/database/mgo/friend.go +++ b/pkg/common/storage/database/mgo/friend.go @@ -53,7 +53,7 @@ func NewFriendMongo(db *mongo.Database) (database.Friend, error) { } func (f *FriendMgo) friendSort() any { - return bson.D{{"is_pinned", -1}, {"friend_nickname", 1}, {"create_time", 1}} + return bson.D{{"is_pinned", -1}, {"create_time", 1}} } // Create inserts multiple friend records. @@ -168,7 +168,7 @@ func (f *FriendMgo) FindInWhoseFriends(ctx context.Context, friendUserID string, // FindFriendUserIDs retrieves a list of friend user IDs for a given owner. func (f *FriendMgo) FindFriendUserIDs(ctx context.Context, ownerUserID string) ([]string, error) { filter := bson.M{"owner_user_id": ownerUserID} - return mongoutil.Find[string](ctx, f.coll, filter, options.Find().SetProjection(bson.M{"_id": 0, "friend_user_id": 1})) + return mongoutil.Find[string](ctx, f.coll, filter, options.Find().SetProjection(bson.M{"_id": 0, "friend_user_id": 1}).SetSort(f.friendSort())) } func (f *FriendMgo) UpdateFriends(ctx context.Context, ownerUserID string, friendUserIDs []string, val map[string]any) error { @@ -204,25 +204,17 @@ func (f *FriendMgo) FindFriendUserID(ctx context.Context, friendUserID string) ( return mongoutil.Find[string](ctx, f.coll, filter, options.Find().SetProjection(bson.M{"_id": 0, "owner_user_id": 1}).SetSort(f.friendSort())) } -func (f *FriendMgo) UpdateFriendUserInfo(ctx context.Context, friendUserID string, nickname string, faceURL string) error { - filter := bson.M{ - "friend_user_id": friendUserID, - } - _, err := mongoutil.UpdateMany(ctx, f.coll, filter, bson.M{"$set": bson.M{"nickname": nickname, "face_url": faceURL}}) - return err -} - -func (f *FriendMgo) SearchFriend(ctx context.Context, ownerUserID, keyword string, pagination pagination.Pagination) (int64, []*model.Friend, error) { - filter := bson.M{ - "owner_user_id": ownerUserID, - } - if keyword != "" { - filter["$or"] = []bson.M{ - {"remark": bson.M{"$regex": keyword, "$options": "i"}}, - {"nickname": bson.M{"$regex": keyword, "$options": "i"}}, - {"friend_user_id": bson.M{"$regex": keyword, "$options": "i"}}, - } - } - opt := options.Find().SetSort(f.friendSort()) - return mongoutil.FindPage[*model.Friend](ctx, f.coll, filter, pagination, opt) -} +//func (f *FriendMgo) SearchFriend(ctx context.Context, ownerUserID, keyword string, pagination pagination.Pagination) (int64, []*model.Friend, error) { +// filter := bson.M{ +// "owner_user_id": ownerUserID, +// } +// if keyword != "" { +// filter["$or"] = []bson.M{ +// {"remark": bson.M{"$regex": keyword, "$options": "i"}}, +// {"nickname": bson.M{"$regex": keyword, "$options": "i"}}, +// {"friend_user_id": bson.M{"$regex": keyword, "$options": "i"}}, +// } +// } +// opt := options.Find().SetSort(f.friendSort()) +// return mongoutil.FindPage[*model.Friend](ctx, f.coll, filter, pagination, opt) +//} diff --git a/pkg/common/storage/database/mgo/user.go b/pkg/common/storage/database/mgo/user.go index 96cb18882..793b8cdc8 100644 --- a/pkg/common/storage/database/mgo/user.go +++ b/pkg/common/storage/database/mgo/user.go @@ -319,3 +319,69 @@ func (u *UserMgo) CountRangeEverydayTotal(ctx context.Context, start time.Time, } return res, nil } + +func (u *UserMgo) SortQuery(ctx context.Context, userIDName map[string]string, asc bool) ([]*model.User, error) { + if len(userIDName) == 0 { + return nil, nil + } + userIDs := make([]string, 0, len(userIDName)) + attached := make(map[string]string) + for userID, name := range userIDName { + userIDs = append(userIDs, userID) + if name == "" { + continue + } + attached[userID] = name + } + var sortValue int + if asc { + sortValue = 1 + } else { + sortValue = -1 + } + if len(attached) == 0 { + filter := bson.M{"user_id": bson.M{"$in": userIDs}} + opt := options.Find().SetSort(bson.M{"nickname": sortValue}) + return mongoutil.Find[*model.User](ctx, u.coll, filter, opt) + } + pipeline := []bson.M{ + { + "$match": bson.M{ + "user_id": bson.M{"$in": userIDs}, + }, + }, + { + "$addFields": bson.M{ + "_query_sort_name": bson.M{ + "$arrayElemAt": []any{ + bson.M{ + "$filter": bson.M{ + "input": bson.M{ + "$objectToArray": attached, + }, + "as": "item", + "cond": bson.M{ + "$eq": []any{"$$item.k", "$user_id"}, + }, + }, + }, + 0, + }, + }, + }, + }, + { + "$addFields": bson.M{ + "_query_sort_name": bson.M{ + "$ifNull": []any{"$_query_sort_name.v", "$nickname"}, + }, + }, + }, + { + "$sort": bson.M{ + "_query_sort_name": sortValue, + }, + }, + } + return mongoutil.Aggregate[*model.User](ctx, u.coll, pipeline) +} diff --git a/pkg/common/storage/database/user.go b/pkg/common/storage/database/user.go index 2e4088620..4ddc8285f 100644 --- a/pkg/common/storage/database/user.go +++ b/pkg/common/storage/database/user.go @@ -39,6 +39,9 @@ type User interface { CountTotal(ctx context.Context, before *time.Time) (count int64, err error) // Get user total quantity every day CountRangeEverydayTotal(ctx context.Context, start time.Time, end time.Time) (map[string]int64, error) + + SortQuery(ctx context.Context, userIDName map[string]string, asc bool) ([]*model.User, error) + // CRUD user command AddUserCommand(ctx context.Context, userID string, Type int32, UUID string, value string, ex string) error DeleteUserCommand(ctx context.Context, userID string, Type int32, UUID string) error From c5f565ff209740e35a81976e427c4fbb26d7d96e Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Thu, 6 Jun 2024 09:57:47 +0800 Subject: [PATCH 26/59] sync option --- internal/rpc/friend/convert.go | 0 internal/rpc/friend/friend.go | 2 +- internal/rpc/friend/sync.go | 8 ++++---- internal/rpc/user/user.go | 4 ---- 4 files changed, 5 insertions(+), 9 deletions(-) delete mode 100644 internal/rpc/friend/convert.go diff --git a/internal/rpc/friend/convert.go b/internal/rpc/friend/convert.go deleted file mode 100644 index e69de29bb..000000000 diff --git a/internal/rpc/friend/friend.go b/internal/rpc/friend/friend.go index 51fae71c4..cbc16cca9 100644 --- a/internal/rpc/friend/friend.go +++ b/internal/rpc/friend/friend.go @@ -275,7 +275,7 @@ func (s *friendServer) GetDesignatedFriends(ctx context.Context, req *relation.G if err != nil { return nil, err } - return &pbfriend.GetDesignatedFriendsResp{ + return &relation.GetDesignatedFriendsResp{ FriendsInfo: friends, }, nil } diff --git a/internal/rpc/friend/sync.go b/internal/rpc/friend/sync.go index 46bc1d66a..9af1c47d3 100644 --- a/internal/rpc/friend/sync.go +++ b/internal/rpc/friend/sync.go @@ -2,12 +2,12 @@ package friend import ( "context" + "github.com/openimsdk/protocol/sdkws" "github.com/openimsdk/open-im-server/v3/internal/rpc/incrversion" "github.com/openimsdk/open-im-server/v3/pkg/authverify" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" "github.com/openimsdk/protocol/relation" - "github.com/openimsdk/tools/errs" ) //func (s *friendServer) SearchFriends(ctx context.Context, req *pbfriend.SearchFriendsReq) (*pbfriend.SearchFriendsResp, error) { @@ -38,7 +38,7 @@ func (s *friendServer) GetIncrementalFriends(ctx context.Context, req *relation. if err := authverify.CheckAccessV3(ctx, req.UserID, s.config.Share.IMAdminUserID); err != nil { return nil, err } - opt := incrversion.Option[*sdkws.FriendInfo, pbfriend.GetIncrementalFriendsResp]{ + opt := incrversion.Option[*sdkws.FriendInfo, relation.GetIncrementalFriendsResp]{ Ctx: ctx, VersionKey: req.UserID, VersionID: req.VersionID, @@ -51,8 +51,8 @@ func (s *friendServer) GetIncrementalFriends(ctx context.Context, req *relation. return s.getFriend(ctx, req.UserID, ids) }, ID: func(elem *sdkws.FriendInfo) string { return elem.FriendUser.UserID }, - Resp: func(version *model.VersionLog, delIDs []string, list []*sdkws.FriendInfo, full bool) *pbfriend.GetIncrementalFriendsResp { - return &pbfriend.GetIncrementalFriendsResp{ + Resp: func(version *model.VersionLog, delIDs []string, list []*sdkws.FriendInfo, full bool) *relation.GetIncrementalFriendsResp { + return &relation.GetIncrementalFriendsResp{ VersionID: version.ID.Hex(), Version: uint64(version.Version), Full: full, diff --git a/internal/rpc/user/user.go b/internal/rpc/user/user.go index b1b20dc40..96f784fab 100644 --- a/internal/rpc/user/user.go +++ b/internal/rpc/user/user.go @@ -16,10 +16,8 @@ package user import ( "context" - "errors" "math/rand" "strings" - "sync" "time" "github.com/openimsdk/open-im-server/v3/internal/rpc/friend" @@ -29,8 +27,6 @@ import ( tablerelation "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" "github.com/openimsdk/open-im-server/v3/pkg/common/webhook" "github.com/openimsdk/open-im-server/v3/pkg/localcache" - "github.com/openimsdk/protocol/group" - "github.com/openimsdk/protocol/relation" "github.com/openimsdk/tools/db/redisutil" "github.com/openimsdk/open-im-server/v3/pkg/authverify" From a1523f47e7c8e762e2231df5e87af0c785c772cb Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Thu, 6 Jun 2024 15:28:03 +0800 Subject: [PATCH 27/59] sync option --- config/redis.yml | 4 +-- internal/rpc/friend/sync.go | 26 -------------- internal/rpc/group/sync.go | 4 --- internal/rpc/incrversion/option.go | 45 ++++++++++++------------ pkg/common/storage/cache/friend.go | 2 +- pkg/common/storage/cache/group.go | 4 +-- pkg/common/storage/cache/redis/friend.go | 20 +++++------ pkg/common/storage/cache/redis/group.go | 42 +++++++++++----------- pkg/common/storage/controller/friend.go | 8 ++--- pkg/common/storage/controller/group.go | 19 +++++----- pkg/common/storage/model/friend.go | 2 -- 11 files changed, 72 insertions(+), 104 deletions(-) diff --git a/config/redis.yml b/config/redis.yml index 6fe0dd02d..404d18953 100644 --- a/config/redis.yml +++ b/config/redis.yml @@ -1,6 +1,6 @@ -address: [ localhost:16379 ] +address: [ 172.16.8.48:16379 ] username: '' password: openIM123 clusterMode: false db: 0 -maxRetry: 10 \ No newline at end of file +maxRetry: 10 diff --git a/internal/rpc/friend/sync.go b/internal/rpc/friend/sync.go index 9af1c47d3..a73cf6a17 100644 --- a/internal/rpc/friend/sync.go +++ b/internal/rpc/friend/sync.go @@ -10,30 +10,6 @@ import ( "github.com/openimsdk/protocol/relation" ) -//func (s *friendServer) SearchFriends(ctx context.Context, req *pbfriend.SearchFriendsReq) (*pbfriend.SearchFriendsResp, error) { -// if err := s.userRpcClient.Access(ctx, req.UserID); err != nil { -// return nil, err -// } -// if req.Keyword == "" { -// total, friends, err := s.friendDatabase.PageOwnerFriends(ctx, req.UserID, req.Pagination) -// if err != nil { -// return nil, err -// } -// return &pbfriend.SearchFriendsResp{ -// Total: total, -// Friends: friendsDB2PB(friends), -// }, nil -// } -// total, friends, err := s.friendDatabase.SearchFriend(ctx, req.UserID, req.Keyword, req.Pagination) -// if err != nil { -// return nil, err -// } -// return &pbfriend.SearchFriendsResp{ -// Total: total, -// Friends: friendsDB2PB(friends), -// }, nil -//} - func (s *friendServer) GetIncrementalFriends(ctx context.Context, req *relation.GetIncrementalFriendsReq) (*relation.GetIncrementalFriendsResp, error) { if err := authverify.CheckAccessV3(ctx, req.UserID, s.config.Share.IMAdminUserID); err != nil { return nil, err @@ -43,10 +19,8 @@ func (s *friendServer) GetIncrementalFriends(ctx context.Context, req *relation. VersionKey: req.UserID, VersionID: req.VersionID, VersionNumber: req.Version, - SyncLimit: s.config.RpcConfig.FriendSyncCount, Version: s.friendDatabase.FindFriendIncrVersion, CacheMaxVersion: s.friendDatabase.FindMaxFriendVersionCache, - SortID: s.friendDatabase.FindSortFriendUserIDs, Find: func(ctx context.Context, ids []string) ([]*sdkws.FriendInfo, error) { return s.getFriend(ctx, req.UserID, ids) }, diff --git a/internal/rpc/group/sync.go b/internal/rpc/group/sync.go index 6e0103c83..b35816a7e 100644 --- a/internal/rpc/group/sync.go +++ b/internal/rpc/group/sync.go @@ -15,10 +15,8 @@ func (s *groupServer) GetIncrementalGroupMember(ctx context.Context, req *pbgrou VersionKey: req.GroupID, VersionID: req.VersionID, VersionNumber: req.Version, - SyncLimit: s.config.RpcConfig.GroupSyncCount, Version: s.db.FindMemberIncrVersion, CacheMaxVersion: s.db.FindMaxGroupMemberVersionCache, - SortID: s.db.FindSortGroupMemberUserIDs, Find: func(ctx context.Context, ids []string) ([]*sdkws.GroupMemberFullInfo, error) { return s.getGroupMembersInfo(ctx, req.GroupID, ids) }, @@ -46,10 +44,8 @@ func (s *groupServer) GetIncrementalJoinGroup(ctx context.Context, req *pbgroup. VersionKey: req.UserID, VersionID: req.VersionID, VersionNumber: req.Version, - SyncLimit: s.config.RpcConfig.GroupSyncCount, Version: s.db.FindJoinIncrVersion, CacheMaxVersion: s.db.FindMaxJoinGroupVersionCache, - SortID: s.db.FindSortJoinGroupIDs, Find: s.getGroupsInfo, ID: func(elem *sdkws.GroupInfo) string { return elem.GroupID }, Resp: func(version *model.VersionLog, delIDs []string, list []*sdkws.GroupInfo, full bool) *pbgroup.GetIncrementalJoinGroupResp { diff --git a/internal/rpc/incrversion/option.go b/internal/rpc/incrversion/option.go index 3146985aa..ceb024a3e 100644 --- a/internal/rpc/incrversion/option.go +++ b/internal/rpc/incrversion/option.go @@ -16,6 +16,8 @@ import ( // return maxSync //} +const syncLimit = 200 + const ( tagQuery = iota + 1 tagFull @@ -23,17 +25,17 @@ const ( ) type Option[A, B any] struct { - Ctx context.Context - VersionKey string - VersionID string - VersionNumber uint64 - SyncLimit int + Ctx context.Context + VersionKey string + VersionID string + VersionNumber uint64 + //SyncLimit int CacheMaxVersion func(ctx context.Context, dId string) (*model.VersionLog, error) Version func(ctx context.Context, dId string, version uint, limit int) (*model.VersionLog, error) - SortID func(ctx context.Context, dId string) ([]string, error) - Find func(ctx context.Context, ids []string) ([]A, error) - ID func(elem A) string - Resp func(version *model.VersionLog, delIDs []string, list []A, full bool) *B + //SortID func(ctx context.Context, dId string) ([]string, error) + Find func(ctx context.Context, ids []string) ([]A, error) + ID func(elem A) string + Resp func(version *model.VersionLog, delIDs []string, list []A, full bool) *B } func (o *Option[A, B]) newError(msg string) error { @@ -47,15 +49,15 @@ func (o *Option[A, B]) check() error { if o.VersionKey == "" { return o.newError("versionKey is empty") } - if o.SyncLimit <= 0 { - return o.newError("invalid synchronization quantity") - } + //if o.SyncLimit <= 0 { + // return o.newError("invalid synchronization quantity") + //} if o.Version == nil { return o.newError("func version is nil") } - if o.SortID == nil { - return o.newError("func allID is nil") - } + //if o.SortID == nil { + // return o.newError("func allID is nil") + //} if o.Find == nil { return o.newError("func find is nil") } @@ -81,7 +83,7 @@ func (o *Option[A, B]) getVersion(tag *int) (*model.VersionLog, error) { if o.CacheMaxVersion == nil { if o.validVersion() { *tag = tagQuery - return o.Version(o.Ctx, o.VersionKey, uint(o.VersionNumber), o.SyncLimit) + return o.Version(o.Ctx, o.VersionKey, uint(o.VersionNumber), syncLimit) } *tag = tagFull return o.Version(o.Ctx, o.VersionKey, 0, 0) @@ -103,7 +105,7 @@ func (o *Option[A, B]) getVersion(tag *int) (*model.VersionLog, error) { return cache, nil } *tag = tagQuery - return o.Version(o.Ctx, o.VersionKey, uint(o.VersionNumber), o.SyncLimit) + return o.Version(o.Ctx, o.VersionKey, uint(o.VersionNumber), syncLimit) } } @@ -131,12 +133,11 @@ func (o *Option[A, B]) Build() (*B, error) { deleteIDs []string changeIDs []string ) - //full := o.VersionID != version.ID.Hex() || version.Full() if full { - changeIDs, err = o.SortID(o.Ctx, o.VersionKey) - if err != nil { - return nil, err - } + //changeIDs, err = o.SortID(o.Ctx, o.VersionKey) + //if err != nil { + // return nil, err + //} } else { deleteIDs, changeIDs = version.DeleteAndChangeIDs() } diff --git a/pkg/common/storage/cache/friend.go b/pkg/common/storage/cache/friend.go index 064e3baae..b451d3675 100644 --- a/pkg/common/storage/cache/friend.go +++ b/pkg/common/storage/cache/friend.go @@ -39,7 +39,7 @@ type FriendCache interface { //DelSortFriendUserIDs(ownerUserIDs ...string) FriendCache - FindSortFriendUserIDs(ctx context.Context, ownerUserID string) ([]string, error) + //FindSortFriendUserIDs(ctx context.Context, ownerUserID string) ([]string, error) //FindFriendIncrVersion(ctx context.Context, ownerUserID string, version uint, limit int) (*relationtb.VersionLog, error) diff --git a/pkg/common/storage/cache/group.go b/pkg/common/storage/cache/group.go index 00a46aaf8..73479bb1b 100644 --- a/pkg/common/storage/cache/group.go +++ b/pkg/common/storage/cache/group.go @@ -59,8 +59,8 @@ type GroupCache interface { GetGroupMemberNum(ctx context.Context, groupID string) (memberNum int64, err error) DelGroupsMemberNum(groupID ...string) GroupCache - FindSortGroupMemberUserIDs(ctx context.Context, groupID string) ([]string, error) - FindSortJoinGroupIDs(ctx context.Context, userID string) ([]string, error) + //FindSortGroupMemberUserIDs(ctx context.Context, groupID string) ([]string, error) + //FindSortJoinGroupIDs(ctx context.Context, userID string) ([]string, error) DelMaxGroupMemberVersion(groupIDs ...string) GroupCache DelMaxJoinGroupVersion(userIDs ...string) GroupCache diff --git a/pkg/common/storage/cache/redis/friend.go b/pkg/common/storage/cache/redis/friend.go index 62fad91cd..01988310c 100644 --- a/pkg/common/storage/cache/redis/friend.go +++ b/pkg/common/storage/cache/redis/friend.go @@ -193,16 +193,16 @@ func (f *FriendCacheRedis) DelMaxFriendVersion(ownerUserIDs ...string) cache.Fri return newFriendCache } -func (f *FriendCacheRedis) FindSortFriendUserIDs(ctx context.Context, ownerUserID string) ([]string, error) { - userIDs, err := f.GetFriendIDs(ctx, ownerUserID) - if err != nil { - return nil, err - } - if len(userIDs) > f.syncCount { - userIDs = userIDs[:f.syncCount] - } - return userIDs, nil -} +//func (f *FriendCacheRedis) FindSortFriendUserIDs(ctx context.Context, ownerUserID string) ([]string, error) { +// userIDs, err := f.GetFriendIDs(ctx, ownerUserID) +// if err != nil { +// return nil, err +// } +// if len(userIDs) > f.syncCount { +// userIDs = userIDs[:f.syncCount] +// } +// return userIDs, nil +//} func (f *FriendCacheRedis) FindMaxFriendVersion(ctx context.Context, ownerUserID string) (*model.VersionLog, error) { return getCache(ctx, f.rcClient, f.getFriendMaxVersionKey(ownerUserID), f.expireTime, func(ctx context.Context) (*model.VersionLog, error) { diff --git a/pkg/common/storage/cache/redis/group.go b/pkg/common/storage/cache/redis/group.go index 9699230ef..4015d5cd9 100644 --- a/pkg/common/storage/cache/redis/group.go +++ b/pkg/common/storage/cache/redis/group.go @@ -405,27 +405,27 @@ func (g *GroupCacheRedis) FindGroupMemberUser(ctx context.Context, groupIDs []st }) } -func (g *GroupCacheRedis) FindSortGroupMemberUserIDs(ctx context.Context, groupID string) ([]string, error) { - userIDs, err := g.GetGroupMemberIDs(ctx, groupID) - if err != nil { - return nil, err - } - if len(userIDs) > g.syncCount { - userIDs = userIDs[:g.syncCount] - } - return userIDs, nil -} - -func (g *GroupCacheRedis) FindSortJoinGroupIDs(ctx context.Context, userID string) ([]string, error) { - groupIDs, err := g.GetJoinedGroupIDs(ctx, userID) - if err != nil { - return nil, err - } - if len(groupIDs) > g.syncCount { - groupIDs = groupIDs[:g.syncCount] - } - return groupIDs, nil -} +//func (g *GroupCacheRedis) FindSortGroupMemberUserIDs(ctx context.Context, groupID string) ([]string, error) { +// userIDs, err := g.GetGroupMemberIDs(ctx, groupID) +// if err != nil { +// return nil, err +// } +// if len(userIDs) > g.syncCount { +// userIDs = userIDs[:g.syncCount] +// } +// return userIDs, nil +//} +// +//func (g *GroupCacheRedis) FindSortJoinGroupIDs(ctx context.Context, userID string) ([]string, error) { +// groupIDs, err := g.GetJoinedGroupIDs(ctx, userID) +// if err != nil { +// return nil, err +// } +// if len(groupIDs) > g.syncCount { +// groupIDs = groupIDs[:g.syncCount] +// } +// return groupIDs, nil +//} func (g *GroupCacheRedis) DelMaxGroupMemberVersion(groupIDs ...string) cache.GroupCache { keys := make([]string, 0, len(groupIDs)) diff --git a/pkg/common/storage/controller/friend.go b/pkg/common/storage/controller/friend.go index e03208df9..42c230598 100644 --- a/pkg/common/storage/controller/friend.go +++ b/pkg/common/storage/controller/friend.go @@ -78,7 +78,7 @@ type FriendDatabase interface { // UpdateFriends updates fields for friends UpdateFriends(ctx context.Context, ownerUserID string, friendUserIDs []string, val map[string]any) (err error) - FindSortFriendUserIDs(ctx context.Context, ownerUserID string) ([]string, error) + //FindSortFriendUserIDs(ctx context.Context, ownerUserID string) ([]string, error) FindFriendIncrVersion(ctx context.Context, ownerUserID string, version uint, limit int) (*model.VersionLog, error) @@ -360,9 +360,9 @@ func (f *friendDatabase) UpdateFriends(ctx context.Context, ownerUserID string, }) } -func (f *friendDatabase) FindSortFriendUserIDs(ctx context.Context, ownerUserID string) ([]string, error) { - return f.cache.FindSortFriendUserIDs(ctx, ownerUserID) -} +//func (f *friendDatabase) FindSortFriendUserIDs(ctx context.Context, ownerUserID string) ([]string, error) { +// return f.cache.FindSortFriendUserIDs(ctx, ownerUserID) +//} func (f *friendDatabase) FindFriendIncrVersion(ctx context.Context, ownerUserID string, version uint, limit int) (*model.VersionLog, error) { return f.friend.FindIncrVersion(ctx, ownerUserID, version, limit) diff --git a/pkg/common/storage/controller/group.go b/pkg/common/storage/controller/group.go index f9ee65955..14406420b 100644 --- a/pkg/common/storage/controller/group.go +++ b/pkg/common/storage/controller/group.go @@ -110,9 +110,8 @@ type GroupDatabase interface { FindMemberIncrVersion(ctx context.Context, groupID string, version uint, limit int) (*model.VersionLog, error) FindJoinIncrVersion(ctx context.Context, userID string, version uint, limit int) (*model.VersionLog, error) - FindSortGroupMemberUserIDs(ctx context.Context, groupID string) ([]string, error) - - FindSortJoinGroupIDs(ctx context.Context, userID string) ([]string, error) + //FindSortGroupMemberUserIDs(ctx context.Context, groupID string) ([]string, error) + //FindSortJoinGroupIDs(ctx context.Context, userID string) ([]string, error) FindMaxGroupMemberVersionCache(ctx context.Context, groupID string) (*model.VersionLog, error) FindMaxJoinGroupVersionCache(ctx context.Context, userID string) (*model.VersionLog, error) @@ -497,13 +496,13 @@ func (g *groupDatabase) FindJoinIncrVersion(ctx context.Context, userID string, return g.groupMemberDB.FindJoinIncrVersion(ctx, userID, version, limit) } -func (g *groupDatabase) FindSortGroupMemberUserIDs(ctx context.Context, groupID string) ([]string, error) { - return g.cache.FindSortGroupMemberUserIDs(ctx, groupID) -} - -func (g *groupDatabase) FindSortJoinGroupIDs(ctx context.Context, userID string) ([]string, error) { - return g.cache.FindSortJoinGroupIDs(ctx, userID) -} +//func (g *groupDatabase) FindSortGroupMemberUserIDs(ctx context.Context, groupID string) ([]string, error) { +// return g.cache.FindSortGroupMemberUserIDs(ctx, groupID) +//} +// +//func (g *groupDatabase) FindSortJoinGroupIDs(ctx context.Context, userID string) ([]string, error) { +// return g.cache.FindSortJoinGroupIDs(ctx, userID) +//} func (g *groupDatabase) FindMaxGroupMemberVersionCache(ctx context.Context, groupID string) (*model.VersionLog, error) { return g.cache.FindMaxGroupMemberVersion(ctx, groupID) diff --git a/pkg/common/storage/model/friend.go b/pkg/common/storage/model/friend.go index c7675e466..60a40d9c2 100644 --- a/pkg/common/storage/model/friend.go +++ b/pkg/common/storage/model/friend.go @@ -22,8 +22,6 @@ import ( type Friend struct { OwnerUserID string `bson:"owner_user_id"` FriendUserID string `bson:"friend_user_id"` - FriendNickname string `bson:"friend_nickname"` - FriendFaceURL string `bson:"friend_face_url"` Remark string `bson:"remark"` CreateTime time.Time `bson:"create_time"` AddSource int32 `bson:"add_source"` From 58c4c13cf1179437f42a9a68c1ec84deaa5b604c Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Fri, 7 Jun 2024 17:45:41 +0800 Subject: [PATCH 28/59] sync --- internal/api/group.go | 12 ++- internal/api/router.go | 2 + internal/rpc/friend/friend.go | 3 - internal/rpc/friend/sync.go | 14 ++-- internal/rpc/group/group.go | 5 +- internal/rpc/group/sync.go | 82 +++++++++++++++---- internal/rpc/incrversion/option.go | 40 ++++----- pkg/common/config/config.go | 6 +- pkg/common/storage/cache/redis/group.go | 3 - pkg/common/storage/controller/group.go | 13 ++- pkg/common/storage/database/group_member.go | 2 +- pkg/common/storage/database/mgo/friend.go | 8 +- .../storage/database/mgo/group_member.go | 18 ++-- .../storage/database/mgo/version_log.go | 18 ++-- pkg/common/storage/database/version_log.go | 2 +- pkg/common/storage/model/version_log.go | 26 ++++-- 16 files changed, 161 insertions(+), 93 deletions(-) diff --git a/internal/api/group.go b/internal/api/group.go index 5b8b9d044..0bf61c787 100644 --- a/internal/api/group.go +++ b/internal/api/group.go @@ -179,10 +179,18 @@ func (o *GroupApi) GetIncrementalGroupMemberBatch(c *gin.Context) { return } resp.List[req.GroupID] = res - changeCount += len(res.Changes) + len(res.DeleteUserIds) - if changeCount >= int(res.SyncCount) { + changeCount += len(res.Insert) + len(res.Delete) + len(res.Update) + if changeCount >= 200 { break } } apiresp.GinSuccess(c, resp) } + +func (o *GroupApi) GetIncrementalGroupMemberUserIDs(c *gin.Context) { + a2r.Call(group.GroupClient.GetIncrementalGroupMemberUserIDs, o.Client, c) +} + +func (o *GroupApi) GetIncrementalJoinGroupIDs(c *gin.Context) { + a2r.Call(group.GroupClient.GetIncrementalJoinGroupIDs, o.Client, c) +} diff --git a/internal/api/router.go b/internal/api/router.go index 445a24694..69c026ffd 100644 --- a/internal/api/router.go +++ b/internal/api/router.go @@ -120,6 +120,8 @@ func newGinRouter(disCov discovery.SvcDiscoveryRegistry, config *Config) *gin.En groupRouterGroup.POST("/get_incremental_join_group", g.GetIncrementalJoinGroup) groupRouterGroup.POST("/get_incremental_group_member", g.GetIncrementalGroupMember) groupRouterGroup.POST("/get_incremental_group_member_batch", g.GetIncrementalGroupMemberBatch) + groupRouterGroup.POST("/get_incremental_group_member_user_ids", g.GetIncrementalGroupMemberUserIDs) + groupRouterGroup.POST("/get_incremental_join_group_ids", g.GetIncrementalJoinGroupIDs) } // certificate authRouterGroup := r.Group("/auth") diff --git a/internal/rpc/friend/friend.go b/internal/rpc/friend/friend.go index cbc16cca9..12107125c 100644 --- a/internal/rpc/friend/friend.go +++ b/internal/rpc/friend/friend.go @@ -64,9 +64,6 @@ type Config struct { } func Start(ctx context.Context, config *Config, client discovery.SvcDiscoveryRegistry, server *grpc.Server) error { - if config.RpcConfig.FriendSyncCount < 1 { - config.RpcConfig.FriendSyncCount = constant.MaxSyncPullNumber - } mgocli, err := mongoutil.NewMongoDB(ctx, config.MongodbConfig.Build()) if err != nil { return err diff --git a/internal/rpc/friend/sync.go b/internal/rpc/friend/sync.go index a73cf6a17..cddd23f89 100644 --- a/internal/rpc/friend/sync.go +++ b/internal/rpc/friend/sync.go @@ -25,14 +25,14 @@ func (s *friendServer) GetIncrementalFriends(ctx context.Context, req *relation. return s.getFriend(ctx, req.UserID, ids) }, ID: func(elem *sdkws.FriendInfo) string { return elem.FriendUser.UserID }, - Resp: func(version *model.VersionLog, delIDs []string, list []*sdkws.FriendInfo, full bool) *relation.GetIncrementalFriendsResp { + Resp: func(version *model.VersionLog, deleteIds []string, insertList, updateList []*sdkws.FriendInfo, full bool) *relation.GetIncrementalFriendsResp { return &relation.GetIncrementalFriendsResp{ - VersionID: version.ID.Hex(), - Version: uint64(version.Version), - Full: full, - SyncCount: uint32(s.config.RpcConfig.FriendSyncCount), - DeleteUserIds: delIDs, - Changes: list, + VersionID: version.ID.Hex(), + Version: uint64(version.Version), + Full: full, + Delete: deleteIds, + Insert: insertList, + Update: updateList, } }, } diff --git a/internal/rpc/group/group.go b/internal/rpc/group/group.go index 1cdcdb54b..4d429b3d3 100644 --- a/internal/rpc/group/group.go +++ b/internal/rpc/group/group.go @@ -77,9 +77,6 @@ type Config struct { } func Start(ctx context.Context, config *Config, client discovery.SvcDiscoveryRegistry, server *grpc.Server) error { - if config.RpcConfig.GroupSyncCount <= 0 { - config.RpcConfig.GroupSyncCount = constant.MaxSyncPullNumber - } mgocli, err := mongoutil.NewMongoDB(ctx, config.MongodbConfig.Build()) if err != nil { return err @@ -104,7 +101,7 @@ func Start(ctx context.Context, config *Config, client discovery.SvcDiscoveryReg msgRpcClient := rpcclient.NewMessageRpcClient(client, config.Share.RpcRegisterName.Msg) conversationRpcClient := rpcclient.NewConversationRpcClient(client, config.Share.RpcRegisterName.Conversation) var gs groupServer - database := controller.NewGroupDatabase(rdb, &config.LocalCacheConfig, groupDB, groupMemberDB, groupRequestDB, mgocli.GetTx(), grouphash.NewGroupHashFromGroupServer(&gs), config.RpcConfig.GroupSyncCount) + database := controller.NewGroupDatabase(rdb, &config.LocalCacheConfig, groupDB, groupMemberDB, groupRequestDB, mgocli.GetTx(), grouphash.NewGroupHashFromGroupServer(&gs)) gs.db = database gs.user = userRpcClient gs.notification = NewGroupNotificationSender(database, &msgRpcClient, &userRpcClient, config, func(ctx context.Context, userIDs []string) ([]notification.CommonUser, error) { diff --git a/internal/rpc/group/sync.go b/internal/rpc/group/sync.go index b35816a7e..4bd34eb4d 100644 --- a/internal/rpc/group/sync.go +++ b/internal/rpc/group/sync.go @@ -2,6 +2,9 @@ package group import ( "context" + "crypto/md5" + "encoding/binary" + "encoding/json" "github.com/openimsdk/open-im-server/v3/internal/rpc/incrversion" "github.com/openimsdk/open-im-server/v3/pkg/authverify" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" @@ -9,6 +12,57 @@ import ( "github.com/openimsdk/protocol/sdkws" ) +func (s *groupServer) idHash(ids []string) uint64 { + if len(ids) == 0 { + return 0 + } + data, _ := json.Marshal(ids) + sum := md5.Sum(data) + return binary.BigEndian.Uint64(sum[:]) +} + +func (s *groupServer) GetIncrementalGroupMemberUserIDs(ctx context.Context, req *pbgroup.GetIncrementalGroupMemberUserIDsReq) (*pbgroup.GetIncrementalGroupMemberUserIDsResp, error) { + vl, err := s.db.FindMaxGroupMemberVersionCache(ctx, req.GroupID) + if err != nil { + return nil, err + } + userIDs, err := s.db.FindGroupMemberUserID(ctx, req.GroupID) + if err != nil { + return nil, err + } + idHash := s.idHash(userIDs) + if req.IdHash == idHash { + userIDs = nil + } + return &pbgroup.GetIncrementalGroupMemberUserIDsResp{ + Version: idHash, + VersionID: vl.ID.Hex(), + Equal: req.IdHash == idHash, + UserIDs: userIDs, + }, nil +} + +func (s *groupServer) GetIncrementalJoinGroupIDs(ctx context.Context, req *pbgroup.GetIncrementalJoinGroupIDsReq) (*pbgroup.GetIncrementalJoinGroupIDsResp, error) { + vl, err := s.db.FindMaxJoinGroupVersionCache(ctx, req.UserID) + if err != nil { + return nil, err + } + groupIDs, err := s.db.FindJoinGroupID(ctx, req.UserID) + if err != nil { + return nil, err + } + idHash := s.idHash(groupIDs) + if req.IdHash == idHash { + groupIDs = nil + } + return &pbgroup.GetIncrementalJoinGroupIDsResp{ + Version: idHash, + VersionID: vl.ID.Hex(), + Equal: req.IdHash == idHash, + GroupIDs: groupIDs, + }, nil +} + func (s *groupServer) GetIncrementalGroupMember(ctx context.Context, req *pbgroup.GetIncrementalGroupMemberReq) (*pbgroup.GetIncrementalGroupMemberResp, error) { opt := incrversion.Option[*sdkws.GroupMemberFullInfo, pbgroup.GetIncrementalGroupMemberResp]{ Ctx: ctx, @@ -21,14 +75,14 @@ func (s *groupServer) GetIncrementalGroupMember(ctx context.Context, req *pbgrou return s.getGroupMembersInfo(ctx, req.GroupID, ids) }, ID: func(elem *sdkws.GroupMemberFullInfo) string { return elem.UserID }, - Resp: func(version *model.VersionLog, delIDs []string, list []*sdkws.GroupMemberFullInfo, full bool) *pbgroup.GetIncrementalGroupMemberResp { + Resp: func(version *model.VersionLog, delIDs []string, insertList, updateList []*sdkws.GroupMemberFullInfo, full bool) *pbgroup.GetIncrementalGroupMemberResp { return &pbgroup.GetIncrementalGroupMemberResp{ - VersionID: version.ID.Hex(), - Version: uint64(version.Version), - Full: full, - SyncCount: uint32(s.config.RpcConfig.GroupSyncCount), - DeleteUserIds: delIDs, - Changes: list, + VersionID: version.ID.Hex(), + Version: uint64(version.Version), + Full: full, + Delete: delIDs, + Insert: insertList, + Update: updateList, } }, } @@ -48,14 +102,14 @@ func (s *groupServer) GetIncrementalJoinGroup(ctx context.Context, req *pbgroup. CacheMaxVersion: s.db.FindMaxJoinGroupVersionCache, Find: s.getGroupsInfo, ID: func(elem *sdkws.GroupInfo) string { return elem.GroupID }, - Resp: func(version *model.VersionLog, delIDs []string, list []*sdkws.GroupInfo, full bool) *pbgroup.GetIncrementalJoinGroupResp { + Resp: func(version *model.VersionLog, delIDs []string, insertList, updateList []*sdkws.GroupInfo, full bool) *pbgroup.GetIncrementalJoinGroupResp { return &pbgroup.GetIncrementalJoinGroupResp{ - VersionID: version.ID.Hex(), - Version: uint64(version.Version), - Full: full, - SyncCount: uint32(s.config.RpcConfig.GroupSyncCount), - DeleteGroupIds: delIDs, - Changes: list, + VersionID: version.ID.Hex(), + Version: uint64(version.Version), + Full: full, + Delete: delIDs, + Insert: insertList, + Update: updateList, } }, } diff --git a/internal/rpc/incrversion/option.go b/internal/rpc/incrversion/option.go index ceb024a3e..f7a71244a 100644 --- a/internal/rpc/incrversion/option.go +++ b/internal/rpc/incrversion/option.go @@ -5,7 +5,6 @@ import ( "fmt" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" "github.com/openimsdk/tools/errs" - "github.com/openimsdk/tools/utils/datautil" "go.mongodb.org/mongo-driver/bson/primitive" ) @@ -35,7 +34,7 @@ type Option[A, B any] struct { //SortID func(ctx context.Context, dId string) ([]string, error) Find func(ctx context.Context, ids []string) ([]A, error) ID func(elem A) string - Resp func(version *model.VersionLog, delIDs []string, list []A, full bool) *B + Resp func(version *model.VersionLog, deleteIds []string, insertList, updateList []A, full bool) *B } func (o *Option[A, B]) newError(msg string) error { @@ -130,31 +129,28 @@ func (o *Option[A, B]) Build() (*B, error) { panic(fmt.Errorf("undefined tag %d", tag)) } var ( - deleteIDs []string - changeIDs []string + insertIds []string + deleteIds []string + updateIds []string ) - if full { - //changeIDs, err = o.SortID(o.Ctx, o.VersionKey) - //if err != nil { - // return nil, err - //} - } else { - deleteIDs, changeIDs = version.DeleteAndChangeIDs() + if !full { + insertIds, deleteIds, updateIds = version.DeleteAndChangeIDs() } - var list []A - if len(changeIDs) > 0 { - list, err = o.Find(o.Ctx, changeIDs) + var ( + insertList []A + updateList []A + ) + if len(insertIds) > 0 { + insertList, err = o.Find(o.Ctx, insertIds) if err != nil { return nil, err } - if (!full) && o.ID != nil && len(changeIDs) != len(list) { - foundIDs := datautil.SliceSetAny(list, o.ID) - for _, id := range changeIDs { - if _, ok := foundIDs[id]; !ok { - deleteIDs = append(deleteIDs, id) - } - } + } + if len(updateIds) > 0 { + updateList, err = o.Find(o.Ctx, updateIds) + if err != nil { + return nil, err } } - return o.Resp(version, deleteIDs, list, full), nil + return o.Resp(version, deleteIds, insertList, updateList, full), nil } diff --git a/pkg/common/config/config.go b/pkg/common/config/config.go index a4177dc06..5313c196a 100644 --- a/pkg/common/config/config.go +++ b/pkg/common/config/config.go @@ -244,8 +244,7 @@ type Friend struct { ListenIP string `mapstructure:"listenIP"` Ports []int `mapstructure:"ports"` } `mapstructure:"rpc"` - Prometheus Prometheus `mapstructure:"prometheus"` - FriendSyncCount int `mapstructure:"friendSyncCount"` + Prometheus Prometheus `mapstructure:"prometheus"` } type Group struct { @@ -254,8 +253,7 @@ type Group struct { ListenIP string `mapstructure:"listenIP"` Ports []int `mapstructure:"ports"` } `mapstructure:"rpc"` - Prometheus Prometheus `mapstructure:"prometheus"` - GroupSyncCount int `mapstructure:"groupSyncCount"` + Prometheus Prometheus `mapstructure:"prometheus"` } type Msg struct { diff --git a/pkg/common/storage/cache/redis/group.go b/pkg/common/storage/cache/redis/group.go index 4015d5cd9..c4598567d 100644 --- a/pkg/common/storage/cache/redis/group.go +++ b/pkg/common/storage/cache/redis/group.go @@ -45,7 +45,6 @@ type GroupCacheRedis struct { expireTime time.Duration rcClient *rockscache.Client groupHash cache.GroupHash - syncCount int } func NewGroupCacheRedis( @@ -56,7 +55,6 @@ func NewGroupCacheRedis( groupRequestDB database.GroupRequest, hashCode cache.GroupHash, opts *rockscache.Options, - syncCount int, ) cache.GroupCache { batchHandler := NewBatchDeleterRedis(rdb, opts, []string{localCache.Group.Topic}) g := localCache.Group @@ -70,7 +68,6 @@ func NewGroupCacheRedis( groupMemberDB: groupMemberDB, groupRequestDB: groupRequestDB, groupHash: hashCode, - syncCount: syncCount, } } diff --git a/pkg/common/storage/controller/group.go b/pkg/common/storage/controller/group.go index 14406420b..cb64bc73a 100644 --- a/pkg/common/storage/controller/group.go +++ b/pkg/common/storage/controller/group.go @@ -117,6 +117,8 @@ type GroupDatabase interface { FindMaxJoinGroupVersionCache(ctx context.Context, userID string) (*model.VersionLog, error) SearchJoinGroup(ctx context.Context, userID string, keyword string, pagination pagination.Pagination) (int64, []*model.Group, error) + + FindJoinGroupID(ctx context.Context, userID string) ([]string, error) } func NewGroupDatabase( @@ -127,14 +129,13 @@ func NewGroupDatabase( groupRequestDB database.GroupRequest, ctxTx tx.Tx, groupHash cache.GroupHash, - syncCount int, ) GroupDatabase { return &groupDatabase{ groupDB: groupDB, groupMemberDB: groupMemberDB, groupRequestDB: groupRequestDB, ctxTx: ctxTx, - cache: redis2.NewGroupCacheRedis(rdb, localCache, groupDB, groupMemberDB, groupRequestDB, groupHash, redis2.GetRocksCacheOptions(), syncCount), + cache: redis2.NewGroupCacheRedis(rdb, localCache, groupDB, groupMemberDB, groupRequestDB, groupHash, redis2.GetRocksCacheOptions()), } } @@ -146,6 +147,10 @@ type groupDatabase struct { cache cache.GroupCache } +func (g *groupDatabase) FindJoinGroupID(ctx context.Context, userID string) ([]string, error) { + return g.cache.GetJoinedGroupIDs(ctx, userID) +} + func (g *groupDatabase) FindGroupMembers(ctx context.Context, groupID string, userIDs []string) ([]*model.GroupMember, error) { return g.cache.GetGroupMembersInfo(ctx, groupID, userIDs) } @@ -243,7 +248,7 @@ func (g *groupDatabase) UpdateGroup(ctx context.Context, groupID string, data ma return err } for _, userID := range userIDs { - if err := g.groupMemberDB.JoinGroupIncrVersion(ctx, userID, []string{groupID}, false); err != nil { + if err := g.groupMemberDB.JoinGroupIncrVersion(ctx, userID, []string{groupID}, model.VersionStateUpdate); err != nil { return err } @@ -276,7 +281,7 @@ func (g *groupDatabase) DismissGroup(ctx context.Context, groupID string, delete } c = c.DelMaxJoinGroupVersion(userIDs...) if len(userIDs) > 0 { - if err := g.groupMemberDB.JoinGroupIncrVersion(ctx, groupID, userIDs, true); err != nil { + if err := g.groupMemberDB.JoinGroupIncrVersion(ctx, groupID, userIDs, model.VersionStateDelete); err != nil { return err } } diff --git a/pkg/common/storage/database/group_member.go b/pkg/common/storage/database/group_member.go index c397eda58..0051d694f 100644 --- a/pkg/common/storage/database/group_member.go +++ b/pkg/common/storage/database/group_member.go @@ -34,7 +34,7 @@ type GroupMember interface { TakeGroupMemberNum(ctx context.Context, groupID string) (count int64, err error) FindUserManagedGroupID(ctx context.Context, userID string) (groupIDs []string, err error) IsUpdateRoleLevel(data map[string]any) bool - JoinGroupIncrVersion(ctx context.Context, userID string, groupIDs []string, deleted bool) error + JoinGroupIncrVersion(ctx context.Context, userID string, groupIDs []string, state int32) error FindMemberIncrVersion(ctx context.Context, groupID string, version uint, limit int) (*model.VersionLog, error) FindJoinIncrVersion(ctx context.Context, userID string, version uint, limit int) (*model.VersionLog, error) } diff --git a/pkg/common/storage/database/mgo/friend.go b/pkg/common/storage/database/mgo/friend.go index 646951d7d..699d9cff6 100644 --- a/pkg/common/storage/database/mgo/friend.go +++ b/pkg/common/storage/database/mgo/friend.go @@ -66,7 +66,7 @@ func (f *FriendMgo) Create(ctx context.Context, friends []*model.Friend) error { mp[friend.OwnerUserID] = append(mp[friend.OwnerUserID], friend.FriendUserID) } for ownerUserID, friendUserIDs := range mp { - if err := f.owner.IncrVersion(ctx, ownerUserID, friendUserIDs, false); err != nil { + if err := f.owner.IncrVersion(ctx, ownerUserID, friendUserIDs, model.VersionStateInsert); err != nil { return err } } @@ -83,7 +83,7 @@ func (f *FriendMgo) Delete(ctx context.Context, ownerUserID string, friendUserID return mongoutil.IncrVersion(func() error { return mongoutil.DeleteOne(ctx, f.coll, filter) }, func() error { - return f.owner.IncrVersion(ctx, ownerUserID, friendUserIDs, true) + return f.owner.IncrVersion(ctx, ownerUserID, friendUserIDs, model.VersionStateDelete) }) } @@ -99,7 +99,7 @@ func (f *FriendMgo) UpdateByMap(ctx context.Context, ownerUserID string, friendU return mongoutil.IncrVersion(func() error { return mongoutil.UpdateOne(ctx, f.coll, filter, bson.M{"$set": args}, true) }, func() error { - return f.owner.IncrVersion(ctx, ownerUserID, []string{friendUserID}, false) + return f.owner.IncrVersion(ctx, ownerUserID, []string{friendUserID}, model.VersionStateUpdate) }) } @@ -189,7 +189,7 @@ func (f *FriendMgo) UpdateFriends(ctx context.Context, ownerUserID string, frien return mongoutil.IncrVersion(func() error { return mongoutil.Ignore(mongoutil.UpdateMany(ctx, f.coll, filter, update)) }, func() error { - return f.owner.IncrVersion(ctx, ownerUserID, friendUserIDs, false) + return f.owner.IncrVersion(ctx, ownerUserID, friendUserIDs, model.VersionStateUpdate) }) } diff --git a/pkg/common/storage/database/mgo/group_member.go b/pkg/common/storage/database/mgo/group_member.go index cb64c87a4..ece1d7941 100644 --- a/pkg/common/storage/database/mgo/group_member.go +++ b/pkg/common/storage/database/mgo/group_member.go @@ -70,7 +70,7 @@ func (g *GroupMemberMgo) Create(ctx context.Context, groupMembers []*model.Group gms[member.GroupID] = append(gms[member.GroupID], member.UserID) } for groupID, userIDs := range gms { - if err := g.member.IncrVersion(ctx, groupID, userIDs, false); err != nil { + if err := g.member.IncrVersion(ctx, groupID, userIDs, model.VersionStateInsert); err != nil { return err } } @@ -81,7 +81,7 @@ func (g *GroupMemberMgo) Create(ctx context.Context, groupMembers []*model.Group gms[member.UserID] = append(gms[member.UserID], member.GroupID) } for userID, groupIDs := range gms { - if err := g.join.IncrVersion(ctx, userID, groupIDs, false); err != nil { + if err := g.join.IncrVersion(ctx, userID, groupIDs, model.VersionStateInsert); err != nil { return err } } @@ -97,12 +97,12 @@ func (g *GroupMemberMgo) Delete(ctx context.Context, groupID string, userIDs []s return mongoutil.IncrVersion(func() error { return mongoutil.DeleteMany(ctx, g.coll, filter) }, func() error { - return g.member.IncrVersion(ctx, groupID, userIDs, true) + return g.member.IncrVersion(ctx, groupID, userIDs, model.VersionStateDelete) }, func() error { if len(userIDs) == 0 { return nil } - return g.member.IncrVersion(ctx, groupID, userIDs, true) + return g.member.IncrVersion(ctx, groupID, userIDs, model.VersionStateDelete) }) } @@ -110,9 +110,9 @@ func (g *GroupMemberMgo) UpdateRoleLevel(ctx context.Context, groupID string, us return mongoutil.IncrVersion(func() error { return g.Update(ctx, groupID, userID, bson.M{"role_level": roleLevel}) }, func() error { - return g.member.IncrVersion(ctx, groupID, []string{userID}, true) + return g.member.IncrVersion(ctx, groupID, []string{userID}, model.VersionStateUpdate) }, func() error { - return g.join.IncrVersion(ctx, groupID, []string{userID}, true) + return g.join.IncrVersion(ctx, groupID, []string{userID}, model.VersionStateUpdate) }) } @@ -123,7 +123,7 @@ func (g *GroupMemberMgo) Update(ctx context.Context, groupID string, userID stri return mongoutil.IncrVersion(func() error { return mongoutil.UpdateOne(ctx, g.coll, bson.M{"group_id": groupID, "user_id": userID}, bson.M{"$set": data}, true) }, func() error { - return g.member.IncrVersion(ctx, groupID, []string{userID}, false) + return g.member.IncrVersion(ctx, groupID, []string{userID}, model.VersionStateUpdate) }) } @@ -174,8 +174,8 @@ func (g *GroupMemberMgo) IsUpdateRoleLevel(data map[string]any) bool { return ok } -func (g *GroupMemberMgo) JoinGroupIncrVersion(ctx context.Context, userID string, groupIDs []string, deleted bool) error { - return g.join.IncrVersion(ctx, userID, groupIDs, deleted) +func (g *GroupMemberMgo) JoinGroupIncrVersion(ctx context.Context, userID string, groupIDs []string, state int32) error { + return g.join.IncrVersion(ctx, userID, groupIDs, state) } func (g *GroupMemberMgo) FindMemberIncrVersion(ctx context.Context, groupID string, version uint, limit int) (*model.VersionLog, error) { diff --git a/pkg/common/storage/database/mgo/version_log.go b/pkg/common/storage/database/mgo/version_log.go index 5629c5c00..8ab11007d 100644 --- a/pkg/common/storage/database/mgo/version_log.go +++ b/pkg/common/storage/database/mgo/version_log.go @@ -36,7 +36,7 @@ func (l *VersionLogMgo) initIndex(ctx context.Context) error { return err } -func (l *VersionLogMgo) IncrVersion(ctx context.Context, dId string, eIds []string, deleted bool) error { +func (l *VersionLogMgo) IncrVersion(ctx context.Context, dId string, eIds []string, state int32) error { if len(eIds) == 0 { return errs.ErrArgs.WrapMsg("elem id is empty", "dId", dId) } @@ -44,19 +44,19 @@ func (l *VersionLogMgo) IncrVersion(ctx context.Context, dId string, eIds []stri return errs.ErrArgs.WrapMsg("elem id is duplicate", "dId", dId, "eIds", eIds) } now := time.Now() - res, err := l.writeLogBatch(ctx, dId, eIds, deleted, now) + res, err := l.writeLogBatch(ctx, dId, eIds, state, now) if err != nil { return err } if res.MatchedCount > 0 { return nil } - if _, err := l.initDoc(ctx, dId, eIds, deleted, now); err == nil { + if _, err := l.initDoc(ctx, dId, eIds, state, now); err == nil { return nil } else if !mongo.IsDuplicateKeyError(err) { return err } - if res, err := l.writeLogBatch(ctx, dId, eIds, deleted, now); err != nil { + if res, err := l.writeLogBatch(ctx, dId, eIds, state, now); err != nil { return err } else if res.MatchedCount == 0 { return errs.ErrInternalServer.WrapMsg("mongodb return value that should not occur", "coll", l.coll.Name(), "dId", dId, "eIds", eIds) @@ -64,7 +64,7 @@ func (l *VersionLogMgo) IncrVersion(ctx context.Context, dId string, eIds []stri return nil } -func (l *VersionLogMgo) initDoc(ctx context.Context, dId string, eIds []string, deleted bool, now time.Time) (*model.VersionLogTable, error) { +func (l *VersionLogMgo) initDoc(ctx context.Context, dId string, eIds []string, state int32, now time.Time) (*model.VersionLogTable, error) { wl := model.VersionLogTable{ ID: primitive.NewObjectID(), DID: dId, @@ -76,7 +76,7 @@ func (l *VersionLogMgo) initDoc(ctx context.Context, dId string, eIds []string, for _, eId := range eIds { wl.Logs = append(wl.Logs, model.VersionLogElem{ EID: eId, - Deleted: deleted, + State: state, Version: database.FirstVersion, LastUpdate: now, }) @@ -85,7 +85,7 @@ func (l *VersionLogMgo) initDoc(ctx context.Context, dId string, eIds []string, return &wl, err } -func (l *VersionLogMgo) writeLogBatch(ctx context.Context, dId string, eIds []string, deleted bool, now time.Time) (*mongo.UpdateResult, error) { +func (l *VersionLogMgo) writeLogBatch(ctx context.Context, dId string, eIds []string, state int32, now time.Time) (*mongo.UpdateResult, error) { if eIds == nil { eIds = []string{} } @@ -97,7 +97,7 @@ func (l *VersionLogMgo) writeLogBatch(ctx context.Context, dId string, eIds []st elems = append(elems, bson.M{ "e_id": eId, "version": "$version", - "deleted": deleted, + "state": state, "last_update": now, }) } @@ -159,7 +159,7 @@ func (l *VersionLogMgo) FindChangeLog(ctx context.Context, dId string, version u } else if !errors.Is(err, mongo.ErrNoDocuments) { return nil, err } - if res, err := l.initDoc(ctx, dId, nil, false, time.Now()); err == nil { + if res, err := l.initDoc(ctx, dId, nil, 0, time.Now()); err == nil { return res.VersionLog(), nil } else if mongo.IsDuplicateKeyError(err) { return l.findChangeLog(ctx, dId, version, limit) diff --git a/pkg/common/storage/database/version_log.go b/pkg/common/storage/database/version_log.go index 3450d2776..c9dc09540 100644 --- a/pkg/common/storage/database/version_log.go +++ b/pkg/common/storage/database/version_log.go @@ -12,7 +12,7 @@ const ( ) type VersionLog interface { - IncrVersion(ctx context.Context, dId string, eIds []string, deleted bool) error + IncrVersion(ctx context.Context, dId string, eIds []string, state int32) error FindChangeLog(ctx context.Context, dId string, version uint, limit int) (*model.VersionLog, error) DeleteAfterUnchangedLog(ctx context.Context, deadline time.Time) error } diff --git a/pkg/common/storage/model/version_log.go b/pkg/common/storage/model/version_log.go index a09f493a8..e1b5fe7c5 100644 --- a/pkg/common/storage/model/version_log.go +++ b/pkg/common/storage/model/version_log.go @@ -1,13 +1,22 @@ package model import ( + "context" + "errors" + "github.com/openimsdk/tools/log" "go.mongodb.org/mongo-driver/bson/primitive" "time" ) +const ( + VersionStateInsert = iota + 1 + VersionStateDelete + VersionStateUpdate +) + type VersionLogElem struct { EID string `bson:"e_id"` - Deleted bool `bson:"deleted"` + State int32 `bson:"state"` Version uint `bson:"version"` LastUpdate time.Time `bson:"last_update"` } @@ -43,12 +52,17 @@ type VersionLog struct { LogLen int `bson:"log_len"` } -func (v *VersionLog) DeleteAndChangeIDs() (delIds []string, changeIds []string) { +func (v *VersionLog) DeleteAndChangeIDs() (insertIds, deleteIds, updateIds []string) { for _, l := range v.Logs { - if l.Deleted { - delIds = append(delIds, l.EID) - } else { - changeIds = append(changeIds, l.EID) + switch l.State { + case VersionStateInsert: + insertIds = append(insertIds, l.EID) + case VersionStateDelete: + deleteIds = append(deleteIds, l.EID) + case VersionStateUpdate: + updateIds = append(updateIds, l.EID) + default: + log.ZError(context.Background(), "invalid version status found", errors.New("dirty database data"), "objID", v.ID.Hex(), "did", v.DID, "elem", l) } } return From caebdf32caf3e9eb051cf49e8cf19cde3f1aecfe Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Tue, 11 Jun 2024 18:38:36 +0800 Subject: [PATCH 29/59] sync --- internal/rpc/group/notification.go | 22 +++++++ pkg/common/cmd/group.go | 3 +- pkg/common/storage/database/mgo/black.go | 2 +- .../storage/database/mgo/conversation.go | 3 +- pkg/common/storage/database/mgo/friend.go | 4 +- .../storage/database/mgo/friend_request.go | 2 +- pkg/common/storage/database/mgo/group.go | 2 +- .../storage/database/mgo/group_member.go | 6 +- .../storage/database/mgo/group_request.go | 2 +- pkg/common/storage/database/mgo/log.go | 2 +- pkg/common/storage/database/mgo/object.go | 2 +- pkg/common/storage/database/mgo/user.go | 2 +- .../storage/database/mgo/version_log.go | 62 +++++++++++-------- .../storage/database/mgo/version_test.go | 39 ++++++++++++ pkg/common/storage/database/name.go | 17 +++++ pkg/common/storage/model/version_log.go | 2 +- pkg/common/storage/versionctx/rpc.go | 14 +++++ pkg/common/storage/versionctx/version.go | 48 ++++++++++++++ 18 files changed, 193 insertions(+), 41 deletions(-) create mode 100644 pkg/common/storage/database/mgo/version_test.go create mode 100644 pkg/common/storage/database/name.go create mode 100644 pkg/common/storage/versionctx/rpc.go create mode 100644 pkg/common/storage/versionctx/version.go diff --git a/internal/rpc/group/notification.go b/internal/rpc/group/notification.go index cfa62c85d..13e593c71 100644 --- a/internal/rpc/group/notification.go +++ b/internal/rpc/group/notification.go @@ -17,7 +17,9 @@ package group import ( "context" "fmt" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/versionctx" "github.com/openimsdk/open-im-server/v3/pkg/rpcclient/notification" "github.com/openimsdk/open-im-server/v3/pkg/authverify" @@ -287,6 +289,15 @@ func (g *GroupNotificationSender) fillOpUser(ctx context.Context, opUser **sdkws return nil } +func (g *GroupNotificationSender) setVersion(ctx context.Context, version *uint64, collName string, id string) { + for _, coll := range versionctx.GetVersionLog(ctx).Get() { + if coll.Name == collName && coll.Doc.DID == id { + *version = uint64(coll.Doc.Version) + return + } + } +} + func (g *GroupNotificationSender) GroupCreatedNotification(ctx context.Context, tips *sdkws.GroupCreatedTips) { var err error defer func() { @@ -297,6 +308,7 @@ func (g *GroupNotificationSender) GroupCreatedNotification(ctx context.Context, if err = g.fillOpUser(ctx, &tips.OpUser, tips.Group.GroupID); err != nil { return } + g.setVersion(ctx, &tips.GroupMemberVersion, database.GroupMemberVersionName, tips.Group.GroupID) g.Notification(ctx, mcontext.GetOpUserID(ctx), tips.Group.GroupID, constant.GroupCreatedNotification, tips) } @@ -380,6 +392,7 @@ func (g *GroupNotificationSender) MemberQuitNotification(ctx context.Context, me return } tips := &sdkws.MemberQuitTips{Group: group, QuitUser: member} + g.setVersion(ctx, &tips.GroupMemberVersion, database.GroupMemberVersionName, member.GroupID) g.Notification(ctx, mcontext.GetOpUserID(ctx), member.GroupID, constant.MemberQuitNotification, tips) } @@ -467,6 +480,7 @@ func (g *GroupNotificationSender) GroupOwnerTransferredNotification(ctx context. if err = g.fillOpUser(ctx, &tips.OpUser, tips.Group.GroupID); err != nil { return } + g.setVersion(ctx, &tips.GroupMemberVersion, database.GroupMemberVersionName, req.GroupID) g.Notification(ctx, mcontext.GetOpUserID(ctx), group.GroupID, constant.GroupOwnerTransferredNotification, tips) } @@ -480,6 +494,7 @@ func (g *GroupNotificationSender) MemberKickedNotification(ctx context.Context, if err = g.fillOpUser(ctx, &tips.OpUser, tips.Group.GroupID); err != nil { return } + g.setVersion(ctx, &tips.GroupMemberVersion, database.GroupMemberVersionName, tips.Group.GroupID) g.Notification(ctx, mcontext.GetOpUserID(ctx), tips.Group.GroupID, constant.MemberKickedNotification, tips) } @@ -503,6 +518,7 @@ func (g *GroupNotificationSender) MemberInvitedNotification(ctx context.Context, } tips := &sdkws.MemberInvitedTips{Group: group, InvitedUserList: users} err = g.fillOpUser(ctx, &tips.OpUser, tips.Group.GroupID) + g.setVersion(ctx, &tips.GroupMemberVersion, database.GroupMemberVersionName, tips.Group.GroupID) g.Notification(ctx, mcontext.GetOpUserID(ctx), group.GroupID, constant.MemberInvitedNotification, tips) } @@ -524,6 +540,7 @@ func (g *GroupNotificationSender) MemberEnterNotification(ctx context.Context, g return } tips := &sdkws.MemberEnterTips{Group: group, EntrantUser: user} + g.setVersion(ctx, &tips.GroupMemberVersion, database.GroupMemberVersionName, tips.Group.GroupID) g.Notification(ctx, mcontext.GetOpUserID(ctx), group.GroupID, constant.MemberEnterNotification, tips) } @@ -564,6 +581,7 @@ func (g *GroupNotificationSender) GroupMemberMutedNotification(ctx context.Conte if err = g.fillOpUser(ctx, &tips.OpUser, tips.Group.GroupID); err != nil { return } + g.setVersion(ctx, &tips.GroupMemberVersion, database.GroupMemberVersionName, tips.Group.GroupID) g.Notification(ctx, mcontext.GetOpUserID(ctx), group.GroupID, constant.GroupMemberMutedNotification, tips) } @@ -588,6 +606,7 @@ func (g *GroupNotificationSender) GroupMemberCancelMutedNotification(ctx context if err = g.fillOpUser(ctx, &tips.OpUser, tips.Group.GroupID); err != nil { return } + g.setVersion(ctx, &tips.GroupMemberVersion, database.GroupMemberVersionName, tips.Group.GroupID) g.Notification(ctx, mcontext.GetOpUserID(ctx), group.GroupID, constant.GroupMemberCancelMutedNotification, tips) } @@ -666,6 +685,7 @@ func (g *GroupNotificationSender) GroupMemberInfoSetNotification(ctx context.Con if err = g.fillOpUser(ctx, &tips.OpUser, tips.Group.GroupID); err != nil { return } + g.setVersion(ctx, &tips.GroupMemberVersion, database.GroupMemberVersionName, tips.Group.GroupID) g.Notification(ctx, mcontext.GetOpUserID(ctx), group.GroupID, constant.GroupMemberInfoSetNotification, tips) } @@ -689,6 +709,7 @@ func (g *GroupNotificationSender) GroupMemberSetToAdminNotification(ctx context. if err = g.fillOpUser(ctx, &tips.OpUser, tips.Group.GroupID); err != nil { return } + g.setVersion(ctx, &tips.GroupMemberVersion, database.GroupMemberVersionName, tips.Group.GroupID) g.Notification(ctx, mcontext.GetOpUserID(ctx), group.GroupID, constant.GroupMemberSetToAdminNotification, tips) } @@ -713,5 +734,6 @@ func (g *GroupNotificationSender) GroupMemberSetToOrdinaryUserNotification(ctx c if err = g.fillOpUser(ctx, &tips.OpUser, tips.Group.GroupID); err != nil { return } + g.setVersion(ctx, &tips.GroupMemberVersion, database.GroupMemberVersionName, tips.Group.GroupID) g.Notification(ctx, mcontext.GetOpUserID(ctx), group.GroupID, constant.GroupMemberSetToOrdinaryUserNotification, tips) } diff --git a/pkg/common/cmd/group.go b/pkg/common/cmd/group.go index f158b8c62..20124be95 100644 --- a/pkg/common/cmd/group.go +++ b/pkg/common/cmd/group.go @@ -19,6 +19,7 @@ import ( "github.com/openimsdk/open-im-server/v3/internal/rpc/group" "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/open-im-server/v3/pkg/common/startrpc" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/versionctx" "github.com/openimsdk/tools/system/program" "github.com/spf13/cobra" ) @@ -58,5 +59,5 @@ func (a *GroupRpcCmd) Exec() error { func (a *GroupRpcCmd) runE() error { return startrpc.Start(a.ctx, &a.groupConfig.Discovery, &a.groupConfig.RpcConfig.Prometheus, a.groupConfig.RpcConfig.RPC.ListenIP, a.groupConfig.RpcConfig.RPC.RegisterIP, a.groupConfig.RpcConfig.RPC.Ports, - a.Index(), a.groupConfig.Share.RpcRegisterName.Group, &a.groupConfig.Share, a.groupConfig, group.Start) + a.Index(), a.groupConfig.Share.RpcRegisterName.Group, &a.groupConfig.Share, a.groupConfig, group.Start, versionctx.EnableVersionCtx()) } diff --git a/pkg/common/storage/database/mgo/black.go b/pkg/common/storage/database/mgo/black.go index cf74cfab1..4a7a35e6f 100644 --- a/pkg/common/storage/database/mgo/black.go +++ b/pkg/common/storage/database/mgo/black.go @@ -27,7 +27,7 @@ import ( ) func NewBlackMongo(db *mongo.Database) (database.Black, error) { - coll := db.Collection("black") + coll := db.Collection(database.BlackName) _, err := coll.Indexes().CreateOne(context.Background(), mongo.IndexModel{ Keys: bson.D{ {Key: "owner_user_id", Value: 1}, diff --git a/pkg/common/storage/database/mgo/conversation.go b/pkg/common/storage/database/mgo/conversation.go index 9c35f841b..b462d3958 100644 --- a/pkg/common/storage/database/mgo/conversation.go +++ b/pkg/common/storage/database/mgo/conversation.go @@ -16,6 +16,7 @@ package mgo import ( "context" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" "time" @@ -29,7 +30,7 @@ import ( ) func NewConversationMongo(db *mongo.Database) (*ConversationMgo, error) { - coll := db.Collection("conversation") + coll := db.Collection(database.ConversationName) _, err := coll.Indexes().CreateOne(context.Background(), mongo.IndexModel{ Keys: bson.D{ {Key: "owner_user_id", Value: 1}, diff --git a/pkg/common/storage/database/mgo/friend.go b/pkg/common/storage/database/mgo/friend.go index 699d9cff6..18d80d47d 100644 --- a/pkg/common/storage/database/mgo/friend.go +++ b/pkg/common/storage/database/mgo/friend.go @@ -34,7 +34,7 @@ type FriendMgo struct { // NewFriendMongo creates a new instance of FriendMgo with the provided MongoDB database. func NewFriendMongo(db *mongo.Database) (database.Friend, error) { - coll := db.Collection("friend") + coll := db.Collection(database.FriendName) _, err := coll.Indexes().CreateOne(context.Background(), mongo.IndexModel{ Keys: bson.D{ {Key: "owner_user_id", Value: 1}, @@ -45,7 +45,7 @@ func NewFriendMongo(db *mongo.Database) (database.Friend, error) { if err != nil { return nil, err } - owner, err := NewVersionLog(db.Collection("friend_version")) + owner, err := NewVersionLog(db.Collection(database.FriendVersionName)) if err != nil { return nil, err } diff --git a/pkg/common/storage/database/mgo/friend_request.go b/pkg/common/storage/database/mgo/friend_request.go index 0d60b213d..4eed2f4a2 100644 --- a/pkg/common/storage/database/mgo/friend_request.go +++ b/pkg/common/storage/database/mgo/friend_request.go @@ -27,7 +27,7 @@ import ( ) func NewFriendRequestMongo(db *mongo.Database) (database.FriendRequest, error) { - coll := db.Collection("friend_request") + coll := db.Collection(database.FriendRequestName) _, err := coll.Indexes().CreateOne(context.Background(), mongo.IndexModel{ Keys: bson.D{ {Key: "from_user_id", Value: 1}, diff --git a/pkg/common/storage/database/mgo/group.go b/pkg/common/storage/database/mgo/group.go index 630bc0291..3be7883af 100644 --- a/pkg/common/storage/database/mgo/group.go +++ b/pkg/common/storage/database/mgo/group.go @@ -30,7 +30,7 @@ import ( ) func NewGroupMongo(db *mongo.Database) (database.Group, error) { - coll := db.Collection("group") + coll := db.Collection(database.GroupName) _, err := coll.Indexes().CreateOne(context.Background(), mongo.IndexModel{ Keys: bson.D{ {Key: "group_id", Value: 1}, diff --git a/pkg/common/storage/database/mgo/group_member.go b/pkg/common/storage/database/mgo/group_member.go index ece1d7941..a9ac262ab 100644 --- a/pkg/common/storage/database/mgo/group_member.go +++ b/pkg/common/storage/database/mgo/group_member.go @@ -29,7 +29,7 @@ import ( ) func NewGroupMember(db *mongo.Database) (database.GroupMember, error) { - coll := db.Collection("group_member") + coll := db.Collection(database.GroupMemberName) _, err := coll.Indexes().CreateOne(context.Background(), mongo.IndexModel{ Keys: bson.D{ {Key: "group_id", Value: 1}, @@ -40,11 +40,11 @@ func NewGroupMember(db *mongo.Database) (database.GroupMember, error) { if err != nil { return nil, errs.Wrap(err) } - member, err := NewVersionLog(db.Collection("group_member_version")) + member, err := NewVersionLog(db.Collection(database.GroupMemberVersionName)) if err != nil { return nil, err } - join, err := NewVersionLog(db.Collection("group_join_version")) + join, err := NewVersionLog(db.Collection(database.GroupJoinVersionName)) if err != nil { return nil, err } diff --git a/pkg/common/storage/database/mgo/group_request.go b/pkg/common/storage/database/mgo/group_request.go index 4ae778527..b1942b708 100644 --- a/pkg/common/storage/database/mgo/group_request.go +++ b/pkg/common/storage/database/mgo/group_request.go @@ -28,7 +28,7 @@ import ( ) func NewGroupRequestMgo(db *mongo.Database) (database.GroupRequest, error) { - coll := db.Collection("group_request") + coll := db.Collection(database.GroupRequestName) _, err := coll.Indexes().CreateOne(context.Background(), mongo.IndexModel{ Keys: bson.D{ {Key: "group_id", Value: 1}, diff --git a/pkg/common/storage/database/mgo/log.go b/pkg/common/storage/database/mgo/log.go index 51715bd77..6ff4c6039 100644 --- a/pkg/common/storage/database/mgo/log.go +++ b/pkg/common/storage/database/mgo/log.go @@ -28,7 +28,7 @@ import ( ) func NewLogMongo(db *mongo.Database) (database.Log, error) { - coll := db.Collection("log") + coll := db.Collection(database.LogName) _, err := coll.Indexes().CreateMany(context.Background(), []mongo.IndexModel{ { Keys: bson.D{ diff --git a/pkg/common/storage/database/mgo/object.go b/pkg/common/storage/database/mgo/object.go index 8ed7b3a56..df4d10ec4 100644 --- a/pkg/common/storage/database/mgo/object.go +++ b/pkg/common/storage/database/mgo/object.go @@ -27,7 +27,7 @@ import ( ) func NewS3Mongo(db *mongo.Database) (database.ObjectInfo, error) { - coll := db.Collection("s3") + coll := db.Collection(database.ObjectName) _, err := coll.Indexes().CreateOne(context.Background(), mongo.IndexModel{ Keys: bson.D{ {Key: "name", Value: 1}, diff --git a/pkg/common/storage/database/mgo/user.go b/pkg/common/storage/database/mgo/user.go index 793b8cdc8..8978e64eb 100644 --- a/pkg/common/storage/database/mgo/user.go +++ b/pkg/common/storage/database/mgo/user.go @@ -31,7 +31,7 @@ import ( ) func NewUserMongo(db *mongo.Database) (database.User, error) { - coll := db.Collection("user") + coll := db.Collection(database.UserName) _, err := coll.Indexes().CreateOne(context.Background(), mongo.IndexModel{ Keys: bson.D{ {Key: "user_id", Value: 1}, diff --git a/pkg/common/storage/database/mgo/version_log.go b/pkg/common/storage/database/mgo/version_log.go index 8ab11007d..b53057fe4 100644 --- a/pkg/common/storage/database/mgo/version_log.go +++ b/pkg/common/storage/database/mgo/version_log.go @@ -5,9 +5,9 @@ import ( "errors" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/versionctx" "github.com/openimsdk/tools/db/mongoutil" "github.com/openimsdk/tools/errs" - "github.com/openimsdk/tools/utils/datautil" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/bson/primitive" "go.mongodb.org/mongo-driver/mongo" @@ -37,34 +37,41 @@ func (l *VersionLogMgo) initIndex(ctx context.Context) error { } func (l *VersionLogMgo) IncrVersion(ctx context.Context, dId string, eIds []string, state int32) error { - if len(eIds) == 0 { - return errs.ErrArgs.WrapMsg("elem id is empty", "dId", dId) + _, err := l.IncrVersionResult(ctx, dId, eIds, state) + return err +} + +func (l *VersionLogMgo) IncrVersionResult(ctx context.Context, dId string, eIds []string, state int32) (*model.VersionLog, error) { + vl, err := l.incrVersionResult(ctx, dId, eIds, state) + if err != nil { + return nil, err } - if datautil.Duplicate(eIds) { - return errs.ErrArgs.WrapMsg("elem id is duplicate", "dId", dId, "eIds", eIds) + versionctx.GetVersionLog(ctx).Append(versionctx.Collection{ + Name: l.coll.Name(), + Doc: vl, + }) + return vl, nil +} + +func (l *VersionLogMgo) incrVersionResult(ctx context.Context, dId string, eIds []string, state int32) (*model.VersionLog, error) { + if len(eIds) == 0 { + return nil, errs.ErrArgs.WrapMsg("elem id is empty", "dId", dId) } now := time.Now() - res, err := l.writeLogBatch(ctx, dId, eIds, state, now) - if err != nil { - return err - } - if res.MatchedCount > 0 { - return nil + if res, err := l.writeLogBatch2(ctx, dId, eIds, state, now); err == nil { + return res, nil + } else if !errors.Is(err, mongo.ErrNoDocuments) { + return nil, err } - if _, err := l.initDoc(ctx, dId, eIds, state, now); err == nil { - return nil + if res, err := l.initDoc(ctx, dId, eIds, state, now); err == nil { + return res, nil } else if !mongo.IsDuplicateKeyError(err) { - return err - } - if res, err := l.writeLogBatch(ctx, dId, eIds, state, now); err != nil { - return err - } else if res.MatchedCount == 0 { - return errs.ErrInternalServer.WrapMsg("mongodb return value that should not occur", "coll", l.coll.Name(), "dId", dId, "eIds", eIds) + return nil, err } - return nil + return l.writeLogBatch2(ctx, dId, eIds, state, now) } -func (l *VersionLogMgo) initDoc(ctx context.Context, dId string, eIds []string, state int32, now time.Time) (*model.VersionLogTable, error) { +func (l *VersionLogMgo) initDoc(ctx context.Context, dId string, eIds []string, state int32, now time.Time) (*model.VersionLog, error) { wl := model.VersionLogTable{ ID: primitive.NewObjectID(), DID: dId, @@ -81,11 +88,13 @@ func (l *VersionLogMgo) initDoc(ctx context.Context, dId string, eIds []string, LastUpdate: now, }) } - _, err := l.coll.InsertOne(ctx, &wl) - return &wl, err + if _, err := l.coll.InsertOne(ctx, &wl); err != nil { + return nil, err + } + return wl.VersionLog(), nil } -func (l *VersionLogMgo) writeLogBatch(ctx context.Context, dId string, eIds []string, state int32, now time.Time) (*mongo.UpdateResult, error) { +func (l *VersionLogMgo) writeLogBatch2(ctx context.Context, dId string, eIds []string, state int32, now time.Time) (*model.VersionLog, error) { if eIds == nil { eIds = []string{} } @@ -142,7 +151,8 @@ func (l *VersionLogMgo) writeLogBatch(ctx context.Context, dId string, eIds []st "$unset": "delete_e_ids", }, } - return mongoutil.UpdateMany(ctx, l.coll, filter, pipeline) + opt := options.FindOneAndUpdate().SetUpsert(false).SetReturnDocument(options.After).SetProjection(bson.M{"logs": 0}) + return mongoutil.FindOneAndUpdate[*model.VersionLog](ctx, l.coll, filter, pipeline, opt) } func (l *VersionLogMgo) findDoc(ctx context.Context, dId string) (*model.VersionLog, error) { @@ -160,7 +170,7 @@ func (l *VersionLogMgo) FindChangeLog(ctx context.Context, dId string, version u return nil, err } if res, err := l.initDoc(ctx, dId, nil, 0, time.Now()); err == nil { - return res.VersionLog(), nil + return res, nil } else if mongo.IsDuplicateKeyError(err) { return l.findChangeLog(ctx, dId, version, limit) } else { diff --git a/pkg/common/storage/database/mgo/version_test.go b/pkg/common/storage/database/mgo/version_test.go new file mode 100644 index 000000000..236c61a2c --- /dev/null +++ b/pkg/common/storage/database/mgo/version_test.go @@ -0,0 +1,39 @@ +package mgo + +import ( + "context" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" + "testing" + "time" +) + +func Result[V any](val V, err error) V { + if err != nil { + panic(err) + } + return val +} + +func Check(err error) { + if err != nil { + panic(err) + } +} + +func TestName(t *testing.T) { + cli := Result(mongo.Connect(context.Background(), options.Client().ApplyURI("mongodb://openIM:openIM123@172.16.8.48:37017/openim_v3?maxPoolSize=100").SetConnectTimeout(5*time.Second))) + coll := cli.Database("openim_v3").Collection("version_test") + tmp, err := NewVersionLog(coll) + if err != nil { + panic(err) + } + vl := tmp.(*VersionLogMgo) + res, err := vl.writeLogBatch2(context.Background(), "100", []string{"1000", "1001", "1003"}, model.VersionStateInsert, time.Now()) + if err != nil { + t.Log(err) + return + } + t.Logf("%+v", res) +} diff --git a/pkg/common/storage/database/name.go b/pkg/common/storage/database/name.go new file mode 100644 index 000000000..986f22a1a --- /dev/null +++ b/pkg/common/storage/database/name.go @@ -0,0 +1,17 @@ +package database + +const ( + BlackName = "black" + ConversationName = "conversation" + FriendName = "friend" + FriendVersionName = "friend_version" + FriendRequestName = "friend_request" + GroupName = "group" + GroupMemberName = "group_member" + GroupMemberVersionName = "group_member_version" + GroupJoinVersionName = "group_join_version" + GroupRequestName = "group_request" + LogName = "log" + ObjectName = "s3" + UserName = "user" +) diff --git a/pkg/common/storage/model/version_log.go b/pkg/common/storage/model/version_log.go index e1b5fe7c5..11a40ef24 100644 --- a/pkg/common/storage/model/version_log.go +++ b/pkg/common/storage/model/version_log.go @@ -38,7 +38,7 @@ func (v *VersionLogTable) VersionLog() *VersionLog { Version: v.Version, Deleted: v.Deleted, LastUpdate: v.LastUpdate, - LogLen: 0, + LogLen: len(v.Logs), } } diff --git a/pkg/common/storage/versionctx/rpc.go b/pkg/common/storage/versionctx/rpc.go new file mode 100644 index 000000000..67b95aebd --- /dev/null +++ b/pkg/common/storage/versionctx/rpc.go @@ -0,0 +1,14 @@ +package versionctx + +import ( + "context" + "google.golang.org/grpc" +) + +func EnableVersionCtx() grpc.ServerOption { + return grpc.ChainUnaryInterceptor(enableVersionCtxInterceptor) +} + +func enableVersionCtxInterceptor(ctx context.Context, req any, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (any, error) { + return handler(WithVersionLog(ctx), req) +} diff --git a/pkg/common/storage/versionctx/version.go b/pkg/common/storage/versionctx/version.go new file mode 100644 index 000000000..5db885640 --- /dev/null +++ b/pkg/common/storage/versionctx/version.go @@ -0,0 +1,48 @@ +package versionctx + +import ( + "context" + tablerelation "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" + "sync" +) + +type Collection struct { + Name string + Doc *tablerelation.VersionLog +} + +type versionKey struct{} + +func WithVersionLog(ctx context.Context) context.Context { + return context.WithValue(ctx, versionKey{}, &VersionLog{}) +} + +func GetVersionLog(ctx context.Context) *VersionLog { + if v, ok := ctx.Value(versionKey{}).(*VersionLog); ok { + return v + } + return nil +} + +type VersionLog struct { + lock sync.Mutex + data []Collection +} + +func (v *VersionLog) Append(data ...Collection) { + if v == nil || len(data) == 0 { + return + } + v.lock.Lock() + defer v.lock.Unlock() + v.data = append(v.data, data...) +} + +func (v *VersionLog) Get() []Collection { + if v == nil { + return nil + } + v.lock.Lock() + defer v.lock.Unlock() + return v.data +} From 1b1027192dc7e6c3ff6d1b27373e743dc0689ae6 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Thu, 13 Jun 2024 09:45:14 +0800 Subject: [PATCH 30/59] go.mod --- go.mod | 6 +++--- go.sum | 10 ++++------ 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index 3ef42ef80..4f4cd4b68 100644 --- a/go.mod +++ b/go.mod @@ -13,8 +13,8 @@ require ( github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/lestrrat-go/file-rotatelogs v2.4.0+incompatible // indirect github.com/mitchellh/mapstructure v1.5.0 - github.com/openimsdk/protocol v0.0.69-alpha.4 - github.com/openimsdk/tools v0.0.49-alpha.24 + github.com/openimsdk/protocol v0.0.69-alpha.9 + github.com/openimsdk/tools v0.0.49-alpha.25 github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 github.com/stretchr/testify v1.9.0 @@ -177,4 +177,4 @@ require ( gopkg.in/ini.v1 v1.67.0 // indirect ) -replace github.com/openimsdk/protocol => /Users/chao/Desktop/project/protocol +//replace github.com/openimsdk/protocol => /Users/chao/Desktop/project/protocol diff --git a/go.sum b/go.sum index 835bce361..5a72e28f2 100644 --- a/go.sum +++ b/go.sum @@ -270,12 +270,10 @@ github.com/onsi/gomega v1.25.0 h1:Vw7br2PCDYijJHSfBOWhov+8cAnUf8MfMaIOV323l6Y= github.com/onsi/gomega v1.25.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdMPGhM= github.com/openimsdk/gomake v0.0.13 h1:xLDe/moqgWpRoptHzI4packAWzs4C16b+sVY+txNJp0= github.com/openimsdk/gomake v0.0.13/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= -github.com/openimsdk/protocol v0.0.69-alpha.3 h1:Uf167FVB5EqYpiy2zBbR63OiK+Njjy99fXYasK6Zi+4= -github.com/openimsdk/protocol v0.0.69-alpha.3/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= -github.com/openimsdk/protocol v0.0.69-alpha.4 h1:QJkOFV5Hlu7CbkHG5smeVw+5fx5DVkpNJWqlAOJxuIY= -github.com/openimsdk/protocol v0.0.69-alpha.4/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= -github.com/openimsdk/tools v0.0.49-alpha.24 h1:lJsqnjTPujnr91LRQ6QmcTliMIa4fMOBSTri6rFz2ek= -github.com/openimsdk/tools v0.0.49-alpha.24/go.mod h1:g7mkHXYUPi0/8aAX8VPMHpnb3hqdV69Jph+bXOGvvNM= +github.com/openimsdk/protocol v0.0.69-alpha.9 h1:Mh1upsxwhWs1y65cfIN2XuRsiKEk4MfpYfA9XrvtX24= +github.com/openimsdk/protocol v0.0.69-alpha.9/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= +github.com/openimsdk/tools v0.0.49-alpha.25 h1:OpRPwDZ2xWX7Zj5kyfZhryu/NfZTrsRVr2GFwu1HQHI= +github.com/openimsdk/tools v0.0.49-alpha.25/go.mod h1:rwsFI1G/nBHNfiNapbven41akRDPBbH4df0Cgy6xueU= github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= github.com/pelletier/go-toml/v2 v2.1.0/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= From ef71d0cf6247b386964dfdd2a6a8b4206cb759af Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Fri, 14 Jun 2024 10:54:24 +0800 Subject: [PATCH 31/59] seq --- pkg/common/storage/cache/cachekey/seq1.go | 19 ++ pkg/common/storage/cache/redis/seq1.go.txt | 246 +++++++++++++++++++++ pkg/common/storage/controller/msg.go | 42 ++-- pkg/common/storage/database/mgo/seq.go | 75 +++++++ pkg/common/storage/database/name.go | 1 + pkg/common/storage/database/seq.go | 10 + pkg/common/storage/model/seq.go | 7 + pkg/msgprocessor/conversation.go | 4 + 8 files changed, 384 insertions(+), 20 deletions(-) create mode 100644 pkg/common/storage/cache/cachekey/seq1.go create mode 100644 pkg/common/storage/cache/redis/seq1.go.txt create mode 100644 pkg/common/storage/database/mgo/seq.go create mode 100644 pkg/common/storage/database/seq.go create mode 100644 pkg/common/storage/model/seq.go diff --git a/pkg/common/storage/cache/cachekey/seq1.go b/pkg/common/storage/cache/cachekey/seq1.go new file mode 100644 index 000000000..df3086d32 --- /dev/null +++ b/pkg/common/storage/cache/cachekey/seq1.go @@ -0,0 +1,19 @@ +package cachekey + +const ( + MallocSeq = "MALLOC_SEQ:" + MallocSeqLock = "MALLOC_SEQ_LOCK:" + MallocMinSeqLock = "MALLOC_MIN_SEQ:" +) + +func GetMallocSeqKey(conversationID string) string { + return MallocSeq + conversationID +} + +func GetMallocSeqLockKey(conversationID string) string { + return MallocSeqLock + conversationID +} + +func GetMallocMinSeqKey(conversationID string) string { + return MallocMinSeqLock + conversationID +} diff --git a/pkg/common/storage/cache/redis/seq1.go.txt b/pkg/common/storage/cache/redis/seq1.go.txt new file mode 100644 index 000000000..9ccb83e66 --- /dev/null +++ b/pkg/common/storage/cache/redis/seq1.go.txt @@ -0,0 +1,246 @@ +package redis + +import ( + "context" + "errors" + "fmt" + "github.com/dtm-labs/rockscache" + "github.com/google/uuid" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/cachekey" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" + "github.com/openimsdk/open-im-server/v3/pkg/msgprocessor" + "github.com/openimsdk/tools/errs" + "github.com/redis/go-redis/v9" + "strconv" + "time" +) + +var errLock = errors.New("lock failed") + +type MallocSeq interface { + Malloc(ctx context.Context, conversationID string, size int64) ([]int64, error) + GetMaxSeq(ctx context.Context, conversationID string) (int64, error) + GetMinSeq(ctx context.Context, conversationID string) (int64, error) + SetMinSeq(ctx context.Context, conversationID string, seq int64) error +} + +func NewSeqCache1(rdb redis.UniversalClient, mgo database.Seq) MallocSeq { + opt := rockscache.NewDefaultOptions() + opt.EmptyExpire = time.Second * 3 + opt.Delay = time.Second / 2 + return &seqCache1{ + rdb: rdb, + mgo: mgo, + rocks: rockscache.NewClient(rdb, opt), + lockExpire: time.Minute * 1, + seqExpire: time.Hour * 24 * 7, + minSeqExpire: time.Hour * 1, + groupMinNum: 1000, + userMinNum: 100, + } +} + +type seqCache1 struct { + rdb redis.UniversalClient + rocks *rockscache.Client + mgo database.Seq + lockExpire time.Duration + seqExpire time.Duration + minSeqExpire time.Duration + groupMinNum int64 + userMinNum int64 +} + +/* +1 +2 +3 +4 +5 +6 +7 +8 +9 +10 +*/ + +func (s *seqCache1) GetMaxSeq(ctx context.Context, conversationID string) (int64, error) { + for i := 0; i < 10; i++ { + res, err := s.rdb.LIndex(ctx, cachekey.GetMallocSeqKey(conversationID), 0).Int64() + if err == nil { + return res, nil + } else if !errors.Is(err, redis.Nil) { + return 0, errs.Wrap(err) + } + + if err := s.mallocSeq(ctx, conversationID, 0, nil); err != nil { + return 0, err + } + } + return 0, errs.New("get max seq timeout") +} + +func (s *seqCache1) unlock(ctx context.Context, key string, owner string) error { + script := ` +local value = redis.call("GET", KEYS[1]) +if value == false then + return 0 +end +if value == ARGV[1] then + redis.call("DEL", KEYS[1]) + return 1 +end +return 2 +` + state, err := s.rdb.Eval(ctx, script, []string{key}, owner).Int() + if err != nil { + return errs.Wrap(err) + } + switch state { + case 0: + return errs.Wrap(redis.Nil) + case 1: + return nil + case 2: + return errs.New("not the lock holder") + default: + return errs.New(fmt.Sprintf("unknown state: %d", state)) + } +} + +func (s *seqCache1) initMallocSeq(ctx context.Context, conversationID string, size int64) ([]int64, error) { + owner := uuid.New().String() + ok, err := s.rdb.SetNX(ctx, cachekey.GetMallocSeqLockKey(conversationID), owner, s.lockExpire).Result() + if err != nil { + return nil, err + } + + seq, err := s.mgo.Malloc(ctx, conversationID, size) + if err != nil { + return nil, err + } + seqs := make([]int64, 0, size) + for i := seq - size + 1; i <= seq; i++ { + seqs = append(seqs, i) + } + return seqs, nil +} + +func (s *seqCache1) GetMinSeq(ctx context.Context, conversationID string) (int64, error) { + return getCache[int64](ctx, s.rocks, cachekey.GetMallocMinSeqKey(conversationID), s.minSeqExpire, func(ctx context.Context) (int64, error) { + return s.mgo.GetMinSeq(ctx, conversationID) + }) +} + +func (s *seqCache1) SetMinSeq(ctx context.Context, conversationID string, seq int64) error { + if err := s.mgo.SetMinSeq(ctx, conversationID, seq); err != nil { + return err + } + return s.deleteMinSeqCache(ctx, conversationID) +} + +func (s *seqCache1) Malloc(ctx context.Context, conversationID string, size int64) ([]int64, error) { + if size <= 0 { + return nil, errs.Wrap(errors.New("size must be greater than 0")) + } + seqKey := cachekey.GetMallocSeqKey(conversationID) + lockKey := cachekey.GetMallocSeqLockKey(conversationID) + for i := 0; i < 10; i++ { + seqs, err := s.lpop(ctx, seqKey, lockKey, size) + if err != nil { + return nil, err + } + if len(seqs) < int(size) { + if err := s.mallocSeq(ctx, conversationID, size, &seqs); err != nil { + return nil, err + } + } + if len(seqs) >= int(size) { + return seqs, nil + } + } + return nil, errs.ErrInternalServer.WrapMsg("malloc seq failed") +} + +func (s *seqCache1) push(ctx context.Context, seqKey string, seqs []int64) error { + script := ` +redis.call("DEL", KEYS[1]) +for i = 2, #ARGV do + redis.call("RPUSH", KEYS[1], ARGV[i]) +end +redis.call("EXPIRE", KEYS[1], ARGV[1]) +return 1 +` + argv := make([]any, 0, 1+len(seqs)) + argv = append(argv, s.seqExpire.Seconds()) + for _, seq := range seqs { + argv = append(argv, seq) + } + err := s.rdb.Eval(ctx, script, []string{seqKey}, argv...).Err() + return errs.Wrap(err) +} + +func (s *seqCache1) lpop(ctx context.Context, seqKey, lockKey string, size int64) ([]int64, error) { + script := ` +local result = redis.call("LRANGE", KEYS[1], 0, ARGV[1]-1) +if #result == 0 then + return result +end +redis.call("LTRIM", KEYS[1], #result, -1) +if redis.call("LLEN", KEYS[1]) == 0 then + redis.call("DEL", KEYS[2]) +end +return result +` + res, err := s.rdb.Eval(ctx, script, []string{seqKey, lockKey}, size).Int64Slice() + if err != nil { + return nil, errs.Wrap(err) + } + return res, nil +} + +func (s *seqCache1) getMongoStepSize(conversationID string, size int64) int64 { + var num int64 + if msgprocessor.IsGroupConversationID(conversationID) { + num = s.groupMinNum + } else { + num = s.userMinNum + } + if size > num { + num += size + } + return num +} + +func (s *seqCache1) mallocSeq(ctx context.Context, conversationID string, size int64, seqs *[]int64) error { + var delMinSeqKey bool + _, err := getCache[string](ctx, s.rocks, cachekey.GetMallocSeqLockKey(conversationID), s.lockExpire, func(ctx context.Context) (string, error) { + res, err := s.mgo.Malloc(ctx, conversationID, s.getMongoStepSize(conversationID, size)) + if err != nil { + return "", err + } + delMinSeqKey = res[0] == 1 + if seqs != nil && size > 0 { + if len(*seqs) > 0 && (*seqs)[len(*seqs)-1]+1 == res[0] { + n := size - int64(len(*seqs)) + *seqs = append(*seqs, res[:n]...) + res = res[n:] + } else { + *seqs = res[:size] + res = res[size:] + } + } + if err := s.push(ctx, cachekey.GetMallocSeqKey(conversationID), res); err != nil { + return "", err + } + return strconv.Itoa(int(time.Now().UnixMicro())), nil + }) + if delMinSeqKey { + s.deleteMinSeqCache(ctx, conversationID) + } + return err +} + +func (s *seqCache1) deleteMinSeqCache(ctx context.Context, conversationID string) error { + return s.rocks.TagAsDeleted2(ctx, cachekey.GetMallocMinSeqKey(conversationID)) +} diff --git a/pkg/common/storage/controller/msg.go b/pkg/common/storage/controller/msg.go index 8eb9e8e6f..f0efe5efd 100644 --- a/pkg/common/storage/controller/msg.go +++ b/pkg/common/storage/controller/msg.go @@ -69,26 +69,26 @@ type CommonMsgDatabase interface { DeleteUserMsgsBySeqs(ctx context.Context, userID string, conversationID string, seqs []int64) error // DeleteMsgsPhysicalBySeqs physically deletes messages by emptying them based on sequence numbers. DeleteMsgsPhysicalBySeqs(ctx context.Context, conversationID string, seqs []int64) error - SetMaxSeq(ctx context.Context, conversationID string, maxSeq int64) error + //SetMaxSeq(ctx context.Context, conversationID string, maxSeq int64) error GetMaxSeqs(ctx context.Context, conversationIDs []string) (map[string]int64, error) GetMaxSeq(ctx context.Context, conversationID string) (int64, error) - SetMinSeq(ctx context.Context, conversationID string, minSeq int64) error + //SetMinSeq(ctx context.Context, conversationID string, minSeq int64) error SetMinSeqs(ctx context.Context, seqs map[string]int64) error - GetMinSeqs(ctx context.Context, conversationIDs []string) (map[string]int64, error) - GetMinSeq(ctx context.Context, conversationID string) (int64, error) - GetConversationUserMinSeq(ctx context.Context, conversationID string, userID string) (int64, error) - GetConversationUserMinSeqs(ctx context.Context, conversationID string, userIDs []string) (map[string]int64, error) - SetConversationUserMinSeq(ctx context.Context, conversationID string, userID string, minSeq int64) error - SetConversationUserMinSeqs(ctx context.Context, conversationID string, seqs map[string]int64) (err error) + //GetMinSeqs(ctx context.Context, conversationIDs []string) (map[string]int64, error) + //GetMinSeq(ctx context.Context, conversationID string) (int64, error) + //GetConversationUserMinSeq(ctx context.Context, conversationID string, userID string) (int64, error) + //GetConversationUserMinSeqs(ctx context.Context, conversationID string, userIDs []string) (map[string]int64, error) + //SetConversationUserMinSeq(ctx context.Context, conversationID string, userID string, minSeq int64) error + //SetConversationUserMinSeqs(ctx context.Context, conversationID string, seqs map[string]int64) (err error) SetUserConversationsMinSeqs(ctx context.Context, userID string, seqs map[string]int64) (err error) SetHasReadSeq(ctx context.Context, userID string, conversationID string, hasReadSeq int64) error GetHasReadSeqs(ctx context.Context, userID string, conversationIDs []string) (map[string]int64, error) GetHasReadSeq(ctx context.Context, userID string, conversationID string) (int64, error) UserSetHasReadSeqs(ctx context.Context, userID string, hasReadSeqs map[string]int64) error - GetMongoMaxAndMinSeq(ctx context.Context, conversationID string) (minSeqMongo, maxSeqMongo int64, err error) - GetConversationMinMaxSeqInMongoAndCache(ctx context.Context, conversationID string) (minSeqMongo, maxSeqMongo, minSeqCache, maxSeqCache int64, err error) + //GetMongoMaxAndMinSeq(ctx context.Context, conversationID string) (minSeqMongo, maxSeqMongo int64, err error) + //GetConversationMinMaxSeqInMongoAndCache(ctx context.Context, conversationID string) (minSeqMongo, maxSeqMongo, minSeqCache, maxSeqCache int64, err error) SetSendMsgStatus(ctx context.Context, id string, status int32) error GetSendMsgStatus(ctx context.Context, id string) (int32, error) SearchMessage(ctx context.Context, req *pbmsg.SearchMessageReq) (total int32, msgData []*sdkws.MsgData, err error) @@ -349,6 +349,8 @@ func (db *commonMsgDatabase) DeleteMessagesFromCache(ctx context.Context, conver } func (db *commonMsgDatabase) BatchInsertChat2Cache(ctx context.Context, conversationID string, msgs []*sdkws.MsgData) (seq int64, isNew bool, err error) { + + // TODO set SEQ currentMaxSeq, err := db.seq.GetMaxSeq(ctx, conversationID) if err != nil && errs.Unwrap(err) != redis.Nil { log.ZError(ctx, "storage.seq.GetMaxSeq", err) @@ -817,9 +819,9 @@ func (db *commonMsgDatabase) CleanUpUserConversationsMsgs(ctx context.Context, u } } -func (db *commonMsgDatabase) SetMaxSeq(ctx context.Context, conversationID string, maxSeq int64) error { - return db.seq.SetMaxSeq(ctx, conversationID, maxSeq) -} +//func (db *commonMsgDatabase) SetMaxSeq(ctx context.Context, conversationID string, maxSeq int64) error { +// return db.seq.SetMaxSeq(ctx, conversationID, maxSeq) +//} func (db *commonMsgDatabase) GetMaxSeqs(ctx context.Context, conversationIDs []string) (map[string]int64, error) { return db.seq.GetMaxSeqs(ctx, conversationIDs) @@ -837,13 +839,13 @@ func (db *commonMsgDatabase) SetMinSeqs(ctx context.Context, seqs map[string]int return db.seq.SetMinSeqs(ctx, seqs) } -func (db *commonMsgDatabase) GetMinSeqs(ctx context.Context, conversationIDs []string) (map[string]int64, error) { - return db.seq.GetMinSeqs(ctx, conversationIDs) -} - -func (db *commonMsgDatabase) GetMinSeq(ctx context.Context, conversationID string) (int64, error) { - return db.seq.GetMinSeq(ctx, conversationID) -} +//func (db *commonMsgDatabase) GetMinSeqs(ctx context.Context, conversationIDs []string) (map[string]int64, error) { +// return db.seq.GetMinSeqs(ctx, conversationIDs) +//} +// +//func (db *commonMsgDatabase) GetMinSeq(ctx context.Context, conversationID string) (int64, error) { +// return db.seq.GetMinSeq(ctx, conversationID) +//} func (db *commonMsgDatabase) GetConversationUserMinSeq(ctx context.Context, conversationID string, userID string) (int64, error) { return db.seq.GetConversationUserMinSeq(ctx, conversationID, userID) diff --git a/pkg/common/storage/database/mgo/seq.go b/pkg/common/storage/database/mgo/seq.go new file mode 100644 index 000000000..8592bf63c --- /dev/null +++ b/pkg/common/storage/database/mgo/seq.go @@ -0,0 +1,75 @@ +package mgo + +import ( + "context" + "errors" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" + "github.com/openimsdk/tools/db/mongoutil" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" +) + +func NewSeqMongo(db *mongo.Database) (*SeqMongo, error) { + coll := db.Collection("seq") + return &SeqMongo{coll: coll}, nil +} + +type SeqMongo struct { + coll *mongo.Collection +} + +func (s *SeqMongo) MallocSeq(ctx context.Context, conversationID string, size int64) (int64, error) { + if size <= 0 { + return 0, errors.New("size must be greater than 0") + } + filter := map[string]any{"conversation_id": conversationID} + update := map[string]any{ + "$inc": map[string]any{"max_seq": size}, + "$set": map[string]any{"min_seq": 1}, + } + opt := options.FindOneAndUpdate().SetUpsert(true).SetReturnDocument(options.After).SetProjection(map[string]any{"_id": 0, "max_seq": 1}) + return mongoutil.FindOneAndUpdate[int64](ctx, s.coll, filter, update, opt) +} + +func (s *SeqMongo) Malloc(ctx context.Context, conversationID string, size int64) ([]int64, error) { + seq, err := s.MallocSeq(ctx, conversationID, size) + if err != nil { + return nil, err + } + seqs := make([]int64, 0, size) + for i := seq - size + 1; i <= seq; i++ { + seqs = append(seqs, i) + } + return seqs, nil +} + +func (s *SeqMongo) GetMaxSeq(ctx context.Context, conversationID string) (int64, error) { + seq, err := mongoutil.FindOne[int64](ctx, s.coll, bson.M{"conversation_id": conversationID}, options.FindOne().SetProjection(map[string]any{"_id": 0, "max_seq": 1})) + if err == nil { + return seq, nil + } else if IsNotFound(err) { + return 0, nil + } else { + return 0, err + } +} + +func (s *SeqMongo) GetMinSeq(ctx context.Context, conversationID string) (int64, error) { + seq, err := mongoutil.FindOne[int64](ctx, s.coll, bson.M{"conversation_id": conversationID}, options.FindOne().SetProjection(map[string]any{"_id": 0, "min_seq": 1})) + if err == nil { + return seq, nil + } else if IsNotFound(err) { + return 0, nil + } else { + return 0, err + } +} + +func (s *SeqMongo) SetMinSeq(ctx context.Context, conversationID string, seq int64) error { + return mongoutil.UpdateOne(ctx, s.coll, bson.M{"conversation_id": conversationID}, bson.M{"$set": bson.M{"min_seq": seq}}, false) +} + +func (s *SeqMongo) GetConversation(ctx context.Context, conversationID string) (*model.Seq, error) { + return mongoutil.FindOne[*model.Seq](ctx, s.coll, bson.M{"conversation_id": conversationID}) +} diff --git a/pkg/common/storage/database/name.go b/pkg/common/storage/database/name.go index 986f22a1a..f496d4b41 100644 --- a/pkg/common/storage/database/name.go +++ b/pkg/common/storage/database/name.go @@ -14,4 +14,5 @@ const ( LogName = "log" ObjectName = "s3" UserName = "user" + SeqName = "seq" ) diff --git a/pkg/common/storage/database/seq.go b/pkg/common/storage/database/seq.go new file mode 100644 index 000000000..869b607a5 --- /dev/null +++ b/pkg/common/storage/database/seq.go @@ -0,0 +1,10 @@ +package database + +import "context" + +type Seq interface { + Malloc(ctx context.Context, conversationID string, size int64) ([]int64, error) + GetMaxSeq(ctx context.Context, conversationID string) (int64, error) + GetMinSeq(ctx context.Context, conversationID string) (int64, error) + SetMinSeq(ctx context.Context, conversationID string, seq int64) error +} diff --git a/pkg/common/storage/model/seq.go b/pkg/common/storage/model/seq.go new file mode 100644 index 000000000..3db009167 --- /dev/null +++ b/pkg/common/storage/model/seq.go @@ -0,0 +1,7 @@ +package model + +type Seq struct { + ConversationID string `bson:"conversation_id"` + MaxSeq int64 `bson:"max_seq"` + MinSeq int64 `bson:"min_seq"` +} diff --git a/pkg/msgprocessor/conversation.go b/pkg/msgprocessor/conversation.go index b369269cc..f8140cc7d 100644 --- a/pkg/msgprocessor/conversation.go +++ b/pkg/msgprocessor/conversation.go @@ -24,6 +24,10 @@ import ( "google.golang.org/protobuf/proto" ) +func IsGroupConversationID(conversationID string) bool { + return strings.HasPrefix(conversationID, "g_") || strings.HasPrefix(conversationID, "sg_") +} + func GetNotificationConversationIDByMsg(msg *sdkws.MsgData) string { switch msg.SessionType { case constant.SingleChatType: From 904842b15da197711bfa7fdb27cb74445ff8b6ee Mon Sep 17 00:00:00 2001 From: icey-yu <1186114839@qq.com> Date: Fri, 14 Jun 2024 17:04:03 +0800 Subject: [PATCH 32/59] update: go mod --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 4f4cd4b68..28e51cdaa 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/lestrrat-go/file-rotatelogs v2.4.0+incompatible // indirect github.com/mitchellh/mapstructure v1.5.0 - github.com/openimsdk/protocol v0.0.69-alpha.9 + github.com/openimsdk/protocol v0.0.69-alpha.10 github.com/openimsdk/tools v0.0.49-alpha.25 github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 diff --git a/go.sum b/go.sum index 5a72e28f2..f29079ec3 100644 --- a/go.sum +++ b/go.sum @@ -270,8 +270,8 @@ github.com/onsi/gomega v1.25.0 h1:Vw7br2PCDYijJHSfBOWhov+8cAnUf8MfMaIOV323l6Y= github.com/onsi/gomega v1.25.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdMPGhM= github.com/openimsdk/gomake v0.0.13 h1:xLDe/moqgWpRoptHzI4packAWzs4C16b+sVY+txNJp0= github.com/openimsdk/gomake v0.0.13/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= -github.com/openimsdk/protocol v0.0.69-alpha.9 h1:Mh1upsxwhWs1y65cfIN2XuRsiKEk4MfpYfA9XrvtX24= -github.com/openimsdk/protocol v0.0.69-alpha.9/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= +github.com/openimsdk/protocol v0.0.69-alpha.10 h1:reuNuQt77a57Sd8mUgy8PqTA+sG6rk6CfzHkE9DvBto= +github.com/openimsdk/protocol v0.0.69-alpha.10/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= github.com/openimsdk/tools v0.0.49-alpha.25 h1:OpRPwDZ2xWX7Zj5kyfZhryu/NfZTrsRVr2GFwu1HQHI= github.com/openimsdk/tools v0.0.49-alpha.25/go.mod h1:rwsFI1G/nBHNfiNapbven41akRDPBbH4df0Cgy6xueU= github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= From c5fa596cdd3a8905a35dc215de86f7ea9ac8ad2b Mon Sep 17 00:00:00 2001 From: icey-yu <1186114839@qq.com> Date: Fri, 14 Jun 2024 17:14:43 +0800 Subject: [PATCH 33/59] refactor: change incremental to full --- internal/api/group.go | 8 ++++---- internal/api/router.go | 4 ++-- internal/rpc/group/sync.go | 25 +++++++------------------ pkg/util/hashutil/id.go | 16 ++++++++++++++++ 4 files changed, 29 insertions(+), 24 deletions(-) create mode 100644 pkg/util/hashutil/id.go diff --git a/internal/api/group.go b/internal/api/group.go index 0bf61c787..91992004c 100644 --- a/internal/api/group.go +++ b/internal/api/group.go @@ -187,10 +187,10 @@ func (o *GroupApi) GetIncrementalGroupMemberBatch(c *gin.Context) { apiresp.GinSuccess(c, resp) } -func (o *GroupApi) GetIncrementalGroupMemberUserIDs(c *gin.Context) { - a2r.Call(group.GroupClient.GetIncrementalGroupMemberUserIDs, o.Client, c) +func (o *GroupApi) GetFullGroupMemberUserIDs(c *gin.Context) { + a2r.Call(group.GroupClient.GetFullGroupMemberUserIDs, o.Client, c) } -func (o *GroupApi) GetIncrementalJoinGroupIDs(c *gin.Context) { - a2r.Call(group.GroupClient.GetIncrementalJoinGroupIDs, o.Client, c) +func (o *GroupApi) GetFullJoinGroupIDs(c *gin.Context) { + a2r.Call(group.GroupClient.GetFullJoinGroupIDs, o.Client, c) } diff --git a/internal/api/router.go b/internal/api/router.go index 69c026ffd..e0ec652d5 100644 --- a/internal/api/router.go +++ b/internal/api/router.go @@ -120,8 +120,8 @@ func newGinRouter(disCov discovery.SvcDiscoveryRegistry, config *Config) *gin.En groupRouterGroup.POST("/get_incremental_join_group", g.GetIncrementalJoinGroup) groupRouterGroup.POST("/get_incremental_group_member", g.GetIncrementalGroupMember) groupRouterGroup.POST("/get_incremental_group_member_batch", g.GetIncrementalGroupMemberBatch) - groupRouterGroup.POST("/get_incremental_group_member_user_ids", g.GetIncrementalGroupMemberUserIDs) - groupRouterGroup.POST("/get_incremental_join_group_ids", g.GetIncrementalJoinGroupIDs) + groupRouterGroup.POST("/get_full_group_member_user_ids", g.GetFullGroupMemberUserIDs) + groupRouterGroup.POST("/get_full_join_group_ids", g.GetFullJoinGroupIDs) } // certificate authRouterGroup := r.Group("/auth") diff --git a/internal/rpc/group/sync.go b/internal/rpc/group/sync.go index 4bd34eb4d..bd6fc6399 100644 --- a/internal/rpc/group/sync.go +++ b/internal/rpc/group/sync.go @@ -2,26 +2,15 @@ package group import ( "context" - "crypto/md5" - "encoding/binary" - "encoding/json" "github.com/openimsdk/open-im-server/v3/internal/rpc/incrversion" "github.com/openimsdk/open-im-server/v3/pkg/authverify" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" + "github.com/openimsdk/open-im-server/v3/pkg/util/hashutil" pbgroup "github.com/openimsdk/protocol/group" "github.com/openimsdk/protocol/sdkws" ) -func (s *groupServer) idHash(ids []string) uint64 { - if len(ids) == 0 { - return 0 - } - data, _ := json.Marshal(ids) - sum := md5.Sum(data) - return binary.BigEndian.Uint64(sum[:]) -} - -func (s *groupServer) GetIncrementalGroupMemberUserIDs(ctx context.Context, req *pbgroup.GetIncrementalGroupMemberUserIDsReq) (*pbgroup.GetIncrementalGroupMemberUserIDsResp, error) { +func (s *groupServer) GetFullGroupMemberUserIDs(ctx context.Context, req *pbgroup.GetFullGroupMemberUserIDsReq) (*pbgroup.GetFullGroupMemberUserIDsResp, error) { vl, err := s.db.FindMaxGroupMemberVersionCache(ctx, req.GroupID) if err != nil { return nil, err @@ -30,11 +19,11 @@ func (s *groupServer) GetIncrementalGroupMemberUserIDs(ctx context.Context, req if err != nil { return nil, err } - idHash := s.idHash(userIDs) + idHash := hashutil.IdHash(userIDs) if req.IdHash == idHash { userIDs = nil } - return &pbgroup.GetIncrementalGroupMemberUserIDsResp{ + return &pbgroup.GetFullGroupMemberUserIDsResp{ Version: idHash, VersionID: vl.ID.Hex(), Equal: req.IdHash == idHash, @@ -42,7 +31,7 @@ func (s *groupServer) GetIncrementalGroupMemberUserIDs(ctx context.Context, req }, nil } -func (s *groupServer) GetIncrementalJoinGroupIDs(ctx context.Context, req *pbgroup.GetIncrementalJoinGroupIDsReq) (*pbgroup.GetIncrementalJoinGroupIDsResp, error) { +func (s *groupServer) GetFullJoinGroupIDs(ctx context.Context, req *pbgroup.GetFullJoinGroupIDsReq) (*pbgroup.GetFullJoinGroupIDsResp, error) { vl, err := s.db.FindMaxJoinGroupVersionCache(ctx, req.UserID) if err != nil { return nil, err @@ -51,11 +40,11 @@ func (s *groupServer) GetIncrementalJoinGroupIDs(ctx context.Context, req *pbgro if err != nil { return nil, err } - idHash := s.idHash(groupIDs) + idHash := hashutil.IdHash(groupIDs) if req.IdHash == idHash { groupIDs = nil } - return &pbgroup.GetIncrementalJoinGroupIDsResp{ + return &pbgroup.GetFullJoinGroupIDsResp{ Version: idHash, VersionID: vl.ID.Hex(), Equal: req.IdHash == idHash, diff --git a/pkg/util/hashutil/id.go b/pkg/util/hashutil/id.go new file mode 100644 index 000000000..52e7f4c6f --- /dev/null +++ b/pkg/util/hashutil/id.go @@ -0,0 +1,16 @@ +package hashutil + +import ( + "crypto/md5" + "encoding/binary" + "encoding/json" +) + +func IdHash(ids []string) uint64 { + if len(ids) == 0 { + return 0 + } + data, _ := json.Marshal(ids) + sum := md5.Sum(data) + return binary.BigEndian.Uint64(sum[:]) +} From f19f6f9112a94bf0eef831f902de573ab06e6297 Mon Sep 17 00:00:00 2001 From: icey-yu <1186114839@qq.com> Date: Fri, 14 Jun 2024 17:17:59 +0800 Subject: [PATCH 34/59] feat: get full friend user ids --- internal/rpc/friend/friend.go | 42 +++++++++++++++++------------------ internal/rpc/friend/sync.go | 26 ++++++++++++++++++++-- 2 files changed, 45 insertions(+), 23 deletions(-) diff --git a/internal/rpc/friend/friend.go b/internal/rpc/friend/friend.go index 12107125c..622e19f42 100644 --- a/internal/rpc/friend/friend.go +++ b/internal/rpc/friend/friend.go @@ -41,7 +41,7 @@ import ( ) type friendServer struct { - friendDatabase controller.FriendDatabase + db controller.FriendDatabase blackDatabase controller.BlackDatabase userRpcClient *rpcclient.UserRpcClient notificationSender *FriendNotificationSender @@ -102,7 +102,7 @@ func Start(ctx context.Context, config *Config, client discovery.SvcDiscoveryReg // Register Friend server with refactored MongoDB and Redis integrations relation.RegisterFriendServer(server, &friendServer{ - friendDatabase: controller.NewFriendDatabase( + db: controller.NewFriendDatabase( friendMongoDB, friendRequestMongoDB, redis.NewFriendCacheRedis(rdb, &config.LocalCacheConfig, friendMongoDB, redis.GetRocksCacheOptions()), @@ -139,14 +139,14 @@ func (s *friendServer) ApplyToAddFriend(ctx context.Context, req *relation.Apply return nil, err } - in1, in2, err := s.friendDatabase.CheckIn(ctx, req.FromUserID, req.ToUserID) + in1, in2, err := s.db.CheckIn(ctx, req.FromUserID, req.ToUserID) if err != nil { return nil, err } if in1 && in2 { return nil, servererrs.ErrRelationshipAlready.WrapMsg("already friends has f") } - if err = s.friendDatabase.AddFriendRequest(ctx, req.FromUserID, req.ToUserID, req.ReqMsg, req.Ex); err != nil { + if err = s.db.AddFriendRequest(ctx, req.FromUserID, req.ToUserID, req.ReqMsg, req.Ex); err != nil { return nil, err } s.notificationSender.FriendApplicationAddNotification(ctx, req) @@ -173,7 +173,7 @@ func (s *friendServer) ImportFriends(ctx context.Context, req *relation.ImportFr return nil, err } - if err := s.friendDatabase.BecomeFriends(ctx, req.OwnerUserID, req.FriendUserIDs, constant.BecomeFriendByImport); err != nil { + if err := s.db.BecomeFriends(ctx, req.OwnerUserID, req.FriendUserIDs, constant.BecomeFriendByImport); err != nil { return nil, err } for _, userID := range req.FriendUserIDs { @@ -205,7 +205,7 @@ func (s *friendServer) RespondFriendApply(ctx context.Context, req *relation.Res if err := s.webhookBeforeAddFriendAgree(ctx, &s.config.WebhooksConfig.BeforeAddFriendAgree, req); err != nil && err != servererrs.ErrCallbackContinue { return nil, err } - err := s.friendDatabase.AgreeFriendRequest(ctx, &friendRequest) + err := s.db.AgreeFriendRequest(ctx, &friendRequest) if err != nil { return nil, err } @@ -213,7 +213,7 @@ func (s *friendServer) RespondFriendApply(ctx context.Context, req *relation.Res return resp, nil } if req.HandleResult == constant.FriendResponseRefuse { - err := s.friendDatabase.RefuseFriendRequest(ctx, &friendRequest) + err := s.db.RefuseFriendRequest(ctx, &friendRequest) if err != nil { return nil, err } @@ -229,11 +229,11 @@ func (s *friendServer) DeleteFriend(ctx context.Context, req *relation.DeleteFri if err := s.userRpcClient.Access(ctx, req.OwnerUserID); err != nil { return nil, err } - _, err = s.friendDatabase.FindFriendsWithError(ctx, req.OwnerUserID, []string{req.FriendUserID}) + _, err = s.db.FindFriendsWithError(ctx, req.OwnerUserID, []string{req.FriendUserID}) if err != nil { return nil, err } - if err := s.friendDatabase.Delete(ctx, req.OwnerUserID, []string{req.FriendUserID}); err != nil { + if err := s.db.Delete(ctx, req.OwnerUserID, []string{req.FriendUserID}); err != nil { return nil, err } s.notificationSender.FriendDeletedNotification(ctx, req) @@ -250,11 +250,11 @@ func (s *friendServer) SetFriendRemark(ctx context.Context, req *relation.SetFri if err := s.userRpcClient.Access(ctx, req.OwnerUserID); err != nil { return nil, err } - _, err = s.friendDatabase.FindFriendsWithError(ctx, req.OwnerUserID, []string{req.FriendUserID}) + _, err = s.db.FindFriendsWithError(ctx, req.OwnerUserID, []string{req.FriendUserID}) if err != nil { return nil, err } - if err := s.friendDatabase.UpdateRemark(ctx, req.OwnerUserID, req.FriendUserID, req.Remark); err != nil { + if err := s.db.UpdateRemark(ctx, req.OwnerUserID, req.FriendUserID, req.Remark); err != nil { return nil, err } s.webhookAfterSetFriendRemark(ctx, &s.config.WebhooksConfig.AfterSetFriendRemark, req) @@ -281,7 +281,7 @@ func (s *friendServer) getFriend(ctx context.Context, ownerUserID string, friend if len(friendUserIDs) == 0 { return nil, nil } - friends, err := s.friendDatabase.FindFriendsWithError(ctx, ownerUserID, friendUserIDs) + friends, err := s.db.FindFriendsWithError(ctx, ownerUserID, friendUserIDs) if err != nil { return nil, err } @@ -292,7 +292,7 @@ func (s *friendServer) getFriend(ctx context.Context, ownerUserID string, friend func (s *friendServer) GetDesignatedFriendsApply(ctx context.Context, req *relation.GetDesignatedFriendsApplyReq, ) (resp *relation.GetDesignatedFriendsApplyResp, err error) { - friendRequests, err := s.friendDatabase.FindBothFriendRequests(ctx, req.FromUserID, req.ToUserID) + friendRequests, err := s.db.FindBothFriendRequests(ctx, req.FromUserID, req.ToUserID) if err != nil { return nil, err } @@ -309,7 +309,7 @@ func (s *friendServer) GetPaginationFriendsApplyTo(ctx context.Context, req *rel if err := s.userRpcClient.Access(ctx, req.UserID); err != nil { return nil, err } - total, friendRequests, err := s.friendDatabase.PageFriendRequestToMe(ctx, req.UserID, req.Pagination) + total, friendRequests, err := s.db.PageFriendRequestToMe(ctx, req.UserID, req.Pagination) if err != nil { return nil, err } @@ -327,7 +327,7 @@ func (s *friendServer) GetPaginationFriendsApplyFrom(ctx context.Context, req *r if err := s.userRpcClient.Access(ctx, req.UserID); err != nil { return nil, err } - total, friendRequests, err := s.friendDatabase.PageFriendRequestFromMe(ctx, req.UserID, req.Pagination) + total, friendRequests, err := s.db.PageFriendRequestFromMe(ctx, req.UserID, req.Pagination) if err != nil { return nil, err } @@ -342,7 +342,7 @@ func (s *friendServer) GetPaginationFriendsApplyFrom(ctx context.Context, req *r // ok. func (s *friendServer) IsFriend(ctx context.Context, req *relation.IsFriendReq) (resp *relation.IsFriendResp, err error) { resp = &relation.IsFriendResp{} - resp.InUser1Friends, resp.InUser2Friends, err = s.friendDatabase.CheckIn(ctx, req.UserID1, req.UserID2) + resp.InUser1Friends, resp.InUser2Friends, err = s.db.CheckIn(ctx, req.UserID1, req.UserID2) if err != nil { return nil, err } @@ -353,7 +353,7 @@ func (s *friendServer) GetPaginationFriends(ctx context.Context, req *relation.G if err := s.userRpcClient.Access(ctx, req.UserID); err != nil { return nil, err } - total, friends, err := s.friendDatabase.PageOwnerFriends(ctx, req.UserID, req.Pagination) + total, friends, err := s.db.PageOwnerFriends(ctx, req.UserID, req.Pagination) if err != nil { return nil, err } @@ -371,7 +371,7 @@ func (s *friendServer) GetFriendIDs(ctx context.Context, req *relation.GetFriend return nil, err } resp = &relation.GetFriendIDsResp{} - resp.FriendIDs, err = s.friendDatabase.FindFriendUserIDs(ctx, req.UserID) + resp.FriendIDs, err = s.db.FindFriendUserIDs(ctx, req.UserID) if err != nil { return nil, err } @@ -389,7 +389,7 @@ func (s *friendServer) GetSpecifiedFriendsInfo(ctx context.Context, req *relatio if err != nil { return nil, err } - friends, err := s.friendDatabase.FindFriendsWithError(ctx, req.OwnerUserID, req.UserIDList) + friends, err := s.db.FindFriendsWithError(ctx, req.OwnerUserID, req.UserIDList) if err != nil { return nil, err } @@ -453,7 +453,7 @@ func (s *friendServer) UpdateFriends( return nil, errs.ErrArgs.WrapMsg("friendIDList repeated") } - _, err := s.friendDatabase.FindFriendsWithError(ctx, req.OwnerUserID, req.FriendUserIDs) + _, err := s.db.FindFriendsWithError(ctx, req.OwnerUserID, req.FriendUserIDs) if err != nil { return nil, err } @@ -469,7 +469,7 @@ func (s *friendServer) UpdateFriends( if req.Ex != nil { val["ex"] = req.Ex.Value } - if err = s.friendDatabase.UpdateFriends(ctx, req.OwnerUserID, req.FriendUserIDs, val); err != nil { + if err = s.db.UpdateFriends(ctx, req.OwnerUserID, req.FriendUserIDs, val); err != nil { return nil, err } diff --git a/internal/rpc/friend/sync.go b/internal/rpc/friend/sync.go index cddd23f89..faaa987f2 100644 --- a/internal/rpc/friend/sync.go +++ b/internal/rpc/friend/sync.go @@ -2,6 +2,7 @@ package friend import ( "context" + "github.com/openimsdk/open-im-server/v3/pkg/util/hashutil" "github.com/openimsdk/protocol/sdkws" "github.com/openimsdk/open-im-server/v3/internal/rpc/incrversion" @@ -10,6 +11,27 @@ import ( "github.com/openimsdk/protocol/relation" ) +func (s *friendServer) GetFullFriendUserIDs(ctx context.Context, req *relation.GetFullFriendUserIDsReq) (*relation.GetFullFriendUserIDsResp, error) { + vl, err := s.db.FindMaxFriendVersionCache(ctx, req.UserID) + if err != nil { + return nil, err + } + userIDs, err := s.db.FindFriendUserIDs(ctx, req.UserID) + if err != nil { + return nil, err + } + idHash := hashutil.IdHash(userIDs) + if req.IdHash == idHash { + userIDs = nil + } + return &relation.GetFullFriendUserIDsResp{ + Version: idHash, + VersionID: vl.ID.Hex(), + Equal: req.IdHash == idHash, + UserIDs: userIDs, + }, nil +} + func (s *friendServer) GetIncrementalFriends(ctx context.Context, req *relation.GetIncrementalFriendsReq) (*relation.GetIncrementalFriendsResp, error) { if err := authverify.CheckAccessV3(ctx, req.UserID, s.config.Share.IMAdminUserID); err != nil { return nil, err @@ -19,8 +41,8 @@ func (s *friendServer) GetIncrementalFriends(ctx context.Context, req *relation. VersionKey: req.UserID, VersionID: req.VersionID, VersionNumber: req.Version, - Version: s.friendDatabase.FindFriendIncrVersion, - CacheMaxVersion: s.friendDatabase.FindMaxFriendVersionCache, + Version: s.db.FindFriendIncrVersion, + CacheMaxVersion: s.db.FindMaxFriendVersionCache, Find: func(ctx context.Context, ids []string) ([]*sdkws.FriendInfo, error) { return s.getFriend(ctx, req.UserID, ids) }, From cee1a496e970cb2f4b64617d102e892ceafd4537 Mon Sep 17 00:00:00 2001 From: icey-yu <1186114839@qq.com> Date: Fri, 14 Jun 2024 18:29:54 +0800 Subject: [PATCH 35/59] feat: api and config --- config/redis.yml | 2 +- internal/api/friend.go | 4 ++++ internal/api/router.go | 1 + 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/config/redis.yml b/config/redis.yml index 404d18953..87abed0e1 100644 --- a/config/redis.yml +++ b/config/redis.yml @@ -1,4 +1,4 @@ -address: [ 172.16.8.48:16379 ] +address: [ localhost:16379 ] username: '' password: openIM123 clusterMode: false diff --git a/internal/api/friend.go b/internal/api/friend.go index 6912fdbbb..9266fe75c 100644 --- a/internal/api/friend.go +++ b/internal/api/friend.go @@ -102,3 +102,7 @@ func (o *FriendApi) GetIncrementalFriends(c *gin.Context) { func (o *FriendApi) GetIncrementalBlacks(c *gin.Context) { a2r.Call(relation.FriendClient.GetIncrementalBlacks, o.Client, c) } + +func (o *FriendApi) GetFullFriendUserIDs(c *gin.Context) { + a2r.Call(relation.FriendClient.GetFullFriendUserIDs, o.Client, c) +} diff --git a/internal/api/router.go b/internal/api/router.go index e0ec652d5..0f46f26ba 100644 --- a/internal/api/router.go +++ b/internal/api/router.go @@ -89,6 +89,7 @@ func newGinRouter(disCov discovery.SvcDiscoveryRegistry, config *Config) *gin.En friendRouterGroup.POST("/get_specified_friends_info", f.GetSpecifiedFriendsInfo) friendRouterGroup.POST("/update_friends", f.UpdateFriends) friendRouterGroup.POST("/get_incremental_friends", f.GetIncrementalFriends) + friendRouterGroup.POST("/get_full_friend_user_ids", f.GetFullFriendUserIDs) } g := NewGroupApi(*groupRpc) groupRouterGroup := r.Group("/group") From fe4842b49690477e38aa0a0c534dcb3d4f4e443f Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Fri, 14 Jun 2024 18:49:05 +0800 Subject: [PATCH 36/59] seq --- pkg/common/storage/cache/redis/seq1.go | 253 ++++++++++++++++++ pkg/common/storage/cache/redis/seq_test.go | 27 ++ pkg/common/storage/database/mgo/seq.go | 21 +- .../mgo/{version_test.go => seq_test.go} | 18 +- pkg/common/storage/database/seq.go | 2 +- 5 files changed, 298 insertions(+), 23 deletions(-) create mode 100644 pkg/common/storage/cache/redis/seq1.go create mode 100644 pkg/common/storage/cache/redis/seq_test.go rename pkg/common/storage/database/mgo/{version_test.go => seq_test.go} (58%) diff --git a/pkg/common/storage/cache/redis/seq1.go b/pkg/common/storage/cache/redis/seq1.go new file mode 100644 index 000000000..6374f53cc --- /dev/null +++ b/pkg/common/storage/cache/redis/seq1.go @@ -0,0 +1,253 @@ +package redis + +import ( + "context" + "fmt" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/cachekey" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database/mgo" + "github.com/openimsdk/open-im-server/v3/pkg/msgprocessor" + "github.com/openimsdk/tools/errs" + "github.com/openimsdk/tools/log" + "github.com/redis/go-redis/v9" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" + "time" +) + +type RedisHash struct { + NextSeq int64 + LastSeq int64 +} + +func NewTestSeq() *SeqMalloc { + mgocli, err := mongo.Connect(context.Background(), options.Client().ApplyURI("mongodb://openIM:openIM123@172.16.8.48:37017/openim_v3?maxPoolSize=100").SetConnectTimeout(5*time.Second)) + if err != nil { + panic(err) + } + model, err := mgo.NewSeqMongo(mgocli.Database("openim_v3")) + if err != nil { + panic(err) + } + opt := &redis.Options{ + Addr: "172.16.8.48:16379", + Password: "openIM123", + DB: 1, + } + rdb := redis.NewClient(opt) + if err := rdb.Ping(context.Background()).Err(); err != nil { + panic(err) + } + return &SeqMalloc{ + rdb: rdb, + mgo: model, + //lockTime: time.Second * 30, + lockTime: time.Second * 60 * 60 * 24 * 1, + dataTime: time.Second * 60 * 60 * 24 * 7, + } +} + +type SeqMalloc struct { + rdb redis.UniversalClient + mgo database.Seq + lockTime time.Duration + dataTime time.Duration +} + +func (s *SeqMalloc) getSeqMallocKey(conversationID string) string { + return cachekey.GetMallocSeqKey(conversationID) +} + +func (s *SeqMalloc) setSeq(ctx context.Context, key string, owner int64, currSeq int64, lastSeq int64) (int64, error) { + if lastSeq < currSeq { + return 0, errs.New("lastSeq must be greater than currSeq") + } + // 0： 成功 + // 1： 成功 锁过期，但未被其他人锁 + // 2： 已经被锁，但是锁的不是自己 + script := ` +local key = KEYS[1] +local lockValue = ARGV[1] +local dataSecond = ARGV[2] +local curr_seq = tonumber(ARGV[3]) +local last_seq = tonumber(ARGV[4]) +if redis.call("EXISTS", key) == 0 then + redis.call("HSET", key, "CURR", curr_seq, "LAST", last_seq) + redis.call("EXPIRE", key, dataSecond) + return 1 +end +if redis.call("HGET", key, "LOCK") ~= lockValue then + return 2 +end +redis.call("HDEL", key, "LOCK") +redis.call("HSET", key, "CURR", curr_seq, "LAST", last_seq) +redis.call("EXPIRE", key, dataSecond) +return 0 +` + result, err := s.rdb.Eval(ctx, script, []string{key}, owner, int64(s.dataTime/time.Second), currSeq, lastSeq).Int64() + if err != nil { + return 0, errs.Wrap(err) + } + return result, nil +} + +// malloc size=0为获取当前seq size>0为分配seq +func (s *SeqMalloc) malloc(ctx context.Context, key string, size int64) ([]int64, error) { + // 0： 成功 + // 1： 需要获取，并加锁 + // 2： 已经被锁 + // 3： 超过最大值，并加锁 + script := ` +local key = KEYS[1] +local size = tonumber(ARGV[1]) +local lockSecond = ARGV[2] +local dataSecond = ARGV[3] +local result = {} +if redis.call("EXISTS", key) == 0 then + local lockValue = math.random(0, 999999999) + redis.call("HSET", key, "LOCK", lockValue) + redis.call("EXPIRE", key, lockSecond) + table.insert(result, 1) + table.insert(result, lockValue) + return result +end +if redis.call("HEXISTS", key, "LOCK") == 1 then + table.insert(result, 2) + return result +end +local curr_seq = tonumber(redis.call("HGET", key, "CURR")) +local last_seq = tonumber(redis.call("HGET", key, "LAST")) +if size == 0 then + table.insert(result, 0) + table.insert(result, curr_seq) + table.insert(result, last_seq) + return result +end +local max_seq = curr_seq + size +if max_seq > last_seq then + local lockValue = math.random(0, 999999999) + redis.call("HSET", key, "LOCK", lockValue) + redis.call("HSET", key, "CURR", last_seq) + redis.call("EXPIRE", key, lockSecond) + table.insert(result, 3) + table.insert(result, curr_seq) + table.insert(result, last_seq) + table.insert(result, lockValue) + return result +end +redis.call("HSET", key, "CURR", max_seq) +table.insert(result, 0) +table.insert(result, curr_seq) +table.insert(result, last_seq) +return result +` + result, err := s.rdb.Eval(ctx, script, []string{key}, size, int64(s.lockTime/time.Second), int64(s.dataTime/time.Second)).Int64Slice() + if err != nil { + return nil, errs.Wrap(err) + } + return result, nil +} + +func (s *SeqMalloc) wait(ctx context.Context) error { + timer := time.NewTimer(time.Second / 4) + defer timer.Stop() + select { + case <-timer.C: + return nil + case <-ctx.Done(): + return ctx.Err() + } +} + +func (s *SeqMalloc) setSeqRetry(ctx context.Context, key string, owner int64, currSeq int64, lastSeq int64) { + for i := 0; i < 10; i++ { + state, err := s.setSeq(ctx, key, owner, currSeq, lastSeq) + if err != nil { + log.ZError(ctx, "set seq cache failed", err, "key", key, "owner", owner, "currSeq", currSeq, "lastSeq", lastSeq, "count", i+1) + if err := s.wait(ctx); err != nil { + return + } + continue + } + switch state { + case 0: // ideal state + case 1: + log.ZWarn(ctx, "set seq cache lock not found", nil, "key", key, "owner", owner, "currSeq", currSeq, "lastSeq", lastSeq) + case 2: + log.ZWarn(ctx, "set seq cache lock to be held by someone else", nil, "key", key, "owner", owner, "currSeq", currSeq, "lastSeq", lastSeq) + default: + log.ZError(ctx, "set seq cache lock unknown state", nil, "key", key, "owner", owner, "currSeq", currSeq, "lastSeq", lastSeq) + } + return + } + log.ZError(ctx, "set seq cache retrying still failed", nil, "key", key, "owner", owner, "currSeq", currSeq, "lastSeq", lastSeq) +} + +func (s *SeqMalloc) getMallocSize(conversationID string, size int64) int64 { + if size == 0 { + return 0 + } + var basicSize int64 + if msgprocessor.IsGroupConversationID(conversationID) { + basicSize = 200 + } else { + basicSize = 50 + } + basicSize += size + return basicSize +} + +func (s *SeqMalloc) Malloc(ctx context.Context, conversationID string, size int64) (int64, error) { + if size < 0 { + return 0, errs.New("size must be greater than 0") + } + key := s.getSeqMallocKey(conversationID) + for i := 0; i < 10; i++ { + states, err := s.malloc(ctx, key, size) + if err != nil { + return 0, err + } + switch states[0] { + case 0: // success + return states[1], nil + case 1: // not found + mallocSize := s.getMallocSize(conversationID, size) + seq, err := s.mgo.Malloc(ctx, conversationID, mallocSize) + if err != nil { + return 0, err + } + s.setSeqRetry(ctx, key, states[1], seq+size, seq+mallocSize) + return seq, nil + case 2: // locked + if err := s.wait(ctx); err != nil { + return 0, err + } + continue + case 3: // exceeded cache max value + currSeq := states[1] + lastSeq := states[2] + mallocSize := s.getMallocSize(conversationID, size) + seq, err := s.mgo.Malloc(ctx, conversationID, mallocSize) + if err != nil { + return 0, err + } + if lastSeq == seq { + s.setSeqRetry(ctx, key, states[3], currSeq+size, seq+mallocSize) + return currSeq, nil + } else { + log.ZWarn(ctx, "malloc seq not equal cache last seq", nil, "conversationID", conversationID, "currSeq", currSeq, "lastSeq", lastSeq, "mallocSeq", seq) + s.setSeqRetry(ctx, key, states[3], seq+size, seq+mallocSize) + return seq, nil + } + default: + log.ZError(ctx, "malloc seq unknown state", nil, "state", states[0], "conversationID", conversationID, "size", size) + return 0, errs.New(fmt.Sprintf("unknown state: %d", states[0])) + } + } + log.ZError(ctx, "malloc seq retrying still failed", nil, "conversationID", conversationID, "size", size) + return 0, errs.New("malloc seq waiting for lock timeout", "conversationID", conversationID, "size", size) +} + +func (s *SeqMalloc) GetMaxSeq(ctx context.Context, conversationID string) (int64, error) { + return s.Malloc(ctx, conversationID, 0) +} diff --git a/pkg/common/storage/cache/redis/seq_test.go b/pkg/common/storage/cache/redis/seq_test.go new file mode 100644 index 000000000..1c11bad0b --- /dev/null +++ b/pkg/common/storage/cache/redis/seq_test.go @@ -0,0 +1,27 @@ +package redis + +import ( + "context" + "testing" + "time" +) + +func TestSeq(t *testing.T) { + ts := NewTestSeq() + for i := 1; i < 1000000; i++ { + var size int64 = 100 + first, err := ts.Malloc(context.Background(), "1", size) + if err != nil { + t.Logf("[%d] %s", i, err) + return + } + t.Logf("[%d] %d -> %d", i, first+1, first+size) + time.Sleep(time.Second / 4) + } +} + +func TestDel(t *testing.T) { + ts := NewTestSeq() + t.Log(ts.GetMaxSeq(context.Background(), "1")) + +} diff --git a/pkg/common/storage/database/mgo/seq.go b/pkg/common/storage/database/mgo/seq.go index 8592bf63c..9b8c86ffe 100644 --- a/pkg/common/storage/database/mgo/seq.go +++ b/pkg/common/storage/database/mgo/seq.go @@ -19,29 +19,24 @@ type SeqMongo struct { coll *mongo.Collection } -func (s *SeqMongo) MallocSeq(ctx context.Context, conversationID string, size int64) (int64, error) { - if size <= 0 { +func (s *SeqMongo) Malloc(ctx context.Context, conversationID string, size int64) (int64, error) { + if size < 0 { return 0, errors.New("size must be greater than 0") } + if size == 0 { + return s.GetMaxSeq(ctx, conversationID) + } filter := map[string]any{"conversation_id": conversationID} update := map[string]any{ "$inc": map[string]any{"max_seq": size}, "$set": map[string]any{"min_seq": 1}, } opt := options.FindOneAndUpdate().SetUpsert(true).SetReturnDocument(options.After).SetProjection(map[string]any{"_id": 0, "max_seq": 1}) - return mongoutil.FindOneAndUpdate[int64](ctx, s.coll, filter, update, opt) -} - -func (s *SeqMongo) Malloc(ctx context.Context, conversationID string, size int64) ([]int64, error) { - seq, err := s.MallocSeq(ctx, conversationID, size) + lastSeq, err := mongoutil.FindOneAndUpdate[int64](ctx, s.coll, filter, update, opt) if err != nil { - return nil, err - } - seqs := make([]int64, 0, size) - for i := seq - size + 1; i <= seq; i++ { - seqs = append(seqs, i) + return 0, err } - return seqs, nil + return lastSeq - size, nil } func (s *SeqMongo) GetMaxSeq(ctx context.Context, conversationID string) (int64, error) { diff --git a/pkg/common/storage/database/mgo/version_test.go b/pkg/common/storage/database/mgo/seq_test.go similarity index 58% rename from pkg/common/storage/database/mgo/version_test.go rename to pkg/common/storage/database/mgo/seq_test.go index 236c61a2c..abae2d1b1 100644 --- a/pkg/common/storage/database/mgo/version_test.go +++ b/pkg/common/storage/database/mgo/seq_test.go @@ -2,7 +2,6 @@ package mgo import ( "context" - "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" "go.mongodb.org/mongo-driver/mongo" "go.mongodb.org/mongo-driver/mongo/options" "testing" @@ -24,16 +23,17 @@ func Check(err error) { func TestName(t *testing.T) { cli := Result(mongo.Connect(context.Background(), options.Client().ApplyURI("mongodb://openIM:openIM123@172.16.8.48:37017/openim_v3?maxPoolSize=100").SetConnectTimeout(5*time.Second))) - coll := cli.Database("openim_v3").Collection("version_test") - tmp, err := NewVersionLog(coll) + tmp, err := NewSeqMongo(cli.Database("openim_v3")) if err != nil { panic(err) } - vl := tmp.(*VersionLogMgo) - res, err := vl.writeLogBatch2(context.Background(), "100", []string{"1000", "1001", "1003"}, model.VersionStateInsert, time.Now()) - if err != nil { - t.Log(err) - return + for i := 0; i < 10; i++ { + var size int64 = 100 + firstSeq, err := tmp.Malloc(context.Background(), "1", size) + if err != nil { + t.Log(err) + return + } + t.Logf("%d -> %d", firstSeq, firstSeq+size-1) } - t.Logf("%+v", res) } diff --git a/pkg/common/storage/database/seq.go b/pkg/common/storage/database/seq.go index 869b607a5..20fae3bb1 100644 --- a/pkg/common/storage/database/seq.go +++ b/pkg/common/storage/database/seq.go @@ -3,7 +3,7 @@ package database import "context" type Seq interface { - Malloc(ctx context.Context, conversationID string, size int64) ([]int64, error) + Malloc(ctx context.Context, conversationID string, size int64) (int64, error) GetMaxSeq(ctx context.Context, conversationID string) (int64, error) GetMinSeq(ctx context.Context, conversationID string) (int64, error) SetMinSeq(ctx context.Context, conversationID string, seq int64) error From a41c8c6a9d4cd7969dd9962866a930661834bf65 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Mon, 17 Jun 2024 15:28:09 +0800 Subject: [PATCH 37/59] group version --- go.mod | 2 +- go.sum | 4 +-- internal/api/group.go | 8 +++--- internal/api/router.go | 4 +-- internal/rpc/friend/friend.go | 5 ++++ internal/rpc/group/notification.go | 28 ++++++++++--------- internal/rpc/group/sync.go | 8 +++--- .../storage/database/mgo/group_member.go | 16 +++++++---- .../storage/database/mgo/version_log.go | 4 +++ pkg/common/storage/database/version_log.go | 1 + 10 files changed, 49 insertions(+), 31 deletions(-) diff --git a/go.mod b/go.mod index 4f4cd4b68..29e2fd43f 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/lestrrat-go/file-rotatelogs v2.4.0+incompatible // indirect github.com/mitchellh/mapstructure v1.5.0 - github.com/openimsdk/protocol v0.0.69-alpha.9 + github.com/openimsdk/protocol v0.0.69-alpha.12 github.com/openimsdk/tools v0.0.49-alpha.25 github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 diff --git a/go.sum b/go.sum index 5a72e28f2..a26e59d9e 100644 --- a/go.sum +++ b/go.sum @@ -270,8 +270,8 @@ github.com/onsi/gomega v1.25.0 h1:Vw7br2PCDYijJHSfBOWhov+8cAnUf8MfMaIOV323l6Y= github.com/onsi/gomega v1.25.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdMPGhM= github.com/openimsdk/gomake v0.0.13 h1:xLDe/moqgWpRoptHzI4packAWzs4C16b+sVY+txNJp0= github.com/openimsdk/gomake v0.0.13/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= -github.com/openimsdk/protocol v0.0.69-alpha.9 h1:Mh1upsxwhWs1y65cfIN2XuRsiKEk4MfpYfA9XrvtX24= -github.com/openimsdk/protocol v0.0.69-alpha.9/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= +github.com/openimsdk/protocol v0.0.69-alpha.12 h1:3ZdwmD1y9vcduIC8o2EZS8Ds/fByqcuEFo+NkcBzgRo= +github.com/openimsdk/protocol v0.0.69-alpha.12/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= github.com/openimsdk/tools v0.0.49-alpha.25 h1:OpRPwDZ2xWX7Zj5kyfZhryu/NfZTrsRVr2GFwu1HQHI= github.com/openimsdk/tools v0.0.49-alpha.25/go.mod h1:rwsFI1G/nBHNfiNapbven41akRDPBbH4df0Cgy6xueU= github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= diff --git a/internal/api/group.go b/internal/api/group.go index 0bf61c787..91992004c 100644 --- a/internal/api/group.go +++ b/internal/api/group.go @@ -187,10 +187,10 @@ func (o *GroupApi) GetIncrementalGroupMemberBatch(c *gin.Context) { apiresp.GinSuccess(c, resp) } -func (o *GroupApi) GetIncrementalGroupMemberUserIDs(c *gin.Context) { - a2r.Call(group.GroupClient.GetIncrementalGroupMemberUserIDs, o.Client, c) +func (o *GroupApi) GetFullGroupMemberUserIDs(c *gin.Context) { + a2r.Call(group.GroupClient.GetFullGroupMemberUserIDs, o.Client, c) } -func (o *GroupApi) GetIncrementalJoinGroupIDs(c *gin.Context) { - a2r.Call(group.GroupClient.GetIncrementalJoinGroupIDs, o.Client, c) +func (o *GroupApi) GetFullJoinGroupIDs(c *gin.Context) { + a2r.Call(group.GroupClient.GetFullJoinGroupIDs, o.Client, c) } diff --git a/internal/api/router.go b/internal/api/router.go index 69c026ffd..d2fc9266f 100644 --- a/internal/api/router.go +++ b/internal/api/router.go @@ -120,8 +120,8 @@ func newGinRouter(disCov discovery.SvcDiscoveryRegistry, config *Config) *gin.En groupRouterGroup.POST("/get_incremental_join_group", g.GetIncrementalJoinGroup) groupRouterGroup.POST("/get_incremental_group_member", g.GetIncrementalGroupMember) groupRouterGroup.POST("/get_incremental_group_member_batch", g.GetIncrementalGroupMemberBatch) - groupRouterGroup.POST("/get_incremental_group_member_user_ids", g.GetIncrementalGroupMemberUserIDs) - groupRouterGroup.POST("/get_incremental_join_group_ids", g.GetIncrementalJoinGroupIDs) + groupRouterGroup.POST("/get_incremental_group_member_user_ids", g.GetFullGroupMemberUserIDs) + groupRouterGroup.POST("/get_incremental_join_group_ids", g.GetFullJoinGroupIDs) } // certificate authRouterGroup := r.Group("/auth") diff --git a/internal/rpc/friend/friend.go b/internal/rpc/friend/friend.go index 12107125c..3d8970515 100644 --- a/internal/rpc/friend/friend.go +++ b/internal/rpc/friend/friend.go @@ -51,6 +51,11 @@ type friendServer struct { webhookClient *webhook.Client } +func (s *friendServer) GetFullFriendUserIDs(ctx context.Context, req *relation.GetFullFriendUserIDsReq) (*relation.GetFullFriendUserIDsResp, error) { + //TODO implement me + panic("implement me") +} + type Config struct { RpcConfig config.Friend RedisConfig config.Redis diff --git a/internal/rpc/group/notification.go b/internal/rpc/group/notification.go index 13e593c71..4fae1ed2b 100644 --- a/internal/rpc/group/notification.go +++ b/internal/rpc/group/notification.go @@ -289,10 +289,12 @@ func (g *GroupNotificationSender) fillOpUser(ctx context.Context, opUser **sdkws return nil } -func (g *GroupNotificationSender) setVersion(ctx context.Context, version *uint64, collName string, id string) { - for _, coll := range versionctx.GetVersionLog(ctx).Get() { +func (g *GroupNotificationSender) setVersion(ctx context.Context, version *uint64, versionID *string, collName string, id string) { + versions := versionctx.GetVersionLog(ctx).Get() + for _, coll := range versions { if coll.Name == collName && coll.Doc.DID == id { *version = uint64(coll.Doc.Version) + *versionID = coll.Doc.ID.Hex() return } } @@ -308,7 +310,7 @@ func (g *GroupNotificationSender) GroupCreatedNotification(ctx context.Context, if err = g.fillOpUser(ctx, &tips.OpUser, tips.Group.GroupID); err != nil { return } - g.setVersion(ctx, &tips.GroupMemberVersion, database.GroupMemberVersionName, tips.Group.GroupID) + g.setVersion(ctx, &tips.GroupMemberVersion, &tips.GroupMemberVersionID, database.GroupMemberVersionName, tips.Group.GroupID) g.Notification(ctx, mcontext.GetOpUserID(ctx), tips.Group.GroupID, constant.GroupCreatedNotification, tips) } @@ -392,7 +394,7 @@ func (g *GroupNotificationSender) MemberQuitNotification(ctx context.Context, me return } tips := &sdkws.MemberQuitTips{Group: group, QuitUser: member} - g.setVersion(ctx, &tips.GroupMemberVersion, database.GroupMemberVersionName, member.GroupID) + g.setVersion(ctx, &tips.GroupMemberVersion, &tips.GroupMemberVersionID, database.GroupMemberVersionName, member.GroupID) g.Notification(ctx, mcontext.GetOpUserID(ctx), member.GroupID, constant.MemberQuitNotification, tips) } @@ -480,7 +482,7 @@ func (g *GroupNotificationSender) GroupOwnerTransferredNotification(ctx context. if err = g.fillOpUser(ctx, &tips.OpUser, tips.Group.GroupID); err != nil { return } - g.setVersion(ctx, &tips.GroupMemberVersion, database.GroupMemberVersionName, req.GroupID) + g.setVersion(ctx, &tips.GroupMemberVersion, &tips.GroupMemberVersionID, database.GroupMemberVersionName, req.GroupID) g.Notification(ctx, mcontext.GetOpUserID(ctx), group.GroupID, constant.GroupOwnerTransferredNotification, tips) } @@ -494,7 +496,7 @@ func (g *GroupNotificationSender) MemberKickedNotification(ctx context.Context, if err = g.fillOpUser(ctx, &tips.OpUser, tips.Group.GroupID); err != nil { return } - g.setVersion(ctx, &tips.GroupMemberVersion, database.GroupMemberVersionName, tips.Group.GroupID) + g.setVersion(ctx, &tips.GroupMemberVersion, &tips.GroupMemberVersionID, database.GroupMemberVersionName, tips.Group.GroupID) g.Notification(ctx, mcontext.GetOpUserID(ctx), tips.Group.GroupID, constant.MemberKickedNotification, tips) } @@ -518,7 +520,7 @@ func (g *GroupNotificationSender) MemberInvitedNotification(ctx context.Context, } tips := &sdkws.MemberInvitedTips{Group: group, InvitedUserList: users} err = g.fillOpUser(ctx, &tips.OpUser, tips.Group.GroupID) - g.setVersion(ctx, &tips.GroupMemberVersion, database.GroupMemberVersionName, tips.Group.GroupID) + g.setVersion(ctx, &tips.GroupMemberVersion, &tips.GroupMemberVersionID, database.GroupMemberVersionName, tips.Group.GroupID) g.Notification(ctx, mcontext.GetOpUserID(ctx), group.GroupID, constant.MemberInvitedNotification, tips) } @@ -540,7 +542,7 @@ func (g *GroupNotificationSender) MemberEnterNotification(ctx context.Context, g return } tips := &sdkws.MemberEnterTips{Group: group, EntrantUser: user} - g.setVersion(ctx, &tips.GroupMemberVersion, database.GroupMemberVersionName, tips.Group.GroupID) + g.setVersion(ctx, &tips.GroupMemberVersion, &tips.GroupMemberVersionID, database.GroupMemberVersionName, tips.Group.GroupID) g.Notification(ctx, mcontext.GetOpUserID(ctx), group.GroupID, constant.MemberEnterNotification, tips) } @@ -581,7 +583,7 @@ func (g *GroupNotificationSender) GroupMemberMutedNotification(ctx context.Conte if err = g.fillOpUser(ctx, &tips.OpUser, tips.Group.GroupID); err != nil { return } - g.setVersion(ctx, &tips.GroupMemberVersion, database.GroupMemberVersionName, tips.Group.GroupID) + g.setVersion(ctx, &tips.GroupMemberVersion, &tips.GroupMemberVersionID, database.GroupMemberVersionName, tips.Group.GroupID) g.Notification(ctx, mcontext.GetOpUserID(ctx), group.GroupID, constant.GroupMemberMutedNotification, tips) } @@ -606,7 +608,7 @@ func (g *GroupNotificationSender) GroupMemberCancelMutedNotification(ctx context if err = g.fillOpUser(ctx, &tips.OpUser, tips.Group.GroupID); err != nil { return } - g.setVersion(ctx, &tips.GroupMemberVersion, database.GroupMemberVersionName, tips.Group.GroupID) + g.setVersion(ctx, &tips.GroupMemberVersion, &tips.GroupMemberVersionID, database.GroupMemberVersionName, tips.Group.GroupID) g.Notification(ctx, mcontext.GetOpUserID(ctx), group.GroupID, constant.GroupMemberCancelMutedNotification, tips) } @@ -685,7 +687,7 @@ func (g *GroupNotificationSender) GroupMemberInfoSetNotification(ctx context.Con if err = g.fillOpUser(ctx, &tips.OpUser, tips.Group.GroupID); err != nil { return } - g.setVersion(ctx, &tips.GroupMemberVersion, database.GroupMemberVersionName, tips.Group.GroupID) + g.setVersion(ctx, &tips.GroupMemberVersion, &tips.GroupMemberVersionID, database.GroupMemberVersionName, tips.Group.GroupID) g.Notification(ctx, mcontext.GetOpUserID(ctx), group.GroupID, constant.GroupMemberInfoSetNotification, tips) } @@ -709,7 +711,7 @@ func (g *GroupNotificationSender) GroupMemberSetToAdminNotification(ctx context. if err = g.fillOpUser(ctx, &tips.OpUser, tips.Group.GroupID); err != nil { return } - g.setVersion(ctx, &tips.GroupMemberVersion, database.GroupMemberVersionName, tips.Group.GroupID) + g.setVersion(ctx, &tips.GroupMemberVersion, &tips.GroupMemberVersionID, database.GroupMemberVersionName, tips.Group.GroupID) g.Notification(ctx, mcontext.GetOpUserID(ctx), group.GroupID, constant.GroupMemberSetToAdminNotification, tips) } @@ -734,6 +736,6 @@ func (g *GroupNotificationSender) GroupMemberSetToOrdinaryUserNotification(ctx c if err = g.fillOpUser(ctx, &tips.OpUser, tips.Group.GroupID); err != nil { return } - g.setVersion(ctx, &tips.GroupMemberVersion, database.GroupMemberVersionName, tips.Group.GroupID) + g.setVersion(ctx, &tips.GroupMemberVersion, &tips.GroupMemberVersionID, database.GroupMemberVersionName, tips.Group.GroupID) g.Notification(ctx, mcontext.GetOpUserID(ctx), group.GroupID, constant.GroupMemberSetToOrdinaryUserNotification, tips) } diff --git a/internal/rpc/group/sync.go b/internal/rpc/group/sync.go index 4bd34eb4d..d435530bd 100644 --- a/internal/rpc/group/sync.go +++ b/internal/rpc/group/sync.go @@ -21,7 +21,7 @@ func (s *groupServer) idHash(ids []string) uint64 { return binary.BigEndian.Uint64(sum[:]) } -func (s *groupServer) GetIncrementalGroupMemberUserIDs(ctx context.Context, req *pbgroup.GetIncrementalGroupMemberUserIDsReq) (*pbgroup.GetIncrementalGroupMemberUserIDsResp, error) { +func (s *groupServer) GetFullGroupMemberUserIDs(ctx context.Context, req *pbgroup.GetFullGroupMemberUserIDsReq) (*pbgroup.GetFullGroupMemberUserIDsResp, error) { vl, err := s.db.FindMaxGroupMemberVersionCache(ctx, req.GroupID) if err != nil { return nil, err @@ -34,7 +34,7 @@ func (s *groupServer) GetIncrementalGroupMemberUserIDs(ctx context.Context, req if req.IdHash == idHash { userIDs = nil } - return &pbgroup.GetIncrementalGroupMemberUserIDsResp{ + return &pbgroup.GetFullGroupMemberUserIDsResp{ Version: idHash, VersionID: vl.ID.Hex(), Equal: req.IdHash == idHash, @@ -42,7 +42,7 @@ func (s *groupServer) GetIncrementalGroupMemberUserIDs(ctx context.Context, req }, nil } -func (s *groupServer) GetIncrementalJoinGroupIDs(ctx context.Context, req *pbgroup.GetIncrementalJoinGroupIDsReq) (*pbgroup.GetIncrementalJoinGroupIDsResp, error) { +func (s *groupServer) GetFullJoinGroupIDs(ctx context.Context, req *pbgroup.GetFullJoinGroupIDsReq) (*pbgroup.GetFullJoinGroupIDsResp, error) { vl, err := s.db.FindMaxJoinGroupVersionCache(ctx, req.UserID) if err != nil { return nil, err @@ -55,7 +55,7 @@ func (s *groupServer) GetIncrementalJoinGroupIDs(ctx context.Context, req *pbgro if req.IdHash == idHash { groupIDs = nil } - return &pbgroup.GetIncrementalJoinGroupIDsResp{ + return &pbgroup.GetFullJoinGroupIDsResp{ Version: idHash, VersionID: vl.ID.Hex(), Equal: req.IdHash == idHash, diff --git a/pkg/common/storage/database/mgo/group_member.go b/pkg/common/storage/database/mgo/group_member.go index a9ac262ab..95c6e51a6 100644 --- a/pkg/common/storage/database/mgo/group_member.go +++ b/pkg/common/storage/database/mgo/group_member.go @@ -96,13 +96,19 @@ func (g *GroupMemberMgo) Delete(ctx context.Context, groupID string, userIDs []s } return mongoutil.IncrVersion(func() error { return mongoutil.DeleteMany(ctx, g.coll, filter) - }, func() error { - return g.member.IncrVersion(ctx, groupID, userIDs, model.VersionStateDelete) }, func() error { if len(userIDs) == 0 { - return nil + return g.member.Delete(ctx, groupID) + } else { + return g.member.IncrVersion(ctx, groupID, userIDs, model.VersionStateDelete) + } + }, func() error { + for _, userID := range userIDs { + if err := g.join.IncrVersion(ctx, userID, []string{groupID}, model.VersionStateDelete); err != nil { + return err + } } - return g.member.IncrVersion(ctx, groupID, userIDs, model.VersionStateDelete) + return nil }) } @@ -112,7 +118,7 @@ func (g *GroupMemberMgo) UpdateRoleLevel(ctx context.Context, groupID string, us }, func() error { return g.member.IncrVersion(ctx, groupID, []string{userID}, model.VersionStateUpdate) }, func() error { - return g.join.IncrVersion(ctx, groupID, []string{userID}, model.VersionStateUpdate) + return g.join.IncrVersion(ctx, userID, []string{groupID}, model.VersionStateUpdate) }) } diff --git a/pkg/common/storage/database/mgo/version_log.go b/pkg/common/storage/database/mgo/version_log.go index b53057fe4..1d70f96f5 100644 --- a/pkg/common/storage/database/mgo/version_log.go +++ b/pkg/common/storage/database/mgo/version_log.go @@ -256,3 +256,7 @@ func (l *VersionLogMgo) DeleteAfterUnchangedLog(ctx context.Context, deadline ti }, }) } + +func (l *VersionLogMgo) Delete(ctx context.Context, dId string) error { + return mongoutil.DeleteOne(ctx, l.coll, bson.M{"d_id": dId}) +} diff --git a/pkg/common/storage/database/version_log.go b/pkg/common/storage/database/version_log.go index c9dc09540..9d7bcc172 100644 --- a/pkg/common/storage/database/version_log.go +++ b/pkg/common/storage/database/version_log.go @@ -15,4 +15,5 @@ type VersionLog interface { IncrVersion(ctx context.Context, dId string, eIds []string, state int32) error FindChangeLog(ctx context.Context, dId string, version uint, limit int) (*model.VersionLog, error) DeleteAfterUnchangedLog(ctx context.Context, deadline time.Time) error + Delete(ctx context.Context, dId string) error } From bd4fb8a1a0e1ec5cbd32b611174d73b9e8db57ab Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Mon, 17 Jun 2024 15:37:13 +0800 Subject: [PATCH 38/59] merge --- internal/rpc/friend/friend.go | 5 ----- 1 file changed, 5 deletions(-) diff --git a/internal/rpc/friend/friend.go b/internal/rpc/friend/friend.go index 619f8743b..622e19f42 100644 --- a/internal/rpc/friend/friend.go +++ b/internal/rpc/friend/friend.go @@ -51,11 +51,6 @@ type friendServer struct { webhookClient *webhook.Client } -func (s *friendServer) GetFullFriendUserIDs(ctx context.Context, req *relation.GetFullFriendUserIDsReq) (*relation.GetFullFriendUserIDsResp, error) { - //TODO implement me - panic("implement me") -} - type Config struct { RpcConfig config.Friend RedisConfig config.Redis From d55d416f6867fd62645895a771ceeb4222e8e3bc Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Mon, 17 Jun 2024 17:41:28 +0800 Subject: [PATCH 39/59] seq --- pkg/common/storage/cache/redis/seq.md | 36 ++++++++++++ pkg/common/storage/cache/redis/seq1.go | 9 +++ pkg/common/storage/cache/redis/seq_test.go | 64 +++++++++++++++++++--- pkg/common/storage/controller/msg.go | 1 - 4 files changed, 102 insertions(+), 8 deletions(-) create mode 100644 pkg/common/storage/cache/redis/seq.md diff --git a/pkg/common/storage/cache/redis/seq.md b/pkg/common/storage/cache/redis/seq.md new file mode 100644 index 000000000..9aa66a326 --- /dev/null +++ b/pkg/common/storage/cache/redis/seq.md @@ -0,0 +1,36 @@ + +### mongo +```go +type Seq struct { + ConversationID string `bson:"conversation_id"` + MaxSeq int64 `bson:"max_seq"` + MinSeq int64 `bson:"min_seq"` +} +``` + +```go +type Seq interface { + Malloc(ctx context.Context, conversationID string, size int64) (int64, error) + GetMaxSeq(ctx context.Context, conversationID string) (int64, error) + GetMinSeq(ctx context.Context, conversationID string) (int64, error) + SetMinSeq(ctx context.Context, conversationID string, seq int64) error +} +``` + +1. Malloc 申请seq数量，返回的已有seq的最大值，消息用的第一个seq是返回值+1 +2. GetMaxSeq 获取申请的seq的最大值，在发消息的seq小于这个值 +3. GetMinSeq 获取最小的seq，用于拉取历史消息 +4. SetMinSeq 设置最小的seq，用于拉取历史消息 + +### redis +```go +type RedisSeq struct { + Curr int64 // 当前的最大seq + Last int64 // mongodb中申请的最大seq + Lock *int64 // 锁，用于在mongodb中申请seq +} +``` + +1. Malloc 申请seq数量，返回的已有seq的最大值，消息用的第一个seq是返回值+1，如果redis中申请数量够用，直接返回，并自增对应数量。如果redis中申请数量不够用，加锁，从mongodb中申请seq。 +2. GetMaxSeq 获取已发消息的最大seq就是Curr的值。如果redis中缓存不存在就通过mongodb获取最大seq。存储在redis中。其中Curr和Last都是这个seq值。 +3. GetMinSeq, SetMinSeq用之前rockscache的方案。 \ No newline at end of file diff --git a/pkg/common/storage/cache/redis/seq1.go b/pkg/common/storage/cache/redis/seq1.go index 6374f53cc..316dea487 100644 --- a/pkg/common/storage/cache/redis/seq1.go +++ b/pkg/common/storage/cache/redis/seq1.go @@ -47,6 +47,12 @@ func NewTestSeq() *SeqMalloc { } } +type RedisSeq struct { + Curr int64 + Last int64 + Lock *int64 +} + type SeqMalloc struct { rdb redis.UniversalClient mgo database.Seq @@ -118,6 +124,7 @@ end local curr_seq = tonumber(redis.call("HGET", key, "CURR")) local last_seq = tonumber(redis.call("HGET", key, "LAST")) if size == 0 then + redis.call("EXPIRE", key, dataSecond) table.insert(result, 0) table.insert(result, curr_seq) table.insert(result, last_seq) @@ -136,6 +143,7 @@ if max_seq > last_seq then return result end redis.call("HSET", key, "CURR", max_seq) +redis.call("EXPIRE", key, dataSecond) table.insert(result, 0) table.insert(result, curr_seq) table.insert(result, last_seq) @@ -219,6 +227,7 @@ func (s *SeqMalloc) Malloc(ctx context.Context, conversationID string, size int6 s.setSeqRetry(ctx, key, states[1], seq+size, seq+mallocSize) return seq, nil case 2: // locked + fmt.Println("locked----->", "conversationID", conversationID, "size", size) if err := s.wait(ctx); err != nil { return 0, err } diff --git a/pkg/common/storage/cache/redis/seq_test.go b/pkg/common/storage/cache/redis/seq_test.go index 1c11bad0b..6fff6019a 100644 --- a/pkg/common/storage/cache/redis/seq_test.go +++ b/pkg/common/storage/cache/redis/seq_test.go @@ -2,22 +2,72 @@ package redis import ( "context" + "strconv" + "sync" + "sync/atomic" "testing" "time" ) func TestSeq(t *testing.T) { ts := NewTestSeq() - for i := 1; i < 1000000; i++ { - var size int64 = 100 - first, err := ts.Malloc(context.Background(), "1", size) - if err != nil { - t.Logf("[%d] %s", i, err) + var ( + wg sync.WaitGroup + speed atomic.Int64 + ) + + const count = 256 + wg.Add(count) + for i := 0; i < count; i++ { + index := i + 1 + go func() { + defer wg.Done() + var size int64 = 1 + cID := strconv.Itoa(index * 100) + for i := 1; ; i++ { + first, err := ts.mgo.Malloc(context.Background(), cID, size) // mongo + //first, err := ts.Malloc(context.Background(), cID, size) // redis + if err != nil { + t.Logf("[%d-%d] %s %s", index, i, cID, err) + return + } + speed.Add(size) + _ = first + //t.Logf("[%d] %d -> %d", i, first+1, first+size) + } + }() + } + + done := make(chan struct{}) + + go func() { + wg.Wait() + close(done) + }() + + ticker := time.NewTicker(time.Second) + + for { + select { + case <-done: + ticker.Stop() return + case <-ticker.C: + value := speed.Swap(0) + t.Logf("speed: %d/s", value) } - t.Logf("[%d] %d -> %d", i, first+1, first+size) - time.Sleep(time.Second / 4) } + + //for i := 1; i < 1000000; i++ { + // var size int64 = 100 + // first, err := ts.Malloc(context.Background(), "1", size) + // if err != nil { + // t.Logf("[%d] %s", i, err) + // return + // } + // t.Logf("[%d] %d -> %d", i, first+1, first+size) + // time.Sleep(time.Second / 4) + //} } func TestDel(t *testing.T) { diff --git a/pkg/common/storage/controller/msg.go b/pkg/common/storage/controller/msg.go index f0efe5efd..c5c5462aa 100644 --- a/pkg/common/storage/controller/msg.go +++ b/pkg/common/storage/controller/msg.go @@ -349,7 +349,6 @@ func (db *commonMsgDatabase) DeleteMessagesFromCache(ctx context.Context, conver } func (db *commonMsgDatabase) BatchInsertChat2Cache(ctx context.Context, conversationID string, msgs []*sdkws.MsgData) (seq int64, isNew bool, err error) { - // TODO set SEQ currentMaxSeq, err := db.seq.GetMaxSeq(ctx, conversationID) if err != nil && errs.Unwrap(err) != redis.Nil { From a2a28b43c508225bbd6fb1b593c1f6584f491a9a Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Tue, 18 Jun 2024 16:46:06 +0800 Subject: [PATCH 40/59] seq --- internal/msgtransfer/init.go | 7 +- internal/rpc/msg/server.go | 7 +- pkg/common/storage/cache/cachekey/seq.go | 16 +- pkg/common/storage/cache/cachekey/seq1.go | 5 - pkg/common/storage/cache/redis/seq.md | 36 --- pkg/common/storage/cache/redis/seq1.go.txt | 246 ------------------ .../redis/{seq1.go => seq_conversation.go} | 109 ++++---- .../cache/redis/seq_conversation_test.go | 109 ++++++++ pkg/common/storage/cache/redis/seq_test.go | 77 ------ pkg/common/storage/cache/seq.go | 14 +- pkg/common/storage/cache/seq_conversation.go | 10 + pkg/common/storage/controller/msg.go | 126 +++------ .../mgo/{seq.go => seq_conversation.go} | 43 ++- .../{seq_test.go => seq_conversation_test.go} | 0 pkg/common/storage/database/name.go | 2 +- pkg/common/storage/database/seq.go | 2 +- pkg/common/storage/model/seq.go | 2 +- tools/seq/internal/main.go | 152 +++++++++++ tools/seq/main.go | 16 ++ 19 files changed, 443 insertions(+), 536 deletions(-) delete mode 100644 pkg/common/storage/cache/redis/seq.md delete mode 100644 pkg/common/storage/cache/redis/seq1.go.txt rename pkg/common/storage/cache/redis/{seq1.go => seq_conversation.go} (68%) create mode 100644 pkg/common/storage/cache/redis/seq_conversation_test.go delete mode 100644 pkg/common/storage/cache/redis/seq_test.go create mode 100644 pkg/common/storage/cache/seq_conversation.go rename pkg/common/storage/database/mgo/{seq.go => seq_conversation.go} (50%) rename pkg/common/storage/database/mgo/{seq_test.go => seq_conversation_test.go} (100%) create mode 100644 tools/seq/internal/main.go create mode 100644 tools/seq/main.go diff --git a/internal/msgtransfer/init.go b/internal/msgtransfer/init.go index 65d04f381..e90c2288c 100644 --- a/internal/msgtransfer/init.go +++ b/internal/msgtransfer/init.go @@ -87,7 +87,12 @@ func Start(ctx context.Context, index int, config *Config) error { if err != nil { return err } - msgDatabase, err := controller.NewCommonMsgDatabase(msgDocModel, msgModel, seqModel, &config.KafkaConfig) + seqConversation, err := mgo.NewSeqConversationMongo(mgocli.GetDB()) + if err != nil { + return err + } + seqConversationCache := redis.NewSeqConversationCacheRedis(rdb, seqConversation) + msgDatabase, err := controller.NewCommonMsgDatabase(msgDocModel, msgModel, seqModel, seqConversationCache, &config.KafkaConfig) if err != nil { return err } diff --git a/internal/rpc/msg/server.go b/internal/rpc/msg/server.go index f1fb28fff..7cffb23a9 100644 --- a/internal/rpc/msg/server.go +++ b/internal/rpc/msg/server.go @@ -91,7 +91,12 @@ func Start(ctx context.Context, config *Config, client discovery.SvcDiscoveryReg userRpcClient := rpcclient.NewUserRpcClient(client, config.Share.RpcRegisterName.User, config.Share.IMAdminUserID) groupRpcClient := rpcclient.NewGroupRpcClient(client, config.Share.RpcRegisterName.Group) friendRpcClient := rpcclient.NewFriendRpcClient(client, config.Share.RpcRegisterName.Friend) - msgDatabase, err := controller.NewCommonMsgDatabase(msgDocModel, msgModel, seqModel, &config.KafkaConfig) + seqConversation, err := mgo.NewSeqConversationMongo(mgocli.GetDB()) + if err != nil { + return err + } + seqConversationCache := redis.NewSeqConversationCacheRedis(rdb, seqConversation) + msgDatabase, err := controller.NewCommonMsgDatabase(msgDocModel, msgModel, seqModel, seqConversationCache, &config.KafkaConfig) if err != nil { return err } diff --git a/pkg/common/storage/cache/cachekey/seq.go b/pkg/common/storage/cache/cachekey/seq.go index 3f0ce98a4..ded0286d8 100644 --- a/pkg/common/storage/cache/cachekey/seq.go +++ b/pkg/common/storage/cache/cachekey/seq.go @@ -15,24 +15,24 @@ package cachekey const ( - maxSeq = "MAX_SEQ:" - minSeq = "MIN_SEQ:" - conversationUserMinSeq = "CON_USER_MIN_SEQ:" - hasReadSeq = "HAS_READ_SEQ:" + MaxSeq = "MAX_SEQ:" + MinSeq = "MIN_SEQ:" + ConversationUserMinSeq = "CON_USER_MIN_SEQ:" + HasReadSeq = "HAS_READ_SEQ:" ) func GetMaxSeqKey(conversationID string) string { - return maxSeq + conversationID + return MaxSeq + conversationID } func GetMinSeqKey(conversationID string) string { - return minSeq + conversationID + return MinSeq + conversationID } func GetHasReadSeqKey(conversationID string, userID string) string { - return hasReadSeq + userID + ":" + conversationID + return HasReadSeq + userID + ":" + conversationID } func GetConversationUserMinSeqKey(conversationID, userID string) string { - return conversationUserMinSeq + conversationID + "u:" + userID + return ConversationUserMinSeq + conversationID + "u:" + userID } diff --git a/pkg/common/storage/cache/cachekey/seq1.go b/pkg/common/storage/cache/cachekey/seq1.go index df3086d32..04274db55 100644 --- a/pkg/common/storage/cache/cachekey/seq1.go +++ b/pkg/common/storage/cache/cachekey/seq1.go @@ -2,7 +2,6 @@ package cachekey const ( MallocSeq = "MALLOC_SEQ:" - MallocSeqLock = "MALLOC_SEQ_LOCK:" MallocMinSeqLock = "MALLOC_MIN_SEQ:" ) @@ -10,10 +9,6 @@ func GetMallocSeqKey(conversationID string) string { return MallocSeq + conversationID } -func GetMallocSeqLockKey(conversationID string) string { - return MallocSeqLock + conversationID -} - func GetMallocMinSeqKey(conversationID string) string { return MallocMinSeqLock + conversationID } diff --git a/pkg/common/storage/cache/redis/seq.md b/pkg/common/storage/cache/redis/seq.md deleted file mode 100644 index 9aa66a326..000000000 --- a/pkg/common/storage/cache/redis/seq.md +++ /dev/null @@ -1,36 +0,0 @@ - -### mongo -```go -type Seq struct { - ConversationID string `bson:"conversation_id"` - MaxSeq int64 `bson:"max_seq"` - MinSeq int64 `bson:"min_seq"` -} -``` - -```go -type Seq interface { - Malloc(ctx context.Context, conversationID string, size int64) (int64, error) - GetMaxSeq(ctx context.Context, conversationID string) (int64, error) - GetMinSeq(ctx context.Context, conversationID string) (int64, error) - SetMinSeq(ctx context.Context, conversationID string, seq int64) error -} -``` - -1. Malloc 申请seq数量，返回的已有seq的最大值，消息用的第一个seq是返回值+1 -2. GetMaxSeq 获取申请的seq的最大值，在发消息的seq小于这个值 -3. GetMinSeq 获取最小的seq，用于拉取历史消息 -4. SetMinSeq 设置最小的seq，用于拉取历史消息 - -### redis -```go -type RedisSeq struct { - Curr int64 // 当前的最大seq - Last int64 // mongodb中申请的最大seq - Lock *int64 // 锁，用于在mongodb中申请seq -} -``` - -1. Malloc 申请seq数量，返回的已有seq的最大值，消息用的第一个seq是返回值+1，如果redis中申请数量够用，直接返回，并自增对应数量。如果redis中申请数量不够用，加锁，从mongodb中申请seq。 -2. GetMaxSeq 获取已发消息的最大seq就是Curr的值。如果redis中缓存不存在就通过mongodb获取最大seq。存储在redis中。其中Curr和Last都是这个seq值。 -3. GetMinSeq, SetMinSeq用之前rockscache的方案。 \ No newline at end of file diff --git a/pkg/common/storage/cache/redis/seq1.go.txt b/pkg/common/storage/cache/redis/seq1.go.txt deleted file mode 100644 index 9ccb83e66..000000000 --- a/pkg/common/storage/cache/redis/seq1.go.txt +++ /dev/null @@ -1,246 +0,0 @@ -package redis - -import ( - "context" - "errors" - "fmt" - "github.com/dtm-labs/rockscache" - "github.com/google/uuid" - "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/cachekey" - "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" - "github.com/openimsdk/open-im-server/v3/pkg/msgprocessor" - "github.com/openimsdk/tools/errs" - "github.com/redis/go-redis/v9" - "strconv" - "time" -) - -var errLock = errors.New("lock failed") - -type MallocSeq interface { - Malloc(ctx context.Context, conversationID string, size int64) ([]int64, error) - GetMaxSeq(ctx context.Context, conversationID string) (int64, error) - GetMinSeq(ctx context.Context, conversationID string) (int64, error) - SetMinSeq(ctx context.Context, conversationID string, seq int64) error -} - -func NewSeqCache1(rdb redis.UniversalClient, mgo database.Seq) MallocSeq { - opt := rockscache.NewDefaultOptions() - opt.EmptyExpire = time.Second * 3 - opt.Delay = time.Second / 2 - return &seqCache1{ - rdb: rdb, - mgo: mgo, - rocks: rockscache.NewClient(rdb, opt), - lockExpire: time.Minute * 1, - seqExpire: time.Hour * 24 * 7, - minSeqExpire: time.Hour * 1, - groupMinNum: 1000, - userMinNum: 100, - } -} - -type seqCache1 struct { - rdb redis.UniversalClient - rocks *rockscache.Client - mgo database.Seq - lockExpire time.Duration - seqExpire time.Duration - minSeqExpire time.Duration - groupMinNum int64 - userMinNum int64 -} - -/* -1 -2 -3 -4 -5 -6 -7 -8 -9 -10 -*/ - -func (s *seqCache1) GetMaxSeq(ctx context.Context, conversationID string) (int64, error) { - for i := 0; i < 10; i++ { - res, err := s.rdb.LIndex(ctx, cachekey.GetMallocSeqKey(conversationID), 0).Int64() - if err == nil { - return res, nil - } else if !errors.Is(err, redis.Nil) { - return 0, errs.Wrap(err) - } - - if err := s.mallocSeq(ctx, conversationID, 0, nil); err != nil { - return 0, err - } - } - return 0, errs.New("get max seq timeout") -} - -func (s *seqCache1) unlock(ctx context.Context, key string, owner string) error { - script := ` -local value = redis.call("GET", KEYS[1]) -if value == false then - return 0 -end -if value == ARGV[1] then - redis.call("DEL", KEYS[1]) - return 1 -end -return 2 -` - state, err := s.rdb.Eval(ctx, script, []string{key}, owner).Int() - if err != nil { - return errs.Wrap(err) - } - switch state { - case 0: - return errs.Wrap(redis.Nil) - case 1: - return nil - case 2: - return errs.New("not the lock holder") - default: - return errs.New(fmt.Sprintf("unknown state: %d", state)) - } -} - -func (s *seqCache1) initMallocSeq(ctx context.Context, conversationID string, size int64) ([]int64, error) { - owner := uuid.New().String() - ok, err := s.rdb.SetNX(ctx, cachekey.GetMallocSeqLockKey(conversationID), owner, s.lockExpire).Result() - if err != nil { - return nil, err - } - - seq, err := s.mgo.Malloc(ctx, conversationID, size) - if err != nil { - return nil, err - } - seqs := make([]int64, 0, size) - for i := seq - size + 1; i <= seq; i++ { - seqs = append(seqs, i) - } - return seqs, nil -} - -func (s *seqCache1) GetMinSeq(ctx context.Context, conversationID string) (int64, error) { - return getCache[int64](ctx, s.rocks, cachekey.GetMallocMinSeqKey(conversationID), s.minSeqExpire, func(ctx context.Context) (int64, error) { - return s.mgo.GetMinSeq(ctx, conversationID) - }) -} - -func (s *seqCache1) SetMinSeq(ctx context.Context, conversationID string, seq int64) error { - if err := s.mgo.SetMinSeq(ctx, conversationID, seq); err != nil { - return err - } - return s.deleteMinSeqCache(ctx, conversationID) -} - -func (s *seqCache1) Malloc(ctx context.Context, conversationID string, size int64) ([]int64, error) { - if size <= 0 { - return nil, errs.Wrap(errors.New("size must be greater than 0")) - } - seqKey := cachekey.GetMallocSeqKey(conversationID) - lockKey := cachekey.GetMallocSeqLockKey(conversationID) - for i := 0; i < 10; i++ { - seqs, err := s.lpop(ctx, seqKey, lockKey, size) - if err != nil { - return nil, err - } - if len(seqs) < int(size) { - if err := s.mallocSeq(ctx, conversationID, size, &seqs); err != nil { - return nil, err - } - } - if len(seqs) >= int(size) { - return seqs, nil - } - } - return nil, errs.ErrInternalServer.WrapMsg("malloc seq failed") -} - -func (s *seqCache1) push(ctx context.Context, seqKey string, seqs []int64) error { - script := ` -redis.call("DEL", KEYS[1]) -for i = 2, #ARGV do - redis.call("RPUSH", KEYS[1], ARGV[i]) -end -redis.call("EXPIRE", KEYS[1], ARGV[1]) -return 1 -` - argv := make([]any, 0, 1+len(seqs)) - argv = append(argv, s.seqExpire.Seconds()) - for _, seq := range seqs { - argv = append(argv, seq) - } - err := s.rdb.Eval(ctx, script, []string{seqKey}, argv...).Err() - return errs.Wrap(err) -} - -func (s *seqCache1) lpop(ctx context.Context, seqKey, lockKey string, size int64) ([]int64, error) { - script := ` -local result = redis.call("LRANGE", KEYS[1], 0, ARGV[1]-1) -if #result == 0 then - return result -end -redis.call("LTRIM", KEYS[1], #result, -1) -if redis.call("LLEN", KEYS[1]) == 0 then - redis.call("DEL", KEYS[2]) -end -return result -` - res, err := s.rdb.Eval(ctx, script, []string{seqKey, lockKey}, size).Int64Slice() - if err != nil { - return nil, errs.Wrap(err) - } - return res, nil -} - -func (s *seqCache1) getMongoStepSize(conversationID string, size int64) int64 { - var num int64 - if msgprocessor.IsGroupConversationID(conversationID) { - num = s.groupMinNum - } else { - num = s.userMinNum - } - if size > num { - num += size - } - return num -} - -func (s *seqCache1) mallocSeq(ctx context.Context, conversationID string, size int64, seqs *[]int64) error { - var delMinSeqKey bool - _, err := getCache[string](ctx, s.rocks, cachekey.GetMallocSeqLockKey(conversationID), s.lockExpire, func(ctx context.Context) (string, error) { - res, err := s.mgo.Malloc(ctx, conversationID, s.getMongoStepSize(conversationID, size)) - if err != nil { - return "", err - } - delMinSeqKey = res[0] == 1 - if seqs != nil && size > 0 { - if len(*seqs) > 0 && (*seqs)[len(*seqs)-1]+1 == res[0] { - n := size - int64(len(*seqs)) - *seqs = append(*seqs, res[:n]...) - res = res[n:] - } else { - *seqs = res[:size] - res = res[size:] - } - } - if err := s.push(ctx, cachekey.GetMallocSeqKey(conversationID), res); err != nil { - return "", err - } - return strconv.Itoa(int(time.Now().UnixMicro())), nil - }) - if delMinSeqKey { - s.deleteMinSeqCache(ctx, conversationID) - } - return err -} - -func (s *seqCache1) deleteMinSeqCache(ctx context.Context, conversationID string) error { - return s.rocks.TagAsDeleted2(ctx, cachekey.GetMallocMinSeqKey(conversationID)) -} diff --git a/pkg/common/storage/cache/redis/seq1.go b/pkg/common/storage/cache/redis/seq_conversation.go similarity index 68% rename from pkg/common/storage/cache/redis/seq1.go rename to pkg/common/storage/cache/redis/seq_conversation.go index 316dea487..034462fd1 100644 --- a/pkg/common/storage/cache/redis/seq1.go +++ b/pkg/common/storage/cache/redis/seq_conversation.go @@ -3,74 +3,68 @@ package redis import ( "context" "fmt" + "github.com/dtm-labs/rockscache" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/cachekey" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" - "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database/mgo" "github.com/openimsdk/open-im-server/v3/pkg/msgprocessor" "github.com/openimsdk/tools/errs" "github.com/openimsdk/tools/log" "github.com/redis/go-redis/v9" - "go.mongodb.org/mongo-driver/mongo" - "go.mongodb.org/mongo-driver/mongo/options" "time" ) -type RedisHash struct { - NextSeq int64 - LastSeq int64 +func NewSeqConversationCacheRedis(rdb redis.UniversalClient, mgo database.SeqConversation) cache.SeqConversationCache { + return &seqConversationCacheRedis{ + rdb: rdb, + mgo: mgo, + lockTime: time.Second * 3, + dataTime: time.Hour * 24 * 365, + minSeqExpireTime: time.Hour, + rocks: rockscache.NewClient(rdb, *GetRocksCacheOptions()), + } } -func NewTestSeq() *SeqMalloc { - mgocli, err := mongo.Connect(context.Background(), options.Client().ApplyURI("mongodb://openIM:openIM123@172.16.8.48:37017/openim_v3?maxPoolSize=100").SetConnectTimeout(5*time.Second)) - if err != nil { - panic(err) - } - model, err := mgo.NewSeqMongo(mgocli.Database("openim_v3")) - if err != nil { - panic(err) - } - opt := &redis.Options{ - Addr: "172.16.8.48:16379", - Password: "openIM123", - DB: 1, - } - rdb := redis.NewClient(opt) - if err := rdb.Ping(context.Background()).Err(); err != nil { - panic(err) - } - return &SeqMalloc{ - rdb: rdb, - mgo: model, - //lockTime: time.Second * 30, - lockTime: time.Second * 60 * 60 * 24 * 1, - dataTime: time.Second * 60 * 60 * 24 * 7, - } +type seqConversationCacheRedis struct { + rdb redis.UniversalClient + mgo database.SeqConversation + rocks *rockscache.Client + lockTime time.Duration + dataTime time.Duration + minSeqExpireTime time.Duration +} + +func (s *seqConversationCacheRedis) getMinSeqKey(conversationID string) string { + return cachekey.GetMallocMinSeqKey(conversationID) } -type RedisSeq struct { - Curr int64 - Last int64 - Lock *int64 +func (s *seqConversationCacheRedis) SetMinSeq(ctx context.Context, conversationID string, seq int64) error { + if err := s.mgo.SetMinSeq(ctx, conversationID, seq); err != nil { + return err + } + if err := s.rocks.TagAsDeleted2(ctx, s.getMinSeqKey(conversationID)); err != nil { + return errs.Wrap(err) + } + return nil } -type SeqMalloc struct { - rdb redis.UniversalClient - mgo database.Seq - lockTime time.Duration - dataTime time.Duration +func (s *seqConversationCacheRedis) GetMinSeq(ctx context.Context, conversationID string) (int64, error) { + return getCache(ctx, s.rocks, s.getMinSeqKey(conversationID), s.minSeqExpireTime, func(ctx context.Context) (int64, error) { + return s.mgo.GetMinSeq(ctx, conversationID) + }) } -func (s *SeqMalloc) getSeqMallocKey(conversationID string) string { +func (s *seqConversationCacheRedis) getSeqMallocKey(conversationID string) string { return cachekey.GetMallocSeqKey(conversationID) } -func (s *SeqMalloc) setSeq(ctx context.Context, key string, owner int64, currSeq int64, lastSeq int64) (int64, error) { +func (s *seqConversationCacheRedis) setSeq(ctx context.Context, key string, owner int64, currSeq int64, lastSeq int64) (int64, error) { if lastSeq < currSeq { return 0, errs.New("lastSeq must be greater than currSeq") } - // 0： 成功 - // 1： 成功 锁过期，但未被其他人锁 - // 2： 已经被锁，但是锁的不是自己 + // 0: success + // 1: success the lock has expired, but has not been locked by anyone else + // 2: already locked, but not by yourself script := ` local key = KEYS[1] local lockValue = ARGV[1] @@ -97,12 +91,12 @@ return 0 return result, nil } -// malloc size=0为获取当前seq size>0为分配seq -func (s *SeqMalloc) malloc(ctx context.Context, key string, size int64) ([]int64, error) { - // 0： 成功 - // 1： 需要获取，并加锁 - // 2： 已经被锁 - // 3： 超过最大值，并加锁 +// malloc size=0 is to get the current seq size>0 is to allocate seq +func (s *seqConversationCacheRedis) malloc(ctx context.Context, key string, size int64) ([]int64, error) { + // 0: success + // 1: need to obtain and lock + // 2: already locked + // 3: exceeded the maximum value and locked script := ` local key = KEYS[1] local size = tonumber(ARGV[1]) @@ -156,7 +150,7 @@ return result return result, nil } -func (s *SeqMalloc) wait(ctx context.Context) error { +func (s *seqConversationCacheRedis) wait(ctx context.Context) error { timer := time.NewTimer(time.Second / 4) defer timer.Stop() select { @@ -167,7 +161,7 @@ func (s *SeqMalloc) wait(ctx context.Context) error { } } -func (s *SeqMalloc) setSeqRetry(ctx context.Context, key string, owner int64, currSeq int64, lastSeq int64) { +func (s *seqConversationCacheRedis) setSeqRetry(ctx context.Context, key string, owner int64, currSeq int64, lastSeq int64) { for i := 0; i < 10; i++ { state, err := s.setSeq(ctx, key, owner, currSeq, lastSeq) if err != nil { @@ -191,13 +185,13 @@ func (s *SeqMalloc) setSeqRetry(ctx context.Context, key string, owner int64, cu log.ZError(ctx, "set seq cache retrying still failed", nil, "key", key, "owner", owner, "currSeq", currSeq, "lastSeq", lastSeq) } -func (s *SeqMalloc) getMallocSize(conversationID string, size int64) int64 { +func (s *seqConversationCacheRedis) getMallocSize(conversationID string, size int64) int64 { if size == 0 { return 0 } var basicSize int64 if msgprocessor.IsGroupConversationID(conversationID) { - basicSize = 200 + basicSize = 100 } else { basicSize = 50 } @@ -205,7 +199,7 @@ func (s *SeqMalloc) getMallocSize(conversationID string, size int64) int64 { return basicSize } -func (s *SeqMalloc) Malloc(ctx context.Context, conversationID string, size int64) (int64, error) { +func (s *seqConversationCacheRedis) Malloc(ctx context.Context, conversationID string, size int64) (int64, error) { if size < 0 { return 0, errs.New("size must be greater than 0") } @@ -227,7 +221,6 @@ func (s *SeqMalloc) Malloc(ctx context.Context, conversationID string, size int6 s.setSeqRetry(ctx, key, states[1], seq+size, seq+mallocSize) return seq, nil case 2: // locked - fmt.Println("locked----->", "conversationID", conversationID, "size", size) if err := s.wait(ctx); err != nil { return 0, err } @@ -257,6 +250,6 @@ func (s *SeqMalloc) Malloc(ctx context.Context, conversationID string, size int6 return 0, errs.New("malloc seq waiting for lock timeout", "conversationID", conversationID, "size", size) } -func (s *SeqMalloc) GetMaxSeq(ctx context.Context, conversationID string) (int64, error) { +func (s *seqConversationCacheRedis) GetMaxSeq(ctx context.Context, conversationID string) (int64, error) { return s.Malloc(ctx, conversationID, 0) } diff --git a/pkg/common/storage/cache/redis/seq_conversation_test.go b/pkg/common/storage/cache/redis/seq_conversation_test.go new file mode 100644 index 000000000..1a40624b8 --- /dev/null +++ b/pkg/common/storage/cache/redis/seq_conversation_test.go @@ -0,0 +1,109 @@ +package redis + +import ( + "context" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database/mgo" + "github.com/redis/go-redis/v9" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" + "strconv" + "sync" + "sync/atomic" + "testing" + "time" +) + +func newTestSeq() *seqConversationCacheRedis { + mgocli, err := mongo.Connect(context.Background(), options.Client().ApplyURI("mongodb://openIM:openIM123@172.16.8.48:37017/openim_v3?maxPoolSize=100").SetConnectTimeout(5*time.Second)) + if err != nil { + panic(err) + } + model, err := mgo.NewSeqConversationMongo(mgocli.Database("openim_v3")) + if err != nil { + panic(err) + } + opt := &redis.Options{ + Addr: "172.16.8.48:16379", + Password: "openIM123", + DB: 1, + } + rdb := redis.NewClient(opt) + if err := rdb.Ping(context.Background()).Err(); err != nil { + panic(err) + } + return NewSeqConversationCacheRedis(rdb, model).(*seqConversationCacheRedis) +} + +func TestSeq(t *testing.T) { + ts := newTestSeq() + var ( + wg sync.WaitGroup + speed atomic.Int64 + ) + + const count = 128 + wg.Add(count) + for i := 0; i < count; i++ { + index := i + 1 + go func() { + defer wg.Done() + var size int64 = 10 + cID := strconv.Itoa(index * 1) + for i := 1; ; i++ { + //first, err := ts.mgo.Malloc(context.Background(), cID, size) // mongo + first, err := ts.Malloc(context.Background(), cID, size) // redis + if err != nil { + t.Logf("[%d-%d] %s %s", index, i, cID, err) + return + } + speed.Add(size) + _ = first + //t.Logf("[%d] %d -> %d", i, first+1, first+size) + } + }() + } + + done := make(chan struct{}) + + go func() { + wg.Wait() + close(done) + }() + + ticker := time.NewTicker(time.Second) + + for { + select { + case <-done: + ticker.Stop() + return + case <-ticker.C: + value := speed.Swap(0) + t.Logf("speed: %d/s", value) + } + } +} + +func TestDel(t *testing.T) { + ts := newTestSeq() + for i := 1; i < 100; i++ { + var size int64 = 100 + first, err := ts.Malloc(context.Background(), "100", size) + if err != nil { + t.Logf("[%d] %s", i, err) + return + } + t.Logf("[%d] %d -> %d", i, first+1, first+size) + time.Sleep(time.Second) + } +} + +func TestSeqMalloc(t *testing.T) { + ts := newTestSeq() + t.Log(ts.GetMaxSeq(context.Background(), "100")) +} + +func TestMinSeq(t *testing.T) { + ts := newTestSeq() + t.Log(ts.GetMinSeq(context.Background(), "10000000")) +} diff --git a/pkg/common/storage/cache/redis/seq_test.go b/pkg/common/storage/cache/redis/seq_test.go deleted file mode 100644 index 6fff6019a..000000000 --- a/pkg/common/storage/cache/redis/seq_test.go +++ /dev/null @@ -1,77 +0,0 @@ -package redis - -import ( - "context" - "strconv" - "sync" - "sync/atomic" - "testing" - "time" -) - -func TestSeq(t *testing.T) { - ts := NewTestSeq() - var ( - wg sync.WaitGroup - speed atomic.Int64 - ) - - const count = 256 - wg.Add(count) - for i := 0; i < count; i++ { - index := i + 1 - go func() { - defer wg.Done() - var size int64 = 1 - cID := strconv.Itoa(index * 100) - for i := 1; ; i++ { - first, err := ts.mgo.Malloc(context.Background(), cID, size) // mongo - //first, err := ts.Malloc(context.Background(), cID, size) // redis - if err != nil { - t.Logf("[%d-%d] %s %s", index, i, cID, err) - return - } - speed.Add(size) - _ = first - //t.Logf("[%d] %d -> %d", i, first+1, first+size) - } - }() - } - - done := make(chan struct{}) - - go func() { - wg.Wait() - close(done) - }() - - ticker := time.NewTicker(time.Second) - - for { - select { - case <-done: - ticker.Stop() - return - case <-ticker.C: - value := speed.Swap(0) - t.Logf("speed: %d/s", value) - } - } - - //for i := 1; i < 1000000; i++ { - // var size int64 = 100 - // first, err := ts.Malloc(context.Background(), "1", size) - // if err != nil { - // t.Logf("[%d] %s", i, err) - // return - // } - // t.Logf("[%d] %d -> %d", i, first+1, first+size) - // time.Sleep(time.Second / 4) - //} -} - -func TestDel(t *testing.T) { - ts := NewTestSeq() - t.Log(ts.GetMaxSeq(context.Background(), "1")) - -} diff --git a/pkg/common/storage/cache/seq.go b/pkg/common/storage/cache/seq.go index 091b318c8..46e33c935 100644 --- a/pkg/common/storage/cache/seq.go +++ b/pkg/common/storage/cache/seq.go @@ -5,13 +5,13 @@ import ( ) type SeqCache interface { - SetMaxSeq(ctx context.Context, conversationID string, maxSeq int64) error - GetMaxSeqs(ctx context.Context, conversationIDs []string) (map[string]int64, error) - GetMaxSeq(ctx context.Context, conversationID string) (int64, error) - SetMinSeq(ctx context.Context, conversationID string, minSeq int64) error - SetMinSeqs(ctx context.Context, seqs map[string]int64) error - GetMinSeqs(ctx context.Context, conversationIDs []string) (map[string]int64, error) - GetMinSeq(ctx context.Context, conversationID string) (int64, error) + //SetMaxSeq(ctx context.Context, conversationID string, maxSeq int64) error + //GetMaxSeqs(ctx context.Context, conversationIDs []string) (map[string]int64, error) + //GetMaxSeq(ctx context.Context, conversationID string) (int64, error) + //SetMinSeq(ctx context.Context, conversationID string, minSeq int64) error + //SetMinSeqs(ctx context.Context, seqs map[string]int64) error + //GetMinSeqs(ctx context.Context, conversationIDs []string) (map[string]int64, error) + //GetMinSeq(ctx context.Context, conversationID string) (int64, error) GetConversationUserMinSeq(ctx context.Context, conversationID string, userID string) (int64, error) GetConversationUserMinSeqs(ctx context.Context, conversationID string, userIDs []string) (map[string]int64, error) SetConversationUserMinSeq(ctx context.Context, conversationID string, userID string, minSeq int64) error diff --git a/pkg/common/storage/cache/seq_conversation.go b/pkg/common/storage/cache/seq_conversation.go new file mode 100644 index 000000000..5d38537a9 --- /dev/null +++ b/pkg/common/storage/cache/seq_conversation.go @@ -0,0 +1,10 @@ +package cache + +import "context" + +type SeqConversationCache interface { + Malloc(ctx context.Context, conversationID string, size int64) (int64, error) + GetMaxSeq(ctx context.Context, conversationID string) (int64, error) + SetMinSeq(ctx context.Context, conversationID string, seq int64) error + GetMinSeq(ctx context.Context, conversationID string) (int64, error) +} diff --git a/pkg/common/storage/controller/msg.go b/pkg/common/storage/controller/msg.go index c5c5462aa..3a29f6786 100644 --- a/pkg/common/storage/controller/msg.go +++ b/pkg/common/storage/controller/msg.go @@ -108,7 +108,7 @@ type CommonMsgDatabase interface { DeleteDocMsgBefore(ctx context.Context, ts int64, doc *model.MsgDocModel) ([]int, error) } -func NewCommonMsgDatabase(msgDocModel database.Msg, msg cache.MsgCache, seq cache.SeqCache, kafkaConf *config.Kafka) (CommonMsgDatabase, error) { +func NewCommonMsgDatabase(msgDocModel database.Msg, msg cache.MsgCache, seq cache.SeqCache, seqConversation cache.SeqConversationCache, kafkaConf *config.Kafka) (CommonMsgDatabase, error) { conf, err := kafka.BuildProducerConfig(*kafkaConf.Build()) if err != nil { return nil, err @@ -129,28 +129,19 @@ func NewCommonMsgDatabase(msgDocModel database.Msg, msg cache.MsgCache, seq cach msgDocDatabase: msgDocModel, msg: msg, seq: seq, + seqConversation: seqConversation, producer: producerToRedis, producerToMongo: producerToMongo, producerToPush: producerToPush, }, nil } -//func InitCommonMsgDatabase(rdb redis.UniversalClient, database *mongo.Database, config *tools.CronTaskConfig) (CommonMsgDatabase, error) { -// msgDocModel, err := database.NewMsgMongo(database) -// if err != nil { -// return nil, err -// } -// //todo MsgCacheTimeout -// msg := cache.NewMsgCache(rdb, 86400, config.RedisConfig.EnablePipeline) -// seq := cache.NewSeqCache(rdb) -// return NewCommonMsgDatabase(msgDocModel, msg, seq, &config.KafkaConfig) -//} - type commonMsgDatabase struct { msgDocDatabase database.Msg msgTable model.MsgDocModel msg cache.MsgCache seq cache.SeqCache + seqConversation cache.SeqConversationCache producer *kafka.Producer producerToMongo *kafka.Producer producerToPush *kafka.Producer @@ -349,10 +340,9 @@ func (db *commonMsgDatabase) DeleteMessagesFromCache(ctx context.Context, conver } func (db *commonMsgDatabase) BatchInsertChat2Cache(ctx context.Context, conversationID string, msgs []*sdkws.MsgData) (seq int64, isNew bool, err error) { - // TODO set SEQ - currentMaxSeq, err := db.seq.GetMaxSeq(ctx, conversationID) - if err != nil && errs.Unwrap(err) != redis.Nil { - log.ZError(ctx, "storage.seq.GetMaxSeq", err) + currentMaxSeq, err := db.seqConversation.Malloc(ctx, conversationID, int64(len(msgs))) + if err != nil { + log.ZError(ctx, "storage.seq.Malloc", err) return 0, false, err } lenList := len(msgs) @@ -362,9 +352,6 @@ func (db *commonMsgDatabase) BatchInsertChat2Cache(ctx context.Context, conversa if lenList < 1 { return 0, false, errs.New("no messages to insert", "minCount", 1).Wrap() } - if errs.Unwrap(err) == redis.Nil { - isNew = true - } lastMaxSeq := currentMaxSeq userSeqMap := make(map[string]int64) for _, m := range msgs { @@ -380,13 +367,6 @@ func (db *commonMsgDatabase) BatchInsertChat2Cache(ctx context.Context, conversa } else { prommetrics.MsgInsertRedisSuccessCounter.Inc() } - - err = db.seq.SetMaxSeq(ctx, conversationID, currentMaxSeq) - if err != nil { - log.ZError(ctx, "storage.seq.SetMaxSeq error", err, "conversationID", conversationID) - prommetrics.SeqSetFailedCounter.Inc() - } - err = db.seq.SetHasReadSeqs(ctx, conversationID, userSeqMap) if err != nil { log.ZError(ctx, "SetHasReadSeqs error", err, "userSeqMap", userSeqMap, "conversationID", conversationID) @@ -519,8 +499,8 @@ func (db *commonMsgDatabase) GetMsgBySeqsRange(ctx context.Context, userID strin if err != nil && errs.Unwrap(err) != redis.Nil { return 0, 0, nil, err } - minSeq, err := db.seq.GetMinSeq(ctx, conversationID) - if err != nil && errs.Unwrap(err) != redis.Nil { + minSeq, err := db.seqConversation.GetMinSeq(ctx, conversationID) + if err != nil { return 0, 0, nil, err } if userMinSeq > minSeq { @@ -531,8 +511,8 @@ func (db *commonMsgDatabase) GetMsgBySeqsRange(ctx context.Context, userID strin log.ZWarn(ctx, "minSeq > end", errs.New("minSeq>end"), "minSeq", minSeq, "end", end) return 0, 0, nil, nil } - maxSeq, err := db.seq.GetMaxSeq(ctx, conversationID) - if err != nil && errs.Unwrap(err) != redis.Nil { + maxSeq, err := db.seqConversation.GetMaxSeq(ctx, conversationID) + if err != nil { return 0, 0, nil, err } log.ZDebug(ctx, "GetMsgBySeqsRange", "userMinSeq", userMinSeq, "conMinSeq", minSeq, "conMaxSeq", maxSeq, "userMaxSeq", userMaxSeq) @@ -572,11 +552,8 @@ func (db *commonMsgDatabase) GetMsgBySeqsRange(ctx context.Context, userID strin var successMsgs []*sdkws.MsgData log.ZDebug(ctx, "GetMsgBySeqsRange", "first seqs", seqs, "newBegin", newBegin, "newEnd", newEnd) cachedMsgs, failedSeqs, err := db.msg.GetMessagesBySeq(ctx, conversationID, seqs) - if err != nil { - if err != redis.Nil { - - log.ZError(ctx, "get message from redis exception", err, "conversationID", conversationID, "seqs", seqs) - } + if err != nil && !errors.Is(err, redis.Nil) { + log.ZError(ctx, "get message from redis exception", err, "conversationID", conversationID, "seqs", seqs) } successMsgs = append(successMsgs, cachedMsgs...) log.ZDebug(ctx, "get msgs from cache", "cachedMsgs", cachedMsgs) @@ -600,12 +577,12 @@ func (db *commonMsgDatabase) GetMsgBySeqs(ctx context.Context, userID string, co if err != nil && errs.Unwrap(err) != redis.Nil { return 0, 0, nil, err } - minSeq, err := db.seq.GetMinSeq(ctx, conversationID) - if err != nil && errs.Unwrap(err) != redis.Nil { + minSeq, err := db.seqConversation.GetMinSeq(ctx, conversationID) + if err != nil { return 0, 0, nil, err } - maxSeq, err := db.seq.GetMaxSeq(ctx, conversationID) - if err != nil && errs.Unwrap(err) != redis.Nil { + maxSeq, err := db.seqConversation.GetMaxSeq(ctx, conversationID) + if err != nil { return 0, 0, nil, err } if userMinSeq < minSeq { @@ -649,7 +626,7 @@ func (db *commonMsgDatabase) DeleteConversationMsgsAndSetMinSeq(ctx context.Cont if minSeq == 0 { return nil } - return db.seq.SetMinSeq(ctx, conversationID, minSeq) + return db.seqConversation.SetMinSeq(ctx, conversationID, minSeq) } func (db *commonMsgDatabase) UserMsgsDestruct(ctx context.Context, userID string, conversationID string, destructTime int64, lastMsgDestructTime time.Time) (seqs []int64, err error) { @@ -803,7 +780,7 @@ func (db *commonMsgDatabase) DeleteMsgsBySeqs(ctx context.Context, conversationI func (db *commonMsgDatabase) CleanUpUserConversationsMsgs(ctx context.Context, user string, conversationIDs []string) { for _, conversationID := range conversationIDs { - maxSeq, err := db.seq.GetMaxSeq(ctx, conversationID) + maxSeq, err := db.seqConversation.GetMaxSeq(ctx, conversationID) if err != nil { if err == redis.Nil { log.ZDebug(ctx, "max seq is nil", "conversationID", conversationID) @@ -812,7 +789,7 @@ func (db *commonMsgDatabase) CleanUpUserConversationsMsgs(ctx context.Context, u } continue } - if err := db.seq.SetMinSeq(ctx, conversationID, maxSeq+1); err != nil { + if err := db.seqConversation.SetMinSeq(ctx, conversationID, maxSeq+1); err != nil { log.ZError(ctx, "set min seq failed", err, "conversationID", conversationID, "minSeq", maxSeq+1) } } @@ -823,29 +800,37 @@ func (db *commonMsgDatabase) CleanUpUserConversationsMsgs(ctx context.Context, u //} func (db *commonMsgDatabase) GetMaxSeqs(ctx context.Context, conversationIDs []string) (map[string]int64, error) { - return db.seq.GetMaxSeqs(ctx, conversationIDs) + result := make(map[string]int64) + for _, conversationID := range conversationIDs { + if result[conversationID] != 0 { + continue + } + seq, err := db.seqConversation.GetMaxSeq(ctx, conversationID) + if err != nil { + return nil, err + } + result[conversationID] = seq + } + return result, nil } func (db *commonMsgDatabase) GetMaxSeq(ctx context.Context, conversationID string) (int64, error) { - return db.seq.GetMaxSeq(ctx, conversationID) + return db.seqConversation.GetMaxSeq(ctx, conversationID) } func (db *commonMsgDatabase) SetMinSeq(ctx context.Context, conversationID string, minSeq int64) error { - return db.seq.SetMinSeq(ctx, conversationID, minSeq) + return db.seqConversation.SetMinSeq(ctx, conversationID, minSeq) } func (db *commonMsgDatabase) SetMinSeqs(ctx context.Context, seqs map[string]int64) error { - return db.seq.SetMinSeqs(ctx, seqs) + for conversationID, seq := range seqs { + if err := db.seqConversation.SetMinSeq(ctx, conversationID, seq); err != nil { + return err + } + } + return nil } -//func (db *commonMsgDatabase) GetMinSeqs(ctx context.Context, conversationIDs []string) (map[string]int64, error) { -// return db.seq.GetMinSeqs(ctx, conversationIDs) -//} -// -//func (db *commonMsgDatabase) GetMinSeq(ctx context.Context, conversationID string) (int64, error) { -// return db.seq.GetMinSeq(ctx, conversationID) -//} - func (db *commonMsgDatabase) GetConversationUserMinSeq(ctx context.Context, conversationID string, userID string) (int64, error) { return db.seq.GetConversationUserMinSeq(ctx, conversationID, userID) } @@ -895,11 +880,11 @@ func (db *commonMsgDatabase) GetConversationMinMaxSeqInMongoAndCache(ctx context if err != nil { return } - minSeqCache, err = db.seq.GetMinSeq(ctx, conversationID) + minSeqCache, err = db.seqConversation.GetMinSeq(ctx, conversationID) if err != nil { return } - maxSeqCache, err = db.seq.GetMaxSeq(ctx, conversationID) + maxSeqCache, err = db.seqConversation.GetMaxSeq(ctx, conversationID) if err != nil { return } @@ -1011,33 +996,8 @@ func (db *commonMsgDatabase) DeleteDocMsgBefore(ctx context.Context, ts int64, d } } -//func (db *commonMsgDatabase) ClearMsg(ctx context.Context, ts int64) (err error) { -// var ( -// docNum int -// msgNum int -// start = time.Now() -// ) -// for { -// msgs, err := db.msgDocDatabase.GetBeforeMsg(ctx, ts, 100) -// if err != nil { -// return err -// } -// if len(msgs) == 0 { -// return nil -// } -// for _, msg := range msgs { -// num, err := db.deleteOneMsg(ctx, ts, msg) -// if err != nil { -// return err -// } -// docNum++ -// msgNum += num -// } -// } -//} - func (db *commonMsgDatabase) setMinSeq(ctx context.Context, conversationID string, seq int64) error { - dbSeq, err := db.seq.GetMinSeq(ctx, conversationID) + dbSeq, err := db.seqConversation.GetMinSeq(ctx, conversationID) if err != nil { if errors.Is(errs.Unwrap(err), redis.Nil) { return nil @@ -1047,5 +1007,5 @@ func (db *commonMsgDatabase) setMinSeq(ctx context.Context, conversationID strin if dbSeq >= seq { return nil } - return db.seq.SetMinSeq(ctx, conversationID, seq) + return db.seqConversation.SetMinSeq(ctx, conversationID, seq) } diff --git a/pkg/common/storage/database/mgo/seq.go b/pkg/common/storage/database/mgo/seq_conversation.go similarity index 50% rename from pkg/common/storage/database/mgo/seq.go rename to pkg/common/storage/database/mgo/seq_conversation.go index 9b8c86ffe..c9a3ac41a 100644 --- a/pkg/common/storage/database/mgo/seq.go +++ b/pkg/common/storage/database/mgo/seq_conversation.go @@ -3,6 +3,7 @@ package mgo import ( "context" "errors" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" "github.com/openimsdk/tools/db/mongoutil" "go.mongodb.org/mongo-driver/bson" @@ -10,16 +11,24 @@ import ( "go.mongodb.org/mongo-driver/mongo/options" ) -func NewSeqMongo(db *mongo.Database) (*SeqMongo, error) { - coll := db.Collection("seq") - return &SeqMongo{coll: coll}, nil +func NewSeqConversationMongo(db *mongo.Database) (database.SeqConversation, error) { + coll := db.Collection(database.SeqConversationName) + _, err := coll.Indexes().CreateOne(context.Background(), mongo.IndexModel{ + Keys: bson.D{ + {Key: "conversation_id", Value: 1}, + }, + }) + if err != nil { + return nil, err + } + return &seqConversationMongo{coll: coll}, nil } -type SeqMongo struct { +type seqConversationMongo struct { coll *mongo.Collection } -func (s *SeqMongo) Malloc(ctx context.Context, conversationID string, size int64) (int64, error) { +func (s *seqConversationMongo) Malloc(ctx context.Context, conversationID string, size int64) (int64, error) { if size < 0 { return 0, errors.New("size must be greater than 0") } @@ -29,7 +38,7 @@ func (s *SeqMongo) Malloc(ctx context.Context, conversationID string, size int64 filter := map[string]any{"conversation_id": conversationID} update := map[string]any{ "$inc": map[string]any{"max_seq": size}, - "$set": map[string]any{"min_seq": 1}, + "$set": map[string]any{"min_seq": int64(0)}, } opt := options.FindOneAndUpdate().SetUpsert(true).SetReturnDocument(options.After).SetProjection(map[string]any{"_id": 0, "max_seq": 1}) lastSeq, err := mongoutil.FindOneAndUpdate[int64](ctx, s.coll, filter, update, opt) @@ -39,7 +48,19 @@ func (s *SeqMongo) Malloc(ctx context.Context, conversationID string, size int64 return lastSeq - size, nil } -func (s *SeqMongo) GetMaxSeq(ctx context.Context, conversationID string) (int64, error) { +func (s *seqConversationMongo) MallocSeq(ctx context.Context, conversationID string, size int64) ([]int64, error) { + first, err := s.Malloc(ctx, conversationID, size) + if err != nil { + return nil, err + } + seqs := make([]int64, 0, size) + for i := int64(0); i < size; i++ { + seqs = append(seqs, first+i+1) + } + return seqs, nil +} + +func (s *seqConversationMongo) GetMaxSeq(ctx context.Context, conversationID string) (int64, error) { seq, err := mongoutil.FindOne[int64](ctx, s.coll, bson.M{"conversation_id": conversationID}, options.FindOne().SetProjection(map[string]any{"_id": 0, "max_seq": 1})) if err == nil { return seq, nil @@ -50,7 +71,7 @@ func (s *SeqMongo) GetMaxSeq(ctx context.Context, conversationID string) (int64, } } -func (s *SeqMongo) GetMinSeq(ctx context.Context, conversationID string) (int64, error) { +func (s *seqConversationMongo) GetMinSeq(ctx context.Context, conversationID string) (int64, error) { seq, err := mongoutil.FindOne[int64](ctx, s.coll, bson.M{"conversation_id": conversationID}, options.FindOne().SetProjection(map[string]any{"_id": 0, "min_seq": 1})) if err == nil { return seq, nil @@ -61,10 +82,10 @@ func (s *SeqMongo) GetMinSeq(ctx context.Context, conversationID string) (int64, } } -func (s *SeqMongo) SetMinSeq(ctx context.Context, conversationID string, seq int64) error { +func (s *seqConversationMongo) SetMinSeq(ctx context.Context, conversationID string, seq int64) error { return mongoutil.UpdateOne(ctx, s.coll, bson.M{"conversation_id": conversationID}, bson.M{"$set": bson.M{"min_seq": seq}}, false) } -func (s *SeqMongo) GetConversation(ctx context.Context, conversationID string) (*model.Seq, error) { - return mongoutil.FindOne[*model.Seq](ctx, s.coll, bson.M{"conversation_id": conversationID}) +func (s *seqConversationMongo) GetConversation(ctx context.Context, conversationID string) (*model.SeqConversation, error) { + return mongoutil.FindOne[*model.SeqConversation](ctx, s.coll, bson.M{"conversation_id": conversationID}) } diff --git a/pkg/common/storage/database/mgo/seq_test.go b/pkg/common/storage/database/mgo/seq_conversation_test.go similarity index 100% rename from pkg/common/storage/database/mgo/seq_test.go rename to pkg/common/storage/database/mgo/seq_conversation_test.go diff --git a/pkg/common/storage/database/name.go b/pkg/common/storage/database/name.go index f496d4b41..68fa5af02 100644 --- a/pkg/common/storage/database/name.go +++ b/pkg/common/storage/database/name.go @@ -14,5 +14,5 @@ const ( LogName = "log" ObjectName = "s3" UserName = "user" - SeqName = "seq" + SeqConversationName = "seq" ) diff --git a/pkg/common/storage/database/seq.go b/pkg/common/storage/database/seq.go index 20fae3bb1..fdbc2f8f3 100644 --- a/pkg/common/storage/database/seq.go +++ b/pkg/common/storage/database/seq.go @@ -2,7 +2,7 @@ package database import "context" -type Seq interface { +type SeqConversation interface { Malloc(ctx context.Context, conversationID string, size int64) (int64, error) GetMaxSeq(ctx context.Context, conversationID string) (int64, error) GetMinSeq(ctx context.Context, conversationID string) (int64, error) diff --git a/pkg/common/storage/model/seq.go b/pkg/common/storage/model/seq.go index 3db009167..1dc75eff1 100644 --- a/pkg/common/storage/model/seq.go +++ b/pkg/common/storage/model/seq.go @@ -1,6 +1,6 @@ package model -type Seq struct { +type SeqConversation struct { ConversationID string `bson:"conversation_id"` MaxSeq int64 `bson:"max_seq"` MinSeq int64 `bson:"min_seq"` diff --git a/tools/seq/internal/main.go b/tools/seq/internal/main.go new file mode 100644 index 000000000..5cfd9fd45 --- /dev/null +++ b/tools/seq/internal/main.go @@ -0,0 +1,152 @@ +package internal + +import ( + "context" + "errors" + "fmt" + "github.com/openimsdk/open-im-server/v3/pkg/common/cmd" + "github.com/openimsdk/open-im-server/v3/pkg/common/config" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/cachekey" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database/mgo" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" + "github.com/openimsdk/tools/db/mongoutil" + "github.com/openimsdk/tools/db/redisutil" + "github.com/redis/go-redis/v9" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" + "gopkg.in/yaml.v3" + "os" + "path/filepath" + "strconv" + "strings" + "time" +) + +const batchSize = 5 + +func readConfig[T any](dir string, name string) (*T, error) { + data, err := os.ReadFile(filepath.Join(dir, name)) + if err != nil { + return nil, err + } + var conf T + if err := yaml.Unmarshal(data, &conf); err != nil { + return nil, err + } + return &conf, nil +} + +func redisKey(rdb redis.UniversalClient, prefix string, fn func(ctx context.Context, key string, delKey map[string]struct{}) error) error { + var ( + cursor uint64 + keys []string + err error + ) + ctx := context.Background() + for { + keys, cursor, err = rdb.Scan(ctx, cursor, prefix+"*", batchSize).Result() + if err != nil { + return err + } + delKey := make(map[string]struct{}) + if len(keys) > 0 { + for _, key := range keys { + if err := fn(ctx, key, delKey); err != nil { + return err + } + } + } + if len(delKey) > 0 { + //if err := rdb.Del(ctx, datautil.Keys(delKey)...).Err(); err != nil { + // return err + //} + } + if cursor == 0 { + return nil + } + } +} + +func Main(conf string) error { + redisConfig, err := readConfig[config.Redis](conf, cmd.RedisConfigFileName) + if err != nil { + return err + } + mongodbConfig, err := readConfig[config.Mongo](conf, cmd.MongodbConfigFileName) + if err != nil { + return err + } + ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) + defer cancel() + rdb, err := redisutil.NewRedisClient(ctx, redisConfig.Build()) + if err != nil { + return err + } + mgocli, err := mongoutil.NewMongoDB(ctx, mongodbConfig.Build()) + if err != nil { + return err + } + if _, err := mgo.NewSeqConversationMongo(mgocli.GetDB()); err != nil { + return err + } + coll := mgocli.GetDB().Collection(database.SeqConversationName) + const prefix = cachekey.MaxSeq + return redisKey(rdb, prefix, func(ctx context.Context, key string, delKey map[string]struct{}) error { + conversationId := strings.TrimPrefix(key, prefix) + delKey[key] = struct{}{} + maxValue, err := rdb.Get(ctx, key).Result() + if err != nil { + return err + } + seq, err := strconv.Atoi(maxValue) + if err != nil { + return fmt.Errorf("invalid max seq %s", maxValue) + } + if seq == 0 { + return nil + } + if seq < 0 { + return fmt.Errorf("invalid max seq %s", maxValue) + } + var ( + minSeq int64 + maxSeq = int64(seq) + ) + minKey := cachekey.MinSeq + conversationId + delKey[minKey] = struct{}{} + minValue, err := rdb.Get(ctx, minKey).Result() + if err == nil { + seq, err := strconv.Atoi(minValue) + if err != nil { + return fmt.Errorf("invalid min seq %s", minValue) + } + if seq < 0 { + return fmt.Errorf("invalid min seq %s", minValue) + } + minSeq = int64(seq) + } else if !errors.Is(err, redis.Nil) { + return err + } + if maxSeq < minSeq { + return fmt.Errorf("invalid max seq %d < min seq %d", maxSeq, minSeq) + } + res, err := mongoutil.FindOne[*model.SeqConversation](ctx, coll, bson.M{"conversation_id": conversationId}, nil) + if err == nil { + if res.MaxSeq < int64(seq) { + _, err = coll.UpdateOne(ctx, bson.M{"conversation_id": conversationId}, bson.M{"$set": bson.M{"max_seq": maxSeq, "min_seq": minSeq}}) + } + return err + } else if errors.Is(err, mongo.ErrNoDocuments) { + res = &model.SeqConversation{ + ConversationID: conversationId, + MaxSeq: maxSeq, + MinSeq: minSeq, + } + _, err := coll.InsertOne(ctx, res) + return err + } else { + return err + } + }) +} diff --git a/tools/seq/main.go b/tools/seq/main.go new file mode 100644 index 000000000..ca5a043e7 --- /dev/null +++ b/tools/seq/main.go @@ -0,0 +1,16 @@ +package main + +import ( + "flag" + "fmt" + "github.com/openimsdk/open-im-server/v3/tools/seq/internal" +) + +func main() { + var config string + flag.StringVar(&config, "redis", "/Users/chao/Desktop/project/open-im-server/config", "config directory") + flag.Parse() + if err := internal.Main(config); err != nil { + fmt.Println("seq task", err) + } +} From cce382d0c37cd954fa38af1936813d8b7f056096 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Tue, 18 Jun 2024 18:06:40 +0800 Subject: [PATCH 41/59] seq --- .../database/mgo/seq_conversation_test.go | 8 +- start-config.yml | 1 + tools/seq/internal/main.go | 80 +++++++++++++++++-- tools/seq/main.go | 11 ++- 4 files changed, 83 insertions(+), 17 deletions(-) diff --git a/pkg/common/storage/database/mgo/seq_conversation_test.go b/pkg/common/storage/database/mgo/seq_conversation_test.go index abae2d1b1..e6466a1c6 100644 --- a/pkg/common/storage/database/mgo/seq_conversation_test.go +++ b/pkg/common/storage/database/mgo/seq_conversation_test.go @@ -15,15 +15,9 @@ func Result[V any](val V, err error) V { return val } -func Check(err error) { - if err != nil { - panic(err) - } -} - func TestName(t *testing.T) { cli := Result(mongo.Connect(context.Background(), options.Client().ApplyURI("mongodb://openIM:openIM123@172.16.8.48:37017/openim_v3?maxPoolSize=100").SetConnectTimeout(5*time.Second))) - tmp, err := NewSeqMongo(cli.Database("openim_v3")) + tmp, err := NewSeqConversationMongo(cli.Database("openim_v3")) if err != nil { panic(err) } diff --git a/start-config.yml b/start-config.yml index a9c412b33..21436d7a9 100644 --- a/start-config.yml +++ b/start-config.yml @@ -14,4 +14,5 @@ serviceBinaries: toolBinaries: - check-free-memory - check-component + - seq maxFileDescriptors: 10000 diff --git a/tools/seq/internal/main.go b/tools/seq/internal/main.go index 5cfd9fd45..54a7d477d 100644 --- a/tools/seq/internal/main.go +++ b/tools/seq/internal/main.go @@ -12,9 +12,11 @@ import ( "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" "github.com/openimsdk/tools/db/mongoutil" "github.com/openimsdk/tools/db/redisutil" + "github.com/openimsdk/tools/utils/datautil" "github.com/redis/go-redis/v9" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" "gopkg.in/yaml.v3" "os" "path/filepath" @@ -23,7 +25,12 @@ import ( "time" ) -const batchSize = 5 +const ( + batchSize = 100 + dataVersionCollection = "data_version" + seqKey = "seq" + seqVersion = 38 +) func readConfig[T any](dir string, name string) (*T, error) { data, err := os.ReadFile(filepath.Join(dir, name)) @@ -37,7 +44,7 @@ func readConfig[T any](dir string, name string) (*T, error) { return &conf, nil } -func redisKey(rdb redis.UniversalClient, prefix string, fn func(ctx context.Context, key string, delKey map[string]struct{}) error) error { +func redisKey(rdb redis.UniversalClient, prefix string, del time.Duration, fn func(ctx context.Context, key string, delKey map[string]struct{}) error) error { var ( cursor uint64 keys []string @@ -58,9 +65,20 @@ func redisKey(rdb redis.UniversalClient, prefix string, fn func(ctx context.Cont } } if len(delKey) > 0 { - //if err := rdb.Del(ctx, datautil.Keys(delKey)...).Err(); err != nil { - // return err - //} + delKeys := datautil.Keys(delKey) + if del < time.Second { + if err := rdb.Del(ctx, datautil.Keys(delKey)...).Err(); err != nil { + return err + } + } else { + pipe := rdb.Pipeline() + for _, key := range delKeys { + pipe.Expire(ctx, key, del) + } + if _, err := pipe.Exec(ctx); err != nil { + return err + } + } } if cursor == 0 { return nil @@ -68,7 +86,7 @@ func redisKey(rdb redis.UniversalClient, prefix string, fn func(ctx context.Cont } } -func Main(conf string) error { +func Main(conf string, del time.Duration) error { redisConfig, err := readConfig[config.Redis](conf, cmd.RedisConfigFileName) if err != nil { return err @@ -87,12 +105,22 @@ func Main(conf string) error { if err != nil { return err } + versionColl := mgocli.GetDB().Collection(dataVersionCollection) + converted, err := CheckVersion(versionColl, seqKey, seqVersion) + if err != nil { + return err + } + if converted { + fmt.Println("[seq] seq data has been converted") + return nil + } if _, err := mgo.NewSeqConversationMongo(mgocli.GetDB()); err != nil { return err } coll := mgocli.GetDB().Collection(database.SeqConversationName) const prefix = cachekey.MaxSeq - return redisKey(rdb, prefix, func(ctx context.Context, key string, delKey map[string]struct{}) error { + fmt.Println("start to convert seq conversation") + err = redisKey(rdb, prefix, del, func(ctx context.Context, key string, delKey map[string]struct{}) error { conversationId := strings.TrimPrefix(key, prefix) delKey[key] = struct{}{} maxValue, err := rdb.Get(ctx, key).Result() @@ -149,4 +177,42 @@ func Main(conf string) error { return err } }) + if err != nil { + return err + } + fmt.Println("convert seq conversation success") + return SetVersion(versionColl, seqKey, seqVersion) +} + +func CheckVersion(coll *mongo.Collection, key string, currentVersion int) (converted bool, err error) { + type VersionTable struct { + Key string `bson:"key"` + Value string `bson:"value"` + } + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + res, err := mongoutil.FindOne[VersionTable](ctx, coll, bson.M{"key": key}) + if err == nil { + ver, err := strconv.Atoi(res.Value) + if err != nil { + return false, fmt.Errorf("version %s parse error %w", res.Value, err) + } + if ver >= currentVersion { + return true, nil + } + return false, nil + } else if errors.Is(err, mongo.ErrNoDocuments) { + return false, nil + } else { + return false, err + } +} + +func SetVersion(coll *mongo.Collection, key string, version int) error { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + option := options.Update().SetUpsert(true) + filter := bson.M{"key": key, "value": strconv.Itoa(version)} + update := bson.M{"$set": bson.M{"key": key, "value": strconv.Itoa(version)}} + return mongoutil.UpdateOne(ctx, coll, filter, update, false, option) } diff --git a/tools/seq/main.go b/tools/seq/main.go index ca5a043e7..6bb4b7657 100644 --- a/tools/seq/main.go +++ b/tools/seq/main.go @@ -4,13 +4,18 @@ import ( "flag" "fmt" "github.com/openimsdk/open-im-server/v3/tools/seq/internal" + "time" ) func main() { - var config string - flag.StringVar(&config, "redis", "/Users/chao/Desktop/project/open-im-server/config", "config directory") + var ( + config string + second int + ) + flag.StringVar(&config, "c", "", "config directory") + flag.IntVar(&second, "sec", 3600*24, "delayed deletion of the original seq key after conversion") flag.Parse() - if err := internal.Main(config); err != nil { + if err := internal.Main(config, time.Duration(second)*time.Second); err != nil { fmt.Println("seq task", err) } } From aff8322bc5ce2929b44666bea03e738bebad7b07 Mon Sep 17 00:00:00 2001 From: Gordon <46924906+FGadvancer@users.noreply.github.com> Date: Tue, 18 Jun 2024 21:04:03 +0800 Subject: [PATCH 42/59] fix: sort by id avoid unstable sort friends. --- pkg/common/storage/database/mgo/friend.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/common/storage/database/mgo/friend.go b/pkg/common/storage/database/mgo/friend.go index 18d80d47d..b74df1c19 100644 --- a/pkg/common/storage/database/mgo/friend.go +++ b/pkg/common/storage/database/mgo/friend.go @@ -53,7 +53,7 @@ func NewFriendMongo(db *mongo.Database) (database.Friend, error) { } func (f *FriendMgo) friendSort() any { - return bson.D{{"is_pinned", -1}, {"create_time", 1}} + return bson.D{{"is_pinned", -1}, {"create_time", 1}, {"_id", 1}} } // Create inserts multiple friend records. From 08cb4a948d63b3f88dcd5512989e4958d56e5310 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Wed, 19 Jun 2024 16:28:28 +0800 Subject: [PATCH 43/59] group --- go.mod | 6 +- go.sum | 4 +- internal/rpc/group/notification.go | 5 ++ internal/rpc/group/sync.go | 54 ++++++++++++-- pkg/common/storage/controller/group.go | 56 +++++++-------- pkg/common/storage/database/group_member.go | 1 + pkg/common/storage/database/mgo/friend.go | 70 +++++++++++++------ .../storage/database/mgo/group_member.go | 6 +- pkg/common/storage/model/friend.go | 18 ++--- 9 files changed, 148 insertions(+), 72 deletions(-) diff --git a/go.mod b/go.mod index 4ab39f751..d00f0b739 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/lestrrat-go/file-rotatelogs v2.4.0+incompatible // indirect github.com/mitchellh/mapstructure v1.5.0 - github.com/openimsdk/protocol v0.0.69-alpha.12 + github.com/openimsdk/protocol v0.0.69-alpha.15 github.com/openimsdk/tools v0.0.49-alpha.25 github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 @@ -176,3 +176,7 @@ require ( golang.org/x/crypto v0.21.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect ) + +//replace ( +// github.com/openimsdk/protocol => /Users/chao/Desktop/project/protocol +//) diff --git a/go.sum b/go.sum index a26e59d9e..4c05500bd 100644 --- a/go.sum +++ b/go.sum @@ -270,8 +270,8 @@ github.com/onsi/gomega v1.25.0 h1:Vw7br2PCDYijJHSfBOWhov+8cAnUf8MfMaIOV323l6Y= github.com/onsi/gomega v1.25.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdMPGhM= github.com/openimsdk/gomake v0.0.13 h1:xLDe/moqgWpRoptHzI4packAWzs4C16b+sVY+txNJp0= github.com/openimsdk/gomake v0.0.13/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= -github.com/openimsdk/protocol v0.0.69-alpha.12 h1:3ZdwmD1y9vcduIC8o2EZS8Ds/fByqcuEFo+NkcBzgRo= -github.com/openimsdk/protocol v0.0.69-alpha.12/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= +github.com/openimsdk/protocol v0.0.69-alpha.15 h1:iiVROC2nXTc9qPOSfE7NZckQAVCZiDxBm9yY36ULdyw= +github.com/openimsdk/protocol v0.0.69-alpha.15/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= github.com/openimsdk/tools v0.0.49-alpha.25 h1:OpRPwDZ2xWX7Zj5kyfZhryu/NfZTrsRVr2GFwu1HQHI= github.com/openimsdk/tools v0.0.49-alpha.25/go.mod h1:rwsFI1G/nBHNfiNapbven41akRDPBbH4df0Cgy6xueU= github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= diff --git a/internal/rpc/group/notification.go b/internal/rpc/group/notification.go index 4fae1ed2b..5d385d8c1 100644 --- a/internal/rpc/group/notification.go +++ b/internal/rpc/group/notification.go @@ -324,6 +324,7 @@ func (g *GroupNotificationSender) GroupInfoSetNotification(ctx context.Context, if err = g.fillOpUser(ctx, &tips.OpUser, tips.Group.GroupID); err != nil { return } + g.setVersion(ctx, &tips.GroupMemberVersion, &tips.GroupMemberVersionID, database.GroupMemberVersionName, tips.Group.GroupID) g.Notification(ctx, mcontext.GetOpUserID(ctx), tips.Group.GroupID, constant.GroupInfoSetNotification, tips, rpcclient.WithRpcGetUserName()) } @@ -337,6 +338,7 @@ func (g *GroupNotificationSender) GroupInfoSetNameNotification(ctx context.Conte if err = g.fillOpUser(ctx, &tips.OpUser, tips.Group.GroupID); err != nil { return } + g.setVersion(ctx, &tips.GroupMemberVersion, &tips.GroupMemberVersionID, database.GroupMemberVersionName, tips.Group.GroupID) g.Notification(ctx, mcontext.GetOpUserID(ctx), tips.Group.GroupID, constant.GroupInfoSetNameNotification, tips) } @@ -350,6 +352,7 @@ func (g *GroupNotificationSender) GroupInfoSetAnnouncementNotification(ctx conte if err = g.fillOpUser(ctx, &tips.OpUser, tips.Group.GroupID); err != nil { return } + g.setVersion(ctx, &tips.GroupMemberVersion, &tips.GroupMemberVersionID, database.GroupMemberVersionName, tips.Group.GroupID) g.Notification(ctx, mcontext.GetOpUserID(ctx), tips.Group.GroupID, constant.GroupInfoSetAnnouncementNotification, tips, rpcclient.WithRpcGetUserName()) } @@ -636,6 +639,7 @@ func (g *GroupNotificationSender) GroupMutedNotification(ctx context.Context, gr if err = g.fillOpUser(ctx, &tips.OpUser, tips.Group.GroupID); err != nil { return } + g.setVersion(ctx, &tips.GroupMemberVersion, &tips.GroupMemberVersionID, database.GroupMemberVersionName, groupID) g.Notification(ctx, mcontext.GetOpUserID(ctx), group.GroupID, constant.GroupMutedNotification, tips) } @@ -663,6 +667,7 @@ func (g *GroupNotificationSender) GroupCancelMutedNotification(ctx context.Conte if err = g.fillOpUser(ctx, &tips.OpUser, tips.Group.GroupID); err != nil { return } + g.setVersion(ctx, &tips.GroupMemberVersion, &tips.GroupMemberVersionID, database.GroupMemberVersionName, groupID) g.Notification(ctx, mcontext.GetOpUserID(ctx), group.GroupID, constant.GroupCancelMutedNotification, tips) } diff --git a/internal/rpc/group/sync.go b/internal/rpc/group/sync.go index bd6fc6399..10d945a7e 100644 --- a/internal/rpc/group/sync.go +++ b/internal/rpc/group/sync.go @@ -4,10 +4,13 @@ import ( "context" "github.com/openimsdk/open-im-server/v3/internal/rpc/incrversion" "github.com/openimsdk/open-im-server/v3/pkg/authverify" + "github.com/openimsdk/open-im-server/v3/pkg/common/servererrs" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" "github.com/openimsdk/open-im-server/v3/pkg/util/hashutil" + "github.com/openimsdk/protocol/constant" pbgroup "github.com/openimsdk/protocol/group" "github.com/openimsdk/protocol/sdkws" + "slices" ) func (s *groupServer) GetFullGroupMemberUserIDs(ctx context.Context, req *pbgroup.GetFullGroupMemberUserIDsReq) (*pbgroup.GetFullGroupMemberUserIDsResp, error) { @@ -53,12 +56,36 @@ func (s *groupServer) GetFullJoinGroupIDs(ctx context.Context, req *pbgroup.GetF } func (s *groupServer) GetIncrementalGroupMember(ctx context.Context, req *pbgroup.GetIncrementalGroupMemberReq) (*pbgroup.GetIncrementalGroupMemberResp, error) { + group, err := s.db.TakeGroup(ctx, req.GroupID) + if err != nil { + return nil, err + } + if group.Status == constant.GroupStatusDismissed { + return nil, servererrs.ErrDismissedAlready.Wrap() + } opt := incrversion.Option[*sdkws.GroupMemberFullInfo, pbgroup.GetIncrementalGroupMemberResp]{ - Ctx: ctx, - VersionKey: req.GroupID, - VersionID: req.VersionID, - VersionNumber: req.Version, - Version: s.db.FindMemberIncrVersion, + Ctx: ctx, + VersionKey: req.GroupID, + VersionID: req.VersionID, + VersionNumber: req.Version, + Version: func(ctx context.Context, groupID string, version uint, limit int) (*model.VersionLog, error) { + vl, err := s.db.FindMemberIncrVersion(ctx, groupID, version, limit) + if err != nil { + return nil, err + } + var ok bool + vl.Logs = slices.DeleteFunc(vl.Logs, func(elem model.VersionLogElem) bool { + if elem.EID == "" { + ok = true + return true + } + return false + }) + if !ok { + group = nil + } + return vl, nil + }, CacheMaxVersion: s.db.FindMaxGroupMemberVersionCache, Find: func(ctx context.Context, ids []string) ([]*sdkws.GroupMemberFullInfo, error) { return s.getGroupMembersInfo(ctx, req.GroupID, ids) @@ -75,7 +102,22 @@ func (s *groupServer) GetIncrementalGroupMember(ctx context.Context, req *pbgrou } }, } - return opt.Build() + resp, err := opt.Build() + if err != nil { + return nil, err + } + if group != nil { + count, err := s.db.FindGroupMemberNum(ctx, group.GroupID) + if err != nil { + return nil, err + } + owner, err := s.db.TakeGroupOwner(ctx, group.GroupID) + if err != nil { + return nil, err + } + resp.Group = s.groupDB2PB(group, owner.UserID, count) + } + return resp, nil } func (s *groupServer) GetIncrementalJoinGroup(ctx context.Context, req *pbgroup.GetIncrementalJoinGroupReq) (*pbgroup.GetIncrementalJoinGroupResp, error) { diff --git a/pkg/common/storage/controller/group.go b/pkg/common/storage/controller/group.go index cb64bc73a..71bf6ead2 100644 --- a/pkg/common/storage/controller/group.go +++ b/pkg/common/storage/controller/group.go @@ -243,17 +243,10 @@ func (g *groupDatabase) UpdateGroup(ctx context.Context, groupID string, data ma if err := g.groupDB.UpdateMap(ctx, groupID, data); err != nil { return err } - userIDs, err := g.cache.GetGroupMemberIDs(ctx, groupID) - if err != nil { + if err := g.groupMemberDB.MemberGroupIncrVersion(ctx, groupID, []string{""}, model.VersionStateUpdate); err != nil { return err } - for _, userID := range userIDs { - if err := g.groupMemberDB.JoinGroupIncrVersion(ctx, userID, []string{groupID}, model.VersionStateUpdate); err != nil { - return err - } - - } - return g.cache.CloneGroupCache().DelGroupsInfo(groupID).DelMaxJoinGroupVersion(userIDs...).ChainExecDel(ctx) + return g.cache.CloneGroupCache().DelGroupsInfo(groupID).DelMaxGroupMemberVersion(groupID).ChainExecDel(ctx) }) } @@ -263,11 +256,11 @@ func (g *groupDatabase) DismissGroup(ctx context.Context, groupID string, delete if err := g.groupDB.UpdateStatus(ctx, groupID, constant.GroupStatusDismissed); err != nil { return err } - userIDs, err := g.cache.GetGroupMemberIDs(ctx, groupID) - if err != nil { - return err - } if deleteMember { + userIDs, err := g.cache.GetGroupMemberIDs(ctx, groupID) + if err != nil { + return err + } if err := g.groupMemberDB.Delete(ctx, groupID, nil); err != nil { return err } @@ -277,13 +270,18 @@ func (g *groupDatabase) DismissGroup(ctx context.Context, groupID string, delete DelGroupMembersHash(groupID). DelGroupAllRoleLevel(groupID). DelGroupMembersInfo(groupID, userIDs...). - DelMaxGroupMemberVersion(groupID) - } - c = c.DelMaxJoinGroupVersion(userIDs...) - if len(userIDs) > 0 { - if err := g.groupMemberDB.JoinGroupIncrVersion(ctx, groupID, userIDs, model.VersionStateDelete); err != nil { + DelMaxGroupMemberVersion(groupID). + DelMaxJoinGroupVersion(userIDs...) + for _, userID := range userIDs { + if err := g.groupMemberDB.JoinGroupIncrVersion(ctx, userID, []string{groupID}, model.VersionStateDelete); err != nil { + return err + } + } + } else { + if err := g.groupMemberDB.MemberGroupIncrVersion(ctx, groupID, []string{""}, model.VersionStateUpdate); err != nil { return err } + c = c.DelMaxGroupMemberVersion(groupID) } return c.DelGroupsInfo(groupID).ChainExecDel(ctx) }) @@ -411,12 +409,16 @@ func (g *groupDatabase) TransferGroupOwner(ctx context.Context, groupID string, return c.DelGroupMembersInfo(groupID, oldOwnerUserID, newOwnerUserID). DelGroupAllRoleLevel(groupID). DelGroupMembersHash(groupID). - DelJoinedGroupID(oldOwnerUserID, newOwnerUserID). + DelMaxGroupMemberVersion(groupID). + DelGroupMemberIDs(groupID). ChainExecDel(ctx) }) } func (g *groupDatabase) UpdateGroupMember(ctx context.Context, groupID string, userID string, data map[string]any) error { + if len(data) == 0 { + return nil + } return g.ctxTx.Transaction(ctx, func(ctx context.Context) error { if err := g.groupMemberDB.Update(ctx, groupID, userID, data); err != nil { return err @@ -424,9 +426,9 @@ func (g *groupDatabase) UpdateGroupMember(ctx context.Context, groupID string, u c := g.cache.CloneGroupCache() c = c.DelGroupMembersInfo(groupID, userID) if g.groupMemberDB.IsUpdateRoleLevel(data) { - c = c.DelGroupAllRoleLevel(groupID) + c = c.DelGroupAllRoleLevel(groupID).DelGroupMemberIDs(groupID) } - c = c.DelMaxGroupMemberVersion(groupID).DelMaxJoinGroupVersion(userID) + c = c.DelMaxGroupMemberVersion(groupID) return c.ChainExecDel(ctx) }) } @@ -439,9 +441,9 @@ func (g *groupDatabase) UpdateGroupMembers(ctx context.Context, data []*common.B return err } if g.groupMemberDB.IsUpdateRoleLevel(item.Map) { - c = c.DelGroupAllRoleLevel(item.GroupID) + c = c.DelGroupAllRoleLevel(item.GroupID).DelGroupMemberIDs(item.GroupID) } - c = c.DelGroupMembersInfo(item.GroupID, item.UserID).DelGroupMembersHash(item.GroupID) + c = c.DelGroupMembersInfo(item.GroupID, item.UserID).DelMaxGroupMemberVersion(item.GroupID).DelGroupMembersHash(item.GroupID) } return c.ChainExecDel(ctx) }) @@ -501,14 +503,6 @@ func (g *groupDatabase) FindJoinIncrVersion(ctx context.Context, userID string, return g.groupMemberDB.FindJoinIncrVersion(ctx, userID, version, limit) } -//func (g *groupDatabase) FindSortGroupMemberUserIDs(ctx context.Context, groupID string) ([]string, error) { -// return g.cache.FindSortGroupMemberUserIDs(ctx, groupID) -//} -// -//func (g *groupDatabase) FindSortJoinGroupIDs(ctx context.Context, userID string) ([]string, error) { -// return g.cache.FindSortJoinGroupIDs(ctx, userID) -//} - func (g *groupDatabase) FindMaxGroupMemberVersionCache(ctx context.Context, groupID string) (*model.VersionLog, error) { return g.cache.FindMaxGroupMemberVersion(ctx, groupID) } diff --git a/pkg/common/storage/database/group_member.go b/pkg/common/storage/database/group_member.go index 0051d694f..c49319649 100644 --- a/pkg/common/storage/database/group_member.go +++ b/pkg/common/storage/database/group_member.go @@ -35,6 +35,7 @@ type GroupMember interface { FindUserManagedGroupID(ctx context.Context, userID string) (groupIDs []string, err error) IsUpdateRoleLevel(data map[string]any) bool JoinGroupIncrVersion(ctx context.Context, userID string, groupIDs []string, state int32) error + MemberGroupIncrVersion(ctx context.Context, groupID string, userIDs []string, state int32) error FindMemberIncrVersion(ctx context.Context, groupID string, version uint, limit int) (*model.VersionLog, error) FindJoinIncrVersion(ctx context.Context, userID string, version uint, limit int) (*model.VersionLog, error) } diff --git a/pkg/common/storage/database/mgo/friend.go b/pkg/common/storage/database/mgo/friend.go index b74df1c19..4c3c4b97b 100644 --- a/pkg/common/storage/database/mgo/friend.go +++ b/pkg/common/storage/database/mgo/friend.go @@ -18,6 +18,8 @@ import ( "context" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" + "go.mongodb.org/mongo-driver/bson/primitive" + "time" "github.com/openimsdk/tools/db/mongoutil" "github.com/openimsdk/tools/db/pagination" @@ -53,11 +55,19 @@ func NewFriendMongo(db *mongo.Database) (database.Friend, error) { } func (f *FriendMgo) friendSort() any { - return bson.D{{"is_pinned", -1}, {"create_time", 1}, {"_id", 1}} + return bson.D{{"is_pinned", -1}, {"_id", 1}} } // Create inserts multiple friend records. func (f *FriendMgo) Create(ctx context.Context, friends []*model.Friend) error { + for i, friend := range friends { + if friend.ID.IsZero() { + friends[i].ID = primitive.NewObjectID() + } + if friend.CreateTime.IsZero() { + friends[i].CreateTime = time.Now() + } + } return mongoutil.IncrVersion(func() error { return mongoutil.InsertMany(ctx, f.coll, friends) }, func() error { @@ -108,13 +118,43 @@ func (f *FriendMgo) UpdateRemark(ctx context.Context, ownerUserID, friendUserID, return f.UpdateByMap(ctx, ownerUserID, friendUserID, map[string]any{"remark": remark}) } +func (f *FriendMgo) fillTime(friends ...*model.Friend) { + for i, friend := range friends { + if friend.CreateTime.IsZero() { + friends[i].CreateTime = friend.ID.Timestamp() + } + } +} + +func (f *FriendMgo) findOne(ctx context.Context, filter any) (*model.Friend, error) { + friend, err := mongoutil.FindOne[*model.Friend](ctx, f.coll, filter) + if err != nil { + return nil, err + } + f.fillTime(friend) + return friend, nil +} + +func (f *FriendMgo) find(ctx context.Context, filter any) ([]*model.Friend, error) { + friends, err := mongoutil.Find[*model.Friend](ctx, f.coll, filter) + if err != nil { + return nil, err + } + f.fillTime(friends...) + return friends, nil +} + +func (f *FriendMgo) findPage(ctx context.Context, filter any, pagination pagination.Pagination, opts ...*options.FindOptions) (int64, []*model.Friend, error) { + return mongoutil.FindPage[*model.Friend](ctx, f.coll, filter, pagination, opts...) +} + // Take retrieves a single friend document. Returns an error if not found. func (f *FriendMgo) Take(ctx context.Context, ownerUserID, friendUserID string) (*model.Friend, error) { filter := bson.M{ "owner_user_id": ownerUserID, "friend_user_id": friendUserID, } - return mongoutil.FindOne[*model.Friend](ctx, f.coll, filter) + return f.findOne(ctx, filter) } // FindUserState finds the friendship status between two users. @@ -125,7 +165,7 @@ func (f *FriendMgo) FindUserState(ctx context.Context, userID1, userID2 string) {"owner_user_id": userID2, "friend_user_id": userID1}, }, } - return mongoutil.Find[*model.Friend](ctx, f.coll, filter) + return f.find(ctx, filter) } // FindFriends retrieves a list of friends for a given owner. Missing friends do not cause an error. @@ -134,7 +174,7 @@ func (f *FriendMgo) FindFriends(ctx context.Context, ownerUserID string, friendU "owner_user_id": ownerUserID, "friend_user_id": bson.M{"$in": friendUserIDs}, } - return mongoutil.Find[*model.Friend](ctx, f.coll, filter) + return f.find(ctx, filter) } // FindReversalFriends finds users who have added the specified user as a friend. @@ -143,14 +183,14 @@ func (f *FriendMgo) FindReversalFriends(ctx context.Context, friendUserID string "owner_user_id": bson.M{"$in": ownerUserIDs}, "friend_user_id": friendUserID, } - return mongoutil.Find[*model.Friend](ctx, f.coll, filter) + return f.find(ctx, filter) } // FindOwnerFriends retrieves a paginated list of friends for a given owner. func (f *FriendMgo) FindOwnerFriends(ctx context.Context, ownerUserID string, pagination pagination.Pagination) (int64, []*model.Friend, error) { filter := bson.M{"owner_user_id": ownerUserID} opt := options.Find().SetSort(f.friendSort()) - return mongoutil.FindPage[*model.Friend](ctx, f.coll, filter, pagination, opt) + return f.findPage(ctx, filter, pagination, opt) } func (f *FriendMgo) FindOwnerFriendUserIds(ctx context.Context, ownerUserID string, limit int) ([]string, error) { @@ -162,7 +202,8 @@ func (f *FriendMgo) FindOwnerFriendUserIds(ctx context.Context, ownerUserID stri // FindInWhoseFriends finds users who have added the specified user as a friend, with pagination. func (f *FriendMgo) FindInWhoseFriends(ctx context.Context, friendUserID string, pagination pagination.Pagination) (int64, []*model.Friend, error) { filter := bson.M{"friend_user_id": friendUserID} - return mongoutil.FindPage[*model.Friend](ctx, f.coll, filter, pagination) + opt := options.Find().SetSort(f.friendSort()) + return f.findPage(ctx, filter, pagination, opt) } // FindFriendUserIDs retrieves a list of friend user IDs for a given owner. @@ -203,18 +244,3 @@ func (f *FriendMgo) FindFriendUserID(ctx context.Context, friendUserID string) ( } return mongoutil.Find[string](ctx, f.coll, filter, options.Find().SetProjection(bson.M{"_id": 0, "owner_user_id": 1}).SetSort(f.friendSort())) } - -//func (f *FriendMgo) SearchFriend(ctx context.Context, ownerUserID, keyword string, pagination pagination.Pagination) (int64, []*model.Friend, error) { -// filter := bson.M{ -// "owner_user_id": ownerUserID, -// } -// if keyword != "" { -// filter["$or"] = []bson.M{ -// {"remark": bson.M{"$regex": keyword, "$options": "i"}}, -// {"nickname": bson.M{"$regex": keyword, "$options": "i"}}, -// {"friend_user_id": bson.M{"$regex": keyword, "$options": "i"}}, -// } -// } -// opt := options.Find().SetSort(f.friendSort()) -// return mongoutil.FindPage[*model.Friend](ctx, f.coll, filter, pagination, opt) -//} diff --git a/pkg/common/storage/database/mgo/group_member.go b/pkg/common/storage/database/mgo/group_member.go index 95c6e51a6..99a5a3ed2 100644 --- a/pkg/common/storage/database/mgo/group_member.go +++ b/pkg/common/storage/database/mgo/group_member.go @@ -117,8 +117,6 @@ func (g *GroupMemberMgo) UpdateRoleLevel(ctx context.Context, groupID string, us return g.Update(ctx, groupID, userID, bson.M{"role_level": roleLevel}) }, func() error { return g.member.IncrVersion(ctx, groupID, []string{userID}, model.VersionStateUpdate) - }, func() error { - return g.join.IncrVersion(ctx, userID, []string{groupID}, model.VersionStateUpdate) }) } @@ -184,6 +182,10 @@ func (g *GroupMemberMgo) JoinGroupIncrVersion(ctx context.Context, userID string return g.join.IncrVersion(ctx, userID, groupIDs, state) } +func (g *GroupMemberMgo) MemberGroupIncrVersion(ctx context.Context, groupID string, userIDs []string, state int32) error { + return g.member.IncrVersion(ctx, groupID, userIDs, state) +} + func (g *GroupMemberMgo) FindMemberIncrVersion(ctx context.Context, groupID string, version uint, limit int) (*model.VersionLog, error) { return g.member.FindChangeLog(ctx, groupID, version, limit) } diff --git a/pkg/common/storage/model/friend.go b/pkg/common/storage/model/friend.go index 60a40d9c2..abcca2f2b 100644 --- a/pkg/common/storage/model/friend.go +++ b/pkg/common/storage/model/friend.go @@ -15,17 +15,19 @@ package model import ( + "go.mongodb.org/mongo-driver/bson/primitive" "time" ) // Friend represents the data structure for a friend relationship in MongoDB. type Friend struct { - OwnerUserID string `bson:"owner_user_id"` - FriendUserID string `bson:"friend_user_id"` - Remark string `bson:"remark"` - CreateTime time.Time `bson:"create_time"` - AddSource int32 `bson:"add_source"` - OperatorUserID string `bson:"operator_user_id"` - Ex string `bson:"ex"` - IsPinned bool `bson:"is_pinned"` + ID primitive.ObjectID `bson:"_id"` + OwnerUserID string `bson:"owner_user_id"` + FriendUserID string `bson:"friend_user_id"` + Remark string `bson:"remark"` + CreateTime time.Time `bson:"create_time"` + AddSource int32 `bson:"add_source"` + OperatorUserID string `bson:"operator_user_id"` + Ex string `bson:"ex"` + IsPinned bool `bson:"is_pinned"` } From f613c6dfc9933e817e2b96a8e8831eb23543a75c Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Wed, 19 Jun 2024 16:52:44 +0800 Subject: [PATCH 44/59] group --- internal/rpc/group/sync.go | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/internal/rpc/group/sync.go b/internal/rpc/group/sync.go index 10d945a7e..3793d0c7e 100644 --- a/internal/rpc/group/sync.go +++ b/internal/rpc/group/sync.go @@ -63,6 +63,7 @@ func (s *groupServer) GetIncrementalGroupMember(ctx context.Context, req *pbgrou if group.Status == constant.GroupStatusDismissed { return nil, servererrs.ErrDismissedAlready.Wrap() } + var hasGroupUpdate bool opt := incrversion.Option[*sdkws.GroupMemberFullInfo, pbgroup.GetIncrementalGroupMemberResp]{ Ctx: ctx, VersionKey: req.GroupID, @@ -73,17 +74,14 @@ func (s *groupServer) GetIncrementalGroupMember(ctx context.Context, req *pbgrou if err != nil { return nil, err } - var ok bool vl.Logs = slices.DeleteFunc(vl.Logs, func(elem model.VersionLogElem) bool { if elem.EID == "" { - ok = true + vl.LogLen-- + hasGroupUpdate = true return true } return false }) - if !ok { - group = nil - } return vl, nil }, CacheMaxVersion: s.db.FindMaxGroupMemberVersionCache, @@ -106,7 +104,7 @@ func (s *groupServer) GetIncrementalGroupMember(ctx context.Context, req *pbgrou if err != nil { return nil, err } - if group != nil { + if resp.Full || hasGroupUpdate { count, err := s.db.FindGroupMemberNum(ctx, group.GroupID) if err != nil { return nil, err From 1ee33f36f71dba904cfe8dd75d02a84fcd347743 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Wed, 19 Jun 2024 17:38:30 +0800 Subject: [PATCH 45/59] group --- go.mod | 2 +- go.sum | 4 ++-- internal/rpc/group/notification.go | 9 +++++++-- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index d00f0b739..34987958f 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/lestrrat-go/file-rotatelogs v2.4.0+incompatible // indirect github.com/mitchellh/mapstructure v1.5.0 - github.com/openimsdk/protocol v0.0.69-alpha.15 + github.com/openimsdk/protocol v0.0.69-alpha.16 github.com/openimsdk/tools v0.0.49-alpha.25 github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 diff --git a/go.sum b/go.sum index 4c05500bd..3ab5684a9 100644 --- a/go.sum +++ b/go.sum @@ -270,8 +270,8 @@ github.com/onsi/gomega v1.25.0 h1:Vw7br2PCDYijJHSfBOWhov+8cAnUf8MfMaIOV323l6Y= github.com/onsi/gomega v1.25.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdMPGhM= github.com/openimsdk/gomake v0.0.13 h1:xLDe/moqgWpRoptHzI4packAWzs4C16b+sVY+txNJp0= github.com/openimsdk/gomake v0.0.13/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= -github.com/openimsdk/protocol v0.0.69-alpha.15 h1:iiVROC2nXTc9qPOSfE7NZckQAVCZiDxBm9yY36ULdyw= -github.com/openimsdk/protocol v0.0.69-alpha.15/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= +github.com/openimsdk/protocol v0.0.69-alpha.16 h1:ciSqm2rjBdpScpkQm3wPjAFv0YbIRp8MITRkDZWVv6c= +github.com/openimsdk/protocol v0.0.69-alpha.16/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= github.com/openimsdk/tools v0.0.49-alpha.25 h1:OpRPwDZ2xWX7Zj5kyfZhryu/NfZTrsRVr2GFwu1HQHI= github.com/openimsdk/tools v0.0.49-alpha.25/go.mod h1:rwsFI1G/nBHNfiNapbven41akRDPBbH4df0Cgy6xueU= github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= diff --git a/internal/rpc/group/notification.go b/internal/rpc/group/notification.go index 5d385d8c1..ee4f00ad3 100644 --- a/internal/rpc/group/notification.go +++ b/internal/rpc/group/notification.go @@ -477,11 +477,16 @@ func (g *GroupNotificationSender) GroupOwnerTransferredNotification(ctx context. } opUserID := mcontext.GetOpUserID(ctx) var member map[string]*sdkws.GroupMemberFullInfo - member, err = g.getGroupMemberMap(ctx, req.GroupID, []string{opUserID, req.NewOwnerUserID}) + member, err = g.getGroupMemberMap(ctx, req.GroupID, []string{opUserID, req.NewOwnerUserID, req.OldOwnerUserID}) if err != nil { return } - tips := &sdkws.GroupOwnerTransferredTips{Group: group, OpUser: member[opUserID], NewGroupOwner: member[req.NewOwnerUserID]} + tips := &sdkws.GroupOwnerTransferredTips{ + Group: group, + OpUser: member[opUserID], + NewGroupOwner: member[req.NewOwnerUserID], + OldGroupOwnerInfo: member[req.OldOwnerUserID], + } if err = g.fillOpUser(ctx, &tips.OpUser, tips.Group.GroupID); err != nil { return } From 924888dbd125de298cb2c4ea74d4307d21f57123 Mon Sep 17 00:00:00 2001 From: Gordon <46924906+FGadvancer@users.noreply.github.com> Date: Wed, 19 Jun 2024 20:17:28 +0800 Subject: [PATCH 46/59] fix: sort by id avoid unstable sort friends. --- pkg/common/storage/database/mgo/version_log.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pkg/common/storage/database/mgo/version_log.go b/pkg/common/storage/database/mgo/version_log.go index 1d70f96f5..8836742f0 100644 --- a/pkg/common/storage/database/mgo/version_log.go +++ b/pkg/common/storage/database/mgo/version_log.go @@ -8,6 +8,7 @@ import ( "github.com/openimsdk/open-im-server/v3/pkg/common/storage/versionctx" "github.com/openimsdk/tools/db/mongoutil" "github.com/openimsdk/tools/errs" + "github.com/openimsdk/tools/log" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/bson/primitive" "go.mongodb.org/mongo-driver/mongo" @@ -169,7 +170,9 @@ func (l *VersionLogMgo) FindChangeLog(ctx context.Context, dId string, version u } else if !errors.Is(err, mongo.ErrNoDocuments) { return nil, err } + log.ZDebug(ctx, "init doc", "dId", dId) if res, err := l.initDoc(ctx, dId, nil, 0, time.Now()); err == nil { + log.ZDebug(ctx, "init doc success", "dId", dId) return res, nil } else if mongo.IsDuplicateKeyError(err) { return l.findChangeLog(ctx, dId, version, limit) From 5006af5b54ba47db5ada3b06b1298836dc5ee064 Mon Sep 17 00:00:00 2001 From: Gordon <46924906+FGadvancer@users.noreply.github.com> Date: Wed, 19 Jun 2024 20:28:09 +0800 Subject: [PATCH 47/59] fix: sort by id avoid unstable sort friends. --- pkg/common/storage/database/mgo/group_member.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pkg/common/storage/database/mgo/group_member.go b/pkg/common/storage/database/mgo/group_member.go index 99a5a3ed2..30f8d63b9 100644 --- a/pkg/common/storage/database/mgo/group_member.go +++ b/pkg/common/storage/database/mgo/group_member.go @@ -18,6 +18,7 @@ import ( "context" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" + "github.com/openimsdk/tools/log" "github.com/openimsdk/protocol/constant" "github.com/openimsdk/tools/db/mongoutil" @@ -187,9 +188,11 @@ func (g *GroupMemberMgo) MemberGroupIncrVersion(ctx context.Context, groupID str } func (g *GroupMemberMgo) FindMemberIncrVersion(ctx context.Context, groupID string, version uint, limit int) (*model.VersionLog, error) { + log.ZDebug(ctx, "find member incr version", "groupID", groupID, "version", version) return g.member.FindChangeLog(ctx, groupID, version, limit) } func (g *GroupMemberMgo) FindJoinIncrVersion(ctx context.Context, userID string, version uint, limit int) (*model.VersionLog, error) { + log.ZDebug(ctx, "find join incr version", "userID", userID, "version", version) return g.join.FindChangeLog(ctx, userID, version, limit) } From 4f359b733556290ca137b4529bc74bd7ea11c59d Mon Sep 17 00:00:00 2001 From: Gordon <46924906+FGadvancer@users.noreply.github.com> Date: Wed, 19 Jun 2024 20:35:32 +0800 Subject: [PATCH 48/59] fix: sort by id avoid unstable sort friends. --- pkg/common/storage/cache/redis/group.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/common/storage/cache/redis/group.go b/pkg/common/storage/cache/redis/group.go index c4598567d..589678c50 100644 --- a/pkg/common/storage/cache/redis/group.go +++ b/pkg/common/storage/cache/redis/group.go @@ -446,7 +446,7 @@ func (g *GroupCacheRedis) DelMaxJoinGroupVersion(userIDs ...string) cache.GroupC func (g *GroupCacheRedis) FindMaxGroupMemberVersion(ctx context.Context, groupID string) (*model.VersionLog, error) { return getCache(ctx, g.rcClient, g.getGroupMemberMaxVersionKey(groupID), g.expireTime, func(ctx context.Context) (*model.VersionLog, error) { - return g.groupMemberDB.FindJoinIncrVersion(ctx, groupID, 0, 0) + return g.groupMemberDB.FindMemberIncrVersion(ctx, groupID, 0, 0) }) } From 03d4564596c746f81ab1e95e61a3cc6532aeace1 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Thu, 20 Jun 2024 14:55:59 +0800 Subject: [PATCH 49/59] user version --- go.mod | 6 +- go.sum | 4 +- internal/rpc/friend/sync.go | 16 +++ internal/rpc/group/group.go | 6 +- internal/rpc/user/user.go | 122 +++++++++++++++------- pkg/common/storage/controller/friend.go | 9 +- pkg/common/storage/controller/group.go | 8 ++ pkg/common/storage/database/friend.go | 2 + pkg/common/storage/database/mgo/friend.go | 4 + 9 files changed, 129 insertions(+), 48 deletions(-) diff --git a/go.mod b/go.mod index 34987958f..2614e0f32 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/lestrrat-go/file-rotatelogs v2.4.0+incompatible // indirect github.com/mitchellh/mapstructure v1.5.0 - github.com/openimsdk/protocol v0.0.69-alpha.16 + github.com/openimsdk/protocol v0.0.69-alpha.17 github.com/openimsdk/tools v0.0.49-alpha.25 github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 @@ -177,6 +177,4 @@ require ( gopkg.in/ini.v1 v1.67.0 // indirect ) -//replace ( -// github.com/openimsdk/protocol => /Users/chao/Desktop/project/protocol -//) +//replace github.com/openimsdk/protocol => /Users/chao/Desktop/project/protocol diff --git a/go.sum b/go.sum index 3ab5684a9..fe4f0c390 100644 --- a/go.sum +++ b/go.sum @@ -270,8 +270,8 @@ github.com/onsi/gomega v1.25.0 h1:Vw7br2PCDYijJHSfBOWhov+8cAnUf8MfMaIOV323l6Y= github.com/onsi/gomega v1.25.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdMPGhM= github.com/openimsdk/gomake v0.0.13 h1:xLDe/moqgWpRoptHzI4packAWzs4C16b+sVY+txNJp0= github.com/openimsdk/gomake v0.0.13/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= -github.com/openimsdk/protocol v0.0.69-alpha.16 h1:ciSqm2rjBdpScpkQm3wPjAFv0YbIRp8MITRkDZWVv6c= -github.com/openimsdk/protocol v0.0.69-alpha.16/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= +github.com/openimsdk/protocol v0.0.69-alpha.17 h1:pEag4ZdlovE+AyLsw1VYFU/3sk6ayvGdPzgufQfKf9M= +github.com/openimsdk/protocol v0.0.69-alpha.17/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= github.com/openimsdk/tools v0.0.49-alpha.25 h1:OpRPwDZ2xWX7Zj5kyfZhryu/NfZTrsRVr2GFwu1HQHI= github.com/openimsdk/tools v0.0.49-alpha.25/go.mod h1:rwsFI1G/nBHNfiNapbven41akRDPBbH4df0Cgy6xueU= github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= diff --git a/internal/rpc/friend/sync.go b/internal/rpc/friend/sync.go index faaa987f2..684894609 100644 --- a/internal/rpc/friend/sync.go +++ b/internal/rpc/friend/sync.go @@ -11,6 +11,22 @@ import ( "github.com/openimsdk/protocol/relation" ) +func (s *friendServer) NotificationUserInfoUpdate(ctx context.Context, req *relation.NotificationUserInfoUpdateReq) (*relation.NotificationUserInfoUpdateResp, error) { + userIDs, err := s.db.FindFriendUserIDs(ctx, req.UserID) + if err != nil { + return nil, err + } + for _, userID := range userIDs { + if err := s.db.OwnerIncrVersion(ctx, userID, []string{req.UserID}, model.VersionStateUpdate); err != nil { + return nil, err + } + } + for _, userID := range userIDs { + s.notificationSender.FriendInfoUpdatedNotification(ctx, req.UserID, userID) + } + return &relation.NotificationUserInfoUpdateResp{}, nil +} + func (s *friendServer) GetFullFriendUserIDs(ctx context.Context, req *relation.GetFullFriendUserIDsReq) (*relation.GetFullFriendUserIDsResp, error) { vl, err := s.db.FindMaxFriendVersionCache(ctx, req.UserID) if err != nil { diff --git a/internal/rpc/group/group.go b/internal/rpc/group/group.go index 4d429b3d3..5b82b8ceb 100644 --- a/internal/rpc/group/group.go +++ b/internal/rpc/group/group.go @@ -132,13 +132,17 @@ func (s *groupServer) NotificationUserInfoUpdate(ctx context.Context, req *pbgro } groupIDs = append(groupIDs, member.GroupID) } + for _, groupID := range groupIDs { + if err := s.db.MemberGroupIncrVersion(ctx, groupID, []string{req.UserID}, model.VersionStateUpdate); err != nil { + return nil, err + } + } for _, groupID := range groupIDs { s.notification.GroupMemberInfoSetNotification(ctx, groupID, req.UserID) } if err = s.db.DeleteGroupMemberHash(ctx, groupIDs); err != nil { return nil, err } - return &pbgroup.NotificationUserInfoUpdateResp{}, nil } diff --git a/internal/rpc/user/user.go b/internal/rpc/user/user.go index 96f784fab..211b360b7 100644 --- a/internal/rpc/user/user.go +++ b/internal/rpc/user/user.go @@ -16,10 +16,7 @@ package user import ( "context" - "math/rand" - "strings" - "time" - + "errors" "github.com/openimsdk/open-im-server/v3/internal/rpc/friend" "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/redis" @@ -27,7 +24,13 @@ import ( tablerelation "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" "github.com/openimsdk/open-im-server/v3/pkg/common/webhook" "github.com/openimsdk/open-im-server/v3/pkg/localcache" + "github.com/openimsdk/protocol/group" + friendpb "github.com/openimsdk/protocol/relation" "github.com/openimsdk/tools/db/redisutil" + "math/rand" + "strings" + "sync" + "time" "github.com/openimsdk/open-im-server/v3/pkg/authverify" "github.com/openimsdk/open-im-server/v3/pkg/common/convert" @@ -120,7 +123,8 @@ func (s *userServer) GetDesignateUsers(ctx context.Context, req *pbuser.GetDesig // deprecated: -// UpdateUserInfo +//UpdateUserInfo + func (s *userServer) UpdateUserInfo(ctx context.Context, req *pbuser.UpdateUserInfoReq) (resp *pbuser.UpdateUserInfoResp, err error) { resp = &pbuser.UpdateUserInfoResp{} err = authverify.CheckAccessV3(ctx, req.UserInfo.UserID, s.config.Share.IMAdminUserID) @@ -131,31 +135,33 @@ func (s *userServer) UpdateUserInfo(ctx context.Context, req *pbuser.UpdateUserI if err := s.webhookBeforeUpdateUserInfo(ctx, &s.config.WebhooksConfig.BeforeUpdateUserInfo, req); err != nil { return nil, err } - data := convert.UserPb2DBMap(req.UserInfo) - if err := s.db.UpdateByMap(ctx, req.UserInfo.UserID, data); err != nil { - return nil, err - } - s.friendNotificationSender.UserInfoUpdatedNotification(ctx, req.UserInfo.UserID) - friends, err := s.friendRpcClient.GetFriendIDs(ctx, req.UserInfo.UserID) + oldUser, err := s.db.GetUserByID(ctx, req.UserInfo.UserID) if err != nil { return nil, err } - if req.UserInfo.Nickname != "" || req.UserInfo.FaceURL != "" { - if err = s.groupRpcClient.NotificationUserInfoUpdate(ctx, req.UserInfo.UserID); err != nil { - return nil, err - } - } - for _, friendID := range friends { - s.friendNotificationSender.FriendInfoUpdatedNotification(ctx, req.UserInfo.UserID, friendID) + if err := s.db.UpdateByMap(ctx, req.UserInfo.UserID, data); err != nil { + return nil, err } + s.friendNotificationSender.UserInfoUpdatedNotification(ctx, req.UserInfo.UserID) + //friends, err := s.friendRpcClient.GetFriendIDs(ctx, req.UserInfo.UserID) + //if err != nil { + // return nil, err + //} + //if req.UserInfo.Nickname != "" || req.UserInfo.FaceURL != "" { + // if err = s.NotificationUserInfoUpdate(ctx, req.UserInfo.UserID,oldUser); err != nil { + // return nil, err + // } + //} + //for _, friendID := range friends { + // s.friendNotificationSender.FriendInfoUpdatedNotification(ctx, req.UserInfo.UserID, friendID) + //} s.webhookAfterUpdateUserInfo(ctx, &s.config.WebhooksConfig.AfterUpdateUserInfo, req) - if err = s.groupRpcClient.NotificationUserInfoUpdate(ctx, req.UserInfo.UserID); err != nil { + if err = s.NotificationUserInfoUpdate(ctx, req.UserInfo.UserID, oldUser); err != nil { return nil, err } return resp, nil } - func (s *userServer) UpdateUserInfoEx(ctx context.Context, req *pbuser.UpdateUserInfoExReq) (resp *pbuser.UpdateUserInfoExResp, err error) { resp = &pbuser.UpdateUserInfoExResp{} err = authverify.CheckAccessV3(ctx, req.UserInfo.UserID, s.config.Share.IMAdminUserID) @@ -165,30 +171,33 @@ func (s *userServer) UpdateUserInfoEx(ctx context.Context, req *pbuser.UpdateUse if err = s.webhookBeforeUpdateUserInfoEx(ctx, &s.config.WebhooksConfig.BeforeUpdateUserInfoEx, req); err != nil { return nil, err } + oldUser, err := s.db.GetUserByID(ctx, req.UserInfo.UserID) + if err != nil { + return nil, err + } data := convert.UserPb2DBMapEx(req.UserInfo) if err = s.db.UpdateByMap(ctx, req.UserInfo.UserID, data); err != nil { return nil, err } s.friendNotificationSender.UserInfoUpdatedNotification(ctx, req.UserInfo.UserID) - friends, err := s.friendRpcClient.GetFriendIDs(ctx, req.UserInfo.UserID) - if err != nil { - return nil, err - } - if req.UserInfo.Nickname != nil || req.UserInfo.FaceURL != nil { - if err := s.groupRpcClient.NotificationUserInfoUpdate(ctx, req.UserInfo.UserID); err != nil { - return nil, err - } - } - for _, friendID := range friends { - s.friendNotificationSender.FriendInfoUpdatedNotification(ctx, req.UserInfo.UserID, friendID) - } + //friends, err := s.friendRpcClient.GetFriendIDs(ctx, req.UserInfo.UserID) + //if err != nil { + // return nil, err + //} + //if req.UserInfo.Nickname != nil || req.UserInfo.FaceURL != nil { + // if err := s.NotificationUserInfoUpdate(ctx, req.UserInfo.UserID); err != nil { + // return nil, err + // } + //} + //for _, friendID := range friends { + // s.friendNotificationSender.FriendInfoUpdatedNotification(ctx, req.UserInfo.UserID, friendID) + //} s.webhookAfterUpdateUserInfoEx(ctx, &s.config.WebhooksConfig.AfterUpdateUserInfoEx, req) - if err := s.groupRpcClient.NotificationUserInfoUpdate(ctx, req.UserInfo.UserID); err != nil { + if err := s.NotificationUserInfoUpdate(ctx, req.UserInfo.UserID, oldUser); err != nil { return nil, err } return resp, nil } - func (s *userServer) SetGlobalRecvMessageOpt(ctx context.Context, req *pbuser.SetGlobalRecvMessageOptReq) (resp *pbuser.SetGlobalRecvMessageOptResp, err error) { resp = &pbuser.SetGlobalRecvMessageOptResp{} if _, err := s.db.FindWithError(ctx, []string{req.UserID}); err != nil { @@ -247,6 +256,7 @@ func (s *userServer) GetPaginationUsers(ctx context.Context, req *pbuser.GetPagi return &pbuser.GetPaginationUsersResp{Total: int32(total), Users: convert.UsersDB2Pb(users)}, err } + } func (s *userServer) UserRegister(ctx context.Context, req *pbuser.UserRegisterReq) (resp *pbuser.UserRegisterResp, err error) { @@ -343,8 +353,7 @@ func (s *userServer) SubscribeOrCancelUsersStatus(ctx context.Context, req *pbus // GetUserStatus Get the online status of the user. func (s *userServer) GetUserStatus(ctx context.Context, req *pbuser.GetUserStatusReq) (resp *pbuser.GetUserStatusResp, - err error, -) { + err error) { onlineStatusList, err := s.db.GetUserStatus(ctx, req.UserIDs) if err != nil { return nil, err @@ -354,8 +363,7 @@ func (s *userServer) GetUserStatus(ctx context.Context, req *pbuser.GetUserStatu // SetUserStatus Synchronize user's online status. func (s *userServer) SetUserStatus(ctx context.Context, req *pbuser.SetUserStatusReq) (resp *pbuser.SetUserStatusResp, - err error, -) { + err error) { err = s.db.SetUserStatus(ctx, req.UserID, req.Status, req.PlatformID) if err != nil { return nil, err @@ -379,8 +387,7 @@ func (s *userServer) SetUserStatus(ctx context.Context, req *pbuser.SetUserStatu // GetSubscribeUsersStatus Get the online status of subscribers. func (s *userServer) GetSubscribeUsersStatus(ctx context.Context, - req *pbuser.GetSubscribeUsersStatusReq, -) (*pbuser.GetSubscribeUsersStatusResp, error) { + req *pbuser.GetSubscribeUsersStatusReq) (*pbuser.GetSubscribeUsersStatusResp, error) { userList, err := s.db.GetAllSubscribeList(ctx, req.UserID) if err != nil { return nil, err @@ -469,6 +476,7 @@ func (s *userServer) ProcessUserCommandUpdate(ctx context.Context, req *pbuser.P } func (s *userServer) ProcessUserCommandGet(ctx context.Context, req *pbuser.ProcessUserCommandGetReq) (*pbuser.ProcessUserCommandGetResp, error) { + err := authverify.CheckAccessV3(ctx, req.UserID, s.config.Share.IMAdminUserID) if err != nil { return nil, err @@ -687,6 +695,40 @@ func (s *userServer) userModelToResp(users []*tablerelation.User, pagination pag return &pbuser.SearchNotificationAccountResp{Total: total, NotificationAccounts: notificationAccounts} } +func (s *userServer) NotificationUserInfoUpdate(ctx context.Context, userID string, oldUser *tablerelation.User) error { + user, err := s.db.GetUserByID(ctx, userID) + if err != nil { + return err + } + if user.Nickname == oldUser.Nickname && user.FaceURL == oldUser.FaceURL { + return nil + } + oldUserInfo := convert.UserDB2Pb(oldUser) + newUserInfo := convert.UserDB2Pb(user) + var wg sync.WaitGroup + var es [2]error + wg.Add(len(es)) + go func() { + defer wg.Done() + _, es[0] = s.groupRpcClient.Client.NotificationUserInfoUpdate(ctx, &group.NotificationUserInfoUpdateReq{ + UserID: userID, + OldUserInfo: oldUserInfo, + NewUserInfo: newUserInfo, + }) + }() + + go func() { + defer wg.Done() + _, es[1] = s.friendRpcClient.Client.NotificationUserInfoUpdate(ctx, &friendpb.NotificationUserInfoUpdateReq{ + UserID: userID, + OldUserInfo: oldUserInfo, + NewUserInfo: newUserInfo, + }) + }() + wg.Wait() + return errors.Join(es[:]...) +} + func (s *userServer) SortQuery(ctx context.Context, req *pbuser.SortQueryReq) (*pbuser.SortQueryResp, error) { users, err := s.db.SortQuery(ctx, req.UserIDName, req.Asc) if err != nil { diff --git a/pkg/common/storage/controller/friend.go b/pkg/common/storage/controller/friend.go index 42c230598..4d3a96f12 100644 --- a/pkg/common/storage/controller/friend.go +++ b/pkg/common/storage/controller/friend.go @@ -86,7 +86,7 @@ type FriendDatabase interface { FindFriendUserID(ctx context.Context, friendUserID string) ([]string, error) - //SearchFriend(ctx context.Context, ownerUserID, keyword string, pagination pagination.Pagination) (int64, []*model.Friend, error) + OwnerIncrVersion(ctx context.Context, ownerUserID string, friendUserIDs []string, state int32) error } type friendDatabase struct { @@ -379,3 +379,10 @@ func (f *friendDatabase) FindFriendUserID(ctx context.Context, friendUserID stri //func (f *friendDatabase) SearchFriend(ctx context.Context, ownerUserID, keyword string, pagination pagination.Pagination) (int64, []*model.Friend, error) { // return f.friend.SearchFriend(ctx, ownerUserID, keyword, pagination) //} + +func (f *friendDatabase) OwnerIncrVersion(ctx context.Context, ownerUserID string, friendUserIDs []string, state int32) error { + if err := f.friend.IncrVersion(ctx, ownerUserID, friendUserIDs, state); err != nil { + return err + } + return f.cache.DelMaxFriendVersion(ownerUserID).ChainExecDel(ctx) +} diff --git a/pkg/common/storage/controller/group.go b/pkg/common/storage/controller/group.go index 71bf6ead2..27b6f4366 100644 --- a/pkg/common/storage/controller/group.go +++ b/pkg/common/storage/controller/group.go @@ -109,6 +109,7 @@ type GroupDatabase interface { FindMemberIncrVersion(ctx context.Context, groupID string, version uint, limit int) (*model.VersionLog, error) FindJoinIncrVersion(ctx context.Context, userID string, version uint, limit int) (*model.VersionLog, error) + MemberGroupIncrVersion(ctx context.Context, groupID string, userIDs []string, state int32) error //FindSortGroupMemberUserIDs(ctx context.Context, groupID string) ([]string, error) //FindSortJoinGroupIDs(ctx context.Context, userID string) ([]string, error) @@ -518,3 +519,10 @@ func (g *groupDatabase) SearchJoinGroup(ctx context.Context, userID string, keyw } return g.groupDB.SearchJoin(ctx, groupIDs, keyword, pagination) } + +func (g *groupDatabase) MemberGroupIncrVersion(ctx context.Context, groupID string, userIDs []string, state int32) error { + if err := g.groupMemberDB.MemberGroupIncrVersion(ctx, groupID, userIDs, state); err != nil { + return err + } + return g.cache.DelMaxGroupMemberVersion(groupID).ChainExecDel(ctx) +} diff --git a/pkg/common/storage/database/friend.go b/pkg/common/storage/database/friend.go index 1c9cd1033..b596411fc 100644 --- a/pkg/common/storage/database/friend.go +++ b/pkg/common/storage/database/friend.go @@ -55,4 +55,6 @@ type Friend interface { //SearchFriend(ctx context.Context, ownerUserID, keyword string, pagination pagination.Pagination) (int64, []*model.Friend, error) FindOwnerFriendUserIds(ctx context.Context, ownerUserID string, limit int) ([]string, error) + + IncrVersion(ctx context.Context, ownerUserID string, friendUserIDs []string, state int32) error } diff --git a/pkg/common/storage/database/mgo/friend.go b/pkg/common/storage/database/mgo/friend.go index 4c3c4b97b..7f456fbda 100644 --- a/pkg/common/storage/database/mgo/friend.go +++ b/pkg/common/storage/database/mgo/friend.go @@ -244,3 +244,7 @@ func (f *FriendMgo) FindFriendUserID(ctx context.Context, friendUserID string) ( } return mongoutil.Find[string](ctx, f.coll, filter, options.Find().SetProjection(bson.M{"_id": 0, "owner_user_id": 1}).SetSort(f.friendSort())) } + +func (f *FriendMgo) IncrVersion(ctx context.Context, ownerUserID string, friendUserIDs []string, state int32) error { + return f.owner.IncrVersion(ctx, ownerUserID, friendUserIDs, state) +} From 2d2941a52645d1bd104054d87b20dbbca862c477 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Thu, 20 Jun 2024 15:56:02 +0800 Subject: [PATCH 50/59] seq --- pkg/common/storage/controller/msg.go | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/pkg/common/storage/controller/msg.go b/pkg/common/storage/controller/msg.go index 3a29f6786..de8c111b3 100644 --- a/pkg/common/storage/controller/msg.go +++ b/pkg/common/storage/controller/msg.go @@ -340,11 +340,6 @@ func (db *commonMsgDatabase) DeleteMessagesFromCache(ctx context.Context, conver } func (db *commonMsgDatabase) BatchInsertChat2Cache(ctx context.Context, conversationID string, msgs []*sdkws.MsgData) (seq int64, isNew bool, err error) { - currentMaxSeq, err := db.seqConversation.Malloc(ctx, conversationID, int64(len(msgs))) - if err != nil { - log.ZError(ctx, "storage.seq.Malloc", err) - return 0, false, err - } lenList := len(msgs) if int64(lenList) > db.msgTable.GetSingleGocMsgNum() { return 0, false, errs.New("message count exceeds limit", "limit", db.msgTable.GetSingleGocMsgNum()).Wrap() @@ -352,6 +347,12 @@ func (db *commonMsgDatabase) BatchInsertChat2Cache(ctx context.Context, conversa if lenList < 1 { return 0, false, errs.New("no messages to insert", "minCount", 1).Wrap() } + currentMaxSeq, err := db.seqConversation.Malloc(ctx, conversationID, int64(len(msgs))) + if err != nil { + log.ZError(ctx, "storage.seq.Malloc", err) + return 0, false, err + } + isNew = currentMaxSeq == 0 lastMaxSeq := currentMaxSeq userSeqMap := make(map[string]int64) for _, m := range msgs { From 8e3890ed8824e5e61994adc50b8253457db7c0af Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Fri, 21 Jun 2024 18:24:30 +0800 Subject: [PATCH 51/59] seq --- pkg/common/storage/cache/redis/seq.go | 64 +++++++++++-------- pkg/common/storage/database/mgo/seq_user.go | 71 +++++++++++++++++++++ pkg/common/storage/database/name.go | 1 + pkg/common/storage/database/seq_user.go | 12 ++++ pkg/common/storage/model/seq_user.go | 9 +++ 5 files changed, 130 insertions(+), 27 deletions(-) create mode 100644 pkg/common/storage/database/mgo/seq_user.go create mode 100644 pkg/common/storage/database/seq_user.go create mode 100644 pkg/common/storage/model/seq_user.go diff --git a/pkg/common/storage/cache/redis/seq.go b/pkg/common/storage/cache/redis/seq.go index 76dd921a5..8624cc78f 100644 --- a/pkg/common/storage/cache/redis/seq.go +++ b/pkg/common/storage/cache/redis/seq.go @@ -75,21 +75,43 @@ func (c *seqCache) getSeqs(ctx context.Context, items []string, getkey func(s st return m, nil } -func (c *seqCache) SetMaxSeq(ctx context.Context, conversationID string, maxSeq int64) error { - return c.setSeq(ctx, conversationID, maxSeq, c.getMaxSeqKey) -} - -func (c *seqCache) GetMaxSeqs(ctx context.Context, conversationIDs []string) (m map[string]int64, err error) { - return c.getSeqs(ctx, conversationIDs, c.getMaxSeqKey) -} - -func (c *seqCache) GetMaxSeq(ctx context.Context, conversationID string) (int64, error) { - return c.getSeq(ctx, conversationID, c.getMaxSeqKey) -} - -func (c *seqCache) SetMinSeq(ctx context.Context, conversationID string, minSeq int64) error { - return c.setSeq(ctx, conversationID, minSeq, c.getMinSeqKey) -} +//func (c *seqCache) SetMaxSeq(ctx context.Context, conversationID string, maxSeq int64) error { +// return c.setSeq(ctx, conversationID, maxSeq, c.getMaxSeqKey) +//} +// +//func (c *seqCache) GetMaxSeqs(ctx context.Context, conversationIDs []string) (m map[string]int64, err error) { +// return c.getSeqs(ctx, conversationIDs, c.getMaxSeqKey) +//} +// +//func (c *seqCache) GetMaxSeq(ctx context.Context, conversationID string) (int64, error) { +// return c.getSeq(ctx, conversationID, c.getMaxSeqKey) +//} +// +//func (c *seqCache) SetMinSeq(ctx context.Context, conversationID string, minSeq int64) error { +// return c.setSeq(ctx, conversationID, minSeq, c.getMinSeqKey) +//} +// +//func (c *seqCache) setSeqs(ctx context.Context, seqs map[string]int64, getkey func(key string) string) error { +// for conversationID, seq := range seqs { +// if err := c.rdb.Set(ctx, getkey(conversationID), seq, 0).Err(); err != nil { +// return errs.Wrap(err) +// } +// } +// return nil +//} +// +// +//func (c *seqCache) GetMinSeqs(ctx context.Context, conversationIDs []string) (map[string]int64, error) { +// return c.getSeqs(ctx, conversationIDs, c.getMinSeqKey) +//} +// +//func (c *seqCache) GetMinSeq(ctx context.Context, conversationID string) (int64, error) { +// return c.getSeq(ctx, conversationID, c.getMinSeqKey) +//} +// +//func (c *seqCache) SetMinSeqs(ctx context.Context, seqs map[string]int64) error { +// return c.setSeqs(ctx, seqs, c.getMinSeqKey) +//} func (c *seqCache) setSeqs(ctx context.Context, seqs map[string]int64, getkey func(key string) string) error { for conversationID, seq := range seqs { @@ -100,18 +122,6 @@ func (c *seqCache) setSeqs(ctx context.Context, seqs map[string]int64, getkey fu return nil } -func (c *seqCache) SetMinSeqs(ctx context.Context, seqs map[string]int64) error { - return c.setSeqs(ctx, seqs, c.getMinSeqKey) -} - -func (c *seqCache) GetMinSeqs(ctx context.Context, conversationIDs []string) (map[string]int64, error) { - return c.getSeqs(ctx, conversationIDs, c.getMinSeqKey) -} - -func (c *seqCache) GetMinSeq(ctx context.Context, conversationID string) (int64, error) { - return c.getSeq(ctx, conversationID, c.getMinSeqKey) -} - func (c *seqCache) GetConversationUserMinSeq(ctx context.Context, conversationID string, userID string) (int64, error) { val, err := c.rdb.Get(ctx, c.getConversationUserMinSeqKey(conversationID, userID)).Int64() if err != nil { diff --git a/pkg/common/storage/database/mgo/seq_user.go b/pkg/common/storage/database/mgo/seq_user.go new file mode 100644 index 000000000..4fa299a7a --- /dev/null +++ b/pkg/common/storage/database/mgo/seq_user.go @@ -0,0 +1,71 @@ +package mgo + +import ( + "context" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" + "github.com/openimsdk/tools/db/mongoutil" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" +) + +func NewSeqUserMongo(db *mongo.Database) (database.SeqUser, error) { + coll := db.Collection(database.SeqConversationName) + _, err := coll.Indexes().CreateOne(context.Background(), mongo.IndexModel{ + Keys: bson.D{ + {Key: "user_id", Value: 1}, + {Key: "conversation_id", Value: 1}, + }, + }) + if err != nil { + return nil, err + } + return &seqUserMongo{coll: coll}, nil +} + +type seqUserMongo struct { + coll *mongo.Collection +} + +func (s *seqUserMongo) setSeq(ctx context.Context, userID string, conversationID string, seq int64, field string) error { + filter := map[string]any{ + "user_id": userID, + "conversation_id": conversationID, + } + update := map[string]any{ + "$set": map[string]any{"field": int64(0)}, + } + opt := options.Update().SetUpsert(true) + return mongoutil.UpdateOne(ctx, s.coll, filter, update, false, opt) +} + +func (s *seqUserMongo) GetMaxSeq(ctx context.Context, userID string, conversationID string) (int64, error) { + + //TODO implement me + panic("implement me") +} + +func (s *seqUserMongo) SetMaxSeq(ctx context.Context, userID string, conversationID string, seq int64) error { + //TODO implement me + panic("implement me") +} + +func (s *seqUserMongo) GetMinSeq(ctx context.Context, userID string, conversationID string) (int64, error) { + //TODO implement me + panic("implement me") +} + +func (s *seqUserMongo) SetMinSeq(ctx context.Context, userID string, conversationID string, seq int64) error { + //TODO implement me + panic("implement me") +} + +func (s *seqUserMongo) GetReadSeq(ctx context.Context, userID string, conversationID string) (int64, error) { + //TODO implement me + panic("implement me") +} + +func (s *seqUserMongo) SetReadSeq(ctx context.Context, userID string, conversationID string, seq int64) error { + //TODO implement me + panic("implement me") +} diff --git a/pkg/common/storage/database/name.go b/pkg/common/storage/database/name.go index 68fa5af02..f47b7a6ee 100644 --- a/pkg/common/storage/database/name.go +++ b/pkg/common/storage/database/name.go @@ -15,4 +15,5 @@ const ( ObjectName = "s3" UserName = "user" SeqConversationName = "seq" + SeqUserName = "seq_user" ) diff --git a/pkg/common/storage/database/seq_user.go b/pkg/common/storage/database/seq_user.go new file mode 100644 index 000000000..3e9b29ec2 --- /dev/null +++ b/pkg/common/storage/database/seq_user.go @@ -0,0 +1,12 @@ +package database + +import "context" + +type SeqUser interface { + GetMaxSeq(ctx context.Context, userID string, conversationID string) (int64, error) + SetMaxSeq(ctx context.Context, userID string, conversationID string, seq int64) error + GetMinSeq(ctx context.Context, userID string, conversationID string) (int64, error) + SetMinSeq(ctx context.Context, userID string, conversationID string, seq int64) error + GetReadSeq(ctx context.Context, userID string, conversationID string) (int64, error) + SetReadSeq(ctx context.Context, userID string, conversationID string, seq int64) error +} diff --git a/pkg/common/storage/model/seq_user.go b/pkg/common/storage/model/seq_user.go new file mode 100644 index 000000000..845996bb8 --- /dev/null +++ b/pkg/common/storage/model/seq_user.go @@ -0,0 +1,9 @@ +package model + +type SeqUser struct { + UserID string `bson:"user_id"` + ConversationID string `bson:"conversation_id"` + MinSeq int64 `bson:"min_seq"` + MaxSeq int64 `bson:"max_seq"` + ReadSeq int64 `bson:"read_seq"` +} From 9b043feca7aeb5abe21c7c4bc996da5d04f2d446 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Mon, 24 Jun 2024 17:34:49 +0800 Subject: [PATCH 52/59] seq user --- internal/msgtransfer/init.go | 8 +- internal/rpc/msg/server.go | 8 +- pkg/common/storage/cache/cachekey/seq.go | 44 +-- pkg/common/storage/cache/cachekey/seq1.go | 14 - pkg/common/storage/cache/redis/seq.go | 183 ---------- pkg/common/storage/cache/redis/seq_user.go | 89 +++++ pkg/common/storage/cache/seq.go | 30 -- pkg/common/storage/cache/seq_user.go | 12 + pkg/common/storage/controller/msg.go | 74 ++-- .../storage/database/mgo/seq_conversation.go | 34 +- .../database/mgo/seq_conversation_test.go | 34 +- pkg/common/storage/database/mgo/seq_user.go | 63 ++-- pkg/common/storage/database/seq.go | 1 + pkg/common/storage/database/seq_user.go | 12 +- tools/seq/internal/main.go | 315 ++++++++++++------ tools/seq/main.go | 3 +- 16 files changed, 482 insertions(+), 442 deletions(-) delete mode 100644 pkg/common/storage/cache/cachekey/seq1.go delete mode 100644 pkg/common/storage/cache/redis/seq.go create mode 100644 pkg/common/storage/cache/redis/seq_user.go delete mode 100644 pkg/common/storage/cache/seq.go create mode 100644 pkg/common/storage/cache/seq_user.go diff --git a/internal/msgtransfer/init.go b/internal/msgtransfer/init.go index e90c2288c..c17abbc30 100644 --- a/internal/msgtransfer/init.go +++ b/internal/msgtransfer/init.go @@ -82,7 +82,6 @@ func Start(ctx context.Context, index int, config *Config) error { client.AddOption(mw.GrpcClient(), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithDefaultServiceConfig(fmt.Sprintf(`{"LoadBalancingPolicy": "%s"}`, "round_robin"))) msgModel := redis.NewMsgCache(rdb) - seqModel := redis.NewSeqCache(rdb) msgDocModel, err := mgo.NewMsgMongo(mgocli.GetDB()) if err != nil { return err @@ -92,7 +91,12 @@ func Start(ctx context.Context, index int, config *Config) error { return err } seqConversationCache := redis.NewSeqConversationCacheRedis(rdb, seqConversation) - msgDatabase, err := controller.NewCommonMsgDatabase(msgDocModel, msgModel, seqModel, seqConversationCache, &config.KafkaConfig) + seqUser, err := mgo.NewSeqUserMongo(mgocli.GetDB()) + if err != nil { + return err + } + seqUserCache := redis.NewSeqUserCacheRedis(rdb, seqUser) + msgDatabase, err := controller.NewCommonMsgDatabase(msgDocModel, msgModel, seqUserCache, seqConversationCache, &config.KafkaConfig) if err != nil { return err } diff --git a/internal/rpc/msg/server.go b/internal/rpc/msg/server.go index 7cffb23a9..de0f698ea 100644 --- a/internal/rpc/msg/server.go +++ b/internal/rpc/msg/server.go @@ -86,7 +86,6 @@ func Start(ctx context.Context, config *Config, client discovery.SvcDiscoveryReg return err } msgModel := redis.NewMsgCache(rdb) - seqModel := redis.NewSeqCache(rdb) conversationClient := rpcclient.NewConversationRpcClient(client, config.Share.RpcRegisterName.Conversation) userRpcClient := rpcclient.NewUserRpcClient(client, config.Share.RpcRegisterName.User, config.Share.IMAdminUserID) groupRpcClient := rpcclient.NewGroupRpcClient(client, config.Share.RpcRegisterName.Group) @@ -96,7 +95,12 @@ func Start(ctx context.Context, config *Config, client discovery.SvcDiscoveryReg return err } seqConversationCache := redis.NewSeqConversationCacheRedis(rdb, seqConversation) - msgDatabase, err := controller.NewCommonMsgDatabase(msgDocModel, msgModel, seqModel, seqConversationCache, &config.KafkaConfig) + seqUser, err := mgo.NewSeqUserMongo(mgocli.GetDB()) + if err != nil { + return err + } + seqUserCache := redis.NewSeqUserCacheRedis(rdb, seqUser) + msgDatabase, err := controller.NewCommonMsgDatabase(msgDocModel, msgModel, seqUserCache, seqConversationCache, &config.KafkaConfig) if err != nil { return err } diff --git a/pkg/common/storage/cache/cachekey/seq.go b/pkg/common/storage/cache/cachekey/seq.go index ded0286d8..b32e78300 100644 --- a/pkg/common/storage/cache/cachekey/seq.go +++ b/pkg/common/storage/cache/cachekey/seq.go @@ -1,38 +1,30 @@ -// Copyright © 2024 OpenIM. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - package cachekey const ( - MaxSeq = "MAX_SEQ:" - MinSeq = "MIN_SEQ:" - ConversationUserMinSeq = "CON_USER_MIN_SEQ:" - HasReadSeq = "HAS_READ_SEQ:" + MallocSeq = "MALLOC_SEQ:" + MallocMinSeqLock = "MALLOC_MIN_SEQ:" + + SeqUserMaxSeq = "SEQ_USER_MAX:" + SeqUserMinSeq = "SEQ_USER_MIN:" + SeqUserReadSeq = "SEQ_USER_READ:" ) -func GetMaxSeqKey(conversationID string) string { - return MaxSeq + conversationID +func GetMallocSeqKey(conversationID string) string { + return MallocSeq + conversationID +} + +func GetMallocMinSeqKey(conversationID string) string { + return MallocMinSeqLock + conversationID } -func GetMinSeqKey(conversationID string) string { - return MinSeq + conversationID +func GetSeqUserMaxSeqKey(conversationID string, userID string) string { + return SeqUserMaxSeq + conversationID + ":" + userID } -func GetHasReadSeqKey(conversationID string, userID string) string { - return HasReadSeq + userID + ":" + conversationID +func GetSeqUserMinSeqKey(conversationID string, userID string) string { + return SeqUserMinSeq + conversationID + ":" + userID } -func GetConversationUserMinSeqKey(conversationID, userID string) string { - return ConversationUserMinSeq + conversationID + "u:" + userID +func GetSeqUserReadSeqKey(conversationID string, userID string) string { + return SeqUserReadSeq + conversationID + ":" + userID } diff --git a/pkg/common/storage/cache/cachekey/seq1.go b/pkg/common/storage/cache/cachekey/seq1.go deleted file mode 100644 index 04274db55..000000000 --- a/pkg/common/storage/cache/cachekey/seq1.go +++ /dev/null @@ -1,14 +0,0 @@ -package cachekey - -const ( - MallocSeq = "MALLOC_SEQ:" - MallocMinSeqLock = "MALLOC_MIN_SEQ:" -) - -func GetMallocSeqKey(conversationID string) string { - return MallocSeq + conversationID -} - -func GetMallocMinSeqKey(conversationID string) string { - return MallocMinSeqLock + conversationID -} diff --git a/pkg/common/storage/cache/redis/seq.go b/pkg/common/storage/cache/redis/seq.go deleted file mode 100644 index 8624cc78f..000000000 --- a/pkg/common/storage/cache/redis/seq.go +++ /dev/null @@ -1,183 +0,0 @@ -// Copyright © 2023 OpenIM. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package redis - -import ( - "context" - "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache" - "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/cachekey" - "github.com/openimsdk/tools/errs" - "github.com/openimsdk/tools/utils/stringutil" - "github.com/redis/go-redis/v9" -) - -func NewSeqCache(rdb redis.UniversalClient) cache.SeqCache { - return &seqCache{rdb: rdb} -} - -type seqCache struct { - rdb redis.UniversalClient -} - -func (c *seqCache) getMaxSeqKey(conversationID string) string { - return cachekey.GetMaxSeqKey(conversationID) -} - -func (c *seqCache) getMinSeqKey(conversationID string) string { - return cachekey.GetMinSeqKey(conversationID) -} - -func (c *seqCache) getHasReadSeqKey(conversationID string, userID string) string { - return cachekey.GetHasReadSeqKey(conversationID, userID) -} - -func (c *seqCache) getConversationUserMinSeqKey(conversationID, userID string) string { - return cachekey.GetConversationUserMinSeqKey(conversationID, userID) -} - -func (c *seqCache) setSeq(ctx context.Context, conversationID string, seq int64, getkey func(conversationID string) string) error { - return errs.Wrap(c.rdb.Set(ctx, getkey(conversationID), seq, 0).Err()) -} - -func (c *seqCache) getSeq(ctx context.Context, conversationID string, getkey func(conversationID string) string) (int64, error) { - val, err := c.rdb.Get(ctx, getkey(conversationID)).Int64() - if err != nil { - return 0, errs.Wrap(err) - } - return val, nil -} - -func (c *seqCache) getSeqs(ctx context.Context, items []string, getkey func(s string) string) (m map[string]int64, err error) { - m = make(map[string]int64, len(items)) - for i, v := range items { - res, err := c.rdb.Get(ctx, getkey(v)).Result() - if err != nil && err != redis.Nil { - return nil, errs.Wrap(err) - } - val := stringutil.StringToInt64(res) - if val != 0 { - m[items[i]] = val - } - } - - return m, nil -} - -//func (c *seqCache) SetMaxSeq(ctx context.Context, conversationID string, maxSeq int64) error { -// return c.setSeq(ctx, conversationID, maxSeq, c.getMaxSeqKey) -//} -// -//func (c *seqCache) GetMaxSeqs(ctx context.Context, conversationIDs []string) (m map[string]int64, err error) { -// return c.getSeqs(ctx, conversationIDs, c.getMaxSeqKey) -//} -// -//func (c *seqCache) GetMaxSeq(ctx context.Context, conversationID string) (int64, error) { -// return c.getSeq(ctx, conversationID, c.getMaxSeqKey) -//} -// -//func (c *seqCache) SetMinSeq(ctx context.Context, conversationID string, minSeq int64) error { -// return c.setSeq(ctx, conversationID, minSeq, c.getMinSeqKey) -//} -// -//func (c *seqCache) setSeqs(ctx context.Context, seqs map[string]int64, getkey func(key string) string) error { -// for conversationID, seq := range seqs { -// if err := c.rdb.Set(ctx, getkey(conversationID), seq, 0).Err(); err != nil { -// return errs.Wrap(err) -// } -// } -// return nil -//} -// -// -//func (c *seqCache) GetMinSeqs(ctx context.Context, conversationIDs []string) (map[string]int64, error) { -// return c.getSeqs(ctx, conversationIDs, c.getMinSeqKey) -//} -// -//func (c *seqCache) GetMinSeq(ctx context.Context, conversationID string) (int64, error) { -// return c.getSeq(ctx, conversationID, c.getMinSeqKey) -//} -// -//func (c *seqCache) SetMinSeqs(ctx context.Context, seqs map[string]int64) error { -// return c.setSeqs(ctx, seqs, c.getMinSeqKey) -//} - -func (c *seqCache) setSeqs(ctx context.Context, seqs map[string]int64, getkey func(key string) string) error { - for conversationID, seq := range seqs { - if err := c.rdb.Set(ctx, getkey(conversationID), seq, 0).Err(); err != nil { - return errs.Wrap(err) - } - } - return nil -} - -func (c *seqCache) GetConversationUserMinSeq(ctx context.Context, conversationID string, userID string) (int64, error) { - val, err := c.rdb.Get(ctx, c.getConversationUserMinSeqKey(conversationID, userID)).Int64() - if err != nil { - return 0, errs.Wrap(err) - } - return val, nil -} - -func (c *seqCache) GetConversationUserMinSeqs(ctx context.Context, conversationID string, userIDs []string) (m map[string]int64, err error) { - return c.getSeqs(ctx, userIDs, func(userID string) string { - return c.getConversationUserMinSeqKey(conversationID, userID) - }) -} - -func (c *seqCache) SetConversationUserMinSeq(ctx context.Context, conversationID string, userID string, minSeq int64) error { - return errs.Wrap(c.rdb.Set(ctx, c.getConversationUserMinSeqKey(conversationID, userID), minSeq, 0).Err()) -} - -func (c *seqCache) SetConversationUserMinSeqs(ctx context.Context, conversationID string, seqs map[string]int64) (err error) { - return c.setSeqs(ctx, seqs, func(userID string) string { - return c.getConversationUserMinSeqKey(conversationID, userID) - }) -} - -func (c *seqCache) SetUserConversationsMinSeqs(ctx context.Context, userID string, seqs map[string]int64) (err error) { - return c.setSeqs(ctx, seqs, func(conversationID string) string { - return c.getConversationUserMinSeqKey(conversationID, userID) - }) -} - -func (c *seqCache) SetHasReadSeq(ctx context.Context, userID string, conversationID string, hasReadSeq int64) error { - return errs.Wrap(c.rdb.Set(ctx, c.getHasReadSeqKey(conversationID, userID), hasReadSeq, 0).Err()) -} - -func (c *seqCache) SetHasReadSeqs(ctx context.Context, conversationID string, hasReadSeqs map[string]int64) error { - return c.setSeqs(ctx, hasReadSeqs, func(userID string) string { - return c.getHasReadSeqKey(conversationID, userID) - }) -} - -func (c *seqCache) UserSetHasReadSeqs(ctx context.Context, userID string, hasReadSeqs map[string]int64) error { - return c.setSeqs(ctx, hasReadSeqs, func(conversationID string) string { - return c.getHasReadSeqKey(conversationID, userID) - }) -} - -func (c *seqCache) GetHasReadSeqs(ctx context.Context, userID string, conversationIDs []string) (map[string]int64, error) { - return c.getSeqs(ctx, conversationIDs, func(conversationID string) string { - return c.getHasReadSeqKey(conversationID, userID) - }) -} - -func (c *seqCache) GetHasReadSeq(ctx context.Context, userID string, conversationID string) (int64, error) { - val, err := c.rdb.Get(ctx, c.getHasReadSeqKey(conversationID, userID)).Int64() - if err != nil { - return 0, err - } - return val, nil -} diff --git a/pkg/common/storage/cache/redis/seq_user.go b/pkg/common/storage/cache/redis/seq_user.go new file mode 100644 index 000000000..3c533cdd8 --- /dev/null +++ b/pkg/common/storage/cache/redis/seq_user.go @@ -0,0 +1,89 @@ +package redis + +import ( + "context" + "github.com/dtm-labs/rockscache" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/cachekey" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" + "github.com/openimsdk/tools/errs" + "github.com/redis/go-redis/v9" + "strconv" + "time" +) + +func NewSeqUserCacheRedis(rdb redis.UniversalClient, mgo database.SeqUser) cache.SeqUser { + return &seqUserCacheRedis{ + rdb: rdb, + mgo: mgo, + readSeqWriteRatio: 100, + expireTime: time.Hour * 24 * 7, + readExpireTime: time.Hour * 24 * 30, + rocks: rockscache.NewClient(rdb, *GetRocksCacheOptions()), + } +} + +type seqUserCacheRedis struct { + rdb redis.UniversalClient + mgo database.SeqUser + rocks *rockscache.Client + expireTime time.Duration + readExpireTime time.Duration + readSeqWriteRatio int64 +} + +func (s *seqUserCacheRedis) getSeqUserMaxSeqKey(conversationID string, userID string) string { + return cachekey.GetSeqUserMaxSeqKey(conversationID, userID) +} + +func (s *seqUserCacheRedis) getSeqUserMinSeqKey(conversationID string, userID string) string { + return cachekey.GetSeqUserMinSeqKey(conversationID, userID) +} + +func (s *seqUserCacheRedis) getSeqUserReadSeqKey(conversationID string, userID string) string { + return cachekey.GetSeqUserReadSeqKey(conversationID, userID) +} + +func (s *seqUserCacheRedis) GetMaxSeq(ctx context.Context, conversationID string, userID string) (int64, error) { + return getCache(ctx, s.rocks, s.getSeqUserMaxSeqKey(conversationID, userID), s.expireTime, func(ctx context.Context) (int64, error) { + return s.mgo.GetMaxSeq(ctx, conversationID, userID) + }) +} + +func (s *seqUserCacheRedis) SetMaxSeq(ctx context.Context, conversationID string, userID string, seq int64) error { + if err := s.mgo.SetMaxSeq(ctx, conversationID, userID, seq); err != nil { + return err + } + return s.rocks.TagAsDeleted2(ctx, s.getSeqUserMaxSeqKey(conversationID, userID)) +} + +func (s *seqUserCacheRedis) GetMinSeq(ctx context.Context, conversationID string, userID string) (int64, error) { + return getCache(ctx, s.rocks, s.getSeqUserMinSeqKey(conversationID, userID), s.expireTime, func(ctx context.Context) (int64, error) { + return s.mgo.GetMaxSeq(ctx, conversationID, userID) + }) +} + +func (s *seqUserCacheRedis) SetMinSeq(ctx context.Context, conversationID string, userID string, seq int64) error { + if err := s.mgo.SetMinSeq(ctx, conversationID, userID, seq); err != nil { + return err + } + return s.rocks.TagAsDeleted2(ctx, s.getSeqUserMinSeqKey(conversationID, userID)) +} + +func (s *seqUserCacheRedis) GetReadSeq(ctx context.Context, conversationID string, userID string) (int64, error) { + return getCache(ctx, s.rocks, s.getSeqUserReadSeqKey(conversationID, userID), s.readExpireTime, func(ctx context.Context) (int64, error) { + return s.mgo.GetMaxSeq(ctx, conversationID, userID) + }) +} + +func (s *seqUserCacheRedis) SetReadSeq(ctx context.Context, conversationID string, userID string, seq int64) error { + if seq%s.readSeqWriteRatio == 0 { + if err := s.mgo.SetReadSeq(ctx, conversationID, userID, seq); err != nil { + return err + } + } + if err := s.rocks.RawSet(ctx, s.getSeqUserReadSeqKey(conversationID, userID), strconv.Itoa(int(seq)), s.readExpireTime); err != nil { + return errs.Wrap(err) + } + return nil +} diff --git a/pkg/common/storage/cache/seq.go b/pkg/common/storage/cache/seq.go deleted file mode 100644 index 46e33c935..000000000 --- a/pkg/common/storage/cache/seq.go +++ /dev/null @@ -1,30 +0,0 @@ -package cache - -import ( - "context" -) - -type SeqCache interface { - //SetMaxSeq(ctx context.Context, conversationID string, maxSeq int64) error - //GetMaxSeqs(ctx context.Context, conversationIDs []string) (map[string]int64, error) - //GetMaxSeq(ctx context.Context, conversationID string) (int64, error) - //SetMinSeq(ctx context.Context, conversationID string, minSeq int64) error - //SetMinSeqs(ctx context.Context, seqs map[string]int64) error - //GetMinSeqs(ctx context.Context, conversationIDs []string) (map[string]int64, error) - //GetMinSeq(ctx context.Context, conversationID string) (int64, error) - GetConversationUserMinSeq(ctx context.Context, conversationID string, userID string) (int64, error) - GetConversationUserMinSeqs(ctx context.Context, conversationID string, userIDs []string) (map[string]int64, error) - SetConversationUserMinSeq(ctx context.Context, conversationID string, userID string, minSeq int64) error - // seqs map: key userID value minSeq - SetConversationUserMinSeqs(ctx context.Context, conversationID string, seqs map[string]int64) (err error) - // seqs map: key conversationID value minSeq - SetUserConversationsMinSeqs(ctx context.Context, userID string, seqs map[string]int64) error - // has read seq - SetHasReadSeq(ctx context.Context, userID string, conversationID string, hasReadSeq int64) error - // k: user, v: seq - SetHasReadSeqs(ctx context.Context, conversationID string, hasReadSeqs map[string]int64) error - // k: conversation, v :seq - UserSetHasReadSeqs(ctx context.Context, userID string, hasReadSeqs map[string]int64) error - GetHasReadSeqs(ctx context.Context, userID string, conversationIDs []string) (map[string]int64, error) - GetHasReadSeq(ctx context.Context, userID string, conversationID string) (int64, error) -} diff --git a/pkg/common/storage/cache/seq_user.go b/pkg/common/storage/cache/seq_user.go new file mode 100644 index 000000000..9e68399a7 --- /dev/null +++ b/pkg/common/storage/cache/seq_user.go @@ -0,0 +1,12 @@ +package cache + +import "context" + +type SeqUser interface { + GetMaxSeq(ctx context.Context, conversationID string, userID string) (int64, error) + SetMaxSeq(ctx context.Context, conversationID string, userID string, seq int64) error + GetMinSeq(ctx context.Context, conversationID string, userID string) (int64, error) + SetMinSeq(ctx context.Context, conversationID string, userID string, seq int64) error + GetReadSeq(ctx context.Context, conversationID string, userID string) (int64, error) + SetReadSeq(ctx context.Context, conversationID string, userID string, seq int64) error +} diff --git a/pkg/common/storage/controller/msg.go b/pkg/common/storage/controller/msg.go index de8c111b3..1e7b5c229 100644 --- a/pkg/common/storage/controller/msg.go +++ b/pkg/common/storage/controller/msg.go @@ -108,7 +108,7 @@ type CommonMsgDatabase interface { DeleteDocMsgBefore(ctx context.Context, ts int64, doc *model.MsgDocModel) ([]int, error) } -func NewCommonMsgDatabase(msgDocModel database.Msg, msg cache.MsgCache, seq cache.SeqCache, seqConversation cache.SeqConversationCache, kafkaConf *config.Kafka) (CommonMsgDatabase, error) { +func NewCommonMsgDatabase(msgDocModel database.Msg, msg cache.MsgCache, seqUser cache.SeqUser, seqConversation cache.SeqConversationCache, kafkaConf *config.Kafka) (CommonMsgDatabase, error) { conf, err := kafka.BuildProducerConfig(*kafkaConf.Build()) if err != nil { return nil, err @@ -128,7 +128,7 @@ func NewCommonMsgDatabase(msgDocModel database.Msg, msg cache.MsgCache, seq cach return &commonMsgDatabase{ msgDocDatabase: msgDocModel, msg: msg, - seq: seq, + seqUser: seqUser, seqConversation: seqConversation, producer: producerToRedis, producerToMongo: producerToMongo, @@ -140,8 +140,8 @@ type commonMsgDatabase struct { msgDocDatabase database.Msg msgTable model.MsgDocModel msg cache.MsgCache - seq cache.SeqCache seqConversation cache.SeqConversationCache + seqUser cache.SeqUser producer *kafka.Producer producerToMongo *kafka.Producer producerToPush *kafka.Producer @@ -339,6 +339,15 @@ func (db *commonMsgDatabase) DeleteMessagesFromCache(ctx context.Context, conver return db.msg.DeleteMessagesFromCache(ctx, conversationID, seqs) } +func (db *commonMsgDatabase) setHasReadSeqs(ctx context.Context, conversationID string, userSeqMap map[string]int64) error { + for userID, seq := range userSeqMap { + if err := db.seqUser.SetReadSeq(ctx, conversationID, userID, seq); err != nil { + return err + } + } + return nil +} + func (db *commonMsgDatabase) BatchInsertChat2Cache(ctx context.Context, conversationID string, msgs []*sdkws.MsgData) (seq int64, isNew bool, err error) { lenList := len(msgs) if int64(lenList) > db.msgTable.GetSingleGocMsgNum() { @@ -368,7 +377,7 @@ func (db *commonMsgDatabase) BatchInsertChat2Cache(ctx context.Context, conversa } else { prommetrics.MsgInsertRedisSuccessCounter.Inc() } - err = db.seq.SetHasReadSeqs(ctx, conversationID, userSeqMap) + err = db.setHasReadSeqs(ctx, conversationID, userSeqMap) if err != nil { log.ZError(ctx, "SetHasReadSeqs error", err, "userSeqMap", userSeqMap, "conversationID", conversationID) prommetrics.SeqSetFailedCounter.Inc() @@ -496,7 +505,7 @@ func (db *commonMsgDatabase) getMsgBySeqsRange(ctx context.Context, userID strin // "userMinSeq" can be set as the same value as the conversation's "maxSeq" at the moment they join the group. // This ensures that their message retrieval starts from the point they joined. func (db *commonMsgDatabase) GetMsgBySeqsRange(ctx context.Context, userID string, conversationID string, begin, end, num, userMaxSeq int64) (int64, int64, []*sdkws.MsgData, error) { - userMinSeq, err := db.seq.GetConversationUserMinSeq(ctx, conversationID, userID) + userMinSeq, err := db.seqUser.GetMinSeq(ctx, conversationID, userID) if err != nil && errs.Unwrap(err) != redis.Nil { return 0, 0, nil, err } @@ -574,7 +583,7 @@ func (db *commonMsgDatabase) GetMsgBySeqsRange(ctx context.Context, userID strin } func (db *commonMsgDatabase) GetMsgBySeqs(ctx context.Context, userID string, conversationID string, seqs []int64) (int64, int64, []*sdkws.MsgData, error) { - userMinSeq, err := db.seq.GetConversationUserMinSeq(ctx, conversationID, userID) + userMinSeq, err := db.seqUser.GetMinSeq(ctx, conversationID, userID) if err != nil && errs.Unwrap(err) != redis.Nil { return 0, 0, nil, err } @@ -672,12 +681,12 @@ func (db *commonMsgDatabase) UserMsgsDestruct(ctx context.Context, userID string log.ZDebug(ctx, "UserMsgsDestruct", "conversationID", conversationID, "userID", userID, "seqs", seqs) if len(seqs) > 0 { userMinSeq := seqs[len(seqs)-1] + 1 - currentUserMinSeq, err := db.seq.GetConversationUserMinSeq(ctx, conversationID, userID) - if err != nil && errs.Unwrap(err) != redis.Nil { + currentUserMinSeq, err := db.seqUser.GetMinSeq(ctx, conversationID, userID) + if err != nil { return nil, err } if currentUserMinSeq < userMinSeq { - if err := db.seq.SetConversationUserMinSeq(ctx, conversationID, userID, userMinSeq); err != nil { + if err := db.seqUser.SetMinSeq(ctx, conversationID, userID, userMinSeq); err != nil { return nil, err } } @@ -832,40 +841,45 @@ func (db *commonMsgDatabase) SetMinSeqs(ctx context.Context, seqs map[string]int return nil } -func (db *commonMsgDatabase) GetConversationUserMinSeq(ctx context.Context, conversationID string, userID string) (int64, error) { - return db.seq.GetConversationUserMinSeq(ctx, conversationID, userID) -} - -func (db *commonMsgDatabase) GetConversationUserMinSeqs(ctx context.Context, conversationID string, userIDs []string) (map[string]int64, error) { - return db.seq.GetConversationUserMinSeqs(ctx, conversationID, userIDs) -} - -func (db *commonMsgDatabase) SetConversationUserMinSeq(ctx context.Context, conversationID string, userID string, minSeq int64) error { - return db.seq.SetConversationUserMinSeq(ctx, conversationID, userID, minSeq) -} - -func (db *commonMsgDatabase) SetConversationUserMinSeqs(ctx context.Context, conversationID string, seqs map[string]int64) (err error) { - return db.seq.SetConversationUserMinSeqs(ctx, conversationID, seqs) -} - func (db *commonMsgDatabase) SetUserConversationsMinSeqs(ctx context.Context, userID string, seqs map[string]int64) error { - return db.seq.SetUserConversationsMinSeqs(ctx, userID, seqs) + for conversationID, seq := range seqs { + if err := db.seqUser.SetMinSeq(ctx, conversationID, userID, seq); err != nil { + return err + } + } + return nil } func (db *commonMsgDatabase) UserSetHasReadSeqs(ctx context.Context, userID string, hasReadSeqs map[string]int64) error { - return db.seq.UserSetHasReadSeqs(ctx, userID, hasReadSeqs) + for conversationID, seq := range hasReadSeqs { + if err := db.seqUser.SetReadSeq(ctx, conversationID, userID, seq); err != nil { + return err + } + } + return nil } func (db *commonMsgDatabase) SetHasReadSeq(ctx context.Context, userID string, conversationID string, hasReadSeq int64) error { - return db.seq.SetHasReadSeq(ctx, userID, conversationID, hasReadSeq) + return db.seqUser.SetReadSeq(ctx, conversationID, userID, hasReadSeq) } func (db *commonMsgDatabase) GetHasReadSeqs(ctx context.Context, userID string, conversationIDs []string) (map[string]int64, error) { - return db.seq.GetHasReadSeqs(ctx, userID, conversationIDs) + cSeq := make(map[string]int64) + for _, conversationID := range conversationIDs { + if _, ok := cSeq[conversationID]; ok { + continue + } + seq, err := db.seqUser.GetReadSeq(ctx, conversationID, userID) + if err != nil { + return nil, err + } + cSeq[conversationID] = seq + } + return cSeq, nil } func (db *commonMsgDatabase) GetHasReadSeq(ctx context.Context, userID string, conversationID string) (int64, error) { - return db.seq.GetHasReadSeq(ctx, userID, conversationID) + return db.seqUser.GetReadSeq(ctx, conversationID, userID) } func (db *commonMsgDatabase) SetSendMsgStatus(ctx context.Context, id string, status int32) error { diff --git a/pkg/common/storage/database/mgo/seq_conversation.go b/pkg/common/storage/database/mgo/seq_conversation.go index c9a3ac41a..7971b7e1a 100644 --- a/pkg/common/storage/database/mgo/seq_conversation.go +++ b/pkg/common/storage/database/mgo/seq_conversation.go @@ -28,6 +28,26 @@ type seqConversationMongo struct { coll *mongo.Collection } +func (s *seqConversationMongo) setSeq(ctx context.Context, conversationID string, seq int64, field string) error { + filter := map[string]any{ + "conversation_id": conversationID, + } + insert := bson.M{ + "conversation_id": conversationID, + "min_seq": 0, + "max_seq": 0, + } + delete(insert, field) + update := map[string]any{ + "$set": bson.M{ + field: seq, + }, + "$setOnInsert": insert, + } + opt := options.Update().SetUpsert(true) + return mongoutil.UpdateOne(ctx, s.coll, filter, update, false, opt) +} + func (s *seqConversationMongo) Malloc(ctx context.Context, conversationID string, size int64) (int64, error) { if size < 0 { return 0, errors.New("size must be greater than 0") @@ -48,16 +68,8 @@ func (s *seqConversationMongo) Malloc(ctx context.Context, conversationID string return lastSeq - size, nil } -func (s *seqConversationMongo) MallocSeq(ctx context.Context, conversationID string, size int64) ([]int64, error) { - first, err := s.Malloc(ctx, conversationID, size) - if err != nil { - return nil, err - } - seqs := make([]int64, 0, size) - for i := int64(0); i < size; i++ { - seqs = append(seqs, first+i+1) - } - return seqs, nil +func (s *seqConversationMongo) SetMaxSeq(ctx context.Context, conversationID string, seq int64) error { + return s.setSeq(ctx, conversationID, seq, "max_seq") } func (s *seqConversationMongo) GetMaxSeq(ctx context.Context, conversationID string) (int64, error) { @@ -83,7 +95,7 @@ func (s *seqConversationMongo) GetMinSeq(ctx context.Context, conversationID str } func (s *seqConversationMongo) SetMinSeq(ctx context.Context, conversationID string, seq int64) error { - return mongoutil.UpdateOne(ctx, s.coll, bson.M{"conversation_id": conversationID}, bson.M{"$set": bson.M{"min_seq": seq}}, false) + return s.setSeq(ctx, conversationID, seq, "min_seq") } func (s *seqConversationMongo) GetConversation(ctx context.Context, conversationID string) (*model.SeqConversation, error) { diff --git a/pkg/common/storage/database/mgo/seq_conversation_test.go b/pkg/common/storage/database/mgo/seq_conversation_test.go index e6466a1c6..5167314da 100644 --- a/pkg/common/storage/database/mgo/seq_conversation_test.go +++ b/pkg/common/storage/database/mgo/seq_conversation_test.go @@ -15,19 +15,23 @@ func Result[V any](val V, err error) V { return val } -func TestName(t *testing.T) { - cli := Result(mongo.Connect(context.Background(), options.Client().ApplyURI("mongodb://openIM:openIM123@172.16.8.48:37017/openim_v3?maxPoolSize=100").SetConnectTimeout(5*time.Second))) - tmp, err := NewSeqConversationMongo(cli.Database("openim_v3")) - if err != nil { - panic(err) - } - for i := 0; i < 10; i++ { - var size int64 = 100 - firstSeq, err := tmp.Malloc(context.Background(), "1", size) - if err != nil { - t.Log(err) - return - } - t.Logf("%d -> %d", firstSeq, firstSeq+size-1) - } +func Mongodb() *mongo.Database { + return Result( + mongo.Connect(context.Background(), + options.Client(). + ApplyURI("mongodb://openIM:openIM123@172.16.8.48:37017/openim_v3?maxPoolSize=100"). + SetConnectTimeout(5*time.Second)), + ).Database("openim_v3") +} + +func TestUserSeq(t *testing.T) { + uSeq := Result(NewSeqUserMongo(Mongodb())).(*seqUserMongo) + t.Log(uSeq.SetMinSeq(context.Background(), "1000", "2000", 4)) +} + +func TestConversationSeq(t *testing.T) { + cSeq := Result(NewSeqConversationMongo(Mongodb())).(*seqConversationMongo) + t.Log(cSeq.SetMaxSeq(context.Background(), "2000", 10)) + t.Log(cSeq.Malloc(context.Background(), "2000", 10)) + t.Log(cSeq.GetMaxSeq(context.Background(), "2000")) } diff --git a/pkg/common/storage/database/mgo/seq_user.go b/pkg/common/storage/database/mgo/seq_user.go index 4fa299a7a..5e0eb2022 100644 --- a/pkg/common/storage/database/mgo/seq_user.go +++ b/pkg/common/storage/database/mgo/seq_user.go @@ -2,6 +2,7 @@ package mgo import ( "context" + "errors" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" "github.com/openimsdk/tools/db/mongoutil" "go.mongodb.org/mongo-driver/bson" @@ -10,7 +11,7 @@ import ( ) func NewSeqUserMongo(db *mongo.Database) (database.SeqUser, error) { - coll := db.Collection(database.SeqConversationName) + coll := db.Collection(database.SeqUserName) _, err := coll.Indexes().CreateOne(context.Background(), mongo.IndexModel{ Keys: bson.D{ {Key: "user_id", Value: 1}, @@ -27,45 +28,65 @@ type seqUserMongo struct { coll *mongo.Collection } -func (s *seqUserMongo) setSeq(ctx context.Context, userID string, conversationID string, seq int64, field string) error { +func (s *seqUserMongo) setSeq(ctx context.Context, conversationID string, userID string, seq int64, field string) error { filter := map[string]any{ "user_id": userID, "conversation_id": conversationID, } + insert := bson.M{ + "user_id": userID, + "conversation_id": conversationID, + "min_seq": 0, + "max_seq": 0, + "read_seq": 0, + } + delete(insert, field) update := map[string]any{ - "$set": map[string]any{"field": int64(0)}, + "$set": bson.M{ + field: seq, + }, + "$setOnInsert": insert, } opt := options.Update().SetUpsert(true) return mongoutil.UpdateOne(ctx, s.coll, filter, update, false, opt) } -func (s *seqUserMongo) GetMaxSeq(ctx context.Context, userID string, conversationID string) (int64, error) { +func (s *seqUserMongo) getSeq(ctx context.Context, conversationID string, userID string, failed string) (int64, error) { + filter := map[string]any{ + "user_id": userID, + "conversation_id": conversationID, + } + opt := options.FindOne().SetProjection(bson.M{"_id": 0, failed: 1}) + seq, err := mongoutil.FindOne[int64](ctx, s.coll, filter, opt) + if err == nil { + return seq, nil + } else if errors.Is(err, mongo.ErrNoDocuments) { + return 0, nil + } else { + return 0, err + } +} - //TODO implement me - panic("implement me") +func (s *seqUserMongo) GetMaxSeq(ctx context.Context, conversationID string, userID string) (int64, error) { + return s.getSeq(ctx, conversationID, userID, "max_seq") } -func (s *seqUserMongo) SetMaxSeq(ctx context.Context, userID string, conversationID string, seq int64) error { - //TODO implement me - panic("implement me") +func (s *seqUserMongo) SetMaxSeq(ctx context.Context, conversationID string, userID string, seq int64) error { + return s.setSeq(ctx, conversationID, userID, seq, "max_seq") } -func (s *seqUserMongo) GetMinSeq(ctx context.Context, userID string, conversationID string) (int64, error) { - //TODO implement me - panic("implement me") +func (s *seqUserMongo) GetMinSeq(ctx context.Context, conversationID string, userID string) (int64, error) { + return s.getSeq(ctx, conversationID, userID, "min_seq") } -func (s *seqUserMongo) SetMinSeq(ctx context.Context, userID string, conversationID string, seq int64) error { - //TODO implement me - panic("implement me") +func (s *seqUserMongo) SetMinSeq(ctx context.Context, conversationID string, userID string, seq int64) error { + return s.setSeq(ctx, conversationID, userID, seq, "min_seq") } -func (s *seqUserMongo) GetReadSeq(ctx context.Context, userID string, conversationID string) (int64, error) { - //TODO implement me - panic("implement me") +func (s *seqUserMongo) GetReadSeq(ctx context.Context, conversationID string, userID string) (int64, error) { + return s.getSeq(ctx, conversationID, userID, "read_seq") } -func (s *seqUserMongo) SetReadSeq(ctx context.Context, userID string, conversationID string, seq int64) error { - //TODO implement me - panic("implement me") +func (s *seqUserMongo) SetReadSeq(ctx context.Context, conversationID string, userID string, seq int64) error { + return s.setSeq(ctx, conversationID, userID, seq, "read_seq") } diff --git a/pkg/common/storage/database/seq.go b/pkg/common/storage/database/seq.go index fdbc2f8f3..cf93b795f 100644 --- a/pkg/common/storage/database/seq.go +++ b/pkg/common/storage/database/seq.go @@ -5,6 +5,7 @@ import "context" type SeqConversation interface { Malloc(ctx context.Context, conversationID string, size int64) (int64, error) GetMaxSeq(ctx context.Context, conversationID string) (int64, error) + SetMaxSeq(ctx context.Context, conversationID string, seq int64) error GetMinSeq(ctx context.Context, conversationID string) (int64, error) SetMinSeq(ctx context.Context, conversationID string, seq int64) error } diff --git a/pkg/common/storage/database/seq_user.go b/pkg/common/storage/database/seq_user.go index 3e9b29ec2..d32d614dd 100644 --- a/pkg/common/storage/database/seq_user.go +++ b/pkg/common/storage/database/seq_user.go @@ -3,10 +3,10 @@ package database import "context" type SeqUser interface { - GetMaxSeq(ctx context.Context, userID string, conversationID string) (int64, error) - SetMaxSeq(ctx context.Context, userID string, conversationID string, seq int64) error - GetMinSeq(ctx context.Context, userID string, conversationID string) (int64, error) - SetMinSeq(ctx context.Context, userID string, conversationID string, seq int64) error - GetReadSeq(ctx context.Context, userID string, conversationID string) (int64, error) - SetReadSeq(ctx context.Context, userID string, conversationID string, seq int64) error + GetMaxSeq(ctx context.Context, conversationID string, userID string) (int64, error) + SetMaxSeq(ctx context.Context, conversationID string, userID string, seq int64) error + GetMinSeq(ctx context.Context, conversationID string, userID string) (int64, error) + SetMinSeq(ctx context.Context, conversationID string, userID string, seq int64) error + GetReadSeq(ctx context.Context, conversationID string, userID string) (int64, error) + SetReadSeq(ctx context.Context, conversationID string, userID string, seq int64) error } diff --git a/tools/seq/internal/main.go b/tools/seq/internal/main.go index 54a7d477d..5200ad5fe 100644 --- a/tools/seq/internal/main.go +++ b/tools/seq/internal/main.go @@ -1,30 +1,38 @@ package internal import ( + "bytes" "context" "errors" "fmt" "github.com/openimsdk/open-im-server/v3/pkg/common/cmd" "github.com/openimsdk/open-im-server/v3/pkg/common/config" - "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/cachekey" - "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database/mgo" - "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" "github.com/openimsdk/tools/db/mongoutil" "github.com/openimsdk/tools/db/redisutil" - "github.com/openimsdk/tools/utils/datautil" "github.com/redis/go-redis/v9" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/mongo" "go.mongodb.org/mongo-driver/mongo/options" "gopkg.in/yaml.v3" "os" + "os/signal" "path/filepath" "strconv" "strings" + "sync" + "sync/atomic" + "syscall" "time" ) +const ( + MaxSeq = "MAX_SEQ:" + MinSeq = "MIN_SEQ:" + ConversationUserMinSeq = "CON_USER_MIN_SEQ:" + HasReadSeq = "HAS_READ_SEQ:" +) + const ( batchSize = 100 dataVersionCollection = "data_version" @@ -44,48 +52,6 @@ func readConfig[T any](dir string, name string) (*T, error) { return &conf, nil } -func redisKey(rdb redis.UniversalClient, prefix string, del time.Duration, fn func(ctx context.Context, key string, delKey map[string]struct{}) error) error { - var ( - cursor uint64 - keys []string - err error - ) - ctx := context.Background() - for { - keys, cursor, err = rdb.Scan(ctx, cursor, prefix+"*", batchSize).Result() - if err != nil { - return err - } - delKey := make(map[string]struct{}) - if len(keys) > 0 { - for _, key := range keys { - if err := fn(ctx, key, delKey); err != nil { - return err - } - } - } - if len(delKey) > 0 { - delKeys := datautil.Keys(delKey) - if del < time.Second { - if err := rdb.Del(ctx, datautil.Keys(delKey)...).Err(); err != nil { - return err - } - } else { - pipe := rdb.Pipeline() - for _, key := range delKeys { - pipe.Expire(ctx, key, del) - } - if _, err := pipe.Exec(ctx); err != nil { - return err - } - } - } - if cursor == 0 { - return nil - } - } -} - func Main(conf string, del time.Duration) error { redisConfig, err := readConfig[config.Redis](conf, cmd.RedisConfigFileName) if err != nil { @@ -117,71 +83,218 @@ func Main(conf string, del time.Duration) error { if _, err := mgo.NewSeqConversationMongo(mgocli.GetDB()); err != nil { return err } - coll := mgocli.GetDB().Collection(database.SeqConversationName) - const prefix = cachekey.MaxSeq - fmt.Println("start to convert seq conversation") - err = redisKey(rdb, prefix, del, func(ctx context.Context, key string, delKey map[string]struct{}) error { - conversationId := strings.TrimPrefix(key, prefix) - delKey[key] = struct{}{} - maxValue, err := rdb.Get(ctx, key).Result() - if err != nil { - return err - } - seq, err := strconv.Atoi(maxValue) - if err != nil { - return fmt.Errorf("invalid max seq %s", maxValue) + cSeq, err := mgo.NewSeqConversationMongo(mgocli.GetDB()) + if err != nil { + return err + } + uSeq, err := mgo.NewSeqUserMongo(mgocli.GetDB()) + if err != nil { + return err + } + uSpitHasReadSeq := func(id string) (conversationID string, userID string, err error) { + // HasReadSeq + userID + ":" + conversationID + arr := strings.Split(id, ":") + if len(arr) != 2 || arr[0] == "" || arr[1] == "" { + return "", "", fmt.Errorf("invalid has read seq id %s", id) } - if seq == 0 { - return nil + userID = arr[0] + conversationID = arr[1] + return + } + uSpitConversationUserMinSeq := func(id string) (conversationID string, userID string, err error) { + // ConversationUserMinSeq + conversationID + "u:" + userID + arr := strings.Split(id, "u:") + if len(arr) != 2 || arr[0] == "" || arr[1] == "" { + return "", "", fmt.Errorf("invalid has read seq id %s", id) } - if seq < 0 { - return fmt.Errorf("invalid max seq %s", maxValue) + conversationID = arr[0] + userID = arr[1] + return + } + + ts := []*taskSeq{ + { + Prefix: MaxSeq, + GetSeq: cSeq.GetMaxSeq, + SetSeq: cSeq.SetMinSeq, + }, + { + Prefix: MinSeq, + GetSeq: cSeq.GetMinSeq, + SetSeq: cSeq.SetMinSeq, + }, + { + Prefix: HasReadSeq, + GetSeq: func(ctx context.Context, id string) (int64, error) { + conversationID, userID, err := uSpitHasReadSeq(id) + if err != nil { + return 0, err + } + return uSeq.GetReadSeq(ctx, conversationID, userID) + }, + SetSeq: func(ctx context.Context, id string, seq int64) error { + conversationID, userID, err := uSpitHasReadSeq(id) + if err != nil { + return err + } + return uSeq.SetReadSeq(ctx, conversationID, userID, seq) + }, + }, + { + Prefix: ConversationUserMinSeq, + GetSeq: func(ctx context.Context, id string) (int64, error) { + conversationID, userID, err := uSpitConversationUserMinSeq(id) + if err != nil { + return 0, err + } + return uSeq.GetMinSeq(ctx, conversationID, userID) + }, + SetSeq: func(ctx context.Context, id string, seq int64) error { + conversationID, userID, err := uSpitConversationUserMinSeq(id) + if err != nil { + return err + } + return uSeq.SetMinSeq(ctx, conversationID, userID, seq) + }, + }, + } + + cancel() + ctx = context.Background() + + var wg sync.WaitGroup + wg.Add(len(ts)) + + for i := range ts { + go func(task *taskSeq) { + defer wg.Done() + err := seqRedisToMongo(ctx, rdb, task.GetSeq, task.SetSeq, task.Prefix, del, &task.Count) + task.End = time.Now() + task.Error = err + }(ts[i]) + } + start := time.Now() + done := make(chan struct{}) + go func() { + wg.Wait() + close(done) + }() + + sigs := make(chan os.Signal, 1) + signal.Notify(sigs, syscall.SIGTERM) + + ticker := time.NewTicker(time.Second) + defer ticker.Stop() + var buf bytes.Buffer + + printTaskInfo := func(now time.Time) { + buf.Reset() + buf.WriteString(now.Format(time.DateTime)) + buf.WriteString(" \n") + for i := range ts { + task := ts[i] + if task.Error == nil { + if task.End.IsZero() { + buf.WriteString(fmt.Sprintf("[%s] converting %s* count %d", now.Sub(start), task.Prefix, atomic.LoadInt64(&task.Count))) + } else { + buf.WriteString(fmt.Sprintf("[%s] success %s* count %d", task.End.Sub(start), task.Prefix, atomic.LoadInt64(&task.Count))) + } + } else { + buf.WriteString(fmt.Sprintf("[%s] failed %s* count %d error %s", task.End.Sub(start), task.Prefix, atomic.LoadInt64(&task.Count), task.Error)) + } + buf.WriteString("\n") } - var ( - minSeq int64 - maxSeq = int64(seq) - ) - minKey := cachekey.MinSeq + conversationId - delKey[minKey] = struct{}{} - minValue, err := rdb.Get(ctx, minKey).Result() - if err == nil { - seq, err := strconv.Atoi(minValue) - if err != nil { - return fmt.Errorf("invalid min seq %s", minValue) + fmt.Println(buf.String()) + } + + for { + select { + case <-ctx.Done(): + return ctx.Err() + case s := <-sigs: + return fmt.Errorf("exit by signal %s", s) + case <-done: + errs := make([]error, 0, len(ts)) + for i := range ts { + task := ts[i] + if task.Error != nil { + errs = append(errs, fmt.Errorf("seq %s failed %w", task.Prefix, task.Error)) + } } - if seq < 0 { - return fmt.Errorf("invalid min seq %s", minValue) + if len(errs) > 0 { + return errors.Join(errs...) } - minSeq = int64(seq) - } else if !errors.Is(err, redis.Nil) { - return err - } - if maxSeq < minSeq { - return fmt.Errorf("invalid max seq %d < min seq %d", maxSeq, minSeq) - } - res, err := mongoutil.FindOne[*model.SeqConversation](ctx, coll, bson.M{"conversation_id": conversationId}, nil) - if err == nil { - if res.MaxSeq < int64(seq) { - _, err = coll.UpdateOne(ctx, bson.M{"conversation_id": conversationId}, bson.M{"$set": bson.M{"max_seq": maxSeq, "min_seq": minSeq}}) + printTaskInfo(time.Now()) + if err := SetVersion(versionColl, seqKey, seqVersion); err != nil { + return fmt.Errorf("set mongodb seq version %w", err) } + return nil + case now := <-ticker.C: + printTaskInfo(now) + } + } +} + +type taskSeq struct { + Prefix string + Count int64 + Error error + End time.Time + GetSeq func(ctx context.Context, id string) (int64, error) + SetSeq func(ctx context.Context, id string, seq int64) error +} + +func seqRedisToMongo(ctx context.Context, rdb redis.UniversalClient, getSeq func(ctx context.Context, id string) (int64, error), setSeq func(ctx context.Context, id string, seq int64) error, prefix string, delAfter time.Duration, count *int64) error { + var ( + cursor uint64 + keys []string + err error + ) + for { + keys, cursor, err = rdb.Scan(ctx, cursor, prefix+"*", batchSize).Result() + if err != nil { return err - } else if errors.Is(err, mongo.ErrNoDocuments) { - res = &model.SeqConversation{ - ConversationID: conversationId, - MaxSeq: maxSeq, - MinSeq: minSeq, + } + if len(keys) > 0 { + for _, key := range keys { + seqStr, err := rdb.Get(ctx, key).Result() + if err != nil { + return fmt.Errorf("redis get %s failed %w", key, err) + } + seq, err := strconv.Atoi(seqStr) + if err != nil { + return fmt.Errorf("invalid %s seq %s", key, seqStr) + } + if seq < 0 { + return fmt.Errorf("invalid %s seq %s", key, seqStr) + } + id := strings.TrimPrefix(key, prefix) + redisSeq := int64(seq) + mongoSeq, err := getSeq(ctx, id) + if err != nil { + return fmt.Errorf("get mongo seq %s failed %w", key, err) + } + if mongoSeq < redisSeq { + if err := setSeq(ctx, id, redisSeq); err != nil { + return fmt.Errorf("set mongo seq %s failed %w", key, err) + } + } + if delAfter > 0 { + if err := rdb.Expire(ctx, key, delAfter).Err(); err != nil { + return fmt.Errorf("redis expire key %s failed %w", key, err) + } + } else { + if err := rdb.Del(ctx, key).Err(); err != nil { + return fmt.Errorf("redis del key %s failed %w", key, err) + } + } + atomic.AddInt64(count, 1) } - _, err := coll.InsertOne(ctx, res) - return err - } else { - return err } - }) - if err != nil { - return err + if cursor == 0 { + return nil + } } - fmt.Println("convert seq conversation success") - return SetVersion(versionColl, seqKey, seqVersion) } func CheckVersion(coll *mongo.Collection, key string, currentVersion int) (converted bool, err error) { diff --git a/tools/seq/main.go b/tools/seq/main.go index 6bb4b7657..399e6d934 100644 --- a/tools/seq/main.go +++ b/tools/seq/main.go @@ -12,10 +12,11 @@ func main() { config string second int ) - flag.StringVar(&config, "c", "", "config directory") + flag.StringVar(&config, "c", "/Users/chao/Desktop/project/open-im-server/config", "config directory") flag.IntVar(&second, "sec", 3600*24, "delayed deletion of the original seq key after conversion") flag.Parse() if err := internal.Main(config, time.Duration(second)*time.Second); err != nil { fmt.Println("seq task", err) } + fmt.Println("seq task success!") } From 2b2a75f19e47daee990d329757ab0a3d3e84fe2f Mon Sep 17 00:00:00 2001 From: Monet Lee Date: Fri, 28 Jun 2024 15:29:03 +0800 Subject: [PATCH 53/59] implement minio expire delete. --- config/openim-crontask.yml | 1 + go.mod | 2 +- go.sum | 4 ++-- internal/rpc/third/s3.go | 12 ++++++++++- internal/tools/cron_task.go | 25 +++++++++++++++++++++-- pkg/common/config/config.go | 6 ++++-- pkg/common/storage/controller/s3.go | 9 ++++++-- pkg/common/storage/database/mgo/object.go | 7 +++++++ pkg/common/storage/database/object.go | 3 +++ pkg/rpcclient/third.go | 4 ++++ 10 files changed, 63 insertions(+), 10 deletions(-) diff --git a/config/openim-crontask.yml b/config/openim-crontask.yml index 9bbccfd25..d2154d263 100644 --- a/config/openim-crontask.yml +++ b/config/openim-crontask.yml @@ -1,2 +1,3 @@ chatRecordsClearTime: "0 2 * * *" retainChatRecords: 365 +fileExpireTime: 90 diff --git a/go.mod b/go.mod index 2614e0f32..a1a51b4fa 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/lestrrat-go/file-rotatelogs v2.4.0+incompatible // indirect github.com/mitchellh/mapstructure v1.5.0 - github.com/openimsdk/protocol v0.0.69-alpha.17 + github.com/openimsdk/protocol v0.0.69-alpha.22 github.com/openimsdk/tools v0.0.49-alpha.25 github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 diff --git a/go.sum b/go.sum index fe4f0c390..e67b30206 100644 --- a/go.sum +++ b/go.sum @@ -270,8 +270,8 @@ github.com/onsi/gomega v1.25.0 h1:Vw7br2PCDYijJHSfBOWhov+8cAnUf8MfMaIOV323l6Y= github.com/onsi/gomega v1.25.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdMPGhM= github.com/openimsdk/gomake v0.0.13 h1:xLDe/moqgWpRoptHzI4packAWzs4C16b+sVY+txNJp0= github.com/openimsdk/gomake v0.0.13/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= -github.com/openimsdk/protocol v0.0.69-alpha.17 h1:pEag4ZdlovE+AyLsw1VYFU/3sk6ayvGdPzgufQfKf9M= -github.com/openimsdk/protocol v0.0.69-alpha.17/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= +github.com/openimsdk/protocol v0.0.69-alpha.22 h1:kifZWVNDkg9diXFJUJ/Q9xFc80cveBhc+1dUXcE9xHQ= +github.com/openimsdk/protocol v0.0.69-alpha.22/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= github.com/openimsdk/tools v0.0.49-alpha.25 h1:OpRPwDZ2xWX7Zj5kyfZhryu/NfZTrsRVr2GFwu1HQHI= github.com/openimsdk/tools v0.0.49-alpha.25/go.mod h1:rwsFI1G/nBHNfiNapbven41akRDPBbH4df0Cgy6xueU= github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= diff --git a/internal/rpc/third/s3.go b/internal/rpc/third/s3.go index 4cb1b81d0..19a519d2e 100644 --- a/internal/rpc/third/s3.go +++ b/internal/rpc/third/s3.go @@ -19,11 +19,12 @@ import ( "encoding/base64" "encoding/hex" "encoding/json" - "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" "path" "strconv" "time" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" + "github.com/google/uuid" "github.com/openimsdk/open-im-server/v3/pkg/common/servererrs" "github.com/openimsdk/protocol/third" @@ -283,6 +284,15 @@ func (t *thirdServer) apiAddress(prefix, name string) string { return prefix + name } +func (t *thirdServer) DeleteOutdatedData(ctx context.Context, req *third.DeleteOutdatedDataReq) (*third.DeleteOutdatedDataResp, error) { + expireTime := time.UnixMilli(req.ExpireTime) + err := t.s3dataBase.DeleteByExpires(ctx, expireTime) + if err != nil { + return nil, err + } + return &third.DeleteOutdatedDataResp{}, nil +} + type FormDataMate struct { Name string `json:"name"` Size int64 `json:"size"` diff --git a/internal/tools/cron_task.go b/internal/tools/cron_task.go index bf037b694..28963dc63 100644 --- a/internal/tools/cron_task.go +++ b/internal/tools/cron_task.go @@ -17,15 +17,17 @@ package tools import ( "context" "fmt" + "os" + "time" + "github.com/openimsdk/open-im-server/v3/pkg/common/config" kdisc "github.com/openimsdk/open-im-server/v3/pkg/common/discoveryregister" "github.com/openimsdk/protocol/msg" + "github.com/openimsdk/protocol/third" "github.com/openimsdk/tools/mcontext" "github.com/openimsdk/tools/mw" "google.golang.org/grpc" "google.golang.org/grpc/credentials/insecure" - "os" - "time" "github.com/openimsdk/tools/errs" "github.com/openimsdk/tools/log" @@ -69,6 +71,25 @@ func Start(ctx context.Context, config *CronTaskConfig) error { if _, err := crontab.AddFunc(config.CronTask.ChatRecordsClearTime, clearFunc); err != nil { return errs.Wrap(err) } + deleteFunc := func() { + now := time.Now() + deleteTime := now.Add(-time.Hour * 24 * time.Duration(config.CronTask.FileExpireTime)) + ctx := mcontext.SetOperationID(ctx, fmt.Sprintf("cron_%d_%d", os.Getpid(), deleteTime.UnixMilli())) + log.ZInfo(ctx, "deleteoutDatedData ", "deletetime", deleteTime, "timestamp", deleteTime.UnixMilli()) + tConn, err := client.GetConn(ctx, config.Share.RpcRegisterName.Third) + if err != nil { + return + } + thirdClient := third.NewThirdClient(tConn) + if _, err := thirdClient.DeleteOutdatedData(ctx, &third.DeleteOutdatedDataReq{ExpireTime: deleteTime.UnixMilli()}); err != nil { + log.ZError(ctx, "cron deleteoutDatedData failed", err, "deleteTime", deleteTime, "cont", time.Since(now)) + return + } + log.ZInfo(ctx, "cron deleteoutDatedData success", "deltime", deleteTime, "cont", time.Since(now)) + } + if _, err := crontab.AddFunc(string(config.CronTask.FileExpireTime), deleteFunc); err != nil { + return errs.Wrap(err) + } log.ZInfo(ctx, "start cron task", "chatRecordsClearTime", config.CronTask.ChatRecordsClearTime) crontab.Start() <-ctx.Done() diff --git a/pkg/common/config/config.go b/pkg/common/config/config.go index 6260dc00f..7e5649987 100644 --- a/pkg/common/config/config.go +++ b/pkg/common/config/config.go @@ -15,14 +15,15 @@ package config import ( + "strings" + "time" + "github.com/openimsdk/tools/db/mongoutil" "github.com/openimsdk/tools/db/redisutil" "github.com/openimsdk/tools/mq/kafka" "github.com/openimsdk/tools/s3/cos" "github.com/openimsdk/tools/s3/minio" "github.com/openimsdk/tools/s3/oss" - "strings" - "time" ) type CacheConfig struct { @@ -107,6 +108,7 @@ type API struct { type CronTask struct { ChatRecordsClearTime string `mapstructure:"chatRecordsClearTime"` RetainChatRecords int `mapstructure:"retainChatRecords"` + FileExpireTime int `mapstructure:"fileExpireTime"` } type OfflinePushConfig struct { diff --git a/pkg/common/storage/controller/s3.go b/pkg/common/storage/controller/s3.go index b0ad61203..5d83e2677 100644 --- a/pkg/common/storage/controller/s3.go +++ b/pkg/common/storage/controller/s3.go @@ -16,11 +16,12 @@ package controller import ( "context" + "path/filepath" + "time" + redis2 "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/redis" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" - "path/filepath" - "time" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache" "github.com/openimsdk/tools/s3" @@ -38,6 +39,7 @@ type S3Database interface { SetObject(ctx context.Context, info *model.Object) error StatObject(ctx context.Context, name string) (*s3.ObjectInfo, error) FormData(ctx context.Context, name string, size int64, contentType string, duration time.Duration) (*s3.FormData, error) + DeleteByExpires(ctx context.Context, duration time.Time) error } func NewS3Database(rdb redis.UniversalClient, s3 s3.Interface, obj database.ObjectInfo) S3Database { @@ -111,3 +113,6 @@ func (s *s3Database) StatObject(ctx context.Context, name string) (*s3.ObjectInf func (s *s3Database) FormData(ctx context.Context, name string, size int64, contentType string, duration time.Duration) (*s3.FormData, error) { return s.s3.FormData(ctx, name, size, contentType, duration) } +func (s *s3Database) DeleteByExpires(ctx context.Context, duration time.Time) error { + return s.db.DeleteByExpires(ctx, duration) +} diff --git a/pkg/common/storage/database/mgo/object.go b/pkg/common/storage/database/mgo/object.go index df4d10ec4..689c869d5 100644 --- a/pkg/common/storage/database/mgo/object.go +++ b/pkg/common/storage/database/mgo/object.go @@ -16,6 +16,8 @@ package mgo import ( "context" + "time" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" @@ -68,3 +70,8 @@ func (o *S3Mongo) Take(ctx context.Context, engine string, name string) (*model. func (o *S3Mongo) Delete(ctx context.Context, engine string, name string) error { return mongoutil.DeleteOne(ctx, o.coll, bson.M{"name": name, "engine": engine}) } +func (o *S3Mongo) DeleteByExpires(ctx context.Context, duration time.Time) error { + return mongoutil.DeleteMany(ctx, o.coll, bson.M{ + "create_time": bson.M{"$lt": duration}, + }) +} diff --git a/pkg/common/storage/database/object.go b/pkg/common/storage/database/object.go index 554f71f35..3752993d8 100644 --- a/pkg/common/storage/database/object.go +++ b/pkg/common/storage/database/object.go @@ -16,6 +16,8 @@ package database import ( "context" + "time" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" ) @@ -23,4 +25,5 @@ type ObjectInfo interface { SetObject(ctx context.Context, obj *model.Object) error Take(ctx context.Context, engine string, name string) (*model.Object, error) Delete(ctx context.Context, engine string, name string) error + DeleteByExpires(ctx context.Context, duration time.Time) error } diff --git a/pkg/rpcclient/third.go b/pkg/rpcclient/third.go index 4c71dff6a..7cdc60d52 100644 --- a/pkg/rpcclient/third.go +++ b/pkg/rpcclient/third.go @@ -41,3 +41,7 @@ func NewThird(discov discovery.SvcDiscoveryRegistry, rpcRegisterName, grafanaUrl } return &Third{discov: discov, Client: client, conn: conn, GrafanaUrl: grafanaUrl} } +func (t *Third) DeleteOutdatedData(ctx context.Context, expires int64) error { + _, err := t.Client.DeleteOutdatedData(ctx, &third.DeleteOutdatedDataReq{ExpireTime: expires}) + return err +} From dac8fba11f325d03c481f9cf0528664b8d3944b2 Mon Sep 17 00:00:00 2001 From: Monet Lee Date: Fri, 28 Jun 2024 17:59:57 +0800 Subject: [PATCH 54/59] implement minio expire delete logic. --- go.mod | 2 +- go.sum | 4 ++-- internal/rpc/third/s3.go | 6 +++++- pkg/common/storage/controller/s3.go | 15 ++++++++++++--- pkg/common/storage/database/mgo/object.go | 4 ++-- pkg/common/storage/database/object.go | 2 +- 6 files changed, 23 insertions(+), 10 deletions(-) diff --git a/go.mod b/go.mod index a1a51b4fa..f8437922c 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( github.com/lestrrat-go/file-rotatelogs v2.4.0+incompatible // indirect github.com/mitchellh/mapstructure v1.5.0 github.com/openimsdk/protocol v0.0.69-alpha.22 - github.com/openimsdk/tools v0.0.49-alpha.25 + github.com/openimsdk/tools v0.0.49-alpha.30 github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 github.com/stretchr/testify v1.9.0 diff --git a/go.sum b/go.sum index e67b30206..e05652d99 100644 --- a/go.sum +++ b/go.sum @@ -272,8 +272,8 @@ github.com/openimsdk/gomake v0.0.13 h1:xLDe/moqgWpRoptHzI4packAWzs4C16b+sVY+txNJ github.com/openimsdk/gomake v0.0.13/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= github.com/openimsdk/protocol v0.0.69-alpha.22 h1:kifZWVNDkg9diXFJUJ/Q9xFc80cveBhc+1dUXcE9xHQ= github.com/openimsdk/protocol v0.0.69-alpha.22/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= -github.com/openimsdk/tools v0.0.49-alpha.25 h1:OpRPwDZ2xWX7Zj5kyfZhryu/NfZTrsRVr2GFwu1HQHI= -github.com/openimsdk/tools v0.0.49-alpha.25/go.mod h1:rwsFI1G/nBHNfiNapbven41akRDPBbH4df0Cgy6xueU= +github.com/openimsdk/tools v0.0.49-alpha.30 h1:iT2+1F8cJmlwKEris25YgK0seiJRUear+wTgc1bzcg8= +github.com/openimsdk/tools v0.0.49-alpha.30/go.mod h1:zc0maZ2ohXlHd0ylY5JnCE8uqq/hslhcfcKa6iO5PCU= github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= github.com/pelletier/go-toml/v2 v2.1.0/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= diff --git a/internal/rpc/third/s3.go b/internal/rpc/third/s3.go index 19a519d2e..e952175ad 100644 --- a/internal/rpc/third/s3.go +++ b/internal/rpc/third/s3.go @@ -286,10 +286,14 @@ func (t *thirdServer) apiAddress(prefix, name string) string { func (t *thirdServer) DeleteOutdatedData(ctx context.Context, req *third.DeleteOutdatedDataReq) (*third.DeleteOutdatedDataResp, error) { expireTime := time.UnixMilli(req.ExpireTime) - err := t.s3dataBase.DeleteByExpires(ctx, expireTime) + models, err := t.s3dataBase.FindByExpires(ctx, expireTime) if err != nil { return nil, err } + for _, model := range models { + t.s3dataBase.DeleteObject(ctx, model.Key) + t.s3dataBase.DeleteSpecifiedData(ctx, model.Engine, model.Key) + } return &third.DeleteOutdatedDataResp{}, nil } diff --git a/pkg/common/storage/controller/s3.go b/pkg/common/storage/controller/s3.go index 5d83e2677..b1dce502b 100644 --- a/pkg/common/storage/controller/s3.go +++ b/pkg/common/storage/controller/s3.go @@ -39,7 +39,9 @@ type S3Database interface { SetObject(ctx context.Context, info *model.Object) error StatObject(ctx context.Context, name string) (*s3.ObjectInfo, error) FormData(ctx context.Context, name string, size int64, contentType string, duration time.Duration) (*s3.FormData, error) - DeleteByExpires(ctx context.Context, duration time.Time) error + FindByExpires(ctx context.Context, duration time.Time) ([]*model.Object, error) + DeleteObject(ctx context.Context, name string) error + DeleteSpecifiedData(ctx context.Context, engine string, name string) error } func NewS3Database(rdb redis.UniversalClient, s3 s3.Interface, obj database.ObjectInfo) S3Database { @@ -113,6 +115,13 @@ func (s *s3Database) StatObject(ctx context.Context, name string) (*s3.ObjectInf func (s *s3Database) FormData(ctx context.Context, name string, size int64, contentType string, duration time.Duration) (*s3.FormData, error) { return s.s3.FormData(ctx, name, size, contentType, duration) } -func (s *s3Database) DeleteByExpires(ctx context.Context, duration time.Time) error { - return s.db.DeleteByExpires(ctx, duration) +func (s *s3Database) FindByExpires(ctx context.Context, duration time.Time) ([]*model.Object, error) { + return s.db.FindByExpires(ctx, duration) +} + +func (s *s3Database) DeleteObject(ctx context.Context, name string) error { + return s.s3.DeleteObject(ctx, name) +} +func (s *s3Database) DeleteSpecifiedData(ctx context.Context, engine string, name string) error { + return s.db.Delete(ctx, engine, name) } diff --git a/pkg/common/storage/database/mgo/object.go b/pkg/common/storage/database/mgo/object.go index 689c869d5..a0abf7a7a 100644 --- a/pkg/common/storage/database/mgo/object.go +++ b/pkg/common/storage/database/mgo/object.go @@ -70,8 +70,8 @@ func (o *S3Mongo) Take(ctx context.Context, engine string, name string) (*model. func (o *S3Mongo) Delete(ctx context.Context, engine string, name string) error { return mongoutil.DeleteOne(ctx, o.coll, bson.M{"name": name, "engine": engine}) } -func (o *S3Mongo) DeleteByExpires(ctx context.Context, duration time.Time) error { - return mongoutil.DeleteMany(ctx, o.coll, bson.M{ +func (o *S3Mongo) FindByExpires(ctx context.Context, duration time.Time) ([]*model.Object, error) { + return mongoutil.Find[*model.Object](ctx, o.coll, bson.M{ "create_time": bson.M{"$lt": duration}, }) } diff --git a/pkg/common/storage/database/object.go b/pkg/common/storage/database/object.go index 3752993d8..44329cbc4 100644 --- a/pkg/common/storage/database/object.go +++ b/pkg/common/storage/database/object.go @@ -25,5 +25,5 @@ type ObjectInfo interface { SetObject(ctx context.Context, obj *model.Object) error Take(ctx context.Context, engine string, name string) (*model.Object, error) Delete(ctx context.Context, engine string, name string) error - DeleteByExpires(ctx context.Context, duration time.Time) error + FindByExpires(ctx context.Context, duration time.Time) ([]*model.Object, error) } From 330611796c390404e3baa46d240b84b459b4ef81 Mon Sep 17 00:00:00 2001 From: Monet Lee Date: Tue, 2 Jul 2024 18:10:27 +0800 Subject: [PATCH 55/59] feat: implement scheduled delete outdated object in minio. --- config/openim-crontask.yml | 2 +- go.mod | 2 +- go.sum | 4 +-- internal/rpc/third/s3.go | 34 ++++++++++++++++++++--- internal/tools/cron_task.go | 20 +++++++------ pkg/common/config/config.go | 6 ++-- pkg/common/storage/controller/s3.go | 31 ++++++++++++++++----- pkg/common/storage/controller/third.go | 3 +- pkg/common/storage/database/mgo/object.go | 6 ++++ pkg/common/storage/database/object.go | 1 + 10 files changed, 81 insertions(+), 28 deletions(-) diff --git a/config/openim-crontask.yml b/config/openim-crontask.yml index d2154d263..3839104a4 100644 --- a/config/openim-crontask.yml +++ b/config/openim-crontask.yml @@ -1,3 +1,3 @@ -chatRecordsClearTime: "0 2 * * *" +cronExecuteTime: "0 2 * * *" retainChatRecords: 365 fileExpireTime: 90 diff --git a/go.mod b/go.mod index f8437922c..49e5b50a4 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( github.com/lestrrat-go/file-rotatelogs v2.4.0+incompatible // indirect github.com/mitchellh/mapstructure v1.5.0 github.com/openimsdk/protocol v0.0.69-alpha.22 - github.com/openimsdk/tools v0.0.49-alpha.30 + github.com/openimsdk/tools v0.0.49-alpha.39 github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 github.com/stretchr/testify v1.9.0 diff --git a/go.sum b/go.sum index e05652d99..a7dd1d632 100644 --- a/go.sum +++ b/go.sum @@ -272,8 +272,8 @@ github.com/openimsdk/gomake v0.0.13 h1:xLDe/moqgWpRoptHzI4packAWzs4C16b+sVY+txNJ github.com/openimsdk/gomake v0.0.13/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= github.com/openimsdk/protocol v0.0.69-alpha.22 h1:kifZWVNDkg9diXFJUJ/Q9xFc80cveBhc+1dUXcE9xHQ= github.com/openimsdk/protocol v0.0.69-alpha.22/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= -github.com/openimsdk/tools v0.0.49-alpha.30 h1:iT2+1F8cJmlwKEris25YgK0seiJRUear+wTgc1bzcg8= -github.com/openimsdk/tools v0.0.49-alpha.30/go.mod h1:zc0maZ2ohXlHd0ylY5JnCE8uqq/hslhcfcKa6iO5PCU= +github.com/openimsdk/tools v0.0.49-alpha.39 h1:bl5+q7xHsc/j1NnkN8/gYmn23RsNNbRizDY58d2EY1w= +github.com/openimsdk/tools v0.0.49-alpha.39/go.mod h1:zc0maZ2ohXlHd0ylY5JnCE8uqq/hslhcfcKa6iO5PCU= github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= github.com/pelletier/go-toml/v2 v2.1.0/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= diff --git a/internal/rpc/third/s3.go b/internal/rpc/third/s3.go index e952175ad..99c21d5bb 100644 --- a/internal/rpc/third/s3.go +++ b/internal/rpc/third/s3.go @@ -24,6 +24,7 @@ import ( "time" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" + "go.mongodb.org/mongo-driver/mongo" "github.com/google/uuid" "github.com/openimsdk/open-im-server/v3/pkg/common/servererrs" @@ -287,12 +288,37 @@ func (t *thirdServer) apiAddress(prefix, name string) string { func (t *thirdServer) DeleteOutdatedData(ctx context.Context, req *third.DeleteOutdatedDataReq) (*third.DeleteOutdatedDataResp, error) { expireTime := time.UnixMilli(req.ExpireTime) models, err := t.s3dataBase.FindByExpires(ctx, expireTime) - if err != nil { - return nil, err + if err != nil && errs.Unwrap(err) != mongo.ErrNoDocuments { + return nil, errs.Wrap(err) + } + needDelObjectKeys := make([]string, 0) + for _, model := range models { + needDelObjectKeys = append(needDelObjectKeys, model.Key) + } + + needDelObjectKeys = datautil.Distinct(needDelObjectKeys) + for _, key := range needDelObjectKeys { + count, err := t.s3dataBase.FindNotDelByS3(ctx, key, expireTime) + if err != nil && errs.Unwrap(err) != mongo.ErrNoDocuments { + return nil, errs.Wrap(err) + } + if int(count) < 1 { + thumbnailKey, err := t.s3dataBase.GetImageThumbnailKey(ctx, key) + if err != nil { + return nil, errs.Wrap(err) + } + t.s3dataBase.DeleteObject(ctx, thumbnailKey) + t.s3dataBase.DelS3Key(ctx, "minio", needDelObjectKeys...) + + t.s3dataBase.DeleteObject(ctx, key) + } } + for _, model := range models { - t.s3dataBase.DeleteObject(ctx, model.Key) - t.s3dataBase.DeleteSpecifiedData(ctx, model.Engine, model.Key) + err := t.s3dataBase.DeleteSpecifiedData(ctx, model.Engine, model.Name) + if err != nil { + return nil, errs.Wrap(err) + } } return &third.DeleteOutdatedDataResp{}, nil } diff --git a/internal/tools/cron_task.go b/internal/tools/cron_task.go index 28963dc63..dcdcf2f40 100644 --- a/internal/tools/cron_task.go +++ b/internal/tools/cron_task.go @@ -41,7 +41,7 @@ type CronTaskConfig struct { } func Start(ctx context.Context, config *CronTaskConfig) error { - log.CInfo(ctx, "CRON-TASK server is initializing", "chatRecordsClearTime", config.CronTask.ChatRecordsClearTime, "msgDestructTime", config.CronTask.RetainChatRecords) + log.CInfo(ctx, "CRON-TASK server is initializing", "chatRecordsClearTime", config.CronTask.CronExecuteTime, "msgDestructTime", config.CronTask.RetainChatRecords) if config.CronTask.RetainChatRecords < 1 { return errs.New("msg destruct time must be greater than 1").Wrap() } @@ -68,29 +68,31 @@ func Start(ctx context.Context, config *CronTaskConfig) error { } log.ZInfo(ctx, "cron clear chat records success", "deltime", deltime, "cont", time.Since(now)) } - if _, err := crontab.AddFunc(config.CronTask.ChatRecordsClearTime, clearFunc); err != nil { + if _, err := crontab.AddFunc(config.CronTask.CronExecuteTime, clearFunc); err != nil { return errs.Wrap(err) } + + tConn, err := client.GetConn(ctx, config.Share.RpcRegisterName.Third) + if err != nil { + return err + } + thirdClient := third.NewThirdClient(tConn) + deleteFunc := func() { now := time.Now() deleteTime := now.Add(-time.Hour * 24 * time.Duration(config.CronTask.FileExpireTime)) ctx := mcontext.SetOperationID(ctx, fmt.Sprintf("cron_%d_%d", os.Getpid(), deleteTime.UnixMilli())) log.ZInfo(ctx, "deleteoutDatedData ", "deletetime", deleteTime, "timestamp", deleteTime.UnixMilli()) - tConn, err := client.GetConn(ctx, config.Share.RpcRegisterName.Third) - if err != nil { - return - } - thirdClient := third.NewThirdClient(tConn) if _, err := thirdClient.DeleteOutdatedData(ctx, &third.DeleteOutdatedDataReq{ExpireTime: deleteTime.UnixMilli()}); err != nil { log.ZError(ctx, "cron deleteoutDatedData failed", err, "deleteTime", deleteTime, "cont", time.Since(now)) return } log.ZInfo(ctx, "cron deleteoutDatedData success", "deltime", deleteTime, "cont", time.Since(now)) } - if _, err := crontab.AddFunc(string(config.CronTask.FileExpireTime), deleteFunc); err != nil { + if _, err := crontab.AddFunc(config.CronTask.CronExecuteTime, deleteFunc); err != nil { return errs.Wrap(err) } - log.ZInfo(ctx, "start cron task", "chatRecordsClearTime", config.CronTask.ChatRecordsClearTime) + log.ZInfo(ctx, "start cron task", "CronExecuteTime", config.CronTask.CronExecuteTime) crontab.Start() <-ctx.Done() return nil diff --git a/pkg/common/config/config.go b/pkg/common/config/config.go index 7e5649987..3d49ff577 100644 --- a/pkg/common/config/config.go +++ b/pkg/common/config/config.go @@ -106,9 +106,9 @@ type API struct { } type CronTask struct { - ChatRecordsClearTime string `mapstructure:"chatRecordsClearTime"` - RetainChatRecords int `mapstructure:"retainChatRecords"` - FileExpireTime int `mapstructure:"fileExpireTime"` + CronExecuteTime string `mapstructure:"cronExecuteTime"` + RetainChatRecords int `mapstructure:"retainChatRecords"` + FileExpireTime int `mapstructure:"fileExpireTime"` } type OfflinePushConfig struct { diff --git a/pkg/common/storage/controller/s3.go b/pkg/common/storage/controller/s3.go index b1dce502b..f32d8803e 100644 --- a/pkg/common/storage/controller/s3.go +++ b/pkg/common/storage/controller/s3.go @@ -19,7 +19,7 @@ import ( "path/filepath" "time" - redis2 "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/redis" + redisCache "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/redis" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" @@ -42,20 +42,25 @@ type S3Database interface { FindByExpires(ctx context.Context, duration time.Time) ([]*model.Object, error) DeleteObject(ctx context.Context, name string) error DeleteSpecifiedData(ctx context.Context, engine string, name string) error + FindNotDelByS3(ctx context.Context, key string, duration time.Time) (int64, error) + DelS3Key(ctx context.Context, engine string, keys ...string) error + GetImageThumbnailKey(ctx context.Context, name string) (string, error) } func NewS3Database(rdb redis.UniversalClient, s3 s3.Interface, obj database.ObjectInfo) S3Database { return &s3Database{ - s3: cont.New(redis2.NewS3Cache(rdb, s3), s3), - cache: redis2.NewObjectCacheRedis(rdb, obj), - db: obj, + s3: cont.New(redisCache.NewS3Cache(rdb, s3), s3), + cache: redisCache.NewObjectCacheRedis(rdb, obj), + s3cache: redisCache.NewS3Cache(rdb, s3), + db: obj, } } type s3Database struct { - s3 *cont.Controller - cache cache.ObjectCache - db database.ObjectInfo + s3 *cont.Controller + cache cache.ObjectCache + s3cache cont.S3Cache + db database.ObjectInfo } func (s *s3Database) PartSize(ctx context.Context, size int64) (int64, error) { @@ -125,3 +130,15 @@ func (s *s3Database) DeleteObject(ctx context.Context, name string) error { func (s *s3Database) DeleteSpecifiedData(ctx context.Context, engine string, name string) error { return s.db.Delete(ctx, engine, name) } + +func (s *s3Database) FindNotDelByS3(ctx context.Context, key string, duration time.Time) (int64, error) { + return s.db.FindNotDelByS3(ctx, key, duration) +} + +func (s *s3Database) DelS3Key(ctx context.Context, engine string, keys ...string) error { + return s.s3cache.DelS3Key(ctx, engine, keys...) +} + +func (s *s3Database) GetImageThumbnailKey(ctx context.Context, name string) (string, error) { + return s.s3.GetImageThumbnailKey(ctx, name) +} diff --git a/pkg/common/storage/controller/third.go b/pkg/common/storage/controller/third.go index 344501466..a9c2ae403 100644 --- a/pkg/common/storage/controller/third.go +++ b/pkg/common/storage/controller/third.go @@ -16,9 +16,10 @@ package controller import ( "context" + "time" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" - "time" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache" "github.com/openimsdk/tools/db/pagination" diff --git a/pkg/common/storage/database/mgo/object.go b/pkg/common/storage/database/mgo/object.go index a0abf7a7a..3ae5b8ec8 100644 --- a/pkg/common/storage/database/mgo/object.go +++ b/pkg/common/storage/database/mgo/object.go @@ -75,3 +75,9 @@ func (o *S3Mongo) FindByExpires(ctx context.Context, duration time.Time) ([]*mod "create_time": bson.M{"$lt": duration}, }) } +func (o *S3Mongo) FindNotDelByS3(ctx context.Context, key string, duration time.Time) (int64, error) { + return mongoutil.Count(ctx, o.coll, bson.M{ + "key": key, + "create_time": bson.M{"$gt": duration}, + }) +} diff --git a/pkg/common/storage/database/object.go b/pkg/common/storage/database/object.go index 44329cbc4..4046da2f3 100644 --- a/pkg/common/storage/database/object.go +++ b/pkg/common/storage/database/object.go @@ -26,4 +26,5 @@ type ObjectInfo interface { Take(ctx context.Context, engine string, name string) (*model.Object, error) Delete(ctx context.Context, engine string, name string) error FindByExpires(ctx context.Context, duration time.Time) ([]*model.Object, error) + FindNotDelByS3(ctx context.Context, key string, duration time.Time) (int64, error) } From 7897044e282496dfbbf0e2e20323db3b70f7edfb Mon Sep 17 00:00:00 2001 From: Monet Lee Date: Wed, 3 Jul 2024 11:03:03 +0800 Subject: [PATCH 56/59] implement FindExpires pagination. --- internal/rpc/third/s3.go | 57 ++++++++++++++--------- pkg/common/storage/common/types.go | 12 +++++ pkg/common/storage/controller/s3.go | 8 ++-- pkg/common/storage/database/mgo/object.go | 7 +-- pkg/common/storage/database/object.go | 3 +- 5 files changed, 57 insertions(+), 30 deletions(-) diff --git a/internal/rpc/third/s3.go b/internal/rpc/third/s3.go index 99c21d5bb..c419cfc8c 100644 --- a/internal/rpc/third/s3.go +++ b/internal/rpc/third/s3.go @@ -23,6 +23,8 @@ import ( "strconv" "time" + "github.com/openimsdk/open-im-server/v3/pkg/common/config" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/common" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" "go.mongodb.org/mongo-driver/mongo" @@ -286,39 +288,48 @@ func (t *thirdServer) apiAddress(prefix, name string) string { } func (t *thirdServer) DeleteOutdatedData(ctx context.Context, req *third.DeleteOutdatedDataReq) (*third.DeleteOutdatedDataResp, error) { + var conf config.Third expireTime := time.UnixMilli(req.ExpireTime) - models, err := t.s3dataBase.FindByExpires(ctx, expireTime) - if err != nil && errs.Unwrap(err) != mongo.ErrNoDocuments { - return nil, errs.Wrap(err) + findPagination := &common.FindPagination{ + PageNumber: 1, + ShowNumber: 1000, } - needDelObjectKeys := make([]string, 0) - for _, model := range models { - needDelObjectKeys = append(needDelObjectKeys, model.Key) - } - - needDelObjectKeys = datautil.Distinct(needDelObjectKeys) - for _, key := range needDelObjectKeys { - count, err := t.s3dataBase.FindNotDelByS3(ctx, key, expireTime) + for { + total, models, err := t.s3dataBase.FindByExpires(ctx, expireTime, findPagination) if err != nil && errs.Unwrap(err) != mongo.ErrNoDocuments { return nil, errs.Wrap(err) } - if int(count) < 1 { - thumbnailKey, err := t.s3dataBase.GetImageThumbnailKey(ctx, key) + needDelObjectKeys := make([]string, 0) + for _, model := range models { + needDelObjectKeys = append(needDelObjectKeys, model.Key) + } + + needDelObjectKeys = datautil.Distinct(needDelObjectKeys) + for _, key := range needDelObjectKeys { + count, err := t.s3dataBase.FindNotDelByS3(ctx, key, expireTime) + if err != nil && errs.Unwrap(err) != mongo.ErrNoDocuments { + return nil, errs.Wrap(err) + } + if int(count) < 1 { + thumbnailKey, err := t.s3dataBase.GetImageThumbnailKey(ctx, key) + if err != nil { + return nil, errs.Wrap(err) + } + t.s3dataBase.DeleteObject(ctx, thumbnailKey) + t.s3dataBase.DelS3Key(ctx, conf.Object.Enable, needDelObjectKeys...) + t.s3dataBase.DeleteObject(ctx, key) + } + } + for _, model := range models { + err := t.s3dataBase.DeleteSpecifiedData(ctx, model.Engine, model.Name) if err != nil { return nil, errs.Wrap(err) } - t.s3dataBase.DeleteObject(ctx, thumbnailKey) - t.s3dataBase.DelS3Key(ctx, "minio", needDelObjectKeys...) - - t.s3dataBase.DeleteObject(ctx, key) } - } - - for _, model := range models { - err := t.s3dataBase.DeleteSpecifiedData(ctx, model.Engine, model.Name) - if err != nil { - return nil, errs.Wrap(err) + if total < int64(findPagination.ShowNumber) { + break } + findPagination.PageNumber++ } return &third.DeleteOutdatedDataResp{}, nil } diff --git a/pkg/common/storage/common/types.go b/pkg/common/storage/common/types.go index 759121158..a19de205e 100644 --- a/pkg/common/storage/common/types.go +++ b/pkg/common/storage/common/types.go @@ -24,3 +24,15 @@ type GroupSimpleUserID struct { Hash uint64 MemberNum uint32 } + +type FindPagination struct { + PageNumber int32 + ShowNumber int32 +} + +func (f *FindPagination) GetPageNumber() int32 { + return f.PageNumber +} +func (f *FindPagination) GetShowNumber() int32 { + return f.ShowNumber +} diff --git a/pkg/common/storage/controller/s3.go b/pkg/common/storage/controller/s3.go index f32d8803e..592659536 100644 --- a/pkg/common/storage/controller/s3.go +++ b/pkg/common/storage/controller/s3.go @@ -24,6 +24,7 @@ import ( "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache" + "github.com/openimsdk/tools/db/pagination" "github.com/openimsdk/tools/s3" "github.com/openimsdk/tools/s3/cont" "github.com/redis/go-redis/v9" @@ -39,7 +40,7 @@ type S3Database interface { SetObject(ctx context.Context, info *model.Object) error StatObject(ctx context.Context, name string) (*s3.ObjectInfo, error) FormData(ctx context.Context, name string, size int64, contentType string, duration time.Duration) (*s3.FormData, error) - FindByExpires(ctx context.Context, duration time.Time) ([]*model.Object, error) + FindByExpires(ctx context.Context, duration time.Time, pagination pagination.Pagination) (total int64, objects []*model.Object, err error) DeleteObject(ctx context.Context, name string) error DeleteSpecifiedData(ctx context.Context, engine string, name string) error FindNotDelByS3(ctx context.Context, key string, duration time.Time) (int64, error) @@ -120,8 +121,9 @@ func (s *s3Database) StatObject(ctx context.Context, name string) (*s3.ObjectInf func (s *s3Database) FormData(ctx context.Context, name string, size int64, contentType string, duration time.Duration) (*s3.FormData, error) { return s.s3.FormData(ctx, name, size, contentType, duration) } -func (s *s3Database) FindByExpires(ctx context.Context, duration time.Time) ([]*model.Object, error) { - return s.db.FindByExpires(ctx, duration) +func (s *s3Database) FindByExpires(ctx context.Context, duration time.Time, pagination pagination.Pagination) (total int64, objects []*model.Object, err error) { + + return s.db.FindByExpires(ctx, duration, pagination) } func (s *s3Database) DeleteObject(ctx context.Context, name string) error { diff --git a/pkg/common/storage/database/mgo/object.go b/pkg/common/storage/database/mgo/object.go index 3ae5b8ec8..4242fbb53 100644 --- a/pkg/common/storage/database/mgo/object.go +++ b/pkg/common/storage/database/mgo/object.go @@ -22,6 +22,7 @@ import ( "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" "github.com/openimsdk/tools/db/mongoutil" + "github.com/openimsdk/tools/db/pagination" "github.com/openimsdk/tools/errs" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/mongo" @@ -70,10 +71,10 @@ func (o *S3Mongo) Take(ctx context.Context, engine string, name string) (*model. func (o *S3Mongo) Delete(ctx context.Context, engine string, name string) error { return mongoutil.DeleteOne(ctx, o.coll, bson.M{"name": name, "engine": engine}) } -func (o *S3Mongo) FindByExpires(ctx context.Context, duration time.Time) ([]*model.Object, error) { - return mongoutil.Find[*model.Object](ctx, o.coll, bson.M{ +func (o *S3Mongo) FindByExpires(ctx context.Context, duration time.Time, pagination pagination.Pagination) (total int64, objects []*model.Object, err error) { + return mongoutil.FindPage[*model.Object](ctx, o.coll, bson.M{ "create_time": bson.M{"$lt": duration}, - }) + }, pagination) } func (o *S3Mongo) FindNotDelByS3(ctx context.Context, key string, duration time.Time) (int64, error) { return mongoutil.Count(ctx, o.coll, bson.M{ diff --git a/pkg/common/storage/database/object.go b/pkg/common/storage/database/object.go index 4046da2f3..8292006a0 100644 --- a/pkg/common/storage/database/object.go +++ b/pkg/common/storage/database/object.go @@ -19,12 +19,13 @@ import ( "time" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" + "github.com/openimsdk/tools/db/pagination" ) type ObjectInfo interface { SetObject(ctx context.Context, obj *model.Object) error Take(ctx context.Context, engine string, name string) (*model.Object, error) Delete(ctx context.Context, engine string, name string) error - FindByExpires(ctx context.Context, duration time.Time) ([]*model.Object, error) + FindByExpires(ctx context.Context, duration time.Time, pagination pagination.Pagination) (total int64, objects []*model.Object, err error) FindNotDelByS3(ctx context.Context, key string, duration time.Time) (int64, error) } From 686ccae18d89e64eb8dfc92bb03ed6fa7140923a Mon Sep 17 00:00:00 2001 From: Monet Lee Date: Wed, 3 Jul 2024 11:10:33 +0800 Subject: [PATCH 57/59] remove unnesseary incr. --- internal/rpc/third/s3.go | 1 - 1 file changed, 1 deletion(-) diff --git a/internal/rpc/third/s3.go b/internal/rpc/third/s3.go index c419cfc8c..7d6d2a375 100644 --- a/internal/rpc/third/s3.go +++ b/internal/rpc/third/s3.go @@ -329,7 +329,6 @@ func (t *thirdServer) DeleteOutdatedData(ctx context.Context, req *third.DeleteO if total < int64(findPagination.ShowNumber) { break } - findPagination.PageNumber++ } return &third.DeleteOutdatedDataResp{}, nil } From a421bd11bb34098f61b549f9fdff9e8358ceb6fd Mon Sep 17 00:00:00 2001 From: Monet Lee Date: Wed, 3 Jul 2024 11:14:38 +0800 Subject: [PATCH 58/59] fix uncorrect args call. --- internal/rpc/third/s3.go | 3 ++- pkg/common/storage/common/types.go | 12 ------------ 2 files changed, 2 insertions(+), 13 deletions(-) diff --git a/internal/rpc/third/s3.go b/internal/rpc/third/s3.go index 7d6d2a375..a1e3d857e 100644 --- a/internal/rpc/third/s3.go +++ b/internal/rpc/third/s3.go @@ -30,6 +30,7 @@ import ( "github.com/google/uuid" "github.com/openimsdk/open-im-server/v3/pkg/common/servererrs" + "github.com/openimsdk/protocol/sdkws" "github.com/openimsdk/protocol/third" "github.com/openimsdk/tools/errs" "github.com/openimsdk/tools/log" @@ -290,7 +291,7 @@ func (t *thirdServer) apiAddress(prefix, name string) string { func (t *thirdServer) DeleteOutdatedData(ctx context.Context, req *third.DeleteOutdatedDataReq) (*third.DeleteOutdatedDataResp, error) { var conf config.Third expireTime := time.UnixMilli(req.ExpireTime) - findPagination := &common.FindPagination{ + findPagination := &sdkws.RequestPagination{ PageNumber: 1, ShowNumber: 1000, } diff --git a/pkg/common/storage/common/types.go b/pkg/common/storage/common/types.go index a19de205e..759121158 100644 --- a/pkg/common/storage/common/types.go +++ b/pkg/common/storage/common/types.go @@ -24,15 +24,3 @@ type GroupSimpleUserID struct { Hash uint64 MemberNum uint32 } - -type FindPagination struct { - PageNumber int32 - ShowNumber int32 -} - -func (f *FindPagination) GetPageNumber() int32 { - return f.PageNumber -} -func (f *FindPagination) GetShowNumber() int32 { - return f.ShowNumber -} From c5cf078cd68e2640b2fcb8a8e0a903d15667a144 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Thu, 4 Jul 2024 10:29:33 +0800 Subject: [PATCH 59/59] resolving conflicts --- go.sum | 8 ++++---- internal/rpc/third/s3.go | 1 - internal/rpc/user/user.go | 5 +++++ pkg/apistruct/manage.go | 2 +- pkg/common/storage/cache/redis/seq.go | 0 5 files changed, 10 insertions(+), 6 deletions(-) delete mode 100644 pkg/common/storage/cache/redis/seq.go diff --git a/go.sum b/go.sum index a5a620a14..f452bcd87 100644 --- a/go.sum +++ b/go.sum @@ -270,10 +270,10 @@ github.com/onsi/gomega v1.25.0 h1:Vw7br2PCDYijJHSfBOWhov+8cAnUf8MfMaIOV323l6Y= github.com/onsi/gomega v1.25.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdMPGhM= github.com/openimsdk/gomake v0.0.14-alpha.5 h1:VY9c5x515lTfmdhhPjMvR3BBRrRquAUCFsz7t7vbv7Y= github.com/openimsdk/gomake v0.0.14-alpha.5/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= -github.com/openimsdk/protocol v0.0.69-alpha.17 h1:pEag4ZdlovE+AyLsw1VYFU/3sk6ayvGdPzgufQfKf9M= -github.com/openimsdk/protocol v0.0.69-alpha.17/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= -github.com/openimsdk/tools v0.0.49-alpha.28 h1:1CfdFxvKzyOIvgNMVMq4ZB2upAJ0evLbbigOhWQzhu8= -github.com/openimsdk/tools v0.0.49-alpha.28/go.mod h1:rwsFI1G/nBHNfiNapbven41akRDPBbH4df0Cgy6xueU= +github.com/openimsdk/protocol v0.0.69-alpha.22 h1:kifZWVNDkg9diXFJUJ/Q9xFc80cveBhc+1dUXcE9xHQ= +github.com/openimsdk/protocol v0.0.69-alpha.22/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= +github.com/openimsdk/tools v0.0.49-alpha.39 h1:bl5+q7xHsc/j1NnkN8/gYmn23RsNNbRizDY58d2EY1w= +github.com/openimsdk/tools v0.0.49-alpha.39/go.mod h1:zc0maZ2ohXlHd0ylY5JnCE8uqq/hslhcfcKa6iO5PCU= github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= github.com/pelletier/go-toml/v2 v2.1.0/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= diff --git a/internal/rpc/third/s3.go b/internal/rpc/third/s3.go index a1e3d857e..54cbc30c1 100644 --- a/internal/rpc/third/s3.go +++ b/internal/rpc/third/s3.go @@ -24,7 +24,6 @@ import ( "time" "github.com/openimsdk/open-im-server/v3/pkg/common/config" - "github.com/openimsdk/open-im-server/v3/pkg/common/storage/common" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" "go.mongodb.org/mongo-driver/mongo" diff --git a/internal/rpc/user/user.go b/internal/rpc/user/user.go index 211b360b7..2fea232b4 100644 --- a/internal/rpc/user/user.go +++ b/internal/rpc/user/user.go @@ -60,6 +60,11 @@ type userServer struct { webhookClient *webhook.Client } +func (s *userServer) SetUserOnlineStatus(ctx context.Context, req *pbuser.SetUserOnlineStatusReq) (*pbuser.SetUserOnlineStatusResp, error) { + //TODO implement me + panic("implement me") +} + type Config struct { RpcConfig config.User RedisConfig config.Redis diff --git a/pkg/apistruct/manage.go b/pkg/apistruct/manage.go index 6ea6a29ed..f4deb9fb1 100644 --- a/pkg/apistruct/manage.go +++ b/pkg/apistruct/manage.go @@ -15,7 +15,7 @@ package apistruct import ( - sdkws "github.com/openimsdk/protocol/sdkws" + "github.com/openimsdk/protocol/sdkws" ) // SendMsg defines the structure for sending messages with various metadata. diff --git a/pkg/common/storage/cache/redis/seq.go b/pkg/common/storage/cache/redis/seq.go deleted file mode 100644 index e69de29bb..000000000