From 9ca38cb4b5f1765c9571d25c3e91e33e98891df2 Mon Sep 17 00:00:00 2001 From: Michael Li Date: Sat, 30 Sep 2023 21:13:26 +0800 Subject: [PATCH] optimize get index trends info logic --- internal/conf/db.go | 1 + internal/conf/setting.go | 1 + internal/core/cs/trends.go | 18 ++++++ internal/dao/jinzhu/gorm.go | 2 + internal/dao/jinzhu/trends.go | 10 +-- internal/servants/web/events.go | 62 ++++++++++++------- internal/servants/web/followship.go | 2 + internal/servants/web/friendship.go | 2 + .../0009_create_view_post_filter.down.sql | 3 - .../mysql/0014_user_relation_view.down.sql | 1 + .../mysql/0014_user_relation_view.up.sql | 6 ++ .../postgres/0013_user_relation_view.down.sql | 1 + .../postgres/0013_user_relation_view.up.sql | 6 ++ .../sqlite3/0014_user_relation_view.down.sql | 1 + .../sqlite3/0014_user_relation_view.up.sql | 6 ++ scripts/paopao-mysql.sql | 8 +++ scripts/paopao-postgres.sql | 8 +++ scripts/paopao-sqlite3.sql | 8 +++ 18 files changed, 118 insertions(+), 28 deletions(-) create mode 100644 scripts/migration/mysql/0014_user_relation_view.down.sql create mode 100644 scripts/migration/mysql/0014_user_relation_view.up.sql create mode 100644 scripts/migration/postgres/0013_user_relation_view.down.sql create mode 100644 scripts/migration/postgres/0013_user_relation_view.up.sql create mode 100644 scripts/migration/sqlite3/0014_user_relation_view.down.sql create mode 100644 scripts/migration/sqlite3/0014_user_relation_view.up.sql diff --git a/internal/conf/db.go b/internal/conf/db.go index b7bb6727..db0a1e43 100644 --- a/internal/conf/db.go +++ b/internal/conf/db.go @@ -40,6 +40,7 @@ const ( TablePostStar = "post_star" TableTag = "tag" TableUser = "user" + TableUserRelation = "user_relation" TableUserMetric = "user_metric" TableWalletRecharge = "wallet_recharge" TableWalletStatement = "wallet_statement" diff --git a/internal/conf/setting.go b/internal/conf/setting.go index 49084330..fe90ffec 100644 --- a/internal/conf/setting.go +++ b/internal/conf/setting.go @@ -368,6 +368,7 @@ func (s *databaseConf) TableNames() (res TableNameMap) { TablePostStar, TableTag, TableUser, + TableUserRelation, TableUserMetric, TableWalletRecharge, TableWalletStatement, diff --git a/internal/core/cs/trends.go b/internal/core/cs/trends.go index 80026d74..5739e8aa 100644 --- a/internal/core/cs/trends.go +++ b/internal/core/cs/trends.go @@ -4,9 +4,27 @@ package cs +import "github.com/rocboss/paopao-ce/pkg/types" + type TrendsItem struct { Username string `json:"username"` Nickname string `json:"nickname"` Avatar string `json:"avatar"` IsFresh bool `json:"is_fresh" gorm:"-"` } + +func DistinctTrends(items []*TrendsItem) []*TrendsItem { + if len(items) == 0 { + return items + } + res := make([]*TrendsItem, 0, len(items)) + set := make(map[string]types.Empty, len(items)) + for _, item := range items { + if _, exist := set[item.Username]; exist { + continue + } + res = append(res, item) + set[item.Username] = types.Empty{} + } + return res +} diff --git a/internal/dao/jinzhu/gorm.go b/internal/dao/jinzhu/gorm.go index 4f50ef3f..0a0cdb27 100644 --- a/internal/dao/jinzhu/gorm.go +++ b/internal/dao/jinzhu/gorm.go @@ -32,6 +32,7 @@ var ( _postStar_ string _tag_ string _user_ string + _userRelation_ string _userMetric_ string _walletRecharge_ string _walletStatement_ string @@ -61,6 +62,7 @@ func initTableName() { _postStar_ = m[conf.TablePostStar] _tag_ = m[conf.TableTag] _user_ = m[conf.TableUser] + _userRelation_ = m[conf.TableUserRelation] _userMetric_ = m[conf.TableUserMetric] _walletRecharge_ = m[conf.TableWalletRecharge] _walletStatement_ = m[conf.TableWalletStatement] diff --git a/internal/dao/jinzhu/trends.go b/internal/dao/jinzhu/trends.go index 44e56c67..e4fa8061 100644 --- a/internal/dao/jinzhu/trends.go +++ b/internal/dao/jinzhu/trends.go @@ -18,16 +18,18 @@ type trendsSrvA struct { func (s *trendsSrvA) GetIndexTrends(userId int64, limit int, offset int) (res []*cs.TrendsItem, total int64, err error) { db := s.db.Table(_user_). - Joins(fmt.Sprintf("JOIN %s c ON c.friend_id=%s.id AND c.user_id=? AND c.status=2 AND c.is_del=0", _contact_, _user_), userId). - Joins(fmt.Sprintf("JOIN %s m ON c.friend_id=m.user_id AND m.tweets_count>0 AND m.is_del=0", _userMetric_)). - Where(fmt.Sprintf("%s.is_del=0", _user_)) + Joins(fmt.Sprintf("JOIN %s r ON r.he_uid=%s.id", _userRelation_, _user_)). + Joins(fmt.Sprintf("JOIN %s m ON r.he_uid=m.user_id", _userMetric_)). + Where("r.user_id=? AND m.tweets_count>0 AND m.is_del=0", userId) if err = db.Count(&total).Error; err != nil || total == 0 { return } if offset >= 0 && limit > 0 { db = db.Limit(limit).Offset(offset) } - err = db.Find(&res).Error + if err = db.Order("r.style ASC").Select("username", "nickname", "avatar").Find(&res).Error; err == nil { + res = cs.DistinctTrends(res) + } return } diff --git a/internal/servants/web/events.go b/internal/servants/web/events.go index 292d2c30..0da09764 100644 --- a/internal/servants/web/events.go +++ b/internal/servants/web/events.go @@ -41,6 +41,10 @@ const ( const ( _trendsActionCreateTweet uint8 = iota _trendsActionDeleteTweet + _trendsActionFollowUser + _trendsActionUnfollowUser + _trendsActionAddFriend + _trendsActionDeleteFriend ) type cacheUnreadMsgEvent struct { @@ -75,18 +79,18 @@ type messageActionEvent struct { type trendsActionEvent struct { event.UnimplementedEvent - ac core.AppCache - ds core.DataService - action uint8 - userId int64 + ac core.AppCache + ds core.DataService + action uint8 + userIds []int64 } -func onTrendsActionEvent(action uint8, userId int64) { +func onTrendsActionEvent(action uint8, userIds ...int64) { events.OnEvent(&trendsActionEvent{ - ac: _ac, - ds: _ds, - action: action, - userId: userId, + ac: _ac, + ds: _ds, + action: action, + userIds: userIds, }) } @@ -246,23 +250,39 @@ func (e *trendsActionEvent) Action() (err error) { switch e.action { case _trendsActionCreateTweet: logrus.Debug("trigger trendsActionEvent by create tweet ") - e.ds.UpdateUserMetric(e.userId, cs.MetricActionCreateTweet) - goto ExpireTrends + e.updateUserMetric(cs.MetricActionCreateTweet) + e.expireFriendTrends() case _trendsActionDeleteTweet: logrus.Debug("trigger trendsActionEvent by delete tweet ") - e.ds.UpdateUserMetric(e.userId, cs.MetricActionDeleteTweet) - goto ExpireTrends + e.updateUserMetric(cs.MetricActionDeleteTweet) + e.expireFriendTrends() + case _trendsActionAddFriend, _trendsActionDeleteFriend, + _trendsActionFollowUser, _trendsActionUnfollowUser: + e.expireMyTrends() default: // nothing - goto JustReturn } -ExpireTrends: - if friendIds, err := e.ds.MyFriendIds(e.userId); err == nil { - for _, id := range friendIds { - e.ac.DelAny(fmt.Sprintf("%s%d:*", conf.PrefixIdxTrends, id)) + return +} + +func (e *trendsActionEvent) updateUserMetric(action uint8) { + for _, userId := range e.userIds { + e.ds.UpdateUserMetric(userId, action) + } +} + +func (e *trendsActionEvent) expireFriendTrends() { + for _, userId := range e.userIds { + if friendIds, err := e.ds.MyFriendIds(userId); err == nil { + for _, id := range friendIds { + e.ac.DelAny(fmt.Sprintf("%s%d:*", conf.PrefixIdxTrends, id)) + } } - return err } -JustReturn: - return +} + +func (e *trendsActionEvent) expireMyTrends() { + for _, userId := range e.userIds { + e.ac.DelAny(fmt.Sprintf("%s%d:*", conf.PrefixIdxTrends, userId)) + } } diff --git a/internal/servants/web/followship.go b/internal/servants/web/followship.go index 224d5cb9..22b41635 100644 --- a/internal/servants/web/followship.go +++ b/internal/servants/web/followship.go @@ -90,6 +90,7 @@ func (s *followshipSrv) UnfollowUser(r *web.UnfollowUserReq) mir.Error { cache.OnCacheMyFollowIdsEvent(s.Ds, r.User.ID) cache.OnExpireIndexTweetEvent(r.User.ID) onMessageActionEvent(_messageActionFollow, r.User.ID) + onTrendsActionEvent(_trendsActionUnfollowUser, r.User.ID) return nil } @@ -108,6 +109,7 @@ func (s *followshipSrv) FollowUser(r *web.FollowUserReq) mir.Error { cache.OnCacheMyFollowIdsEvent(s.Ds, r.User.ID) cache.OnExpireIndexTweetEvent(r.User.ID) onMessageActionEvent(_messageActionFollow, r.User.ID) + onTrendsActionEvent(_trendsActionFollowUser, r.User.ID) return nil } diff --git a/internal/servants/web/friendship.go b/internal/servants/web/friendship.go index fe533c38..e0436b72 100644 --- a/internal/servants/web/friendship.go +++ b/internal/servants/web/friendship.go @@ -57,6 +57,7 @@ func (s *friendshipSrv) DeleteFriend(req *web.DeleteFriendReq) mir.Error { } // 触发用户关系缓存更新事件 // cache.OnCacheMyFriendIdsEvent(s.Ds, req.User.ID, req.UserId) + onTrendsActionEvent(_trendsActionDeleteFriend, req.User.ID, req.UserId) return nil } @@ -93,6 +94,7 @@ func (s *friendshipSrv) AddFriend(req *web.AddFriendReq) mir.Error { } // 触发用户关系缓存更新事件 // cache.OnCacheMyFriendIdsEvent(s.Ds, req.User.ID, req.UserId) + onTrendsActionEvent(_trendsActionAddFriend, req.User.ID, req.UserId) return nil } diff --git a/scripts/migration/mysql/0009_create_view_post_filter.down.sql b/scripts/migration/mysql/0009_create_view_post_filter.down.sql index b1550d2e..e0343c6d 100644 --- a/scripts/migration/mysql/0009_create_view_post_filter.down.sql +++ b/scripts/migration/mysql/0009_create_view_post_filter.down.sql @@ -1,5 +1,2 @@ DROP VIEW IF EXISTS p_post_by_media; DROP VIEW IF EXISTS p_post_by_comment; - - - diff --git a/scripts/migration/mysql/0014_user_relation_view.down.sql b/scripts/migration/mysql/0014_user_relation_view.down.sql new file mode 100644 index 00000000..2db1b265 --- /dev/null +++ b/scripts/migration/mysql/0014_user_relation_view.down.sql @@ -0,0 +1 @@ +DROP VIEW IF EXISTS p_user_relation; diff --git a/scripts/migration/mysql/0014_user_relation_view.up.sql b/scripts/migration/mysql/0014_user_relation_view.up.sql new file mode 100644 index 00000000..d821659d --- /dev/null +++ b/scripts/migration/mysql/0014_user_relation_view.up.sql @@ -0,0 +1,6 @@ +CREATE VIEW p_user_relation AS +SELECT user_id, friend_id he_uid, 5 AS style +FROM p_contact WHERE status=2 AND is_del=0 +UNION +SELECT user_id, follow_id he_uid, 10 AS style +FROM p_following WHERE is_del=0; diff --git a/scripts/migration/postgres/0013_user_relation_view.down.sql b/scripts/migration/postgres/0013_user_relation_view.down.sql new file mode 100644 index 00000000..2db1b265 --- /dev/null +++ b/scripts/migration/postgres/0013_user_relation_view.down.sql @@ -0,0 +1 @@ +DROP VIEW IF EXISTS p_user_relation; diff --git a/scripts/migration/postgres/0013_user_relation_view.up.sql b/scripts/migration/postgres/0013_user_relation_view.up.sql new file mode 100644 index 00000000..d821659d --- /dev/null +++ b/scripts/migration/postgres/0013_user_relation_view.up.sql @@ -0,0 +1,6 @@ +CREATE VIEW p_user_relation AS +SELECT user_id, friend_id he_uid, 5 AS style +FROM p_contact WHERE status=2 AND is_del=0 +UNION +SELECT user_id, follow_id he_uid, 10 AS style +FROM p_following WHERE is_del=0; diff --git a/scripts/migration/sqlite3/0014_user_relation_view.down.sql b/scripts/migration/sqlite3/0014_user_relation_view.down.sql new file mode 100644 index 00000000..2db1b265 --- /dev/null +++ b/scripts/migration/sqlite3/0014_user_relation_view.down.sql @@ -0,0 +1 @@ +DROP VIEW IF EXISTS p_user_relation; diff --git a/scripts/migration/sqlite3/0014_user_relation_view.up.sql b/scripts/migration/sqlite3/0014_user_relation_view.up.sql new file mode 100644 index 00000000..d821659d --- /dev/null +++ b/scripts/migration/sqlite3/0014_user_relation_view.up.sql @@ -0,0 +1,6 @@ +CREATE VIEW p_user_relation AS +SELECT user_id, friend_id he_uid, 5 AS style +FROM p_contact WHERE status=2 AND is_del=0 +UNION +SELECT user_id, follow_id he_uid, 10 AS style +FROM p_following WHERE is_del=0; diff --git a/scripts/paopao-mysql.sql b/scripts/paopao-mysql.sql index 041eac2d..fddeaaac 100644 --- a/scripts/paopao-mysql.sql +++ b/scripts/paopao-mysql.sql @@ -503,4 +503,12 @@ FROM WHERE P.is_del = 0; +DROP VIEW IF EXISTS p_user_relation; +CREATE VIEW p_user_relation AS +SELECT user_id, friend_id he_uid, 5 AS style +FROM p_contact WHERE status=2 AND is_del=0 +UNION +SELECT user_id, follow_id he_uid, 10 AS style +FROM p_following WHERE is_del=0; + SET FOREIGN_KEY_CHECKS = 1; diff --git a/scripts/paopao-postgres.sql b/scripts/paopao-postgres.sql index 34736ce4..276abdde 100644 --- a/scripts/paopao-postgres.sql +++ b/scripts/paopao-postgres.sql @@ -415,3 +415,11 @@ FROM C JOIN p_post P ON C.post_id = P.ID WHERE P.is_del = 0; + +DROP VIEW IF EXISTS p_user_relation; +CREATE VIEW p_user_relation AS +SELECT user_id, friend_id he_uid, 5 AS style +FROM p_contact WHERE status=2 AND is_del=0 +UNION +SELECT user_id, follow_id he_uid, 10 AS style +FROM p_following WHERE is_del=0; diff --git a/scripts/paopao-sqlite3.sql b/scripts/paopao-sqlite3.sql index 49fd68c0..32fbd9e2 100644 --- a/scripts/paopao-sqlite3.sql +++ b/scripts/paopao-sqlite3.sql @@ -457,6 +457,14 @@ FROM WHERE P.is_del = 0; +DROP VIEW IF EXISTS p_user_relation; +CREATE VIEW p_user_relation AS +SELECT user_id, friend_id he_uid, 5 AS style +FROM p_contact WHERE status=2 AND is_del=0 +UNION +SELECT user_id, follow_id he_uid, 10 AS style +FROM p_following WHERE is_del=0; + -- ---------------------------- -- Indexes structure for table p_attachment -- ----------------------------