You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
Open-IM-Server/pkg/localcache/cache.go

130 lines
3.2 KiB

// Copyright © 2024 OpenIM. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
feat: add local cache for high frequency reads (#2036) * feat: msg local cache * feat: msg local cache * feat: msg local cache * feat: msg local cache * feat: msg local cache * feat: msg local cache * fix: mongo * fix: mongo * fix: mongo * openim.yaml * localcache * localcache * localcache * localcache * localcache * localcache * localcache * localcache * localcache * local cache * local cache * local cache * local cache * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * feat: cache add single-flight and timing-wheel. * feat: local cache * feat: local cache * feat: local cache * feat: cache add single-flight and timing-wheel. * feat: local cache * feat: local cache * feat: local cache * feat: local cache * feat: local cache * feat: local cache * feat: local cache * feat: local cache * feat: local cache * feat: local cache * feat: local cache * feat: local cache * feat: local cache * feat: local cache * feat: local cache * feat: local cache * feat: local cache * feat: msg rpc local cache * feat: msg rpc local cache * feat: msg rpc local cache * feat: msg rpc local cache * feat: msg rpc local cache * feat: msg rpc local cache * refactor: refactor the code of push and optimization. * cicd: robot automated Change * refactor: rename cache. * merge * fix: refactor project dir avoid import cycle. * update tools * merge * feat: conversation FindRecvMsgNotNotifyUserIDs * feat: conversation FindRecvMsgNotNotifyUserIDs * feat: conversation FindRecvMsgNotNotifyUserIDs * merge * merge the latest main --------- Co-authored-by: Gordon <46924906+FGadvancer@users.noreply.github.com> Co-authored-by: withchao <withchao@users.noreply.github.com>
8 months ago
package localcache
import (
"context"
"hash/fnv"
"unsafe"
3.6.1 code conventions (#2202) * refactor: webhooks update. * Adjust configuration settings * Adjust configuration settings * Adjust configuration settings * feat: s3 api addr * refactor: webhooks update. * Adjust configuration settings * Adjust configuration settings * Adjust configuration settings * Adjust configuration settings * Adjust configuration settings * Adjust configuration settings * Adjust configuration settings * refactor: webhooks update. * refactor: kafka update. * Simplify the Docker Compose configuration, remove unnecessary environment variables, and eliminate the gateway service. * refactor: kafka update. * refactor: kafka update. * Simplify the Docker Compose configuration, remove unnecessary environment variables, and eliminate the gateway service. * Simplify the Docker Compose configuration, remove unnecessary environment variables, and eliminate the gateway service. * Windows can compile and run. * Windows can compile and run. * refactor: kafka update. * feat: msg cache split * refactor: webhooks update * refactor: webhooks update * refactor: friends update * refactor: group update * refactor: third update * refactor: api update * refactor: crontab update * refactor: msggateway update * mage * mage * refactor: all module update. * check * refactor: all module update. * load config * load config * load config * load config * refactor: all module update. * refactor: all module update. * refactor: all module update. * refactor: all module update. * refactor: all module update. * Optimize Docker configuration and script. * refactor: all module update. * Optimize Docker configuration and script. * Optimize Docker configuration and script. * Optimize Docker configuration and script. * refactor: all module update. * Optimize Docker configuration and script. * refactor: all module update. * refactor: all module update. * Optimize Docker configuration and script. * Optimize Docker configuration and script. * Optimize Docker configuration and script. * Optimize Docker configuration and script. * Optimize Docker configuration and script. * Optimize Docker configuration and script. * update tools * update tools * Optimize Docker configuration and script. * Optimize Docker configuration and script. * Optimize Docker configuration and script. * Optimize Docker configuration and script. * Optimize Docker configuration and script. * Optimize Docker configuration and script. * Optimize Docker configuration and script. * Optimize Docker configuration and script. * Optimize Docker configuration and script. * Optimize Docker configuration and script. * Optimize Docker configuration and script. * update protocol * Optimize Docker configuration and script. * Optimize Docker configuration and script. * refactor: all module update. * Optimize Docker configuration and script. * Optimize Docker configuration and script. * Optimize Docker configuration and script. * Optimize Docker configuration and script. * refactor: api remove token auth by redis directly. * Code Refactoring * refactor: websocket auth change to call rpc of auth. * refactor: kick online user and remove token change to call auth rpc. * refactor: kick online user and remove token change to call auth rpc. * refactor: remove msggateway redis. * refactor: cmd update. * refactor: cmd update. * refactor: cmd update. * refactor: cmd update. * refactor: cmd update. * refactor: cmd update. * refactor: cmd update. * refactor: cmd update. * refactor: cmd update. * refactor: cmd update. * refactor: cmd update. * refactor: cmd update. * refactor: cmd update. * refactor: cmd update. * refactor: cmd update. * refactor: cmd update. * refactor: cmd update. * refactor: cmd update. * refactor: cmd update. * refactor: cmd update. * refactor: cmd update. * refactor: cmd update. * refactor: cmd update. * refactor: cmd update. * refactor: cmd update. * refactor: cmd update. * refactor webhook * refactor: cmd update. * refactor: cmd update. * refactor: cmd update. * refactor: cmd update. * refactor webhook * refactor: cmd update. * refactor: cmd update. * fix: runtime: goroutine stack exceeds * refactor: cmd update. * refactor notification * refactor notification * refactor * refactor: cmd update. * refactor: cmd update. * refactor * refactor * refactor * protojson * protojson * protojson * go mod * wrapperspb * refactor: cmd update. * refactor: cmd update. * refactor: cmd update. * refactor: context update. * refactor: websocket update info. * refactor: websocket update info. * refactor: websocket update info. * refactor: websocket update info. * refactor: api name change. * refactor: debug info. * refactor: debug info. * refactor: debug info. * fix: update file * refactor * refactor * refactor: debug info. * refactor: debug info. * refactor: debug info. * refactor: debug info. * refactor: debug info. * refactor: debug info. * fix: callback update. * fix: callback update. * refactor * fix: update message. * fix: msg cache timeout. * refactor * refactor * fix: push update. * fix: push update. * fix: push update. * fix: push update. * fix: push update. * fix: push update. * refactor * refactor * fix: push update. * fix: websocket handle error remove when upgrade error. * fix: priority url * fix: minio config * refactor: add zk logger. * refactor * fix: minio config * refactor * refactor * refactor * refactor * refactor: remove zk logger. * refactor: update tools version. * refactor * refactor: update server version to 3.7.0. * refactor * refactor * refactor * refactor * refactor * refactor * refactor * refactor * refactor * refactor * refactor * refactor * refactor * refactor: zk log debug. * refactor: zk log debug. * refactor: zk log debug. * refactor: zk log debug. * refactor: zk log debug. * refactor * refactor * refactor * refactor: log level change. * refactor: 3.7.0 code conventions. --------- Co-authored-by: skiffer-git <44203734@qq.com> Co-authored-by: withchao <993506633@qq.com>
7 months ago
"github.com/openimsdk/open-im-server/v3/pkg/localcache/link"
"github.com/openimsdk/open-im-server/v3/pkg/localcache/lru"
feat: add local cache for high frequency reads (#2036) * feat: msg local cache * feat: msg local cache * feat: msg local cache * feat: msg local cache * feat: msg local cache * feat: msg local cache * fix: mongo * fix: mongo * fix: mongo * openim.yaml * localcache * localcache * localcache * localcache * localcache * localcache * localcache * localcache * localcache * local cache * local cache * local cache * local cache * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * feat: cache add single-flight and timing-wheel. * feat: local cache * feat: local cache * feat: local cache * feat: cache add single-flight and timing-wheel. * feat: local cache * feat: local cache * feat: local cache * feat: local cache * feat: local cache * feat: local cache * feat: local cache * feat: local cache * feat: local cache * feat: local cache * feat: local cache * feat: local cache * feat: local cache * feat: local cache * feat: local cache * feat: local cache * feat: local cache * feat: msg rpc local cache * feat: msg rpc local cache * feat: msg rpc local cache * feat: msg rpc local cache * feat: msg rpc local cache * feat: msg rpc local cache * refactor: refactor the code of push and optimization. * cicd: robot automated Change * refactor: rename cache. * merge * fix: refactor project dir avoid import cycle. * update tools * merge * feat: conversation FindRecvMsgNotNotifyUserIDs * feat: conversation FindRecvMsgNotNotifyUserIDs * feat: conversation FindRecvMsgNotNotifyUserIDs * merge * merge the latest main --------- Co-authored-by: Gordon <46924906+FGadvancer@users.noreply.github.com> Co-authored-by: withchao <withchao@users.noreply.github.com>
8 months ago
)
type Cache[V any] interface {
Get(ctx context.Context, key string, fetch func(ctx context.Context) (V, error)) (V, error)
GetLink(ctx context.Context, key string, fetch func(ctx context.Context) (V, error), link ...string) (V, error)
Del(ctx context.Context, key ...string)
DelLocal(ctx context.Context, key ...string)
Stop()
}
feat: new features merged (#2409) * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * new mongo * new mongo * new mongo * new mongo * new mongo * new mongo * new mongo * new mongo * friend incr sync * friend incr sync * friend incr sync * friend incr sync * friend incr sync * mage * optimization version log * optimization version log * sync * sync * sync * group sync * sync option * sync option * refactor: replace `friend` package with `realtion`. * refactor: update lastest commit to relation. * sync option * sync option * sync option * sync * sync * go.mod * seq * update: go mod * refactor: change incremental to full * feat: get full friend user ids * feat: api and config * seq * group version * merge * seq * seq * seq * fix: sort by id avoid unstable sort friends. * group * group * group * fix: sort by id avoid unstable sort friends. * fix: sort by id avoid unstable sort friends. * fix: sort by id avoid unstable sort friends. * user version * seq * seq * seq user * user online * implement minio expire delete. * user online * config * fix * fix * implement minio expire delete logic. * online cache * online cache * online cache * online cache * online cache * online cache * online cache * online cache * online cache * online cache * online cache * online cache * feat: implement scheduled delete outdated object in minio. * update gomake version * update gomake version * implement FindExpires pagination. * remove unnesseary incr. * fix uncorrect args call. * online push * online push * online push * resolving conflicts * resolving conflicts * test * api prommetrics * api prommetrics * api prommetrics * api prommetrics * api prommetrics * rpc prommetrics * rpc prommetrics * online status * online status * online status * online status * sub * conversation version incremental * merge seq * merge online * merge online * merge online * merge seq * GetOwnerConversation * fix: change incremental syncer router name. * rockscache batch get * rockscache seq batch get * fix: GetMsgDocModelByIndex bug * update go.mod * update go.mod * merge * feat: prometheus * feat: prometheus --------- Co-authored-by: withchao <withchao@users.noreply.github.com> Co-authored-by: Monet Lee <monet_lee@163.com> Co-authored-by: OpenIM-Gordon <46924906+FGadvancer@users.noreply.github.com> Co-authored-by: icey-yu <1186114839@qq.com>
4 months ago
func LRUStringHash(key string) uint64 {
h := fnv.New64a()
h.Write(*(*[]byte)(unsafe.Pointer(&key)))
return h.Sum64()
}
feat: add local cache for high frequency reads (#2036) * feat: msg local cache * feat: msg local cache * feat: msg local cache * feat: msg local cache * feat: msg local cache * feat: msg local cache * fix: mongo * fix: mongo * fix: mongo * openim.yaml * localcache * localcache * localcache * localcache * localcache * localcache * localcache * localcache * localcache * local cache * local cache * local cache * local cache * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * feat: cache add single-flight and timing-wheel. * feat: local cache * feat: local cache * feat: local cache * feat: cache add single-flight and timing-wheel. * feat: local cache * feat: local cache * feat: local cache * feat: local cache * feat: local cache * feat: local cache * feat: local cache * feat: local cache * feat: local cache * feat: local cache * feat: local cache * feat: local cache * feat: local cache * feat: local cache * feat: local cache * feat: local cache * feat: local cache * feat: msg rpc local cache * feat: msg rpc local cache * feat: msg rpc local cache * feat: msg rpc local cache * feat: msg rpc local cache * feat: msg rpc local cache * refactor: refactor the code of push and optimization. * cicd: robot automated Change * refactor: rename cache. * merge * fix: refactor project dir avoid import cycle. * update tools * merge * feat: conversation FindRecvMsgNotNotifyUserIDs * feat: conversation FindRecvMsgNotNotifyUserIDs * feat: conversation FindRecvMsgNotNotifyUserIDs * merge * merge the latest main --------- Co-authored-by: Gordon <46924906+FGadvancer@users.noreply.github.com> Co-authored-by: withchao <withchao@users.noreply.github.com>
8 months ago
func New[V any](opts ...Option) Cache[V] {
opt := defaultOption()
for _, o := range opts {
o(opt)
}
c := cache[V]{opt: opt}
if opt.localSlotNum > 0 && opt.localSlotSize > 0 {
createSimpleLRU := func() lru.LRU[string, V] {
if opt.expirationEvict {
return lru.NewExpirationLRU[string, V](opt.localSlotSize, opt.localSuccessTTL, opt.localFailedTTL, opt.target, c.onEvict)
} else {
return lru.NewLayLRU[string, V](opt.localSlotSize, opt.localSuccessTTL, opt.localFailedTTL, opt.target, c.onEvict)
}
}
if opt.localSlotNum == 1 {
c.local = createSimpleLRU()
} else {
feat: new features merged (#2409) * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * new mongo * new mongo * new mongo * new mongo * new mongo * new mongo * new mongo * new mongo * friend incr sync * friend incr sync * friend incr sync * friend incr sync * friend incr sync * mage * optimization version log * optimization version log * sync * sync * sync * group sync * sync option * sync option * refactor: replace `friend` package with `realtion`. * refactor: update lastest commit to relation. * sync option * sync option * sync option * sync * sync * go.mod * seq * update: go mod * refactor: change incremental to full * feat: get full friend user ids * feat: api and config * seq * group version * merge * seq * seq * seq * fix: sort by id avoid unstable sort friends. * group * group * group * fix: sort by id avoid unstable sort friends. * fix: sort by id avoid unstable sort friends. * fix: sort by id avoid unstable sort friends. * user version * seq * seq * seq user * user online * implement minio expire delete. * user online * config * fix * fix * implement minio expire delete logic. * online cache * online cache * online cache * online cache * online cache * online cache * online cache * online cache * online cache * online cache * online cache * online cache * feat: implement scheduled delete outdated object in minio. * update gomake version * update gomake version * implement FindExpires pagination. * remove unnesseary incr. * fix uncorrect args call. * online push * online push * online push * resolving conflicts * resolving conflicts * test * api prommetrics * api prommetrics * api prommetrics * api prommetrics * api prommetrics * rpc prommetrics * rpc prommetrics * online status * online status * online status * online status * sub * conversation version incremental * merge seq * merge online * merge online * merge online * merge seq * GetOwnerConversation * fix: change incremental syncer router name. * rockscache batch get * rockscache seq batch get * fix: GetMsgDocModelByIndex bug * update go.mod * update go.mod * merge * feat: prometheus * feat: prometheus --------- Co-authored-by: withchao <withchao@users.noreply.github.com> Co-authored-by: Monet Lee <monet_lee@163.com> Co-authored-by: OpenIM-Gordon <46924906+FGadvancer@users.noreply.github.com> Co-authored-by: icey-yu <1186114839@qq.com>
4 months ago
c.local = lru.NewSlotLRU[string, V](opt.localSlotNum, LRUStringHash, createSimpleLRU)
feat: add local cache for high frequency reads (#2036) * feat: msg local cache * feat: msg local cache * feat: msg local cache * feat: msg local cache * feat: msg local cache * feat: msg local cache * fix: mongo * fix: mongo * fix: mongo * openim.yaml * localcache * localcache * localcache * localcache * localcache * localcache * localcache * localcache * localcache * local cache * local cache * local cache * local cache * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * feat: cache add single-flight and timing-wheel. * feat: local cache * feat: local cache * feat: local cache * feat: cache add single-flight and timing-wheel. * feat: local cache * feat: local cache * feat: local cache * feat: local cache * feat: local cache * feat: local cache * feat: local cache * feat: local cache * feat: local cache * feat: local cache * feat: local cache * feat: local cache * feat: local cache * feat: local cache * feat: local cache * feat: local cache * feat: local cache * feat: msg rpc local cache * feat: msg rpc local cache * feat: msg rpc local cache * feat: msg rpc local cache * feat: msg rpc local cache * feat: msg rpc local cache * refactor: refactor the code of push and optimization. * cicd: robot automated Change * refactor: rename cache. * merge * fix: refactor project dir avoid import cycle. * update tools * merge * feat: conversation FindRecvMsgNotNotifyUserIDs * feat: conversation FindRecvMsgNotNotifyUserIDs * feat: conversation FindRecvMsgNotNotifyUserIDs * merge * merge the latest main --------- Co-authored-by: Gordon <46924906+FGadvancer@users.noreply.github.com> Co-authored-by: withchao <withchao@users.noreply.github.com>
8 months ago
}
if opt.linkSlotNum > 0 {
c.link = link.New(opt.linkSlotNum)
}
}
return &c
}
type cache[V any] struct {
opt *option
link link.Link
local lru.LRU[string, V]
}
func (c *cache[V]) onEvict(key string, value V) {
if c.link != nil {
lks := c.link.Del(key)
for k := range lks {
if key != k { // prevent deadlock
c.local.Del(k)
}
}
}
}
func (c *cache[V]) del(key ...string) {
if c.local == nil {
return
}
for _, k := range key {
c.local.Del(k)
if c.link != nil {
lks := c.link.Del(k)
for k := range lks {
c.local.Del(k)
}
}
}
}
func (c *cache[V]) Get(ctx context.Context, key string, fetch func(ctx context.Context) (V, error)) (V, error) {
return c.GetLink(ctx, key, fetch)
}
func (c *cache[V]) GetLink(ctx context.Context, key string, fetch func(ctx context.Context) (V, error), link ...string) (V, error) {
if c.local != nil {
return c.local.Get(key, func() (V, error) {
if len(link) > 0 {
c.link.Link(key, link...)
}
return fetch(ctx)
})
} else {
return fetch(ctx)
}
}
func (c *cache[V]) Del(ctx context.Context, key ...string) {
for _, fn := range c.opt.delFn {
fn(ctx, key...)
}
c.del(key...)
}
func (c *cache[V]) DelLocal(ctx context.Context, key ...string) {
c.del(key...)
}
func (c *cache[V]) Stop() {
c.local.Stop()
}