You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
Open-IM-Server/pkg/localcache/lru/lru_expiration.go

115 lines
2.7 KiB

// Copyright © 2024 OpenIM. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
feat: add local cache for high frequency reads (#2036) * feat: msg local cache * feat: msg local cache * feat: msg local cache * feat: msg local cache * feat: msg local cache * feat: msg local cache * fix: mongo * fix: mongo * fix: mongo * openim.yaml * localcache * localcache * localcache * localcache * localcache * localcache * localcache * localcache * localcache * local cache * local cache * local cache * local cache * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * feat: cache add single-flight and timing-wheel. * feat: local cache * feat: local cache * feat: local cache * feat: cache add single-flight and timing-wheel. * feat: local cache * feat: local cache * feat: local cache * feat: local cache * feat: local cache * feat: local cache * feat: local cache * feat: local cache * feat: local cache * feat: local cache * feat: local cache * feat: local cache * feat: local cache * feat: local cache * feat: local cache * feat: local cache * feat: local cache * feat: msg rpc local cache * feat: msg rpc local cache * feat: msg rpc local cache * feat: msg rpc local cache * feat: msg rpc local cache * feat: msg rpc local cache * refactor: refactor the code of push and optimization. * cicd: robot automated Change * refactor: rename cache. * merge * fix: refactor project dir avoid import cycle. * update tools * merge * feat: conversation FindRecvMsgNotNotifyUserIDs * feat: conversation FindRecvMsgNotNotifyUserIDs * feat: conversation FindRecvMsgNotNotifyUserIDs * merge * merge the latest main --------- Co-authored-by: Gordon <46924906+FGadvancer@users.noreply.github.com> Co-authored-by: withchao <withchao@users.noreply.github.com>
10 months ago
package lru
import (
"sync"
"time"
"github.com/hashicorp/golang-lru/v2/expirable"
feat: add local cache for high frequency reads (#2036) * feat: msg local cache * feat: msg local cache * feat: msg local cache * feat: msg local cache * feat: msg local cache * feat: msg local cache * fix: mongo * fix: mongo * fix: mongo * openim.yaml * localcache * localcache * localcache * localcache * localcache * localcache * localcache * localcache * localcache * local cache * local cache * local cache * local cache * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * feat: cache add single-flight and timing-wheel. * feat: local cache * feat: local cache * feat: local cache * feat: cache add single-flight and timing-wheel. * feat: local cache * feat: local cache * feat: local cache * feat: local cache * feat: local cache * feat: local cache * feat: local cache * feat: local cache * feat: local cache * feat: local cache * feat: local cache * feat: local cache * feat: local cache * feat: local cache * feat: local cache * feat: local cache * feat: local cache * feat: msg rpc local cache * feat: msg rpc local cache * feat: msg rpc local cache * feat: msg rpc local cache * feat: msg rpc local cache * feat: msg rpc local cache * refactor: refactor the code of push and optimization. * cicd: robot automated Change * refactor: rename cache. * merge * fix: refactor project dir avoid import cycle. * update tools * merge * feat: conversation FindRecvMsgNotNotifyUserIDs * feat: conversation FindRecvMsgNotNotifyUserIDs * feat: conversation FindRecvMsgNotNotifyUserIDs * merge * merge the latest main --------- Co-authored-by: Gordon <46924906+FGadvancer@users.noreply.github.com> Co-authored-by: withchao <withchao@users.noreply.github.com>
10 months ago
)
func NewExpirationLRU[K comparable, V any](size int, successTTL, failedTTL time.Duration, target Target, onEvict EvictCallback[K, V]) LRU[K, V] {
var cb expirable.EvictCallback[K, *expirationLruItem[V]]
if onEvict != nil {
cb = func(key K, value *expirationLruItem[V]) {
onEvict(key, value.value)
}
}
core := expirable.NewLRU[K, *expirationLruItem[V]](size, cb, successTTL)
return &ExpirationLRU[K, V]{
core: core,
successTTL: successTTL,
failedTTL: failedTTL,
target: target,
}
}
type expirationLruItem[V any] struct {
lock sync.RWMutex
err error
value V
}
type ExpirationLRU[K comparable, V any] struct {
lock sync.Mutex
core *expirable.LRU[K, *expirationLruItem[V]]
successTTL time.Duration
failedTTL time.Duration
target Target
}
Fix err (#2608) * refactor: refactor workflows contents. * add tool workflows. * update field. * fix: remove chat error. * Fix err. * fix error. * remove cn comment. * update workflows files. * update infra config. * move workflows. * feat: update bot. * fix: solve uncorrect outdated msg get. * update get docIDs logic. * update * update skip logic. * fix * update. * fix: delay deleteObject func. * remove unused content. * update log type. * feat: implement request batch count limit. * update * update * feat: add rocksTimeout * feat: wrap logs * feat: add logs * feat: listen config * feat: enable listen TIME_WAIT port * feat: add logs * feat: cache batch * chore: enable fullUserCache * feat: push rpc num * feat: push err * feat: with operationID * feat: sleep * feat: change 1s * feat: change log * feat: implement Getbatch in rpcCache. * feat: print getOnline cost * feat: change log * feat: change kafka and push config * feat: del interface * feat: fix err * feat: change config * feat: go mod * feat: change config * feat: change config * feat: add sleep in push * feat: warn logs * feat: logs * feat: logs * feat: change port * feat: start config * feat: remove port reuse * feat: prometheus config * feat: prometheus config * feat: prometheus config * feat: add long time send msg to grafana * feat: init * feat: init * feat: implement offline push. * feat: batch get user online * feat: implement batch Push spilt * update go mod * Revert "feat: change port" This reverts commit 06d5e944 * feat: change port * feat: change config * feat: implement kafka producer and consumer. * update format, * add PushMQ log. * feat: get all online users and init push * feat: lock in online cache * feat: config * fix: init online status * fix: add logs * fix: userIDs * fix: add logs * feat: update Handler logic. * update MQ logic. * update * update * fix: method name * fix: update OfflinePushConsumerHandler. * fix: prommetrics * fix: add logs * fix: ctx * fix: log * fix: config * feat: change port * fix: atomic online cache status --------- Co-authored-by: Monet Lee <monet_lee@163.com>
4 months ago
func (x *ExpirationLRU[K, V]) GetBatch(keys []K, fetch func(keys []K) (map[K]V, error)) (map[K]V, error) {
//TODO implement me
panic("implement me")
}
feat: add local cache for high frequency reads (#2036) * feat: msg local cache * feat: msg local cache * feat: msg local cache * feat: msg local cache * feat: msg local cache * feat: msg local cache * fix: mongo * fix: mongo * fix: mongo * openim.yaml * localcache * localcache * localcache * localcache * localcache * localcache * localcache * localcache * localcache * local cache * local cache * local cache * local cache * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * feat: cache add single-flight and timing-wheel. * feat: local cache * feat: local cache * feat: local cache * feat: cache add single-flight and timing-wheel. * feat: local cache * feat: local cache * feat: local cache * feat: local cache * feat: local cache * feat: local cache * feat: local cache * feat: local cache * feat: local cache * feat: local cache * feat: local cache * feat: local cache * feat: local cache * feat: local cache * feat: local cache * feat: local cache * feat: local cache * feat: msg rpc local cache * feat: msg rpc local cache * feat: msg rpc local cache * feat: msg rpc local cache * feat: msg rpc local cache * feat: msg rpc local cache * refactor: refactor the code of push and optimization. * cicd: robot automated Change * refactor: rename cache. * merge * fix: refactor project dir avoid import cycle. * update tools * merge * feat: conversation FindRecvMsgNotNotifyUserIDs * feat: conversation FindRecvMsgNotNotifyUserIDs * feat: conversation FindRecvMsgNotNotifyUserIDs * merge * merge the latest main --------- Co-authored-by: Gordon <46924906+FGadvancer@users.noreply.github.com> Co-authored-by: withchao <withchao@users.noreply.github.com>
10 months ago
func (x *ExpirationLRU[K, V]) Get(key K, fetch func() (V, error)) (V, error) {
x.lock.Lock()
v, ok := x.core.Get(key)
if ok {
x.lock.Unlock()
x.target.IncrGetSuccess()
v.lock.RLock()
defer v.lock.RUnlock()
return v.value, v.err
} else {
v = &expirationLruItem[V]{}
x.core.Add(key, v)
v.lock.Lock()
x.lock.Unlock()
defer v.lock.Unlock()
v.value, v.err = fetch()
if v.err == nil {
x.target.IncrGetSuccess()
} else {
x.target.IncrGetFailed()
x.core.Remove(key)
}
return v.value, v.err
}
}
func (x *ExpirationLRU[K, V]) Del(key K) bool {
x.lock.Lock()
ok := x.core.Remove(key)
x.lock.Unlock()
if ok {
x.target.IncrDelHit()
} else {
x.target.IncrDelNotFound()
}
return ok
}
feat: new features merged (#2409) * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * new mongo * new mongo * new mongo * new mongo * new mongo * new mongo * new mongo * new mongo * friend incr sync * friend incr sync * friend incr sync * friend incr sync * friend incr sync * mage * optimization version log * optimization version log * sync * sync * sync * group sync * sync option * sync option * refactor: replace `friend` package with `realtion`. * refactor: update lastest commit to relation. * sync option * sync option * sync option * sync * sync * go.mod * seq * update: go mod * refactor: change incremental to full * feat: get full friend user ids * feat: api and config * seq * group version * merge * seq * seq * seq * fix: sort by id avoid unstable sort friends. * group * group * group * fix: sort by id avoid unstable sort friends. * fix: sort by id avoid unstable sort friends. * fix: sort by id avoid unstable sort friends. * user version * seq * seq * seq user * user online * implement minio expire delete. * user online * config * fix * fix * implement minio expire delete logic. * online cache * online cache * online cache * online cache * online cache * online cache * online cache * online cache * online cache * online cache * online cache * online cache * feat: implement scheduled delete outdated object in minio. * update gomake version * update gomake version * implement FindExpires pagination. * remove unnesseary incr. * fix uncorrect args call. * online push * online push * online push * resolving conflicts * resolving conflicts * test * api prommetrics * api prommetrics * api prommetrics * api prommetrics * api prommetrics * rpc prommetrics * rpc prommetrics * online status * online status * online status * online status * sub * conversation version incremental * merge seq * merge online * merge online * merge online * merge seq * GetOwnerConversation * fix: change incremental syncer router name. * rockscache batch get * rockscache seq batch get * fix: GetMsgDocModelByIndex bug * update go.mod * update go.mod * merge * feat: prometheus * feat: prometheus --------- Co-authored-by: withchao <withchao@users.noreply.github.com> Co-authored-by: Monet Lee <monet_lee@163.com> Co-authored-by: OpenIM-Gordon <46924906+FGadvancer@users.noreply.github.com> Co-authored-by: icey-yu <1186114839@qq.com>
5 months ago
func (x *ExpirationLRU[K, V]) SetHas(key K, value V) bool {
x.lock.Lock()
defer x.lock.Unlock()
if x.core.Contains(key) {
x.core.Add(key, &expirationLruItem[V]{value: value})
return true
}
return false
}
Fix err (#2608) * refactor: refactor workflows contents. * add tool workflows. * update field. * fix: remove chat error. * Fix err. * fix error. * remove cn comment. * update workflows files. * update infra config. * move workflows. * feat: update bot. * fix: solve uncorrect outdated msg get. * update get docIDs logic. * update * update skip logic. * fix * update. * fix: delay deleteObject func. * remove unused content. * update log type. * feat: implement request batch count limit. * update * update * feat: add rocksTimeout * feat: wrap logs * feat: add logs * feat: listen config * feat: enable listen TIME_WAIT port * feat: add logs * feat: cache batch * chore: enable fullUserCache * feat: push rpc num * feat: push err * feat: with operationID * feat: sleep * feat: change 1s * feat: change log * feat: implement Getbatch in rpcCache. * feat: print getOnline cost * feat: change log * feat: change kafka and push config * feat: del interface * feat: fix err * feat: change config * feat: go mod * feat: change config * feat: change config * feat: add sleep in push * feat: warn logs * feat: logs * feat: logs * feat: change port * feat: start config * feat: remove port reuse * feat: prometheus config * feat: prometheus config * feat: prometheus config * feat: add long time send msg to grafana * feat: init * feat: init * feat: implement offline push. * feat: batch get user online * feat: implement batch Push spilt * update go mod * Revert "feat: change port" This reverts commit 06d5e944 * feat: change port * feat: change config * feat: implement kafka producer and consumer. * update format, * add PushMQ log. * feat: get all online users and init push * feat: lock in online cache * feat: config * fix: init online status * fix: add logs * fix: userIDs * fix: add logs * feat: update Handler logic. * update MQ logic. * update * update * fix: method name * fix: update OfflinePushConsumerHandler. * fix: prommetrics * fix: add logs * fix: ctx * fix: log * fix: config * feat: change port * fix: atomic online cache status --------- Co-authored-by: Monet Lee <monet_lee@163.com>
4 months ago
func (x *ExpirationLRU[K, V]) Set(key K, value V) {
x.lock.Lock()
defer x.lock.Unlock()
x.core.Add(key, &expirationLruItem[V]{value: value})
}
feat: add local cache for high frequency reads (#2036) * feat: msg local cache * feat: msg local cache * feat: msg local cache * feat: msg local cache * feat: msg local cache * feat: msg local cache * fix: mongo * fix: mongo * fix: mongo * openim.yaml * localcache * localcache * localcache * localcache * localcache * localcache * localcache * localcache * localcache * local cache * local cache * local cache * local cache * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * feat: cache add single-flight and timing-wheel. * feat: local cache * feat: local cache * feat: local cache * feat: cache add single-flight and timing-wheel. * feat: local cache * feat: local cache * feat: local cache * feat: local cache * feat: local cache * feat: local cache * feat: local cache * feat: local cache * feat: local cache * feat: local cache * feat: local cache * feat: local cache * feat: local cache * feat: local cache * feat: local cache * feat: local cache * feat: local cache * feat: msg rpc local cache * feat: msg rpc local cache * feat: msg rpc local cache * feat: msg rpc local cache * feat: msg rpc local cache * feat: msg rpc local cache * refactor: refactor the code of push and optimization. * cicd: robot automated Change * refactor: rename cache. * merge * fix: refactor project dir avoid import cycle. * update tools * merge * feat: conversation FindRecvMsgNotNotifyUserIDs * feat: conversation FindRecvMsgNotNotifyUserIDs * feat: conversation FindRecvMsgNotNotifyUserIDs * merge * merge the latest main --------- Co-authored-by: Gordon <46924906+FGadvancer@users.noreply.github.com> Co-authored-by: withchao <withchao@users.noreply.github.com>
10 months ago
func (x *ExpirationLRU[K, V]) Stop() {
}