ironbee
/
frostfs-node
forked from TrueCloudLab/frostfs-node
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[#5] frost-node: Used generic cache 

Signed-off-by: Evgenii Stratonikov <e.stratonikov@yadro.com>
carpawell/fix/multiple-cache-update-requests-FROST
Evgenii Stratonikov 2022-12-30 13:23:41 +03:00 committed by fyrchik
parent 9936b112b8
commit fdb0affc31
3 changed files with 72 additions and 100 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

163
cmd/frostfs-node/cache.go
View File

@ -12,35 +12,35 @@ import (
cid "github.com/TrueCloudLab/frostfs-sdk-go/container/id" cid "github.com/TrueCloudLab/frostfs-sdk-go/container/id"
netmapSDK "github.com/TrueCloudLab/frostfs-sdk-go/netmap" netmapSDK "github.com/TrueCloudLab/frostfs-sdk-go/netmap"
"github.com/TrueCloudLab/frostfs-sdk-go/user" "github.com/TrueCloudLab/frostfs-sdk-go/user"
lru "github.com/hashicorp/golang-lru" lru "github.com/hashicorp/golang-lru/v2"
) )
type netValueReader func(interface{}) (interface{}, error) type netValueReader[K any, V any] func(K) (V, error)
type valueWithTime struct { type valueWithTime[V any] struct {
v interface{} v V
t time.Time t time.Time
// cached error in order to not repeat failed request for some time // cached error in order to not repeat failed request for some time
e error e error
} }
// entity that provides TTL cache interface. // entity that provides TTL cache interface.
type ttlNetCache struct { type ttlNetCache[K comparable, V any] struct {
ttl time.Duration ttl time.Duration
sz int sz int
cache *lru.Cache cache *lru.Cache[K, *valueWithTime[V]]
netRdr netValueReader netRdr netValueReader[K, V]
} }
// complicates netValueReader with TTL caching mechanism. // complicates netValueReader with TTL caching mechanism.
func newNetworkTTLCache(sz int, ttl time.Duration, netRdr netValueReader) *ttlNetCache { func newNetworkTTLCache[K comparable, V any](sz int, ttl time.Duration, netRdr netValueReader[K, V]) *ttlNetCache[K, V] {
cache, err := lru.New(sz) cache, err := lru.New[K, *valueWithTime[V]](sz)
fatalOnErr(err) fatalOnErr(err)
return &ttlNetCache{ return &ttlNetCache[K, V]{
ttl: ttl, ttl: ttl,
sz: sz, sz: sz,
cache: cache, cache: cache,
@ -53,47 +53,45 @@ func newNetworkTTLCache(sz int, ttl time.Duration, netRdr netValueReader) *ttlNe
// updates the value from the network on cache miss or by TTL. // updates the value from the network on cache miss or by TTL.
// //
// returned value should not be modified. // returned value should not be modified.
func (c *ttlNetCache) get(key interface{}) (interface{}, error) { func (c *ttlNetCache[K, V]) get(key K) (V, error) {
val, ok := c.cache.Peek(key) val, ok := c.cache.Peek(key)
if ok { if ok {
valWithTime := val.(*valueWithTime) if time.Since(val.t) < c.ttl {
return val.v, val.e
if time.Since(valWithTime.t) < c.ttl {
return valWithTime.v, valWithTime.e
} }
c.cache.Remove(key) c.cache.Remove(key)
} }
val, err := c.netRdr(key) v, err := c.netRdr(key)
c.set(key, val, err) c.set(key, v, err)
return val, err return v, err
} }
func (c *ttlNetCache) set(k, v interface{}, e error) { func (c *ttlNetCache[K, V]) set(k K, v V, e error) {
c.cache.Add(k, &valueWithTime{ c.cache.Add(k, &valueWithTime[V]{
v: v, v: v,
t: time.Now(), t: time.Now(),
e: e, e: e,
}) })
} }
func (c *ttlNetCache) remove(key interface{}) { func (c *ttlNetCache[K, V]) remove(key K) {
c.cache.Remove(key) c.cache.Remove(key)
} }
// entity that provides LRU cache interface. // entity that provides LRU cache interface.
type lruNetCache struct { type lruNetCache struct {
cache *lru.Cache cache *lru.Cache[uint64, *netmapSDK.NetMap]
netRdr netValueReader netRdr netValueReader[uint64, *netmapSDK.NetMap]
} }
// newNetworkLRUCache returns wrapper over netValueReader with LRU cache. // newNetworkLRUCache returns wrapper over netValueReader with LRU cache.
func newNetworkLRUCache(sz int, netRdr netValueReader) *lruNetCache { func newNetworkLRUCache(sz int, netRdr netValueReader[uint64, *netmapSDK.NetMap]) *lruNetCache {
cache, err := lru.New(sz) cache, err := lru.New[uint64, *netmapSDK.NetMap](sz)
fatalOnErr(err) fatalOnErr(err)
return &lruNetCache{ return &lruNetCache{
@ -107,7 +105,7 @@ func newNetworkLRUCache(sz int, netRdr netValueReader) *lruNetCache {
// updates the value from the network on cache miss. // updates the value from the network on cache miss.
// //
// returned value should not be modified. // returned value should not be modified.
func (c *lruNetCache) get(key interface{}) (interface{}, error) { func (c *lruNetCache) get(key uint64) (*netmapSDK.NetMap, error) {
val, ok := c.cache.Get(key) val, ok := c.cache.Get(key)
if ok { if ok {
return val, nil return val, nil
@ -125,73 +123,53 @@ func (c *lruNetCache) get(key interface{}) (interface{}, error) {
// wrapper over TTL cache of values read from the network // wrapper over TTL cache of values read from the network
// that implements container storage. // that implements container storage.
type ttlContainerStorage ttlNetCache type ttlContainerStorage struct {
*ttlNetCache[cid.ID, *container.Container]
}
func newCachedContainerStorage(v container.Source, ttl time.Duration) *ttlContainerStorage { func newCachedContainerStorage(v container.Source, ttl time.Duration) ttlContainerStorage {
const containerCacheSize = 100 const containerCacheSize = 100
lruCnrCache := newNetworkTTLCache(containerCacheSize, ttl, func(key interface{}) (interface{}, error) { lruCnrCache := newNetworkTTLCache[cid.ID, *container.Container](containerCacheSize, ttl, func(id cid.ID) (*container.Container, error) {
var id cid.ID
err := id.DecodeString(key.(string))
if err != nil {
return nil, err
}
return v.Get(id) return v.Get(id)
}) })
return (*ttlContainerStorage)(lruCnrCache) return ttlContainerStorage{lruCnrCache}
} }
func (s *ttlContainerStorage) handleRemoval(cnr cid.ID) { func (s ttlContainerStorage) handleRemoval(cnr cid.ID) {
(*ttlNetCache)(s).set(cnr.EncodeToString(), nil, apistatus.ContainerNotFound{}) s.set(cnr, nil, apistatus.ContainerNotFound{})
} }
// Get returns container value from the cache. If value is missing in the cache // Get returns container value from the cache. If value is missing in the cache
// or expired, then it returns value from side chain and updates the cache. // or expired, then it returns value from side chain and updates the cache.
func (s *ttlContainerStorage) Get(cnr cid.ID) (*container.Container, error) { func (s ttlContainerStorage) Get(cnr cid.ID) (*container.Container, error) {
val, err := (*ttlNetCache)(s).get(cnr.EncodeToString()) return s.get(cnr)
if err != nil {
return nil, err
}
return val.(*container.Container), nil
} }
type ttlEACLStorage ttlNetCache type ttlEACLStorage struct {
*ttlNetCache[cid.ID, *container.EACL]
}
func newCachedEACLStorage(v container.EACLSource, ttl time.Duration) *ttlEACLStorage { func newCachedEACLStorage(v container.EACLSource, ttl time.Duration) ttlEACLStorage {
const eaclCacheSize = 100 const eaclCacheSize = 100
lruCnrCache := newNetworkTTLCache(eaclCacheSize, ttl, func(key interface{}) (interface{}, error) { lruCnrCache := newNetworkTTLCache(eaclCacheSize, ttl, func(id cid.ID) (*container.EACL, error) {
var id cid.ID
err := id.DecodeString(key.(string))
if err != nil {
return nil, err
}
return v.GetEACL(id) return v.GetEACL(id)
}) })
return (*ttlEACLStorage)(lruCnrCache) return ttlEACLStorage{lruCnrCache}
} }
// GetEACL returns eACL value from the cache. If value is missing in the cache // GetEACL returns eACL value from the cache. If value is missing in the cache
// or expired, then it returns value from side chain and updates cache. // or expired, then it returns value from side chain and updates cache.
func (s *ttlEACLStorage) GetEACL(cnr cid.ID) (*container.EACL, error) { func (s ttlEACLStorage) GetEACL(cnr cid.ID) (*container.EACL, error) {
val, err := (*ttlNetCache)(s).get(cnr.EncodeToString()) return s.get(cnr)
if err != nil {
return nil, err
}
return val.(*container.EACL), nil
} }
// InvalidateEACL removes cached eACL value. // InvalidateEACL removes cached eACL value.
func (s *ttlEACLStorage) InvalidateEACL(cnr cid.ID) { func (s ttlEACLStorage) InvalidateEACL(cnr cid.ID) {
(*ttlNetCache)(s).remove(cnr.EncodeToString()) s.remove(cnr)
} }
type lruNetmapSource struct { type lruNetmapSource struct {
@ -203,8 +181,8 @@ type lruNetmapSource struct {
func newCachedNetmapStorage(s netmap.State, v netmap.Source) netmap.Source { func newCachedNetmapStorage(s netmap.State, v netmap.Source) netmap.Source {
const netmapCacheSize = 10 const netmapCacheSize = 10
lruNetmapCache := newNetworkLRUCache(netmapCacheSize, func(key interface{}) (interface{}, error) { lruNetmapCache := newNetworkLRUCache(netmapCacheSize, func(key uint64) (*netmapSDK.NetMap, error) {
return v.GetNetMapByEpoch(key.(uint64)) return v.GetNetMapByEpoch(key)
}) })
return &lruNetmapSource{ return &lruNetmapSource{
@ -227,7 +205,7 @@ func (s *lruNetmapSource) getNetMapByEpoch(epoch uint64) (*netmapSDK.NetMap, err
return nil, err return nil, err
} }
return val.(*netmapSDK.NetMap), nil return val, nil
} }
func (s *lruNetmapSource) Epoch() (uint64, error) { func (s *lruNetmapSource) Epoch() (uint64, error) {
@ -236,7 +214,9 @@ func (s *lruNetmapSource) Epoch() (uint64, error) {
// wrapper over TTL cache of values read from the network // wrapper over TTL cache of values read from the network
// that implements container lister. // that implements container lister.
type ttlContainerLister ttlNetCache type ttlContainerLister struct {
*ttlNetCache[string, *cacheItemContainerList]
}
// value type for ttlNetCache used by ttlContainerLister. // value type for ttlNetCache used by ttlContainerLister.
type cacheItemContainerList struct { type cacheItemContainerList struct {
@ -246,14 +226,11 @@ type cacheItemContainerList struct {
list []cid.ID list []cid.ID
} }
func newCachedContainerLister(c *cntClient.Client, ttl time.Duration) *ttlContainerLister { func newCachedContainerLister(c *cntClient.Client, ttl time.Duration) ttlContainerLister {
const containerListerCacheSize = 100 const containerListerCacheSize = 100
lruCnrListerCache := newNetworkTTLCache(containerListerCacheSize, ttl, func(key interface{}) (interface{}, error) { lruCnrListerCache := newNetworkTTLCache(containerListerCacheSize, ttl, func(strID string) (*cacheItemContainerList, error) {
var ( var id *user.ID
id *user.ID
strID = key.(string)
)
if strID != "" { if strID != "" {
id = new(user.ID) id = new(user.ID)
@ -274,28 +251,24 @@ func newCachedContainerLister(c *cntClient.Client, ttl time.Duration) *ttlContai
}, nil }, nil
}) })
return (*ttlContainerLister)(lruCnrListerCache) return ttlContainerLister{lruCnrListerCache}
} }
// List returns list of container IDs from the cache. If list is missing in the // List returns list of container IDs from the cache. If list is missing in the
// cache or expired, then it returns container IDs from side chain and updates // cache or expired, then it returns container IDs from side chain and updates
// the cache. // the cache.
func (s *ttlContainerLister) List(id *user.ID) ([]cid.ID, error) { func (s ttlContainerLister) List(id *user.ID) ([]cid.ID, error) {
var str string var str string
if id != nil { if id != nil {
str = id.EncodeToString() str = id.EncodeToString()
} }
val, err := (*ttlNetCache)(s).get(str) item, err := s.get(str)
if err != nil { if err != nil {
return nil, err return nil, err
} }
// panic on typecast below is OK since developer must be careful,
// runtime can do nothing with wrong type occurrence
item := val.(*cacheItemContainerList)
item.mtx.RLock() item.mtx.RLock()
res := make([]cid.ID, len(item.list)) res := make([]cid.ID, len(item.list))
copy(res, item.list) copy(res, item.list)
@ -314,16 +287,14 @@ func (s *ttlContainerLister) List(id *user.ID) ([]cid.ID, error) {
func (s *ttlContainerLister) update(owner user.ID, cnr cid.ID, add bool) { func (s *ttlContainerLister) update(owner user.ID, cnr cid.ID, add bool) {
strOwner := owner.EncodeToString() strOwner := owner.EncodeToString()
val, ok := (*ttlNetCache)(s).cache.Get(strOwner) val, ok := s.cache.Get(strOwner)
if !ok { if !ok {
// we could cache the single cnr but in this case we will disperse // we could cache the single cnr but in this case we will disperse
// with the Sidechain a lot // with the Sidechain a lot
return return
} }
// panic on typecast below is OK since developer must be careful, item := val.v
// runtime can do nothing with wrong type occurrence
item := val.(*valueWithTime).v.(*cacheItemContainerList)
item.mtx.Lock() item.mtx.Lock()
{ {
@ -349,9 +320,11 @@ func (s *ttlContainerLister) update(owner user.ID, cnr cid.ID, add bool) {
item.mtx.Unlock() item.mtx.Unlock()
} }
type cachedIRFetcher ttlNetCache type cachedIRFetcher struct {
*ttlNetCache[struct{}, [][]byte]
}
func newCachedIRFetcher(f interface{ InnerRingKeys() ([][]byte, error) }) *cachedIRFetcher { func newCachedIRFetcher(f interface{ InnerRingKeys() ([][]byte, error) }) cachedIRFetcher {
const ( const (
irFetcherCacheSize = 1 // we intend to store only one value irFetcherCacheSize = 1 // we intend to store only one value
@ -364,25 +337,25 @@ func newCachedIRFetcher(f interface{ InnerRingKeys() ([][]byte, error) }) *cache
irFetcherCacheTTL = 30 * time.Second irFetcherCacheTTL = 30 * time.Second
) )
irFetcherCache := newNetworkTTLCache(irFetcherCacheSize, irFetcherCacheTTL, irFetcherCache := newNetworkTTLCache[struct{}, [][]byte](irFetcherCacheSize, irFetcherCacheTTL,
func(key interface{}) (interface{}, error) { func(_ struct{}) ([][]byte, error) {
return f.InnerRingKeys() return f.InnerRingKeys()
}, },
) )
return (*cachedIRFetcher)(irFetcherCache) return cachedIRFetcher{irFetcherCache}
} }
// InnerRingKeys returns cached list of Inner Ring keys. If keys are missing in // InnerRingKeys returns cached list of Inner Ring keys. If keys are missing in
// the cache or expired, then it returns keys from side chain and updates // the cache or expired, then it returns keys from side chain and updates
// the cache. // the cache.
func (f *cachedIRFetcher) InnerRingKeys() ([][]byte, error) { func (f cachedIRFetcher) InnerRingKeys() ([][]byte, error) {
val, err := (*ttlNetCache)(f).get("") val, err := f.get(struct{}{})
if err != nil { if err != nil {
return nil, err return nil, err
} }
return val.([][]byte), nil return val, nil
} }
type ttlMaxObjectSizeCache struct { type ttlMaxObjectSizeCache struct {

4
cmd/frostfs-node/container.go
View File

@ -657,8 +657,8 @@ type morphContainerWriter struct {
neoClient *cntClient.Client neoClient *cntClient.Client
cacheEnabled bool cacheEnabled bool
eacls *ttlEACLStorage eacls ttlEACLStorage
lists *ttlContainerLister lists ttlContainerLister
} }
func (m morphContainerWriter) Put(cnr containerCore.Container) (*cid.ID, error) { func (m morphContainerWriter) Put(cnr containerCore.Container) (*cid.ID, error) {

5
go.mod
View File

@ -13,7 +13,7 @@ require (
github.com/flynn-archive/go-shlex v0.0.0-20150515145356-3f9db97f8568 github.com/flynn-archive/go-shlex v0.0.0-20150515145356-3f9db97f8568
github.com/google/go-github/v39 v39.2.0 github.com/google/go-github/v39 v39.2.0
github.com/google/uuid v1.3.0 github.com/google/uuid v1.3.0
github.com/hashicorp/golang-lru v0.5.4 github.com/hashicorp/golang-lru/v2 v2.0.1
github.com/klauspost/compress v1.15.13 github.com/klauspost/compress v1.15.13
github.com/mitchellh/go-homedir v1.1.0 github.com/mitchellh/go-homedir v1.1.0
github.com/mr-tron/base58 v1.2.0 github.com/mr-tron/base58 v1.2.0
@ -39,8 +39,6 @@ require (
gopkg.in/yaml.v3 v3.0.1 gopkg.in/yaml.v3 v3.0.1
) )
require github.com/hashicorp/golang-lru/v2 v2.0.1
require ( require (
github.com/TrueCloudLab/frostfs-crypto v0.5.0 // indirect github.com/TrueCloudLab/frostfs-crypto v0.5.0 // indirect
github.com/TrueCloudLab/rfc6979 v0.3.0 // indirect github.com/TrueCloudLab/rfc6979 v0.3.0 // indirect
@ -56,6 +54,7 @@ require (
github.com/golang/snappy v0.0.3 // indirect github.com/golang/snappy v0.0.3 // indirect
github.com/google/go-querystring v1.1.0 // indirect github.com/google/go-querystring v1.1.0 // indirect
github.com/gorilla/websocket v1.4.2 // indirect github.com/gorilla/websocket v1.4.2 // indirect
github.com/hashicorp/golang-lru v0.5.4 // indirect
github.com/hashicorp/hcl v1.0.0 // indirect github.com/hashicorp/hcl v1.0.0 // indirect
github.com/holiman/uint256 v1.2.0 // indirect github.com/holiman/uint256 v1.2.0 // indirect
github.com/inconshreveable/mousetrap v1.0.1 // indirect github.com/inconshreveable/mousetrap v1.0.1 // indirect